<commit_before>"""
Tests for the mock of the add target endpoint.
"""

import base64
import io
import json
from string import hexdigits
from typing import Any, Dict, Union
from urllib.parse import urljoin

import pytest
import requests
from _pytest.fixtures import SubRequest
from PIL import Image
from requests import codes
from requests_mock import POST

from common.constants import ResultCodes
from tests.mock_vws.utils import assert_vws_failure, assert_vws_response
from tests.utils import VuforiaServerCredentials
from vws._request_utils import authorization_header, rfc_1123_date

def assert_valid_target_id(target_id: str) -> None:
    """
    Assert that a given Target ID is in a valid format.

    Args:
        target_id: The Target ID to check.

    Raises:
        AssertionError: The Target ID is not in a valid format.
    """
    assert len(target_id) == 32
    assert all(char in hexdigits for char in target_id)

def _image_file(file_format: str, color_space: str) -> io.BytesIO:
    """
    Return an image file in the given format and color space.

    Args:
        file_format: See
            http://pillow.readthedocs.io/en/3.1.x/handbook/image-file-formats.html
        color_space: One of "L", "RGB", or "CMYK". "L" means greyscale.
    """
    image_buffer = io.BytesIO()
    width = 1
    height = 1
    image = Image.new(color_space, (width, height))
    image.save(image_buffer, file_format)
    image_buffer.seek(0)
    return image_buffer

@pytest.fixture
def png_rgb() -> io.BytesIO:
    """
    Return a PNG file in the RGB color space.
    """
    return _image_file(file_format='PNG', color_space='RGB')

@pytest.fixture
def png_greyscale() -> io.BytesIO:
    """
    Return a PNG file in the greyscale color space.
    """
    return _image_file(file_format='PNG', color_space='L')

@pytest.fixture
def jpeg_cmyk() -> io.BytesIO:
    """
    Return a PNG file in the CMYK color space.
    """
    return _image_file(file_format='JPEG', color_space='CMYK')

@pytest.fixture
def jpeg_rgb() -> io.BytesIO:
    """
    Return a JPEG file in the RGB color space.
    """
    return _image_file(file_format='JPEG', color_space='RGB')

@pytest.fixture
def tiff_rgb() -> io.BytesIO:
    """
    Return a TIFF file in the RGB color space.

    This is given as an option which is not supported by Vuforia as Vuforia
    supports only JPEG and PNG files.
    """
    return _image_file(file_format='TIFF', color_space='RGB')

@pytest.fixture(params=['png_rgb', 'jpeg_rgb', 'png_greyscale'])
def image_file(request: SubRequest) -> io.BytesIO:
    """
    Return an image file.
    """
    return request.getfixturevalue(request.param)

def add_target(
    vuforia_server_credentials: VuforiaServerCredentials,
    data: Dict[str, Any],
    content_type: str='application/json',
) -> requests.Response:
    """
    Helper to make a request to the endpoint to add a target.

    Args:
        vuforia_server_credentials: The credentials to use to connect to
            Vuforia.
        data: The data to send, in JSON format, to the endpoint.
        content_type: The `Content-Type` header to use.

    Returns:
        The response returned by the API.
    """
    date = rfc_1123_date()
    request_path = '/targets'

    content = bytes(json.dumps(data), encoding='utf-8')

    authorization_string = authorization_header(
        access_key=vuforia_server_credentials.access_key,
        secret_key=vuforia_server_credentials.secret_key,
        method=POST,
        content=content,
        content_type=content_type,
        date=date,
        request_path=request_path,
    )

    headers = {
        "Authorization": authorization_string,
        "Date": date,
        'Content-Type': content_type,
    }

    response = requests.request(
        method=POST,
        url=urljoin('https://vws.vuforia.com/', request_path),
        headers=headers,
        data=content,
    )

    return response

@pytest.mark.usefixtures('verify_mock_vuforia')
class TestSuccess:
    """
    Tests for the mock of the add target endpoint at `POST /targets`.
    """

    @pytest.mark.parametrize('content_type', [
        # This is the documented required content type:
        'application/json',
        # Other content types also work.
        'other/content_type',
    ], ids=['Documented Content-Type', 'Undocumented Content-Type'])
    @pytest.mark.parametrize('name', [
        'a',
        'a' * 64,
    ], ids=['Short name', 'Long name'])
    @pytest.mark.parametrize('width', [0, 0.1],
                             ids=['Zero width', 'Float width'])
    def test_created(self,
                     vuforia_server_credentials: VuforiaServerCredentials,
                     image_file: io.BytesIO,  # noqa: E501 pylint: disable=redefined-outer-name
                     content_type: str,
                     name: str,
                     width: Union[int, float],
                     ) -> None:
        """It is possible to get a `TargetCreated` response."""
        image_data = image_file.read()
        image_data_encoded = base64.b64encode(image_data).decode('ascii')

        data = {
            'name': name,
            'width': width,
            'image': image_data_encoded,
        }

        response = add_target(
            vuforia_server_credentials=vuforia_server_credentials,
            data=data,
            content_type=content_type,
        )

        assert_vws_response(
            response=response,
            status_code=codes.CREATED,
            result_code=ResultCodes.TARGET_CREATED,
        )
        expected_keys = {'result_code', 'transaction_id', 'target_id'}
        assert response.json().keys() == expected_keys
        assert_valid_target_id(target_id=response.json()['target_id'])

@pytest.mark.usefixtures('verify_mock_vuforia')
class TestMissingData:
    """
    Tests for giving incomplete data.
    """

    @pytest.mark.parametrize('data_to_remove', ['name', 'width', 'image'])
    def test_missing_data(self,
                          vuforia_server_credentials:
                          VuforiaServerCredentials,
                          png_rgb: io.BytesIO,  # noqa: E501 pylint: disable=redefined-outer-name
                          data_to_remove: str,
                          ) -> None:
        """
        `name`, `width` and `image` are all required.
        """
        image_data = png_rgb.read()
        image_data_encoded = base64.b64encode(image_data).decode('ascii')

        data = {
            'name': 'example_name',
            'width': 1,
            'image': image_data_encoded,
        }
        data.pop(data_to_remove)

        response = add_target(
            vuforia_server_credentials=vuforia_server_credentials,
            data=data,
        )

        assert_vws_failure(
            response=response,
            status_code=codes.BAD_REQUEST,
            result_code=ResultCodes.FAIL,
        )

@pytest.mark.usefixtures('verify_mock_vuforia')
class TestWidth:
    """
    Tests for the target width field.
    """

    @pytest.mark.parametrize(
        'width',
        [-1, '10'],
        ids=['Negative', 'Wrong Type'],
    )
    def test_width_invalid(self,
                           vuforia_server_credentials:
                           VuforiaServerCredentials,
                           png_rgb: io.BytesIO,  # noqa: E501 pylint: disable=redefined-outer-name
                           width: Any) -> None:
        """
        The width must be a non-negative number.
        """
        image_data = png_rgb.read()
        image_data_encoded = base64.b64encode(image_data).decode('ascii')

        data = {
            'name': 'example_name',
            'width': width,
            'image': image_data_encoded,
        }

        response = add_target(
            vuforia_server_credentials=vuforia_server_credentials,
            data=data,
        )

        assert_vws_failure(
            response=response,
            status_code=codes.BAD_REQUEST,
            result_code=ResultCodes.FAIL,
        )

@pytest.mark.usefixtures('verify_mock_vuforia')
class TestTargetName:
    """
    Tests for the target name field.
    """

    @pytest.mark.parametrize(
        'name',
        [1, '', 'a' * 65],
        ids=['Wrong Type', 'Empty', 'Too Long'],
    )
    def test_name_invalid(self,
                          name: str,
                          png_rgb: io.BytesIO,  # noqa: E501 pylint: disable=redefined-outer-name
                          vuforia_server_credentials: VuforiaServerCredentials
                          ) -> None:
        """
        A target's name must be a string of length 0 < N < 65.
        """
        image_data = png_rgb.read()
        image_data_encoded = base64.b64encode(image_data).decode('ascii')

        data = {
            'name': name,
            'width': 1,
            'image': image_data_encoded,
        }

        response = add_target(
            vuforia_server_credentials=vuforia_server_credentials,
            data=data,
        )

        assert_vws_failure(
            response=response,
            status_code=codes.BAD_REQUEST,
            result_code=ResultCodes.FAIL,
        )

    def test_existing_target_name(self,
                                  png_rgb: io.BytesIO,  # noqa: E501 pylint: disable=redefined-outer-name
                                  vuforia_server_credentials:
                                  VuforiaServerCredentials) -> None:
        """
        Only one target can have a given name.
        """
        image_data = png_rgb.read()
        image_data_encoded = base64.b64encode(image_data).decode('ascii')

        data = {
            'name': 'example_name',
            'width': 1,
            'image': image_data_encoded,
        }

        add_target(
            vuforia_server_credentials=vuforia_server_credentials,
            data=data,
        )

        response = add_target(
            vuforia_server_credentials=vuforia_server_credentials,
            data=data,
        )

        assert_vws_failure(
            response=response,
            status_code=codes.FORBIDDEN,
            result_code=ResultCodes.TARGET_NAME_EXIST,
        )

@pytest.mark.usefixtures('verify_mock_vuforia')
class TestInvalidImage:
    """
    Tests for giving images which do not conform to the specifications
    detailed in "Supported Images" on
    https://library.vuforia.com/articles/Training/Image-Target-Guide
    """

    def test_invalid_type(self,
                          tiff_rgb: io.BytesIO,  # noqa: E501 pylint: disable=redefined-outer-name
                          vuforia_server_credentials: VuforiaServerCredentials,
                          ) -> None:
        """
        A `BAD_REQUEST` response is returned if an image which is not a JPEG
        or PNG file is given.
        """
        image_data = tiff_rgb.read()
        image_data_encoded = base64.b64encode(image_data).decode('ascii')

        data = {
            'name': 'example_name',
            'width': 1,
            'image': image_data_encoded,
        }

        response = add_target(
            vuforia_server_credentials=vuforia_server_credentials,
            data=data,
        )

        assert_vws_failure(
            response=response,
            status_code=codes.UNPROCESSABLE_ENTITY,
            result_code=ResultCodes.BAD_IMAGE,
        )

    def test_wrong_color_space(self,
                               jpeg_cmyk: io.BytesIO,  # noqa: E501 pylint: disable=redefined-outer-name
                               vuforia_server_credentials:
                               VuforiaServerCredentials,
                               ) -> None:
        """
        A `BAD_REQUEST` response is returned if an image which is not in the
        greyscale or RGB color space.
        """
        image_data = jpeg_cmyk.read()
        image_data_encoded = base64.b64encode(image_data).decode('ascii')

        data = {
            'name': 'example_name',
            'width': 1,
            'image': image_data_encoded,
        }

        response = add_target(
            vuforia_server_credentials=vuforia_server_credentials,
            data=data,
        )

        assert_vws_failure(
            response=response,
            status_code=codes.UNPROCESSABLE_ENTITY,
            result_code=ResultCodes.BAD_IMAGE,
        )

@pytest.mark.usefixtures('verify_mock_vuforia')
class TestNotMandatoryFields:
    """
    Tests for passing data which is not mandatory to the endpoint.
    """

    def test_invalid_extra_data(self,
                                vuforia_server_credentials:
                                VuforiaServerCredentials,
                                png_rgb: io.BytesIO,  # noqa: E501 pylint: disable=redefined-outer-name
                                ) -> None:
        """
        A `BAD_REQUEST` response is returned when unexpected data is given.
        """
        image_data = png_rgb.read()
        image_data_encoded = base64.b64encode(image_data).decode('ascii')

        data = {
            'name': 'example_name',
            'width': 1,
            'image': image_data_encoded,
            'extra_thing': 1,
        }

        response = add_target(
            vuforia_server_credentials=vuforia_server_credentials,
            data=data,
        )

        assert_vws_failure(
            response=response,
            status_code=codes.BAD_REQUEST,
            result_code=ResultCodes.FAIL,
        )
<commit_msg>Explain test split<commit_after>"""
Tests for the mock of the add target endpoint.
"""

import base64
import io
import json
from string import hexdigits
from typing import Any, Dict, Union
from urllib.parse import urljoin

import pytest
import requests
from _pytest.fixtures import SubRequest
from PIL import Image
from requests import codes
from requests_mock import POST

from common.constants import ResultCodes
from tests.mock_vws.utils import assert_vws_failure, assert_vws_response
from tests.utils import VuforiaServerCredentials
from vws._request_utils import authorization_header, rfc_1123_date

def assert_valid_target_id(target_id: str) -> None:
    """
    Assert that a given Target ID is in a valid format.

    Args:
        target_id: The Target ID to check.

    Raises:
        AssertionError: The Target ID is not in a valid format.
    """
    assert len(target_id) == 32
    assert all(char in hexdigits for char in target_id)

def _image_file(file_format: str, color_space: str) -> io.BytesIO:
    """
    Return an image file in the given format and color space.

    Args:
        file_format: See
            http://pillow.readthedocs.io/en/3.1.x/handbook/image-file-formats.html
        color_space: One of "L", "RGB", or "CMYK". "L" means greyscale.
    """
    image_buffer = io.BytesIO()
    width = 1
    height = 1
    image = Image.new(color_space, (width, height))
    image.save(image_buffer, file_format)
    image_buffer.seek(0)
    return image_buffer

@pytest.fixture
def png_rgb() -> io.BytesIO:
    """
    Return a PNG file in the RGB color space.
    """
    return _image_file(file_format='PNG', color_space='RGB')

@pytest.fixture
def png_greyscale() -> io.BytesIO:
    """
    Return a PNG file in the greyscale color space.
    """
    return _image_file(file_format='PNG', color_space='L')

@pytest.fixture
def jpeg_cmyk() -> io.BytesIO:
    """
    Return a PNG file in the CMYK color space.
    """
    return _image_file(file_format='JPEG', color_space='CMYK')

@pytest.fixture
def jpeg_rgb() -> io.BytesIO:
    """
    Return a JPEG file in the RGB color space.
    """
    return _image_file(file_format='JPEG', color_space='RGB')

@pytest.fixture
def tiff_rgb() -> io.BytesIO:
    """
    Return a TIFF file in the RGB color space.

    This is given as an option which is not supported by Vuforia as Vuforia
    supports only JPEG and PNG files.
    """
    return _image_file(file_format='TIFF', color_space='RGB')

@pytest.fixture(params=['png_rgb', 'jpeg_rgb', 'png_greyscale'])
def image_file(request: SubRequest) -> io.BytesIO:
    """
    Return an image file.
    """
    return request.getfixturevalue(request.param)

def add_target(
    vuforia_server_credentials: VuforiaServerCredentials,
    data: Dict[str, Any],
    content_type: str='application/json',
) -> requests.Response:
    """
    Helper to make a request to the endpoint to add a target.

    Args:
        vuforia_server_credentials: The credentials to use to connect to
            Vuforia.
        data: The data to send, in JSON format, to the endpoint.
        content_type: The `Content-Type` header to use.

    Returns:
        The response returned by the API.
    """
    date = rfc_1123_date()
    request_path = '/targets'

    content = bytes(json.dumps(data), encoding='utf-8')

    authorization_string = authorization_header(
        access_key=vuforia_server_credentials.access_key,
        secret_key=vuforia_server_credentials.secret_key,
        method=POST,
        content=content,
        content_type=content_type,
        date=date,
        request_path=request_path,
    )

    headers = {
        "Authorization": authorization_string,
        "Date": date,
        'Content-Type': content_type,
    }

    response = requests.request(
        method=POST,
        url=urljoin('https://vws.vuforia.com/', request_path),
        headers=headers,
        data=content,
    )

    return response

@pytest.mark.usefixtures('verify_mock_vuforia')
class TestSuccess:
    """
    Tests for the mock of the add target endpoint at `POST /targets`.

    Each success creates a target which must be deleted before the next test.
    This is a slow operation so these tests are limited and not parametrized
    in a matrix.
    """

    @pytest.mark.parametrize('content_type', [
        # This is the documented required content type:
        'application/json',
        # Other content types also work.
        'other/content_type',
    ], ids=['Documented Content-Type', 'Undocumented Content-Type'])
    @pytest.mark.parametrize('name', [
        'a',
        'a' * 64,
    ], ids=['Short name', 'Long name'])
    @pytest.mark.parametrize('width', [0, 0.1],
                             ids=['Zero width', 'Float width'])
    def test_created(self,
                     vuforia_server_credentials: VuforiaServerCredentials,
                     image_file: io.BytesIO,  # noqa: E501 pylint: disable=redefined-outer-name
                     content_type: str,
                     name: str,
                     width: Union[int, float],
                     ) -> None:
        """It is possible to get a `TargetCreated` response."""
        image_data = image_file.read()
        image_data_encoded = base64.b64encode(image_data).decode('ascii')

        data = {
            'name': name,
            'width': width,
            'image': image_data_encoded,
        }

        response = add_target(
            vuforia_server_credentials=vuforia_server_credentials,
            data=data,
            content_type=content_type,
        )

        assert_vws_response(
            response=response,
            status_code=codes.CREATED,
            result_code=ResultCodes.TARGET_CREATED,
        )
        expected_keys = {'result_code', 'transaction_id', 'target_id'}
        assert response.json().keys() == expected_keys
        assert_valid_target_id(target_id=response.json()['target_id'])

@pytest.mark.usefixtures('verify_mock_vuforia')
class TestMissingData:
    """
    Tests for giving incomplete data.
    """

    @pytest.mark.parametrize('data_to_remove', ['name', 'width', 'image'])
    def test_missing_data(self,
                          vuforia_server_credentials:
                          VuforiaServerCredentials,
                          png_rgb: io.BytesIO,  # noqa: E501 pylint: disable=redefined-outer-name
                          data_to_remove: str,
                          ) -> None:
        """
        `name`, `width` and `image` are all required.
        """
        image_data = png_rgb.read()
        image_data_encoded = base64.b64encode(image_data).decode('ascii')

        data = {
            'name': 'example_name',
            'width': 1,
            'image': image_data_encoded,
        }
        data.pop(data_to_remove)

        response = add_target(
            vuforia_server_credentials=vuforia_server_credentials,
            data=data,
        )

        assert_vws_failure(
            response=response,
            status_code=codes.BAD_REQUEST,
            result_code=ResultCodes.FAIL,
        )

@pytest.mark.usefixtures('verify_mock_vuforia')
class TestWidth:
    """
    Tests for the target width field.
    """

    @pytest.mark.parametrize(
        'width',
        [-1, '10'],
        ids=['Negative', 'Wrong Type'],
    )
    def test_width_invalid(self,
                           vuforia_server_credentials:
                           VuforiaServerCredentials,
                           png_rgb: io.BytesIO,  # noqa: E501 pylint: disable=redefined-outer-name
                           width: Any) -> None:
        """
        The width must be a non-negative number.
        """
        image_data = png_rgb.read()
        image_data_encoded = base64.b64encode(image_data).decode('ascii')

        data = {
            'name': 'example_name',
            'width': width,
            'image': image_data_encoded,
        }

        response = add_target(
            vuforia_server_credentials=vuforia_server_credentials,
            data=data,
        )

        assert_vws_failure(
            response=response,
            status_code=codes.BAD_REQUEST,
            result_code=ResultCodes.FAIL,
        )

@pytest.mark.usefixtures('verify_mock_vuforia')
class TestTargetName:
    """
    Tests for the target name field.
    """

    @pytest.mark.parametrize(
        'name',
        [1, '', 'a' * 65],
        ids=['Wrong Type', 'Empty', 'Too Long'],
    )
    def test_name_invalid(self,
                          name: str,
                          png_rgb: io.BytesIO,  # noqa: E501 pylint: disable=redefined-outer-name
                          vuforia_server_credentials: VuforiaServerCredentials
                          ) -> None:
        """
        A target's name must be a string of length 0 < N < 65.
        """
        image_data = png_rgb.read()
        image_data_encoded = base64.b64encode(image_data).decode('ascii')

        data = {
            'name': name,
            'width': 1,
            'image': image_data_encoded,
        }

        response = add_target(
            vuforia_server_credentials=vuforia_server_credentials,
            data=data,
        )

        assert_vws_failure(
            response=response,
            status_code=codes.BAD_REQUEST,
            result_code=ResultCodes.FAIL,
        )

    def test_existing_target_name(self,
                                  png_rgb: io.BytesIO,  # noqa: E501 pylint: disable=redefined-outer-name
                                  vuforia_server_credentials:
                                  VuforiaServerCredentials) -> None:
        """
        Only one target can have a given name.
        """
        image_data = png_rgb.read()
        image_data_encoded = base64.b64encode(image_data).decode('ascii')

        data = {
            'name': 'example_name',
            'width': 1,
            'image': image_data_encoded,
        }

        add_target(
            vuforia_server_credentials=vuforia_server_credentials,
            data=data,
        )

        response = add_target(
            vuforia_server_credentials=vuforia_server_credentials,
            data=data,
        )

        assert_vws_failure(
            response=response,
            status_code=codes.FORBIDDEN,
            result_code=ResultCodes.TARGET_NAME_EXIST,
        )

@pytest.mark.usefixtures('verify_mock_vuforia')
class TestInvalidImage:
    """
    Tests for giving images which do not conform to the specifications
    detailed in "Supported Images" on
    https://library.vuforia.com/articles/Training/Image-Target-Guide
    """

    def test_invalid_type(self,
                          tiff_rgb: io.BytesIO,  # noqa: E501 pylint: disable=redefined-outer-name
                          vuforia_server_credentials: VuforiaServerCredentials,
                          ) -> None:
        """
        A `BAD_REQUEST` response is returned if an image which is not a JPEG
        or PNG file is given.
        """
        image_data = tiff_rgb.read()
        image_data_encoded = base64.b64encode(image_data).decode('ascii')

        data = {
            'name': 'example_name',
            'width': 1,
            'image': image_data_encoded,
        }

        response = add_target(
            vuforia_server_credentials=vuforia_server_credentials,
            data=data,
        )

        assert_vws_failure(
            response=response,
            status_code=codes.UNPROCESSABLE_ENTITY,
            result_code=ResultCodes.BAD_IMAGE,
        )

    def test_wrong_color_space(self,
                               jpeg_cmyk: io.BytesIO,  # noqa: E501 pylint: disable=redefined-outer-name
                               vuforia_server_credentials:
                               VuforiaServerCredentials,
                               ) -> None:
        """
        A `BAD_REQUEST` response is returned if an image which is not in the
        greyscale or RGB color space.
        """
        image_data = jpeg_cmyk.read()
        image_data_encoded = base64.b64encode(image_data).decode('ascii')

        data = {
            'name': 'example_name',
            'width': 1,
            'image': image_data_encoded,
        }

        response = add_target(
            vuforia_server_credentials=vuforia_server_credentials,
            data=data,
        )

        assert_vws_failure(
            response=response,
            status_code=codes.UNPROCESSABLE_ENTITY,
            result_code=ResultCodes.BAD_IMAGE,
        )

@pytest.mark.usefixtures('verify_mock_vuforia')
class TestNotMandatoryFields:
    """
    Tests for passing data which is not mandatory to the endpoint.
    """

    def test_invalid_extra_data(self,
                                vuforia_server_credentials:
                                VuforiaServerCredentials,
                                png_rgb: io.BytesIO,  # noqa: E501 pylint: disable=redefined-outer-name
                                ) -> None:
        """
        A `BAD_REQUEST` response is returned when unexpected data is given.
        """
        image_data = png_rgb.read()
        image_data_encoded = base64.b64encode(image_data).decode('ascii')

        data = {
            'name': 'example_name',
            'width': 1,
            'image': image_data_encoded,
            'extra_thing': 1,
        }

        response = add_target(
            vuforia_server_credentials=vuforia_server_credentials,
            data=data,
        )

        assert_vws_failure(
            response=response,
            status_code=codes.BAD_REQUEST,
            result_code=ResultCodes.FAIL,
        )
<|endoftext|>


<commit_before># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import pytest

from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import (create_exec_option_dimension_from_dict,
    create_parquet_dimension)

# Test with denial of reservations at varying frequency.
DEBUG_ACTION_DIMS = [None,
  '-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@0.1',
  '-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@0.5',
  '-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@0.9',
  '-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0']

@pytest.mark.xfail(pytest.config.option.testing_remote_cluster,
                   reason='Queries may not spill on larger clusters')
class TestSpillingDebugActionDimensions(ImpalaTestSuite):
  @classmethod
  def get_workload(self):
    return 'functional-query'

  @classmethod
  def add_test_dimensions(cls):
    super(TestSpillingDebugActionDimensions, cls).add_test_dimensions()
    cls.ImpalaTestMatrix.clear_constraints()
    cls.ImpalaTestMatrix.add_dimension(create_parquet_dimension('tpch'))
    # Tests are calibrated so that they can execute and spill with this page size.
    cls.ImpalaTestMatrix.add_dimension(
        create_exec_option_dimension_from_dict({'default_spillable_buffer_size' : ['256k'],
          'debug_action' : DEBUG_ACTION_DIMS}))

  def test_spilling(self, vector):
    self.run_test_case('QueryTest/spilling', vector)

  def test_spilling_aggs(self, vector):
    self.run_test_case('QueryTest/spilling-aggs', vector)

  def test_spilling_large_rows(self, vector, unique_database):
    """Test that we can process large rows in spilling operators, with or without
       spilling to disk"""
    self.run_test_case('QueryTest/spilling-large-rows', vector, unique_database)

  def test_spilling_naaj(self, vector):
    """Test spilling null-aware anti-joins"""
    self.run_test_case('QueryTest/spilling-naaj', vector)

  def test_spilling_sorts_exhaustive(self, vector):
    if self.exploration_strategy() != 'exhaustive':
      pytest.skip("only run large sorts on exhaustive")
    self.run_test_case('QueryTest/spilling-sorts-exhaustive', vector)

@pytest.mark.xfail(pytest.config.option.testing_remote_cluster,
                   reason='Queries may not spill on larger clusters')
class TestSpillingNoDebugActionDimensions(ImpalaTestSuite):
  """Spilling tests to which we don't want to apply the debug_action dimension."""
  @classmethod
  def get_workload(self):
    return 'functional-query'

  @classmethod
  def add_test_dimensions(cls):
    super(TestSpillingNoDebugActionDimensions, cls).add_test_dimensions()
    cls.ImpalaTestMatrix.clear_constraints()
    cls.ImpalaTestMatrix.add_dimension(create_parquet_dimension('tpch'))
    # Tests are calibrated so that they can execute and spill with this page size.
    cls.ImpalaTestMatrix.add_dimension(
        create_exec_option_dimension_from_dict({'default_spillable_buffer_size' : ['256k']}))

  def test_spilling_naaj_no_deny_reservation(self, vector):
    """
    Null-aware anti-join tests that depend on getting more than the minimum reservation
    and therefore will not reliably pass with the deny reservation debug action enabled.
    """
    self.run_test_case('QueryTest/spilling-naaj-no-deny-reservation', vector)

  def test_spilling_query_options(self, vector):
    """Test that spilling-related query options work end-to-end. These tests rely on
      setting debug_action to alternative values via query options."""
    self.run_test_case('QueryTest/spilling-query-options', vector)

<commit_msg>Move some test_spilling debug actions to exhaustive<commit_after># Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import pytest

from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.test_dimensions import (create_exec_option_dimension_from_dict,
    create_parquet_dimension)

# Test with denial of reservations at varying frequency.
# Always test with the minimal amount of spilling and running with the absolute minimum
# memory requirement.
CORE_DEBUG_ACTION_DIMS = [None,
  '-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@1.0']

# Test with different frequency of denial on exhaustive to try and exercise more
# interesting code paths.
EXHAUSTIVE_DEBUG_ACTION_DIMS = [
  '-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@0.1',
  '-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@0.5',
  '-1:OPEN:SET_DENY_RESERVATION_PROBABILITY@0.9']

@pytest.mark.xfail(pytest.config.option.testing_remote_cluster,
                   reason='Queries may not spill on larger clusters')
class TestSpillingDebugActionDimensions(ImpalaTestSuite):
  @classmethod
  def get_workload(self):
    return 'functional-query'

  @classmethod
  def add_test_dimensions(cls):
    super(TestSpillingDebugActionDimensions, cls).add_test_dimensions()
    cls.ImpalaTestMatrix.clear_constraints()
    cls.ImpalaTestMatrix.add_dimension(create_parquet_dimension('tpch'))
    debug_action_dims = CORE_DEBUG_ACTION_DIMS
    if cls.exploration_strategy() == 'exhaustive':
      debug_action_dims = CORE_DEBUG_ACTION_DIMS + EXHAUSTIVE_DEBUG_ACTION_DIMS
    # Tests are calibrated so that they can execute and spill with this page size.
    cls.ImpalaTestMatrix.add_dimension(
        create_exec_option_dimension_from_dict({'default_spillable_buffer_size' : ['256k'],
          'debug_action' : debug_action_dims}))

  def test_spilling(self, vector):
    self.run_test_case('QueryTest/spilling', vector)

  def test_spilling_aggs(self, vector):
    self.run_test_case('QueryTest/spilling-aggs', vector)

  def test_spilling_large_rows(self, vector, unique_database):
    """Test that we can process large rows in spilling operators, with or without
       spilling to disk"""
    self.run_test_case('QueryTest/spilling-large-rows', vector, unique_database)

  def test_spilling_naaj(self, vector):
    """Test spilling null-aware anti-joins"""
    self.run_test_case('QueryTest/spilling-naaj', vector)

  def test_spilling_sorts_exhaustive(self, vector):
    if self.exploration_strategy() != 'exhaustive':
      pytest.skip("only run large sorts on exhaustive")
    self.run_test_case('QueryTest/spilling-sorts-exhaustive', vector)

@pytest.mark.xfail(pytest.config.option.testing_remote_cluster,
                   reason='Queries may not spill on larger clusters')
class TestSpillingNoDebugActionDimensions(ImpalaTestSuite):
  """Spilling tests to which we don't want to apply the debug_action dimension."""
  @classmethod
  def get_workload(self):
    return 'functional-query'

  @classmethod
  def add_test_dimensions(cls):
    super(TestSpillingNoDebugActionDimensions, cls).add_test_dimensions()
    cls.ImpalaTestMatrix.clear_constraints()
    cls.ImpalaTestMatrix.add_dimension(create_parquet_dimension('tpch'))
    # Tests are calibrated so that they can execute and spill with this page size.
    cls.ImpalaTestMatrix.add_dimension(
        create_exec_option_dimension_from_dict({'default_spillable_buffer_size' : ['256k']}))

  def test_spilling_naaj_no_deny_reservation(self, vector):
    """
    Null-aware anti-join tests that depend on getting more than the minimum reservation
    and therefore will not reliably pass with the deny reservation debug action enabled.
    """
    self.run_test_case('QueryTest/spilling-naaj-no-deny-reservation', vector)

  def test_spilling_query_options(self, vector):
    """Test that spilling-related query options work end-to-end. These tests rely on
      setting debug_action to alternative values via query options."""
    self.run_test_case('QueryTest/spilling-query-options', vector)

<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from .test_configuration_common import ConfigTester
from .test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask

if is_torch_available():
    from transformers import (
        DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        DistilBertConfig,
        DistilBertForMaskedLM,
        DistilBertForMultipleChoice,
        DistilBertForQuestionAnswering,
        DistilBertForSequenceClassification,
        DistilBertForTokenClassification,
        DistilBertModel,
    )

    class DistilBertModelTester(object):
        def __init__(
            self,
            parent,
            batch_size=13,
            seq_length=7,
            is_training=True,
            use_input_mask=True,
            use_token_type_ids=False,
            use_labels=True,
            vocab_size=99,
            hidden_size=32,
            num_hidden_layers=5,
            num_attention_heads=4,
            intermediate_size=37,
            hidden_act="gelu",
            hidden_dropout_prob=0.1,
            attention_probs_dropout_prob=0.1,
            max_position_embeddings=512,
            type_vocab_size=16,
            type_sequence_label_size=2,
            initializer_range=0.02,
            num_labels=3,
            num_choices=4,
            scope=None,
        ):
            self.parent = parent
            self.batch_size = batch_size
            self.seq_length = seq_length
            self.is_training = is_training
            self.use_input_mask = use_input_mask
            self.use_token_type_ids = use_token_type_ids
            self.use_labels = use_labels
            self.vocab_size = vocab_size
            self.hidden_size = hidden_size
            self.num_hidden_layers = num_hidden_layers
            self.num_attention_heads = num_attention_heads
            self.intermediate_size = intermediate_size
            self.hidden_act = hidden_act
            self.hidden_dropout_prob = hidden_dropout_prob
            self.attention_probs_dropout_prob = attention_probs_dropout_prob
            self.max_position_embeddings = max_position_embeddings
            self.type_vocab_size = type_vocab_size
            self.type_sequence_label_size = type_sequence_label_size
            self.initializer_range = initializer_range
            self.num_labels = num_labels
            self.num_choices = num_choices
            self.scope = scope

        def prepare_config_and_inputs(self):
            input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)

            input_mask = None
            if self.use_input_mask:
                input_mask = random_attention_mask([self.batch_size, self.seq_length])

            sequence_labels = None
            token_labels = None
            choice_labels = None
            if self.use_labels:
                sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size)
                token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels)
                choice_labels = ids_tensor([self.batch_size], self.num_choices)

            config = DistilBertConfig(
                vocab_size=self.vocab_size,
                dim=self.hidden_size,
                n_layers=self.num_hidden_layers,
                n_heads=self.num_attention_heads,
                hidden_dim=self.intermediate_size,
                hidden_act=self.hidden_act,
                dropout=self.hidden_dropout_prob,
                attention_dropout=self.attention_probs_dropout_prob,
                max_position_embeddings=self.max_position_embeddings,
                initializer_range=self.initializer_range,
            )

            return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels

        def create_and_check_distilbert_model(
            self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
        ):
            model = DistilBertModel(config=config)
            model.to(torch_device)
            model.eval()
            result = model(input_ids, input_mask)
            result = model(input_ids)
            self.parent.assertEqual(
                result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size)
            )

        def create_and_check_distilbert_for_masked_lm(
            self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
        ):
            model = DistilBertForMaskedLM(config=config)
            model.to(torch_device)
            model.eval()
            result = model(input_ids, attention_mask=input_mask, labels=token_labels)
            self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size))

        def create_and_check_distilbert_for_question_answering(
            self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
        ):
            model = DistilBertForQuestionAnswering(config=config)
            model.to(torch_device)
            model.eval()
            result = model(
                input_ids, attention_mask=input_mask, start_positions=sequence_labels, end_positions=sequence_labels
            )
            self.parent.assertEqual(result.start_logits.shape, (self.batch_size, self.seq_length))
            self.parent.assertEqual(result.end_logits.shape, (self.batch_size, self.seq_length))

        def create_and_check_distilbert_for_sequence_classification(
            self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
        ):
            config.num_labels = self.num_labels
            model = DistilBertForSequenceClassification(config)
            model.to(torch_device)
            model.eval()
            result = model(input_ids, attention_mask=input_mask, labels=sequence_labels)
            self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_labels))

        def create_and_check_distilbert_for_token_classification(
            self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
        ):
            config.num_labels = self.num_labels
            model = DistilBertForTokenClassification(config=config)
            model.to(torch_device)
            model.eval()

            result = model(input_ids, attention_mask=input_mask, labels=token_labels)
            self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.num_labels))

        def create_and_check_distilbert_for_multiple_choice(
            self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
        ):
            config.num_choices = self.num_choices
            model = DistilBertForMultipleChoice(config=config)
            model.to(torch_device)
            model.eval()
            multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
            multiple_choice_input_mask = input_mask.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
            result = model(
                multiple_choice_inputs_ids,
                attention_mask=multiple_choice_input_mask,
                labels=choice_labels,
            )
            self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_choices))

        def prepare_config_and_inputs_for_common(self):
            config_and_inputs = self.prepare_config_and_inputs()
            (config, input_ids, input_mask, sequence_labels, token_labels, choice_labels) = config_and_inputs
            inputs_dict = {"input_ids": input_ids, "attention_mask": input_mask}
            return config, inputs_dict

@require_torch
class DistilBertModelTest(ModelTesterMixin, unittest.TestCase):

    all_model_classes = (
        (
            DistilBertModel,
            DistilBertForMaskedLM,
            DistilBertForMultipleChoice,
            DistilBertForQuestionAnswering,
            DistilBertForSequenceClassification,
            DistilBertForTokenClassification,
        )
        if is_torch_available()
        else None
    )
    test_pruning = True
    test_torchscript = True
    test_resize_embeddings = True

    def setUp(self):
        self.model_tester = DistilBertModelTester(self)
        self.config_tester = ConfigTester(self, config_class=DistilBertConfig, dim=37)

    def test_config(self):
        self.config_tester.run_common_tests()

    def test_distilbert_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_distilbert_model(*config_and_inputs)

    def test_for_masked_lm(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_distilbert_for_masked_lm(*config_and_inputs)

    def test_for_question_answering(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_distilbert_for_question_answering(*config_and_inputs)

    def test_for_sequence_classification(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_distilbert_for_sequence_classification(*config_and_inputs)

    def test_for_token_classification(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_distilbert_for_token_classification(*config_and_inputs)

    def test_for_multiple_choice(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_distilbert_for_multiple_choice(*config_and_inputs)

    @slow
    def test_model_from_pretrained(self):
        for model_name in DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            model = DistilBertModel.from_pretrained(model_name)
            self.assertIsNotNone(model)
<commit_msg>Added Integration testing for DistilBert model from issue #9948' (#9995)<commit_after># coding=utf-8
# Copyright 2020 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from .test_configuration_common import ConfigTester
from .test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask

if is_torch_available():
    import torch

    from transformers import (
        DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        DistilBertConfig,
        DistilBertForMaskedLM,
        DistilBertForMultipleChoice,
        DistilBertForQuestionAnswering,
        DistilBertForSequenceClassification,
        DistilBertForTokenClassification,
        DistilBertModel,
    )

    class DistilBertModelTester(object):
        def __init__(
            self,
            parent,
            batch_size=13,
            seq_length=7,
            is_training=True,
            use_input_mask=True,
            use_token_type_ids=False,
            use_labels=True,
            vocab_size=99,
            hidden_size=32,
            num_hidden_layers=5,
            num_attention_heads=4,
            intermediate_size=37,
            hidden_act="gelu",
            hidden_dropout_prob=0.1,
            attention_probs_dropout_prob=0.1,
            max_position_embeddings=512,
            type_vocab_size=16,
            type_sequence_label_size=2,
            initializer_range=0.02,
            num_labels=3,
            num_choices=4,
            scope=None,
        ):
            self.parent = parent
            self.batch_size = batch_size
            self.seq_length = seq_length
            self.is_training = is_training
            self.use_input_mask = use_input_mask
            self.use_token_type_ids = use_token_type_ids
            self.use_labels = use_labels
            self.vocab_size = vocab_size
            self.hidden_size = hidden_size
            self.num_hidden_layers = num_hidden_layers
            self.num_attention_heads = num_attention_heads
            self.intermediate_size = intermediate_size
            self.hidden_act = hidden_act
            self.hidden_dropout_prob = hidden_dropout_prob
            self.attention_probs_dropout_prob = attention_probs_dropout_prob
            self.max_position_embeddings = max_position_embeddings
            self.type_vocab_size = type_vocab_size
            self.type_sequence_label_size = type_sequence_label_size
            self.initializer_range = initializer_range
            self.num_labels = num_labels
            self.num_choices = num_choices
            self.scope = scope

        def prepare_config_and_inputs(self):
            input_ids = ids_tensor([self.batch_size, self.seq_length], self.vocab_size)

            input_mask = None
            if self.use_input_mask:
                input_mask = random_attention_mask([self.batch_size, self.seq_length])

            sequence_labels = None
            token_labels = None
            choice_labels = None
            if self.use_labels:
                sequence_labels = ids_tensor([self.batch_size], self.type_sequence_label_size)
                token_labels = ids_tensor([self.batch_size, self.seq_length], self.num_labels)
                choice_labels = ids_tensor([self.batch_size], self.num_choices)

            config = DistilBertConfig(
                vocab_size=self.vocab_size,
                dim=self.hidden_size,
                n_layers=self.num_hidden_layers,
                n_heads=self.num_attention_heads,
                hidden_dim=self.intermediate_size,
                hidden_act=self.hidden_act,
                dropout=self.hidden_dropout_prob,
                attention_dropout=self.attention_probs_dropout_prob,
                max_position_embeddings=self.max_position_embeddings,
                initializer_range=self.initializer_range,
            )

            return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels

        def create_and_check_distilbert_model(
            self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
        ):
            model = DistilBertModel(config=config)
            model.to(torch_device)
            model.eval()
            result = model(input_ids, input_mask)
            result = model(input_ids)
            self.parent.assertEqual(
                result.last_hidden_state.shape, (self.batch_size, self.seq_length, self.hidden_size)
            )

        def create_and_check_distilbert_for_masked_lm(
            self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
        ):
            model = DistilBertForMaskedLM(config=config)
            model.to(torch_device)
            model.eval()
            result = model(input_ids, attention_mask=input_mask, labels=token_labels)
            self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.vocab_size))

        def create_and_check_distilbert_for_question_answering(
            self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
        ):
            model = DistilBertForQuestionAnswering(config=config)
            model.to(torch_device)
            model.eval()
            result = model(
                input_ids, attention_mask=input_mask, start_positions=sequence_labels, end_positions=sequence_labels
            )
            self.parent.assertEqual(result.start_logits.shape, (self.batch_size, self.seq_length))
            self.parent.assertEqual(result.end_logits.shape, (self.batch_size, self.seq_length))

        def create_and_check_distilbert_for_sequence_classification(
            self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
        ):
            config.num_labels = self.num_labels
            model = DistilBertForSequenceClassification(config)
            model.to(torch_device)
            model.eval()
            result = model(input_ids, attention_mask=input_mask, labels=sequence_labels)
            self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_labels))

        def create_and_check_distilbert_for_token_classification(
            self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
        ):
            config.num_labels = self.num_labels
            model = DistilBertForTokenClassification(config=config)
            model.to(torch_device)
            model.eval()

            result = model(input_ids, attention_mask=input_mask, labels=token_labels)
            self.parent.assertEqual(result.logits.shape, (self.batch_size, self.seq_length, self.num_labels))

        def create_and_check_distilbert_for_multiple_choice(
            self, config, input_ids, input_mask, sequence_labels, token_labels, choice_labels
        ):
            config.num_choices = self.num_choices
            model = DistilBertForMultipleChoice(config=config)
            model.to(torch_device)
            model.eval()
            multiple_choice_inputs_ids = input_ids.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
            multiple_choice_input_mask = input_mask.unsqueeze(1).expand(-1, self.num_choices, -1).contiguous()
            result = model(
                multiple_choice_inputs_ids,
                attention_mask=multiple_choice_input_mask,
                labels=choice_labels,
            )
            self.parent.assertEqual(result.logits.shape, (self.batch_size, self.num_choices))

        def prepare_config_and_inputs_for_common(self):
            config_and_inputs = self.prepare_config_and_inputs()
            (config, input_ids, input_mask, sequence_labels, token_labels, choice_labels) = config_and_inputs
            inputs_dict = {"input_ids": input_ids, "attention_mask": input_mask}
            return config, inputs_dict

@require_torch
class DistilBertModelTest(ModelTesterMixin, unittest.TestCase):

    all_model_classes = (
        (
            DistilBertModel,
            DistilBertForMaskedLM,
            DistilBertForMultipleChoice,
            DistilBertForQuestionAnswering,
            DistilBertForSequenceClassification,
            DistilBertForTokenClassification,
        )
        if is_torch_available()
        else None
    )
    test_pruning = True
    test_torchscript = True
    test_resize_embeddings = True

    def setUp(self):
        self.model_tester = DistilBertModelTester(self)
        self.config_tester = ConfigTester(self, config_class=DistilBertConfig, dim=37)

    def test_config(self):
        self.config_tester.run_common_tests()

    def test_distilbert_model(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_distilbert_model(*config_and_inputs)

    def test_for_masked_lm(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_distilbert_for_masked_lm(*config_and_inputs)

    def test_for_question_answering(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_distilbert_for_question_answering(*config_and_inputs)

    def test_for_sequence_classification(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_distilbert_for_sequence_classification(*config_and_inputs)

    def test_for_token_classification(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_distilbert_for_token_classification(*config_and_inputs)

    def test_for_multiple_choice(self):
        config_and_inputs = self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_distilbert_for_multiple_choice(*config_and_inputs)

    @slow
    def test_model_from_pretrained(self):
        for model_name in DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
            model = DistilBertModel.from_pretrained(model_name)
            self.assertIsNotNone(model)

@require_torch
class DistilBertModelIntergrationTest(unittest.TestCase):
    @slow
    def test_inference_no_head_absolute_embedding(self):
        model = DistilBertModel.from_pretrained("distilbert-base-uncased")
        input_ids = torch.tensor([[0, 345, 232, 328, 740, 140, 1695, 69, 6078, 1588, 2]])
        output = model(input_ids)[0]
        expected_shape = torch.Size((1, 11, 768))
        self.assertEqual(output.shape, expected_shape)
        expected_slice = torch.tensor(
            [[[0.4026, -0.2919, 0.3902], [0.3828, -0.2129, 0.3563], [0.3919, -0.2287, 0.3438]]]
        )

        self.assertTrue(torch.allclose(output[:, :3, :3], expected_slice, atol=1e-4))
<|endoftext|>


<commit_before>from __future__ import print_function, division, absolute_import
from fontTools.misc.py23 import *

import os
import unittest

from fontTools import ttLib
from fontTools.pens.ttGlyphPen import TTGlyphPen

class TTGlyphPenTest(unittest.TestCase):

    def runEndToEnd(self, filename):
        font = ttLib.TTFont()
        ttx_path = os.path.join(
            os.path.abspath(os.path.dirname(os.path.realpath(__file__))),
            '..', 'ttLib', 'testdata', filename)
        font.importXML(ttx_path, quiet=True)

        glyphSet = font.getGlyphSet()
        glyfTable = font['glyf']
        pen = TTGlyphPen(font.getGlyphSet())

        for name in font.getGlyphOrder():
            oldGlyph = glyphSet[name]
            oldGlyph.draw(pen)
            oldGlyph = oldGlyph._glyph
            newGlyph = pen.glyph()

            if hasattr(oldGlyph, 'program'):
                newGlyph.program = oldGlyph.program

            self.assertEqual(
                oldGlyph.compile(glyfTable), newGlyph.compile(glyfTable))

    def test_e2e_linesAndSimpleComponents(self):
        self.runEndToEnd('TestTTF-Regular.ttx')

    def test_e2e_curvesAndComponentTransforms(self):
        self.runEndToEnd('TestTTFComplex-Regular.ttx')

if __name__ == '__main__':
    unittest.main()
<commit_msg>[TTGlyphPen] Add unit tests<commit_after>from __future__ import print_function, division, absolute_import
from fontTools.misc.py23 import *

import os
import unittest

from fontTools import ttLib
from fontTools.pens.ttGlyphPen import TTGlyphPen

class TTGlyphPenTest(unittest.TestCase):

    def runEndToEnd(self, filename):
        font = ttLib.TTFont()
        ttx_path = os.path.join(
            os.path.abspath(os.path.dirname(os.path.realpath(__file__))),
            '..', 'ttLib', 'testdata', filename)
        font.importXML(ttx_path, quiet=True)

        glyphSet = font.getGlyphSet()
        glyfTable = font['glyf']
        pen = TTGlyphPen(font.getGlyphSet())

        for name in font.getGlyphOrder():
            oldGlyph = glyphSet[name]
            oldGlyph.draw(pen)
            oldGlyph = oldGlyph._glyph
            newGlyph = pen.glyph()

            if hasattr(oldGlyph, 'program'):
                newGlyph.program = oldGlyph.program

            self.assertEqual(
                oldGlyph.compile(glyfTable), newGlyph.compile(glyfTable))

    def test_e2e_linesAndSimpleComponents(self):
        self.runEndToEnd('TestTTF-Regular.ttx')

    def test_e2e_curvesAndComponentTransforms(self):
        self.runEndToEnd('TestTTFComplex-Regular.ttx')

    def test_moveTo_errorWithinContour(self):
        pen = TTGlyphPen(None)
        pen.moveTo((0, 0))
        with self.assertRaises(AssertionError):
            pen.moveTo((1, 0))

    def test_closePath_ignoresAnchors(self):
        pen = TTGlyphPen(None)
        pen.moveTo((0, 0))
        pen.closePath()
        self.assertFalse(pen.points)

    def test_endPath_sameAsClosePath(self):
        pen = TTGlyphPen(None)

        pen.moveTo((0, 0))
        pen.lineTo((0, 1))
        pen.lineTo((1, 0))
        pen.closePath()
        closePathGlyph = pen.glyph()

        pen.moveTo((0, 0))
        pen.lineTo((0, 1))
        pen.lineTo((1, 0))
        pen.endPath()
        endPathGlyph = pen.glyph()

        endPathGlyph.program = closePathGlyph.program
        self.assertEqual(closePathGlyph, endPathGlyph)

    def test_glyph_errorOnUnendedContour(self):
        pen = TTGlyphPen(None)
        pen.moveTo((0, 0))
        with self.assertRaises(AssertionError):
            pen.glyph()

    def test_glyph_decomposes(self):
        componentName = 'a'
        glyphSet = {}
        pen = TTGlyphPen(glyphSet)

        pen.moveTo((0, 0))
        pen.lineTo((0, 1))
        pen.lineTo((1, 0))
        pen.closePath()
        glyphSet[componentName] = _TestGlyph(pen.glyph())

        pen.moveTo((0, 0))
        pen.lineTo((0, 1))
        pen.lineTo((1, 0))
        pen.closePath()
        pen.addComponent(componentName, (1, 0, 0, 1, 2, 0))
        compositeGlyph = pen.glyph()

        pen.moveTo((0, 0))
        pen.lineTo((0, 1))
        pen.lineTo((1, 0))
        pen.closePath()
        pen.moveTo((2, 0))
        pen.lineTo((2, 1))
        pen.lineTo((3, 0))
        pen.closePath()
        plainGlyph = pen.glyph()

        plainGlyph.program = compositeGlyph.program
        self.assertEqual(plainGlyph, compositeGlyph)

class _TestGlyph(object):
    def __init__(self, glyph):
        self.coordinates = glyph.coordinates

    def draw(self, pen):
        pen.moveTo(self.coordinates[0])
        for point in self.coordinates[1:]:
            pen.lineTo(point)
        pen.closePath()

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>#! /usr/bin/env python
# Last Change: Wed Jun 13 07:00 PM 2007 J

# For now, just test that all mode/dim execute correctly

import sys
import os
from numpy.testing import *

import numpy as N

set_package_path()
from pyem import GMM, GM, EM
restore_path()

set_local_path()
# import modules that are located in the same directory as this file.
from testcommon import DEF_DEC
curpath = sys.path[0]
restore_path()

def load_dataset(filename):
    from scipy.io import loadmat
    dic = loadmat(os.path.join(curpath, filename), squeeze_me = False)
    dic['w0'] = dic['w0'].squeeze()
    dic['w'] = dic['w'].squeeze()
    dic['tw'] = dic['tw'].squeeze()
    return dic

class EmTest(NumpyTestCase):
    def _create_model_and_run_em(self, d, k, mode, nframes):
        #+++++++++++++++++++++++++++++++++++++++++++++++++
        # Generate a model with k components, d dimensions
        #+++++++++++++++++++++++++++++++++++++++++++++++++
        w, mu, va   = GM.gen_param(d, k, mode, spread = 1.5)
        gm          = GM.fromvalues(w, mu, va)
        # Sample nframes frames  from the model
        data        = gm.sample(nframes)

        #++++++++++++++++++++++++++++++++++++++++++
        # Approximate the models with classical EM
        #++++++++++++++++++++++++++++++++++++++++++
        # Init the model
        lgm = GM(d, k, mode)
        gmm = GMM(lgm, 'kmean')

        em  = EM()
        lk  = em.train(data, gmm)

class test_full_run(EmTest):
    """This class only tests whether the algorithms runs. Do not check the
    results."""
    def test_1d(self, level = 1):
        d       = 1
        k       = 2
        mode    = 'full'
        nframes = int(1e2)

        #seed(1)
        self._create_model_and_run_em(d, k, mode, nframes)

    def test_2d(self, level = 1):
        d       = 2
        k       = 2
        mode    = 'full'
        nframes = int(1e2)

        #seed(1)
        self._create_model_and_run_em(d, k, mode, nframes)

    def test_5d(self, level = 1):
        d       = 5
        k       = 3
        mode    = 'full'
        nframes = int(1e2)

        #seed(1)
        self._create_model_and_run_em(d, k, mode, nframes)

class test_diag_run(EmTest):
    """This class only tests whether the algorithms runs. Do not test the
    results."""
    def test_1d(self, level = 1):
        d       = 1
        k       = 2
        mode    = 'diag'
        nframes = int(1e2)

        #seed(1)
        self._create_model_and_run_em(d, k, mode, nframes)

    def test_2d(self, level = 1):
        d       = 2
        k       = 2
        mode    = 'diag'
        nframes = int(1e2)

        #seed(1)
        self._create_model_and_run_em(d, k, mode, nframes)

    def test_5d(self, level = 1):
        d       = 5
        k       = 3
        mode    = 'diag'
        nframes = int(1e2)

        #seed(1)
        self._create_model_and_run_em(d, k, mode, nframes)

class test_datasets(EmTest):
    """This class tests whether the EM algorithms works using pre-computed
    datasets."""
    def test_1d_full(self, level = 1):
        d = 1
        k = 4
        mode = 'full'
        # Data are exactly the same than in diagonal mode, just test that
        # calling full mode works even in 1d, even if it is kind of stupid to
        # do so
        dic = load_dataset('diag_1d_4k.mat')

        gm = GM.fromvalues(dic['w0'], dic['mu0'], dic['va0'])
        gmm = GMM(gm, 'test')
        EM().train(dic['data'], gmm)

        assert_array_almost_equal(gmm.gm.w, dic['w'], DEF_DEC)
        assert_array_almost_equal(gmm.gm.mu, dic['mu'], DEF_DEC)
        assert_array_almost_equal(gmm.gm.va, dic['va'], DEF_DEC)

    def test_1d_diag(self, level = 1):
        d = 1
        k = 4
        mode = 'diag'
        dic = load_dataset('diag_1d_4k.mat')

        gm = GM.fromvalues(dic['w0'], dic['mu0'], dic['va0'])
        gmm = GMM(gm, 'test')
        EM().train(dic['data'], gmm)

        assert_array_equal(gmm.gm.w, dic['w'], DEF_DEC)
        assert_array_equal(gmm.gm.mu, dic['mu'], DEF_DEC)
        assert_array_equal(gmm.gm.va, dic['va'], DEF_DEC)

    def test_2d_full(self, level = 1):
        d = 2
        k = 3
        mode = 'full'
        dic = load_dataset('full_2d_3k.mat')

        gm = GM.fromvalues(dic['w0'], dic['mu0'], dic['va0'])
        gmm = GMM(gm, 'test')
        EM().train(dic['data'], gmm)

        assert_array_equal(gmm.gm.w, dic['w'], DEF_DEC)
        assert_array_equal(gmm.gm.mu, dic['mu'], DEF_DEC)
        assert_array_equal(gmm.gm.va, dic['va'], DEF_DEC)

    def test_2d_diag(self, level = 1):
        d = 2
        k = 3
        mode = 'diag'
        dic = load_dataset('diag_2d_3k.mat')

        gm = GM.fromvalues(dic['w0'], dic['mu0'], dic['va0'])
        gmm = GMM(gm, 'test')
        EM().train(dic['data'], gmm)

        assert_array__almost_equal(gmm.gm.w, dic['w'], DEF_DEC)
        assert_array__almost_equal(gmm.gm.mu, dic['mu'], DEF_DEC)
        assert_array__almost_equal(gmm.gm.va, dic['va'], DEF_DEC)

class test_log_domain(EmTest):
    """This class tests whether the GMM works in log domain."""
    def _test_common(self, d, k, mode):
        dic = load_dataset('%s_%dd_%dk.mat' % (mode, d, k))

        gm = GM.fromvalues(dic['w0'], dic['mu0'], dic['va0'])
        gmm = GMM(gm, 'test')

        a, na = gmm.compute_responsabilities(dic['data'])
        la, nla = gmm.compute_log_responsabilities(dic['data'])

        ta = N.log(a)
        tna = N.log(na)
        if not N.all(N.isfinite(ta)):
            print "precision problem for %s, %dd, %dk, test need fixing" % (mode, d, k)
        else:
            assert_array_almost_equal(ta, la, DEF_DEC)

        if not N.all(N.isfinite(tna)):
            print "precision problem for %s, %dd, %dk, test need fixing" % (mode, d, k)
        else:
            assert_array_almost_equal(tna, nla, DEF_DEC)

    def test_2d_diag(self, level = 1):
        d = 2
        k = 3
        mode = 'diag'
        self._test_common(d, k, mode)

    def test_1d_full(self, level = 1):
        d = 1
        k = 4
        mode = 'diag'
        self._test_common(d, k, mode)

    def test_2d_full(self, level = 1):
        d = 2
        k = 3
        mode = 'full'
        self._test_common(d, k, mode)

if __name__ == "__main__":
    NumpyTest().run()
<commit_msg>Trivial fix for typo in pyem tests.<commit_after>#! /usr/bin/env python
# Last Change: Wed Jun 13 07:00 PM 2007 J

# For now, just test that all mode/dim execute correctly

import sys
import os
from numpy.testing import *

import numpy as N

set_package_path()
from pyem import GMM, GM, EM
restore_path()

set_local_path()
# import modules that are located in the same directory as this file.
from testcommon import DEF_DEC
curpath = sys.path[0]
restore_path()

def load_dataset(filename):
    from scipy.io import loadmat
    dic = loadmat(os.path.join(curpath, filename), squeeze_me = False)
    dic['w0'] = dic['w0'].squeeze()
    dic['w'] = dic['w'].squeeze()
    dic['tw'] = dic['tw'].squeeze()
    return dic

class EmTest(NumpyTestCase):
    def _create_model_and_run_em(self, d, k, mode, nframes):
        #+++++++++++++++++++++++++++++++++++++++++++++++++
        # Generate a model with k components, d dimensions
        #+++++++++++++++++++++++++++++++++++++++++++++++++
        w, mu, va   = GM.gen_param(d, k, mode, spread = 1.5)
        gm          = GM.fromvalues(w, mu, va)
        # Sample nframes frames  from the model
        data        = gm.sample(nframes)

        #++++++++++++++++++++++++++++++++++++++++++
        # Approximate the models with classical EM
        #++++++++++++++++++++++++++++++++++++++++++
        # Init the model
        lgm = GM(d, k, mode)
        gmm = GMM(lgm, 'kmean')

        em  = EM()
        lk  = em.train(data, gmm)

class test_full_run(EmTest):
    """This class only tests whether the algorithms runs. Do not check the
    results."""
    def test_1d(self, level = 1):
        d       = 1
        k       = 2
        mode    = 'full'
        nframes = int(1e2)

        #seed(1)
        self._create_model_and_run_em(d, k, mode, nframes)

    def test_2d(self, level = 1):
        d       = 2
        k       = 2
        mode    = 'full'
        nframes = int(1e2)

        #seed(1)
        self._create_model_and_run_em(d, k, mode, nframes)

    def test_5d(self, level = 1):
        d       = 5
        k       = 3
        mode    = 'full'
        nframes = int(1e2)

        #seed(1)
        self._create_model_and_run_em(d, k, mode, nframes)

class test_diag_run(EmTest):
    """This class only tests whether the algorithms runs. Do not test the
    results."""
    def test_1d(self, level = 1):
        d       = 1
        k       = 2
        mode    = 'diag'
        nframes = int(1e2)

        #seed(1)
        self._create_model_and_run_em(d, k, mode, nframes)

    def test_2d(self, level = 1):
        d       = 2
        k       = 2
        mode    = 'diag'
        nframes = int(1e2)

        #seed(1)
        self._create_model_and_run_em(d, k, mode, nframes)

    def test_5d(self, level = 1):
        d       = 5
        k       = 3
        mode    = 'diag'
        nframes = int(1e2)

        #seed(1)
        self._create_model_and_run_em(d, k, mode, nframes)

class test_datasets(EmTest):
    """This class tests whether the EM algorithms works using pre-computed
    datasets."""
    def test_1d_full(self, level = 1):
        d = 1
        k = 4
        mode = 'full'
        # Data are exactly the same than in diagonal mode, just test that
        # calling full mode works even in 1d, even if it is kind of stupid to
        # do so
        dic = load_dataset('diag_1d_4k.mat')

        gm = GM.fromvalues(dic['w0'], dic['mu0'], dic['va0'])
        gmm = GMM(gm, 'test')
        EM().train(dic['data'], gmm)

        assert_array_almost_equal(gmm.gm.w, dic['w'], DEF_DEC)
        assert_array_almost_equal(gmm.gm.mu, dic['mu'], DEF_DEC)
        assert_array_almost_equal(gmm.gm.va, dic['va'], DEF_DEC)

    def test_1d_diag(self, level = 1):
        d = 1
        k = 4
        mode = 'diag'
        dic = load_dataset('diag_1d_4k.mat')

        gm = GM.fromvalues(dic['w0'], dic['mu0'], dic['va0'])
        gmm = GMM(gm, 'test')
        EM().train(dic['data'], gmm)

        assert_array_almost_equal(gmm.gm.w, dic['w'], DEF_DEC)
        assert_array_almost_equal(gmm.gm.mu, dic['mu'], DEF_DEC)
        assert_array_almost_equal(gmm.gm.va, dic['va'], DEF_DEC)

    def test_2d_full(self, level = 1):
        d = 2
        k = 3
        mode = 'full'
        dic = load_dataset('full_2d_3k.mat')

        gm = GM.fromvalues(dic['w0'], dic['mu0'], dic['va0'])
        gmm = GMM(gm, 'test')
        EM().train(dic['data'], gmm)

        assert_array_almost_equal(gmm.gm.w, dic['w'], DEF_DEC)
        assert_array_almost_equal(gmm.gm.mu, dic['mu'], DEF_DEC)
        assert_array_almost_equal(gmm.gm.va, dic['va'], DEF_DEC)

    def test_2d_diag(self, level = 1):
        d = 2
        k = 3
        mode = 'diag'
        dic = load_dataset('diag_2d_3k.mat')

        gm = GM.fromvalues(dic['w0'], dic['mu0'], dic['va0'])
        gmm = GMM(gm, 'test')
        EM().train(dic['data'], gmm)

        assert_array_almost_equal(gmm.gm.w, dic['w'], DEF_DEC)
        assert_array_almost_equal(gmm.gm.mu, dic['mu'], DEF_DEC)
        assert_array_almost_equal(gmm.gm.va, dic['va'], DEF_DEC)

class test_log_domain(EmTest):
    """This class tests whether the GMM works in log domain."""
    def _test_common(self, d, k, mode):
        dic = load_dataset('%s_%dd_%dk.mat' % (mode, d, k))

        gm = GM.fromvalues(dic['w0'], dic['mu0'], dic['va0'])
        gmm = GMM(gm, 'test')

        a, na = gmm.compute_responsabilities(dic['data'])
        la, nla = gmm.compute_log_responsabilities(dic['data'])

        ta = N.log(a)
        tna = N.log(na)
        if not N.all(N.isfinite(ta)):
            print "precision problem for %s, %dd, %dk, test need fixing" % (mode, d, k)
        else:
            assert_array_almost_equal(ta, la, DEF_DEC)

        if not N.all(N.isfinite(tna)):
            print "precision problem for %s, %dd, %dk, test need fixing" % (mode, d, k)
        else:
            assert_array_almost_equal(tna, nla, DEF_DEC)

    def test_2d_diag(self, level = 1):
        d = 2
        k = 3
        mode = 'diag'
        self._test_common(d, k, mode)

    def test_1d_full(self, level = 1):
        d = 1
        k = 4
        mode = 'diag'
        self._test_common(d, k, mode)

    def test_2d_full(self, level = 1):
        d = 2
        k = 3
        mode = 'full'
        self._test_common(d, k, mode)

if __name__ == "__main__":
    NumpyTest().run()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Nicole Thomas <nicole@saltstack.com>`
'''

# Import Salt Libs
from salt.modules import cmdmod
from salt.exceptions import CommandExecutionError
from salt.log import LOG_LEVELS

# Import Salt Testing Libs
from salttesting import TestCase, skipIf
from salttesting.mock import (
    mock_open,
    MagicMock,
    NO_MOCK,
    NO_MOCK_REASON,
    patch
)
from salttesting.helpers import ensure_in_syspath

ensure_in_syspath('../../')

cmdmod.__grains__ = {}

DEFAULT_SHELL = 'foo/bar'
MOCK_SHELL_FILE = '# List of acceptable shells\n' \
                  '\n'\
                  '/bin/bash\n'

@skipIf(NO_MOCK, NO_MOCK_REASON)
class CMDMODTestCase(TestCase):
    '''
    Unit tests for the salt.modules.cmdmod module
    '''

    mock_loglevels = {'info': 'foo', 'all': 'bar', 'critical': 'bar',
                      'trace': 'bar', 'garbage': 'bar', 'error': 'bar',
                      'debug': 'bar', 'warning': 'bar', 'quiet': 'bar'}

    def test_render_cmd_no_template(self):
        '''
        Tests return when template=None
        '''
        self.assertEqual(cmdmod._render_cmd('foo', 'bar', None),
                         ('foo', 'bar'))

    def test_render_cmd_unavailable_engine(self):
        '''
        Tests CommandExecutionError raised when template isn't in the
        template registry
        '''
        self.assertRaises(CommandExecutionError,
                          cmdmod._render_cmd,
                          'boo', 'bar', 'baz')

    def test_check_loglevel_bad_level(self):
        '''
        Tests return of providing an invalid loglevel option
        '''
        with patch.dict(LOG_LEVELS, self.mock_loglevels):
            self.assertEqual(cmdmod._check_loglevel(level='bad_loglevel'), 'foo')

    def test_check_loglevel_bad_level_not_str(self):
        '''
        Tests the return of providing an invalid loglevel option that is not a string
        '''
        with patch.dict(LOG_LEVELS, self.mock_loglevels):
            self.assertEqual(cmdmod._check_loglevel(level=1000), 'foo')

    def test_check_loglevel_quiet(self):
        '''
        Tests the return of providing a loglevel of 'quiet'
        '''
        with patch.dict(LOG_LEVELS, self.mock_loglevels):
            self.assertEqual(cmdmod._check_loglevel(level='quiet'), None)

    def test_check_loglevel_utils_quite(self):
        '''
        Tests the return of quiet=True
        '''
        with patch.dict(LOG_LEVELS, self.mock_loglevels):
            self.assertEqual(cmdmod._check_loglevel(quiet=True), None)

    def test_parse_env_not_env(self):
        '''
        Tests the return of an env that is not an env
        '''
        self.assertEqual(cmdmod._parse_env(None), {})

    def test_parse_env_list(self):
        '''
        Tests the return of an env that is a list
        '''
        ret = {'foo': None, 'bar': None}
        self.assertEqual(ret, cmdmod._parse_env(['foo', 'bar']))

    def test_parse_env_dict(self):
        '''
        Test the return of an env that is not a dict
        '''
        self.assertEqual(cmdmod._parse_env('test'), {})

    @patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True))
    @patch('salt.utils.is_windows', MagicMock(return_value=False))
    @patch('os.path.isfile', MagicMock(return_value=False))
    def test_run_shell_is_not_file(self):
        '''
        Tests error raised when shell is not available after _is_valid_shell error msg
        and os.path.isfile returns False
        '''
        self.assertRaises(CommandExecutionError, cmdmod._run, 'foo', 'bar')

    @patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True))
    @patch('salt.utils.is_windows', MagicMock(return_value=False))
    @patch('os.path.isfile', MagicMock(return_value=True))
    @patch('os.access', MagicMock(return_value=False))
    def test_run_shell_file_no_access(self):
        '''
        Tests error raised when shell is not available after _is_valid_shell error msg,
        os.path.isfile returns True, but os.access returns False
        '''
        self.assertRaises(CommandExecutionError, cmdmod._run, 'foo', 'bar')

    @patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True))
    @patch('salt.utils.is_windows', MagicMock(return_value=True))
    def test_run_runas_with_windows(self):
        '''
        Tests error raised when runas is passed on windows
        '''
        with patch.dict(cmdmod.__grains__, {'os': 'fake_os'}):
            self.assertRaises(CommandExecutionError,
                              cmdmod._run,
                              'foo', 'bar', runas='baz')

    @patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True))
    @patch('salt.utils.is_windows', MagicMock(return_value=False))
    @patch('os.path.isfile', MagicMock(return_value=True))
    @patch('os.access', MagicMock(return_value=True))
    def test_run_user_not_available(self):
        '''
        Tests return when runas user is not available
        '''
        self.assertRaises(CommandExecutionError, cmdmod._run, 'foo', 'bar', runas='baz')

    @patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True))
    @patch('salt.utils.is_windows', MagicMock(return_value=False))
    @patch('os.path.isfile', MagicMock(return_value=True))
    @patch('os.access', MagicMock(return_value=True))
    def test_run_zero_umask(self):
        '''
        Tests error raised when umask is set to zero
        '''
        self.assertRaises(CommandExecutionError, cmdmod._run, 'foo', 'bar', umask=0)

    @patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True))
    @patch('salt.utils.is_windows', MagicMock(return_value=False))
    @patch('os.path.isfile', MagicMock(return_value=True))
    @patch('os.access', MagicMock(return_value=True))
    def test_run_invalid_umask(self):
        '''
        Tests error raised when an invalid umask is given
        '''
        self.assertRaises(CommandExecutionError, cmdmod._run, 'foo', 'bar', umask='baz')

    @patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True))
    @patch('salt.utils.is_windows', MagicMock(return_value=False))
    @patch('os.path.isfile', MagicMock(return_value=True))
    @patch('os.access', MagicMock(return_value=True))
    def test_run_invalid_cwd_not_abs_path(self):
        '''
        Tests error raised when cwd is not an absolute path
        '''
        self.assertRaises(CommandExecutionError, cmdmod._run, 'foo', 'bar')

    @patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True))
    @patch('salt.utils.is_windows', MagicMock(return_value=False))
    @patch('os.path.isfile', MagicMock(return_value=True))
    @patch('os.access', MagicMock(return_value=True))
    @patch('os.path.isabs', MagicMock(return_value=True))
    def test_run_invalid_cwd_not_dir(self):
        '''
        Tests error raised when cwd is not a dir
        '''
        self.assertRaises(CommandExecutionError, cmdmod._run, 'foo', 'bar')

    @patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True))
    @patch('salt.utils.is_windows', MagicMock(return_value=False))
    @patch('os.path.isfile', MagicMock(return_value=True))
    @patch('os.access', MagicMock(return_value=True))
    @patch('salt.utils.timed_subprocess.TimedProc', MagicMock(side_effect=OSError))
    def test_run_no_vt_os_error(self):
        '''
        Tests error raised when not useing vt and OSError is provided
        '''
        self.assertRaises(CommandExecutionError, cmdmod._run, 'foo')

    @patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True))
    @patch('salt.utils.is_windows', MagicMock(return_value=False))
    @patch('os.path.isfile', MagicMock(return_value=True))
    @patch('os.access', MagicMock(return_value=True))
    @patch('salt.utils.timed_subprocess.TimedProc', MagicMock(side_effect=IOError))
    def test_run_no_vt_io_error(self):
        '''
        Tests error raised when not useing vt and IOError is provided
        '''
        self.assertRaises(CommandExecutionError, cmdmod._run, 'foo')

    @patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True))
    @patch('salt.utils.is_windows', MagicMock(return_value=False))
    @patch('os.path.isfile', MagicMock(return_value=True))
    @patch('os.access', MagicMock(return_value=True))
    def test_run(self):
        '''
        Tests end result when a command is not found
        '''
        ret = cmdmod._run('foo', use_vt=True).get('stderr')
        self.assertIn('foo', ret)

    @patch('salt.utils.is_windows', MagicMock(return_value=True))
    def test_is_valid_shell_windows(self):
        '''
        Tests return if running on windows
        '''
        self.assertTrue(cmdmod._is_valid_shell('foo'))

    @patch('os.path.exists', MagicMock(return_value=False))
    def test_is_valid_shell_none(self):
        '''
        Tests return of when os.path.exists(/etc/shells) isn't available
        '''
        self.assertIsNone(cmdmod._is_valid_shell('foo'))

    @patch('os.path.exists', MagicMock(return_value=True))
    def test_is_valid_shell_available(self):
        '''
        Tests return when provided shell is available
        '''
        with patch('salt.utils.fopen', mock_open(read_data=MOCK_SHELL_FILE)):
            self.assertTrue(cmdmod._is_valid_shell('/bin/bash'))

    @patch('os.path.exists', MagicMock(return_value=True))
    def test_is_valid_shell_unavailable(self):
        '''
        Tests return when provided shell is not available
        '''
        with patch('salt.utils.fopen', mock_open(read_data=MOCK_SHELL_FILE)):
            self.assertFalse(cmdmod._is_valid_shell('foo'))

if __name__ == '__main__':
    from integration import run_tests
    run_tests(CMDMODTestCase, needs_daemon=False)
<commit_msg>Py3 compatibility fixes.<commit_after># -*- coding: utf-8 -*-
'''
    :codeauthor: :email:`Nicole Thomas <nicole@saltstack.com>`
'''

# Import python libs
from __future__ import absolute_import

# Import Salt Libs
from salt.modules import cmdmod
from salt.exceptions import CommandExecutionError
from salt.log import LOG_LEVELS

# Import Salt Testing Libs
from salttesting import TestCase, skipIf
from salttesting.mock import (
    mock_open,
    MagicMock,
    NO_MOCK,
    NO_MOCK_REASON,
    patch
)
from salttesting.helpers import ensure_in_syspath

ensure_in_syspath('../../')

cmdmod.__grains__ = {}

DEFAULT_SHELL = 'foo/bar'
MOCK_SHELL_FILE = '# List of acceptable shells\n' \
                  '\n'\
                  '/bin/bash\n'

@skipIf(NO_MOCK, NO_MOCK_REASON)
class CMDMODTestCase(TestCase):
    '''
    Unit tests for the salt.modules.cmdmod module
    '''

    mock_loglevels = {'info': 'foo', 'all': 'bar', 'critical': 'bar',
                      'trace': 'bar', 'garbage': 'bar', 'error': 'bar',
                      'debug': 'bar', 'warning': 'bar', 'quiet': 'bar'}

    def test_render_cmd_no_template(self):
        '''
        Tests return when template=None
        '''
        self.assertEqual(cmdmod._render_cmd('foo', 'bar', None),
                         ('foo', 'bar'))

    def test_render_cmd_unavailable_engine(self):
        '''
        Tests CommandExecutionError raised when template isn't in the
        template registry
        '''
        self.assertRaises(CommandExecutionError,
                          cmdmod._render_cmd,
                          'boo', 'bar', 'baz')

    def test_check_loglevel_bad_level(self):
        '''
        Tests return of providing an invalid loglevel option
        '''
        with patch.dict(LOG_LEVELS, self.mock_loglevels):
            self.assertEqual(cmdmod._check_loglevel(level='bad_loglevel'), 'foo')

    def test_check_loglevel_bad_level_not_str(self):
        '''
        Tests the return of providing an invalid loglevel option that is not a string
        '''
        with patch.dict(LOG_LEVELS, self.mock_loglevels):
            self.assertEqual(cmdmod._check_loglevel(level=1000), 'foo')

    def test_check_loglevel_quiet(self):
        '''
        Tests the return of providing a loglevel of 'quiet'
        '''
        with patch.dict(LOG_LEVELS, self.mock_loglevels):
            self.assertEqual(cmdmod._check_loglevel(level='quiet'), None)

    def test_check_loglevel_utils_quite(self):
        '''
        Tests the return of quiet=True
        '''
        with patch.dict(LOG_LEVELS, self.mock_loglevels):
            self.assertEqual(cmdmod._check_loglevel(quiet=True), None)

    def test_parse_env_not_env(self):
        '''
        Tests the return of an env that is not an env
        '''
        self.assertEqual(cmdmod._parse_env(None), {})

    def test_parse_env_list(self):
        '''
        Tests the return of an env that is a list
        '''
        ret = {'foo': None, 'bar': None}
        self.assertEqual(ret, cmdmod._parse_env(['foo', 'bar']))

    def test_parse_env_dict(self):
        '''
        Test the return of an env that is not a dict
        '''
        self.assertEqual(cmdmod._parse_env('test'), {})

    @patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True))
    @patch('salt.utils.is_windows', MagicMock(return_value=False))
    @patch('os.path.isfile', MagicMock(return_value=False))
    def test_run_shell_is_not_file(self):
        '''
        Tests error raised when shell is not available after _is_valid_shell error msg
        and os.path.isfile returns False
        '''
        self.assertRaises(CommandExecutionError, cmdmod._run, 'foo', 'bar')

    @patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True))
    @patch('salt.utils.is_windows', MagicMock(return_value=False))
    @patch('os.path.isfile', MagicMock(return_value=True))
    @patch('os.access', MagicMock(return_value=False))
    def test_run_shell_file_no_access(self):
        '''
        Tests error raised when shell is not available after _is_valid_shell error msg,
        os.path.isfile returns True, but os.access returns False
        '''
        self.assertRaises(CommandExecutionError, cmdmod._run, 'foo', 'bar')

    @patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True))
    @patch('salt.utils.is_windows', MagicMock(return_value=True))
    def test_run_runas_with_windows(self):
        '''
        Tests error raised when runas is passed on windows
        '''
        with patch.dict(cmdmod.__grains__, {'os': 'fake_os'}):
            self.assertRaises(CommandExecutionError,
                              cmdmod._run,
                              'foo', 'bar', runas='baz')

    @patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True))
    @patch('salt.utils.is_windows', MagicMock(return_value=False))
    @patch('os.path.isfile', MagicMock(return_value=True))
    @patch('os.access', MagicMock(return_value=True))
    def test_run_user_not_available(self):
        '''
        Tests return when runas user is not available
        '''
        self.assertRaises(CommandExecutionError, cmdmod._run, 'foo', 'bar', runas='baz')

    @patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True))
    @patch('salt.utils.is_windows', MagicMock(return_value=False))
    @patch('os.path.isfile', MagicMock(return_value=True))
    @patch('os.access', MagicMock(return_value=True))
    def test_run_zero_umask(self):
        '''
        Tests error raised when umask is set to zero
        '''
        self.assertRaises(CommandExecutionError, cmdmod._run, 'foo', 'bar', umask=0)

    @patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True))
    @patch('salt.utils.is_windows', MagicMock(return_value=False))
    @patch('os.path.isfile', MagicMock(return_value=True))
    @patch('os.access', MagicMock(return_value=True))
    def test_run_invalid_umask(self):
        '''
        Tests error raised when an invalid umask is given
        '''
        self.assertRaises(CommandExecutionError, cmdmod._run, 'foo', 'bar', umask='baz')

    @patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True))
    @patch('salt.utils.is_windows', MagicMock(return_value=False))
    @patch('os.path.isfile', MagicMock(return_value=True))
    @patch('os.access', MagicMock(return_value=True))
    def test_run_invalid_cwd_not_abs_path(self):
        '''
        Tests error raised when cwd is not an absolute path
        '''
        self.assertRaises(CommandExecutionError, cmdmod._run, 'foo', 'bar')

    @patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True))
    @patch('salt.utils.is_windows', MagicMock(return_value=False))
    @patch('os.path.isfile', MagicMock(return_value=True))
    @patch('os.access', MagicMock(return_value=True))
    @patch('os.path.isabs', MagicMock(return_value=True))
    def test_run_invalid_cwd_not_dir(self):
        '''
        Tests error raised when cwd is not a dir
        '''
        self.assertRaises(CommandExecutionError, cmdmod._run, 'foo', 'bar')

    @patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True))
    @patch('salt.utils.is_windows', MagicMock(return_value=False))
    @patch('os.path.isfile', MagicMock(return_value=True))
    @patch('os.access', MagicMock(return_value=True))
    @patch('salt.utils.timed_subprocess.TimedProc', MagicMock(side_effect=OSError))
    def test_run_no_vt_os_error(self):
        '''
        Tests error raised when not useing vt and OSError is provided
        '''
        self.assertRaises(CommandExecutionError, cmdmod._run, 'foo')

    @patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True))
    @patch('salt.utils.is_windows', MagicMock(return_value=False))
    @patch('os.path.isfile', MagicMock(return_value=True))
    @patch('os.access', MagicMock(return_value=True))
    @patch('salt.utils.timed_subprocess.TimedProc', MagicMock(side_effect=IOError))
    def test_run_no_vt_io_error(self):
        '''
        Tests error raised when not useing vt and IOError is provided
        '''
        self.assertRaises(CommandExecutionError, cmdmod._run, 'foo')

    @patch('salt.modules.cmdmod._is_valid_shell', MagicMock(return_value=True))
    @patch('salt.utils.is_windows', MagicMock(return_value=False))
    @patch('os.path.isfile', MagicMock(return_value=True))
    @patch('os.access', MagicMock(return_value=True))
    def test_run(self):
        '''
        Tests end result when a command is not found
        '''
        ret = cmdmod._run('foo', use_vt=True).get('stderr')
        self.assertIn('foo', ret)

    @patch('salt.utils.is_windows', MagicMock(return_value=True))
    def test_is_valid_shell_windows(self):
        '''
        Tests return if running on windows
        '''
        self.assertTrue(cmdmod._is_valid_shell('foo'))

    @patch('os.path.exists', MagicMock(return_value=False))
    def test_is_valid_shell_none(self):
        '''
        Tests return of when os.path.exists(/etc/shells) isn't available
        '''
        self.assertIsNone(cmdmod._is_valid_shell('foo'))

    @patch('os.path.exists', MagicMock(return_value=True))
    def test_is_valid_shell_available(self):
        '''
        Tests return when provided shell is available
        '''
        with patch('salt.utils.fopen', mock_open(read_data=MOCK_SHELL_FILE)):
            self.assertTrue(cmdmod._is_valid_shell('/bin/bash'))

    @patch('os.path.exists', MagicMock(return_value=True))
    def test_is_valid_shell_unavailable(self):
        '''
        Tests return when provided shell is not available
        '''
        with patch('salt.utils.fopen', mock_open(read_data=MOCK_SHELL_FILE)):
            self.assertFalse(cmdmod._is_valid_shell('foo'))

if __name__ == '__main__':
    from integration import run_tests
    run_tests(CMDMODTestCase, needs_daemon=False)
<|endoftext|>