<commit_before>#!/usr/bin/env python
#
# Copyright 2011, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Tests for dispatch module."""

import os
import unittest

import set_sys_path  # Update sys.path to locate mod_pywebsocket module.

from mod_pywebsocket import dispatch
from mod_pywebsocket import handshake
from test import mock

_TEST_HANDLERS_DIR = os.path.join(
        os.path.split(__file__)[0], 'testdata', 'handlers')

_TEST_HANDLERS_SUB_DIR = os.path.join(_TEST_HANDLERS_DIR, 'sub')

class DispatcherTest(unittest.TestCase):
    """A unittest for dispatch module."""

    def test_normalize_path(self):
        self.assertEqual(os.path.abspath('/a/b').replace('\\', '/'),
                         dispatch._normalize_path('/a/b'))
        self.assertEqual(os.path.abspath('/a/b').replace('\\', '/'),
                         dispatch._normalize_path('\\a\\b'))
        self.assertEqual(os.path.abspath('/a/b').replace('\\', '/'),
                         dispatch._normalize_path('/a/c/../b'))
        self.assertEqual(os.path.abspath('abc').replace('\\', '/'),
                         dispatch._normalize_path('abc'))

    def test_converter(self):
        converter = dispatch._create_path_to_resource_converter('/a/b')
        self.assertEqual('/h', converter('/a/b/h_wsh.py'))
        self.assertEqual('/c/h', converter('/a/b/c/h_wsh.py'))
        self.assertEqual(None, converter('/a/b/h.py'))
        self.assertEqual(None, converter('a/b/h_wsh.py'))

        converter = dispatch._create_path_to_resource_converter('a/b')
        self.assertEqual('/h', converter(dispatch._normalize_path(
            'a/b/h_wsh.py')))

        converter = dispatch._create_path_to_resource_converter('/a/b///')
        self.assertEqual('/h', converter('/a/b/h_wsh.py'))
        self.assertEqual('/h', converter(dispatch._normalize_path(
            '/a/b/../b/h_wsh.py')))

        converter = dispatch._create_path_to_resource_converter(
            '/a/../a/b/../b/')
        self.assertEqual('/h', converter('/a/b/h_wsh.py'))

        converter = dispatch._create_path_to_resource_converter(r'\a\b')
        self.assertEqual('/h', converter(r'\a\b\h_wsh.py'))
        self.assertEqual('/h', converter(r'/a/b/h_wsh.py'))

    def test_enumerate_handler_file_paths(self):
        paths = list(
            dispatch._enumerate_handler_file_paths(_TEST_HANDLERS_DIR))
        paths.sort()
        self.assertEqual(8, len(paths))
        expected_paths = [
                os.path.join(_TEST_HANDLERS_DIR, 'abort_by_user_wsh.py'),
                os.path.join(_TEST_HANDLERS_DIR, 'blank_wsh.py'),
                os.path.join(_TEST_HANDLERS_DIR, 'origin_check_wsh.py'),
                os.path.join(_TEST_HANDLERS_DIR, 'sub',
                             'exception_in_transfer_wsh.py'),
                os.path.join(_TEST_HANDLERS_DIR, 'sub', 'non_callable_wsh.py'),
                os.path.join(_TEST_HANDLERS_DIR, 'sub', 'plain_wsh.py'),
                os.path.join(_TEST_HANDLERS_DIR, 'sub',
                             'wrong_handshake_sig_wsh.py'),
                os.path.join(_TEST_HANDLERS_DIR, 'sub',
                             'wrong_transfer_sig_wsh.py'),
                ]
        for expected, actual in zip(expected_paths, paths):
            self.assertEqual(expected, actual)

    def test_source_handler_file(self):
        self.assertRaises(
            dispatch.DispatchException, dispatch._source_handler_file, '')
        self.assertRaises(
            dispatch.DispatchException, dispatch._source_handler_file, 'def')
        self.assertRaises(
            dispatch.DispatchException, dispatch._source_handler_file, '1/0')
        self.failUnless(dispatch._source_handler_file(
                'def web_socket_do_extra_handshake(request):pass\n'
                'def web_socket_transfer_data(request):pass\n'))

    def test_source_warnings(self):
        dispatcher = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None)
        warnings = dispatcher.source_warnings()
        warnings.sort()
        expected_warnings = [
                (os.path.realpath(os.path.join(
                    _TEST_HANDLERS_DIR, 'blank_wsh.py')) +
                 ': web_socket_do_extra_handshake is not defined.'),
                (os.path.realpath(os.path.join(
                    _TEST_HANDLERS_DIR, 'sub', 'non_callable_wsh.py')) +
                 ': web_socket_do_extra_handshake is not callable.'),
                (os.path.realpath(os.path.join(
                    _TEST_HANDLERS_DIR, 'sub', 'wrong_handshake_sig_wsh.py')) +
                 ': web_socket_do_extra_handshake is not defined.'),
                (os.path.realpath(os.path.join(
                    _TEST_HANDLERS_DIR, 'sub', 'wrong_transfer_sig_wsh.py')) +
                 ': web_socket_transfer_data is not defined.'),
                ]
        self.assertEquals(4, len(warnings))
        for expected, actual in zip(expected_warnings, warnings):
            self.assertEquals(expected, actual)

    def test_do_extra_handshake(self):
        dispatcher = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None)
        request = mock.MockRequest()
        request.ws_resource = '/origin_check'
        request.ws_origin = 'http://example.com'
        dispatcher.do_extra_handshake(request)  # Must not raise exception.

        request.ws_origin = 'http://bad.example.com'
        try:
            dispatcher.do_extra_handshake(request)
            self.fail('Could not catch HandshakeException with 403 status')
        except handshake.HandshakeException, e:
            self.assertEquals(403, e.status)
        except Exception, e:
            self.fail('Unexpected exception: %r' % e)

    def test_abort_extra_handshake(self):
        dispatcher = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None)
        request = mock.MockRequest()
        request.ws_resource = '/abort_by_user'
        self.assertRaises(handshake.AbortedByUserException,
                          dispatcher.do_extra_handshake, request)

    def test_transfer_data(self):
        dispatcher = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None)

        request = mock.MockRequest(connection=mock.MockConn('\xff\x00'))
        request.ws_resource = '/origin_check'
        request.ws_protocol = 'p1'
        dispatcher.transfer_data(request)
        self.assertEqual('origin_check_wsh.py is called for /origin_check, p1'
                         '\xff\x00',
                         request.connection.written_data())

        request = mock.MockRequest(connection=mock.MockConn('\xff\x00'))
        request.ws_resource = '/sub/plain'
        request.ws_protocol = None
        dispatcher.transfer_data(request)
        self.assertEqual('sub/plain_wsh.py is called for /sub/plain, None'
                         '\xff\x00',
                         request.connection.written_data())

        request = mock.MockRequest(connection=mock.MockConn('\xff\x00'))
        request.ws_resource = '/sub/plain?'
        request.ws_protocol = None
        dispatcher.transfer_data(request)
        self.assertEqual('sub/plain_wsh.py is called for /sub/plain?, None'
                         '\xff\x00',
                         request.connection.written_data())

        request = mock.MockRequest(connection=mock.MockConn('\xff\x00'))
        request.ws_resource = '/sub/plain?q=v'
        request.ws_protocol = None
        dispatcher.transfer_data(request)
        self.assertEqual('sub/plain_wsh.py is called for /sub/plain?q=v, None'
                         '\xff\x00',
                         request.connection.written_data())

    def test_transfer_data_no_handler(self):
        dispatcher = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None)
        for resource in ['/blank', '/sub/non_callable',
                         '/sub/no_wsh_at_the_end', '/does/not/exist']:
            request = mock.MockRequest(connection=mock.MockConn(''))
            request.ws_resource = resource
            request.ws_protocol = 'p2'
            try:
                dispatcher.transfer_data(request)
                self.fail()
            except dispatch.DispatchException, e:
                self.failUnless(str(e).find('No handler') != -1)
            except Exception:
                self.fail()

    def test_transfer_data_handler_exception(self):
        dispatcher = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None)
        request = mock.MockRequest(connection=mock.MockConn(''))
        request.ws_resource = '/sub/exception_in_transfer'
        request.ws_protocol = 'p3'
        try:
            dispatcher.transfer_data(request)
            self.fail()
        except Exception, e:
            self.failUnless(str(e).find('Intentional') != -1,
                            'Unexpected exception: %s' % e)

    def test_abort_transfer_data(self):
        dispatcher = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None)
        request = mock.MockRequest()
        request.ws_resource = '/abort_by_user'
        self.assertRaises(handshake.AbortedByUserException,
                          dispatcher.transfer_data, request)

    def test_scan_dir(self):
        disp = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None)
        self.assertEqual(4, len(disp._handler_suite_map))
        self.failUnless('/origin_check' in disp._handler_suite_map)
        self.failUnless(
            '/sub/exception_in_transfer' in disp._handler_suite_map)
        self.failUnless('/sub/plain' in disp._handler_suite_map)

    def test_scan_sub_dir(self):
        disp = dispatch.Dispatcher(_TEST_HANDLERS_DIR, _TEST_HANDLERS_SUB_DIR)
        self.assertEqual(2, len(disp._handler_suite_map))
        self.failIf('/origin_check' in disp._handler_suite_map)
        self.failUnless(
            '/sub/exception_in_transfer' in disp._handler_suite_map)
        self.failUnless('/sub/plain' in disp._handler_suite_map)

    def test_scan_sub_dir_as_root(self):
        disp = dispatch.Dispatcher(_TEST_HANDLERS_SUB_DIR,
                                   _TEST_HANDLERS_SUB_DIR)
        self.assertEqual(2, len(disp._handler_suite_map))
        self.failIf('/origin_check' in disp._handler_suite_map)
        self.failIf('/sub/exception_in_transfer' in disp._handler_suite_map)
        self.failIf('/sub/plain' in disp._handler_suite_map)
        self.failUnless('/exception_in_transfer' in disp._handler_suite_map)
        self.failUnless('/plain' in disp._handler_suite_map)

    def test_scan_dir_must_under_root(self):
        dispatch.Dispatcher('a/b', 'a/b/c')  # OK
        dispatch.Dispatcher('a/b///', 'a/b')  # OK
        self.assertRaises(dispatch.DispatchException,
                          dispatch.Dispatcher, 'a/b/c', 'a/b')

    def test_resource_path_alias(self):
        disp = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None)
        disp.add_resource_path_alias('/', '/origin_check')
        self.assertEqual(5, len(disp._handler_suite_map))
        self.failUnless('/origin_check' in disp._handler_suite_map)
        self.failUnless(
            '/sub/exception_in_transfer' in disp._handler_suite_map)
        self.failUnless('/sub/plain' in disp._handler_suite_map)
        self.failUnless('/' in disp._handler_suite_map)
        self.assertRaises(dispatch.DispatchException,
                          disp.add_resource_path_alias, '/alias', '/not-exist')

if __name__ == '__main__':
    unittest.main()

# vi:sts=4 sw=4 et
<commit_msg>Support test_dispatch.py in ActivePython<commit_after>#!/usr/bin/env python
#
# Copyright 2011, Google Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#     * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

"""Tests for dispatch module."""

import os
import unittest

import set_sys_path  # Update sys.path to locate mod_pywebsocket module.

from mod_pywebsocket import dispatch
from mod_pywebsocket import handshake
from test import mock

_TEST_HANDLERS_DIR = os.path.join(
        os.path.split(__file__)[0], 'testdata', 'handlers')

_TEST_HANDLERS_SUB_DIR = os.path.join(_TEST_HANDLERS_DIR, 'sub')

class DispatcherTest(unittest.TestCase):
    """A unittest for dispatch module."""

    def test_normalize_path(self):
        self.assertEqual(os.path.abspath('/a/b').replace('\\', '/'),
                         dispatch._normalize_path('/a/b'))
        self.assertEqual(os.path.abspath('/a/b').replace('\\', '/'),
                         dispatch._normalize_path('\\a\\b'))
        self.assertEqual(os.path.abspath('/a/b').replace('\\', '/'),
                         dispatch._normalize_path('/a/c/../b'))
        self.assertEqual(os.path.abspath('abc').replace('\\', '/'),
                         dispatch._normalize_path('abc'))

    def test_converter(self):
        converter = dispatch._create_path_to_resource_converter('/a/b')
        # Python built by MSC inserts a drive name like 'C:\' via realpath().
        # Converter Generator expands provided path using realpath() and uses
        # the path including a drive name to verify the prefix.
        os_root = os.path.realpath('/')
        self.assertEqual('/h', converter(os_root + 'a/b/h_wsh.py'))
        self.assertEqual('/c/h', converter(os_root + 'a/b/c/h_wsh.py'))
        self.assertEqual(None, converter(os_root + 'a/b/h.py'))
        self.assertEqual(None, converter('a/b/h_wsh.py'))

        converter = dispatch._create_path_to_resource_converter('a/b')
        self.assertEqual('/h', converter(dispatch._normalize_path(
            'a/b/h_wsh.py')))

        converter = dispatch._create_path_to_resource_converter('/a/b///')
        self.assertEqual('/h', converter(os_root + 'a/b/h_wsh.py'))
        self.assertEqual('/h', converter(dispatch._normalize_path(
            '/a/b/../b/h_wsh.py')))

        converter = dispatch._create_path_to_resource_converter(
            '/a/../a/b/../b/')
        self.assertEqual('/h', converter(os_root + 'a/b/h_wsh.py'))

        converter = dispatch._create_path_to_resource_converter(r'\a\b')
        self.assertEqual('/h', converter(os_root + r'a\b\h_wsh.py'))
        self.assertEqual('/h', converter(os_root + r'a/b/h_wsh.py'))

    def test_enumerate_handler_file_paths(self):
        paths = list(
            dispatch._enumerate_handler_file_paths(_TEST_HANDLERS_DIR))
        paths.sort()
        self.assertEqual(8, len(paths))
        expected_paths = [
                os.path.join(_TEST_HANDLERS_DIR, 'abort_by_user_wsh.py'),
                os.path.join(_TEST_HANDLERS_DIR, 'blank_wsh.py'),
                os.path.join(_TEST_HANDLERS_DIR, 'origin_check_wsh.py'),
                os.path.join(_TEST_HANDLERS_DIR, 'sub',
                             'exception_in_transfer_wsh.py'),
                os.path.join(_TEST_HANDLERS_DIR, 'sub', 'non_callable_wsh.py'),
                os.path.join(_TEST_HANDLERS_DIR, 'sub', 'plain_wsh.py'),
                os.path.join(_TEST_HANDLERS_DIR, 'sub',
                             'wrong_handshake_sig_wsh.py'),
                os.path.join(_TEST_HANDLERS_DIR, 'sub',
                             'wrong_transfer_sig_wsh.py'),
                ]
        for expected, actual in zip(expected_paths, paths):
            self.assertEqual(expected, actual)

    def test_source_handler_file(self):
        self.assertRaises(
            dispatch.DispatchException, dispatch._source_handler_file, '')
        self.assertRaises(
            dispatch.DispatchException, dispatch._source_handler_file, 'def')
        self.assertRaises(
            dispatch.DispatchException, dispatch._source_handler_file, '1/0')
        self.failUnless(dispatch._source_handler_file(
                'def web_socket_do_extra_handshake(request):pass\n'
                'def web_socket_transfer_data(request):pass\n'))

    def test_source_warnings(self):
        dispatcher = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None)
        warnings = dispatcher.source_warnings()
        warnings.sort()
        expected_warnings = [
                (os.path.realpath(os.path.join(
                    _TEST_HANDLERS_DIR, 'blank_wsh.py')) +
                 ': web_socket_do_extra_handshake is not defined.'),
                (os.path.realpath(os.path.join(
                    _TEST_HANDLERS_DIR, 'sub', 'non_callable_wsh.py')) +
                 ': web_socket_do_extra_handshake is not callable.'),
                (os.path.realpath(os.path.join(
                    _TEST_HANDLERS_DIR, 'sub', 'wrong_handshake_sig_wsh.py')) +
                 ': web_socket_do_extra_handshake is not defined.'),
                (os.path.realpath(os.path.join(
                    _TEST_HANDLERS_DIR, 'sub', 'wrong_transfer_sig_wsh.py')) +
                 ': web_socket_transfer_data is not defined.'),
                ]
        self.assertEquals(4, len(warnings))
        for expected, actual in zip(expected_warnings, warnings):
            self.assertEquals(expected, actual)

    def test_do_extra_handshake(self):
        dispatcher = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None)
        request = mock.MockRequest()
        request.ws_resource = '/origin_check'
        request.ws_origin = 'http://example.com'
        dispatcher.do_extra_handshake(request)  # Must not raise exception.

        request.ws_origin = 'http://bad.example.com'
        try:
            dispatcher.do_extra_handshake(request)
            self.fail('Could not catch HandshakeException with 403 status')
        except handshake.HandshakeException, e:
            self.assertEquals(403, e.status)
        except Exception, e:
            self.fail('Unexpected exception: %r' % e)

    def test_abort_extra_handshake(self):
        dispatcher = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None)
        request = mock.MockRequest()
        request.ws_resource = '/abort_by_user'
        self.assertRaises(handshake.AbortedByUserException,
                          dispatcher.do_extra_handshake, request)

    def test_transfer_data(self):
        dispatcher = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None)

        request = mock.MockRequest(connection=mock.MockConn('\xff\x00'))
        request.ws_resource = '/origin_check'
        request.ws_protocol = 'p1'
        dispatcher.transfer_data(request)
        self.assertEqual('origin_check_wsh.py is called for /origin_check, p1'
                         '\xff\x00',
                         request.connection.written_data())

        request = mock.MockRequest(connection=mock.MockConn('\xff\x00'))
        request.ws_resource = '/sub/plain'
        request.ws_protocol = None
        dispatcher.transfer_data(request)
        self.assertEqual('sub/plain_wsh.py is called for /sub/plain, None'
                         '\xff\x00',
                         request.connection.written_data())

        request = mock.MockRequest(connection=mock.MockConn('\xff\x00'))
        request.ws_resource = '/sub/plain?'
        request.ws_protocol = None
        dispatcher.transfer_data(request)
        self.assertEqual('sub/plain_wsh.py is called for /sub/plain?, None'
                         '\xff\x00',
                         request.connection.written_data())

        request = mock.MockRequest(connection=mock.MockConn('\xff\x00'))
        request.ws_resource = '/sub/plain?q=v'
        request.ws_protocol = None
        dispatcher.transfer_data(request)
        self.assertEqual('sub/plain_wsh.py is called for /sub/plain?q=v, None'
                         '\xff\x00',
                         request.connection.written_data())

    def test_transfer_data_no_handler(self):
        dispatcher = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None)
        for resource in ['/blank', '/sub/non_callable',
                         '/sub/no_wsh_at_the_end', '/does/not/exist']:
            request = mock.MockRequest(connection=mock.MockConn(''))
            request.ws_resource = resource
            request.ws_protocol = 'p2'
            try:
                dispatcher.transfer_data(request)
                self.fail()
            except dispatch.DispatchException, e:
                self.failUnless(str(e).find('No handler') != -1)
            except Exception:
                self.fail()

    def test_transfer_data_handler_exception(self):
        dispatcher = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None)
        request = mock.MockRequest(connection=mock.MockConn(''))
        request.ws_resource = '/sub/exception_in_transfer'
        request.ws_protocol = 'p3'
        try:
            dispatcher.transfer_data(request)
            self.fail()
        except Exception, e:
            self.failUnless(str(e).find('Intentional') != -1,
                            'Unexpected exception: %s' % e)

    def test_abort_transfer_data(self):
        dispatcher = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None)
        request = mock.MockRequest()
        request.ws_resource = '/abort_by_user'
        self.assertRaises(handshake.AbortedByUserException,
                          dispatcher.transfer_data, request)

    def test_scan_dir(self):
        disp = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None)
        self.assertEqual(4, len(disp._handler_suite_map))
        self.failUnless('/origin_check' in disp._handler_suite_map)
        self.failUnless(
            '/sub/exception_in_transfer' in disp._handler_suite_map)
        self.failUnless('/sub/plain' in disp._handler_suite_map)

    def test_scan_sub_dir(self):
        disp = dispatch.Dispatcher(_TEST_HANDLERS_DIR, _TEST_HANDLERS_SUB_DIR)
        self.assertEqual(2, len(disp._handler_suite_map))
        self.failIf('/origin_check' in disp._handler_suite_map)
        self.failUnless(
            '/sub/exception_in_transfer' in disp._handler_suite_map)
        self.failUnless('/sub/plain' in disp._handler_suite_map)

    def test_scan_sub_dir_as_root(self):
        disp = dispatch.Dispatcher(_TEST_HANDLERS_SUB_DIR,
                                   _TEST_HANDLERS_SUB_DIR)
        self.assertEqual(2, len(disp._handler_suite_map))
        self.failIf('/origin_check' in disp._handler_suite_map)
        self.failIf('/sub/exception_in_transfer' in disp._handler_suite_map)
        self.failIf('/sub/plain' in disp._handler_suite_map)
        self.failUnless('/exception_in_transfer' in disp._handler_suite_map)
        self.failUnless('/plain' in disp._handler_suite_map)

    def test_scan_dir_must_under_root(self):
        dispatch.Dispatcher('a/b', 'a/b/c')  # OK
        dispatch.Dispatcher('a/b///', 'a/b')  # OK
        self.assertRaises(dispatch.DispatchException,
                          dispatch.Dispatcher, 'a/b/c', 'a/b')

    def test_resource_path_alias(self):
        disp = dispatch.Dispatcher(_TEST_HANDLERS_DIR, None)
        disp.add_resource_path_alias('/', '/origin_check')
        self.assertEqual(5, len(disp._handler_suite_map))
        self.failUnless('/origin_check' in disp._handler_suite_map)
        self.failUnless(
            '/sub/exception_in_transfer' in disp._handler_suite_map)
        self.failUnless('/sub/plain' in disp._handler_suite_map)
        self.failUnless('/' in disp._handler_suite_map)
        self.assertRaises(dispatch.DispatchException,
                          disp.add_resource_path_alias, '/alias', '/not-exist')

if __name__ == '__main__':
    unittest.main()

# vi:sts=4 sw=4 et
<|endoftext|>


<commit_before>from concurrent.futures import ThreadPoolExecutor
from collections import namedtuple

import docker

from tornado.log import app_log

from tornado import gen, web

ContainerConfig = namedtuple('ContainerConfig', [
    'image', 'ipython_executable', 'mem_limit', 'cpu_shares', 'container_ip', 'container_port'
])

# Number of times to retry API calls before giving up.
RETRIES = 5

class AsyncDockerClient():
    '''Completely ridiculous wrapper for a Docker client that returns futures
    on every single docker method called on it, configured with an executor.
    If no executor is passed, it defaults ThreadPoolExecutor(max_workers=2).
    '''
    def __init__(self, docker_client, executor=None):
        if executor is None:
            executor = ThreadPoolExecutor(max_workers=2)
        self._docker_client = docker_client
        self.executor = executor

    def __getattr__(self, name):
        '''Creates a function, based on docker_client.name that returns a
        Future. If name is not a callable, returns the attribute directly.
        '''
        fn = getattr(self._docker_client, name)

        # Make sure it really is a function first
        if not callable(fn):
            return fn

        def method(*args, **kwargs):
            return self.executor.submit(fn, *args, **kwargs)

        return method

class DockerSpawner():
    def __init__(self,
                 docker_host='unix://var/run/docker.sock',
                 version='1.12',
                 timeout=20,
                 max_workers=64):

        blocking_docker_client = docker.Client(base_url=docker_host,
                                               version=version,
                                               timeout=timeout)

        executor = ThreadPoolExecutor(max_workers=max_workers)

        async_docker_client = AsyncDockerClient(blocking_docker_client,
                                                executor)
        self.docker_client = async_docker_client

    @gen.coroutine
    def create_notebook_server(self, base_path, container_config):
        '''Creates a notebook_server running off of `base_path`.

        Returns the container_id, ip, port in a Future.'''

        templates = ['/srv/ga',
                     '/srv/ipython/IPython/html',
                     '/srv/ipython/IPython/html/templates']

        tornado_settings = {'template_path': templates}

        ipython_args = [
                "notebook", "--no-browser",
                "--port {}".format(container_config.container_port),
                "--ip=0.0.0.0",
                "--NotebookApp.base_url=/{}".format(base_path),
                "--NotebookApp.tornado_settings=\"{}\"".format(tornado_settings)
        ]

        ipython_command = container_config.ipython_executable + " " + " ".join(ipython_args)

        command = [
            "/bin/sh",
            "-c",
            ipython_command
        ]

        resp = yield self._with_retries(RETRIES,
                                        self.docker_client.create_container,
                                        image=container_config.image,
                                        command=command,
                                        mem_limit=container_config.mem_limit,
                                        cpu_shares=container_config.cpu_shares)

        docker_warnings = resp['Warnings']
        if docker_warnings is not None:
            app_log.warn(docker_warnings)

        container_id = resp['Id']
        app_log.info("Created container {}".format(container_id))

        port_bindings = {
            container_config.container_port: (container_config.container_ip,)
        }
        yield self._with_retries(RETRIES,
                                 self.docker_client.start,
                                 container_id,
                                 port_bindings=port_bindings)

        container_network = yield self._with_retries(RETRIES,
                                                     self.docker_client.port,
                                                     container_id,
                                                     container_config.container_port)

        host_port = container_network[0]['HostPort']
        host_ip = container_network[0]['HostIp']

        raise gen.Return((container_id, host_ip, int(host_port)))

    @gen.coroutine
    def shutdown_notebook_server(self, container_id, alive=True):
        '''Gracefully stop a running container.'''

        if alive:
            yield self._with_retries(RETRIES, self.docker_client.stop, container_id)
        yield self._with_retries(RETRIES, self.docker_client.remove_container, container_id)

    @gen.coroutine
    def list_notebook_servers(self, container_config, all=True):
        '''List containers that were launched from a specific image.'''

        existing = yield self._with_retries(RETRIES,
                                            self.docker_client.containers,
                                            all=all,
                                            trunc=False)

        untagged_image = container_config.image.split(':')[0]
        def has_matching_image(container):
            return container['Image'].split(':')[0] == untagged_image

        matching = [container for container in existing if has_matching_image(container)]
        raise gen.Return(matching)

    @gen.coroutine
    def _with_retries(self, max_tries, fn, *args, **kwargs):
        '''Attempt a Docker API call.

        If an error occurs, retry up to "max_tries" times before letting the exception propagate
        up the stack.'''

        try:
            result = yield fn(*args, **kwargs)
            raise gen.Return(result)
        except docker.errors.APIError as e:
            app_log.error("Encountered a Docker error (%i retries remain): %s", max_tries, e)
            if max_tries > 0:
                result = yield self._with_retries(max_tries - 1, fn, *args, **kwargs)
                raise gen.Return(result)
            else:
                raise e
<commit_msg>Remove max_tries and default it to RETRIES.<commit_after>from concurrent.futures import ThreadPoolExecutor
from collections import namedtuple

import docker

from tornado.log import app_log

from tornado import gen, web

ContainerConfig = namedtuple('ContainerConfig', [
    'image', 'ipython_executable', 'mem_limit', 'cpu_shares', 'container_ip', 'container_port'
])

# Number of times to retry API calls before giving up.
RETRIES = 5

class AsyncDockerClient():
    '''Completely ridiculous wrapper for a Docker client that returns futures
    on every single docker method called on it, configured with an executor.
    If no executor is passed, it defaults ThreadPoolExecutor(max_workers=2).
    '''
    def __init__(self, docker_client, executor=None):
        if executor is None:
            executor = ThreadPoolExecutor(max_workers=2)
        self._docker_client = docker_client
        self.executor = executor

    def __getattr__(self, name):
        '''Creates a function, based on docker_client.name that returns a
        Future. If name is not a callable, returns the attribute directly.
        '''
        fn = getattr(self._docker_client, name)

        # Make sure it really is a function first
        if not callable(fn):
            return fn

        def method(*args, **kwargs):
            return self.executor.submit(fn, *args, **kwargs)

        return method

class DockerSpawner():
    def __init__(self,
                 docker_host='unix://var/run/docker.sock',
                 version='1.12',
                 timeout=20,
                 max_workers=64):

        blocking_docker_client = docker.Client(base_url=docker_host,
                                               version=version,
                                               timeout=timeout)

        executor = ThreadPoolExecutor(max_workers=max_workers)

        async_docker_client = AsyncDockerClient(blocking_docker_client,
                                                executor)
        self.docker_client = async_docker_client

    @gen.coroutine
    def create_notebook_server(self, base_path, container_config):
        '''Creates a notebook_server running off of `base_path`.

        Returns the container_id, ip, port in a Future.'''

        templates = ['/srv/ga',
                     '/srv/ipython/IPython/html',
                     '/srv/ipython/IPython/html/templates']

        tornado_settings = {'template_path': templates}

        ipython_args = [
                "notebook", "--no-browser",
                "--port {}".format(container_config.container_port),
                "--ip=0.0.0.0",
                "--NotebookApp.base_url=/{}".format(base_path),
                "--NotebookApp.tornado_settings=\"{}\"".format(tornado_settings)
        ]

        ipython_command = container_config.ipython_executable + " " + " ".join(ipython_args)

        command = [
            "/bin/sh",
            "-c",
            ipython_command
        ]

        resp = yield self._with_retries(self.docker_client.create_container,
                                        image=container_config.image,
                                        command=command,
                                        mem_limit=container_config.mem_limit,
                                        cpu_shares=container_config.cpu_shares)

        docker_warnings = resp['Warnings']
        if docker_warnings is not None:
            app_log.warn(docker_warnings)

        container_id = resp['Id']
        app_log.info("Created container {}".format(container_id))

        port_bindings = {
            container_config.container_port: (container_config.container_ip,)
        }
        yield self._with_retries(self.docker_client.start,
                                 container_id,
                                 port_bindings=port_bindings)

        container_network = yield self._with_retries(self.docker_client.port,
                                                     container_id,
                                                     container_config.container_port)

        host_port = container_network[0]['HostPort']
        host_ip = container_network[0]['HostIp']

        raise gen.Return((container_id, host_ip, int(host_port)))

    @gen.coroutine
    def shutdown_notebook_server(self, container_id, alive=True):
        '''Gracefully stop a running container.'''

        if alive:
            yield self._with_retries(self.docker_client.stop, container_id)
        yield self._with_retries(self.docker_client.remove_container, container_id)

    @gen.coroutine
    def list_notebook_servers(self, container_config, all=True):
        '''List containers that were launched from a specific image.'''

        existing = yield self._with_retries(self.docker_client.containers,
                                            all=all,
                                            trunc=False)

        untagged_image = container_config.image.split(':')[0]
        def has_matching_image(container):
            return container['Image'].split(':')[0] == untagged_image

        matching = [container for container in existing if has_matching_image(container)]
        raise gen.Return(matching)

    @gen.coroutine
    def _with_retries(self, fn, *args, **kwargs):
        '''Attempt a Docker API call.

        If an error occurs, retry up to "max_tries" times before letting the exception propagate
        up the stack.'''

        max_tries = kwargs.get('max_tries', RETRIES)
        try:
            if 'max_tries' in kwargs:
                del kwargs['max_tries']
            result = yield fn(*args, **kwargs)
            raise gen.Return(result)
        except docker.errors.APIError as e:
            app_log.error("Encountered a Docker error (%i retries remain): %s", max_tries, e)
            if max_tries > 0:
                kwargs['max_tries'] = max_tries - 1
                result = yield self._with_retries(fn, *args, **kwargs)
                raise gen.Return(result)
            else:
                raise e
<|endoftext|>


<commit_before>from sqlshare_rest.util.db import get_backend
from sqlshare_rest.models import Query
from django.utils import timezone

def process_queue():
    filtered = Query.objects.filter(is_finished=False)

    try:
        oldest_query = filtered.order_by('id')[:1].get()
    except Query.DoesNotExist:
        return

    backend = get_backend()
    try:
        res = backend.run_query(oldest_query.sql, oldest_query.owner)
    except Exception as ex:
        oldest_query.has_error = True
        oldest_query.error = str(ex)

    oldest_query.is_finished = True
    oldest_query.date_finished = timezone.now()
    print "Finished: ", oldest_query.date_finished
    oldest_query.save()
<commit_msg>Remove a print statement that was dumb and breaking python3<commit_after>from sqlshare_rest.util.db import get_backend
from sqlshare_rest.models import Query
from django.utils import timezone

def process_queue():
    filtered = Query.objects.filter(is_finished=False)

    try:
        oldest_query = filtered.order_by('id')[:1].get()
    except Query.DoesNotExist:
        return

    backend = get_backend()
    try:
        res = backend.run_query(oldest_query.sql, oldest_query.owner)
    except Exception as ex:
        oldest_query.has_error = True
        oldest_query.error = str(ex)

    oldest_query.is_finished = True
    oldest_query.date_finished = timezone.now()
    oldest_query.save()
<|endoftext|>


<commit_before>import os
import sys
sys.path.insert(0, os.path.join(".."))

import unittest
from corenlp_xml.document import Document, Sentence, Token, TokenList
from collections import OrderedDict
from nltk import Tree

class TestDocument(unittest.TestCase):

    def setUp(self):
        with open("test.xml", "r") as xml_file:
            self._document = Document(xml_file.read())

    def test_sentiment(self):
        self.assertIsNone(self._document._sentiment, "Sentiment should be lazy-loaded")
        expected = 1.2173913043478262
        self.assertEquals(expected, self._document.sentiment, "Sentiment should be returned for public property")
        self.assertIsNotNone(self._document._sentiment, "Sentiment should be memoized")
        self.assertEquals(expected, self._document._sentiment, "Sentiment should be memoized")

    def test_sentences(self):
        self.assertIsNone(self._document._sentences_dict, "Sentences should be lazy-loaded")
        sentences = self._document._get_sentences_dict().values()
        self.assertIsNotNone(self._document._sentences_dict, "Sentences should be memoized")
        self.assertGreater(len(sentences), 0, "We should have sentences")
        for sentence in sentences:
            self.assertIsInstance(sentence, Sentence, "Sentences should be a list of only sentences")
        self.assertEquals(self._document.sentences, sentences, "Sentences property should work")
        self.assertIsInstance(self._document._sentences_dict, OrderedDict, "Protected sentences should be ordered")

    def test_get_sentence_by_id(self):
        sentence = self._document.get_sentence_by_id(1)
        self.assertIsInstance(sentence, Sentence, "Should return a Sentence instance")
        self.assertEquals(sentence.id, 1, "Sentence returned should have the appropriate ID")
        self.assertIsNone(self._document.get_sentence_by_id(-1), "If the ID doesn't exist, we should get None")

class TestSentence(unittest.TestCase):

    """ Tests the Sentence class """

    def setUp(self):
        """ It would probably be a good idea to look into Mock, eventually """
        with open("test.xml", "r") as xml_file:
            self._document = Document(xml_file.read())
            self._sentence = self._document.sentences[0]

    def test_id(self):
        """ Value isn't None because it's initialized when you create a Document. Hence why mocks might be nice. """
        self.assertEquals(1, self._sentence.id, "ID should be an int")
        self.assertIsNotNone(self._sentence._id, "id property should be memoized")

    def test_sentiment(self):
        self.assertIsNone(self._sentence._sentiment, "Sentiment should be lazy-loaded")
        self.assertEquals(1, self._sentence.sentiment, "Sentiment should be an int")

    def test_tokens(self):
        self.assertIsNone(self._sentence._tokens_dict, "Tokens should be lazy-loaded")
        self.assertGreater(len(self._sentence.tokens), 0, "Tokens should be generated")
        self.assertIsInstance(self._sentence.tokens, TokenList, "Tokens should be a tokenlist")
        self.assertEquals(str(self._sentence.tokens),
                          " ".join([token.word for token in self._sentence.token]),
                          "toString function of tokenlist should be words")
        for token in self._sentence.tokens:
            self.assertIsInstance(token, Token, "Tokens should all be of class Token")
        self.assertIsNotNone(self._sentence._tokens_dict, "Tokens should be memoized")
        self.assertIsInstance(self._sentence._tokens_dict, OrderedDict, "Protected tokens should be ordered")

    def test_get_token_by_id(self):
        token = self._sentence.get_token_by_id(1)
        self.assertIsInstance(token, Token, "Should return a Token instance")
        self.assertEquals(token.id, 1, "Token returned should have the appropriate ID")
        self.assertIsNone(self._sentence.get_token_by_id(-1), "If the ID doesn't exist, we should get None")

    def test_parse(self):
        self.assertIsNone(self._sentence._parse, "Parse should be lazy-loaded")
        parse = self._sentence.parse
        self.assertIsInstance(parse, Tree, "Parse should be an nltk.Tree instance")
        self.assertIsInstance(self._sentence._parse, Tree, "Parse should be memoized")

class TestToken(unittest.TestCase):

    """ Tests the Token class """
    def setUp(self):
        """ It would probably be a good idea to look into Mock, eventually """
        with open("test.xml", "r") as xml_file:
            self._document = Document(xml_file.read())
            self._sentence = self._document.sentences[0]
            self._token = self._sentence.tokens[0]

    def test_id(self):
        """ Value isn't None because it's initialized when getting tokens from sent. Hence why mocks might be nice. """
        self.assertIsNotNone(self._token._id, "id property should be memoized")
        self.assertEquals(1, self._token.id)

    def test_word(self):
        self.assertIsNone(self._token._word, "Word should be lazy-loaded")
        # BREAKAGE WARNING: depends on the current state of xml file test.xml
        self.assertEquals("Taking", self._token.word, "Word should be string value of word")
        self.assertIsNotNone(self._token._word, "Word property should be memoized")

    def test_lemma(self):
        self.assertIsNone(self._token._lemma, "Lemma should be lazy-loaded")
        # BREAKAGE WARNING: depends on the current state of xml file test.xml
        self.assertEquals("take", self._token.lemma, "Word should be string value of word")
        self.assertIsNotNone(self._token._lemma, "Lemma property should be memoized")

    def test_character_offset_begin(self):
        self.assertIsNone(self._token._character_offset_begin, "Character offset being should be lazy-loaded")
        self.assertIsNotNone(self._token.character_offset_begin, "Character offset should be accessible")
        self.assertIsNotNone(self._token._character_offset_begin, "Character offset property should be memoized")

    def test_character_offset_end(self):
        self.assertIsNone(self._token._character_offset_end, "Character offset being should be lazy-loaded")
        self.assertIsNotNone(self._token.character_offset_end, "Character offset should be accessible")
        self.assertIsNotNone(self._token._character_offset_end, "Character offset property should be memoized")

    def test_pos(self):
        self.assertIsNone(self._token._pos, "POS should be lazy-loaded")
        # BREAKAGE WARNING: depends on the current state of xml file test.xml
        self.assertEquals("VBG", self._token.pos, "POS should be a POS, yo")
        self.assertIsNotNone(self._token._pos, "POS property should be memoized")

    def test_ner(self):
        self.assertIsNone(self._token._ner, "NER should be lazy-loaded")
        # BREAKAGE WARNING: depends on the current state of xml file test.xml
        self.assertEquals("O", self._token.ner, "NER should be a NER, yo")
        self.assertIsNotNone(self._token._ner, "NER property should be memoized")

    def test_speaker(self):
        self.assertIsNone(self._token._speaker, "Speaker should be lazy-loaded")
        # BREAKAGE WARNING: depends on the current state of xml file test.xml
        self.assertEquals("PER0", self._token.speaker, "Speaker should be a speaker son")
        self.assertIsNotNone(self._token._speaker, "speaker property should be memoized")

def suite():
    """
    Generates test suite
    """
    test_suite = unittest.TestSuite()
    test_suite.addTest(unittest.TestLoader().loadTestsFromTestCase(TestDocument))
    test_suite.addTest(unittest.TestLoader().loadTestsFromTestCase(TestSentence))
    test_suite.addTest(unittest.TestLoader().loadTestsFromTestCase(TestToken))
    return test_suite

if __name__ == "__main__":
    unittest.TextTestRunner(verbosity=1).run(suite())<commit_msg>fix broken test<commit_after>import os
import sys
sys.path.insert(0, os.path.join(".."))

import unittest
from corenlp_xml.document import Document, Sentence, Token, TokenList
from collections import OrderedDict
from nltk import Tree

class TestDocument(unittest.TestCase):

    def setUp(self):
        with open("test.xml", "r") as xml_file:
            self._document = Document(xml_file.read())

    def test_sentiment(self):
        self.assertIsNone(self._document._sentiment, "Sentiment should be lazy-loaded")
        expected = 1.2173913043478262
        self.assertEquals(expected, self._document.sentiment, "Sentiment should be returned for public property")
        self.assertIsNotNone(self._document._sentiment, "Sentiment should be memoized")
        self.assertEquals(expected, self._document._sentiment, "Sentiment should be memoized")

    def test_sentences(self):
        self.assertIsNone(self._document._sentences_dict, "Sentences should be lazy-loaded")
        sentences = self._document._get_sentences_dict().values()
        self.assertIsNotNone(self._document._sentences_dict, "Sentences should be memoized")
        self.assertGreater(len(sentences), 0, "We should have sentences")
        for sentence in sentences:
            self.assertIsInstance(sentence, Sentence, "Sentences should be a list of only sentences")
        self.assertEquals(self._document.sentences, sentences, "Sentences property should work")
        self.assertIsInstance(self._document._sentences_dict, OrderedDict, "Protected sentences should be ordered")

    def test_get_sentence_by_id(self):
        sentence = self._document.get_sentence_by_id(1)
        self.assertIsInstance(sentence, Sentence, "Should return a Sentence instance")
        self.assertEquals(sentence.id, 1, "Sentence returned should have the appropriate ID")
        self.assertIsNone(self._document.get_sentence_by_id(-1), "If the ID doesn't exist, we should get None")

class TestSentence(unittest.TestCase):

    """ Tests the Sentence class """

    def setUp(self):
        """ It would probably be a good idea to look into Mock, eventually """
        with open("test.xml", "r") as xml_file:
            self._document = Document(xml_file.read())
            self._sentence = self._document.sentences[0]

    def test_id(self):
        """ Value isn't None because it's initialized when you create a Document. Hence why mocks might be nice. """
        self.assertEquals(1, self._sentence.id, "ID should be an int")
        self.assertIsNotNone(self._sentence._id, "id property should be memoized")

    def test_sentiment(self):
        self.assertIsNone(self._sentence._sentiment, "Sentiment should be lazy-loaded")
        self.assertEquals(1, self._sentence.sentiment, "Sentiment should be an int")

    def test_tokens(self):
        self.assertIsNone(self._sentence._tokens_dict, "Tokens should be lazy-loaded")
        self.assertGreater(len(self._sentence.tokens), 0, "Tokens should be generated")
        self.assertIsInstance(self._sentence.tokens, TokenList, "Tokens should be a tokenlist")
        self.assertEquals(str(self._sentence.tokens),
                          " ".join([token.word for token in self._sentence.tokens]),
                          "toString function of tokenlist should be words")
        for token in self._sentence.tokens:
            self.assertIsInstance(token, Token, "Tokens should all be of class Token")
        self.assertIsNotNone(self._sentence._tokens_dict, "Tokens should be memoized")
        self.assertIsInstance(self._sentence._tokens_dict, OrderedDict, "Protected tokens should be ordered")

    def test_get_token_by_id(self):
        token = self._sentence.get_token_by_id(1)
        self.assertIsInstance(token, Token, "Should return a Token instance")
        self.assertEquals(token.id, 1, "Token returned should have the appropriate ID")
        self.assertIsNone(self._sentence.get_token_by_id(-1), "If the ID doesn't exist, we should get None")

    def test_parse(self):
        self.assertIsNone(self._sentence._parse, "Parse should be lazy-loaded")
        parse = self._sentence.parse
        self.assertIsInstance(parse, Tree, "Parse should be an nltk.Tree instance")
        self.assertIsInstance(self._sentence._parse, Tree, "Parse should be memoized")

class TestToken(unittest.TestCase):

    """ Tests the Token class """
    def setUp(self):
        """ It would probably be a good idea to look into Mock, eventually """
        with open("test.xml", "r") as xml_file:
            self._document = Document(xml_file.read())
            self._sentence = self._document.sentences[0]
            self._token = self._sentence.tokens[0]

    def test_id(self):
        """ Value isn't None because it's initialized when getting tokens from sent. Hence why mocks might be nice. """
        self.assertIsNotNone(self._token._id, "id property should be memoized")
        self.assertEquals(1, self._token.id)

    def test_word(self):
        self.assertIsNone(self._token._word, "Word should be lazy-loaded")
        # BREAKAGE WARNING: depends on the current state of xml file test.xml
        self.assertEquals("Taking", self._token.word, "Word should be string value of word")
        self.assertIsNotNone(self._token._word, "Word property should be memoized")

    def test_lemma(self):
        self.assertIsNone(self._token._lemma, "Lemma should be lazy-loaded")
        # BREAKAGE WARNING: depends on the current state of xml file test.xml
        self.assertEquals("take", self._token.lemma, "Word should be string value of word")
        self.assertIsNotNone(self._token._lemma, "Lemma property should be memoized")

    def test_character_offset_begin(self):
        self.assertIsNone(self._token._character_offset_begin, "Character offset being should be lazy-loaded")
        self.assertIsNotNone(self._token.character_offset_begin, "Character offset should be accessible")
        self.assertIsNotNone(self._token._character_offset_begin, "Character offset property should be memoized")

    def test_character_offset_end(self):
        self.assertIsNone(self._token._character_offset_end, "Character offset being should be lazy-loaded")
        self.assertIsNotNone(self._token.character_offset_end, "Character offset should be accessible")
        self.assertIsNotNone(self._token._character_offset_end, "Character offset property should be memoized")

    def test_pos(self):
        self.assertIsNone(self._token._pos, "POS should be lazy-loaded")
        # BREAKAGE WARNING: depends on the current state of xml file test.xml
        self.assertEquals("VBG", self._token.pos, "POS should be a POS, yo")
        self.assertIsNotNone(self._token._pos, "POS property should be memoized")

    def test_ner(self):
        self.assertIsNone(self._token._ner, "NER should be lazy-loaded")
        # BREAKAGE WARNING: depends on the current state of xml file test.xml
        self.assertEquals("O", self._token.ner, "NER should be a NER, yo")
        self.assertIsNotNone(self._token._ner, "NER property should be memoized")

    def test_speaker(self):
        self.assertIsNone(self._token._speaker, "Speaker should be lazy-loaded")
        # BREAKAGE WARNING: depends on the current state of xml file test.xml
        self.assertEquals("PER0", self._token.speaker, "Speaker should be a speaker son")
        self.assertIsNotNone(self._token._speaker, "speaker property should be memoized")

def suite():
    """
    Generates test suite
    """
    test_suite = unittest.TestSuite()
    test_suite.addTest(unittest.TestLoader().loadTestsFromTestCase(TestDocument))
    test_suite.addTest(unittest.TestLoader().loadTestsFromTestCase(TestSentence))
    test_suite.addTest(unittest.TestLoader().loadTestsFromTestCase(TestToken))
    return test_suite

if __name__ == "__main__":
    unittest.TextTestRunner(verbosity=1).run(suite())<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Bot to import and source statements about Surface Water Bodies.

Author: Lokal_Profil
License: MIT

usage:
    python swb_import.py [OPTIONS]

&params;
"""
from __future__ import unicode_literals
from builtins import dict

import pywikibot

import wikidataStuff.helpers as helpers
from wikidataStuff.WikidataStuff import WikidataStuff as WdS

from WFD.WFDBase import WfdBot, UnmappedValueError

parameter_help = """\
SwbBot options (may be omitted):
-year              Year to which the WFD data applies.
-new               if present new items are created on Wikidata, otherwise
                   only updates are processed.
-in_file           path to the data file
-mappings          path to the mappings file (if not "mappings.json")
-cutoff            number items to process before stopping (if not then all)

Can also handle any pywikibot options. Most importantly:
-simulate          don't write to database
-dir               directory in which user_config is located
-help              output all available options
"""
docuReplacements = {'&params;': parameter_help}
EDIT_SUMMARY = 'importing #SWB using data from #WFD'

class SwbBot(WfdBot):
    """Bot to enrich/create info on Wikidata for SWB objects."""

    def __init__(self, mappings, year, new=False, cutoff=None):
        """
        Initialise the SwbBot.

        :param mappings: dict holding data for any offline mappings
        :param year: year of the report (used to date certain statements).
        :param new: whether to also create new items
        :param cutoff: the number of items to process before stopping. None
            being interpreted as all.
        """
        super(SwbBot, self).__init__(mappings, year, new, cutoff,
                                     EDIT_SUMMARY)

        self.eu_swb_p = 'P2856'  # eu_cd
        self.eu_rbd_p = 'P2965'  # euRBDCode

        self.swb_cats = mappings.get('surfaceWaterBodyCategory')
        self.impact_types = mappings.get('swSignificantImpactType')

        self.load_known_items()

    def load_known_items(self):
        """
        Load existing eu_swb and eu_rbd items.

        The loaded values are Qids (without q-prefixes)
        """
        self.swb_items = helpers.fill_cache_wdqs(self.eu_swb_p)
        self.rbd_items = helpers.fill_cache_wdqs(self.eu_rbd_p)

    def set_common_values(self, data):
        """
        Set and validate values shared by every SWB in the dataset.

        :param data: dict of all the SWB:s in the RBD
        """
        super(SwbBot, self).set_common_values(data)
        try:
            rbd_q = self.rbd_items.get[data.get('euRBDCode')]
            self.rbd = self.wd.QtoItemPage(rbd_q)
        except KeyError:
            raise UnmappedValueError('online rbd objects',
                                     data.get('euRBDCode'))

        self.descriptions = self.mappings.get('descriptions').get('SWB')
        WfdBot.validate_mapping(self.descriptions, self.langs, 'descriptions')

    def process_all_swb(self, data):
        """
        Handle all the surface water bodies in a single RBD.

        Only increments counter when an swb is updated.

        :param data: dict of all the swb:s in the RBD
        """
        count = 0
        for entry_data in data:
            if self.cutoff and count >= self.cutoff:
                break
            eu_swb_code = entry_data.get('euSurfaceWaterBodyCode')
            item = None
            if eu_swb_code in self.swb_items:
                item = self.wd.QtoItemPage(self.swb_items[eu_swb_code])

            if item or self.new:
                self.process_single_swb(entry_data, item)
                count += 1

    def process_single_swb(self, data, item):
        """
        Process a swb (whether item exists or not).

        :param data: dict of data for a single swb
        :param item: Wikidata item associated with a swb, or None if one
            should be created.
        """
        item = item or self.create_new_swb_item(data)
        item.exists()  # load the item contents

        # Determine claims
        name = data.get('surfaceWaterBodyName')  # in English per page 33
        protoclaims = self.make_protoclaims(data)

        # Upload claims
        self.wd.addLabelOrAlias('en', name, item)
        self.commit_claims(protoclaims, item)

    def create_new_swb_item(self, data):
        """
        Create a new swb item with some basic info and return.

        The new item is created with a label, descriptions and
        an eu_swb_code claim. This allows the item to be found in a second
        pass should the script crash before adding the protoclaims.

        The eu_swb_code is added without reference (this is handled
        during the protoclaim stage)
        The added label is in english per p.33 of the WFD specifications.
        The protoclaim stage should recognise that the label has already been
        added and thus not add it again.

        :param data: dict of data for a single swb
        :return: pywikibot.ItemPage
        """
        name = data.get('surfaceWaterBodyName')
        desc = self.make_descriptions(self.descriptions)
        claim = self.wd.make_simple_claim(
            self.eu_swb_p, data.get('euSurfaceWaterBodyCode'))

        item_data = {
            "labels": {
                "en": {
                    "language": "en",
                    "value": name
                }
            },
            "descriptions": desc,
            "claims": [claim.toJSON(), ]
        }

        return self.wd.make_new_item(item_data, EDIT_SUMMARY)

    def make_protoclaims(self, data):
        """
        Construct potential claims for an entry.

        :param data: dict of data for a single swb
        """
        protoclaims = dict()

        # P31: surfaceWaterBodyCategory
        swb_cat = self.swb_cats.get(data.get('surfaceWaterBodyCategory'))
        protoclaims['P31'] = WdS.Statement(self.wd.QtoItemPage(swb_cat))

        # self.eu_swb_p: euSurfaceWaterBodyCode
        protoclaims[self.eu_swb_p] = WdS.Statement(
            data.get('euSurfaceWaterBodyCode'))

        # P17: country
        protoclaims['P17'] = WdS.Statement(self.country)

        # P361: parent RBD
        protoclaims['P361'] = WdS.Statement(self.rbd)

        # P3643: swSignificantImpactType
        protoclaims['P3643'] = self.make_significant_impact_type(data)

        return protoclaims

    def make_significant_impact_type(self, data):
        """
        Construct statements for swSignificantImpactType data.

        Uses self.year as timepoint.

        Sets the 'novalue' statement if there are no impact types in the data.

        @todo: This needs to be made aware of pre-existing claim to be able to
               update the data. Specifically:
               * If a claim was previously present but is not any more:
                 add an end time as qualifier to that value.
               * If a claim was previously present and still is:
                 no change needed (but we should possibly us start time instead
                 of time point?
               * In both cases the reference can still be added.

        :param data: dict of data for a single swb
        """
        claims = []
        for impact in data.get('swSignificantImpactType'):
            impact = impact.split(' - ')[0]
            impact_q = self.impact_types.get(impact)
            claims.append(
                WdS.Statement(self.wd.QtoItemPage(impact_q)).addQualifier(
                    WdS.Qualifier('P585', self.year)))
        if not claims:
            # none were added for this year
            claims.append(
                WdS.Statement('novalue', special=True).addQualifier(
                    WdS.Qualifier('P585', self.year)))
        return claims

    @staticmethod
    def main(*args):
        """Command line entry point."""
        mappings = 'mappings.json'
        force_path = __file__
        in_file = None
        new = False
        cutoff = None
        year = '2016'

        # Load pywikibot args and handle local args
        for arg in pywikibot.handle_args(args):
            option, sep, value = arg.partition(':')
            if option == '-in_file':
                in_file = value
            elif option == '-mappings':
                mappings = value
                force_path = None
            elif option == '-new':
                new = True
            elif option == '-year':
                year = value
            elif option == '-cutoff':
                cutoff = int(value)

        # require in_file
        if not in_file:
            raise pywikibot.Error('An in_file must be specified')

        # load and validate data and mappings
        mappings = helpers.load_json_file(mappings, force_path)
        data = WfdBot.load_data(in_file, key='SWB')
        validate_indata(data, mappings)

        # initialise SwbBot object
        bot = SwbBot(mappings, year, new=new, cutoff=cutoff)
        bot.set_common_values(data)

        bot.process_all_swb(data)

def validate_indata(data, mappings):
    """
    Validate that all encountered values needing to be mapped are.

    :param data: the source data from the xml
    :param mappings: dict holding data for any offline mappings
    """
    # validate mapping of each <surfaceWaterBodyCategory> and
    # <swSignificantImpactType>
    swb_cats = set()
    impacts = set()
    for swb in data.get('SurfaceWaterBody'):
        swb_cats.add(swb.get('surfaceWaterBodyCategory'))
        impacts |= set(helpers.listify(swb.get('swSignificantImpactType')))
    impacts = [impact.split(' - ')[0] for impact in impacts]

    WfdBot.validate_mapping(
        mappings.get('surfaceWaterBodyCategory'), swb_cats,
        'surfaceWaterBodyCategory')
    WfdBot.validate_mapping(
        mappings.get('swSignificantImpactType'), impacts,
        'swSignificantImpactType')

if __name__ == "__main__":
    SwbBot.main()
<commit_msg>Do not use non-names for Label or Alias (#7)<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
Bot to import and source statements about Surface Water Bodies.

Author: Lokal_Profil
License: MIT

usage:
    python swb_import.py [OPTIONS]

&params;
"""
from __future__ import unicode_literals
from builtins import dict

import pywikibot

import wikidataStuff.helpers as helpers
from wikidataStuff.WikidataStuff import WikidataStuff as WdS

from WFD.WFDBase import WfdBot, UnmappedValueError

parameter_help = """\
SwbBot options (may be omitted):
-year              Year to which the WFD data applies.
-new               if present new items are created on Wikidata, otherwise
                   only updates are processed.
-in_file           path to the data file
-mappings          path to the mappings file (if not "mappings.json")
-cutoff            number items to process before stopping (if not then all)

Can also handle any pywikibot options. Most importantly:
-simulate          don't write to database
-dir               directory in which user_config is located
-help              output all available options
"""
docuReplacements = {'&params;': parameter_help}
EDIT_SUMMARY = 'importing #SWB using data from #WFD'

class SwbBot(WfdBot):
    """Bot to enrich/create info on Wikidata for SWB objects."""

    def __init__(self, mappings, year, new=False, cutoff=None):
        """
        Initialise the SwbBot.

        :param mappings: dict holding data for any offline mappings
        :param year: year of the report (used to date certain statements).
        :param new: whether to also create new items
        :param cutoff: the number of items to process before stopping. None
            being interpreted as all.
        """
        super(SwbBot, self).__init__(mappings, year, new, cutoff,
                                     EDIT_SUMMARY)

        self.eu_swb_p = 'P2856'  # eu_cd
        self.eu_rbd_p = 'P2965'  # euRBDCode

        self.swb_cats = mappings.get('surfaceWaterBodyCategory')
        self.impact_types = mappings.get('swSignificantImpactType')

        self.load_known_items()

    def load_known_items(self):
        """
        Load existing eu_swb and eu_rbd items.

        The loaded values are Qids (without q-prefixes)
        """
        self.swb_items = helpers.fill_cache_wdqs(self.eu_swb_p)
        self.rbd_items = helpers.fill_cache_wdqs(self.eu_rbd_p)

    def set_common_values(self, data):
        """
        Set and validate values shared by every SWB in the dataset.

        :param data: dict of all the SWB:s in the RBD
        """
        super(SwbBot, self).set_common_values(data)
        try:
            rbd_q = self.rbd_items.get[data.get('euRBDCode')]
            self.rbd = self.wd.QtoItemPage(rbd_q)
        except KeyError:
            raise UnmappedValueError('online rbd objects',
                                     data.get('euRBDCode'))

        self.descriptions = self.mappings.get('descriptions').get('SWB')
        WfdBot.validate_mapping(self.descriptions, self.langs, 'descriptions')

    def process_all_swb(self, data):
        """
        Handle all the surface water bodies in a single RBD.

        Only increments counter when an swb is updated.

        :param data: dict of all the swb:s in the RBD
        """
        count = 0
        for entry_data in data:
            if self.cutoff and count >= self.cutoff:
                break
            eu_swb_code = entry_data.get('euSurfaceWaterBodyCode')
            item = None
            if eu_swb_code in self.swb_items:
                item = self.wd.QtoItemPage(self.swb_items[eu_swb_code])

            if item or self.new:
                self.process_single_swb(entry_data, item)
                count += 1

    def process_single_swb(self, data, item):
        """
        Process a swb (whether item exists or not).

        :param data: dict of data for a single swb
        :param item: Wikidata item associated with a swb, or None if one
            should be created.
        """
        bad_names = ('not applicable', )  # known (lower case) non-names
        item = item or self.create_new_swb_item(data)
        item.exists()  # load the item contents

        # Determine claims
        name = data.get('surfaceWaterBodyName')  # in English per page 33
        if name and name.lower() in bad_names:
            name = None
        protoclaims = self.make_protoclaims(data)

        # Upload claims
        if name:
            self.wd.addLabelOrAlias('en', name, item)
        self.commit_claims(protoclaims, item)

    def create_new_swb_item(self, data):
        """
        Create a new swb item with some basic info and return.

        The new item is created with a label, descriptions and
        an eu_swb_code claim. This allows the item to be found in a second
        pass should the script crash before adding the protoclaims.

        The eu_swb_code is added without reference (this is handled
        during the protoclaim stage)
        The added label is in english per p.33 of the WFD specifications.
        The protoclaim stage should recognise that the label has already been
        added and thus not add it again.

        :param data: dict of data for a single swb
        :return: pywikibot.ItemPage
        """
        name = data.get('surfaceWaterBodyName')
        desc = self.make_descriptions(self.descriptions)
        claim = self.wd.make_simple_claim(
            self.eu_swb_p, data.get('euSurfaceWaterBodyCode'))

        item_data = {
            "labels": {
                "en": {
                    "language": "en",
                    "value": name
                }
            },
            "descriptions": desc,
            "claims": [claim.toJSON(), ]
        }

        return self.wd.make_new_item(item_data, EDIT_SUMMARY)

    def make_protoclaims(self, data):
        """
        Construct potential claims for an entry.

        :param data: dict of data for a single swb
        """
        protoclaims = dict()

        # P31: surfaceWaterBodyCategory
        swb_cat = self.swb_cats.get(data.get('surfaceWaterBodyCategory'))
        protoclaims['P31'] = WdS.Statement(self.wd.QtoItemPage(swb_cat))

        # self.eu_swb_p: euSurfaceWaterBodyCode
        protoclaims[self.eu_swb_p] = WdS.Statement(
            data.get('euSurfaceWaterBodyCode'))

        # P17: country
        protoclaims['P17'] = WdS.Statement(self.country)

        # P361: parent RBD
        protoclaims['P361'] = WdS.Statement(self.rbd)

        # P3643: swSignificantImpactType
        protoclaims['P3643'] = self.make_significant_impact_type(data)

        return protoclaims

    def make_significant_impact_type(self, data):
        """
        Construct statements for swSignificantImpactType data.

        Uses self.year as timepoint.

        Sets the 'novalue' statement if there are no impact types in the data.

        @todo: This needs to be made aware of pre-existing claim to be able to
               update the data. Specifically:
               * If a claim was previously present but is not any more:
                 add an end time as qualifier to that value.
               * If a claim was previously present and still is:
                 no change needed (but we should possibly us start time instead
                 of time point?
               * In both cases the reference can still be added.

        :param data: dict of data for a single swb
        """
        claims = []
        for impact in data.get('swSignificantImpactType'):
            impact = impact.split(' - ')[0]
            impact_q = self.impact_types.get(impact)
            claims.append(
                WdS.Statement(self.wd.QtoItemPage(impact_q)).addQualifier(
                    WdS.Qualifier('P585', self.year)))
        if not claims:
            # none were added for this year
            claims.append(
                WdS.Statement('novalue', special=True).addQualifier(
                    WdS.Qualifier('P585', self.year)))
        return claims

    @staticmethod
    def main(*args):
        """Command line entry point."""
        mappings = 'mappings.json'
        force_path = __file__
        in_file = None
        new = False
        cutoff = None
        year = '2016'

        # Load pywikibot args and handle local args
        for arg in pywikibot.handle_args(args):
            option, sep, value = arg.partition(':')
            if option == '-in_file':
                in_file = value
            elif option == '-mappings':
                mappings = value
                force_path = None
            elif option == '-new':
                new = True
            elif option == '-year':
                year = value
            elif option == '-cutoff':
                cutoff = int(value)

        # require in_file
        if not in_file:
            raise pywikibot.Error('An in_file must be specified')

        # load and validate data and mappings
        mappings = helpers.load_json_file(mappings, force_path)
        data = WfdBot.load_data(in_file, key='SWB')
        validate_indata(data, mappings)

        # initialise SwbBot object
        bot = SwbBot(mappings, year, new=new, cutoff=cutoff)
        bot.set_common_values(data)

        bot.process_all_swb(data)

def validate_indata(data, mappings):
    """
    Validate that all encountered values needing to be mapped are.

    :param data: the source data from the xml
    :param mappings: dict holding data for any offline mappings
    """
    # validate mapping of each <surfaceWaterBodyCategory> and
    # <swSignificantImpactType>
    swb_cats = set()
    impacts = set()
    for swb in data.get('SurfaceWaterBody'):
        swb_cats.add(swb.get('surfaceWaterBodyCategory'))
        impacts |= set(helpers.listify(swb.get('swSignificantImpactType')))
    impacts = [impact.split(' - ')[0] for impact in impacts]

    WfdBot.validate_mapping(
        mappings.get('surfaceWaterBodyCategory'), swb_cats,
        'surfaceWaterBodyCategory')
    WfdBot.validate_mapping(
        mappings.get('swSignificantImpactType'), impacts,
        'swSignificantImpactType')

if __name__ == "__main__":
    SwbBot.main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# pylint: disable=unused-variable
"""These test the natcmp() function.

Note that these tests are only relevant for Python version < 3.
"""
import sys
from functools import partial
from mock import patch

import pytest
from hypothesis import given
from hypothesis.strategies import floats, integers, lists

from natsort import ns

from natsort.compat.py23 import py23_cmp

PY_VERSION = float(sys.version[:3])

if PY_VERSION < 3:
    from natsort import natcmp

class Comparable(object):
    """Stub class for testing natcmp functionality."""
    def __init__(self, value):
        self.value = value

    def __cmp__(self, other):
        return natcmp(self.value, other.value)

@pytest.mark.skipif(PY_VERSION >= 3.0, reason='cmp() deprecated in Python 3')
def test__classes_can_be_compared():
    one = Comparable("1")
    two = Comparable("2")
    another_two = Comparable("2")
    ten = Comparable("10")

    assert ten > two == another_two > one

@pytest.mark.skipif(PY_VERSION >= 3.0, reason='cmp() deprecated in Python 3')
def test__keys_are_being_cached():
    natcmp.cached_keys = {}
    assert len(natcmp.cached_keys) == 0
    natcmp(0, 0)
    assert len(natcmp.cached_keys) == 1
    natcmp(0, 0)
    assert len(natcmp.cached_keys) == 1
    natcmp(0, 0, alg=ns.L)
    assert len(natcmp.cached_keys) == 2

    natcmp(0, 0, alg=ns.L)
    assert len(natcmp.cached_keys) == 2

    with patch('natsort.compat.locale.dumb_sort', return_value=True):
        natcmp(0, 0, alg=ns.L)

    assert len(natcmp.cached_keys) == 3

    with patch('natsort.compat.locale.dumb_sort', return_value=True):
        natcmp(0, 0, alg=ns.L)

    assert len(natcmp.cached_keys) == 3

@pytest.mark.skipif(PY_VERSION >= 3.0, reason='cmp() deprecated in Python 3')
def test__illegal_algorithm_raises_error():
    try:
        natcmp(0, 0, alg="Just random stuff")
        assert False

    except ValueError:
        assert True

    except Exception:
        assert False

@pytest.mark.skipif(PY_VERSION >= 3.0, reason='cmp() deprecated in Python 3')
def test__classes_can_utilize_max_or_min():
    comparables = [Comparable(i) for i in range(10)]

    assert max(comparables) == comparables[-1]
    assert min(comparables) == comparables[0]

@pytest.mark.skipif(PY_VERSION >= 3.0, reason='cmp() deprecated in Python 3')
@given(integers(), integers())
def test__natcmp_works_the_same_for_integers_as_cmp(x, y):
    assert py23_cmp(x, y) == natcmp(x, y)

@pytest.mark.skipif(PY_VERSION >= 3.0, reason='cmp() deprecated in Python 3')
@given(floats(allow_nan=False), floats(allow_nan=False))
def test__natcmp_works_the_same_for_floats_as_cmp(x, y):
    assert py23_cmp(x, y) == natcmp(x, y)

@pytest.mark.skipif(PY_VERSION >= 3.0, reason='cmp() deprecated in Python 3')
@given(lists(elements=integers()))
def test_sort_strings_with_numbers(a_list):
    strings = [str(var) for var in a_list]
    natcmp_sorted = sorted(strings, cmp=partial(natcmp, alg=ns.REAL))

    assert sorted(a_list) == [int(var) for var in natcmp_sorted]
<commit_msg>Fixed patching for Python > 3<commit_after># -*- coding: utf-8 -*-
# pylint: disable=unused-variable
"""These test the natcmp() function.

Note that these tests are only relevant for Python version < 3.
"""
import sys
from functools import partial
from compat.mock import patch

import pytest
from hypothesis import given
from hypothesis.strategies import floats, integers, lists

from natsort import ns

from natsort.compat.py23 import py23_cmp

PY_VERSION = float(sys.version[:3])

if PY_VERSION < 3:
    from natsort import natcmp

class Comparable(object):
    """Stub class for testing natcmp functionality."""
    def __init__(self, value):
        self.value = value

    def __cmp__(self, other):
        return natcmp(self.value, other.value)

@pytest.mark.skipif(PY_VERSION >= 3.0, reason='cmp() deprecated in Python 3')
def test__classes_can_be_compared():
    one = Comparable("1")
    two = Comparable("2")
    another_two = Comparable("2")
    ten = Comparable("10")

    assert ten > two == another_two > one

@pytest.mark.skipif(PY_VERSION >= 3.0, reason='cmp() deprecated in Python 3')
def test__keys_are_being_cached():
    natcmp.cached_keys = {}
    assert len(natcmp.cached_keys) == 0
    natcmp(0, 0)
    assert len(natcmp.cached_keys) == 1
    natcmp(0, 0)
    assert len(natcmp.cached_keys) == 1
    natcmp(0, 0, alg=ns.L)
    assert len(natcmp.cached_keys) == 2

    natcmp(0, 0, alg=ns.L)
    assert len(natcmp.cached_keys) == 2

    with patch('natsort.compat.locale.dumb_sort', return_value=True):
        natcmp(0, 0, alg=ns.L)

    assert len(natcmp.cached_keys) == 3

    with patch('natsort.compat.locale.dumb_sort', return_value=True):
        natcmp(0, 0, alg=ns.L)

    assert len(natcmp.cached_keys) == 3

@pytest.mark.skipif(PY_VERSION >= 3.0, reason='cmp() deprecated in Python 3')
def test__illegal_algorithm_raises_error():
    try:
        natcmp(0, 0, alg="Just random stuff")
        assert False

    except ValueError:
        assert True

    except Exception:
        assert False

@pytest.mark.skipif(PY_VERSION >= 3.0, reason='cmp() deprecated in Python 3')
def test__classes_can_utilize_max_or_min():
    comparables = [Comparable(i) for i in range(10)]

    assert max(comparables) == comparables[-1]
    assert min(comparables) == comparables[0]

@pytest.mark.skipif(PY_VERSION >= 3.0, reason='cmp() deprecated in Python 3')
@given(integers(), integers())
def test__natcmp_works_the_same_for_integers_as_cmp(x, y):
    assert py23_cmp(x, y) == natcmp(x, y)

@pytest.mark.skipif(PY_VERSION >= 3.0, reason='cmp() deprecated in Python 3')
@given(floats(allow_nan=False), floats(allow_nan=False))
def test__natcmp_works_the_same_for_floats_as_cmp(x, y):
    assert py23_cmp(x, y) == natcmp(x, y)

@pytest.mark.skipif(PY_VERSION >= 3.0, reason='cmp() deprecated in Python 3')
@given(lists(elements=integers()))
def test_sort_strings_with_numbers(a_list):
    strings = [str(var) for var in a_list]
    natcmp_sorted = sorted(strings, cmp=partial(natcmp, alg=ns.REAL))

    assert sorted(a_list) == [int(var) for var in natcmp_sorted]
<|endoftext|>


<commit_before>"""Collection of tools that can be used while testing the laserchicken software."""
import datetime as dt

import numpy as np

from laserchicken import keys

def generate_tiny_test_point_cloud():
    """Generate a simple but valid point cloud with 3 points."""
    pc = {keys.point: {'x': {'type': 'double', 'data': np.array([1, 2, 3], dtype=np.float64)},
                       'y': {'type': 'double', 'data': np.array([2, 3, 4], dtype=np.float64)},
                       'z': {'type': 'double', 'data': np.array([3, 4, 5], dtype=np.float64)}}}
    return pc

class SimpleTestData(object):
    """Test data within this class should all be in sync (reflect the same data)."""

    @staticmethod
    def get_point_cloud():
        """Get the point cloud data."""
        # This simple_test_point cloud and the simple_test_header should be in sync. Some tests depend on it.
        pc = {keys.point: {'x': {'type': 'float', 'data': np.array([1, 2, 3])},
                           'y': {'type': 'float', 'data': np.array([20, 30, 40])},
                           'z': {'type': 'float', 'data': np.array([300, 400, 500])}}}
        return pc

    @staticmethod
    def get_header():
        """Get the ply header."""
        # This simple_test_header cloud and the simple_test_point should be in sync. Some tests depend on it.
        header = """ply
format ascii 1.0
element vertex 3
property float x
property float y
property float z
"""
        return header

    @staticmethod
    def get_data():
        """Get the data in ply format."""
        data = """1 20 300
2 30 400
3 40 500
"""
        return data

class ComplexTestData(object):
    """Test data within this class should all be in sync (reflect the same data)."""

    comments = [{"module": "filter", "time": str(dt.datetime(2018, 1, 18, 16, 1, 0))},
                {"module": "filter", "time": str(dt.datetime(2018, 2, 4, 13, 11, 0))}]

    def get_point_cloud(self):
        """Get the point cloud data."""
        # This complex_test_point cloud and the complex_test_header should be in sync. Some tests depend on it.
        pc = {keys.point: {'x': {'type': 'float', 'data': np.array([1, 2, 3, 4, 5], dtype=np.float)},
                           'y': {'type': 'float', 'data': np.array([2, 3, 4, 5, 6], dtype=np.float)},
                           'z': {'type': 'float', 'data': np.array([3, 4, 5, 6, 7], dtype=np.float)},
                           'return': {'type': 'int', 'data': np.array([1, 1, 2, 2, 1], dtype=np.int)}
                           },
              keys.point_cloud: {'offset': {'type': 'double', 'data': 12.1}},
              keys.provenance: self.comments
              }
        return pc

    def get_header(self):
        """Get the ply header."""
        # This complex_test_header cloud and the complex_test_point should be in sync. Some tests depend on it.
        header = ("""ply
format ascii 1.0
comment [
comment {},
comment {}
comment ]
element vertex 5
property float x
property float y
property float z
property int return
element pointcloud 1
property double offset
""").format(self.comments[0], self.comments[1])
        return header

    @staticmethod
    def get_data():
        """Get the data in ply format."""
        data = """1 2 3 1
2 3 4 1
3 4 5 2
4 5 6 2
5 6 7 1
12.1
"""
        return data

    @staticmethod
    def get_wkt_polygon_around_first_point_only():
        """Get a polygon in wkt format that surrounds only the first point."""
        return "POLYGON(( 1.5 10.0, 1.5 -10.0, -1.5 -10.0, -1.5 1.0, 1.5 10.0 ))"

def create_point_cloud(x, y, z, normalized_z=None):
    """
    Create a point cloud object given only the x y z values.

    :param x: x attribute values
    :param y: y attribute values
    :param z: z attribute values
    :param normalized_z: optional normalized z attribute values
    :return: point cloud object
    """
    point_cloud = {keys.point: {'x': {'type': 'float', 'data': np.array(x)}, 'y': {'type': 'float', 'data': np.array(y)},
                            'z': {'type': 'float', 'data': np.array(z)}}, keys.point_cloud: {},
               keys.provenance: [{'time': (dt.datetime(2018, 1, 18, 16, 1, 0)), 'module': 'filter'}]}
    if normalized_z is not None:
        point_cloud[keys.point][keys.normalized_height] = {'type': 'float', 'data': np.array(normalized_z)}
    return point_cloud

def create_points_in_xy_grid(z_function):
    """Create 100 points in a grid of 10 by 10."""
    n_points = 100
    points = np.zeros((n_points, 3))
    for i in range(n_points):
        x = i % np.sqrt(n_points)
        y = np.floor(i / np.sqrt(n_points))
        points[i] = np.array((x, y, z_function(x, y)))
    return n_points, points
<commit_msg>fix expected output of test data to have the right type<commit_after>"""Collection of tools that can be used while testing the laserchicken software."""
import datetime as dt

import numpy as np

from laserchicken import keys

def generate_tiny_test_point_cloud():
    """Generate a simple but valid point cloud with 3 points."""
    pc = {keys.point: {'x': {'type': 'double', 'data': np.array([1, 2, 3], dtype=np.float64)},
                       'y': {'type': 'double', 'data': np.array([2, 3, 4], dtype=np.float64)},
                       'z': {'type': 'double', 'data': np.array([3, 4, 5], dtype=np.float64)}}}
    return pc

class SimpleTestData(object):
    """Test data within this class should all be in sync (reflect the same data)."""

    @staticmethod
    def get_point_cloud():
        """Get the point cloud data."""
        # This simple_test_point cloud and the simple_test_header should be in sync. Some tests depend on it.
        pc = {keys.point: {'x': {'type': 'float', 'data': np.array([1, 2, 3])},
                           'y': {'type': 'float', 'data': np.array([20, 30, 40])},
                           'z': {'type': 'float', 'data': np.array([300, 400, 500])}}}
        return pc

    @staticmethod
    def get_header():
        """Get the ply header."""
        # This simple_test_header cloud and the simple_test_point should be in sync. Some tests depend on it.
        header = """ply
format ascii 1.0
element vertex 3
property float x
property float y
property float z
"""
        return header

    @staticmethod
    def get_data():
        """Get the data in ply format."""
        data = """1 20 300
2 30 400
3 40 500
"""
        return data

class ComplexTestData(object):
    """Test data within this class should all be in sync (reflect the same data)."""

    comments = [{"module": "filter", "time": str(dt.datetime(2018, 1, 18, 16, 1, 0))},
                {"module": "filter", "time": str(dt.datetime(2018, 2, 4, 13, 11, 0))}]

    def get_point_cloud(self):
        """Get the point cloud data."""
        # This complex_test_point cloud and the complex_test_header should be in sync. Some tests depend on it.
        pc = {keys.point: {'x': {'type': 'float', 'data': np.array([1, 2, 3, 4, 5], dtype=np.float)},
                           'y': {'type': 'float', 'data': np.array([2, 3, 4, 5, 6], dtype=np.float)},
                           'z': {'type': 'float', 'data': np.array([3, 4, 5, 6, 7], dtype=np.float)},
                           'return': {'type': 'int', 'data': np.array([1, 1, 2, 2, 1], dtype=np.int)}
                           },
              keys.point_cloud: {'offset': {'type': 'double', 'data': 12.1}},
              keys.provenance: self.comments
              }
        return pc

    def get_header(self):
        """Get the ply header."""
        # This complex_test_header cloud and the complex_test_point should be in sync. Some tests depend on it.
        header = ("""ply
format ascii 1.0
comment [
comment {},
comment {}
comment ]
element vertex 5
property float x
property float y
property float z
property int return
element pointcloud 1
property double offset
""").format(self.comments[0], self.comments[1])
        return header

    @staticmethod
    def get_data():
        """Get the data in ply format."""
        data = """1.0 2.0 3.0 1
2.0 3.0 4.0 1
3.0 4.0 5.0 2
4.0 5.0 6.0 2
5.0 6.0 7.0 1
12.1
"""
        return data

    @staticmethod
    def get_wkt_polygon_around_first_point_only():
        """Get a polygon in wkt format that surrounds only the first point."""
        return "POLYGON(( 1.5 10.0, 1.5 -10.0, -1.5 -10.0, -1.5 1.0, 1.5 10.0 ))"

def create_point_cloud(x, y, z, normalized_z=None):
    """
    Create a point cloud object given only the x y z values.

    :param x: x attribute values
    :param y: y attribute values
    :param z: z attribute values
    :param normalized_z: optional normalized z attribute values
    :return: point cloud object
    """
    point_cloud = {keys.point: {'x': {'type': 'float', 'data': np.array(x)}, 'y': {'type': 'float', 'data': np.array(y)},
                            'z': {'type': 'float', 'data': np.array(z)}}, keys.point_cloud: {},
               keys.provenance: [{'time': (dt.datetime(2018, 1, 18, 16, 1, 0)), 'module': 'filter'}]}
    if normalized_z is not None:
        point_cloud[keys.point][keys.normalized_height] = {'type': 'float', 'data': np.array(normalized_z)}
    return point_cloud

def create_points_in_xy_grid(z_function):
    """Create 100 points in a grid of 10 by 10."""
    n_points = 100
    points = np.zeros((n_points, 3))
    for i in range(n_points):
        x = i % np.sqrt(n_points)
        y = np.floor(i / np.sqrt(n_points))
        points[i] = np.array((x, y, z_function(x, y)))
    return n_points, points
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from opps.views.tests.test_generic_detail import *
<commit_msg>Add test_generic_list on tests views<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from opps.views.tests.test_generic_detail import *
from opps.views.tests.test_generic_list import *
<|endoftext|>