<commit_before>import asyncio
import base64
import hashlib
import os
import socket
import unittest

import aiohttp
from aiohttp import web, websocket

WS_KEY = b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11"

class TestWebWebSocketFunctional(unittest.TestCase):

    def setUp(self):
        self.loop = asyncio.new_event_loop()
        asyncio.set_event_loop(None)

    def tearDown(self):
        self.loop.close()

    def find_unused_port(self):
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.bind(('127.0.0.1', 0))
        port = s.getsockname()[1]
        s.close()
        return port

    @asyncio.coroutine
    def create_server(self, method, path, handler):
        app = web.Application(loop=self.loop)
        app.router.add_route(method, path, handler)

        port = self.find_unused_port()
        srv = yield from self.loop.create_server(
            app.make_handler(), '127.0.0.1', port)
        url = "http://127.0.0.1:{}".format(port) + path
        self.addCleanup(srv.close)
        return app, srv, url

    @asyncio.coroutine
    def connect_ws(self, url, protocol='chat'):
        sec_key = base64.b64encode(os.urandom(16))

        conn = aiohttp.TCPConnector(loop=self.loop)
        self.addCleanup(conn.close)
        # send request
        response = yield from aiohttp.request(
            'get', url,
            headers={
                'UPGRADE': 'WebSocket',
                'CONNECTION': 'Upgrade',
                'SEC-WEBSOCKET-VERSION': '13',
                'SEC-WEBSOCKET-PROTOCOL': protocol,
                'SEC-WEBSOCKET-KEY': sec_key.decode(),
            },
            connector=conn,
            loop=self.loop)
        self.addCleanup(response.close, True)

        self.assertEqual(101, response.status)
        self.assertEqual(response.headers.get('upgrade', '').lower(),
                         'websocket')
        self.assertEqual(response.headers.get('connection', '').lower(),
                         'upgrade')

        key = response.headers.get('sec-websocket-accept', '').encode()
        match = base64.b64encode(hashlib.sha1(sec_key + WS_KEY).digest())
        self.assertEqual(key, match)

        # switch to websocket protocol
        connection = response.connection
        reader = connection.reader.set_parser(websocket.WebSocketParser)
        writer = websocket.WebSocketWriter(connection.writer)

        return response, reader, writer

    def test_send_recv_text(self):

        closed = asyncio.Future(loop=self.loop)

        @asyncio.coroutine
        def handler(request):
            ws = web.WebSocketResponse()
            yield from ws.prepare(request)
            msg = yield from ws.receive_str()
            ws.send_str(msg+'/answer')
            yield from ws.close()
            closed.set_result(1)
            return ws

        @asyncio.coroutine
        def go():
            _, _, url = yield from self.create_server('GET', '/', handler)
            resp, reader, writer = yield from self.connect_ws(url)
            writer.send('ask')
            msg = yield from reader.read()
            self.assertEqual(msg.tp, websocket.MSG_TEXT)
            self.assertEqual('ask/answer', msg.data)

            msg = yield from reader.read()
            self.assertEqual(msg.tp, websocket.MSG_CLOSE)
            self.assertEqual(msg.data, 1000)
            self.assertEqual(msg.extra, '')

            writer.close()

            yield from closed
            resp.close()

        self.loop.run_until_complete(go())

    def test_send_recv_bytes(self):

        closed = asyncio.Future(loop=self.loop)

        @asyncio.coroutine
        def handler(request):
            ws = web.WebSocketResponse()
            yield from ws.prepare(request)

            msg = yield from ws.receive_bytes()
            ws.send_bytes(msg+b'/answer')
            yield from ws.close()
            closed.set_result(1)
            return ws

        @asyncio.coroutine
        def go():
            _, _, url = yield from self.create_server('GET', '/', handler)
            resp, reader, writer = yield from self.connect_ws(url)
            writer.send(b'ask', binary=True)
            msg = yield from reader.read()
            self.assertEqual(msg.tp, websocket.MSG_BINARY)
            self.assertEqual(b'ask/answer', msg.data)

            msg = yield from reader.read()
            self.assertEqual(msg.tp, websocket.MSG_CLOSE)
            self.assertEqual(msg.data, 1000)
            self.assertEqual(msg.extra, '')

            writer.close()
            yield from closed
            resp.close()

        self.loop.run_until_complete(go())

    def test_auto_pong_with_closing_by_peer(self):

        closed = asyncio.Future(loop=self.loop)

        @asyncio.coroutine
        def handler(request):
            ws = web.WebSocketResponse()
            yield from ws.prepare(request)
            yield from ws.receive()

            msg = yield from ws.receive()
            self.assertEqual(msg.tp, web.MsgType.close)
            self.assertEqual(msg.data, 1000)
            self.assertEqual(msg.extra, 'exit message')
            closed.set_result(None)
            return ws

        @asyncio.coroutine
        def go():
            _, _, url = yield from self.create_server('GET', '/', handler)
            resp, reader, writer = yield from self.connect_ws(url)
            writer.ping()
            writer.send('ask')

            msg = yield from reader.read()
            self.assertEqual(msg.tp, websocket.MSG_PONG)
            writer.close(1000, 'exit message')
            yield from closed
            resp.close()

        self.loop.run_until_complete(go())

    def test_ping(self):

        closed = asyncio.Future(loop=self.loop)

        @asyncio.coroutine
        def handler(request):
            ws = web.WebSocketResponse()
            yield from ws.prepare(request)

            ws.ping('data')
            yield from ws.receive()
            closed.set_result(None)
            return ws

        @asyncio.coroutine
        def go():
            _, _, url = yield from self.create_server('GET', '/', handler)
            resp, reader, writer = yield from self.connect_ws(url)
            msg = yield from reader.read()
            self.assertEqual(msg.tp, websocket.MSG_PING)
            self.assertEqual(msg.data, b'data')
            writer.pong()
            writer.close(2, 'exit message')
            yield from closed
            resp.close()

        self.loop.run_until_complete(go())

    def test_client_ping(self):

        closed = asyncio.Future(loop=self.loop)

        @asyncio.coroutine
        def handler(request):
            ws = web.WebSocketResponse()
            yield from ws.prepare(request)

            yield from ws.receive()
            closed.set_result(None)
            return ws

        @asyncio.coroutine
        def go():
            _, _, url = yield from self.create_server('GET', '/', handler)
            resp, reader, writer = yield from self.connect_ws(url)
            writer.ping('data')
            msg = yield from reader.read()
            self.assertEqual(msg.tp, websocket.MSG_PONG)
            self.assertEqual(msg.data, b'data')
            writer.pong()
            writer.close()
            yield from closed
            resp.close()

        self.loop.run_until_complete(go())

    def test_pong(self):

        closed = asyncio.Future(loop=self.loop)

        @asyncio.coroutine
        def handler(request):
            ws = web.WebSocketResponse(autoping=False)
            yield from ws.prepare(request)

            msg = yield from ws.receive()
            self.assertEqual(msg.tp, web.MsgType.ping)
            ws.pong('data')

            msg = yield from ws.receive()
            self.assertEqual(msg.tp, web.MsgType.close)
            self.assertEqual(msg.data, 1000)
            self.assertEqual(msg.extra, 'exit message')
            closed.set_result(None)
            return ws

        @asyncio.coroutine
        def go():
            _, _, url = yield from self.create_server('GET', '/', handler)
            resp, reader, writer = yield from self.connect_ws(url)
            writer.ping('data')
            msg = yield from reader.read()
            self.assertEqual(msg.tp, websocket.MSG_PONG)
            self.assertEqual(msg.data, b'data')
            writer.close(1000, 'exit message')

            yield from closed
            resp.close()

        self.loop.run_until_complete(go())

    def test_change_status(self):

        closed = asyncio.Future(loop=self.loop)

        @asyncio.coroutine
        def handler(request):
            ws = web.WebSocketResponse()
            ws.set_status(200)
            self.assertEqual(200, ws.status)
            yield from ws.prepare(request)
            self.assertEqual(101, ws.status)
            yield from ws.close()
            closed.set_result(None)
            return ws

        @asyncio.coroutine
        def go():
            _, _, url = yield from self.create_server('GET', '/', handler)
            resp, _, writer = yield from self.connect_ws(url)
            writer.close()
            yield from closed
            resp.close()

        self.loop.run_until_complete(go())

    def test_handle_protocol(self):

        closed = asyncio.Future(loop=self.loop)

        @asyncio.coroutine
        def handler(request):
            ws = web.WebSocketResponse(protocols=('foo', 'bar'))
            yield from ws.prepare(request)
            yield from ws.close()
            self.assertEqual('bar', ws.protocol)
            closed.set_result(None)
            return ws

        @asyncio.coroutine
        def go():
            _, _, url = yield from self.create_server('GET', '/', handler)
            resp, _, writer = yield from self.connect_ws(url, 'eggs, bar')
            writer.close()

            yield from closed
            resp.close()

        self.loop.run_until_complete(go())

    def test_server_close_handshake(self):

        closed = asyncio.Future(loop=self.loop)

        @asyncio.coroutine
        def handler(request):
            ws = web.WebSocketResponse(protocols=('foo', 'bar'))
            yield from ws.prepare(request)
            yield from ws.close()
            closed.set_result(None)
            return ws

        @asyncio.coroutine
        def go():
            _, _, url = yield from self.create_server('GET', '/', handler)
            resp, reader, writer = yield from self.connect_ws(url, 'eggs, bar')

            msg = yield from reader.read()
            self.assertEqual(msg.tp, websocket.MSG_CLOSE)
            writer.close()
            yield from closed
            resp.close()

        self.loop.run_until_complete(go())

    def test_client_close_handshake(self):

        closed = asyncio.Future(loop=self.loop)

        @asyncio.coroutine
        def handler(request):
            ws = web.WebSocketResponse(
                autoclose=False, protocols=('foo', 'bar'))
            yield from ws.prepare(request)

            msg = yield from ws.receive()
            self.assertEqual(msg.tp, web.MsgType.close)
            self.assertFalse(ws.closed)
            yield from ws.close()
            self.assertTrue(ws.closed)
            self.assertEqual(ws.close_code, 1007)

            msg = yield from ws.receive()
            self.assertEqual(msg.tp, web.MsgType.closed)

            closed.set_result(None)
            return ws

        @asyncio.coroutine
        def go():
            _, _, url = yield from self.create_server('GET', '/', handler)
            resp, reader, writer = yield from self.connect_ws(url, 'eggs, bar')

            writer.close(code=1007)
            msg = yield from reader.read()
            self.assertEqual(msg.tp, websocket.MSG_CLOSE)
            yield from closed
            resp.close()

        self.loop.run_until_complete(go())

    def test_server_close_handshake_server_eats_client_messages(self):

        closed = asyncio.Future(loop=self.loop)

        @asyncio.coroutine
        def handler(request):
            ws = web.WebSocketResponse(protocols=('foo', 'bar'))
            yield from ws.prepare(request)
            yield from ws.close()
            closed.set_result(None)
            return ws

        @asyncio.coroutine
        def go():
            _, _, url = yield from self.create_server('GET', '/', handler)
            response, reader, writer = yield from self.connect_ws(
                url, 'eggs, bar')

            msg = yield from reader.read()
            self.assertEqual(msg.tp, websocket.MSG_CLOSE)

            writer.send('text')
            writer.send(b'bytes', binary=True)
            writer.ping()

            writer.close()
            yield from closed

            response.close()

        self.loop.run_until_complete(go())

    def test_receive_msg(self):
        @asyncio.coroutine
        def handler(request):
            ws = web.WebSocketResponse()
            yield from ws.prepare(request)

            with self.assertWarns(DeprecationWarning):
                msg = yield from ws.receive_msg()
                self.assertEqual(msg.data, b'data')
            yield from ws.close()
            return ws

        @asyncio.coroutine
        def go():
            _, _, url = yield from self.create_server('GET', '/', handler)
            resp = yield from aiohttp.ws_connect(url, loop=self.loop)
            resp.send_bytes(b'data')
            yield from resp.close()

        self.loop.run_until_complete(go())
<commit_msg>Cleanup tests: drop extra warning about chat websocket protocol<commit_after>import asyncio
import base64
import hashlib
import os
import socket
import unittest

import aiohttp
from aiohttp import web, websocket

WS_KEY = b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11"

class TestWebWebSocketFunctional(unittest.TestCase):

    def setUp(self):
        self.loop = asyncio.new_event_loop()
        asyncio.set_event_loop(None)

    def tearDown(self):
        self.loop.close()

    def find_unused_port(self):
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.bind(('127.0.0.1', 0))
        port = s.getsockname()[1]
        s.close()
        return port

    @asyncio.coroutine
    def create_server(self, method, path, handler):
        app = web.Application(loop=self.loop)
        app.router.add_route(method, path, handler)

        port = self.find_unused_port()
        srv = yield from self.loop.create_server(
            app.make_handler(), '127.0.0.1', port)
        url = "http://127.0.0.1:{}".format(port) + path
        self.addCleanup(srv.close)
        return app, srv, url

    @asyncio.coroutine
    def connect_ws(self, url, protocol=''):
        sec_key = base64.b64encode(os.urandom(16))

        conn = aiohttp.TCPConnector(loop=self.loop)
        self.addCleanup(conn.close)
        # send request
        response = yield from aiohttp.request(
            'get', url,
            headers={
                'UPGRADE': 'WebSocket',
                'CONNECTION': 'Upgrade',
                'SEC-WEBSOCKET-VERSION': '13',
                'SEC-WEBSOCKET-PROTOCOL': protocol,
                'SEC-WEBSOCKET-KEY': sec_key.decode(),
            },
            connector=conn,
            loop=self.loop)
        self.addCleanup(response.close, True)

        self.assertEqual(101, response.status)
        self.assertEqual(response.headers.get('upgrade', '').lower(),
                         'websocket')
        self.assertEqual(response.headers.get('connection', '').lower(),
                         'upgrade')

        key = response.headers.get('sec-websocket-accept', '').encode()
        match = base64.b64encode(hashlib.sha1(sec_key + WS_KEY).digest())
        self.assertEqual(key, match)

        # switch to websocket protocol
        connection = response.connection
        reader = connection.reader.set_parser(websocket.WebSocketParser)
        writer = websocket.WebSocketWriter(connection.writer)

        return response, reader, writer

    def test_send_recv_text(self):

        closed = asyncio.Future(loop=self.loop)

        @asyncio.coroutine
        def handler(request):
            ws = web.WebSocketResponse()
            yield from ws.prepare(request)
            msg = yield from ws.receive_str()
            ws.send_str(msg+'/answer')
            yield from ws.close()
            closed.set_result(1)
            return ws

        @asyncio.coroutine
        def go():
            _, _, url = yield from self.create_server('GET', '/', handler)
            resp, reader, writer = yield from self.connect_ws(url)
            writer.send('ask')
            msg = yield from reader.read()
            self.assertEqual(msg.tp, websocket.MSG_TEXT)
            self.assertEqual('ask/answer', msg.data)

            msg = yield from reader.read()
            self.assertEqual(msg.tp, websocket.MSG_CLOSE)
            self.assertEqual(msg.data, 1000)
            self.assertEqual(msg.extra, '')

            writer.close()

            yield from closed
            resp.close()

        self.loop.run_until_complete(go())

    def test_send_recv_bytes(self):

        closed = asyncio.Future(loop=self.loop)

        @asyncio.coroutine
        def handler(request):
            ws = web.WebSocketResponse()
            yield from ws.prepare(request)

            msg = yield from ws.receive_bytes()
            ws.send_bytes(msg+b'/answer')
            yield from ws.close()
            closed.set_result(1)
            return ws

        @asyncio.coroutine
        def go():
            _, _, url = yield from self.create_server('GET', '/', handler)
            resp, reader, writer = yield from self.connect_ws(url)
            writer.send(b'ask', binary=True)
            msg = yield from reader.read()
            self.assertEqual(msg.tp, websocket.MSG_BINARY)
            self.assertEqual(b'ask/answer', msg.data)

            msg = yield from reader.read()
            self.assertEqual(msg.tp, websocket.MSG_CLOSE)
            self.assertEqual(msg.data, 1000)
            self.assertEqual(msg.extra, '')

            writer.close()
            yield from closed
            resp.close()

        self.loop.run_until_complete(go())

    def test_auto_pong_with_closing_by_peer(self):

        closed = asyncio.Future(loop=self.loop)

        @asyncio.coroutine
        def handler(request):
            ws = web.WebSocketResponse()
            yield from ws.prepare(request)
            yield from ws.receive()

            msg = yield from ws.receive()
            self.assertEqual(msg.tp, web.MsgType.close)
            self.assertEqual(msg.data, 1000)
            self.assertEqual(msg.extra, 'exit message')
            closed.set_result(None)
            return ws

        @asyncio.coroutine
        def go():
            _, _, url = yield from self.create_server('GET', '/', handler)
            resp, reader, writer = yield from self.connect_ws(url)
            writer.ping()
            writer.send('ask')

            msg = yield from reader.read()
            self.assertEqual(msg.tp, websocket.MSG_PONG)
            writer.close(1000, 'exit message')
            yield from closed
            resp.close()

        self.loop.run_until_complete(go())

    def test_ping(self):

        closed = asyncio.Future(loop=self.loop)

        @asyncio.coroutine
        def handler(request):
            ws = web.WebSocketResponse()
            yield from ws.prepare(request)

            ws.ping('data')
            yield from ws.receive()
            closed.set_result(None)
            return ws

        @asyncio.coroutine
        def go():
            _, _, url = yield from self.create_server('GET', '/', handler)
            resp, reader, writer = yield from self.connect_ws(url)
            msg = yield from reader.read()
            self.assertEqual(msg.tp, websocket.MSG_PING)
            self.assertEqual(msg.data, b'data')
            writer.pong()
            writer.close(2, 'exit message')
            yield from closed
            resp.close()

        self.loop.run_until_complete(go())

    def test_client_ping(self):

        closed = asyncio.Future(loop=self.loop)

        @asyncio.coroutine
        def handler(request):
            ws = web.WebSocketResponse()
            yield from ws.prepare(request)

            yield from ws.receive()
            closed.set_result(None)
            return ws

        @asyncio.coroutine
        def go():
            _, _, url = yield from self.create_server('GET', '/', handler)
            resp, reader, writer = yield from self.connect_ws(url)
            writer.ping('data')
            msg = yield from reader.read()
            self.assertEqual(msg.tp, websocket.MSG_PONG)
            self.assertEqual(msg.data, b'data')
            writer.pong()
            writer.close()
            yield from closed
            resp.close()

        self.loop.run_until_complete(go())

    def test_pong(self):

        closed = asyncio.Future(loop=self.loop)

        @asyncio.coroutine
        def handler(request):
            ws = web.WebSocketResponse(autoping=False)
            yield from ws.prepare(request)

            msg = yield from ws.receive()
            self.assertEqual(msg.tp, web.MsgType.ping)
            ws.pong('data')

            msg = yield from ws.receive()
            self.assertEqual(msg.tp, web.MsgType.close)
            self.assertEqual(msg.data, 1000)
            self.assertEqual(msg.extra, 'exit message')
            closed.set_result(None)
            return ws

        @asyncio.coroutine
        def go():
            _, _, url = yield from self.create_server('GET', '/', handler)
            resp, reader, writer = yield from self.connect_ws(url)
            writer.ping('data')
            msg = yield from reader.read()
            self.assertEqual(msg.tp, websocket.MSG_PONG)
            self.assertEqual(msg.data, b'data')
            writer.close(1000, 'exit message')

            yield from closed
            resp.close()

        self.loop.run_until_complete(go())

    def test_change_status(self):

        closed = asyncio.Future(loop=self.loop)

        @asyncio.coroutine
        def handler(request):
            ws = web.WebSocketResponse()
            ws.set_status(200)
            self.assertEqual(200, ws.status)
            yield from ws.prepare(request)
            self.assertEqual(101, ws.status)
            yield from ws.close()
            closed.set_result(None)
            return ws

        @asyncio.coroutine
        def go():
            _, _, url = yield from self.create_server('GET', '/', handler)
            resp, _, writer = yield from self.connect_ws(url)
            writer.close()
            yield from closed
            resp.close()

        self.loop.run_until_complete(go())

    def test_handle_protocol(self):

        closed = asyncio.Future(loop=self.loop)

        @asyncio.coroutine
        def handler(request):
            ws = web.WebSocketResponse(protocols=('foo', 'bar'))
            yield from ws.prepare(request)
            yield from ws.close()
            self.assertEqual('bar', ws.protocol)
            closed.set_result(None)
            return ws

        @asyncio.coroutine
        def go():
            _, _, url = yield from self.create_server('GET', '/', handler)
            resp, _, writer = yield from self.connect_ws(url, 'eggs, bar')
            writer.close()

            yield from closed
            resp.close()

        self.loop.run_until_complete(go())

    def test_server_close_handshake(self):

        closed = asyncio.Future(loop=self.loop)

        @asyncio.coroutine
        def handler(request):
            ws = web.WebSocketResponse(protocols=('foo', 'bar'))
            yield from ws.prepare(request)
            yield from ws.close()
            closed.set_result(None)
            return ws

        @asyncio.coroutine
        def go():
            _, _, url = yield from self.create_server('GET', '/', handler)
            resp, reader, writer = yield from self.connect_ws(url, 'eggs, bar')

            msg = yield from reader.read()
            self.assertEqual(msg.tp, websocket.MSG_CLOSE)
            writer.close()
            yield from closed
            resp.close()

        self.loop.run_until_complete(go())

    def test_client_close_handshake(self):

        closed = asyncio.Future(loop=self.loop)

        @asyncio.coroutine
        def handler(request):
            ws = web.WebSocketResponse(
                autoclose=False, protocols=('foo', 'bar'))
            yield from ws.prepare(request)

            msg = yield from ws.receive()
            self.assertEqual(msg.tp, web.MsgType.close)
            self.assertFalse(ws.closed)
            yield from ws.close()
            self.assertTrue(ws.closed)
            self.assertEqual(ws.close_code, 1007)

            msg = yield from ws.receive()
            self.assertEqual(msg.tp, web.MsgType.closed)

            closed.set_result(None)
            return ws

        @asyncio.coroutine
        def go():
            _, _, url = yield from self.create_server('GET', '/', handler)
            resp, reader, writer = yield from self.connect_ws(url, 'eggs, bar')

            writer.close(code=1007)
            msg = yield from reader.read()
            self.assertEqual(msg.tp, websocket.MSG_CLOSE)
            yield from closed
            resp.close()

        self.loop.run_until_complete(go())

    def test_server_close_handshake_server_eats_client_messages(self):

        closed = asyncio.Future(loop=self.loop)

        @asyncio.coroutine
        def handler(request):
            ws = web.WebSocketResponse(protocols=('foo', 'bar'))
            yield from ws.prepare(request)
            yield from ws.close()
            closed.set_result(None)
            return ws

        @asyncio.coroutine
        def go():
            _, _, url = yield from self.create_server('GET', '/', handler)
            response, reader, writer = yield from self.connect_ws(
                url, 'eggs, bar')

            msg = yield from reader.read()
            self.assertEqual(msg.tp, websocket.MSG_CLOSE)

            writer.send('text')
            writer.send(b'bytes', binary=True)
            writer.ping()

            writer.close()
            yield from closed

            response.close()

        self.loop.run_until_complete(go())

    def test_receive_msg(self):
        @asyncio.coroutine
        def handler(request):
            ws = web.WebSocketResponse()
            yield from ws.prepare(request)

            with self.assertWarns(DeprecationWarning):
                msg = yield from ws.receive_msg()
                self.assertEqual(msg.data, b'data')
            yield from ws.close()
            return ws

        @asyncio.coroutine
        def go():
            _, _, url = yield from self.create_server('GET', '/', handler)
            resp = yield from aiohttp.ws_connect(url, loop=self.loop)
            resp.send_bytes(b'data')
            yield from resp.close()

        self.loop.run_until_complete(go())
<|endoftext|>


<commit_before>
'''
Check for positional offsets between the VLA and SD datasets.
'''

from image_registration import register_images
from spectral_cube import Projection, SpectralCube
from uvcombine.uvcombine import match_flux_units

from astropy import log
import astropy.units as u
import numpy as np

from .progressbar import _map_context
from .io_utils import create_huge_fits
from .spectra_shifter import fourier_shift

def cube_registration(cube, target_cube, verbose=True, num_cores=1,
                      **kwargs):
    '''
    Use the DFT method in image_registration to find the offset for each
    plane. Returns the offset for each plane, which can be used to find the
    average offset between the cubes.
    '''

    # We're doing this per-channel, so both cubes need to have the same
    # spectral axis
    if not np.allclose(cube.spectral_axis.value,
                       target_cube.spectral_axis.to(cube.spectral_axis.unit).value):
        raise Warning("The spectral axes do not match. Spectrally regrid the "
                      "cubes to be the same before feathering.")

    cubegen = ((i, cube[i], target_cube[i]) for i in xrange(cube.shape[0]))

    with _map_context(num_cores, verbose=verbose,
                      num_jobs=cube.shape[0]) as map:

        offsets = [x for x in map(_register, cubegen)]

    channels = np.array([x[0] for x in offsets])
    offsets = np.array([x[1] for x in offsets])

    return offsets[channels]

def _register(args):
    '''
    Match units, convolve, and regrid, then register the channels.
    '''

    chan, plane, targ_plane = args

    # TODO: Should be using match_flux_units, but the Slices don't have
    # WCS spectral information anymore. That's an issue whenever the
    # units as Jy/beam.

    # header = plane.header.copy()
    # if header['BUNIT'] == "Jy":
    #     header.update({"BUNIT": "Jy/beam"})
    # targ_header = targ_plane.header.copy()
    # if targ_header['BUNIT'] == "Jy":
    #     targ_header.update({"BUNIT": "Jy/beam"})

    # plane_match = match_flux_units(plane.value, header,
    #                                targ_header)

    # Convert back to a projection for the next steps
    # new_plane = Projection(plane_match.value, unit=plane_match.unit,
    #                        wcs=plane.wcs, header=plane.header)

    # Hard-wired for my HI VLA to SD comparisons right now.

    hi_freq = 1.42040575177 * u.GHz

    if plane.unit == u.K and targ_plane.unit != u.K:
        targ_plane = targ_plane.to(u.K, targ_plane.beam.jtok_equiv(hi_freq))
    elif plane.unit != u.K and targ_plane.unit == u.K:
        plane = plane.to(u.K, plane.beam.jtok_equiv(hi_freq))

    plane_shape = plane.shape
    targ_plane_shape = targ_plane.shape

    # Convolve whichever has the larger beam
    if targ_plane.beam != plane.beam:
        try:
            plane = plane.convolve_to(targ_plane.beam)
        except ValueError:
            targ_plane = targ_plane.convolve_to(plane.beam)

    targ_plane = targ_plane.reproject(plane.header)

    # Now register
    # Assume that the image are reasonably close -- 20% of the smallest
    # spatial axis
    maxoff = min(targ_plane.shape) * 0.2
    # Upsample by the ratio between the image sizes.
    usfac = min(targ_plane_shape) / float(min(plane_shape))
    if usfac < 1:
        usfac = 1 / usfac
    # register_image return dx, dy. To be consistent throughout this package,
    # swap the output so it is dy, dx
    output = register_images(targ_plane.value, plane.value, zeromean=True,
                             nthreads=1, maxoff=maxoff, usfac=int(usfac))[::-1]

    return chan, output

def spatial_shift_cube(cube, dy, dx, verbose=True, save_shifted=True,
                       save_name=None, num_cores=1, chunk=100):
    '''
    Shift a SpectralCube by dy and dx in the spatial dimensions.
    '''

    if save_shifted:
        if save_name is None:
            raise TypeError("save_name must be given the save the shifted "
                            "cube.")

        output_hdu = create_huge_fits(save_name, cube.header, return_hdu=True)
    else:
        newcube = np.empty(cube.shape, dtype=cube[:1, 0, 0].dtype)
        newmask = np.empty(cube.shape, dtype=bool)

    num_chans = cube.shape[0]
    channels = np.arange(num_chans)
    chunked_channels = \
        np.array_split(channels,
                       [chunk * i for i in xrange(num_chans / chunk)])
    if chunked_channels[-1].size == 0:
        chunked_channels = chunked_channels[:-1]

    for i, chunk_chans in enumerate(chunked_channels):

        log.info("On chunk {0} of {1}".format(i + 1, len(chunked_channels)))

        changen = ((chan, cube[chan], dy, dx) for chan in chunk_chans)

        if not save_shifted:
            maskgen = ((chan,
                        cube.mask.include(view=(slice(chan, chan + 1))),
                        dy, dx) for chan in chunk_chans)

        with _map_context(num_cores, verbose=verbose)as map:
            log.info("Shifting array")
            output = map(_shifter, changen)

            if not save_shifted:
                log.info("Shifting mask")
                output_mask = map(_shifter, maskgen)

        for chan, shifted_arr in output:
            if save_shifted:
                output_hdu[0].data[chan] = shifted_arr
            else:
                newcube[chan] = shifted_arr

        if not save_shifted:
            for chan, shifted_mask in output_mask:
                newmask[chan] = shifted_mask > 0.5

        if save_shifted:
            output_hdu.flush()

    if save_shifted:
        output_hdu.close()
    else:
        return SpectralCube(newcube, wcs=cube.wcs,
                            header=cube.header, meta=cube.meta,
                            mask=cube.mask)

def _shifter(args):

    chan, arr, y_shift, x_shift = args

    if y_shift != 0:
        arr_shift = fourier_shift(arr, y_shift, axis=0)
    else:
        arr_shift = arr

    if x_shift != 0:
        arr_shift = fourier_shift(arr_shift, x_shift, axis=1)

    return chan, arr_shift
<commit_msg>Set rest frequency<commit_after>
'''
Check for positional offsets between the VLA and SD datasets.
'''

from image_registration import register_images
from spectral_cube import Projection, SpectralCube
from uvcombine.uvcombine import match_flux_units

from astropy import log
import astropy.units as u
import numpy as np

from .progressbar import _map_context
from .io_utils import create_huge_fits
from .spectra_shifter import fourier_shift

def cube_registration(cube, target_cube, verbose=True, num_cores=1,
                      restfreq=1.42040575177 * u.GHz,
                      **kwargs):
    '''
    Use the DFT method in image_registration to find the offset for each
    plane. Returns the offset for each plane, which can be used to find the
    average offset between the cubes.
    '''

    # We're doing this per-channel, so both cubes need to have the same
    # spectral axis
    if not np.allclose(cube.spectral_axis.value,
                       target_cube.spectral_axis.to(cube.spectral_axis.unit).value):
        raise Warning("The spectral axes do not match. Spectrally regrid the "
                      "cubes to be the same before feathering.")

    freqs = cube.with_spectral_unit(u.Hz, velocity_convention='radio',
                                    rest_value=restfreq).spectral_axis

    cubegen = ((i, cube[i], target_cube[i], freqs[i])
               for i in xrange(cube.shape[0]))

    with _map_context(num_cores, verbose=verbose,
                      num_jobs=cube.shape[0]) as map:

        offsets = [x for x in map(_register, cubegen)]

    channels = np.array([x[0] for x in offsets])
    offsets = np.array([x[1] for x in offsets])

    return offsets[channels]

def _register(args):
    '''
    Match units, convolve, and regrid, then register the channels.
    '''

    chan, plane, targ_plane, freq = args

    # TODO: Should be using match_flux_units, but the Slices don't have
    # WCS spectral information anymore. That's an issue whenever the
    # units as Jy/beam.

    # header = plane.header.copy()
    # if header['BUNIT'] == "Jy":
    #     header.update({"BUNIT": "Jy/beam"})
    # targ_header = targ_plane.header.copy()
    # if targ_header['BUNIT'] == "Jy":
    #     targ_header.update({"BUNIT": "Jy/beam"})

    # plane_match = match_flux_units(plane.value, header,
    #                                targ_header)

    # Convert back to a projection for the next steps
    # new_plane = Projection(plane_match.value, unit=plane_match.unit,
    #                        wcs=plane.wcs, header=plane.header)

    # Hard-wired for my HI VLA to SD comparisons right now.

    if plane.unit == u.K and targ_plane.unit != u.K:
        targ_plane = targ_plane.to(u.K, targ_plane.beam.jtok_equiv(freq))
    elif plane.unit != u.K and targ_plane.unit == u.K:
        plane = plane.to(u.K, plane.beam.jtok_equiv(freq))

    plane_shape = plane.shape
    targ_plane_shape = targ_plane.shape

    # Convolve whichever has the larger beam
    if targ_plane.beam != plane.beam:
        try:
            plane = plane.convolve_to(targ_plane.beam)
        except ValueError:
            targ_plane = targ_plane.convolve_to(plane.beam)

    targ_plane = targ_plane.reproject(plane.header)

    # Now register
    # Assume that the image are reasonably close -- 20% of the smallest
    # spatial axis
    maxoff = min(targ_plane.shape) * 0.2
    # Upsample by the ratio between the image sizes.
    usfac = min(targ_plane_shape) / float(min(plane_shape))
    if usfac < 1:
        usfac = 1 / usfac
    # register_image return dx, dy. To be consistent throughout this package,
    # swap the output so it is dy, dx
    output = register_images(targ_plane.value, plane.value, zeromean=True,
                             nthreads=1, maxoff=maxoff, usfac=int(usfac))[::-1]

    return chan, output

def spatial_shift_cube(cube, dy, dx, verbose=True, save_shifted=True,
                       save_name=None, num_cores=1, chunk=100):
    '''
    Shift a SpectralCube by dy and dx in the spatial dimensions.
    '''

    if save_shifted:
        if save_name is None:
            raise TypeError("save_name must be given the save the shifted "
                            "cube.")

        output_hdu = create_huge_fits(save_name, cube.header, return_hdu=True)
    else:
        newcube = np.empty(cube.shape, dtype=cube[:1, 0, 0].dtype)
        newmask = np.empty(cube.shape, dtype=bool)

    num_chans = cube.shape[0]
    channels = np.arange(num_chans)
    chunked_channels = \
        np.array_split(channels,
                       [chunk * i for i in xrange(num_chans / chunk)])
    if chunked_channels[-1].size == 0:
        chunked_channels = chunked_channels[:-1]

    for i, chunk_chans in enumerate(chunked_channels):

        log.info("On chunk {0} of {1}".format(i + 1, len(chunked_channels)))

        changen = ((chan, cube[chan], dy, dx) for chan in chunk_chans)

        if not save_shifted:
            maskgen = ((chan,
                        cube.mask.include(view=(slice(chan, chan + 1))),
                        dy, dx) for chan in chunk_chans)

        with _map_context(num_cores, verbose=verbose)as map:
            log.info("Shifting array")
            output = map(_shifter, changen)

            if not save_shifted:
                log.info("Shifting mask")
                output_mask = map(_shifter, maskgen)

        for chan, shifted_arr in output:
            if save_shifted:
                output_hdu[0].data[chan] = shifted_arr
            else:
                newcube[chan] = shifted_arr

        if not save_shifted:
            for chan, shifted_mask in output_mask:
                newmask[chan] = shifted_mask > 0.5

        if save_shifted:
            output_hdu.flush()

    if save_shifted:
        output_hdu.close()
    else:
        return SpectralCube(newcube, wcs=cube.wcs,
                            header=cube.header, meta=cube.meta,
                            mask=cube.mask)

def _shifter(args):

    chan, arr, y_shift, x_shift = args

    if y_shift != 0:
        arr_shift = fourier_shift(arr, y_shift, axis=0)
    else:
        arr_shift = arr

    if x_shift != 0:
        arr_shift = fourier_shift(arr_shift, x_shift, axis=1)

    return chan, arr_shift
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: Arne Neumann <discoursegraphs.programming@arne.cl>

"""Basic tests for the ``rs3tree`` module"""

import os

import pytest

from discoursegraphs import t
from discoursegraphs.readwrite.rst.rs3 import RSTTree
import discoursegraphs as dg

# files to test
"""
['eins-zwei-drei-(joint-eins-and-zwei-and-drei).rs3',

 'eins-zwei-drei-(circ-(circ-eins-to-zwei)-from-drei).rs3',
 'foo-bar-only-segments.rs3',
 'foo-bar-circ-foo-to-bar.rs3',
 'eins-zwei-drei-(circ-(joint-eins-and-zwei)-to-drei).rs3',
 'eins-zwei-drei-(elab-eins-from-(joint-zwei-and-drei).rs3',
 'foo-bar-foo-conj-bar.rs3',
 'eins-zwei-drei-(circ-eins-from-(joint-zwei-and-drei).rs3',
 'eins-zwei-drei-only-segments.rs3',
 'eins-zwei-drei-(circ-(circ-eins-to-zwei)-to-drei.rs3',
 'eins-zwei-drei-(circ-(joint-eins-and-zwei)-from-drei).rs3',
 'eins-zwei-drei-(circ-eins-to-(joint-zwei-and-drei).rs3',
 'foo-bar-elab-foo-to-bar.rs3',
 'foo-bar-circ-bar-to-foo.rs3',

 'foo-bar-foo-joint-bar.rs3',
 'foo-bar-elab-bar-to-foo.rs3',
 'eins-zwei-drei-(circ-(circ-eins-from-zwei)-from-drei).rs3',
 'eins-zwei-drei-(circ-(circ-eins-from-zwei)-to-drei).rs3']"""

def get_filepath(rs3tree_example_filename):
    """Return the absolute path of an example file."""
    return os.path.join(
        dg.DATA_ROOT_DIR, 'rs3tree', rs3tree_example_filename)

def test_segments_only_trees():
    """Files without a single root must get a virtual one."""

    # minimal case: file without any segments
    rs3_file = get_filepath("empty.rs3")
    produced = RSTTree(rs3_file)
    expected = t("")

    assert expected == produced.tree

    # one lonely segment
    rs3_file = get_filepath('only-one-segment.rs3')
    produced = RSTTree(rs3_file)
    expected = t("foo")

    assert expected == produced.tree

    # two segments w/out a root
    rs3_file = get_filepath("foo-bar-only-segments.rs3")
    produced = RSTTree(rs3_file)
    expected = t("virtual-root",
                 [("N", "foo"), ("N", "bar")])

    assert expected == produced.tree

    # three segments w/out a root
    rs3_file = get_filepath('eins-zwei-drei-only-segments.rs3')
    produced = RSTTree(rs3_file)
    expected = t("virtual-root",
                 [("N", "eins"), ("N", "zwei"), ("N", "drei")])

    assert expected == produced.tree
<commit_msg>rs3tree: added more tests<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Author: Arne Neumann <discoursegraphs.programming@arne.cl>

"""Basic tests for the ``rs3tree`` module"""

import os

import pytest

from discoursegraphs import t
from discoursegraphs.readwrite.rst.rs3 import RSTTree
import discoursegraphs as dg

def example2tree(rs3tree_example_filename):
    """Return the absolute path of an example file."""
    filepath = os.path.join(dg.DATA_ROOT_DIR, 'rs3tree',
                            rs3tree_example_filename)
    return RSTTree(filepath)

def test_segments_only_trees():
    """Files without a single root must get a virtual one."""

    # minimal case: file without any segments
    produced = example2tree("empty.rs3")
    expected = t("")

    assert expected == produced.tree

    # two segments w/out a root
    produced = example2tree("foo-bar-only-segments.rs3")
    expected = t("virtual-root",
                 [("N", "foo"), ("N", "bar")])

    assert expected == produced.tree

    # three segments w/out a root
    produced = example2tree('eins-zwei-drei-only-segments.rs3')
    expected = t("virtual-root",
                 [("N", "eins"), ("N", "zwei"), ("N", "drei")])

    assert expected == produced.tree

    # FIXME: one lonely segment; produces (N (foo))
    produced = example2tree('only-one-segment.rs3')
    expected = t("foo")

    assert expected == produced.tree

def test_single_nucsat_relation():
    # FIXME: only contains root node
    produced = example2tree("foo-bar-circ-foo-to-bar.rs3")
    expected = t("circumstance", [
        ("S", "foo"),
        ("N", "bar")])

    assert expected == produced.tree

    # FIXME: only contains root node
    produced = example2tree("foo-bar-elab-foo-to-bar.rs3")
    expected = t("elaboration", [
        ("S", "foo"),
        ("N", "bar")])

    assert expected == produced.tree

    # FIXME: only contains root node
    produced = example2tree("foo-bar-circ-bar-to-foo.rs3")
    expected = t("circumstance", [
        ("N", "foo"),
        ("S", "bar")])

    assert expected == produced.tree

    # FIXME: only contains root node
    produced = example2tree("foo-bar-elab-bar-to-foo.rs3")
    expected = t("elaboration", [
        ("N", "foo"),
        ("S", "bar")])

    assert expected == produced.tree

def test_single_nucnuc_relation():
    # FIXME: superfluous N root
    produced = example2tree("foo-bar-foo-joint-bar.rs3")
    expected = t("joint", [
        ("N", "foo"),
        ("N", "bar")])

    assert expected == produced.tree

    # FIXME: superfluous N root
    produced = example2tree("foo-bar-foo-conj-bar.rs3")
    expected = t("conjunction", [
        ("N", "foo"),
        ("N", "bar")])

    assert expected == produced.tree

    # FIXME: superfluous N root
    produced = example2tree('eins-zwei-drei-(joint-eins-and-zwei-and-drei).rs3')
    expected = t("joint", [
        ("N", "eins"),
        ("N", "zwei"),
        ("N", "drei")])

    assert expected == produced.tree

def test_nested_nucsat_relation():
    # correct
    produced = example2tree('eins-zwei-drei-(circ-(circ-eins-from-zwei)-from-drei).rs3')
    expected = t("circumstance", [
        ("N", [
            ("circumstance", [
                ("N", "eins"),
                ("S", "zwei")])]),
        ("S", "drei")])

    assert expected == produced.tree

    # correct
    produced = example2tree('eins-zwei-drei-(circ-(circ-eins-to-zwei)-from-drei).rs3')
    expected = t("circumstance", [
        ("N", [
            ("circumstance", [
                ("S", "eins"),
                ("N", "zwei")])]),
        ("S", "drei")])

    assert expected == produced.tree

    # FIXME: only contains root node
    produced = example2tree('eins-zwei-drei-(circ-(circ-eins-from-zwei)-to-drei).rs3')
    expected = t("circumstance", [
        ("S", [
            ("circumstance", [
                ("N", "eins"),
                ("S", "zwei")])]),
        ("N", "drei")])

    assert expected == produced.tree

    # FIXME: only contains root node
    produced = example2tree('eins-zwei-drei-(circ-(circ-eins-to-zwei)-to-drei.rs3')
    expected = t("circumstance", [
        ("S", [
            ("circumstance", [
                ("S", "eins"),
                ("N", "zwei")])]),
        ("N", "drei")])

    assert expected == produced.tree

def test_nested_nucsat_nucnuc_relation():
    # CORRECT
    produced = example2tree('eins-zwei-drei-(circ-eins-to-(joint-zwei-and-drei).rs3')
    expected = t("circumstance", [
        ("S", "eins"),
        ("N", [
            ("joint", [
                ("N", "zwei"),
                ("N", "drei")])])])

    assert expected == produced.tree

    # CORRECT
    produced = example2tree('eins-zwei-drei-(circ-(joint-eins-and-zwei)-from-drei).rs3')
    expected = t("circumstance", [
        ("N", [
            ("joint", [
                ("N", "eins"),
                ("N", "zwei")
            ])
        ]),
        ("S", "drei")])

    assert expected == produced.tree

    # FIXME, only contains root node
    produced = example2tree('eins-zwei-drei-(circ-eins-from-(joint-zwei-and-drei).rs3')
    expected = t("circumstance", [
        ("N", "eins"),
        ("S", [
            ("joint", [
                ("N", "zwei"),
                ("N", "drei")])])])

    assert expected == produced.tree

    # FIXME, only contains root node
    produced = example2tree('eins-zwei-drei-(circ-(joint-eins-and-zwei)-to-drei).rs3')
    expected = t("circumstance", [
        ("S", [
            ("joint", [
                ("N", "eins"),
                ("N", "zwei")
            ])
        ]),
        ("N", "drei")])

    assert expected == produced.tree

    # FIXME, only root node
    produced = example2tree('eins-zwei-drei-(elab-eins-from-(joint-zwei-and-drei).rs3')
    expected = t("elaboration", [
        ("N", [
            ("joint", [
                ("N", "eins"),
                ("N", "zwei")
            ])
        ]),
        ("S", "drei")])

    assert expected == produced.tree
<|endoftext|>


<commit_before># use imagemagick to convert
# them all to a sequence of thumbnail images
# requires sudo apt-get install imagemagick

import os
import os.path
import time
from subprocess import Popen

os.system('mkdir -p static/thumbs')
os.system('mkdir -p tmp') # for intermediate files

relpath = "pdf"
allFiles = os.listdir(relpath)
pdfs = [x for x in allFiles if x.endswith(".pdf")]

for i,p in enumerate(pdfs):
  fullpath = os.path.join(relpath, p)
  outpath = os.path.join('static', 'thumbs', p + '.jpg')

  if os.path.isfile(outpath):
    print 'skipping %s, exists.' % (fullpath, )
    continue

  print "%d/%d processing %s" % (i, len(pdfs), p)

  # take first 8 pages of the pdf ([0-7]), since 9th page are references
  # tile them horizontally, use JPEG compression 80, trim the borders for each image
  #cmd = "montage %s[0-7] -mode Concatenate -tile x1 -quality 80 -resize x230 -trim %s" % (fullpath, "thumbs/" + f + ".jpg")
  #print "EXEC: " + cmd

  # nvm, below using a roundabout alternative that is worse and requires temporary files, yuck!
  # but i found that it succeeds more often. I can't remember wha thappened anymore but I remember
  # that the version above, while more elegant, had some problem with it on some pdfs. I think.

  # erase previous intermediate files test-*.png
  if os.path.isfile('tmp/test-0.png'):
    for i in xrange(8):
      f = 'tmp/test-%d.png' % (i,)
      f2= 'tmp/testbuf-%d.png' % (i,)
      if os.path.isfile(f):
        cmd = 'mv %s %s' % (f, f2)
        os.system(cmd)
        # okay originally I was going to issue an rm call, but I am too terrified of
        # running scripted rm queries, so what we will do is instead issue a "mv" call
        # to rename the files. That's a bit safer, right? We have to do this because if
        # some papers are shorter than 8 pages, then results from previous paper will
        # "leek" over to this result, through the intermediate files.

  # spawn async. convert can unfortunately enter an infinite loop, have to handle this
  pp = Popen(['convert', "%s[0-7]" % (fullpath, ), "-thumbnail", "x156", "tmp/test.png"])
  t0 = time.time()
  while time.time() - t0 < 15: # give it 15 seconds deadline
    ret = pp.poll()
    if not (ret is None):
      # process terminated
      break
    time.sleep(0.1)
  ret = pp.poll()
  if ret is None:
    # we did not terminate in 5 seconds
    pp.terminate() # give up

  if not os.path.isfile('tmp/test-0.png'):
    # failed to render pdf, replace with missing image
    os.system('cp %s %s' % ('static/thumbs/missing.jpg', outpath))
    print 'could not render pdf, creating a missing image placeholder'
  else:
    cmd = "montage -mode concatenate -quality 80 -tile x1 tmp/test-*.png %s" % (outpath, )
    print cmd
    os.system(cmd)

  time.sleep(0.01) # silly way for allowing for ctrl+c termination
<commit_msg>removed temporary thumbnail iteration<commit_after># use imagemagick to convert
# them all to a sequence of thumbnail images
# requires sudo apt-get install imagemagick

import os
import os.path
import time
from subprocess import Popen

os.system('mkdir -p static/thumbs')

relpath = "pdf"
allFiles = os.listdir(relpath)
pdfs = [x for x in allFiles if x.endswith(".pdf")]

for i,p in enumerate(pdfs):
  fullpath = os.path.join(relpath, p)
  outpath = os.path.join('static', 'thumbs', p + '.jpg')

  if os.path.isfile(outpath):
    print 'skipping %s, exists.' % (fullpath, )
    continue

  print "%d/%d processing %s" % (i, len(pdfs), p)

  # spawn async. convert can unfortunately enter an infinite loop, have to handle this
  # convert:
  #	- path/to/file.pdf[start-end] : operate on file, pages from start to end included
  #	- thumbnail [w]x[h]: create a [Width]X[height] thumbnail of the page
  # 	- quality [0-100]: output png quality
  #	- +append : append an image sequence left to right
  #	- path/to/output.png
  pp = Popen([
  	'convert',
	"%s[0-7]" % (fullpath, ),
	"-thumbnail",
	"x156",
	"-quality",
	"80",
	"+append",
	"%s" % (outpath, ),
	])

  t0 = time.time()
  while time.time() - t0 < 15: # give it 15 seconds deadline
    ret = pp.poll()
    if not (ret is None):
      # process terminated
      break
    time.sleep(0.1)
  ret = pp.poll()
  if ret is None:
    # we did not terminate in 5 seconds
    pp.terminate() # give up

  if not os.path.isfile(outpath):
    # failed to render pdf, replace with missing image
    os.system('cp %s %s' % ('static/thumbs/missing.jpg', outpath))
    print 'could not render pdf, creating a missing image placeholder'

  time.sleep(0.01) # silly way for allowing for ctrl+c termination
<|endoftext|>


<commit_before>"""
Sources: Tess Snyder, https://docs.python.org/2/tutorial/datastructures.html

"""
from ggame import App, Color, LineStyle, Sprite, RectangleAsset, CircleAsset, EllipseAsset, PolygonAsset
import random

black = Color(0x000000, 1.0)
white = Color(0xffffff, 1.0)
clear = Color(0xffffff, 0.0)
red = Color(0xff0000, 1.0)
blue = Color(0x0000ff, 1.0)
whiteline = LineStyle(1,white)
thinline = LineStyle(1, black)
thickline = LineStyle(5, black)
medline = LineStyle(4, black)
oline = LineStyle(8, blue)
xline = LineStyle(8, red)

vline = RectangleAsset(10, 310, thinline, black)
hline = RectangleAsset(310, 10, thinline, black)
oshape = CircleAsset(30, oline, clear)
xshape = PolygonAsset([(5,5),(35,35),(65,5),(5,65),(35,35),(65,65)], xline, red)

w = PolygonAsset([(5,5),(15,30),(25,15),(35,30),(45,5),(35,30),(25,15),(15,30)], thickline, clear)
i = PolygonAsset([(5,0),(5,30)], thickline, clear)
n = PolygonAsset([(5,30),(5,5),(25,25),(25,0),(25,25),(5,5),(5,30)], thickline, clear)
s = CircleAsset(7.5, medline, clear)
scover = CircleAsset(5, whiteline, white)

Sprite(w, (400, 160))
Sprite(i, (455, 165))
Sprite(n, (470, 165))
Sprite(s, (513, 174))
Sprite(s, (513, 187))
Sprite(scover, (520,174))
Sprite(scover, (505,187))

Sprite(vline, (140, 40))
Sprite(vline, (240, 40))
Sprite(hline, (40, 140))
Sprite(hline, (40, 240))
osprites = [ ]
xsprites = [ ]

for x in [95, 195, 295]:
    for y in [95, 195, 295]:
        so = Sprite(oshape, (x,y))
        so.visible = False
        osprites.append(so)

for x in [60, 160, 260]:
    for y in [60, 160, 260]:
        sx = Sprite(xshape, (x,y))
        sx.visible = False
        xsprites.append(sx)

choice = input("Would you like to be X's or O's? ")
player = 0

while player==0:
    if choice=="x":
        player = xsprites
        comp = osprites

    elif choice=="o":
        player = osprites
        comp = xsprites

    else:
        choice = input("Invalid input. Try again. Would you like to be X's or O's? ")

class Ttt(App):

    def __init__(self):
        super().__init__()
        self.listenMouseEvent( 'click', self.click)

    def click(self, event):
        x = event.x
        y = event.y
        for s in player:
            if abs(x - s.x) <= 40 and abs(y - s.y) <= 40:
                s.visible = True
                player.remove(s)

                for sprite in comp:
                    if abs(sprite.x - s.x) <= 40 and abs(sprite.y - s.y) <= 40:
                        comp.remove(sprite)

                xorandom = random.choice(comp)
                xorandom.visible = True

                comp.remove(xorandom)
                for spr in player:
                    if abs(spr.x - xorandom.x) <= 40 and abs(xorandom.y - spr.y) <= 40:
                        player.remove(spr)

myapp = Ttt()
myapp.run()
<commit_msg>Updated from Brython Server: 4/27/2016 1:29:01 PM<commit_after>"""
Sources: Tess Snyder, https://docs.python.org/2/tutorial/datastructures.html

"""
from ggame import App, Color, LineStyle, Sprite, RectangleAsset, CircleAsset, EllipseAsset, PolygonAsset
import random

black = Color(0x000000, 1.0)
white = Color(0xffffff, 1.0)
clear = Color(0xffffff, 0.0)
red = Color(0xff0000, 1.0)
blue = Color(0x0000ff, 1.0)
whiteline = LineStyle(1,white)
thinline = LineStyle(1, black)
thickline = LineStyle(5, black)
medline = LineStyle(4, black)
oline = LineStyle(8, blue)
xline = LineStyle(8, red)

vline = RectangleAsset(10, 310, thinline, black)
hline = RectangleAsset(310, 10, thinline, black)
oshape = CircleAsset(30, oline, clear)
xshape = PolygonAsset([(5,5),(35,35),(65,5),(5,65),(35,35),(65,65)], xline, red)

w = PolygonAsset([(5,5),(15,30),(25,15),(35,30),(45,5),(35,30),(25,15),(15,30)], thickline, clear)
i = PolygonAsset([(5,0),(5,30)], thickline, clear)
n = PolygonAsset([(5,30),(5,5),(25,25),(25,0),(25,25),(5,5),(5,30)], thickline, clear)
s = CircleAsset(7.5, medline, clear)
scover = CircleAsset(5, whiteline, white)

Sprite(w, (400, 160))
Sprite(i, (455, 165))
Sprite(n, (470, 165))
Sprite(s, (513, 174))
Sprite(s, (513, 187))
Sprite(scover, (520,174))
Sprite(scover, (503,187))

Sprite(vline, (140, 40))
Sprite(vline, (240, 40))
Sprite(hline, (40, 140))
Sprite(hline, (40, 240))
osprites = [ ]
xsprites = [ ]

for x in [95, 195, 295]:
    for y in [95, 195, 295]:
        so = Sprite(oshape, (x,y))
        so.visible = False
        osprites.append(so)

for x in [60, 160, 260]:
    for y in [60, 160, 260]:
        sx = Sprite(xshape, (x,y))
        sx.visible = False
        xsprites.append(sx)

choice = input("Would you like to be X's or O's? ")
player = 0

while player==0:
    if choice=="x":
        player = xsprites
        comp = osprites

    elif choice=="o":
        player = osprites
        comp = xsprites

    else:
        choice = input("Invalid input. Try again. Would you like to be X's or O's? ")

class Ttt(App):

    def __init__(self):
        super().__init__()
        self.listenMouseEvent( 'click', self.click)

    def click(self, event):
        x = event.x
        y = event.y
        for s in player:
            if abs(x - s.x) <= 40 and abs(y - s.y) <= 40:
                s.visible = True
                player.remove(s)

                for sprite in comp:
                    if abs(sprite.x - s.x) <= 40 and abs(sprite.y - s.y) <= 40:
                        comp.remove(sprite)

                xorandom = random.choice(comp)
                xorandom.visible = True

                comp.remove(xorandom)
                for spr in player:
                    if abs(spr.x - xorandom.x) <= 40 and abs(xorandom.y - spr.y) <= 40:
                        player.remove(spr)

myapp = Ttt()
myapp.run()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import NoAlertPresentException
import unittest, time, re
import random
class TestRegister(unittest.TestCase):

	username = "test_user_" + str(random.randint(1, 10000000))

	def setUp(self):
		self.driver = webdriver.Firefox()
		self.driver.implicitly_wait(30)
		self.base_url = "http://localhost:8000"
		self.verificationErrors = []
		self.accept_next_alert = True

	def test_register_normal(self):
		driver = self.driver
		driver.get(self.base_url + "/login")
		driver.find_element_by_id("reg").click()
		driver.find_element_by_id("first_name").clear()
		driver.find_element_by_id("first_name").send_keys("test")
		driver.find_element_by_id("last_name").clear()
		driver.find_element_by_id("last_name").send_keys("user")
		driver.find_element_by_id("password").clear()
		driver.find_element_by_id("password").send_keys("Aa1aaaa!")
		driver.find_element_by_id("confirm").clear()
		driver.find_element_by_id("confirm").send_keys("Aa1aaaa!")
		driver.find_element_by_id("email").clear()
		driver.find_element_by_id("email").send_keys(self.username + "@example.com")
		driver.find_element_by_id("submit").click()
		self.assertEqual("http://localhost:8000/", driver.current_url)
		# Logout the user so that login test can pass
		driver.find_element_by_xpath("//div[@id='bs-example-navbar-collapse-1']/ul[2]/li/a").click()
		driver.find_element_by_link_text("Logout").click()
		self.assertEqual("http://localhost:8000/login", driver.current_url)

	def test_register_bad_password(self):
		driver = self.driver
		driver.get(self.base_url + "/login")
		driver.find_element_by_id("reg").click()
		driver.find_element_by_id("first_name").clear()
		driver.find_element_by_id("first_name").send_keys("test")
		driver.find_element_by_id("last_name").clear()
		driver.find_element_by_id("last_name").send_keys("user")
		driver.find_element_by_id("password").clear()
		driver.find_element_by_id("password").send_keys("a")
		driver.find_element_by_id("confirm").clear()
		driver.find_element_by_id("confirm").send_keys("a")
		driver.find_element_by_id("email").clear()
		driver.find_element_by_id("email").send_keys(self.username + "@example.com")
		driver.find_element_by_id("submit").click()
		self.assertEqual("Password is too weak, make sure it meets all of the required conditions", driver.find_element_by_xpath("//div[2]/div/div[2]").text)
		self.assertEqual("Password must be between 8 and 64 characters long", driver.find_element_by_css_selector("div.alert.alert-danger").text)

	def test_register_bad_email_form(self):
		driver = self.driver
		driver.get(self.base_url + "/login")
		driver.find_element_by_id("reg").click()
		driver.find_element_by_id("first_name").clear()
		driver.find_element_by_id("first_name").send_keys("test")
		driver.find_element_by_id("last_name").clear()
		driver.find_element_by_id("last_name").send_keys("user")
		driver.find_element_by_id("password").clear()
		driver.find_element_by_id("password").send_keys("Aa1aaaa!")
		driver.find_element_by_id("confirm").clear()
		driver.find_element_by_id("confirm").send_keys("Aa1aaaa!")
		driver.find_element_by_id("email").clear()
		driver.find_element_by_id("email").send_keys(self.username + "example.com")
		driver.find_element_by_id("submit").click()
		self.assertEqual("Invalid email address. It must be *@*.* or similar.", driver.find_element_by_css_selector("div.alert.alert-danger").text)

	def test_register_bad_email_short(self):
		driver = self.driver
		driver.get(self.base_url + "/login")
		driver.find_element_by_id("reg").click()
		driver.find_element_by_id("first_name").clear()
		driver.find_element_by_id("first_name").send_keys("test")
		driver.find_element_by_id("last_name").clear()
		driver.find_element_by_id("last_name").send_keys("user")
		driver.find_element_by_id("password").clear()
		driver.find_element_by_id("password").send_keys("Aa1aaaa!")
		driver.find_element_by_id("confirm").clear()
		driver.find_element_by_id("confirm").send_keys("Aa1aaaa!")
		driver.find_element_by_id("email").clear()
		driver.find_element_by_id("email").send_keys("e@e.c")
		driver.find_element_by_id("submit").click()
		self.assertEqual("Email must be between 6 and 64 characters long", driver.find_element_by_css_selector("div.alert.alert-danger").text)

	def is_element_present(self, how, what):
		try: self.driver.find_element(by=how, value=what)
		except NoSuchElementException as e: return False
		return True

	def is_alert_present(self):
		try: self.driver.switch_to_alert()
		except NoAlertPresentException as e: return False
		return True

	def close_alert_and_get_its_text(self):
		try:
			alert = self.driver.switch_to_alert()
			alert_text = alert.text
			if self.accept_next_alert:
				alert.accept()
			else:
				alert.dismiss()
			return alert_text
		finally: self.accept_next_alert = True

	def tearDown(self):
		self.driver.quit()
		self.assertEqual([], self.verificationErrors)

if __name__ == "__main__":
	unittest.main()
<commit_msg>fixing possible test failure due to screen size<commit_after># -*- coding: utf-8 -*-
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import NoAlertPresentException
import unittest, time, re
import random
class TestRegister(unittest.TestCase):

	username = "test_user_" + str(random.randint(1, 10000000))

	def setUp(self):
		self.driver = webdriver.Firefox()
		self.driver.maximize_window()
		self.driver.implicitly_wait(30)
		self.base_url = "http://localhost:8000"
		self.verificationErrors = []
		self.accept_next_alert = True

	def test_register_normal(self):
		driver = self.driver
		driver.get(self.base_url + "/login")
		driver.find_element_by_id("reg").click()
		driver.find_element_by_id("first_name").clear()
		driver.find_element_by_id("first_name").send_keys("test")
		driver.find_element_by_id("last_name").clear()
		driver.find_element_by_id("last_name").send_keys("user")
		driver.find_element_by_id("password").clear()
		driver.find_element_by_id("password").send_keys("Aa1aaaa!")
		driver.find_element_by_id("confirm").clear()
		driver.find_element_by_id("confirm").send_keys("Aa1aaaa!")
		driver.find_element_by_id("email").clear()
		driver.find_element_by_id("email").send_keys(self.username + "@example.com")
		driver.find_element_by_id("submit").click()
		self.assertEqual("http://localhost:8000/", driver.current_url)
		# Logout the user so that login test can pass
		driver.find_element_by_xpath("//div[@id='bs-example-navbar-collapse-1']/ul[2]/li/a").click()
		driver.find_element_by_link_text("Logout").click()
		self.assertEqual("http://localhost:8000/login", driver.current_url)

	def test_register_bad_password(self):
		driver = self.driver
		driver.get(self.base_url + "/login")
		driver.find_element_by_id("reg").click()
		driver.find_element_by_id("first_name").clear()
		driver.find_element_by_id("first_name").send_keys("test")
		driver.find_element_by_id("last_name").clear()
		driver.find_element_by_id("last_name").send_keys("user")
		driver.find_element_by_id("password").clear()
		driver.find_element_by_id("password").send_keys("a")
		driver.find_element_by_id("confirm").clear()
		driver.find_element_by_id("confirm").send_keys("a")
		driver.find_element_by_id("email").clear()
		driver.find_element_by_id("email").send_keys(self.username + "@example.com")
		driver.find_element_by_id("submit").click()
		self.assertEqual("Password is too weak, make sure it meets all of the required conditions", driver.find_element_by_xpath("//div[2]/div/div[2]").text)
		self.assertEqual("Password must be between 8 and 64 characters long", driver.find_element_by_css_selector("div.alert.alert-danger").text)

	def test_register_bad_email_form(self):
		driver = self.driver
		driver.get(self.base_url + "/login")
		driver.find_element_by_id("reg").click()
		driver.find_element_by_id("first_name").clear()
		driver.find_element_by_id("first_name").send_keys("test")
		driver.find_element_by_id("last_name").clear()
		driver.find_element_by_id("last_name").send_keys("user")
		driver.find_element_by_id("password").clear()
		driver.find_element_by_id("password").send_keys("Aa1aaaa!")
		driver.find_element_by_id("confirm").clear()
		driver.find_element_by_id("confirm").send_keys("Aa1aaaa!")
		driver.find_element_by_id("email").clear()
		driver.find_element_by_id("email").send_keys(self.username + "example.com")
		driver.find_element_by_id("submit").click()
		self.assertEqual("Invalid email address. It must be *@*.* or similar.", driver.find_element_by_css_selector("div.alert.alert-danger").text)

	def test_register_bad_email_short(self):
		driver = self.driver
		driver.get(self.base_url + "/login")
		driver.find_element_by_id("reg").click()
		driver.find_element_by_id("first_name").clear()
		driver.find_element_by_id("first_name").send_keys("test")
		driver.find_element_by_id("last_name").clear()
		driver.find_element_by_id("last_name").send_keys("user")
		driver.find_element_by_id("password").clear()
		driver.find_element_by_id("password").send_keys("Aa1aaaa!")
		driver.find_element_by_id("confirm").clear()
		driver.find_element_by_id("confirm").send_keys("Aa1aaaa!")
		driver.find_element_by_id("email").clear()
		driver.find_element_by_id("email").send_keys("e@e.c")
		driver.find_element_by_id("submit").click()
		self.assertEqual("Email must be between 6 and 64 characters long", driver.find_element_by_css_selector("div.alert.alert-danger").text)

	def is_element_present(self, how, what):
		try: self.driver.find_element(by=how, value=what)
		except NoSuchElementException as e: return False
		return True

	def is_alert_present(self):
		try: self.driver.switch_to_alert()
		except NoAlertPresentException as e: return False
		return True

	def close_alert_and_get_its_text(self):
		try:
			alert = self.driver.switch_to_alert()
			alert_text = alert.text
			if self.accept_next_alert:
				alert.accept()
			else:
				alert.dismiss()
			return alert_text
		finally: self.accept_next_alert = True

	def tearDown(self):
		self.driver.quit()
		self.assertEqual([], self.verificationErrors)

if __name__ == "__main__":
	unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# This work was created by participants in the DataONE project, and is
# jointly copyrighted by participating institutions in DataONE. For
# more information on DataONE, see our web site at http://dataone.org.
#
#   Copyright 2013 DataONE
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
:mod:`setup`
============

:Synopsis: Create egg.
:Author: DataONE (Dahl)
:Notes:
  The script should be run in the current directory with ./setup.py.

  The setup() parameters are described here:
  http://pythonhosted.org/setuptools/setuptools.html
  http://docs.python.org/2/distutils/setupscript.html

  This setup script excludes unversioned python modules from the build if pysvn
  is installed. The only way I found to do this was to write a list of exclude
  directives to MANIFEST.in before calling setup().

  Unversioned data packages are automatically excluded as only versioned files
  are included with the "include_package_data = True" parameter.

  When creating the package, some warnings on the form:
  warning: no previously-included files found matching '<file>'
  will be generated, as the MANIFEST.in file will contain exclude directives for
  files that would not have been included
"""
import locale
import os
import re
import setuptools
import sys

from setuptools import setup, find_packages

# py2exe only needs to be present when building an executable on Windows.
# When py2exe is not present, the py2exe specific options below are ignored.
try:
  import py2exe
except ImportError:
  pass

try:
  import pysvn
except ImportError:
  has_svn = False
else:
  has_svn = True

import d1_client_onedrive

# http://www.py2exe.org/index.cgi/WorkingWithVariousPackagesAndModules
#
# lxml
#
# if missing _elementhpath, either pull whole lxml library in packages=..., or
# put "from lxml import _elementhpath as _dummy" somewhere in code; in both
# cases also pull gzip in packages=...

# Windows executable setup
if sys.platform == 'win32':
  opts = {
    'py2exe': {
      'packages': [
        'd1_client_onedrive',
        'd1_client_onedrive.impl',
        'd1_client_onedrive.impl.drivers',
        'd1_client_onedrive.impl.drivers.dokan',
        'd1_client_onedrive.impl.drivers.fuse',
        'd1_client_onedrive.impl.resolver',
        'rdflib.plugins',
        'lxml'
      ],
      'skip_archive': True,
    }
  }
  extra_opts = dict(console=['d1_client_onedrive/onedrive.py'], )
# Mac App setup
elif sys.platform == 'darwin':
  opts = dict(
    py2app=dict(
      argv_emulation=True,
      iconfile='mac/mac_dataone.icns',
      packages=['rdflib', 'lxml'],
      site_packages=True,
      resources=['d1_client_onedrive/impl/d1.icon']
      #resources = ['mime_mappings.csv',]
    )
  )
  extra_opts = dict(
    #app = ['d1_client_onedrive/onedrive.py'],
    app=['mac/start_app.py'],
    setup_requires=['py2app'],
  )
# Normal setup
else:
  opts = dict()
  extra_opts = dict()

# Metadata.
name = 'dataone.generic_member_node'
version = d1_client_onedrive.__version__
description = 'DataONE Generic Member Node (GMN)'
author = 'DataONE Project'
author_email = 'developers@dataone.org'
url = 'http://dataone.org'
license = 'Apache License, Version 2.0'

def main():
  # Enable SVN to work with filenames containing Unicode.
  locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')

  if has_svn:
    create_manifest_in()

  setuptools.setup(
    # Metadata
    name=name,
    version=version,
    description=description,
    author=author,
    author_email=author_email,
    url=url,
    license=license,
    # Dependencies that are available through PyPI.
    install_requires=[
      'dataone.common == 1.1.3',
      'dataone.libclient == 1.2.3',
      'dataone.workspace_client == 0.0.2',
      'fusepy',
      'rdflib',
    ],
    data_files=[
      ('d1_client', ['./mime_mappings.csv']),
      (
        '', [
          './workspace.xml'
        ]
      ),
    ],
    # Options for py2exe and py2app.
    options=opts,
    **extra_opts
  )

def create_manifest_in():
  with open('MANIFEST.in', 'w') as f:
    for p in get_unversioned_python_modules():
      f.write(u'exclude {0}\n'.format(p).encode('utf-8'))

def get_unversioned_python_modules():
  paths = []
  client = pysvn.Client()
  for s in client.status('.'):
    if not os.path.isfile(s.path):
      continue
    if not os.path.splitext(s.path)[1] == '.py':
      continue
    if s.is_versioned:
      continue
    paths.append(s.path)
  return paths

if __name__ == '__main__':
  main()
<commit_msg>Fixed name and description (copy/paste error)<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# This work was created by participants in the DataONE project, and is
# jointly copyrighted by participating institutions in DataONE. For
# more information on DataONE, see our web site at http://dataone.org.
#
#   Copyright 2013 DataONE
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
:mod:`setup`
============

:Synopsis: Create egg.
:Author: DataONE (Dahl)
:Notes:
  The script should be run in the current directory with ./setup.py.

  The setup() parameters are described here:
  http://pythonhosted.org/setuptools/setuptools.html
  http://docs.python.org/2/distutils/setupscript.html

  This setup script excludes unversioned python modules from the build if pysvn
  is installed. The only way I found to do this was to write a list of exclude
  directives to MANIFEST.in before calling setup().

  Unversioned data packages are automatically excluded as only versioned files
  are included with the "include_package_data = True" parameter.

  When creating the package, some warnings on the form:
  warning: no previously-included files found matching '<file>'
  will be generated, as the MANIFEST.in file will contain exclude directives for
  files that would not have been included
"""
import locale
import os
import re
import setuptools
import sys

from setuptools import setup, find_packages

# py2exe only needs to be present when building an executable on Windows.
# When py2exe is not present, the py2exe specific options below are ignored.
try:
  import py2exe
except ImportError:
  pass

try:
  import pysvn
except ImportError:
  has_svn = False
else:
  has_svn = True

import d1_client_onedrive

# http://www.py2exe.org/index.cgi/WorkingWithVariousPackagesAndModules
#
# lxml
#
# if missing _elementhpath, either pull whole lxml library in packages=..., or
# put "from lxml import _elementhpath as _dummy" somewhere in code; in both
# cases also pull gzip in packages=...

# Windows executable setup
if sys.platform == 'win32':
  opts = {
    'py2exe': {
      'packages': [
        'd1_client_onedrive',
        'd1_client_onedrive.impl',
        'd1_client_onedrive.impl.drivers',
        'd1_client_onedrive.impl.drivers.dokan',
        'd1_client_onedrive.impl.drivers.fuse',
        'd1_client_onedrive.impl.resolver',
        'rdflib.plugins',
        'lxml'
      ],
      'skip_archive': True,
    }
  }
  extra_opts = dict(console=['d1_client_onedrive/onedrive.py'], )
# Mac App setup
elif sys.platform == 'darwin':
  opts = dict(
    py2app=dict(
      argv_emulation=True,
      iconfile='mac/mac_dataone.icns',
      packages=['rdflib', 'lxml'],
      site_packages=True,
      resources=['d1_client_onedrive/impl/d1.icon']
      #resources = ['mime_mappings.csv',]
    )
  )
  extra_opts = dict(
    #app = ['d1_client_onedrive/onedrive.py'],
    app=['mac/start_app.py'],
    setup_requires=['py2app'],
  )
# Normal setup
else:
  opts = dict()
  extra_opts = dict()

# Metadata.
name = ' dataone.onedrive'
version = d1_client_onedrive.__version__
description = 'Filesystem access to the DataONE Workspace'
author = 'DataONE Project'
author_email = 'developers@dataone.org'
url = 'http://dataone.org'
license = 'Apache License, Version 2.0'

def main():
  # Enable SVN to work with filenames containing Unicode.
  locale.setlocale(locale.LC_ALL, 'en_US.UTF-8')

  if has_svn:
    create_manifest_in()

  setuptools.setup(
    # Metadata
    name=name,
    version=version,
    description=description,
    author=author,
    author_email=author_email,
    url=url,
    license=license,
    # Dependencies that are available through PyPI.
    install_requires=[
      'dataone.common == 1.1.3',
      'dataone.libclient == 1.2.3',
      'dataone.workspace_client == 0.0.2',
      'fusepy',
      'rdflib',
    ],
    data_files=[
      ('d1_client', ['./mime_mappings.csv']),
      (
        '', [
          './workspace.xml'
        ]
      ),
    ],
    # Options for py2exe and py2app.
    options=opts,
    **extra_opts
  )

def create_manifest_in():
  with open('MANIFEST.in', 'w') as f:
    for p in get_unversioned_python_modules():
      f.write(u'exclude {0}\n'.format(p).encode('utf-8'))

def get_unversioned_python_modules():
  paths = []
  client = pysvn.Client()
  for s in client.status('.'):
    if not os.path.isfile(s.path):
      continue
    if not os.path.splitext(s.path)[1] == '.py':
      continue
    if s.is_versioned:
      continue
    paths.append(s.path)
  return paths

if __name__ == '__main__':
  main()
<|endoftext|>


<commit_before>import asyncio

from ._testutil import BaseTest, run_until_complete

class TaskCancellationTest(BaseTest):

    @run_until_complete
    def test_future_cancellation(self):
        conn = yield from self.create_connection(
            ('localhost', 6379), loop=self.loop)

        ts = self.loop.time()
        fut = conn.execute('BLPOP', 'some-list', 5)
        with self.assertRaises(asyncio.TimeoutError):
            yield from asyncio.wait_for(fut, 1, loop=self.loop)
        self.assertTrue(fut.cancelled())

        # XXX: Connection becomes available only after timeout expires
        yield from conn.execute('TIME')
        dt = self.loop.time() - ts
        self.assertAlmostEqual(dt, 5.0, delta=1)
<commit_msg>more stable task_cancellation_test<commit_after>import asyncio

from ._testutil import BaseTest, run_until_complete

class TaskCancellationTest(BaseTest):

    @run_until_complete
    def test_future_cancellation(self):
        conn = yield from self.create_connection(
            ('localhost', 6379), loop=self.loop)

        ts = self.loop.time()
        fut = conn.execute('BLPOP', 'some-list', 5)
        with self.assertRaises(asyncio.TimeoutError):
            yield from asyncio.wait_for(fut, 1, loop=self.loop)
        self.assertTrue(fut.cancelled())

        # NOTE: Connection becomes available only after timeout expires
        yield from conn.execute('TIME')
        dt = int(self.loop.time() - ts)
        self.assertIn(dt, {4, 5, 6})
        # self.assertAlmostEqual(dt, 5.0, delta=1)  # this fails too often
<|endoftext|>


<commit_before>import unittest
import docker
import requests.exceptions
import tempfile
import os
import shutil
import subprocess

# Tests for crawlers in kraken crawlers configuration.

# Tests conducted with a single container running.
class SingleContainerTests(unittest.TestCase):

    def setUp(self):
        self.docker = docker.Client(base_url='unix://var/run/docker.sock', version='auto')
        try:
            if len(self.docker.containers()) != 0:
                raise Exception("Sorry, this test requires a machine with no docker containers running.")
        except requests.exceptions.ConnectionError as e:
            print "Error connecting to docker daemon, are you in the docker group? You need to be in the docker group."

        self.docker.pull(repository='alpine', tag='latest')
        self.container = self.docker.create_container(image='alpine:latest', command='/bin/sleep 60')
        self.tempd = tempfile.mkdtemp(prefix='crawlertest.')
        self.docker.start(container=self.container['Id'])

    def tearDown(self):
        self.docker.stop(container=self.container['Id'])
        self.docker.remove_container(container=self.container['Id'])

        shutil.rmtree(self.tempd)

    def testCrawlContainer(self):
        env = os.environ.copy()
        mypath = os.path.dirname(os.path.realpath(__file__))
        os.makedirs(self.tempd + '/out')

        #crawler itself needs to be root
        process = subprocess.Popen(
            [
                '/usr/bin/python2.7', mypath + '/../crawler/crawler.py',
                '--url', 'file://' + self.tempd + '/out/crawler',
                '--features', 'cpu,memory,interface',
                '--crawlContainers', 'ALL',
                '--format', 'graphite',
                '--crawlmode', 'OUTCONTAINER',
                '--numprocesses', '1'
            ],
            env=env)
        stdout, stderr = process.communicate()
        assert process.returncode == 0

        print stderr
        print stdout

        subprocess.call(['/bin/chmod', '-R', '777', self.tempd])

        files = os.listdir(self.tempd + '/out')
        assert len(files) == 1

        f = open(self.tempd + '/out/' + files[0], 'r')
        output = f.read()
        print output # only printed if the test fails
        assert 'interface-lo.if_octets.tx' in output
        assert 'cpu-0.cpu-idle' in output
        assert 'memory.memory-used' in output
        f.close()

if __name__ == '__main__':
    unittest.main()
<commit_msg>The test_crawler_container should call python call directly<commit_after>import unittest
import docker
import requests.exceptions
import tempfile
import os
import shutil
import subprocess

# Tests for crawlers in kraken crawlers configuration.

import crawler.crawlutils

# Tests conducted with a single container running.

class SingleContainerTests(unittest.TestCase):

    def setUp(self):
        self.docker = docker.Client(
            base_url='unix://var/run/docker.sock', version='auto')
        try:
            if len(self.docker.containers()) != 0:
                raise Exception(
                    "Sorry, this test requires a machine with no docker containers running.")
        except requests.exceptions.ConnectionError as e:
            print "Error connecting to docker daemon, are you in the docker group? You need to be in the docker group."

        self.docker.pull(repository='alpine', tag='latest')
        self.container = self.docker.create_container(
            image='alpine:latest', command='/bin/sleep 60')
        self.tempd = tempfile.mkdtemp(prefix='crawlertest.')
        self.docker.start(container=self.container['Id'])

    def tearDown(self):
        self.docker.stop(container=self.container['Id'])
        self.docker.remove_container(container=self.container['Id'])

        shutil.rmtree(self.tempd)

    def testCrawlContainer1(self):
        env = os.environ.copy()
        mypath = os.path.dirname(os.path.realpath(__file__))
        os.makedirs(self.tempd + '/out')

	# Adding every single option (even if not used in this test), to make
	# writing other tests easier
	options = {
	    'features': 'cpu,memory,interface',
	    'format': 'graphite',
	    'crawlmode': 'OUTCONTAINER',
	    'urls': ['file://' + self.tempd + '/out/crawler'],
	    'options': {
		'load': {},
		'process': {},
		'metric': {},
		'logcrawler': {
		    'log_types_file': 'd464347c-3b99-11e5-b0e9-062dcffc249f.type-mapping',
		    'host_log_basedir': '/var/log/crawler_container_logs/',
		    'default_log_files': [{
			'type': None,
			'name': '/var/log/messages'
		    }, {
			'type': None,
			'name': '/etc/csf_env.properties'
		    }]
		},
		'file': {
		    'avoid_setns': False,
		    'exclude_dirs': ['boot', 'dev', 'proc', 'sys', 'mnt', 'tmp', 'var/cache', 'usr/share/man', 'usr/share/doc', 'usr/share/mime'],
		    'root_dir': '/'
		},
		'mountpoint': 'Undefined',
		'disk': {},
		'environment': 'cloudsight',
		'memory': {},
		'config': {
		    'avoid_setns': False,
		    'exclude_dirs': ['dev', 'proc', 'mnt', 'tmp', 'var/cache', 'usr/share/man', 'usr/share/doc', 'usr/share/mime'],
		    'root_dir': '/',
		    'discover_config_files': True,
		    'known_config_files': ['etc/passwd', 'etc/group', 'etc/hosts', 'etc/hostname', 'etc/mtab', 'etc/fstab', 'etc/aliases', 'etc/ssh/ssh_config', 'etc/ssh/sshd_config', 'etc/sudoers']
		},
		'metadata': {
		    'extra_metadata': {},
		    'container_long_id_to_namespace_map': {},
		    'extra_metadata_for_all': False
		},
		'dockerhistory': {},
		'compress': False,
		'interface': {},
		'_test_crash': {},
		'package': {
		    'avoid_setns': False
		},
		'docker_containers_list': 'ALL',
		'partition_strategy': {
		    'args': {
			'process_id': 0,
			'num_processes': 1
		    },
		    'name': 'equally_by_pid'
		},
		'connection': {},
		'_test_infinite_loop': {},
		'dockerinspect': {},
		'dockerps': {},
		'link_container_log_files': False,
		'os': {
		    'avoid_setns': False
		},
		'cpu': {}
	    },
	    'overwrite': False
	}

        crawler.crawlutils.snapshot(**options)

        subprocess.call(['/bin/chmod', '-R', '777', self.tempd])

        files = os.listdir(self.tempd + '/out')
        assert len(files) == 1

        f = open(self.tempd + '/out/' + files[0], 'r')
        output = f.read()
        print output  # only printed if the test fails
        assert 'interface-lo.if_octets.tx' in output
        assert 'cpu-0.cpu-idle' in output
        assert 'memory.memory-used' in output
        f.close()

    def testCrawlContainer2(self):
        env = os.environ.copy()
        mypath = os.path.dirname(os.path.realpath(__file__))
        os.makedirs(self.tempd + '/out')

        # crawler itself needs to be root
        process = subprocess.Popen(
            [
                '/usr/bin/python2.7', mypath + '/../crawler/crawler.py',
                '--url', 'file://' + self.tempd + '/out/crawler',
                '--features', 'cpu,memory,interface',
                '--crawlContainers', 'ALL',
                '--format', 'graphite',
                '--crawlmode', 'OUTCONTAINER',
                '--numprocesses', '1'
            ],
            env=env)
        stdout, stderr = process.communicate()
        assert process.returncode == 0

        print stderr
        print stdout

        subprocess.call(['/bin/chmod', '-R', '777', self.tempd])

        files = os.listdir(self.tempd + '/out')
        assert len(files) == 1

        f = open(self.tempd + '/out/' + files[0], 'r')
        output = f.read()
        print output  # only printed if the test fails
        assert 'interface-lo.if_octets.tx' in output
        assert 'cpu-0.cpu-idle' in output
        assert 'memory.memory-used' in output
        f.close()

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from decimal import Decimal
from sqlalchemy_i18n import find_translations
from tests import TestCase

class TestFindTranslations(TestCase):
    locales = ['en', 'fi', 'sv']

    def test_with_full_confidence(self):
        article = self.Article()
        article.translations['en'].name = u'Some article'
        article.translations['en'].content = u'Some content'
        article.translations['fi'].name = u'Joku artikkeli'
        article.translations['fi'].content = u'Jotain tekstii'
        self.session.add(article)
        self.session.commit()

        article = self.Article()
        article.translations['en'].name = u'Some article'
        self.session.add(article)
        self.session.commit()
        assert dict(find_translations(article, 'name', 'fi')) == {
            u'Joku artikkeli': Decimal(1.0)
        }

    def test_with_varying_confidence(self):
        article = self.Article()
        article.translations['en'].name = u'Some article'
        article.translations['en'].content = u'Some content'
        article.translations['fi'].name = u'Joku artikkeli'
        article.translations['fi'].content = u'Jotain tekstii'
        self.session.add(article)
        article = self.Article()
        article.translations['en'].name = u'Some article'
        article.translations['en'].content = u'Some content'
        article.translations['fi'].name = u'Joku toinen artikkeli'
        article.translations['fi'].content = u'Jotain muuta tekstii'
        self.session.add(article)

        self.session.commit()

        article = self.Article()
        article.translations['en'].name = u'Some article'
        self.session.add(article)
        self.session.commit()

        assert dict(find_translations(article, 'name', 'fi').all()) == {
            u'Joku artikkeli': Decimal(0.5),
            u'Joku toinen artikkeli': Decimal(0.5)
        }
<commit_msg>Fixed py2.6 compatibility<commit_after># -*- coding: utf-8 -*-
from decimal import Decimal
from sqlalchemy_i18n import find_translations
from tests import TestCase

class TestFindTranslations(TestCase):
    locales = ['en', 'fi', 'sv']

    def test_with_full_confidence(self):
        article = self.Article()
        article.translations['en'].name = u'Some article'
        article.translations['en'].content = u'Some content'
        article.translations['fi'].name = u'Joku artikkeli'
        article.translations['fi'].content = u'Jotain tekstii'
        self.session.add(article)
        self.session.commit()

        article = self.Article()
        article.translations['en'].name = u'Some article'
        self.session.add(article)
        self.session.commit()
        assert dict(find_translations(article, 'name', 'fi')) == {
            u'Joku artikkeli': Decimal('1.0')
        }

    def test_with_varying_confidence(self):
        article = self.Article()
        article.translations['en'].name = u'Some article'
        article.translations['en'].content = u'Some content'
        article.translations['fi'].name = u'Joku artikkeli'
        article.translations['fi'].content = u'Jotain tekstii'
        self.session.add(article)
        article = self.Article()
        article.translations['en'].name = u'Some article'
        article.translations['en'].content = u'Some content'
        article.translations['fi'].name = u'Joku toinen artikkeli'
        article.translations['fi'].content = u'Jotain muuta tekstii'
        self.session.add(article)

        self.session.commit()

        article = self.Article()
        article.translations['en'].name = u'Some article'
        self.session.add(article)
        self.session.commit()

        assert dict(find_translations(article, 'name', 'fi').all()) == {
            u'Joku artikkeli': Decimal('0.5'),
            u'Joku toinen artikkeli': Decimal('0.5')
        }
<|endoftext|>