<commit_before>import unittest

import numpy

import chainer
from chainer import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition

if cuda.available:
    cuda.init()

def _check_forward(e1, e2, f, y_expect):
    e1 = chainer.Variable(e1)
    e2 = chainer.Variable(e2)
    y = f(e1, e2)
    gradient_check.assert_allclose(y_expect, y.data)

def _check_backward(e1, e2, y_grad, f, bias):
    e1 = chainer.Variable(e1)
    e2 = chainer.Variable(e2)
    y = f(e1, e2)
    y.grad = y_grad
    y.backward()

    func = y.creator
    f = lambda: func.forward((e1.data, e2.data))

    ge1, ge2, gW = gradient_check.numerical_grad(
        f, (e1.data, e2.data, func.W), (y.grad,), eps=1e-2)

    gradient_check.assert_allclose(ge1, e1.grad)
    gradient_check.assert_allclose(ge2, e2.grad)
    gradient_check.assert_allclose(gW, func.gW)

    if bias:
        gV1, gV2, gb = gradient_check.numerical_grad(
            f, (func.V1, func.V2, func.b),
            (y.grad,), eps=1e-2)
        gradient_check.assert_allclose(gV1, func.gV1)
        gradient_check.assert_allclose(gV2, func.gV2)
        gradient_check.assert_allclose(gb, func.gb)

def _batch_to_gpu(*xs):
    return tuple(cuda.to_gpu(x) for x in xs)

class TestTensorNetwork(unittest.TestCase):

    in_shape = (2, 3)
    out_size = 4
    batch_size = 10

    def setUp(self):
        self.f = functions.TensorNetwork(
            self.in_shape[0], self.in_shape[1], self.out_size)
        self.f.W = numpy.random.uniform(
            -1, 1, self.f.W.shape).astype(numpy.float32)
        self.f.V1 = numpy.random.uniform(
            -1, 1, self.f.V1.shape).astype(numpy.float32)
        self.f.V2 = numpy.random.uniform(
            -1, 1, self.f.V2.shape).astype(numpy.float32)
        self.f.b = numpy.random.uniform(
            -1, 1, self.f.b.shape).astype(numpy.float32)
        self.f.zero_grads()

        W = self.f.W.copy()
        V1 = self.f.V1.copy()
        V2 = self.f.V2.copy()
        b = self.f.b.copy()

        self.e1 = numpy.random.uniform(
            -1, 1, (self.batch_size, self.in_shape[0])).astype(numpy.float32)
        self.e2 = numpy.random.uniform(
            -1, 1, (self.batch_size, self.in_shape[1])).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (self.batch_size, self.out_size)).astype(numpy.float32)

        self.y = (
            numpy.einsum('ij,ik,jkl->il', self.e1, self.e2, W) +
            self.e1.dot(V1) + self.e2.dot(V2) + b)

    @condition.retry(3)
    def test_forward_cpu(self):
        _check_forward(self.e1, self.e2, self.f, self.y)

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.f.to_gpu()
        _check_forward(cuda.to_gpu(self.e1),
                       cuda.to_gpu(self.e2),
                       self.f, self.y)

    @condition.retry(3)
    def test_backward_cpu(self):
        _check_backward(self.e1, self.e2, self.gy, self.f, True)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.f.to_gpu()
        _check_backward(cuda.to_gpu(self.e1),
                        cuda.to_gpu(self.e2),
                        cuda.to_gpu(self.gy),
                        self.f, True)

class TestTensorNetworkWOBias(unittest.TestCase):

    in_shape = (2, 3)
    out_size = 4
    batch_size = 10

    def setUp(self):
        self.f = functions.TensorNetwork(
            self.in_shape[0], self.in_shape[1], self.out_size, True)
        self.f.W = numpy.random.uniform(
            -1, 1, self.f.W.shape).astype(numpy.float32)
        self.f.zero_grads()

        W = self.f.W.copy()

        self.e1 = numpy.random.uniform(
            -1, 1, (self.batch_size, self.in_shape[0])).astype(numpy.float32)
        self.e2 = numpy.random.uniform(
            -1, 1, (self.batch_size, self.in_shape[1])).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (self.batch_size, self.out_size)).astype(numpy.float32)

        self.y = numpy.einsum('ij,ik,jkl->il', self.e1, self.e2, W)

    @condition.retry(3)
    def test_forward_cpu(self):
        _check_forward(self.e1, self.e2, self.f, self.y)

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.f.to_gpu()
        _check_forward(cuda.to_gpu(self.e1),
                       cuda.to_gpu(self.e2),
                       self.f, self.y)

    @condition.retry(3)
    def test_backward_cpu(self):
        _check_backward(self.e1, self.e2, self.gy, self.f, False)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.f.to_gpu()
        _check_backward(cuda.to_gpu(self.e1),
                        cuda.to_gpu(self.e2),
                        cuda.to_gpu(self.gy),
                        self.f, False)

class InitByInitialParameter(unittest.TestCase):

    in_shape = (2, 3)
    out_size = 4
    batch_size = 10

    def setUp(self):
        self.W = numpy.random.uniform(
            -1, 1,
            (self.in_shape[0], self.in_shape[1], self.out_size)
        ).astype(numpy.float32)
        self.V1 = numpy.random.uniform(
            -1, 1, (self.in_shape[0], self.out_size)).astype(numpy.float32)
        self.V2 = numpy.random.uniform(
            -1, 1, (self.in_shape[1], self.out_size)).astype(numpy.float32)
        self.b = numpy.random.uniform(
            -1, 1, (self.out_size,)).astype(numpy.float32)

class NormalInitialParameter(InitByInitialParameter):

    def check_normal(self, initialW, initial_bias, nobias):
        functions.TensorNetwork(
            self.in_shape[0], self.in_shape[1], self.out_size, nobias,
            initialW, initial_bias)

    def test_normal_cpu_bias(self):
        self.check_normal(self.W, (self.V1, self.V2, self.b), False)

    def test_normal_cpu_nobias(self):
        self.check_normal(self.W, None, False)

    @attr.gpu
    def test_normal_gpu_bias(self):
        self.check_normal(cuda.to_gpu(self.W),
                          _batch_to_gpu(self.V1, self.V2, self.b),
                          False)

    @attr.gpu
    def test_normal_gpu_nobias(self):
        self.check_normal(cuda.to_gpu(self.W), None, False)

class InvalidInitialParameter(InitByInitialParameter):

    def setUp(self):
        super(InvalidInitialParameter, self).setUp()
        self.invalidW = numpy.random.uniform(
            -1, 1,
            (self.in_shape[0]+1, self.in_shape[1], self.out_size)
        ).astype(numpy.float32)
        self.invalidV1 = numpy.random.uniform(
            -1, 1, (self.in_shape[0]+1, self.out_size)).astype(numpy.float32)
        self.invalidV2 = numpy.random.uniform(
            -1, 1, (self.in_shape[1]+1, self.out_size)).astype(numpy.float32)
        self.invalidb = numpy.random.uniform(
            -1, 1, (self.out_size+1,)).astype(numpy.float32)

    def check_invalid(self, initialW, initial_bias, nobias):
        with self.assertRaises(AssertionError):
            functions.TensorNetwork(
                self.in_shape[0], self.in_shape[1], self.out_size, nobias,
                initialW, initial_bias)

    def test_invalidW_cpu(self):
        self.check_invalid(self.invalidW, (self.V1, self.V2, self.b), False)
        self.check_invalid(self.invalidW, None, True)

    @attr.gpu
    def test_invalidW_gpu(self):
        invalidW = cuda.to_gpu(self.invalidW)
        self.check_invalid(invalidW,
                           _batch_to_gpu(self.V1, self.V2, self.b),
                           False)
        self.check_invalid(invalidW, None, True)

    def test_invalidV1_cpu(self):
        self.check_invalid(self.W, (self.invalidV1, self.V2, self.b), False)

    @attr.gpu
    def test_invaliV1_gpu(self):
        self.check_invalid(cuda.to_gpu(self.W),
                           _batch_to_gpu(self.invalidV1, self.V2, self.b),
                           False)

    def test_invalidV2_cpu(self):
        self.check_invalid(self.W, (self.V1, self.invalidV2, self.b), False)

    @attr.gpu
    def test_invaliV2_gpu(self):
        self.check_invalid(cuda.to_gpu(self.W),
                           _batch_to_gpu(self.V1, self.invalidV2, self.b),
                           False)

    def test_invalidb_cpu(self):
        self.check_invalid(self.W, (self.V1, self.V2, self.invalidb), False)

    @attr.gpu
    def test_invalib_gpu(self):
        self.check_invalid(cuda.to_gpu(self.W),
                           _batch_to_gpu(self.V1, self.V2, self.invalidb),
                           False)

testing.run_module(__name__, __file__)
<commit_msg>Format code<commit_after>import unittest

import numpy

import chainer
from chainer import cuda
from chainer import functions
from chainer import gradient_check
from chainer import testing
from chainer.testing import attr
from chainer.testing import condition

if cuda.available:
    cuda.init()

def _check_forward(e1, e2, f, y_expect):
    e1 = chainer.Variable(e1)
    e2 = chainer.Variable(e2)
    y = f(e1, e2)
    gradient_check.assert_allclose(y_expect, y.data)

def _check_backward(e1, e2, y_grad, f, bias):
    e1 = chainer.Variable(e1)
    e2 = chainer.Variable(e2)
    y = f(e1, e2)
    y.grad = y_grad
    y.backward()

    func = y.creator
    f = lambda: func.forward((e1.data, e2.data))

    ge1, ge2, gW = gradient_check.numerical_grad(
        f, (e1.data, e2.data, func.W), (y.grad,), eps=1e-2)

    gradient_check.assert_allclose(ge1, e1.grad)
    gradient_check.assert_allclose(ge2, e2.grad)
    gradient_check.assert_allclose(gW, func.gW)

    if bias:
        gV1, gV2, gb = gradient_check.numerical_grad(
            f, (func.V1, func.V2, func.b),
            (y.grad,), eps=1e-2)
        gradient_check.assert_allclose(gV1, func.gV1)
        gradient_check.assert_allclose(gV2, func.gV2)
        gradient_check.assert_allclose(gb, func.gb)

def _batch_to_gpu(*xs):
    return tuple(cuda.to_gpu(x) for x in xs)

class TestTensorNetwork(unittest.TestCase):

    in_shape = (2, 3)
    out_size = 4
    batch_size = 10

    def setUp(self):
        self.f = functions.TensorNetwork(
            self.in_shape[0], self.in_shape[1], self.out_size)
        self.f.W = numpy.random.uniform(
            -1, 1, self.f.W.shape).astype(numpy.float32)
        self.f.V1 = numpy.random.uniform(
            -1, 1, self.f.V1.shape).astype(numpy.float32)
        self.f.V2 = numpy.random.uniform(
            -1, 1, self.f.V2.shape).astype(numpy.float32)
        self.f.b = numpy.random.uniform(
            -1, 1, self.f.b.shape).astype(numpy.float32)
        self.f.zero_grads()

        W = self.f.W.copy()
        V1 = self.f.V1.copy()
        V2 = self.f.V2.copy()
        b = self.f.b.copy()

        self.e1 = numpy.random.uniform(
            -1, 1, (self.batch_size, self.in_shape[0])).astype(numpy.float32)
        self.e2 = numpy.random.uniform(
            -1, 1, (self.batch_size, self.in_shape[1])).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (self.batch_size, self.out_size)).astype(numpy.float32)

        self.y = (
            numpy.einsum('ij,ik,jkl->il', self.e1, self.e2, W) +
            self.e1.dot(V1) + self.e2.dot(V2) + b)

    @condition.retry(3)
    def test_forward_cpu(self):
        _check_forward(self.e1, self.e2, self.f, self.y)

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.f.to_gpu()
        _check_forward(cuda.to_gpu(self.e1),
                       cuda.to_gpu(self.e2),
                       self.f, self.y)

    @condition.retry(3)
    def test_backward_cpu(self):
        _check_backward(self.e1, self.e2, self.gy, self.f, True)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.f.to_gpu()
        _check_backward(cuda.to_gpu(self.e1),
                        cuda.to_gpu(self.e2),
                        cuda.to_gpu(self.gy),
                        self.f, True)

class TestTensorNetworkWOBias(unittest.TestCase):

    in_shape = (2, 3)
    out_size = 4
    batch_size = 10

    def setUp(self):
        self.f = functions.TensorNetwork(
            self.in_shape[0], self.in_shape[1], self.out_size, True)
        self.f.W = numpy.random.uniform(
            -1, 1, self.f.W.shape).astype(numpy.float32)
        self.f.zero_grads()

        W = self.f.W.copy()

        self.e1 = numpy.random.uniform(
            -1, 1, (self.batch_size, self.in_shape[0])).astype(numpy.float32)
        self.e2 = numpy.random.uniform(
            -1, 1, (self.batch_size, self.in_shape[1])).astype(numpy.float32)
        self.gy = numpy.random.uniform(
            -1, 1, (self.batch_size, self.out_size)).astype(numpy.float32)

        self.y = numpy.einsum('ij,ik,jkl->il', self.e1, self.e2, W)

    @condition.retry(3)
    def test_forward_cpu(self):
        _check_forward(self.e1, self.e2, self.f, self.y)

    @attr.gpu
    @condition.retry(3)
    def test_forward_gpu(self):
        self.f.to_gpu()
        _check_forward(cuda.to_gpu(self.e1),
                       cuda.to_gpu(self.e2),
                       self.f, self.y)

    @condition.retry(3)
    def test_backward_cpu(self):
        _check_backward(self.e1, self.e2, self.gy, self.f, False)

    @attr.gpu
    @condition.retry(3)
    def test_backward_gpu(self):
        self.f.to_gpu()
        _check_backward(cuda.to_gpu(self.e1), cuda.to_gpu(self.e2),
                        cuda.to_gpu(self.gy), self.f, False)

class InitByInitialParameter(unittest.TestCase):

    in_shape = (2, 3)
    out_size = 4
    batch_size = 10

    def setUp(self):
        self.W = numpy.random.uniform(
            -1, 1,
            (self.in_shape[0], self.in_shape[1], self.out_size)
        ).astype(numpy.float32)
        self.V1 = numpy.random.uniform(
            -1, 1, (self.in_shape[0], self.out_size)).astype(numpy.float32)
        self.V2 = numpy.random.uniform(
            -1, 1, (self.in_shape[1], self.out_size)).astype(numpy.float32)
        self.b = numpy.random.uniform(
            -1, 1, (self.out_size,)).astype(numpy.float32)

class NormalInitialParameter(InitByInitialParameter):

    def check_normal(self, initialW, initial_bias, nobias):
        functions.TensorNetwork(
            self.in_shape[0], self.in_shape[1], self.out_size, nobias,
            initialW, initial_bias)

    def test_normal_cpu_bias(self):
        self.check_normal(self.W, (self.V1, self.V2, self.b), False)

    def test_normal_cpu_nobias(self):
        self.check_normal(self.W, None, False)

    @attr.gpu
    def test_normal_gpu_bias(self):
        self.check_normal(cuda.to_gpu(self.W),
                          _batch_to_gpu(self.V1, self.V2, self.b), False)

    @attr.gpu
    def test_normal_gpu_nobias(self):
        self.check_normal(cuda.to_gpu(self.W), None, False)

class InvalidInitialParameter(InitByInitialParameter):

    def setUp(self):
        super(InvalidInitialParameter, self).setUp()
        self.invalidW = numpy.random.uniform(
            -1, 1,
            (self.in_shape[0]+1, self.in_shape[1], self.out_size)
        ).astype(numpy.float32)
        self.invalidV1 = numpy.random.uniform(
            -1, 1, (self.in_shape[0]+1, self.out_size)).astype(numpy.float32)
        self.invalidV2 = numpy.random.uniform(
            -1, 1, (self.in_shape[1]+1, self.out_size)).astype(numpy.float32)
        self.invalidb = numpy.random.uniform(
            -1, 1, (self.out_size+1,)).astype(numpy.float32)

    def check_invalid(self, initialW, initial_bias, nobias):
        with self.assertRaises(AssertionError):
            functions.TensorNetwork(
                self.in_shape[0], self.in_shape[1], self.out_size, nobias,
                initialW, initial_bias)

    def test_invalidW_cpu(self):
        self.check_invalid(self.invalidW, (self.V1, self.V2, self.b), False)
        self.check_invalid(self.invalidW, None, True)

    @attr.gpu
    def test_invalidW_gpu(self):
        invalidW = cuda.to_gpu(self.invalidW)
        self.check_invalid(invalidW,
                           _batch_to_gpu(self.V1, self.V2, self.b), False)
        self.check_invalid(invalidW, None, True)

    def test_invalidV1_cpu(self):
        self.check_invalid(self.W, (self.invalidV1, self.V2, self.b), False)

    @attr.gpu
    def test_invaliV1_gpu(self):
        self.check_invalid(cuda.to_gpu(self.W),
                           _batch_to_gpu(self.invalidV1, self.V2, self.b),
                           False)

    def test_invalidV2_cpu(self):
        self.check_invalid(self.W, (self.V1, self.invalidV2, self.b), False)

    @attr.gpu
    def test_invaliV2_gpu(self):
        self.check_invalid(cuda.to_gpu(self.W),
                           _batch_to_gpu(self.V1, self.invalidV2, self.b),
                           False)

    def test_invalidb_cpu(self):
        self.check_invalid(self.W, (self.V1, self.V2, self.invalidb), False)

    @attr.gpu
    def test_invalib_gpu(self):
        self.check_invalid(cuda.to_gpu(self.W),
                           _batch_to_gpu(self.V1, self.V2, self.invalidb),
                           False)

testing.run_module(__name__, __file__)
<|endoftext|>


<commit_before>"""Test praw.models.LiveThread"""
from praw.const import API_PATH
from praw.exceptions import APIException
from praw.models import LiveThread, LiveUpdate, Redditor, RedditorList
import mock
import pytest

from ... import IntegrationTest

class TestLiveThread(IntegrationTest):
    @mock.patch('time.sleep', return_value=None)
    def test_contributor(self, _):
        thread = LiveThread(self.reddit, 'ukaeu1ik4sw5')
        with self.recorder.use_cassette('TestLiveThread_test_contributor'):
            contributors = thread.contributor()
        assert isinstance(contributors, RedditorList)
        assert len(contributors) > 0

    @mock.patch('time.sleep', return_value=None)
    def test_contributor__with_manage_permission(self, _):
        # see issue #710 for more info
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'xyu8kmjvfrww')
        url = API_PATH['live_contributors'].format(id=thread.id)
        with self.recorder.use_cassette(
                'TestLiveThread_test_contributor__with_manage_permission'):
            data = thread._reddit.request('GET', url)
            contributors = thread.contributor()
        assert isinstance(data, list)
        assert isinstance(contributors, RedditorList)
        assert len(contributors) > 0

    @mock.patch('time.sleep', return_value=None)
    def test_init(self, _):
        thread = LiveThread(self.reddit, 'ukaeu1ik4sw5')
        with self.recorder.use_cassette('TestLiveThread_test_init'):
            assert thread.title == 'reddit updates'

    @mock.patch('time.sleep', return_value=None)
    def test_updates(self, _):
        thread = LiveThread(self.reddit, 'ukaeu1ik4sw5')
        with self.recorder.use_cassette('TestLiveThread_test_updates'):
            for update in thread.updates(limit=None):
                assert update.thread == thread
        assert update.body.startswith('Small change:')

class TestLiveContributorRelationship(IntegrationTest):
    @mock.patch('time.sleep', return_value=None)
    def test_invite__already_invited(self, _):
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'xyu8kmjvfrww')
        with self.recorder.use_cassette(
                'TestLiveContributorRelationship_'
                'test_invite__already_invited'):
            thread.contributor.invite('nmtake')
            with pytest.raises(APIException) as excinfo:
                thread.contributor.invite('nmtake')
        assert excinfo.value.error_type == 'LIVEUPDATE_ALREADY_CONTRIBUTOR'

    @mock.patch('time.sleep', return_value=None)
    def test_invite__empty_list(self, _):
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'xyu8kmjvfrww')
        with self.recorder.use_cassette(
                'TestLiveContributorRelationship_test_invite__empty_list'):
            thread.contributor.invite('nmtake', [])

    @mock.patch('time.sleep', return_value=None)
    def test_invite__limited(self, _):
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'xyu8kmjvfrww')
        with self.recorder.use_cassette(
                'TestLiveContributorRelationship_test_invite__limited'):
            thread.contributor.invite('nmtake', ['manage', 'edit'])

    @mock.patch('time.sleep', return_value=None)
    def test_invite__none(self, _):
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'xyu8kmjvfrww')
        with self.recorder.use_cassette(
                'TestLiveContributorRelationship_test_invite__none'):
            thread.contributor.invite('nmtake', None)

    @mock.patch('time.sleep', return_value=None)
    def test_invite__redditor(self, _):
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'xyu8kmjvfrww')
        redditor = Redditor(self.reddit,
                            _data={'name': 'nmtake', 'id': 'll32z'})
        with self.recorder.use_cassette(
                'TestLiveContributorRelationship_test_invite__redditor'):
            thread.contributor.invite(redditor)

    @mock.patch('time.sleep', return_value=None)
    def test_leave(self, _):
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'xyu8kmjvfrww')
        with self.recorder.use_cassette(
                'TestLiveContributorRelationship_test_leave'):
            thread.contributor.leave()

    @mock.patch('time.sleep', return_value=None)
    def test_remove__fullname(self, _):
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'xyu8kmjvfrww')
        with self.recorder.use_cassette(
                'TestLiveContributorRelationship_test_remove__fullname'):
            thread.contributor.remove('t2_ll32z')

    @mock.patch('time.sleep', return_value=None)
    def test_remove__redditor(self, _):
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'xyu8kmjvfrww')
        redditor = Redditor(self.reddit,
                            _data={'name': 'nmtake', 'id': 'll32z'})
        with self.recorder.use_cassette(
                'TestLiveContributorRelationship_test_remove__redditor'):
            thread.contributor.remove(redditor)

    @mock.patch('time.sleep', return_value=None)
    def test_remove_invite__fullname(self, _):
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'xyu8kmjvfrww')
        with self.recorder.use_cassette(
                'TestLiveContributorRelationship_'
                'test_remove_invite__fullname'):
            thread.contributor.remove_invite('t2_ll32z')

    @mock.patch('time.sleep', return_value=None)
    def test_remove_invite__redditor(self, _):
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'xyu8kmjvfrww')
        redditor = Redditor(self.reddit,
                            _data={'name': 'nmtake', 'id': 'll32z'})
        with self.recorder.use_cassette(
                'TestLiveContributorRelationship_'
                'test_remove_invite__redditor'):
            thread.contributor.remove_invite(redditor)

class TestLiveThreadContribution(IntegrationTest):
    @mock.patch('time.sleep', return_value=None)
    def test_add(self, _):
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'xyu8kmjvfrww')
        with self.recorder.use_cassette(
                'TestLiveThreadContribution_add'):
            thread.contrib.add('* `LiveThreadContribution.add() test`')

    @mock.patch('time.sleep', return_value=None)
    def test_close(self, _):
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'ya2tmqiyb064')
        with self.recorder.use_cassette(
                'TestLiveThreadContribution_close'):
            thread.contrib.close()

class TestLiveUpdateContribution(IntegrationTest):
    @mock.patch('time.sleep', return_value=None)
    def test_remove(self, _):
        self.reddit.read_only = False
        update = LiveUpdate(self.reddit, 'xyu8kmjvfrww',
                            '5d556760-dbee-11e6-9f46-0e78de675452')
        with self.recorder.use_cassette(
                'TestLiveUpdateContribution_remove'):
            update.contrib.remove()
<commit_msg>Remove unnecessary mock for sleep() from tests<commit_after>"""Test praw.models.LiveThread"""
from praw.const import API_PATH
from praw.exceptions import APIException
from praw.models import LiveThread, LiveUpdate, Redditor, RedditorList
import mock
import pytest

from ... import IntegrationTest

class TestLiveThread(IntegrationTest):
    def test_contributor(self):
        thread = LiveThread(self.reddit, 'ukaeu1ik4sw5')
        with self.recorder.use_cassette('TestLiveThread_test_contributor'):
            contributors = thread.contributor()
        assert isinstance(contributors, RedditorList)
        assert len(contributors) > 0

    @mock.patch('time.sleep', return_value=None)
    def test_contributor__with_manage_permission(self, _):
        # see issue #710 for more info
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'xyu8kmjvfrww')
        url = API_PATH['live_contributors'].format(id=thread.id)
        with self.recorder.use_cassette(
                'TestLiveThread_test_contributor__with_manage_permission'):
            data = thread._reddit.request('GET', url)
            contributors = thread.contributor()
        assert isinstance(data, list)
        assert isinstance(contributors, RedditorList)
        assert len(contributors) > 0

    def test_init(self):
        thread = LiveThread(self.reddit, 'ukaeu1ik4sw5')
        with self.recorder.use_cassette('TestLiveThread_test_init'):
            assert thread.title == 'reddit updates'

    @mock.patch('time.sleep', return_value=None)
    def test_updates(self, _):
        thread = LiveThread(self.reddit, 'ukaeu1ik4sw5')
        with self.recorder.use_cassette('TestLiveThread_test_updates'):
            for update in thread.updates(limit=None):
                assert update.thread == thread
        assert update.body.startswith('Small change:')

class TestLiveContributorRelationship(IntegrationTest):
    @mock.patch('time.sleep', return_value=None)
    def test_invite__already_invited(self, _):
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'xyu8kmjvfrww')
        with self.recorder.use_cassette(
                'TestLiveContributorRelationship_'
                'test_invite__already_invited'):
            thread.contributor.invite('nmtake')
            with pytest.raises(APIException) as excinfo:
                thread.contributor.invite('nmtake')
        assert excinfo.value.error_type == 'LIVEUPDATE_ALREADY_CONTRIBUTOR'

    def test_invite__empty_list(self):
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'xyu8kmjvfrww')
        with self.recorder.use_cassette(
                'TestLiveContributorRelationship_test_invite__empty_list'):
            thread.contributor.invite('nmtake', [])

    def test_invite__limited(self):
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'xyu8kmjvfrww')
        with self.recorder.use_cassette(
                'TestLiveContributorRelationship_test_invite__limited'):
            thread.contributor.invite('nmtake', ['manage', 'edit'])

    def test_invite__none(self):
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'xyu8kmjvfrww')
        with self.recorder.use_cassette(
                'TestLiveContributorRelationship_test_invite__none'):
            thread.contributor.invite('nmtake', None)

    def test_invite__redditor(self):
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'xyu8kmjvfrww')
        redditor = Redditor(self.reddit,
                            _data={'name': 'nmtake', 'id': 'll32z'})
        with self.recorder.use_cassette(
                'TestLiveContributorRelationship_test_invite__redditor'):
            thread.contributor.invite(redditor)

    def test_leave(self):
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'xyu8kmjvfrww')
        with self.recorder.use_cassette(
                'TestLiveContributorRelationship_test_leave'):
            thread.contributor.leave()

    def test_remove__fullname(self):
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'xyu8kmjvfrww')
        with self.recorder.use_cassette(
                'TestLiveContributorRelationship_test_remove__fullname'):
            thread.contributor.remove('t2_ll32z')

    def test_remove__redditor(self):
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'xyu8kmjvfrww')
        redditor = Redditor(self.reddit,
                            _data={'name': 'nmtake', 'id': 'll32z'})
        with self.recorder.use_cassette(
                'TestLiveContributorRelationship_test_remove__redditor'):
            thread.contributor.remove(redditor)

    def test_remove_invite__fullname(self):
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'xyu8kmjvfrww')
        with self.recorder.use_cassette(
                'TestLiveContributorRelationship_'
                'test_remove_invite__fullname'):
            thread.contributor.remove_invite('t2_ll32z')

    def test_remove_invite__redditor(self):
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'xyu8kmjvfrww')
        redditor = Redditor(self.reddit,
                            _data={'name': 'nmtake', 'id': 'll32z'})
        with self.recorder.use_cassette(
                'TestLiveContributorRelationship_'
                'test_remove_invite__redditor'):
            thread.contributor.remove_invite(redditor)

class TestLiveThreadContribution(IntegrationTest):
    def test_add(self):
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'xyu8kmjvfrww')
        with self.recorder.use_cassette(
                'TestLiveThreadContribution_add'):
            thread.contrib.add('* `LiveThreadContribution.add() test`')

    def test_close(self):
        self.reddit.read_only = False
        thread = LiveThread(self.reddit, 'ya2tmqiyb064')
        with self.recorder.use_cassette(
                'TestLiveThreadContribution_close'):
            thread.contrib.close()

class TestLiveUpdateContribution(IntegrationTest):
    @mock.patch('time.sleep', return_value=None)
    def test_remove(self, _):
        self.reddit.read_only = False
        update = LiveUpdate(self.reddit, 'xyu8kmjvfrww',
                            '5d556760-dbee-11e6-9f46-0e78de675452')
        with self.recorder.use_cassette(
                'TestLiveUpdateContribution_remove'):
            update.contrib.remove()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# dials.reflection_viewer.py
#
#  Copyright (C) 2014 Diamond Light Source
#
#  Author: Luis Fuentes-Montero (Luiso)
#
#  This code is distributed under the BSD license, a copy of which is
#  included in the root directory of this package.
from __future__ import division

from dials.array_family import flex
from from_flex_to_wxbitmap import wxbitmap_convert
import wx

class show_3d(wx.App):
  def OnInit(self, lst):
    self.frame = MyFrame(None, title="Bitmaps")
    print "______________________________________________________________"
    self.frame.set_bmp(lst)
    print lst
    print "______________________________________________________________"
    self.SetTopWindow(self.frame)
    self.frame.Show()
    return True

class MyFrame(wx.Frame):
  def __init__(self, parent, id = wx.ID_ANY, title = "",
               pos=wx.DefaultPosition, size=wx.DefaultSize,
               style = wx.DEFAULT_FRAME_STYLE,
               name = "MyFrame"):
    super(MyFrame, self).__init__(parent, id, title,
                                  pos, size, style, name)
    # Attributes
    self.panel = wx.Panel(self)

  def set_bmp(self, bmp_in):
    self.bitmap = wx.StaticBitmap(self.panel, bitmap = bmp_in[1])

if(__name__ == "__main__"):
  size_xy = 5
  data2d = flex.double(flex.grid(1, size_xy, size_xy),15)
  data2d[0, 1, 1] = 50

  for row in range(size_xy):
    for col in range(size_xy):
      data2d[0,row, col] += row * 2
      data2d[0,row, col] += col * 2

  mask2d = flex.int(flex.grid(1, size_xy, size_xy),size_xy)
  mask2d[0, 1, 1] = 5

  #testing wxbitmap_convert as a class
  a = wxbitmap_convert(data2d)
  print "calling obj", a.get_np()
  print a.__doc__

  datasize_xyd = flex.double(flex.grid(size_xy, size_xy, size_xy),15)
  datasize_xyd[1, 1, 1] = 50

  for frm in range(size_xy):
    for row in range(size_xy):
      for col in range(size_xy):
        datasize_xyd[frm, row, col] += (row * 2 + col * 2 + frm * 2)
        #datasize_xyd[0,row, col] += col * 2

  masksize_xyd = flex.int(flex.grid(size_xy, size_xy, size_xy),size_xy)
  masksize_xyd[0, 1, 1] = 5

  #testing wxbitmap_convert as a function
  print wxbitmap_convert(datasize_xyd).get_np()

  app = show_3d(redirect=False, wxbitmap_convert(datasize_xyd).get_wxbitmap_lst())
  app.MainLoop()
<commit_msg>previous commit was a typo<commit_after>#!/usr/bin/env python
#
# dials.reflection_viewer.py
#
#  Copyright (C) 2014 Diamond Light Source
#
#  Author: Luis Fuentes-Montero (Luiso)
#
#  This code is distributed under the BSD license, a copy of which is
#  included in the root directory of this package.
from __future__ import division

from dials.array_family import flex
from from_flex_to_wxbitmap import wxbitmap_convert
import wx

class show_3d(wx.App):
  def OnInit(self):
    self.frame = MyFrame(None, title="Bitmaps")
    print "______________________________________________________________"
    return True

  def in_lst(self, lst):
    self.frame.set_bmp(lst)
    print "in_lst"
    print lst
    self.SetTopWindow(self.frame)
    self.frame.Show()

class MyFrame(wx.Frame):
  def __init__(self, parent, id = wx.ID_ANY, title = "",
               pos=wx.DefaultPosition, size=wx.DefaultSize,
               style = wx.DEFAULT_FRAME_STYLE,
               name = "MyFrame"):
    super(MyFrame, self).__init__(parent, id, title,
                                  pos, size, style, name)
    # Attributes
    self.panel = wx.Panel(self)

  def set_bmp(self, bmp_in):
    self.bitmap = wx.StaticBitmap(self.panel, bitmap = bmp_in[1])

if(__name__ == "__main__"):
  size_xy = 5
  data2d = flex.double(flex.grid(1, size_xy, size_xy),15)
  data2d[0, 1, 1] = 50

  for row in range(size_xy):
    for col in range(size_xy):
      data2d[0,row, col] += row * 2
      data2d[0,row, col] += col * 2

  mask2d = flex.int(flex.grid(1, size_xy, size_xy),size_xy)
  mask2d[0, 1, 1] = 5

  #testing wxbitmap_convert as a class
  a = wxbitmap_convert(data2d)
  print "calling obj", a.get_np()
  print a.__doc__

  datasize_xyd = flex.double(flex.grid(size_xy, size_xy, size_xy),15)
  datasize_xyd[1, 1, 1] = 50

  for frm in range(size_xy):
    for row in range(size_xy):
      for col in range(size_xy):
        datasize_xyd[frm, row, col] += (row * 2 + col * 2 + frm * 2)
        #datasize_xyd[0,row, col] += col * 2

  masksize_xyd = flex.int(flex.grid(size_xy, size_xy, size_xy),size_xy)
  masksize_xyd[0, 1, 1] = 5

  #testing wxbitmap_convert as a function
  print wxbitmap_convert(datasize_xyd).get_np()

  app = show_3d(redirect=False)
  app.in_lst(wxbitmap_convert(datasize_xyd).get_wxbitmap_lst())
  app.MainLoop()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
'''
Test AnsibleGate State Module
'''
from __future__ import absolute_import, unicode_literals, print_function

# Import python libraries
import os
import shutil
import tempfile
import yaml

# Import salt libraries
import salt.utils.files
import salt.utils.path

# Import testing libraries
from tests.support.case import ModuleCase
from tests.support.helpers import destructiveTest
from tests.support.mixins import SaltReturnAssertsMixin
from tests.support.runtests import RUNTIME_VARS
from tests.support.unit import skipIf

@destructiveTest
@skipIf(not salt.utils.path.which('ansible-playbook'), 'ansible-playbook is not installed')
class AnsiblePlaybooksTestCase(ModuleCase, SaltReturnAssertsMixin):
    '''
    Test ansible.playbooks states
    '''
    def setUp(self):
        priv_file = os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'key_test')
        data = {
            'all': {
                'hosts': {
                    'localhost': {
                        'ansible_host': '127.0.0.1',
                        'ansible_port': 2827,
                        'ansible_user': RUNTIME_VARS.RUNNING_TESTS_USER,
                        'ansible_ssh_private_key_file': priv_file,
                        'ansible_ssh_extra_args': (
                            '-o StrictHostKeyChecking=false '
                            '-o UserKnownHostsFile=/dev/null '
                        )
                    },
                },
            },
        }
        self.tempdir = tempfile.mkdtemp()
        self.inventory = self.tempdir + 'inventory'
        with salt.utils.files.fopen(self.inventory, 'w') as yaml_file:
            yaml.dump(data, yaml_file, default_flow_style=False)

    def tearDown(self):
        shutil.rmtree(self.tempdir)
        delattr(self, 'tempdir')
        delattr(self, 'inventory')

    def test_ansible_playbook(self):
        ret = self.run_state(
            'ansible.playbooks',
            name='remove.yml',
            git_repo='git://github.com/gtmanfred/playbooks.git',
            ansible_kwargs={'inventory': self.inventory}
        )
        self.assertSaltTrueReturn(ret)
        ret = self.run_state(
            'ansible.playbooks',
            name='install.yml',
            gitrepo='git://github.com/gtmanfred/playbooks.git',
            ansible_kwargs={'inventory': self.inventory}
        )
        self.assertSaltTrueReturn(ret)
<commit_msg>gitrepo -> git_repo in test<commit_after># -*- coding: utf-8 -*-
'''
Test AnsibleGate State Module
'''
from __future__ import absolute_import, unicode_literals, print_function

# Import python libraries
import os
import shutil
import tempfile
import yaml

# Import salt libraries
import salt.utils.files
import salt.utils.path

# Import testing libraries
from tests.support.case import ModuleCase
from tests.support.helpers import destructiveTest
from tests.support.mixins import SaltReturnAssertsMixin
from tests.support.runtests import RUNTIME_VARS
from tests.support.unit import skipIf

@destructiveTest
@skipIf(not salt.utils.path.which('ansible-playbook'), 'ansible-playbook is not installed')
class AnsiblePlaybooksTestCase(ModuleCase, SaltReturnAssertsMixin):
    '''
    Test ansible.playbooks states
    '''
    def setUp(self):
        priv_file = os.path.join(RUNTIME_VARS.TMP_CONF_DIR, 'key_test')
        data = {
            'all': {
                'hosts': {
                    'localhost': {
                        'ansible_host': '127.0.0.1',
                        'ansible_port': 2827,
                        'ansible_user': RUNTIME_VARS.RUNNING_TESTS_USER,
                        'ansible_ssh_private_key_file': priv_file,
                        'ansible_ssh_extra_args': (
                            '-o StrictHostKeyChecking=false '
                            '-o UserKnownHostsFile=/dev/null '
                        )
                    },
                },
            },
        }
        self.tempdir = tempfile.mkdtemp()
        self.inventory = self.tempdir + 'inventory'
        with salt.utils.files.fopen(self.inventory, 'w') as yaml_file:
            yaml.dump(data, yaml_file, default_flow_style=False)

    def tearDown(self):
        shutil.rmtree(self.tempdir)
        delattr(self, 'tempdir')
        delattr(self, 'inventory')

    def test_ansible_playbook(self):
        ret = self.run_state(
            'ansible.playbooks',
            name='remove.yml',
            git_repo='git://github.com/gtmanfred/playbooks.git',
            ansible_kwargs={'inventory': self.inventory}
        )
        self.assertSaltTrueReturn(ret)
        ret = self.run_state(
            'ansible.playbooks',
            name='install.yml',
            git_repo='git://github.com/gtmanfred/playbooks.git',
            ansible_kwargs={'inventory': self.inventory}
        )
        self.assertSaltTrueReturn(ret)
<|endoftext|>


<commit_before>import re
import requests
from lxml.html import parse
from requests.exceptions import ConnectionError

DO_FAIL=False  # fail on error

def is_root_link(link):
    pattern = re.compile("^/$")
    return pattern.match(link)

def is_mailto_link(link):
    pattern = re.compile("^mailto:.*")
    return pattern.match(link)

def is_internal_link(link):
    pattern = re.compile("^/.*")
    return pattern.match(link)

def is_in_page_link(link):
    pattern = re.compile("^#.*")
    return pattern.match(link)

def get_links(doc):
    return [x for x in [y.get("href") for y in doc.cssselect("a")] if not (
            is_root_link(x)
            or is_mailto_link(x))]

def verify_link(link):
    if link[0] == "#":
        # local link on page
        return
    print("verifying "+link)
    try:
        result = requests.get(link, timeout=20, verify=False)
        if result.status_code == 200:
            print(link+" ==> OK")
        elif result.status_code == 307:
            print(link+" ==> REDIRECT")
        else:
            print("ERROR: link `"+link+"` failed with status "
                  , result.status_code)

            if DO_FAIL:
                raise Exception("Failed verify")
    except ConnectionError as ex:
        print("ERROR: ", link, ex)
        if DO_FAIL:
            raise ex

def verify_static_file(link):
    print("verifying "+link)
    try:
        result = requests.get(link, timeout=20, verify=False)
        if (result.status_code == 200 and
                result.content.find(bytes("Error: 404 Not Found", "utf-8")) <= 0):
            print(link+" ==> OK")
        else:
            print("ERROR: link {}".format(link))
            raise Exception("Failed verify")
    except ConnectionError as ex:
        print("ERROR: ", link, ex)

def check_page(host, start_url):
    print("")
    print("Checking links host "+host+" in page `"+start_url+"`")
    doc = parse(start_url).getroot()
    links = get_links(doc)
    in_page_links = list(filter(is_in_page_link, links))
    internal_links = list(filter(is_internal_link, links))
    external_links = [x for x in links if not (is_internal_link(x) or is_in_page_link(x))]

    for link in internal_links:
        verify_link(host+link)

    for link in external_links:
        verify_link(link)

def check_links(args_obj, parser):
    print("")
    print("Checking links")
    host = args_obj.host

    # Check the home page
    check_page(host, host)

    # Check traits page
    check_page(
        host,
        host+"/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P")

def check_packaged_js_files(args_obj, parser):
    host = args_obj.host
    js_files = [
        # Datatables Extensions:
        "/css/DataTablesExtensions/buttonsBootstrap/css/buttons.bootstrap.css",
        "/js/DataTablesExtensions/buttons/js/dataTables.buttons.min.js",
        "/css/DataTablesExtensions/buttonStyles/css/buttons.dataTables.min.css",
        "/js/DataTablesExtensions/buttons/js/dataTables.buttons.min.js",
        "/js/DataTablesExtensions/colResize/dataTables.colResize.js",
        "/js/DataTablesExtensions/colReorder/js/dataTables.colReorder.js",
        "/js/DataTablesExtensions/buttons/js/buttons.colVis.min.js",
        "/js/DataTables/js/jquery.dataTables.js",
        "/css/DataTablesExtensions/scroller/css/scroller.dataTables.min.css",
        # Datatables plugins:
        "/js/DataTablesExtensions/plugins/sorting/natural.js",
        "/js/DataTablesExtensions/plugins/sorting/scientific.js",
        # Other js libraries
        "/js/chroma/chroma.min.js",
        "/js/d3-tip/d3-tip.js",
        "/js/d3js/d3.min.js",
        "/js/js_alt/underscore.min.js",
        "/js/nvd3/nv.d3.min.css",
        "/js/qtip2/jquery.qtip.min.js",
        "/js/js_alt/md5.min.js",
        "/css/nouislider/nouislider.min.css",
    ]

    print("Checking links")
    for link in js_files:
        verify_static_file(host+link)
<commit_msg>add nouislider.js to mechanical-rob<commit_after>import re
import requests
from lxml.html import parse
from requests.exceptions import ConnectionError

DO_FAIL=False  # fail on error

def is_root_link(link):
    pattern = re.compile("^/$")
    return pattern.match(link)

def is_mailto_link(link):
    pattern = re.compile("^mailto:.*")
    return pattern.match(link)

def is_internal_link(link):
    pattern = re.compile("^/.*")
    return pattern.match(link)

def is_in_page_link(link):
    pattern = re.compile("^#.*")
    return pattern.match(link)

def get_links(doc):
    return [x for x in [y.get("href") for y in doc.cssselect("a")] if not (
            is_root_link(x)
            or is_mailto_link(x))]

def verify_link(link):
    if link[0] == "#":
        # local link on page
        return
    print("verifying "+link)
    try:
        result = requests.get(link, timeout=20, verify=False)
        if result.status_code == 200:
            print(link+" ==> OK")
        elif result.status_code == 307:
            print(link+" ==> REDIRECT")
        else:
            print("ERROR: link `"+link+"` failed with status "
                  , result.status_code)

            if DO_FAIL:
                raise Exception("Failed verify")
    except ConnectionError as ex:
        print("ERROR: ", link, ex)
        if DO_FAIL:
            raise ex

def verify_static_file(link):
    print("verifying "+link)
    try:
        result = requests.get(link, timeout=20, verify=False)
        if (result.status_code == 200 and
                result.content.find(bytes("Error: 404 Not Found", "utf-8")) <= 0):
            print(link+" ==> OK")
        else:
            print("ERROR: link {}".format(link))
            raise Exception("Failed verify")
    except ConnectionError as ex:
        print("ERROR: ", link, ex)

def check_page(host, start_url):
    print("")
    print("Checking links host "+host+" in page `"+start_url+"`")
    doc = parse(start_url).getroot()
    links = get_links(doc)
    in_page_links = list(filter(is_in_page_link, links))
    internal_links = list(filter(is_internal_link, links))
    external_links = [x for x in links if not (is_internal_link(x) or is_in_page_link(x))]

    for link in internal_links:
        verify_link(host+link)

    for link in external_links:
        verify_link(link)

def check_links(args_obj, parser):
    print("")
    print("Checking links")
    host = args_obj.host

    # Check the home page
    check_page(host, host)

    # Check traits page
    check_page(
        host,
        host+"/show_trait?trait_id=1435395_s_at&dataset=HC_M2_0606_P")

def check_packaged_js_files(args_obj, parser):
    host = args_obj.host
    js_files = [
        # Datatables Extensions:
        "/css/DataTablesExtensions/buttonsBootstrap/css/buttons.bootstrap.css",
        "/js/DataTablesExtensions/buttons/js/dataTables.buttons.min.js",
        "/css/DataTablesExtensions/buttonStyles/css/buttons.dataTables.min.css",
        "/js/DataTablesExtensions/buttons/js/dataTables.buttons.min.js",
        "/js/DataTablesExtensions/colResize/dataTables.colResize.js",
        "/js/DataTablesExtensions/colReorder/js/dataTables.colReorder.js",
        "/js/DataTablesExtensions/buttons/js/buttons.colVis.min.js",
        "/js/DataTables/js/jquery.dataTables.js",
        "/css/DataTablesExtensions/scroller/css/scroller.dataTables.min.css",
        # Datatables plugins:
        "/js/DataTablesExtensions/plugins/sorting/natural.js",
        "/js/DataTablesExtensions/plugins/sorting/scientific.js",
        # Other js libraries
        "/js/chroma/chroma.min.js",
        "/js/d3-tip/d3-tip.js",
        "/js/d3js/d3.min.js",
        "/js/js_alt/underscore.min.js",
        "/js/nvd3/nv.d3.min.css",
        "/js/qtip2/jquery.qtip.min.js",
        "/js/js_alt/md5.min.js",
        "/css/nouislider/nouislider.min.css",
        "/js/nouislider/nouislider.js",
    ]

    print("Checking links")
    for link in js_files:
        verify_static_file(host+link)
<|endoftext|>


<commit_before># Copyright (c) 2015 Uber Technologies, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import base64
import json
import random

import pytest
import tornado
import tornado.gen

from tchannel import TChannel, Response
from tchannel.tornado import Request
from tchannel.zipkin.annotation import Endpoint
from tchannel.zipkin.annotation import client_send
from tchannel.zipkin.tcollector import TCollector
from tchannel.zipkin.tcollector import Response as TResponse
from tchannel.zipkin.trace import Trace, _uniq_id
from tchannel.zipkin.tracers import TChannelZipkinTracer
from tchannel.zipkin.zipkin_trace import ZipkinTraceHook
from tests.mock_server import MockServer
from tornado.concurrent import Future

try:
    from cStringIO import StringIO
except:
    from StringIO import StringIO

def register(tchannel):

    @tchannel.raw.register('endpoint1')
    @tornado.gen.coroutine
    def handler1(request):
        hostport = request.headers

        res = yield tchannel.raw(
            service='handler2',
            hostport=hostport,
            endpoint="endpoint2",
        )

        raise tornado.gen.Return(Response(res.body, "from handler1"))

    @tchannel.raw.register('endpoint2')
    @tornado.gen.coroutine
    def handler2(request):
        return "from handler2"

    @tchannel.thrift.register(TCollector)
    def submit(request):
        span = request.body.span
        ok = request.transport.shard_key == base64.b64encode(span.traceId)
        return TResponse(ok=ok)

trace_buf = StringIO()

@pytest.yield_fixture
def trace_server():
    with MockServer() as server:
        register(server.tchannel)
        server.tchannel.hooks.register(
            ZipkinTraceHook(
                dst=trace_buf,
                sample_rate=1,
            ),
        )
        yield server

@pytest.mark.gen_test
def test_zipkin_trace(trace_server):
    endpoint = b'endpoint1'
    zipkin_tracer = ZipkinTraceHook(dst=trace_buf, sample_rate=1)
    tchannel = TChannel(name='test')
    tchannel.hooks.register(zipkin_tracer)

    hostport = 'localhost:%d' % trace_server.port

    response = yield tchannel.raw(
        service='test-client',
        hostport=hostport,
        endpoint=endpoint,
        headers=hostport,
        trace=True,
    )

    header = response.headers
    body = response.body
    assert header == "from handler1"
    assert body == "from handler2"
    traces = []
    for trace in trace_buf.getvalue().split("\n"):
        if trace:
            traces.append(json.loads(trace))

    parent_span_id = object()
    trace_id = traces[0][0][u'trace_id']

    assert traces

    for trace in traces:
        assert trace_id == trace[0][u'trace_id']
        if trace[0][u'name'] == u'endpoint2':
            parent_span_id = trace[0][u'parent_span_id']
        else:
            span_id = trace[0][u'span_id']

    assert parent_span_id == span_id

@pytest.mark.gen_test
def test_tcollector_submit(trace_server):
    tchannel = TChannel(name='test', known_peers=[trace_server.hostport])

    trace = Trace(
        name='endpoint', endpoint=Endpoint('1.0.0.1', 1111, 'tcollector')
    )
    anns = [client_send()]

    results = yield TChannelZipkinTracer(tchannel).record([(trace, anns)])

    assert results[0].body.ok is True

@pytest.mark.gen_test
def test_tcollector_submit_never_retry():

    def submit(request):
        count[0] += 1
        if f.running():
            f.set_result(None)
        raise Exception()

    count = [0]
    f = Future()

    zipkin_server = TChannel('zipkin')
    zipkin_server.thrift.register(TCollector, handler=submit)
    zipkin_server.listen()

    zipkin_server1 = TChannel('server')
    zipkin_server1.thrift.register(TCollector, handler=submit)

    @zipkin_server1.raw.register
    def hello(request):
        return 'hello'

    zipkin_server1.listen()

    client = TChannel('client', known_peers=[zipkin_server.hostport])
    client.hooks.register(ZipkinTraceHook(tchannel=client, sample_rate=1))
    yield client.raw(
        service='server',
        endpoint='hello',
        hostport=zipkin_server1.hostport,
        body='body',
        trace=True,
    )

    yield f
    assert count[0] == 1

@pytest.mark.gen_test
def test_zipkin_trace_sampling():
    run_times = 100000
    sample_rate = 1.0 - random.random()
    hook = ZipkinTraceHook(sample_rate=sample_rate)
    count = 0
    for _ in range(run_times):
        if hook._lucky(_uniq_id()):
            count += 1

    assert 0.9 * run_times * sample_rate <= count
    assert count <= run_times * sample_rate * 1.1

@pytest.mark.gen_test
def test_zipkin_trace_zero_sampling():
    run_times = 100000
    hook = ZipkinTraceHook(sample_rate=0)

    request = Request()
    request.tracing.traceflags = True
    for _ in range(run_times):
        hook.before_send_request(request)

    assert not request.tracing.annotations

<commit_msg>fix the lint<commit_after># Copyright (c) 2015 Uber Technologies, Inc.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.

import base64
import json
import random

import pytest
import tornado
import tornado.gen

from tchannel import TChannel, Response
from tchannel.tornado import Request
from tchannel.zipkin.annotation import Endpoint
from tchannel.zipkin.annotation import client_send
from tchannel.zipkin.tcollector import TCollector
from tchannel.zipkin.tcollector import Response as TResponse
from tchannel.zipkin.trace import Trace, _uniq_id
from tchannel.zipkin.tracers import TChannelZipkinTracer
from tchannel.zipkin.zipkin_trace import ZipkinTraceHook
from tests.mock_server import MockServer
from tornado.concurrent import Future

try:
    from cStringIO import StringIO
except:
    from StringIO import StringIO

def register(tchannel):

    @tchannel.raw.register('endpoint1')
    @tornado.gen.coroutine
    def handler1(request):
        hostport = request.headers

        res = yield tchannel.raw(
            service='handler2',
            hostport=hostport,
            endpoint="endpoint2",
        )

        raise tornado.gen.Return(Response(res.body, "from handler1"))

    @tchannel.raw.register('endpoint2')
    @tornado.gen.coroutine
    def handler2(request):
        return "from handler2"

    @tchannel.thrift.register(TCollector)
    def submit(request):
        span = request.body.span
        ok = request.transport.shard_key == base64.b64encode(span.traceId)
        return TResponse(ok=ok)

trace_buf = StringIO()

@pytest.yield_fixture
def trace_server():
    with MockServer() as server:
        register(server.tchannel)
        server.tchannel.hooks.register(
            ZipkinTraceHook(
                dst=trace_buf,
                sample_rate=1,
            ),
        )
        yield server

@pytest.mark.gen_test
def test_zipkin_trace(trace_server):
    endpoint = b'endpoint1'
    zipkin_tracer = ZipkinTraceHook(dst=trace_buf, sample_rate=1)
    tchannel = TChannel(name='test')
    tchannel.hooks.register(zipkin_tracer)

    hostport = 'localhost:%d' % trace_server.port

    response = yield tchannel.raw(
        service='test-client',
        hostport=hostport,
        endpoint=endpoint,
        headers=hostport,
        trace=True,
    )

    header = response.headers
    body = response.body
    assert header == "from handler1"
    assert body == "from handler2"
    traces = []
    for trace in trace_buf.getvalue().split("\n"):
        if trace:
            traces.append(json.loads(trace))

    parent_span_id = object()
    trace_id = traces[0][0][u'trace_id']

    assert traces

    for trace in traces:
        assert trace_id == trace[0][u'trace_id']
        if trace[0][u'name'] == u'endpoint2':
            parent_span_id = trace[0][u'parent_span_id']
        else:
            span_id = trace[0][u'span_id']

    assert parent_span_id == span_id

@pytest.mark.gen_test
def test_tcollector_submit(trace_server):
    tchannel = TChannel(name='test', known_peers=[trace_server.hostport])

    trace = Trace(
        name='endpoint', endpoint=Endpoint('1.0.0.1', 1111, 'tcollector')
    )
    anns = [client_send()]

    results = yield TChannelZipkinTracer(tchannel).record([(trace, anns)])

    assert results[0].body.ok is True

@pytest.mark.gen_test
def test_tcollector_submit_never_retry():

    def submit(request):
        count[0] += 1
        if f.running():
            f.set_result(None)
        raise Exception()

    count = [0]
    f = Future()

    zipkin_server = TChannel('zipkin')
    zipkin_server.thrift.register(TCollector, handler=submit)
    zipkin_server.listen()

    zipkin_server1 = TChannel('server')
    zipkin_server1.thrift.register(TCollector, handler=submit)

    @zipkin_server1.raw.register
    def hello(request):
        return 'hello'

    zipkin_server1.listen()

    client = TChannel('client', known_peers=[zipkin_server.hostport])
    client.hooks.register(ZipkinTraceHook(tchannel=client, sample_rate=1))
    yield client.raw(
        service='server',
        endpoint='hello',
        hostport=zipkin_server1.hostport,
        body='body',
        trace=True,
    )

    yield f
    assert count[0] == 1

@pytest.mark.gen_test
def test_zipkin_trace_sampling():
    run_times = 100000
    sample_rate = 1.0 - random.random()
    hook = ZipkinTraceHook(sample_rate=sample_rate)
    count = 0
    for _ in range(run_times):
        if hook._lucky(_uniq_id()):
            count += 1

    assert 0.9 * run_times * sample_rate <= count
    assert count <= run_times * sample_rate * 1.1

@pytest.mark.gen_test
def test_zipkin_trace_zero_sampling():
    run_times = 100000
    hook = ZipkinTraceHook(sample_rate=0)

    request = Request()
    request.tracing.traceflags = True
    for _ in range(run_times):
        hook.before_send_request(request)

    assert not request.tracing.annotations
<|endoftext|>


<commit_before>#!/usr/bin/python2
# vim: set fileencoding=utf-8

from __future__ import print_function

import socket, struct, fcntl, sys
from os import getpid
from time import time
from pyjavaproperties import Properties
from uuid import uuid4
import zmq
import json

# this is the file that holds the lib's configuration
CONFIGURATION_PROPERTIES = '/srv/vismo/config.properties'
#CONFIGURATION_PROPERTIES = 'config.properties'

###
##
## Utilities
##
###

def get_public_ip(iface='eth0'):
    """
        Return a tuple whose first element contains the first non loopback interface
        and the second elements is the first not inet6 address of that interface.
    """

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sockfd = sock.fileno()
    SIOCGIFADDR = 0x8915
    ifreq = struct.pack('16sH14s', iface, socket.AF_INET, '\x00'*14)

    try:
        res = fcntl.ioctl(sockfd, SIOCGIFADDR, ifreq)
    except:
        return None
    finally:
        sock.close()

    ip = struct.unpack('16sH2x4s8x', res)[2]

    return socket.inet_ntoa(ip)

def is_effectively_zero(n):
    return abs(n) <= 5e-9

def log(msg):
    print('vismo-local-event-dispatcher: {0}'.format(msg), file=sys.stderr)

def log_and_raise(msg, excp):
    print('vismo-local-event-dispatcher: {0}'.format(msg), file=sys.stderr)
    raise excp(msg)

_MAX_LENGTH = 80
def safe_repr(obj, short=False):
    try:
        result = repr(obj)
    except Exception:
        result = object.__repr__(obj)
    if not short or len(result) < _MAX_LENGTH:
        return result
    return result[:_MAX_LENGTH] + ' [truncated]...'

## core

class EventDispatcher(object):
    """
        An event dispatcher is used to pass around events to
        interested parties.
    """

    def __init__(self, service_name, sock):
        self.service_name = service_name
        self.sock = sock

    def send(self, **event):
        """Actually dispatch the event."""

        self.sock.send(event)

    def calculate_event_time_difference(self, e1, e2):
        """
            Calculates the time difference of the two events. It is assumed
            that e1 ``happens before'' e2, so that a positive difference is
            returned.

            It is also assumed that the event timestamps are in millis and
            the returned value is in seconds.
        """

        return (float(e2['timestamp']) - float(e1['timestamp'])) / 1000.0

    def calculate_mean_value_per_time_unit(self, val, time_diff):
        if is_effectively_zero(time_diff):
            return 0.0

        return float(val) / float(time_diff)

class VismoEventDispatcher(EventDispatcher):
    """
        This the bridge that handles the event generation code (the caller
        of this library) with the event distribution code (the main
        monitoring instance). This instance talks directly to the locally
        running vismo instance.

        The services of this library are called by the Object Service on
        each request/response cycle. Since each request in object service
        is handled by a different process, we know that each event belongs
        to the same request/response.
    """

    def __init__(self, service_name):
        self.load_configuration()
        sock = self.create_push_socket(self.producers_point)
        super(VismoEventDispatcher, self).__init__(service_name, sock)
        self.start_request_event = None
        self.start_response_event = None
        self.end_response_event = None
        self.id = str(uuid4())

    def load_configuration(self):
        p = Properties()
        p.load(open(CONFIGURATION_PROPERTIES))
        self.producers_point = p['producers.point']
        # FIXME: auto acquire the name according to the machine's ip
        self.cluster_name = 'test'
        self.iface = 'eth0'
        self.ip = get_public_ip(self.iface)

    def create_push_socket(self, end_point):
        ctx = zmq.Context()
        sock = ctx.socket(zmq.PUSH)
        sock.setsockopt(zmq.LINGER, 0)
        sock.connect(end_point)
        log('connecting to endpoint={0}'.format(end_point))

        return sock

    def send(self, **event):
        """
            This is the single public entry point to the object.
        """

        self.cleanup_event(event)
        self.add_basic_fields(event)
        log('sending: {0}'.format(event))
        self.handle_event(event)

    def cleanup_event(self, event):
        """
            For consistency reasons, use dashes and full nouns.
        """

        if 'content_size' in event:
            event['content-size'] = event['content_size']
            del event['content_size']
        if 'obj' in event:
            event['object'] = event['obj']
            del event['obj']
        if 'topic' in event:
            del event['topic']

    def add_basic_fields(self, event):
        event['timestamp'] = int(1000 * time())
        event['originating-machine'] = self.ip
        event['originating-service'] = self.service_name
        event['originating-cluster'] = self.cluster_name
        event['id'] = self.id

    def _sock_send(self, event):
        self.sock.send(json.dumps(event))

    def handle_event(self, event):
        """
            This does most of the work. It accumulates the interesting events,
            and upon arrival of all events, makes some request calculations and
            sends the event to the main monitoring instance.
        """

        # has the event anything to do with request/response?
        if 'tag' not in event:
            self._sock_send(event)
            return

        tag = event['tag']

        if tag == 'start-request':
            self.start_request_event = event
        elif tag == 'start-response':
            self.start_response_event = event
        elif tag == 'end-response':
            self.end_response_event = event
        else:
            log_and_raise('handed event with incomprehensible tag: {0}, event: {1}'.format(tag, event), ValueError)

        ## TODO: handle object services failures => availability
        # if we have all the events
        if self.start_request_event and self.start_response_event and self.end_response_event:
            main_event = dict(self.end_response_event)

            main_event['transaction-latency'] = self.calculate_latency()
            main_event['transaction-duration'] = self.calculate_transaction_duration()

            if 'content-size' not in main_event or main_event['content-size'] is None:
                main_event['transaction-throughput'] = 0
            else:
                main_event['transaction-throughput'] = self.calculate_throughput(main_event['content-size'], main_event['transaction-duration'])
            # TODO: Calculate availability and other Niki's required stuff

            # not needed
            del main_event['status']
            del main_event['tag']

            self._sock_send(event)
            self._sock_send(main_event)
        else:
            # send anyway
            self._sock_send(event)

    def calculate_latency(self):
        """
            Latency here is defined as the time duration from the time the system received
            the start request event till the time the system is ready to start serving the
            response, as seen by the start response event.
        """

        return self.calculate_event_time_difference(self.start_request_event, self.start_response_event)

    def calculate_transaction_duration(self):
        """
            Transaction duration is the time spent serving the request. More accurately, is the
            duration from the time the system received the start request event till the time the
            system received the end response event.
        """

        return self.calculate_event_time_difference(self.start_request_event, self.end_response_event)

    def calculate_throughput(self, content_size, transaction_time):
        """
            Throughput is defined as the the number of bytes served in the unit of time.
            Transaction size is measured in bytes, transaction_time in seconds.
            Output value is in bytes per second.
        """

        return self.calculate_mean_value_per_time_unit(content_size, transaction_time)

if __name__ == '__main__':
    import unittest
    from time import sleep

    class MyTestCase(unittest.TestCase):
        """
            This is used to reproduce to methods missing from
            python 2.6.6
        """

        def assertGreater(self, a, b, msg=None):
            """Just like self.assertTrue(a > b), but with a nicer default message."""
            if not a > b:
                standardMsg = '%s not greater than %s' % (safe_repr(a), safe_repr(b))
                self.fail(self._formatMessage(msg, standardMsg))

        def assertAlmostEqual(self, first, second, places=None, msg=None, delta=None):
            """Fail if the two objects are unequal as determined by their
               difference rounded to the given number of decimal places
               (default 7) and comparing to zero, or by comparing that the
               between the two objects is more than the given delta.

               Note that decimal places (from zero) are usually not the same
               as significant digits (measured from the most signficant digit).

               If the two objects compare equal then they will automatically
               compare almost equal.
            """
            if first == second:
                # shortcut
                return
            if delta is not None and places is not None:
                raise TypeError("specify delta or places not both")

            if delta is not None:
                if abs(first - second) <= delta:
                    return

                standardMsg = '%s != %s within %s delta' % (safe_repr(first),
                                                            safe_repr(second),
                                                            safe_repr(delta))
            else:
                if places is None:
                    places = 7

                if round(abs(second-first), places) == 0:
                    return

                standardMsg = '%s != %s within %r places' % (safe_repr(first),
                                                              safe_repr(second),
                                                              places)
            msg = self._formatMessage(msg, standardMsg)
            raise self.failureException(msg)

    class EventDispatcherTest(MyTestCase):
        def setUp(self):
            self.content_size = 1000 # in bytes
            self.obj = 'ofdesire'
            self.success_status = 'SUCCESS'
            self.fail_status = 'FAIL'
            self.time_till_start_of_response = 0.1 # in seconds
            self.time_till_end_of_response = 0.1 # in seconds
            self.delta = 0.005
            self.sent_events = []
            self.sock = object()
            def sock(): pass
            sock.send = lambda e: self.sent_events.append(e)
            self.dispatcher = EventDispatcher('foo', sock)

        def send_event(self, **args):
            args['timestamp'] = int(1000 * time())
            args['content-size'] = args['content_size']
            del args['content_size']
            self.dispatcher.send(**args)

        def send_start_request(self):
            self.send_event(tag='start-request', content_size=self.content_size, obj=self.obj, status=self.success_status)

        def send_start_response(self):
            self.send_event(tag='start-response', content_size=self.content_size, obj=self.obj, status=self.success_status)

        def send_end_response(self):
            self.send_event(tag='end-response', content_size=self.content_size, obj=self.obj, status=self.success_status)

        def perform_full_request_response_event_generation(self):
            self.send_start_request()
            sleep(self.time_till_start_of_response)
            self.send_start_response()
            sleep(self.time_till_end_of_response)
            self.send_end_response()

        def test_that_event_was_sent(self):
            self.send_event(tag='start-request', content_size=1000, obj='ofdesire', status=1)
            self.assertEquals(self.content_size, self.sent_events[0]['content-size'])

        def test_event_latency(self):
            self.perform_full_request_response_event_generation()

            latency = self.dispatcher.calculate_event_time_difference(self.sent_events[0], self.sent_events[1])
            self.assertGreater(latency, 0)
            self.assertAlmostEqual(self.time_till_start_of_response, latency, delta=self.delta)

        def test_event_throughput(self):
            self.perform_full_request_response_event_generation()

            throughput = self.dispatcher.calculate_mean_value_per_time_unit(self.sent_events[0]['content-size'], self.time_till_start_of_response + self.time_till_end_of_response)
            self.assertAlmostEqual(self.content_size / (self.time_till_start_of_response + self.time_till_end_of_response), throughput, delta=self.delta)

        def test_event_transaction_time(self):
            self.perform_full_request_response_event_generation()

            transaction_time = self.dispatcher.calculate_event_time_difference(self.sent_events[0], self.sent_events[2])
            self.assertGreater(transaction_time, 0)
            self.assertAlmostEqual(self.time_till_start_of_response + self.time_till_end_of_response, transaction_time, delta=self.delta)

    unittest.main()

    mon = VismoEventDispatcher('foo')
    mon.send(tag='start-request', obj='ofdesire', status=1)
    sleep(1)
    mon.send(tag='start-response', obj='ofdesire', status=2)
    sleep(1)
    mon.send(tag='end-response', content_size=1000, obj='ofdesire', status=3)

<commit_msg>relying on http method, not some adhoc topic stuff<commit_after>#!/usr/bin/python2
# vim: set fileencoding=utf-8

from __future__ import print_function

import socket, struct, fcntl, sys
from os import getpid
from time import time
from pyjavaproperties import Properties
from uuid import uuid4
import zmq
import json

# this is the file that holds the lib's configuration
CONFIGURATION_PROPERTIES = '/srv/vismo/config.properties'
#CONFIGURATION_PROPERTIES = 'config.properties'

###
##
## Utilities
##
###

def get_public_ip(iface='eth0'):
    """
        Return a tuple whose first element contains the first non loopback interface
        and the second elements is the first not inet6 address of that interface.
    """

    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sockfd = sock.fileno()
    SIOCGIFADDR = 0x8915
    ifreq = struct.pack('16sH14s', iface, socket.AF_INET, '\x00'*14)

    try:
        res = fcntl.ioctl(sockfd, SIOCGIFADDR, ifreq)
    except:
        return None
    finally:
        sock.close()

    ip = struct.unpack('16sH2x4s8x', res)[2]

    return socket.inet_ntoa(ip)

def is_effectively_zero(n):
    return abs(n) <= 5e-9

def log(msg):
    print('vismo-local-event-dispatcher: {0}'.format(msg), file=sys.stderr)

def log_and_raise(msg, excp):
    print('vismo-local-event-dispatcher: {0}'.format(msg), file=sys.stderr)
    raise excp(msg)

_MAX_LENGTH = 80
def safe_repr(obj, short=False):
    try:
        result = repr(obj)
    except Exception:
        result = object.__repr__(obj)
    if not short or len(result) < _MAX_LENGTH:
        return result
    return result[:_MAX_LENGTH] + ' [truncated]...'

## core

class EventDispatcher(object):
    """
        An event dispatcher is used to pass around events to
        interested parties.
    """

    def __init__(self, service_name, sock):
        self.service_name = service_name
        self.sock = sock

    def send(self, **event):
        """Actually dispatch the event."""

        self.sock.send(event)

    def calculate_event_time_difference(self, e1, e2):
        """
            Calculates the time difference of the two events. It is assumed
            that e1 ``happens before'' e2, so that a positive difference is
            returned.

            It is also assumed that the event timestamps are in millis and
            the returned value is in seconds.
        """

        return (float(e2['timestamp']) - float(e1['timestamp'])) / 1000.0

    def calculate_mean_value_per_time_unit(self, val, time_diff):
        if is_effectively_zero(time_diff):
            return 0.0

        return float(val) / float(time_diff)

class VismoEventDispatcher(EventDispatcher):
    """
        This the bridge that handles the event generation code (the caller
        of this library) with the event distribution code (the main
        monitoring instance). This instance talks directly to the locally
        running vismo instance.

        The services of this library are called by the Object Service on
        each request/response cycle. Since each request in object service
        is handled by a different process, we know that each event belongs
        to the same request/response.
    """

    def __init__(self, service_name):
        self.load_configuration()
        sock = self.create_push_socket(self.producers_point)
        super(VismoEventDispatcher, self).__init__(service_name, sock)
        self.start_request_event = None
        self.start_response_event = None
        self.end_response_event = None
        self.id = str(uuid4())

    def load_configuration(self):
        p = Properties()
        p.load(open(CONFIGURATION_PROPERTIES))
        self.producers_point = p['producers.point']
        # FIXME: auto acquire the name according to the machine's ip
        self.cluster_name = 'test'
        self.iface = 'eth0'
        self.ip = get_public_ip(self.iface)

    def create_push_socket(self, end_point):
        ctx = zmq.Context()
        sock = ctx.socket(zmq.PUSH)
        sock.setsockopt(zmq.LINGER, 0)
        sock.connect(end_point)
        log('connecting to endpoint={0}'.format(end_point))

        return sock

    def send(self, **event):
        """
            This is the single public entry point to the object.
        """

        self.cleanup_event(event)
        self.add_basic_fields(event)
        log('sending: {0}'.format(event))
        self.handle_event(event)

    def cleanup_event(self, event):
        """
            For consistency reasons, use dashes and full nouns.
        """

        def object_service_topic_to_http_op(topic):
            if topic == 'reads':
                return 'GET'
            elif topic == 'writes':
                return 'PUT'
            elif topic == 'deletes':
                return 'DELETE'
            else:
                log('got unknown topic: {0}'.format(topic))
                return topic

        if 'content_size' in event:
            event['content-size'] = event['content_size']
            del event['content_size']
        if 'obj' in event:
            event['object'] = event['obj']
            del event['obj']
        if 'topic' in event:
            event['operation'] = object_service_topic_to_http_op(event['topic'])
            del event['topic']

    def add_basic_fields(self, event):
        event['timestamp'] = int(1000 * time())
        event['originating-machine'] = self.ip
        event['originating-service'] = self.service_name
        event['originating-cluster'] = self.cluster_name
        event['id'] = self.id

    def _sock_send(self, event):
        self.sock.send(json.dumps(event))

    def handle_event(self, event):
        """
            This does most of the work. It accumulates the interesting events,
            and upon arrival of all events, makes some request calculations and
            sends the event to the main monitoring instance.
        """

        # has the event anything to do with request/response?
        if 'tag' not in event:
            self._sock_send(event)
            return

        tag = event['tag']

        if tag == 'start-request':
            self.start_request_event = event
        elif tag == 'start-response':
            self.start_response_event = event
        elif tag == 'end-response':
            self.end_response_event = event
        else:
            log_and_raise('handed event with incomprehensible tag: {0}, event: {1}'.format(tag, event), ValueError)

        ## TODO: handle object services failures => availability
        # if we have all the events
        if self.start_request_event and self.start_response_event and self.end_response_event:
            main_event = dict(self.end_response_event)

            main_event['transaction-latency'] = self.calculate_latency()
            main_event['transaction-duration'] = self.calculate_transaction_duration()

            if 'content-size' not in main_event or main_event['content-size'] is None:
                main_event['transaction-throughput'] = 0
            else:
                main_event['transaction-throughput'] = self.calculate_throughput(main_event['content-size'], main_event['transaction-duration'])
            # TODO: Calculate availability and other Niki's required stuff

            # not needed
            del main_event['status']
            del main_event['tag']

            self._sock_send(event)
            self._sock_send(main_event)
        else:
            # send anyway
            self._sock_send(event)

    def calculate_latency(self):
        """
            Latency here is defined as the time duration from the time the system received
            the start request event till the time the system is ready to start serving the
            response, as seen by the start response event.
        """

        return self.calculate_event_time_difference(self.start_request_event, self.start_response_event)

    def calculate_transaction_duration(self):
        """
            Transaction duration is the time spent serving the request. More accurately, is the
            duration from the time the system received the start request event till the time the
            system received the end response event.
        """

        return self.calculate_event_time_difference(self.start_request_event, self.end_response_event)

    def calculate_throughput(self, content_size, transaction_time):
        """
            Throughput is defined as the the number of bytes served in the unit of time.
            Transaction size is measured in bytes, transaction_time in seconds.
            Output value is in bytes per second.
        """

        return self.calculate_mean_value_per_time_unit(content_size, transaction_time)

if __name__ == '__main__':
    import unittest
    from time import sleep

    class MyTestCase(unittest.TestCase):
        """
            This is used to reproduce to methods missing from
            python 2.6.6
        """

        def assertGreater(self, a, b, msg=None):
            """Just like self.assertTrue(a > b), but with a nicer default message."""
            if not a > b:
                standardMsg = '%s not greater than %s' % (safe_repr(a), safe_repr(b))
                self.fail(self._formatMessage(msg, standardMsg))

        def assertAlmostEqual(self, first, second, places=None, msg=None, delta=None):
            """Fail if the two objects are unequal as determined by their
               difference rounded to the given number of decimal places
               (default 7) and comparing to zero, or by comparing that the
               between the two objects is more than the given delta.

               Note that decimal places (from zero) are usually not the same
               as significant digits (measured from the most signficant digit).

               If the two objects compare equal then they will automatically
               compare almost equal.
            """
            if first == second:
                # shortcut
                return
            if delta is not None and places is not None:
                raise TypeError("specify delta or places not both")

            if delta is not None:
                if abs(first - second) <= delta:
                    return

                standardMsg = '%s != %s within %s delta' % (safe_repr(first),
                                                            safe_repr(second),
                                                            safe_repr(delta))
            else:
                if places is None:
                    places = 7

                if round(abs(second-first), places) == 0:
                    return

                standardMsg = '%s != %s within %r places' % (safe_repr(first),
                                                              safe_repr(second),
                                                              places)
            msg = self._formatMessage(msg, standardMsg)
            raise self.failureException(msg)

    class EventDispatcherTest(MyTestCase):
        def setUp(self):
            self.content_size = 1000 # in bytes
            self.obj = 'ofdesire'
            self.success_status = 'SUCCESS'
            self.fail_status = 'FAIL'
            self.time_till_start_of_response = 0.1 # in seconds
            self.time_till_end_of_response = 0.1 # in seconds
            self.delta = 0.005
            self.sent_events = []
            self.sock = object()
            def sock(): pass
            sock.send = lambda e: self.sent_events.append(e)
            self.dispatcher = EventDispatcher('foo', sock)

        def send_event(self, **args):
            args['timestamp'] = int(1000 * time())
            args['content-size'] = args['content_size']
            del args['content_size']
            self.dispatcher.send(**args)

        def send_start_request(self):
            self.send_event(tag='start-request', content_size=self.content_size, obj=self.obj, status=self.success_status)

        def send_start_response(self):
            self.send_event(tag='start-response', content_size=self.content_size, obj=self.obj, status=self.success_status)

        def send_end_response(self):
            self.send_event(tag='end-response', content_size=self.content_size, obj=self.obj, status=self.success_status)

        def perform_full_request_response_event_generation(self):
            self.send_start_request()
            sleep(self.time_till_start_of_response)
            self.send_start_response()
            sleep(self.time_till_end_of_response)
            self.send_end_response()

        def test_that_event_was_sent(self):
            self.send_event(tag='start-request', content_size=1000, obj='ofdesire', status=1)
            self.assertEquals(self.content_size, self.sent_events[0]['content-size'])

        def test_event_latency(self):
            self.perform_full_request_response_event_generation()

            latency = self.dispatcher.calculate_event_time_difference(self.sent_events[0], self.sent_events[1])
            self.assertGreater(latency, 0)
            self.assertAlmostEqual(self.time_till_start_of_response, latency, delta=self.delta)

        def test_event_throughput(self):
            self.perform_full_request_response_event_generation()

            throughput = self.dispatcher.calculate_mean_value_per_time_unit(self.sent_events[0]['content-size'], self.time_till_start_of_response + self.time_till_end_of_response)
            self.assertAlmostEqual(self.content_size / (self.time_till_start_of_response + self.time_till_end_of_response), throughput, delta=self.delta)

        def test_event_transaction_time(self):
            self.perform_full_request_response_event_generation()

            transaction_time = self.dispatcher.calculate_event_time_difference(self.sent_events[0], self.sent_events[2])
            self.assertGreater(transaction_time, 0)
            self.assertAlmostEqual(self.time_till_start_of_response + self.time_till_end_of_response, transaction_time, delta=self.delta)

    unittest.main()

    mon = VismoEventDispatcher('foo')
    mon.send(tag='start-request', obj='ofdesire', status=1, operation='GET')
    sleep(1)
    mon.send(tag='start-response', obj='ofdesire', status=2, operation='GET')
    sleep(1)
    mon.send(tag='end-response', content_size=1000, obj='ofdesire', status=3, operation='GET')

<|endoftext|>


<commit_before>import pytest
import random
import os
from multiprocessing import Process, Queue
from keras.utils.test_utils import keras_test
from keras.utils.test_utils import layer_test
from keras.models import Sequential
from keras import applications
from keras import backend as K

pytestmark = pytest.mark.skipif(
    os.environ.get('CORE_CHANGED', 'True') == 'False' and
    os.environ.get('APP_CHANGED', 'True') == 'False',
    reason='Runs only when the relevant files have been modified.')

MOBILENET_LIST = [(applications.MobileNet, 1024),
                  (applications.MobileNetV2, 1280)]
DENSENET_LIST = [(applications.DenseNet121, 1024),
                 (applications.DenseNet169, 1664),
                 (applications.DenseNet201, 1920)]
NASNET_LIST = [(applications.NASNetMobile, 1056),
               (applications.NASNetLarge, 4032)]

def _get_output_shape(model_fn):
    if K.backend() == 'cntk':
        # Create model in a subprocess so that
        # the memory consumed by InceptionResNetV2 will be
        # released back to the system after this test
        # (to deal with OOM error on CNTK backend).
        # TODO: remove the use of multiprocessing from these tests
        # once a memory clearing mechanism
        # is implemented in the CNTK backend.
        def target(queue):
            model = model_fn()
            queue.put(model.output_shape)
        queue = Queue()
        p = Process(target=target, args=(queue,))
        p.start()
        p.join()
        # The error in a subprocess won't propagate
        # to the main process, so we check if the model
        # is successfully created by checking if the output shape
        # has been put into the queue
        assert not queue.empty(), 'Model creation failed.'
        return queue.get_nowait()
    else:
        model = model_fn()
        return model.output_shape

@keras_test
def _test_application_basic(app, last_dim=1000):
    output_shape = _get_output_shape(lambda: app(weights=None))
    assert output_shape == (None, last_dim)

@keras_test
def _test_application_notop(app, last_dim):
    output_shape = _get_output_shape(
        lambda: app(weights=None, include_top=False))
    assert output_shape == (None, None, None, last_dim)

@keras_test
def _test_application_variable_input_channels(app, last_dim):
    if K.image_data_format() == 'channels_first':
        input_shape = (1, None, None)
    else:
        input_shape = (None, None, 1)
    output_shape = _get_output_shape(
        lambda: app(weights=None, include_top=False, input_shape=input_shape))
    assert output_shape == (None, None, None, last_dim)

    if K.image_data_format() == 'channels_first':
        input_shape = (4, None, None)
    else:
        input_shape = (None, None, 4)
    output_shape = _get_output_shape(
        lambda: app(weights=None, include_top=False, input_shape=input_shape))
    assert output_shape == (None, None, None, last_dim)

@keras_test
def _test_app_pooling(app, last_dim):
    output_shape = _get_output_shape(
        lambda: app(weights=None,
                    include_top=False,
                    pooling=random.choice(['avg', 'max'])))
    assert output_shape == (None, last_dim)

def test_resnet50():
    app = applications.ResNet50
    last_dim = 2048
    _test_application_basic(app)
    _test_application_notop(app, last_dim)
    _test_application_variable_input_channels(app, last_dim)
    _test_app_pooling(app, last_dim)

def test_vgg():
    app = random.choice([applications.VGG16, applications.VGG19])
    last_dim = 512
    _test_application_basic(app)
    _test_application_notop(app, last_dim)
    _test_application_variable_input_channels(app, last_dim)
    _test_app_pooling(app, last_dim)

def test_xception():
    app = applications.Xception
    last_dim = 2048
    _test_application_basic(app)
    _test_application_notop(app, last_dim)
    _test_application_variable_input_channels(app, last_dim)
    _test_app_pooling(app, last_dim)

def test_inceptionv3():
    app = applications.InceptionV3
    last_dim = 2048
    _test_application_basic(app)
    _test_application_notop(app, last_dim)
    _test_application_variable_input_channels(app, last_dim)
    _test_app_pooling(app, last_dim)

def test_inceptionresnetv2():
    app = applications.InceptionResNetV2
    last_dim = 1536
    _test_application_basic(app)
    _test_application_notop(app, last_dim)
    _test_application_variable_input_channels(app, last_dim)
    _test_app_pooling(app, last_dim)

def test_mobilenet():
    app, last_dim = random.choice(MOBILENET_LIST)
    _test_application_basic(app)
    _test_application_notop(app, last_dim)
    _test_application_variable_input_channels(app, last_dim)
    _test_app_pooling(app, last_dim)

def test_densenet():
    app, last_dim = random.choice(DENSENET_LIST)
    _test_application_basic(app)
    _test_application_notop(app, last_dim)
    _test_application_variable_input_channels(app, last_dim)
    _test_app_pooling(app, last_dim)

@pytest.mark.skipif((K.backend() != 'tensorflow'),
                    reason='NASNets are supported only on TensorFlow')
def test_nasnet():
    app, last_dim = random.choice(NASNET_LIST)
    _test_application_basic(app)
    _test_application_notop(app, last_dim)
    _test_application_variable_input_channels(app, last_dim)
    _test_app_pooling(app, last_dim)

if __name__ == '__main__':
    pytest.main([__file__])
<commit_msg>Reduce tests for applications (#10346)<commit_after>import pytest
import random
import os
from multiprocessing import Process, Queue
from keras.utils.test_utils import keras_test
from keras.utils.test_utils import layer_test
from keras.models import Sequential
from keras import applications
from keras import backend as K

pytestmark = pytest.mark.skipif(
    os.environ.get('CORE_CHANGED', 'True') == 'False' and
    os.environ.get('APP_CHANGED', 'True') == 'False',
    reason='Runs only when the relevant files have been modified.')

MODEL_LIST = [
    (applications.ResNet50, 2048),
    (applications.VGG16, 512),
    (applications.VGG19, 512),
    (applications.Xception, 2048),
    (applications.InceptionV3, 2048),
    (applications.InceptionResNetV2, 1536),
    (applications.MobileNet, 1024),
    (applications.MobileNetV2, 1280),
    (applications.DenseNet121, 1024),
    (applications.DenseNet169, 1664),
    (applications.DenseNet201, 1920)
    # TODO: enable nasnet tests if they support Theano and CNTK
    # (applications.NASNetMobile, 1056),
    # (applications.NASNetLarge, 4032)
]

def _get_output_shape(model_fn):
    if K.backend() == 'cntk':
        # Create model in a subprocess so that
        # the memory consumed by InceptionResNetV2 will be
        # released back to the system after this test
        # (to deal with OOM error on CNTK backend).
        # TODO: remove the use of multiprocessing from these tests
        # once a memory clearing mechanism
        # is implemented in the CNTK backend.
        def target(queue):
            model = model_fn()
            queue.put(model.output_shape)
        queue = Queue()
        p = Process(target=target, args=(queue,))
        p.start()
        p.join()
        # The error in a subprocess won't propagate
        # to the main process, so we check if the model
        # is successfully created by checking if the output shape
        # has been put into the queue
        assert not queue.empty(), 'Model creation failed.'
        return queue.get_nowait()
    else:
        model = model_fn()
        return model.output_shape

@keras_test
def _test_application_basic(app, last_dim=1000):
    output_shape = _get_output_shape(lambda: app(weights=None))
    assert output_shape == (None, last_dim)

@keras_test
def _test_application_notop(app, last_dim):
    output_shape = _get_output_shape(
        lambda: app(weights=None, include_top=False))
    assert output_shape == (None, None, None, last_dim)

def test_applications():
    for _ in range(3):
        app, last_dim = random.choice(MODEL_LIST)
        _test_application_basic(app)
        _test_application_notop(app, last_dim)

if __name__ == '__main__':
    pytest.main([__file__])
<|endoftext|>