<commit_before>"""Tests for UnauthenticatedReddit class."""

from __future__ import print_function, unicode_literals

import os
import sys
import time
from six.moves.urllib.parse import urlparse

from praw.helpers import submissions_between
from praw.errors import PRAWException
from .helper import PRAWTest, betamax, teardown_on_keyboard_interrupt

def mock_time():
    return 1448923624.0

real_time = time.time

class TestHelperSubmissionsBetween(PRAWTest):
    def setUp(self):
        PRAWTest.setUp(self)
        time.time = mock_time
        self.verbosity = 3
        if os.getenv('TRAVIS') and not hasattr(sys, 'pypy_version_info'):
            self.verbosity = 0

    def tearDown(self):
        PRAWTest.tearDown(self)
        # make sure nose reports correct running time
        time.time = real_time

    @betamax()
    @teardown_on_keyboard_interrupt
    def test_submissions_between_raises_correctly(self):
        with self.assertRaises(PRAWException):
            list(submissions_between(self.r,
                                     self.sr,
                                     extra_cloudsearch_fields={'self': 'yes'},
                                     verbosity=self.verbosity))

    @betamax()
    @teardown_on_keyboard_interrupt
    def test_submissions_between_order(self):
        all_subs = list(submissions_between(self.r,
                                            self.sr,
                                            highest_timestamp=time.time(),
                                            verbosity=self.verbosity))

        for i in range(len(all_subs) - 1):
            self.assertGreaterEqual(all_subs[i].created_utc,
                                    all_subs[i + 1].created_utc)

        sr_obj = self.r.get_subreddit(self.sr)
        all_subs_sr_object = list(
            submissions_between(self.r,
                                sr_obj,
                                verbosity=self.verbosity)
        )

        self.assertEqual(all_subs, all_subs_sr_object)

        all_subs_reversed = list(submissions_between(self.r,
                                                     sr_obj,
                                                     newest_first=False,
                                                     verbosity=self.verbosity))

        self.assertEqual(all_subs, list(reversed(all_subs_reversed)))

    @betamax()
    @teardown_on_keyboard_interrupt
    def test_submissions_between_with_filters(self):
        all_subs = list(submissions_between(self.r,
                                            self.sr,
                                            verbosity=self.verbosity))
        t1 = 1420000000
        t2 = 1441111111

        t1_t2_subs = list(submissions_between(self.r,
                                              self.sr,
                                              lowest_timestamp=t1,
                                              highest_timestamp=t2,
                                              verbosity=self.verbosity))

        def filter_subs(subs,
                        lowest_timestamp=0,
                        highest_timestamp=10**10,
                        criterion=None):
            filtered = [s for s in subs
                        if s.created_utc <= highest_timestamp and
                        s.created_utc >= lowest_timestamp and
                        (criterion is None or criterion(s))]
            # make sure we never accidentally craft a bad test case
            self.assertGreater(len(filtered), 0)
            return filtered

        t1_t2_subs_canon = filter_subs(all_subs, t1, t2)
        self.assertEqual(t1_t2_subs, t1_t2_subs_canon)

        self_subs = list(
            submissions_between(self.r,
                                self.sr,
                                extra_cloudsearch_fields={"self": "1"},
                                verbosity=self.verbosity)
        )
        self_subs_canon = filter_subs(all_subs,
                                      criterion=lambda s: s.is_self)
        self.assertEqual(self_subs, self_subs_canon)

        def wa_criterion(s):
            return not s.is_self and \
                urlparse(s.url).netloc == "web.archive.org"

        wa_cs_fields = {"self": "0",
                        "site": "web.archive.org"}

        subs_wa = list(
            submissions_between(self.r,
                                self.sr,
                                extra_cloudsearch_fields=wa_cs_fields,
                                verbosity=self.verbosity)
        )

        subs_wa_canon = filter_subs(all_subs, criterion=wa_criterion)
        self.assertEqual(subs_wa, subs_wa_canon)

        patu_cs_fields = {"self": "1",
                          "author": "PyAPITestUser2",
                          "title": "test"}

        def patu_criterion(s):
            return s.is_self and \
                s.author.name == "PyAPITestUser2" and\
                "test" in s.title.lower()

        subs_patu = list(
            submissions_between(self.r,
                                self.sr,
                                extra_cloudsearch_fields=patu_cs_fields,
                                verbosity=self.verbosity)
        )

        subs_patu_canon = filter_subs(all_subs, criterion=patu_criterion)
        self.assertEqual(subs_patu, subs_patu_canon)
<commit_msg>Disable submissions_between debug output in tests.<commit_after>"""Tests for UnauthenticatedReddit class."""

from __future__ import print_function, unicode_literals

import os
import sys
import time
from six.moves.urllib.parse import urlparse

from praw.helpers import submissions_between
from praw.errors import PRAWException
from .helper import PRAWTest, betamax, teardown_on_keyboard_interrupt

def mock_time():
    return 1448923624.0

real_time = time.time

class TestHelperSubmissionsBetween(PRAWTest):
    def setUp(self):
        PRAWTest.setUp(self)
        time.time = mock_time
        self.verbosity = 0

    def tearDown(self):
        PRAWTest.tearDown(self)
        # make sure nose reports correct running time
        time.time = real_time

    @betamax()
    @teardown_on_keyboard_interrupt
    def test_submissions_between_raises_correctly(self):
        with self.assertRaises(PRAWException):
            list(submissions_between(self.r,
                                     self.sr,
                                     extra_cloudsearch_fields={'self': 'yes'},
                                     verbosity=self.verbosity))

    @betamax()
    @teardown_on_keyboard_interrupt
    def test_submissions_between_order(self):
        all_subs = list(submissions_between(self.r,
                                            self.sr,
                                            highest_timestamp=time.time(),
                                            verbosity=self.verbosity))

        for i in range(len(all_subs) - 1):
            self.assertGreaterEqual(all_subs[i].created_utc,
                                    all_subs[i + 1].created_utc)

        sr_obj = self.r.get_subreddit(self.sr)
        all_subs_sr_object = list(
            submissions_between(self.r,
                                sr_obj,
                                verbosity=self.verbosity)
        )

        self.assertEqual(all_subs, all_subs_sr_object)

        all_subs_reversed = list(submissions_between(self.r,
                                                     sr_obj,
                                                     newest_first=False,
                                                     verbosity=self.verbosity))

        self.assertEqual(all_subs, list(reversed(all_subs_reversed)))

    @betamax()
    @teardown_on_keyboard_interrupt
    def test_submissions_between_with_filters(self):
        all_subs = list(submissions_between(self.r,
                                            self.sr,
                                            verbosity=self.verbosity))
        t1 = 1420000000
        t2 = 1441111111

        t1_t2_subs = list(submissions_between(self.r,
                                              self.sr,
                                              lowest_timestamp=t1,
                                              highest_timestamp=t2,
                                              verbosity=self.verbosity))

        def filter_subs(subs,
                        lowest_timestamp=0,
                        highest_timestamp=10**10,
                        criterion=None):
            filtered = [s for s in subs
                        if s.created_utc <= highest_timestamp and
                        s.created_utc >= lowest_timestamp and
                        (criterion is None or criterion(s))]
            # make sure we never accidentally craft a bad test case
            self.assertGreater(len(filtered), 0)
            return filtered

        t1_t2_subs_canon = filter_subs(all_subs, t1, t2)
        self.assertEqual(t1_t2_subs, t1_t2_subs_canon)

        self_subs = list(
            submissions_between(self.r,
                                self.sr,
                                extra_cloudsearch_fields={"self": "1"},
                                verbosity=self.verbosity)
        )
        self_subs_canon = filter_subs(all_subs,
                                      criterion=lambda s: s.is_self)
        self.assertEqual(self_subs, self_subs_canon)

        def wa_criterion(s):
            return not s.is_self and \
                urlparse(s.url).netloc == "web.archive.org"

        wa_cs_fields = {"self": "0",
                        "site": "web.archive.org"}

        subs_wa = list(
            submissions_between(self.r,
                                self.sr,
                                extra_cloudsearch_fields=wa_cs_fields,
                                verbosity=self.verbosity)
        )

        subs_wa_canon = filter_subs(all_subs, criterion=wa_criterion)
        self.assertEqual(subs_wa, subs_wa_canon)

        patu_cs_fields = {"self": "1",
                          "author": "PyAPITestUser2",
                          "title": "test"}

        def patu_criterion(s):
            return s.is_self and \
                s.author.name == "PyAPITestUser2" and\
                "test" in s.title.lower()

        subs_patu = list(
            submissions_between(self.r,
                                self.sr,
                                extra_cloudsearch_fields=patu_cs_fields,
                                verbosity=self.verbosity)
        )

        subs_patu_canon = filter_subs(all_subs, criterion=patu_criterion)
        self.assertEqual(subs_patu, subs_patu_canon)
<|endoftext|>


<commit_before>"""
Test OLPC extensions to MUC invitations.
"""

import dbus

from twisted.words.xish import domish, xpath

from gabbletest import go, make_result_iq, exec_test
from servicetest import call_async, EventPattern

def test(q, bus, conn, stream):
    conn.Connect()

    q.expect('dbus-signal', signal='StatusChanged', args=[0, 1])

    handles = {}
    handles['bob'] = conn.RequestHandles(1, ['bob@localhost'])[0]

    buddy_iface = dbus.Interface(conn, 'org.laptop.Telepathy.BuddyInfo')
    act_prop_iface = dbus.Interface(conn, 'org.laptop.Telepathy.ActivityProperties')
    call_async(q, buddy_iface, 'GetActivities', handles['bob'])

    event = q.expect('stream-iq', iq_type='get', to='bob@localhost')
    # Bob has no activities
    event.stanza['type'] = 'result'
    event.stanza['to'] = 'test@localhost'
    event.stanza['from'] = 'bob@localhost'
    stream.send(event.stanza)

    event = q.expect('dbus-return', method='GetActivities')
    # initially, Bob has no activities
    assert event.value == ([],)

    # Bob sends an activity properties message
    message = domish.Element(('jabber:client', 'message'))
    message['from'] = 'bob@localhost'
    message['to'] = 'test@localhost'
    properties = message.addElement(
        ('http://laptop.org/xmpp/activity-properties', 'properties'))
    properties['room'] = 'chat@conf.localhost'
    properties['activity'] = 'foo_id'
    property = properties.addElement((None, 'property'))
    property['type'] = 'str'
    property['name'] = 'color'
    property.addContent('#ffff00,#00ffff')
    property = properties.addElement((None, 'property'))
    property['type'] = 'bool'
    property['name'] = 'private'
    property.addContent('1')

    stream.send(message)

    event = q.expect('dbus-signal', signal='ActivityPropertiesChanged')
    handles['chat'], props = event.args
    assert props == {'color': '#ffff00,#00ffff', 'private' : True}

    event = q.expect('dbus-signal', signal='ActivitiesChanged')
    assert event.args[0] == handles['bob']
    acts = event.args[1]
    assert len(acts) == 1
    assert acts[0] == ('foo_id', handles['chat'])

    props = act_prop_iface.GetProperties(handles['chat'])
    assert props == {'color': '#ffff00,#00ffff', 'private' : True}

    # Bobs invites us to the activity
    message = domish.Element((None, 'message'))
    message['from'] = 'chat@conf.localhost'
    message['to'] = 'test@localhost'
    x = message.addElement(('http://jabber.org/protocol/muc#user', 'x'))
    invite = x.addElement((None, 'invite'))
    invite['from'] = 'bob@localhost'
    reason = invite.addElement((None, 'reason'))
    reason.addContent('No good reason')

    stream.send(message)

    event = q.expect('dbus-signal', signal='NewChannel')
    assert event.args[1] == 'org.freedesktop.Telepathy.Channel.Type.Text'

    assert event.args[2] == 2   # handle type
    assert event.args[3] == handles['chat']   # handle

    text_chan = bus.get_object(conn.bus_name, event.args[0])
    group_iface = dbus.Interface(text_chan,
            'org.freedesktop.Telepathy.Channel.Interface.Group')

    members = group_iface.GetAllMembers()[0]
    local_pending = group_iface.GetAllMembers()[1]
    remote_pending = group_iface.GetAllMembers()[2]

    assert len(members) == 1
    assert conn.InspectHandles(1, members)[0] == 'bob@localhost'
    assert len(local_pending) == 1
    # FIXME: the username-part-is-nickname assumption
    assert conn.InspectHandles(1, local_pending)[0] == \
            'chat@conf.localhost/test'
    assert len(remote_pending) == 0

    handles['chat_self'] = group_iface.GetSelfHandle()
    assert handles['chat_self'] == local_pending[0]

    # by now, we should have picked up the extra activity properties
    call_async(q, buddy_iface, 'GetActivities', handles['bob'])

    event = q.expect('stream-iq', iq_type='get', to='bob@localhost')
    # Bob still has no (public) activities
    event.stanza['type'] = 'result'
    event.stanza['to'] = 'test@localhost'
    event.stanza['from'] = 'bob@localhost'
    stream.send(event.stanza)

    event = q.expect('dbus-return', method='GetActivities')

    assert event.value == ([('foo_id', handles['chat'])],)

    # OK, now accept the invitation
    call_async(q, group_iface, 'AddMembers', [handles['chat_self']], 'Oh, OK then')

    _, event, _ = q.expect_many(
        EventPattern('stream-presence', to='chat@conf.localhost/test'),
        EventPattern('dbus-signal', signal='MembersChanged'),
        EventPattern('dbus-return', method='AddMembers')
        )

    assert event.args == ['', [], [handles['bob']], [],
            [handles['chat_self']], 0, handles['chat_self']]

    # Send presence for own membership of room.
    presence = domish.Element((None, 'presence'))
    presence['from'] = 'chat@conf.localhost/test'
    x = presence.addElement(('http://jabber.org/protocol/muc#user', 'x'))
    item = x.addElement('item')
    item['affiliation'] = 'owner'
    item['role'] = 'moderator'
    stream.send(presence)

    event = q.expect('dbus-signal', signal='MembersChanged')
    assert event.args == ['', [handles['chat_self']], [], [], [], 0, 0]

    call_async(q, buddy_iface, 'SetActivities', [('foo_id', handles['chat'])])

    event = q.expect('stream-iq', iq_type='set')
    event.stanza['type'] = 'result'
    event.stanza['to'] = 'test@localhost'
    event.stanza['from'] = 'test@localhost'
    stream.send(event.stanza)

    q.expect('dbus-return', method='SetActivities')
    call_async(q, act_prop_iface, 'SetProperties',
        handles['chat'], {'color': '#ffff00,#00ffff', 'private': True})

    event = q.expect('dbus-signal', signal='ActivityPropertiesChanged')
    chat_handle, props = event.args
    assert chat_handle == handles['chat']
    assert props == {'color': '#ffff00,#00ffff', 'private' : True}

    q.expect('dbus-return', method='SetProperties')
    # Test sending an invitation
    handles['alice'] = conn.RequestHandles(1,
        ['alice@localhost'])[0]
    call_async(q, group_iface, 'AddMembers', [handles['alice']],
            'I want to test invitations')

    event = q.expect('stream-message', to='alice@localhost')
    message = event.stanza

    properties = xpath.queryForNodes('/message/properties', message)
    assert (properties is not None and len(properties) == 1), repr(properties)
    assert properties[0].uri == 'http://laptop.org/xmpp/activity-properties'
    assert properties[0]['room'] == 'chat@conf.localhost'
    assert properties[0]['activity'] == 'foo_id'

    property = xpath.queryForNodes('/properties/property', properties[0])
    assert (property is not None and len(property) == 2), repr(property)
    seen = set()
    for p in property:
        seen.add(p['name'])
        if p['name'] == 'color':
            assert p['type'] == 'str'
            assert str(p) == '#ffff00,#00ffff'
        elif p['name'] == 'private':
            assert p['type'] == 'bool'
            assert str(p) == '1'
        else:
            assert False, 'Unexpected property %s' % p['name']
    assert 'color' in seen, seen
    assert 'private' in seen, seen

    event = q.expect('stream-message', to='chat@conf.localhost')
    message = event.stanza

    x = xpath.queryForNodes('/message/x', message)
    assert (x is not None and len(x) == 1), repr(x)
    assert x[0].uri == 'http://jabber.org/protocol/muc#user'

    invites = xpath.queryForNodes('/x/invite', x[0])
    assert (invites is not None and len(invites) == 1), repr(invites)
    assert invites[0]['to'] == 'alice@localhost'

    reasons = xpath.queryForNodes('/invite/reason', invites[0])
    assert (reasons is not None and len(reasons) == 1), repr(reasons)
    assert str(reasons[0]) == 'I want to test invitations'

    call_async(q, act_prop_iface, 'SetProperties',
        handles['chat'], {'color': '#f00baa,#f00baa', 'private': True})

    event = q.expect('stream-message', to='alice@localhost')
    message = event.stanza

    properties = xpath.queryForNodes('/message/properties', message)
    assert (properties is not None and len(properties) == 1), repr(properties)
    assert properties[0].uri == 'http://laptop.org/xmpp/activity-properties'
    assert properties[0]['room'] == 'chat@conf.localhost'
    assert properties[0]['activity'] == 'foo_id'

    property = xpath.queryForNodes('/properties/property', properties[0])
    assert (property is not None and len(property) == 2), repr(property)
    seen = set()
    for p in property:
        seen.add(p['name'])
        if p['name'] == 'color':
            assert p['type'] == 'str'
            assert str(p) == '#f00baa,#f00baa'
        elif p['name'] == 'private':
            assert p['type'] == 'bool'
            assert str(p) == '1'
        else:
            assert False, 'Unexpected property %s' % p['name']
    assert 'color' in seen, seen
    assert 'private' in seen, seen

    event = q.expect('dbus-signal', signal='ActivityPropertiesChanged')
    chat_handle, props = event.args
    assert chat_handle == handles['chat']
    assert props == {'color': '#f00baa,#f00baa', 'private' : True}

    q.expect('dbus-return', method='SetProperties')

    conn.Disconnect()

    q.expect('dbus-signal', signal='StatusChanged', args=[2, 1])

if __name__ == '__main__':
    exec_test(test)
<commit_msg>olpc-muc-invitation.py: use ns.ps<commit_after>"""
Test OLPC extensions to MUC invitations.
"""

import dbus

from twisted.words.xish import domish, xpath

from gabbletest import go, make_result_iq, exec_test
from servicetest import call_async, EventPattern
import ns

def test(q, bus, conn, stream):
    conn.Connect()

    q.expect('dbus-signal', signal='StatusChanged', args=[0, 1])

    handles = {}
    handles['bob'] = conn.RequestHandles(1, ['bob@localhost'])[0]

    buddy_iface = dbus.Interface(conn, 'org.laptop.Telepathy.BuddyInfo')
    act_prop_iface = dbus.Interface(conn, 'org.laptop.Telepathy.ActivityProperties')
    call_async(q, buddy_iface, 'GetActivities', handles['bob'])

    event = q.expect('stream-iq', iq_type='get', to='bob@localhost')
    # Bob has no activities
    event.stanza['type'] = 'result'
    event.stanza['to'] = 'test@localhost'
    event.stanza['from'] = 'bob@localhost'
    stream.send(event.stanza)

    event = q.expect('dbus-return', method='GetActivities')
    # initially, Bob has no activities
    assert event.value == ([],)

    # Bob sends an activity properties message
    message = domish.Element(('jabber:client', 'message'))
    message['from'] = 'bob@localhost'
    message['to'] = 'test@localhost'
    properties = message.addElement(
        (ns.OLPC_ACTIVITY_PROPS, 'properties'))
    properties['room'] = 'chat@conf.localhost'
    properties['activity'] = 'foo_id'
    property = properties.addElement((None, 'property'))
    property['type'] = 'str'
    property['name'] = 'color'
    property.addContent('#ffff00,#00ffff')
    property = properties.addElement((None, 'property'))
    property['type'] = 'bool'
    property['name'] = 'private'
    property.addContent('1')

    stream.send(message)

    event = q.expect('dbus-signal', signal='ActivityPropertiesChanged')
    handles['chat'], props = event.args
    assert props == {'color': '#ffff00,#00ffff', 'private' : True}

    event = q.expect('dbus-signal', signal='ActivitiesChanged')
    assert event.args[0] == handles['bob']
    acts = event.args[1]
    assert len(acts) == 1
    assert acts[0] == ('foo_id', handles['chat'])

    props = act_prop_iface.GetProperties(handles['chat'])
    assert props == {'color': '#ffff00,#00ffff', 'private' : True}

    # Bobs invites us to the activity
    message = domish.Element((None, 'message'))
    message['from'] = 'chat@conf.localhost'
    message['to'] = 'test@localhost'
    x = message.addElement((ns.MUC_USER, 'x'))
    invite = x.addElement((None, 'invite'))
    invite['from'] = 'bob@localhost'
    reason = invite.addElement((None, 'reason'))
    reason.addContent('No good reason')

    stream.send(message)

    event = q.expect('dbus-signal', signal='NewChannel')
    assert event.args[1] == 'org.freedesktop.Telepathy.Channel.Type.Text'

    assert event.args[2] == 2   # handle type
    assert event.args[3] == handles['chat']   # handle

    text_chan = bus.get_object(conn.bus_name, event.args[0])
    group_iface = dbus.Interface(text_chan,
            'org.freedesktop.Telepathy.Channel.Interface.Group')

    members = group_iface.GetAllMembers()[0]
    local_pending = group_iface.GetAllMembers()[1]
    remote_pending = group_iface.GetAllMembers()[2]

    assert len(members) == 1
    assert conn.InspectHandles(1, members)[0] == 'bob@localhost'
    assert len(local_pending) == 1
    # FIXME: the username-part-is-nickname assumption
    assert conn.InspectHandles(1, local_pending)[0] == \
            'chat@conf.localhost/test'
    assert len(remote_pending) == 0

    handles['chat_self'] = group_iface.GetSelfHandle()
    assert handles['chat_self'] == local_pending[0]

    # by now, we should have picked up the extra activity properties
    call_async(q, buddy_iface, 'GetActivities', handles['bob'])

    event = q.expect('stream-iq', iq_type='get', to='bob@localhost')
    # Bob still has no (public) activities
    event.stanza['type'] = 'result'
    event.stanza['to'] = 'test@localhost'
    event.stanza['from'] = 'bob@localhost'
    stream.send(event.stanza)

    event = q.expect('dbus-return', method='GetActivities')

    assert event.value == ([('foo_id', handles['chat'])],)

    # OK, now accept the invitation
    call_async(q, group_iface, 'AddMembers', [handles['chat_self']], 'Oh, OK then')

    _, event, _ = q.expect_many(
        EventPattern('stream-presence', to='chat@conf.localhost/test'),
        EventPattern('dbus-signal', signal='MembersChanged'),
        EventPattern('dbus-return', method='AddMembers')
        )

    assert event.args == ['', [], [handles['bob']], [],
            [handles['chat_self']], 0, handles['chat_self']]

    # Send presence for own membership of room.
    presence = domish.Element((None, 'presence'))
    presence['from'] = 'chat@conf.localhost/test'
    x = presence.addElement((ns.MUC_USER, 'x'))
    item = x.addElement('item')
    item['affiliation'] = 'owner'
    item['role'] = 'moderator'
    stream.send(presence)

    event = q.expect('dbus-signal', signal='MembersChanged')
    assert event.args == ['', [handles['chat_self']], [], [], [], 0, 0]

    call_async(q, buddy_iface, 'SetActivities', [('foo_id', handles['chat'])])

    event = q.expect('stream-iq', iq_type='set')
    event.stanza['type'] = 'result'
    event.stanza['to'] = 'test@localhost'
    event.stanza['from'] = 'test@localhost'
    stream.send(event.stanza)

    q.expect('dbus-return', method='SetActivities')
    call_async(q, act_prop_iface, 'SetProperties',
        handles['chat'], {'color': '#ffff00,#00ffff', 'private': True})

    event = q.expect('dbus-signal', signal='ActivityPropertiesChanged')
    chat_handle, props = event.args
    assert chat_handle == handles['chat']
    assert props == {'color': '#ffff00,#00ffff', 'private' : True}

    q.expect('dbus-return', method='SetProperties')
    # Test sending an invitation
    handles['alice'] = conn.RequestHandles(1,
        ['alice@localhost'])[0]
    call_async(q, group_iface, 'AddMembers', [handles['alice']],
            'I want to test invitations')

    event = q.expect('stream-message', to='alice@localhost')
    message = event.stanza

    properties = xpath.queryForNodes('/message/properties', message)
    assert (properties is not None and len(properties) == 1), repr(properties)
    assert properties[0].uri == ns.OLPC_ACTIVITY_PROPS
    assert properties[0]['room'] == 'chat@conf.localhost'
    assert properties[0]['activity'] == 'foo_id'

    property = xpath.queryForNodes('/properties/property', properties[0])
    assert (property is not None and len(property) == 2), repr(property)
    seen = set()
    for p in property:
        seen.add(p['name'])
        if p['name'] == 'color':
            assert p['type'] == 'str'
            assert str(p) == '#ffff00,#00ffff'
        elif p['name'] == 'private':
            assert p['type'] == 'bool'
            assert str(p) == '1'
        else:
            assert False, 'Unexpected property %s' % p['name']
    assert 'color' in seen, seen
    assert 'private' in seen, seen

    event = q.expect('stream-message', to='chat@conf.localhost')
    message = event.stanza

    x = xpath.queryForNodes('/message/x', message)
    assert (x is not None and len(x) == 1), repr(x)
    assert x[0].uri == ns.MUC_USER

    invites = xpath.queryForNodes('/x/invite', x[0])
    assert (invites is not None and len(invites) == 1), repr(invites)
    assert invites[0]['to'] == 'alice@localhost'

    reasons = xpath.queryForNodes('/invite/reason', invites[0])
    assert (reasons is not None and len(reasons) == 1), repr(reasons)
    assert str(reasons[0]) == 'I want to test invitations'

    call_async(q, act_prop_iface, 'SetProperties',
        handles['chat'], {'color': '#f00baa,#f00baa', 'private': True})

    event = q.expect('stream-message', to='alice@localhost')
    message = event.stanza

    properties = xpath.queryForNodes('/message/properties', message)
    assert (properties is not None and len(properties) == 1), repr(properties)
    assert properties[0].uri == ns.OLPC_ACTIVITY_PROPS
    assert properties[0]['room'] == 'chat@conf.localhost'
    assert properties[0]['activity'] == 'foo_id'

    property = xpath.queryForNodes('/properties/property', properties[0])
    assert (property is not None and len(property) == 2), repr(property)
    seen = set()
    for p in property:
        seen.add(p['name'])
        if p['name'] == 'color':
            assert p['type'] == 'str'
            assert str(p) == '#f00baa,#f00baa'
        elif p['name'] == 'private':
            assert p['type'] == 'bool'
            assert str(p) == '1'
        else:
            assert False, 'Unexpected property %s' % p['name']
    assert 'color' in seen, seen
    assert 'private' in seen, seen

    event = q.expect('dbus-signal', signal='ActivityPropertiesChanged')
    chat_handle, props = event.args
    assert chat_handle == handles['chat']
    assert props == {'color': '#f00baa,#f00baa', 'private' : True}

    q.expect('dbus-return', method='SetProperties')

    conn.Disconnect()

    q.expect('dbus-signal', signal='StatusChanged', args=[2, 1])

if __name__ == '__main__':
    exec_test(test)
<|endoftext|>


<commit_before>#!/usr/bin/python
import os
import sys
import time
import json
import logging
import urllib2
from threading import Thread, Event
from optparse import OptionParser

from npsgd import model_manager
from npsgd.config import config
from npsgd.model_task import ModelTask
from npsgd.model_manager import modelManager
import npsgd.email_manager

class TaskKeepAliveThread(Thread):
    """Model keep alive thread.

    Periodically sends a keepalive (heartbeat) to the server while we are working
    on a model task so that it doesn't expire our task id.
    """

    def __init__(self, keepAliveRequest, taskId):
        Thread.__init__(self)
        self.done             = Event()
        self.keepAliveRequest = keepAliveRequest
        self.taskId           = taskId
        self.daemon           = True

    def run(self):
        fails = 0
        while True:
            self.done.wait(config.keepAliveInterval)
            if self.done.isSet():
                break

            try:
                logging.info("Making heartbeat request '%s'", self.keepAliveRequest)
                response = urllib2.urlopen("%s/%s" % (self.keepAliveRequest, self.taskId))
            except urllib2.URLError, e:
                logging.error("Heartbeat failed to make connection to %s", self.keepAliveRequest)
                fails += 1

class NPSGDWorker(object):
    """Worker class for executing models and sending out result emails.

    """
    def __init__(self, serverAddress, serverPort):
        self.baseRequest          = "http://%s:%s" % (serverAddress, serverPort)
        self.infoRequest          = "%s/worker_info"      % self.baseRequest
        self.taskRequest          = "%s/worker_work_task" % self.baseRequest
        self.failedTaskRequest    = "%s/worker_failed_task" % self.baseRequest
        self.hasTaskRequest       = "%s/worker_has_task" % self.baseRequest
        self.succeedTaskRequest    = "%s/worker_succeed_task" % self.baseRequest
        self.taskKeepAliveRequest = "%s/worker_keep_alive_task" % self.baseRequest
        self.requestTimeout  = 100
        self.supportedModels = ["test"]
        self.requestErrors   = 0
        self.maxErrors       = 3
        self.errorSleepTime    = 10
        self.requestSleepTime = 10

    def getServerInfo(self):
        try:
            response = urllib2.urlopen(self.infoRequest)
        except urllib2.URLError, e:
            logging.error("Failed to make initial connection to %s", self.baseRequest)
            return

        logging.info("Got initial response from server")

    def loop(self):
        logging.info("Entering event loop")
        while True:
            try:
                self.handleEvents()
            except Exception:
                logging.exception("Unhandled exception in event loop!")

    def handleEvents(self):
            try:
                logging.info("Polling %s for tasks" % self.taskRequest)
                response = urllib2.urlopen(self.taskRequest)
            except urllib2.URLError, e:
                self.requestErrors += 1
                logging.error("Error making worker request to server, attempt #%d", self.requestErrors + 1)
                time.sleep(self.errorSleepTime)
                return

            try:
                decodedResponse = json.load(response)
            except ValueError, e:
                self.requestErrors += 1
                logging.error("Error decoding server response, attempt #%d", self.requestErrors +1)
                time.sleep(self.errorSleepTime)
                return

            self.requestErrors = 0
            self.processResponse(decodedResponse)
            time.sleep(self.requestSleepTime)

    def processResponse(self, response):
        if "status" in response:
            if response["status"] == "empty_queue":
                logging.info("No tasks available on server")
        elif "task" in response:
            self.processTask(response["task"])

    def notifyFailedTask(self, taskId):
        try:
            logging.info("Notifying server of failed task with id %s", taskId)
            response = urllib2.urlopen("%s/%s" % (self.failedTaskRequest, taskId))
        except urllib2.URLError, e:
            logging.error("Failed to communicate failed task to server %s", self.baseRequest)

    def notifySucceedTask(self, taskId):
        try:
            logging.info("Notifying server of succeeded task with id %s", taskId)
            response = urllib2.urlopen("%s/%s" % (self.succeedTaskRequest, taskId))
        except urllib2.URLError, e:
            logging.error("Failed to communicate succeeded task to server %s", self.baseRequest)

    def serverHasTask(self, taskId):
        try:
            logging.info("Making has task request for %s", taskId)
            response = urllib2.urlopen("%s/%s" % (self.hasTaskRequest, taskId))
        except urllib2.URLError, e:
            logging.error("Failed to make has task request to server %s", self.baseRequest)
            raise RuntimeError(e)

        try:
            decodedResponse = json.load(response)
        except ValueError, e:
            logging.error("Bad response from server for has task")
            raise RuntimeError(e)

        if "response" in decodedResponse and decodedResponse["response"] in ["yes", "no"]:
            return decodedResponse["response"] == "yes"
        else:
            logging.error("Malformed response from server")
            raise RuntimeError("Malformed response from server for 'has task'")

    def processTask(self, taskDict):
        taskId = None
        if "taskId" in taskDict:
            taskId = taskDict["taskId"]

        try:
            try:
                model = modelManager.getModel(taskDict["modelName"], taskDict["modelVersion"])
                logging.info("Creating a model task for '%s'", taskDict["modelName"])
                taskObject = model.fromDict(taskDict)
            except KeyError, e:
                logging.warning("Was unable to deserialize model task")
                if taskId:
                    self.notifyFailedTask(taskId)
                return

            keepAliveThread = TaskKeepAliveThread(self.taskKeepAliveRequest, taskObject.taskId)
            keepAliveThread.start()
            try:
                resultsEmail = taskObject.run()
                logging.info("Model finished running, sending email")
                if self.serverHasTask(taskObject.taskId):
                    npsgd.email_manager.blockingEmailSend(resultsEmail)
                    logging.info("Email sent, model is 100% complete!")
                    self.notifySucceedTask(taskObject.taskId)
                else:
                    logging.warning("Skipping task completion since the server forgot about our task")

            except RuntimeError, e:
                logging.error("Some kind of error during processing model task, notifying server of failure")
                logging.exception(e)
                self.notifyFailedTask(taskObject.taskId)

            finally:
                keepAliveThread.done.set()

        except: #If all else fails, notify the server that we are going down
            if taskId:
                self.notifyFailedTask(taskId)
            raise

def main():
    parser = OptionParser()
    parser.add_option('-c', '--config', dest="config",
            help="Configuration file path", type="string", default="config.cfg")

    parser.add_option('-l', '--log-filename', dest='log',
                        help="Log filename (use '-' for stderr)", default="-")

    (options, args) = parser.parse_args()

    config.loadConfig(options.config)
    config.setupLogging(options.log)
    model_manager.setupModels()
    model_manager.startScannerThread()

    worker = NPSGDWorker(config.queueServerAddress, config.queueServerPort)
    logging.info("NPSGD Worker booted up, going into event loop")
    worker.getServerInfo()
    worker.loop()

if __name__ == "__main__":
    main()
<commit_msg>Add better error logging to worker<commit_after>#!/usr/bin/python
import os
import sys
import time
import json
import logging
import urllib2
from threading import Thread, Event
from optparse import OptionParser

from npsgd import model_manager
from npsgd.config import config
from npsgd.model_task import ModelTask
from npsgd.model_manager import modelManager
import npsgd.email_manager

class TaskKeepAliveThread(Thread):
    """Model keep alive thread.

    Periodically sends a keepalive (heartbeat) to the server while we are working
    on a model task so that it doesn't expire our task id.
    """

    def __init__(self, keepAliveRequest, taskId):
        Thread.__init__(self)
        self.done             = Event()
        self.keepAliveRequest = keepAliveRequest
        self.taskId           = taskId
        self.daemon           = True

    def run(self):
        fails = 0
        while True:
            self.done.wait(config.keepAliveInterval)
            if self.done.isSet():
                break

            try:
                logging.info("Making heartbeat request '%s'", self.keepAliveRequest)
                response = urllib2.urlopen("%s/%s" % (self.keepAliveRequest, self.taskId))
            except urllib2.URLError, e:
                logging.error("Heartbeat failed to make connection to %s", self.keepAliveRequest)
                fails += 1

class NPSGDWorker(object):
    """Worker class for executing models and sending out result emails.

    """
    def __init__(self, serverAddress, serverPort):
        self.baseRequest          = "http://%s:%s" % (serverAddress, serverPort)
        self.infoRequest          = "%s/worker_info"      % self.baseRequest
        self.taskRequest          = "%s/worker_work_task" % self.baseRequest
        self.failedTaskRequest    = "%s/worker_failed_task" % self.baseRequest
        self.hasTaskRequest       = "%s/worker_has_task" % self.baseRequest
        self.succeedTaskRequest    = "%s/worker_succeed_task" % self.baseRequest
        self.taskKeepAliveRequest = "%s/worker_keep_alive_task" % self.baseRequest
        self.requestTimeout  = 100
        self.supportedModels = ["test"]
        self.requestErrors   = 0
        self.maxErrors       = 3
        self.errorSleepTime    = 10
        self.requestSleepTime = 10

    def getServerInfo(self):
        try:
            response = urllib2.urlopen(self.infoRequest)
        except urllib2.URLError, e:
            logging.error("Failed to make initial connection to %s", self.baseRequest)
            return

        logging.info("Got initial response from server")

    def loop(self):
        logging.info("Entering event loop")
        while True:
            try:
                self.handleEvents()
            except Exception:
                logging.exception("Unhandled exception in event loop!")

    def handleEvents(self):
            try:
                logging.info("Polling %s for tasks" % self.taskRequest)
                response = urllib2.urlopen(self.taskRequest)
            except urllib2.URLError, e:
                self.requestErrors += 1
                logging.error("Error making worker request to server, attempt #%d", self.requestErrors + 1)
                time.sleep(self.errorSleepTime)
                return

            try:
                decodedResponse = json.load(response)
            except ValueError, e:
                self.requestErrors += 1
                logging.error("Error decoding server response, attempt #%d", self.requestErrors +1)
                time.sleep(self.errorSleepTime)
                return

            self.requestErrors = 0
            self.processResponse(decodedResponse)
            time.sleep(self.requestSleepTime)

    def processResponse(self, response):
        if "status" in response:
            if response["status"] == "empty_queue":
                logging.info("No tasks available on server")
        elif "task" in response:
            self.processTask(response["task"])

    def notifyFailedTask(self, taskId):
        try:
            logging.info("Notifying server of failed task with id %s", taskId)
            response = urllib2.urlopen("%s/%s" % (self.failedTaskRequest, taskId))
        except urllib2.URLError, e:
            logging.error("Failed to communicate failed task to server %s", self.baseRequest)

    def notifySucceedTask(self, taskId):
        try:
            logging.info("Notifying server of succeeded task with id %s", taskId)
            response = urllib2.urlopen("%s/%s" % (self.succeedTaskRequest, taskId))
        except urllib2.URLError, e:
            logging.error("Failed to communicate succeeded task to server %s", self.baseRequest)

    def serverHasTask(self, taskId):
        try:
            logging.info("Making has task request for %s", taskId)
            response = urllib2.urlopen("%s/%s" % (self.hasTaskRequest, taskId))
        except urllib2.URLError, e:
            logging.error("Failed to make has task request to server %s", self.baseRequest)
            raise RuntimeError(e)

        try:
            decodedResponse = json.load(response)
        except ValueError, e:
            logging.error("Bad response from server for has task")
            raise RuntimeError(e)

        if "response" in decodedResponse and decodedResponse["response"] in ["yes", "no"]:
            return decodedResponse["response"] == "yes"
        else:
            logging.error("Malformed response from server")
            raise RuntimeError("Malformed response from server for 'has task'")

    def processTask(self, taskDict):
        taskId = None
        if "taskId" in taskDict:
            taskId = taskDict["taskId"]

        try:
            try:
                model = modelManager.getModel(taskDict["modelName"], taskDict["modelVersion"])
                logging.info("Creating a model task for '%s'", taskDict["modelName"])
                taskObject = model.fromDict(taskDict)
            except KeyError, e:
                logging.warning("Was unable to deserialize model task (%s), model task: %s", e, taskDict)
                if taskId:
                    self.notifyFailedTask(taskId)
                return

            keepAliveThread = TaskKeepAliveThread(self.taskKeepAliveRequest, taskObject.taskId)
            keepAliveThread.start()
            try:
                resultsEmail = taskObject.run()
                logging.info("Model finished running, sending email")
                if self.serverHasTask(taskObject.taskId):
                    npsgd.email_manager.blockingEmailSend(resultsEmail)
                    logging.info("Email sent, model is 100% complete!")
                    self.notifySucceedTask(taskObject.taskId)
                else:
                    logging.warning("Skipping task completion since the server forgot about our task")

            except RuntimeError, e:
                logging.error("Some kind of error during processing model task, notifying server of failure")
                logging.exception(e)
                self.notifyFailedTask(taskObject.taskId)

            finally:
                keepAliveThread.done.set()

        except: #If all else fails, notify the server that we are going down
            if taskId:
                self.notifyFailedTask(taskId)
            raise

def main():
    parser = OptionParser()
    parser.add_option('-c', '--config', dest="config",
            help="Configuration file path", type="string", default="config.cfg")

    parser.add_option('-l', '--log-filename', dest='log',
                        help="Log filename (use '-' for stderr)", default="-")

    (options, args) = parser.parse_args()

    config.loadConfig(options.config)
    config.setupLogging(options.log)
    model_manager.setupModels()
    model_manager.startScannerThread()

    worker = NPSGDWorker(config.queueServerAddress, config.queueServerPort)
    logging.info("NPSGD Worker booted up, going into event loop")
    worker.getServerInfo()
    worker.loop()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>"""
    This file is copied/apdated from https://github.com/berkeleydeeprlcourse/homework/tree/master/hw3
"""
import sys
import pickle
import numpy as np
from collections import namedtuple
from itertools import count
import random
import gym.spaces

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.autograd as autograd

from utils.replay_buffer import ReplayBuffer
from utils.gym import get_wrapper_by_name

USE_CUDA = torch.cuda.is_available()
dtype = torch.cuda.FloatTensor if torch.cuda.is_available() else torch.FloatTensor

class Variable(autograd.Variable):
    def __init__(self, data, *args, **kwargs):
        if USE_CUDA:
            data = data.cuda()
        super(Variable, self).__init__(data, *args, **kwargs)

"""
    OptimizerSpec containing following attributes
        constructor: The optimizer constructor ex: RMSprop
        kwargs: {Dict} arguments for constructing optimizer
"""
OptimizerSpec = namedtuple("OptimizerSpec", ["constructor", "kwargs"])

Statistic = {
    "bellman_errors": [],
    "mean_episode_rewards": [],
    "best_mean_episode_rewards": []
}

def dqn_learing(
    env,
    q_func,
    optimizer_spec,
    exploration,
    stopping_criterion=None,
    replay_buffer_size=1000000,
    batch_size=32,
    gamma=0.99,
    learning_starts=50000,
    learning_freq=4,
    frame_history_len=4,
    target_update_freq=10000,
    grad_norm_clipping=10):

    """Run Deep Q-learning algorithm.

    You can specify your own convnet using q_func.

    All schedules are w.r.t. total number of steps taken in the environment.

    Parameters
    ----------
    env: gym.Env
        gym environment to train on.
    q_func: function
        Model to use for computing the q function. It should accept the
        following named arguments:
            input_channel: int
                number of channel of input.
            num_actions: int
                number of actions
    optimizer_spec: OptimizerSpec
        Specifying the constructor and kwargs, as well as learning rate schedule
        for the optimizer
    exploration: Schedule (defined in utils.schedule)
        schedule for probability of chosing random action.
    stopping_criterion: (env) -> bool
        should return true when it's ok for the RL algorithm to stop.
        takes in env and the number of steps executed so far.
    replay_buffer_size: int
        How many memories to store in the replay buffer.
    batch_size: int
        How many transitions to sample each time experience is replayed.
    gamma: float
        Discount Factor
    learning_starts: int
        After how many environment steps to start replaying experiences
    learning_freq: int
        How many steps of environment to take between every experience replay
    frame_history_len: int
        How many past frames to include as input to the model.
    target_update_freq: int
        How many experience replay rounds (not steps!) to perform between
        each update to the target Q network
    """
    assert type(env.observation_space) == gym.spaces.Box
    assert type(env.action_space)      == gym.spaces.Discrete

    ###############
    # BUILD MODEL #
    ###############

    if len(env.observation_space.shape) == 1:
        # This means we are running on low-dimensional observations (e.g. RAM)
        input_arg = env.observation_space.shape[0]
    else:
        img_h, img_w, img_c = env.observation_space.shape
        input_arg = frame_history_len * img_c
    num_actions = env.action_space.n

    # Construct an epilson greedy policy with given exploration schedule
    def select_epilson_greedy_action(model, obs, t):
        sample = random.random()
        eps_threshold = exploration.value(t)
        if sample > eps_threshold:
            obs = torch.from_numpy(obs).type(dtype).unsqueeze(0) / 255.0
            # Use volatile = True if variable is only used in inference mode, i.e. don’t save the history
            return model(Variable(obs, volatile=True)).data.max(1)[1].cpu()
        else:
            return torch.IntTensor([[random.randrange(num_actions)]])

    # Initialize target q function and q function
    Q = q_func(input_arg, num_actions).type(dtype)
    target_Q = q_func(input_arg, num_actions).type(dtype)

    # Construct Q network optimizer function
    optimizer = optimizer_spec.constructor(Q.parameters(), **optimizer_spec.kwargs)

    # Construct the replay buffer
    replay_buffer = ReplayBuffer(replay_buffer_size, frame_history_len)

    ###############
    # RUN ENV     #
    ###############
    num_param_updates = 0
    bellman_error = -float('nan')
    mean_episode_reward = -float('nan')
    best_mean_episode_reward = -float('inf')
    last_obs = env.reset()
    LOG_EVERY_N_STEPS = 10000

    for t in count():
        ### Check stopping criterion
        if stopping_criterion is not None and stopping_criterion(env):
            break

        ### Step the env and store the transition
        # Store lastest observation in replay memory and last_idx can be used to store action, reward, done
        last_idx = replay_buffer.store_frame(last_obs)
        # encode_recent_observation will take the latest observation
        # that you pushed into the buffer and compute the corresponding
        # input that should be given to a Q network by appending some
        # previous frames.
        recent_observations = replay_buffer.encode_recent_observation()

        # Choose random action if not yet start learning
        if t > learning_starts:
            action = select_epilson_greedy_action(Q, recent_observations, t)[0, 0]
        else:
            action = random.randrange(num_actions)
        # Advance one step
        obs, reward, done, _ = env.step(action)
        # clip rewards between -1 and 1
        reward = max(-1.0, min(reward, 1.0))
        # Store other info in replay memory
        replay_buffer.store_effect(last_idx, action, reward, done)
        # Resets the environment when reaching an episode boundary.
        if done:
            obs = env.reset()
        last_obs = obs

        ### Perform experience replay and train the network.
        # Note that this is only done if the replay buffer contains enough samples
        # for us to learn something useful -- until then, the model will not be
        # initialized and random actions should be taken
        if (t > learning_starts and
                t % learning_freq == 0 and
                replay_buffer.can_sample(batch_size)):
            # Use the replay buffer to sample a batch of transitions
            # Note: done_mask[i] is 1 if the next state corresponds to the end of an episode,
            # in which case there is no Q-value at the next state; at the end of an
            # episode, only the current state reward contributes to the target
            obs_batch, act_batch, rew_batch, next_obs_batch, done_mask = replay_buffer.sample(batch_size)
            # Convert numpy nd_array to torch variables for calculation
            obs_batch = Variable(torch.from_numpy(obs_batch).type(dtype)) / 255.0
            act_batch = Variable(torch.from_numpy(act_batch).long())
            rew_batch = Variable(torch.from_numpy(rew_batch))
            next_obs_batch = Variable(torch.from_numpy(next_obs_batch).type(dtype)) / 255.0
            not_done_mask = Variable(torch.from_numpy(1 - done_mask)).type(dtype)

            if USE_CUDA:
                act_batch = act_batch.cuda()
                rew_batch = rew_batch.cuda()

            # Compute current Q value, q_func takes only state and output value for every state-action pair
            # We choose Q based on action taken.
            current_Q_values = Q(obs_batch).gather(1, act_batch.unsqueeze(1))
            # Compute next Q value based on which action gives max Q values
            next_max_q = target_Q(next_obs_batch).detach().max(1)[0]
            next_Q_values = not_done_mask * next_max_q
            # Detach variable from the current graph since we don't want gradients for next Q to propagated
            # Compute Bellman error, use huber loss to mitigate outlier impact
            target_Q_values = rew_batch + (gamma * next_Q_values)
            bellman_error = (target_Q_values - current_Q_values)
            # clip the bellman error between [-1 , 1]
            clipped_bellman_error = bellman_error.clamp(-1, 1)
            # Note: clipped_bellman_delta * -1 will be right gradient
            d_error = clipped_bellman_error * -1.0
            # Clear previous gradients before backward pass
            optimizer.zero_grad()
            # run backward pass
            current_Q_values.backward(d_error.data.unsqueeze(1))

            # Perfom the update
            optimizer.step()
            num_param_updates += 1

            # Periodically update the target network by Q network to target Q network
            if num_param_updates % target_update_freq == 0:
                target_Q.load_state_dict(Q.state_dict())

        ### 4. Log progress and keep track of statistics
        episode_rewards = get_wrapper_by_name(env, "Monitor").get_episode_rewards()
        if len(episode_rewards) > 0:
            mean_episode_reward = np.mean(episode_rewards[-100:])
        if len(episode_rewards) > 100:
            best_mean_episode_reward = max(best_mean_episode_reward, mean_episode_reward)

        Statistic["bellman_errors"].append(bellman_error)
        Statistic["mean_episode_rewards"].append(mean_episode_reward)
        Statistic["best_mean_episode_rewards"].append(best_mean_episode_reward)

        if t % LOG_EVERY_N_STEPS == 0 and t > learning_starts:
            print("Timestep %d" % (t,))
            print("mean reward (100 episodes) %f" % mean_episode_reward)
            print("best mean reward %f" % best_mean_episode_reward)
            print("episodes %d" % len(episode_rewards))
            print("exploration %f" % exploration.value(t))
            sys.stdout.flush()

            # Dump statistics to pickle
            with open('statistics.pkl', 'wb') as f:
                pickle.dump(Statistic, f)
                print("Saved to %s" % 'statistics.pkl')
<commit_msg>We can't save bellman error because it will cause out of memory<commit_after>"""
    This file is copied/apdated from https://github.com/berkeleydeeprlcourse/homework/tree/master/hw3
"""
import sys
import pickle
import numpy as np
from collections import namedtuple
from itertools import count
import random
import gym.spaces

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.autograd as autograd

from utils.replay_buffer import ReplayBuffer
from utils.gym import get_wrapper_by_name

USE_CUDA = torch.cuda.is_available()
dtype = torch.cuda.FloatTensor if torch.cuda.is_available() else torch.FloatTensor

class Variable(autograd.Variable):
    def __init__(self, data, *args, **kwargs):
        if USE_CUDA:
            data = data.cuda()
        super(Variable, self).__init__(data, *args, **kwargs)

"""
    OptimizerSpec containing following attributes
        constructor: The optimizer constructor ex: RMSprop
        kwargs: {Dict} arguments for constructing optimizer
"""
OptimizerSpec = namedtuple("OptimizerSpec", ["constructor", "kwargs"])

Statistic = {
    "mean_episode_rewards": [],
    "best_mean_episode_rewards": []
}

def dqn_learing(
    env,
    q_func,
    optimizer_spec,
    exploration,
    stopping_criterion=None,
    replay_buffer_size=1000000,
    batch_size=32,
    gamma=0.99,
    learning_starts=50000,
    learning_freq=4,
    frame_history_len=4,
    target_update_freq=10000,
    grad_norm_clipping=10):

    """Run Deep Q-learning algorithm.

    You can specify your own convnet using q_func.

    All schedules are w.r.t. total number of steps taken in the environment.

    Parameters
    ----------
    env: gym.Env
        gym environment to train on.
    q_func: function
        Model to use for computing the q function. It should accept the
        following named arguments:
            input_channel: int
                number of channel of input.
            num_actions: int
                number of actions
    optimizer_spec: OptimizerSpec
        Specifying the constructor and kwargs, as well as learning rate schedule
        for the optimizer
    exploration: Schedule (defined in utils.schedule)
        schedule for probability of chosing random action.
    stopping_criterion: (env) -> bool
        should return true when it's ok for the RL algorithm to stop.
        takes in env and the number of steps executed so far.
    replay_buffer_size: int
        How many memories to store in the replay buffer.
    batch_size: int
        How many transitions to sample each time experience is replayed.
    gamma: float
        Discount Factor
    learning_starts: int
        After how many environment steps to start replaying experiences
    learning_freq: int
        How many steps of environment to take between every experience replay
    frame_history_len: int
        How many past frames to include as input to the model.
    target_update_freq: int
        How many experience replay rounds (not steps!) to perform between
        each update to the target Q network
    """
    assert type(env.observation_space) == gym.spaces.Box
    assert type(env.action_space)      == gym.spaces.Discrete

    ###############
    # BUILD MODEL #
    ###############

    if len(env.observation_space.shape) == 1:
        # This means we are running on low-dimensional observations (e.g. RAM)
        input_arg = env.observation_space.shape[0]
    else:
        img_h, img_w, img_c = env.observation_space.shape
        input_arg = frame_history_len * img_c
    num_actions = env.action_space.n

    # Construct an epilson greedy policy with given exploration schedule
    def select_epilson_greedy_action(model, obs, t):
        sample = random.random()
        eps_threshold = exploration.value(t)
        if sample > eps_threshold:
            obs = torch.from_numpy(obs).type(dtype).unsqueeze(0) / 255.0
            # Use volatile = True if variable is only used in inference mode, i.e. don’t save the history
            return model(Variable(obs, volatile=True)).data.max(1)[1].cpu()
        else:
            return torch.IntTensor([[random.randrange(num_actions)]])

    # Initialize target q function and q function
    Q = q_func(input_arg, num_actions).type(dtype)
    target_Q = q_func(input_arg, num_actions).type(dtype)

    # Construct Q network optimizer function
    optimizer = optimizer_spec.constructor(Q.parameters(), **optimizer_spec.kwargs)

    # Construct the replay buffer
    replay_buffer = ReplayBuffer(replay_buffer_size, frame_history_len)

    ###############
    # RUN ENV     #
    ###############
    num_param_updates = 0
    mean_episode_reward = -float('nan')
    best_mean_episode_reward = -float('inf')
    last_obs = env.reset()
    LOG_EVERY_N_STEPS = 10000

    for t in count():
        ### Check stopping criterion
        if stopping_criterion is not None and stopping_criterion(env):
            break

        ### Step the env and store the transition
        # Store lastest observation in replay memory and last_idx can be used to store action, reward, done
        last_idx = replay_buffer.store_frame(last_obs)
        # encode_recent_observation will take the latest observation
        # that you pushed into the buffer and compute the corresponding
        # input that should be given to a Q network by appending some
        # previous frames.
        recent_observations = replay_buffer.encode_recent_observation()

        # Choose random action if not yet start learning
        if t > learning_starts:
            action = select_epilson_greedy_action(Q, recent_observations, t)[0, 0]
        else:
            action = random.randrange(num_actions)
        # Advance one step
        obs, reward, done, _ = env.step(action)
        # clip rewards between -1 and 1
        reward = max(-1.0, min(reward, 1.0))
        # Store other info in replay memory
        replay_buffer.store_effect(last_idx, action, reward, done)
        # Resets the environment when reaching an episode boundary.
        if done:
            obs = env.reset()
        last_obs = obs

        ### Perform experience replay and train the network.
        # Note that this is only done if the replay buffer contains enough samples
        # for us to learn something useful -- until then, the model will not be
        # initialized and random actions should be taken
        if (t > learning_starts and
                t % learning_freq == 0 and
                replay_buffer.can_sample(batch_size)):
            # Use the replay buffer to sample a batch of transitions
            # Note: done_mask[i] is 1 if the next state corresponds to the end of an episode,
            # in which case there is no Q-value at the next state; at the end of an
            # episode, only the current state reward contributes to the target
            obs_batch, act_batch, rew_batch, next_obs_batch, done_mask = replay_buffer.sample(batch_size)
            # Convert numpy nd_array to torch variables for calculation
            obs_batch = Variable(torch.from_numpy(obs_batch).type(dtype) / 255.0)
            act_batch = Variable(torch.from_numpy(act_batch).long())
            rew_batch = Variable(torch.from_numpy(rew_batch))
            next_obs_batch = Variable(torch.from_numpy(next_obs_batch).type(dtype) / 255.0)
            not_done_mask = Variable(torch.from_numpy(1 - done_mask)).type(dtype)

            if USE_CUDA:
                act_batch = act_batch.cuda()
                rew_batch = rew_batch.cuda()

            # Compute current Q value, q_func takes only state and output value for every state-action pair
            # We choose Q based on action taken.
            current_Q_values = Q(obs_batch).gather(1, act_batch.unsqueeze(1))
            # Compute next Q value based on which action gives max Q values
            next_max_q = target_Q(next_obs_batch).detach().max(1)[0]
            next_Q_values = not_done_mask * next_max_q
            # Detach variable from the current graph since we don't want gradients for next Q to propagated
            # Compute Bellman error, use huber loss to mitigate outlier impact
            target_Q_values = rew_batch + (gamma * next_Q_values)
            bellman_error = target_Q_values - current_Q_values
            # clip the bellman error between [-1 , 1]
            clipped_bellman_error = bellman_error.clamp(-1, 1)
            # Note: clipped_bellman_delta * -1 will be right gradient
            d_error = clipped_bellman_error * -1.0
            # Clear previous gradients before backward pass
            optimizer.zero_grad()
            # run backward pass
            current_Q_values.backward(d_error.data.unsqueeze(1))

            # Perfom the update
            optimizer.step()
            num_param_updates += 1

            # Periodically update the target network by Q network to target Q network
            if num_param_updates % target_update_freq == 0:
                target_Q.load_state_dict(Q.state_dict())

        ### 4. Log progress and keep track of statistics
        episode_rewards = get_wrapper_by_name(env, "Monitor").get_episode_rewards()
        if len(episode_rewards) > 0:
            mean_episode_reward = np.mean(episode_rewards[-100:])
        if len(episode_rewards) > 100:
            best_mean_episode_reward = max(best_mean_episode_reward, mean_episode_reward)

        Statistic["mean_episode_rewards"].append(mean_episode_reward)
        Statistic["best_mean_episode_rewards"].append(best_mean_episode_reward)

        if t % LOG_EVERY_N_STEPS == 0 and t > learning_starts:
            print("Timestep %d" % (t,))
            print("mean reward (100 episodes) %f" % mean_episode_reward)
            print("best mean reward %f" % best_mean_episode_reward)
            print("episodes %d" % len(episode_rewards))
            print("exploration %f" % exploration.value(t))
            sys.stdout.flush()

            # Dump statistics to pickle
            with open('statistics.pkl', 'wb') as f:
                pickle.dump(Statistic, f)
                print("Saved to %s" % 'statistics.pkl')
<|endoftext|>


<commit_before>from ddt import ddt, data
import mock
from rest_framework import test, status

from waldur_core.structure.tests import factories as structure_factories
from waldur_core.structure.tests import fixtures as structure_fixtures
from waldur_core.structure import models as structure_models
from waldur_core.users import models as user_models

from .. import models
from . import factories

class ExpertBidBaseTest(test.APITransactionTestCase):

    def setUp(self):
        self.expert_fixture = structure_fixtures.ProjectFixture()
        self.expert_manager = self.expert_fixture.owner
        self.expert_provider = factories.ExpertProviderFactory(customer=self.expert_fixture.customer)

        self.project_fixture = structure_fixtures.ProjectFixture()
        self.project = self.project_fixture.project
        self.expert_request = factories.ExpertRequestFactory(project=self.project)
        self.expert = self.project_fixture.admin

class ExpertBidListTest(ExpertBidBaseTest):
    def setUp(self):
        super(ExpertBidListTest, self).setUp()
        self.expert_bid = factories.ExpertBidFactory(
            request=self.expert_request,
            team=self.expert_fixture.project,
        )

    def test_expert_manager_can_see_owned_bid(self):
        self.client.force_authenticate(self.expert_manager)
        response = self.client.get(factories.ExpertBidFactory.get_list_url())
        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(len(response.data), 1)

    def test_other_expert_manager_can_not_see_expert_bid(self):
        fixture = structure_fixtures.ProjectFixture()
        factories.ExpertProviderFactory(customer=fixture.customer)
        self.client.force_authenticate(fixture.owner)
        response = self.client.get(factories.ExpertBidFactory.get_list_url())
        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(len(response.data), 0)

    def test_team_members_list_is_rendered_for_bid(self):
        team_member = self.expert_fixture.manager
        self.client.force_authenticate(self.expert_fixture.staff)

        response = self.client.get(factories.ExpertBidFactory.get_url(self.expert_bid))
        self.assertEqual(len(response.data['team_members']), 1)

        actual_member = response.data['team_members'][0]
        self.assertEqual(actual_member['username'], team_member.username)

class ExpertBidCreateTest(ExpertBidBaseTest):
    action_url = factories.ExpertBidFactory.get_list_url()

    def test_expert_manager_can_create_expert_bid(self):
        self.client.force_authenticate(self.expert_manager)

        response = self.client.post(self.action_url, self._get_payload())

        self.assertEqual(response.status_code, status.HTTP_201_CREATED)

    def test_project_manager_can_not_create_expert_bid(self):
        self.client.force_authenticate(self.project_fixture.manager)

        response = self.client.post(self.action_url, self._get_payload())

        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)

    def test_bid_can_be_created_for_pending_request_only(self):
        self.expert_request.state = models.ExpertRequest.States.ACTIVE
        self.expert_request.save()
        self.client.force_authenticate(self.expert_manager)

        response = self.client.post(self.action_url, self._get_payload())

        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertIn('request', response.data)

    def test_team_should_have_at_least_one_member(self):
        self.project.remove_user(self.expert)
        self.client.force_authenticate(self.expert_manager)

        response = self.client.post(self.action_url, self._get_payload())

        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)

    def test_organization_cannot_submit_bid_second_time_for_the_same_request(self):
        self.client.force_authenticate(self.expert_manager)

        response = self.client.post(self.action_url, self._get_payload())
        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
        second_project = structure_factories.ProjectFactory(customer=self.project.customer)
        second_project.add_user(structure_factories.UserFactory(), structure_models.CustomerRole.OWNER)
        response = self.client.post(self.action_url, self._get_payload(second_project))

        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)

    def test_organization_can_submit_second_bid_for_second_request(self):
        self.client.force_authenticate(self.expert_manager)

        response = self.client.post(self.action_url, self._get_payload())
        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
        second_project = structure_factories.ProjectFactory(customer=self.project.customer)
        second_project.add_user(structure_factories.UserFactory(), structure_models.CustomerRole.OWNER)
        second_request = factories.ExpertRequestFactory(project=self.project)
        response = self.client.post(self.action_url, self._get_payload(second_project, second_request))

        self.assertEqual(response.status_code, status.HTTP_201_CREATED)

    def test_when_expert_bid_is_created_event_is_emitted(self):
        with mock.patch('logging.LoggerAdapter.info') as mocked_info:
            expert_bid = factories.ExpertBidFactory()
            template = 'User {user_username} with full name {user_full_name} has created ' \
                       'bid for request {request_name} under {customer_name} / {project_name}.'
            context = {
                'user_username': expert_bid.user.username,
                'user_full_name': expert_bid.user.full_name,
                'request_name': expert_bid.request.name,
                'customer_name': expert_bid.request.project.customer.name,
                'project_name': expert_bid.request.project.name,
            }
            expected_message = template.format(**context)
            actual_message = mocked_info.call_args_list[-1][0][0]
            self.assertEqual(expected_message, actual_message)

    def _get_payload(self, project=None, request=None):
        if not project:
            project = self.project

        return {
            'request': factories.ExpertRequestFactory.get_url(request or self.expert_request),
            'team': structure_factories.ProjectFactory.get_url(project),
            'price': 100.00,
        }

class ExpertBidAcceptTest(ExpertBidBaseTest):

    def setUp(self):
        super(ExpertBidAcceptTest, self).setUp()
        self.team = self.expert_fixture.project
        self.expert_bid = factories.ExpertBidFactory(request=self.expert_request, team=self.team)

    def test_staff_can_accept_bid(self):
        self.client.force_authenticate(self.project_fixture.staff)
        response = self.accept_bid()
        self.assertEqual(response.status_code, status.HTTP_200_OK)

    def test_customer_owner_can_accept_bid(self):
        self.client.force_authenticate(self.project_fixture.owner)
        response = self.accept_bid()
        self.assertEqual(response.status_code, status.HTTP_200_OK)

    def test_project_manager_can_not_accept_bid(self):
        self.client.force_authenticate(self.project_fixture.manager)
        response = self.accept_bid()
        self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND, response.data)

    def test_customer_owner_can_not_accept_bid_if_request_is_not_pending(self):
        self.expert_request.state = models.ExpertRequest.States.ACTIVE
        self.expert_request.save()
        self.client.force_authenticate(self.project_fixture.owner)
        response = self.accept_bid()
        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)

    def test_when_bid_accepted_request_becomes_active(self):
        self.client.force_authenticate(self.project_fixture.owner)
        response = self.accept_bid()
        self.expert_request.refresh_from_db()
        self.assertEqual(self.expert_request.state, models.ExpertRequest.States.ACTIVE)

    def test_when_bid_accepted_event_is_emitted(self):
        self.client.force_authenticate(self.project_fixture.owner)
        with mock.patch('logging.LoggerAdapter.info') as mocked_info:
            response = self.accept_bid()
            template = 'Expert request {expert_request_name} has been activated.'
            context = {
                'expert_request_name': self.expert_bid.request.name,
            }
            expected_message = template.format(**context)
            actual_message = mocked_info.call_args_list[-1][0][0]
            self.assertEqual(expected_message, actual_message)

    def test_when_bid_accepted_invitations_for_expert_team_members_are_created(self):
        # Arrange
        expert_users = structure_factories.UserFactory.create_batch(3)
        for user in expert_users:
            self.team.add_user(user, structure_models.ProjectRole.ADMINISTRATOR)

        # Act
        self.client.force_authenticate(self.project_fixture.owner)
        self.accept_bid()

        # Assert
        invitations = self.get_invitations(expert_users)
        self.assertEqual(len(invitations), len(expert_users))

    @mock.patch('waldur_core.users.tasks.send_invitation')
    def test_when_bid_accepted_invitations_for_expert_team_members_are_created(self, mocked_task):
        # Arrange
        expert_users = structure_factories.UserFactory.create_batch(3)
        for user in expert_users:
            self.team.add_user(user, structure_models.ProjectRole.ADMINISTRATOR)

        # Act
        self.client.force_authenticate(self.project_fixture.owner)
        self.accept_bid()

        # Assert
        invitations = self.get_invitations(expert_users)
        self.assertEqual(len(invitations), len(expert_users))

        calls = [
            mock.call(invitation.uuid.hex, self.project_fixture.owner.full_name)
            for invitation in invitations
        ]
        mocked_task.delay.assert_has_calls(calls)

    @mock.patch('waldur_mastermind.experts.tasks.send_new_contract')
    def test_when_bid_accepted_notification_emails_for_customer_owners_are_sent(self, mocked_task):
        self.client.force_authenticate(self.project_fixture.owner)
        self.accept_bid()
        mocked_task.delay.assert_called_once_with(self.expert_request.uuid.hex)

    def test_when_bid_accepted_contract_is_created(self):
        self.client.force_authenticate(self.project_fixture.owner)
        self.accept_bid()
        self.expert_request.refresh_from_db()
        self.assertIsNotNone(self.expert_request.contract)

    def get_invitations(self, users):
        return user_models.Invitation.objects.filter(
            project=self.project,
            customer=self.project.customer,
            created_by=self.project_fixture.owner,
            email__in=[user.email for user in users],
            project_role=structure_models.ProjectRole.ADMINISTRATOR,
        )

    def accept_bid(self):
        url = factories.ExpertBidFactory.get_url(self.expert_bid, 'accept')
        return self.client.post(url)

@ddt
class ExpertBidDeleteTest(ExpertBidBaseTest):

    def setUp(self):
        super(ExpertBidDeleteTest, self).setUp()
        self.team = self.expert_fixture.project
        self.expert_bid = factories.ExpertBidFactory(request=self.expert_request, team=self.team)

    @data('staff', 'owner')
    def test_authorized_user_can_delete_bid_for_pending_request(self, username):
        response = self.delete_bid(username)
        self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT, response.data)

    @data('staff', 'owner')
    def test_authorized_user_can_not_delete_bid_for_active_request(self, username):
        self.expert_request.state = models.ExpertRequest.States.ACTIVE
        self.expert_request.save()

        response = self.delete_bid(username)
        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)

    @data('user', 'manager')
    def test_unauthorized_user_can_not_delete_bid(self, username):
        response = self.delete_bid(username)
        self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)

    def delete_bid(self, username):
        user = getattr(self.expert_fixture, username)
        self.client.force_authenticate(user)

        url = factories.ExpertBidFactory.get_url(self.expert_bid)
        return self.client.delete(url)
<commit_msg>Fix unit test [WAL-1332]<commit_after>from ddt import ddt, data
import mock
from rest_framework import test, status

from waldur_core.structure.tests import factories as structure_factories
from waldur_core.structure.tests import fixtures as structure_fixtures
from waldur_core.structure import models as structure_models
from waldur_core.users import models as user_models

from .. import models
from . import factories

class ExpertBidBaseTest(test.APITransactionTestCase):

    def setUp(self):
        self.expert_fixture = structure_fixtures.ProjectFixture()
        self.expert_manager = self.expert_fixture.owner
        self.expert_provider = factories.ExpertProviderFactory(customer=self.expert_fixture.customer)

        self.project_fixture = structure_fixtures.ProjectFixture()
        self.project = self.project_fixture.project
        self.expert_request = factories.ExpertRequestFactory(project=self.project)
        self.expert = self.project_fixture.admin

class ExpertBidListTest(ExpertBidBaseTest):
    def setUp(self):
        super(ExpertBidListTest, self).setUp()
        self.expert_bid = factories.ExpertBidFactory(
            request=self.expert_request,
            team=self.expert_fixture.project,
        )

    def test_expert_manager_can_see_owned_bid(self):
        self.client.force_authenticate(self.expert_manager)
        response = self.client.get(factories.ExpertBidFactory.get_list_url())
        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(len(response.data), 1)

    def test_other_expert_manager_can_not_see_expert_bid(self):
        fixture = structure_fixtures.ProjectFixture()
        factories.ExpertProviderFactory(customer=fixture.customer)
        self.client.force_authenticate(fixture.owner)
        response = self.client.get(factories.ExpertBidFactory.get_list_url())
        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertEqual(len(response.data), 0)

    def test_team_members_list_is_rendered_for_bid(self):
        team_member = self.expert_fixture.manager
        self.client.force_authenticate(self.expert_fixture.staff)

        response = self.client.get(factories.ExpertBidFactory.get_url(self.expert_bid))
        self.assertEqual(len(response.data['team_members']), 1)

        actual_member = response.data['team_members'][0]
        self.assertEqual(actual_member['username'], team_member.username)

class ExpertBidCreateTest(ExpertBidBaseTest):
    action_url = factories.ExpertBidFactory.get_list_url()

    def test_expert_manager_can_create_expert_bid(self):
        self.client.force_authenticate(self.expert_manager)

        response = self.client.post(self.action_url, self._get_payload())

        self.assertEqual(response.status_code, status.HTTP_201_CREATED)

    def test_project_manager_can_not_create_expert_bid(self):
        self.client.force_authenticate(self.project_fixture.manager)

        response = self.client.post(self.action_url, self._get_payload())

        self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)

    def test_bid_can_be_created_for_pending_request_only(self):
        self.expert_request.state = models.ExpertRequest.States.ACTIVE
        self.expert_request.save()
        self.client.force_authenticate(self.expert_manager)

        response = self.client.post(self.action_url, self._get_payload())

        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
        self.assertIn('request', response.data)

    def test_team_should_have_at_least_one_member(self):
        self.project.remove_user(self.expert)
        self.client.force_authenticate(self.expert_manager)

        response = self.client.post(self.action_url, self._get_payload())

        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)

    def test_organization_cannot_submit_bid_second_time_for_the_same_request(self):
        self.client.force_authenticate(self.expert_manager)

        response = self.client.post(self.action_url, self._get_payload())
        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
        second_project = structure_factories.ProjectFactory(customer=self.project.customer)
        second_project.add_user(structure_factories.UserFactory(), structure_models.CustomerRole.OWNER)
        response = self.client.post(self.action_url, self._get_payload(second_project))

        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)

    def test_organization_can_submit_second_bid_for_second_request(self):
        self.client.force_authenticate(self.expert_manager)

        response = self.client.post(self.action_url, self._get_payload())
        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
        second_project = structure_factories.ProjectFactory(customer=self.project.customer)
        second_project.add_user(structure_factories.UserFactory(), structure_models.CustomerRole.OWNER)
        second_request = factories.ExpertRequestFactory(project=self.project)
        response = self.client.post(self.action_url, self._get_payload(second_project, second_request))

        self.assertEqual(response.status_code, status.HTTP_201_CREATED)

    def test_when_expert_bid_is_created_event_is_emitted(self):
        with mock.patch('logging.LoggerAdapter.info') as mocked_info:
            expert_bid = factories.ExpertBidFactory()
            template = 'User {user_username} with full name {user_full_name} has created ' \
                       'bid for request {request_name} under {customer_name} / {project_name}.'
            context = {
                'user_username': expert_bid.user.username,
                'user_full_name': expert_bid.user.full_name,
                'request_name': expert_bid.request.name,
                'customer_name': expert_bid.request.project.customer.name,
                'project_name': expert_bid.request.project.name,
            }
            expected_message = template.format(**context)
            actual_message = mocked_info.call_args_list[-1][0][0]
            self.assertEqual(expected_message, actual_message)

    def _get_payload(self, project=None, request=None):
        if not project:
            project = self.project

        return {
            'request': factories.ExpertRequestFactory.get_url(request or self.expert_request),
            'team': structure_factories.ProjectFactory.get_url(project),
            'price': 100.00,
        }

class ExpertBidAcceptTest(ExpertBidBaseTest):

    def setUp(self):
        super(ExpertBidAcceptTest, self).setUp()
        self.team = self.expert_fixture.project
        self.expert_bid = factories.ExpertBidFactory(request=self.expert_request, team=self.team)

    def test_staff_can_accept_bid(self):
        self.client.force_authenticate(self.project_fixture.staff)
        response = self.accept_bid()
        self.assertEqual(response.status_code, status.HTTP_200_OK)

    def test_customer_owner_can_accept_bid(self):
        self.client.force_authenticate(self.project_fixture.owner)
        response = self.accept_bid()
        self.assertEqual(response.status_code, status.HTTP_200_OK)

    def test_project_manager_can_not_accept_bid(self):
        self.client.force_authenticate(self.project_fixture.manager)
        response = self.accept_bid()
        self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND, response.data)

    def test_customer_owner_can_not_accept_bid_if_request_is_not_pending(self):
        self.expert_request.state = models.ExpertRequest.States.ACTIVE
        self.expert_request.save()
        self.client.force_authenticate(self.project_fixture.owner)
        response = self.accept_bid()
        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)

    def test_when_bid_accepted_request_becomes_active(self):
        self.client.force_authenticate(self.project_fixture.owner)
        response = self.accept_bid()
        self.expert_request.refresh_from_db()
        self.assertEqual(self.expert_request.state, models.ExpertRequest.States.ACTIVE)

    def test_when_bid_accepted_event_is_emitted(self):
        self.client.force_authenticate(self.project_fixture.owner)
        with mock.patch('logging.LoggerAdapter.info') as mocked_info:
            response = self.accept_bid()
            template = 'Expert request {expert_request_name} has been activated.'
            context = {
                'expert_request_name': self.expert_bid.request.name,
            }
            expected_message = template.format(**context)
            actual_message = mocked_info.call_args_list[-1][0][0]
            self.assertEqual(expected_message, actual_message)

    def test_when_bid_accepted_invitations_for_expert_team_members_are_created(self):
        # Arrange
        expert_users = structure_factories.UserFactory.create_batch(3)
        for user in expert_users:
            self.team.add_user(user, structure_models.ProjectRole.ADMINISTRATOR)

        # Act
        self.client.force_authenticate(self.project_fixture.owner)
        self.accept_bid()

        # Assert
        invitations = self.get_invitations(expert_users)
        self.assertEqual(len(invitations), len(expert_users))

    @mock.patch('waldur_core.users.tasks.send_invitation')
    def test_when_bid_accepted_invitations_for_expert_team_members_are_created(self, mocked_task):
        # Arrange
        expert_users = structure_factories.UserFactory.create_batch(3)
        for user in expert_users:
            self.team.add_user(user, structure_models.ProjectRole.ADMINISTRATOR)

        # Act
        self.client.force_authenticate(self.project_fixture.owner)
        self.accept_bid()

        # Assert
        invitations = self.get_invitations(expert_users)
        self.assertEqual(len(invitations), len(expert_users))

        calls = [
            mock.call(invitation.uuid.hex, self.project_fixture.owner.full_name)
            for invitation in invitations
        ]
        mocked_task.delay.assert_has_calls(calls)

    @mock.patch('waldur_mastermind.experts.tasks.send_new_contract')
    def test_when_bid_accepted_notification_emails_for_customer_owners_are_sent(self, mocked_task):
        self.client.force_authenticate(self.project_fixture.owner)
        self.accept_bid()
        mocked_task.delay.assert_called_once_with(self.expert_request.uuid.hex)

    def test_when_bid_accepted_contract_is_created(self):
        self.client.force_authenticate(self.project_fixture.owner)
        self.accept_bid()
        self.assertTrue(models.ExpertContract.objects.filter(request=self.expert_request).exists())

    def get_invitations(self, users):
        return user_models.Invitation.objects.filter(
            project=self.project,
            customer=self.project.customer,
            created_by=self.project_fixture.owner,
            email__in=[user.email for user in users],
            project_role=structure_models.ProjectRole.ADMINISTRATOR,
        )

    def accept_bid(self):
        url = factories.ExpertBidFactory.get_url(self.expert_bid, 'accept')
        return self.client.post(url)

@ddt
class ExpertBidDeleteTest(ExpertBidBaseTest):

    def setUp(self):
        super(ExpertBidDeleteTest, self).setUp()
        self.team = self.expert_fixture.project
        self.expert_bid = factories.ExpertBidFactory(request=self.expert_request, team=self.team)

    @data('staff', 'owner')
    def test_authorized_user_can_delete_bid_for_pending_request(self, username):
        response = self.delete_bid(username)
        self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT, response.data)

    @data('staff', 'owner')
    def test_authorized_user_can_not_delete_bid_for_active_request(self, username):
        self.expert_request.state = models.ExpertRequest.States.ACTIVE
        self.expert_request.save()

        response = self.delete_bid(username)
        self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)

    @data('user', 'manager')
    def test_unauthorized_user_can_not_delete_bid(self, username):
        response = self.delete_bid(username)
        self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND)

    def delete_bid(self, username):
        user = getattr(self.expert_fixture, username)
        self.client.force_authenticate(user)

        url = factories.ExpertBidFactory.get_url(self.expert_bid)
        return self.client.delete(url)
<|endoftext|>