<commit_before>#!/usr/bin/env python
import json

from pprint import pprint
from time import time

from flask import Blueprint, Flask, Response, request
from flask import session, jsonify, url_for, current_app
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm.exc import UnmappedInstanceError
from sqlalchemy import or_

from db.database import db_session
from db.models import Nodes, Roles, Clusters, Tasks
from errors import (
    http_bad_request,
    http_conflict,
    http_not_found,
    http_not_implemented)

tasks = Blueprint('tasks', __name__)

@tasks.route('/', methods=['GET', 'POST'])
def list_tasks():
    if request.method == 'POST':
        # TODO(shep): sanity check action and payload
        fields = {'node_id': None,
                  'action': None,
                  'payload': 'json',
                  'state': 'pending',
                  'result': 'json',
                  'completed': None,
                  'expires': None}

        for k,v in fields.iteritems():
            if v == 'json':
                if k in request.json:
                    fields[k] = json.dumps(request.json[k])
                else:
                    fields[k] = None
            else:
                if k in request.json:
                    fields[k] = request.json[k]

        task = Tasks(node_id=fields['node_id'], action=fields['action'],
                     payload=fields['payload'], state=fields['state'],
                     result=fields['result'], submitted=int(time()),
                     completed=fields['completed'], expires=fields['expires'])
        db_session.add(task)
        msg = {'status': 500, 'message': 'Internal Error'}
        try:
            db_session.commit()
            # FIXME(shep): add a ref
            href = request.base_url + str(task.id)
            tmp = dict()
            for col in task.__table__.columns.keys():
                if col == 'payload' or col == 'result':
                    val = getattr(task, col)
                    tmp[col] = val if (val is None) else json.loads(val)
                else:
                    tmp[col] = getattr(task, col)
            msg = {'status': 201, 'message': 'Task Created',
                   'ref': href, 'task': tmp}
        except IntegrityError, e:
            db_session.rollback()
            return http_conflict(e)

        resp = jsonify(msg)
        resp.status_code = 201
        resp.headers['Location'] = href
    else:
        task_list = {"tasks": []}

        # FIXME(rp): need api selectable filters here
        for row in Tasks.query.filter(or_(Tasks.state == 'pending',
                                          Tasks.state == 'running')):
            tmp = dict()
            for col in row.__table__.columns.keys():
                if col == 'payload' or col == 'result':
                    val = getattr(row, col)
                    tmp[col] = val if (val is None) else json.loads(val)
                else:
                    tmp[col] = getattr(row, col)
            task_list['tasks'].append(tmp)
        resp = jsonify(task_list)
    return resp

@tasks.route('/<task_id>', methods=['GET', 'PUT'])
def task_by_id(task_id):
    if request.method == 'PUT':
        # NOTE: We probably can't rename hosts -- it affect chef...
        # Think on this.  Also, probably should do a get_node_status
        # to make sure it's happy in the config management
        r = Tasks.query.filter_by(id=node_id).first()
        if 'action' in request.json:
            r.action = request.json['action']
        if 'payload' in request.json:
            r.payload = jason.dumps(request.json['payload'])
        if 'state' in request.json:
            r.state = request.json['state']
        if 'result' in request.json:
            r.result = json.dumps(request.json['result'])
        #TODO(shep): this is an un-excepted db call
        db_session.commit()
        task = dict()
        for col in r.__table__.columns.keys():
            if col == 'payload' or col == 'result':
                val = getattr(r, col)
                task[col] = val if (val is None) else json.loads(val)
            else:
                task[col] = getattr(r, col)
        resp = jsonify(task)
    else:
        row = Tasks.query.filter_by(id=task_id).first()
        if row is None:
            return http_not_found()
        else:
            task = dict()
            for col in row.__table__.columns.keys():
                if col == 'payload' or col == 'result':
                    val = getattr(row, col)
                    task[col] = val if (val is None) else json.loads(val)
                else:
                    task[col] = getattr(row, col)

            resp = jsonify(task)
    return resp
<commit_msg>fix tasks PUT<commit_after>#!/usr/bin/env python
import json

from pprint import pprint
from time import time

from flask import Blueprint, Flask, Response, request
from flask import session, jsonify, url_for, current_app
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm.exc import UnmappedInstanceError
from sqlalchemy import or_

from db.database import db_session
from db.models import Nodes, Roles, Clusters, Tasks
from errors import (
    http_bad_request,
    http_conflict,
    http_not_found,
    http_not_implemented)

tasks = Blueprint('tasks', __name__)

@tasks.route('/', methods=['GET', 'POST'])
def list_tasks():
    if request.method == 'POST':
        # TODO(shep): sanity check action and payload
        fields = {'node_id': None,
                  'action': None,
                  'payload': 'json',
                  'state': 'pending',
                  'result': 'json',
                  'completed': None,
                  'expires': None}

        for k,v in fields.iteritems():
            if v == 'json':
                if k in request.json:
                    fields[k] = json.dumps(request.json[k])
                else:
                    fields[k] = None
            else:
                if k in request.json:
                    fields[k] = request.json[k]

        task = Tasks(node_id=fields['node_id'], action=fields['action'],
                     payload=fields['payload'], state=fields['state'],
                     result=fields['result'], submitted=int(time()),
                     completed=fields['completed'], expires=fields['expires'])
        db_session.add(task)
        msg = {'status': 500, 'message': 'Internal Error'}
        try:
            db_session.commit()
            # FIXME(shep): add a ref
            href = request.base_url + str(task.id)
            tmp = dict()
            for col in task.__table__.columns.keys():
                if col == 'payload' or col == 'result':
                    val = getattr(task, col)
                    tmp[col] = val if (val is None) else json.loads(val)
                else:
                    tmp[col] = getattr(task, col)
            msg = {'status': 201, 'message': 'Task Created',
                   'ref': href, 'task': tmp}
        except IntegrityError, e:
            db_session.rollback()
            return http_conflict(e)

        resp = jsonify(msg)
        resp.status_code = 201
        resp.headers['Location'] = href
    else:
        task_list = {"tasks": []}

        # FIXME(rp): need api selectable filters here
        for row in Tasks.query.filter(or_(Tasks.state == 'pending',
                                          Tasks.state == 'running')):
            tmp = dict()
            for col in row.__table__.columns.keys():
                if col == 'payload' or col == 'result':
                    val = getattr(row, col)
                    tmp[col] = val if (val is None) else json.loads(val)
                else:
                    tmp[col] = getattr(row, col)
            task_list['tasks'].append(tmp)
        resp = jsonify(task_list)
    return resp

@tasks.route('/<task_id>', methods=['GET', 'PUT'])
def task_by_id(task_id):
    if request.method == 'PUT':
        # NOTE: We probably can't rename hosts -- it affect chef...
        # Think on this.  Also, probably should do a get_node_status
        # to make sure it's happy in the config management
        r = Tasks.query.filter_by(id=task_id).first()
        if 'action' in request.json:
            r.action = request.json['action']
        if 'payload' in request.json:
            r.payload = jason.dumps(request.json['payload'])
        if 'state' in request.json:
            r.state = request.json['state']
        if 'result' in request.json:
            r.result = json.dumps(request.json['result'])
        #TODO(shep): this is an un-excepted db call
        db_session.commit()
        task = dict()
        for col in r.__table__.columns.keys():
            if col == 'payload' or col == 'result':
                val = getattr(r, col)
                task[col] = val if (val is None) else json.loads(val)
            else:
                task[col] = getattr(r, col)
        resp = jsonify(task)
    else:
        row = Tasks.query.filter_by(id=task_id).first()
        if row is None:
            return http_not_found()
        else:
            task = dict()
            for col in row.__table__.columns.keys():
                if col == 'payload' or col == 'result':
                    val = getattr(row, col)
                    task[col] = val if (val is None) else json.loads(val)
                else:
                    task[col] = getattr(row, col)

            resp = jsonify(task)
    return resp
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json

from mock import ANY, Mock, call

from twisted.internet import defer

from synapse.api.errors import AuthError
from synapse.types import UserID

from tests import unittest
from tests.unittest import override_config
from tests.utils import register_federation_servlets

# Some local users to test with
U_APPLE = UserID.from_string("@apple:test")
U_BANANA = UserID.from_string("@banana:test")

# Remote user
U_ONION = UserID.from_string("@onion:farm")

# Test room id
ROOM_ID = "a-room"

def _expect_edu_transaction(edu_type, content, origin="test"):
    return {
        "origin": origin,
        "origin_server_ts": 1000000,
        "pdus": [],
        "edus": [{"edu_type": edu_type, "content": content}],
    }

def _make_edu_transaction_json(edu_type, content):
    return json.dumps(_expect_edu_transaction(edu_type, content)).encode("utf8")

class TypingNotificationsTestCase(unittest.HomeserverTestCase):
    servlets = [register_federation_servlets]

    def make_homeserver(self, reactor, clock):
        # we mock out the keyring so as to skip the authentication check on the
        # federation API call.
        mock_keyring = Mock(spec=["verify_json_for_server"])
        mock_keyring.verify_json_for_server.return_value = defer.succeed(True)

        # we mock out the federation client too
        mock_federation_client = Mock(spec=["put_json"])
        mock_federation_client.put_json.return_value = defer.succeed((200, "OK"))

        datastores = Mock()
        datastores.main = Mock(
            spec=[
                # Bits that Federation needs
                "prep_send_transaction",
                "delivered_txn",
                "get_received_txn_response",
                "set_received_txn_response",
                "get_destination_retry_timings",
                "get_devices_by_remote",
                "maybe_store_room_on_invite",
                # Bits that user_directory needs
                "get_user_directory_stream_pos",
                "get_current_state_deltas",
                "get_device_updates_by_remote",
            ]
        )

        # the tests assume that we are starting at unix time 1000
        reactor.pump((1000,))

        hs = self.setup_test_homeserver(
            notifier=Mock(),
            http_client=mock_federation_client,
            keyring=mock_keyring,
            replication_streams={},
        )

        hs.datastores = datastores

        return hs

    def prepare(self, reactor, clock, hs):
        mock_notifier = hs.get_notifier()
        self.on_new_event = mock_notifier.on_new_event

        self.handler = hs.get_typing_handler()

        self.event_source = hs.get_event_sources().sources["typing"]

        self.datastore = hs.get_datastore()
        retry_timings_res = {
            "destination": "",
            "retry_last_ts": 0,
            "retry_interval": 0,
            "failure_ts": None,
        }
        self.datastore.get_destination_retry_timings.return_value = defer.succeed(
            retry_timings_res
        )

        self.datastore.get_device_updates_by_remote.return_value = defer.succeed(
            (0, [])
        )

        def get_received_txn_response(*args):
            return defer.succeed(None)

        self.datastore.get_received_txn_response = get_received_txn_response

        self.room_members = []

        def check_user_in_room(room_id, user_id):
            if user_id not in [u.to_string() for u in self.room_members]:
                raise AuthError(401, "User is not in the room")
            return defer.succeed(None)

        hs.get_auth().check_user_in_room = check_user_in_room

        def get_joined_hosts_for_room(room_id):
            return {member.domain for member in self.room_members}

        self.datastore.get_joined_hosts_for_room = get_joined_hosts_for_room

        def get_current_users_in_room(room_id):
            return defer.succeed({str(u) for u in self.room_members})

        hs.get_state_handler().get_current_users_in_room = get_current_users_in_room

        self.datastore.get_user_directory_stream_pos.return_value = (
            # we deliberately return a non-None stream pos to avoid doing an initial_spam
            defer.succeed(1)
        )

        self.datastore.get_current_state_deltas.return_value = (0, None)

        self.datastore.get_to_device_stream_token = lambda: 0
        self.datastore.get_new_device_msgs_for_remote = lambda *args, **kargs: defer.succeed(
            ([], 0)
        )
        self.datastore.delete_device_msgs_for_remote = lambda *args, **kargs: None
        self.datastore.set_received_txn_response = lambda *args, **kwargs: defer.succeed(
            None
        )

    def test_started_typing_local(self):
        self.room_members = [U_APPLE, U_BANANA]

        self.assertEquals(self.event_source.get_current_key(), 0)

        self.get_success(
            self.handler.started_typing(
                target_user=U_APPLE, auth_user=U_APPLE, room_id=ROOM_ID, timeout=20000
            )
        )

        self.on_new_event.assert_has_calls([call("typing_key", 1, rooms=[ROOM_ID])])

        self.assertEquals(self.event_source.get_current_key(), 1)
        events = self.get_success(
            self.event_source.get_new_events(room_ids=[ROOM_ID], from_key=0)
        )
        self.assertEquals(
            events[0],
            [
                {
                    "type": "m.typing",
                    "room_id": ROOM_ID,
                    "content": {"user_ids": [U_APPLE.to_string()]},
                }
            ],
        )

    @override_config({"send_federation": True})
    def test_started_typing_remote_send(self):
        self.room_members = [U_APPLE, U_ONION]

        self.get_success(
            self.handler.started_typing(
                target_user=U_APPLE, auth_user=U_APPLE, room_id=ROOM_ID, timeout=20000
            )
        )

        put_json = self.hs.get_http_client().put_json
        put_json.assert_called_once_with(
            "farm",
            path="/_matrix/federation/v1/send/1000000",
            data=_expect_edu_transaction(
                "m.typing",
                content={
                    "room_id": ROOM_ID,
                    "user_id": U_APPLE.to_string(),
                    "typing": True,
                },
            ),
            json_data_callback=ANY,
            long_retries=True,
            backoff_on_404=True,
            try_trailing_slash_on_400=True,
        )

    def test_started_typing_remote_recv(self):
        self.room_members = [U_APPLE, U_ONION]

        self.assertEquals(self.event_source.get_current_key(), 0)

        (request, channel) = self.make_request(
            "PUT",
            "/_matrix/federation/v1/send/1000000",
            _make_edu_transaction_json(
                "m.typing",
                content={
                    "room_id": ROOM_ID,
                    "user_id": U_ONION.to_string(),
                    "typing": True,
                },
            ),
            federation_auth_origin=b"farm",
        )
        self.render(request)
        self.assertEqual(channel.code, 200)

        self.on_new_event.assert_has_calls([call("typing_key", 1, rooms=[ROOM_ID])])

        self.assertEquals(self.event_source.get_current_key(), 1)
        events = self.get_success(
            self.event_source.get_new_events(room_ids=[ROOM_ID], from_key=0)
        )
        self.assertEquals(
            events[0],
            [
                {
                    "type": "m.typing",
                    "room_id": ROOM_ID,
                    "content": {"user_ids": [U_ONION.to_string()]},
                }
            ],
        )

    @override_config({"send_federation": True})
    def test_stopped_typing(self):
        self.room_members = [U_APPLE, U_BANANA, U_ONION]

        # Gut-wrenching
        from synapse.handlers.typing import RoomMember

        member = RoomMember(ROOM_ID, U_APPLE.to_string())
        self.handler._member_typing_until[member] = 1002000
        self.handler._room_typing[ROOM_ID] = {U_APPLE.to_string()}

        self.assertEquals(self.event_source.get_current_key(), 0)

        self.get_success(
            self.handler.stopped_typing(
                target_user=U_APPLE, auth_user=U_APPLE, room_id=ROOM_ID
            )
        )

        self.on_new_event.assert_has_calls([call("typing_key", 1, rooms=[ROOM_ID])])

        put_json = self.hs.get_http_client().put_json
        put_json.assert_called_once_with(
            "farm",
            path="/_matrix/federation/v1/send/1000000",
            data=_expect_edu_transaction(
                "m.typing",
                content={
                    "room_id": ROOM_ID,
                    "user_id": U_APPLE.to_string(),
                    "typing": False,
                },
            ),
            json_data_callback=ANY,
            long_retries=True,
            backoff_on_404=True,
            try_trailing_slash_on_400=True,
        )

        self.assertEquals(self.event_source.get_current_key(), 1)
        events = self.get_success(
            self.event_source.get_new_events(room_ids=[ROOM_ID], from_key=0)
        )
        self.assertEquals(
            events[0],
            [{"type": "m.typing", "room_id": ROOM_ID, "content": {"user_ids": []}}],
        )

    def test_typing_timeout(self):
        self.room_members = [U_APPLE, U_BANANA]

        self.assertEquals(self.event_source.get_current_key(), 0)

        self.get_success(
            self.handler.started_typing(
                target_user=U_APPLE, auth_user=U_APPLE, room_id=ROOM_ID, timeout=10000
            )
        )

        self.on_new_event.assert_has_calls([call("typing_key", 1, rooms=[ROOM_ID])])
        self.on_new_event.reset_mock()

        self.assertEquals(self.event_source.get_current_key(), 1)
        events = self.get_success(
            self.event_source.get_new_events(room_ids=[ROOM_ID], from_key=0)
        )
        self.assertEquals(
            events[0],
            [
                {
                    "type": "m.typing",
                    "room_id": ROOM_ID,
                    "content": {"user_ids": [U_APPLE.to_string()]},
                }
            ],
        )

        self.reactor.pump([16])

        self.on_new_event.assert_has_calls([call("typing_key", 2, rooms=[ROOM_ID])])

        self.assertEquals(self.event_source.get_current_key(), 2)
        events = self.get_success(
            self.event_source.get_new_events(room_ids=[ROOM_ID], from_key=1)
        )
        self.assertEquals(
            events[0],
            [{"type": "m.typing", "room_id": ROOM_ID, "content": {"user_ids": []}}],
        )

        # SYN-230 - see if we can still set after timeout

        self.get_success(
            self.handler.started_typing(
                target_user=U_APPLE, auth_user=U_APPLE, room_id=ROOM_ID, timeout=10000
            )
        )

        self.on_new_event.assert_has_calls([call("typing_key", 3, rooms=[ROOM_ID])])
        self.on_new_event.reset_mock()

        self.assertEquals(self.event_source.get_current_key(), 3)
        events = self.get_success(
            self.event_source.get_new_events(room_ids=[ROOM_ID], from_key=0)
        )
        self.assertEquals(
            events[0],
            [
                {
                    "type": "m.typing",
                    "room_id": ROOM_ID,
                    "content": {"user_ids": [U_APPLE.to_string()]},
                }
            ],
        )
<commit_msg>Fix tests<commit_after># -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import json

from mock import ANY, Mock, call

from twisted.internet import defer

from synapse.api.errors import AuthError
from synapse.types import UserID

from tests import unittest
from tests.unittest import override_config
from tests.utils import register_federation_servlets

# Some local users to test with
U_APPLE = UserID.from_string("@apple:test")
U_BANANA = UserID.from_string("@banana:test")

# Remote user
U_ONION = UserID.from_string("@onion:farm")

# Test room id
ROOM_ID = "a-room"

def _expect_edu_transaction(edu_type, content, origin="test"):
    return {
        "origin": origin,
        "origin_server_ts": 1000000,
        "pdus": [],
        "edus": [{"edu_type": edu_type, "content": content}],
    }

def _make_edu_transaction_json(edu_type, content):
    return json.dumps(_expect_edu_transaction(edu_type, content)).encode("utf8")

class TypingNotificationsTestCase(unittest.HomeserverTestCase):
    servlets = [register_federation_servlets]

    def make_homeserver(self, reactor, clock):
        # we mock out the keyring so as to skip the authentication check on the
        # federation API call.
        mock_keyring = Mock(spec=["verify_json_for_server"])
        mock_keyring.verify_json_for_server.return_value = defer.succeed(True)

        # we mock out the federation client too
        mock_federation_client = Mock(spec=["put_json"])
        mock_federation_client.put_json.return_value = defer.succeed((200, "OK"))

        datastores = Mock()
        datastores.main = Mock(
            spec=[
                # Bits that Federation needs
                "prep_send_transaction",
                "delivered_txn",
                "get_received_txn_response",
                "set_received_txn_response",
                "get_destination_retry_timings",
                "get_devices_by_remote",
                "maybe_store_room_on_invite",
                # Bits that user_directory needs
                "get_user_directory_stream_pos",
                "get_current_state_deltas",
                "get_device_updates_by_remote",
            ]
        )

        # the tests assume that we are starting at unix time 1000
        reactor.pump((1000,))

        hs = self.setup_test_homeserver(
            notifier=Mock(),
            http_client=mock_federation_client,
            keyring=mock_keyring,
            replication_streams={},
        )

        hs.datastores = datastores

        return hs

    def prepare(self, reactor, clock, hs):
        mock_notifier = hs.get_notifier()
        self.on_new_event = mock_notifier.on_new_event

        self.handler = hs.get_typing_handler()

        self.event_source = hs.get_event_sources().sources["typing"]

        self.datastore = hs.get_datastore()
        retry_timings_res = {
            "destination": "",
            "retry_last_ts": 0,
            "retry_interval": 0,
            "failure_ts": None,
        }
        self.datastore.get_destination_retry_timings.return_value = defer.succeed(
            retry_timings_res
        )

        self.datastore.get_device_updates_by_remote.return_value = defer.succeed(
            (0, [])
        )

        def get_received_txn_response(*args):
            return defer.succeed(None)

        self.datastore.get_received_txn_response = get_received_txn_response

        self.room_members = []

        def check_user_in_room(room_id, user_id):
            if user_id not in [u.to_string() for u in self.room_members]:
                raise AuthError(401, "User is not in the room")
            return defer.succeed(None)

        hs.get_auth().check_user_in_room = check_user_in_room

        def get_joined_hosts_for_room(room_id):
            return {member.domain for member in self.room_members}

        self.datastore.get_joined_hosts_for_room = get_joined_hosts_for_room

        def get_users_in_room(room_id):
            return defer.succeed({str(u) for u in self.room_members})

        self.datastore.get_users_in_room = get_users_in_room

        self.datastore.get_user_directory_stream_pos.return_value = (
            # we deliberately return a non-None stream pos to avoid doing an initial_spam
            defer.succeed(1)
        )

        self.datastore.get_current_state_deltas.return_value = (0, None)

        self.datastore.get_to_device_stream_token = lambda: 0
        self.datastore.get_new_device_msgs_for_remote = lambda *args, **kargs: defer.succeed(
            ([], 0)
        )
        self.datastore.delete_device_msgs_for_remote = lambda *args, **kargs: None
        self.datastore.set_received_txn_response = lambda *args, **kwargs: defer.succeed(
            None
        )

    def test_started_typing_local(self):
        self.room_members = [U_APPLE, U_BANANA]

        self.assertEquals(self.event_source.get_current_key(), 0)

        self.get_success(
            self.handler.started_typing(
                target_user=U_APPLE, auth_user=U_APPLE, room_id=ROOM_ID, timeout=20000
            )
        )

        self.on_new_event.assert_has_calls([call("typing_key", 1, rooms=[ROOM_ID])])

        self.assertEquals(self.event_source.get_current_key(), 1)
        events = self.get_success(
            self.event_source.get_new_events(room_ids=[ROOM_ID], from_key=0)
        )
        self.assertEquals(
            events[0],
            [
                {
                    "type": "m.typing",
                    "room_id": ROOM_ID,
                    "content": {"user_ids": [U_APPLE.to_string()]},
                }
            ],
        )

    @override_config({"send_federation": True})
    def test_started_typing_remote_send(self):
        self.room_members = [U_APPLE, U_ONION]

        self.get_success(
            self.handler.started_typing(
                target_user=U_APPLE, auth_user=U_APPLE, room_id=ROOM_ID, timeout=20000
            )
        )

        put_json = self.hs.get_http_client().put_json
        put_json.assert_called_once_with(
            "farm",
            path="/_matrix/federation/v1/send/1000000",
            data=_expect_edu_transaction(
                "m.typing",
                content={
                    "room_id": ROOM_ID,
                    "user_id": U_APPLE.to_string(),
                    "typing": True,
                },
            ),
            json_data_callback=ANY,
            long_retries=True,
            backoff_on_404=True,
            try_trailing_slash_on_400=True,
        )

    def test_started_typing_remote_recv(self):
        self.room_members = [U_APPLE, U_ONION]

        self.assertEquals(self.event_source.get_current_key(), 0)

        (request, channel) = self.make_request(
            "PUT",
            "/_matrix/federation/v1/send/1000000",
            _make_edu_transaction_json(
                "m.typing",
                content={
                    "room_id": ROOM_ID,
                    "user_id": U_ONION.to_string(),
                    "typing": True,
                },
            ),
            federation_auth_origin=b"farm",
        )
        self.render(request)
        self.assertEqual(channel.code, 200)

        self.on_new_event.assert_has_calls([call("typing_key", 1, rooms=[ROOM_ID])])

        self.assertEquals(self.event_source.get_current_key(), 1)
        events = self.get_success(
            self.event_source.get_new_events(room_ids=[ROOM_ID], from_key=0)
        )
        self.assertEquals(
            events[0],
            [
                {
                    "type": "m.typing",
                    "room_id": ROOM_ID,
                    "content": {"user_ids": [U_ONION.to_string()]},
                }
            ],
        )

    @override_config({"send_federation": True})
    def test_stopped_typing(self):
        self.room_members = [U_APPLE, U_BANANA, U_ONION]

        # Gut-wrenching
        from synapse.handlers.typing import RoomMember

        member = RoomMember(ROOM_ID, U_APPLE.to_string())
        self.handler._member_typing_until[member] = 1002000
        self.handler._room_typing[ROOM_ID] = {U_APPLE.to_string()}

        self.assertEquals(self.event_source.get_current_key(), 0)

        self.get_success(
            self.handler.stopped_typing(
                target_user=U_APPLE, auth_user=U_APPLE, room_id=ROOM_ID
            )
        )

        self.on_new_event.assert_has_calls([call("typing_key", 1, rooms=[ROOM_ID])])

        put_json = self.hs.get_http_client().put_json
        put_json.assert_called_once_with(
            "farm",
            path="/_matrix/federation/v1/send/1000000",
            data=_expect_edu_transaction(
                "m.typing",
                content={
                    "room_id": ROOM_ID,
                    "user_id": U_APPLE.to_string(),
                    "typing": False,
                },
            ),
            json_data_callback=ANY,
            long_retries=True,
            backoff_on_404=True,
            try_trailing_slash_on_400=True,
        )

        self.assertEquals(self.event_source.get_current_key(), 1)
        events = self.get_success(
            self.event_source.get_new_events(room_ids=[ROOM_ID], from_key=0)
        )
        self.assertEquals(
            events[0],
            [{"type": "m.typing", "room_id": ROOM_ID, "content": {"user_ids": []}}],
        )

    def test_typing_timeout(self):
        self.room_members = [U_APPLE, U_BANANA]

        self.assertEquals(self.event_source.get_current_key(), 0)

        self.get_success(
            self.handler.started_typing(
                target_user=U_APPLE, auth_user=U_APPLE, room_id=ROOM_ID, timeout=10000
            )
        )

        self.on_new_event.assert_has_calls([call("typing_key", 1, rooms=[ROOM_ID])])
        self.on_new_event.reset_mock()

        self.assertEquals(self.event_source.get_current_key(), 1)
        events = self.get_success(
            self.event_source.get_new_events(room_ids=[ROOM_ID], from_key=0)
        )
        self.assertEquals(
            events[0],
            [
                {
                    "type": "m.typing",
                    "room_id": ROOM_ID,
                    "content": {"user_ids": [U_APPLE.to_string()]},
                }
            ],
        )

        self.reactor.pump([16])

        self.on_new_event.assert_has_calls([call("typing_key", 2, rooms=[ROOM_ID])])

        self.assertEquals(self.event_source.get_current_key(), 2)
        events = self.get_success(
            self.event_source.get_new_events(room_ids=[ROOM_ID], from_key=1)
        )
        self.assertEquals(
            events[0],
            [{"type": "m.typing", "room_id": ROOM_ID, "content": {"user_ids": []}}],
        )

        # SYN-230 - see if we can still set after timeout

        self.get_success(
            self.handler.started_typing(
                target_user=U_APPLE, auth_user=U_APPLE, room_id=ROOM_ID, timeout=10000
            )
        )

        self.on_new_event.assert_has_calls([call("typing_key", 3, rooms=[ROOM_ID])])
        self.on_new_event.reset_mock()

        self.assertEquals(self.event_source.get_current_key(), 3)
        events = self.get_success(
            self.event_source.get_new_events(room_ids=[ROOM_ID], from_key=0)
        )
        self.assertEquals(
            events[0],
            [
                {
                    "type": "m.typing",
                    "room_id": ROOM_ID,
                    "content": {"user_ids": [U_APPLE.to_string()]},
                }
            ],
        )
<|endoftext|>


<commit_before>import socket
import struct
import threading
from . import packets

from terrabot.data.player import Player
from terrabot.data.world import World
from terrabot.util.worlddrawer import draw_world
from .events import Events, EventManager

class TerraBot(object):
    """A bot for a terraria server"""

    # Defaults to 7777, because that is the default port for the server
    def __init__(self, ip, port=7777, protocol=156, name="Terrabot"):
        super(TerraBot, self).__init__()

        self.HOST = ip
        self.PORT = port
        self.ADDR = (self.HOST, self.PORT)

        self.protocol = protocol
        self.running = False

        self.writeThread = threading.Thread(target=self.read_packets)
        self.writeThread.daemon = True

        self.readThread = threading.Thread(target=self.write_packets)
        self.readThread.daemon = True

        self.client = None
        self.writeQueue = []

        self.world = World()
        self.player = Player(name)
        self._evman = EventManager()

        self._evman.method_on_event(Events.PlayerID, self.received_player_id)
        self._evman.method_on_event(Events.Initialized, self.initialized)
        self._evman.method_on_event(Events.Login, self.logged_in)
        # self.event_manager.method_on_event(events.Events.)

    """Connects to the server and starts the main loop"""
    def start(self):
        if not self.writeThread.isAlive() and not self.readThread.isAlive():
            self.client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self.client.connect(self.ADDR)
            self.running = True
            self.writeThread.start()
            self.readThread.start()
            self.add_packet(packets.Packet1(self.protocol))

    def read_packets(self):
        while self.running:
            packet_length = self.client.recv(2)
            if len(packet_length) < 2:
                self.stop()
                continue
            packet_length = struct.unpack("<h", packet_length)[0] - 2

            data = self.client.recv(packet_length)
            packno = data[0]

            try:
                parser = "Packet" + format(packno, 'x').upper() + "Parser"
                packet_class = getattr(packets, parser)
                packet_class().parse(self.world, self.player, data, self._evman)
            except AttributeError as e:
                pass

            if packno == 2:
                self.stop()
                continue

    def item_owner_changed(self, id, data):
        if self.player.logged_in:
            self.add_packet(packets.Packet16(data[0], data[1]))

    def received_player_id(self, event_id, data):
        self.add_packet(packets.Packet4(self.player))
        self.add_packet(packets.Packet10(self.player))
        self.add_packet(packets.Packet2A(self.player))
        self.add_packet(packets.Packet32(self.player))
        for i in range(0, 83):
            self.add_packet(packets.Packet5(self.player, i))
        self.add_packet(packets.Packet6())

    def initialized(self, event, data):
        self.add_packet(packets.Packet8(self.player, self.world))

    def logged_in(self, event, data):
        self.add_packet(packets.PacketC(self.player, self.world))
        self.add_packet(packets.Packet19(self.player))
        self._evman.method_on_event(Events.ItemOwnerChanged, self.item_owner_changed)

    def write_packets(self):
        while self.running:
            if len(self.writeQueue) > 0:
                self.writeQueue[0].send(self.client)
                self.writeQueue.pop(0)

    def teleport(x, y):
        self.add_packet(packets.Packet41(self.id, x, y))

    def message(self, msg):
        if self.player.logged_in:
            self.add_packet(packets.Packet19(self.player, msg))

    """Returns the event manager of this bot
       A function is used, so I can change the name internally without
       affecting bots
       """
    def get_event_manager(self):
        return self._evman

    def add_packet(self, packet):
        self.writeQueue.append(packet)

    def stop(self):
        self.running = False
<commit_msg>Added missing argument and temp. position update<commit_after>import socket
import struct
import threading
from . import packets

from terrabot.data.player import Player
from terrabot.data.world import World
from terrabot.util.worlddrawer import draw_world
from .events import Events, EventManager

class TerraBot(object):
    """A bot for a terraria server"""

    # Defaults to 7777, because that is the default port for the server
    def __init__(self, ip, port=7777, protocol=156, name="Terrabot"):
        super(TerraBot, self).__init__()

        self.HOST = ip
        self.PORT = port
        self.ADDR = (self.HOST, self.PORT)

        self.protocol = protocol
        self.running = False

        self.writeThread = threading.Thread(target=self.read_packets)
        self.writeThread.daemon = True

        self.readThread = threading.Thread(target=self.write_packets)
        self.readThread.daemon = True

        self.client = None
        self.writeQueue = []

        self.world = World()
        self.player = Player(name)
        self._evman = EventManager()

        self._evman.method_on_event(Events.PlayerID, self.received_player_id)
        self._evman.method_on_event(Events.Initialized, self.initialized)
        self._evman.method_on_event(Events.Login, self.logged_in)
        # self.event_manager.method_on_event(events.Events.)

    """Connects to the server and starts the main loop"""
    def start(self):
        if not self.writeThread.isAlive() and not self.readThread.isAlive():
            self.client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self.client.connect(self.ADDR)
            self.running = True
            self.writeThread.start()
            self.readThread.start()
            self.add_packet(packets.Packet1(self.protocol))

    def read_packets(self):
        while self.running:
            packet_length = self.client.recv(2)
            if len(packet_length) < 2:
                self.stop()
                continue
            packet_length = struct.unpack("<h", packet_length)[0] - 2

            data = self.client.recv(packet_length)
            packno = data[0]

            try:
                parser = "Packet" + format(packno, 'x').upper() + "Parser"
                packet_class = getattr(packets, parser)
                packet_class().parse(self.world, self.player, data, self._evman)
            except AttributeError as e:
                print(e)

            if packno == 2:
                self.stop()
                continue

    def item_owner_changed(self, id, data):
        if self.player.logged_in:
            self.add_packet(packets.Packet16(data[0], data[1]))

    def received_player_id(self, event_id, data):
        self.add_packet(packets.Packet4(self.player))
        self.add_packet(packets.Packet10(self.player))
        self.add_packet(packets.Packet2A(self.player))
        self.add_packet(packets.Packet32(self.player))
        for i in range(0, 83):
            self.add_packet(packets.Packet5(self.player, i))
        self.add_packet(packets.Packet6())

    def initialized(self, event, data):
        self.add_packet(packets.Packet8(self.player, self.world))

    def logged_in(self, event, data):
        self.add_packet(packets.PacketC(self.player, self.world))
        self.add_packet(packets.Packet19(self.player))
        self._evman.method_on_event(Events.ItemOwnerChanged, self.item_owner_changed)

    def write_packets(self):
        while self.running:
            if len(self.writeQueue) > 0:
                self.writeQueue[0].send(self.client)
                self.writeQueue.pop(0)

    def teleport(self, x, y):
        self.add_packet(packets.Packet41(self.player.playerID, x, y))

        #Temporary fix until I parse player update data!
        self.player.x = x
        self.player.y = y

    def message(self, msg):
        if self.player.logged_in:
            self.add_packet(packets.Packet19(self.player, msg))

    """Returns the event manager of this bot
       A function is used, so I can change the name internally without
       affecting bots
       """
    def get_event_manager(self):
        return self._evman

    def add_packet(self, packet):
        self.writeQueue.append(packet)

    def stop(self):
        self.running = False
<|endoftext|>


<commit_before>import ctypes
import logging
import math
import random
import sys
import threading
import time
import traceback

from .api import Job, ConcurrencyError, RetriableError

__all__ = [
    'ResourceManager', 'BasicResourceManager',
    'InterruptJob', 'Worker', 'JobChannel'
]

class InterruptJob(Exception):
    pass

class _RequeueRequested(Exception):
    pass

class JobContext:
    def __init__(self, worker_id, job):
        self.worker_id = worker_id
        self.job = job
        self.outdated = False
        self.requeue_requested = False
        self.requeue_run_at = None
        self.requeue_on_error = False

    def update(self, job):
        self.job = job
        self.outdated = False

    @property
    def cancelled(self):
        return self.job.status == Job.CANCELLED

    @property
    def revoked(self):
        return self.job.worker_id != self.worker_id

    def requeue_job(self, run_at):
        self.requeue_requested = True
        self.requeue_run_at = run_at

class JobChannel:
    def __init__(self, ctx):
        self.__ctx = ctx

    @property
    def job(self):
        return self.__ctx.job

    @property
    def cancelled(self):
        return self.__ctx.cancelled

    @property
    def revoked(self):
        return self.__ctx.revoked

    @property
    def cancelled_or_revoked(self):
        return self.cancelled or self.revoked

    def interrupt_if_requested(self):
        if self.cancelled or self.revoked:
            raise InterruptJob

    def requeue_job(self, run_at=None):
        self.__ctx.requeue_job(run_at)
        raise _RequeueRequested

    def requeue_job_on_error(self):
        self.__ctx.requeue_on_error = True

class WorkerThread(threading.Thread):
    def __init__(self, *args, **kwargs):
        super(WorkerThread, self).__init__(*args, **kwargs)
        self.interrupt_requested = False
        self.interrupted = False
        self.exc_info = None

    @property
    def has_failed(self):
        return self.exc_info is not None

    def run(self):
        try:
            super(WorkerThread, self).run()
        except InterruptJob:
            self.interrupted = True
        except _RequeueRequested:
            pass
        except Exception:
            self.exc_info = sys.exc_info()

def _substract_resources(r1, r2):
    result = r1.copy()
    for k in r2:
        v = r1[k] - r2[k]
        if v > 0:
            result[k] = v
        else:
            result.pop(k)
    return result

def _add_resources(r1, r2):
    result = r1.copy()
    for k in r2:
        result[k] = r1.get(k, 0) + r2[k]
    return result

class ResourceManager:
    def get_current_resources(self):
        pass

    def acquire(self):
        pass

    def reclaim(self, resources):
        pass

class BasicResourceManager(ResourceManager):
    def __init__(self, resources):
        self._resources = resources

    def get_current_resources(self):
        return self._resources.copy()

    def acquire(self):
        result = self._resources.copy()
        self._resources = {}
        return result

    def reclaim(self, resources):
        self._resources = _add_resources(self._resources, resources)

class Worker:
    def __init__(self, id_, resources, worker_func, controller, *, interrupt_via_exception=False):
        self._id = id_
        self._resources = resources
        self._worker_func = worker_func
        self._controller = controller
        self._interrupt_via_exception = interrupt_via_exception

        # current job and worker thread
        self._job_ctx = None
        self._worker_thread = None

        self._stop = threading.Event()
        self._force_stop = threading.Event()

        # TODO: handle exception in _main_loop
        self._main_loop_thread = threading.Thread(target=self._loop)

        self.logger = logging.getLogger(__name__)

    @property
    def id(self):
        return self._id

    @property
    def is_running(self):
        return self._main_loop_thread.is_alive()

    @property
    def is_busy(self):
        return self._job_ctx is not None

    def start(self):
        self.logger.info('[%s] Available resources %r', self._id, self._resources.get_current_resources())
        self.logger.info('[%s] Starting worker...', self._id)
        self._main_loop_thread.start()

    def request_stop(self):
        self.logger.info('[%s] Got request to stop...', self._id)
        self._stop.set()

    def join(self, timeout=None):
        if timeout is None:
            # join without timeout will block signal handling
            while self._main_loop_thread.is_alive():
                self._main_loop_thread.join(0.1)
        else:
            self._main_loop_thread.join(timeout)

    def stop(self):
        self.request_stop()
        self.join()

    def _reclaim_resources(self, resources):
        self._resources.reclaim(resources)
        self.logger.debug('[%s] Reclaimed resources: %r', self._id, resources)

    def _reset_current_job(self):
        assert self._job_ctx is not None
        assert self._worker_thread is not None
        assert not self._worker_thread.is_alive()

        self._reclaim_resources(self._job_ctx.job.reqs)
        self._job_ctx = None
        self._worker_thread = None

    def _should_requeue_current_job(self):
        assert self._job_ctx is not None
        assert self._worker_thread is not None
        assert not self._worker_thread.is_alive()

        return self._job_ctx.requeue_requested or \
            self._worker_thread.has_failed and self._job_ctx.requeue_on_error

    def _loop(self):
        #
        retry_delay = 0
        #
        while True:
            if self._stop.is_set() and self._job_ctx is None:
                break

            if self._force_stop.is_set():
                break

            if retry_delay:
                # aka exponential backoff for retriable errors
                time.sleep(retry_delay)

            try:
                if self._job_ctx is None:
                    self._try_acquire_job()

                elif self._job_ctx.outdated:
                    self._try_update_current_job()

                elif self._job_ctx.cancelled or self._job_ctx.revoked:
                    self._wait_for_worker_thread_and_cleanup()

                elif self._worker_thread.is_alive():
                    self._try_heartbeat_current_job()

                elif self._should_requeue_current_job():
                    self._try_requeue_current_job()

                else:  # worker has finished processing the job
                    self._try_finalize_current_job()

            except RetriableError:
                retry_delay = 1 if retry_delay == 0 else min(10, retry_delay * 2)
            else:
                retry_delay = 0

    def _try_acquire_job(self):
        # lock resources from manager
        resources = self._resources.acquire()
        self.logger.debug('[%s] Acquired resources: %r', self._id, resources)
        try:
            job = self._controller.acquire_job(resources, self._id)
        except ConcurrencyError:
            job = None

        if job:
            self._job_ctx = JobContext(self.id, job)
            self.logger.info('[%s] Acquired job %s', self._id, self._job_ctx.job.id)
            self._worker_thread = WorkerThread(target=self._worker_func, args=(JobChannel(self._job_ctx),))
            self._worker_thread.daemon = True  # to make force stop possible
            self._worker_thread.start()
            # reclaim leftovers
            self._reclaim_resources(_substract_resources(resources, job.reqs))
        else:
            self._reclaim_resources(resources)
            time.sleep(math.e - random.random())

    def _try_update_current_job(self):
        job = self._controller.get_job(self._job_ctx.job.id)

        self._job_ctx.update(job)

        if self._job_ctx.cancelled:
            self.logger.info('[%s] It seems job %s was cancelled', self._id, self._job_ctx.job.id)

        if self._job_ctx.revoked:
            self.logger.info('[%s] It seems job %s was taken from us', self._id, self._job_ctx.job.id)

    def _wait_for_worker_thread_and_cleanup(self):
        if self._worker_thread.is_alive():
            if self._interrupt_via_exception and not self._worker_thread.interrupt_requested:
                ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(self._worker_thread.ident),
                                                           ctypes.py_object(InterruptJob))
                self._worker_thread.interrupt_requested = True
                self.logger.info('[%s] Worker thread with job %s was interrupted via exception',
                                 self._id, self._job_ctx.job.id)

            self._worker_thread.join(math.pi - random.random())
        else:
            self.logger.info('[%s] Processing of job %s was terminated', self._id, self._job_ctx.job.id)
            self._reset_current_job()

    def _try_heartbeat_current_job(self):
        try:
            job = self._controller.heartbeat_job(self._job_ctx.job.id, self._job_ctx.job.version)
        except ConcurrencyError:
            job = None

        if job:
            self._job_ctx.update(job)
            self._worker_thread.join(math.pi - random.random())
        else:
            self._job_ctx.outdated = True
            self.logger.info('[%s] Failed to heartbeat job %s due to version mismatch',
                             self._id, self._job_ctx.job.id)

    def _try_requeue_current_job(self):
        assert not self._worker_thread.is_alive()
        # requeue current job with new run_at time
        try:
            self._controller.requeue_job(self._job_ctx.job.id, self._job_ctx.job.version,
                                         run_at=self._job_ctx.requeue_run_at)
        except ConcurrencyError:
            self._job_ctx.outdated = True
            self.logger.info('[%s] Failed to mark job %s as completed due to version mismatch',
                             self._id, self._job_ctx.job.id)
        else:
            self.logger.info('[%s] Job %s has been requeued', self._id, self._job_ctx.job.id)
            self._reset_current_job()

    def _try_finalize_current_job(self):
        assert not self._worker_thread.is_alive()
        # worker thread has finished, we should mark job as COMPLETED
        if self._worker_thread.has_failed:
            reason = str(self._worker_thread.exc_info[1])
            tback = ''.join(traceback.format_exception(*self._worker_thread.exc_info))
            worker_exception = {'reason': reason, 'traceback': tback}
        else:
            worker_exception = None

        try:
            job = self._controller.finalize_job(self._job_ctx.job.id, self._job_ctx.job.version,
                                                worker_exception=worker_exception)
        except ConcurrencyError:
            job = None

        if job:
            self.logger.info('[%s] Job %s has been processed', self._id, self._job_ctx.job.id)
            self._reset_current_job()
        else:
            self._job_ctx.outdated = True
            self.logger.info('[%s] Failed to mark job %s as completed due to version mismatch',
                             self._id, self._job_ctx.job.id)
<commit_msg>Make BasicResourceManager threadsafe<commit_after>import ctypes
import logging
import math
import random
import sys
import threading
import time
import traceback

from .api import Job, ConcurrencyError, RetriableError

__all__ = [
    'ResourceManager', 'BasicResourceManager',
    'InterruptJob', 'Worker', 'JobChannel'
]

class InterruptJob(Exception):
    pass

class _RequeueRequested(Exception):
    pass

class JobContext:
    def __init__(self, worker_id, job):
        self.worker_id = worker_id
        self.job = job
        self.outdated = False
        self.requeue_requested = False
        self.requeue_run_at = None
        self.requeue_on_error = False

    def update(self, job):
        self.job = job
        self.outdated = False

    @property
    def cancelled(self):
        return self.job.status == Job.CANCELLED

    @property
    def revoked(self):
        return self.job.worker_id != self.worker_id

    def requeue_job(self, run_at):
        self.requeue_requested = True
        self.requeue_run_at = run_at

class JobChannel:
    def __init__(self, ctx):
        self.__ctx = ctx

    @property
    def job(self):
        return self.__ctx.job

    @property
    def cancelled(self):
        return self.__ctx.cancelled

    @property
    def revoked(self):
        return self.__ctx.revoked

    @property
    def cancelled_or_revoked(self):
        return self.cancelled or self.revoked

    def interrupt_if_requested(self):
        if self.cancelled or self.revoked:
            raise InterruptJob

    def requeue_job(self, run_at=None):
        self.__ctx.requeue_job(run_at)
        raise _RequeueRequested

    def requeue_job_on_error(self):
        self.__ctx.requeue_on_error = True

class WorkerThread(threading.Thread):
    def __init__(self, *args, **kwargs):
        super(WorkerThread, self).__init__(*args, **kwargs)
        self.interrupt_requested = False
        self.interrupted = False
        self.exc_info = None

    @property
    def has_failed(self):
        return self.exc_info is not None

    def run(self):
        try:
            super(WorkerThread, self).run()
        except InterruptJob:
            self.interrupted = True
        except _RequeueRequested:
            pass
        except Exception:
            self.exc_info = sys.exc_info()

def substract_resources(r1, r2):
    result = r1.copy()
    for k in r2:
        v = r1[k] - r2[k]
        if v > 0:
            result[k] = v
        else:
            result.pop(k)
    return result

class ResourceManager:
    def get_current_resources(self):
        pass

    def acquire(self):
        pass

    def reclaim(self, resources):
        pass

class BasicResourceManager(ResourceManager):
    def __init__(self, resources):
        self._resources = resources
        self._lock = threading.Lock()

    def get_current_resources(self):
        with self._lock:
            return self._resources.copy()

    def acquire(self):
        with self._lock:
            result = self._resources.copy()
            self._resources = {}
        return result

    def reclaim(self, resources):
        with self._lock:
            for k in resources:
                self._resources[k] = self._resources.get(k, 0) + resources[k]

class Worker:
    def __init__(self, id_, resources, worker_func, controller, *, interrupt_via_exception=False):
        self._id = id_
        self._resources = resources
        self._worker_func = worker_func
        self._controller = controller
        self._interrupt_via_exception = interrupt_via_exception

        # current job and worker thread
        self._job_ctx = None
        self._worker_thread = None

        self._stop = threading.Event()
        self._force_stop = threading.Event()

        # TODO: handle exception in _main_loop
        self._main_loop_thread = threading.Thread(target=self._loop)

        self.logger = logging.getLogger(__name__)

    @property
    def id(self):
        return self._id

    @property
    def is_running(self):
        return self._main_loop_thread.is_alive()

    @property
    def is_busy(self):
        return self._job_ctx is not None

    def start(self):
        self.logger.info('[%s] Available resources %r', self._id, self._resources.get_current_resources())
        self.logger.info('[%s] Starting worker...', self._id)
        self._main_loop_thread.start()

    def request_stop(self):
        self.logger.info('[%s] Got request to stop...', self._id)
        self._stop.set()

    def join(self, timeout=None):
        if timeout is None:
            # join without timeout will block signal handling
            while self._main_loop_thread.is_alive():
                self._main_loop_thread.join(0.1)
        else:
            self._main_loop_thread.join(timeout)

    def stop(self):
        self.request_stop()
        self.join()

    def _reclaim_resources(self, resources):
        self._resources.reclaim(resources)
        self.logger.debug('[%s] Reclaimed resources: %r', self._id, resources)

    def _reset_current_job(self):
        assert self._job_ctx is not None
        assert self._worker_thread is not None
        assert not self._worker_thread.is_alive()

        self._reclaim_resources(self._job_ctx.job.reqs)
        self._job_ctx = None
        self._worker_thread = None

    def _should_requeue_current_job(self):
        assert self._job_ctx is not None
        assert self._worker_thread is not None
        assert not self._worker_thread.is_alive()

        return self._job_ctx.requeue_requested or \
            self._worker_thread.has_failed and self._job_ctx.requeue_on_error

    def _loop(self):
        #
        retry_delay = 0
        #
        while True:
            if self._stop.is_set() and self._job_ctx is None:
                break

            if self._force_stop.is_set():
                break

            if retry_delay:
                # aka exponential backoff for retriable errors
                time.sleep(retry_delay)

            try:
                if self._job_ctx is None:
                    self._try_acquire_job()

                elif self._job_ctx.outdated:
                    self._try_update_current_job()

                elif self._job_ctx.cancelled or self._job_ctx.revoked:
                    self._wait_for_worker_thread_and_cleanup()

                elif self._worker_thread.is_alive():
                    self._try_heartbeat_current_job()

                elif self._should_requeue_current_job():
                    self._try_requeue_current_job()

                else:  # worker has finished processing the job
                    self._try_finalize_current_job()

            except RetriableError:
                retry_delay = 1 if retry_delay == 0 else min(10, retry_delay * 2)
            else:
                retry_delay = 0

    def _try_acquire_job(self):
        # lock resources from manager
        resources = self._resources.acquire()
        self.logger.debug('[%s] Acquired resources: %r', self._id, resources)
        try:
            job = self._controller.acquire_job(resources, self._id)
        except ConcurrencyError:
            job = None

        if job:
            self._job_ctx = JobContext(self.id, job)
            self.logger.info('[%s] Acquired job %s', self._id, self._job_ctx.job.id)
            self._worker_thread = WorkerThread(target=self._worker_func, args=(JobChannel(self._job_ctx),))
            self._worker_thread.daemon = True  # to make force stop possible
            self._worker_thread.start()
            # reclaim leftovers
            self._reclaim_resources(substract_resources(resources, job.reqs))
        else:
            self._reclaim_resources(resources)
            time.sleep(math.e - random.random())

    def _try_update_current_job(self):
        job = self._controller.get_job(self._job_ctx.job.id)

        self._job_ctx.update(job)

        if self._job_ctx.cancelled:
            self.logger.info('[%s] It seems job %s was cancelled', self._id, self._job_ctx.job.id)

        if self._job_ctx.revoked:
            self.logger.info('[%s] It seems job %s was taken from us', self._id, self._job_ctx.job.id)

    def _wait_for_worker_thread_and_cleanup(self):
        if self._worker_thread.is_alive():
            if self._interrupt_via_exception and not self._worker_thread.interrupt_requested:
                ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(self._worker_thread.ident),
                                                           ctypes.py_object(InterruptJob))
                self._worker_thread.interrupt_requested = True
                self.logger.info('[%s] Worker thread with job %s was interrupted via exception',
                                 self._id, self._job_ctx.job.id)

            self._worker_thread.join(math.pi - random.random())
        else:
            self.logger.info('[%s] Processing of job %s was terminated', self._id, self._job_ctx.job.id)
            self._reset_current_job()

    def _try_heartbeat_current_job(self):
        try:
            job = self._controller.heartbeat_job(self._job_ctx.job.id, self._job_ctx.job.version)
        except ConcurrencyError:
            job = None

        if job:
            self._job_ctx.update(job)
            self._worker_thread.join(math.pi - random.random())
        else:
            self._job_ctx.outdated = True
            self.logger.info('[%s] Failed to heartbeat job %s due to version mismatch',
                             self._id, self._job_ctx.job.id)

    def _try_requeue_current_job(self):
        assert not self._worker_thread.is_alive()
        # requeue current job with new run_at time
        try:
            self._controller.requeue_job(self._job_ctx.job.id, self._job_ctx.job.version,
                                         run_at=self._job_ctx.requeue_run_at)
        except ConcurrencyError:
            self._job_ctx.outdated = True
            self.logger.info('[%s] Failed to mark job %s as completed due to version mismatch',
                             self._id, self._job_ctx.job.id)
        else:
            self.logger.info('[%s] Job %s has been requeued', self._id, self._job_ctx.job.id)
            self._reset_current_job()

    def _try_finalize_current_job(self):
        assert not self._worker_thread.is_alive()
        # worker thread has finished, we should mark job as COMPLETED
        if self._worker_thread.has_failed:
            reason = str(self._worker_thread.exc_info[1])
            tback = ''.join(traceback.format_exception(*self._worker_thread.exc_info))
            worker_exception = {'reason': reason, 'traceback': tback}
        else:
            worker_exception = None

        try:
            job = self._controller.finalize_job(self._job_ctx.job.id, self._job_ctx.job.version,
                                                worker_exception=worker_exception)
        except ConcurrencyError:
            job = None

        if job:
            self.logger.info('[%s] Job %s has been processed', self._id, self._job_ctx.job.id)
            self._reset_current_job()
        else:
            self._job_ctx.outdated = True
            self.logger.info('[%s] Failed to mark job %s as completed due to version mismatch',
                             self._id, self._job_ctx.job.id)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright 2011 Marc-Antoine Ruel. All Rights Reserved. Use of this
# source code is governed by a BSD-style license that can be found in the
# LICENSE file.

"""Updates the config files without destroying anything."""

import difflib
import optparse
import os
import shutil
import subprocess
import sys
import tempfile

BIN_PUB_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, BIN_PUB_DIR)

import maruel

def diff(diff_cmd, dst, basename, content):
    """Diffs the destination file with the content it should be."""
    old_size = 0
    if os.path.isfile(dst):
        old_size = os.stat(dst).st_size
    print(
        'Diffing %-16s  new: %5d bytes  current: %5d bytes' % (
            basename, len(content), old_size))
    handle, name = tempfile.mkstemp()
    try:
        try:
            os.write(handle, content)
        finally:
            os.close(handle)
        result = subprocess.call(
            diff_cmd + [name, dst], shell=sys.platform=='win32')
        if result:
            return result
        # Look if the source content changed.
        actual_content = maruel.read(name)
        if actual_content != content:
            # The user modified the source. For now just quit early.
            print('You modified the source')
            for i in difflib.unified_diff(
                    content.splitlines(True),
                    actual_content.splitlines(True),
                    fromfile=basename,
                    tofile=basename):
                sys.stdout.write(i)
            return 1
    finally:
        os.remove(name)
    return 0

def update_config(files, diff_cmd):
    """Processes files in files and copy them in the $HOME directory."""
    home_dir = os.environ.get('HOME') or os.environ['USERPROFILE']
    to_diff = []
    ok_files = []
    for basename in sorted(files):
        dst = os.path.join(home_dir, basename)
        dst_dir = os.path.dirname(dst)
        if not os.path.isdir(dst_dir):
            os.makedirs(dst_dir)
        content = files[basename]
        if not os.path.isfile(dst):
            if os.path.isdir(dst):
                print('Skipping file which is directory at destination %s' %
                        basename)
                continue
            print('Copying %s' % basename)
            maruel.write(dst, content)
            ok_files.append(basename)
        else:
            if maruel.read(dst) != content:
                to_diff.append(basename)
            else:
                ok_files.append(basename)

    for basename in to_diff:
        result = diff(
            diff_cmd, os.path.join(home_dir, basename), basename, files[basename])
        if result:
            return result
    print('The following files are ok:')
    for f in ok_files:
      print('  ' + f)

def load_files(config_dir, files):
    """Loads all the files content into dictionnary |files|.

    Appends the content if an entry is already present.
    """
    blacklist = [r'.*README.md$', r'.*\.swp$']
    for basename in maruel.walk(config_dir, [r'.*'], blacklist):
        src = os.path.join(config_dir, basename)
        if os.path.isfile(src):
            if basename not in files:
                files[basename] = maruel.read(src)
            else:
                files[basename] += maruel.read(src)

def main():
    parser = optparse.OptionParser()
    parser.add_option('-u', action='store_true', help='Unified diff')
    options, args = parser.parse_args()
    if args:
      parser.error('Unsupported args %s' % args)

    files = {}
    load_files(os.path.join(BIN_PUB_DIR, 'configs'), files)

    # Look for ~/bin/bin_pub pattern.
    if os.path.basename(os.path.dirname(BIN_PUB_DIR)) == 'bin':
      load_files(os.path.join(BIN_PUB_DIR, '..', 'configs'), files)

    cmd = ['vim', '-d']
    if options.u:
        cmd = ['diff', '-u']
    return update_config(files, cmd)

if __name__ == '__main__':
    sys.exit(main())

# vim: ts=4:sw=4:tw=80:et:
<commit_msg>update_config.py: fix for python3<commit_after>#!/usr/bin/env python3
# Copyright 2011 Marc-Antoine Ruel. All Rights Reserved. Use of this
# source code is governed by a BSD-style license that can be found in the
# LICENSE file.

"""Updates the config files without destroying anything."""

import difflib
import optparse
import os
import shutil
import subprocess
import sys
import tempfile

BIN_PUB_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, BIN_PUB_DIR)

import maruel

def diff(diff_cmd, dst, basename, content):
    """Diffs the destination file with the content it should be."""
    old_size = 0
    if os.path.isfile(dst):
        old_size = os.stat(dst).st_size
    print(
        'Diffing %-16s  new: %5d bytes  current: %5d bytes' % (
            basename, len(content), old_size))
    handle, name = tempfile.mkstemp()
    try:
        try:
            os.write(handle, content.encode('utf-8'))
        finally:
            os.close(handle)
        result = subprocess.call(
            diff_cmd + [name, dst], shell=sys.platform=='win32')
        if result:
            return result
        # Look if the source content changed.
        actual_content = maruel.read(name)
        if actual_content != content:
            # The user modified the source. For now just quit early.
            print('You modified the source')
            for i in difflib.unified_diff(
                    content.splitlines(True),
                    actual_content.splitlines(True),
                    fromfile=basename,
                    tofile=basename):
                sys.stdout.write(i)
            return 1
    finally:
        os.remove(name)
    return 0

def update_config(files, diff_cmd):
    """Processes files in files and copy them in the $HOME directory."""
    home_dir = os.environ.get('HOME') or os.environ['USERPROFILE']
    to_diff = []
    ok_files = []
    for basename in sorted(files):
        dst = os.path.join(home_dir, basename)
        dst_dir = os.path.dirname(dst)
        if not os.path.isdir(dst_dir):
            os.makedirs(dst_dir)
        content = files[basename]
        if not os.path.isfile(dst):
            if os.path.isdir(dst):
                print('Skipping file which is directory at destination %s' %
                        basename)
                continue
            print('Copying %s' % basename)
            maruel.write(dst, content)
            ok_files.append(basename)
        else:
            if maruel.read(dst) != content:
                to_diff.append(basename)
            else:
                ok_files.append(basename)

    for basename in to_diff:
        result = diff(
            diff_cmd, os.path.join(home_dir, basename), basename, files[basename])
        if result:
            return result
    print('The following files are ok:')
    for f in ok_files:
      print('  ' + f)

def load_files(config_dir, files):
    """Loads all the files content into dictionnary |files|.

    Appends the content if an entry is already present.
    """
    blacklist = [r'.*README.md$', r'.*\.swp$']
    for basename in maruel.walk(config_dir, [r'.*'], blacklist):
        src = os.path.join(config_dir, basename)
        if os.path.isfile(src):
            if basename not in files:
                files[basename] = maruel.read(src)
            else:
                files[basename] += maruel.read(src)

def main():
    parser = optparse.OptionParser()
    parser.add_option('-u', action='store_true', help='Unified diff')
    options, args = parser.parse_args()
    if args:
      parser.error('Unsupported args %s' % args)

    files = {}
    load_files(os.path.join(BIN_PUB_DIR, 'configs'), files)

    # Look for ~/bin/bin_pub pattern.
    if os.path.basename(os.path.dirname(BIN_PUB_DIR)) == 'bin':
      load_files(os.path.join(BIN_PUB_DIR, '..', 'configs'), files)

    cmd = ['vim', '-d']
    if options.u:
        cmd = ['diff', '-u']
    return update_config(files, cmd)

if __name__ == '__main__':
    sys.exit(main())

# vim: ts=4:sw=4:tw=80:et:
<|endoftext|>


<commit_before>"""
TeSS to iAnn events sync v 0.0

This module aims to provide a schedulable process to make automatic sync
from TeSS to iAnn event registry.
"""
from dateutil.parser import parse
from pytz import UTC as utc
from datetime import datetime
import pysolr
import urllib2
import json
from docs import conf
import logging
import daemon
import time
import click
import os
import sys

WELCOME_MSJ = 'ELIXIR TeSS to iAnn events synchronizer script V 0.0'

def init():
    logging.basicConfig(filename=conf.LOG_FILE, level=logging.INFO,
                        format='%(levelname)s[%(asctime)s]: %(message)s',
                        datefmt='%m/%d/%Y %I:%M:%S %p', filemode='a')

def map_tess_to_iann(tess_event=None):
    """
    Map an TeSS event to a iAnn event
    :param tess_event: dictionary that represents an event in TeSS
    :return: dictionary that represents an event in iAnn
    """
    iann_event = dict()
    if not tess_event:
        return iann_event
    for tess_field, iann_field in conf.TESS_TO_IANN_MAPPER.iteritems():
        if tess_field in tess_event and tess_event[tess_field]:
            iann_event[iann_field] = tess_event[tess_field]
    iann_event['category'] = iann_event['category'] + ['event'] if 'category' in iann_event else ['event']
    for index, item in enumerate(iann_event['category']):
        if item == 'workshops_and_courses':
            iann_event['category'][index] = 'course'
        if item == 'meetings_and_conferences':
            iann_event['category'][index] = 'meeting'
    return iann_event

def get_tess_events_from_url(query_url):
    """
    Gets the events data for a given TeSS query URL
    :param query_url: query url
    :return: all the events retrieved for the page
    """
    # Execute the HTTP query
    logging.info("Query to TeSS web service: " + query_url)
    page_data = urllib2.urlopen(query_url)
    # Load the JSON data into a Python dict
    tess_events = json.load(page_data)
    return tess_events

def get_tess_all_events(start_dt, expired=False):
    """
    Retrieves all the data available from TeSS
    :param start_dt: date to start harversting events
    :param expired: flag to indicate if expired events must be fetched
    :return: all the events available from TeSS
    """
    page = 1
    tess_events = []
    end_dt = utc.localize(datetime.now())
    start_dt = start_dt
    logging.info('Starting harvesting of TeSS events data from ' + str(start_dt) + ' to ' + str(end_dt))
    # Repeat the data gathering for every possible page until there is no results
    while True:
        logging.info('Retrieving TeSS data for page # %d' % page)
        # Generate the URL to make the query
        query_url = conf.TESS_URL + 'events.json' + '?' + 'page=%d' % page
        if expired:
            query_url += '&include_expired=true'
        page_results = get_tess_events_from_url(query_url)
        if not page_results:
            logging.info('Finishing harvesting')
            break
        tess_events += page_results
        page += 1
    logging.info('Performing conversion from TeSS events to iAnn events')
    # Iterate over the result set and map the TeSS events fields to iAnn events fields
    iann_events = [map_tess_to_iann(tess_event) for tess_event in tess_events
                   if start_dt < parse(tess_event['updated_at']) < end_dt]
    logging.info('Conversion done! %d updated events retrieved and converted' % len(iann_events))

    return dict(start=start_dt, end=end_dt, events=iann_events)

def push_to_iann(events):
    """
    Adds data to iAnn Solr from a Solr data structure
    :param events: list of events to be pushed to iAnn Solr
    """
    # Instantiates Solr service
    solr = pysolr.Solr(conf.IANN_URL, timeout=10)
    # Add Solr documents to service
    solr.add(events)

@click.command()
@click.option('--delay', default=10, help='Seconds between executions when the script is run as a daemon (eg. 60)')
@click.option('--log', default=conf.LOG_FILE, help='Log file path, if not defined will use the one on the conf.py')
@click.option('--tess_url', default=conf.TESS_URL, help='TeSS service URL, if not defined will use the one on conf.py')
@click.option('--iann_url', default=conf.IANN_URL, help='iAnn Solr URL, if not defined will use the one on conf.py')
@click.option('--daemonize', is_flag=True, help='Flag to run the script as a daemon')
@click.option('--include_expired', is_flag=True, help='Flag to fetch expired events from TeSS')
@click.option('--start', default=None, help='Start date')
def run(delay, log, tess_url, iann_url, daemonize, start, include_expired):
    """
    ELIXIR TeSS to iAnn events synchronizer script V 0.0
    Script to get TeSS events and push them to iAnn. It can be used as a batch process or as a daemon process.
    """
    conf.LOG_FILE = log
    conf.TESS_URL = tess_url
    conf.IANN_URL = iann_url
    log_start = None
    for line in reversed(list(open(conf.LOG_FILE))):
        if 'Last End:' in line:
            log_start = line.split('Last End:')[1]
            break
    if start:
        start = utc.localize(parse(start))
    elif log_start:
        start = parse(log_start)
    else:
        start = utc.localize(parse('2000-01-01'))
    click.secho(WELCOME_MSJ, fg='yellow', bg='red', bold=True)
    if not daemonize:
        click.secho('Fetching events from TeSS', fg='blue', bold=True)
        init()
        results = get_tess_all_events(start, include_expired)
        push_to_iann(results['events'])
        click.secho('Done!', fg='blue', bold=True)
        logging.info('Last End:' + str(results['end']))
        logging.info('/**************************************************************************************/')
        return
    click.secho('Fetching events from TeSS every %d seconds' % delay, fg='blue', bold=True)
    with daemon.DaemonContext(stdout=sys.stdout, stderr=sys.stdout):
        init()
        click.secho('Process ID: %d' % os.getpid(), fg='red', bold=True, blink=True)
        while True:
            results = get_tess_all_events(start, include_expired)
            push_to_iann(results['events'])
            start = results['end']
            logging.info('Last End:' + str(start))
            logging.info('/**************************************************************************************/')
            time.sleep(delay)

if __name__ == "__main__":
    try:
        run()
    except AttributeError:
        print "Try 'python tess_to_iann.py --help' to get information about usage"

<commit_msg>Adding failed to update Solr exception<commit_after>"""
TeSS to iAnn events sync v 0.0

This module aims to provide a schedulable process to make automatic sync
from TeSS to iAnn event registry.
"""
from dateutil.parser import parse
from pytz import UTC as utc
from datetime import datetime
import pysolr
import urllib2
import json
from docs import conf
import logging
import daemon
import time
import click
import os
import sys

WELCOME_MSJ = 'ELIXIR TeSS to iAnn events synchronizer script V 0.0'

def init():
    logging.basicConfig(filename=conf.LOG_FILE, level=logging.INFO,
                        format='%(levelname)s[%(asctime)s]: %(message)s',
                        datefmt='%m/%d/%Y %I:%M:%S %p', filemode='a')

def map_tess_to_iann(tess_event=None):
    """
    Map an TeSS event to a iAnn event
    :param tess_event: dictionary that represents an event in TeSS
    :return: dictionary that represents an event in iAnn
    """
    iann_event = dict()
    if not tess_event:
        return iann_event
    for tess_field, iann_field in conf.TESS_TO_IANN_MAPPER.iteritems():
        if tess_field in tess_event and tess_event[tess_field]:
            iann_event[iann_field] = tess_event[tess_field]
    iann_event['category'] = iann_event['category'] + ['event'] if 'category' in iann_event else ['event']
    for index, item in enumerate(iann_event['category']):
        if item == 'workshops_and_courses':
            iann_event['category'][index] = 'course'
        if item == 'meetings_and_conferences':
            iann_event['category'][index] = 'meeting'
    return iann_event

def get_tess_events_from_url(query_url):
    """
    Gets the events data for a given TeSS query URL
    :param query_url: query url
    :return: all the events retrieved for the page
    """
    # Execute the HTTP query
    logging.info("Query to TeSS web service: " + query_url)
    page_data = urllib2.urlopen(query_url)
    # Load the JSON data into a Python dict
    tess_events = json.load(page_data)
    return tess_events

def get_tess_all_events(start_dt, expired=False):
    """
    Retrieves all the data available from TeSS
    :param start_dt: date to start harversting events
    :param expired: flag to indicate if expired events must be fetched
    :return: all the events available from TeSS
    """
    page = 1
    tess_events = []
    end_dt = utc.localize(datetime.now())
    start_dt = start_dt
    logging.info('Starting harvesting of TeSS events data from ' + str(start_dt) + ' to ' + str(end_dt))
    # Repeat the data gathering for every possible page until there is no results
    while True:
        logging.info('Retrieving TeSS data for page # %d' % page)
        # Generate the URL to make the query
        query_url = conf.TESS_URL + 'events.json' + '?' + 'page=%d' % page
        if expired:
            query_url += '&include_expired=true'
        page_results = get_tess_events_from_url(query_url)
        if not page_results:
            logging.info('Finishing harvesting')
            break
        tess_events += page_results
        page += 1
    logging.info('Performing conversion from TeSS events to iAnn events')
    # Iterate over the result set and map the TeSS events fields to iAnn events fields
    iann_events = [map_tess_to_iann(tess_event) for tess_event in tess_events
                   if start_dt < parse(tess_event['updated_at']) < end_dt]
    logging.info('Conversion done! %d updated events retrieved and converted' % len(iann_events))

    return dict(start=start_dt, end=end_dt, events=iann_events)

def push_to_iann(events):
    """
    Adds data to iAnn Solr from a Solr data structure
    :param events: list of events to be pushed to iAnn Solr
    """
    # Instantiates Solr service
    solr = pysolr.Solr(conf.IANN_URL, timeout=10)
    # Add Solr documents to service
    solr.add(events)

@click.command()
@click.option('--delay', default=10, help='Seconds between executions when the script is run as a daemon (eg. 60)')
@click.option('--log', default=conf.LOG_FILE, help='Log file path, if not defined will use the one on the conf.py')
@click.option('--tess_url', default=conf.TESS_URL, help='TeSS service URL, if not defined will use the one on conf.py')
@click.option('--iann_url', default=conf.IANN_URL, help='iAnn Solr URL, if not defined will use the one on conf.py')
@click.option('--daemonize', is_flag=True, help='Flag to run the script as a daemon')
@click.option('--include_expired', is_flag=True, help='Flag to fetch expired events from TeSS')
@click.option('--start', default=None, help='Start date')
def run(delay, log, tess_url, iann_url, daemonize, start, include_expired):
    """
    ELIXIR TeSS to iAnn events synchronizer script V 0.0
    Script to get TeSS events and push them to iAnn. It can be used as a batch process or as a daemon process.
    """
    conf.LOG_FILE = log
    conf.TESS_URL = tess_url
    conf.IANN_URL = iann_url
    log_start = None
    for line in reversed(list(open(conf.LOG_FILE))):
        if 'Last End:' in line:
            log_start = line.split('Last End:')[1]
            break
    if start:
        start = utc.localize(parse(start))
    elif log_start:
        start = parse(log_start)
    else:
        start = utc.localize(parse('2000-01-01'))
    click.secho(WELCOME_MSJ, fg='yellow', bg='red', bold=True)
    if not daemonize:
        click.secho('Fetching events from TeSS', fg='blue', bold=True)
        init()
        results = get_tess_all_events(start, include_expired)
        try:
            push_to_iann(results['events'])
            click.secho('Done!', fg='blue', bold=True)
            logging.info('Last End:' + str(results['end']))
            logging.info('/**************************************************************************************/')
        except pysolr.SolrError:
            logging.info('There was an error trying to push the events to iAnn Solr')
            logging.info('/**************************************************************************************/')

        return
    click.secho('Fetching events from TeSS every %d seconds' % delay, fg='blue', bold=True)
    with daemon.DaemonContext(stdout=sys.stdout, stderr=sys.stdout):
        init()
        click.secho('Process ID: %d' % os.getpid(), fg='red', bold=True, blink=True)
        while True:
            results = get_tess_all_events(start, include_expired)
            try:
                push_to_iann(results['events'])
                start = results['end']
                logging.info('Last End:' + str(start))
                logging.info('/**************************************************************************************/')
            except pysolr.SolrError:
                logging.info('There was an error trying to push the events to iAnn Solr, will try again in '
                             + str(delay) + ' seconds')
                logging.info('/**************************************************************************************/')
            time.sleep(delay)

if __name__ == "__main__":
    try:
        run()
    except AttributeError:
        print "Try 'python tess_to_iann.py --help' to get information about usage"

<|endoftext|>