<commit_before># Copyright (c) 2012 OpenStack Foundation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import copy
import itertools
import operator

from neutron_lib import constants
from neutron_lib import exceptions
from oslo_config import cfg
from oslo_db import exception as db_exc
from oslo_log import log as logging
import oslo_messaging
from oslo_utils import excutils

from neutron._i18n import _, _LW
from neutron.callbacks import resources
from neutron.common import constants as n_const
from neutron.common import exceptions as n_exc
from neutron.common import utils
from neutron.db import api as db_api
from neutron.db import provisioning_blocks
from neutron.extensions import portbindings
from neutron import manager
from neutron.plugins.common import utils as p_utils
from neutron.quota import resource_registry

LOG = logging.getLogger(__name__)

class DhcpRpcCallback(object):
    """DHCP agent RPC callback in plugin implementations.

    This class implements the server side of an rpc interface.  The client
    side of this interface can be found in
    neutron.agent.dhcp.agent.DhcpPluginApi.  For more information about
    changing rpc interfaces, see doc/source/devref/rpc_api.rst.
    """

    # API version history:
    #     1.0 - Initial version.
    #     1.1 - Added get_active_networks_info, create_dhcp_port,
    #           and update_dhcp_port methods.
    #     1.2 - Removed get_dhcp_port. When removing a method (Making a
    #           backwards incompatible change) you would normally bump the
    #           major version. However, since the method was unused in the
    #           RPC client for many releases, it should be OK to bump the
    #           minor release instead and claim RPC compatibility with the
    #           last few client versions.
    #     1.3 - Removed release_port_fixed_ip. It's not used by reference DHCP
    #           agent since Juno, so similar rationale for not bumping the
    #           major version as above applies here too.
    #     1.4 - Removed update_lease_expiration. It's not used by reference
    #           DHCP agent since Juno, so similar rationale for not bumping the
    #           major version as above applies here too.
    #     1.5 - Added dhcp_ready_on_ports.
    #     1.6 - Removed get_active_networks. It's not used by reference
    #           DHCP agent since Havana, so similar rationale for not bumping
    #           the major version as above applies here too.

    target = oslo_messaging.Target(
        namespace=n_const.RPC_NAMESPACE_DHCP_PLUGIN,
        version='1.6')

    def _get_active_networks(self, context, **kwargs):
        """Retrieve and return a list of the active networks."""
        host = kwargs.get('host')
        plugin = manager.NeutronManager.get_plugin()
        if utils.is_extension_supported(
            plugin, constants.DHCP_AGENT_SCHEDULER_EXT_ALIAS):
            if cfg.CONF.network_auto_schedule:
                plugin.auto_schedule_networks(context, host)
            nets = plugin.list_active_networks_on_active_dhcp_agent(
                context, host)
        else:
            filters = dict(admin_state_up=[True])
            nets = plugin.get_networks(context, filters=filters)
        return nets

    def _port_action(self, plugin, context, port, action):
        """Perform port operations taking care of concurrency issues."""
        try:
            if action == 'create_port':
                return p_utils.create_port(plugin, context, port)
            elif action == 'update_port':
                return plugin.update_port(context, port['id'], port)
            else:
                msg = _('Unrecognized action')
                raise exceptions.Invalid(message=msg)
        except (db_exc.DBError,
                exceptions.NetworkNotFound,
                exceptions.SubnetNotFound,
                exceptions.IpAddressGenerationFailure) as e:
            with excutils.save_and_reraise_exception(reraise=False) as ctxt:
                if isinstance(e, exceptions.IpAddressGenerationFailure):
                    # Check if the subnet still exists and if it does not,
                    # this is the reason why the ip address generation failed.
                    # In any other unlikely event re-raise
                    try:
                        subnet_id = port['port']['fixed_ips'][0]['subnet_id']
                        plugin.get_subnet(context, subnet_id)
                    except exceptions.SubnetNotFound:
                        pass
                    else:
                        ctxt.reraise = True
                net_id = port['port']['network_id']
                LOG.warning(_LW("Action %(action)s for network %(net_id)s "
                                "could not complete successfully: %(reason)s"),
                            {"action": action, "net_id": net_id, 'reason': e})

    def _group_by_network_id(self, res):
        grouped = {}
        keyfunc = operator.itemgetter('network_id')
        for net_id, values in itertools.groupby(sorted(res, key=keyfunc),
                                                keyfunc):
            grouped[net_id] = list(values)
        return grouped

    def get_active_networks_info(self, context, **kwargs):
        """Returns all the networks/subnets/ports in system."""
        host = kwargs.get('host')
        LOG.debug('get_active_networks_info from %s', host)
        networks = self._get_active_networks(context, **kwargs)
        plugin = manager.NeutronManager.get_plugin()
        filters = {'network_id': [network['id'] for network in networks]}
        ports = plugin.get_ports(context, filters=filters)
        filters['enable_dhcp'] = [True]
        # NOTE(kevinbenton): we sort these because the agent builds tags
        # based on position in the list and has to restart the process if
        # the order changes.
        subnets = sorted(plugin.get_subnets(context, filters=filters),
                         key=operator.itemgetter('id'))

        grouped_subnets = self._group_by_network_id(subnets)
        grouped_ports = self._group_by_network_id(ports)
        for network in networks:
            network['subnets'] = grouped_subnets.get(network['id'], [])
            network['ports'] = grouped_ports.get(network['id'], [])

        return networks

    def get_network_info(self, context, **kwargs):
        """Retrieve and return extended information about a network."""
        network_id = kwargs.get('network_id')
        host = kwargs.get('host')
        LOG.debug('Network %(network_id)s requested from '
                  '%(host)s', {'network_id': network_id,
                               'host': host})
        plugin = manager.NeutronManager.get_plugin()
        try:
            network = plugin.get_network(context, network_id)
        except exceptions.NetworkNotFound:
            LOG.debug("Network %s could not be found, it might have "
                      "been deleted concurrently.", network_id)
            return
        filters = dict(network_id=[network_id])
        # NOTE(kevinbenton): we sort these because the agent builds tags
        # based on position in the list and has to restart the process if
        # the order changes.
        network['subnets'] = sorted(
            plugin.get_subnets(context, filters=filters),
            key=operator.itemgetter('id'))
        network['ports'] = plugin.get_ports(context, filters=filters)
        return network

    @db_api.retry_db_errors
    def release_dhcp_port(self, context, **kwargs):
        """Release the port currently being used by a DHCP agent."""
        host = kwargs.get('host')
        network_id = kwargs.get('network_id')
        device_id = kwargs.get('device_id')

        LOG.debug('DHCP port deletion for %(network_id)s request from '
                  '%(host)s',
                  {'network_id': network_id, 'host': host})
        plugin = manager.NeutronManager.get_plugin()
        plugin.delete_ports_by_device_id(context, device_id, network_id)

    @db_api.retry_db_errors
    @resource_registry.mark_resources_dirty
    def create_dhcp_port(self, context, **kwargs):
        """Create and return dhcp port information.

        If an expected failure occurs, a None port is returned.

        """
        host = kwargs.get('host')
        # Note(pbondar): Create deep copy of port to prevent operating
        # on changed dict if RetryRequest is raised
        port = copy.deepcopy(kwargs.get('port'))
        LOG.debug('Create dhcp port %(port)s '
                  'from %(host)s.',
                  {'port': port,
                   'host': host})

        port['port']['device_owner'] = constants.DEVICE_OWNER_DHCP
        port['port'][portbindings.HOST_ID] = host
        if 'mac_address' not in port['port']:
            port['port']['mac_address'] = constants.ATTR_NOT_SPECIFIED
        plugin = manager.NeutronManager.get_plugin()
        return self._port_action(plugin, context, port, 'create_port')

    @db_api.retry_db_errors
    def update_dhcp_port(self, context, **kwargs):
        """Update the dhcp port."""
        host = kwargs.get('host')
        port = kwargs.get('port')
        port['id'] = kwargs.get('port_id')
        port['port'][portbindings.HOST_ID] = host
        plugin = manager.NeutronManager.get_plugin()
        old_port = plugin.get_port(context, port['id'])
        if (old_port['device_id'] != n_const.DEVICE_ID_RESERVED_DHCP_PORT
            and old_port['device_id'] !=
            utils.get_dhcp_agent_device_id(port['port']['network_id'], host)):
            raise n_exc.DhcpPortInUse(port_id=port['id'])
        LOG.debug('Update dhcp port %(port)s '
                  'from %(host)s.',
                  {'port': port,
                   'host': host})
        return self._port_action(plugin, context, port, 'update_port')

    def dhcp_ready_on_ports(self, context, port_ids):
        for port_id in port_ids:
            provisioning_blocks.provisioning_complete(
                context, port_id, resources.PORT,
                provisioning_blocks.DHCP_ENTITY)
<commit_msg>Add retry decorator to dhcp_ready_on_ports<commit_after># Copyright (c) 2012 OpenStack Foundation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import copy
import itertools
import operator

from neutron_lib import constants
from neutron_lib import exceptions
from oslo_config import cfg
from oslo_db import exception as db_exc
from oslo_log import log as logging
import oslo_messaging
from oslo_utils import excutils

from neutron._i18n import _, _LW
from neutron.callbacks import resources
from neutron.common import constants as n_const
from neutron.common import exceptions as n_exc
from neutron.common import utils
from neutron.db import api as db_api
from neutron.db import provisioning_blocks
from neutron.extensions import portbindings
from neutron import manager
from neutron.plugins.common import utils as p_utils
from neutron.quota import resource_registry

LOG = logging.getLogger(__name__)

class DhcpRpcCallback(object):
    """DHCP agent RPC callback in plugin implementations.

    This class implements the server side of an rpc interface.  The client
    side of this interface can be found in
    neutron.agent.dhcp.agent.DhcpPluginApi.  For more information about
    changing rpc interfaces, see doc/source/devref/rpc_api.rst.
    """

    # API version history:
    #     1.0 - Initial version.
    #     1.1 - Added get_active_networks_info, create_dhcp_port,
    #           and update_dhcp_port methods.
    #     1.2 - Removed get_dhcp_port. When removing a method (Making a
    #           backwards incompatible change) you would normally bump the
    #           major version. However, since the method was unused in the
    #           RPC client for many releases, it should be OK to bump the
    #           minor release instead and claim RPC compatibility with the
    #           last few client versions.
    #     1.3 - Removed release_port_fixed_ip. It's not used by reference DHCP
    #           agent since Juno, so similar rationale for not bumping the
    #           major version as above applies here too.
    #     1.4 - Removed update_lease_expiration. It's not used by reference
    #           DHCP agent since Juno, so similar rationale for not bumping the
    #           major version as above applies here too.
    #     1.5 - Added dhcp_ready_on_ports.
    #     1.6 - Removed get_active_networks. It's not used by reference
    #           DHCP agent since Havana, so similar rationale for not bumping
    #           the major version as above applies here too.

    target = oslo_messaging.Target(
        namespace=n_const.RPC_NAMESPACE_DHCP_PLUGIN,
        version='1.6')

    def _get_active_networks(self, context, **kwargs):
        """Retrieve and return a list of the active networks."""
        host = kwargs.get('host')
        plugin = manager.NeutronManager.get_plugin()
        if utils.is_extension_supported(
            plugin, constants.DHCP_AGENT_SCHEDULER_EXT_ALIAS):
            if cfg.CONF.network_auto_schedule:
                plugin.auto_schedule_networks(context, host)
            nets = plugin.list_active_networks_on_active_dhcp_agent(
                context, host)
        else:
            filters = dict(admin_state_up=[True])
            nets = plugin.get_networks(context, filters=filters)
        return nets

    def _port_action(self, plugin, context, port, action):
        """Perform port operations taking care of concurrency issues."""
        try:
            if action == 'create_port':
                return p_utils.create_port(plugin, context, port)
            elif action == 'update_port':
                return plugin.update_port(context, port['id'], port)
            else:
                msg = _('Unrecognized action')
                raise exceptions.Invalid(message=msg)
        except (db_exc.DBError,
                exceptions.NetworkNotFound,
                exceptions.SubnetNotFound,
                exceptions.IpAddressGenerationFailure) as e:
            with excutils.save_and_reraise_exception(reraise=False) as ctxt:
                if isinstance(e, exceptions.IpAddressGenerationFailure):
                    # Check if the subnet still exists and if it does not,
                    # this is the reason why the ip address generation failed.
                    # In any other unlikely event re-raise
                    try:
                        subnet_id = port['port']['fixed_ips'][0]['subnet_id']
                        plugin.get_subnet(context, subnet_id)
                    except exceptions.SubnetNotFound:
                        pass
                    else:
                        ctxt.reraise = True
                net_id = port['port']['network_id']
                LOG.warning(_LW("Action %(action)s for network %(net_id)s "
                                "could not complete successfully: %(reason)s"),
                            {"action": action, "net_id": net_id, 'reason': e})

    def _group_by_network_id(self, res):
        grouped = {}
        keyfunc = operator.itemgetter('network_id')
        for net_id, values in itertools.groupby(sorted(res, key=keyfunc),
                                                keyfunc):
            grouped[net_id] = list(values)
        return grouped

    def get_active_networks_info(self, context, **kwargs):
        """Returns all the networks/subnets/ports in system."""
        host = kwargs.get('host')
        LOG.debug('get_active_networks_info from %s', host)
        networks = self._get_active_networks(context, **kwargs)
        plugin = manager.NeutronManager.get_plugin()
        filters = {'network_id': [network['id'] for network in networks]}
        ports = plugin.get_ports(context, filters=filters)
        filters['enable_dhcp'] = [True]
        # NOTE(kevinbenton): we sort these because the agent builds tags
        # based on position in the list and has to restart the process if
        # the order changes.
        subnets = sorted(plugin.get_subnets(context, filters=filters),
                         key=operator.itemgetter('id'))

        grouped_subnets = self._group_by_network_id(subnets)
        grouped_ports = self._group_by_network_id(ports)
        for network in networks:
            network['subnets'] = grouped_subnets.get(network['id'], [])
            network['ports'] = grouped_ports.get(network['id'], [])

        return networks

    def get_network_info(self, context, **kwargs):
        """Retrieve and return extended information about a network."""
        network_id = kwargs.get('network_id')
        host = kwargs.get('host')
        LOG.debug('Network %(network_id)s requested from '
                  '%(host)s', {'network_id': network_id,
                               'host': host})
        plugin = manager.NeutronManager.get_plugin()
        try:
            network = plugin.get_network(context, network_id)
        except exceptions.NetworkNotFound:
            LOG.debug("Network %s could not be found, it might have "
                      "been deleted concurrently.", network_id)
            return
        filters = dict(network_id=[network_id])
        # NOTE(kevinbenton): we sort these because the agent builds tags
        # based on position in the list and has to restart the process if
        # the order changes.
        network['subnets'] = sorted(
            plugin.get_subnets(context, filters=filters),
            key=operator.itemgetter('id'))
        network['ports'] = plugin.get_ports(context, filters=filters)
        return network

    @db_api.retry_db_errors
    def release_dhcp_port(self, context, **kwargs):
        """Release the port currently being used by a DHCP agent."""
        host = kwargs.get('host')
        network_id = kwargs.get('network_id')
        device_id = kwargs.get('device_id')

        LOG.debug('DHCP port deletion for %(network_id)s request from '
                  '%(host)s',
                  {'network_id': network_id, 'host': host})
        plugin = manager.NeutronManager.get_plugin()
        plugin.delete_ports_by_device_id(context, device_id, network_id)

    @db_api.retry_db_errors
    @resource_registry.mark_resources_dirty
    def create_dhcp_port(self, context, **kwargs):
        """Create and return dhcp port information.

        If an expected failure occurs, a None port is returned.

        """
        host = kwargs.get('host')
        # Note(pbondar): Create deep copy of port to prevent operating
        # on changed dict if RetryRequest is raised
        port = copy.deepcopy(kwargs.get('port'))
        LOG.debug('Create dhcp port %(port)s '
                  'from %(host)s.',
                  {'port': port,
                   'host': host})

        port['port']['device_owner'] = constants.DEVICE_OWNER_DHCP
        port['port'][portbindings.HOST_ID] = host
        if 'mac_address' not in port['port']:
            port['port']['mac_address'] = constants.ATTR_NOT_SPECIFIED
        plugin = manager.NeutronManager.get_plugin()
        return self._port_action(plugin, context, port, 'create_port')

    @db_api.retry_db_errors
    def update_dhcp_port(self, context, **kwargs):
        """Update the dhcp port."""
        host = kwargs.get('host')
        port = kwargs.get('port')
        port['id'] = kwargs.get('port_id')
        port['port'][portbindings.HOST_ID] = host
        plugin = manager.NeutronManager.get_plugin()
        old_port = plugin.get_port(context, port['id'])
        if (old_port['device_id'] != n_const.DEVICE_ID_RESERVED_DHCP_PORT
            and old_port['device_id'] !=
            utils.get_dhcp_agent_device_id(port['port']['network_id'], host)):
            raise n_exc.DhcpPortInUse(port_id=port['id'])
        LOG.debug('Update dhcp port %(port)s '
                  'from %(host)s.',
                  {'port': port,
                   'host': host})
        return self._port_action(plugin, context, port, 'update_port')

    @db_api.retry_db_errors
    def dhcp_ready_on_ports(self, context, port_ids):
        for port_id in port_ids:
            provisioning_blocks.provisioning_complete(
                context, port_id, resources.PORT,
                provisioning_blocks.DHCP_ENTITY)
<|endoftext|>


<commit_before><commit_msg>make old way default<commit_after><|endoftext|>


<commit_before>import argparse
import logging.config

# parse args and setup logging before anything else

arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('token')
arg_parser.add_argument('--log', help='Log level', default='WARNING')
arg_parser.add_argument('--state', help='Bot state file', default='bot.json')
args = arg_parser.parse_args()

LOG_FMT = '%(asctime)s P#%(process)d T#%(thread)d [%(name)s/%(levelname)s] %(message)s'

# attempt to use colorlog, if available
try:
    import colorlog
    import sys

    formatter = colorlog.ColoredFormatter(
        fmt='%(log_color)s' + LOG_FMT + '%(reset)s',
        log_colors={
            'DEBUG': 'blue',
            'INFO': 'green',
            'WARNING': 'yellow',
            'ERROR': 'red',
            'CRITICAL': 'white,bg_red',
        }
    )

    handler = colorlog.StreamHandler(sys.stdout)
    handler.setFormatter(formatter)
    logging.root.addHandler(handler)
    logging.root.setLevel(args.log)

# otherwise just stick with basic logging
except:
    logging.basicConfig(
        level=args.log,
        format=LOG_FMT)

log = logging.getLogger(__name__)

import asyncio
import time

from cogbot.cog_bot import CogBot
from cogbot.cog_bot_state import CogBotState

# TODO Consider switching to the library rewrite: https://github.com/Rapptz/discord.py/tree/rewrite
# The rewrite has auto-reconnect behaviour: https://github.com/Rapptz/discord.py/blob/rewrite/discord/client.py#L405
# This would make our hacky crash workarounds trivial.

def _attempt_logout(loop, bot):
    try:
        log.warning('Attempting clean logout...')
        loop.run_until_complete(bot.logout())

        log.info('Gathering leftover tasks...')
        pending = asyncio.Task.all_tasks(loop=loop)
        gathered = asyncio.gather(*pending, loop=loop)

        log.info('Cancelling leftover tasks...')
        gathered.cancel()

        log.info('Allowing cancelled tasks to finish...')
        try:
            loop.run_until_complete(gathered)
        except:
            pass

    except Exception as ex:
        log.exception('Encountered an error while attempting to logout')
        log.critical('Forcibly terminating with system exit')
        exit()

def run():
    state = CogBotState(args.state)

    loop = asyncio.get_event_loop()

    last_death: type = None

    while True:
        log.info('Starting bot...')
        bot = CogBot(state=state, loop=loop)

        if last_death and state.notify_on_recovery and state.managers:
            log.warning(f'Notifying {len(state.managers)} managers of crash recovery...')

            try:
                reason = last_death.__name__
            except:
                reason = None

            message = 'Hello! I\'ve just recovered from a fatal crash caused by'
            message += f': `{reason}`' if reason else ' an unknown error.'

            for manager in state.managers:
                bot.queue_message(bot.get_user_info, manager, message)

        try:
            loop.run_until_complete(bot.start(args.token))

        except KeyboardInterrupt:
            log.info('Keyboard interrupt detected')
            _attempt_logout(loop, bot)
            break

        except Exception as ex:
            last_death = ex
            log.exception('Encountered a fatal exception')
            _attempt_logout(loop, bot)

        log.info('Closing event loop...')
        loop.close()

        log.warning(f'Restarting bot in {state.recovery_delay} seconds...')
        time.sleep(state.recovery_delay)

        log.info('Opening a new event loop...')
        loop = asyncio.new_event_loop()

    log.info('Closing event loop for good...')
    loop.close()

    log.warning('Bot successfully terminated')

log.info('Hello!')
run()
log.info('Goodbye!')
<commit_msg>Adjust log format<commit_after>import argparse
import logging.config

# parse args and setup logging before anything else

arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('token')
arg_parser.add_argument('--log', help='Log level', default='WARNING')
arg_parser.add_argument('--state', help='Bot state file', default='bot.json')
args = arg_parser.parse_args()

LOG_FMT = '%(asctime)s [%(name)s/%(levelname)s] %(message)s'

# attempt to use colorlog, if available
try:
    import colorlog
    import sys

    formatter = colorlog.ColoredFormatter(
        fmt='%(log_color)s' + LOG_FMT + '%(reset)s',
        log_colors={
            'DEBUG': 'blue',
            'INFO': 'green',
            'WARNING': 'yellow',
            'ERROR': 'red',
            'CRITICAL': 'white,bg_red',
        }
    )

    handler = colorlog.StreamHandler(sys.stdout)
    handler.setFormatter(formatter)
    logging.root.addHandler(handler)
    logging.root.setLevel(args.log)

# otherwise just stick with basic logging
except:
    logging.basicConfig(
        level=args.log,
        format=LOG_FMT)

log = logging.getLogger(__name__)

import asyncio
import time

from cogbot.cog_bot import CogBot
from cogbot.cog_bot_state import CogBotState

# TODO Consider switching to the library rewrite: https://github.com/Rapptz/discord.py/tree/rewrite
# The rewrite has auto-reconnect behaviour: https://github.com/Rapptz/discord.py/blob/rewrite/discord/client.py#L405
# This would make our hacky crash workarounds trivial.

def _attempt_logout(loop, bot):
    try:
        log.warning('Attempting clean logout...')
        loop.run_until_complete(bot.logout())

        log.info('Gathering leftover tasks...')
        pending = asyncio.Task.all_tasks(loop=loop)
        gathered = asyncio.gather(*pending, loop=loop)

        log.info('Cancelling leftover tasks...')
        gathered.cancel()

        log.info('Allowing cancelled tasks to finish...')
        try:
            loop.run_until_complete(gathered)
        except:
            pass

    except Exception as ex:
        log.exception('Encountered an error while attempting to logout')
        log.critical('Forcibly terminating with system exit')
        exit()

def run():
    state = CogBotState(args.state)

    loop = asyncio.get_event_loop()

    last_death: type = None

    while True:
        log.info('Starting bot...')
        bot = CogBot(state=state, loop=loop)

        if last_death and state.notify_on_recovery and state.managers:
            log.warning(f'Notifying {len(state.managers)} managers of crash recovery...')

            try:
                reason = last_death.__name__
            except:
                reason = None

            message = 'Hello! I\'ve just recovered from a fatal crash caused by'
            message += f': `{reason}`' if reason else ' an unknown error.'

            for manager in state.managers:
                bot.queue_message(bot.get_user_info, manager, message)

        try:
            loop.run_until_complete(bot.start(args.token))

        except KeyboardInterrupt:
            log.info('Keyboard interrupt detected')
            _attempt_logout(loop, bot)
            break

        except Exception as ex:
            last_death = ex
            log.exception('Encountered a fatal exception')
            _attempt_logout(loop, bot)

        log.info('Closing event loop...')
        loop.close()

        log.warning(f'Restarting bot in {state.recovery_delay} seconds...')
        time.sleep(state.recovery_delay)

        log.info('Opening a new event loop...')
        loop = asyncio.new_event_loop()

    log.info('Closing event loop for good...')
    loop.close()

    log.warning('Bot successfully terminated')

log.info('Hello!')
run()
log.info('Goodbye!')
<|endoftext|>


<commit_before>from django.db import connections, models
from django.db.models.query import RawQuerySet

from corehq.sql_db.routers import (
    HINT_PARTITION_VALUE,
    HINT_PLPROXY_READ,
    HINT_USING,
    db_for_read_write,
)
from corehq.util.exceptions import AccessRestricted

def raise_access_restricted(queryset):
    if not queryset._db and 'partition_value' not in queryset._hints and 'using' not in queryset._hints:
        raise AccessRestricted("This model can be partitioned. Please specify which"
            "partitioned database with `using` or look for an appropriate dbaccessors "
            "method.")

class RequireDBManager(models.Manager):
    """
    A Model Manager which requires that .using() is called before any
    other methods like .filter() or .get().
    """

    def get_queryset(self):
        queryset = super(RequireDBManager, self).get_queryset()
        raise_access_restricted(queryset)
        return queryset

    @staticmethod
    def get_db(partition_value):
        from corehq.sql_db.util import get_db_alias_for_partitioned_doc
        return get_db_alias_for_partitioned_doc(partition_value)

    def partitioned_get(self, partition_value, **kwargs):
        """Get a partioned model from it's database

        If the lookup should be performed using a different lookup value then it
        should be supplied as a keyword argument:

            model.objects.partitioned_get(partition_value, other_field=value)

        is equivalent to:

            db = get_db_alias_for_partitioned_doc(partition_value)
            model.objects.using(db).get(other_field=value)

        :param partition_value: The value that is used to partition the model;
                                this value will be used to select the database
        """
        if not kwargs:
            kwargs = {
                self.model.partition_attr: partition_value
            }
        return self.partitioned_query(partition_value).get(**kwargs)

    def partitioned_query(self, partition_value=None):
        """Shortcut to get a queryset for a partitioned database.
        Equivalent to:

            db = get_db_alias_for_partitioned_doc(partition_value)
            qs = Model.objects.using(db)
        """
        return self.db_manager(hints={HINT_PARTITION_VALUE: partition_value})

    def using(self, alias):
        return self.db_manager(hints={HINT_USING: alias}).get_queryset()

    def plproxy_read(self, raw_query, params=None):
        return RawQuerySet(raw_query, model=self.model, params=params, hints={HINT_PLPROXY_READ: True})

class PartitionedModel(models.Model):
    """
    This makes it so that you must specify a database name when
    using model methods .save() and .delete(), or when running
    queries through the model Manager.

    Useful to prevent accidental queries with Models whose partitioning
    is determined in Python rather than PL/Proxy.
    """

    objects = RequireDBManager()

    @classmethod
    def get_plproxy_cursor(cls, readonly=False):
        db = db_for_read_write(cls, write=not readonly, hints={HINT_PLPROXY_READ: True})
        return connections[db].cursor()

    @classmethod
    def get_cursor_for_partition_value(cls, partition_value, readonly=False):
        db = db_for_read_write(cls, write=not readonly, hints={HINT_PARTITION_VALUE: partition_value})
        return connections[db].cursor()

    @classmethod
    def get_cursor_for_partition_db(cls, db_alias, readonly=False):
        db = db_for_read_write(cls, write=not readonly, hints={HINT_USING: db_alias})
        return connections[db].cursor()

    @property
    def partition_attr(self):
        raise NotImplementedError

    @property
    def partition_value(self):
        return getattr(self, self.partition_attr)

    @property
    def db(self):
        """The partitioned database for this object"""
        assert self.partition_value, 'Partitioned model must have a partition value'
        return RequireDBManager.get_db(self.partition_value)

    class Meta(object):
        abstract = True

    def save(self, *args, **kwargs):
        self._add_routing(kwargs)
        return super(PartitionedModel, self).save(*args, **kwargs)

    def delete(self, *args, **kwargs):
        self._add_routing(kwargs)
        return super(PartitionedModel, self).delete(*args, **kwargs)

    def _add_routing(self, kwargs):
        if 'using' in kwargs:
            assert kwargs['using'] == self.db, \
                "using=%r, expected %r" % (kwargs['using'], self.db)
        else:
            kwargs['using'] = self.db
<commit_msg>remove routing code in save functions<commit_after>from django.db import connections, models
from django.db.models.query import RawQuerySet

from corehq.sql_db.routers import (
    HINT_PARTITION_VALUE,
    HINT_PLPROXY_READ,
    HINT_USING,
    db_for_read_write,
)
from corehq.util.exceptions import AccessRestricted

def raise_access_restricted(queryset):
    if not queryset._db and 'partition_value' not in queryset._hints and 'using' not in queryset._hints:
        raise AccessRestricted("This model can be partitioned. Please specify which"
            "partitioned database with `using` or look for an appropriate dbaccessors "
            "method.")

class RequireDBManager(models.Manager):
    """
    A Model Manager which requires that .using() is called before any
    other methods like .filter() or .get().
    """

    def get_queryset(self):
        queryset = super(RequireDBManager, self).get_queryset()
        raise_access_restricted(queryset)
        return queryset

    @staticmethod
    def get_db(partition_value):
        from corehq.sql_db.util import get_db_alias_for_partitioned_doc
        return get_db_alias_for_partitioned_doc(partition_value)

    def partitioned_get(self, partition_value, **kwargs):
        """Get a partioned model from it's database

        If the lookup should be performed using a different lookup value then it
        should be supplied as a keyword argument:

            model.objects.partitioned_get(partition_value, other_field=value)

        is equivalent to:

            db = get_db_alias_for_partitioned_doc(partition_value)
            model.objects.using(db).get(other_field=value)

        :param partition_value: The value that is used to partition the model;
                                this value will be used to select the database
        """
        if not kwargs:
            kwargs = {
                self.model.partition_attr: partition_value
            }
        return self.partitioned_query(partition_value).get(**kwargs)

    def partitioned_query(self, partition_value=None):
        """Shortcut to get a queryset for a partitioned database.
        Equivalent to:

            db = get_db_alias_for_partitioned_doc(partition_value)
            qs = Model.objects.using(db)
        """
        return self.db_manager(hints={HINT_PARTITION_VALUE: partition_value})

    def using(self, alias):
        return self.db_manager(hints={HINT_USING: alias}).get_queryset()

    def plproxy_read(self, raw_query, params=None):
        return RawQuerySet(raw_query, model=self.model, params=params, hints={HINT_PLPROXY_READ: True})

class PartitionedModel(models.Model):
    """
    This makes it so that you must specify a database name when
    using model methods .save() and .delete(), or when running
    queries through the model Manager.

    Useful to prevent accidental queries with Models whose partitioning
    is determined in Python rather than PL/Proxy.
    """

    objects = RequireDBManager()

    @classmethod
    def get_plproxy_cursor(cls, readonly=False):
        db = db_for_read_write(cls, write=not readonly, hints={HINT_PLPROXY_READ: True})
        return connections[db].cursor()

    @classmethod
    def get_cursor_for_partition_value(cls, partition_value, readonly=False):
        db = db_for_read_write(cls, write=not readonly, hints={HINT_PARTITION_VALUE: partition_value})
        return connections[db].cursor()

    @classmethod
    def get_cursor_for_partition_db(cls, db_alias, readonly=False):
        db = db_for_read_write(cls, write=not readonly, hints={HINT_USING: db_alias})
        return connections[db].cursor()

    @property
    def partition_attr(self):
        raise NotImplementedError

    @property
    def partition_value(self):
        return getattr(self, self.partition_attr)

    @property
    def db(self):
        """The partitioned database for this object"""
        assert self.partition_value, 'Partitioned model must have a partition value'
        return RequireDBManager.get_db(self.partition_value)

    class Meta(object):
        abstract = True
<|endoftext|>


<commit_before>from __future__ import absolute_import

import warnings

from raven.exceptions import InvalidDsn
from raven.transport.threaded import ThreadedHTTPTransport
from raven.utils import six
from raven.utils.urlparse import parse_qsl, urlparse

ERR_UNKNOWN_SCHEME = 'Unsupported Sentry DSN scheme: {0}'

DEFAULT_TRANSPORT = ThreadedHTTPTransport

class RemoteConfig(object):
    def __init__(self, base_url=None, project=None, public_key=None,
                 secret_key=None, transport=None, options=None):
        if base_url:
            base_url = base_url.rstrip('/')
            store_endpoint = '%s/api/%s/store/' % (base_url, project)
        else:
            store_endpoint = None

        self.base_url = base_url
        self.project = project
        self.public_key = public_key
        self.secret_key = secret_key
        self.options = options or {}
        self.store_endpoint = store_endpoint

        self._transport_cls = transport or DEFAULT_TRANSPORT

    def __unicode__(self):
        return six.text_type(self.base_url)

    def is_active(self):
        return all([self.base_url, self.project, self.public_key, self.secret_key])

    # TODO(dcramer): we dont want transports bound to a URL
    def get_transport(self):
        if not self.store_endpoint:
            return

        if not hasattr(self, '_transport'):
            parsed = urlparse(self.store_endpoint)
            self._transport = self._transport_cls(parsed, **self.options)
        return self._transport

    def get_public_dsn(self):
        url = urlparse(self.base_url)
        netloc = url.hostname
        if url.port:
            netloc += ':%s' % url.port
        return '//%s@%s%s/%s' % (self.public_key, netloc, url.path, self.project)

    @classmethod
    def from_string(cls, value, transport=None, transport_registry=None):
        url = urlparse(value)

        if url.scheme not in ('http', 'https'):
            warnings.warn('Transport selection via DSN is deprecated. You should explicitly pass the transport class to Client() instead.')

        if transport is None:
            if not transport_registry:
                from raven.transport import TransportRegistry, default_transports
                transport_registry = TransportRegistry(default_transports)

            if not transport_registry.supported_scheme(url.scheme):
                raise InvalidDsn(ERR_UNKNOWN_SCHEME.format(url.scheme))

            transport = transport_registry.get_transport_cls(url.scheme)

        netloc = url.hostname
        if url.port:
            netloc += ':%s' % url.port

        path_bits = url.path.rsplit('/', 1)
        if len(path_bits) > 1:
            path = path_bits[0]
        else:
            path = ''
        project = path_bits[-1]

        if not all([netloc, project, url.username, url.password]):
            raise InvalidDsn('Invalid Sentry DSN: %r' % url.geturl())

        base_url = '%s://%s%s' % (url.scheme.rsplit('+', 1)[-1], netloc, path)

        return cls(
            base_url=base_url,
            project=project,
            public_key=url.username,
            secret_key=url.password,
            options=dict(parse_qsl(url.query)),
            transport=transport,
        )
<commit_msg>Improve unsupported DSN scheme error<commit_after>from __future__ import absolute_import

import warnings

from raven.exceptions import InvalidDsn
from raven.transport.threaded import ThreadedHTTPTransport
from raven.utils import six
from raven.utils.urlparse import parse_qsl, urlparse

ERR_UNKNOWN_SCHEME = 'Unsupported Sentry DSN scheme: {0} ({1})'

DEFAULT_TRANSPORT = ThreadedHTTPTransport

class RemoteConfig(object):
    def __init__(self, base_url=None, project=None, public_key=None,
                 secret_key=None, transport=None, options=None):
        if base_url:
            base_url = base_url.rstrip('/')
            store_endpoint = '%s/api/%s/store/' % (base_url, project)
        else:
            store_endpoint = None

        self.base_url = base_url
        self.project = project
        self.public_key = public_key
        self.secret_key = secret_key
        self.options = options or {}
        self.store_endpoint = store_endpoint

        self._transport_cls = transport or DEFAULT_TRANSPORT

    def __unicode__(self):
        return six.text_type(self.base_url)

    def is_active(self):
        return all([self.base_url, self.project, self.public_key, self.secret_key])

    # TODO(dcramer): we dont want transports bound to a URL
    def get_transport(self):
        if not self.store_endpoint:
            return

        if not hasattr(self, '_transport'):
            parsed = urlparse(self.store_endpoint)
            self._transport = self._transport_cls(parsed, **self.options)
        return self._transport

    def get_public_dsn(self):
        url = urlparse(self.base_url)
        netloc = url.hostname
        if url.port:
            netloc += ':%s' % url.port
        return '//%s@%s%s/%s' % (self.public_key, netloc, url.path, self.project)

    @classmethod
    def from_string(cls, value, transport=None, transport_registry=None):
        url = urlparse(value)

        if url.scheme not in ('http', 'https'):
            warnings.warn('Transport selection via DSN is deprecated. You should explicitly pass the transport class to Client() instead.')

        if transport is None:
            if not transport_registry:
                from raven.transport import TransportRegistry, default_transports
                transport_registry = TransportRegistry(default_transports)

            if not transport_registry.supported_scheme(url.scheme):
                raise InvalidDsn(ERR_UNKNOWN_SCHEME.format(url.scheme, value))

            transport = transport_registry.get_transport_cls(url.scheme)

        netloc = url.hostname
        if url.port:
            netloc += ':%s' % url.port

        path_bits = url.path.rsplit('/', 1)
        if len(path_bits) > 1:
            path = path_bits[0]
        else:
            path = ''
        project = path_bits[-1]

        if not all([netloc, project, url.username, url.password]):
            raise InvalidDsn('Invalid Sentry DSN: %r' % url.geturl())

        base_url = '%s://%s%s' % (url.scheme.rsplit('+', 1)[-1], netloc, path)

        return cls(
            base_url=base_url,
            project=project,
            public_key=url.username,
            secret_key=url.password,
            options=dict(parse_qsl(url.query)),
            transport=transport,
        )
<|endoftext|>


<commit_before>import sys
import os

# ------------------------------------------------------------------------------
# User Authentication - Liftpass is a single user application (for now)
# ------------------------------------------------------------------------------
UserKey = b'$UserKey'
UserSecret = b'$UserSecret'

# ------------------------------------------------------------------------------
# Paths
# ------------------------------------------------------------------------------
BasePath = os.path.abspath('./')
sys.path.append(BasePath)

# Where application data is stored
DataPath = os.path.join(BasePath,'data/')

# ------------------------------------------------------------------------------
# Analytics Storage - Where SDK updates are stored before being processed
# ------------------------------------------------------------------------------
AnalyticsStorage = {
	'engine': 'core.storage.filesystem',
	'path': os.path.join(DataPath, 'analytics/')
}

# ------------------------------------------------------------------------------
# Content Database - Where application content and settings are stored
# ------------------------------------------------------------------------------
ContentDatabase = {
	'address': 'sqlite:///%s/content.db'%DataPath,
	'debug': False
}

# ------------------------------------------------------------------------------
# Monitoring  - records server activity and performance (not yet supported)
# ------------------------------------------------------------------------------
MonitorEngine = None

# ------------------------------------------------------------------------------
# Debug Terminal - caches user updates for debuging
# ------------------------------------------------------------------------------
DashboardTerminal = {
	'engine': 'core.terminal.local',
	'path': os.path.join(DataPath, 'terminal/')
}

# ------------------------------------------------------------------------------
# API Interface Service
# ------------------------------------------------------------------------------
APIServer = {
	'address': '127.0.0.1',
	'port': 9090,
	'debug': True,
	'cors': True
}
<commit_msg>Updated config file<commit_after>import sys
import os

# ------------------------------------------------------------------------------
# User Authentication - Liftpass is a single user application (for now)
# ------------------------------------------------------------------------------
UserKey = b'$UserKey'
UserSecret = b'$UserSecret'

# ------------------------------------------------------------------------------
# Paths
# ------------------------------------------------------------------------------
BasePath = os.path.abspath('./')
sys.path.append(BasePath)

# Where application data is stored
DataPath = os.path.join(BasePath,'data/')

# ------------------------------------------------------------------------------
# Analytics Storage - Where SDK updates are stored before being processed
# ------------------------------------------------------------------------------
AnalyticsStorage = {
	'engine': 'core.storage.filesystem',
	'path': os.path.join(DataPath, 'analytics/')
}

# ------------------------------------------------------------------------------
# Content Database - Where application content and settings are stored
# ------------------------------------------------------------------------------
ContentDatabase = {
	'address': 'sqlite:///%s/content.db'%DataPath,
	'debug': False
}

# ------------------------------------------------------------------------------
# Monitoring  - records server activity and performance (not yet supported)
# ------------------------------------------------------------------------------
MonitorEngine = None

# ------------------------------------------------------------------------------
# Debug Terminal - caches user updates for debuging
# ------------------------------------------------------------------------------
DashboardTerminal = {
	'engine': 'core.terminal.local',
	'path': os.path.join(DataPath, 'terminal/')
}

# ------------------------------------------------------------------------------
# API Interface Service
# ------------------------------------------------------------------------------
APIServer = {
	'address': '127.0.0.1',
	'port': 9090,
	'cors': True
}
<|endoftext|>


<commit_before>import time
import collections

import synapse.link as s_link
import synapse.async as s_async
import synapse.daemon as s_daemon
import synapse.eventbus as s_eventbus
import synapse.telepath as s_telepath

import synapse.lib.scope as s_scope
import synapse.lib.service as s_service
import synapse.lib.threads as s_threads

from synapse.tests.common import *
import logging
logger = logging.getLogger(__name__)

class Foo(s_eventbus.EventBus):

    def bar(self, x, y):
        return x + y

    def baz(self, x, y):
        raise ValueError('derp')

    def echo(self, x):
        return x

    def speed(self):
        return

    @s_telepath.clientside
    def localthing(self, x):
        return self.echo(x)

class TelePathTest(SynTest):

    def getFooServ(self):
        dmon = s_daemon.Daemon()

        link = dmon.listen('tcp://127.0.0.1:0/foo')
        dmon.share('foo', Foo())

        return dmon, link

    def getFooEnv(self, url='tcp://127.0.0.1:0/foo'):
        env = TstEnv()
        env.add('dmon', s_daemon.Daemon(), fini=True)
        env.add('link', env.dmon.listen(url))

        env.dmon.share('foo', Foo())

        return env

    def test_telepath_basics(self):

        env = self.getFooEnv()

        foo = s_telepath.openlink(env.link)

        self.true(s_telepath.isProxy(foo))
        self.false(s_telepath.isProxy(self))

        # Test magic methods
        self.true(bool(foo) is True)
        self.true(foo == foo)
        self.false(foo == 1)
        self.true(foo != 1)

        s = time.time()
        for i in range(1000):
            foo.speed()

        e = time.time()

        self.eq(foo.bar(10, 20), 30)
        self.raises(NoSuchMeth, foo.faz, 10, 20)
        self.raises(SynErr, foo.baz, 10, 20)

        foo.fini()
        env.fini()

    def test_telepath_chop(self):

        dmon, link = self.getFooServ()

        port = link[1].get('port')

        foo = s_telepath.openurl('tcp://localhost:%d/foo' % (port,))

        self.eq(foo.bar(10, 20), 30)

        foo.fini()
        dmon.fini()

    def test_telepath_nosuchobj(self):
        dmon, link = self.getFooServ()
        port = link[1].get('port')

        newp = s_telepath.openurl('tcp://localhost:%d/newp' % (port,))
        self.raises(SynErr, newp.foo)

        newp.fini()
        dmon.fini()

    def test_telepath_call(self):
        dmon, link = self.getFooServ()

        foo = s_telepath.openlink(link)

        job = foo.call('bar', 10, 20)
        self.nn(job)

        self.eq(foo.syncjob(job), 30)

        foo.fini()
        dmon.fini()

    def test_telepath_push(self):

        # override default timeout=None for tests
        with s_scope.enter({'syntimeout': 3}):

            env = self.getFooEnv()

            port = env.link[1].get('port')

            prox0 = s_telepath.openurl('tcp://127.0.0.1/', port=port)  # type: s_telepath.Proxy

            prox0.push('foo1', Foo())

            prox1 = s_telepath.openurl('tcp://127.0.0.1/foo1', port=port)  # type: s_telepath.Proxy

            self.eq(prox1.bar(10, 20), 30)

            wait0 = env.dmon.waiter(1, 'tele:push:fini')

            prox0.fini()

            self.nn(wait0.wait(timeout=2))

            self.raises(NoSuchObj, prox1.bar, 10, 20)

            prox1.fini()

            env.fini()

    def test_telepath_callx(self):

        class Baz:
            def faz(self, x, y=10):
                return '%d:%d' % (x, y)

        env = self.getFooEnv()
        env.dmon.share('baz', Baz())

        port = env.link[1].get('port')
        foo = s_telepath.openurl('tcp://127.0.0.1/foo', port=port)

        # make sure proxy is working normally...
        self.eq(foo.bar(10, 20), 30)

        # carry out a cross item task
        job = foo.callx('baz', ('faz', (30,), {'y': 40}), )

        self.eq(foo.syncjob(job), '30:40')
        foo.fini()

    def test_telepath_fakesync(self):
        env = self.getFooEnv()
        port = env.link[1].get('port')

        class DeadLock(s_eventbus.EventBus):

            def hork(self):
                self.fire('foo:bar')

            def bar(self, x, y):
                return x + y

        dead = DeadLock()
        env.dmon.share('dead', dead)

        data = {}
        evt = threading.Event()

        prox = s_telepath.openurl('tcp://127.0.0.1/dead', port=port)
        def foobar(mesg):
            data['foobar'] = prox.bar(10, 20)
            evt.set()

        prox.on('foo:bar', foobar)

        prox.hork()

        evt.wait(timeout=2)

        self.eq(data.get('foobar'), 30)

        prox.fini()
        dead.fini()
        env.fini()

    def test_telepath_reshare(self):
        env0 = self.getFooEnv()
        env1 = self.getFooEnv()

        port = env0.link[1].get('port')
        prox0 = s_telepath.openurl('tcp://127.0.0.1/foo', port=port)

        env1.dmon.share('bar', prox0)

        port = env1.link[1].get('port')
        prox1 = s_telepath.openurl('tcp://127.0.0.1/bar', port=port)

        self.eq(prox1.bar(33, 44), 77)

        prox0.fini()
        prox1.fini()

        env0.fini()
        env1.fini()

    def test_telepath_eval(self):

        foo = s_telepath.evalurl('ctor://synapse.tests.test_telepath.Foo()')
        self.eq(foo.bar(10, 20), 30)

        env0 = self.getFooEnv()
        port = env0.link[1].get('port')

        foo = s_telepath.evalurl('tcp://127.0.0.1/foo', port=port)
        self.eq(foo.bar(10, 20), 30)

        foo.fini()
        env0.fini()

    def test_telepath_reconnect(self):
        tenv = self.getFooEnv()

        port = tenv.link[1].get('port')
        prox = s_telepath.openurl('tcp://127.0.0.1/foo', port=port)

        url = 'tcp://127.0.0.1:%d/foo' % (port,)
        self.eq(prox.bar(10, 20), 30)

        data = {}
        def _onHehe(mesg):
            data['hehe'] = data.get('hehe', 0) + 1
            data['haha'] = mesg[1].get('haha')

        prox.on('hehe', _onHehe)
        prox.fire('hehe', haha=1)
        self.eq(data.get('hehe'), 1)
        self.eq(data.get('haha'), 1)

        waiter = self.getTestWait(prox, 1, 'tele:sock:init')
        # shut down the daemon
        tenv.dmon.fini()

        dmon = s_daemon.Daemon()
        dmon.share('foo', Foo())
        dmon.listen(url)

        waiter.wait()

        self.eq(prox.bar(10, 20), 30)

        prox.fire('hehe', haha=3)
        self.eq(data.get('hehe'), 2)
        self.eq(data.get('haha'), 3)

        prox.fini()
        dmon.fini()
        tenv.fini()

    def test_telepath_yielder(self):

        class YieldTest:

            def woot(self):
                yield 10
                yield 20
                yield 30

        dmon = s_daemon.Daemon()
        link = dmon.listen('tcp://127.0.0.1:0/hehe')

        dmon.share('hehe', YieldTest())

        prox = s_telepath.openlink(link)

        items = []

        for item in prox.woot():
            items.append(item)

        self.eq(tuple(items), (10, 20, 30))

        self.eq(len(dmon._dmon_yields), 0)

        prox.fini()
        dmon.fini()

    def test_telepath_server_badvers(self):

        dmon = s_daemon.Daemon()
        link = dmon.listen('tcp://127.0.0.1:0/')

        rlay = s_link.getLinkRelay(link)

        jid = guid()
        sock = rlay.connect()

        sock.tx(tufo('tele:syn', jid=jid, vers=(0, 0)))

        mesg = sock.recvobj()

        sock.fini()

        self.eq(mesg[0], 'job:done')
        self.eq(mesg[1].get('err'), 'BadMesgVers')

    def test_telepath_gzip(self):

        with s_daemon.Daemon() as dmon:

            link = dmon.listen('tcp://127.0.0.1:0/foo')

            dmon.share('foo', Foo())

            with s_telepath.openlink(link) as prox:

                wait0 = self.getTestWait(prox, 1, 'sock:gzip')
                #wait1 = self.getTestWait(dmon., 1, 'sock:gzip')

                ping = 'V' * 50000
                self.eq(prox.echo(ping), ping)

                wait0.wait(timeout=2)

    def test_telepath_events(self):
        with s_daemon.Daemon() as dmon:
            with s_service.SvcBus() as sbus:
                urlt = 'tcp://127.0.0.1:%d'
                url = urlt % (0)
                link = dmon.listen(url)
                port = link[1].get('port')
                url = urlt % (port)
                dmon.share('sbus', sbus)
                proxy0 = s_telepath.openurl(url + '/sbus')
                proxy1 = s_telepath.openurl(url + '/sbus')

                counters = collections.defaultdict(int)

                def count(name):
                    def onmesg(mesg):
                        counters[name] += 1
                    return onmesg

                proxy0.on('foo:bar', count('p0'))
                proxy1.on('foo:bar', count('p1'))

                func = count('f0')
                proxy1.on('foo:bar', func, tag='hehe')

                wait = proxy1.waiter(1, 'foo:bar')
                proxy0.fire('foo:bar', tag='tagu', tufo=('iden', {'prop': 'valu'}))
                self.nn(wait.wait(timeout=2))

                self.eq(counters['p0'], 1)
                self.eq(counters['p1'], 1)
                self.eq(counters['f0'], 0)

                wait = proxy1.waiter(1, 'foo:bar')
                proxy0.fire('foo:bar', tag='hehe', tufo=('iden', {'prop': 'valu'}))
                self.nn(wait.wait(timeout=2))

                self.eq(counters['p0'], 2)
                self.eq(counters['p1'], 2)
                self.eq(counters['f0'], 1)

                proxy1.off('foo:bar', func)

                wait = proxy1.waiter(1, 'foo:bar')
                proxy0.fire('foo:bar', tag='hehe', tufo=('iden', {'prop': 'valu'}))
                self.nn(wait.wait(timeout=2))

                self.eq(counters['p0'], 3)
                self.eq(counters['p1'], 3)
                self.eq(counters['f0'], 1)

                proxy0.fini()
                proxy1.fini()

    def test_telepath_clientside(self):

        with s_daemon.Daemon() as dmon:

            link = dmon.listen('tcp://127.0.0.1:0/')
            port = link[1].get('port')

            dmon.share('foo', Foo())

            with s_telepath.openurl('tcp://127.0.0.1/foo', port=port) as prox:
                self.eq(prox.localthing(20), 20)
                self.eq(prox.localthing(30), 30)

    def test_telepath_reqproxy(self):

        self.raises(MustBeProxy, s_telepath.reqIsProxy, 'woot')

        with s_daemon.Daemon() as dmon:

            dmon.share('foo', 'woot')

            link = dmon.listen('tcp://127.0.0.1:0/')
            port = link[1].get('port')

            with s_telepath.openurl('tcp://127.0.0.1/foo', port=port) as foo:
                self.raises(MustBeLocal, s_telepath.reqNotProxy, foo)

    def test_telepath_reminder(self):

        evnt = threading.Event()
        class Mind(s_eventbus.EventBus):

            def __init__(self):
                s_eventbus.EventBus.__init__(self)
                self.sent = []
                self.on('foo:bar', self._onFooBar)

            def _onFooBar(self, mesg):
                self.sent.append(mesg)
                evnt.set()

            def woot(self):
                s_telepath.reminder('foo:bar', name='hehe')
                return 10

        mind = Mind()

        with s_daemon.Daemon() as dmon:

            dmon.share('mind', mind)

            link = dmon.listen('tcp://127.0.0.1:0/')

            port = link[1].get('port')

            prox = s_telepath.openurl('tcp://127.0.0.1/mind', port=port)

            self.eq(prox.woot(), 10)

            # reach in and squish the socket...
            prox._tele_sock.fini()
            self.true(evnt.wait(timeout=1))
            self.eq(mind.sent[0][0], 'foo:bar')

            prox.fini()
<commit_msg>Ensure the Proxy's resources have been fini'd.<commit_after>import time
import collections

import synapse.link as s_link
import synapse.async as s_async
import synapse.daemon as s_daemon
import synapse.eventbus as s_eventbus
import synapse.telepath as s_telepath

import synapse.lib.scope as s_scope
import synapse.lib.service as s_service
import synapse.lib.threads as s_threads

from synapse.tests.common import *
import logging
logger = logging.getLogger(__name__)

class Foo(s_eventbus.EventBus):

    def bar(self, x, y):
        return x + y

    def baz(self, x, y):
        raise ValueError('derp')

    def echo(self, x):
        return x

    def speed(self):
        return

    @s_telepath.clientside
    def localthing(self, x):
        return self.echo(x)

class TelePathTest(SynTest):

    def getFooServ(self):
        dmon = s_daemon.Daemon()

        link = dmon.listen('tcp://127.0.0.1:0/foo')
        dmon.share('foo', Foo())

        return dmon, link

    def getFooEnv(self, url='tcp://127.0.0.1:0/foo'):
        env = TstEnv()
        env.add('dmon', s_daemon.Daemon(), fini=True)
        env.add('link', env.dmon.listen(url))

        env.dmon.share('foo', Foo())

        return env

    def test_telepath_basics(self):

        env = self.getFooEnv()

        foo = s_telepath.openlink(env.link)

        self.true(s_telepath.isProxy(foo))
        self.false(s_telepath.isProxy(self))

        # Test magic methods
        self.true(bool(foo) is True)
        self.true(foo == foo)
        self.false(foo == 1)
        self.true(foo != 1)

        s = time.time()
        for i in range(1000):
            foo.speed()

        e = time.time()

        self.eq(foo.bar(10, 20), 30)
        self.raises(NoSuchMeth, foo.faz, 10, 20)
        self.raises(SynErr, foo.baz, 10, 20)

        foo.fini()
        # We have fini'd the Proxy resources
        self.true(foo._tele_boss.isfini)
        self.true(foo._tele_sock.isfini)
        self.true(foo._tele_plex.isfini)

        env.fini()

    def test_telepath_chop(self):

        dmon, link = self.getFooServ()

        port = link[1].get('port')

        foo = s_telepath.openurl('tcp://localhost:%d/foo' % (port,))

        self.eq(foo.bar(10, 20), 30)

        foo.fini()
        dmon.fini()

    def test_telepath_nosuchobj(self):
        dmon, link = self.getFooServ()
        port = link[1].get('port')

        newp = s_telepath.openurl('tcp://localhost:%d/newp' % (port,))
        self.raises(SynErr, newp.foo)

        newp.fini()
        dmon.fini()

    def test_telepath_call(self):
        dmon, link = self.getFooServ()

        foo = s_telepath.openlink(link)

        job = foo.call('bar', 10, 20)
        self.nn(job)

        self.eq(foo.syncjob(job), 30)

        foo.fini()
        dmon.fini()

    def test_telepath_push(self):

        # override default timeout=None for tests
        with s_scope.enter({'syntimeout': 3}):

            env = self.getFooEnv()

            port = env.link[1].get('port')

            prox0 = s_telepath.openurl('tcp://127.0.0.1/', port=port)  # type: s_telepath.Proxy

            prox0.push('foo1', Foo())

            prox1 = s_telepath.openurl('tcp://127.0.0.1/foo1', port=port)  # type: s_telepath.Proxy

            self.eq(prox1.bar(10, 20), 30)

            wait0 = env.dmon.waiter(1, 'tele:push:fini')

            prox0.fini()

            self.nn(wait0.wait(timeout=2))

            self.raises(NoSuchObj, prox1.bar, 10, 20)

            prox1.fini()

            env.fini()

    def test_telepath_callx(self):

        class Baz:
            def faz(self, x, y=10):
                return '%d:%d' % (x, y)

        env = self.getFooEnv()
        env.dmon.share('baz', Baz())

        port = env.link[1].get('port')
        foo = s_telepath.openurl('tcp://127.0.0.1/foo', port=port)

        # make sure proxy is working normally...
        self.eq(foo.bar(10, 20), 30)

        # carry out a cross item task
        job = foo.callx('baz', ('faz', (30,), {'y': 40}), )

        self.eq(foo.syncjob(job), '30:40')
        foo.fini()

    def test_telepath_fakesync(self):
        env = self.getFooEnv()
        port = env.link[1].get('port')

        class DeadLock(s_eventbus.EventBus):

            def hork(self):
                self.fire('foo:bar')

            def bar(self, x, y):
                return x + y

        dead = DeadLock()
        env.dmon.share('dead', dead)

        data = {}
        evt = threading.Event()

        prox = s_telepath.openurl('tcp://127.0.0.1/dead', port=port)
        def foobar(mesg):
            data['foobar'] = prox.bar(10, 20)
            evt.set()

        prox.on('foo:bar', foobar)

        prox.hork()

        evt.wait(timeout=2)

        self.eq(data.get('foobar'), 30)

        prox.fini()
        dead.fini()
        env.fini()

    def test_telepath_reshare(self):
        env0 = self.getFooEnv()
        env1 = self.getFooEnv()

        port = env0.link[1].get('port')
        prox0 = s_telepath.openurl('tcp://127.0.0.1/foo', port=port)

        env1.dmon.share('bar', prox0)

        port = env1.link[1].get('port')
        prox1 = s_telepath.openurl('tcp://127.0.0.1/bar', port=port)

        self.eq(prox1.bar(33, 44), 77)

        prox0.fini()
        prox1.fini()

        env0.fini()
        env1.fini()

    def test_telepath_eval(self):

        foo = s_telepath.evalurl('ctor://synapse.tests.test_telepath.Foo()')
        self.eq(foo.bar(10, 20), 30)

        env0 = self.getFooEnv()
        port = env0.link[1].get('port')

        foo = s_telepath.evalurl('tcp://127.0.0.1/foo', port=port)
        self.eq(foo.bar(10, 20), 30)

        foo.fini()
        env0.fini()

    def test_telepath_reconnect(self):
        tenv = self.getFooEnv()

        port = tenv.link[1].get('port')
        prox = s_telepath.openurl('tcp://127.0.0.1/foo', port=port)

        url = 'tcp://127.0.0.1:%d/foo' % (port,)
        self.eq(prox.bar(10, 20), 30)

        data = {}
        def _onHehe(mesg):
            data['hehe'] = data.get('hehe', 0) + 1
            data['haha'] = mesg[1].get('haha')

        prox.on('hehe', _onHehe)
        prox.fire('hehe', haha=1)
        self.eq(data.get('hehe'), 1)
        self.eq(data.get('haha'), 1)

        waiter = self.getTestWait(prox, 1, 'tele:sock:init')
        # shut down the daemon
        tenv.dmon.fini()

        dmon = s_daemon.Daemon()
        dmon.share('foo', Foo())
        dmon.listen(url)

        waiter.wait()

        self.eq(prox.bar(10, 20), 30)

        prox.fire('hehe', haha=3)
        self.eq(data.get('hehe'), 2)
        self.eq(data.get('haha'), 3)

        prox.fini()
        dmon.fini()
        tenv.fini()

    def test_telepath_yielder(self):

        class YieldTest:

            def woot(self):
                yield 10
                yield 20
                yield 30

        dmon = s_daemon.Daemon()
        link = dmon.listen('tcp://127.0.0.1:0/hehe')

        dmon.share('hehe', YieldTest())

        prox = s_telepath.openlink(link)

        items = []

        for item in prox.woot():
            items.append(item)

        self.eq(tuple(items), (10, 20, 30))

        self.eq(len(dmon._dmon_yields), 0)

        prox.fini()
        dmon.fini()

    def test_telepath_server_badvers(self):

        dmon = s_daemon.Daemon()
        link = dmon.listen('tcp://127.0.0.1:0/')

        rlay = s_link.getLinkRelay(link)

        jid = guid()
        sock = rlay.connect()

        sock.tx(tufo('tele:syn', jid=jid, vers=(0, 0)))

        mesg = sock.recvobj()

        sock.fini()

        self.eq(mesg[0], 'job:done')
        self.eq(mesg[1].get('err'), 'BadMesgVers')

    def test_telepath_gzip(self):

        with s_daemon.Daemon() as dmon:

            link = dmon.listen('tcp://127.0.0.1:0/foo')

            dmon.share('foo', Foo())

            with s_telepath.openlink(link) as prox:

                wait0 = self.getTestWait(prox, 1, 'sock:gzip')
                #wait1 = self.getTestWait(dmon., 1, 'sock:gzip')

                ping = 'V' * 50000
                self.eq(prox.echo(ping), ping)

                wait0.wait(timeout=2)

    def test_telepath_events(self):
        with s_daemon.Daemon() as dmon:
            with s_service.SvcBus() as sbus:
                urlt = 'tcp://127.0.0.1:%d'
                url = urlt % (0)
                link = dmon.listen(url)
                port = link[1].get('port')
                url = urlt % (port)
                dmon.share('sbus', sbus)
                proxy0 = s_telepath.openurl(url + '/sbus')
                proxy1 = s_telepath.openurl(url + '/sbus')

                counters = collections.defaultdict(int)

                def count(name):
                    def onmesg(mesg):
                        counters[name] += 1
                    return onmesg

                proxy0.on('foo:bar', count('p0'))
                proxy1.on('foo:bar', count('p1'))

                func = count('f0')
                proxy1.on('foo:bar', func, tag='hehe')

                wait = proxy1.waiter(1, 'foo:bar')
                proxy0.fire('foo:bar', tag='tagu', tufo=('iden', {'prop': 'valu'}))
                self.nn(wait.wait(timeout=2))

                self.eq(counters['p0'], 1)
                self.eq(counters['p1'], 1)
                self.eq(counters['f0'], 0)

                wait = proxy1.waiter(1, 'foo:bar')
                proxy0.fire('foo:bar', tag='hehe', tufo=('iden', {'prop': 'valu'}))
                self.nn(wait.wait(timeout=2))

                self.eq(counters['p0'], 2)
                self.eq(counters['p1'], 2)
                self.eq(counters['f0'], 1)

                proxy1.off('foo:bar', func)

                wait = proxy1.waiter(1, 'foo:bar')
                proxy0.fire('foo:bar', tag='hehe', tufo=('iden', {'prop': 'valu'}))
                self.nn(wait.wait(timeout=2))

                self.eq(counters['p0'], 3)
                self.eq(counters['p1'], 3)
                self.eq(counters['f0'], 1)

                proxy0.fini()
                proxy1.fini()

    def test_telepath_clientside(self):

        with s_daemon.Daemon() as dmon:

            link = dmon.listen('tcp://127.0.0.1:0/')
            port = link[1].get('port')

            dmon.share('foo', Foo())

            with s_telepath.openurl('tcp://127.0.0.1/foo', port=port) as prox:
                self.eq(prox.localthing(20), 20)
                self.eq(prox.localthing(30), 30)

    def test_telepath_reqproxy(self):

        self.raises(MustBeProxy, s_telepath.reqIsProxy, 'woot')

        with s_daemon.Daemon() as dmon:

            dmon.share('foo', 'woot')

            link = dmon.listen('tcp://127.0.0.1:0/')
            port = link[1].get('port')

            with s_telepath.openurl('tcp://127.0.0.1/foo', port=port) as foo:
                self.raises(MustBeLocal, s_telepath.reqNotProxy, foo)

    def test_telepath_reminder(self):

        evnt = threading.Event()
        class Mind(s_eventbus.EventBus):

            def __init__(self):
                s_eventbus.EventBus.__init__(self)
                self.sent = []
                self.on('foo:bar', self._onFooBar)

            def _onFooBar(self, mesg):
                self.sent.append(mesg)
                evnt.set()

            def woot(self):
                s_telepath.reminder('foo:bar', name='hehe')
                return 10

        mind = Mind()

        with s_daemon.Daemon() as dmon:

            dmon.share('mind', mind)

            link = dmon.listen('tcp://127.0.0.1:0/')

            port = link[1].get('port')

            prox = s_telepath.openurl('tcp://127.0.0.1/mind', port=port)

            self.eq(prox.woot(), 10)

            # reach in and squish the socket...
            prox._tele_sock.fini()
            self.true(evnt.wait(timeout=1))
            self.eq(mind.sent[0][0], 'foo:bar')

            prox.fini()
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

from VtkRenderer import *
from LidarTransforms import *
import numpy as np
import sys
import os
from VideoReader import VideoReader
import cv2
import cv

global count

class LDRGrabberCallback:

    def __init__(self):
        pass

    def execute(self, iren, event):
        global count
        t = time.time()

        (success, img) = reader.getNextFrame()
        if success == False:
            return

        (frame, ldr_file) = map_file.readline().rstrip().split(' ')
        if count % 3 == 0:
            count += 1
            return
        ldr_file = sys.argv[3] + '/' + ldr_file

        renderer = iren.GetRenderWindow().GetRenderers().GetFirstRenderer()

        pts = loadLDR(ldr_file)

        count += 1
        self.pointCloud = VtkPointCloud(pts[:, 0:3], pts[:, 3])

        if count > 2:
            renderer.RemoveActor(self.actor)

        self.actor = self.pointCloud.get_vtk_cloud(zMin=0, zMax=255)
        renderer.AddActor(self.actor)

        if count == 2:
            renderer.ResetCamera()

        iren.GetRenderWindow().Render()

        img = cv2.resize(img, (640, 480))
        cv2.imshow('display', img)
        cv2.waitKey(1)

        print time.time() - t

if __name__ == '__main__':
    reader = VideoReader(sys.argv[1])
    count = 1
    map_file = open(sys.argv[2], 'r')

    renderer = vtk.vtkRenderer()
    renderer.SetBackground(0., 0., 0.)
    renderer.ResetCamera()

    # Render Window
    renderWindow = vtk.vtkRenderWindow()
    renderWindow.AddRenderer(renderer)
    renderWindow.SetSize(640, 480)

    # Interactor
    renderWindowInteractor = vtk.vtkRenderWindowInteractor()
    renderWindowInteractor.SetRenderWindow(renderWindow)
    mouseInteractor = vtk.vtkInteractorStyleTrackballCamera()
    renderWindowInteractor.SetInteractorStyle(mouseInteractor)
    renderWindow.Render()

    cb = LDRGrabberCallback()
    renderWindowInteractor.AddObserver('TimerEvent', cb.execute)
    timerId = renderWindowInteractor.CreateRepeatingTimer(10)
    renderWindowInteractor.Start()

<commit_msg>File loading and processing done in a separate thread<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import cv2
import numpy as np
import os
import Queue
import sys
import threading
from LidarTransforms import *
from VideoReader import VideoReader
from VtkRenderer import *

global count

class LDRGrabberCallback:

    def __init__(self, queue_manager):
        self.queue = queue_manager.queue
        self.manager = queue_manager
        bg_thread = threading.Thread(target=queue_manager.loadNext)
        bg_thread.daemon = True
        bg_thread.start()

    def execute(self, iren, event):
        try:

            frame_cloud = self.queue.get()
            pts = frame_cloud['pts']
            img = frame_cloud['img']

            global count
            count += 1

            self.pointCloud = VtkPointCloud(pts[:, 0:3], pts[:, 3])

            t = time.time()
            if count > 2:
                renderer.RemoveActor(self.actor)
            self.actor = self.pointCloud.get_vtk_cloud(zMin=0, zMax=255)
            renderer.AddActor(self.actor)

            # Initially set the camera frame

            if count == 2:
                renderer.ResetCamera()
            iren.GetRenderWindow().Render()

            cv2.imshow('display', img)
            cv2.waitKey(1)

            print "Display:",  time.time() - t
        except Queue.Empty:
            if self.manager.finished == True:
                return

class FrameCloudManager:

    def __init__(self, video_file, frame_folder, map_file):
        self.reader = VideoReader(video_file)
        self.map_file = open(map_file, 'r')
        self.frame_folder = frame_folder
        self.queue = Queue.Queue()
        self.finished = False

    def loadNext(self):
        while self.finished == False:
            t = time.time()
            (success, img) = self.reader.getNextFrame()
            if success == False:
                self.finished = True
                return

            img = cv2.resize(img, (640, 480))

            (frame, ldr_file) = \
                self.map_file.readline().rstrip().split(' ')
            ldr_file = self.frame_folder + '/' + ldr_file
            pts = loadLDR(ldr_file)

            self.queue.put({'img': img, 'pts': pts})
            print "Load:", time.time() - t

if __name__ == '__main__':
    if len(sys.argv) < 4:
        print """
        Usage:
            testPointCloudFrameCompare.py <video file>.avi <frame folder>/ <map_file>.out
        """
    else:

        frame_cloud_manager = FrameCloudManager(sys.argv[1], sys.argv[2],
                sys.argv[3])
        global count
        count = 1

        renderer = vtk.vtkRenderer()
        renderer.SetBackground(0., 0., 0.)
        renderer.ResetCamera()

        # Render Window
        renderWindow = vtk.vtkRenderWindow()
        renderWindow.AddRenderer(renderer)
        renderWindow.SetSize(640, 480)

        # Interactor
        renderWindowInteractor = vtk.vtkRenderWindowInteractor()
        renderWindowInteractor.SetRenderWindow(renderWindow)
        mouseInteractor = vtk.vtkInteractorStyleTrackballCamera()
        renderWindowInteractor.SetInteractorStyle(mouseInteractor)
        renderWindow.Render()

        cb = LDRGrabberCallback(frame_cloud_manager)
        renderWindowInteractor.AddObserver('TimerEvent', cb.execute)
        timerId = renderWindowInteractor.CreateRepeatingTimer(10)
        renderWindowInteractor.Start()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os, re, time
from json import dump as write_json
from twisted.internet.defer import inlineCallbacks, returnValue
from datetime import datetime, timedelta
from gazouilleur import config
from gazouilleur.lib.mongo import SingleMongo, find_stats, count_followers, find_last_followers, sortasc, sortdesc
from gazouilleur.lib.log import loggerr
from gazouilleur.lib.utils import *

class Stats(object):

    def __init__(self, user):
        self.user = user
        try:
            self.url = '%s/' % config.URL_STATS.rstrip('/')
        except:
            self.url = None
        self.templates = os.path.join("web", "templates")

    @inlineCallbacks
    def print_last(self):
        now = timestamp_hour(datetime.today())
        since = now - timedelta(days=30)
        stats = yield find_stats({'user': self.user, 'timestamp': {'$gte': since}}, filter=sortdesc('timestamp'))
        if not len(stats):
            returnValue()
        stat = stats[0]
        stat["followers"] = yield count_followers(self.user)
        rts = 0
        fols = 0
        twts = 0
        delays = {1: 'hour', 6: '6 hours', 24: 'day', 7*24: 'week', 30*24: 'month'}
        order = delays.keys()
        order.sort()
        olds = {'tweets': {}, 'followers': {}, 'rts': {}}
        for s in stats:
            d = now - s['timestamp']
            delay = d.seconds / 3600 + d.days * 24
            fols = stat['followers'] - s['followers']
            twts = stat['tweets'] - s['tweets']
            for i in order:
                if delay == i:
                    if 'stats%sH' % i not in olds['tweets']:
                        olds['tweets']['stats%sH' % i] = twts if twts not in olds['tweets'].values() else 0
                    if 'stats%sH' % i not in olds['followers']:
                        olds['followers']['stats%sH' % i] = fols if fols not in olds['followers'].values() else 0
                    if 'stats%sH' % i not in olds['rts']:
                        olds['rts']['stats%sH' % i] = rts if rts not in olds['rts'].values() else 0
            rts += s['rts_last_hour']
        olds['rts']['stats1H'] = stat['rts_last_hour']
        for i in order:
            if rts and 'stats%sH' % i not in olds['rts'] and rts not in olds['rts'].values():
                olds['rts']['stats%sH' % i] = rts
                rts = 0
            if fols and 'stats%sH' % i not in olds['followers']  and fols not in olds['followers'].values():
                olds['followers']['stats%sH' % i] = fols
                fols = 0
            if twts and 'stats%sH' % i not in olds['tweets'] and twts not in olds['tweets'].values():
                olds['tweets']['stats%sH' % i] = twts
                twts = 0
        res = []
        if stat['tweets']:
            res.append("Tweets: %d total" % stat['tweets'] + " ; ".join([""]+["%d last %s" %  (olds['tweets']['stats%sH' % i], delays[i]) for i in order if 'stats%sH' % i in olds['tweets'] and olds['tweets']['stats%sH' % i]]))
        textrts = ["%d last %s" % (olds['rts']['stats%sH' % i], delays[i]) for i in order if 'stats%sH' % i in olds['rts'] and olds['rts']['stats%sH' % i]]
        if textrts:
            res.append("RTs: " + " ; ".join(textrts))
        if stat['followers']:
            res.append("Followers: %d total" % stat['followers'] + " ; ".join([""]+["%+d last %s" % (olds['followers']['stats%sH' % i], delays[i]) for i in order if 'stats%sH' % i in olds['followers'] and olds['followers']['stats%sH' % i]]))
            recent = yield find_last_followers(self.user)
            if recent:
                res.append("Recent follower%s: %s" % ("s include" if len(recent) > 1 else "", format_4_followers(recent)))
        if self.url and res:
            res.append("More details: %sstatic_stats_%s.html" % (self.url, self.user))
        returnValue([(True, "[Stats] %s" % m) for m in res])

    @inlineCallbacks
    def dump_data(self):
        if not self.url:
            returnValue(False)
        stats = yield find_stats({'user': self.user}, filter=sortasc('timestamp'), timeout=120)
        dates = [s['timestamp'] for s in stats]
        tweets = [s['tweets'] for s in stats]
        tweets_diff = [a - b for a, b in zip(tweets[1:],tweets[:-1])]
        followers = [s['followers'] for s in stats]
        followers_diff = [a - b for a, b in zip(followers[1:], followers[:-1])]
        rts_diff = [s['rts_last_hour'] for s in stats]
        rts = []
        n = 0
        for a in rts_diff:
            n += a
            rts.append(n)

        jsondata = {}
        imax = len(dates) - 1
        for i, date in enumerate(dates):
            ts = int(time.mktime(date.timetuple()))
            jsondata[ts] = { 'tweets': tweets[i], 'followers': followers[i], 'rts': rts[i] }
            if i < imax:
                jsondata[ts].update({ 'tweets_diff': tweets_diff[i], 'followers_diff': followers_diff[i], 'rts_diff': rts_diff[i+1] })

        try:
            jsondir = os.path.join('web', 'data')
            if not os.path.exists(jsondir):
                os.makedirs(jsondir)
            with open(os.path.join(jsondir, 'stats_%s.json' % self.user), 'w') as outfile:
                write_json(jsondata, outfile)
        except IOError as e:
            loggerr("Could not write web/data/stats_%s.json : %s" % (self.user, e), action="stats")

        try:
            from plots import CumulativeCurve, DailyHistogram, WeekPunchCard
            imgdir = os.path.join('web', 'img')
            if not os.path.exists(imgdir):
                os.makedirs(imgdir)
            CumulativeCurve(dates, tweets, 'Total tweets', imgdir, 'tweets_%s' % self.user)
            CumulativeCurve(dates, followers, 'Total followers', imgdir, 'followers_%s' % self.user)
            CumulativeCurve(dates, rts, 'Total RTs since %s' % dates[0], imgdir, 'rts_%s' % self.user)
            DailyHistogram(dates[:-1], tweets_diff, 'New tweets', imgdir, 'new_tweets_%s' % self.user)
            DailyHistogram(dates[:-1], followers_diff, 'New followers', imgdir, 'new_followers_%s' % self.user)
            DailyHistogram(dates[:-1], rts_diff[1:], 'New RTs', imgdir, 'new_rts_%s' % self.user)
            WeekPunchCard(dates[:-1], tweets_diff, 'Tweets punchcard', imgdir, 'tweets_card_%s' % self.user)
            WeekPunchCard(dates[:-1], followers_diff, 'Followers punchcard', imgdir, 'followers_card_%s' % self.user)
            WeekPunchCard(dates[:-1], rts_diff[1:], 'RTs punchcard', imgdir, 'rts_card_%s' % self.user)
        except Exception as e:
            loggerr("Could not write images in web/img for %s : %s" % (self.user, e), action="stats")

        data = {'user': self.user, 'url': self.url}
        self.render_template("static_stats.html", self.user, data)
        returnValue(True)

    def render_template(self, template, name, data):
        outfile = template.replace('.html', '_%s.html' % name)
        try:
            import codecs
            import pystache
            from contextlib import nested
            ofile = os.path.join("web", outfile)
            with nested(open(os.path.join(self.templates, template), "r"), codecs.open(ofile, "w", encoding="utf-8")) as (temp, generated):
                generated.write(pystache.Renderer(string_encoding='utf8').render(temp.read(), data))
            os.chmod(ofile, 0o644)
            return True
        except IOError as e:
            loggerr("Could not write web/%s from %s/%s : %s" % (outfile, self.templates, template, e), action="stats")
            return False

    @inlineCallbacks
    def digest(self, hours, channel):
        now = datetime.today()
        since = now - timedelta(hours=hours)
        re_chan = re.compile(r'^#*%s$' % channel.lower(), re.I)
        query = {'channel': re_chan, 'timestamp': {'$gte': since}}
        data = {
            "channel": channel,
            "t0": clean_date(since),
            "t1": clean_date(now),
            "news": [],
            "imgs": [],
            "tweets": []
        }

        news = yield SingleMongo('news', 'find', query, fields=['sourcename', 'source', 'link', 'message'], filter=sortasc('sourcename')+sortasc('timestamp'))
        lastsource = ""
        for n in news:
            source = n["sourcename"]
            if source != lastsource:
                lastsource = source
                data["news"].append({
                    "name": source,
                    "link": n["link"],
                    "elements": []
                })
            data["news"][-1]["elements"].append({
                "text": n["message"],
                "link": n["link"]
            })
        del(news)

        tweets = yield SingleMongo('tweets', 'find', query, fields=['screenname', 'message', 'link'], filter=sortasc('timestamp'))
        links = {}
        imgs = {}
        filters = yield SingleMongo('filters', 'find', {'channel': re_chan}, fields=['keyword'])
        filters = [keyword['keyword'].lower() for keyword in filters]
        for t in tweets:
            skip = False
            tuser_low = t['screenname'].lower()
            if "@%s" % tuser_low in filters:
                continue
            msg_low = t["message"].lower()
            if not ((self.user and self.user in msg_low) or self.user == tuser_low):
                for k in filters:
                    if k in msg_low:
                        skip = True
                        break
            if skip: continue
            for link in URL_REGEX.findall(t["message"]):
                link, _ = clean_url(link[2])
                if not link.startswith("http"):
                    continue
                tid = re_twitmedia.search(link)
                if tid:
                    tid = tid.group(1)
                    if tid not in imgs:
                        imgs[tid] = 1
                        data["imgs"].append({"id": tid})
                    continue
                if re_tweet.match(link):
                    continue
                if link not in links:
                    links[link] = {
                        "link": link,
                        "first": ("%s: %s" % (t["screenname"], t["message"].replace(link, ""))),
                        "firstlink": t["link"],
                        "count": 0
                    }
                links[link]["count"] += 1

        del(tweets)
        data["tweets"] = sorted(links.values(), key=lambda x: "%06d-%s" % (10**6-x['count'], x['link']))
        del(links)

        filename = "%s_%s_%s" % (channel.lstrip("#"), data["t0"].replace(" ", "+"), data["t1"].replace(" ", "+"))
        if not self.render_template("digest.html", filename, data):
            returnValue("Wooops could not generate html for %s..." % filename)
        returnValue("Digest for the last %s hours available at %sdigest_%s.html" % (hours, self.url, filename))

re_tweet = re.compile(r'https?://twitter\.com/\S+/statuse?s?/\d+$')
re_twitmedia = re.compile(r'^https?://twitter\.com/\S+/statuse?s?/(\d+)/(photo|video)/\d+$')
def clean_date(d):
    d = d.isoformat()
    return d[:d.find(":", 15)].replace("T", " ")
<commit_msg>better sorting<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os, re, time
from json import dump as write_json
from twisted.internet.defer import inlineCallbacks, returnValue
from datetime import datetime, timedelta
from gazouilleur import config
from gazouilleur.lib.mongo import SingleMongo, find_stats, count_followers, find_last_followers, sortasc, sortdesc
from gazouilleur.lib.log import loggerr
from gazouilleur.lib.utils import *

class Stats(object):

    def __init__(self, user):
        self.user = user
        try:
            self.url = '%s/' % config.URL_STATS.rstrip('/')
        except:
            self.url = None
        self.templates = os.path.join("web", "templates")

    @inlineCallbacks
    def print_last(self):
        now = timestamp_hour(datetime.today())
        since = now - timedelta(days=30)
        stats = yield find_stats({'user': self.user, 'timestamp': {'$gte': since}}, filter=sortdesc('timestamp'))
        if not len(stats):
            returnValue()
        stat = stats[0]
        stat["followers"] = yield count_followers(self.user)
        rts = 0
        fols = 0
        twts = 0
        delays = {1: 'hour', 6: '6 hours', 24: 'day', 7*24: 'week', 30*24: 'month'}
        order = delays.keys()
        order.sort()
        olds = {'tweets': {}, 'followers': {}, 'rts': {}}
        for s in stats:
            d = now - s['timestamp']
            delay = d.seconds / 3600 + d.days * 24
            fols = stat['followers'] - s['followers']
            twts = stat['tweets'] - s['tweets']
            for i in order:
                if delay == i:
                    if 'stats%sH' % i not in olds['tweets']:
                        olds['tweets']['stats%sH' % i] = twts if twts not in olds['tweets'].values() else 0
                    if 'stats%sH' % i not in olds['followers']:
                        olds['followers']['stats%sH' % i] = fols if fols not in olds['followers'].values() else 0
                    if 'stats%sH' % i not in olds['rts']:
                        olds['rts']['stats%sH' % i] = rts if rts not in olds['rts'].values() else 0
            rts += s['rts_last_hour']
        olds['rts']['stats1H'] = stat['rts_last_hour']
        for i in order:
            if rts and 'stats%sH' % i not in olds['rts'] and rts not in olds['rts'].values():
                olds['rts']['stats%sH' % i] = rts
                rts = 0
            if fols and 'stats%sH' % i not in olds['followers']  and fols not in olds['followers'].values():
                olds['followers']['stats%sH' % i] = fols
                fols = 0
            if twts and 'stats%sH' % i not in olds['tweets'] and twts not in olds['tweets'].values():
                olds['tweets']['stats%sH' % i] = twts
                twts = 0
        res = []
        if stat['tweets']:
            res.append("Tweets: %d total" % stat['tweets'] + " ; ".join([""]+["%d last %s" %  (olds['tweets']['stats%sH' % i], delays[i]) for i in order if 'stats%sH' % i in olds['tweets'] and olds['tweets']['stats%sH' % i]]))
        textrts = ["%d last %s" % (olds['rts']['stats%sH' % i], delays[i]) for i in order if 'stats%sH' % i in olds['rts'] and olds['rts']['stats%sH' % i]]
        if textrts:
            res.append("RTs: " + " ; ".join(textrts))
        if stat['followers']:
            res.append("Followers: %d total" % stat['followers'] + " ; ".join([""]+["%+d last %s" % (olds['followers']['stats%sH' % i], delays[i]) for i in order if 'stats%sH' % i in olds['followers'] and olds['followers']['stats%sH' % i]]))
            recent = yield find_last_followers(self.user)
            if recent:
                res.append("Recent follower%s: %s" % ("s include" if len(recent) > 1 else "", format_4_followers(recent)))
        if self.url and res:
            res.append("More details: %sstatic_stats_%s.html" % (self.url, self.user))
        returnValue([(True, "[Stats] %s" % m) for m in res])

    @inlineCallbacks
    def dump_data(self):
        if not self.url:
            returnValue(False)
        stats = yield find_stats({'user': self.user}, filter=sortasc('timestamp'), timeout=120)
        dates = [s['timestamp'] for s in stats]
        tweets = [s['tweets'] for s in stats]
        tweets_diff = [a - b for a, b in zip(tweets[1:],tweets[:-1])]
        followers = [s['followers'] for s in stats]
        followers_diff = [a - b for a, b in zip(followers[1:], followers[:-1])]
        rts_diff = [s['rts_last_hour'] for s in stats]
        rts = []
        n = 0
        for a in rts_diff:
            n += a
            rts.append(n)

        jsondata = {}
        imax = len(dates) - 1
        for i, date in enumerate(dates):
            ts = int(time.mktime(date.timetuple()))
            jsondata[ts] = { 'tweets': tweets[i], 'followers': followers[i], 'rts': rts[i] }
            if i < imax:
                jsondata[ts].update({ 'tweets_diff': tweets_diff[i], 'followers_diff': followers_diff[i], 'rts_diff': rts_diff[i+1] })

        try:
            jsondir = os.path.join('web', 'data')
            if not os.path.exists(jsondir):
                os.makedirs(jsondir)
            with open(os.path.join(jsondir, 'stats_%s.json' % self.user), 'w') as outfile:
                write_json(jsondata, outfile)
        except IOError as e:
            loggerr("Could not write web/data/stats_%s.json : %s" % (self.user, e), action="stats")

        try:
            from plots import CumulativeCurve, DailyHistogram, WeekPunchCard
            imgdir = os.path.join('web', 'img')
            if not os.path.exists(imgdir):
                os.makedirs(imgdir)
            CumulativeCurve(dates, tweets, 'Total tweets', imgdir, 'tweets_%s' % self.user)
            CumulativeCurve(dates, followers, 'Total followers', imgdir, 'followers_%s' % self.user)
            CumulativeCurve(dates, rts, 'Total RTs since %s' % dates[0], imgdir, 'rts_%s' % self.user)
            DailyHistogram(dates[:-1], tweets_diff, 'New tweets', imgdir, 'new_tweets_%s' % self.user)
            DailyHistogram(dates[:-1], followers_diff, 'New followers', imgdir, 'new_followers_%s' % self.user)
            DailyHistogram(dates[:-1], rts_diff[1:], 'New RTs', imgdir, 'new_rts_%s' % self.user)
            WeekPunchCard(dates[:-1], tweets_diff, 'Tweets punchcard', imgdir, 'tweets_card_%s' % self.user)
            WeekPunchCard(dates[:-1], followers_diff, 'Followers punchcard', imgdir, 'followers_card_%s' % self.user)
            WeekPunchCard(dates[:-1], rts_diff[1:], 'RTs punchcard', imgdir, 'rts_card_%s' % self.user)
        except Exception as e:
            loggerr("Could not write images in web/img for %s : %s" % (self.user, e), action="stats")

        data = {'user': self.user, 'url': self.url}
        self.render_template("static_stats.html", self.user, data)
        returnValue(True)

    def render_template(self, template, name, data):
        outfile = template.replace('.html', '_%s.html' % name)
        try:
            import codecs
            import pystache
            from contextlib import nested
            ofile = os.path.join("web", outfile)
            with nested(open(os.path.join(self.templates, template), "r"), codecs.open(ofile, "w", encoding="utf-8")) as (temp, generated):
                generated.write(pystache.Renderer(string_encoding='utf8').render(temp.read(), data))
            os.chmod(ofile, 0o644)
            return True
        except IOError as e:
            loggerr("Could not write web/%s from %s/%s : %s" % (outfile, self.templates, template, e), action="stats")
            return False

    @inlineCallbacks
    def digest(self, hours, channel):
        now = datetime.today()
        since = now - timedelta(hours=hours)
        re_chan = re.compile(r'^#*%s$' % channel.lower(), re.I)
        query = {'channel': re_chan, 'timestamp': {'$gte': since}}
        data = {
            "channel": channel,
            "t0": clean_date(since),
            "t1": clean_date(now),
            "news": [],
            "imgs": [],
            "tweets": []
        }

        news = yield SingleMongo('news', 'find', query, fields=['sourcename', 'source', 'link', 'message'], filter=sortasc('sourcename')+sortasc('timestamp'))
        lastsource = ""
        for n in news:
            source = n["sourcename"]
            if source != lastsource:
                lastsource = source
                data["news"].append({
                    "name": source,
                    "link": n["link"],
                    "elements": []
                })
            data["news"][-1]["elements"].append({
                "text": n["message"],
                "link": n["link"]
            })
        del(news)

        tweets = yield SingleMongo('tweets', 'find', query, fields=['screenname', 'message', 'link'], filter=sortasc('id'))
        links = {}
        imgs = {}
        filters = yield SingleMongo('filters', 'find', {'channel': re_chan}, fields=['keyword'])
        filters = [keyword['keyword'].lower() for keyword in filters]
        for t in tweets:
            skip = False
            tuser_low = t['screenname'].lower()
            if "@%s" % tuser_low in filters:
                continue
            msg_low = t["message"].lower()
            if not ((self.user and self.user in msg_low) or self.user == tuser_low):
                for k in filters:
                    if k in msg_low:
                        skip = True
                        break
            if skip: continue
            for link in URL_REGEX.findall(t["message"]):
                link, _ = clean_url(link[2])
                if not link.startswith("http"):
                    continue
                tid = re_twitmedia.search(link)
                if tid:
                    tid = tid.group(1)
                    if tid not in imgs:
                        imgs[tid] = 1
                        data["imgs"].append({"id": tid})
                    continue
                if re_tweet.match(link):
                    continue
                if link not in links:
                    links[link] = {
                        "link": link,
                        "first": ("%s: %s" % (t["screenname"], t["message"].replace(link, ""))),
                        "firstlink": t["link"],
                        "count": 0
                    }
                links[link]["count"] += 1

        del(tweets)
        data["tweets"] = sorted(links.values(), key=lambda x: "%06d-%s" % (10**6-x['count'], x['link']))
        del(links)

        filename = "%s_%s_%s" % (channel.lstrip("#"), data["t0"].replace(" ", "+"), data["t1"].replace(" ", "+"))
        if not self.render_template("digest.html", filename, data):
            returnValue("Wooops could not generate html for %s..." % filename)
        returnValue("Digest for the last %s hours available at %sdigest_%s.html" % (hours, self.url, filename))

re_tweet = re.compile(r'https?://twitter\.com/\S+/statuse?s?/\d+$')
re_twitmedia = re.compile(r'^https?://twitter\.com/\S+/statuse?s?/(\d+)/(photo|video)/\d+$')
def clean_date(d):
    d = d.isoformat()
    return d[:d.find(":", 15)].replace("T", " ")
<|endoftext|>