<commit_before>################################################################################
import cherrypy
import filewrapper
import libtorrent
import math
import time
import utils

################################################################################
class DownloaderMonitor(cherrypy.process.plugins.Monitor):
    ############################################################################
    def __init__(self, bus, http_config, torrent_config):
        cherrypy.process.plugins.Monitor.__init__(self, bus, self._background_task, frequency=1)

        self.http_config     = http_config
        self.torrent_config  = torrent_config
        self.session         = None
        self.connections     = []
        self.monitor_running = False
        self.torrent_handles = []
        self.last_cleanup    = time.time()

    ############################################################################
    def start(self):
        cherrypy.process.plugins.Monitor.start(self)

        self.bus.log('[Downloader] Starting session')
        self.session = libtorrent.session()
        self.session.set_alert_mask(libtorrent.alert.category_t.error_notification | libtorrent.alert.category_t.status_notification | libtorrent.alert.category_t.storage_notification)
        self.session.start_dht()
        self.session.start_lsd()
        self.session.start_upnp()
        self.session.start_natpmp()
        self.session.listen_on(self.torrent_config['port'], self.torrent_config['port'])

        # Session settings
        session_settings = self.session.settings()
        session_settings.announce_to_all_tiers = True
        session_settings.announce_to_all_trackers = True
        session_settings.connection_speed = 100
        session_settings.peer_connect_timeout = 2
        session_settings.rate_limit_ip_overhead = True
        session_settings.request_timeout = 5
        session_settings.torrent_connect_boost = 100

        if self.torrent_config['max_download_rate'] > 0:
            session_settings.download_rate_limit = self.torrent_config['max_download_rate'] * 1024
        if self.torrent_config['max_upload_rate'] > 0:
            session_settings.upload_rate_limit = self.torrent_config['max_upload_rate'] * 1024
        self.session.set_settings(session_settings)

        # Encryption settings
        encryption_settings = libtorrent.pe_settings()
        encryption_settings.out_enc_policy = libtorrent.enc_policy(libtorrent.enc_policy.forced)
        encryption_settings.in_enc_policy = libtorrent.enc_policy(libtorrent.enc_policy.forced)
        encryption_settings.allowed_enc_level = libtorrent.enc_level.both
        encryption_settings.prefer_rc4 = True
        self.session.set_pe_settings(encryption_settings)

    ############################################################################
    def stop(self):
        if not self.monitor_running:
            return

        self.bus.log('[Downloader] Stopping session')

        for torrent_handle in self.torrent_handles:
            self.remove_torrent(torrent_handle)

        self.session.stop_natpmp()
        self.session.stop_upnp()
        self.session.stop_lsd()
        self.session.stop_dht()

        self.monitor_running = False
        time.sleep(1)
        cherrypy.process.plugins.Monitor.stop(self)

    ############################################################################
    def add_torrent(self, uri, download_dir):
        add_torrent_params                 = {}
        add_torrent_params['url']          = uri
        add_torrent_params['save_path']    = download_dir
        add_torrent_params['storage_mode'] = libtorrent.storage_mode_t.storage_mode_sparse
        add_torrent_params['auto_managed'] = False

        torrent_handle = self.session.add_torrent(add_torrent_params)
        torrent_handle.set_sequential_download(True)
        torrent_handle.resume()
        self.bus.connection_monitor.add_torrent(str(torrent_handle.info_hash()))

        if torrent_handle not in self.torrent_handles:
            self.torrent_handles.append(torrent_handle)

        return { 'name': torrent_handle.name(), 'info_hash': str(torrent_handle.info_hash()) }

    ############################################################################
    def remove_torrent(self, torrent_handle):
        remove_torrent_flags = libtorrent.options_t.delete_files if not self.torrent_config['keep_files'] else 0
        self.session.remove_torrent(torrent_handle, remove_torrent_flags)
        self.torrent_handles.remove(torrent_handle)

    ############################################################################
    def remove_paused_torrents(self):
        torrent_handles_to_remove = []
        for torrent_handle in self.torrent_handles:
            if torrent_handle.status().paused:
                torrent_handles_to_remove.append(torrent_handle)

        for torrent_handle in torrent_handles_to_remove:
            self.remove_torrent(torrent_handle)

    ############################################################################
    def get_status(self):
        result = {}

        if self.session:
            result['session'] = self.torrent_config

            result['session']['connection_sets'] = []
            for info_hash, connection_set in self.bus.connection_monitor.torrent_connections.iteritems():
                connection_set_status = { 'info_hash': info_hash, 'timestamp': connection_set['timestamp'], 'connections':[] }

                for connection in connection_set['set']:
                    connection_status = { 'info_hash': info_hash, 'remote_ip': connection.raddr[0], 'remote_port': connection.raddr[1] }
                    connection_set_status['connections'].append(connection_status)

                result['session']['connection_sets'].append(connection_set_status)

            result['session']['torrents'] = []
            for torrent_handle in self.torrent_handles:
                torrent_status = torrent_handle.status()

                torrent = {}

                try:
                    torrent['paused']        = torrent_status.paused
                    torrent['state']         = str(torrent_status.state)
                    torrent['state_index']   = int(torrent_status.state)
                    torrent['progress']      = math.trunc(torrent_status.progress * 100.0) / 100.0
                    torrent['download_rate'] = torrent_status.download_rate / 1024
                    torrent['upload_rate']   = torrent_status.upload_rate / 1024
                    torrent['num_seeds']     = torrent_status.num_seeds
                    torrent['total_seeds']   = torrent_status.num_complete
                    torrent['num_peers']     = torrent_status.num_peers
                    torrent['total_peers']   = torrent_status.num_incomplete
                    torrent['info_hash']     = str(torrent_handle.info_hash())

                    video_file = self._get_video_file_from_torrent(torrent_handle)
                    if video_file:
                        torrent['video_file']                          = {}
                        torrent['video_file']['path']                  = video_file.path
                        torrent['video_file']['size']                  = video_file.size
                        torrent['video_file']['start_piece_index']     = utils.piece_from_offset(torrent_handle, video_file.offset)
                        torrent['video_file']['end_piece_index']       = utils.piece_from_offset(torrent_handle, video_file.offset + video_file.size)
                        torrent['video_file']['total_pieces']          = utils.get_video_file_total_pieces(torrent_handle, video_file)
                        torrent['video_file']['preload_buffer_pieces'] = utils.get_preload_buffer_piece_count(torrent_handle, video_file)
                        torrent['video_file']['is_ready_fast']         = self.is_video_file_ready(torrent_handle, True, False)
                        torrent['video_file']['is_ready_slow']         = self.is_video_file_ready(torrent_handle, False, False)
                        torrent['video_file']['complete_pieces']       = utils.get_video_file_complete_pieces(torrent_handle, video_file)

                        piece_map = ''
                        for piece_index in range(torrent['video_file']['start_piece_index'], torrent['video_file']['end_piece_index'] + 1):
                            if torrent_handle.have_piece(piece_index):
                                piece_map = piece_map + '*'
                            elif torrent_handle.piece_priority(piece_index) == 0:
                                piece_map = piece_map + '0'
                            elif torrent_handle.piece_priority(piece_index) == 7:
                                piece_map = piece_map + '7'
                            else:
                                piece_map = piece_map + '.'
                        torrent['video_file']['piece_map'] = piece_map
                except RuntimeError:
                    pass

                result['session']['torrents'].append(torrent)

        return result

    ###########################################################################
    def is_video_file_ready_from_info_hash(self, info_hash, is_fast, log_enabled=True):
        for torrent_handle in self.torrent_handles:
            if str(torrent_handle.info_hash()) == info_hash:
                return self.is_video_file_ready(torrent_handle, is_fast, log_enabled)
        raise RuntimeError

    ###########################################################################
    def is_video_file_ready(self, torrent_handle, is_fast, log_enabled=True):
        if torrent_handle:
            status     = torrent_handle.status()
            video_file = self._get_video_file_from_torrent(torrent_handle)

            if int(status.state) >= 3 and video_file:
                complete_pieces = utils.get_video_file_complete_pieces(torrent_handle, video_file)
                total_pieces    = utils.get_video_file_total_pieces(torrent_handle, video_file)
                needed_pieces   = utils.get_preload_buffer_piece_count(torrent_handle, video_file)

                if is_fast or complete_pieces >= needed_pieces:
                    return True
                else:
                    if log_enabled:
                        self.bus.log('[Downloader] Not enough pieces yet: {0}/{1} (total: {2}) @ {3} kB/s'.format(complete_pieces, needed_pieces, total_pieces, status.download_rate / 1024))
            else:
                if log_enabled:
                    self.bus.log('[Downloader] Not ready yet: {0}'.format(str(status.state)))
        else:
            if log_enabled:
                self.bus.log('[Downloader] Not ready yet')

        return False

    ###########################################################################
    def get_video_file(self, info_hash):
        for torrent_handle in self.torrent_handles:
            if str(torrent_handle.info_hash()) == info_hash:
                video_file = self._get_video_file_from_torrent(torrent_handle)
                if video_file:
                    if torrent_handle.status().paused:
                        torrent_handle.resume()
                    return filewrapper.FileWrapper(self.bus, torrent_handle, video_file)
                else:
                    return None
        raise RuntimeError

    ############################################################################
    def _background_task(self):
        self.monitor_running = True

        while not self.session and self.monitor_running:
            time.sleep(0.1)

        while self.monitor_running:
            for torrent_handle in self.torrent_handles:
                if not self.bus.connection_monitor.has_video_connections(str(torrent_handle.info_hash())):
                    if not torrent_handle.status().paused:
                        torrent_handle.pause()

            if (time.time() - self.last_cleanup) > 18000: # 5 hours
                self.remove_paused_torrents()
                self.last_cleanup = time.time()

            self.session.wait_for_alert(1000)
            alert = self.session.pop_alert()
            if alert:
                if alert.what() in ('cache_flushed_alert', 'external_ip_alert', 'hash_failed_alert', 'metadata_failed_alert', 'tracker_error_alert'):
                    continue

                self.bus.log('[Downloader][{0}] {1}'.format(alert.what(), alert.message()))

                if isinstance(alert, libtorrent.metadata_received_alert):
                    video_file = self._get_video_file_from_torrent(alert.handle)

                    if not video_file:
                        self.bus.log('[Downloader] No video file found, removing torrent')
                        self.remove_torrent(torrent_handle)

    ############################################################################
    def _get_video_file_from_torrent(self, torrent_handle):
        video_file = None

        try:
            for file in torrent_handle.get_torrent_info().files():
                if file.path.endswith('.mkv') or file.path.endswith('.mp4') or file.path.endswith('.avi'):
                    if not video_file or video_file.size < file.size:
                        video_file = file
        except RuntimeError:
            pass

        return video_file
<commit_msg>Fixed shutdown issue<commit_after>################################################################################
import cherrypy
import filewrapper
import libtorrent
import math
import time
import utils

################################################################################
class DownloaderMonitor(cherrypy.process.plugins.Monitor):
    ############################################################################
    def __init__(self, bus, http_config, torrent_config):
        cherrypy.process.plugins.Monitor.__init__(self, bus, self._background_task, frequency=1)

        self.http_config     = http_config
        self.torrent_config  = torrent_config
        self.session         = None
        self.connections     = []
        self.monitor_running = False
        self.torrent_handles = []
        self.last_cleanup    = time.time()

        self.expected_alert_types    = []
        self.expected_alert_received = False

    ############################################################################
    def start(self):
        cherrypy.process.plugins.Monitor.start(self)

        self.bus.log('[Downloader] Starting session')
        self.session = libtorrent.session()
        self.session.set_alert_mask(libtorrent.alert.category_t.error_notification | libtorrent.alert.category_t.status_notification | libtorrent.alert.category_t.storage_notification)
        self.session.start_dht()
        self.session.start_lsd()
        self.session.start_upnp()
        self.session.start_natpmp()
        self.session.listen_on(self.torrent_config['port'], self.torrent_config['port'])

        # Session settings
        session_settings = self.session.settings()
        session_settings.announce_to_all_tiers = True
        session_settings.announce_to_all_trackers = True
        session_settings.connection_speed = 100
        session_settings.peer_connect_timeout = 2
        session_settings.rate_limit_ip_overhead = True
        session_settings.request_timeout = 5
        session_settings.torrent_connect_boost = 100

        if self.torrent_config['max_download_rate'] > 0:
            session_settings.download_rate_limit = self.torrent_config['max_download_rate'] * 1024
        if self.torrent_config['max_upload_rate'] > 0:
            session_settings.upload_rate_limit = self.torrent_config['max_upload_rate'] * 1024
        self.session.set_settings(session_settings)

        # Encryption settings
        encryption_settings = libtorrent.pe_settings()
        encryption_settings.out_enc_policy = libtorrent.enc_policy(libtorrent.enc_policy.forced)
        encryption_settings.in_enc_policy = libtorrent.enc_policy(libtorrent.enc_policy.forced)
        encryption_settings.allowed_enc_level = libtorrent.enc_level.both
        encryption_settings.prefer_rc4 = True
        self.session.set_pe_settings(encryption_settings)

    ############################################################################
    def stop(self):
        if not self.monitor_running:
            return

        self.bus.log('[Downloader] Stopping session')

        torrent_handles_to_remove = list(self.torrent_handles)
        for torrent_handle in torrent_handles_to_remove:
            self.remove_torrent(torrent_handle, True)

        self.session.stop_natpmp()
        self.session.stop_upnp()
        self.session.stop_lsd()
        self.session.stop_dht()

        self.monitor_running = False
        cherrypy.process.plugins.Monitor.stop(self)

    ############################################################################
    def add_torrent(self, uri, download_dir):
        add_torrent_params                 = {}
        add_torrent_params['url']          = uri
        add_torrent_params['save_path']    = download_dir
        add_torrent_params['storage_mode'] = libtorrent.storage_mode_t.storage_mode_sparse
        add_torrent_params['auto_managed'] = False

        torrent_handle = self.session.add_torrent(add_torrent_params)
        torrent_handle.set_sequential_download(True)
        torrent_handle.resume()
        self.bus.connection_monitor.add_torrent(str(torrent_handle.info_hash()))

        if torrent_handle not in self.torrent_handles:
            self.torrent_handles.append(torrent_handle)

        return { 'name': torrent_handle.name(), 'info_hash': str(torrent_handle.info_hash()) }

    ############################################################################
    def remove_torrent(self, torrent_handle, wait_for_alert=False):
        remove_torrent_flags = libtorrent.options_t.delete_files if not self.torrent_config['keep_files'] else 0
        self.session.remove_torrent(torrent_handle, remove_torrent_flags)

        if wait_for_alert:
            self.expected_alert_received = False
            self.expected_alert_types    = ['torrent_removed_alert'] if self.torrent_config['keep_files'] else ['torrent_deleted_alert', 'torrent_delete_failed_alert']

            while not self.expected_alert_received:
                time.sleep(0.1)

        self.torrent_handles.remove(torrent_handle)

    ############################################################################
    def remove_paused_torrents(self):
        torrent_handles_to_remove = []
        for torrent_handle in self.torrent_handles:
            if torrent_handle.status().paused:
                torrent_handles_to_remove.append(torrent_handle)

        for torrent_handle in torrent_handles_to_remove:
            self.remove_torrent(torrent_handle)

    ############################################################################
    def get_status(self):
        result = {}

        if self.session:
            result['session'] = self.torrent_config

            result['session']['connection_sets'] = []
            for info_hash, connection_set in self.bus.connection_monitor.torrent_connections.iteritems():
                connection_set_status = { 'info_hash': info_hash, 'timestamp': connection_set['timestamp'], 'connections':[] }

                for connection in connection_set['set']:
                    connection_status = { 'info_hash': info_hash, 'remote_ip': connection.raddr[0], 'remote_port': connection.raddr[1] }
                    connection_set_status['connections'].append(connection_status)

                result['session']['connection_sets'].append(connection_set_status)

            result['session']['torrents'] = []
            for torrent_handle in self.torrent_handles:
                torrent_status = torrent_handle.status()

                torrent = {}

                try:
                    torrent['paused']        = torrent_status.paused
                    torrent['state']         = str(torrent_status.state)
                    torrent['state_index']   = int(torrent_status.state)
                    torrent['progress']      = math.trunc(torrent_status.progress * 100.0) / 100.0
                    torrent['download_rate'] = torrent_status.download_rate / 1024
                    torrent['upload_rate']   = torrent_status.upload_rate / 1024
                    torrent['num_seeds']     = torrent_status.num_seeds
                    torrent['total_seeds']   = torrent_status.num_complete
                    torrent['num_peers']     = torrent_status.num_peers
                    torrent['total_peers']   = torrent_status.num_incomplete
                    torrent['info_hash']     = str(torrent_handle.info_hash())

                    video_file = self._get_video_file_from_torrent(torrent_handle)
                    if video_file:
                        torrent['video_file']                          = {}
                        torrent['video_file']['path']                  = video_file.path
                        torrent['video_file']['size']                  = video_file.size
                        torrent['video_file']['start_piece_index']     = utils.piece_from_offset(torrent_handle, video_file.offset)
                        torrent['video_file']['end_piece_index']       = utils.piece_from_offset(torrent_handle, video_file.offset + video_file.size)
                        torrent['video_file']['total_pieces']          = utils.get_video_file_total_pieces(torrent_handle, video_file)
                        torrent['video_file']['preload_buffer_pieces'] = utils.get_preload_buffer_piece_count(torrent_handle, video_file)
                        torrent['video_file']['is_ready_fast']         = self.is_video_file_ready(torrent_handle, True, False)
                        torrent['video_file']['is_ready_slow']         = self.is_video_file_ready(torrent_handle, False, False)
                        torrent['video_file']['complete_pieces']       = utils.get_video_file_complete_pieces(torrent_handle, video_file)

                        piece_map = ''
                        for piece_index in range(torrent['video_file']['start_piece_index'], torrent['video_file']['end_piece_index'] + 1):
                            if torrent_handle.have_piece(piece_index):
                                piece_map = piece_map + '*'
                            elif torrent_handle.piece_priority(piece_index) == 0:
                                piece_map = piece_map + '0'
                            elif torrent_handle.piece_priority(piece_index) == 7:
                                piece_map = piece_map + '7'
                            else:
                                piece_map = piece_map + '.'
                        torrent['video_file']['piece_map'] = piece_map
                except RuntimeError:
                    pass

                result['session']['torrents'].append(torrent)

        return result

    ###########################################################################
    def is_video_file_ready_from_info_hash(self, info_hash, is_fast, log_enabled=True):
        for torrent_handle in self.torrent_handles:
            if str(torrent_handle.info_hash()) == info_hash:
                return self.is_video_file_ready(torrent_handle, is_fast, log_enabled)
        raise RuntimeError

    ###########################################################################
    def is_video_file_ready(self, torrent_handle, is_fast, log_enabled=True):
        if torrent_handle:
            status     = torrent_handle.status()
            video_file = self._get_video_file_from_torrent(torrent_handle)

            if int(status.state) >= 3 and video_file:
                complete_pieces = utils.get_video_file_complete_pieces(torrent_handle, video_file)
                total_pieces    = utils.get_video_file_total_pieces(torrent_handle, video_file)
                needed_pieces   = utils.get_preload_buffer_piece_count(torrent_handle, video_file)

                if is_fast or complete_pieces >= needed_pieces:
                    return True
                else:
                    if log_enabled:
                        self.bus.log('[Downloader] Not enough pieces yet: {0}/{1} (total: {2}) @ {3} kB/s'.format(complete_pieces, needed_pieces, total_pieces, status.download_rate / 1024))
            else:
                if log_enabled:
                    self.bus.log('[Downloader] Not ready yet: {0}'.format(str(status.state)))
        else:
            if log_enabled:
                self.bus.log('[Downloader] Not ready yet')

        return False

    ###########################################################################
    def get_video_file(self, info_hash):
        for torrent_handle in self.torrent_handles:
            if str(torrent_handle.info_hash()) == info_hash:
                video_file = self._get_video_file_from_torrent(torrent_handle)
                if video_file:
                    if torrent_handle.status().paused:
                        torrent_handle.resume()
                    return filewrapper.FileWrapper(self.bus, torrent_handle, video_file)
                else:
                    return None
        raise RuntimeError

    ############################################################################
    def _background_task(self):
        self.monitor_running = True

        while not self.session and self.monitor_running:
            time.sleep(0.1)

        while self.monitor_running:
            for torrent_handle in self.torrent_handles:
                if not self.bus.connection_monitor.has_video_connections(str(torrent_handle.info_hash())):
                    if not torrent_handle.status().paused:
                        torrent_handle.pause()

            if (time.time() - self.last_cleanup) > 18000: # 5 hours
                self.remove_paused_torrents()
                self.last_cleanup = time.time()

            self._alert_pump()

    ############################################################################
    def _alert_pump(self):
        while True:
            self.session.wait_for_alert(1000)
            alert = self.session.pop_alert()
            if alert:
                if alert.what() in ('cache_flushed_alert', 'external_ip_alert', 'hash_failed_alert', 'metadata_failed_alert', 'tracker_error_alert'):
                    continue

                self.bus.log('[Downloader][{0}] {1}'.format(alert.what(), alert.message()))

                if isinstance(alert, libtorrent.metadata_received_alert):
                    video_file = self._get_video_file_from_torrent(alert.handle)

                    if not video_file:
                        self.bus.log('[Downloader] No video file found, removing torrent')
                        self.remove_torrent(torrent_handle)

                elif alert.what() in self.expected_alert_types:
                    self.expected_alert_received = True
                    self.expected_alert_types    = []
                    break

            if not self.expected_alert_types:
                break

    ############################################################################
    def _get_video_file_from_torrent(self, torrent_handle):
        video_file = None

        try:
            for file in torrent_handle.get_torrent_info().files():
                if file.path.endswith('.mkv') or file.path.endswith('.mp4') or file.path.endswith('.avi'):
                    if not video_file or video_file.size < file.size:
                        video_file = file
        except RuntimeError:
            pass

        return video_file
<|endoftext|>


<commit_before># -*- test-case-name: vumi_message_store.tests.test_riak_backend -*-

"""
Fake in-memory Riak manager that implements enough for the message store.
"""

# TODO: Sync and async.

import base64
import json

from twisted.internet import reactor
from twisted.internet.defer import Deferred, inlineCallbacks, returnValue
from vumi.persist.model import ModelProxy
from vumi.utils import flatten_generator

class FakeMemoryRiakManager(object):
    """
    This is an in-memory implementation of a subset of the Riak manager's
    functionality.

    NOTE: This is the minimal subset necessary for MessageStoreRiakBackend.
    """

    def __init__(self, state, store_versions=None):
        self._state = state
        self._is_sync = state._is_sync
        if self._is_sync:
            self.call_decorator = flatten_generator
        else:
            self.call_decorator = inlineCallbacks
        self.store_versions = store_versions or {}
        # Wrap the methods that need to handle async stuff.
        self.load = self.call_decorator(self._internal_load)
        self.store = self.call_decorator(self._internal_store)

    def proxy(self, modelcls):
        """
        Return a real proxy using the fake manager.
        """
        return ModelProxy(self, modelcls)

    def bucket_name(self, modelcls_or_obj):
        return modelcls_or_obj.bucket

    def bucket_for_modelcls(self, modelcls):
        """
        Return a fake bucket object for the given model class.
        """
        bucket_name = self.bucket_name(modelcls)
        return FakeRiakBucket(self._state, bucket_name)

    def riak_object(self, modelcls, key):
        """
        Create and return a fake RiakObject.
        """
        bucket_name = self.bucket_name(modelcls)
        riak_object = FakeRiakObject(self._state, bucket_name, key)
        riak_object.set_content_type("application/json")
        riak_object.set_data({'$VERSION': modelcls.VERSION})
        return riak_object

    def _internal_store(self, modelobj):
        """
        This method gets wrapped in a sync or async decorator in __init__().
        """
        riak_object = modelobj._riak_object
        modelcls = type(modelobj)
        model_name = "%s.%s" % (modelcls.__module__, modelcls.__name__)
        store_version = self.store_versions.get(model_name, modelcls.VERSION)
        # Run reverse migrators until we have the correct version of the data.
        data_version = riak_object.get_data().get('$VERSION', None)
        while data_version != store_version:
            migrator = modelcls.MIGRATOR(
                modelcls, self, data_version, reverse=True)
            riak_object = migrator(riak_object).get_riak_object()
            data_version = riak_object.get_data().get('$VERSION', None)
        yield riak_object.store()
        returnValue(modelobj)

    def _internal_load(self, modelcls, key, result=None):
        """
        This method gets wrapped in a sync or async decorator in __init__().
        """
        assert result is None
        riak_object = self.riak_object(modelcls, key)
        yield riak_object.reload()
        was_migrated = False

        # Run migrators until we have the correct version of the data.
        while riak_object.get_data() is not None:
            data_version = riak_object.get_data().get('$VERSION', None)
            if data_version == modelcls.VERSION:
                obj = modelcls(self, key, _riak_object=riak_object)
                obj.was_migrated = was_migrated
                returnValue(obj)
            migrator = modelcls.MIGRATOR(modelcls, self, data_version)
            riak_object = migrator(riak_object).get_riak_object()
            was_migrated = True
        returnValue(None)

    def index_keys(self, model, index_name, start_value, end_value=None,
                   return_terms=None):
        bucket = self.bucket_for_modelcls(model)
        return bucket.get_index(
            index_name, start_value, end_value, return_terms=return_terms)

    def index_keys_page(self, model, index_name, start_value, end_value=None,
                        return_terms=None, max_results=None,
                        continuation=None):
        bucket = self.bucket_for_modelcls(model)
        return bucket.get_index_page(
            index_name, start_value, end_value, return_terms=return_terms,
            max_results=max_results, continuation=continuation)

class FakeRiakState(object):
    """
    An object to hold fake Riak state.
    """

    def __init__(self, is_sync):
        self._buckets = {}
        self._is_sync = is_sync
        self._delayed_calls = []

    def teardown(self):
        for delayed in self._delayed_calls:
            if not (delayed.cancelled or delayed.called):
                delayed.cancel()

    def _with_async_delay(self, func, *args, **kw):
        """
        If in async mode, add some delay to catch code that doesn't properly
        wait for the deferred to fire.
        """
        if self._is_sync:
            return func(*args, **kw)

        # Add some latency to catch things that don't wait on deferreds. We
        # can't use deferLater() here because we want to keep track of the
        # delayed call object.
        d = Deferred()
        delayed = reactor.callLater(
            0.002, lambda _: d.callback(func(*args, **kw)), None)
        self._delayed_calls.append(delayed)
        return d

    def _get_bucket(self, bucket_name):
        """
        Get the data for a bucket.
        """
        return self._buckets.setdefault(bucket_name, {
            "objects": {},
            "indexes": {},
        })

    def store_object(self, fake_object):
        """
        Store an object.
        """
        bucket = self._get_bucket(fake_object._bucket_name)
        bucket["objects"][fake_object.key] = {
            "content_type": fake_object._current_data["content_type"],
            "data": json.dumps(fake_object._current_data["data"]),
        }
        indexes = {}
        for index_name, index_value in fake_object._current_data["indexes"]:
            indexes.setdefault(index_name, []).append(
                (index_value, fake_object.key))
        for index_name, index_values in indexes.iteritems():
            old_indexes = bucket["indexes"].get(index_name, [])
            new_indexes = [
                (value, key) for value, key in old_indexes
                if key != fake_object.key]
            new_indexes.extend(index_values)
            new_indexes.sort()
            bucket["indexes"][index_name] = new_indexes

    def _reload_object_state(self, fake_object):
        """
        Set an object's data from stored state.
        """
        bucket = self._get_bucket(fake_object._bucket_name)
        obj_state = bucket["objects"].get(fake_object.key)
        if obj_state is None:
            fake_object._current_data["data"] = None
            return
        fake_object._current_data["data"] = json.loads(obj_state["data"])
        fake_object._current_data["content_type"] = obj_state["content_type"]
        fake_object._current_data["indexes"] = set()
        for index_name, index_data in bucket["indexes"].iteritems():
            for index_value, key in index_data:
                if key == fake_object.key:
                    fake_object._current_data["indexes"].add(
                        (index_name, index_value))

    def get_index_page(self, bucket_name, index_name, start_value, end_value,
                       return_terms=False, max_results=None,
                       continuation=None):
        """
        Perform an index query.
        """
        # Get all matching results
        index = self._get_bucket(bucket_name)["indexes"].get(index_name, [])
        if end_value is None:
            in_range = lambda v: v[0] == start_value
        else:
            in_range = lambda v: start_value <= v[0] <= end_value
        results = filter(in_range, index)

        # Drop all results we've returned previously, if any.
        if continuation is not None:
            continuation = tuple(json.loads(base64.b64decode(continuation)))
            while results and results[0] < continuation:
                results.pop(0)

        # If we're not paginated, we're done.
        if max_results is None:
            return self._return_index_page(
                bucket_name, index_name, start_value, end_value, return_terms,
                max_results, results, continuation=None)

        # Truncate the results and build the continuation token.
        continuation = None
        truncated_results = results[:max_results]
        if len(truncated_results) < len(results):
            continuation = base64.b64encode(
                json.dumps(list(results[max_results])))
        return self._return_index_page(
            bucket_name, index_name, start_value, end_value, return_terms,
            max_results, truncated_results, continuation=continuation)

    def _return_index_page(self, bucket_name, index_name, start_value,
                           end_value, return_terms, max_results, results,
                           continuation):
        """
        Construct and return an index page object.
        """
        if not return_terms:
            results = [key for _, key in results]
        return FakeMemoryIndexPage(self, continuation, results, {
            "bucket_name": bucket_name,
            "index_name": index_name,
            "start_value": start_value,
            "end_value": end_value,
            "return_terms": return_terms,
            "max_results": max_results,
        })

class FakeMemoryIndexPage(object):
    """
    Index page implementation for traversing in-memory indexes.
    """
    def __init__(self, state, continuation, results, params):
        self._state = state
        self._continuation = continuation
        self._results = results
        self._params = params

    def next_page(self):
        assert self.has_next_page()
        return self._state.get_index_page(
            continuation=self._continuation, **self._params)

    def has_next_page(self):
        return self._continuation is not None

    def __iter__(self):
        return iter(self._results)

class FakeRiakBucket(object):
    """
    A fake implementation of a Riak bucket object using in-memory state.

    NOTE: This is the minimal subset necessary for MessageStoreRiakBackend.
    """

    def __init__(self, state, bucket_name):
        self._state = state
        self._bucket_name = bucket_name

    # Methods that "touch the network".

    def get_index(self, index_name, start_value, end_value=None,
                  return_terms=None):
        return self.get_index_page(
            index_name, start_value, end_value, return_terms=return_terms)

    def get_index_page(self, index_name, start_value, end_value=None,
                       return_terms=None, max_results=None, continuation=None):
        return self._state._with_async_delay(
            self._state.get_index_page,
            self._bucket_name, index_name, start_value, end_value,
            return_terms=return_terms, max_results=max_results,
            continuation=continuation)

class FakeRiakObject(object):
    """
    A fake implementation of a Riak object using in-memory state.

    NOTE: This is the minimal subset necessary for MessageStoreRiakBackend.
    """

    def __init__(self, state, bucket_name, key):
        self._state = state
        self._bucket_name = bucket_name
        self._key = key
        self._current_data = {
            "content_type": None,
            "data": None,
            "indexes": set(),
        }

    @property
    def key(self):
        return self._key

    def set_content_type(self, content_type):
        self._current_data["content_type"] = content_type

    def get_data(self):
        return self._current_data["data"]

    def set_data(self, data):
        self._current_data["data"] = data

    def set_data_field(self, key, value):
        self._current_data["data"][key] = value

    def delete_data_field(self, key):
        del self._current_data["data"][key]

    def get_indexes(self):
        return self._current_data["indexes"]

    def add_index(self, index_name, index_value):
        self._current_data["indexes"].add((index_name, index_value))

    def remove_index(self, index_name, index_value=None):
        indexes = self._current_data["indexes"]
        if index_name is None:
            assert index_value is None
            indexes.clear()
        elif index_value is None:
            for index in [(k, v) for k, v in indexes if k == index_name]:
                indexes.remove(index)
        else:
            indexes.remove((index_name, index_value))

    # Methods that "touch the network".

    def _reload(self):
        new_obj = type(self)(self._state, self._bucket_name, self._key)
        new_obj._current_data = self._current_data
        self._state._reload_object_state(new_obj)
        return new_obj

    def _store(self):
        self._state.store_object(self)
        return self._reload()

    def store(self):
        return self._state._with_async_delay(self._store)

    def reload(self):
        return self._state._with_async_delay(self._reload)
<commit_msg>Remove unnecessary list().<commit_after># -*- test-case-name: vumi_message_store.tests.test_riak_backend -*-

"""
Fake in-memory Riak manager that implements enough for the message store.
"""

# TODO: Sync and async.

import base64
import json

from twisted.internet import reactor
from twisted.internet.defer import Deferred, inlineCallbacks, returnValue
from vumi.persist.model import ModelProxy
from vumi.utils import flatten_generator

class FakeMemoryRiakManager(object):
    """
    This is an in-memory implementation of a subset of the Riak manager's
    functionality.

    NOTE: This is the minimal subset necessary for MessageStoreRiakBackend.
    """

    def __init__(self, state, store_versions=None):
        self._state = state
        self._is_sync = state._is_sync
        if self._is_sync:
            self.call_decorator = flatten_generator
        else:
            self.call_decorator = inlineCallbacks
        self.store_versions = store_versions or {}
        # Wrap the methods that need to handle async stuff.
        self.load = self.call_decorator(self._internal_load)
        self.store = self.call_decorator(self._internal_store)

    def proxy(self, modelcls):
        """
        Return a real proxy using the fake manager.
        """
        return ModelProxy(self, modelcls)

    def bucket_name(self, modelcls_or_obj):
        return modelcls_or_obj.bucket

    def bucket_for_modelcls(self, modelcls):
        """
        Return a fake bucket object for the given model class.
        """
        bucket_name = self.bucket_name(modelcls)
        return FakeRiakBucket(self._state, bucket_name)

    def riak_object(self, modelcls, key):
        """
        Create and return a fake RiakObject.
        """
        bucket_name = self.bucket_name(modelcls)
        riak_object = FakeRiakObject(self._state, bucket_name, key)
        riak_object.set_content_type("application/json")
        riak_object.set_data({'$VERSION': modelcls.VERSION})
        return riak_object

    def _internal_store(self, modelobj):
        """
        This method gets wrapped in a sync or async decorator in __init__().
        """
        riak_object = modelobj._riak_object
        modelcls = type(modelobj)
        model_name = "%s.%s" % (modelcls.__module__, modelcls.__name__)
        store_version = self.store_versions.get(model_name, modelcls.VERSION)
        # Run reverse migrators until we have the correct version of the data.
        data_version = riak_object.get_data().get('$VERSION', None)
        while data_version != store_version:
            migrator = modelcls.MIGRATOR(
                modelcls, self, data_version, reverse=True)
            riak_object = migrator(riak_object).get_riak_object()
            data_version = riak_object.get_data().get('$VERSION', None)
        yield riak_object.store()
        returnValue(modelobj)

    def _internal_load(self, modelcls, key, result=None):
        """
        This method gets wrapped in a sync or async decorator in __init__().
        """
        assert result is None
        riak_object = self.riak_object(modelcls, key)
        yield riak_object.reload()
        was_migrated = False

        # Run migrators until we have the correct version of the data.
        while riak_object.get_data() is not None:
            data_version = riak_object.get_data().get('$VERSION', None)
            if data_version == modelcls.VERSION:
                obj = modelcls(self, key, _riak_object=riak_object)
                obj.was_migrated = was_migrated
                returnValue(obj)
            migrator = modelcls.MIGRATOR(modelcls, self, data_version)
            riak_object = migrator(riak_object).get_riak_object()
            was_migrated = True
        returnValue(None)

    def index_keys(self, model, index_name, start_value, end_value=None,
                   return_terms=None):
        bucket = self.bucket_for_modelcls(model)
        return bucket.get_index(
            index_name, start_value, end_value, return_terms=return_terms)

    def index_keys_page(self, model, index_name, start_value, end_value=None,
                        return_terms=None, max_results=None,
                        continuation=None):
        bucket = self.bucket_for_modelcls(model)
        return bucket.get_index_page(
            index_name, start_value, end_value, return_terms=return_terms,
            max_results=max_results, continuation=continuation)

class FakeRiakState(object):
    """
    An object to hold fake Riak state.
    """

    def __init__(self, is_sync):
        self._buckets = {}
        self._is_sync = is_sync
        self._delayed_calls = []

    def teardown(self):
        for delayed in self._delayed_calls:
            if not (delayed.cancelled or delayed.called):
                delayed.cancel()

    def _with_async_delay(self, func, *args, **kw):
        """
        If in async mode, add some delay to catch code that doesn't properly
        wait for the deferred to fire.
        """
        if self._is_sync:
            return func(*args, **kw)

        # Add some latency to catch things that don't wait on deferreds. We
        # can't use deferLater() here because we want to keep track of the
        # delayed call object.
        d = Deferred()
        delayed = reactor.callLater(
            0.002, lambda _: d.callback(func(*args, **kw)), None)
        self._delayed_calls.append(delayed)
        return d

    def _get_bucket(self, bucket_name):
        """
        Get the data for a bucket.
        """
        return self._buckets.setdefault(bucket_name, {
            "objects": {},
            "indexes": {},
        })

    def store_object(self, fake_object):
        """
        Store an object.
        """
        bucket = self._get_bucket(fake_object._bucket_name)
        bucket["objects"][fake_object.key] = {
            "content_type": fake_object._current_data["content_type"],
            "data": json.dumps(fake_object._current_data["data"]),
        }
        indexes = {}
        for index_name, index_value in fake_object._current_data["indexes"]:
            indexes.setdefault(index_name, []).append(
                (index_value, fake_object.key))
        for index_name, index_values in indexes.iteritems():
            old_indexes = bucket["indexes"].get(index_name, [])
            new_indexes = [
                (value, key) for value, key in old_indexes
                if key != fake_object.key]
            new_indexes.extend(index_values)
            new_indexes.sort()
            bucket["indexes"][index_name] = new_indexes

    def _reload_object_state(self, fake_object):
        """
        Set an object's data from stored state.
        """
        bucket = self._get_bucket(fake_object._bucket_name)
        obj_state = bucket["objects"].get(fake_object.key)
        if obj_state is None:
            fake_object._current_data["data"] = None
            return
        fake_object._current_data["data"] = json.loads(obj_state["data"])
        fake_object._current_data["content_type"] = obj_state["content_type"]
        fake_object._current_data["indexes"] = set()
        for index_name, index_data in bucket["indexes"].iteritems():
            for index_value, key in index_data:
                if key == fake_object.key:
                    fake_object._current_data["indexes"].add(
                        (index_name, index_value))

    def get_index_page(self, bucket_name, index_name, start_value, end_value,
                       return_terms=False, max_results=None,
                       continuation=None):
        """
        Perform an index query.
        """
        # Get all matching results
        index = self._get_bucket(bucket_name)["indexes"].get(index_name, [])
        if end_value is None:
            in_range = lambda v: v[0] == start_value
        else:
            in_range = lambda v: start_value <= v[0] <= end_value
        results = filter(in_range, index)

        # Drop all results we've returned previously, if any.
        if continuation is not None:
            continuation = tuple(json.loads(base64.b64decode(continuation)))
            while results and results[0] < continuation:
                results.pop(0)

        # If we're not paginated, we're done.
        if max_results is None:
            return self._return_index_page(
                bucket_name, index_name, start_value, end_value, return_terms,
                max_results, results, continuation=None)

        # Truncate the results and build the continuation token.
        continuation = None
        truncated_results = results[:max_results]
        if len(truncated_results) < len(results):
            continuation = base64.b64encode(json.dumps(results[max_results]))
        return self._return_index_page(
            bucket_name, index_name, start_value, end_value, return_terms,
            max_results, truncated_results, continuation=continuation)

    def _return_index_page(self, bucket_name, index_name, start_value,
                           end_value, return_terms, max_results, results,
                           continuation):
        """
        Construct and return an index page object.
        """
        if not return_terms:
            results = [key for _, key in results]
        return FakeMemoryIndexPage(self, continuation, results, {
            "bucket_name": bucket_name,
            "index_name": index_name,
            "start_value": start_value,
            "end_value": end_value,
            "return_terms": return_terms,
            "max_results": max_results,
        })

class FakeMemoryIndexPage(object):
    """
    Index page implementation for traversing in-memory indexes.
    """
    def __init__(self, state, continuation, results, params):
        self._state = state
        self._continuation = continuation
        self._results = results
        self._params = params

    def next_page(self):
        assert self.has_next_page()
        return self._state.get_index_page(
            continuation=self._continuation, **self._params)

    def has_next_page(self):
        return self._continuation is not None

    def __iter__(self):
        return iter(self._results)

class FakeRiakBucket(object):
    """
    A fake implementation of a Riak bucket object using in-memory state.

    NOTE: This is the minimal subset necessary for MessageStoreRiakBackend.
    """

    def __init__(self, state, bucket_name):
        self._state = state
        self._bucket_name = bucket_name

    # Methods that "touch the network".

    def get_index(self, index_name, start_value, end_value=None,
                  return_terms=None):
        return self.get_index_page(
            index_name, start_value, end_value, return_terms=return_terms)

    def get_index_page(self, index_name, start_value, end_value=None,
                       return_terms=None, max_results=None, continuation=None):
        return self._state._with_async_delay(
            self._state.get_index_page,
            self._bucket_name, index_name, start_value, end_value,
            return_terms=return_terms, max_results=max_results,
            continuation=continuation)

class FakeRiakObject(object):
    """
    A fake implementation of a Riak object using in-memory state.

    NOTE: This is the minimal subset necessary for MessageStoreRiakBackend.
    """

    def __init__(self, state, bucket_name, key):
        self._state = state
        self._bucket_name = bucket_name
        self._key = key
        self._current_data = {
            "content_type": None,
            "data": None,
            "indexes": set(),
        }

    @property
    def key(self):
        return self._key

    def set_content_type(self, content_type):
        self._current_data["content_type"] = content_type

    def get_data(self):
        return self._current_data["data"]

    def set_data(self, data):
        self._current_data["data"] = data

    def set_data_field(self, key, value):
        self._current_data["data"][key] = value

    def delete_data_field(self, key):
        del self._current_data["data"][key]

    def get_indexes(self):
        return self._current_data["indexes"]

    def add_index(self, index_name, index_value):
        self._current_data["indexes"].add((index_name, index_value))

    def remove_index(self, index_name, index_value=None):
        indexes = self._current_data["indexes"]
        if index_name is None:
            assert index_value is None
            indexes.clear()
        elif index_value is None:
            for index in [(k, v) for k, v in indexes if k == index_name]:
                indexes.remove(index)
        else:
            indexes.remove((index_name, index_value))

    # Methods that "touch the network".

    def _reload(self):
        new_obj = type(self)(self._state, self._bucket_name, self._key)
        new_obj._current_data = self._current_data
        self._state._reload_object_state(new_obj)
        return new_obj

    def _store(self):
        self._state.store_object(self)
        return self._reload()

    def store(self):
        return self._state._with_async_delay(self._store)

    def reload(self):
        return self._state._with_async_delay(self._reload)
<|endoftext|>


<commit_before>from flask import Flask
from flask import session
from flask import Response
from flask import request
from flask import redirect, url_for
from flask import render_template

from api.core import app, db
from api.models.user import User
from api.models.page import Page
from api.models.event import Event
from api.models.question import Question
from api.models.group import Group

from api.controllers.nlp import *

import urllib
import requests
import json
import datetime

from gensim import corpora, models, similarities

@app.route('/dashboard/page/<page_id>')
def dashboard_page(page_id):
    page = db.session.query(Page).filter(
        Page.page_id == page_id
    ).first()

    events = db.session.query(Event).filter(
        Event.page_id == page.id
    ).all()

    return render_template('listing.html', events=events, page_id=page_id)

@app.route('/dashboard/event/<event_id>')
def dashboard_event(event_id):
    event = db.session.query(Event).filter(
        Event.id == event_id
    ).first()

    page = db.session.query(Page).filter(
        Page.id == event.page_id
    ).first()

    return render_template('dashboard.html', page_id=page.page_id, event=event)

@app.route('/dashboard/<group_id>/details')
def dashboard_reply(group_id):
    group = db.session.query(Group).filter(
        Group.id == group_id
    ).first()

    users = db.session.query(Question).filter(
        Question.group_id == group_id
    ).group_by(Question.user).all()

    return Response(json.dumps({
        'users': [i.profile for i in users],
        'question': group.question
    }), mimetype='application/json')

@app.route('/dashboard/event/<event_id>/retrieve')
def dashboard_retrieve(event_id):
    event = db.session.query(Event).filter(
        Event.id == event_id
    ).first()

    if event.fb_post_id is None:
        return Response(json.dumps({
            'questions': []
        }), mimetype='application/json')

    page = db.session.query(Page).filter(
        Page.id == event.page_id
    ).first()

    params = {
        'access_token': page.token
    }
    params = urllib.urlencode(params)
    questions = json.loads(
        urllib.urlopen('https://graph.facebook.com/' + event.fb_post_id + '/comments?%s' % params).read()
    )

    for question in questions['data']:
        new_question = {
            'fb_id':    question['id'],
            'user':     question['from']['id'],
            'question': question['message'],
            'event_id': event.id
        }

        db_q = db.session.query(Question).filter(
            Question.fb_id == question['id']
        ).first()

        if not db_q:
            groups = db.session.query(Group).filter(
                Group.event_id == event.id
            ).all()

            g = None
            if groups:
                g = match_group(new_question['question'], groups, 0.9)

            if g is None:
                g = {
                    'event_id': event.id,
                    'question': new_question['question']
                }

                # Get Sentiment
                params = {
                    'apikey':     '2ccd6f653c1e4253b6ac5ee0dadb284bde58331e',
                    'text':       new_question['question'],
                    'outputMode': 'json'
                }
                params = urllib.urlencode(params)

                sentiment = json.loads(
                    urllib.urlopen('http://access.alchemyapi.com/calls/text/TextGetTextSentiment?%s' % params).read()
                )

                if 'docSentiment' in sentiment and 'type' in sentiment['docSentiment']:
                    if sentiment['docSentiment']['type']:
                        g['sentiment'] = sentiment['docSentiment']['type']

                g = Group(**g)

                db.session.add(g)
                db.session.commit()

            new_question['group_id'] = g.id

            q = Question(**new_question)

            db.session.add(q)
            db.session.commit()

    return Response(json.dumps({
        'status': questions
    }), mimetype='application/json')

@app.route('/dashboard/event/<event_id>/delete')
def dashboard_delete(event_id):
    event = db.session.query(Event).filter(
        Event.id == event_id
    ).first()

    page = db.session.query(Page).filter(
        Page.id == event.page_id
    ).first()

    db.session.delete(event)
    db.session.commit()

    return redirect('/dashboard/page/%s' % page.page_id)

@app.route('/dashboard/event/<event_id>/polling')
def dashboard_polling(event_id):
    event = db.session.query(Event).filter(
        Event.id == event_id
    ).first()

    page = db.session.query(Page).filter(
        Page.id == event.page_id
    ).first()

    dashboard_retrieve(event_id)

    aGroups = db.session.query(Group).filter(
        Group.event_id == event_id
    ).filter(
        Group.status == 1
    ).all()

    uGroups = db.session.query(Group).filter(
        Group.event_id == event_id
    ).filter(
        Group.status == 0
    ).all()

    questions = db.session.query(Question).filter(
        Question.event_id == event.id
    ).count()

    users = db.session.query(Question).filter(
        Question.event_id == event.id
    ).group_by(
        Question.user
    ).count()

    return Response(json.dumps({
        'questionsNumber': questions,
        'usersOverview': users,
        'answeredQuestions': [i.serialize for i in aGroups],
        'unansweredQuestions': [i.serialize for i in uGroups],
        'totalPositive': 0,
        'totalNegative': 0,
        'totalNeutral': 0
    }), mimetype='application/json')

@app.route('/dashboard/event/<event_id>/publish')
def dashboard_publish(event_id):
    event = db.session.query(Event).filter(
        Event.id == event_id
    ).first()

    page = db.session.query(Page).filter(
        Page.id == event.page_id
    ).first()

    params = {
        'access_token':       page.token,
        'to':                 page.page_id,
        'message':            'This is an awesome post. Deal with it!',
        'format':             'json',
        'suppress_http_code': 1,
        'method':             'post'
    }
    params = urllib.urlencode(params)

    data = json.loads(
        urllib.urlopen('https://graph.facebook.com/' + page.page_id + '/feed?%s' % params).read()
    )

    if 'id' in data:
        event.fb_post_id = data['id']
        event.status = 1
        db.session.commit()

    return redirect('/dashboard/page/%s' % page.page_id)

@app.route('/dashboard/new', methods=['POST'])
def dashboard_new():
    form = request.form

    page_id = form['session-page']

    page = db.session.query(Page).filter(
        Page.page_id == page_id
    ).first()

    new_event = {
        'page_id': page.id,
        'title': form['session-title'],
        'message': 'This is a random message'
    }

    if form['session-start-date']:
        new_event['start_date'] = datetime.datetime.strptime(form['session-start-date'], '%Y/%m/%d %H:%M')

    if form['session-end-date']:
        new_event['end_date'] = datetime.datetime.strptime(form['session-end-date'], '%Y/%m/%d %H:%M')

    db_event = Event(**new_event)

    db.session.add(db_event)
    db.session.commit()

    return redirect('/dashboard/page/%s' % page_id)

@app.route('/dashboard/page/<page_id>/<access_token>')
def dashboard_page_token(page_id, access_token):
    fb_page = {
        'user_id': session['USER_ID'],
        'page_id': page_id,
        'token': access_token
    }

    db_page = db.session.query(Page).filter(
        Page.page_id == page_id
    ).first()

    if not db_page:
        db_page = Page(**fb_page)

        db.session.add(db_page)
    else:
        db_page.token = access_token

    db.session.commit()

    return redirect('/dashboard/page/%s' % page_id)
<commit_msg>Prop commit<commit_after>from flask import Flask
from flask import session
from flask import Response
from flask import request
from flask import redirect, url_for
from flask import render_template

from api.core import app, db
from api.models.user import User
from api.models.page import Page
from api.models.event import Event
from api.models.question import Question
from api.models.group import Group

from api.controllers.nlp import *

import urllib
import requests
import json
import datetime

from gensim import corpora, models, similarities

@app.route('/dashboard/page/<page_id>')
def dashboard_page(page_id):
    page = db.session.query(Page).filter(
        Page.page_id == page_id
    ).first()

    events = db.session.query(Event).filter(
        Event.page_id == page.id
    ).all()

    return render_template('listing.html', events=events, page_id=page_id)

@app.route('/dashboard/event/<event_id>')
def dashboard_event(event_id):
    event = db.session.query(Event).filter(
        Event.id == event_id
    ).first()

    page = db.session.query(Page).filter(
        Page.id == event.page_id
    ).first()

    return render_template('dashboard.html', page_id=page.page_id, event=event)

@app.route('/dashboard/<group_id>/details')
def dashboard_details(group_id):
    group = db.session.query(Group).filter(
        Group.id == group_id
    ).first()

    users = db.session.query(Question).filter(
        Question.group_id == group_id
    ).group_by(Question.user).all()

    return Response(json.dumps({
        'users': [i.profile for i in users],
        'question': group.question
    }), mimetype='application/json')

@app.route('/dashboard/<group_id>/reply')
def dashboard_reply(group_id):
    data = request.form

    group = db.session.query(Group).filter(
        Group.id == group_id
    ).first()

    questions = db.session.query(Question).filter(
        Question.group_id == group.id
    ).all()

    for q in questions:
        params = {
            'access_token':       session['PAGE_TOKEN'],
            'message':            data['message'],
            'format':             'json',
            'suppress_http_code': 1,
            'method':             'post'
        }
        params = urllib.urlencode(params)

        data = json.loads(
            urllib.urlopen('https://graph.facebook.com/' + q.fb_id + '/comments?%s' % params).read()
        )

    return Response(json.dumps({
        'status': 'ok'
    }), mimetype='application/json')

@app.route('/dashboard/event/<event_id>/retrieve')
def dashboard_retrieve(event_id):
    event = db.session.query(Event).filter(
        Event.id == event_id
    ).first()

    if event.fb_post_id is None:
        return Response(json.dumps({
            'questions': []
        }), mimetype='application/json')

    page = db.session.query(Page).filter(
        Page.id == event.page_id
    ).first()

    params = {
        'access_token': page.token
    }
    params = urllib.urlencode(params)
    questions = json.loads(
        urllib.urlopen('https://graph.facebook.com/' + event.fb_post_id + '/comments?%s' % params).read()
    )

    for question in questions['data']:
        new_question = {
            'fb_id':    question['id'],
            'user':     question['from']['id'],
            'question': question['message'],
            'event_id': event.id
        }

        db_q = db.session.query(Question).filter(
            Question.fb_id == question['id']
        ).first()

        if not db_q:
            groups = db.session.query(Group).filter(
                Group.event_id == event.id
            ).all()

            g = None
            if groups:
                g = match_group(new_question['question'], groups, 0.9)

            if g is None:
                g = {
                    'event_id': event.id,
                    'question': new_question['question']
                }

                # Get Sentiment
                params = {
                    'apikey':     '2ccd6f653c1e4253b6ac5ee0dadb284bde58331e',
                    'text':       new_question['question'],
                    'outputMode': 'json'
                }
                params = urllib.urlencode(params)

                sentiment = json.loads(
                    urllib.urlopen('http://access.alchemyapi.com/calls/text/TextGetTextSentiment?%s' % params).read()
                )

                if 'docSentiment' in sentiment and 'type' in sentiment['docSentiment']:
                    if sentiment['docSentiment']['type']:
                        g['sentiment'] = sentiment['docSentiment']['type']

                g = Group(**g)

                db.session.add(g)
                db.session.commit()

            new_question['group_id'] = g.id

            q = Question(**new_question)

            db.session.add(q)
            db.session.commit()

    return Response(json.dumps({
        'status': questions
    }), mimetype='application/json')

@app.route('/dashboard/event/<event_id>/delete')
def dashboard_delete(event_id):
    event = db.session.query(Event).filter(
        Event.id == event_id
    ).first()

    page = db.session.query(Page).filter(
        Page.id == event.page_id
    ).first()

    db.session.delete(event)
    db.session.commit()

    return redirect('/dashboard/page/%s' % page.page_id)

@app.route('/dashboard/event/<event_id>/polling')
def dashboard_polling(event_id):
    event = db.session.query(Event).filter(
        Event.id == event_id
    ).first()

    page = db.session.query(Page).filter(
        Page.id == event.page_id
    ).first()

    dashboard_retrieve(event_id)

    aGroups = db.session.query(Group).filter(
        Group.event_id == event_id
    ).filter(
        Group.status == 1
    ).all()

    uGroups = db.session.query(Group).filter(
        Group.event_id == event_id
    ).filter(
        Group.status == 0
    ).all()

    questions = db.session.query(Question).filter(
        Question.event_id == event.id
    ).count()

    users = db.session.query(Question).filter(
        Question.event_id == event.id
    ).group_by(
        Question.user
    ).count()

    return Response(json.dumps({
        'questionsNumber': questions,
        'usersOverview': users,
        'answeredQuestions': [i.serialize for i in aGroups],
        'unansweredQuestions': [i.serialize for i in uGroups],
        'totalPositive': 0,
        'totalNegative': 0,
        'totalNeutral': 0
    }), mimetype='application/json')

@app.route('/dashboard/event/<event_id>/publish')
def dashboard_publish(event_id):
    event = db.session.query(Event).filter(
        Event.id == event_id
    ).first()

    page = db.session.query(Page).filter(
        Page.id == event.page_id
    ).first()

    params = {
        'access_token':       page.token,
        'to':                 page.page_id,
        'message':            'This is an awesome post. Deal with it!',
        'format':             'json',
        'suppress_http_code': 1,
        'method':             'post'
    }
    params = urllib.urlencode(params)

    data = json.loads(
        urllib.urlopen('https://graph.facebook.com/' + page.page_id + '/feed?%s' % params).read()
    )

    if 'id' in data:
        event.fb_post_id = data['id']
        event.status = 1
        db.session.commit()

    return redirect('/dashboard/page/%s' % page.page_id)

@app.route('/dashboard/new', methods=['POST'])
def dashboard_new():
    form = request.form

    page_id = form['session-page']

    page = db.session.query(Page).filter(
        Page.page_id == page_id
    ).first()

    new_event = {
        'page_id': page.id,
        'title': form['session-title'],
        'message': 'This is a random message'
    }

    if form['session-start-date']:
        new_event['start_date'] = datetime.datetime.strptime(form['session-start-date'], '%Y/%m/%d %H:%M')

    if form['session-end-date']:
        new_event['end_date'] = datetime.datetime.strptime(form['session-end-date'], '%Y/%m/%d %H:%M')

    db_event = Event(**new_event)

    db.session.add(db_event)
    db.session.commit()

    return redirect('/dashboard/page/%s' % page_id)

@app.route('/dashboard/page/<page_id>/<access_token>')
def dashboard_page_token(page_id, access_token):
    fb_page = {
        'user_id': session['USER_ID'],
        'page_id': page_id,
        'token': access_token
    }

    db_page = db.session.query(Page).filter(
        Page.page_id == page_id
    ).first()

    if not db_page:
        db_page = Page(**fb_page)

        db.session.add(db_page)
    else:
        db_page.token = access_token

    session['PAGE_TOKEN'] = access_token

    db.session.commit()

    return redirect('/dashboard/page/%s' % page_id)
<|endoftext|>


<commit_before>import datetime

from ..web import base
from .. import util
from .. import validators
from ..auth import groupauth
from ..dao import containerstorage
from .containerhandler import ContainerHandler

class GroupHandler(base.RequestHandler):

    def __init__(self, request=None, response=None):
        super(GroupHandler, self).__init__(request, response)
        self.storage = containerstorage.GroupStorage()

    def get(self, _id):
        group = self._get_group(_id)
        permchecker = groupauth.default(self, group)
        result = permchecker(self.storage.exec_op)('GET', _id)
        if not self.superuser_request and not self.is_true('join_avatars'):
            self._filter_roles([result], self.uid, self.user_site)
        if self.is_true('join_avatars'):
            ContainerHandler.join_user_info([result])
        return result

    def delete(self, _id):
        if _id == 'unknown':
            self.abort(400, 'The group "unknown" can\'t be deleted as it is integral within the API')
        group = self._get_group(_id)
        permchecker = groupauth.default(self, group)
        result = permchecker(self.storage.exec_op)('DELETE', _id)
        if result.deleted_count == 1:
            return {'deleted': result.deleted_count}
        else:
            self.abort(404, 'Group {} not removed'.format(_id))
        return result

    def get_all(self, uid=None):
        projection = {'name': 1, 'created': 1, 'modified': 1}
        permchecker = groupauth.list_permission_checker(self, uid)
        results = permchecker(self.storage.exec_op)('GET', projection=projection)
        if not self.superuser_request and not self.is_true('join_avatars'):
            self._filter_roles(results, self.uid, self.user_site)
        if self.is_true('join_avatars'):
            results = ContainerHandler.join_user_info(results)
        return results

    def put(self, _id):
        group = self._get_group(_id)
        permchecker = groupauth.default(self, group)
        payload = self.request.json_body
        mongo_schema_uri = validators.schema_uri('mongo', 'group.json')
        mongo_validator = validators.decorator_from_schema_path(mongo_schema_uri)
        payload_schema_uri = validators.schema_uri('input', 'group-update.json')
        payload_validator = validators.from_schema_path(payload_schema_uri)
        payload_validator(payload, 'PUT')
        payload['modified'] = datetime.datetime.utcnow()
        result = mongo_validator(permchecker(self.storage.exec_op))('PUT', _id=_id, payload=payload)
        if result.modified_count == 1:
            return {'modified': result.modified_count}
        else:
            self.abort(404, 'Group {} not updated'.format(_id))

    def post(self):
        permchecker = groupauth.default(self, None)
        payload = self.request.json_body
        mongo_schema_uri = validators.schema_uri('mongo', 'group.json')
        mongo_validator = validators.decorator_from_schema_path(mongo_schema_uri)
        payload_schema_uri = validators.schema_uri('input', 'group-new.json')
        payload_validator = validators.from_schema_path(payload_schema_uri)
        payload_validator(payload, 'POST')
        payload['created'] = payload['modified'] = datetime.datetime.utcnow()
        payload['roles'] = [{'_id': self.uid, 'access': 'admin', 'site': self.user_site}] if self.uid else []
        result = mongo_validator(permchecker(self.storage.exec_op))('POST', payload=payload)
        if result.acknowledged:
            if result.upserted_id:
                return {'_id': result.upserted_id}
            else:
                self.response.status_int = 201
                return {'_id': payload['_id']}
        else:
            self.abort(404, 'Group {} not updated'.format(payload['_id']))

    def _get_group(self, _id):
        group = self.storage.get_container(_id)
        if group is not None:
            return group
        else:
            self.abort(404, 'Group {} not found'.format(_id))

    def _filter_roles(self, results, uid, site):
        """
        if the user is not admin only her role is returned.
        """
        for result in results:
            user_perm = util.user_perm(result.get('roles', []), uid, site)
            if user_perm.get('access') != 'admin':
                result['roles'] = [user_perm] if user_perm else []
        return results
<commit_msg>re-add roles and tags to group projection<commit_after>import datetime

from ..web import base
from .. import util
from .. import validators
from ..auth import groupauth
from ..dao import containerstorage
from .containerhandler import ContainerHandler

class GroupHandler(base.RequestHandler):

    def __init__(self, request=None, response=None):
        super(GroupHandler, self).__init__(request, response)
        self.storage = containerstorage.GroupStorage()

    def get(self, _id):
        group = self._get_group(_id)
        permchecker = groupauth.default(self, group)
        result = permchecker(self.storage.exec_op)('GET', _id)
        if not self.superuser_request and not self.is_true('join_avatars'):
            self._filter_roles([result], self.uid, self.user_site)
        if self.is_true('join_avatars'):
            ContainerHandler.join_user_info([result])
        return result

    def delete(self, _id):
        if _id == 'unknown':
            self.abort(400, 'The group "unknown" can\'t be deleted as it is integral within the API')
        group = self._get_group(_id)
        permchecker = groupauth.default(self, group)
        result = permchecker(self.storage.exec_op)('DELETE', _id)
        if result.deleted_count == 1:
            return {'deleted': result.deleted_count}
        else:
            self.abort(404, 'Group {} not removed'.format(_id))
        return result

    def get_all(self, uid=None):
        projection = {'name': 1, 'created': 1, 'modified': 1, 'roles': 1, 'tags': 1}
        permchecker = groupauth.list_permission_checker(self, uid)
        results = permchecker(self.storage.exec_op)('GET', projection=projection)
        if not self.superuser_request and not self.is_true('join_avatars'):
            self._filter_roles(results, self.uid, self.user_site)
        if self.is_true('join_avatars'):
            results = ContainerHandler.join_user_info(results)
        return results

    def put(self, _id):
        group = self._get_group(_id)
        permchecker = groupauth.default(self, group)
        payload = self.request.json_body
        mongo_schema_uri = validators.schema_uri('mongo', 'group.json')
        mongo_validator = validators.decorator_from_schema_path(mongo_schema_uri)
        payload_schema_uri = validators.schema_uri('input', 'group-update.json')
        payload_validator = validators.from_schema_path(payload_schema_uri)
        payload_validator(payload, 'PUT')
        payload['modified'] = datetime.datetime.utcnow()
        result = mongo_validator(permchecker(self.storage.exec_op))('PUT', _id=_id, payload=payload)
        if result.modified_count == 1:
            return {'modified': result.modified_count}
        else:
            self.abort(404, 'Group {} not updated'.format(_id))

    def post(self):
        permchecker = groupauth.default(self, None)
        payload = self.request.json_body
        mongo_schema_uri = validators.schema_uri('mongo', 'group.json')
        mongo_validator = validators.decorator_from_schema_path(mongo_schema_uri)
        payload_schema_uri = validators.schema_uri('input', 'group-new.json')
        payload_validator = validators.from_schema_path(payload_schema_uri)
        payload_validator(payload, 'POST')
        payload['created'] = payload['modified'] = datetime.datetime.utcnow()
        payload['roles'] = [{'_id': self.uid, 'access': 'admin', 'site': self.user_site}] if self.uid else []
        result = mongo_validator(permchecker(self.storage.exec_op))('POST', payload=payload)
        if result.acknowledged:
            if result.upserted_id:
                return {'_id': result.upserted_id}
            else:
                self.response.status_int = 201
                return {'_id': payload['_id']}
        else:
            self.abort(404, 'Group {} not updated'.format(payload['_id']))

    def _get_group(self, _id):
        group = self.storage.get_container(_id)
        if group is not None:
            return group
        else:
            self.abort(404, 'Group {} not found'.format(_id))

    def _filter_roles(self, results, uid, site):
        """
        if the user is not admin only her role is returned.
        """
        for result in results:
            user_perm = util.user_perm(result.get('roles', []), uid, site)
            if user_perm.get('access') != 'admin':
                result['roles'] = [user_perm] if user_perm else []
        return results
<|endoftext|>


<commit_before>import ConfigParser
import datetime
import json
import logging
import MySQLdb
import os
import urllib2

from dirq.queue import Queue, QueueError
from rest_framework.pagination import PaginationSerializer
from django.conf import settings
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from rest_framework.response import Response
from rest_framework.views import APIView

class CloudRecordView(APIView):
    """
    Submit Cloud Accounting Records.
    .../cloud/record

    Will save Cloud Accounting Records for later loading.
    """

    def post(self, request, format=None):
        """
        Submit Cloud Accounting Records.

        .../cloud/record

        Will save Cloud Accounting Records for later loading.
        """
        logger = logging.getLogger(__name__)

        try:
            empaid = request.META['HTTP_EMPA_ID']
        except KeyError:
            empaid = 'noid'

        logger.info("Received message. ID = %s", empaid)

        try:
            signer = request.META['SSL_CLIENT_S_DN']
        except KeyError:
            logger.error("No DN supplied in header")
            return Response(status=401)

        # authorise DNs here
        if not self._signer_is_valid(signer):
            logger.error("%s not a valid provider" % signer)
            return Response(status=403)

        if "_content" in request.POST.dict():
            # then POST likely to come via the rest api framework
            # hence use the content of request.POST as message
            body = request.POST.get('_content')

        else:
            # POST likely to comes through a browser client or curl
            # hence use request.body as message
            body = request.body

        logger.debug("Message body received: %s", body)

        for header in request.META:
            logger.debug("%s: %s", header, request.META[header])

        # taken from ssm2
        QSCHEMA = {'body': 'string',
                   'signer': 'string',
                   'empaid': 'string?'}

        inqpath = os.path.join(settings.QPATH, 'incoming')
        inq = Queue(inqpath, schema=QSCHEMA)

        try:
            name = inq.add({'body': body,
                            'signer': signer,
                            'empaid': empaid})
        except QueueError as err:
            logger.error("Could not save %s/%s: %s", inqpath, name, err)

            response = "Data could not be saved to disk, please try again."
            return Response(response, status=500)

        logger.info("Message saved to in queue as %s/%s", inqpath, name)

        response = "Data successfully saved for future loading."
        return Response(response, status=202)

###############################################################################
#                                                                             #
# Helper methods                                                              #
#                                                                             #
###############################################################################

    def _signer_is_valid(self, signer):
        site_list = urllib2.Request('http://indigo.cloud.plgrid.pl/cmdb/service/list')
        site_list = urllib2.urlopen(site_list)

        site_json = json.loads(site_list.read())

        for site in range(len(site_json['rows'])):
            try:
                if signer in site_json['rows'][site]['value']['hostname']:
                    return True
            except KeyError:
                pass

        # If we have not returned while in for loop
        # then site must be invalid
        return False
<commit_msg>Raise an exception if site data cannot be found<commit_after>import ConfigParser
import datetime
import httplib
import json
import logging
import MySQLdb
import os
import urllib2

from dirq.queue import Queue, QueueError
from rest_framework.pagination import PaginationSerializer
from django.conf import settings
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from rest_framework.response import Response
from rest_framework.views import APIView

class CloudRecordView(APIView):
    """
    Submit Cloud Accounting Records.
    .../cloud/record

    Will save Cloud Accounting Records for later loading.
    """

    def post(self, request, format=None):
        """
        Submit Cloud Accounting Records.

        .../cloud/record

        Will save Cloud Accounting Records for later loading.
        """
        logger = logging.getLogger(__name__)

        try:
            empaid = request.META['HTTP_EMPA_ID']
        except KeyError:
            empaid = 'noid'

        logger.info("Received message. ID = %s", empaid)

        try:
            signer = request.META['SSL_CLIENT_S_DN']
        except KeyError:
            logger.error("No DN supplied in header")
            return Response(status=401)

        # authorise DNs here
        if not self._signer_is_valid(signer):
            logger.error("%s not a valid provider" % signer)
            return Response(status=403)

        if "_content" in request.POST.dict():
            # then POST likely to come via the rest api framework
            # hence use the content of request.POST as message
            body = request.POST.get('_content')

        else:
            # POST likely to comes through a browser client or curl
            # hence use request.body as message
            body = request.body

        logger.debug("Message body received: %s", body)

        for header in request.META:
            logger.debug("%s: %s", header, request.META[header])

        # taken from ssm2
        QSCHEMA = {'body': 'string',
                   'signer': 'string',
                   'empaid': 'string?'}

        inqpath = os.path.join(settings.QPATH, 'incoming')
        inq = Queue(inqpath, schema=QSCHEMA)

        try:
            name = inq.add({'body': body,
                            'signer': signer,
                            'empaid': empaid})
        except QueueError as err:
            logger.error("Could not save %s/%s: %s", inqpath, name, err)

            response = "Data could not be saved to disk, please try again."
            return Response(response, status=500)

        logger.info("Message saved to in queue as %s/%s", inqpath, name)

        response = "Data successfully saved for future loading."
        return Response(response, status=202)

###############################################################################
#                                                                             #
# Helper methods                                                              #
#                                                                             #
###############################################################################

    def _signer_is_valid(self, signer):
        logger = logging.getLogger(__name__)

        site_list = urllib2.Request('http://indigo.cloud.plgrid.pl/cmdb/service/list')
        site_list = urllib2.urlopen(site_list)

        try:
            site_json = json.loads(site_list.read())
        except ValueError:
            logger.error("List of providers could not be retrieved.")
            return False

        for site in range(len(site_json['rows'])):
            try:
                if signer in site_json['rows'][site]['value']['hostname']:
                    return True
            except KeyError:
                pass

        # If we have not returned while in for loop
        # then site must be invalid
        logger.info('Site is not found on list of providers')
        return False
<|endoftext|>