<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""Este módulo contiene al bot y lo ejecuta si se corre el script."""

import platform
import re
import sqlite3
import sys

import aiohttp
import discord
import peewee
import yaml

import modules.commands
from modules import logger
from modules.base.command import Command
from modules.reaction_hook import reaction_hook

__author__ = 'Nicolás Santisteban, Jonathan Gutiérrez'
__license__ = 'MIT'
__version__ = '0.5.0-dev.0'
__status__ = "Desarrollo"

class Alexis(discord.Client):
    """Contiene al bot e inicializa su funcionamiento."""
    def __init__(self, **options):
        super().__init__(**options)
        self.http_session = aiohttp.ClientSession(loop=self.loop)

        self.log = logger.get_logger('Alexis')
        self.initialized = False
        self.config = {}
        self.sharedcfg = {}
        self.cmds = {}
        self.swhandlers = {}
        self.mention_handlers = []

        self.db = peewee.SqliteDatabase('database.db')
        self.db.connect()

        self.load_config()

        # Regex de mención (incluye nicks)
        self.rx_mention = None

        # El ID del último en enviar un mensaje (omite PM)
        self.last_author = None

    """Inicializa al bot"""
    def init(self):
        self.log.info('"Alexis Bot" versión %s de %s.', __version__, __status__.lower())
        self.log.info('Python %s en %s.', sys.version, sys.platform)
        self.log.info(platform.uname())
        self.log.info('Soporte SQLite3 para versión %s.', sqlite3.sqlite_version)
        self.log.info('discord.py versión %s.', discord.__version__)
        self.log.info('------')

        # Cargar (instanciar clases de) comandos
        self.log.debug('Cargando comandos...')
        cmd_instances = []
        db_models = []

        for cmd in modules.commands.classes:
            cmd_instances.append(cmd(self))

        # Guardar instancias de módulos de comandos
        for i in cmd_instances:
            db_models += i.db_models

            # Comandos
            names = [i.name] if isinstance(i.name, str) else list(i.name)
            for name in names:
                name = name.strip()
                if name == '':
                    continue

                if name not in self.cmds:
                    self.cmds[name] = []

                self.cmds[name].append(i)

            # Handlers startswith
            if isinstance(i.swhandler, str) or isinstance(i.swhandler, list):
                swh = [i.swhandler] if isinstance(i.swhandler, str) else i.swhandler
                for swtext in swh:
                    swtext = swtext.strip()
                    if swtext == '':
                        continue

                    if swtext not in self.swhandlers:
                        self.swhandlers[swtext] = []

                    self.swhandlers[swtext].append(i)

            # Comandos que se activan con una mención
            if isinstance(i.mention_handler, bool) and i.mention_handler:
                self.mention_handlers.append(i)

            # Tasks
            if i.run_task:
                self.loop.create_task(i.task())

        self.log.info('Inicializando base de datos...')
        self.db.create_tables(db_models, True)

        self.log.debug('Comandos cargados: ' + ', '.join(self.cmds.keys()))
        self.log.info('Conectando...')

        try:
            self.run(self.config['token'])
        except Exception as ex:
            self.log.exception(ex)
            raise

    """Esto se ejecuta cuando el bot está conectado y listo"""
    async def on_ready(self):
        self.log.info('Conectado como:')
        self.log.info(self.user.name)
        self.log.info(self.user.id)
        self.log.info('------')
        await self.change_presence(game=discord.Game(name=self.config['playing']))

        self.rx_mention = re.compile('^<@!?{}>'.format(self.user.id))
        self.initialized = True

    """Método ejecutado cada vez que se recibe un mensaje"""
    async def on_message(self, message):
        if not self.initialized:
            return

        # Info sobre el mensaje
        text = message.content
        author = Command.final_name(message.author)
        chan = message.channel
        is_pm = message.server is None
        is_owner = self.is_owner(message.author, message.server)
        own_message = message.author.id == self.user.id

        # Mandar PMs al log
        if is_pm:
            if own_message:
                self.log.info('[PM] (-> %s): %s', message.channel.user, text)
            else:
                self.log.info('[PM] %s: %s', author, text)

        # Command handler
        if text.startswith(self.config['command_prefix']) and len(text) > 1:
            cmd = text.split(' ')[0][1:]
            if cmd in self.cmds:
                self.log.debug('[command] %s sent message: "%s" command %s', message.author, text, cmd)
                for i in self.cmds[cmd]:
                    if i.owner_only and not is_owner:
                        await self.send_message(chan, i.owner_error)
                    elif not i.allow_pm and is_pm:
                        await self.send_message(chan, i.pm_error)
                    else:
                        await i.handle(message, i.parse(message))
                return

        # 'startswith' handlers
        for swtext in self.swhandlers.keys():
            if message.content.startswith(swtext):
                for cmd in self.swhandlers[swtext]:
                    if cmd.owner_only and not is_owner:
                        await self.send_message(chan, cmd.owner_error)
                    elif not cmd.allow_pm and is_pm:
                        await self.send_message(chan, cmd.pm_error)
                    else:
                        await cmd.handle(message, cmd.parse(message))

        # Mention handlers
        if self.user.mentioned_in(message):
            for i in self.mention_handlers:
                if i.owner_only and not is_owner:
                    await self.send_message(chan, i.owner_error)
                elif not i.allow_pm and is_pm:
                    await self.send_message(chan, i.pm_error)
                else:
                    await i.handle(message, i.parse(message))

    """Esta función es llamada cuando un mensaje recibe una reacción"""
    async def on_reaction_add(self, reaction, user):
        await reaction_hook(self, reaction, user)

    def load_config(self):
        try:
            with open('config.yml', 'r') as file:
                config = yaml.safe_load(file)

            # Completar info con defaults
            if 'owners' not in config:
                config['owners'] = []
            if 'default_memes' not in config:
                config['default_memes'] = []
            if 'frases' not in config:
                config['frases'] = []
            if 'starboard_reactions' not in config or not isinstance(config['starboard_reactions'], int):
                config['starboard_reactions'] = 5
            if 'command_prefix' not in config or not isinstance(config['command_prefix'], str):
                config['command_prefix'] = '!'

            self.config = config
            return True
        except Exception as ex:
            self.log.exception(ex)
            return False

    def is_owner(self, member, server):
        if server is None:
            return False

        if member.id in self.config['owners']:
            return True

        for role in member.roles:
            owner_role = server.id + "@" + role.id
            if owner_role in self.config['owners']:
                return True

        return False

    async def send_message(self, destination, content=None, **kwargs):
        svid = destination.server.id if isinstance(destination, discord.Channel) else 'PM?'
        msg = 'Sending message "{}" to {}#{}'.format(content, svid, destination)
        if isinstance(kwargs.get('embed'), discord.Embed):
            msg += ' (with embed: {})'.format(kwargs.get('embed').to_dict())

        self.log.debug(msg)
        await super(Alexis, self).send_message(destination, content, **kwargs)

if __name__ == '__main__':
    ale = Alexis()
    ale.init()
    ale.http_session.close()
<commit_msg>Cambiar versión<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""Este módulo contiene al bot y lo ejecuta si se corre el script."""

import platform
import re
import sqlite3
import sys

import aiohttp
import discord
import peewee
import yaml

import modules.commands
from modules import logger
from modules.base.command import Command
from modules.reaction_hook import reaction_hook

__author__ = 'Nicolás Santisteban, Jonathan Gutiérrez'
__license__ = 'MIT'
__version__ = '0.5.1'
__status__ = "Desarrollo"

class Alexis(discord.Client):
    """Contiene al bot e inicializa su funcionamiento."""
    def __init__(self, **options):
        super().__init__(**options)
        self.http_session = aiohttp.ClientSession(loop=self.loop)

        self.log = logger.get_logger('Alexis')
        self.initialized = False
        self.config = {}
        self.sharedcfg = {}
        self.cmds = {}
        self.swhandlers = {}
        self.mention_handlers = []

        self.db = peewee.SqliteDatabase('database.db')
        self.db.connect()

        self.load_config()

        # Regex de mención (incluye nicks)
        self.rx_mention = None

        # El ID del último en enviar un mensaje (omite PM)
        self.last_author = None

    """Inicializa al bot"""
    def init(self):
        self.log.info('"Alexis Bot" versión %s de %s.', __version__, __status__.lower())
        self.log.info('Python %s en %s.', sys.version, sys.platform)
        self.log.info(platform.uname())
        self.log.info('Soporte SQLite3 para versión %s.', sqlite3.sqlite_version)
        self.log.info('discord.py versión %s.', discord.__version__)
        self.log.info('------')

        # Cargar (instanciar clases de) comandos
        self.log.debug('Cargando comandos...')
        cmd_instances = []
        db_models = []

        for cmd in modules.commands.classes:
            cmd_instances.append(cmd(self))

        # Guardar instancias de módulos de comandos
        for i in cmd_instances:
            db_models += i.db_models

            # Comandos
            names = [i.name] if isinstance(i.name, str) else list(i.name)
            for name in names:
                name = name.strip()
                if name == '':
                    continue

                if name not in self.cmds:
                    self.cmds[name] = []

                self.cmds[name].append(i)

            # Handlers startswith
            if isinstance(i.swhandler, str) or isinstance(i.swhandler, list):
                swh = [i.swhandler] if isinstance(i.swhandler, str) else i.swhandler
                for swtext in swh:
                    swtext = swtext.strip()
                    if swtext == '':
                        continue

                    if swtext not in self.swhandlers:
                        self.swhandlers[swtext] = []

                    self.swhandlers[swtext].append(i)

            # Comandos que se activan con una mención
            if isinstance(i.mention_handler, bool) and i.mention_handler:
                self.mention_handlers.append(i)

            # Tasks
            if i.run_task:
                self.loop.create_task(i.task())

        self.log.info('Inicializando base de datos...')
        self.db.create_tables(db_models, True)

        self.log.debug('Comandos cargados: ' + ', '.join(self.cmds.keys()))
        self.log.info('Conectando...')

        try:
            self.run(self.config['token'])
        except Exception as ex:
            self.log.exception(ex)
            raise

    """Esto se ejecuta cuando el bot está conectado y listo"""
    async def on_ready(self):
        self.log.info('Conectado como:')
        self.log.info(self.user.name)
        self.log.info(self.user.id)
        self.log.info('------')
        await self.change_presence(game=discord.Game(name=self.config['playing']))

        self.rx_mention = re.compile('^<@!?{}>'.format(self.user.id))
        self.initialized = True

    """Método ejecutado cada vez que se recibe un mensaje"""
    async def on_message(self, message):
        if not self.initialized:
            return

        # Info sobre el mensaje
        text = message.content
        author = Command.final_name(message.author)
        chan = message.channel
        is_pm = message.server is None
        is_owner = self.is_owner(message.author, message.server)
        own_message = message.author.id == self.user.id

        # Mandar PMs al log
        if is_pm:
            if own_message:
                self.log.info('[PM] (-> %s): %s', message.channel.user, text)
            else:
                self.log.info('[PM] %s: %s', author, text)

        # Command handler
        if text.startswith(self.config['command_prefix']) and len(text) > 1:
            cmd = text.split(' ')[0][1:]
            if cmd in self.cmds:
                self.log.debug('[command] %s sent message: "%s" command %s', message.author, text, cmd)
                for i in self.cmds[cmd]:
                    if i.owner_only and not is_owner:
                        await self.send_message(chan, i.owner_error)
                    elif not i.allow_pm and is_pm:
                        await self.send_message(chan, i.pm_error)
                    else:
                        await i.handle(message, i.parse(message))
                return

        # 'startswith' handlers
        for swtext in self.swhandlers.keys():
            if message.content.startswith(swtext):
                for cmd in self.swhandlers[swtext]:
                    if cmd.owner_only and not is_owner:
                        await self.send_message(chan, cmd.owner_error)
                    elif not cmd.allow_pm and is_pm:
                        await self.send_message(chan, cmd.pm_error)
                    else:
                        await cmd.handle(message, cmd.parse(message))

        # Mention handlers
        if self.user.mentioned_in(message):
            for i in self.mention_handlers:
                if i.owner_only and not is_owner:
                    await self.send_message(chan, i.owner_error)
                elif not i.allow_pm and is_pm:
                    await self.send_message(chan, i.pm_error)
                else:
                    await i.handle(message, i.parse(message))

    """Esta función es llamada cuando un mensaje recibe una reacción"""
    async def on_reaction_add(self, reaction, user):
        await reaction_hook(self, reaction, user)

    def load_config(self):
        try:
            with open('config.yml', 'r') as file:
                config = yaml.safe_load(file)

            # Completar info con defaults
            if 'owners' not in config:
                config['owners'] = []
            if 'default_memes' not in config:
                config['default_memes'] = []
            if 'frases' not in config:
                config['frases'] = []
            if 'starboard_reactions' not in config or not isinstance(config['starboard_reactions'], int):
                config['starboard_reactions'] = 5
            if 'command_prefix' not in config or not isinstance(config['command_prefix'], str):
                config['command_prefix'] = '!'

            self.config = config
            return True
        except Exception as ex:
            self.log.exception(ex)
            return False

    def is_owner(self, member, server):
        if server is None:
            return False

        if member.id in self.config['owners']:
            return True

        for role in member.roles:
            owner_role = server.id + "@" + role.id
            if owner_role in self.config['owners']:
                return True

        return False

    async def send_message(self, destination, content=None, **kwargs):
        svid = destination.server.id if isinstance(destination, discord.Channel) else 'PM?'
        msg = 'Sending message "{}" to {}#{}'.format(content, svid, destination)
        if isinstance(kwargs.get('embed'), discord.Embed):
            msg += ' (with embed: {})'.format(kwargs.get('embed').to_dict())

        self.log.debug(msg)
        await super(Alexis, self).send_message(destination, content, **kwargs)

if __name__ == '__main__':
    ale = Alexis()
    ale.init()
    ale.http_session.close()
<|endoftext|>


<commit_before>from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data
from stp_core.loop.eventually import eventually
from plenum.server.monitor import Monitor
from plenum.test.helper import sdk_send_random_and_check

def testPostingThroughput(postingStatsEnabled,
                          decreasedMonitoringTimeouts,
                          looper,
                          txnPoolNodeSet,
                          sdk_wallet_client, sdk_pool_handle):
    """
    The throughput after `DashboardUpdateFreq` seconds and before sending any
    requests should be zero.
    Send `n` requests in less than `ThroughputWindowSize` seconds and the
    throughput till `ThroughputWindowSize` should consider those `n` requests.
    After `ThroughputWindowSize` seconds the throughput should be zero
    Test `totalRequests` too.
    """

    config = decreasedMonitoringTimeouts

    # We are sleeping for this window size, because we need to clear previous
    # values that were being stored for this much time in tests
    looper.runFor(config.ThroughputWindowSize)

    reqCount = 10
    for node in txnPoolNodeSet:
        assert node.monitor.highResThroughput == 0
        assert node.monitor.totalRequests == 0

    sdk_send_random_and_check(looper,
                              txnPoolNodeSet,
                              sdk_pool_handle,
                              sdk_wallet_client,
                              reqCount)
    ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)

    for node in txnPoolNodeSet:
        assert len(node.monitor.orderedRequestsInLast) == reqCount
        assert node.monitor.highResThroughput > 0
        assert node.monitor.totalRequests == reqCount
        # TODO: Add implementation to actually call firebase plugin
        # and test if firebase plugin is sending total request count
        # if node is primary

    looper.runFor(config.DashboardUpdateFreq)

    for node in txnPoolNodeSet:
        node.monitor.spylog.count(Monitor.sendThroughput.__name__) > 0

    # Run for latency window duration so that `orderedRequestsInLast`
    # becomes empty
    looper.runFor(config.ThroughputWindowSize)

    def chk():
        for node in txnPoolNodeSet:
            assert len(node.monitor.orderedRequestsInLast) == 0
            assert node.monitor.highResThroughput == 0
            assert node.monitor.totalRequests == reqCount

    timeout = config.ThroughputWindowSize
    looper.run(eventually(chk, retryWait=1, timeout=timeout))

def testPostingLatency(postingStatsEnabled,
                       decreasedMonitoringTimeouts,
                       looper,
                       txnPoolNodeSet,
                       sdk_wallet_client, sdk_pool_handle):
    """
    The latencies (master as well as average of backups) after
    `DashboardUpdateFreq` seconds and before sending any requests should be zero.
    Send `n` requests in less than `LatencyWindowSize` seconds and the
    latency till `LatencyWindowSize` should consider those `n` requests.
    After `LatencyWindowSize` seconds the latencies should be zero
    """

    config = decreasedMonitoringTimeouts

    # Run for latency window duration so that `latenciesByMasterInLast` and
    # `latenciesByBackupsInLast` become empty
    looper.runFor(config.LatencyWindowSize)
    reqCount = 10
    for node in txnPoolNodeSet:
        assert node.monitor.masterLatency == 0
        assert node.monitor.avgBackupLatency == 0

    sdk_send_random_and_check(looper,
                              txnPoolNodeSet,
                              sdk_pool_handle,
                              sdk_wallet_client,
                              reqCount)

    for node in txnPoolNodeSet:
        assert node.monitor.masterLatency > 0
        assert node.monitor.avgBackupLatency > 0

    looper.runFor(config.DashboardUpdateFreq)

    for node in txnPoolNodeSet:
        node.monitor.spylog.count(Monitor.sendLatencies.__name__) > 0

    # Run for latency window duration so that `latenciesByMasterInLast` and
    # `latenciesByBackupsInLast` become empty
    looper.runFor(config.LatencyWindowSize)

    def chk():
        for node in txnPoolNodeSet:
            assert node.monitor.masterLatency == 0
            assert node.monitor.avgBackupLatency == 0

    timeout = config.LatencyWindowSize
    looper.run(eventually(chk, retryWait=1, timeout=timeout))
<commit_msg>[New SDK] Try to fix testPostingThroughput<commit_after>import pytest
from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data
from stp_core.loop.eventually import eventually
from plenum.server.monitor import Monitor
from plenum.test.helper import sdk_send_random_and_check

@pytest.fixture(scope='module')
def tconf(tconf):
    old_val = tconf.Max3PCBatchSize
    tconf.Max3PCBatchSize = 1
    yield tconf
    tconf.Max3PCBatchSize = old_val

def testPostingThroughput(postingStatsEnabled,
                          decreasedMonitoringTimeouts,
                          looper,
                          txnPoolNodeSet,
                          sdk_wallet_client, sdk_pool_handle):
    """
    The throughput after `DashboardUpdateFreq` seconds and before sending any
    requests should be zero.
    Send `n` requests in less than `ThroughputWindowSize` seconds and the
    throughput till `ThroughputWindowSize` should consider those `n` requests.
    After `ThroughputWindowSize` seconds the throughput should be zero
    Test `totalRequests` too.
    """

    config = decreasedMonitoringTimeouts

    # We are sleeping for this window size, because we need to clear previous
    # values that were being stored for this much time in tests
    looper.runFor(config.ThroughputWindowSize)

    reqCount = 10
    for node in txnPoolNodeSet:
        assert node.monitor.highResThroughput == 0
        assert node.monitor.totalRequests == 0

    sdk_send_random_and_check(looper,
                              txnPoolNodeSet,
                              sdk_pool_handle,
                              sdk_wallet_client,
                              reqCount)
    ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet)

    for node in txnPoolNodeSet:
        assert len(node.monitor.orderedRequestsInLast) == reqCount
        assert node.monitor.highResThroughput > 0
        assert node.monitor.totalRequests == reqCount
        # TODO: Add implementation to actually call firebase plugin
        # and test if firebase plugin is sending total request count
        # if node is primary

    looper.runFor(config.DashboardUpdateFreq)

    for node in txnPoolNodeSet:
        node.monitor.spylog.count(Monitor.sendThroughput.__name__) > 0

    # Run for latency window duration so that `orderedRequestsInLast`
    # becomes empty
    looper.runFor(config.ThroughputWindowSize)

    def chk():
        for node in txnPoolNodeSet:
            assert len(node.monitor.orderedRequestsInLast) == 0
            assert node.monitor.highResThroughput == 0
            assert node.monitor.totalRequests == reqCount

    timeout = config.ThroughputWindowSize
    looper.run(eventually(chk, retryWait=1, timeout=timeout))

def testPostingLatency(postingStatsEnabled,
                       decreasedMonitoringTimeouts,
                       looper,
                       txnPoolNodeSet,
                       sdk_wallet_client, sdk_pool_handle):
    """
    The latencies (master as well as average of backups) after
    `DashboardUpdateFreq` seconds and before sending any requests should be zero.
    Send `n` requests in less than `LatencyWindowSize` seconds and the
    latency till `LatencyWindowSize` should consider those `n` requests.
    After `LatencyWindowSize` seconds the latencies should be zero
    """

    config = decreasedMonitoringTimeouts

    # Run for latency window duration so that `latenciesByMasterInLast` and
    # `latenciesByBackupsInLast` become empty
    looper.runFor(config.LatencyWindowSize)
    reqCount = 10
    for node in txnPoolNodeSet:
        assert node.monitor.masterLatency == 0
        assert node.monitor.avgBackupLatency == 0

    sdk_send_random_and_check(looper,
                              txnPoolNodeSet,
                              sdk_pool_handle,
                              sdk_wallet_client,
                              reqCount)

    for node in txnPoolNodeSet:
        assert node.monitor.masterLatency > 0
        assert node.monitor.avgBackupLatency > 0

    looper.runFor(config.DashboardUpdateFreq)

    for node in txnPoolNodeSet:
        node.monitor.spylog.count(Monitor.sendLatencies.__name__) > 0

    # Run for latency window duration so that `latenciesByMasterInLast` and
    # `latenciesByBackupsInLast` become empty
    looper.runFor(config.LatencyWindowSize)

    def chk():
        for node in txnPoolNodeSet:
            assert node.monitor.masterLatency == 0
            assert node.monitor.avgBackupLatency == 0

    timeout = config.LatencyWindowSize
    looper.run(eventually(chk, retryWait=1, timeout=timeout))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Run system tests for the project
"""
import os
import sys
from os.path import dirname
import importlib
import unittest
import logging

from drift.systesthelper import setup_tenant
from drift.utils import get_config

try:
    from teamcity import is_running_under_teamcity
    from teamcity.unittestpy import TeamcityTestRunner
except ImportError:
    is_running_under_teamcity = None
    TeamcityTestRunner = None

def get_options(parser):
    parser.add_argument(
        "--db",
        help="Database to create and retain for later tests",
        default=None
    )
    parser.add_argument(
        "--tests",
        help="Comma separated list of tests to run (partial match)",
        default=None
    )
    parser.add_argument(
        "--verbose",
        help="Verbose test output",
        action="store_true"
    )
    parser.add_argument(
        "--logging",
        help="Enable log output",
        action="store_true"
    )
    parser.add_argument(
        "--target",
        help="The address of the server to run the tests against",
        default=None
    )
    parser.add_argument(
        "--failfast",
        "-f",
        help="Exit once we encounter a failed test",
        default=False,
        action="store_true"
    )
    parser.add_argument(
        "--preview",
        "-p",
        help="Only list out which tests would be run.",
        action="store_true"
    )

def run_command(args):
    pick_tests = []
    if args.tests:
        pick_tests = [t.lower() for t in args.tests.split(",")]
        print "Picking tests {}".format(pick_tests)

    # Set up a mock tenant so we can bootstrap the app and inspect
    # the modules within.
    setup_tenant()
    conf = get_config()
    test_modules = []

    for app in conf.drift_app['apps']:
        m = importlib.import_module(app)
        path = dirname(m.__file__)
        tests_path = os.path.join(path, "tests")
        if not os.path.exists(tests_path):
            print "No tests found for app '{}'".format(app)
            continue
        if not os.path.exists(os.path.join(tests_path, "__init__.py")):
            print "No tests found for app '{}' (missing __init__.py)".format(app)
            continue
        n = 0
        for filename in os.listdir(tests_path):
            if filename.endswith(".py") and not filename.startswith("__"):
                test_module_name = app + ".tests." + filename[:-3]
                test_modules.append(test_module_name)
                n += 1
        print "app '{}' has {} test modules".format(app, n)

    suites = {}
    for module_name in test_modules:
        # first import it to see if we get any errors
        m = importlib.import_module(module_name)
        suites[module_name] = unittest.defaultTestLoader.loadTestsFromName(module_name)

    tests_to_run = []
    tests_to_skip = []
    for module_name, suite in suites.iteritems():
        for test_cases in suite:
            for t in test_cases:
                if pick_tests:
                    for p in pick_tests:
                        if p in str(t).lower():
                            tests_to_run.append(t)
                        else:
                            tests_to_skip.append(t)
                else:
                    tests_to_run.append(t)

    print "Running {} test(s) from {} module(s)".format(len(tests_to_run), len(suites))
    print "Skipping {} test(s)".format(len(tests_to_skip))
    if pick_tests:
        print "Just running the following tests:"
        if not tests_to_run:
            print "   No tests found!"
        for t in tests_to_run:
            print "   {}".format(t)

    if args.preview:
        return

    test_suite = unittest.TestSuite(tests_to_run)
    verbosity = 1
    if args.verbose:
        verbosity = 2

    if not args.logging:
        logging.disable(logging.WARNING)

    cls = unittest.TextTestRunner
    if is_running_under_teamcity and TeamcityTestRunner:
        if is_running_under_teamcity():
            cls = TeamcityTestRunner
    results = cls(verbosity=verbosity, failfast=args.failfast).run(test_suite)

    # if a tenant was not specified on the commandline we destroy it

    if not results.wasSuccessful():
        sys.exit(1)
<commit_msg>Fixing 'runsystest' logic so it jives with pytest.<commit_after># -*- coding: utf-8 -*-
"""
Run system tests for the project
"""
import os
import sys
from os.path import dirname
import importlib
import unittest
import logging

from drift.systesthelper import setup_tenant
from drift.utils import get_config

try:
    from teamcity import is_running_under_teamcity
    from teamcity.unittestpy import TeamcityTestRunner
except ImportError:
    is_running_under_teamcity = None
    TeamcityTestRunner = None

def get_options(parser):
    parser.add_argument(
        "--db",
        help="Database to create and retain for later tests",
        default=None
    )
    parser.add_argument(
        "--tests",
        help="Comma separated list of tests to run (partial match)",
        default=None
    )
    parser.add_argument(
        "--verbose",
        help="Verbose test output",
        action="store_true"
    )
    parser.add_argument(
        "--logging",
        help="Enable log output",
        action="store_true"
    )
    parser.add_argument(
        "--target",
        help="The address of the server to run the tests against",
        default=None
    )
    parser.add_argument(
        "--failfast",
        "-f",
        help="Exit once we encounter a failed test",
        default=False,
        action="store_true"
    )
    parser.add_argument(
        "--preview",
        "-p",
        help="Only list out which tests would be run.",
        action="store_true"
    )

def run_command(args):
    pick_tests = []
    if args.tests:
        pick_tests = [t.lower() for t in args.tests.split(",")]
        print "Picking tests {}".format(pick_tests)

    # Set up a mock tenant so we can bootstrap the app and inspect
    # the modules within.
    setup_tenant()
    conf = get_config()
    test_modules = []

    for app in conf.drift_app['apps']:
        m = importlib.import_module(app)
        path = dirname(m.__file__)
        tests_path = os.path.join(path, "tests")
        if not os.path.exists(tests_path):
            print "No tests found for app '{}'".format(app)
            continue
        if not os.path.exists(os.path.join(tests_path, "__init__.py")):
            print "No tests found for app '{}' (missing __init__.py)".format(app)
            continue
        n = 0
        for filename in os.listdir(tests_path):
            if filename.endswith(".py") and not filename.startswith("__"):
                test_module_name = app + ".tests." + filename[:-3]
                test_modules.append(test_module_name)
                n += 1
        print "app '{}' has {} test modules".format(app, n)

    suites = {}
    for module_name in test_modules[:]:
        # first import it to see if we get any errors
        try:
            m = importlib.import_module(module_name)
        except ImportError as e:
            # HACK: the 'runsystest' command will be deprecated in favour of pytest, but in
            # order to get pytest to work, some rearrangement of modules was neccessary which
            # blew up here. This work-around works well enough for the time being.
            if 'No module named' not in str(e):
                raise
        else:
            suites[module_name] = unittest.defaultTestLoader.loadTestsFromName(module_name)

    tests_to_run = []
    tests_to_skip = []
    for module_name, suite in suites.iteritems():
        for test_cases in suite:
            for t in test_cases:
                if pick_tests:
                    for p in pick_tests:
                        if p in str(t).lower():
                            tests_to_run.append(t)
                        else:
                            tests_to_skip.append(t)
                else:
                    tests_to_run.append(t)

    print "Running {} test(s) from {} module(s)".format(len(tests_to_run), len(suites))
    print "Skipping {} test(s)".format(len(tests_to_skip))
    if pick_tests:
        print "Just running the following tests:"
        if not tests_to_run:
            print "   No tests found!"
        for t in tests_to_run:
            print "   {}".format(t)

    if args.preview:
        return

    test_suite = unittest.TestSuite(tests_to_run)
    verbosity = 1
    if args.verbose:
        verbosity = 2

    if not args.logging:
        logging.disable(logging.WARNING)

    cls = unittest.TextTestRunner
    if is_running_under_teamcity and TeamcityTestRunner:
        if is_running_under_teamcity():
            cls = TeamcityTestRunner
    results = cls(verbosity=verbosity, failfast=args.failfast).run(test_suite)

    # if a tenant was not specified on the commandline we destroy it

    if not results.wasSuccessful():
        sys.exit(1)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Build the documentation.

import errno, os, re, sys
from subprocess import check_call, CalledProcessError, Popen, PIPE, STDOUT

versions = ['1.0.0', '1.1.0', '2.0.0', '3.0.2', '4.0.0', '4.1.0', '5.0.0', '5.1.0', '5.2.0', '5.2.1', '5.3.0', '6.0.0', '6.1.0', '6.1.1', '6.1.2', '6.2.0', '6.2.1', '7.0.0', '7.0.1', '7.0.2', '7.0.3', '7.1.0', '7.1.1', '7.1.2', '7.1.3', '8.0.0', '8.0.1']

class Pip:
  def __init__(self, venv_dir):
    self.path = os.path.join(venv_dir, 'bin', 'pip')

  def install(self, package, commit=None):
    "Install package using pip."
    if commit:
      package = 'git+https://github.com/{0}.git@{1}'.format(package, commit)
    print('Installing {0}'.format(package))
    check_call([self.path, 'install', package])

def create_build_env(venv_dir='virtualenv'):
  # Create virtualenv.
  if not os.path.exists(venv_dir):
    check_call(['python3', '-m', 'venv', venv_dir])
  # Install Sphinx and Breathe. Require the exact version of Sphinx which is
  # compatible with Breathe.
  pip = Pip(venv_dir)
  pip.install('wheel')
  pip.install('six')
  pip.install('sphinx-doc/sphinx', 'v3.3.0')
  pip.install('michaeljones/breathe', 'v4.25.0')

def build_docs(version='dev', **kwargs):
  doc_dir = kwargs.get('doc_dir', os.path.dirname(os.path.realpath(__file__)))
  work_dir = kwargs.get('work_dir', '.')
  include_dir = kwargs.get(
      'include_dir', os.path.join(os.path.dirname(doc_dir), 'include', 'fmt'))
  # Build docs.
  cmd = ['doxygen', '-']
  p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=STDOUT)
  doxyxml_dir = os.path.join(work_dir, 'doxyxml')
  out, _ = p.communicate(input=r'''
      PROJECT_NAME      = fmt
      GENERATE_LATEX    = NO
      GENERATE_MAN      = NO
      GENERATE_RTF      = NO
      CASE_SENSE_NAMES  = NO
      INPUT             = {0}/chrono.h {0}/color.h {0}/core.h {0}/compile.h \
                          {0}/format.h {0}/os.h {0}/ostream.h {0}/printf.h \
                          {0}/xchar.h
      QUIET             = YES
      JAVADOC_AUTOBRIEF = YES
      AUTOLINK_SUPPORT  = NO
      GENERATE_HTML     = NO
      GENERATE_XML      = YES
      XML_OUTPUT        = {1}
      ALIASES           = "rst=\verbatim embed:rst"
      ALIASES          += "endrst=\endverbatim"
      MACRO_EXPANSION   = YES
      PREDEFINED        = _WIN32=1 \
                          __linux__=1 \
                          FMT_USE_VARIADIC_TEMPLATES=1 \
                          FMT_USE_RVALUE_REFERENCES=1 \
                          FMT_USE_USER_DEFINED_LITERALS=1 \
                          FMT_USE_ALIAS_TEMPLATES=1 \
                          FMT_API= \
                          "FMT_BEGIN_NAMESPACE=namespace fmt {{" \
                          "FMT_END_NAMESPACE=}}" \
                          "FMT_STRING_ALIAS=1" \
                          "FMT_VARIADIC(...)=" \
                          "FMT_VARIADIC_W(...)=" \
                          "FMT_DOC=1"
      EXCLUDE_SYMBOLS   = fmt::formatter fmt::printf_formatter fmt::arg_join \
                          fmt::basic_format_arg::handle
    '''.format(include_dir, doxyxml_dir).encode('UTF-8'))
  out = out.decode('utf-8')
  internal_symbols = [
    'fmt::detail::.*',
    'basic_data<>',
    'fmt::type_identity',
    'fmt::dynamic_formatter'
  ]
  noisy_warnings = [
    'warning: (Compound|Member .* of class) (' + '|'.join(internal_symbols) + \
      ') is not documented.',
    'warning: Internal inconsistency: .* does not belong to any container!'
  ]
  for w in noisy_warnings:
      out = re.sub('.*' + w + '\n', '', out)
  print(out)
  if p.returncode != 0:
    raise CalledProcessError(p.returncode, cmd)

  html_dir = os.path.join(work_dir, 'html')
  main_versions = reversed(versions[-3:])
  check_call([os.path.join(work_dir, 'virtualenv', 'bin', 'sphinx-build'),
              '-Dbreathe_projects.format=' + os.path.abspath(doxyxml_dir),
              '-Dversion=' + version, '-Drelease=' + version,
              '-Aversion=' + version, '-Aversions=' + ','.join(main_versions),
              '-b', 'html', doc_dir, html_dir])
  try:
    check_call(['lessc', '--verbose', '--clean-css',
                '--include-path=' + os.path.join(doc_dir, 'bootstrap'),
                os.path.join(doc_dir, 'fmt.less'),
                os.path.join(html_dir, '_static', 'fmt.css')])
  except OSError as e:
    if e.errno != errno.ENOENT:
      raise
    print('lessc not found; make sure that Less (http://lesscss.org/) ' +
          'is installed')
    sys.exit(1)
  return html_dir

if __name__ == '__main__':
  create_build_env()
  build_docs(sys.argv[1])
<commit_msg>Fix docs<commit_after>#!/usr/bin/env python3
# Build the documentation.

import errno, os, re, sys
from subprocess import check_call, CalledProcessError, Popen, PIPE, STDOUT

versions = ['1.0.0', '1.1.0', '2.0.0', '3.0.2', '4.0.0', '4.1.0', '5.0.0', '5.1.0', '5.2.0', '5.2.1', '5.3.0', '6.0.0', '6.1.0', '6.1.1', '6.1.2', '6.2.0', '6.2.1', '7.0.0', '7.0.1', '7.0.2', '7.0.3', '7.1.0', '7.1.1', '7.1.2', '7.1.3', '8.0.0', '8.0.1']

class Pip:
  def __init__(self, venv_dir):
    self.path = os.path.join(venv_dir, 'bin', 'pip')

  def install(self, package, commit=None):
    "Install package using pip."
    if commit:
      package = 'git+https://github.com/{0}.git@{1}'.format(package, commit)
    print('Installing {0}'.format(package))
    check_call([self.path, 'install', package])

def create_build_env(venv_dir='virtualenv'):
  # Create virtualenv.
  if not os.path.exists(venv_dir):
    check_call(['python3', '-m', 'venv', venv_dir])
  # Install Sphinx and Breathe. Require the exact version of Sphinx which is
  # compatible with Breathe.
  pip = Pip(venv_dir)
  pip.install('wheel')
  pip.install('six')
  pip.install('sphinx-doc/sphinx', 'v3.3.0')
  pip.install('michaeljones/breathe', 'v4.25.0')

def build_docs(version='dev', **kwargs):
  doc_dir = kwargs.get('doc_dir', os.path.dirname(os.path.realpath(__file__)))
  work_dir = kwargs.get('work_dir', '.')
  include_dir = kwargs.get(
      'include_dir', os.path.join(os.path.dirname(doc_dir), 'include', 'fmt'))
  # Build docs.
  cmd = ['doxygen', '-']
  p = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=STDOUT)
  doxyxml_dir = os.path.join(work_dir, 'doxyxml')
  out, _ = p.communicate(input=r'''
      PROJECT_NAME      = fmt
      GENERATE_LATEX    = NO
      GENERATE_MAN      = NO
      GENERATE_RTF      = NO
      CASE_SENSE_NAMES  = NO
      INPUT             = {0}/chrono.h {0}/color.h {0}/core.h {0}/compile.h \
                          {0}/format.h {0}/os.h {0}/ostream.h {0}/printf.h \
                          {0}/xchar.h
      QUIET             = YES
      JAVADOC_AUTOBRIEF = YES
      AUTOLINK_SUPPORT  = NO
      GENERATE_HTML     = NO
      GENERATE_XML      = YES
      XML_OUTPUT        = {1}
      ALIASES           = "rst=\verbatim embed:rst"
      ALIASES          += "endrst=\endverbatim"
      MACRO_EXPANSION   = YES
      PREDEFINED        = _WIN32=1 \
                          __linux__=1 \
                          FMT_ENABLE_IF(...)= \
                          FMT_USE_VARIADIC_TEMPLATES=1 \
                          FMT_USE_RVALUE_REFERENCES=1 \
                          FMT_USE_USER_DEFINED_LITERALS=1 \
                          FMT_USE_ALIAS_TEMPLATES=1 \
                          FMT_API= \
                          "FMT_BEGIN_NAMESPACE=namespace fmt {{" \
                          "FMT_END_NAMESPACE=}}" \
                          "FMT_STRING_ALIAS=1" \
                          "FMT_VARIADIC(...)=" \
                          "FMT_VARIADIC_W(...)=" \
                          "FMT_DOC=1"
      EXCLUDE_SYMBOLS   = fmt::formatter fmt::printf_formatter fmt::arg_join \
                          fmt::basic_format_arg::handle
    '''.format(include_dir, doxyxml_dir).encode('UTF-8'))
  out = out.decode('utf-8')
  internal_symbols = [
    'fmt::detail::.*',
    'basic_data<>',
    'fmt::type_identity',
    'fmt::dynamic_formatter'
  ]
  noisy_warnings = [
    'warning: (Compound|Member .* of class) (' + '|'.join(internal_symbols) + \
      ') is not documented.',
    'warning: Internal inconsistency: .* does not belong to any container!'
  ]
  for w in noisy_warnings:
      out = re.sub('.*' + w + '\n', '', out)
  print(out)
  if p.returncode != 0:
    raise CalledProcessError(p.returncode, cmd)

  html_dir = os.path.join(work_dir, 'html')
  main_versions = reversed(versions[-3:])
  check_call([os.path.join(work_dir, 'virtualenv', 'bin', 'sphinx-build'),
              '-Dbreathe_projects.format=' + os.path.abspath(doxyxml_dir),
              '-Dversion=' + version, '-Drelease=' + version,
              '-Aversion=' + version, '-Aversions=' + ','.join(main_versions),
              '-b', 'html', doc_dir, html_dir])
  try:
    check_call(['lessc', '--verbose', '--clean-css',
                '--include-path=' + os.path.join(doc_dir, 'bootstrap'),
                os.path.join(doc_dir, 'fmt.less'),
                os.path.join(html_dir, '_static', 'fmt.css')])
  except OSError as e:
    if e.errno != errno.ENOENT:
      raise
    print('lessc not found; make sure that Less (http://lesscss.org/) ' +
          'is installed')
    sys.exit(1)
  return html_dir

if __name__ == '__main__':
  create_build_env()
  build_docs(sys.argv[1])
<|endoftext|>


<commit_before>import slackclient
import markovify
import os
import time
import argparse
import re
import sys

STATE_SIZE = 2
USER_ID_REGEX = r"\<\@(.+?)\>"

projectname = "andrey3000"

if os.name != "posix":
    import win32com
    from win32com.shell import shellcon, shell
    homedir = shell.SHGetFolderPath(0, shellcon.CSIDL_APPDATA, 0, 0)
else:
    homedir = os.path.join(os.path.expanduser("~"), ".config")
settingsdirectory = os.path.join(homedir, projectname)
if not os.path.isdir(settingsdirectory):
    os.makedirs(settingsdirectory)

chains_dir = os.path.join(settingsdirectory, "chains")
if not os.path.isdir(chains_dir):
    os.makedirs(chains_dir)

class ArgumentParserError(Exception):
    pass

class ThrowingArgumentParser(argparse.ArgumentParser):
    def error(self, message):
        raise ArgumentParserError(message)

def get_markov(user_id):
    try:
        with open(os.path.join(chains_dir, user_id), "r") as infile:
            return markovify.Text.from_json(infile.read())
    except:
        return markovify.Text("", state_size=STATE_SIZE)

def save_markov(user_id, markov):
    with open(os.path.join(chains_dir, user_id), "w") as outfile:
        outfile.write(markov.to_json())

def parse_command(command, message):
    parser = ThrowingArgumentParser()
    subparsers = parser.add_subparsers()
    impersonate_subparser = subparsers.add_parser("impersonate", description="Impersonate the given user")
    impersonate_subparser.set_defaults(function="impersonate")
    impersonate_subparser.add_argument("user", default=None)

    try:
        args, other_args = parser.parse_known_args(command)

        if args.function == "impersonate":
            if args.user is None:
                return "Who should I impersonate?"
            if args.user.lower() == "me":
                uid = message.get("user")
            elif args.user in ("<!everyone>", "everyone"):
                uid = "everyone"
            else:
                m = re.match(USER_ID_REGEX, args.user)
                if not m:
                    return "I don't recognize user {}".format(args.user)
                uid = m.group(1)
            markov = get_markov(uid)
            try:
                sentence = markov.make_sentence()
            except:
                sentence = "Could not impersonate <@{}>, not enough data".format(uid)
            if not sentence:
                sentence = "Could not impersonate <@{}>, not enough data".format(uid)
            return sentence
        else:
            raise ArgumentParserError("Unkown command")
    except ArgumentParserError as e:
        return "Unknown command"

def main(argv=None):
    if argv is None:
        argv = sys.argv[1:]

    parser = argparse.ArgumentParser()
    parser.add_argument("--key", "-k", default=None, help="Key to use to connect to slack")

    args = parser.parse_args(argv)

    slack_token = None

    try:
        with open(os.path.join(os.path.dirname(__file__), "andrey.key")) as infile:
            slack_token = infile.read().strip()
    except:
        pass

    try:
        slack_token = os.environ["andrey_key"]
    except:
        pass

    if args.key is not None:
        slack_token = args.key

    if not slack_token:
        print "No slack token found!"
        return -1

    client = slackclient.SlackClient(slack_token)

    channels = client.api_call("channels.list")

    if not client.rtm_connect():
        raise Exception("Failed to connect to slack")

    username = client.server.username
    user_id = None
    for uid, user_data in client.server.users.items():
        if user_data.name == username:
            user_id = uid
            break
    assert user_id is not None, "Unable to find own user id"

    while True:
        for slack_message in client.rtm_read():
            if slack_message.get("type") == "message":
                text = slack_message.get("text")
                if text is None:
                    continue
                if not text.startswith("<@{}>".format(user_id)):
                    if slack_message.get("user") == user_id:
                        continue
                    try:
                        chain = markovify.Text(str(text), state_size=STATE_SIZE)
                        old_chain = get_markov(slack_message.get("user"))
                        new_chain = markovify.combine([old_chain, chain])
                        save_markov(slack_message.get("user"), new_chain)

                        old_chain = get_markov("everyone")
                        new_chain = markovify.combine([old_chain, chain])
                        save_markov("everyone", new_chain)
                    except Exception as e:
                        print e
                        print text
                else:
                    message = parse_command(text.split()[1:], slack_message)
                    client.rtm_send_message(slack_message.get("channel"), message)

        time.sleep(.25)

if __name__ == "__main__":
    exit(main())<commit_msg>Added function to let andrey write in the style of a given text<commit_after>import slackclient
import markovify
import os
import time
import argparse
import re
import sys

STATE_SIZE = 2
USER_ID_REGEX = r"\<\@(.+?)\>"

projectname = "andrey3000"

if os.name != "posix":
    import win32com
    from win32com.shell import shellcon, shell
    homedir = shell.SHGetFolderPath(0, shellcon.CSIDL_APPDATA, 0, 0)
else:
    homedir = os.path.join(os.path.expanduser("~"), ".config")
settingsdirectory = os.path.join(homedir, projectname)
if not os.path.isdir(settingsdirectory):
    os.makedirs(settingsdirectory)

chains_dir = os.path.join(settingsdirectory, "chains")
if not os.path.isdir(chains_dir):
    os.makedirs(chains_dir)

texts_dir = os.path.join(settingsdirectory, "texts")
if not os.path.isdir(texts_dir):
    os.makedirs(texts_dir)

class ArgumentParserError(Exception):
    pass

class ThrowingArgumentParser(argparse.ArgumentParser):
    def error(self, message):
        raise ArgumentParserError(message)

def get_markov(user_id, path=chains_dir):
    try:
        with open(os.path.join(path, user_id), "r") as infile:
            return markovify.Text.from_json(infile.read())
    except:
        return markovify.Text("", state_size=STATE_SIZE)

def save_markov(user_id, markov, path=chains_dir):
    with open(os.path.join(path, user_id), "w") as outfile:
        outfile.write(markov.to_json())

def parse_command(command, message):
    parser = ThrowingArgumentParser()
    subparsers = parser.add_subparsers()
    impersonate_subparser = subparsers.add_parser("impersonate", description="Impersonate the given user")
    impersonate_subparser.set_defaults(function="impersonate")
    impersonate_subparser.add_argument("user", default=None)

    write_subparser = subparsers.add_parser("write", description="Write from a saved text")
    write_subparser.set_defaults(function="write")
    write_subparser.add_argument("name", default=[], nargs="+")

    try:
        args, other_args = parser.parse_known_args(command)

        if args.function == "impersonate":
            if args.user is None:
                return "Who should I impersonate?"
            if args.user.lower() == "me":
                uid = message.get("user")
            elif args.user in ("<!everyone>", "everyone"):
                uid = "everyone"
            else:
                m = re.match(USER_ID_REGEX, args.user)
                if not m:
                    return "I don't recognize user {}".format(args.user)
                uid = m.group(1)
            markov = get_markov(uid)
            try:
                sentence = markov.make_sentence()
            except:
                sentence = "Could not impersonate <@{}>, not enough data".format(uid)
            if not sentence:
                sentence = "Could not impersonate <@{}>, not enough data".format(uid)
            return sentence
        elif args.function == "write":
            name = "_".join(args.name).lower()
            markov = get_markov(name, path=texts_dir)
            try:
                sentence = markov.make_sentence()
            except:
                sentence = "I do not know how to write {}".format(" ".join(args.name))
            if not sentence:
                sentence = "I do not know how to write {}".format(" ".join(args.name))
            return sentence
        else:
            raise ArgumentParserError("Unkown command")
    except ArgumentParserError as e:
        return "Unknown command"

def main(argv=None):
    if argv is None:
        argv = sys.argv[1:]

    parser = argparse.ArgumentParser()
    parser.set_defaults(function=None)
    subparsers = parser.add_subparsers()
    add_text_subparser = subparsers.add_parser("add_text", description="Add a text for andrey to write excerpts for")
    add_text_subparser.set_defaults(function="add_text")

    add_text_subparser.add_argument("text_file", help="File to read text from")
    add_text_subparser.add_argument("name", help="Name to save text as")

    run_subparser = subparsers.add_parser("run", description="Add a text for andrey to write excerpts for")
    run_subparser.set_defaults(function="run")

    run_subparser.add_argument("--key", "-k", default=None, help="Key to use to connect to slack")

    if not argv or argv[0] not in [command for command in subparsers.choices]:
        argv = ["run"] + argv

    args = parser.parse_args(argv)

    if args.function == "run":
        slack_token = None

        try:
            with open(os.path.join(os.path.dirname(__file__), "andrey.key")) as infile:
                slack_token = infile.read().strip()
        except:
            pass

        try:
            slack_token = os.environ["andrey_key"]
        except:
            pass

        if args.key is not None:
            slack_token = args.key

        if not slack_token:
            print "No slack token found!"
            return -1

        client = slackclient.SlackClient(slack_token)

        channels = client.api_call("channels.list")

        if not client.rtm_connect():
            raise Exception("Failed to connect to slack")

        username = client.server.username
        user_id = None
        for uid, user_data in client.server.users.items():
            if user_data.name == username:
                user_id = uid
                break
        assert user_id is not None, "Unable to find own user id"

        while True:
            for slack_message in client.rtm_read():
                if slack_message.get("type") == "message":
                    text = slack_message.get("text")
                    if text is None:
                        continue
                    if not text.startswith("<@{}>".format(user_id)):
                        if slack_message.get("user") == user_id:
                            continue
                        try:
                            chain = markovify.Text(str(text), state_size=STATE_SIZE)
                            old_chain = get_markov(slack_message.get("user"))
                            new_chain = markovify.combine([old_chain, chain])
                            save_markov(slack_message.get("user"), new_chain)

                            old_chain = get_markov("everyone")
                            new_chain = markovify.combine([old_chain, chain])
                            save_markov("everyone", new_chain)
                        except Exception as e:
                            print e
                            print text
                    else:
                        message = parse_command(text.split()[1:], slack_message)
                        client.rtm_send_message(slack_message.get("channel"), message)

            time.sleep(.25)

    elif args.function == "add_text":
        with open(args.text_file, "r") as infile:
            data = infile.read()
        m = markovify.Text(data)
        name = "_".join(args.name.split(" ")).lower()
        save_markov(name, m, path=texts_dir)

if __name__ == "__main__":
    exit(main())<|endoftext|>


<commit_before>#from google.appengine.ext import webapp
import webapp2
from google.appengine.api import memcache
from google.appengine.ext import db
from google.appengine.ext.webapp.util import run_wsgi_app
from google.appengine.ext.webapp import template

import base64
import os
import simplejson as json
import logging

# See https://developers.google.com/appengine/docs/python/runtime#The_Environment
try:
  VERSION = os.environ['CURRENT_VERSION_ID']
except:
  VERSION = ''

MEMCACHE_ENABLED = True

class ImageRecord(db.Model):
  # key = photo_id
  photo_id = db.StringProperty()
  title = db.StringProperty()
  date = db.StringProperty()
  folder = db.StringProperty()
  library_url = db.StringProperty()
  width = db.IntegerProperty()
  height = db.IntegerProperty()
  description = db.StringProperty()
  note = db.StringProperty()

class ThumbnailRecord(db.Model):
  # key = photo_id
  image = db.BlobProperty()

def GetImageRecords(photo_ids):
  """Queries the ImageRecord db, w/ memcaching. Returns photo_id -> rec dict"""
  # check if we've got the whole thing in memcache
  keys_no_prefix = photo_ids[:]
  if MEMCACHE_ENABLED:
    multi_key = VERSION + 'MIR' + ','.join(photo_ids)
    recs = memcache.get(multi_key)
    if recs: return recs

    key_prefix = VERSION + 'IR'

    record_map = memcache.get_multi(keys_no_prefix, key_prefix=key_prefix)
    missing_ids = list(set(keys_no_prefix) - set(record_map.keys()))
    if not missing_ids: return record_map
  else:
    missing_ids = keys_no_prefix
    record_map = {}

  config = db.create_config(read_policy=db.EVENTUAL_CONSISTENCY)
  db_recs = ImageRecord.get_by_key_name(missing_ids, config=config)

  memcache_map = {}
  for id, r in zip(missing_ids, db_recs):
    record_map[id] = r
    memcache_map[id] = r

  if MEMCACHE_ENABLED and memcache_map:
    memcache.add_multi(memcache_map, key_prefix=key_prefix)
    memcache.add(multi_key, record_map)
  return record_map

class RecordFetcher(webapp2.RequestHandler):
  def post(self):
    self.get()

  def get(self):
    """Responds to AJAX requests for record information."""
    photo_ids = self.request.get_all("id")
    default_response = {
      'title': '(not here yet!)',
      'date': '1926 Feb. 18',
      'folder': 'S.F. Streets / Alemany Boulevard',
      'library_url': 'http://sflib1.sfpl.org:82/record=b1000001~S0',
      'width': 600,
      'height': 500
    }

    rs = GetImageRecords(photo_ids)
    response = {}
    for id, r in rs.iteritems():
      if not r:
        #self.response.out.write("no record for '%s'" % id)
        # This is just to aid local testing:
        response[id] = default_response.copy()
        orig_id = id.split('-')[0]
      else:
        title = r.title
        if r.description:
          title += '; ' + r.description
        if r.note:
          title += '; ' + r.note
        response[id] = {
          'title': title,
          'date': r.date,
          'folder': r.folder,
          'width': r.width,
          'height': r.height
        }
    self.response.headers['Cache-Control'] = 'public; max-age=2592000'
    self.response.out.write(json.dumps(response))

class AddEgg(webapp2.RequestHandler):
  def get(self):
    image = ImageRecord(key_name="egg")
    image.photo_id = 'egg'
    image.title = 'Old S.F. developers with their handywork'
    image.date = 'August 17, 2011'
    image.folder = 'The New York Collection'
    image.library_url = 'http://danvk.org/'
    image.put()

class AddDims(webapp2.RequestHandler):
  def get(self):
    self.post()

  def post(self):
    recs = self.request.get_all('d')
    photo_id_to_dims = {}
    for rec in recs:
      photo_id, width_str, height_str = rec.split(',')
      width = int(width_str)
      height = int(height_str)
      photo_id_to_dims[photo_id] = (width, height)

    config = db.create_config(read_policy=db.EVENTUAL_CONSISTENCY)
    db_recs = ImageRecord.get_by_key_name(photo_id_to_dims.keys(), config=config)

    for image in db_recs:
      width, height = photo_id_to_dims[image.photo_id]
      if image.width == width and image.height == height:
        continue
      image.width = width
      image.height = height
      image.put()

    self.response.out.write('Added %d dimensions' % len(db_recs))

kProps = ['photo_id', 'title', 'date', 'folder', 'note', 'library_url', 'description', 'note']
kIntProps = ['width', 'height']

class UploadHandler(webapp2.RequestHandler):
  def post(self):
    """Adds a new image to the DB."""
    self.response.headers.add_header('Content-type', 'text/plain')
    recs = self.request.get_all('r')
    for rec_json in recs:
      rec = json.loads(rec_json)
      id = rec['photo_id']
      assert id

      verb = 'Updated'
      record = ImageRecord.get_by_key_name(id)
      if not record:
        verb = 'Added'
        record = ImageRecord(key_name=id)

      props = record.properties()
      for field in kProps:
        if field in rec:
          props[field].__set__(record, rec[field])
      for field in kIntProps:
        if field in rec:
          props[field].__set__(record, int(rec[field]))

      record.put()
      self.response.out.write('%s image record %s\n' % (verb, id))

class RootHandler(webapp2.RequestHandler):
  def get(self):
    self.response.headers['Content-type'] = 'text/html'
    self.response.out.write(open('static/viewer.html').read())

app = webapp2.WSGIApplication(
                              [
                               ('/', RootHandler),
                               ('/info', RecordFetcher),
                               ('/upload', UploadHandler),
                               #('/thumb.*', ThumbnailFetcher),
                               ('/addegg', AddEgg),
                               #('/adddims', AddDims),
                              ],
                              debug=True)

# def main():
#     run_wsgi_app(application)
#
# if __name__ == "__main__":
#     main()

<commit_msg>restore password :(<commit_after>#from google.appengine.ext import webapp
import webapp2
from google.appengine.api import memcache
from google.appengine.ext import db
from google.appengine.ext.webapp.util import run_wsgi_app
from google.appengine.ext.webapp import template

import base64
import os
import simplejson as json
import logging

# See https://developers.google.com/appengine/docs/python/runtime#The_Environment
try:
  VERSION = os.environ['CURRENT_VERSION_ID']
except:
  VERSION = ''

MEMCACHE_ENABLED = True

class ImageRecord(db.Model):
  # key = photo_id
  photo_id = db.StringProperty()
  title = db.StringProperty()
  date = db.StringProperty()
  folder = db.StringProperty()
  library_url = db.StringProperty()
  width = db.IntegerProperty()
  height = db.IntegerProperty()
  description = db.StringProperty()
  note = db.StringProperty()

class ThumbnailRecord(db.Model):
  # key = photo_id
  image = db.BlobProperty()

def GetImageRecords(photo_ids):
  """Queries the ImageRecord db, w/ memcaching. Returns photo_id -> rec dict"""
  # check if we've got the whole thing in memcache
  keys_no_prefix = photo_ids[:]
  if MEMCACHE_ENABLED:
    multi_key = VERSION + 'MIR' + ','.join(photo_ids)
    recs = memcache.get(multi_key)
    if recs: return recs

    key_prefix = VERSION + 'IR'

    record_map = memcache.get_multi(keys_no_prefix, key_prefix=key_prefix)
    missing_ids = list(set(keys_no_prefix) - set(record_map.keys()))
    if not missing_ids: return record_map
  else:
    missing_ids = keys_no_prefix
    record_map = {}

  config = db.create_config(read_policy=db.EVENTUAL_CONSISTENCY)
  db_recs = ImageRecord.get_by_key_name(missing_ids, config=config)

  memcache_map = {}
  for id, r in zip(missing_ids, db_recs):
    record_map[id] = r
    memcache_map[id] = r

  if MEMCACHE_ENABLED and memcache_map:
    memcache.add_multi(memcache_map, key_prefix=key_prefix)
    memcache.add(multi_key, record_map)
  return record_map

class RecordFetcher(webapp2.RequestHandler):
  def post(self):
    self.get()

  def get(self):
    """Responds to AJAX requests for record information."""
    photo_ids = self.request.get_all("id")
    default_response = {
      'title': '(not here yet!)',
      'date': '1926 Feb. 18',
      'folder': 'S.F. Streets / Alemany Boulevard',
      'library_url': 'http://sflib1.sfpl.org:82/record=b1000001~S0',
      'width': 600,
      'height': 500
    }

    rs = GetImageRecords(photo_ids)
    response = {}
    for id, r in rs.iteritems():
      if not r:
        #self.response.out.write("no record for '%s'" % id)
        # This is just to aid local testing:
        response[id] = default_response.copy()
        orig_id = id.split('-')[0]
      else:
        title = r.title
        if r.description:
          title += '; ' + r.description
        if r.note:
          title += '; ' + r.note
        response[id] = {
          'title': title,
          'date': r.date,
          'folder': r.folder,
          'width': r.width,
          'height': r.height
        }
    self.response.headers['Cache-Control'] = 'public; max-age=2592000'
    self.response.out.write(json.dumps(response))

class AddEgg(webapp2.RequestHandler):
  def get(self):
    image = ImageRecord(key_name="egg")
    image.photo_id = 'egg'
    image.title = 'Old S.F. developers with their handywork'
    image.date = 'August 17, 2011'
    image.folder = 'The New York Collection'
    image.library_url = 'http://danvk.org/'
    image.put()

class AddDims(webapp2.RequestHandler):
  def get(self):
    self.post()

  def post(self):
    recs = self.request.get_all('d')
    photo_id_to_dims = {}
    for rec in recs:
      photo_id, width_str, height_str = rec.split(',')
      width = int(width_str)
      height = int(height_str)
      photo_id_to_dims[photo_id] = (width, height)

    config = db.create_config(read_policy=db.EVENTUAL_CONSISTENCY)
    db_recs = ImageRecord.get_by_key_name(photo_id_to_dims.keys(), config=config)

    for image in db_recs:
      width, height = photo_id_to_dims[image.photo_id]
      if image.width == width and image.height == height:
        continue
      image.width = width
      image.height = height
      image.put()

    self.response.out.write('Added %d dimensions' % len(db_recs))

kProps = ['photo_id', 'title', 'date', 'folder', 'note', 'library_url', 'description', 'note']
kIntProps = ['width', 'height']

class UploadHandler(webapp2.RequestHandler):
  def post(self):
    """Adds a new image to the DB."""
    self.response.headers.add_header('Content-type', 'text/plain')
    recs = self.request.get_all('r')
    for rec_json in recs:
      rec = json.loads(rec_json)
      id = rec['photo_id']
      assert id

      verb = 'Updated'
      record = ImageRecord.get_by_key_name(id)
      if not record:
        verb = 'Added'
        record = ImageRecord(key_name=id)

      props = record.properties()
      for field in kProps:
        if field in rec:
          props[field].__set__(record, rec[field])
      for field in kIntProps:
        if field in rec:
          props[field].__set__(record, int(rec[field]))

      record.put()
      self.response.out.write('%s image record %s\n' % (verb, id))

# From http://stackoverflow.com/questions/9461085/password-protect-static-page-appengine-howto
def basicAuth(func):
  def callf(webappRequest, *args, **kwargs):
    # Parse the header to extract a user/password combo.
    # We're expecting something like "Basic XZxgZRTpbjpvcGVuIHYlc4FkZQ=="
    auth_header = webappRequest.request.headers.get('Authorization')

    if auth_header == None:
      webappRequest.response.set_status(401, message="Authorization Required")
      webappRequest.response.headers['WWW-Authenticate'] = 'Basic realm="OldNYC"'
    else:
      # Isolate the encoded user/passwd and decode it
      auth_parts = auth_header.split(' ')
      user_pass_parts = base64.b64decode(auth_parts[1]).split(':')
      user_arg = user_pass_parts[0]
      pass_arg = user_pass_parts[1]

      if user_arg != "robert" or pass_arg != "moses":
        webappRequest.response.set_status(401, message="Authorization Required")
        webappRequest.response.headers['WWW-Authenticate'] = 'Basic realm="Secure Area"'
        # Rendering a 401 Error page is a good way to go...
        self.response.out.write('sorry!', {})
      else:
        return func(webappRequest, *args, **kwargs)

  return callf

class RootHandler(webapp2.RequestHandler):
  @basicAuth
  def get(self):
    logging.info('hello')
    self.response.headers['Content-type'] = 'text/html'
    self.response.out.write(open('static/viewer.html').read())

app = webapp2.WSGIApplication(
                              [
                               ('/', RootHandler),
                               ('/info', RecordFetcher),
                               ('/upload', UploadHandler),
                               #('/thumb.*', ThumbnailFetcher),
                               ('/addegg', AddEgg),
                               #('/adddims', AddDims),
                              ],
                              debug=True)

# def main():
#     run_wsgi_app(application)
#
# if __name__ == "__main__":
#     main()

<|endoftext|>


<commit_before>"""
    viewer.git
    ~~~~~~~~~~

    Interface to git.

    :copyright: © 2014 by Petr Zemek <s3rvac@gmail.com> and contributors
    :license: BSD, see LICENSE for more details
"""

import subprocess

from .utils import chdir

class Commit:
    """A representation of a git commit."""
    VALID_HASH_LENGTH = 40
    VALID_HASH_CHARACTERS = set('0123456789abcdef')

    def __init__(self, hash, author, email, date):
        """Creates a commit with the given data.

         - hash: identifier of the commit (string of VALID_HASH_LENGTH
                 containing only VALID_HASH_CHARACTERS)
         - author: author of the commit (string)
         - email: email of the author (string)
         - date: date the commit was authored (a date object)

        The hash is normalized so that it contains only lowercase characters.
        If the hash has invalid length or contains invalid characters after the
        normalization, ValueError is raised.
        """
        self.hash = hash
        self.author = author
        self.email = email
        self.date = date

        self._normalize_hash()
        self._validate_hash()

    def short_hash(self, length=8):
        """Returns a shorter version of the hash."""
        return self.hash[:length]

    def __eq__(self, other):
        return (self.hash == other.hash and
                self.author == other.author and
                self.email == other.email and
                self.date == other.date)

    def __ne__(self, other):
        return not self == other

    def __repr__(self):
        return '{}({!r}, {!r}, {!r}, {!r})'.format(
            self.__class__.__name__,
            self.hash,
            self.author,
            self.email,
            self.date)

    def _normalize_hash(self):
        self.hash = self.hash.lower()

    def _validate_hash(self):
        self._validate_hash_length()
        self._validate_hash_characters()

    def _validate_hash_length(self):
        if len(self.hash) != self.VALID_HASH_LENGTH:
            raise ValueError(
                "hash '{}' has invalid length {} (expected {})".format(
                    self.hash, len(self.hash), self.VALID_HASH_LENGTH))

    def _validate_hash_characters(self):
        hash_characters = set(self.hash)
        invalid_characters = hash_characters - self.VALID_HASH_CHARACTERS
        if invalid_characters:
            raise ValueError(
                "hash '{}' contains invalid character(s): '{}'".format(
                    self.hash, ''.join(invalid_characters)))

class Branch:
    """A representation of a git branch."""

    def __init__(self, name, commit, unmerged_commits):
        """Constructs a branch with the given data.

         - name: name of the branch (string)
         - commit: current commit in the branch (a Commit object)
         - unmerged_commits: commits that have not been merged to the master
                             branch (a list of Commit objects)
        """
        self.name = name
        self.commit = commit
        self.unmerged_commits = unmerged_commits

class BaseGitError(Exception):
    """A base class for all exception raised by the Git class."""
    pass

class GitCmdError(BaseGitError):
    """An exception that is raised when a git command fails."""
    pass

class GitBinaryNotFoundError(GitCmdError):
    """An exception that is raised when the git binary was not found."""
    pass

class Git:
    """Interface to a git repository.

    The methods in this class may raise the following exceptions:

     - FileNotFoundError: If the repository path path does not exist or cannot be entered.

     - GitBinaryNotFoundError: If the git binary (i.e. git) is not found.

     - GitCmdError: If there is an error when running a git command.
    """

    def __init__(self, path):
        """Creates an interface to a git repository in the given path.

        The path is then accessible through the path attribute.

        See the class description for a list of exceptions that this method may
        raise.
        """
        self.path = path
        self._verify_repository_existence()

    def run_git_cmd(self, args):
        """Runs the git command with the given arguments in the repository and
        returns the output.

        args has to be a sequence of parameters passed to git.

        See the class description for a list of exceptions that this method may
        raise.
        """
        with chdir(self.path):
            try:
                return subprocess.check_output(['git'] + list(args))
            # When a command is not found or cannot be executed,
            # subprocess.check_output() raises OSError.
            except OSError:
                raise GitBinaryNotFoundError(
                    "'git' is not installed or cannot be executed")
            except subprocess.CalledProcessError as ex:
                raise GitCmdError(ex.output)

    def _verify_repository_existence(self):
        self.run_git_cmd(['status'])
<commit_msg>git: Wrap a long line in the description of Git.<commit_after>"""
    viewer.git
    ~~~~~~~~~~

    Interface to git.

    :copyright: © 2014 by Petr Zemek <s3rvac@gmail.com> and contributors
    :license: BSD, see LICENSE for more details
"""

import subprocess

from .utils import chdir

class Commit:
    """A representation of a git commit."""
    VALID_HASH_LENGTH = 40
    VALID_HASH_CHARACTERS = set('0123456789abcdef')

    def __init__(self, hash, author, email, date):
        """Creates a commit with the given data.

         - hash: identifier of the commit (string of VALID_HASH_LENGTH
                 containing only VALID_HASH_CHARACTERS)
         - author: author of the commit (string)
         - email: email of the author (string)
         - date: date the commit was authored (a date object)

        The hash is normalized so that it contains only lowercase characters.
        If the hash has invalid length or contains invalid characters after the
        normalization, ValueError is raised.
        """
        self.hash = hash
        self.author = author
        self.email = email
        self.date = date

        self._normalize_hash()
        self._validate_hash()

    def short_hash(self, length=8):
        """Returns a shorter version of the hash."""
        return self.hash[:length]

    def __eq__(self, other):
        return (self.hash == other.hash and
                self.author == other.author and
                self.email == other.email and
                self.date == other.date)

    def __ne__(self, other):
        return not self == other

    def __repr__(self):
        return '{}({!r}, {!r}, {!r}, {!r})'.format(
            self.__class__.__name__,
            self.hash,
            self.author,
            self.email,
            self.date)

    def _normalize_hash(self):
        self.hash = self.hash.lower()

    def _validate_hash(self):
        self._validate_hash_length()
        self._validate_hash_characters()

    def _validate_hash_length(self):
        if len(self.hash) != self.VALID_HASH_LENGTH:
            raise ValueError(
                "hash '{}' has invalid length {} (expected {})".format(
                    self.hash, len(self.hash), self.VALID_HASH_LENGTH))

    def _validate_hash_characters(self):
        hash_characters = set(self.hash)
        invalid_characters = hash_characters - self.VALID_HASH_CHARACTERS
        if invalid_characters:
            raise ValueError(
                "hash '{}' contains invalid character(s): '{}'".format(
                    self.hash, ''.join(invalid_characters)))

class Branch:
    """A representation of a git branch."""

    def __init__(self, name, commit, unmerged_commits):
        """Constructs a branch with the given data.

         - name: name of the branch (string)
         - commit: current commit in the branch (a Commit object)
         - unmerged_commits: commits that have not been merged to the master
                             branch (a list of Commit objects)
        """
        self.name = name
        self.commit = commit
        self.unmerged_commits = unmerged_commits

class BaseGitError(Exception):
    """A base class for all exception raised by the Git class."""
    pass

class GitCmdError(BaseGitError):
    """An exception that is raised when a git command fails."""
    pass

class GitBinaryNotFoundError(GitCmdError):
    """An exception that is raised when the git binary was not found."""
    pass

class Git:
    """Interface to a git repository.

    The methods in this class may raise the following exceptions:

     - FileNotFoundError: If the repository path path does not exist or cannot
                          be entered.

     - GitBinaryNotFoundError: If the git binary (i.e. git) is not found.

     - GitCmdError: If there is an error when running a git command.
    """

    def __init__(self, path):
        """Creates an interface to a git repository in the given path.

        The path is then accessible through the path attribute.

        See the class description for a list of exceptions that this method may
        raise.
        """
        self.path = path
        self._verify_repository_existence()

    def run_git_cmd(self, args):
        """Runs the git command with the given arguments in the repository and
        returns the output.

        args has to be a sequence of parameters passed to git.

        See the class description for a list of exceptions that this method may
        raise.
        """
        with chdir(self.path):
            try:
                return subprocess.check_output(['git'] + list(args))
            # When a command is not found or cannot be executed,
            # subprocess.check_output() raises OSError.
            except OSError:
                raise GitBinaryNotFoundError(
                    "'git' is not installed or cannot be executed")
            except subprocess.CalledProcessError as ex:
                raise GitCmdError(ex.output)

    def _verify_repository_existence(self):
        self.run_git_cmd(['status'])
<|endoftext|>


<commit_before>"""
kombu.transport.mongodb
=======================

MongoDB transport.

:copyright: (c) 2010 - 2013 by Flavio Percoco Premoli.
:license: BSD, see LICENSE for more details.

"""
from __future__ import absolute_import

import pymongo

from pymongo import errors
from pymongo import MongoClient, uri_parser

from kombu.five import Empty
from kombu.syn import _detect_environment
from kombu.utils.encoding import bytes_to_str
from kombu.utils.json import loads, dumps

from . import virtual

DEFAULT_HOST = '127.0.0.1'
DEFAULT_PORT = 27017

DEFAULT_MESSAGES_COLLECTION = 'messages'
DEFAULT_ROUTING_COLLECTION = 'messages.routing'
DEFAULT_BROADCAST_COLLECTION = 'messages.broadcast'

class BroadcastCursor(object):
    """Cursor for broadcast queues."""

    def __init__(self, cursor):
        self._cursor = cursor

        self.purge(rewind=False)

    def get_size(self):
        return self._cursor.count() - self._offset

    def close(self):
        self._cursor.close()

    def purge(self, rewind=True):
        if rewind:
            self._cursor.rewind()

        # Fast forward the cursor past old events
        self._offset = self._cursor.count()
        self._cursor = self._cursor.skip(self._offset)

    def __iter__(self):
        return self

    def __next__(self):
        while True:
            try:
                msg = next(self._cursor)
            except pymongo.errors.OperationFailure as exc:
                # In some cases tailed cursor can become invalid
                # and have to be reinitalized
                if 'not valid at server' in exc.message:
                    self.purge()

                    continue

                raise
            else:
                break

        self._offset += 1

        return msg
    next = __next__

class Channel(virtual.Channel):
    _client = None
    supports_fanout = True
    _fanout_queues = {}

    def __init__(self, *vargs, **kwargs):
        super(Channel, self).__init__(*vargs, **kwargs)

        self._broadcast_cursors = {}

    def _new_queue(self, queue, **kwargs):
        pass

    def _get(self, queue):
        if queue in self._fanout_queues:
            try:
                msg = next(self.get_broadcast_cursor(queue))
            except StopIteration:
                msg = None
        else:
            msg = self.get_messages().find_and_modify(
                query={'queue': queue},
                sort=[('priority', pymongo.ASCENDING),
                      ('_id', pymongo.ASCENDING)],
                remove=True,
            )

        if msg is None:
            raise Empty()

        return loads(bytes_to_str(msg['payload']))

    def _size(self, queue):
        if queue in self._fanout_queues:
            return self.get_broadcast_cursor(queue).get_size()

        return self.get_messages().find({'queue': queue}).count()

    def _put(self, queue, message, **kwargs):
        self.get_messages().insert({
            'payload': dumps(message),
            'queue': queue,
            'priority': self._get_message_priority(message, reverse=True),
        })

    def _purge(self, queue):
        size = self._size(queue)

        if queue in self._fanout_queues:
            self.get_broadcaset_cursor(queue).purge()
        else:
            self.get_messages().remove({'queue': queue})

        return size

    def _parse_uri(self, scheme='mongodb://'):
        # See mongodb uri documentation:
        # http://docs.mongodb.org/manual/reference/connection-string/
        client = self.connection.client
        hostname = client.hostname

        if not hostname.startswith(scheme):
            hostname = scheme + hostname

        if not hostname[len(scheme):]:
            hostname += DEFAULT_HOST

        if client.userid and '@' not in hostname:
            head, tail = hostname.split('://')

            credentials = client.userid
            if client.password:
                credentials += ':' + client.password

            hostname = head + '://' + credentials + '@' + tail

        port = client.port if client.port is not None else DEFAULT_PORT

        parsed = uri_parser.parse_uri(hostname, port)

        dbname = parsed['database'] or client.virtual_host

        if dbname in ('/', None):
            dbname = 'kombu_default'

        options = {
            'auto_start_request': True,
            'ssl': client.ssl,
            'connectTimeoutMS': (int(client.connect_timeout * 1000)
                                 if client.connect_timeout else None),
        }
        options.update(client.transport_options)
        options.update(parsed['options'])

        return hostname, dbname, options

    def _open(self, scheme='mongodb://'):
        hostname, dbname, options = self._parse_uri(scheme=scheme)

        mongoconn = MongoClient(
            host=hostname, ssl=options['ssl'],
            auto_start_request=options['auto_start_request'],
            connectTimeoutMS=options['connectTimeoutMS'],
            use_greenlets=_detect_environment() != 'default',
        )
        database = mongoconn[dbname]

        version = mongoconn.server_info()['version']
        if tuple(map(int, version.split('.')[:2])) < (1, 3):
            raise NotImplementedError(
                'Kombu requires MongoDB version 1.3+ (server is {0})'.format(
                    version))

        self._create_broadcast(database, options)

        self._client = database

    def _create_broadcast(self, database, options):
        '''Create capped collection for broadcast messages.'''
        if DEFAULT_BROADCAST_COLLECTION in database.collection_names():
            return

        capsize = options.get('capped_queue_size') or 100000
        database.create_collection(DEFAULT_BROADCAST_COLLECTION,
                                   size=capsize, capped=True)

    def _ensure_indexes(self):
        '''Ensure indexes on collections.'''
        self.get_messages().ensure_index(
            [('queue', 1), ('priority', 1), ('_id', 1)], background=True,
        )
        self.get_broadcast().ensure_index([('queue', 1)])
        self.get_routing().ensure_index([('queue', 1), ('exchange', 1)])

    def get_table(self, exchange):
        """Get table of bindings for ``exchange``."""
        # TODO Store a more complete exchange metatable in the
        #      routing collection
        localRoutes = frozenset(self.state.exchanges[exchange]['table'])
        brokerRoutes = self.get_messages().routing.find(
            {'exchange': exchange}
        )

        return localRoutes | frozenset((r['routing_key'],
                                        r['pattern'],
                                        r['queue']) for r in brokerRoutes)

    def _put_fanout(self, exchange, message, routing_key, **kwargs):
        """Deliver fanout message."""
        self.get_broadcast().insert({'payload': dumps(message),
                                     'queue': exchange})

    def _queue_bind(self, exchange, routing_key, pattern, queue):
        if self.typeof(exchange).type == 'fanout':
            self.create_broadcast_cursor(exchange, routing_key, pattern, queue)
            self._fanout_queues[queue] = exchange

        meta = {'exchange': exchange,
                'queue': queue,
                'routing_key': routing_key,
                'pattern': pattern}
        self.get_routing().update(meta, meta, upsert=True)

    def queue_delete(self, queue, **kwargs):
        self.get_routing().remove({'queue': queue})

        super(Channel, self).queue_delete(queue, **kwargs)

        if queue in self._fanout_queues:
            try:
                cursor = self._broadcast_cursors.pop(queue)
            except KeyError:
                pass
            else:
                cursor.close()

                self._fanout_queues.pop(queue)

    @property
    def client(self):
        if self._client is None:
            self._open()
            self._ensure_indexes()

        return self._client

    def get_messages(self):
        return self.client[DEFAULT_MESSAGES_COLLECTION]

    def get_routing(self):
        return self.client[DEFAULT_ROUTING_COLLECTION]

    def get_broadcast(self):
        return self.client[DEFAULT_BROADCAST_COLLECTION]

    def get_broadcast_cursor(self, queue):
        try:
            return self._broadcast_cursors[queue]
        except KeyError:
            # Cursor may be absent when Channel created more than once.
            # _fanout_queues is a class-level mutable attribute so it's
            # shared over all Channel instances.
            return self.create_broadcast_cursor(
                self._fanout_queues[queue], None, None, queue,
            )

    def create_broadcast_cursor(self, exchange, routing_key, pattern, queue):
        cursor = self.get_broadcast().find(
            query={'queue': exchange},
            sort=[('$natural', 1)],
            tailable=True,
        )
        ret = self._broadcast_cursors[queue] = BroadcastCursor(cursor)
        return ret

class Transport(virtual.Transport):
    Channel = Channel

    can_parse_url = True
    polling_interval = 1
    default_port = DEFAULT_PORT
    connection_errors = (
        virtual.Transport.connection_errors + (errors.ConnectionFailure, )
    )
    channel_errors = (
        virtual.Transport.channel_errors + (
            errors.ConnectionFailure,
            errors.OperationFailure)
    )
    driver_type = 'mongodb'
    driver_name = 'pymongo'

    def driver_version(self):
        return pymongo.version
<commit_msg>mongodb transport - evaluate connection<commit_after>"""
kombu.transport.mongodb
=======================

MongoDB transport.

:copyright: (c) 2010 - 2013 by Flavio Percoco Premoli.
:license: BSD, see LICENSE for more details.

"""
from __future__ import absolute_import

import pymongo

from pymongo import errors
from pymongo import MongoClient, uri_parser

from kombu.five import Empty
from kombu.syn import _detect_environment
from kombu.utils.encoding import bytes_to_str
from kombu.utils.json import loads, dumps

from . import virtual

DEFAULT_HOST = '127.0.0.1'
DEFAULT_PORT = 27017

DEFAULT_MESSAGES_COLLECTION = 'messages'
DEFAULT_ROUTING_COLLECTION = 'messages.routing'
DEFAULT_BROADCAST_COLLECTION = 'messages.broadcast'

class BroadcastCursor(object):
    """Cursor for broadcast queues."""

    def __init__(self, cursor):
        self._cursor = cursor

        self.purge(rewind=False)

    def get_size(self):
        return self._cursor.count() - self._offset

    def close(self):
        self._cursor.close()

    def purge(self, rewind=True):
        if rewind:
            self._cursor.rewind()

        # Fast forward the cursor past old events
        self._offset = self._cursor.count()
        self._cursor = self._cursor.skip(self._offset)

    def __iter__(self):
        return self

    def __next__(self):
        while True:
            try:
                msg = next(self._cursor)
            except pymongo.errors.OperationFailure as exc:
                # In some cases tailed cursor can become invalid
                # and have to be reinitalized
                if 'not valid at server' in exc.message:
                    self.purge()

                    continue

                raise
            else:
                break

        self._offset += 1

        return msg
    next = __next__

class Channel(virtual.Channel):
    _client = None
    supports_fanout = True
    _fanout_queues = {}

    def __init__(self, *vargs, **kwargs):
        super(Channel, self).__init__(*vargs, **kwargs)

        self._broadcast_cursors = {}

        # Evaluate connection
        self.client

    def _new_queue(self, queue, **kwargs):
        pass

    def _get(self, queue):
        if queue in self._fanout_queues:
            try:
                msg = next(self.get_broadcast_cursor(queue))
            except StopIteration:
                msg = None
        else:
            msg = self.get_messages().find_and_modify(
                query={'queue': queue},
                sort=[('priority', pymongo.ASCENDING),
                      ('_id', pymongo.ASCENDING)],
                remove=True,
            )

        if msg is None:
            raise Empty()

        return loads(bytes_to_str(msg['payload']))

    def _size(self, queue):
        if queue in self._fanout_queues:
            return self.get_broadcast_cursor(queue).get_size()

        return self.get_messages().find({'queue': queue}).count()

    def _put(self, queue, message, **kwargs):
        self.get_messages().insert({
            'payload': dumps(message),
            'queue': queue,
            'priority': self._get_message_priority(message, reverse=True),
        })

    def _purge(self, queue):
        size = self._size(queue)

        if queue in self._fanout_queues:
            self.get_broadcaset_cursor(queue).purge()
        else:
            self.get_messages().remove({'queue': queue})

        return size

    def _parse_uri(self, scheme='mongodb://'):
        # See mongodb uri documentation:
        # http://docs.mongodb.org/manual/reference/connection-string/
        client = self.connection.client
        hostname = client.hostname

        if not hostname.startswith(scheme):
            hostname = scheme + hostname

        if not hostname[len(scheme):]:
            hostname += DEFAULT_HOST

        if client.userid and '@' not in hostname:
            head, tail = hostname.split('://')

            credentials = client.userid
            if client.password:
                credentials += ':' + client.password

            hostname = head + '://' + credentials + '@' + tail

        port = client.port if client.port is not None else DEFAULT_PORT

        parsed = uri_parser.parse_uri(hostname, port)

        dbname = parsed['database'] or client.virtual_host

        if dbname in ('/', None):
            dbname = 'kombu_default'

        options = {
            'auto_start_request': True,
            'ssl': client.ssl,
            'connectTimeoutMS': (int(client.connect_timeout * 1000)
                                 if client.connect_timeout else None),
        }
        options.update(client.transport_options)
        options.update(parsed['options'])

        return hostname, dbname, options

    def _open(self, scheme='mongodb://'):
        hostname, dbname, options = self._parse_uri(scheme=scheme)

        mongoconn = MongoClient(
            host=hostname, ssl=options['ssl'],
            auto_start_request=options['auto_start_request'],
            connectTimeoutMS=options['connectTimeoutMS'],
            use_greenlets=_detect_environment() != 'default',
        )
        database = mongoconn[dbname]

        version = mongoconn.server_info()['version']
        if tuple(map(int, version.split('.')[:2])) < (1, 3):
            raise NotImplementedError(
                'Kombu requires MongoDB version 1.3+ (server is {0})'.format(
                    version))

        self._create_broadcast(database, options)

        self._client = database

    def _create_broadcast(self, database, options):
        '''Create capped collection for broadcast messages.'''
        if DEFAULT_BROADCAST_COLLECTION in database.collection_names():
            return

        capsize = options.get('capped_queue_size') or 100000
        database.create_collection(DEFAULT_BROADCAST_COLLECTION,
                                   size=capsize, capped=True)

    def _ensure_indexes(self):
        '''Ensure indexes on collections.'''
        self.get_messages().ensure_index(
            [('queue', 1), ('priority', 1), ('_id', 1)], background=True,
        )
        self.get_broadcast().ensure_index([('queue', 1)])
        self.get_routing().ensure_index([('queue', 1), ('exchange', 1)])

    def get_table(self, exchange):
        """Get table of bindings for ``exchange``."""
        # TODO Store a more complete exchange metatable in the
        #      routing collection
        localRoutes = frozenset(self.state.exchanges[exchange]['table'])
        brokerRoutes = self.get_messages().routing.find(
            {'exchange': exchange}
        )

        return localRoutes | frozenset((r['routing_key'],
                                        r['pattern'],
                                        r['queue']) for r in brokerRoutes)

    def _put_fanout(self, exchange, message, routing_key, **kwargs):
        """Deliver fanout message."""
        self.get_broadcast().insert({'payload': dumps(message),
                                     'queue': exchange})

    def _queue_bind(self, exchange, routing_key, pattern, queue):
        if self.typeof(exchange).type == 'fanout':
            self.create_broadcast_cursor(exchange, routing_key, pattern, queue)
            self._fanout_queues[queue] = exchange

        meta = {'exchange': exchange,
                'queue': queue,
                'routing_key': routing_key,
                'pattern': pattern}
        self.get_routing().update(meta, meta, upsert=True)

    def queue_delete(self, queue, **kwargs):
        self.get_routing().remove({'queue': queue})

        super(Channel, self).queue_delete(queue, **kwargs)

        if queue in self._fanout_queues:
            try:
                cursor = self._broadcast_cursors.pop(queue)
            except KeyError:
                pass
            else:
                cursor.close()

                self._fanout_queues.pop(queue)

    @property
    def client(self):
        if self._client is None:
            self._open()
            self._ensure_indexes()

        return self._client

    def get_messages(self):
        return self.client[DEFAULT_MESSAGES_COLLECTION]

    def get_routing(self):
        return self.client[DEFAULT_ROUTING_COLLECTION]

    def get_broadcast(self):
        return self.client[DEFAULT_BROADCAST_COLLECTION]

    def get_broadcast_cursor(self, queue):
        try:
            return self._broadcast_cursors[queue]
        except KeyError:
            # Cursor may be absent when Channel created more than once.
            # _fanout_queues is a class-level mutable attribute so it's
            # shared over all Channel instances.
            return self.create_broadcast_cursor(
                self._fanout_queues[queue], None, None, queue,
            )

    def create_broadcast_cursor(self, exchange, routing_key, pattern, queue):
        cursor = self.get_broadcast().find(
            query={'queue': exchange},
            sort=[('$natural', 1)],
            tailable=True,
        )
        ret = self._broadcast_cursors[queue] = BroadcastCursor(cursor)
        return ret

class Transport(virtual.Transport):
    Channel = Channel

    can_parse_url = True
    polling_interval = 1
    default_port = DEFAULT_PORT
    connection_errors = (
        virtual.Transport.connection_errors + (errors.ConnectionFailure, )
    )
    channel_errors = (
        virtual.Transport.channel_errors + (
            errors.ConnectionFailure,
            errors.OperationFailure)
    )
    driver_type = 'mongodb'
    driver_name = 'pymongo'

    def driver_version(self):
        return pymongo.version
<|endoftext|>