<commit_before>#!/usr/bin/env python

from argparse import ArgumentParser
from ServerConfig import General
from ServerConfig import Storage
from ServerConfig import Microbench
from ServerConfig import TellStore
from mbclient import startMBClient
from mbserver import startMBServer
from storage import *
from observer import *
from functools import partial
from stop_java_unmount_memfs import stop_java_unmount_memfs
import time
import os
import sys
import signal

import logging

logging.basicConfig()

def exitGracefully(signal, frame):
    stop_java_unmount_memfs()
    sys.exit(0)

def sqliteOut():
    storage = Storage.storage().__class__.__name__.lower()
    if Storage.storage == TellStore:
        storage = storage + "_{0}".format(TellStore.approach)
#     else:
#         print "Error: current storage not supported"
#         exit(1)
    numStorages = len(Storage.servers) + len(Storage.servers1)
    numMBServers = len(Microbench.servers0) + len(Microbench.servers1)
    numClients = Microbench.clients
    numAnalytical = Microbench.analyticalClients
    res = "{0}_{1}storages_{2}clients_{3}scans".format(storage, numStorages, numClients, numAnalytical)
    res += "_{0}infinioBatch".format(Microbench.infinioBatch)
    return res

def runMBench(outdir, onlyPopulation = False):
    # do as for many experiments we have to run
    ## start storages

    if Storage.storage == TellStore:
        stObserver = Observer("Initialize network server")
    if Storage.storage == Cassandra:
        stObserver = Observer("No host ID found")
    else:
        stObserver = None

    if stObserver == None:
       storageClients = startStorage([])
    else:
       storageClients = startStorage([stObserver])
       # wait for notification that they have started
       stObserver.waitFor(len(Storage.servers) + len(Storage.servers1))

    print "Storage started"

    ## start microbenchmark server
    mbObserver = Observer("Started mbench server")
    serverClients = startMBServer([mbObserver])
    mbObserver.waitFor(len(Microbench.servers0) + len(Microbench.servers1))
    time.sleep(2)
    print "Server started... Continue with population"

    clients = Microbench.clients
    Microbench.clients = 10*(len(Microbench.servers0) + len(Microbench.servers1)) - 1
    res = startMBClient(True, "{0}/{1}_population".format(outdir, sqliteOut()))
    Microbench.clients = clients
    if res != 0:
        print "Population failed"
        exit(res)
    print "Population done"
    if not onlyPopulation:
        res = startMBClient(False, "{0}/{1}".format(outdir, sqliteOut()))
        if res != 0:
            print "Benchmark failed"
            exit(res)

    for client in serverClients:
        client.kill()

    for client in storageClients:
        client.kill()

    for client in serverClients:
        client.join()

    for client in storageClients:
        client.join()

    if (Storage.storage == Cassandra or Storage.storage == Hadoop or Storage.storage == Hbase):
        stop_java_unmount_memfs()

def configForAnalytics():
    Microbench.analyticalClients = 1
    Microbench.clients = 0
    Microbench.infinioBatch = 16
    if len(Microbench.servers0) > 0:
        Microbench.servers1 = []
        while len(Microbench.servers0) > 1:
            del Microbench.servers0[-1]

def configGetPut():
    Microbench.analyticalClients = 0
    Microbench.clientThreads = 4
    Microbench.clients = 10*(len(Microbench.servers0) + len(Microbench.servers1)) - 1
    Microbench.threads = 1 if Storage.storage == TellStore else 4
    Microbench.insertProb = 0.166
    Microbench.updateProb = 0.166
    Microbench.deleteProb = 0.166

def configMixed():
    configGetPut()
    Microbench.analyticalClients = 1

def experiment1a(outdir):
    # edit Microbenchmark class
    configGetPut()
    runMBench(outdir)

def experiment1a_singlebatch(outdir):
    configGetPut()
    old = Microbench.infinioBatch
    Microbench.infinioBatch = 1
    runMBench(outdir)
    Microbench.infinioBatch = old

def experiment1b(outdir):
    configGetPut()
    Microbench.insertProb = 0.0
    Microbench.updateProb = 0.0
    Microbench.deleteProb = 0.0
    runMBench(outdir)

def experiment2a(outdir):
    configForAnalytics()
    runMBench(outdir)

def experiment3(outdir):
    configGetPut()
    Microbench.analyticalClients = 1
    runMBench(outdir)

def varyBatching(experiment, outdir):
    for i in [1,2,4,8,16,32,64]:
        Microbench.infinioBatch = i
        experiment(outdir)
    Microbench.infinioBatch = 16

def scalingExperiment(experiment, outdir, numNodes):
    Storage.master = 'euler07'
    Storage.servers = []
    servers = ['euler04', 'euler05', 'euler06', 'euler02']
    servers.reverse()
    mservers0 = ['euler03', 'euler08', 'euler09', 'euler10', 'euler11', 'euler01']
    mservers1 = servers + ['euler03', 'euler08', 'euler09', 'euler10', 'euler11', 'euler01']
    mservers0.reverse()
    mservers1.reverse()
    Microbench.servers0 = []
    Microbench.servers1 = []
    while len(Storage.servers) < numNodes:
        Storage.servers.append(servers.pop())
        while 3*len(Storage.servers) > len(Microbench.servers0) + len(Microbench.servers1):
            if len(mservers0) != 0:
                Microbench.servers0.append(mservers0.pop())
            else:
                Microbench.servers1.append(mservers1.pop())
    experiment(outdir)

def runOnTell(experiment, outdir, numNodes):
    Storage.storage = TellStore
    for approach in ["columnmap", "rowstore", "logstructured"]:
        TellStore.approach = approach
        TellStore.setDefaultMemorySize()
        for num in numNodes:
            experiment(outdir, num)

def runOnOthers(experiment, outdir, numNodes):
    for num in numNodes:
       experiment(outdir,num)

def runAllBenchmarks(outdir, experiments):
    if Storage.storage == TellStore:
        runOn = runOnTell
    else:
        runOn = runOnOthers
    if len(experiments) == 0 or "experiment1a" in experiments:
        print "#######################################"
        print " RUN EXPERIMENT 1a"
        print "#######################################"
        o = '{0}/experiment1a'.format(outdir)
        if os.path.isdir(o):
            raise RuntimeError('{0} exists'.format(o))
        os.mkdir(o)
        runOn(partial(scalingExperiment, experiment1a), o, [1,2,3,4])
    if len(experiments) == 0 or "experiment1b" in experiments:
        # Experiment 1b
        print "#######################################"
        print " RUN EXPERIMENT 1b"
        print "#######################################"
        o = '{0}/experiment1b'.format(outdir)
        if os.path.isdir(o):
            raise RuntimeError('{0} exists'.format(o))
        os.mkdir(o)
        runOn(partial(scalingExperiment, experiment1b), o, [1,2,3,4])
    if (len(experiments) == 0 or "experiment1c" in experiments) and Storage.storage == TellStore:
        # Experiment 1c
        # No experiment needed here (inserts are measured for all experiments)
        # o = '{0}/experiment1c'.format(outdir)
        # if os.path.isdir(o):
        #     raise RuntimeError('{0} exists'.format(o))
        # os.mkdir(o)
        # runOnTell(partial(scalingExperiment, experiment1c), o, [1,2,3,4])
        # Experiment 1d
        print "#######################################"
        print " RUN EXPERIMENT 1d"
        print "#######################################"
        o = '{0}/experiment1d'.format(outdir)
        if os.path.isdir(o):
            raise RuntimeError('{0} exists'.format(o))
        os.mkdir(o)
        runOn(partial(scalingExperiment, partial(varyBatching, experiment1a)), o, [2])
    if len(experiments) == 0 or "experiment2a" in experiments:
        # Experiment 2a
        print "#######################################"
        print " RUN EXPERIMENT 2a"
        print "#######################################"
        o = '{0}/experiment2a'.format(outdir)
        if os.path.isdir(o):
            raise RuntimeError('{0} exists'.format(o))
        os.mkdir(o)
        runOn(partial(scalingExperiment, experiment2a), o, [2])
    if len(experiments) == 0 or "experiment3" in experiments:
        # Experiment 3
        print "#######################################"
        print " RUN EXPERIMENT 3"
        print "#######################################"
        o = '{0}/experiment3'.format(outdir)
        if os.path.isdir(o):
            raise RuntimeError('{0} exists'.format(o))
        os.mkdir(o)
        runOn(partial(scalingExperiment, experiment3), o, [1, 2, 3, 4])

if __name__ == "__main__":
    signal.signal(signal.SIGINT, exitGracefully)
    out = 'results'
    parser = ArgumentParser()
    parser.add_argument("-o", help="Output directory", default=out)
    parser.add_argument('experiments', metavar='E', type=str, nargs='*', help='Experiments to run (none defaults to all)')
    args = parser.parse_args()
    if not os.path.isdir(out):
        os.mkdir(out)
    runAllBenchmarks(out, args.experiments)

<commit_msg>put experiment2a back to what it is supposed to be (scaling from 1 to 4)<commit_after>#!/usr/bin/env python

from argparse import ArgumentParser
from ServerConfig import General
from ServerConfig import Storage
from ServerConfig import Microbench
from ServerConfig import TellStore
from mbclient import startMBClient
from mbserver import startMBServer
from storage import *
from observer import *
from functools import partial
from stop_java_unmount_memfs import stop_java_unmount_memfs
import time
import os
import sys
import signal

import logging

logging.basicConfig()

def exitGracefully(signal, frame):
    stop_java_unmount_memfs()
    sys.exit(0)

def sqliteOut():
    storage = Storage.storage().__class__.__name__.lower()
    if Storage.storage == TellStore:
        storage = storage + "_{0}".format(TellStore.approach)
#     else:
#         print "Error: current storage not supported"
#         exit(1)
    numStorages = len(Storage.servers) + len(Storage.servers1)
    numMBServers = len(Microbench.servers0) + len(Microbench.servers1)
    numClients = Microbench.clients
    numAnalytical = Microbench.analyticalClients
    res = "{0}_{1}storages_{2}clients_{3}scans".format(storage, numStorages, numClients, numAnalytical)
    res += "_{0}infinioBatch".format(Microbench.infinioBatch)
    return res

def runMBench(outdir, onlyPopulation = False):
    # do as for many experiments we have to run
    ## start storages

    if Storage.storage == TellStore:
        stObserver = Observer("Initialize network server")
    if Storage.storage == Cassandra:
        stObserver = Observer("No host ID found")
    else:
        stObserver = None

    if stObserver == None:
       storageClients = startStorage([])
    else:
       storageClients = startStorage([stObserver])
       # wait for notification that they have started
       stObserver.waitFor(len(Storage.servers) + len(Storage.servers1))

    print "Storage started"

    ## start microbenchmark server
    mbObserver = Observer("Started mbench server")
    serverClients = startMBServer([mbObserver])
    mbObserver.waitFor(len(Microbench.servers0) + len(Microbench.servers1))
    time.sleep(2)
    print "Server started... Continue with population"

    clients = Microbench.clients
    Microbench.clients = 10*(len(Microbench.servers0) + len(Microbench.servers1)) - 1
    res = startMBClient(True, "{0}/{1}_population".format(outdir, sqliteOut()))
    Microbench.clients = clients
    if res != 0:
        print "Population failed"
        exit(res)
    print "Population done"
    if not onlyPopulation:
        res = startMBClient(False, "{0}/{1}".format(outdir, sqliteOut()))
        if res != 0:
            print "Benchmark failed"
            exit(res)

    for client in serverClients:
        client.kill()

    for client in storageClients:
        client.kill()

    for client in serverClients:
        client.join()

    for client in storageClients:
        client.join()

    if (Storage.storage == Cassandra or Storage.storage == Hadoop or Storage.storage == Hbase):
        stop_java_unmount_memfs()

def configForAnalytics():
    Microbench.analyticalClients = 1
    Microbench.clients = 0
    Microbench.infinioBatch = 16
    if len(Microbench.servers0) > 0:
        Microbench.servers1 = []
        while len(Microbench.servers0) > 1:
            del Microbench.servers0[-1]

def configGetPut():
    Microbench.analyticalClients = 0
    Microbench.clientThreads = 4
    Microbench.clients = 10*(len(Microbench.servers0) + len(Microbench.servers1)) - 1
    Microbench.threads = 1 if Storage.storage == TellStore else 4
    Microbench.insertProb = 0.166
    Microbench.updateProb = 0.166
    Microbench.deleteProb = 0.166

def configMixed():
    configGetPut()
    Microbench.analyticalClients = 1

def experiment1a(outdir):
    # edit Microbenchmark class
    configGetPut()
    runMBench(outdir)

def experiment1a_singlebatch(outdir):
    configGetPut()
    old = Microbench.infinioBatch
    Microbench.infinioBatch = 1
    runMBench(outdir)
    Microbench.infinioBatch = old

def experiment1b(outdir):
    configGetPut()
    Microbench.insertProb = 0.0
    Microbench.updateProb = 0.0
    Microbench.deleteProb = 0.0
    runMBench(outdir)

def experiment2a(outdir):
    configForAnalytics()
    runMBench(outdir)

def experiment3(outdir):
    configGetPut()
    Microbench.analyticalClients = 1
    runMBench(outdir)

def varyBatching(experiment, outdir):
    for i in [1,2,4,8,16,32,64]:
        Microbench.infinioBatch = i
        experiment(outdir)
    Microbench.infinioBatch = 16

def scalingExperiment(experiment, outdir, numNodes):
    Storage.master = 'euler07'
    Storage.servers = []
    servers = ['euler04', 'euler05', 'euler06', 'euler02']
    servers.reverse()
    mservers0 = ['euler03', 'euler08', 'euler09', 'euler10', 'euler11', 'euler01']
    mservers1 = servers + ['euler03', 'euler08', 'euler09', 'euler10', 'euler11', 'euler01']
    mservers0.reverse()
    mservers1.reverse()
    Microbench.servers0 = []
    Microbench.servers1 = []
    while len(Storage.servers) < numNodes:
        Storage.servers.append(servers.pop())
        while 3*len(Storage.servers) > len(Microbench.servers0) + len(Microbench.servers1):
            if len(mservers0) != 0:
                Microbench.servers0.append(mservers0.pop())
            else:
                Microbench.servers1.append(mservers1.pop())
    experiment(outdir)

def runOnTell(experiment, outdir, numNodes):
    Storage.storage = TellStore
    for approach in ["columnmap", "rowstore", "logstructured"]:
        TellStore.approach = approach
        TellStore.setDefaultMemorySize()
        for num in numNodes:
            experiment(outdir, num)

def runOnOthers(experiment, outdir, numNodes):
    for num in numNodes:
       experiment(outdir,num)

def runAllBenchmarks(outdir, experiments):
    if Storage.storage == TellStore:
        runOn = runOnTell
    else:
        runOn = runOnOthers
    if len(experiments) == 0 or "experiment1a" in experiments:
        print "#######################################"
        print " RUN EXPERIMENT 1a"
        print "#######################################"
        o = '{0}/experiment1a'.format(outdir)
        if os.path.isdir(o):
            raise RuntimeError('{0} exists'.format(o))
        os.mkdir(o)
        runOn(partial(scalingExperiment, experiment1a), o, [1,2,3,4])
    if len(experiments) == 0 or "experiment1b" in experiments:
        # Experiment 1b
        print "#######################################"
        print " RUN EXPERIMENT 1b"
        print "#######################################"
        o = '{0}/experiment1b'.format(outdir)
        if os.path.isdir(o):
            raise RuntimeError('{0} exists'.format(o))
        os.mkdir(o)
        runOn(partial(scalingExperiment, experiment1b), o, [1,2,3,4])
    if (len(experiments) == 0 or "experiment1c" in experiments) and Storage.storage == TellStore:
        # Experiment 1c
        # No experiment needed here (inserts are measured for all experiments)
        # o = '{0}/experiment1c'.format(outdir)
        # if os.path.isdir(o):
        #     raise RuntimeError('{0} exists'.format(o))
        # os.mkdir(o)
        # runOnTell(partial(scalingExperiment, experiment1c), o, [1,2,3,4])
        # Experiment 1d
        print "#######################################"
        print " RUN EXPERIMENT 1d"
        print "#######################################"
        o = '{0}/experiment1d'.format(outdir)
        if os.path.isdir(o):
            raise RuntimeError('{0} exists'.format(o))
        os.mkdir(o)
        runOn(partial(scalingExperiment, partial(varyBatching, experiment1a)), o, [2])
    if len(experiments) == 0 or "experiment2a" in experiments:
        # Experiment 2a
        print "#######################################"
        print " RUN EXPERIMENT 2a"
        print "#######################################"
        o = '{0}/experiment2a'.format(outdir)
        if os.path.isdir(o):
            raise RuntimeError('{0} exists'.format(o))
        os.mkdir(o)
        runOn(partial(scalingExperiment, experiment2a), o, [1,2,3,4])
    if len(experiments) == 0 or "experiment3" in experiments:
        # Experiment 3
        print "#######################################"
        print " RUN EXPERIMENT 3"
        print "#######################################"
        o = '{0}/experiment3'.format(outdir)
        if os.path.isdir(o):
            raise RuntimeError('{0} exists'.format(o))
        os.mkdir(o)
        runOn(partial(scalingExperiment, experiment3), o, [1, 2, 3, 4])

if __name__ == "__main__":
    signal.signal(signal.SIGINT, exitGracefully)
    out = 'results'
    parser = ArgumentParser()
    parser.add_argument("-o", help="Output directory", default=out)
    parser.add_argument('experiments', metavar='E', type=str, nargs='*', help='Experiments to run (none defaults to all)')
    args = parser.parse_args()
    if not os.path.isdir(out):
        os.mkdir(out)
    runAllBenchmarks(out, args.experiments)

<|endoftext|>


<commit_before>import logging
import config
import json
import discord

import discord.ext.commands as commands

def setup(bot):
    bot.add_cog(Admin(bot))

# Helper functions
def isOwner(context):
    'Check whether or not the user is the owner of the bot'
    return context.message.author.id == config.OWNER_ID

def isAdmin(context):
    '''Check whether or not the user is an admin of the bot'''
    if isOwner(context):
        return True
    else:
        with open('adminlist.json') as data_file:
            adminList = json.load(data_file)
        return context.message.author.id in adminList['admins']

class Admin:
    '''A cog defining commands for controlling the
    bot's operation such as stopping the bot'''
    def __init__(self, bot):
        self.logger = logging.getLogger(__name__)
        self.bot = bot

    def loadjson(self, jsonname):
        try:
            with open(jsonname) as data_file:
                data = json.load(data_file)
                self.logger.info('Json successfully loaded.')
            return data
        except FileNotFoundError:
            return {'admins' : []}

    def savejson(self, data, jsonname):
        try:
            with open(jsonname, 'wt') as outfile:
                json.dump(data, outfile)
            self.logger.info('Json successfully saved.')
        except TypeError as e:
            self.logger.error(e)

    @commands.command()
    @commands.check(isOwner)
    async def addadmin(self, ID : str):
        '''Makes a user admin via ID'''
        self.data = self.loadjson('adminlist.json')
        if ID.isdecimal() and not ID in self.data['admins'] and ID != config.OWNER_ID:
            self.data['admins'].append(ID)
            self.savejson(self.data, 'adminlist.json')
            self.logger.info('%s added as an admin' % ID)
            await self.bot.say('%s added as an admin.' % ID)
        else:
            self.logger.warning('User entered invalid ID.')
            await self.bot.say('You entered an invalid ID.')

    @commands.command()
    @commands.check(isOwner)
    async def removeadmin(self, ID : str):
        '''Removes a user from the admin list via ID'''
        self.data = self.loadjson('adminlist.json')
        if ID in self.data['admins']:
            self.data['admins'].remove(ID)
            self.savejson(self.data, 'adminlist.json')
            self.logger.info('%s removed from the admin list.' % ID)
            await self.bot.say('%s has been removed from the admin list.' % ID)
        else:
            self.logger.warning('User entered invalid ID: %s', ID)
            await self.bot.say('You entered an invalid ID.')

    @commands.command()
    @commands.check(isAdmin)
    async def listadmins(self):
        '''Prints a list of the admin ID's to the chat.'''
        self.data = self.loadjson('adminlist.json')
        self.logger.info('User listed admin ID\'s')
        await self.bot.say('**-------- ADMINS --------**')
        for admin in self.data['admins']:
            try:
                user = await self.bot.get_user_info(admin)
                self.logger.info("%s - %s", user.name, user.id)
                await self.bot.say('**User:** %s, **ID:** %s' % (user.name, user.id))
            except discord.errors.NotFound as e:
                self.logger.info('NotFound - %s', admin)
                await self.bot.say('**User:** NotFound (Is this a valid ID?), **ID:** %s' % admin)

    @commands.command()
    @commands.check(isAdmin)
    async def isadmin(self):
        '''Checks if the user is an admin'''
        self.logger.info('User checked if they were an admin')
        await self.bot.say('You are an admin! Congratulations but you still suck.')<commit_msg>Admin system overhaul<commit_after>import logging
from util import is_owner

import discord.ext.commands as commands

def setup(bot):
    bot.add_cog(Admin(bot))

def is_admin(context):
    'Check whether or not the user is an admin of the server'
    top_role = str(context.message.author.top_role)
    admin_role = "admin"
    if is_owner(context):
        return True
    else:
        return top_role == admin_role

class Admin:
    '''A cog defining commands for controlling the
    bot's operation such as stopping the bot'''
    def __init__(self, bot):
        self.logger = logging.getLogger(__name__)
        self.bot = bot

    @commands.command(pass_context=True)
    @commands.check(is_owner)
    async def promote(self, ctx, promotee: str):
        '''Command that promotes a user via ID or name to a role called
        "admin", given that it has the permissions and the role exists.
        ID is most accurate, then Name#0000, then Name.'''
        if ctx.message.server is not None:
            server = ctx.message.server
            self.logger.info('Kappa1')
            if promotee.isdecimal():
                member = server.get_member(promotee)
                self.logger.info('User entered valid ID: %s', promotee)
            else:
                member = server.get_member_named(promotee)
                self.logger.info('User entered valid name: %s', promotee)
            if member is not None:
                await self.bot.say('Invalid member name/ID entered.')
                self.logger.warning('User entered invalid Name or ID')
            else:
                roles = ['admin']
                await server.add_roles(member, roles)
                await self.bot.say('%s promoted to admin!' % member.name)
        else:
            await self.bot.say('You can\'t promote someone in a private chat.')
            self.logger.warning('User tried to promote in PM')
<|endoftext|>


<commit_before>import logging
import asyncio
import time
import random
import collections

import discord
import markovify
from discord.ext import commands

from .common import Cog

log = logging.getLogger(__name__)

def make_textmodel(texter, data):
    texter.model = markovify.NewlineText(data, texter.chain_length)

async def make_texter(chain_length, data, texter_id):
    texter = Texter(texter_id, chain_length)
    await texter.fill(data)
    return texter

class TexterFail(Exception):
    pass

class Texter:
    """Texter - Main texter class.

    This class holds information about a markov chain generator.
    """
    __slots__ = ('loop', 'id', 'refcount', 'chain_length', \
        'model', 'wordcount', 'linecount', 'time_taken')

    def __init__(self, texter_id, chain_length=1, loop=None):
        if loop is None:
            loop = asyncio.get_event_loop()

        self.id = texter_id
        self.refcount = 1
        self.wordcount = 0
        self.linecount = 0

        self.chain_length = chain_length
        self.loop = loop
        self.model = None

        #: Time taken to make this texter
        self.time_taken = 0

    def __repr__(self):
        return f'<Texter refcount={self.refcount} wordcount={self.wordcount}>'

    async def fill(self, data):
        """Fill a texter with its text model."""
        t_start = time.monotonic()

        future_textmodel = self.loop.run_in_executor(None, make_textmodel, self, data)
        await future_textmodel

        self.wordcount = data.count(' ') + 1
        self.linecount = data.count('\n')

        delta = round((time.monotonic() - t_start) * 1000, 2)
        self.time_taken = delta

        log.info(f"Texter.fill: {self.linecount} lines, {self.wordcount} words, {delta}ms")

    def _sentence(self, char_limit):
        """Get a sentence from a initialized texter."""
        text = 'None'
        if char_limit is not None:
            text = self.model.make_short_sentence(char_limit)
        else:
            text = self.model.make_sentence()

        return text

    async def sentence(self, char_limit=None):
        """Get a sentence from a initialized texter."""
        if self.refcount <= 4:
            # max value refcount can be is 5
            self.refcount += 1

        res = None
        count = 0
        while res is None:
            if count > 3: break
            future_sentence = self.loop.run_in_executor(None, self._sentence, char_limit)
            res = await future_sentence
            count += 1

        return str(res)

    def clear(self):
        del self.model, self.refcount, self.chain_length, self.loop

class Speak(Cog):
    def __init__(self, bot):
        super().__init__(bot)
        self.text_generators = {}
        self.generating = {}

        self.coll_task = self.bot.loop.create_task(self.coll_task_func())

        self.st_gen_totalms = 1
        self.st_gen_count = 1

        self.st_txc_totalms = 1
        self.st_txc_runs = 1

    def __unload(self):
        """Remove all texters from memory"""
        to_del = []
        for tx in self.text_generators.values():
            tx.clear()
            to_del.append(tx.id)

        for tx_id in to_del:
            del self.text_generators[tx_id]

    async def coll_task_func(self):
        try:
            while True:
                await self.texter_collection()
                await asyncio.sleep(60)
        except asyncio.CancelledError:
            pass

    async def texter_collection(self):
        """Free memory by collecting unused Texters."""
        amount = len(self.text_generators)
        if amount < 1:
            return

        t_start = time.monotonic()
        cleaned = 0

        for texter in list(self.text_generators.values()):
            if texter.refcount < 1:
                texter.clear()
                cleaned += 1
                del self.text_generators[texter.id]
            else:
                texter.refcount -= 1

        t_end = time.monotonic()

        if cleaned > 0:
            delta = round((t_end - t_start) * 1000, 2)

            self.st_txc_runs += 1
            self.st_txc_totalms += delta

            log.info(f'[tx:coll] {amount} -> {amount - cleaned}, {delta}ms')

    async def get_messages(self, guild, amount=2000) -> list:
        channel_id = await self.config.cfg_get(guild, 'speak_channel')
        channel = guild.get_channel(channel_id)
        if channel is None:
            raise TexterFail('Channel to read messages not found, use "j!help speakchan"?')

        self.generating[guild.id] = True
        try:
            messages = []
            async for message in channel.history(limit=amount):
                author = message.author
                if author == self.bot.user:
                    continue

                if author.bot:
                    continue

                content = message.clean_content
                if content.startswith('j!'):
                    continue

                messages.append(message.clean_content)

            self.generating[guild.id] = False
            return messages
        except discord.Forbidden:
            log.info(f'[get_messages] Forbidden from {guild.name}[{guild.id}]')
            self.generating[guild.id] = False
            raise TexterFail("Can't read from the channel, setup your permissions!")

    async def get_messages_str(self, guild, amount=2000):
        m = await self.get_messages(guild, amount)
        return '\n'.join(m)

    async def new_texter(self, guild):
        guild_messages = await self.get_messages_str(guild)
        new_texter = await make_texter(1, guild_messages, guild.id)

        self.st_txc_totalms += new_texter.time_taken
        self.st_txc_runs += 1

        self.text_generators[guild.id] = new_texter

    async def get_texter(self, guild):
        if guild.id not in self.text_generators:
            await self.new_texter(guild)

        return self.text_generators[guild.id]

    async def make_sentence(self, ctx, char_limit=None):
        with ctx.typing():
            try:
                texter = await self.get_texter(ctx.guild)
            except TexterFail as err:
                raise self.SayException(f'Failed to generate a texter: `{err.args[0]!r}`')

        sentence = await texter.sentence(char_limit)
        return sentence

    async def on_message(self, message):
        ctx = await self.bot.get_context(message)
        if message.author.bot:
            return

        if not isinstance(ctx.channel, discord.TextChannel):
            return

        if await self.bot.is_blocked(message.author.id):
            return

        if await self.bot.is_blocked_guild(message.guild.id):
            return

        prob = await self.config.cfg_get(ctx.guild, 'autoreply_prob')
        if prob is None:
            log.warning('[autoreply] how can autoreply_prob be none??')
            return

        force = False
        for prefix in self.bot.config.SPEAK_PREFIXES:
            if message.content.startswith(prefix):
                log.info('[autoreply:speak_prefix] %s(%d), %s(%d) - %r', \
                    message.guild, message.guild.id, message.author, message.author.id, message.content)
                force = True

        if not force:
            if random.random() > prob:
                return

        if self.generating.get(ctx.guild.id):
            return

        sentence = await self.make_sentence(ctx)
        await ctx.send(sentence)

    @commands.command()
    @commands.is_owner()
    async def texclean(self, ctx, amount: int = 1):
        """Clean texters."""
        before = len(self.text_generators)
        t_start = time.monotonic()

        for i in range(amount):
            await self.texter_collection()

        after = len(self.text_generators)
        t_end = time.monotonic()

        delta = round((t_end - t_start) * 1000, 2)
        await ctx.send(f"`{before} => {after}, cleaned {before-after}, took {delta}ms`")

    @commands.command()
    @commands.is_owner()
    async def ntexter(self, ctx, guild_id: int = None):
        """Create a new texter for a guild, overwrites existing one"""
        if guild_id is None:
            guild_id = ctx.guild.id

        guild = self.bot.get_guild(guild_id)
        t1 = time.monotonic()
        await self.new_texter(guild)
        t2 = time.monotonic()

        delta = round((t2 - t1), 2)
        await ctx.send(f'Took {delta} seconds loading texter.')

    @commands.command(aliases=['spt'])
    @commands.guild_only()
    async def speaktrigger(self, ctx):
        """Force your Texter to say a sentence.

        If the texter is still being generated, this command
        does nothing while it isn't completly generated.
        """
        if self.generating.get(ctx.guild.id):
            return

        sentence = await self.make_sentence(ctx)
        await ctx.send(sentence)

    @commands.command(hidden=True)
    async def covfefe(self, ctx):
        """covfefe."""
        await ctx.send("Despite the constant negative press covfefe.")

    @commands.command(aliases=['jw'])
    @commands.guild_only()
    async def jwormhole(self, ctx):
        """lul wormhole"""
        res = await self.make_sentence(ctx)
        await ctx.send(f'<@127296623779774464> wormhole send {res}')

    @commands.command()
    @commands.guild_only()
    async def madlibs(self, ctx, *, inputstr: str):
        """Changes any "---" in the input to a 12-letter generated sentence"""

        inputstr = inputstr.replace('@everyone', '@\u200beveryone')
        inputstr = inputstr.replace('@here', '@\u200bhere')

        splitted = inputstr.split()
        if splitted.count('---') < 1:
            await ctx.send(":no_entry_sign: you can't just make josé say whatever you want! :no_entry_sign:")
            return

        if splitted.count('---') > 5:
            await ctx.send("thats a .......... lot")
            return

        res = []

        for word in splitted:
            if word == '---':
                res.append(await self.make_sentence(ctx, 12))
            else:
                res.append(word)

        await ctx.send(' '.join(res))

    @commands.command()
    @commands.is_owner()
    async def txstress(self, ctx):
        """Stress test texters LUL"""
        t1 = time.monotonic()
        async def create_tx(guild):
            try:
                return await self.new_texter(guild)
            except TexterFail:
                await ctx.send(f'Failed for `{guild!s}[{guild.id}]`')
        for guild in self.bot.guilds:
            await create_tx(guild)
        t2 = time.monotonic()

        delta = round((t2 - t1), 2)
        await ctx.send(f'Generated {len(txs)} texters in {delta} seconds')

    @commands.command()
    async def txstat(self, ctx):
        """Show statistics about all texters"""
        tg = self.text_generators

        refcounts = collections.Counter()
        for gid, tx in tg.items():
            refcounts[tx.refcount] += 1

        res = ['refcount | texters']
        res += [f'{r}        | {txc}' for (r, txc) in refcounts.most_common()]
        res = '\n'.join(res)
        await ctx.send(f'```{res}```')

    @commands.command()
    @commands.is_owner()
    async def alltx(self, ctx):
        """Get info about all loaded texters."""
        em = discord.Embed(colour=discord.Colour.blurple())
        em.description = ''
        for guild_id, tx in self.text_generators.items():
            guild = self.bot.get_guild(guild_id)
            em.description += f'**{guild!s}**, gid={guild_id} - `refcount={tx.refcount}, words={tx.wordcount} lines={tx.linecount}`\n'

        await ctx.send(embed=em)

def setup(bot):
    bot.add_cog(Speak(bot))
<commit_msg>speak: fix very bad typo<commit_after>import logging
import asyncio
import time
import random
import collections

import discord
import markovify
from discord.ext import commands

from .common import Cog

log = logging.getLogger(__name__)

def make_textmodel(texter, data):
    texter.model = markovify.NewlineText(data, texter.chain_length)

async def make_texter(chain_length, data, texter_id):
    texter = Texter(texter_id, chain_length)
    await texter.fill(data)
    return texter

class TexterFail(Exception):
    pass

class Texter:
    """Texter - Main texter class.

    This class holds information about a markov chain generator.
    """
    __slots__ = ('loop', 'id', 'refcount', 'chain_length', \
        'model', 'wordcount', 'linecount', 'time_taken')

    def __init__(self, texter_id, chain_length=1, loop=None):
        if loop is None:
            loop = asyncio.get_event_loop()

        self.id = texter_id
        self.refcount = 1
        self.wordcount = 0
        self.linecount = 0

        self.chain_length = chain_length
        self.loop = loop
        self.model = None

        #: Time taken to make this texter
        self.time_taken = 0

    def __repr__(self):
        return f'<Texter refcount={self.refcount} wordcount={self.wordcount}>'

    async def fill(self, data):
        """Fill a texter with its text model."""
        t_start = time.monotonic()

        future_textmodel = self.loop.run_in_executor(None, make_textmodel, self, data)
        await future_textmodel

        self.wordcount = data.count(' ') + 1
        self.linecount = data.count('\n')

        delta = round((time.monotonic() - t_start) * 1000, 2)
        self.time_taken = delta

        log.info(f"Texter.fill: {self.linecount} lines, {self.wordcount} words, {delta}ms")

    def _sentence(self, char_limit):
        """Get a sentence from a initialized texter."""
        text = 'None'
        if char_limit is not None:
            text = self.model.make_short_sentence(char_limit)
        else:
            text = self.model.make_sentence()

        return text

    async def sentence(self, char_limit=None):
        """Get a sentence from a initialized texter."""
        if self.refcount <= 4:
            # max value refcount can be is 5
            self.refcount += 1

        res = None
        count = 0
        while res is None:
            if count > 3: break
            future_sentence = self.loop.run_in_executor(None, self._sentence, char_limit)
            res = await future_sentence
            count += 1

        return str(res)

    def clear(self):
        del self.model, self.refcount, self.chain_length, self.loop

class Speak(Cog):
    def __init__(self, bot):
        super().__init__(bot)
        self.text_generators = {}
        self.generating = {}

        self.coll_task = self.bot.loop.create_task(self.coll_task_func())

        self.st_gen_totalms = 1
        self.st_gen_count = 1

        self.st_txc_totalms = 1
        self.st_txc_runs = 1

    def __unload(self):
        """Remove all texters from memory"""
        to_del = []
        for tx in self.text_generators.values():
            tx.clear()
            to_del.append(tx.id)

        for tx_id in to_del:
            del self.text_generators[tx_id]

    async def coll_task_func(self):
        try:
            while True:
                await self.texter_collection()
                await asyncio.sleep(60)
        except asyncio.CancelledError:
            pass

    async def texter_collection(self):
        """Free memory by collecting unused Texters."""
        amount = len(self.text_generators)
        if amount < 1:
            return

        t_start = time.monotonic()
        cleaned = 0

        for texter in list(self.text_generators.values()):
            if texter.refcount < 1:
                texter.clear()
                cleaned += 1
                del self.text_generators[texter.id]
            else:
                texter.refcount -= 1

        t_end = time.monotonic()

        if cleaned > 0:
            delta = round((t_end - t_start) * 1000, 2)

            self.st_txc_totalms += delta

            log.info(f'[tx:coll] {amount} -> {amount - cleaned}, {delta}ms')

        self.st_txc_runs += 1

    async def get_messages(self, guild, amount=2000) -> list:
        channel_id = await self.config.cfg_get(guild, 'speak_channel')
        channel = guild.get_channel(channel_id)
        if channel is None:
            raise TexterFail('Channel to read messages not found, use "j!help speakchan"?')

        self.generating[guild.id] = True
        try:
            messages = []
            async for message in channel.history(limit=amount):
                author = message.author
                if author == self.bot.user:
                    continue

                if author.bot:
                    continue

                content = message.clean_content
                if content.startswith('j!'):
                    continue

                messages.append(message.clean_content)

            self.generating[guild.id] = False
            return messages
        except discord.Forbidden:
            log.info(f'[get_messages] Forbidden from {guild.name}[{guild.id}]')
            self.generating[guild.id] = False
            raise TexterFail("Can't read from the channel, setup your permissions!")

    async def get_messages_str(self, guild, amount=2000):
        m = await self.get_messages(guild, amount)
        return '\n'.join(m)

    async def new_texter(self, guild):
        guild_messages = await self.get_messages_str(guild)
        new_texter = await make_texter(1, guild_messages, guild.id)

        self.st_gen_totalms += new_texter.time_taken
        self.st_gen_runs += 1

        self.text_generators[guild.id] = new_texter

    async def get_texter(self, guild):
        if guild.id not in self.text_generators:
            await self.new_texter(guild)

        return self.text_generators[guild.id]

    async def make_sentence(self, ctx, char_limit=None):
        with ctx.typing():
            try:
                texter = await self.get_texter(ctx.guild)
            except TexterFail as err:
                raise self.SayException(f'Failed to generate a texter: `{err.args[0]!r}`')

        sentence = await texter.sentence(char_limit)
        return sentence

    async def on_message(self, message):
        ctx = await self.bot.get_context(message)
        if message.author.bot:
            return

        if not isinstance(ctx.channel, discord.TextChannel):
            return

        if await self.bot.is_blocked(message.author.id):
            return

        if await self.bot.is_blocked_guild(message.guild.id):
            return

        prob = await self.config.cfg_get(ctx.guild, 'autoreply_prob')
        if prob is None:
            log.warning('[autoreply] how can autoreply_prob be none??')
            return

        force = False
        for prefix in self.bot.config.SPEAK_PREFIXES:
            if message.content.startswith(prefix):
                log.info('[autoreply:speak_prefix] %s(%d), %s(%d) - %r', \
                    message.guild, message.guild.id, message.author, message.author.id, message.content)
                force = True

        if not force:
            if random.random() > prob:
                return

        if self.generating.get(ctx.guild.id):
            return

        sentence = await self.make_sentence(ctx)
        await ctx.send(sentence)

    @commands.command()
    @commands.is_owner()
    async def texclean(self, ctx, amount: int = 1):
        """Clean texters."""
        before = len(self.text_generators)
        t_start = time.monotonic()

        for i in range(amount):
            await self.texter_collection()

        after = len(self.text_generators)
        t_end = time.monotonic()

        delta = round((t_end - t_start) * 1000, 2)
        await ctx.send(f"`{before} => {after}, cleaned {before-after}, took {delta}ms`")

    @commands.command()
    @commands.is_owner()
    async def ntexter(self, ctx, guild_id: int = None):
        """Create a new texter for a guild, overwrites existing one"""
        if guild_id is None:
            guild_id = ctx.guild.id

        guild = self.bot.get_guild(guild_id)
        t1 = time.monotonic()
        await self.new_texter(guild)
        t2 = time.monotonic()

        delta = round((t2 - t1), 2)
        await ctx.send(f'Took {delta} seconds loading texter.')

    @commands.command(aliases=['spt'])
    @commands.guild_only()
    async def speaktrigger(self, ctx):
        """Force your Texter to say a sentence.

        If the texter is still being generated, this command
        does nothing while it isn't completly generated.
        """
        if self.generating.get(ctx.guild.id):
            return

        sentence = await self.make_sentence(ctx)
        await ctx.send(sentence)

    @commands.command(hidden=True)
    async def covfefe(self, ctx):
        """covfefe."""
        await ctx.send("Despite the constant negative press covfefe.")

    @commands.command(aliases=['jw'])
    @commands.guild_only()
    async def jwormhole(self, ctx):
        """lul wormhole"""
        res = await self.make_sentence(ctx)
        await ctx.send(f'<@127296623779774464> wormhole send {res}')

    @commands.command()
    @commands.guild_only()
    async def madlibs(self, ctx, *, inputstr: str):
        """Changes any "---" in the input to a 12-letter generated sentence"""

        inputstr = inputstr.replace('@everyone', '@\u200beveryone')
        inputstr = inputstr.replace('@here', '@\u200bhere')

        splitted = inputstr.split()
        if splitted.count('---') < 1:
            await ctx.send(":no_entry_sign: you can't just make josé say whatever you want! :no_entry_sign:")
            return

        if splitted.count('---') > 5:
            await ctx.send("thats a .......... lot")
            return

        res = []

        for word in splitted:
            if word == '---':
                res.append(await self.make_sentence(ctx, 12))
            else:
                res.append(word)

        await ctx.send(' '.join(res))

    @commands.command()
    @commands.is_owner()
    async def txstress(self, ctx):
        """Stress test texters LUL"""
        t1 = time.monotonic()
        async def create_tx(guild):
            try:
                return await self.new_texter(guild)
            except TexterFail:
                await ctx.send(f'Failed for `{guild!s}[{guild.id}]`')
        for guild in self.bot.guilds:
            await create_tx(guild)
        t2 = time.monotonic()

        delta = round((t2 - t1), 2)
        await ctx.send(f'Generated {len(txs)} texters in {delta} seconds')

    @commands.command()
    async def txstat(self, ctx):
        """Show statistics about all texters"""
        tg = self.text_generators

        refcounts = collections.Counter()
        for gid, tx in tg.items():
            refcounts[tx.refcount] += 1

        res = ['refcount | texters']
        res += [f'{r}        | {txc}' for (r, txc) in refcounts.most_common()]
        res = '\n'.join(res)
        await ctx.send(f'```{res}```')

    @commands.command()
    @commands.is_owner()
    async def alltx(self, ctx):
        """Get info about all loaded texters."""
        em = discord.Embed(colour=discord.Colour.blurple())
        em.description = ''
        for guild_id, tx in self.text_generators.items():
            guild = self.bot.get_guild(guild_id)
            em.description += f'**{guild!s}**, gid={guild_id} - `refcount={tx.refcount}, words={tx.wordcount} lines={tx.linecount}`\n'

        await ctx.send(embed=em)

def setup(bot):
    bot.add_cog(Speak(bot))
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Copyright 2015 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import argparse
import json
import nose
import os
import signal
import subprocess
import sys
import unittest
import urllib
import urllib2

LOCAL_TEST_PORT = 9002

REGRESSION_TEST_URLS = [
        'http://www.blackanddecker.fr',
        'http://www.google.com',
        'http://dota2.gamepedia.com/',
        'http://www.orange.fr',
        'http://librarian.codes',
        'http://fredrikthalberg.com',
        'http://harleykwyn.com',
    ]

class PwsTest(unittest.TestCase):
    _HOST = None  # Set in main()
    _ENABLE_EXPERIMENTAL = False

    @property
    def HOST(self):
        PwsTest._HOST

    @property
    def ENABLE_EXPERIMENTAL(self):
        PwsTest._ENABLE_EXPERIMENTAL

    def request(self, params=None, payload=None):
        """
        Makes an http request to our endpoint

        If payload is None, this performs a GET request.
        Otherwise, the payload is json-serialized and a POST request is sent.

        """
        JSON = getattr(self, 'JSON', False)
        url = '{}/{}'.format(self.HOST, self.PATH)
        if params:
            url += '?{}'.format(urllib.urlencode(params))
        args = [url]
        if payload is not None:
            args.append(json.dumps(payload))
        req = urllib2.Request(*args)
        req.add_header("Content-Type", "application/json")
        response = urllib2.urlopen(req)
        data = response.read()
        if JSON:
            data = json.loads(data)
            # Print so we have something nice to look at when we fail
            print json.dumps(data, indent=2)
        else:
            print data
        return response.code, data

class TestResolveScan(PwsTest):
    PATH = 'resolve-scan'
    JSON = True

    def call(self, values):
        return self.request(payload=values)[1]

    def test_demo_data(self):
        result = self.call({
            'objects': [
                { 'url': 'http://www.caltrain.com/schedules/realtime/stations/mountainviewstation-mobile.html' },
                { 'url': 'http://benfry.com/distellamap/' },
                { 'url': 'http://en.wikipedia.org/wiki/Le_D%C3%A9jeuner_sur_l%E2%80%99herbe' },
                { 'url': 'http://sfmoma.org' }
            ]
        })
        self.assertIn('metadata', result)
        self.assertEqual(len(result['metadata']), 4)
        self.assertIn('description', result['metadata'][0])
        self.assertIn('title', result['metadata'][0])
        self.assertIn('url', result['metadata'][0])
        self.assertIn('displayUrl', result['metadata'][0])
        self.assertIn('rank', result['metadata'][0])
        self.assertIn('id', result['metadata'][0])
        self.assertIn('icon', result['metadata'][0])

    def test_invalid_data(self):
        result = self.call({
            'objects': [
                { 'url': 'http://totallybadurlthatwontwork.com/' },
                { 'usdf': 'http://badkeys' },
            ]
        })
        self.assertIn('metadata', result)
        self.assertEqual(len(result['metadata']), 0)
        #self.assertEqual(len(result['unresolved']), 1)

    def test_rssi_ranking(self):
        result = self.call({
            'objects': [
                {
                    'url': 'http://www.caltrain.com/schedules/realtime/stations/mountainviewstation-mobile.html',
                    'rssi': -75,
                    'txpower': -22,
                },
                {
                    'url': 'http://benfry.com/distellamap/',
                    'rssi': -95,
                    'txpower': -63,
                },
                {
                    'url': 'http://en.wikipedia.org/wiki/Le_D%C3%A9jeuner_sur_l%E2%80%99herbe',
                    'rssi': -61,
                    'txpower': -22,
                },
                {
                    'url': 'http://sfmoma.org',
                    'rssi': -74,
                    'txpower': -22,
                },
            ]
        })
        self.assertIn('metadata', result)
        self.assertEqual(len(result['metadata']), 4)
        self.assertEqual(result['metadata'][0]['id'],
                         'http://benfry.com/distellamap/')
        self.assertEqual(result['metadata'][1]['id'],
                         'http://en.wikipedia.org/wiki/'
                         'Le_D%C3%A9jeuner_sur_l%E2%80%99herbe')
        self.assertEqual(result['metadata'][2]['id'],
                         'http://sfmoma.org')
        self.assertEqual(result['metadata'][3]['id'],
                         'http://www.caltrain.com/schedules/realtime/'
                         'stations/mountainviewstation-mobile.html')

    def test_url_which_redirects(self):
        result = self.call({
            'objects': [
                {
                    'url': 'http://goo.gl/KYvLwO',
                },
            ]
        })
        self.assertIn('metadata', result)
        self.assertEqual(len(result['metadata']), 1)

        beaconResult = result['metadata'][0]

        self.assertEqual(beaconResult['id'],
                         'http://goo.gl/KYvLwO')
        self.assertEqual(beaconResult['url'],
                         'https://github.com/Google/physical-web')
        self.assertEqual(beaconResult['displayUrl'],
                         'https://github.com/Google/physical-web')

    def test_redirect_with_rssi_tx_power(self):
        if not self.ENABLE_EXPERIMENTAL:
            return

        result = self.call({
            'objects': [
                {
                    'url': '{}/experimental/googl/KYvLwO'.format(self.HOST),
                    'rssi': -41,
                    'txpower': -22
                },
                {
                    'url': '{}/experimental/googl/r8iJqW'.format(self.HOST),
                    'rssi': -91,
                    'txpower': -22
                },
            ]
        })
        self.assertIn('metadata', result)
        self.assertEqual(len(result['metadata']), 1)
        self.assertEqual(result['metadata'][0]['url'],
                         'https://github.com/Google/physical-web')

    def test_regression_urls(self):
        result = self.call({
            'objects': [ {'url': url} for url in REGRESSION_TEST_URLS ]
        })
        self.assertIn('metadata', result)
        self.assertEqual(len(result['metadata']), len(REGRESSION_TEST_URLS))

        for beaconResult in result['metadata']:
            self.assertIn('description', beaconResult)
            self.assertIn('title', beaconResult)
            self.assertIn('url', beaconResult)
            self.assertIn('rank', beaconResult)
            self.assertIn('id', beaconResult)
            self.assertIn('icon', beaconResult)

    def test_invalid_rssi(self):
        result = self.call({
            'objects': [{
                'url': 'http://github.com/google/physical-web/',
                'rssi': 127,
                'txpower': -41
                }]
        })
        self.assertIn('metadata', result)
        self.assertEqual(len(result['metadata']), 1)

        beaconResult = result['metadata'][0]

        self.assertIn('description', beaconResult)
        self.assertIn('title', beaconResult)
        self.assertIn('url', beaconResult)
        self.assertIn('rank', beaconResult)
        self.assertIn('id', beaconResult)
        self.assertIn('icon', beaconResult)

        self.assertEqual(1000, beaconResult['rank'])

class TestShortenUrl(PwsTest):
    PATH = 'shorten-url'
    JSON = True

    def call(self, values):
        return self.request(payload=values)[1]

    def test_github_url(self):
        result = self.call({
            'longUrl': 'http://www.github.com/Google/physical-web'
        })
        self.assertIn('kind', result)
        self.assertIn('id', result)
        self.assertIn('longUrl', result)
        self.assertTrue(result['id'].startswith('http://goo.gl/'))

class RefreshUrl(PwsTest):
    PATH = 'refresh-url'

    def call(self, url):
        params = {'url': url}
        return self.request(params=params, payload='')[1]

    def test_github_url(self):
        result = self.call('https://github.com/google/physical-web')
        self.assertEqual(result, '')

class TestGo(PwsTest):
    PATH = 'go'

    def call(self, url):
        params = {'url': url}
        return self.request(params=params)[0]

    def test_github_url(self):
        result = self.call('https://github.com/google/physical-web')
        self.assertEqual(result, 200)

def main():
    """The main routine."""
    # Parse arguments
    local_url = 'http://localhost:{}'.format(LOCAL_TEST_PORT)
    parser = argparse.ArgumentParser(description='Run web-service tests')
    parser.add_argument(
            '-e', '--endpoint', dest='endpoint', default='auto',
            help='Which server to test against.\n'
                 'auto:  {} (server starts automatically)\n'
                 'local: http://localhost:8080\n'
                 'prod:  https://url-caster.appspot.com\n'
                 'dev:   https://url-caster-dev.appspot.com\n'
                 '*:     Other values interpreted literally'
                 .format(local_url))
    parser.add_argument('-x', '--experimental', dest='experimental', action='store_true', default=False)
    args = parser.parse_args()

    # Setup the endpoint
    endpoint = args.endpoint
    server = None
    if endpoint.lower() == 'auto':
        endpoint = local_url
        print 'Starting local server...',
        server = subprocess.Popen([
            'dev_appserver.py', os.path.dirname(__file__),
            '--port', str(LOCAL_TEST_PORT),
            '--admin_port', str(LOCAL_TEST_PORT + 1),
        ], bufsize=1, stderr=subprocess.PIPE, preexec_fn=os.setsid)
        # Wait for the server to start up
        while True:
            line = server.stderr.readline()
            if 'Unable to bind' in line:
                print 'Rogue server already running.'
                return 1
            if 'running at: {}'.format(local_url) in line:
                break
        print 'done'
    elif endpoint.lower() == 'local':
        endpoint = 'http://localhost:8080'
    elif endpoint.lower() == 'prod':
        endpoint = 'https://url-caster.appspot.com'
    elif endpoint.lower() == 'dev':
        endpoint = 'https://url-caster-dev.appspot.com'
    PwsTest.HOST = endpoint
    PwsTest.ENABLE_EXPERIMENTAL = args.experimental

    # Run the tests
    try:
        nose.runmodule()
    finally:
        # Teardown the endpoint
        if server:
            os.killpg(os.getpgid(server.pid), signal.SIGINT)
            server.wait()

    # We should never get here since nose.runmodule will call exit
    return 0

if __name__ == '__main__':
    try:
        exit(main())
    except KeyboardInterrupt:
        sys.stderr.write('Exiting due to KeyboardInterrupt!\n')
<commit_msg>Add tests for invalid urls<commit_after>#!/usr/bin/env python
#
# Copyright 2015 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import argparse
import json
import nose
import os
import signal
import subprocess
import sys
import unittest
import urllib
import urllib2

LOCAL_TEST_PORT = 9002

REGRESSION_TEST_URLS = [
        'http://www.blackanddecker.fr',
        'http://www.google.com',
        'http://dota2.gamepedia.com/',
        'http://www.orange.fr',
        'http://librarian.codes',
        'http://fredrikthalberg.com',
        'http://harleykwyn.com',
    ]

REGRESSION_TEST_BAD_URLS = [
        'http://google.com/asdfasdfasdfasdfa',
        'http://www.',
    ]

class PwsTest(unittest.TestCase):
    _HOST = None  # Set in main()
    _ENABLE_EXPERIMENTAL = False

    @property
    def HOST(self):
        PwsTest._HOST

    @property
    def ENABLE_EXPERIMENTAL(self):
        PwsTest._ENABLE_EXPERIMENTAL

    def request(self, params=None, payload=None):
        """
        Makes an http request to our endpoint

        If payload is None, this performs a GET request.
        Otherwise, the payload is json-serialized and a POST request is sent.

        """
        JSON = getattr(self, 'JSON', False)
        url = '{}/{}'.format(self.HOST, self.PATH)
        if params:
            url += '?{}'.format(urllib.urlencode(params))
        args = [url]
        if payload is not None:
            args.append(json.dumps(payload))
        req = urllib2.Request(*args)
        req.add_header("Content-Type", "application/json")
        response = urllib2.urlopen(req)
        data = response.read()
        if JSON:
            data = json.loads(data)
            # Print so we have something nice to look at when we fail
            print json.dumps(data, indent=2)
        else:
            print data
        return response.code, data

class TestResolveScan(PwsTest):
    PATH = 'resolve-scan'
    JSON = True

    def call(self, values):
        return self.request(payload=values)[1]

    def test_demo_data(self):
        result = self.call({
            'objects': [
                { 'url': 'http://www.caltrain.com/schedules/realtime/stations/mountainviewstation-mobile.html' },
                { 'url': 'http://benfry.com/distellamap/' },
                { 'url': 'http://en.wikipedia.org/wiki/Le_D%C3%A9jeuner_sur_l%E2%80%99herbe' },
                { 'url': 'http://sfmoma.org' }
            ]
        })
        self.assertIn('metadata', result)
        self.assertEqual(len(result['metadata']), 4)
        self.assertIn('description', result['metadata'][0])
        self.assertIn('title', result['metadata'][0])
        self.assertIn('url', result['metadata'][0])
        self.assertIn('displayUrl', result['metadata'][0])
        self.assertIn('rank', result['metadata'][0])
        self.assertIn('id', result['metadata'][0])
        self.assertIn('icon', result['metadata'][0])

    def test_invalid_data(self):
        result = self.call({
            'objects': [
                { 'url': 'http://totallybadurlthatwontwork.com/' },
                { 'usdf': 'http://badkeys' },
            ]
        })
        self.assertIn('metadata', result)
        self.assertEqual(len(result['metadata']), 0)
        #self.assertEqual(len(result['unresolved']), 1)

    def test_rssi_ranking(self):
        result = self.call({
            'objects': [
                {
                    'url': 'http://www.caltrain.com/schedules/realtime/stations/mountainviewstation-mobile.html',
                    'rssi': -75,
                    'txpower': -22,
                },
                {
                    'url': 'http://benfry.com/distellamap/',
                    'rssi': -95,
                    'txpower': -63,
                },
                {
                    'url': 'http://en.wikipedia.org/wiki/Le_D%C3%A9jeuner_sur_l%E2%80%99herbe',
                    'rssi': -61,
                    'txpower': -22,
                },
                {
                    'url': 'http://sfmoma.org',
                    'rssi': -74,
                    'txpower': -22,
                },
            ]
        })
        self.assertIn('metadata', result)
        self.assertEqual(len(result['metadata']), 4)
        self.assertEqual(result['metadata'][0]['id'],
                         'http://benfry.com/distellamap/')
        self.assertEqual(result['metadata'][1]['id'],
                         'http://en.wikipedia.org/wiki/'
                         'Le_D%C3%A9jeuner_sur_l%E2%80%99herbe')
        self.assertEqual(result['metadata'][2]['id'],
                         'http://sfmoma.org')
        self.assertEqual(result['metadata'][3]['id'],
                         'http://www.caltrain.com/schedules/realtime/'
                         'stations/mountainviewstation-mobile.html')

    def test_url_which_redirects(self):
        result = self.call({
            'objects': [
                {
                    'url': 'http://goo.gl/KYvLwO',
                },
            ]
        })
        self.assertIn('metadata', result)
        self.assertEqual(len(result['metadata']), 1)

        beaconResult = result['metadata'][0]

        self.assertEqual(beaconResult['id'],
                         'http://goo.gl/KYvLwO')
        self.assertEqual(beaconResult['url'],
                         'https://github.com/Google/physical-web')
        self.assertEqual(beaconResult['displayUrl'],
                         'https://github.com/Google/physical-web')

    def test_redirect_with_rssi_tx_power(self):
        if not self.ENABLE_EXPERIMENTAL:
            return

        result = self.call({
            'objects': [
                {
                    'url': '{}/experimental/googl/KYvLwO'.format(self.HOST),
                    'rssi': -41,
                    'txpower': -22
                },
                {
                    'url': '{}/experimental/googl/r8iJqW'.format(self.HOST),
                    'rssi': -91,
                    'txpower': -22
                },
            ]
        })
        self.assertIn('metadata', result)
        self.assertEqual(len(result['metadata']), 1)
        self.assertEqual(result['metadata'][0]['url'],
                         'https://github.com/Google/physical-web')

    def test_regression_urls(self):
        result = self.call({
            'objects': [ {'url': url} for url in REGRESSION_TEST_URLS ]
        })
        self.assertIn('metadata', result)
        self.assertEqual(len(result['metadata']), len(REGRESSION_TEST_URLS))

        for beaconResult in result['metadata']:
            self.assertIn('description', beaconResult)
            self.assertIn('title', beaconResult)
            self.assertIn('url', beaconResult)
            self.assertIn('rank', beaconResult)
            self.assertIn('id', beaconResult)
            self.assertIn('icon', beaconResult)

    def test_regression_bad_urls(self):
        result = self.call({
            'objects': [ {'url': url} for url in REGRESSION_TEST_BAD_URLS ]
        })
        self.assertIn('metadata', result)
        self.assertEqual(len(result['metadata']), 0)

    def test_invalid_rssi(self):
        result = self.call({
            'objects': [{
                'url': 'http://github.com/google/physical-web/',
                'rssi': 127,
                'txpower': -41
                }]
        })
        self.assertIn('metadata', result)
        self.assertEqual(len(result['metadata']), 1)

        beaconResult = result['metadata'][0]

        self.assertIn('description', beaconResult)
        self.assertIn('title', beaconResult)
        self.assertIn('url', beaconResult)
        self.assertIn('rank', beaconResult)
        self.assertIn('id', beaconResult)
        self.assertIn('icon', beaconResult)

        self.assertEqual(1000, beaconResult['rank'])

class TestShortenUrl(PwsTest):
    PATH = 'shorten-url'
    JSON = True

    def call(self, values):
        return self.request(payload=values)[1]

    def test_github_url(self):
        result = self.call({
            'longUrl': 'http://www.github.com/Google/physical-web'
        })
        self.assertIn('kind', result)
        self.assertIn('id', result)
        self.assertIn('longUrl', result)
        self.assertTrue(result['id'].startswith('http://goo.gl/'))

class RefreshUrl(PwsTest):
    PATH = 'refresh-url'

    def call(self, url):
        params = {'url': url}
        return self.request(params=params, payload='')[1]

    def test_github_url(self):
        result = self.call('https://github.com/google/physical-web')
        self.assertEqual(result, '')

class TestGo(PwsTest):
    PATH = 'go'

    def call(self, url):
        params = {'url': url}
        return self.request(params=params)[0]

    def test_github_url(self):
        result = self.call('https://github.com/google/physical-web')
        self.assertEqual(result, 200)

def main():
    """The main routine."""
    # Parse arguments
    local_url = 'http://localhost:{}'.format(LOCAL_TEST_PORT)
    parser = argparse.ArgumentParser(description='Run web-service tests')
    parser.add_argument(
            '-e', '--endpoint', dest='endpoint', default='auto',
            help='Which server to test against.\n'
                 'auto:  {} (server starts automatically)\n'
                 'local: http://localhost:8080\n'
                 'prod:  https://url-caster.appspot.com\n'
                 'dev:   https://url-caster-dev.appspot.com\n'
                 '*:     Other values interpreted literally'
                 .format(local_url))
    parser.add_argument('-x', '--experimental', dest='experimental', action='store_true', default=False)
    args = parser.parse_args()

    # Setup the endpoint
    endpoint = args.endpoint
    server = None
    if endpoint.lower() == 'auto':
        endpoint = local_url
        print 'Starting local server...',
        server = subprocess.Popen([
            'dev_appserver.py', os.path.dirname(__file__),
            '--port', str(LOCAL_TEST_PORT),
            '--admin_port', str(LOCAL_TEST_PORT + 1),
        ], bufsize=1, stderr=subprocess.PIPE, preexec_fn=os.setsid)
        # Wait for the server to start up
        while True:
            line = server.stderr.readline()
            if 'Unable to bind' in line:
                print 'Rogue server already running.'
                return 1
            if 'running at: {}'.format(local_url) in line:
                break
        print 'done'
    elif endpoint.lower() == 'local':
        endpoint = 'http://localhost:8080'
    elif endpoint.lower() == 'prod':
        endpoint = 'https://url-caster.appspot.com'
    elif endpoint.lower() == 'dev':
        endpoint = 'https://url-caster-dev.appspot.com'
    PwsTest.HOST = endpoint
    PwsTest.ENABLE_EXPERIMENTAL = args.experimental

    # Run the tests
    try:
        nose.runmodule()
    finally:
        # Teardown the endpoint
        if server:
            os.killpg(os.getpgid(server.pid), signal.SIGINT)
            server.wait()

    # We should never get here since nose.runmodule will call exit
    return 0

if __name__ == '__main__':
    try:
        exit(main())
    except KeyboardInterrupt:
        sys.stderr.write('Exiting due to KeyboardInterrupt!\n')
<|endoftext|>


<commit_before>import json
import re
import requests

from .payload import *
from .template import *

# I agree with him : http://stackoverflow.com/a/36937/3843242
class NotificationType:
    REGULAR = 'REGULAR'
    SILENT_PUSH = 'SILENT_PUSH'
    NO_PUSH = 'NO_PUSH'

class SenderAction:
    TYPING_ON = 'typing_on'
    TYPING_OFF = 'typing_off'
    MARK_SEEN = 'mark_seen'

class Event(object):
    def __init__(self, messaging=None):
        if messaging is None:
            messaging = {}

        self.messaging = messaging
        self.matched_callbacks = []

    @property
    def sender_id(self):
        return self.messaging.get("sender", {}).get("id", None)

    @property
    def recipient_id(self):
        return self.messaging.get("recipient", {}).get("id", None)

    @property
    def timestamp(self):
        return self.messaging.get("timestamp", None)

    @property
    def message(self):
        return self.messaging.get("message", {})

    @property
    def message_text(self):
        return self.message.get("text", None)

    @property
    def message_attachments(self):
        return self.message.get("attachments", [])

    @property
    def quick_reply(self):
        return self.messaging.get("message", {}).get("quick_reply", {})

    @property
    def postback(self):
        return self.messaging.get("postback", {})

    @property
    def optin(self):
        return self.messaging.get("optin", {})

    @property
    def account_linking(self):
        return self.messaging.get("account_linking", {})

    @property
    def delivery(self):
        return self.messaging.get("delivery", {})

    @property
    def read(self):
        return self.messaging.get("read", {})

    @property
    def message_mid(self):
        return self.messaging.get("message", {}).get("mid", None)

    @property
    def message_seq(self):
        return self.messaging.get("message", {}).get("seq", None)

    @property
    def is_optin(self):
        return 'optin' in self.messaging

    @property
    def is_message(self):
        return 'message' in self.messaging

    @property
    def is_text_message(self):
        return self.messaging.get("message", {}).get("text", None) is not None

    @property
    def is_attachment_message(self):
        return self.messaging.get("message", {}).get("attachments", None) is not None

    @property
    def is_echo(self):
        return self.messaging.get("message", {}).get("is_echo", None) is not None

    @property
    def is_delivery(self):
        return 'delivery' in self.messaging

    @property
    def is_postback(self):
        return 'postback' in self.messaging

    @property
    def is_read(self):
        return 'read' in self.messaging

    @property
    def is_account_linking(self):
        return 'account_linking' in self.messaging

    @property
    def is_quick_reply(self):
        return self.messaging.get("message", {}).get("quick_reply", None) is not None

    @property
    def quick_reply_payload(self):
        return self.messaging.get("message", {}).get("quick_reply", {}).get("payload", '')

    @property
    def postback_payload(self):
        return self.messaging.get("postback", {}).get("payload", '')

class Page(object):
    def __init__(self, page_access_token, **options):
        self.page_access_token = page_access_token
        self._after_send = options.pop('after_send', None)
        self._page_id = None
        self._page_name = None

    # webhook_handlers contains optin, message, echo, delivery, postback, read, account_linking.
    # these are only set by decorators
    _webhook_handlers = {}

    _quick_reply_callbacks = {}
    _button_callbacks = {}

    _quick_reply_callbacks_key_regex = {}
    _button_callbacks_key_regex = {}

    _after_send = None

    def _call_handler(self, name, func, *args, **kwargs):
        if func is not None:
            func(*args, **kwargs)
        elif name in self._webhook_handlers:
            self._webhook_handlers[name](*args, **kwargs)
        else:
            print("there's no %s handler" % name)

    def handle_webhook(self, payload, optin=None, message=None, echo=None, delivery=None,
                       postback=None, read=None, account_linking=None):
        data = json.loads(payload)

        # Make sure this is a page subscription
        if data.get("object") != "page":
            print("Webhook failed, only support page subscription")
            return False

        # Iterate over each entry
        # There may be multiple if batched
        def get_events(data):
            for entry in data.get("entry"):
                for messaging in entry.get("messaging"):
                    event = Event(messaging)
                    yield event

        for event in get_events(data):
            if event.is_optin:
                self._call_handler('optin', optin, event)
            elif event.is_echo:
                self._call_handler('echo', echo, event)
            elif event.is_quick_reply:
                event.matched_callbacks = self.get_quick_reply_callbacks(event)
                self._call_handler('message', message, event)
                for callback in event.matched_callbacks:
                    callback(event.quick_reply_payload, event)
            elif event.is_message and not event.is_echo and not event.is_quick_reply:
                self._call_handler('message', message, event)
            elif event.is_delivery:
                self._call_handler('delivery', delivery, event)
            elif event.is_postback:
                event.matched_callbacks = self.get_postback_callbacks(event)
                self._call_handler('postback', postback, event)
                for callback in event.matched_callbacks:
                    callback(event.postback_payload, event)
            elif event.is_read:
                self._call_handler('read', read, event)
            elif event.is_account_linking:
                self._call_handler('account_linking', account_linking, event)
            else:
                print("Webhook received unknown messagingEvent:", event)

    @property
    def page_id(self):
        if self._page_id is None:
            self._fetch_page_info()

        return self._page_id

    @property
    def page_name(self):
        if self._page_name is None:
            self._fetch_page_info()

        return self._page_name

    def _fetch_page_info(self):
        r = requests.get("https://graph.facebook.com/v2.6/me",
                         params={"access_token": self.page_access_token},
                         headers={'Content-type': 'application/json'})

        if r.status_code != requests.codes.ok:
            print(r.text)
            return

        data = json.loads(r.text)
        if 'id' not in data or 'name' not in data:
            raise ValueError('Could not fetch data : GET /v2.6/me')

        self._page_id = data['id']
        self._page_name = data['name']

    def get_user_profile(self, fb_user_id):
        r = requests.get("https://graph.facebook.com/v2.6/%s" % fb_user_id,
                         params={"access_token": self.page_access_token},
                         headers={'Content-type': 'application/json'})

        if r.status_code != requests.codes.ok:
            print(r.text)
            return

        return json.loads(r.text)

    def _send(self, payload, callback=None):
        r = requests.post("https://graph.facebook.com/v2.6/me/messages",
                          params={"access_token": self.page_access_token},
                          data=payload.to_json(),
                          headers={'Content-type': 'application/json'})

        if r.status_code != requests.codes.ok:
            print(r.text)

        if callback is not None:
            callback(payload, r)

        if self._after_send is not None:
            self._after_send(payload, r)

        return r

    def send(self, recipient_id, message, quick_replies=None, metadata=None,
             notification_type=None, callback=None):
        text = message if isinstance(message, str) else message.encode('utf-8') if isinstance(message,unicode) else None
        attachment = message if not isinstance(message, str) else None

        payload = Payload(recipient=Recipient(id=recipient_id),
                          message=Message(text=text,
                                          attachment=attachment,
                                          quick_replies=quick_replies,
                                          metadata=metadata),
                          notification_type=notification_type)

        return self._send(payload, callback=callback)

    def typing_on(self, recipient_id):
        payload = Payload(recipient=Recipient(id=recipient_id),
                          sender_action=SenderAction.TYPING_ON)

        self._send(payload)

    def typing_off(self, recipient_id):
        payload = Payload(recipient=Recipient(id=recipient_id),
                          sender_action=SenderAction.TYPING_OFF)

        self._send(payload)

    def mark_seen(self, recipient_id):
        payload = Payload(recipient=Recipient(id=recipient_id),
                          sender_action=SenderAction.MARK_SEEN)

        self._send(payload)

    """
    thread settings
    """

    def _send_thread_settings(self, data):
        r = requests.post("https://graph.facebook.com/v2.6/me/thread_settings",
                          params={"access_token": self.page_access_token},
                          data=data,
                          headers={'Content-type': 'application/json'})

        if r.status_code != requests.codes.ok:
            print(r.text)

    def greeting(self, text):
        if not text or not isinstance(text, str):
            raise ValueError("greeting text error")

        self._send_thread_settings(json.dumps({
            'setting_type': 'greeting',
            'greeting': {
                'text': text
            }
        }))

    def show_starting_button(self, payload):
        if not payload or not isinstance(payload, str):
            raise ValueError("show_starting_button payload error")

        self._send_thread_settings(json.dumps({
            "setting_type": "call_to_actions",
            "thread_state": "new_thread",
            "call_to_actions": [{
                "payload": payload
            }]
        }))

    def hide_starting_button(self):
        self._send_thread_settings(json.dumps({
            "setting_type": "call_to_actions",
            "thread_state": "new_thread"
        }))

    def show_persistent_menu(self, buttons):
        if not buttons or not isinstance(buttons, list):
            raise ValueError('show_persistent_menu buttons error')

        buttons = Buttons.convert_shortcut_buttons(buttons)

        buttons_dict = []
        for button in buttons:
            if isinstance(button, ButtonWeb):
                buttons_dict.append({
                    "type": "web_url",
                    "title": button.title,
                    "url": button.url
                })
            elif isinstance(button, ButtonPostBack):
                buttons_dict.append({
                    "type": "postback",
                    "title": button.title,
                    "payload": button.payload
                })
            else:
                raise ValueError('show_persistent_menu button type must be "url" or "postback"')

        self._send_thread_settings(json.dumps({
            "setting_type": "call_to_actions",
            "thread_state": "existing_thread",
            "call_to_actions": buttons_dict
        }))

    def hide_persistent_menu(self):
        self._send_thread_settings(json.dumps({
            "setting_type": "call_to_actions",
            "thread_state": "existing_thread"
        }))

    """
    decorations
    """

    def handle_optin(self, func):
        self._webhook_handlers['optin'] = func

    def handle_message(self, func):
        self._webhook_handlers['message'] = func

    def handle_echo(self, func):
        self._webhook_handlers['echo'] = func

    def handle_delivery(self, func):
        self._webhook_handlers['delivery'] = func

    def handle_postback(self, func):
        self._webhook_handlers['postback'] = func

    def handle_read(self, func):
        self._webhook_handlers['read'] = func

    def handle_account_linking(self, func):
        self._webhook_handlers['account_linking'] = func

    def after_send(self, func):
        self._after_send = func

    _callback_default_types = ['QUICK_REPLY', 'POSTBACK']

    def callback(self, payloads=None, types=None):
        if types is None:
            types = self._callback_default_types

        if not isinstance(types, list):
            raise ValueError('callback types must be list')

        for type in types:
            if type not in self._callback_default_types:
                raise ValueError('callback types must be "QUICK_REPLY" or "POSTBACK"')

        def wrapper(func):
            if payloads is None:
                return func

            for payload in payloads:
                if 'QUICK_REPLY' in types:
                    self._quick_reply_callbacks[payload] = func
                if 'POSTBACK' in types:
                    self._button_callbacks[payload] = func

            return func

        return wrapper

    def get_quick_reply_callbacks(self, event):
        callbacks = []
        for key in self._quick_reply_callbacks.keys():
            if key not in self._quick_reply_callbacks_key_regex:
                self._quick_reply_callbacks_key_regex[key] = re.compile(key + '$')

            if self._quick_reply_callbacks_key_regex[key].match(event.quick_reply_payload):
                callbacks.append(self._quick_reply_callbacks[key])

        return callbacks

    def get_postback_callbacks(self, event):
        callbacks = []
        for key in self._button_callbacks.keys():
            if key not in self._button_callbacks_key_regex:
                self._button_callbacks_key_regex[key] = re.compile(key + '$')

            if self._button_callbacks_key_regex[key].match(event.postback_payload):
                callbacks.append(self._button_callbacks[key])

        return callbacks
<commit_msg>close #16, fix unicode issue on python2.x<commit_after>import sys
import json
import re
import requests

from .payload import *
from .template import *

# I agree with him : http://stackoverflow.com/a/36937/3843242
class NotificationType:
    REGULAR = 'REGULAR'
    SILENT_PUSH = 'SILENT_PUSH'
    NO_PUSH = 'NO_PUSH'

class SenderAction:
    TYPING_ON = 'typing_on'
    TYPING_OFF = 'typing_off'
    MARK_SEEN = 'mark_seen'

class Event(object):
    def __init__(self, messaging=None):
        if messaging is None:
            messaging = {}

        self.messaging = messaging
        self.matched_callbacks = []

    @property
    def sender_id(self):
        return self.messaging.get("sender", {}).get("id", None)

    @property
    def recipient_id(self):
        return self.messaging.get("recipient", {}).get("id", None)

    @property
    def timestamp(self):
        return self.messaging.get("timestamp", None)

    @property
    def message(self):
        return self.messaging.get("message", {})

    @property
    def message_text(self):
        return self.message.get("text", None)

    @property
    def message_attachments(self):
        return self.message.get("attachments", [])

    @property
    def quick_reply(self):
        return self.messaging.get("message", {}).get("quick_reply", {})

    @property
    def postback(self):
        return self.messaging.get("postback", {})

    @property
    def optin(self):
        return self.messaging.get("optin", {})

    @property
    def account_linking(self):
        return self.messaging.get("account_linking", {})

    @property
    def delivery(self):
        return self.messaging.get("delivery", {})

    @property
    def read(self):
        return self.messaging.get("read", {})

    @property
    def message_mid(self):
        return self.messaging.get("message", {}).get("mid", None)

    @property
    def message_seq(self):
        return self.messaging.get("message", {}).get("seq", None)

    @property
    def is_optin(self):
        return 'optin' in self.messaging

    @property
    def is_message(self):
        return 'message' in self.messaging

    @property
    def is_text_message(self):
        return self.messaging.get("message", {}).get("text", None) is not None

    @property
    def is_attachment_message(self):
        return self.messaging.get("message", {}).get("attachments", None) is not None

    @property
    def is_echo(self):
        return self.messaging.get("message", {}).get("is_echo", None) is not None

    @property
    def is_delivery(self):
        return 'delivery' in self.messaging

    @property
    def is_postback(self):
        return 'postback' in self.messaging

    @property
    def is_read(self):
        return 'read' in self.messaging

    @property
    def is_account_linking(self):
        return 'account_linking' in self.messaging

    @property
    def is_quick_reply(self):
        return self.messaging.get("message", {}).get("quick_reply", None) is not None

    @property
    def quick_reply_payload(self):
        return self.messaging.get("message", {}).get("quick_reply", {}).get("payload", '')

    @property
    def postback_payload(self):
        return self.messaging.get("postback", {}).get("payload", '')

class Page(object):
    def __init__(self, page_access_token, **options):
        self.page_access_token = page_access_token
        self._after_send = options.pop('after_send', None)
        self._page_id = None
        self._page_name = None

    # webhook_handlers contains optin, message, echo, delivery, postback, read, account_linking.
    # these are only set by decorators
    _webhook_handlers = {}

    _quick_reply_callbacks = {}
    _button_callbacks = {}

    _quick_reply_callbacks_key_regex = {}
    _button_callbacks_key_regex = {}

    _after_send = None

    def _call_handler(self, name, func, *args, **kwargs):
        if func is not None:
            func(*args, **kwargs)
        elif name in self._webhook_handlers:
            self._webhook_handlers[name](*args, **kwargs)
        else:
            print("there's no %s handler" % name)

    def handle_webhook(self, payload, optin=None, message=None, echo=None, delivery=None,
                       postback=None, read=None, account_linking=None):
        data = json.loads(payload)

        # Make sure this is a page subscription
        if data.get("object") != "page":
            print("Webhook failed, only support page subscription")
            return False

        # Iterate over each entry
        # There may be multiple if batched
        def get_events(data):
            for entry in data.get("entry"):
                for messaging in entry.get("messaging"):
                    event = Event(messaging)
                    yield event

        for event in get_events(data):
            if event.is_optin:
                self._call_handler('optin', optin, event)
            elif event.is_echo:
                self._call_handler('echo', echo, event)
            elif event.is_quick_reply:
                event.matched_callbacks = self.get_quick_reply_callbacks(event)
                self._call_handler('message', message, event)
                for callback in event.matched_callbacks:
                    callback(event.quick_reply_payload, event)
            elif event.is_message and not event.is_echo and not event.is_quick_reply:
                self._call_handler('message', message, event)
            elif event.is_delivery:
                self._call_handler('delivery', delivery, event)
            elif event.is_postback:
                event.matched_callbacks = self.get_postback_callbacks(event)
                self._call_handler('postback', postback, event)
                for callback in event.matched_callbacks:
                    callback(event.postback_payload, event)
            elif event.is_read:
                self._call_handler('read', read, event)
            elif event.is_account_linking:
                self._call_handler('account_linking', account_linking, event)
            else:
                print("Webhook received unknown messagingEvent:", event)

    @property
    def page_id(self):
        if self._page_id is None:
            self._fetch_page_info()

        return self._page_id

    @property
    def page_name(self):
        if self._page_name is None:
            self._fetch_page_info()

        return self._page_name

    def _fetch_page_info(self):
        r = requests.get("https://graph.facebook.com/v2.6/me",
                         params={"access_token": self.page_access_token},
                         headers={'Content-type': 'application/json'})

        if r.status_code != requests.codes.ok:
            print(r.text)
            return

        data = json.loads(r.text)
        if 'id' not in data or 'name' not in data:
            raise ValueError('Could not fetch data : GET /v2.6/me')

        self._page_id = data['id']
        self._page_name = data['name']

    def get_user_profile(self, fb_user_id):
        r = requests.get("https://graph.facebook.com/v2.6/%s" % fb_user_id,
                         params={"access_token": self.page_access_token},
                         headers={'Content-type': 'application/json'})

        if r.status_code != requests.codes.ok:
            print(r.text)
            return

        return json.loads(r.text)

    def _send(self, payload, callback=None):
        r = requests.post("https://graph.facebook.com/v2.6/me/messages",
                          params={"access_token": self.page_access_token},
                          data=payload.to_json(),
                          headers={'Content-type': 'application/json'})

        if r.status_code != requests.codes.ok:
            print(r.text)

        if callback is not None:
            callback(payload, r)

        if self._after_send is not None:
            self._after_send(payload, r)

        return r

    def send(self, recipient_id, message, quick_replies=None, metadata=None,
             notification_type=None, callback=None):
        if sys.version_info >= (3, 0):
            text = message if isinstance(message, str) else None
        else:
            text = message if isinstance(message, str) else message.encode('utf-8') if isinstance(message, unicode) else None

        attachment = message if not text else None

        payload = Payload(recipient=Recipient(id=recipient_id),
                          message=Message(text=text,
                                          attachment=attachment,
                                          quick_replies=quick_replies,
                                          metadata=metadata),
                          notification_type=notification_type)

        return self._send(payload, callback=callback)

    def typing_on(self, recipient_id):
        payload = Payload(recipient=Recipient(id=recipient_id),
                          sender_action=SenderAction.TYPING_ON)

        self._send(payload)

    def typing_off(self, recipient_id):
        payload = Payload(recipient=Recipient(id=recipient_id),
                          sender_action=SenderAction.TYPING_OFF)

        self._send(payload)

    def mark_seen(self, recipient_id):
        payload = Payload(recipient=Recipient(id=recipient_id),
                          sender_action=SenderAction.MARK_SEEN)

        self._send(payload)

    """
    thread settings
    """

    def _send_thread_settings(self, data):
        r = requests.post("https://graph.facebook.com/v2.6/me/thread_settings",
                          params={"access_token": self.page_access_token},
                          data=data,
                          headers={'Content-type': 'application/json'})

        if r.status_code != requests.codes.ok:
            print(r.text)

    def greeting(self, text):
        if not text or not isinstance(text, str):
            raise ValueError("greeting text error")

        self._send_thread_settings(json.dumps({
            'setting_type': 'greeting',
            'greeting': {
                'text': text
            }
        }))

    def show_starting_button(self, payload):
        if not payload or not isinstance(payload, str):
            raise ValueError("show_starting_button payload error")

        self._send_thread_settings(json.dumps({
            "setting_type": "call_to_actions",
            "thread_state": "new_thread",
            "call_to_actions": [{
                "payload": payload
            }]
        }))

    def hide_starting_button(self):
        self._send_thread_settings(json.dumps({
            "setting_type": "call_to_actions",
            "thread_state": "new_thread"
        }))

    def show_persistent_menu(self, buttons):
        if not buttons or not isinstance(buttons, list):
            raise ValueError('show_persistent_menu buttons error')

        buttons = Buttons.convert_shortcut_buttons(buttons)

        buttons_dict = []
        for button in buttons:
            if isinstance(button, ButtonWeb):
                buttons_dict.append({
                    "type": "web_url",
                    "title": button.title,
                    "url": button.url
                })
            elif isinstance(button, ButtonPostBack):
                buttons_dict.append({
                    "type": "postback",
                    "title": button.title,
                    "payload": button.payload
                })
            else:
                raise ValueError('show_persistent_menu button type must be "url" or "postback"')

        self._send_thread_settings(json.dumps({
            "setting_type": "call_to_actions",
            "thread_state": "existing_thread",
            "call_to_actions": buttons_dict
        }))

    def hide_persistent_menu(self):
        self._send_thread_settings(json.dumps({
            "setting_type": "call_to_actions",
            "thread_state": "existing_thread"
        }))

    """
    decorations
    """

    def handle_optin(self, func):
        self._webhook_handlers['optin'] = func

    def handle_message(self, func):
        self._webhook_handlers['message'] = func

    def handle_echo(self, func):
        self._webhook_handlers['echo'] = func

    def handle_delivery(self, func):
        self._webhook_handlers['delivery'] = func

    def handle_postback(self, func):
        self._webhook_handlers['postback'] = func

    def handle_read(self, func):
        self._webhook_handlers['read'] = func

    def handle_account_linking(self, func):
        self._webhook_handlers['account_linking'] = func

    def after_send(self, func):
        self._after_send = func

    _callback_default_types = ['QUICK_REPLY', 'POSTBACK']

    def callback(self, payloads=None, types=None):
        if types is None:
            types = self._callback_default_types

        if not isinstance(types, list):
            raise ValueError('callback types must be list')

        for type in types:
            if type not in self._callback_default_types:
                raise ValueError('callback types must be "QUICK_REPLY" or "POSTBACK"')

        def wrapper(func):
            if payloads is None:
                return func

            for payload in payloads:
                if 'QUICK_REPLY' in types:
                    self._quick_reply_callbacks[payload] = func
                if 'POSTBACK' in types:
                    self._button_callbacks[payload] = func

            return func

        return wrapper

    def get_quick_reply_callbacks(self, event):
        callbacks = []
        for key in self._quick_reply_callbacks.keys():
            if key not in self._quick_reply_callbacks_key_regex:
                self._quick_reply_callbacks_key_regex[key] = re.compile(key + '$')

            if self._quick_reply_callbacks_key_regex[key].match(event.quick_reply_payload):
                callbacks.append(self._quick_reply_callbacks[key])

        return callbacks

    def get_postback_callbacks(self, event):
        callbacks = []
        for key in self._button_callbacks.keys():
            if key not in self._button_callbacks_key_regex:
                self._button_callbacks_key_regex[key] = re.compile(key + '$')

            if self._button_callbacks_key_regex[key].match(event.postback_payload):
                callbacks.append(self._button_callbacks[key])

        return callbacks
<|endoftext|>