<commit_before>"""
bots.py: Spawn virtual users/bots on a PyLink server and make them interact
with things.
"""

import sys
import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import utils
from log import log

@utils.add_cmd
def spawnclient(irc, source, args):
    """<nick> <ident> <host>

    Admin-only. Spawns the specified PseudoClient on the PyLink server.
    Note: this doesn't check the validity of any fields you give it!"""
    utils.checkAuthenticated(irc, source, allowOper=False)
    try:
        nick, ident, host = args[:3]
    except ValueError:
        irc.reply("Error: Not enough arguments. Needs 3: nick, user, host.")
        return
    irc.proto.spawnClient(nick, ident, host, manipulatable=True)

@utils.add_cmd
def quit(irc, source, args):
    """<target> [<reason>]

    Admin-only. Quits the PyLink client with nick <target>, if one exists."""
    utils.checkAuthenticated(irc, source, allowOper=False)
    try:
        nick = args[0]
    except IndexError:
        irc.reply("Error: Not enough arguments. Needs 1-2: nick, reason (optional).")
        return
    if irc.pseudoclient.uid == irc.nickToUid(nick):
        irc.reply("Error: Cannot quit the main PyLink PseudoClient!")
        return
    u = irc.nickToUid(nick)
    quitmsg =  ' '.join(args[1:]) or 'Client Quit'
    if not utils.isManipulatableClient(irc, u):
        irc.reply("Error: Cannot force quit a protected PyLink services client.")
        return
    irc.proto.quit(u, quitmsg)
    irc.callHooks([u, 'PYLINK_BOTSPLUGIN_QUIT', {'text': quitmsg, 'parse_as': 'QUIT'}])

def joinclient(irc, source, args):
    """<target> <channel1>,[<channel2>], etc.

    Admin-only. Joins <target>, the nick of a PyLink client, to a comma-separated list of channels."""
    utils.checkAuthenticated(irc, source, allowOper=False)
    try:
        nick = args[0]
        clist = args[1].split(',')
        if not clist:
            raise IndexError
    except IndexError:
        irc.reply("Error: Not enough arguments. Needs 2: nick, comma separated list of channels.")
        return
    u = irc.nickToUid(nick)
    if not utils.isManipulatableClient(irc, u):
        irc.reply("Error: Cannot force join a protected PyLink services client.")
        return
    for channel in clist:
        if not utils.isChannel(channel):
            irc.reply("Error: Invalid channel name %r." % channel)
            return
        irc.proto.join(u, channel)
        irc.callHooks([u, 'PYLINK_BOTSPLUGIN_JOIN', {'channel': channel, 'users': [u],
                                                'modes': irc.channels[channel].modes,
                                                'parse_as': 'JOIN'}])
utils.add_cmd(joinclient, name='join')

@utils.add_cmd
def nick(irc, source, args):
    """<target> <newnick>

    Admin-only. Changes the nick of <target>, a PyLink client, to <newnick>."""
    utils.checkAuthenticated(irc, source, allowOper=False)
    try:
        nick = args[0]
        newnick = args[1]
    except IndexError:
        irc.reply("Error: Not enough arguments. Needs 2: nick, newnick.")
        return
    u = irc.nickToUid(nick)
    if newnick in ('0', u):
        newnick = u
    elif not utils.isNick(newnick):
        irc.reply('Error: Invalid nickname %r.' % newnick)
        return
    elif not utils.isManipulatableClient(irc, u):
        irc.reply("Error: Cannot force nick changes for a protected PyLink services client.")
        return
    irc.proto.nick(u, newnick)
    irc.callHooks([u, 'PYLINK_BOTSPLUGIN_NICK', {'newnick': newnick, 'oldnick': nick, 'parse_as': 'NICK'}])

@utils.add_cmd
def part(irc, source, args):
    """<target> <channel1>,[<channel2>],... [<reason>]

    Admin-only. Parts <target>, the nick of a PyLink client, from a comma-separated list of channels."""
    utils.checkAuthenticated(irc, source, allowOper=False)
    try:
        nick = args[0]
        clist = args[1].split(',')
        reason = ' '.join(args[2:])
    except IndexError:
        irc.reply("Error: Not enough arguments. Needs 2: nick, comma separated list of channels.")
        return
    u = irc.nickToUid(nick)
    if not utils.isManipulatableClient(irc, u):
        irc.reply("Error: Cannot force part a protected PyLink services client.")
        return
    for channel in clist:
        if not utils.isChannel(channel):
            irc.reply("Error: Invalid channel name %r." % channel)
            return
        irc.proto.part(u, channel, reason)
    irc.callHooks([u, 'PYLINK_BOTSPLUGIN_PART', {'channels': clist, 'text': reason, 'parse_as': 'PART'}])

@utils.add_cmd
def msg(irc, source, args):
    """<source> <target> <text>

    Admin-only. Sends message <text> from <source>, where <source> is the nick of a PyLink client."""
    utils.checkAuthenticated(irc, source, allowOper=False)
    try:
        msgsource, target, text = args[0], args[1], ' '.join(args[2:])
    except IndexError:
        irc.reply('Error: Not enough arguments. Needs 3: source nick, target, text.')
        return
    sourceuid = irc.nickToUid(msgsource)
    if not sourceuid:
        irc.reply('Error: Unknown user %r.' % msgsource)
        return
    if not utils.isChannel(target):
        real_target = irc.nickToUid(target)
        if real_target is None:
            irc.reply('Error: Unknown user %r.' % target)
            return
    else:
        real_target = target
    if not text:
        irc.reply('Error: No text given.')
        return
    irc.proto.message(sourceuid, real_target, text)
    irc.callHooks([sourceuid, 'PYLINK_BOTSPLUGIN_MSG', {'target': real_target, 'text': text, 'parse_as': 'PRIVMSG'}])
<commit_msg>bots: make source client names optional (Closes #161)<commit_after>"""
bots.py: Spawn virtual users/bots on a PyLink server and make them interact
with things.
"""

import sys
import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import utils
from log import log

@utils.add_cmd
def spawnclient(irc, source, args):
    """<nick> <ident> <host>

    Admin-only. Spawns the specified PseudoClient on the PyLink server.
    Note: this doesn't check the validity of any fields you give it!"""
    utils.checkAuthenticated(irc, source, allowOper=False)
    try:
        nick, ident, host = args[:3]
    except ValueError:
        irc.reply("Error: Not enough arguments. Needs 3: nick, user, host.")
        return
    irc.proto.spawnClient(nick, ident, host, manipulatable=True)

@utils.add_cmd
def quit(irc, source, args):
    """<target> [<reason>]

    Admin-only. Quits the PyLink client with nick <target>, if one exists."""
    utils.checkAuthenticated(irc, source, allowOper=False)

    try:
        nick = args[0]
    except IndexError:
        irc.reply("Error: Not enough arguments. Needs 1-2: nick, reason (optional).")
        return
    if irc.pseudoclient.uid == irc.nickToUid(nick):
        irc.reply("Error: Cannot quit the main PyLink PseudoClient!")
        return

    u = irc.nickToUid(nick)

    quitmsg =  ' '.join(args[1:]) or 'Client Quit'

    if not utils.isManipulatableClient(irc, u):
        irc.reply("Error: Cannot force quit a protected PyLink services client.")
        return

    irc.proto.quit(u, quitmsg)
    irc.callHooks([u, 'PYLINK_BOTSPLUGIN_QUIT', {'text': quitmsg, 'parse_as': 'QUIT'}])

def joinclient(irc, source, args):
    """[<target>] <channel1>,[<channel2>], etc.

    Admin-only. Joins <target>, the nick of a PyLink client, to a comma-separated list of channels. If <target> is not given, it defaults to the main PyLink client."""
    utils.checkAuthenticated(irc, source, allowOper=False)

    try:
        # Check if the first argument is an existing PyLink client. If it is not,
        # then assume that the first argument was actually the channels being joined.
        u = irc.nickToUid(args[0])

        if not irc.isInternalClient(u):  # First argument isn't one of our clients
            raise IndexError

        clist = args[1]
    except IndexError:  # No nick was given; shift arguments one to the left.
        u = irc.pseudoclient.uid
        try:
            clist = args[0]
        except IndexError:
            irc.reply("Error: Not enough arguments. Needs 1-2: nick (optional), comma separated list of channels.")
            return

    clist = clist.split(',')
    if not clist:
        irc.reply("Error: No valid channels given.")
        return

    if not utils.isManipulatableClient(irc, u):
        irc.reply("Error: Cannot force join a protected PyLink services client.")
        return

    for channel in clist:
        if not utils.isChannel(channel):
            irc.reply("Error: Invalid channel name %r." % channel)
            return
        irc.proto.join(u, channel)

        # Call a join hook manually so other plugins like relay can understand it.
        irc.callHooks([u, 'PYLINK_BOTSPLUGIN_JOIN', {'channel': channel, 'users': [u],
                                                'modes': irc.channels[channel].modes,
                                                'parse_as': 'JOIN'}])
utils.add_cmd(joinclient, name='join')

@utils.add_cmd
def nick(irc, source, args):
    """[<target>] <newnick>

    Admin-only. Changes the nick of <target>, a PyLink client, to <newnick>. If <target> is not given, it defaults to the main PyLink client."""
    utils.checkAuthenticated(irc, source, allowOper=False)

    try:
        nick = args[0]
        newnick = args[1]
    except IndexError:
        try:
            nick = irc.pseudoclient.nick
            newnick = args[0]
        except IndexError:
            irc.reply("Error: Not enough arguments. Needs 1-2: nick (optional), newnick.")
            return
    u = irc.nickToUid(nick)

    if newnick in ('0', u):  # Allow /nick 0 to work
        newnick = u

    elif not utils.isNick(newnick):
        irc.reply('Error: Invalid nickname %r.' % newnick)
        return

    elif not utils.isManipulatableClient(irc, u):
        irc.reply("Error: Cannot force nick changes for a protected PyLink services client.")
        return

    irc.proto.nick(u, newnick)
    # Ditto above: manually send a NICK change hook payload to other plugins.
    irc.callHooks([u, 'PYLINK_BOTSPLUGIN_NICK', {'newnick': newnick, 'oldnick': nick, 'parse_as': 'NICK'}])

@utils.add_cmd
def part(irc, source, args):
    """[<target>] <channel1>,[<channel2>],... [<reason>]

    Admin-only. Parts <target>, the nick of a PyLink client, from a comma-separated list of channels. If <target> is not given, it defaults to the main PyLink client."""
    utils.checkAuthenticated(irc, source, allowOper=False)

    try:
        nick = args[0]
        clist = args[1]
        # For the part message, join all remaining arguments into one text string
        reason = ' '.join(args[2:])

        # First, check if the first argument is an existing PyLink client. If it is not,
        # then assume that the first argument was actually the channels being parted.
        u = irc.nickToUid(nick)
        if not irc.isInternalClient(u):  # First argument isn't one of our clients
            raise IndexError

    except IndexError:  # No nick was given; shift arguments one to the left.
        u = irc.pseudoclient.uid

        try:
            clist = args[0]
        except IndexError:
            irc.reply("Error: Not enough arguments. Needs 1-2: nick (optional), comma separated list of channels.")
            return
        reason = ' '.join(args[1:])

    clist = clist.split(',')
    if not clist:
        irc.reply("Error: No valid channels given.")
        return

    if not utils.isManipulatableClient(irc, u):
        irc.reply("Error: Cannot force part a protected PyLink services client.")
        return

    for channel in clist:
        if not utils.isChannel(channel):
            irc.reply("Error: Invalid channel name %r." % channel)
            return
        irc.proto.part(u, channel, reason)

    irc.callHooks([u, 'PYLINK_BOTSPLUGIN_PART', {'channels': clist, 'text': reason, 'parse_as': 'PART'}])

@utils.add_cmd
def msg(irc, source, args):
    """[<source>] <target> <text>

    Admin-only. Sends message <text> from <source>, where <source> is the nick of a PyLink client. If <source> is not given, it defaults to the main PyLink client."""
    utils.checkAuthenticated(irc, source, allowOper=False)

    # Because we want the source nick to be optional, this argument parsing gets a bit tricky.
    try:
        msgsource = args[0]
        target = args[1]
        text = ' '.join(args[2:])

        # First, check if the first argument is an existing PyLink client. If it is not,
        # then assume that the first argument was actually the message TARGET.
        sourceuid = irc.nickToUid(msgsource)
        if not irc.isInternalClient(sourceuid):  # First argument isn't one of our clients
            raise IndexError

        if not text:
            raise IndexError
    except IndexError:
        try:
            sourceuid = irc.pseudoclient.uid
            target = args[0]
            text = ' '.join(args[1:])
        except IndexError:
            irc.reply('Error: Not enough arguments. Needs 3: source nick, target, text.')
            return

    if not text:
        irc.reply('Error: No text given.')
        return

    if not utils.isChannel(target):
        # Convert nick of the message target to a UID, if the target isn't a channel
        real_target = irc.nickToUid(target)
        if real_target is None:  # Unknown target user, if target isn't a valid channel name
            irc.reply('Error: Unknown user %r.' % target)
            return
    else:
        real_target = target

    irc.proto.message(sourceuid, real_target, text)
    irc.callHooks([sourceuid, 'PYLINK_BOTSPLUGIN_MSG', {'target': real_target, 'text': text, 'parse_as': 'PRIVMSG'}])
<|endoftext|>


<commit_before>import html
import re

import discord
import peewee
from discord import Embed

from bot import Command, BaseModel, categories
from bot.utils import text_cut, auto_int
from bot.regex import pat_channel, pat_subreddit

class RedditFollow(Command):
    __author__ = 'makzk'
    __version__ = '1.1.5'

    def __init__(self, bot):
        super().__init__(bot)
        self.name = 'reddit'
        self.aliases = ['redditor']
        self.help = '$[reddit-help]'
        self.format = '$[reddit-format]'
        self.db_models = [Redditor, Post, ChannelFollow]
        self.chans = {}
        self.allow_pm = False
        self.owner_only = True
        self.category = categories.STAFF
        self.schedule = (self.load_task, 15)

    def on_loaded(self):
        self.load_channels()

    async def handle(self, cmd):
        if cmd.cmdname == 'redditor':
            cmd.args = ['posts'] + cmd.args
            cmd.argc = len(cmd.args)
            cmd.cmdname = 'reddit'

        if cmd.argc < 1:
            await cmd.answer('$[format]: $[reddit-format]')
            return

        if cmd.args[0] in ['set', 'follow', 'remove', 'unfollow']:
            if cmd.argc < 2:
                await cmd.answer('$[format]: $[reddit-format-set]')
                return
            else:
                if not pat_subreddit.match(cmd.args[1]):
                    await cmd.answer('$[reddit-error-sub-name]')
                    return

                if cmd.argc > 2:
                    chan_match = pat_channel.match(cmd.args[2])
                    if not chan_match:
                        await cmd.answer('$[format]: $[reddit-format-set]')
                        return

                    chan_id = auto_int(chan_match.group(1))
                    channel = cmd.message.guild.get_channel(chan_id)
                    if channel is None:
                        await cmd.answer('$[reddit-error-channel-not-found]')
                        return
                else:
                    channel = cmd.message.channel

            if cmd.args[0] in ['set', 'follow']:
                chan, created = ChannelFollow.get_or_create(
                    subreddit=cmd.args[1], serverid=cmd.message.guild.id, channelid=channel.id)

                if created:
                    msg = '$[reddit-sub-added-here]' if channel.id == cmd.channel.id else '$[reddit-sub-added]'
                    await cmd.answer(msg, locales={'sub': cmd.args[1]})
                    if chan.subreddit not in self.chans:
                        self.chans[chan.subreddit] = []

                    self.chans[chan.subreddit].append((chan.serverid, chan.channelid))
                    return
                else:
                    await cmd.answer('$[reddit-error-sub-already-added]')
                    return
            else:
                try:
                    asd = ChannelFollow.get(ChannelFollow.subreddit == cmd.args[1],
                                            ChannelFollow.serverid == cmd.message.guild.id,
                                            ChannelFollow.channelid == channel.id)
                    asd.delete_instance()

                    if cmd.args[1] in self.chans:
                        for tup in self.chans[cmd.args[1]]:
                            if tup == (cmd.message.guild.id, channel.id):
                                self.chans[cmd.args[1]].remove(tup)
                                break
                        if len(self.chans[cmd.args[1]]) == 0:
                            del self.chans[cmd.args[1]]

                    await cmd.answer('$[reddit-sub-removed]')
                    return
                except ChannelFollow.DoesNotExist:
                    await cmd.answer('$[reddit-error-not-followed]')
                    return
        elif cmd.args[0] == 'list':
            res = ChannelFollow.select().where(ChannelFollow.serverid == cmd.message.guild.id)
            resp = []
            for chan in res:
                resp.append('- **{}** ➡ <#{}>'.format(chan.subreddit, chan.channelid))

            if len(res) == 0:
                await cmd.answer('$[reddit-no-following-subs]')
            else:
                await cmd.answer('$[reddit-feeds-title]:\n{}'.format('\n'.join(resp)))
        elif cmd.args[0] == 'posts':
            if cmd.argc < 2:
                await cmd.answer('$[format]: $[reddit-format-posts]')
                return

            user = cmd.args[1]
            if user.startswith('/u/'):
                user = user[3:]
            if user.startswith('u/'):
                user = user[2:]

            num_posts = self.get_user_posts(user)

            if num_posts > 0:
                await cmd.answer('$[reddit-user-posts-count]', locales={
                    'num': num_posts, 'redditor': user, 's': ['s', ''][bool(num_posts == 1)]})
            else:
                await cmd.answer('$[reddit-user-no-posts]', locales={'redditor': user})
        else:
            await cmd.answer('$[format]: $[reddit-format]')

    async def load_task(self):
        post_id = ''
        for (subname, chans) in self.chans.items():
            subchannels = [chanid for svid, chanid in chans]
            posts = await self.get_posts(subname)

            if len(posts) == 0:
                continue

            data = posts[0]

            try:
                exists = Post.get(Post.id == data['id'])
            except Post.DoesNotExist:
                exists = False

            redditor = None
            if 'author' in data:
                redditor, _ = Redditor.get_or_create(name=data['author'].lower())

            while data['id'] != post_id and not exists:
                subname = data.get('subreddit') or data.get('display_name')
                embed = self.post_to_embed(data)
                if embed is None:
                    break

                for channel in subchannels:
                    chan = self.bot.get_channel(auto_int(channel))
                    if chan is not None:
                        try:
                            await self.bot.send_message(chan, content='$[reddit-message-title]', embed=embed,
                                                        locales={'sub': subname})
                        except discord.Forbidden:
                            self.log.debug('Could not sent a r/%s post to %s (%s) due to missing permissions',
                                           subname, chan.guild.name, chan.guild.id)
                    else:
                        self.log.warning('Channel ID %s not found for subreddit subscription r/%s', channel, subname)

                post_id = data['id']
                if not exists:
                    with self.bot.db.atomic():
                        Post.create(id=post_id, permalink=data['permalink'], over_18=data['over_18'])

                        if redditor is not None:
                            Redditor.update(posts=Redditor.posts + 1).where(
                                Redditor.name == data['author'].lower()).execute()

    def load_channels(self):
        for chan in ChannelFollow.select():
            if chan.subreddit not in self.chans:
                self.chans[chan.subreddit] = []

            self.chans[chan.subreddit].append((chan.serverid, chan.channelid))

    def get_user_posts(self, user):
        if not re.match('^[a-zA-Z0-9_-]*$', user):
            return None

        redditor, _ = Redditor.get_or_create(name=user.lower())
        return redditor.posts

    async def get_posts(self, sub, since=0):
        url = 'https://www.reddit.com/r/{}/new.json'.format(sub)
        req = self.http.get(url)
        async with req as r:
            if not r.status == 200:
                return []

            posts = []
            data = await r.json()
            for post in data['data']['children']:
                if since < post['data']['created']:
                    posts.append(post['data'])

            return posts

    def post_to_embed(self, post):
        author = '($[])'
        author_url = 'https://www.reddit.com/'
        if 'author' in post:
            author = '/u/' + post['author']
            author_url += 'user/' + post['author']

        if 'permalink' not in post:
            return None

        embed = Embed()
        embed.title = text_cut(post['title'], 256)
        embed.url = 'https://www.reddit.com' + post['permalink']
        embed.set_author(name=author, url=author_url)

        if post['is_self']:
            embed.description = text_cut(post['selftext'], 2048)
        elif post['media']:
            if 'preview' in post:
                embed.set_image(url=html.unescape(post['preview']['images'][0]['source']['url']))
            elif post['thumbnail'] != 'default' and post['thumbnail'].startswith('http'):
                embed.set_thumbnail(url=html.unescape(post['thumbnail']))
            embed.description = "$[reddit-media-link]: " + post['url']
        elif 'preview' in post:
            embed.set_image(url=html.unescape(post['preview']['images'][0]['source']['url']))
        elif post['thumbnail'] != '' and post['thumbnail'] != 'default' and post['thumbnail'].startswith('http'):
            embed.set_thumbnail(url=html.unescape(post['thumbnail']))

        return embed

class Post(BaseModel):
    id = peewee.CharField()
    permalink = peewee.CharField(null=True)
    over_18 = peewee.BooleanField(default=False)

class Redditor(BaseModel):
    name = peewee.TextField()
    posts = peewee.IntegerField(default=0)

class ChannelFollow(BaseModel):
    subreddit = peewee.TextField()
    serverid = peewee.TextField()
    channelid = peewee.TextField()
<commit_msg>Add channel info to permissions error on reddit post<commit_after>import html
import re

import discord
import peewee
from discord import Embed

from bot import Command, BaseModel, categories
from bot.utils import text_cut, auto_int
from bot.regex import pat_channel, pat_subreddit

class RedditFollow(Command):
    __author__ = 'makzk'
    __version__ = '1.1.5'

    def __init__(self, bot):
        super().__init__(bot)
        self.name = 'reddit'
        self.aliases = ['redditor']
        self.help = '$[reddit-help]'
        self.format = '$[reddit-format]'
        self.db_models = [Redditor, Post, ChannelFollow]
        self.chans = {}
        self.allow_pm = False
        self.owner_only = True
        self.category = categories.STAFF
        self.schedule = (self.load_task, 15)

    def on_loaded(self):
        self.load_channels()

    async def handle(self, cmd):
        if cmd.cmdname == 'redditor':
            cmd.args = ['posts'] + cmd.args
            cmd.argc = len(cmd.args)
            cmd.cmdname = 'reddit'

        if cmd.argc < 1:
            await cmd.answer('$[format]: $[reddit-format]')
            return

        if cmd.args[0] in ['set', 'follow', 'remove', 'unfollow']:
            if cmd.argc < 2:
                await cmd.answer('$[format]: $[reddit-format-set]')
                return
            else:
                if not pat_subreddit.match(cmd.args[1]):
                    await cmd.answer('$[reddit-error-sub-name]')
                    return

                if cmd.argc > 2:
                    chan_match = pat_channel.match(cmd.args[2])
                    if not chan_match:
                        await cmd.answer('$[format]: $[reddit-format-set]')
                        return

                    chan_id = auto_int(chan_match.group(1))
                    channel = cmd.message.guild.get_channel(chan_id)
                    if channel is None:
                        await cmd.answer('$[reddit-error-channel-not-found]')
                        return
                else:
                    channel = cmd.message.channel

            if cmd.args[0] in ['set', 'follow']:
                chan, created = ChannelFollow.get_or_create(
                    subreddit=cmd.args[1], serverid=cmd.message.guild.id, channelid=channel.id)

                if created:
                    msg = '$[reddit-sub-added-here]' if channel.id == cmd.channel.id else '$[reddit-sub-added]'
                    await cmd.answer(msg, locales={'sub': cmd.args[1]})
                    if chan.subreddit not in self.chans:
                        self.chans[chan.subreddit] = []

                    self.chans[chan.subreddit].append((chan.serverid, chan.channelid))
                    return
                else:
                    await cmd.answer('$[reddit-error-sub-already-added]')
                    return
            else:
                try:
                    asd = ChannelFollow.get(ChannelFollow.subreddit == cmd.args[1],
                                            ChannelFollow.serverid == cmd.message.guild.id,
                                            ChannelFollow.channelid == channel.id)
                    asd.delete_instance()

                    if cmd.args[1] in self.chans:
                        for tup in self.chans[cmd.args[1]]:
                            if tup == (cmd.message.guild.id, channel.id):
                                self.chans[cmd.args[1]].remove(tup)
                                break
                        if len(self.chans[cmd.args[1]]) == 0:
                            del self.chans[cmd.args[1]]

                    await cmd.answer('$[reddit-sub-removed]')
                    return
                except ChannelFollow.DoesNotExist:
                    await cmd.answer('$[reddit-error-not-followed]')
                    return
        elif cmd.args[0] == 'list':
            res = ChannelFollow.select().where(ChannelFollow.serverid == cmd.message.guild.id)
            resp = []
            for chan in res:
                resp.append('- **{}** ➡ <#{}>'.format(chan.subreddit, chan.channelid))

            if len(res) == 0:
                await cmd.answer('$[reddit-no-following-subs]')
            else:
                await cmd.answer('$[reddit-feeds-title]:\n{}'.format('\n'.join(resp)))
        elif cmd.args[0] == 'posts':
            if cmd.argc < 2:
                await cmd.answer('$[format]: $[reddit-format-posts]')
                return

            user = cmd.args[1]
            if user.startswith('/u/'):
                user = user[3:]
            if user.startswith('u/'):
                user = user[2:]

            num_posts = self.get_user_posts(user)

            if num_posts > 0:
                await cmd.answer('$[reddit-user-posts-count]', locales={
                    'num': num_posts, 'redditor': user, 's': ['s', ''][bool(num_posts == 1)]})
            else:
                await cmd.answer('$[reddit-user-no-posts]', locales={'redditor': user})
        else:
            await cmd.answer('$[format]: $[reddit-format]')

    async def load_task(self):
        post_id = ''
        for (subname, chans) in self.chans.items():
            subchannels = [chanid for svid, chanid in chans]
            posts = await self.get_posts(subname)

            if len(posts) == 0:
                continue

            data = posts[0]

            try:
                exists = Post.get(Post.id == data['id'])
            except Post.DoesNotExist:
                exists = False

            redditor = None
            if 'author' in data:
                redditor, _ = Redditor.get_or_create(name=data['author'].lower())

            while data['id'] != post_id and not exists:
                subname = data.get('subreddit') or data.get('display_name')
                embed = self.post_to_embed(data)
                if embed is None:
                    break

                for channel in subchannels:
                    chan = self.bot.get_channel(auto_int(channel))
                    if chan is not None:
                        try:
                            await self.bot.send_message(chan, content='$[reddit-message-title]', embed=embed,
                                                        locales={'sub': subname})
                        except discord.Forbidden:
                            self.log.debug('Could not sent a r/%s post to %s (%s) #%s (%s) due to missing permissions',
                                           subname, chan.guild.name, chan.guild.id, chan.name, chan.id)
                    else:
                        self.log.warning('Channel ID %s not found for subreddit subscription r/%s', channel, subname)

                post_id = data['id']
                if not exists:
                    with self.bot.db.atomic():
                        Post.create(id=post_id, permalink=data['permalink'], over_18=data['over_18'])

                        if redditor is not None:
                            Redditor.update(posts=Redditor.posts + 1).where(
                                Redditor.name == data['author'].lower()).execute()

    def load_channels(self):
        for chan in ChannelFollow.select():
            if chan.subreddit not in self.chans:
                self.chans[chan.subreddit] = []

            self.chans[chan.subreddit].append((chan.serverid, chan.channelid))

    def get_user_posts(self, user):
        if not re.match('^[a-zA-Z0-9_-]*$', user):
            return None

        redditor, _ = Redditor.get_or_create(name=user.lower())
        return redditor.posts

    async def get_posts(self, sub, since=0):
        url = 'https://www.reddit.com/r/{}/new.json'.format(sub)
        req = self.http.get(url)
        async with req as r:
            if not r.status == 200:
                return []

            posts = []
            data = await r.json()
            for post in data['data']['children']:
                if since < post['data']['created']:
                    posts.append(post['data'])

            return posts

    def post_to_embed(self, post):
        author = '($[])'
        author_url = 'https://www.reddit.com/'
        if 'author' in post:
            author = '/u/' + post['author']
            author_url += 'user/' + post['author']

        if 'permalink' not in post:
            return None

        embed = Embed()
        embed.title = text_cut(post['title'], 256)
        embed.url = 'https://www.reddit.com' + post['permalink']
        embed.set_author(name=author, url=author_url)

        if post['is_self']:
            embed.description = text_cut(post['selftext'], 2048)
        elif post['media']:
            if 'preview' in post:
                embed.set_image(url=html.unescape(post['preview']['images'][0]['source']['url']))
            elif post['thumbnail'] != 'default' and post['thumbnail'].startswith('http'):
                embed.set_thumbnail(url=html.unescape(post['thumbnail']))
            embed.description = "$[reddit-media-link]: " + post['url']
        elif 'preview' in post:
            embed.set_image(url=html.unescape(post['preview']['images'][0]['source']['url']))
        elif post['thumbnail'] != '' and post['thumbnail'] != 'default' and post['thumbnail'].startswith('http'):
            embed.set_thumbnail(url=html.unescape(post['thumbnail']))

        return embed

class Post(BaseModel):
    id = peewee.CharField()
    permalink = peewee.CharField(null=True)
    over_18 = peewee.BooleanField(default=False)

class Redditor(BaseModel):
    name = peewee.TextField()
    posts = peewee.IntegerField(default=0)

class ChannelFollow(BaseModel):
    subreddit = peewee.TextField()
    serverid = peewee.TextField()
    channelid = peewee.TextField()
<|endoftext|>


<commit_before>"""
Watstock's market prediction neural network.
"""

from __future__ import print_function
import time
import os
import numpy as np
import pandas as pd
import math
import json

from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers import GRU
from keras.layers import Dropout
from keras.layers import Activation
from keras.callbacks import EarlyStopping

from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import MinMaxScaler

# fixing random generator
np.random.seed(1234)

def create_dataset(df, timesteps=1):
  """Converts a dataframe of values into a matrix"""
  dataX, dataY = [], []
  for i in range(len(df)-timesteps):
    dataX.append(df[i:(i+timesteps), :])
    dataY.append(df[i + timesteps, :])

  return np.array(dataX), np.array(dataY)

def split_dataset(dataframe, timesteps=90, testset=30):
  # split into train and test datasets
  testset_ratio = 1.0 * (testset + timesteps)/len(dataframe)
  train_size = int(len(dataframe) * (1 - testset_ratio))
  test_size = len(dataframe) - train_size
  train_df, test_df = dataframe[0:train_size,:], dataframe[train_size:,:]

  # Reshape into X=...,t-3,t-2,t-1,t and Y=t+1
  X_train, Y_train = create_dataset(train_df, timesteps)
  X_test, Y_test = create_dataset(test_df, timesteps)

  # Reshape inputs from [samples, features] to [samples, timesteps, features] format
  X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], X_train.shape[2]))
  X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], X_test.shape[2]))

  # Leave only columns for prediction
  Y_train = Y_train[:,-1:]
  Y_test = Y_test[:,-1:]

  return X_train, Y_train, X_test, Y_test

def build_model(layers, sequence_length, dropout=None):
  model = Sequential()

  for i in xrange(len(layers) - 2):
    return_sequences = i < len(layers) - 3
    layer = None
    if i == 0:
      layer = GRU(layers[i+1],
                  input_shape=(sequence_length, layers[i]),
                  return_sequences=return_sequences)
    else:
      layer = GRU(layers[i+1],
                  return_sequences=return_sequences)

    model.add(layer)

    # adding dropout
    if dropout != None:
      model.add(Dropout(dropout))

  model.add(Dense(layers[-1]))
  model.add(Activation('linear')) # Since we are doing a regression, its activation is linear

  model.compile(loss='mse', optimizer='rmsprop')

  return model

def train_model(model, data, batch_size=1, epochs=100, valset=30):
  X_train, Y_train = data

  early_stopping = EarlyStopping(monitor='val_loss', patience=10, verbose=0)
  val_ratio = 1.0 * valset / len(X_train)

  start_time = time.time()
  model.fit(X_train,
            Y_train,
            batch_size=batch_size,
            nb_epoch=epochs,
            verbose=1,
            validation_split=val_ratio,
            callbacks=[early_stopping])

  return (time.time() - start_time)

def save_prediction(data):

  # serialize
  json_data = json.dumps(data, sort_keys=True, indent=4, separators=(',', ': '))

  # build file name
  current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
  output_name = 'results/%s_%s.json' % (data['symbol'], current_time)

  with open(output_name, 'w') as file:
    file.write(json_data)

def symbol_to_path(symbol, base_dir="data"):
  """Return CSV file path given ticker symbol."""
  return os.path.join(base_dir, "{}.csv".format(str(symbol)))

def get_data(symbol, dates, usecols=['Date', 'Adj Close']):
  """Read stock data (adjusted close) for given symbols from CSV files."""
  df = pd.DataFrame(index=dates)

  df_temp = pd.read_csv(symbol_to_path(symbol), index_col='Date',
                        parse_dates=True, usecols=usecols,
                        na_values=['nan'])
  df = df.join(df_temp)

  return df

def main():

  # params
  symbol = 'AAPL'
  date_from = '2006-12-05'
  date_to = '2016-12-05'
  tsteps = 15
  testset = 30
  layers = [300, 300, 100]

  batch_size = 10
  epochs = 500
  dropout = 0.2

  print('Symbol:', symbol)
  print('Time steps:', tsteps)
  print('Test set:', testset)

  # Define a date range
  dates = pd.date_range(date_from, date_to)

  # Get stock data
  df = get_data(symbol, dates, usecols=['Date', 'Adj Close'])
  df = df.dropna()

  # Add sentiment data
  # df_sentiment = get_data('AOS-AAPL', dates, usecols=['Date', 'Article Sentiment', 'Impact Score'])
  # df = df.join(df_sentiment)

  # Add day of year data
  # df_dayofyear = pd.to_datetime(df.index.values).dayofyear
  # dayofyear_df = pd.DataFrame(data=df_dayofyear, index=df.index.values, columns=['Day of year'])
  # df = df.join(dayofyear_df)

  # Reordering columns so Ads Close is the last
  #df = df[['Day of year', 'Volume', 'Adj Close']]
  # df = df[['Article Sentiment', 'Impact Score', 'Volume', 'Adj Close']]

  date_from = df.index[0].strftime('%Y-%m-%d')
  date_to = df.index[-1].strftime('%Y-%m-%d')
  print('Date from:', date_from)
  print('Date to:', date_to)

  features = df.shape[1]
  print('Features:', df.columns.values)

  # Normalize the dataset
  dataset = df.values
  dataset = dataset.astype('float32')

  scaler = MinMaxScaler(feature_range=(0, 1))
  dataset = scaler.fit_transform(dataset)

  # Split into train and test sets
  X_train, Y_train, X_test, Y_test = split_dataset(dataset, timesteps=tsteps, testset=testset)
  print('Train set:', len(X_train), ', test set:', len(X_test))

  # Create and fit the RNN
  architecture = [features] + layers + [1]
  print('Architecture:', architecture)

  model = build_model(architecture, sequence_length=tsteps, dropout=dropout)
  train_duration = train_model(model, (X_train, Y_train), batch_size=batch_size, epochs=epochs)

  # Make predictions
  print('Predicting...')
  Y_train_prediction = model.predict(X_train, batch_size=batch_size)
  Y_test_prediction = model.predict(X_test, batch_size=batch_size)

  # Invert predictions
  df_shape = np.zeros((X_train.shape[0], X_train.shape[2]))
  df_shape[:,-1:] = Y_train_prediction
  Y_train_prediction = scaler.inverse_transform(df_shape)[:,-1]

  df_shape = np.zeros((X_train.shape[0], X_train.shape[2]))
  df_shape[:,-1:] = Y_train
  Y_train = scaler.inverse_transform(df_shape)[:,-1]

  df_shape = np.zeros((X_test.shape[0], X_test.shape[2]))
  df_shape[:,-1:] = Y_test_prediction
  Y_test_prediction = scaler.inverse_transform(df_shape)[:,-1]

  df_shape = np.zeros((X_test.shape[0], X_test.shape[2]))
  df_shape[:,-1:] = Y_test
  Y_test = scaler.inverse_transform(df_shape)[:,-1]

  # Calculate accuracy as Mean absolute percentage error
  train_error_ds = abs(Y_train - Y_train_prediction) / Y_train
  test_error_df = abs(Y_test - Y_test_prediction) / Y_test
  print('Train Mean Absolute Percentage Error:', train_error_ds.mean() * 100)
  print('Test Mean Absolute Percentage Error:', test_error_df.mean() * 100)

  test_index = df.index[-Y_test.shape[0]:].strftime('%Y-%m-%d')

  prediction = {
    'symbol': symbol,
    'date_from': date_from,
    'date_to': date_to,
    'train_set': len(X_train),
    'test_set': len(X_test),
    'dates': test_index.tolist(),
    'features': df.columns.values.tolist(),
    'price': Y_test.tolist(),
    'prediction': Y_test_prediction.tolist(),
    'sequence_length': tsteps,
    'architecture': architecture,
    'train_accuracy': (1 - train_error_ds.mean()) * 100,
    'test_accuracy': (1 - test_error_df.mean()) * 100,
    'train_duration': train_duration
  }

  print(prediction)

  print('Saving prediction...')
  save_prediction(prediction)

if __name__ == "__main__":
  main()<commit_msg>val_set, batch_size and dropout parameters serialization<commit_after>"""
Watstock's market prediction neural network.
"""

from __future__ import print_function
import time
import os
import numpy as np
import pandas as pd
import math
import json

from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers import GRU
from keras.layers import Dropout
from keras.layers import Activation
from keras.callbacks import EarlyStopping

from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import MinMaxScaler

# fixing random generator
np.random.seed(1234)

def create_dataset(df, timesteps=1):
  """Converts a dataframe of values into a matrix"""
  dataX, dataY = [], []
  for i in range(len(df)-timesteps):
    dataX.append(df[i:(i+timesteps), :])
    dataY.append(df[i + timesteps, :])

  return np.array(dataX), np.array(dataY)

def split_dataset(dataframe, timesteps=90, testset=30):
  # split into train and test datasets
  testset_ratio = 1.0 * (testset + timesteps)/len(dataframe)
  train_size = int(len(dataframe) * (1 - testset_ratio))
  test_size = len(dataframe) - train_size
  train_df, test_df = dataframe[0:train_size,:], dataframe[train_size:,:]

  # Reshape into X=...,t-3,t-2,t-1,t and Y=t+1
  X_train, Y_train = create_dataset(train_df, timesteps)
  X_test, Y_test = create_dataset(test_df, timesteps)

  # Reshape inputs from [samples, features] to [samples, timesteps, features] format
  X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1], X_train.shape[2]))
  X_test = np.reshape(X_test, (X_test.shape[0], X_test.shape[1], X_test.shape[2]))

  # Leave only columns for prediction
  Y_train = Y_train[:,-1:]
  Y_test = Y_test[:,-1:]

  return X_train, Y_train, X_test, Y_test

def build_model(layers, sequence_length, dropout=None):
  model = Sequential()

  for i in xrange(len(layers) - 2):
    return_sequences = i < len(layers) - 3
    layer = None
    if i == 0:
      layer = GRU(layers[i+1],
                  input_shape=(sequence_length, layers[i]),
                  return_sequences=return_sequences)
    else:
      layer = GRU(layers[i+1],
                  return_sequences=return_sequences)

    model.add(layer)

    # adding dropout
    if dropout != None:
      model.add(Dropout(dropout))

  model.add(Dense(layers[-1]))
  model.add(Activation('linear')) # Since we are doing a regression, its activation is linear

  model.compile(loss='mse', optimizer='rmsprop')

  return model

def train_model(model, data, batch_size=1, epochs=100, valset=30):
  X_train, Y_train = data

  early_stopping = EarlyStopping(monitor='val_loss', patience=10, verbose=0)
  val_ratio = 1.0 * valset / len(X_train)

  start_time = time.time()
  model.fit(X_train,
            Y_train,
            batch_size=batch_size,
            nb_epoch=epochs,
            verbose=1,
            validation_split=val_ratio,
            callbacks=[early_stopping])

  return (time.time() - start_time)

def save_prediction(data):

  # serialize
  json_data = json.dumps(data, sort_keys=True, indent=4, separators=(',', ': '))

  # build file name
  current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
  output_name = 'results/%s_%s.json' % (data['symbol'], current_time)

  with open(output_name, 'w') as file:
    file.write(json_data)

def symbol_to_path(symbol, base_dir="data"):
  """Return CSV file path given ticker symbol."""
  return os.path.join(base_dir, "{}.csv".format(str(symbol)))

def get_data(symbol, dates, usecols=['Date', 'Adj Close']):
  """Read stock data (adjusted close) for given symbols from CSV files."""
  df = pd.DataFrame(index=dates)

  df_temp = pd.read_csv(symbol_to_path(symbol), index_col='Date',
                        parse_dates=True, usecols=usecols,
                        na_values=['nan'])
  df = df.join(df_temp)

  return df

def main():

  # params
  symbol = 'AAPL'
  date_from = '2006-12-05'
  date_to = '2016-12-05'
  tsteps = 15
  testset = 30
  layers = [300, 300, 100]

  batch_size = 10
  valset = 30
  epochs = 500
  dropout = 0.2

  print('Symbol:', symbol)
  print('Time steps:', tsteps)
  print('Test set:', testset)

  # Define a date range
  dates = pd.date_range(date_from, date_to)

  # Get stock data
  df = get_data(symbol, dates, usecols=['Date', 'Adj Close'])
  df = df.dropna()

  # Add sentiment data
  # df_sentiment = get_data('AOS-AAPL', dates, usecols=['Date', 'Article Sentiment', 'Impact Score'])
  # df = df.join(df_sentiment)

  # Add day of year data
  # df_dayofyear = pd.to_datetime(df.index.values).dayofyear
  # dayofyear_df = pd.DataFrame(data=df_dayofyear, index=df.index.values, columns=['Day of year'])
  # df = df.join(dayofyear_df)

  # Reordering columns so Ads Close is the last
  #df = df[['Day of year', 'Volume', 'Adj Close']]
  # df = df[['Article Sentiment', 'Impact Score', 'Volume', 'Adj Close']]

  date_from = df.index[0].strftime('%Y-%m-%d')
  date_to = df.index[-1].strftime('%Y-%m-%d')
  print('Date from:', date_from)
  print('Date to:', date_to)

  features = df.shape[1]
  print('Features:', df.columns.values)

  # Normalize the dataset
  dataset = df.values
  dataset = dataset.astype('float32')

  scaler = MinMaxScaler(feature_range=(0, 1))
  dataset = scaler.fit_transform(dataset)

  # Split into train and test sets
  X_train, Y_train, X_test, Y_test = split_dataset(dataset, timesteps=tsteps, testset=testset)
  print('Train set:', len(X_train), ', test set:', len(X_test))

  # Create and fit the RNN
  architecture = [features] + layers + [1]
  print('Architecture:', architecture)

  model = build_model(architecture, sequence_length=tsteps, dropout=dropout)
  train_duration = train_model(model, (X_train, Y_train), batch_size=batch_size, epochs=epochs, valset=valset)

  # Make predictions
  print('Predicting...')
  Y_train_prediction = model.predict(X_train, batch_size=batch_size)
  Y_test_prediction = model.predict(X_test, batch_size=batch_size)

  # Invert predictions
  df_shape = np.zeros((X_train.shape[0], X_train.shape[2]))
  df_shape[:,-1:] = Y_train_prediction
  Y_train_prediction = scaler.inverse_transform(df_shape)[:,-1]

  df_shape = np.zeros((X_train.shape[0], X_train.shape[2]))
  df_shape[:,-1:] = Y_train
  Y_train = scaler.inverse_transform(df_shape)[:,-1]

  df_shape = np.zeros((X_test.shape[0], X_test.shape[2]))
  df_shape[:,-1:] = Y_test_prediction
  Y_test_prediction = scaler.inverse_transform(df_shape)[:,-1]

  df_shape = np.zeros((X_test.shape[0], X_test.shape[2]))
  df_shape[:,-1:] = Y_test
  Y_test = scaler.inverse_transform(df_shape)[:,-1]

  # Calculate accuracy as Mean absolute percentage error
  train_error_ds = abs(Y_train - Y_train_prediction) / Y_train
  test_error_df = abs(Y_test - Y_test_prediction) / Y_test
  print('Train Mean Absolute Percentage Error:', train_error_ds.mean() * 100)
  print('Test Mean Absolute Percentage Error:', test_error_df.mean() * 100)

  test_index = df.index[-Y_test.shape[0]:].strftime('%Y-%m-%d')

  prediction = {
    'symbol': symbol,
    'date_from': date_from,
    'date_to': date_to,
    'train_set': len(X_train),
    'val_set': valset,
    'test_set': len(X_test),
    'dates': test_index.tolist(),
    'features': df.columns.values.tolist(),
    'price': Y_test.tolist(),
    'prediction': Y_test_prediction.tolist(),
    'sequence_length': tsteps,
    'architecture': architecture,
    'dropout': dropout,
    'train_accuracy': (1 - train_error_ds.mean()) * 100,
    'test_accuracy': (1 - test_error_df.mean()) * 100,
    'train_duration': train_duration,
    'batch_size': batch_size
  }

  print(prediction)

  print('Saving prediction...')
  save_prediction(prediction)

if __name__ == "__main__":
  main()<|endoftext|>


<commit_before>import os, syslog
import pygame
import time
import pywapi
import string
import logging

# *** Not to be used for commercial use without permission!
# if you want to buy the icons for commercial use please send me a note - http://vclouds.deviantart.com/ ***
#
# Credit goes to Jamie Jackson http://blog.jacobean.net/?p=1016 for his nice Tutorial and great Parts of his Code, which i re-use.
# Credit goes to Merlin the Red for his great pictures: http://vclouds.deviantart.com/art/plain-weather-icons-157162192 , which i use.
#  ^-> I use his svg files and bring them to ~ 328x328 px
#
# I modify the original code for my 4 inch display and that it work in two 'screens'. First shows the currentday, Secound shows the forecast.

installPathImgBig = "/opt/PiTFTWeather/img/big/"
installPathImgSmall = "/opt/PiTFTWeather/img/small/"

# location for Cottbus, Brandenburg, Germany on weather.com
weatherDotComLocationCode = 'GMXX0171'

class PyLcd :
    screen = None;
    colourBlack = (0, 0, 0)

    def __init__(self):
        "Ininitializes a new pygame screen using the framebuffer"
        # Based on "Python GUI in Linux frame buffer"
        # http://www.karoltomala.com/blog/?p=679
        disp_no = os.getenv("DISPLAY")
        if disp_no:
            print "I'm running under X display = {0}".format(disp_no)

        os.putenv('SDL_FBDEV', '/dev/fb1')

        # Select frame buffer driver
        # Make sure that SDL_VIDEODRIVER is set
        driver = 'fbcon'
        if not os.getenv('SDL_VIDEODRIVER'):
            os.putenv('SDL_VIDEODRIVER', driver)
        try:
            pygame.display.init()
        except Exception as e:
           print "exception: %s" % (e,)
        except pygame.error:
            print 'Driver: {0} failed.'.format(driver)
            exit(0)

        size = (pygame.display.Info().current_w, pygame.display.Info().current_h)
        self.screen = pygame.display.set_mode(size, pygame.FULLSCREEN)
        # Clear the screen to start
        self.screen.fill((0, 0, 0))
        # Initialise font support
        pygame.font.init()
        # Render the screen
        pygame.display.update()

    def __del__(self):
        logging.info("Ende.")
        pygame.display.quit()
        #print "Destructor pygame display shuts down."

# Create an instance of the PyScope class
lcd = PyLcd()

pygame.mouse.set_visible(False)

# font colours
colourWhite = (255, 255, 255)
colourBlack = (0, 0, 0)

# update interval
updateRate = 3600 # seconds, for server call
betweenTime = 20 # seconds, befor screen switching

screenTimeOffset = 20

# set up the fonts
# choose the font
fontpath = pygame.font.match_font('dejavusansmono')
# set up 2 sizes
font = pygame.font.Font(fontpath, 36)
fontS2 = pygame.font.Font(fontpath, 28)
fontSm = pygame.font.Font(fontpath, 18)

# possbile
states = ["initial", "screen1", "screen2", "network"]
state = "initial"
weather_com_result = {}
h = 0
w = 0

# starting class
class PygameWeather(object):
    # class variable shared by all instances
    variabel_0 = 'test'
    # more vars

    # def evaluateInformation()
    # def updateScreen1
    # def updateScreen2

    # call weather server, if no connection or error return old data
    def callServer( self, mydict ):
       old_values = mydict.copy()
       weather_com_result = pywapi.get_weather_from_weather_com(weatherDotComLocationCode)
       #check if the dictionary has error keep old data; else return new data
       if weather_com_result.has_key('error'):
           logging.info(weather_com_result['error'])
           return old_values
       else:
           return weather_com_result

    # to see the clock with secounds
    def showClock(self):
      updated = time.strftime("%H:%M:%S")
      text_surface = fontS2.render(updated, True, colourWhite)
      lcd.screen.blit(text_surface, (10, 260))

    #todo: seperate code in state-machine in run and put other stuff in seperate functions
    # implement run method
    def run(self):
       global betweenTime
       global updateRate
       global screenTimeOffset
       global weather_com_result
       global state
       global w
       global h
       global forecastIcons
       quit = False
       while not quit:
                for event in pygame.event.get():
                    if event.type == QUIT: #sth wrong here if pygame.QUIT
                      sys.exit()
                      quit = True
                    elif event.type == KEYDOWN and event.key == K_ESCAPE:
                      quit = True
                      logging.info("ESC Key was pressed")

                    if quit is True:
                      print "Escape Button was pressed."
                      return

                # retrieve data from weather.com and keep old values if no connection
                if state == "initial":
                  weather_com_result = self.callServer( weather_com_result )
                  state = "screen1"
                  # if weather_com_result is empty check TODO: FIXME
                  # run alternative data, infunction

                if betweenTime >= updateRate:
                  betweenTime = 0
                  state = "network"
                  logging.info("1 hour is over, Calling server...")
                  weather_com_result = self.callServer( weather_com_result )
                  logging.info("Calling server successful")
                  state = "screen1"
                  # if weather_com_result is empty check TODO: FIXME
                  # run alternative data
                #

                # extract current data for today
                today = weather_com_result['forecasts'][0]['day_of_week'][0:3] + " " \
                    + weather_com_result['forecasts'][0]['date'][4:] + " " \
                    + weather_com_result['forecasts'][0]['date'][:3]
                windSpeed = weather_com_result['current_conditions']['wind']['speed']
                currWind = "{}km/h ".format(windSpeed) + weather_com_result['current_conditions']['wind']['text']
                currTemp = weather_com_result['current_conditions']['temperature'] + u'\N{DEGREE SIGN}' + "C"
                currPress = weather_com_result['current_conditions']['barometer']['reading'][:-3] + "mb"
                uv = "UV {}".format(weather_com_result['current_conditions']['uv']['text'])
                humid = "Hum {}%".format(weather_com_result['current_conditions']['humidity'])
                currTempFeeling = "(" + weather_com_result['current_conditions']['feels_like'] + u'\N{DEGREE SIGN}' + "C)"
                todayDesc = "It is " + weather_com_result['current_conditions']['text'].lower() + " today."

                # summary and description of forecast data
                forecastDesc = ["Day", "Max", "Min", "   ", "   "] # forecastDesc = ["Day", "Max", "Min", "Hum", "Kmh"]

                # extract forecast data
                forecastDays = {}
                forecaseHighs = {}
                forecaseLows = {}
                forecastPrecips = {}
                forecastWinds = {}
                forecastIcons = {}

                for i in range(0, 5):
                    if not(weather_com_result['forecasts'][i]):
                        break

                    forecastDays[i] = weather_com_result['forecasts'][i]['day_of_week'][0:3]
                    forecaseHighs[i] = weather_com_result['forecasts'][i]['high'] + u'\N{DEGREE SIGN}' + "C"
                    forecaseLows[i] = weather_com_result['forecasts'][i]['low'] + u'\N{DEGREE SIGN}' + "C"
                    forecastPrecips[i] = weather_com_result['forecasts'][i]['day']['chance_precip'] + "%"
                    forecastWinds[i] = weather_com_result['forecasts'][i]['day']['wind']['speed'] + \
                        weather_com_result['forecasts'][i]['day']['wind']['text']
                    forecastIcons[i] = installPathImgSmall + (weather_com_result['forecasts'][i]['day']['icon']) + ".png"

                # 1. screen, dayInformation
                # blank the screen
                lcd.screen.fill(colourBlack)

                # Render the weather logo at 0,0
                icon = installPathImgBig + (weather_com_result['current_conditions']['icon']) + ".png"
                logo = pygame.image.load(icon).convert()
                w = logo.get_width() - 50
                h = logo.get_height() - 50
                logo = pygame.transform.scale(logo, (w,h))
                lcd.screen.blit(logo, (0, 0))

                # set the anchor for the current weather data text
                textAnchorX = 260 # 310 war ok bei 328 Bild
                textAnchorY = 5
                textYoffset = 40

                # add current weather data text artifacts to the screen
                text_surface = font.render(today, True, colourWhite)
                lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))
                textAnchorY+=textYoffset
                text_surface = font.render(currTemp, True, colourWhite)
                lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))
                # my code
                # feeled temp
                textAnchorY+=textYoffset
                text_surface = font.render(currTempFeeling, True, colourWhite)
                lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))
                # my code
                textAnchorY+=textYoffset
                text_surface = font.render(currWind, True, colourWhite)
                lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))
                textAnchorY+=textYoffset
                text_surface = font.render(currPress, True, colourWhite)
                lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))
                textAnchorY+=textYoffset
                text_surface = font.render(uv, True, colourWhite)
                lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))
                textAnchorY+=textYoffset
                text_surface = font.render(humid, True, colourWhite)
                lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))

                # wait between screen changes screenTimeOffset alias first betweenTime
                pygame.display.update()

                # wait
                time.sleep(screenTimeOffset)
                betweenTime += screenTimeOffset

                # blank the screen after screenTimeOffset
                lcd.screen.fill(colourBlack)
                pygame.display.update()

                # 2. screen
                # state = screen2
                # blank the screen
                lcd.screen.fill(colourBlack)
                pygame.display.update()

                # set X axis text anchor for the forecast text
                textAnchorX = 0
                textXoffset = 75 #100
                textAnchorY = 10
                #pygame.draw.line(lcd.screen.get_surface(), colourWhite, (50,10), (450,10),4)

                # add summy of the values in one row
                for i in range(0,5):
                  text_surface = fontS2.render(forecastDesc[i], True, colourWhite)
                  lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))
                  textAnchorY+=textYoffset

                textAnchorX+=70

                # add each days forecast text
                for i in range(0, 5):
                    textAnchorY = 10
                    text_surface = fontS2.render(forecastDays[i], True, colourWhite)
                    lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))
                    textAnchorY+=textYoffset
                    text_surface = fontS2.render(forecaseHighs[i], True, colourWhite)
                    lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))
                    textAnchorY+=textYoffset
                    text_surface = fontS2.render(forecaseLows[i], True, colourWhite)
                    lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))
                    textAnchorY+=textYoffset
                    #text_surface = fontS2.render(forecastPrecips[i], True, colourWhite)
                    #lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))
                    #textAnchorY+=textYoffset
                    #text_surface = fontS2.render(forecastWinds[i], True, colourWhite)
                    #lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))
                    #textAnchorX+=textXoffset
                    try:
                      logo = pygame.image.load(forecastIcons[i]).convert()
                      w = logo.get_width()
                      h = logo.get_height()
                      logo = pygame.transform.scale(logo, (w,h))
                      lcd.screen.blit(logo, (textAnchorX, textAnchorY))
                      textAnchorX+=textXoffset
                    except pygame.error as message:
                      logging.warn(forecastIcons)
                      str = "err width: {}" .format(w)
                      str = str + " height: {}" .format(h)
                      logging.warn(str)
                      logging.warn(message)
                      textAnchorX+=textXoffset
                      #w = logo.get_width() - 150
                      #h = logo.get_height() - 150

                # today desc under the table
                textAnchorY = 220
                textAnchorX = 10
                text_surface = fontS2.render(todayDesc, True, colourWhite)
                lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))

                # update when information
                textAnchorY+=textYoffset
                updated = time.strftime("%H:%M") #time.strftime("%H:%M:%S")
                text_surface = fontS2.render(updated, True, colourWhite)
                lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))

                # update screen with forecast text
                pygame.display.update()

                # wait
                time.sleep(screenTimeOffset)
                betweenTime += screenTimeOffset

                # blank the screen after screenTimeOffset
                lcd.screen.fill(colourBlack)
                pygame.display.update()

                # Wait
                #time.sleep(updateRate)

                #i = 0
                #while i < screenTimeOffset:
                  #global betweenTime
                  #time.sleep(1)
                  #i = i + 1
                  #lcd.screen.fill(colourBlack)
                  #pygame.display.update()
                  #self.showClock()
                  #pygame.display.update()

                #betweenTime+=screenTimeOffset

if __name__ == '__main__':
  logging.basicConfig(level=logging.DEBUG, filename="logfile", filemode="a+", format="%(asctime)s %(message)s")
  try:
    PygameWeather().run()
  except Exception as e:
    #print "\n except: %s" % (e,)
    logging.warn(e)

<commit_msg>try to cleanup code test with two vars, to not using global<commit_after>import os, syslog
import pygame
import time
import pywapi
import string
import logging

# *** Not to be used for commercial use without permission!
# if you want to buy the icons for commercial use please send me a note - http://vclouds.deviantart.com/ ***
#
# Credit goes to Jamie Jackson http://blog.jacobean.net/?p=1016 for his nice Tutorial and great Parts of his Code, which i re-use.
# Credit goes to Merlin the Red for his great pictures: http://vclouds.deviantart.com/art/plain-weather-icons-157162192 , which i use.
#  ^-> I use his svg files and bring them to ~ 328x328 px
#
# I modify the original code for my 4 inch display and that it work in two 'screens'. First shows the currentday, Secound shows the forecast.

installPathImgBig = "/opt/PiTFTWeather/img/big/"
installPathImgSmall = "/opt/PiTFTWeather/img/small/"

# location for Cottbus, Brandenburg, Germany on weather.com
weatherDotComLocationCode = 'GMXX0171'

class PyLcd :
    screen = None;
    colourBlack = (0, 0, 0)

    def __init__(self):
        "Ininitializes a new pygame screen using the framebuffer"
        # Based on "Python GUI in Linux frame buffer"
        # http://www.karoltomala.com/blog/?p=679
        disp_no = os.getenv("DISPLAY")
        if disp_no:
            print "I'm running under X display = {0}".format(disp_no)

        os.putenv('SDL_FBDEV', '/dev/fb1')

        # Select frame buffer driver
        # Make sure that SDL_VIDEODRIVER is set
        driver = 'fbcon'
        if not os.getenv('SDL_VIDEODRIVER'):
            os.putenv('SDL_VIDEODRIVER', driver)
        try:
            pygame.display.init()
        except Exception as e:
           print "exception: %s" % (e,)
        except pygame.error:
            print 'Driver: {0} failed.'.format(driver)
            exit(0)

        size = (pygame.display.Info().current_w, pygame.display.Info().current_h)
        self.screen = pygame.display.set_mode(size, pygame.FULLSCREEN)
        # Clear the screen to start
        self.screen.fill((0, 0, 0))
        # Initialise font support
        pygame.font.init()
        # Render the screen
        pygame.display.update()

    def __del__(self):
        logging.info("Ende.")
        pygame.display.quit()
        #print "Destructor pygame display shuts down."

# Create an instance of the PyScope class
lcd = PyLcd()

pygame.mouse.set_visible(False)

# font colours
colourWhite = (255, 255, 255)
colourBlack = (0, 0, 0)

screenTimeOffset = 20

# set up the fonts
# choose the font
fontpath = pygame.font.match_font('dejavusansmono')
# set up 2 sizes
font = pygame.font.Font(fontpath, 36)
fontS2 = pygame.font.Font(fontpath, 28)
fontSm = pygame.font.Font(fontpath, 18)

# possbile
states = ["initial", "screen1", "screen2", "network"]
state = "initial"
weather_com_result = {}
h = 0
w = 0

# starting class
class PygameWeather(object):
    """This class uses pygame and show on console Weatherinformation graphicaly"""

    # class variable not shared by all instances
    # update interval
    updateRate = 3600 # seconds, for server call
    betweenTime = 20 # seconds, befor screen switching
    # more vars

    # def evaluateInformation()
    # def updateScreen1
    # def updateScreen2

    # call weather server, if no connection or error return old data
    def callServer( self, mydict ):
       old_values = mydict.copy()
       weather_com_result = pywapi.get_weather_from_weather_com(weatherDotComLocationCode)
       #check if the dictionary has error keep old data; else return new data
       if weather_com_result.has_key('error'):
           logging.info(weather_com_result['error'])
           return old_values
       else:
           return weather_com_result

    # to see the clock with secounds
    def showClock(self):
      updated = time.strftime("%H:%M:%S")
      text_surface = fontS2.render(updated, True, colourWhite)
      lcd.screen.blit(text_surface, (10, 260))

    #todo: seperate code in state-machine in run and put other stuff in seperate functions
    # implement run method
    def run(self):
       #global betweenTime
       #global updateRate
       global screenTimeOffset
       global weather_com_result
       global state
       global w
       global h
       global forecastIcons
       quit = False
       while not quit:
                for event in pygame.event.get():
                    if event.type == QUIT: #sth wrong here if pygame.QUIT
                      sys.exit()
                      quit = True
                    elif event.type == KEYDOWN and event.key == K_ESCAPE:
                      quit = True
                      logging.info("ESC Key was pressed")

                    if quit is True:
                      print "Escape Button was pressed."
                      return

                # retrieve data from weather.com and keep old values if no connection
                if state == "initial":
                  weather_com_result = self.callServer( weather_com_result )
                  state = "screen1"
                  # if weather_com_result is empty check TODO: FIXME
                  # run alternative data, infunction

                if self.betweenTime >= self.updateRate:
                  self.betweenTime = 0
                  state = "network"
                  logging.info("1 hour is over, Calling server...")
                  weather_com_result = self.callServer( weather_com_result )
                  logging.info("Calling server successful")
                  state = "screen1"
                  # if weather_com_result is empty check TODO: FIXME
                  # run alternative data
                #

                # extract current data for today
                today = weather_com_result['forecasts'][0]['day_of_week'][0:3] + " " \
                    + weather_com_result['forecasts'][0]['date'][4:] + " " \
                    + weather_com_result['forecasts'][0]['date'][:3]
                windSpeed = weather_com_result['current_conditions']['wind']['speed']
                currWind = "{}km/h ".format(windSpeed) + weather_com_result['current_conditions']['wind']['text']
                currTemp = weather_com_result['current_conditions']['temperature'] + u'\N{DEGREE SIGN}' + "C"
                currPress = weather_com_result['current_conditions']['barometer']['reading'][:-3] + "mb"
                uv = "UV {}".format(weather_com_result['current_conditions']['uv']['text'])
                humid = "Hum {}%".format(weather_com_result['current_conditions']['humidity'])
                currTempFeeling = "(" + weather_com_result['current_conditions']['feels_like'] + u'\N{DEGREE SIGN}' + "C)"
                todayDesc = "It is " + weather_com_result['current_conditions']['text'].lower() + " today."

                # summary and description of forecast data
                forecastDesc = ["Day", "Max", "Min", "   ", "   "] # forecastDesc = ["Day", "Max", "Min", "Hum", "Kmh"]

                # extract forecast data
                forecastDays = {}
                forecaseHighs = {}
                forecaseLows = {}
                forecastPrecips = {}
                forecastWinds = {}
                forecastIcons = {}

                for i in range(0, 5):
                    if not(weather_com_result['forecasts'][i]):
                        break

                    forecastDays[i] = weather_com_result['forecasts'][i]['day_of_week'][0:3]
                    forecaseHighs[i] = weather_com_result['forecasts'][i]['high'] + u'\N{DEGREE SIGN}' + "C"
                    forecaseLows[i] = weather_com_result['forecasts'][i]['low'] + u'\N{DEGREE SIGN}' + "C"
                    forecastPrecips[i] = weather_com_result['forecasts'][i]['day']['chance_precip'] + "%"
                    forecastWinds[i] = weather_com_result['forecasts'][i]['day']['wind']['speed'] + \
                        weather_com_result['forecasts'][i]['day']['wind']['text']
                    forecastIcons[i] = installPathImgSmall + (weather_com_result['forecasts'][i]['day']['icon']) + ".png"

                # 1. screen, dayInformation
                # blank the screen
                lcd.screen.fill(colourBlack)

                # Render the weather logo at 0,0
                icon = installPathImgBig + (weather_com_result['current_conditions']['icon']) + ".png"
                logo = pygame.image.load(icon).convert()
                w = logo.get_width() - 50
                h = logo.get_height() - 50
                logo = pygame.transform.scale(logo, (w,h))
                lcd.screen.blit(logo, (0, 0))

                # set the anchor for the current weather data text
                textAnchorX = 260 # 310 war ok bei 328 Bild
                textAnchorY = 5
                textYoffset = 40

                # add current weather data text artifacts to the screen
                text_surface = font.render(today, True, colourWhite)
                lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))
                textAnchorY+=textYoffset
                text_surface = font.render(currTemp, True, colourWhite)
                lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))
                # my code
                # feeled temp
                textAnchorY+=textYoffset
                text_surface = font.render(currTempFeeling, True, colourWhite)
                lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))
                # my code
                textAnchorY+=textYoffset
                text_surface = font.render(currWind, True, colourWhite)
                lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))
                textAnchorY+=textYoffset
                text_surface = font.render(currPress, True, colourWhite)
                lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))
                textAnchorY+=textYoffset
                text_surface = font.render(uv, True, colourWhite)
                lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))
                textAnchorY+=textYoffset
                text_surface = font.render(humid, True, colourWhite)
                lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))

                # wait between screen changes screenTimeOffset alias first betweenTime
                pygame.display.update()

                # wait
                time.sleep(screenTimeOffset)
                self.betweenTime += screenTimeOffset

                # blank the screen after screenTimeOffset
                lcd.screen.fill(colourBlack)
                pygame.display.update()

                # 2. screen
                # state = screen2
                # blank the screen
                lcd.screen.fill(colourBlack)
                pygame.display.update()

                # set X axis text anchor for the forecast text
                textAnchorX = 0
                textXoffset = 75 #100
                textAnchorY = 10
                #pygame.draw.line(lcd.screen.get_surface(), colourWhite, (50,10), (450,10),4)

                # add summy of the values in one row
                for i in range(0,5):
                  text_surface = fontS2.render(forecastDesc[i], True, colourWhite)
                  lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))
                  textAnchorY+=textYoffset

                textAnchorX+=70

                # add each days forecast text
                for i in range(0, 5):
                    textAnchorY = 10
                    text_surface = fontS2.render(forecastDays[i], True, colourWhite)
                    lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))
                    textAnchorY+=textYoffset
                    text_surface = fontS2.render(forecaseHighs[i], True, colourWhite)
                    lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))
                    textAnchorY+=textYoffset
                    text_surface = fontS2.render(forecaseLows[i], True, colourWhite)
                    lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))
                    textAnchorY+=textYoffset
                    #text_surface = fontS2.render(forecastPrecips[i], True, colourWhite)
                    #lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))
                    #textAnchorY+=textYoffset
                    #text_surface = fontS2.render(forecastWinds[i], True, colourWhite)
                    #lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))
                    #textAnchorX+=textXoffset
                    try:
                      logo = pygame.image.load(forecastIcons[i]).convert()
                      w = logo.get_width()
                      h = logo.get_height()
                      logo = pygame.transform.scale(logo, (w,h))
                      lcd.screen.blit(logo, (textAnchorX, textAnchorY))
                      textAnchorX+=textXoffset
                    except pygame.error as message:
                      logging.warn(forecastIcons)
                      str = "err width: {}" .format(w)
                      str = str + " height: {}" .format(h)
                      logging.warn(str)
                      logging.warn(message)
                      textAnchorX+=textXoffset
                      #w = logo.get_width() - 150
                      #h = logo.get_height() - 150

                # today desc under the table
                textAnchorY = 220
                textAnchorX = 10
                text_surface = fontS2.render(todayDesc, True, colourWhite)
                lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))

                # update when information
                textAnchorY+=textYoffset
                updated = time.strftime("%H:%M") #time.strftime("%H:%M:%S")
                text_surface = fontS2.render(updated, True, colourWhite)
                lcd.screen.blit(text_surface, (textAnchorX, textAnchorY))

                # update screen with forecast text
                pygame.display.update()

                # wait
                time.sleep(screenTimeOffset)
                self.betweenTime += screenTimeOffset

                # blank the screen after screenTimeOffset
                lcd.screen.fill(colourBlack)
                pygame.display.update()

                # Wait
                #time.sleep(updateRate)

                #i = 0
                #while i < screenTimeOffset:
                  #global betweenTime
                  #time.sleep(1)
                  #i = i + 1
                  #lcd.screen.fill(colourBlack)
                  #pygame.display.update()
                  #self.showClock()
                  #pygame.display.update()

                #betweenTime+=screenTimeOffset

if __name__ == '__main__':
  logging.basicConfig(level=logging.DEBUG, filename="logfile", filemode="a+", format="%(asctime)s %(message)s")
  try:
    PygameWeather().run()
  except Exception as e:
    #print "\n except: %s" % (e,)
    logging.warn(e)

<|endoftext|>


<commit_before>#!/usr/bin/env python

from bs4 import BeautifulSoup
import phabricator
import json
import datetime
import dateutil.parser
import os
import string
import pickle

MWCORETEAM_PHID = "PHID-PROJ-oft3zinwvih7bgdhpfgj"
WORKBOARD_HTML_CACHE = '/home/robla/2014/phabworkboard-data/html'
WORKBOARD_PICKLE_CACHE = '/home/robla/2014/phabworkboard-data/pickles'

# Scrape the HTML for a Phabricator workboard, and return a simple dict
# that represents the workboard.  The HTML is pre-retrieved via cron job
# that snarfs the HTML as much as hourly.  This function accesses the
# cache via timestamp, which is rounded to the nearest hour in the file
# name.
def parse_workboard_html(wbtime):
    # File name will look something like workboard-2014-12-22T00.html,
    # which corresponds to midnight on 2014-12-22
    filename = 'workboard-{:%Y-%m-%dT%H%Z}.html'.format(wbtime)
    htmlhandle = open(os.path.join(WORKBOARD_HTML_CACHE, filename))
    soup = BeautifulSoup(htmlhandle)
    columns = soup.find_all(class_="phui-workpanel-view")
    retval = {}
    for col in columns:
        state = col.find(class_="phui-action-header-title").strings.next()
        objnames = col.find_all(class_="phui-object-item-objname")
        for objname in objnames:
            retval[objname.string] = state
    return retval

# Take data structures representing the task states in two workboards,
# and return a dict that contains the tasks for which the state changed
# between the two workboards.  The contents of each item in the dict
# should be a tuple with the old state and the new state.
def get_workboard_diff(old_workboard, new_workboard):
    allkeys = list(set(old_workboard.keys()).union(new_workboard.keys()))
    diff = {}
    for key in allkeys:
        oldvalue = old_workboard.get(key)
        newvalue = new_workboard.get(key)
        if oldvalue != newvalue:
            diff[key] = (oldvalue, newvalue)
    return diff

def get_activity_for_tasks(phab, phabcache, tasks):
    tasknums = [int(string.lstrip(x, "T")) for x in tasks]
    activityquery = lambda: phab.maniphest.gettasktransactions(ids=tasknums)
    activity = phabcache.get("gettasktransactions", activityquery)
    return activity

class PhabCache:

    def __init__(self, cachedir):
        self.cachedir = cachedir
        return

    def get(self, key, apicall):
        picklefile = os.path.join(self.cachedir, key + ".pickle")
        try:
            result = pickle.load(open(picklefile))
        except IOError:
            result = apicall()
            pickle.dump(result, open(picklefile, "wb"))
        return result

# Return an item if it's relevant to our current search, or {} if it isn't.
# Also return any PHIDs that need to be resolved.
def get_filtered_transactions_for_task(taskfeed):
    transactions = []
    phids = set()
    for tact in taskfeed:
        item = {}
        item['transactionType'] = tact["transactionType"]
        item['timestamp'] = int(tact['dateCreated'])
        item['authorPHID'] = tact['authorPHID']
        phids.add(item['authorPHID'])
        if (tact["transactionType"] == "status"):
            item['oldValue'] = tact['oldValue']
            item['newValue'] = tact['newValue']
        elif (tact["transactionType"] == "reassign"):
            item['oldValue'] = tact['oldValue']
            if tact['oldValue']:
                phids.add(item['oldValue'])
            item['newValue'] = tact['newValue']
            if tact['newValue']:
                phids.add(item['newValue'])
        elif (tact["transactionType"] == "projectcolumn" and
                tact["oldValue"]["projectPHID"] == MWCORETEAM_PHID):
            oldvalphids = tact['oldValue']['columnPHIDs']
            if isinstance(oldvalphids, dict):
                item['oldValue'] = oldvalphids.values()[0]
                phids.add(item['oldValue'])
            else:
                item['oldValue'] = None
            item['newValue'] = tact['newValue']['columnPHIDs'][0]
            phids.add(item['newValue'])
        else:
            item = {}
        if item:
            transactions.append(item)
    return transactions, phids

def process_transactions(transactions):
    taskstate = {}
    taskstate['actorset'] = set()
    for tact in transactions:
        #time = datetime.datetime.fromtimestamp(
        #    tact['timestamp']).strftime("%Y-%m-%d %H:%M UTC")
        #author = phidquery[tact['authorPHID']]['name']
        if tact['authorPHID']:
            taskstate['actorset'].add(tact['authorPHID'])
        if tact['transactionType'] == 'projectcolumn':
            taskstate['column'] = tact['newValue']
        elif tact['transactionType'] == 'status':
            taskstate['status'] = tact['newValue']
        elif tact['transactionType'] == 'reassign':
            taskstate['assignee'] = tact['newValue']
            if tact['newValue']:
                taskstate['actorset'].add(tact['newValue'])
    return taskstate

def render_transaction(tact, phidquery):
    time = datetime.datetime.fromtimestamp(
        tact['timestamp']).strftime("%Y-%m-%d %H:%M UTC")
    author = phidquery[tact['authorPHID']]['name']
    if tact['transactionType'] == 'projectcolumn':
        if tact['oldValue']:
            oldcolumn = phidquery[tact['oldValue']]['name']
        else:
            oldcolumn = '(none)'
        newcolumn = phidquery[tact['newValue']]['name']
        retval = "  {0} {1} Column: '{2}' '{3}'".format(
            time, author, oldcolumn, newcolumn)
    elif tact['transactionType'] == 'status':
        oldstatus = str(tact['oldValue'])
        newstatus = tact['newValue']
        retval = "  {0} {1} Status: '{2}' '{3}'".format(
            time, author, oldstatus, newstatus)
    elif tact['transactionType'] == 'reassign':
        if tact['oldValue']:
            oldassignee = phidquery[tact['oldValue']]['name']
        else:
            oldassignee = '(unassigned)'
        if tact['newValue']:
            newassignee = phidquery[tact['newValue']]['name']
        else:
            newassignee = '(unassigned)'
        retval = "  {0} {1} Assignee: '{2}' '{3}'".format(
            time, author, oldassignee, newassignee)
    return retval

def main():
    phab = phabricator.Phabricator()
    phabcache = PhabCache(WORKBOARD_PICKLE_CACHE)
    start = dateutil.parser.parse("2014-12-22T0:00PST")
    end = dateutil.parser.parse("2014-12-23T0:00PST")
    old_workboard = parse_workboard_html(start)
    new_workboard = parse_workboard_html(end)
    diff = get_workboard_diff(old_workboard, new_workboard)
    allkeys = list(set(old_workboard.keys()).union(new_workboard.keys()))
    activity = get_activity_for_tasks(phab, phabcache, allkeys)

    transactions = {}
    phids = set()
    for tasknum, taskfeed in activity.iteritems():
        tacts, newphids = get_filtered_transactions_for_task(taskfeed)
        transactions[tasknum] = tacts
        phids.update(newphids)

    actortasks = {}
    taskstate = {}
    for task in transactions.keys():
        taskstate[task] = process_transactions(transactions[task])
        for actor in taskstate[task]['actorset']:
            assert actor
            if actortasks.get(actor):
                actortasks[actor].append(task)
            else:
                actortasks[actor] = [task]

    phidapifunc = lambda: phab.phid.query(phids=list(phids))
    phidquery = phabcache.get("phidquery", phidapifunc)
    for actor, tasklist in actortasks.iteritems():
        print "Actor: " + phidquery[actor]['name']
        for task in tasklist:
            print "  Task T{0}".format(task)
            for tact in transactions[task]:
                print "  " + render_transaction(tact, phidquery)

if __name__ == "__main__":
    main()
<commit_msg>Filter transactions by start and end time<commit_after>#!/usr/bin/env python

from bs4 import BeautifulSoup
import phabricator
import json
import datetime
import dateutil.parser
import os
import string
import pickle

MWCORETEAM_PHID = "PHID-PROJ-oft3zinwvih7bgdhpfgj"
WORKBOARD_HTML_CACHE = '/home/robla/2014/phabworkboard-data/html'
WORKBOARD_PICKLE_CACHE = '/home/robla/2014/phabworkboard-data/pickles'

# Scrape the HTML for a Phabricator workboard, and return a simple dict
# that represents the workboard.  The HTML is pre-retrieved via cron job
# that snarfs the HTML as much as hourly.  This function accesses the
# cache via timestamp, which is rounded to the nearest hour in the file
# name.
def parse_workboard_html(wbtime):
    # File name will look something like workboard-2014-12-22T00.html,
    # which corresponds to midnight on 2014-12-22
    filename = 'workboard-{:%Y-%m-%dT%H%Z}.html'.format(wbtime)
    htmlhandle = open(os.path.join(WORKBOARD_HTML_CACHE, filename))
    soup = BeautifulSoup(htmlhandle)
    columns = soup.find_all(class_="phui-workpanel-view")
    retval = {}
    for col in columns:
        state = col.find(class_="phui-action-header-title").strings.next()
        objnames = col.find_all(class_="phui-object-item-objname")
        for objname in objnames:
            retval[objname.string] = state
    return retval

# Take data structures representing the task states in two workboards,
# and return a dict that contains the tasks for which the state changed
# between the two workboards.  The contents of each item in the dict
# should be a tuple with the old state and the new state.
def get_workboard_diff(old_workboard, new_workboard):
    allkeys = list(set(old_workboard.keys()).union(new_workboard.keys()))
    diff = {}
    for key in allkeys:
        oldvalue = old_workboard.get(key)
        newvalue = new_workboard.get(key)
        if oldvalue != newvalue:
            diff[key] = (oldvalue, newvalue)
    return diff

def get_activity_for_tasks(phab, phabcache, tasks):
    tasknums = [int(string.lstrip(x, "T")) for x in tasks]
    activityquery = lambda: phab.maniphest.gettasktransactions(ids=tasknums)
    activity = phabcache.get("gettasktransactions", activityquery)
    return activity

class PhabCache:

    def __init__(self, cachedir):
        self.cachedir = cachedir
        return

    def get(self, key, apicall):
        picklefile = os.path.join(self.cachedir, key + ".pickle")
        try:
            result = pickle.load(open(picklefile))
        except IOError:
            result = apicall()
            pickle.dump(result, open(picklefile, "wb"))
        return result

# Return an item if it's relevant to our current search, or {} if it isn't.
# Also return any PHIDs that need to be resolved.
def get_filtered_transactions_for_task(taskfeed):
    transactions = []
    phids = set()
    for tact in taskfeed:
        item = {}
        item['transactionType'] = tact["transactionType"]
        item['timestamp'] = int(tact['dateCreated'])
        item['authorPHID'] = tact['authorPHID']
        phids.add(item['authorPHID'])
        if (tact["transactionType"] == "status"):
            item['oldValue'] = tact['oldValue']
            item['newValue'] = tact['newValue']
        elif (tact["transactionType"] == "reassign"):
            item['oldValue'] = tact['oldValue']
            if tact['oldValue']:
                phids.add(item['oldValue'])
            item['newValue'] = tact['newValue']
            if tact['newValue']:
                phids.add(item['newValue'])
        elif (tact["transactionType"] == "projectcolumn" and
                tact["oldValue"]["projectPHID"] == MWCORETEAM_PHID):
            oldvalphids = tact['oldValue']['columnPHIDs']
            if isinstance(oldvalphids, dict):
                item['oldValue'] = oldvalphids.values()[0]
                phids.add(item['oldValue'])
            else:
                item['oldValue'] = None
            item['newValue'] = tact['newValue']['columnPHIDs'][0]
            phids.add(item['newValue'])
        else:
            item = {}
        if item:
            transactions.append(item)
    return transactions, phids

def process_transactions(transactions, start, end):
    taskstate = {}
    taskstate['actorset'] = set()
    for tact in transactions:
        ttime = datetime.datetime.fromtimestamp(tact['timestamp'],
            dateutil.tz.tzutc())
        if ttime > end:
            break
        if tact['authorPHID']:
            if ttime > start and ttime < end:
                taskstate['actorset'].add(tact['authorPHID'])
        if tact['transactionType'] == 'projectcolumn':
            taskstate['column'] = tact['newValue']
        elif tact['transactionType'] == 'status':
            taskstate['status'] = tact['newValue']
        elif tact['transactionType'] == 'reassign':
            taskstate['assignee'] = tact['newValue']
            if tact['newValue']:
                taskstate['actorset'].add(tact['newValue'])
    return taskstate

def render_transaction(tact, phidquery):
    time = datetime.datetime.fromtimestamp(
        tact['timestamp']).strftime("%Y-%m-%d %H:%M UTC")
    author = phidquery[tact['authorPHID']]['name']
    if tact['transactionType'] == 'projectcolumn':
        if tact['oldValue']:
            oldcolumn = phidquery[tact['oldValue']]['name']
        else:
            oldcolumn = '(none)'
        newcolumn = phidquery[tact['newValue']]['name']
        retval = "  {0} {1} Column: '{2}' '{3}'".format(
            time, author, oldcolumn, newcolumn)
    elif tact['transactionType'] == 'status':
        oldstatus = str(tact['oldValue'])
        newstatus = tact['newValue']
        retval = "  {0} {1} Status: '{2}' '{3}'".format(
            time, author, oldstatus, newstatus)
    elif tact['transactionType'] == 'reassign':
        if tact['oldValue']:
            oldassignee = phidquery[tact['oldValue']]['name']
        else:
            oldassignee = '(unassigned)'
        if tact['newValue']:
            newassignee = phidquery[tact['newValue']]['name']
        else:
            newassignee = '(unassigned)'
        retval = "  {0} {1} Assignee: '{2}' '{3}'".format(
            time, author, oldassignee, newassignee)
    return retval

def main():
    phab = phabricator.Phabricator()
    phabcache = PhabCache(WORKBOARD_PICKLE_CACHE)
    start = dateutil.parser.parse("2014-12-22T0:00PST")
    end = dateutil.parser.parse("2014-12-23T0:00PST")
    old_workboard = parse_workboard_html(start)
    new_workboard = parse_workboard_html(end)
    diff = get_workboard_diff(old_workboard, new_workboard)
    allkeys = list(set(old_workboard.keys()).union(new_workboard.keys()))
    activity = get_activity_for_tasks(phab, phabcache, allkeys)

    transactions = {}
    phids = set()
    for tasknum, taskfeed in activity.iteritems():
        tacts, newphids = get_filtered_transactions_for_task(taskfeed)
        transactions[tasknum] = tacts
        phids.update(newphids)

    actortasks = {}
    taskstate = {}
    for task in transactions.keys():
        taskstate[task] = process_transactions(transactions[task], start, end)
        for actor in taskstate[task]['actorset']:
            assert actor
            if actortasks.get(actor):
                actortasks[actor].append(task)
            else:
                actortasks[actor] = [task]

    phidapifunc = lambda: phab.phid.query(phids=list(phids))
    phidquery = phabcache.get("phidquery", phidapifunc)
    for actor, tasklist in actortasks.iteritems():
        print "Actor: " + phidquery[actor]['name']
        for task in tasklist:
            print "  Task T{0}".format(task)
            for tact in transactions[task]:
                ttime = datetime.datetime.fromtimestamp(tact['timestamp'],
                    dateutil.tz.tzutc())
                if ttime > start and ttime < end:
                    print "  " + render_transaction(tact, phidquery)

if __name__ == "__main__":
    main()
<|endoftext|>