<commit_before>#!/usr/bin/env python

import configparser
import pickle
import os
import urllib
import logging
try:
    from slackclient import SlackClient
except:
    import slack

import sys
import click
import requests
from bs4 import BeautifulSoup
from bs4 import Tag

from moduleContent import *
from moduleQueue import *

class moduleSlack(Content,Queue):

    def __init__(self):
        super().__init__()
        self.service = None
        self.sc = None
        self.keys = []

    def setClient(self, slackCredentials=None):
        self.setSlackClient(slackCredentials)

    def setSlackClient(self, slackCredentials):
        self.service = 'slack'
        config = configparser.ConfigParser()
        if not slackCredentials:
            slackCredentials = CONFIGDIR + '/.rssSlack'
        config.read(slackCredentials)

        self.slack_token = config["Slack"].get('oauth-token')
        self.user_slack_token = config["Slack"].get('user-oauth-token')

        try:
            self.sc = SlackClient(self.slack_token)
        except:
            self.sc = slack.WebClient(token=self.slack_token)

        config = configparser.ConfigParser()
        config.read(CONFIGDIR + '/.rssBlogs')
        section = "Blog7"

        url = config.get(section, "url")
        self.setUrl(url)
        self.setSocialNetworks(config, section)
        if ('buffer' in config.options(section)):
            self.setBufferapp(config.get(section, "buffer"))

        if ('cache' in config.options(section)):
            self.setProgram(config.get(section, "cache"))

    def getSlackClient(self):
        return self.sc

    def setPosts(self, channel='links'):
        logging.info("  Setting posts")
        self.posts = []
        theChannel = self.getChanId(channel)
        history = self.sc.api_call( "channels.history", count=1000, channel=theChannel)
        if 'messages' in history:
            self.posts = history['messages']
        else:
            self.posts = []

    def getTitle(self, i):
        post = self.getPosts()[i]
        return(self.getPostTitle(post))

    def getLink(self, i):
        post = self.getPosts()[i]
        return(self.getPostLink(post))

    def getPostTitle(self, post):
        if 'attachments' in post:
            return(post['attachments'][0]['title'])
        else:
            text = post['text']
            if text.startswith('<'):
                title = post['text'][1:-1]
            else:
                pos = text.find('<')
                title=text[:pos]
            return(title)

    def getPostLink(self, post):
        if 'attachments' in post:
            return(post['attachments'][0]['original_url'])
        else:
            text = post['text']
            if text.startswith('<'):
                url = post['text'][1:-1]
            else:
                # Some people include URLs in the title of the page
                pos = text.rfind('<')
                url=text[pos+1:-1]
            return(url)

    #def isForMe(self, args):
    #    return ((self.service[0].capitalize() in args.split()[0])
    #           or (args[0] == '*'))

    def publish(self, j):
        logging.info("Publishing %d"% j)
        post = self.obtainPostData(j)
        logging.info("Publishing %s"% post[0])
        update = ''
        title = self.getTitle(j)
        url = self.getLink(j)
        logging.info("Title: %s" % str(title))
        logging.info("Url: %s" % str(url))
        if self.getBuffer():
            for profile in self.getSocialNetworks():
                if profile[0] in self.getBufferapp():
                    lenMax = self.len(profile)
                    logging.info("   getBuffer %s" % profile)
                    socialNetwork = (profile,self.getSocialNetworks()[profile])
                    listPosts = []
                    listPosts.append((title, url))
                    update = update + self.buffer[socialNetwork].addPosts(listPosts)
                    update = update + '\n'

        if self.getProgram():
            for profile in self.getSocialNetworks():
                if profile[0] in self.getProgram():
                    lenMax = self.len(profile)
                    socialNetwork = (profile,self.getSocialNetworks()[profile])

                    listP = self.cache[socialNetwork].getPosts()
                    listPsts = self.obtainPostData(j)
                    listP = listP + [listPsts]
                    self.cache[socialNetwork].posts = listP
                    update = update + self.cache[socialNetwork].updatePostsCache()
                    update = update + '\n'
        t = moduleTumblr.moduleTumblr()
        t.setClient('fernand0')
        # We need to publish it in the Tumblr blog since we won't publish it by
        # usuarl means (it is deleted from queue).
        update = update + t.publishPost(title, url, '')['display_text']
        # Res: {'id': 187879917068, 'state': 'queued', 'display_text': 'Added to queue.'}

        theChannel = self.getChanId("links")
        res = self.deletePost(self.getId(j), theChannel)
        logging.info("Res: %s" % str(res))
        update = update + str(res['ok'])

        logging.info("Publishing title: %s" % title)
        logging.info("Update before return %s"% update)
        return(update)

    def getId(self, i):
        post = self.getPosts()[i]
        return(post['ts'])

    def getKeys(self):
        return(self.keys)

    def setKeys(self, keys):
        self.keys = keys

    def delete(self, j, theChannel=None):
        logging.info("Deleting id %s" % j)
        if not theChannel:
            theChannel = self.getChanId("links")
        idPost = self.getId(j)
        self.sc.token = self.user_slack_token
        result = self.sc.api_call("chat.delete", channel=theChannel, ts=idPost)
        self.sc.token = self.slack_token
        logging.info(result)
        return(result['ok'])

    def deletePost(self, idPost, theChannel):
        #theChannel or the name of the channel?
        logging.info("Deleting id %s from %s" % (idPost, theChannel))

        result = self.sc.api_call("chat.delete", channel=theChannel, ts=idPost)

        logging.info(result)
        return(result)

    def getChanId(self, name):
        logging.info("getChanId %s"% self.service)

        self.sc.token = self.user_slack_token
        chanList = self.sc.api_call("channels.list")['channels']
        self.sc.token = self.slack_token
        for channel in chanList:
            if channel['name_normalized'] == name:
                return(channel['id'])
        return(None)

    def extractDataMessage(self, i):
        logging.info("Service %s"% self.service)
        (theTitle, theLink, firstLink, theImage, theSummary, content, theSummaryLinks, theContent, theLinks, comment) = (None, None, None, None, None, None, None, None, None, None)

        if i < len(self.getPosts()):
            theTitle = self.getTitle[0]
            theLink = self.getLink[1]

            theLinks = None
            content = None
            theContent = None
            firstLink = theLink
            theImage = None
            theSummary = None

            theSummaryLinks = None
            comment = None

        return (theTitle, theLink, firstLink, theImage, theSummary, content, theSummaryLinks, theContent, theLinks, comment)

    def obtainPostData(self, i, debug=False):
        if not self.posts:
            self.setPosts()

        posts = self.getPosts()
        if not posts:
            return (None, None, None, None, None, None, None, None, None, None)

        post = posts[i]
        if 'attachments' in post:
            post = post['attachments'][0]

        theContent = ''
        url = ''
        firstLink = ''
        logging.debug("i %d", i)
        logging.debug("post %s", post)

        theTitle = self.getTitle(i)
        theLink = self.getLink(i)
        print(theTitle)
        print(theLink)
        if theLink.find('tumblr')>0:
            theTitle = post['text']
        firstLink = theLink
        if 'text' in post:
            content = post['text']
        else:
            content = theLink
        theSummary = content
        theSummaryLinks = content
        if 'image_url' in post:
            theImage = post['image_url']
        elif 'thumb_url' in post:
            theImage = post['thumb_url']
        else:
            logging.info("Fail image")
            logging.debug("Fail image %s", post)
            theImage = ''

        if 'original_url' in post:
            theLink = post['original_url']
        elif url:
            theLink = url
        else:
            theLink = self.getPostLink(post)

        if ('comment' in post):
            comment = post['comment']
        else:
            comment = ""

        #print("content", content)
        theSummaryLinks = ""

        if not content.startswith('http'):
            soup = BeautifulSoup(content, 'lxml')
            link = soup.a
            if link:
                firstLink = link.get('href')
                if firstLink:
                    if firstLink[0] != 'h':
                        firstLink = theLink

        if not firstLink:
            firstLink = theLink

        if 'image_url' in post:
            theImage = post['image_url']
        else:
            theImage = None
        theLinks = theSummaryLinks
        theSummaryLinks = theContent + theLinks

        theContent = ""
        theSummaryLinks = ""
        #if self.getLinksToAvoid():
        #    (theContent, theSummaryLinks) = self.extractLinks(soup, self.getLinkstoavoid())
        #else:
        #    (theContent, theSummaryLinks) = self.extractLinks(soup, "")

        if 'image_url' in post:
            theImage = post['image_url']
        else:
            theImage = None
        theLinks = theSummaryLinks
        theSummaryLinks = theContent + theLinks

        logging.debug("=========")
        logging.debug("Results: ")
        logging.debug("=========")
        logging.debug("Title:     ", theTitle)
        logging.debug("Link:      ", theLink)
        logging.debug("First Link:", firstLink)
        logging.debug("Summary:   ", content[:200])
        logging.debug("Sum links: ", theSummaryLinks)
        logging.debug("the Links"  , theLinks)
        logging.debug("Comment:   ", comment)
        logging.debug("Image;     ", theImage)
        logging.debug("Post       ", theTitle + " " + theLink)
        logging.debug("==============================================")
        logging.debug("")

        return (theTitle, theLink, firstLink, theImage, theSummary, content, theSummaryLinks, theContent, theLinks, comment)

    def publishPost(self, chan, msg):
        theChan = self.getChanId(chan)
        logging.info("Publishing %s" % msg)
        try:
            self.sc.token = self.user_slack_token
            result = self.sc.api_call("chat.postMessage",
                channel = theChan, text = msg)
            self.sc.token = self.slack_token
        except:
            result = self.sc.chat_postMessage(channel=theChan,
                    text=msg)
        logging.info(result)
        return(result)

    def getBots(self):
        if not self.posts:
            self.setPosts('tavern-of-the-bots')
        msgs = {}
        for msg in self.getPosts():
            if msg['text'].find('Hello')>=0:
                posN = msg['text'].find('Name:')+6
                posFN = msg['text'].find('"',posN)
                posI = msg['text'].find('IP:')+4
                posFI = msg['text'].find(' ',posI+1)-1
                posC = msg['text'].find('[')
                name = msg['text'][posN:posFN]
                ip = msg['text'][posI:posFI]
                command = msg['text'][posC+1:posC+2]
                #print("IP %s %s" % (ip,name))
                if name not in msgs:
                    msgs[name] = (ip, command)
        theBots = ""
        for name in msgs:
            theBots = theBots + "[%s] %16s %s\n"%(msgs[name][1], msgs[name][0], name)
        return(theBots)

def main():
    CHANNEL = 'tavern-of-the-bots'

    import moduleTumblr
    import moduleSlack

    config = configparser.ConfigParser()
    config.read(CONFIGDIR + '/.rssBlogs')

    site = moduleSlack.moduleSlack()
    section = "Blog7"

    url = config.get(section, "url")
    site.setUrl(url)

    SLACKCREDENTIALS = os.path.expanduser(CONFIGDIR + '/.rssSlack')
    site.setSlackClient(SLACKCREDENTIALS)

    theChannel = site.getChanId(CHANNEL)
    print("the Channel %s" % theChannel)

    site.setPosts('links')
    site.setPosts('tavern-of-the-bots')
    print(site.getPosts())
    rep = site.publishPost('tavern-of-the-bots', 'hello')
    wait = input('Delete %s?' % rep)
    site.deletePost(rep['ts'], theChannel)
    sys.exit()

    site.setSocialNetworks(config, section)

    if ('buffer' in config.options(section)):
        site.setBufferapp(config.get(section, "buffer"))

    if ('cache' in config.options(section)):
        site.setProgram(config.get(section, "cache"))

    theChannel = site.getChanId("links")

    i = 0
    listLinks = ""

    lastUrl = ''
    for i, post in enumerate(site.getPosts()):
        url = site.getLink(i)
        if urllib.parse.urlparse(url).netloc == lastUrl:
            listLinks = listLinks + "%d>> %s\n" % (i, url)
        else:
            listLinks = listLinks + "%d) %s\n" % (i, url)
        lastUrl = urllib.parse.urlparse(url).netloc
        print(site.getTitle(i))
        print(site.getLink(i))
        print(site.getPostTitle(post))
        print(site.getPostLink(post))
        i = i + 1

    numEntries = i
    click.echo_via_pager(listLinks)
    i = input("Which one? [x] to exit ")
    if i == 'x':
        sys.exit()

    elem = int(i)
    print(site.getPosts()[elem])

    action = input("Delete [d], publish [p], exit [x] ")

    if action == 'x':
        sys.exit()
    elif action == 'p':
        if site.getBufferapp():
            for profile in site.getSocialNetworks():
                if profile[0] in site.getBufferapp():
                    lenMax = site.len(profile)
                    print("   getBuffer %s" % profile)
                    socialNetwork = (profile,site.getSocialNetworks()[profile])
                    title = site.getTitle(elem)
                    url = site.getLink(elem)
                    listPosts = []
                    listPosts.append((title, url))
                    site.buffer[socialNetwork].addPosts(listPosts)

        if site.getProgram():
            for profile in site.getSocialNetworks():
                if profile[0] in site.getProgram():
                    lenMax = site.len(profile)
                    print("   getProgram %s" % profile)

                    socialNetwork = (profile,site.getSocialNetworks()[profile])

                    listP = site.cache[socialNetwork].getPosts()
                    #site.cache[socialNetwork].posts = site.cache[socialNetwork].posts[:8]
                    #listP = site.cache[socialNetwork].getPosts()
                    #for i,l in enumerate(listP):
                    #    print(i, l)
                    #site.cache[socialNetwork].updatePostsCache()
                    listPsts = site.obtainPostData(elem)
                    listP = listP + [listPsts]
                    #for i,l in enumerate(listP):
                    #    print(i, l)
                    #sys.exit()
                    site.cache[socialNetwork].posts = listP
                    site.cache[socialNetwork].updatePostsCache()
        t = moduleTumblr.moduleTumblr()
        t.setClient('fernand0')
        # We need to publish it in the Tumblr blog since we won't publish it by
        # usuarl means (it is deleted from queue).
        t.publishPost(title, url, '')

    site.deletePost(site.getId(j), theChannel)
    #print(outputData['Slack']['pending'][elem][8])

if __name__ == '__main__':
    main()
<commit_msg>Improving output for moduleSlack<commit_after>#!/usr/bin/env python

import configparser
import pickle
import os
import urllib
import logging
try:
    from slackclient import SlackClient
except:
    import slack

import sys
import time
import click
import requests
from bs4 import BeautifulSoup
from bs4 import Tag

from moduleContent import *
from moduleQueue import *

class moduleSlack(Content,Queue):

    def __init__(self):
        super().__init__()
        self.service = None
        self.sc = None
        self.keys = []

    def setClient(self, slackCredentials=None):
        self.setSlackClient(slackCredentials)

    def setSlackClient(self, slackCredentials):
        self.service = 'slack'
        config = configparser.ConfigParser()
        if not slackCredentials:
            slackCredentials = CONFIGDIR + '/.rssSlack'
        config.read(slackCredentials)

        self.slack_token = config["Slack"].get('oauth-token')
        self.user_slack_token = config["Slack"].get('user-oauth-token')

        try:
            self.sc = SlackClient(self.slack_token)
        except:
            self.sc = slack.WebClient(token=self.slack_token)

        config = configparser.ConfigParser()
        config.read(CONFIGDIR + '/.rssBlogs')
        section = "Blog7"

        url = config.get(section, "url")
        self.setUrl(url)
        self.setSocialNetworks(config, section)
        if ('buffer' in config.options(section)):
            self.setBufferapp(config.get(section, "buffer"))

        if ('cache' in config.options(section)):
            self.setProgram(config.get(section, "cache"))

    def getSlackClient(self):
        return self.sc

    def setPosts(self, channel='links'):
        logging.info("  Setting posts")
        self.posts = []
        theChannel = self.getChanId(channel)
        history = self.sc.api_call( "channels.history", count=1000, channel=theChannel)
        if 'messages' in history:
            self.posts = history['messages']
        else:
            self.posts = []

    def getTitle(self, i):
        post = self.getPosts()[i]
        return(self.getPostTitle(post))

    def getLink(self, i):
        post = self.getPosts()[i]
        return(self.getPostLink(post))

    def getPostTitle(self, post):
        if 'attachments' in post:
            return(post['attachments'][0]['title'])
        else:
            text = post['text']
            if text.startswith('<'):
                title = post['text'][1:-1]
            else:
                pos = text.find('<')
                title=text[:pos]
            return(title)

    def getPostLink(self, post):
        if 'attachments' in post:
            return(post['attachments'][0]['original_url'])
        else:
            text = post['text']
            if text.startswith('<'):
                url = post['text'][1:-1]
            else:
                # Some people include URLs in the title of the page
                pos = text.rfind('<')
                url=text[pos+1:-1]
            return(url)

    #def isForMe(self, args):
    #    return ((self.service[0].capitalize() in args.split()[0])
    #           or (args[0] == '*'))

    def publish(self, j):
        logging.info("Publishing %d"% j)
        post = self.obtainPostData(j)
        logging.info("Publishing %s"% post[0])
        update = ''
        title = self.getTitle(j)
        url = self.getLink(j)
        logging.info("Title: %s" % str(title))
        logging.info("Url: %s" % str(url))
        if self.getBuffer():
            for profile in self.getSocialNetworks():
                if profile[0] in self.getBufferapp():
                    lenMax = self.len(profile)
                    logging.info("   getBuffer %s" % profile)
                    socialNetwork = (profile,self.getSocialNetworks()[profile])
                    listPosts = []
                    listPosts.append((title, url))
                    update = update + self.buffer[socialNetwork].addPosts(listPosts)
                    update = update + '\n'

        if self.getProgram():
            for profile in self.getSocialNetworks():
                if profile[0] in self.getProgram():
                    lenMax = self.len(profile)
                    socialNetwork = (profile,self.getSocialNetworks()[profile])

                    listP = self.cache[socialNetwork].getPosts()
                    listPsts = self.obtainPostData(j)
                    listP = listP + [listPsts]
                    self.cache[socialNetwork].posts = listP
                    update = update + self.cache[socialNetwork].updatePostsCache()
                    update = update + '\n'
        t = moduleTumblr.moduleTumblr()
        t.setClient('fernand0')
        # We need to publish it in the Tumblr blog since we won't publish it by
        # usuarl means (it is deleted from queue).
        update = update + t.publishPost(title, url, '')['display_text']
        # Res: {'id': 187879917068, 'state': 'queued', 'display_text': 'Added to queue.'}

        theChannel = self.getChanId("links")
        res = self.deletePost(self.getId(j), theChannel)
        logging.info("Res: %s" % str(res))
        update = update + str(res['ok'])

        logging.info("Publishing title: %s" % title)
        logging.info("Update before return %s"% update)
        return(update)

    def getId(self, i):
        post = self.getPosts()[i]
        return(post['ts'])

    def getKeys(self):
        return(self.keys)

    def setKeys(self, keys):
        self.keys = keys

    def delete(self, j, theChannel=None):
        logging.info("Deleting id %s" % j)
        if not theChannel:
            theChannel = self.getChanId("links")
        idPost = self.getId(j)
        self.sc.token = self.user_slack_token
        result = self.sc.api_call("chat.delete", channel=theChannel, ts=idPost)
        self.sc.token = self.slack_token
        logging.info(result)
        return(result['ok'])

    def deletePost(self, idPost, theChannel):
        #theChannel or the name of the channel?
        logging.info("Deleting id %s from %s" % (idPost, theChannel))

        result = self.sc.api_call("chat.delete", channel=theChannel, ts=idPost)

        logging.info(result)
        return(result)

    def getChanId(self, name):
        logging.info("getChanId %s"% self.service)

        self.sc.token = self.user_slack_token
        chanList = self.sc.api_call("channels.list")['channels']
        self.sc.token = self.slack_token
        for channel in chanList:
            if channel['name_normalized'] == name:
                return(channel['id'])
        return(None)

    def extractDataMessage(self, i):
        logging.info("Service %s"% self.service)
        (theTitle, theLink, firstLink, theImage, theSummary, content, theSummaryLinks, theContent, theLinks, comment) = (None, None, None, None, None, None, None, None, None, None)

        if i < len(self.getPosts()):
            theTitle = self.getTitle[0]
            theLink = self.getLink[1]

            theLinks = None
            content = None
            theContent = None
            firstLink = theLink
            theImage = None
            theSummary = None

            theSummaryLinks = None
            comment = None

        return (theTitle, theLink, firstLink, theImage, theSummary, content, theSummaryLinks, theContent, theLinks, comment)

    def obtainPostData(self, i, debug=False):
        if not self.posts:
            self.setPosts()

        posts = self.getPosts()
        if not posts:
            return (None, None, None, None, None, None, None, None, None, None)

        post = posts[i]
        if 'attachments' in post:
            post = post['attachments'][0]

        theContent = ''
        url = ''
        firstLink = ''
        logging.debug("i %d", i)
        logging.debug("post %s", post)

        theTitle = self.getTitle(i)
        theLink = self.getLink(i)
        print(theTitle)
        print(theLink)
        if theLink.find('tumblr')>0:
            theTitle = post['text']
        firstLink = theLink
        if 'text' in post:
            content = post['text']
        else:
            content = theLink
        theSummary = content
        theSummaryLinks = content
        if 'image_url' in post:
            theImage = post['image_url']
        elif 'thumb_url' in post:
            theImage = post['thumb_url']
        else:
            logging.info("Fail image")
            logging.debug("Fail image %s", post)
            theImage = ''

        if 'original_url' in post:
            theLink = post['original_url']
        elif url:
            theLink = url
        else:
            theLink = self.getPostLink(post)

        if ('comment' in post):
            comment = post['comment']
        else:
            comment = ""

        #print("content", content)
        theSummaryLinks = ""

        if not content.startswith('http'):
            soup = BeautifulSoup(content, 'lxml')
            link = soup.a
            if link:
                firstLink = link.get('href')
                if firstLink:
                    if firstLink[0] != 'h':
                        firstLink = theLink

        if not firstLink:
            firstLink = theLink

        if 'image_url' in post:
            theImage = post['image_url']
        else:
            theImage = None
        theLinks = theSummaryLinks
        theSummaryLinks = theContent + theLinks

        theContent = ""
        theSummaryLinks = ""
        #if self.getLinksToAvoid():
        #    (theContent, theSummaryLinks) = self.extractLinks(soup, self.getLinkstoavoid())
        #else:
        #    (theContent, theSummaryLinks) = self.extractLinks(soup, "")

        if 'image_url' in post:
            theImage = post['image_url']
        else:
            theImage = None
        theLinks = theSummaryLinks
        theSummaryLinks = theContent + theLinks

        logging.debug("=========")
        logging.debug("Results: ")
        logging.debug("=========")
        logging.debug("Title:     ", theTitle)
        logging.debug("Link:      ", theLink)
        logging.debug("First Link:", firstLink)
        logging.debug("Summary:   ", content[:200])
        logging.debug("Sum links: ", theSummaryLinks)
        logging.debug("the Links"  , theLinks)
        logging.debug("Comment:   ", comment)
        logging.debug("Image;     ", theImage)
        logging.debug("Post       ", theTitle + " " + theLink)
        logging.debug("==============================================")
        logging.debug("")

        return (theTitle, theLink, firstLink, theImage, theSummary, content, theSummaryLinks, theContent, theLinks, comment)

    def publishPost(self, chan, msg):
        theChan = self.getChanId(chan)
        logging.info("Publishing %s" % msg)
        try:
            self.sc.token = self.user_slack_token
            result = self.sc.api_call("chat.postMessage",
                channel = theChan, text = msg)
            self.sc.token = self.slack_token
        except:
            result = self.sc.chat_postMessage(channel=theChan,
                    text=msg)
        logging.info(result)
        return(result)

    def getBots(self):
        if not self.posts:
            self.setPosts('tavern-of-the-bots')
        msgs = {}
        for msg in reversed(self.getPosts()):
            if msg['text'].find('Hello')>=0:
                posN = msg['text'].find('Name:')+6
                posFN = msg['text'].find('"',posN)
                posI = msg['text'].find('IP:')+4
                posFI = msg['text'].find(' ',posI+1)-1
                posC = msg['text'].find('[')
                name = msg['text'][posN:posFN]
                ip = msg['text'][posI:posFI]
                command = msg['text'][posC+1:posC+2]
                #print("IP %s %s" % (ip,name))
                if name not in msgs:
                    theTime = "%d-%d-%d"%time.localtime(float(msg['ts']))[:3]
                    msgs[name] = (ip, command, theTime)
        theBots = []
        for name in msgs:
            a, b, c = msgs[name]
            theBots.append("{2} [{1}] {0} {3}".format(a,b,c,name))
        return(theBots)

def main():
    CHANNEL = 'tavern-of-the-bots'

    import moduleTumblr
    import moduleSlack

    config = configparser.ConfigParser()
    config.read(CONFIGDIR + '/.rssBlogs')

    site = moduleSlack.moduleSlack()
    section = "Blog7"

    url = config.get(section, "url")
    site.setUrl(url)

    SLACKCREDENTIALS = os.path.expanduser(CONFIGDIR + '/.rssSlack')
    site.setSlackClient(SLACKCREDENTIALS)

    theChannel = site.getChanId(CHANNEL)
    print("the Channel %s" % theChannel)

    site.setPosts('links')
    site.setPosts('tavern-of-the-bots')
    print(site.getPosts())
    print(site.getBots())
    print(site.sc.api_call('channels.list'))
    sys.exit()
    rep = site.publishPost('tavern-of-the-bots', 'hello')
    wait = input('Delete %s?' % rep)
    site.deletePost(rep['ts'], theChannel)
    sys.exit()

    site.setSocialNetworks(config, section)

    if ('buffer' in config.options(section)):
        site.setBufferapp(config.get(section, "buffer"))

    if ('cache' in config.options(section)):
        site.setProgram(config.get(section, "cache"))

    theChannel = site.getChanId("links")

    i = 0
    listLinks = ""

    lastUrl = ''
    for i, post in enumerate(site.getPosts()):
        url = site.getLink(i)
        if urllib.parse.urlparse(url).netloc == lastUrl:
            listLinks = listLinks + "%d>> %s\n" % (i, url)
        else:
            listLinks = listLinks + "%d) %s\n" % (i, url)
        lastUrl = urllib.parse.urlparse(url).netloc
        print(site.getTitle(i))
        print(site.getLink(i))
        print(site.getPostTitle(post))
        print(site.getPostLink(post))
        i = i + 1

    numEntries = i
    click.echo_via_pager(listLinks)
    i = input("Which one? [x] to exit ")
    if i == 'x':
        sys.exit()

    elem = int(i)
    print(site.getPosts()[elem])

    action = input("Delete [d], publish [p], exit [x] ")

    if action == 'x':
        sys.exit()
    elif action == 'p':
        if site.getBufferapp():
            for profile in site.getSocialNetworks():
                if profile[0] in site.getBufferapp():
                    lenMax = site.len(profile)
                    print("   getBuffer %s" % profile)
                    socialNetwork = (profile,site.getSocialNetworks()[profile])
                    title = site.getTitle(elem)
                    url = site.getLink(elem)
                    listPosts = []
                    listPosts.append((title, url))
                    site.buffer[socialNetwork].addPosts(listPosts)

        if site.getProgram():
            for profile in site.getSocialNetworks():
                if profile[0] in site.getProgram():
                    lenMax = site.len(profile)
                    print("   getProgram %s" % profile)

                    socialNetwork = (profile,site.getSocialNetworks()[profile])

                    listP = site.cache[socialNetwork].getPosts()
                    #site.cache[socialNetwork].posts = site.cache[socialNetwork].posts[:8]
                    #listP = site.cache[socialNetwork].getPosts()
                    #for i,l in enumerate(listP):
                    #    print(i, l)
                    #site.cache[socialNetwork].updatePostsCache()
                    listPsts = site.obtainPostData(elem)
                    listP = listP + [listPsts]
                    #for i,l in enumerate(listP):
                    #    print(i, l)
                    #sys.exit()
                    site.cache[socialNetwork].posts = listP
                    site.cache[socialNetwork].updatePostsCache()
        t = moduleTumblr.moduleTumblr()
        t.setClient('fernand0')
        # We need to publish it in the Tumblr blog since we won't publish it by
        # usuarl means (it is deleted from queue).
        t.publishPost(title, url, '')

    site.deletePost(site.getId(j), theChannel)
    #print(outputData['Slack']['pending'][elem][8])

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import configparser
import pickle
import os
import urllib
import logging
from pdfrw import PdfReader
from slackclient import SlackClient
import sys
import click
import requests
from bs4 import BeautifulSoup
from bs4 import Tag

from configMod import *
from moduleContent import *

class moduleSlack(Content):

    def __init__(self):
        super().__init__()
        self.sc = None
        self.keys = []

    def setSlackClient(self, slackCredentials):
        config = configparser.ConfigParser()
        config.read(slackCredentials)

        slack_token = config["Slack"].get('api-key')

        self.sc = SlackClient(slack_token)

    def getSlackClient(self):
        return self.sc

    def setPosts(self, channel='links'):
        logging.info("  Setting posts")
        self.posts = []
        theChannel = self.getChanId(channel)
        history = self.sc.api_call( "channels.history", count=1000, channel=theChannel)
        if 'messages' in history:
            self.posts = history['messages']
        else:
            self.posts = []

        #outputData = {}
        #serviceName = 'Slack'
        #outputData[serviceName] = {'sent': [], 'pending': []}
        #for post in self.getPosts():
        #    if 'attachments' in post:
        #        outputData[serviceName]['pending'].append(
        #            (post['text'][1:-1], post['attachments'][0]['title'], '', '', '', '', '', '', post['ts'], ''))
        #    else:
        #        #print(post)
        #        outputData[serviceName]['pending'].append(
        #            (post['text'][1:-1], '', '', '', '', '', '', '', post['ts'], ''))
        #self.postsFormatted = outputData

    def getTitle(self, i):
        post = self.getPosts()[i]
        return(self.getPostTitle(post))

    def getLink(self, i):
        post = self.getPosts()[i]
        return(self.getPostLink(post))

    def getPostTitle(self, post):
        if 'attachments' in post:
            return(post['attachments'][0]['title'])
        else:
            text = post['text']
            if text.startswith('<'):
                title = post['text'][1:-1]
            else:
                pos = text.find('<')
                title=text[:pos]
            return(title)

    def getPostLink(self, post):
        if 'attachments' in post:
            return(post['attachments'][0]['original_url'])
        else:
            text = post['text']
            if text.startswith('<'):
                url = post['text'][1:-1]
            else:
                pos = text.find('<')
                url=text[pos+1:-1]
            return(url)

    def getId(self, i):
        post = self.getPosts()[i]
        return(post['ts'])

    def getKeys(self):
        return(self.keys)

    def setKeys(self, keys):
        self.keys = keys

    def deletePost(self, idPost, theChannel):
        #theChannel or the name of the channel?
        logging.info("Deleting id %s" % idPost)

        result = self.sc.api_call("chat.delete", channel=theChannel, ts=idPost)

        logging.info(result)
        return(result)

    def getChanId(self, name):
        chanList = self.sc.api_call("channels.list")['channels']
        for channel in chanList:
            if channel['name_normalized'] == name:
                return(channel['id'])
        return(None)

    def obtainPostData(self, i, debug=False):
        if not self.posts:
            self.setPosts()

        posts = self.getPosts()
        if not posts:
            return (None, None, None, None, None, None, None, None, None, None)

        post = posts[i]
        if 'attachments' in post:
            post = post['attachments'][0]

        theContent = ''
        url = ''
        firstLink = ''
        logging.debug("i %d", i)
        logging.debug("post %s", post)

        theTitle = self.getTitle(i)
        theLink = self.getLink(i)
        if theLink.find('tumblr')>0:
            theTitle = post['text']
        firstLink = theLink
        if 'text' in post:
            content = post['text']
        else:
            content = theLink
        theSummary = content
        theSummaryLinks = content
        if 'image_url' in post:
            theImage = post['image_url']
        elif 'thumb_url' in post:
            theImage = post['thumb_url']
        else:
            logging.info("Fail image")
            logging.info("Fail image %s", post)
            theImage = ''
        #elif 'text' in post:
        #    if post['text'].startswith('<h'):
        #        # It's an url
        #        url = post['text'][1:-1]
        #        req = requests.get(url)
        #
        #        if req.text.find('403 Forbidden')>=0:
        #            theTitle = url
        #            theSummary = url
        #            content = url
        #            theDescription = url
        #        else:
        #            if url.lower().endswith('pdf'):
        #                nameFile = '/tmp/kkkkk.pdf'
        #                with open(nameFile,'wb') as f:
        #                    f.write(req.content)
        #                theTitle = PdfReader(nameFile).Info.Title
        #                if theTitle:
        #                    theTitle = theTitle[1:-1]
        #                else:
        #                    theTitle = url
        #                theUrl = url
        #                theSummary = ''
        #                content = theSummary
        #                theDescription = theSummary
        #            else:
        #                soup = BeautifulSoup(req.text, 'lxml')
        #                #print("soup", soup)
        #                theTitle = soup.title
        #                if theTitle:
        #                    theTitle = str(theTitle.string)
        #                else:
        #                    # The last part of the path, without the dot part, and
        #                    # capitized
        #                    urlP = urllib.parse.urlparse(url)
        #                    theTitle = os.path.basename(urlP.path).split('.')[0].capitalize()
        #                theSummary = str(soup.body)
        #                content = theSummary
        #                theDescription = theSummary
        #    else:
        #        theSummary = post['text']
        #        content = post['text']
        #        theDescription = post['text']
        #        theTitle = post['text']
        #else:
        #    theSummary = post['title']
        #    content = post['title']
        #    theDescription = post['title']

        if 'original_url' in post:
            theLink = post['original_url']
        elif url:
            theLink = url
        else:
            theLink = self.getPostLink(post)

        if ('comment' in post):
            comment = post['comment']
        else:
            comment = ""

        #print("content", content)
        theSummaryLinks = ""

        soup = BeautifulSoup(content, 'lxml')
        if not content.startswith('http'):
            link = soup.a
            if link:
                firstLink = link.get('href')
                if firstLink:
                    if firstLink[0] != 'h':
                        firstLink = theLink

        if not firstLink:
            firstLink = theLink

        if 'image_url' in post:
            theImage = post['image_url']
        else:
            theImage = None
        theLinks = theSummaryLinks
        theSummaryLinks = theContent + theLinks

        theContent = ""
        theSummaryLinks = ""
        #if self.getLinksToAvoid():
        #    (theContent, theSummaryLinks) = self.extractLinks(soup, self.getLinkstoavoid())
        #else:
        #    (theContent, theSummaryLinks) = self.extractLinks(soup, "")

        if 'image_url' in post:
            theImage = post['image_url']
        else:
            theImage = None
        theLinks = theSummaryLinks
        theSummaryLinks = theContent + theLinks

        logging.debug("=========")
        logging.debug("Results: ")
        logging.debug("=========")
        logging.debug("Title:     ", theTitle)
        logging.debug("Link:      ", theLink)
        logging.debug("First Link:", firstLink)
        logging.debug("Summary:   ", content[:200])
        logging.debug("Sum links: ", theSummaryLinks)
        logging.debug("the Links"  , theLinks)
        logging.debug("Comment:   ", comment)
        logging.debug("Image;     ", theImage)
        logging.debug("Post       ", theTitle + " " + theLink)
        logging.debug("==============================================")
        logging.debug("")

        return (theTitle, theLink, firstLink, theImage, theSummary, content, theSummaryLinks, theContent, theLinks, comment)

    def publishPost(self, chan, msg):
        theChan = self.getChanId(chan)
        logging.info("Publishing %s" % msg)
        result = self.sc.api_call("chat.postMessage",
                channel = theChan, text = msg)
        logging.info(result)
        return(result)

def main():
    CHANNEL = 'tavern-of-the-bots'

    import moduleTumblr
    import moduleSlack

    config = configparser.ConfigParser()
    config.read(CONFIGDIR + '/.rssBlogs')

    site = moduleSlack.moduleSlack()
    section = "Blog7"

    url = config.get(section, "url")
    site.setUrl(url)

    SLACKCREDENTIALS = os.path.expanduser(CONFIGDIR + '/.rssSlack')
    site.setSlackClient(SLACKCREDENTIALS)

    theChannel = site.getChanId(CHANNEL)
    site.setPosts('links')

    site.setSocialNetworks(config, section)

    if ('bufferapp' in config.options(section)):
        site.setBufferapp(config.get(section, "bufferapp"))

    if ('program' in config.options(section)):
        site.setProgram(config.get(section, "program"))

    theChannel = site.getChanId("links")

    i = 0
    listLinks = ""

    lastUrl = ''
    for i, post in enumerate(site.getPosts()):
        url = site.getLink(i)
        if urllib.parse.urlparse(url).netloc == lastUrl:
            listLinks = listLinks + "%d>> %s\n" % (i, url)
        else:
            listLinks = listLinks + "%d) %s\n" % (i, url)
        lastUrl = urllib.parse.urlparse(url).netloc
        print(site.getTitle(i))
        print(site.getLink(i))
        print(site.getPostTitle(post))
        print(site.getPostLink(post))
        i = i + 1

    numEntries = i
    click.echo_via_pager(listLinks)
    i = input("Which one? [x] to exit ")
    if i == 'x':
        sys.exit()

    elem = int(i)
    print(site.getPosts()[elem])

    action = input("Delete [d], publish [p], exit [x] ")

    if action == 'x':
        sys.exit()
    elif action == 'p':
        if site.getBufferapp():
            for profile in site.getSocialNetworks():
                if profile[0] in site.getBufferapp():
                    lenMax = site.len(profile)
                    print("   getBuffer %s" % profile)
                    socialNetwork = (profile,site.getSocialNetworks()[profile])
                    title = site.getTitle(elem)
                    url = site.getLink(elem)
                    listPosts = []
                    listPosts.append((title, url))
                    site.buffer[socialNetwork].addPosts(listPosts)

        if site.getProgram():
            for profile in site.getSocialNetworks():
                if profile[0] in site.getProgram():
                    lenMax = site.len(profile)
                    print("   getProgram %s" % profile)

                    socialNetwork = (profile,site.getSocialNetworks()[profile])

                    listP = site.cache[socialNetwork].getPosts()
                    #site.cache[socialNetwork].posts = site.cache[socialNetwork].posts[:8]
                    #listP = site.cache[socialNetwork].getPosts()
                    #for i,l in enumerate(listP):
                    #    print(i, l)
                    #site.cache[socialNetwork].updatePostsCache()
                    listPsts = site.obtainPostData(elem)
                    listP = listP + [listPsts]
                    #for i,l in enumerate(listP):
                    #    print(i, l)
                    #sys.exit()
                    site.cache[socialNetwork].posts = listP
                    site.cache[socialNetwork].updatePostsCache()
        t = moduleTumblr.moduleTumblr()
        t.setClient('fernand0')
        # We need to publish it in the Tumblr blog since we won't publish it by
        # usuarl means (it is deleted from queue).
        t.publishPost(title, url, '')

    site.deletePost(site.getId(elem), theChannel)
    #print(outputData['Slack']['pending'][elem][8])

if __name__ == '__main__':
    main()
<commit_msg>Cleaning moduleSlack code<commit_after>#!/usr/bin/env python

import configparser
import pickle
import os
import urllib
import logging
from pdfrw import PdfReader
from slackclient import SlackClient
import sys
import click
import requests
from bs4 import BeautifulSoup
from bs4 import Tag

from moduleContent import *

class moduleSlack(Content):

    def __init__(self):
        super().__init__()
        self.sc = None
        self.keys = []

    def setSlackClient(self, slackCredentials):
        config = configparser.ConfigParser()
        config.read(slackCredentials)

        slack_token = config["Slack"].get('api-key')

        self.sc = SlackClient(slack_token)

    def getSlackClient(self):
        return self.sc

    def setPosts(self, channel='links'):
        logging.info("  Setting posts")
        self.posts = []
        theChannel = self.getChanId(channel)
        history = self.sc.api_call( "channels.history", count=1000, channel=theChannel)
        if 'messages' in history:
            self.posts = history['messages']
        else:
            self.posts = []

        #outputData = {}
        #serviceName = 'Slack'
        #outputData[serviceName] = {'sent': [], 'pending': []}
        #for post in self.getPosts():
        #    if 'attachments' in post:
        #        outputData[serviceName]['pending'].append(
        #            (post['text'][1:-1], post['attachments'][0]['title'], '', '', '', '', '', '', post['ts'], ''))
        #    else:
        #        #print(post)
        #        outputData[serviceName]['pending'].append(
        #            (post['text'][1:-1], '', '', '', '', '', '', '', post['ts'], ''))
        #self.postsFormatted = outputData

    def getTitle(self, i):
        post = self.getPosts()[i]
        return(self.getPostTitle(post))

    def getLink(self, i):
        post = self.getPosts()[i]
        return(self.getPostLink(post))

    def getPostTitle(self, post):
        if 'attachments' in post:
            return(post['attachments'][0]['title'])
        else:
            text = post['text']
            if text.startswith('<'):
                title = post['text'][1:-1]
            else:
                pos = text.find('<')
                title=text[:pos]
            return(title)

    def getPostLink(self, post):
        if 'attachments' in post:
            return(post['attachments'][0]['original_url'])
        else:
            text = post['text']
            if text.startswith('<'):
                url = post['text'][1:-1]
            else:
                pos = text.find('<')
                url=text[pos+1:-1]
            return(url)

    def getId(self, i):
        post = self.getPosts()[i]
        return(post['ts'])

    def getKeys(self):
        return(self.keys)

    def setKeys(self, keys):
        self.keys = keys

    def deletePost(self, idPost, theChannel):
        #theChannel or the name of the channel?
        logging.info("Deleting id %s" % idPost)

        result = self.sc.api_call("chat.delete", channel=theChannel, ts=idPost)

        logging.info(result)
        return(result)

    def getChanId(self, name):
        chanList = self.sc.api_call("channels.list")['channels']
        for channel in chanList:
            if channel['name_normalized'] == name:
                return(channel['id'])
        return(None)

    def obtainPostData(self, i, debug=False):
        if not self.posts:
            self.setPosts()

        posts = self.getPosts()
        if not posts:
            return (None, None, None, None, None, None, None, None, None, None)

        post = posts[i]
        if 'attachments' in post:
            post = post['attachments'][0]

        theContent = ''
        url = ''
        firstLink = ''
        logging.debug("i %d", i)
        logging.debug("post %s", post)

        theTitle = self.getTitle(i)
        theLink = self.getLink(i)
        if theLink.find('tumblr')>0:
            theTitle = post['text']
        firstLink = theLink
        if 'text' in post:
            content = post['text']
        else:
            content = theLink
        theSummary = content
        theSummaryLinks = content
        if 'image_url' in post:
            theImage = post['image_url']
        elif 'thumb_url' in post:
            theImage = post['thumb_url']
        else:
            logging.info("Fail image")
            logging.info("Fail image %s", post)
            theImage = ''
        #elif 'text' in post:
        #    if post['text'].startswith('<h'):
        #        # It's an url
        #        url = post['text'][1:-1]
        #        req = requests.get(url)
        #
        #        if req.text.find('403 Forbidden')>=0:
        #            theTitle = url
        #            theSummary = url
        #            content = url
        #            theDescription = url
        #        else:
        #            if url.lower().endswith('pdf'):
        #                nameFile = '/tmp/kkkkk.pdf'
        #                with open(nameFile,'wb') as f:
        #                    f.write(req.content)
        #                theTitle = PdfReader(nameFile).Info.Title
        #                if theTitle:
        #                    theTitle = theTitle[1:-1]
        #                else:
        #                    theTitle = url
        #                theUrl = url
        #                theSummary = ''
        #                content = theSummary
        #                theDescription = theSummary
        #            else:
        #                soup = BeautifulSoup(req.text, 'lxml')
        #                #print("soup", soup)
        #                theTitle = soup.title
        #                if theTitle:
        #                    theTitle = str(theTitle.string)
        #                else:
        #                    # The last part of the path, without the dot part, and
        #                    # capitized
        #                    urlP = urllib.parse.urlparse(url)
        #                    theTitle = os.path.basename(urlP.path).split('.')[0].capitalize()
        #                theSummary = str(soup.body)
        #                content = theSummary
        #                theDescription = theSummary
        #    else:
        #        theSummary = post['text']
        #        content = post['text']
        #        theDescription = post['text']
        #        theTitle = post['text']
        #else:
        #    theSummary = post['title']
        #    content = post['title']
        #    theDescription = post['title']

        if 'original_url' in post:
            theLink = post['original_url']
        elif url:
            theLink = url
        else:
            theLink = self.getPostLink(post)

        if ('comment' in post):
            comment = post['comment']
        else:
            comment = ""

        #print("content", content)
        theSummaryLinks = ""

        soup = BeautifulSoup(content, 'lxml')
        if not content.startswith('http'):
            link = soup.a
            if link:
                firstLink = link.get('href')
                if firstLink:
                    if firstLink[0] != 'h':
                        firstLink = theLink

        if not firstLink:
            firstLink = theLink

        if 'image_url' in post:
            theImage = post['image_url']
        else:
            theImage = None
        theLinks = theSummaryLinks
        theSummaryLinks = theContent + theLinks

        theContent = ""
        theSummaryLinks = ""
        #if self.getLinksToAvoid():
        #    (theContent, theSummaryLinks) = self.extractLinks(soup, self.getLinkstoavoid())
        #else:
        #    (theContent, theSummaryLinks) = self.extractLinks(soup, "")

        if 'image_url' in post:
            theImage = post['image_url']
        else:
            theImage = None
        theLinks = theSummaryLinks
        theSummaryLinks = theContent + theLinks

        logging.debug("=========")
        logging.debug("Results: ")
        logging.debug("=========")
        logging.debug("Title:     ", theTitle)
        logging.debug("Link:      ", theLink)
        logging.debug("First Link:", firstLink)
        logging.debug("Summary:   ", content[:200])
        logging.debug("Sum links: ", theSummaryLinks)
        logging.debug("the Links"  , theLinks)
        logging.debug("Comment:   ", comment)
        logging.debug("Image;     ", theImage)
        logging.debug("Post       ", theTitle + " " + theLink)
        logging.debug("==============================================")
        logging.debug("")

        return (theTitle, theLink, firstLink, theImage, theSummary, content, theSummaryLinks, theContent, theLinks, comment)

    def publishPost(self, chan, msg):
        theChan = self.getChanId(chan)
        logging.info("Publishing %s" % msg)
        result = self.sc.api_call("chat.postMessage",
                channel = theChan, text = msg)
        logging.info(result)
        return(result)

def main():
    CHANNEL = 'tavern-of-the-bots'

    import moduleTumblr
    import moduleSlack

    config = configparser.ConfigParser()
    config.read(CONFIGDIR + '/.rssBlogs')

    site = moduleSlack.moduleSlack()
    section = "Blog7"

    url = config.get(section, "url")
    site.setUrl(url)

    SLACKCREDENTIALS = os.path.expanduser(CONFIGDIR + '/.rssSlack')
    site.setSlackClient(SLACKCREDENTIALS)

    theChannel = site.getChanId(CHANNEL)
    site.setPosts('links')

    site.setSocialNetworks(config, section)

    if ('bufferapp' in config.options(section)):
        site.setBufferapp(config.get(section, "bufferapp"))

    if ('program' in config.options(section)):
        site.setProgram(config.get(section, "program"))

    theChannel = site.getChanId("links")

    i = 0
    listLinks = ""

    lastUrl = ''
    for i, post in enumerate(site.getPosts()):
        url = site.getLink(i)
        if urllib.parse.urlparse(url).netloc == lastUrl:
            listLinks = listLinks + "%d>> %s\n" % (i, url)
        else:
            listLinks = listLinks + "%d) %s\n" % (i, url)
        lastUrl = urllib.parse.urlparse(url).netloc
        print(site.getTitle(i))
        print(site.getLink(i))
        print(site.getPostTitle(post))
        print(site.getPostLink(post))
        i = i + 1

    numEntries = i
    click.echo_via_pager(listLinks)
    i = input("Which one? [x] to exit ")
    if i == 'x':
        sys.exit()

    elem = int(i)
    print(site.getPosts()[elem])

    action = input("Delete [d], publish [p], exit [x] ")

    if action == 'x':
        sys.exit()
    elif action == 'p':
        if site.getBufferapp():
            for profile in site.getSocialNetworks():
                if profile[0] in site.getBufferapp():
                    lenMax = site.len(profile)
                    print("   getBuffer %s" % profile)
                    socialNetwork = (profile,site.getSocialNetworks()[profile])
                    title = site.getTitle(elem)
                    url = site.getLink(elem)
                    listPosts = []
                    listPosts.append((title, url))
                    site.buffer[socialNetwork].addPosts(listPosts)

        if site.getProgram():
            for profile in site.getSocialNetworks():
                if profile[0] in site.getProgram():
                    lenMax = site.len(profile)
                    print("   getProgram %s" % profile)

                    socialNetwork = (profile,site.getSocialNetworks()[profile])

                    listP = site.cache[socialNetwork].getPosts()
                    #site.cache[socialNetwork].posts = site.cache[socialNetwork].posts[:8]
                    #listP = site.cache[socialNetwork].getPosts()
                    #for i,l in enumerate(listP):
                    #    print(i, l)
                    #site.cache[socialNetwork].updatePostsCache()
                    listPsts = site.obtainPostData(elem)
                    listP = listP + [listPsts]
                    #for i,l in enumerate(listP):
                    #    print(i, l)
                    #sys.exit()
                    site.cache[socialNetwork].posts = listP
                    site.cache[socialNetwork].updatePostsCache()
        t = moduleTumblr.moduleTumblr()
        t.setClient('fernand0')
        # We need to publish it in the Tumblr blog since we won't publish it by
        # usuarl means (it is deleted from queue).
        t.publishPost(title, url, '')

    site.deletePost(site.getId(elem), theChannel)
    #print(outputData['Slack']['pending'][elem][8])

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># coding=utf8
"""
api.py - Willie Freies Labor API Module
Licensed under a Mozilla Public License 2.0.
"""
from willie.module import commands
import urllib, urllib2, json
from datetime import datetime

def configure(config):
	"""
	|  [api]       | example  | purpose               |
	| ------------ | -------- | --------------------- |
	| api_user     | testuser | Sets the API user     |
	| api_password | testpw   | Sets the API password |
	"""
	if config.option('Configure API module', False):
		config.add_section('api')
		config.interactive_add('api', 'api_user', 'API user')
		config.interactive_add('api', 'api_password', 'API password')

def api(bot, path, params=None):
	"""Performs an API call."""
	apiUrl = 'https://freieslabor.org/api/%s' % path
	passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
	passman.add_password(None, apiUrl, \
		bot.config.api.api_user, bot.config.api.api_password)
	opener = urllib2.build_opener(urllib2.HTTPBasicAuthHandler(passman))
	urllib2.install_opener(opener)
	# POST or GET
	if params:
		data = urllib.urlencode(params)
		req = urllib2.Request(apiUrl, data)
	else:
		req = urllib2.Request(apiUrl)

	response = urllib2.urlopen(req)
	return json.loads(response.read())

def apiError(err, msg):
	"""Generates error messages for API/HTTP errors."""
	try:
		response = json.loads(err.read())
		return '%s (%s)' % (msg, response['status'])
	except ValueError:
		return msg

@commands('open', 'auf')
def openLab(bot, trigger):
	"""Sets the lab status to "open"."""
	try:
		# check whether status has changed
		oldStatus = api(bot, 'room')
		if oldStatus['open']:
			bot.say('Door status already "open".')
		elif not oldStatus['open']:
			# change status
			response = api(bot, 'room', { 'open': True })
			if response['success']:
				bot.say('Door status changed to "open".')
			else:
				# raise error with API error message
				raise ApiException(response['status'])

	except (urllib2.HTTPError, ApiException), e:
		bot.say(apiError(e, "Couldn't set room status"))

@commands('close', 'zu')
def closeLab(bot, trigger):
	"""Sets the lab status to "closed"."""
	try:
		# check whether status has changed
		oldStatus = api(bot, 'room')
		if not oldStatus['open']:
			bot.say('Door status already "closed".')
		elif oldStatus['open']:
			# change status
			response = api(bot, 'room', { 'open': False })
			if response['success']:
				bot.say('Door status changed to "closed".')
			else:
				# raise error with API error message
				raise ApiException(response['status'])

	except (urllib2.HTTPError, ApiException), e:
		bot.say(apiError(e, "Couldn't set room status"))

@commands('room', 'door', 'status')
def labStatus(bot, trigger):
	try:
		response = api(bot, 'room')
		since = datetime.fromtimestamp(response['since'])
		sinceStr = since.strftime('%a, %H:%M')
		statusStr = 'open' if response['open'] else 'closed'

		bot.say('The door is %s since %s.' % (statusStr, sinceStr))
	except urllib2.HTTPError, e:
		bot.say(apiError(e, "Couldn't get room status"))

class ApiException(Exception):
	"""Custom API exception."""
	pass

<commit_msg>added docstring for room command<commit_after># coding=utf8
"""
api.py - Willie Freies Labor API Module
Licensed under a Mozilla Public License 2.0.
"""
from willie.module import commands
import urllib, urllib2, json
from datetime import datetime

def configure(config):
	"""
	|  [api]       | example  | purpose               |
	| ------------ | -------- | --------------------- |
	| api_user     | testuser | Sets the API user     |
	| api_password | testpw   | Sets the API password |
	"""
	if config.option('Configure API module', False):
		config.add_section('api')
		config.interactive_add('api', 'api_user', 'API user')
		config.interactive_add('api', 'api_password', 'API password')

def api(bot, path, params=None):
	"""Performs an API call."""
	apiUrl = 'https://freieslabor.org/api/%s' % path
	passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
	passman.add_password(None, apiUrl, \
		bot.config.api.api_user, bot.config.api.api_password)
	opener = urllib2.build_opener(urllib2.HTTPBasicAuthHandler(passman))
	urllib2.install_opener(opener)
	# POST or GET
	if params:
		data = urllib.urlencode(params)
		req = urllib2.Request(apiUrl, data)
	else:
		req = urllib2.Request(apiUrl)

	response = urllib2.urlopen(req)
	return json.loads(response.read())

def apiError(err, msg):
	"""Generates error messages for API/HTTP errors."""
	try:
		response = json.loads(err.read())
		return '%s (%s)' % (msg, response['status'])
	except ValueError:
		return msg

@commands('open', 'auf')
def openLab(bot, trigger):
	"""Sets the lab status to "open"."""
	try:
		# check whether status has changed
		oldStatus = api(bot, 'room')
		if oldStatus['open']:
			bot.say('Door status already "open".')
		elif not oldStatus['open']:
			# change status
			response = api(bot, 'room', { 'open': True })
			if response['success']:
				bot.say('Door status changed to "open".')
			else:
				# raise error with API error message
				raise ApiException(response['status'])

	except (urllib2.HTTPError, ApiException), e:
		bot.say(apiError(e, "Couldn't set room status"))

@commands('close', 'zu')
def closeLab(bot, trigger):
	"""Sets the lab status to "closed"."""
	try:
		# check whether status has changed
		oldStatus = api(bot, 'room')
		if not oldStatus['open']:
			bot.say('Door status already "closed".')
		elif oldStatus['open']:
			# change status
			response = api(bot, 'room', { 'open': False })
			if response['success']:
				bot.say('Door status changed to "closed".')
			else:
				# raise error with API error message
				raise ApiException(response['status'])

	except (urllib2.HTTPError, ApiException), e:
		bot.say(apiError(e, "Couldn't set room status"))

@commands('room', 'door', 'status')
def labStatus(bot, trigger):
	"""Returns the lab door status."""
	try:
		response = api(bot, 'room')
		since = datetime.fromtimestamp(response['since'])
		sinceStr = since.strftime('%a, %H:%M')
		statusStr = 'open' if response['open'] else 'closed'

		bot.say('The door is %s since %s.' % (statusStr, sinceStr))
	except urllib2.HTTPError, e:
		bot.say(apiError(e, "Couldn't get room status"))

class ApiException(Exception):
	"""Custom API exception."""
	pass

<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
url.py - Willie URL title module
Copyright 2010-2011, Michael Yanovich, yanovich.net, Kenneth Sham
Copyright 2012 Edward Powell
Licensed under the Eiffel Forum License 2.

http://willie.dftba.net
"""

import re
from htmlentitydefs import name2codepoint
import web
import unicodedata
import urlparse

url_finder = None
r_entity = re.compile(r'&[A-Za-z0-9#]+;')
INVALID_WEBSITE = 0x01

def configure(config):
    if config.option('Exclude certain URLs from automatic title display', True):
        config.add_list('url_exclude', 'Enter regular expressions for each URL you would like to exclude.',
            'Regex:')
        config.interactive_add('url_exclusion_char',
            'Prefix to suppress URL titling', '!')

        chunk = ("url_exclude = %s\nurl_exclusion_char = '%s'\n" %
                (config.url_exclude, config.url_exclusion_char))
        return chunk
    else: return ''

def setup(willie):
    global url_finder
    # Set up empty url exclusion list and default exclusion character
    if not hasattr(willie.config, 'url_exclude'):
        willie.config.set_attr('url_exclude', [])
    else:
        for s in willie.config.url_exclude:
            if isinstance(s, basestring):
                willie.config.url_exclude.remove(s)
                willie.config.url_exclude.append(re.compile(s))
            #Otherwise, it's probably already compiled by another module.

    if not hasattr(willie.config, 'url_exclusion_char'):
        willie.config.set_attr('url_exclusion_char', '!')

    url_finder = re.compile(r'(?u)(%s?(http|https|ftp)(://\S+))' %
        (willie.config.url_exclusion_char))
    # We want the exclusion list to be pre-compiled, since url parsing gets
    # called a /lot/, and it's annoying when it's laggy.

def find_title(url):
    """
    This finds the title when provided with a string of a URL.
    """
    uri = url

    if not uri and hasattr(self, 'last_seen_uri'):
        uri = self.last_seen_uri.get(origin.sender)

    if not re.search('^((https?)|(ftp))://', uri):
        uri = 'http://' + uri

    if "twitter.com" in uri:
        uri = uri.replace('#!', '?_escaped_fragment_=')

    content = web.get(uri)
    regex = re.compile('<(/?)title( [^>]+)?>', re.IGNORECASE)
    content = regex.sub(r'<\1title>',content)
    regex = re.compile('[\'"]<title>[\'"]', re.IGNORECASE)
    content = regex.sub('',content)
    start = content.find('<title>')
    if start == -1: return
    end = content.find('</title>', start)
    if end == -1: return
    content = content[start+7:end]
    content = content.strip('\n').rstrip().lstrip()
    title = content

    if len(title) > 200:
        title = title[:200] + '[...]'

    def e(m):
        entity = m.group()
        if entity.startswith('&#x'):
            cp = int(entity[3:-1],16)
            return unichr(cp).encode('utf-8')
        elif entity.startswith('&#'):
            cp = int(entity[2:-1])
            return unichr(cp).encode('utf-8')
        else:
            char = name2codepoint[entity[1:-1]]
            return unichr(char).encode('utf-8')

    title = r_entity.sub(e, title)

    if title:
        title = uni_decode(title)
    else: title = 'None'

    title = title.replace('\n', '')
    title = title.replace('\r', '')

    def remove_spaces(x):
        if "  " in x:
            x = x.replace("  ", " ")
            return remove_spaces(x)
        else:
            return x

    title = remove_spaces (title)

    re_dcc = re.compile(r'(?i)dcc\ssend')
    title = re.sub(re_dcc, '', title)

    if title:
        return title

def getTLD (url):
    idx = 7
    if url.startswith('https://'): idx = 8
    elif url.startswith('ftp://'): idx = 6
    u = url[idx:]
    f = u.find('/')
    if f == -1: u = url
    else: u = url[0:idx] + u[0:f]
    return u

def get_results(willie, text):
    a = re.findall(url_finder, text)
    display = [ ]
    for match in a:
        match = match[0]
        if (match.startswith(willie.config.url_exclusion_char) or
                any(pattern.find(match) for pattern in willie.config.url_exclude)):
            continue
        url = uni_encode(match)
        url = uni_decode(url)
        url = iriToUri(url)
        try:
            page_title = find_title(url)
        except:
            page_title = None # if it can't access the site fail silently
        display.append([page_title, url])
    return display

def show_title_auto (willie, trigger):
    if trigger.startswith('.title '):
        return
    if len(re.findall("\([\d]+\sfiles\sin\s[\d]+\sdirs\)", trigger)) == 1: return
    try:
        results = get_results(willie, trigger)
    except: return
    if results is None: return

    k = 1
    for r in results:
        if k > 3: break
        k += 1

        if r[0] is None:
            continue
        else: r[1] = getTLD(r[1])
        willie.say('[ %s ] - %s' % (r[0], r[1]))
show_title_auto.rule = '(?u).*((http|https)(://\S+)).*'
show_title_auto.priority = 'high'

def show_title_demand (willie, trigger):
    #try:
    results = get_results(trigger)
    #except: return
    if results is None: return

    for r in results:
        if r[0] is None: continue
        r[1] = getTLD(r[1])
        willie.say('[ %s ] - %s' % (r[0], r[1]))
show_title_demand.commands = ['title']
show_title_demand.priority = 'high'

#Tools formerly in unicode.py

def uni_decode(bytes):
    try:
        text = bytes.decode('utf-8')
    except UnicodeDecodeError:
        try:
            text = bytes.decode('iso-8859-1')
        except UnicodeDecodeError:
            text = bytes.decode('cp1252')
    return text

def uni_encode(bytes):
    try:
        text = bytes.encode('utf-8')
    except UnicodeEncodeError:
        try:
            text = bytes.encode('iso-8859-1')
        except UnicodeEncodeError:
            text = bytes.encode('cp1252')
    return text

def urlEncodeNonAscii(b):
    return re.sub('[\x80-\xFF]', lambda c: '%%%02x' % ord(c.group(0)), b)

def iriToUri(iri):
    parts = urlparse.urlparse(iri)
    return urlparse.urlunparse(
        part.encode('idna') if parti == 1 else urlEncodeNonAscii(part.encode('utf-8'))
        for parti, part in enumerate(parts)
    )

if __name__ == '__main__':
    print __doc__.strip()
<commit_msg>[url] Properly fix url title exclusions<commit_after>#!/usr/bin/env python
"""
url.py - Willie URL title module
Copyright 2010-2011, Michael Yanovich, yanovich.net, Kenneth Sham
Copyright 2012 Edward Powell
Licensed under the Eiffel Forum License 2.

http://willie.dftba.net
"""

import re
from htmlentitydefs import name2codepoint
import web
import unicodedata
import urlparse

url_finder = None
r_entity = re.compile(r'&[A-Za-z0-9#]+;')
INVALID_WEBSITE = 0x01

def configure(config):
    if config.option('Exclude certain URLs from automatic title display', True):
        config.add_list('url_exclude', 'Enter regular expressions for each URL you would like to exclude.',
            'Regex:')
        config.interactive_add('url_exclusion_char',
            'Prefix to suppress URL titling', '!')

        chunk = ("url_exclude = %s\nurl_exclusion_char = '%s'\n" %
                (config.url_exclude, config.url_exclusion_char))
        return chunk
    else: return ''

def setup(willie):
    global url_finder
    # Set up empty url exclusion list and default exclusion character
    if not hasattr(willie.config, 'url_exclude'):
        willie.config.set_attr('url_exclude', [])
    else:
        for s in willie.config.url_exclude:
            if isinstance(s, basestring):
                willie.config.url_exclude.remove(s)
                willie.config.url_exclude.append(re.compile(s))
            #Otherwise, it's probably already compiled by another module.

    if not hasattr(willie.config, 'url_exclusion_char'):
        willie.config.set_attr('url_exclusion_char', '!')

    url_finder = re.compile(r'(?u)(%s?(http|https|ftp)(://\S+))' %
        (willie.config.url_exclusion_char))
    # We want the exclusion list to be pre-compiled, since url parsing gets
    # called a /lot/, and it's annoying when it's laggy.

def find_title(url):
    """
    This finds the title when provided with a string of a URL.
    """
    uri = url

    if not uri and hasattr(self, 'last_seen_uri'):
        uri = self.last_seen_uri.get(origin.sender)

    if not re.search('^((https?)|(ftp))://', uri):
        uri = 'http://' + uri

    if "twitter.com" in uri:
        uri = uri.replace('#!', '?_escaped_fragment_=')

    content = web.get(uri)
    regex = re.compile('<(/?)title( [^>]+)?>', re.IGNORECASE)
    content = regex.sub(r'<\1title>',content)
    regex = re.compile('[\'"]<title>[\'"]', re.IGNORECASE)
    content = regex.sub('',content)
    start = content.find('<title>')
    if start == -1: return
    end = content.find('</title>', start)
    if end == -1: return
    content = content[start+7:end]
    content = content.strip('\n').rstrip().lstrip()
    title = content

    if len(title) > 200:
        title = title[:200] + '[...]'

    def e(m):
        entity = m.group()
        if entity.startswith('&#x'):
            cp = int(entity[3:-1],16)
            return unichr(cp).encode('utf-8')
        elif entity.startswith('&#'):
            cp = int(entity[2:-1])
            return unichr(cp).encode('utf-8')
        else:
            char = name2codepoint[entity[1:-1]]
            return unichr(char).encode('utf-8')

    title = r_entity.sub(e, title)

    if title:
        title = uni_decode(title)
    else: title = 'None'

    title = title.replace('\n', '')
    title = title.replace('\r', '')

    def remove_spaces(x):
        if "  " in x:
            x = x.replace("  ", " ")
            return remove_spaces(x)
        else:
            return x

    title = remove_spaces (title)

    re_dcc = re.compile(r'(?i)dcc\ssend')
    title = re.sub(re_dcc, '', title)

    if title:
        return title

def getTLD (url):
    idx = 7
    if url.startswith('https://'): idx = 8
    elif url.startswith('ftp://'): idx = 6
    u = url[idx:]
    f = u.find('/')
    if f == -1: u = url
    else: u = url[0:idx] + u[0:f]
    return u

def get_results(willie, text):
    a = re.findall(url_finder, text)
    display = [ ]
    for match in a:
        match = match[0]
        #for pattern in willie.config.url_exclude:
        #    if pattern.findall(match):
        #        print pattern.pattern
        #        print dir(pattern)
        #        continue
        if (match.startswith(willie.config.url_exclusion_char) or
                any(pattern.findall(match) for pattern in willie.config.url_exclude)):
            continue
        url = uni_encode(match)
        url = uni_decode(url)
        url = iriToUri(url)
        try:
            page_title = find_title(url)
        except:
            page_title = None # if it can't access the site fail silently
        display.append([page_title, url])
    return display

def show_title_auto (willie, trigger):
    if trigger.startswith('.title '):
        return
    if len(re.findall("\([\d]+\sfiles\sin\s[\d]+\sdirs\)", trigger)) == 1: return
    try:
        results = get_results(willie, trigger)
    except: return
    if results is None: return

    k = 1
    for r in results:
        if k > 3: break
        k += 1

        if r[0] is None:
            continue
        else: r[1] = getTLD(r[1])
        willie.say('[ %s ] - %s' % (r[0], r[1]))
show_title_auto.rule = '(?u).*((http|https)(://\S+)).*'
show_title_auto.priority = 'high'

def show_title_demand (willie, trigger):
    #try:
    results = get_results(trigger)
    #except: return
    if results is None: return

    for r in results:
        if r[0] is None: continue
        r[1] = getTLD(r[1])
        willie.say('[ %s ] - %s' % (r[0], r[1]))
show_title_demand.commands = ['title']
show_title_demand.priority = 'high'

#Tools formerly in unicode.py

def uni_decode(bytes):
    try:
        text = bytes.decode('utf-8')
    except UnicodeDecodeError:
        try:
            text = bytes.decode('iso-8859-1')
        except UnicodeDecodeError:
            text = bytes.decode('cp1252')
    return text

def uni_encode(bytes):
    try:
        text = bytes.encode('utf-8')
    except UnicodeEncodeError:
        try:
            text = bytes.encode('iso-8859-1')
        except UnicodeEncodeError:
            text = bytes.encode('cp1252')
    return text

def urlEncodeNonAscii(b):
    return re.sub('[\x80-\xFF]', lambda c: '%%%02x' % ord(c.group(0)), b)

def iriToUri(iri):
    parts = urlparse.urlparse(iri)
    return urlparse.urlunparse(
        part.encode('idna') if parti == 1 else urlEncodeNonAscii(part.encode('utf-8'))
        for parti, part in enumerate(parts)
    )

if __name__ == '__main__':
    print __doc__.strip()
<|endoftext|>


<commit_before>from django.db import models
from django.db.models import signals as model_signals
from django.db.models.base import ModelBase

from mptt import signals
from mptt.managers import TreeManager

class MPTTOptions(object):
    """
    Options class for MPTT models. Use this as an inner class called MPTTMeta:

    class MyModel(MPTTModel):
        class MPTTMeta:
            order_insertion_by = ['name']
            parent_attr = 'myparent'
        ...
    """

    order_insertion_by = []
    tree_manager_attr = 'tree'
    left_attr = 'lft'
    right_attr = 'rght'
    tree_id_attr = 'tree_id'
    level_attr = 'level'
    parent_attr = 'parent'

    def __init__(self, opts):
        # Override defaults with options provided
        if opts:
            for key, value in opts.__dict__.iteritems():
                setattr(self, key, value)

        # Normalize order_insertion_by to a list
        if isinstance(self.order_insertion_by, basestring):
            self.order_insertion_by = [self.order_insertion_by]
        elif self.order_insertion_by is None:
            self.order_insertion_by = []

    def __iter__(self):
        return iter([(k, v) for (k, v) in self.__dict__.items() if not k.startswith('_')])

    # Helper methods for accessing tree attributes on models.
    def get_raw_field_value(self, instance, field_name):
        """
        Gets the value of the given fieldname for the instance.
        This is not the same as getattr().
        This function will return IDs for foreignkeys etc, rather than doing
        a database query.
        """
        field = instance._meta.get_field(field_name)
        return field.value_from_object(instance)

    def set_raw_field_value(self, instance, field_name, value):
        """
        Sets the value of the given fieldname for the instance.
        This is not the same as setattr().
        This function requires an ID for a foreignkey (etc) rather than an instance.
        """
        field = instance._meta.get_field(field_name)
        setattr(instance, field.attname, value)

class MPTTModelBase(ModelBase):
    """
    Metaclass for MPTT models
    """

    def __new__(meta, class_name, bases, class_dict):
        """
        Create subclasses of MPTTModel. This:
         - adds the MPTT fields to the class
         - adds a TreeManager to the model
        """
        mptt_opts = class_dict.pop('MPTTMeta', None)
        class_dict['_mptt_meta'] = MPTTOptions(mptt_opts)
        cls = ModelBase.__new__(meta, class_name, bases, class_dict)

        try:
            MPTTModel
        except NameError:
            # We're defining the base class right now, so don't do anything
            # We only want to add this stuff to the subclasses.
            # (Otherwise if field names are customized, we'll end up adding two
            # copies)
            pass
        else:
            for key in ('left_attr', 'right_attr', 'tree_id_attr', 'level_attr'):
                field_name = getattr(cls._mptt_meta, key)
                try:
                    cls._meta.get_field(field_name)
                except models.FieldDoesNotExist:
                    field = models.PositiveIntegerField(db_index=True, editable=False)
                    field.contribute_to_class(cls, field_name)

            # Add a custom tree manager
            manager = TreeManager(cls._mptt_meta)
            manager.contribute_to_class(cls, cls._mptt_meta.tree_manager_attr)
            setattr(cls, '_tree_manager', getattr(cls, cls._mptt_meta.tree_manager_attr))

            # Set up signal receivers
            model_signals.post_init.connect(signals.post_init, sender=cls)
            model_signals.pre_save.connect(signals.pre_save, sender=cls)

        return cls

class MPTTModel(models.Model):
    """
    Base class for tree models.
    """

    __metaclass__ = MPTTModelBase

    class Meta:
        abstract = True

    def _mpttfield(self, fieldname):
        translated_fieldname = getattr(self._mptt_meta, '%s_attr' % fieldname)
        return getattr(self, translated_fieldname)

    def get_ancestors(self, ascending=False):
        """
        Creates a ``QuerySet`` containing the ancestors of this model
        instance.

        This defaults to being in descending order (root ancestor first,
        immediate parent last); passing ``True`` for the ``ascending``
        argument will reverse the ordering (immediate parent first, root
        ancestor last).
        """
        if self.is_root_node():
            return self._tree_manager.none()

        opts = self._mptt_meta

        order_by = opts.left_attr
        if ascending:
            order_by = '-%s' % order_by

        qs = self._tree_manager._mptt_filter(
            left__lt=self._mpttfield('left'),
            right__gt=self._mpttfield('right'),
            tree_id=self._mpttfield('tree_id'),
        )
        return qs.order_by(order_by)

    def get_children(self):
        """
        Returns a ``QuerySet`` containing the immediate children of this
        model instance, in tree order.

        The benefit of using this method over the reverse relation
        provided by the ORM to the instance's children is that a
        database query can be avoided in the case where the instance is
        a leaf node (it has no children).

        If called from a template where the tree has been walked by the
        ``cache_tree_children`` filter, no database query is required.
        """

        if hasattr(self, '_cached_children'):
            return self._cached_children
        else:
            if self.is_leaf_node():
                return self._tree_manager.none()

            return self._tree_manager._mptt_filter(parent=self)

    def get_descendants(self, include_self=False):
        """
        Creates a ``QuerySet`` containing descendants of this model
        instance, in tree order.

        If ``include_self`` is ``True``, the ``QuerySet`` will also
        include this model instance.
        """
        if not include_self and self.is_leaf_node():
            return self._tree_manager.none()

        opts = self._mptt_meta
        left = getattr(self, opts.left_attr)
        right = getattr(self, opts.right_attr)

        if not include_self:
            left += 1
            right -= 1

        return self._tree_manager._mptt_filter(
            tree_id=self._mpttfield('tree_id'),
            left__gte=left,
            left__lte=right
        )

    def get_descendant_count(self):
        """
        Returns the number of descendants this model instance has.
        """
        return (self._mpttfield('right') -
                self._mpttfield('left') - 1) / 2

    def get_leafnodes(self, include_self=False):
        """
        Creates a ``QuerySet`` containing leafnodes of this model
        instance, in tree order.

        If ``include_self`` is ``True``, the ``QuerySet`` will also
        include this model instance (if it is a leaf node)
        """
        descendants = self.get_descendants(include_self=include_self)

        return self._tree_manager._mptt_filter(descendants,
            left=models.F(self._mptt_meta.right_attr)-1
        )

    def get_next_sibling(self, **filters):
        """
        Returns this model instance's next sibling in the tree, or
        ``None`` if it doesn't have a next sibling.
        """
        qs = self._tree_manager.filter(**filters)
        if self.is_root_node():
            qs = self._tree_manager._mptt_filter(qs,
                parent__isnull=True,
                tree_id__gt=self._mpttfield('tree_id'),
            )
        else:
            qs = self._tree_manager._mptt_filter(qs,
                parent__id=getattr(self, '%s_id' % self._mptt_meta.parent_attr),
                left__gt=self._mpttfield('right'),
            )

        siblings = qs[:1]
        return siblings[0] if siblings else None

    def get_previous_sibling(self, **filters):
        """
        Returns this model instance's previous sibling in the tree, or
        ``None`` if it doesn't have a previous sibling.
        """
        opts = self._mptt_meta
        qs = self._tree_manager.filter(**filters)
        if self.is_root_node():
            qs = self._tree_manager._mptt_filter(qs,
                parent__isnull=True,
                tree_id__lt=self._mpttfield('tree_id'),
            )
            qs = qs.order_by('-%s' % opts.tree_id_attr)
        else:
            qs = self._tree_manager._mptt_filter(qs,
                parent__id=getattr(self, '%s_id' % opts.parent_attr),
                right__lt=self._mpttfield('left'),
            )
            qs = qs.order_by('-%s' % opts.right_attr)

        siblings = qs[:1]
        return siblings[0] if siblings else None

    def get_root(self):
        """
        Returns the root node of this model instance's tree.
        """
        if self.is_root_node():
            return self

        return self._tree_manager._mptt_filter(
            tree_id=self._mpttfield('tree_id'),
            parent__isnull=True
        ).get()

    def get_siblings(self, include_self=False):
        """
        Creates a ``QuerySet`` containing siblings of this model
        instance. Root nodes are considered to be siblings of other root
        nodes.

        If ``include_self`` is ``True``, the ``QuerySet`` will also
        include this model instance.
        """
        if self.is_root_node():
            queryset = self._tree_manager._mptt_filter(parent__isnull=True)
        else:
            parent_id = getattr(self, '%s_id' % self._mptt_meta.parent_attr)
            queryset = self._tree_manager._mptt_filter(parent__id=parent_id)
        if not include_self:
            queryset = queryset.exclude(pk=self.pk)
        return queryset

    def get_level(self):
        """
        Returns the level of this node (distance from root)
        """
        return getattr(self, self._mptt_meta.level_attr)

    def insert_at(self, target, position='first-child', save=False):
        """
        Convenience method for calling ``TreeManager.insert_node`` with this
        model instance.
        """
        self._tree_manager.insert_node(self, target, position, save)

    def is_child_node(self):
        """
        Returns ``True`` if this model instance is a child node, ``False``
        otherwise.
        """
        return not self.is_root_node()

    def is_leaf_node(self):
        """
        Returns ``True`` if this model instance is a leaf node (it has no
        children), ``False`` otherwise.
        """
        return not self.get_descendant_count()

    def is_root_node(self):
        """
        Returns ``True`` if this model instance is a root node,
        ``False`` otherwise.
        """
        return getattr(self, '%s_id' % self._mptt_meta.parent_attr) is None

    def move_to(self, target, position='first-child'):
        """
        Convenience method for calling ``TreeManager.move_node`` with this
        model instance.
        """
        self._tree_manager.move_node(self, target, position)

    def delete(self, *args, **kwargs):
        tree_width = (self._mpttfield('right') -
                      self._mpttfield('left') + 1)
        target_right = self._mpttfield('right')
        tree_id = self._mpttfield('tree_id')
        self._tree_manager._close_gap(tree_width, target_right, tree_id)
        super(MPTTModel, self).delete(*args, **kwargs)

<commit_msg>for backward compatibility with existing libraries, copy the attributes from _mptt_meta into _meta. To be removed in 0.5 - all new code should use _mptt_meta<commit_after>from django.db import models
from django.db.models import signals as model_signals
from django.db.models.base import ModelBase

from mptt import signals
from mptt.managers import TreeManager

class MPTTOptions(object):
    """
    Options class for MPTT models. Use this as an inner class called MPTTMeta:

    class MyModel(MPTTModel):
        class MPTTMeta:
            order_insertion_by = ['name']
            parent_attr = 'myparent'
        ...
    """

    order_insertion_by = []
    tree_manager_attr = 'tree'
    left_attr = 'lft'
    right_attr = 'rght'
    tree_id_attr = 'tree_id'
    level_attr = 'level'
    parent_attr = 'parent'

    def __init__(self, opts):
        # Override defaults with options provided
        if opts:
            for key, value in opts.__dict__.iteritems():
                setattr(self, key, value)

        # Normalize order_insertion_by to a list
        if isinstance(self.order_insertion_by, basestring):
            self.order_insertion_by = [self.order_insertion_by]
        elif self.order_insertion_by is None:
            self.order_insertion_by = []

    def __iter__(self):
        return iter([(k, v) for (k, v) in self.__dict__.items() if not k.startswith('_')])

    # Helper methods for accessing tree attributes on models.
    def get_raw_field_value(self, instance, field_name):
        """
        Gets the value of the given fieldname for the instance.
        This is not the same as getattr().
        This function will return IDs for foreignkeys etc, rather than doing
        a database query.
        """
        field = instance._meta.get_field(field_name)
        return field.value_from_object(instance)

    def set_raw_field_value(self, instance, field_name, value):
        """
        Sets the value of the given fieldname for the instance.
        This is not the same as setattr().
        This function requires an ID for a foreignkey (etc) rather than an instance.
        """
        field = instance._meta.get_field(field_name)
        setattr(instance, field.attname, value)

class MPTTModelBase(ModelBase):
    """
    Metaclass for MPTT models
    """

    def __new__(meta, class_name, bases, class_dict):
        """
        Create subclasses of MPTTModel. This:
         - adds the MPTT fields to the class
         - adds a TreeManager to the model
        """
        mptt_opts = class_dict.pop('MPTTMeta', None)
        class_dict['_mptt_meta'] = MPTTOptions(mptt_opts)
        cls = ModelBase.__new__(meta, class_name, bases, class_dict)

        # For backwards compatibility with existing libraries, we copy the
        # _mptt_meta options into _meta.
        # This will be removed in 0.5.
        # All new code should use _mptt_meta rather than _meta for tree attributes.
        for attr in ('left_attr', 'right_attr', 'tree_id_attr', 'level_attr', 'parent_attr',
                    'tree_manager_attr', 'order_insertion_by'):
            setattr(cls._meta, attr, getattr(cls._mptt_meta, attr))

        try:
            MPTTModel
        except NameError:
            # We're defining the base class right now, so don't do anything
            # We only want to add this stuff to the subclasses.
            # (Otherwise if field names are customized, we'll end up adding two
            # copies)
            pass
        else:
            for key in ('left_attr', 'right_attr', 'tree_id_attr', 'level_attr'):
                field_name = getattr(cls._mptt_meta, key)
                try:
                    cls._meta.get_field(field_name)
                except models.FieldDoesNotExist:
                    field = models.PositiveIntegerField(db_index=True, editable=False)
                    field.contribute_to_class(cls, field_name)

            # Add a custom tree manager
            manager = TreeManager(cls._mptt_meta)
            manager.contribute_to_class(cls, cls._mptt_meta.tree_manager_attr)
            setattr(cls, '_tree_manager', getattr(cls, cls._mptt_meta.tree_manager_attr))

            # Set up signal receivers
            model_signals.post_init.connect(signals.post_init, sender=cls)
            model_signals.pre_save.connect(signals.pre_save, sender=cls)

        return cls

class MPTTModel(models.Model):
    """
    Base class for tree models.
    """

    __metaclass__ = MPTTModelBase

    class Meta:
        abstract = True

    def _mpttfield(self, fieldname):
        translated_fieldname = getattr(self._mptt_meta, '%s_attr' % fieldname)
        return getattr(self, translated_fieldname)

    def get_ancestors(self, ascending=False):
        """
        Creates a ``QuerySet`` containing the ancestors of this model
        instance.

        This defaults to being in descending order (root ancestor first,
        immediate parent last); passing ``True`` for the ``ascending``
        argument will reverse the ordering (immediate parent first, root
        ancestor last).
        """
        if self.is_root_node():
            return self._tree_manager.none()

        opts = self._mptt_meta

        order_by = opts.left_attr
        if ascending:
            order_by = '-%s' % order_by

        qs = self._tree_manager._mptt_filter(
            left__lt=self._mpttfield('left'),
            right__gt=self._mpttfield('right'),
            tree_id=self._mpttfield('tree_id'),
        )
        return qs.order_by(order_by)

    def get_children(self):
        """
        Returns a ``QuerySet`` containing the immediate children of this
        model instance, in tree order.

        The benefit of using this method over the reverse relation
        provided by the ORM to the instance's children is that a
        database query can be avoided in the case where the instance is
        a leaf node (it has no children).

        If called from a template where the tree has been walked by the
        ``cache_tree_children`` filter, no database query is required.
        """

        if hasattr(self, '_cached_children'):
            return self._cached_children
        else:
            if self.is_leaf_node():
                return self._tree_manager.none()

            return self._tree_manager._mptt_filter(parent=self)

    def get_descendants(self, include_self=False):
        """
        Creates a ``QuerySet`` containing descendants of this model
        instance, in tree order.

        If ``include_self`` is ``True``, the ``QuerySet`` will also
        include this model instance.
        """
        if not include_self and self.is_leaf_node():
            return self._tree_manager.none()

        opts = self._mptt_meta
        left = getattr(self, opts.left_attr)
        right = getattr(self, opts.right_attr)

        if not include_self:
            left += 1
            right -= 1

        return self._tree_manager._mptt_filter(
            tree_id=self._mpttfield('tree_id'),
            left__gte=left,
            left__lte=right
        )

    def get_descendant_count(self):
        """
        Returns the number of descendants this model instance has.
        """
        return (self._mpttfield('right') -
                self._mpttfield('left') - 1) / 2

    def get_leafnodes(self, include_self=False):
        """
        Creates a ``QuerySet`` containing leafnodes of this model
        instance, in tree order.

        If ``include_self`` is ``True``, the ``QuerySet`` will also
        include this model instance (if it is a leaf node)
        """
        descendants = self.get_descendants(include_self=include_self)

        return self._tree_manager._mptt_filter(descendants,
            left=models.F(self._mptt_meta.right_attr)-1
        )

    def get_next_sibling(self, **filters):
        """
        Returns this model instance's next sibling in the tree, or
        ``None`` if it doesn't have a next sibling.
        """
        qs = self._tree_manager.filter(**filters)
        if self.is_root_node():
            qs = self._tree_manager._mptt_filter(qs,
                parent__isnull=True,
                tree_id__gt=self._mpttfield('tree_id'),
            )
        else:
            qs = self._tree_manager._mptt_filter(qs,
                parent__id=getattr(self, '%s_id' % self._mptt_meta.parent_attr),
                left__gt=self._mpttfield('right'),
            )

        siblings = qs[:1]
        return siblings[0] if siblings else None

    def get_previous_sibling(self, **filters):
        """
        Returns this model instance's previous sibling in the tree, or
        ``None`` if it doesn't have a previous sibling.
        """
        opts = self._mptt_meta
        qs = self._tree_manager.filter(**filters)
        if self.is_root_node():
            qs = self._tree_manager._mptt_filter(qs,
                parent__isnull=True,
                tree_id__lt=self._mpttfield('tree_id'),
            )
            qs = qs.order_by('-%s' % opts.tree_id_attr)
        else:
            qs = self._tree_manager._mptt_filter(qs,
                parent__id=getattr(self, '%s_id' % opts.parent_attr),
                right__lt=self._mpttfield('left'),
            )
            qs = qs.order_by('-%s' % opts.right_attr)

        siblings = qs[:1]
        return siblings[0] if siblings else None

    def get_root(self):
        """
        Returns the root node of this model instance's tree.
        """
        if self.is_root_node():
            return self

        return self._tree_manager._mptt_filter(
            tree_id=self._mpttfield('tree_id'),
            parent__isnull=True
        ).get()

    def get_siblings(self, include_self=False):
        """
        Creates a ``QuerySet`` containing siblings of this model
        instance. Root nodes are considered to be siblings of other root
        nodes.

        If ``include_self`` is ``True``, the ``QuerySet`` will also
        include this model instance.
        """
        if self.is_root_node():
            queryset = self._tree_manager._mptt_filter(parent__isnull=True)
        else:
            parent_id = getattr(self, '%s_id' % self._mptt_meta.parent_attr)
            queryset = self._tree_manager._mptt_filter(parent__id=parent_id)
        if not include_self:
            queryset = queryset.exclude(pk=self.pk)
        return queryset

    def get_level(self):
        """
        Returns the level of this node (distance from root)
        """
        return getattr(self, self._mptt_meta.level_attr)

    def insert_at(self, target, position='first-child', save=False):
        """
        Convenience method for calling ``TreeManager.insert_node`` with this
        model instance.
        """
        self._tree_manager.insert_node(self, target, position, save)

    def is_child_node(self):
        """
        Returns ``True`` if this model instance is a child node, ``False``
        otherwise.
        """
        return not self.is_root_node()

    def is_leaf_node(self):
        """
        Returns ``True`` if this model instance is a leaf node (it has no
        children), ``False`` otherwise.
        """
        return not self.get_descendant_count()

    def is_root_node(self):
        """
        Returns ``True`` if this model instance is a root node,
        ``False`` otherwise.
        """
        return getattr(self, '%s_id' % self._mptt_meta.parent_attr) is None

    def move_to(self, target, position='first-child'):
        """
        Convenience method for calling ``TreeManager.move_node`` with this
        model instance.
        """
        self._tree_manager.move_node(self, target, position)

    def delete(self, *args, **kwargs):
        tree_width = (self._mpttfield('right') -
                      self._mpttfield('left') + 1)
        target_right = self._mpttfield('right')
        tree_id = self._mpttfield('tree_id')
        self._tree_manager._close_gap(tree_width, target_right, tree_id)
        super(MPTTModel, self).delete(*args, **kwargs)

<|endoftext|>