<commit_before>import datetime
import asyncio
import re
import os
import requests
import json
import uuid
import random
import calendar
import time
import libs.TelepotObjects as TelepotObjects
import libs.SerializableDict as SerializableDict
import libs.StorageObjects as StorageObjects
import libs.Loggiz as Loggiz
import libs.Models as Models
import telepot.helper
'''
Copyright (c) 2016, Egil Hasting
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:

1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
'''
__author__ = "Egil Hasting"
__copyright__ = "Copyright 2016"
__credits__ = ["Egil Hasting"]
__license__ = "BSD"
__version__ = "1.0.0"
__maintainer__ = "Egil Hasting"
__email__ = "egil.hasting@higen.org"
__status__ = "Production"

class GeneralMessageEvent(object):
    def __init__(self, keyword, bot, dbobject, messageobject):
        self.keyword = keyword
        self.dbobject = dbobject
        self.messageobject = messageobject
        self.bot = bot
        self.isvalid = True
        self._validate()

    def _validate(self):
        if self.keyword is None:
            self.isvalid = False
        if not isinstance(self.messageobject, TelepotObjects.MessageObject):
            self.isvalid = False
        if not isinstance(self.dbobject, Models.StaticModels):
            self.isvalid = False
        if not isinstance(self.bot, telepot.telepot.helper.Sender):
            self.isvalid = False
        if not self.isvalid:
            Loggiz.L.err("GeneralMessageEvent() - Object did not validate")

    @asyncio.coroutine
    def run(self):
        raise NotImplementedError()

class WebSearchDuckDuckGo(GeneralMessageEvent):
    def __init__(self, keyword, bot, dbobject, messageobject):
        GeneralMessageEvent.__init__(self, keyword, bot, dbobject, messageobject)

    def _generate_url(self, searchstring):
        searchstring = searchstring.replace(" ", "+")
        print(searchstring)
        return "http://api.duckduckgo.com/?q={}&format=json".format(searchstring)

    @asyncio.coroutine
    def run(self):
        found = None
        m = re.search('^({})\s(.*)[$\s.]+$'.format(self.keyword), self.messageobject.text)
        if m is not None:
            found = m.groups()
        else:
            m = re.search('^({})\s(.*)$'.format(self.keyword), self.messageobject.text)
            if m is not None:
                found = m.groups()
        if found is not None:
            r = requests.get(self._generate_url(found[1]))
            if r.status_code == 200:
                searchresult = r.json()
                resultcount = len(searchresult["RelatedTopics"])
                outputstring = "{} (found: {})\n".format(searchresult["Heading"], resultcount)
                limitcounter = 0
                for article in searchresult["RelatedTopics"]:
                    outputstring += article.get("Result", "") + "\n"
                    d = article.get("Result", "")
                    if d == "":
                        print(article)
                    limitcounter += 1
                    if limitcounter == 3:
                        break
                yield from self.bot.sendMessage("{}".format(outputstring), parse_mode="HTML")
        else:
            Loggiz.L.err("Missing search term")

class Time(GeneralMessageEvent):
    def __init__(self, keyword, bot, dbobject, messageobject):
        GeneralMessageEvent.__init__(self, keyword, bot, dbobject, messageobject)

    @asyncio.coroutine
    def run(self):
        currenttz = time.tzname
        out = "Current Time\n"
        out += "Norway: " + str(time.strftime('%X %x %Z')) + "\n"
        os.environ['TZ'] = 'US/Eastern'
        time.tzset()
        out += "Baltimore: " + str(time.strftime('%X %x %Z')) + "\n"
        Loggiz.L.info(out)
        yield from self.bot.sendMessage("{}".format(out), parse_mode="HTML")

class Stats(GeneralMessageEvent):
    def __init__(self, keyword, bot, dbobject, messageobject):
        GeneralMessageEvent.__init__(self, keyword, bot, dbobject, messageobject)
        self.seen = self.dbobject.Get("seenlog")
        self.uindex = self.dbobject.Get("userindex")

    @asyncio.coroutine
    def run(self):
        users = self.seen.usercounter.Get()
        data = users.rawdict()
        output_string = "Most Active User Stats (by words):\n\n"
        place = 1
        for key, user in sorted(data, key=self.sort_by_word):
            username = key
            if username == "":
                continue
            usercountobject = SerializableDict.UserObject(user)
            output_string += "[{}] {}: {} (Lines: {})\n".format(place, username, usercountobject.wordcounter, usercountobject.counter)
            place += 1
        Loggiz.L.info(output_string)
        yield from self.bot.sendMessage("{}".format(output_string), parse_mode="HTML")

    def sort_by_word(self, userdict):
        usercountobject = SerializableDict.UserObject(userdict[1])
        if not isinstance(usercountobject.wordcounter, int):
            return 1
        Loggiz.L.info(usercountobject.wordcounter)
        return usercountobject.wordcounter

class Help(GeneralMessageEvent):
    def __init__(self, keyword, bot, dbobject, messageobject):
        GeneralMessageEvent.__init__(self, keyword, bot, dbobject, messageobject)

    @asyncio.coroutine
    def run(self, commands):
        output_string = "Available commands\n"
        output_string += commands
        yield from self.bot.sendMessage("{}".format(output_string), parse_mode="HTML")

    @classmethod
    def sort_by_word(cls, userdict):
        usercountobject = SerializableDict.UserObject(userdict)
        if usercountobject.wordcounter == "":
            return 0
        return usercountobject.wordcounter

class Counter(GeneralMessageEvent):
    def __init__(self, keyword, bot, dbobject, messageobject):
        GeneralMessageEvent.__init__(self, keyword, bot, dbobject, messageobject)
        self.seen = self.dbobject.Get("seenlog")

    @asyncio.coroutine
    def run(self):
        user = self.seen.usercounter.Get()
        usercount = user.get(self.messageobject.mfrom.username)
        usercountobject = SerializableDict.UserObject(usercount)

        # Line counter
        if usercountobject.counter == "":
            usercountobject.counter = 1
        else:
            usercountobject.counter = usercountobject.counter + 1
        # Word counter
        currentwordcount = re.findall('\w+', self.messageobject.text.lower())
        print("Words: {}".format(len(currentwordcount)))
        if usercountobject.wordcounter == "":
            usercountobject.wordcounter = len(currentwordcount)
        else:
            usercountobject.wordcounter = usercountobject.wordcounter + len(currentwordcount)
        # Last seen
        usercountobject.modified = str(datetime.datetime.now().replace(microsecond=0))
        # Metadata
        usercountobject.firstname = self.messageobject.mfrom.first_name
        usercountobject.lastname = self.messageobject.mfrom.last_name
        usercountobject.username = self.messageobject.mfrom.username
        # Store object to dictionary and back to DB
        user.set(self.messageobject.mfrom.username, usercountobject.SaveObject())
        self.seen.usercounter.Set(user)

class Seen(GeneralMessageEvent):
    def __init__(self, keyword, bot, dbobject, messageobject):
        GeneralMessageEvent.__init__(self, keyword, bot, dbobject, messageobject)
        self.seendb = self.dbobject.Get("seenlog")

    @asyncio.coroutine
    def run(self):
        Loggiz.L.info("Gettings Stats")
        Loggiz.L.Print(self.messageobject.text)
        user = self.seendb.usercounter.Get()
        found = None
        m = re.search('^({})\s(.*)[$\s.]+$'.format(self.keyword), self.messageobject.text)
        if m is not None:
            found = m.groups()
        else:
            m = re.search('^({})\s(.*)$'.format(self.keyword), self.messageobject.text)
            if m is not None:
                found = m.groups()
        if found is not None:
            username = found[1]
            if username.startswith("@"):
                # remove @
                username = username[1:]

            fetchseenuser = user.get(username)
            userseenobject = SerializableDict.UserObject(fetchseenuser)
            if userseenobject.modified != "":
                yield from self.bot.sendMessage("hey! {} was last seen {} (lines/words: {}/{})".format(username, userseenobject.modified, userseenobject.counter, userseenobject.wordcounter))
            else:
                Loggiz.L.Print("Did not find any user!")

class QuoteBase(GeneralMessageEvent):
    def __init__(self, keyword, bot, dbobject, messageobject):
        GeneralMessageEvent.__init__(self, keyword, bot, dbobject, messageobject)
        self.uindex = self.dbobject.Get("userindex")

class AddQuote(QuoteBase):
    def __init__(self, keyword, bot, dbobject, messageobject):
        QuoteBase.__init__(self, keyword, bot, dbobject, messageobject)

    @asyncio.coroutine
    def run(self):
        new_quote_index = str(uuid.uuid4())

        m = re.search('^({})\s+(.*): (.*)$'.format(self.keyword), self.messageobject.text)
        if m is not None:
            found = m.groups()
            if len(found) != 3:
                yield from self.bot.sendMessage('[USAGE] !addquote <username>: <quote>')
            if found[1] not in self.uindex.index.Get():
                tmplist = self.uindex.index.Get()
                tmplist.append(found[1])
                self.uindex.index.Set(tmplist)
                Loggiz.L.info("user/nick added to index")
            quotetext = StorageObjects.ComnodeObject("quotestext.{}".format(new_quote_index), "str", desc="", hidden=False)
            quotetext.Set(found[2])

            quotemetausername = StorageObjects.ComnodeObject("quotemap.{}".format(found[1]), "list", desc="", hidden=False)
            qmun = quotemetausername.Get()
            qmun.append(new_quote_index)
            quotemetausername.Set(qmun)
            yield from self.bot.sendMessage("Quote from {} added with id {}".format(found[1], new_quote_index))
        else:
            yield from self.bot.sendMessage('[USAGE] !addquote <username>: <quote>')

class Quote(QuoteBase):
    def __init__(self, keyword, bot, dbobject, messageobject):
        QuoteBase.__init__(self, keyword, bot, dbobject, messageobject)

    def get_quote(self, username):
        quotemetausername = StorageObjects.ComnodeObject("quotemap.{}".format(username), "list", desc="", hidden=False)
        qmun = quotemetausername.Get()
        if len(qmun) > 0:
            random.seed(calendar.timegm(time.gmtime()))
            foundindex = random.randrange(0, len(qmun))
            Loggiz.L.info("found: {}, total: {}".format(foundindex, len(qmun)))
            if len(qmun) == foundindex:
                foundindex = foundindex - 1
            quotetext = StorageObjects.ComnodeObject("quotestext.{}".format(qmun[foundindex]), "str", desc="", hidden=False)
            return "{}: {}".format(username, quotetext.Get())
        else:
            return None

    @asyncio.coroutine
    def run(self):
        quoteoutput = None
        found = None
        qsentence = re.search('^.*({})\s(.*)[$\s.]+$'.format(self.keyword), self.messageobject.text)
        qdirect = re.search('.*({})\s(.*)$'.format(self.keyword), self.messageobject.text)
        qsingle = re.search('^({})$'.format(self.keyword), self.messageobject.text)
        if qsentence is not None:
            found = qsentence.groups()
            Loggiz.L.info("quote found in sentence")
            quoteoutput = self.get_quote(found[1])
        elif qdirect is not None:
            found = qdirect.groups()
            Loggiz.L.info("quote found direct")
            quoteoutput = self.get_quote(found[1])
        elif qsingle is not None:
            found = qsingle.groups()
            Loggiz.L.info("quote found single")
            userindexlength = len(self.uindex.index.Get())
            if userindexlength == 0:
                return
            random.seed(calendar.timegm(time.gmtime()))
            luckyuser = random.randrange(0, userindexlength)
            if len(self.uindex.index.Get()) == luckyuser:
                luckyuser = luckyuser - 1
            quoteoutput = self.get_quote(self.uindex.index.Get()[luckyuser])
        if found is not None:
            if quoteoutput is not None:
                yield from self.bot.sendMessage(quoteoutput)

if __name__ == '__main__':
    pass<commit_msg>time zone feat<commit_after>import datetime
import asyncio
import re
import os
import requests
import json
import uuid
import random
import calendar
import time
import libs.TelepotObjects as TelepotObjects
import libs.SerializableDict as SerializableDict
import libs.StorageObjects as StorageObjects
import libs.Loggiz as Loggiz
import libs.Models as Models
import telepot.helper
'''
Copyright (c) 2016, Egil Hasting
All rights reserved.
Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:

1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation and/or
other materials provided with the distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
'''
__author__ = "Egil Hasting"
__copyright__ = "Copyright 2016"
__credits__ = ["Egil Hasting"]
__license__ = "BSD"
__version__ = "1.0.0"
__maintainer__ = "Egil Hasting"
__email__ = "egil.hasting@higen.org"
__status__ = "Production"

class GeneralMessageEvent(object):
    def __init__(self, keyword, bot, dbobject, messageobject):
        self.keyword = keyword
        self.dbobject = dbobject
        self.messageobject = messageobject
        self.bot = bot
        self.isvalid = True
        self._validate()

    def _validate(self):
        if self.keyword is None:
            self.isvalid = False
        if not isinstance(self.messageobject, TelepotObjects.MessageObject):
            self.isvalid = False
        if not isinstance(self.dbobject, Models.StaticModels):
            self.isvalid = False
        if not isinstance(self.bot, telepot.telepot.helper.Sender):
            self.isvalid = False
        if not self.isvalid:
            Loggiz.L.err("GeneralMessageEvent() - Object did not validate")

    @asyncio.coroutine
    def run(self):
        raise NotImplementedError()

class WebSearchDuckDuckGo(GeneralMessageEvent):
    def __init__(self, keyword, bot, dbobject, messageobject):
        GeneralMessageEvent.__init__(self, keyword, bot, dbobject, messageobject)

    def _generate_url(self, searchstring):
        searchstring = searchstring.replace(" ", "+")
        print(searchstring)
        return "http://api.duckduckgo.com/?q={}&format=json".format(searchstring)

    @asyncio.coroutine
    def run(self):
        found = None
        m = re.search('^({})\s(.*)[$\s.]+$'.format(self.keyword), self.messageobject.text)
        if m is not None:
            found = m.groups()
        else:
            m = re.search('^({})\s(.*)$'.format(self.keyword), self.messageobject.text)
            if m is not None:
                found = m.groups()
        if found is not None:
            r = requests.get(self._generate_url(found[1]))
            if r.status_code == 200:
                searchresult = r.json()
                resultcount = len(searchresult["RelatedTopics"])
                outputstring = "{} (found: {})\n".format(searchresult["Heading"], resultcount)
                limitcounter = 0
                for article in searchresult["RelatedTopics"]:
                    outputstring += article.get("Result", "") + "\n"
                    d = article.get("Result", "")
                    if d == "":
                        print(article)
                    limitcounter += 1
                    if limitcounter == 3:
                        break
                yield from self.bot.sendMessage("{}".format(outputstring), parse_mode="HTML")
        else:
            Loggiz.L.err("Missing search term")

class Time(GeneralMessageEvent):
    def __init__(self, keyword, bot, dbobject, messageobject):
        GeneralMessageEvent.__init__(self, keyword, bot, dbobject, messageobject)

    @asyncio.coroutine
    def run(self):
        currenttz = time.tzname
        out = "Current Time\n"
        out += "Norway: " + str(time.strftime('%X %x %Z')) + "\n"
        os.environ['TZ'] = 'US/Eastern'
        time.tzset()
        out += "Baltimore: " + str(time.strftime('%X %x %Z')) + "\n"
        Loggiz.L.info(out)
        yield from self.bot.sendMessage("{}".format(out), parse_mode="HTML")

class Stats(GeneralMessageEvent):
    def __init__(self, keyword, bot, dbobject, messageobject):
        GeneralMessageEvent.__init__(self, keyword, bot, dbobject, messageobject)
        self.seen = self.dbobject.Get("seenlog")
        self.uindex = self.dbobject.Get("userindex")

    @asyncio.coroutine
    def run(self):
        users = self.seen.usercounter.Get()
        data = users.rawdict()
        output_string = "Most Active User Stats (by words):\n\n"
        place = 1
        for key, user in sorted(data, key=self.sort_by_word, reverse=True):
            username = key
            if username == "":
                continue
            usercountobject = SerializableDict.UserObject(user)
            output_string += "[{}] {}: {} (Lines: {})\n".format(place, username, usercountobject.wordcounter, usercountobject.counter)
            place += 1
        Loggiz.L.info(output_string)
        yield from self.bot.sendMessage("{}".format(output_string), parse_mode="HTML")

    def sort_by_word(self, userdict):
        usercountobject = SerializableDict.UserObject(userdict[1])
        if not isinstance(usercountobject.wordcounter, int):
            return 1
        Loggiz.L.info(usercountobject.wordcounter)
        return usercountobject.wordcounter

class Help(GeneralMessageEvent):
    def __init__(self, keyword, bot, dbobject, messageobject):
        GeneralMessageEvent.__init__(self, keyword, bot, dbobject, messageobject)

    @asyncio.coroutine
    def run(self, commands):
        output_string = "Available commands\n"
        output_string += commands
        yield from self.bot.sendMessage("{}".format(output_string), parse_mode="HTML")

    @classmethod
    def sort_by_word(cls, userdict):
        usercountobject = SerializableDict.UserObject(userdict)
        if usercountobject.wordcounter == "":
            return 0
        return usercountobject.wordcounter

class Counter(GeneralMessageEvent):
    def __init__(self, keyword, bot, dbobject, messageobject):
        GeneralMessageEvent.__init__(self, keyword, bot, dbobject, messageobject)
        self.seen = self.dbobject.Get("seenlog")

    @asyncio.coroutine
    def run(self):
        user = self.seen.usercounter.Get()
        usercount = user.get(self.messageobject.mfrom.username)
        usercountobject = SerializableDict.UserObject(usercount)

        # Line counter
        if usercountobject.counter == "":
            usercountobject.counter = 1
        else:
            usercountobject.counter = usercountobject.counter + 1
        # Word counter
        currentwordcount = re.findall('\w+', self.messageobject.text.lower())
        print("Words: {}".format(len(currentwordcount)))
        if usercountobject.wordcounter == "":
            usercountobject.wordcounter = len(currentwordcount)
        else:
            usercountobject.wordcounter = usercountobject.wordcounter + len(currentwordcount)
        # Last seen
        usercountobject.modified = str(datetime.datetime.now().replace(microsecond=0))
        # Metadata
        usercountobject.firstname = self.messageobject.mfrom.first_name
        usercountobject.lastname = self.messageobject.mfrom.last_name
        usercountobject.username = self.messageobject.mfrom.username
        # Store object to dictionary and back to DB
        user.set(self.messageobject.mfrom.username, usercountobject.SaveObject())
        self.seen.usercounter.Set(user)

class Seen(GeneralMessageEvent):
    def __init__(self, keyword, bot, dbobject, messageobject):
        GeneralMessageEvent.__init__(self, keyword, bot, dbobject, messageobject)
        self.seendb = self.dbobject.Get("seenlog")

    @asyncio.coroutine
    def run(self):
        Loggiz.L.info("Gettings Stats")
        Loggiz.L.Print(self.messageobject.text)
        user = self.seendb.usercounter.Get()
        found = None
        m = re.search('^({})\s(.*)[$\s.]+$'.format(self.keyword), self.messageobject.text)
        if m is not None:
            found = m.groups()
        else:
            m = re.search('^({})\s(.*)$'.format(self.keyword), self.messageobject.text)
            if m is not None:
                found = m.groups()
        if found is not None:
            username = found[1]
            if username.startswith("@"):
                # remove @
                username = username[1:]

            fetchseenuser = user.get(username)
            userseenobject = SerializableDict.UserObject(fetchseenuser)
            if userseenobject.modified != "":
                yield from self.bot.sendMessage("hey! {} was last seen {} (lines/words: {}/{})".format(username, userseenobject.modified, userseenobject.counter, userseenobject.wordcounter))
            else:
                Loggiz.L.Print("Did not find any user!")

class QuoteBase(GeneralMessageEvent):
    def __init__(self, keyword, bot, dbobject, messageobject):
        GeneralMessageEvent.__init__(self, keyword, bot, dbobject, messageobject)
        self.uindex = self.dbobject.Get("userindex")

class AddQuote(QuoteBase):
    def __init__(self, keyword, bot, dbobject, messageobject):
        QuoteBase.__init__(self, keyword, bot, dbobject, messageobject)

    @asyncio.coroutine
    def run(self):
        new_quote_index = str(uuid.uuid4())

        m = re.search('^({})\s+(.*): (.*)$'.format(self.keyword), self.messageobject.text)
        if m is not None:
            found = m.groups()
            if len(found) != 3:
                yield from self.bot.sendMessage('[USAGE] !addquote <username>: <quote>')
            if found[1] not in self.uindex.index.Get():
                tmplist = self.uindex.index.Get()
                tmplist.append(found[1])
                self.uindex.index.Set(tmplist)
                Loggiz.L.info("user/nick added to index")
            quotetext = StorageObjects.ComnodeObject("quotestext.{}".format(new_quote_index), "str", desc="", hidden=False)
            quotetext.Set(found[2])

            quotemetausername = StorageObjects.ComnodeObject("quotemap.{}".format(found[1]), "list", desc="", hidden=False)
            qmun = quotemetausername.Get()
            qmun.append(new_quote_index)
            quotemetausername.Set(qmun)
            yield from self.bot.sendMessage("Quote from {} added with id {}".format(found[1], new_quote_index))
        else:
            yield from self.bot.sendMessage('[USAGE] !addquote <username>: <quote>')

class Quote(QuoteBase):
    def __init__(self, keyword, bot, dbobject, messageobject):
        QuoteBase.__init__(self, keyword, bot, dbobject, messageobject)

    def get_quote(self, username):
        quotemetausername = StorageObjects.ComnodeObject("quotemap.{}".format(username), "list", desc="", hidden=False)
        qmun = quotemetausername.Get()
        if len(qmun) > 0:
            random.seed(calendar.timegm(time.gmtime()))
            foundindex = random.randrange(0, len(qmun))
            Loggiz.L.info("found: {}, total: {}".format(foundindex, len(qmun)))
            if len(qmun) == foundindex:
                foundindex = foundindex - 1
            quotetext = StorageObjects.ComnodeObject("quotestext.{}".format(qmun[foundindex]), "str", desc="", hidden=False)
            return "{}: {}".format(username, quotetext.Get())
        else:
            return None

    @asyncio.coroutine
    def run(self):
        quoteoutput = None
        found = None
        qsentence = re.search('^.*({})\s(.*)[$\s.]+$'.format(self.keyword), self.messageobject.text)
        qdirect = re.search('.*({})\s(.*)$'.format(self.keyword), self.messageobject.text)
        qsingle = re.search('^({})$'.format(self.keyword), self.messageobject.text)
        if qsentence is not None:
            found = qsentence.groups()
            Loggiz.L.info("quote found in sentence")
            quoteoutput = self.get_quote(found[1])
        elif qdirect is not None:
            found = qdirect.groups()
            Loggiz.L.info("quote found direct")
            quoteoutput = self.get_quote(found[1])
        elif qsingle is not None:
            found = qsingle.groups()
            Loggiz.L.info("quote found single")
            userindexlength = len(self.uindex.index.Get())
            if userindexlength == 0:
                return
            random.seed(calendar.timegm(time.gmtime()))
            luckyuser = random.randrange(0, userindexlength)
            if len(self.uindex.index.Get()) == luckyuser:
                luckyuser = luckyuser - 1
            quoteoutput = self.get_quote(self.uindex.index.Get()[luckyuser])
        if found is not None:
            if quoteoutput is not None:
                yield from self.bot.sendMessage(quoteoutput)

if __name__ == '__main__':
    pass<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
rdfalchemy.py - a Simple API for RDF

Requires rdflib <http://www.rdflib.net/> version 2.3 ??.

"""

__license__ = """
Copyright (c) 2005-2007 Philip Cooper <Philip.Cooper@openvest.com>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
__version__ = "0.1dev"

from rdflib import ConjunctiveGraph
from rdflib import Literal, BNode, Namespace, URIRef
from itertools import chain
import re
import logging

console = logging.StreamHandler()
## console.setLevel(logging.DEBUG)    ## <- the debug level goes here
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
console.setFormatter(formatter)
log=logging.getLogger('rdfAlchemy')
##log.setLevel(logging.DEBUG)
log.addHandler(console)

RDF  =Namespace("http://www.w3.org/1999/02/22-rdf-syntax-ns#")
RDFS =Namespace("http://www.w3.org/2000/01/rdf-schema#")
OWL  =Namespace("http://www.w3.org/2002/07/owl#")

re_ns_n = re.compile('(.*[/#])(.*)')

# Look into caching as in:
# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/276643
# Note: Non data descriptors (get only) lookup in obj.__dict__ first
#       Data descriptors (get and set) use the __get__ first

# helper function, might be somewhere in rdflib I need to look for it there
def getList(sub, pred=None, db=None):
    """Attempts to return a list from sub (subject that is)
    passed in if it is a Collection or a Container (Bag,Seq or Alt)"""
    if not db:
        if isinstance(sub,rdfObject):
            db=sub.db
        else:
            db=rdfObject.db
    if pred:
        if isinstance(sub, rdfObject):
            subUri=sub.resUri
        base = db.value(subUri,pred,any=True)
    else:
        # if there was no predicate assume a base node was passed in
        base=sub
    if type(base) != BNode:
        # Doesn't look like a list or a collection, just return multiple values (or an error?)
        val=[o for o in db.objects(subUri, pred)]
        return val
    members=[]
    first = db.value(base, RDF.first)
    # OK let's work at returning a list if there is an RDF.first
    if first:
        while first:
            members.append(first)
            base = db.value(base, RDF.rest)
            first = db.value(base, RDF.first)
        return members
    # OK let's work at returning a Collection (Seq,Bag or Alt) if was no RDF.first
    else:
        i=1
        first=db.value(base, RDF._1)
        if not first:
            raise AttributeError, "Not a list, or collection but another type of BNode"
        while first:
            members.append(first)
            i += 1
            first=db.value(base, RDF['_%d'%i])
        return members

class rdflibAbstract(object):
    """Abstract base class for descriptors
    Descriptors are to map class instance variables to predicates
    optional cacheName is where to store items
    range_type is the rdf:type of the range of this predicate"""
    def __init__(self, pred, cacheName=None, range_type=None):
        self.pred = pred
        self.name = cacheName or pred
        self.range_type = range_type

    @property
    def range_class(self):
        """return the class that this descriptor is mapped to through the range_type"""
        if self.range_type:
            try:
                return self._mappedClass
            except AttributeError:
                log.warn("Descriptor %s has range of: %s but not yet mapped"%(self, self.range_type))
                return rdfObject
        else:
            return rdfObject

class rdflibSingle(rdflibAbstract):
    '''This is a Discriptor
    Takes a the URI of the predicate at initialization
    Expects to return a single item
    on Assignment will set that value to the
    ONLY triple with that subject,predicate pair'''
    def __init__(self, pred, cacheName=None, range_type=None):
        super(rdflibSingle, self).__init__(pred, cacheName, range_type)

    def __get__(self, obj, cls):
        if obj is None:
            return self
        if self.name in obj.__dict__:
            return obj.__dict__[self.name]
        log.debug("Geting with descriptor %s for %s"%(self.pred,obj.resUri))
        val=obj.__getitem__(self.pred)
        if isinstance(val, BNode) or isinstance(val,URIRef):
            val = self.range_class(val)
        obj.__dict__[self.name]= val
        return val

    def __set__(self, obj, value):
        log.debug("SET with descriptor value %s of type %s"%(value,type(value)))
        #setattr(obj, self.name, value)  #this recurses indefinatly
        obj.__dict__[self.name]= value
        if isinstance(value,Literal):
            o = value
        elif isinstance(value,str) or isinstance(value,unicode):
            o = Literal(value,)
        elif isinstance(value,int) or isinstance(value,float):
            o = Literal(str(value),)
        else:
            raise NotImplimented
        obj.db.set((obj.resUri,self.pred, o))
        #return None

    def __delete__(self, obj):
        # if this is a bnode like a list or a container a lot more should
        # be done ala getList above
        log.debug("DELETE with descriptor for %s on %s"%(self.pred, obj.resUri))
        return obj.db.remove((obj.resUri,self.pred, None))

class rdflibMultiple(rdflibAbstract):
    '''This is a Discriptor
       Expects to return a list of values (could be a list of one)'''
    def __init__(self, pred, cacheName=None, range_type=None):
        super(rdflibMultiple, self).__init__(pred, cacheName=None, range_type=None)

    def __get__(self, obj, cls):
        if obj is None:
            return self
        val=[o for o in obj.db.objects(obj.resUri, self.pred)]
        # check to see if this is a Container or Collection
        # if so, return collection as a list
        if len(val) == 1 \
           and (obj.db.value(o,RDF.first) or obj.db.value(o,RDF._1)):
                  val=getList(obj, self.pred)
        val=[((isinstance(v,BNode) or isinstance(v,URIRef)) and self.range_class(v) or v) for v in val]
        setattr(obj, self.name, val)
        try:
            log.info("Geting %s for %s"%(obj.db.qname(self.pred),obj.db.qname(obj.resUri)))
        except:
            log.info("Geting %s for %s"%(self.pred,obj.resUri))
        return val

class rdfObject(object):
    db=ConjunctiveGraph()
    """Default graph for access to instances of this type"""
    rdf_type=None
    """rdf:type of instances of this class"""
    def __init__(self, resUri):
        if isinstance(resUri, rdfObject):
            self.resUri=resUri.resUri
            self.db=resUri.db
        elif isinstance(resUri, BNode) or isinstance(resUri, URIRef):
            self.resUri=resUri
        elif resUri[0]=="<" and resUri[-1]==">":
            self.resUri=URIRef(resUri[1:-1])
        elif resUri.startswith("_:"):
            self.resUri=BNode(resUri[2:])
        else:
            raise AttributeError("cannot construct rdfObject from %s"%(str(resUri)))

        rdftype = list(self.db.objects(self.resUri, RDF.type))
        if len(rdftype)==1:
            self.namespace, trash = re_ns_n.match(rdftype[0]).groups()
            self.namespace=Namespace(self.namespace)
        elif isinstance(self.resUri,URIRef):
            ns_n =  re_ns_n.match(self.resUri)
            if ns_n:
                self.namespace, self.name = ns_n.groups()
                self.namespace=Namespace(self.namespace)

    def get_by(cls, **kwargs):
        """Class Method, returns a single instance of the class
        by a single kwarg.  the keyword must be a descriptor of the
        class.
        example:
            bigBlue = Company.get_by(symbol='IBM')

        OWL Note:
            the keyword should map to an rdf predicate
            that is of type owl:InverseFunctional"""
        if len(kwargs) != 1:
            raise ValueError("get_by did not want %i args"%(len(kwargs)))
        key,value = kwargs.items()[0]
        pred=cls.__dict__[key].pred
        uri=cls.db.value(None,pred,Literal(value))
        if uri:
            return cls(uri)
        else:
            raise LookupError("%s = %s not found"%(key,value))
    get_by=classmethod(get_by)
    #short term hack.  Need to go to a sqlalchemy 0.4 style query method
    query_get_by=get_by

    def filter_by(cls, **kwargs):
        """Class method returns a generator over classs instances
        meeting the kwargs conditions.

        Each keyword must be a class descriptor

        filter by RDF.type == cls.rdf_type is implicit

        Order helps, the first keyword should be the most restrictive
        """
        filters = []
        for key,value in kwargs.items():
            try:
                pred=cls.__dict__[key].pred
            except LookupError :
                raise LookupError ("%s not a valid descriptor for %s" % (key,cls))
            # try to make the value be OK for the triple query as an object
            if isinstance(value, rdfObject):
                obj = rdfObject.resUri
            elif isinstance(value, URIRef) or isinstance(value,BNode):
                obj = value
            else:
                obj = Literal(value)
            filters.append((pred,obj))
        # make sure we filter by type
        if not (RDF.type,cls.rdf_type) in filters:
            filters.append((RDF.type,cls.rdf_type))
        pred, obj = filters[0]
        print "Checking %s, %s" % (pred,obj)
        for sub in cls.db.subjects(pred,obj):
            print "maybe %s" % sub
            for pred,obj in filters[1:]:
                print "Checking %s, %s" % (pred,obj)
                if not list(cls.db.triples((sub,pred,obj))):
                    print "Not %s" % s
                    continue
            yield cls(sub)
    filter_by=classmethod(filter_by)

    def ClassInstances(cls):
        """return a generator for instances of this rdf:type
        you can look in MyClass.rdf_type to see the predicate being used"""
        for i in cls.db.subjects(RDF.type, cls.rdf_type):
            yield cls(i)
    ClassInstances=classmethod(ClassInstances)

    def GetRandom(cls):
        """for develoment just returns a random instance of this class"""
        from random import randint
        xii=list(cls.ClassInstances())
        return xii[randint(0,len(xii)-1)]
    GetRandom=classmethod(GetRandom)

    def __repr__(self):
        return "<%s -> %s>"%(self.__class__, self.resUri)

    def __getitem__(self, pred):
        #log.debug("Geting with __getitem__ %s for %s"%(self.db.qname(pred),self.db.qname(self.resUri)))
        log.debug("Geting with __getitem__ %s for %s"%(pred,self.resUri))
        val=self.db.value(self.resUri,pred)
        if isinstance(val,Literal):
            val =  val.datatype and val.toPython() or unicode(val)
        elif isinstance(val, BNode) or isinstance(val,URIRef):
            val=rdfObject(val)
        return val

    def ppo(self,db=None):
        """Like pretty print...
        Return a 'pretty predicate,object' of self
        returning all predicate object pairs with qnames"""
        db = db or self.db
        for p,o in db.predicate_objects(self.resUri):
            print "%20s = %s"% (db.qname(p),str(o))
<commit_msg>Fixed the rang_type really this time<commit_after>#!/usr/bin/env python
"""
rdfalchemy.py - a Simple API for RDF

Requires rdflib <http://www.rdflib.net/> version 2.3 ??.

"""

__license__ = """
Copyright (c) 2005-2007 Philip Cooper <Philip.Cooper@openvest.com>

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""
__version__ = "0.1dev"

from rdflib import ConjunctiveGraph
from rdflib import Literal, BNode, Namespace, URIRef
from itertools import chain
import re
import logging

console = logging.StreamHandler()
## console.setLevel(logging.DEBUG)    ## <- the debug level goes here
formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
console.setFormatter(formatter)
log=logging.getLogger('rdfAlchemy')
##log.setLevel(logging.DEBUG)
log.addHandler(console)

RDF  =Namespace("http://www.w3.org/1999/02/22-rdf-syntax-ns#")
RDFS =Namespace("http://www.w3.org/2000/01/rdf-schema#")
OWL  =Namespace("http://www.w3.org/2002/07/owl#")

re_ns_n = re.compile('(.*[/#])(.*)')

# Look into caching as in:
# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/276643
# Note: Non data descriptors (get only) lookup in obj.__dict__ first
#       Data descriptors (get and set) use the __get__ first

# helper function, might be somewhere in rdflib I need to look for it there
def getList(sub, pred=None, db=None):
    """Attempts to return a list from sub (subject that is)
    passed in if it is a Collection or a Container (Bag,Seq or Alt)"""
    if not db:
        if isinstance(sub,rdfObject):
            db=sub.db
        else:
            db=rdfObject.db
    if pred:
        if isinstance(sub, rdfObject):
            subUri=sub.resUri
        base = db.value(subUri,pred,any=True)
    else:
        # if there was no predicate assume a base node was passed in
        base=sub
    if type(base) != BNode:
        # Doesn't look like a list or a collection, just return multiple values (or an error?)
        val=[o for o in db.objects(subUri, pred)]
        return val
    members=[]
    first = db.value(base, RDF.first)
    # OK let's work at returning a list if there is an RDF.first
    if first:
        while first:
            members.append(first)
            base = db.value(base, RDF.rest)
            first = db.value(base, RDF.first)
        return members
    # OK let's work at returning a Collection (Seq,Bag or Alt) if was no RDF.first
    else:
        i=1
        first=db.value(base, RDF._1)
        if not first:
            raise AttributeError, "Not a list, or collection but another type of BNode"
        while first:
            members.append(first)
            i += 1
            first=db.value(base, RDF['_%d'%i])
        return members

class rdflibAbstract(object):
    """Abstract base class for descriptors
    Descriptors are to map class instance variables to predicates
    optional cacheName is where to store items
    range_type is the rdf:type of the range of this predicate"""
    def __init__(self, pred, cacheName=None, range_type=None):
        self.pred = pred
        self.name = cacheName or pred
        self.range_type = range_type

    @property
    def range_class(self):
        """return the class that this descriptor is mapped to through the range_type"""
        if self.range_type:
            try:
                return self._mappedClass
            except AttributeError:
                log.warn("Descriptor %s has range of: %s but not yet mapped"%(self, self.range_type))
                return rdfObject
        else:
            return rdfObject

class rdflibSingle(rdflibAbstract):
    '''This is a Discriptor
    Takes a the URI of the predicate at initialization
    Expects to return a single item
    on Assignment will set that value to the
    ONLY triple with that subject,predicate pair'''
    def __init__(self, pred, cacheName=None, range_type=None):
        super(rdflibSingle, self).__init__(pred, cacheName, range_type)

    def __get__(self, obj, cls):
        if obj is None:
            return self
        if self.name in obj.__dict__:
            return obj.__dict__[self.name]
        log.debug("Geting with descriptor %s for %s"%(self.pred,obj.resUri))
        val=obj.__getitem__(self.pred)
        if isinstance(val, BNode) or isinstance(val,URIRef):
            val = self.range_class(val)
        obj.__dict__[self.name]= val
        return val

    def __set__(self, obj, value):
        log.debug("SET with descriptor value %s of type %s"%(value,type(value)))
        #setattr(obj, self.name, value)  #this recurses indefinatly
        obj.__dict__[self.name]= value
        if isinstance(value,Literal):
            o = value
        elif isinstance(value,str) or isinstance(value,unicode):
            o = Literal(value,)
        elif isinstance(value,int) or isinstance(value,float):
            o = Literal(str(value),)
        else:
            raise NotImplimented
        obj.db.set((obj.resUri,self.pred, o))
        #return None

    def __delete__(self, obj):
        # if this is a bnode like a list or a container a lot more should
        # be done ala getList above
        log.debug("DELETE with descriptor for %s on %s"%(self.pred, obj.resUri))
        return obj.db.remove((obj.resUri,self.pred, None))

class rdflibMultiple(rdflibAbstract):
    '''This is a Discriptor
       Expects to return a list of values (could be a list of one)'''
    def __init__(self, pred, cacheName=None, range_type=None):
        super(rdflibMultiple, self).__init__(pred, cacheName, range_type)

    def __get__(self, obj, cls):
        if obj is None:
            return self
        val=[o for o in obj.db.objects(obj.resUri, self.pred)]
        # check to see if this is a Container or Collection
        # if so, return collection as a list
        if len(val) == 1 \
           and (obj.db.value(o,RDF.first) or obj.db.value(o,RDF._1)):
                  val=getList(obj, self.pred)
        val=[((isinstance(v,BNode) or isinstance(v,URIRef)) and self.range_class(v) or v) for v in val]
        setattr(obj, self.name, val)
        try:
            log.info("Geting %s for %s"%(obj.db.qname(self.pred),obj.db.qname(obj.resUri)))
        except:
            log.info("Geting %s for %s"%(self.pred,obj.resUri))
        return val

class rdfObject(object):
    db=ConjunctiveGraph()
    """Default graph for access to instances of this type"""
    rdf_type=None
    """rdf:type of instances of this class"""
    def __init__(self, resUri):
        if isinstance(resUri, rdfObject):
            self.resUri=resUri.resUri
            self.db=resUri.db
        elif isinstance(resUri, BNode) or isinstance(resUri, URIRef):
            self.resUri=resUri
        elif resUri[0]=="<" and resUri[-1]==">":
            self.resUri=URIRef(resUri[1:-1])
        elif resUri.startswith("_:"):
            self.resUri=BNode(resUri[2:])
        else:
            raise AttributeError("cannot construct rdfObject from %s"%(str(resUri)))

        rdftype = list(self.db.objects(self.resUri, RDF.type))
        if len(rdftype)==1:
            self.namespace, trash = re_ns_n.match(rdftype[0]).groups()
            self.namespace=Namespace(self.namespace)
        elif isinstance(self.resUri,URIRef):
            ns_n =  re_ns_n.match(self.resUri)
            if ns_n:
                self.namespace, self.name = ns_n.groups()
                self.namespace=Namespace(self.namespace)

    def get_by(cls, **kwargs):
        """Class Method, returns a single instance of the class
        by a single kwarg.  the keyword must be a descriptor of the
        class.
        example:
            bigBlue = Company.get_by(symbol='IBM')

        OWL Note:
            the keyword should map to an rdf predicate
            that is of type owl:InverseFunctional"""
        if len(kwargs) != 1:
            raise ValueError("get_by did not want %i args"%(len(kwargs)))
        key,value = kwargs.items()[0]
        pred=cls.__dict__[key].pred
        uri=cls.db.value(None,pred,Literal(value))
        if uri:
            return cls(uri)
        else:
            raise LookupError("%s = %s not found"%(key,value))
    get_by=classmethod(get_by)
    #short term hack.  Need to go to a sqlalchemy 0.4 style query method
    query_get_by=get_by

    def filter_by(cls, **kwargs):
        """Class method returns a generator over classs instances
        meeting the kwargs conditions.

        Each keyword must be a class descriptor

        filter by RDF.type == cls.rdf_type is implicit

        Order helps, the first keyword should be the most restrictive
        """
        filters = []
        for key,value in kwargs.items():
            try:
                pred=cls.__dict__[key].pred
            except LookupError :
                raise LookupError ("%s not a valid descriptor for %s" % (key,cls))
            # try to make the value be OK for the triple query as an object
            if isinstance(value, rdfObject):
                obj = rdfObject.resUri
            elif isinstance(value, URIRef) or isinstance(value,BNode):
                obj = value
            else:
                obj = Literal(value)
            filters.append((pred,obj))
        # make sure we filter by type
        if not (RDF.type,cls.rdf_type) in filters:
            filters.append((RDF.type,cls.rdf_type))
        pred, obj = filters[0]
        print "Checking %s, %s" % (pred,obj)
        for sub in cls.db.subjects(pred,obj):
            print "maybe %s" % sub
            for pred,obj in filters[1:]:
                print "Checking %s, %s" % (pred,obj)
                if not list(cls.db.triples((sub,pred,obj))):
                    print "Not %s" % s
                    continue
            yield cls(sub)
    filter_by=classmethod(filter_by)

    def ClassInstances(cls):
        """return a generator for instances of this rdf:type
        you can look in MyClass.rdf_type to see the predicate being used"""
        for i in cls.db.subjects(RDF.type, cls.rdf_type):
            yield cls(i)
    ClassInstances=classmethod(ClassInstances)

    def GetRandom(cls):
        """for develoment just returns a random instance of this class"""
        from random import randint
        xii=list(cls.ClassInstances())
        return xii[randint(0,len(xii)-1)]
    GetRandom=classmethod(GetRandom)

    def __repr__(self):
        return "<%s -> %s>"%(self.__class__, self.resUri)

    def __getitem__(self, pred):
        #log.debug("Geting with __getitem__ %s for %s"%(self.db.qname(pred),self.db.qname(self.resUri)))
        log.debug("Geting with __getitem__ %s for %s"%(pred,self.resUri))
        val=self.db.value(self.resUri,pred)
        if isinstance(val,Literal):
            val =  val.datatype and val.toPython() or unicode(val)
        elif isinstance(val, BNode) or isinstance(val,URIRef):
            val=rdfObject(val)
        return val

    def ppo(self,db=None):
        """Like pretty print...
        Return a 'pretty predicate,object' of self
        returning all predicate object pairs with qnames"""
        db = db or self.db
        for p,o in db.predicate_objects(self.resUri):
            print "%20s = %s"% (db.qname(p),str(o))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import json

import re
from urllib import urlencode
import urllib
import urllib2
from urllib2 import HTTPCookieProcessor
from urllib2 import HTTPError
from urllib2 import ProxyHandler
from urllib2 import Request
from urllib2 import build_opener
from bs4 import BeautifulSoup
from datetime import datetime
import MySQLdb

default_header = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0',}

sites = [
     'http://www.pornhub.com/video/search?search=loira',
    # 'http://www.hellxx.com/index.php?module=videos&tag=d',
    # 'http://www.redtube.com/?search=s',
    # 'http://pt.xhamster.com/search.php?from=&new=&q=asasaa&qcat=video',
    # 'http://novinhasdozapzap.com/'
]

content = urllib2.urlopen(sites[0]).read()

soup = BeautifulSoup(content)
conn  = MySQLdb.connect("localhost","root","29367253","tocadasnovinhas" )
for video in soup.find("ul",{"class":"videos"}):
    img = video.find("img")
    time = video.find("var")
    video = video.find("a")
    if type(img) is not int:
        alt = img['alt']
        thumb = img['data-mediumthumb']
        href = video['href']
        tempo = video.contents[0]
        pornolandia = sites[0].split('/')[2]
        siteDetalhes = ''.join([pornolandia,href])
        sock = urllib.urlopen("http://"+siteDetalhes)
        htmlSource = sock.read()
        soup = BeautifulSoup(htmlSource)
        for i in soup.findAll("div",{"class":"video-action-sub-tab"}):
            print i.find("textarea")
        # for detalhes in divDetalhes:
        #     inputs = detalhes.find("input")
        #     embed =  inputs.get('value')
        #
        # x = conn.cursor()
        # try:
        #     x.execute("""INSERT INTO video (name,description,img,embed,category,tempo) VALUES (%s,%s,%s,%s,%s,%s)""",(alt.lower().replace(" ","-"),alt,thumb,divDetalhes,'loiras',tempo))
        #     conn.commit()
        # except:
        #     conn.rollback()
<commit_msg>teste<commit_after># -*- coding: utf-8 -*-
import json

import re
from urllib import urlencode
import urllib
import urllib2
from urllib2 import HTTPCookieProcessor
from urllib2 import HTTPError
from urllib2 import ProxyHandler
from urllib2 import Request
from urllib2 import build_opener
from bs4 import BeautifulSoup
from datetime import datetime
import MySQLdb

default_header = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0',}

sites = [
     'http://www.pornhub.com/video/search?search=loira',
    # 'http://www.hellxx.com/index.php?module=videos&tag=d',
    # 'http://www.redtube.com/?search=s',
    # 'http://pt.xhamster.com/search.php?from=&new=&q=asasaa&qcat=video',
    # 'http://novinhasdozapzap.com/'
]

content = urllib2.urlopen(sites[0]).read()

soup = BeautifulSoup(content)
conn  = MySQLdb.connect("localhost","root","29367253","tocadasnovinhas" )
for video in soup.find("ul",{"class":"videos"}):
    img = video.find("img")
    time = video.find("var")
    video = video.find("a")
    if type(img) is not int:
        alt = img['alt']
        thumb = img['data-mediumthumb']
        href = video['href']
        tempo = video.contents[0]
        print href
        # pornolandia = sites[0].split('/')[2]
        # siteDetalhes = ''.join([pornolandia,href])
        #
        # iframe = '<iframe src="http://www.pornhub.com/embed/'++'" frameborder="0" width="560" height="340" scrolling="no"></iframe>'
        #
        #
        # x = conn.cursor()
        # try:
        #     x.execute("""INSERT INTO video (name,description,img,embed,category,tempo) VALUES (%s,%s,%s,%s,%s,%s)""",(alt.lower().replace(" ","-"),alt,thumb,divDetalhes,'loiras',tempo))
        #     conn.commit()
        # except:
        #     conn.rollback()
<|endoftext|>


<commit_before>registry = {}

def register(model, fields, order='pk', filter=False, results=5):
    registry[str(model)] = (model, fields, results, order, filter)

def search_for_string(search_string):
    search_string = search_string.lower()
    matches = []
    for key in registry:
        model, fields, results, order, filter_by = registry[key]
        # partial application didn't seem sane in python ... so:
        if filter_by:
            if callable(filter_by):
                filter_by = filter_by()
            objects = model.objects.filter(filter_by)
        else:
            objects = model.objects.all()
        counter = 0
        for object in objects.order_by(order):
            for field in fields:
                try:
                    searchee = getattr(object, field)
                except AttributeError:
                    pass
                if callable(searchee):
                    searchee = searchee()
                if search_string in searchee.lower():
                    matches.append(object)
                    counter += 1
                    if counter >= results:
                        break

    return matches
<commit_msg>fixed last commit<commit_after>registry = {}

def register(model, fields, order='pk', filter=False, results=5):
    registry[str(model)] = (model, fields, results, order, filter)

class LoopBreak(Exception): pass

def search_for_string(search_string):
    search_string = search_string.lower()
    matches = []
    for key in registry:
        model, fields, results, order, filter_by = registry[key]
        # partial application didn't seem sane in python ... so:
        if filter_by:
            if callable(filter_by):
                filter_by = filter_by()
            objects = model.objects.filter(filter_by)
        else:
            objects = model.objects.all()
        counter = 0
        try:
            for object in objects.order_by(order):
                for field in fields:
                    try:
                        searchee = getattr(object, field)
                    except AttributeError:
                        pass
                    if callable(searchee):
                        searchee = searchee()
                    if search_string in searchee.lower():
                        matches.append(object)
                        counter += 1
                        if counter >= results:
                            raise LoopBreak()
        except LoopBreak:
            pass

    return matches
<|endoftext|>


<commit_before>#! /usr/bin/env python

# Main controller. Sarah Burke Spolaor June 2015
#
# Based on frb_trigger_controller.py by P. Demorest, 2015/02
#
# Listen for OBS packets having a certain 'triggered archiving'
# intent, and perform some as-yet-unspecified action when these
# are recieved.
#

"""
This branch of the code is testing the addition of an option that
archives data as soon as it comes into the CBE, rather than at the end
in queue_monitor.

Need to take care of:

 - Will need no_archive to be functioning so scripts need to include
   this option.

 - Do NOT do any hardlinking at the end in q_m -- otherwise there will
   be double copies pushed into the archive.

 - Delete all BDFs in no_archive at the end.

 - Do not touch bunker or archive.

---------------------- THIS IS THE VERSION I'M DOING -----------------------------

OTHERWISE we can run in an otherproject mode where we ourselves
hardlink a copy of the BDFs to no_archive and then at the end just
delete all the ones from no_archive - never touching the stuff in the
other bunker or archive directories. This would allow us to test on
other datasets, too, without interruption to anyone else.

For this, the following changes are needed:

 - Add option to mcaf_monitor and to queue_monitor.

 - In mcaf_monitor (?), determine BDF list and hardlink them from bunker.

 - In queue_monitor, don't hardlink to archive, just remove BDFs.

 - Error-check for input tags, i.e. make sure mcaf_mon and queue_mon both have the same option set, and make sure that we're not setting both "archive_all" and "handsoff_archive"

 - Throughout ensure compatibility with test and production versions.

 ---------------------------------------------------------------------------------
"""

import datetime
import os
import asyncore
import click
import logging
import sdmreader

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

from realfast import queue_monitor, rtutils, mcaf_library

# set up
rtparams_default = os.path.join(os.path.join(os.path.split(os.path.split(mcaf_library.__file__)[0])[0], 'conf'), 'rtpipe_cbe.conf') # install system puts conf files here. used by queue_rtpipe.py
default_bdfdir = '/lustre/evla/wcbe/data/no_archive'
telcaldir = '/home/mchammer/evladata/telcal'  # then yyyy/mm
workdir = os.getcwd()     # assuming we start in workdir
redishost = os.uname()[1]  # assuming we start on redis host

class FRBController(object):
    """Listens for OBS packets and tells FRB processing about any
    notable scans."""

    def __init__(self, intent='', project='', production=False, verbose=False, nrao_controls_archiving=False, rtparams='', slow=0):
        # Mode can be project, intent
        self.intent = intent
        self.project = project
        self.production = production
        self.nrao_controls_archiving = nrao_controls_archiving
        self.verbose = verbose
        self.rtparams = rtparams
        self.slow = slow

    def add_sdminfo(self, sdminfo):
        config = mcaf_library.MCAST_Config(sdminfo=sdminfo)

        # !!! Wrapper here to deal with potential subscans?

        # Check if MCAST message is simply telling us the obs is finished
        if config.obsComplete:
            logger.info("Received finalMessage=True; This observation has completed.")
            # if completing the desired SB, then do a final rsync
            if self.project in config.projectID and self.production:
                logger.info("Final rsync to make workdir copy of SDM %sd complete." % (config.sdmLocation.rstrip('/')))
                rtutils.rsync(config.sdmLocation.rstrip('/'), workdir)  # final transfer to ensure complete SDM in workdir

        # Otherwise check if project is the one we want.
        elif self.project in config.projectID:  # project of interest
            sdmlocation = config.sdmLocation.rstrip('/')
            filename = os.path.join(workdir, os.path.basename(sdmlocation))   # new full-path filename
            scan = int(config.scan)

            if self.intent in config.intentString:
                logger.info("Scan %d has desired intent (%s) and project (%s)" % (scan, self.intent, self.project))
                bdfloc = os.path.join(default_bdfdir, os.path.basename(config.bdfLocation))

                # If we're not controlling the archiving (as would
                # occur if we wanted to make BDFs instantly flowing to
                # the archive, or if we wanted to e.g. test by
                # piggybacking on another (NON-SCIENCE!)
                # observation...
                #
                # We need to then make our own copy in no_archive of
                # the BDF.
                if self.nrao_controls_archiving:
                    # Link from bunker to no_archive
                    bdfFROM = os.path.join('/lustre/evla/wcbe/data/bunker/',os.path.basename(config.bdfLocation))
                    bdfTO   = os.path.join(bdfArchdir, os.path.basename(bdfFROM))
                    print "Note BDF autostring from MCAF is %s" & config.bdfLocation
                    if not production:
                        logger.info('TEST MODE. Would hardlink %s to %s' % ( bdfFROM, bdfTO ))
                        touch( bdfTO + ".test" )
                    else:
                        logger.debug('Hardlinking %s to %s for parallel use by realfast.' % ( bdfFROM, bdfTO ))
                        logger.debug('***********WARNING: QUEUE_MONITOR SHOULD HAVE -noarch TURNED ON!!!')
                        os.link( bdfFROM, bdfTO )

                # If we're not in listening mode, prepare data and submit to queue system
                if self.production:
                    assert len(sdmlocation) and isinstance(sdmlocation, str), 'sdmlocation empty or not a string?'
                    assert 'mchammer' not in filename, 'filename %s is SDM original!'

                    # check that SDM is usable by rtpipe. Currently checks spw order and duplicates.
                    if rtutils.check_spw(sdmlocation, scan) and os.path.exists(bdfloc):
                        # 1) copy data into place
                        rtutils.rsync(sdmlocation, workdir)

                        logger.info("Processing sdm %s, scan %d..." % (filename, scan))
                        logger.debug("BDF is in %s\n" % (bdfloc))

                        # 2) find telcalfile (use timeout to wait for it to be written)
                        telcalfile = rtutils.gettelcalfile(telcaldir, filename, timeout=60)

                        # 3) if cal available and bdf exists, submit search job and add tail job to monitoring queue
                        if telcalfile:
                            logger.info('Submitting job to rtutils.search with args: %s %s %s %s %s %s %s %s' % ('default', filename, self.rtparams, '', str([scan]), telcalfile, redishost, default_bdfdir))
                            lastjob = rtutils.search('default', filename, self.rtparams, '', [scan], telcalfile=telcalfile, redishost=redishost, bdfdir=default_bdfdir)
                            rtutils.addjob(lastjob.id)
                        else:
                            logger.info('No calibration available. No job submitted.')
                    else:
                        logger.info("Not submitting scan %d of sdm %s. bdf not found or cannot be processed by rtpipe." % (scan, os.path.basename(sdmlocation)))

            # each CAL and TARGET scan gets integrated to MS for slow transients search, if in production mode and slow timescale set
            if (self.slow > 0) and ( ('CALIBRATE' in config.intentString) or ('TARGET' in config.intentString) and self.production):
                rtutils.rsync(sdmlocation, workdir)
                logger.info('Creating measurement set for %s scan %d' % (filename, scan))
                sc,sr = sdmreader.read_metadata(filename, scan, bdfdir=default_bdfdir)
                rtutils.linkbdfs(filename, sc, default_bdfdir)
                rtutils.integrate(filename, str(scan), self.slow, redishost)

@click.command()
@click.option('--intent', '-i', default='', help='Intent to trigger on')
@click.option('--project', '-p', default='', help='Project name to trigger on')
@click.option('--production', help='Run the code (not just print, etc)', is_flag=True)
@click.option('--verbose', '-v', help='More verbose output', is_flag=True)
@click.option('--nrao_controls_archiving', '-N', help='NRAO controls archiving; i.e. we make our own BDF hardlinks to no_archive and do no archiving at the end. If this is selected, queue_monitor MUST ALSO be run with this same option set.', is_flag=True)
@click.option('--rtparams', help='Parameter file for rtpipe. Default is rtpipe_cbe_conf.', default=rtparams_default)
@click.option('--slow', help='Integrate scans to MS with ints of this many seconds.', default=0)
def monitor(intent, project, production, verbose, nrao_controls_archiving, rtparams, slow):
    """ Monitor of mcaf observation files.
    Scans that match intent and project are searched (if in production mode).
    Blocking function.
    """

    # Set up verbosity level for log
    if verbose:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)

    logger.info('mcaf_monitor started')
    logger.info('Looking for intent = %s, project = %s' % (intent, project))
    logger.debug('Running in verbose mode')

    if not production:
        logger.info('Running in test mode')
    else:
        logger.info('Running in production mode')

    if nrao_controls_archiving:
        logger.info('*** NRAO CONTROLLING ARCHIVING. ***')
        logger.info('*** Will link and clean BDFs in no_archive directory without ever touching archive directory.')

    # This starts the receiving/handling loop
    controller = FRBController(intent=intent, project=project, production=production, verbose=verbose, rtparams=rtparams, slow=slow)
    sdminfo_client = mcaf_library.SdminfoClient(controller)
    try:
        asyncore.loop()
    except KeyboardInterrupt:
        # Just exit without the trace barf
        logger.info('Escaping mcaf_monitor')

def testrtpipe(filename, paramfile):
    """ Function for a quick test of rtpipe and queue system
    filename should have full path.
    """

    import sdmreader
    sc,sr = sdmreader.read_metadata(filename, bdfdir=default_bdfdir)
    scan = sc.keys()[0]
    telcalfile = rtutils.gettelcalfile(telcaldir, filename, timeout=60)
    lastjob = rtutils.search('default', filename, paramfile, '', [scan], telcalfile=telcalfile, redishost=redishost, bdfdir=default_bdfdir)
    return lastjob
<commit_msg>fix & to %<commit_after>#! /usr/bin/env python

# Main controller. Sarah Burke Spolaor June 2015
#
# Based on frb_trigger_controller.py by P. Demorest, 2015/02
#
# Listen for OBS packets having a certain 'triggered archiving'
# intent, and perform some as-yet-unspecified action when these
# are recieved.
#

"""
This branch of the code is testing the addition of an option that
archives data as soon as it comes into the CBE, rather than at the end
in queue_monitor.

Need to take care of:

 - Will need no_archive to be functioning so scripts need to include
   this option.

 - Do NOT do any hardlinking at the end in q_m -- otherwise there will
   be double copies pushed into the archive.

 - Delete all BDFs in no_archive at the end.

 - Do not touch bunker or archive.

---------------------- THIS IS THE VERSION I'M DOING -----------------------------

OTHERWISE we can run in an otherproject mode where we ourselves
hardlink a copy of the BDFs to no_archive and then at the end just
delete all the ones from no_archive - never touching the stuff in the
other bunker or archive directories. This would allow us to test on
other datasets, too, without interruption to anyone else.

For this, the following changes are needed:

 - Add option to mcaf_monitor and to queue_monitor.

 - In mcaf_monitor (?), determine BDF list and hardlink them from bunker.

 - In queue_monitor, don't hardlink to archive, just remove BDFs.

 - Error-check for input tags, i.e. make sure mcaf_mon and queue_mon both have the same option set, and make sure that we're not setting both "archive_all" and "handsoff_archive"

 - Throughout ensure compatibility with test and production versions.

 ---------------------------------------------------------------------------------
"""

import datetime
import os
import asyncore
import click
import logging
import sdmreader

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

from realfast import queue_monitor, rtutils, mcaf_library

# set up
rtparams_default = os.path.join(os.path.join(os.path.split(os.path.split(mcaf_library.__file__)[0])[0], 'conf'), 'rtpipe_cbe.conf') # install system puts conf files here. used by queue_rtpipe.py
default_bdfdir = '/lustre/evla/wcbe/data/no_archive'
telcaldir = '/home/mchammer/evladata/telcal'  # then yyyy/mm
workdir = os.getcwd()     # assuming we start in workdir
redishost = os.uname()[1]  # assuming we start on redis host

class FRBController(object):
    """Listens for OBS packets and tells FRB processing about any
    notable scans."""

    def __init__(self, intent='', project='', production=False, verbose=False, nrao_controls_archiving=False, rtparams='', slow=0):
        # Mode can be project, intent
        self.intent = intent
        self.project = project
        self.production = production
        self.nrao_controls_archiving = nrao_controls_archiving
        self.verbose = verbose
        self.rtparams = rtparams
        self.slow = slow

    def add_sdminfo(self, sdminfo):
        config = mcaf_library.MCAST_Config(sdminfo=sdminfo)

        # !!! Wrapper here to deal with potential subscans?

        # Check if MCAST message is simply telling us the obs is finished
        if config.obsComplete:
            logger.info("Received finalMessage=True; This observation has completed.")
            # if completing the desired SB, then do a final rsync
            if self.project in config.projectID and self.production:
                logger.info("Final rsync to make workdir copy of SDM %sd complete." % (config.sdmLocation.rstrip('/')))
                rtutils.rsync(config.sdmLocation.rstrip('/'), workdir)  # final transfer to ensure complete SDM in workdir

        # Otherwise check if project is the one we want.
        elif self.project in config.projectID:  # project of interest
            sdmlocation = config.sdmLocation.rstrip('/')
            filename = os.path.join(workdir, os.path.basename(sdmlocation))   # new full-path filename
            scan = int(config.scan)

            if self.intent in config.intentString:
                logger.info("Scan %d has desired intent (%s) and project (%s)" % (scan, self.intent, self.project))
                bdfloc = os.path.join(default_bdfdir, os.path.basename(config.bdfLocation))

                # If we're not controlling the archiving (as would
                # occur if we wanted to make BDFs instantly flowing to
                # the archive, or if we wanted to e.g. test by
                # piggybacking on another (NON-SCIENCE!)
                # observation...
                #
                # We need to then make our own copy in no_archive of
                # the BDF.
                if self.nrao_controls_archiving:
                    # Link from bunker to no_archive
                    bdfFROM = os.path.join('/lustre/evla/wcbe/data/bunker/',os.path.basename(config.bdfLocation))
                    bdfTO   = os.path.join(bdfArchdir, os.path.basename(bdfFROM))
                    logger.debug("Note BDF autostring from MCAF is %s" % config.bdfLocation)
                    if not production:
                        logger.info('TEST MODE. Would hardlink %s to %s' % ( bdfFROM, bdfTO ))
                        touch( bdfTO + ".test" )
                    else:
                        logger.debug('Hardlinking %s to %s for parallel use by realfast.' % ( bdfFROM, bdfTO ))
                        logger.debug('***********WARNING: QUEUE_MONITOR SHOULD HAVE -N TURNED ON!!!')
                        os.link( bdfFROM, bdfTO )

                # If we're not in listening mode, prepare data and submit to queue system
                if self.production:
                    assert len(sdmlocation) and isinstance(sdmlocation, str), 'sdmlocation empty or not a string?'
                    assert 'mchammer' not in filename, 'filename %s is SDM original!'

                    # check that SDM is usable by rtpipe. Currently checks spw order and duplicates.
                    if rtutils.check_spw(sdmlocation, scan) and os.path.exists(bdfloc):
                        # 1) copy data into place
                        rtutils.rsync(sdmlocation, workdir)

                        logger.info("Processing sdm %s, scan %d..." % (filename, scan))
                        logger.debug("BDF is in %s\n" % (bdfloc))

                        # 2) find telcalfile (use timeout to wait for it to be written)
                        telcalfile = rtutils.gettelcalfile(telcaldir, filename, timeout=60)

                        # 3) if cal available and bdf exists, submit search job and add tail job to monitoring queue
                        if telcalfile:
                            logger.info('Submitting job to rtutils.search with args: %s %s %s %s %s %s %s %s' % ('default', filename, self.rtparams, '', str([scan]), telcalfile, redishost, default_bdfdir))
                            lastjob = rtutils.search('default', filename, self.rtparams, '', [scan], telcalfile=telcalfile, redishost=redishost, bdfdir=default_bdfdir)
                            rtutils.addjob(lastjob.id)
                        else:
                            logger.info('No calibration available. No job submitted.')
                    else:
                        logger.info("Not submitting scan %d of sdm %s. bdf not found or cannot be processed by rtpipe." % (scan, os.path.basename(sdmlocation)))

            # each CAL and TARGET scan gets integrated to MS for slow transients search, if in production mode and slow timescale set
            if (self.slow > 0) and ( ('CALIBRATE' in config.intentString) or ('TARGET' in config.intentString) and self.production):
                rtutils.rsync(sdmlocation, workdir)
                logger.info('Creating measurement set for %s scan %d' % (filename, scan))
                sc,sr = sdmreader.read_metadata(filename, scan, bdfdir=default_bdfdir)
                rtutils.linkbdfs(filename, sc, default_bdfdir)
                rtutils.integrate(filename, str(scan), self.slow, redishost)

@click.command()
@click.option('--intent', '-i', default='', help='Intent to trigger on')
@click.option('--project', '-p', default='', help='Project name to trigger on')
@click.option('--production', help='Run the code (not just print, etc)', is_flag=True)
@click.option('--verbose', '-v', help='More verbose output', is_flag=True)
@click.option('--nrao_controls_archiving', '-N', help='NRAO controls archiving; i.e. we make our own BDF hardlinks to no_archive and do no archiving at the end. If this is selected, queue_monitor MUST ALSO be run with this same option set.', is_flag=True)
@click.option('--rtparams', help='Parameter file for rtpipe. Default is rtpipe_cbe_conf.', default=rtparams_default)
@click.option('--slow', help='Integrate scans to MS with ints of this many seconds.', default=0)
def monitor(intent, project, production, verbose, nrao_controls_archiving, rtparams, slow):
    """ Monitor of mcaf observation files.
    Scans that match intent and project are searched (if in production mode).
    Blocking function.
    """

    # Set up verbosity level for log
    if verbose:
        logger.setLevel(logging.DEBUG)
    else:
        logger.setLevel(logging.INFO)

    logger.info('mcaf_monitor started')
    logger.info('Looking for intent = %s, project = %s' % (intent, project))
    logger.debug('Running in verbose mode')

    if not production:
        logger.info('Running in test mode')
    else:
        logger.info('Running in production mode')

    if nrao_controls_archiving:
        logger.info('*** NRAO CONTROLLING ARCHIVING. ***')
        logger.info('*** Will link and clean BDFs in no_archive directory without ever touching archive directory.')

    # This starts the receiving/handling loop
    controller = FRBController(intent=intent, project=project, production=production, verbose=verbose, rtparams=rtparams, slow=slow)
    sdminfo_client = mcaf_library.SdminfoClient(controller)
    try:
        asyncore.loop()
    except KeyboardInterrupt:
        # Just exit without the trace barf
        logger.info('Escaping mcaf_monitor')

def testrtpipe(filename, paramfile):
    """ Function for a quick test of rtpipe and queue system
    filename should have full path.
    """

    import sdmreader
    sc,sr = sdmreader.read_metadata(filename, bdfdir=default_bdfdir)
    scan = sc.keys()[0]
    telcalfile = rtutils.gettelcalfile(telcaldir, filename, timeout=60)
    lastjob = rtutils.search('default', filename, paramfile, '', [scan], telcalfile=telcalfile, redishost=redishost, bdfdir=default_bdfdir)
    return lastjob
<|endoftext|>


<commit_before>import sublime
import sublime_plugin
import subprocess
import re
import os
import tempfile
import time
import threading

REGION_KEY = 'cjttest'

class DiffView(sublime_plugin.WindowCommand):
    diff_base = ''

    def run(self):
        self.window.last_diff = self
        self.last_hunk_index = 0
        self.preview = None

        # Use show_input_panel as show_quick_panel doesn't allow arbitrary data
        self.window.show_input_panel("Diff against? [HEAD]", self.diff_base, self.do_diff, None, None)

    def do_diff(self, diff_base):
        if diff_base == '':
            diff_base = 'HEAD'
        self.diff_base = diff_base

        # Record the original layout
        self.orig_layout = self.window.get_layout()

        # Create the diff parser
        self.parser = DiffParser(self.diff_base)

        # Show the list of changed hunks
        self.list_changed_hunks()

    def list_changed_hunks(self):
        self.window.show_quick_panel(
            [h.description for h in self.parser.changed_hunks],
            self.show_hunk_diff,
            0,
            self.last_hunk_index,
            self.preview_hunk)

    def show_hunk_diff(self, index):
        print("show_hunk_diff: {}".format(index))
        self.window.active_view().erase_regions(REGION_KEY)
        if index == -1:
            if self.preview:
                self.preview.close()
                self.preview = None
            return
        self.last_hunk_index = index
        hunk = self.parser.changed_hunks[index]
        self.window.open_file(hunk.filespec(), sublime.ENCODED_POSITION)

    def preview_hunk(self, index):
        self.window.active_view().erase_regions(REGION_KEY)
        hunk = self.parser.changed_hunks[index]
        already_exists = self.window.find_open_file(hunk.file_diff.filename)
        self.preview = self.window.open_file(hunk.filespec(), sublime.TRANSIENT | sublime.ENCODED_POSITION)
        if already_exists:
            self.preview = None
        hunk.file_diff.add_regions(self.window.active_view())

class DiffHunksList(sublime_plugin.WindowCommand):
    def run(self):
        if hasattr(self.window, 'last_diff'):
            self.window.last_diff.list_changed_hunks()

class DiffParser(object):
    STAT_CHANGED_FILE = re.compile('\s*([\w\.\-\/]+)\s*\|')

    def __init__(self, diff_base):
        self.git_base = git_command(['rev-parse', '--show-toplevel']).rstrip()
        self.diff_base = diff_base
        self.changed_files = self._get_changed_files()
        self.changed_hunks = []
        for f in self.changed_files:
            self.changed_hunks += f.get_hunks()

    def _get_changed_files(self):
        files = []
        for line in git_command(['diff', '--stat', self.diff_base]).split('\n'):
            match = self.STAT_CHANGED_FILE.match(line)
            if match:
                filename = match.group(1)
                abs_filename = os.path.join(self.git_base, filename)
                files.append(FileDiff(match.group(1), abs_filename, self.diff_base))
        return files

class FileDiff(object):
    HUNK_MATCH = re.compile('\r?\n@@ \-(\d+),?(\d*) \+(\d+),?(\d*) @@')

    def __init__(self, filename, abs_filename, diff_args):
        self.filename = filename
        self.abs_filename = abs_filename
        self.old_file = 'UNDEFINED'
        self.diff_args = diff_args
        self.diff_text = ''
        self.hunks = []

    def get_hunks(self):
        if not self.hunks:
            self.parse_diff()
        return self.hunks

    def parse_diff(self):
        if not self.diff_text:
            self.diff_text = git_command(['diff', self.diff_args, '-U0', '--', self.filename])
            hunks = self.HUNK_MATCH.split(self.diff_text)

            # First item is the header - drop it
            hunks.pop(0)
            match_len = 5
            while len(hunks) >= match_len:
                self.hunks.append(HunkDiff(self, hunks[:match_len]))
                hunks = hunks[match_len:]

    def add_regions(self, view):
        # TODO: Don't currently know how to show DELETE hunks sensibly
        regions = [h.get_region(view) for h in self.hunks if h.type != "DEL"]
        view.add_regions(REGION_KEY, regions, "comment")

class HunkDiff(object):
    LINE_DELIM_MATCH = re.compile('\r?\n~\r?\n')
    ADD_LINE_MATCH = re.compile('^\+(.*)')
    DEL_LINE_MATCH = re.compile('^\-(.*)')

    def __init__(self, file_diff, match):
        self.file_diff = file_diff
        self.old_line_start = int(match[0])
        self.old_hunk_len = 1
        if len(match[1]) > 0:
            self.old_hunk_len = int(match[1])
        self.new_line_start = int(match[2])
        self.new_hunk_len = 1
        if len(match[3]) > 0:
            self.new_hunk_len = int(match[3])
        self.hunk_diff_text = match[4]
        self.description = "{}:{}".format(file_diff.filename, self.new_line_start)

        if self.old_hunk_len == 0:
            self.type = "ADD"
        elif self.new_hunk_len == 0:
            self.type = "DEL"
        else:
            self.type = "MOD"

    def parse_diff(self):
        # Need to track line number (and character position) as we run through the regions.
        # Multiline adds and deletes are spread over multiple regions.
        old_line_num = self.old_line_start
        new_line_num = self.new_line_start
        for region in self.LINE_DELIM_MATCH.split(self.hunk_diff_text):
            print("$$$$" + region + "&&&&")
            add_match = self.ADD_LINE_MATCH.match(region)
            del_match = self.DEL_LINE_MATCH.match(region)

    def filespec(self):
        return "{}:{}".format(self.file_diff.abs_filename, self.new_line_start)

    def get_region(self, view):
        # TODO - differentiate between ADD (old_hunk_len == 0), DEL (new_hunk_len == 0) and MOD (both > 0)
        return sublime.Region(
            view.text_point(self.new_line_start - 1, 0),
            view.text_point(self.new_line_start + self.new_hunk_len - 1, 0))

def git_command(args):
    p = subprocess.Popen(['git'] + args,
                         stdout=subprocess.PIPE,
                         shell=True)
    out, err = p.communicate()
    return out.decode('utf-8')
<commit_msg>Prett much working<commit_after>import sublime
import sublime_plugin
import subprocess
import re
import os
import tempfile
import time
import threading

ADD_REGION_KEY = 'diffview-highlight-addition'
MOD_REGION_KEY = 'diffview-highlight-modification'
DEL_REGION_KEY = 'diffview-highlight-deletion'

class DiffView(sublime_plugin.WindowCommand):
    diff_base = ''

    def run(self):
        self.window.last_diff = self
        self.last_hunk_index = 0
        self.preview = None

        # Use show_input_panel as show_quick_panel doesn't allow arbitrary data
        self.window.show_input_panel("Diff against? [HEAD]", self.diff_base, self.do_diff, None, None)

    def do_diff(self, diff_base):
        if diff_base == '':
            diff_base = 'HEAD'
        self.diff_base = diff_base

        # Record the original layout
        self.orig_layout = self.window.get_layout()

        # Create the diff parser
        self.parser = DiffParser(self.diff_base)

        # Show the list of changed hunks
        self.list_changed_hunks()

    def list_changed_hunks(self):
        self.window.show_quick_panel(
            [h.description for h in self.parser.changed_hunks],
            self.show_hunk_diff,
            0,
            self.last_hunk_index,
            self.preview_hunk)

    def show_hunk_diff(self, index):
        self.window.active_view().erase_regions(ADD_REGION_KEY)
        self.window.active_view().erase_regions(MOD_REGION_KEY)
        self.window.active_view().erase_regions(DEL_REGION_KEY)

        if index == -1:
            if self.preview:
                self.preview.close()
                self.preview = None
            return
        self.last_hunk_index = index
        hunk = self.parser.changed_hunks[index]
        self.window.open_file(hunk.filespec(), sublime.ENCODED_POSITION)

    def preview_hunk(self, index):
        self.window.active_view().erase_regions(ADD_REGION_KEY)
        self.window.active_view().erase_regions(MOD_REGION_KEY)
        self.window.active_view().erase_regions(DEL_REGION_KEY)

        hunk = self.parser.changed_hunks[index]
        already_exists = self.window.find_open_file(hunk.file_diff.filename)
        self.preview = self.window.open_file(hunk.filespec(), sublime.TRANSIENT | sublime.ENCODED_POSITION)
        if already_exists:
            self.preview = None
        hunk.file_diff.add_regions(self.window.active_view())

class DiffHunksList(sublime_plugin.WindowCommand):
    def run(self):
        if hasattr(self.window, 'last_diff'):
            self.window.last_diff.list_changed_hunks()

class DiffParser(object):
    STAT_CHANGED_FILE = re.compile('\s*([\w\.\-\/]+)\s*\|')

    def __init__(self, diff_base):
        self.git_base = git_command(['rev-parse', '--show-toplevel']).rstrip()
        self.diff_base = diff_base
        self.changed_files = self._get_changed_files()
        self.changed_hunks = []
        for f in self.changed_files:
            self.changed_hunks += f.get_hunks()

    def _get_changed_files(self):
        files = []
        for line in git_command(['diff', '--stat', self.diff_base]).split('\n'):
            match = self.STAT_CHANGED_FILE.match(line)
            if match:
                filename = match.group(1)
                abs_filename = os.path.join(self.git_base, filename)
                files.append(FileDiff(match.group(1), abs_filename, self.diff_base))
        return files

class FileDiff(object):
    HUNK_MATCH = re.compile('\r?\n@@ \-(\d+),?(\d*) \+(\d+),?(\d*) @@')

    def __init__(self, filename, abs_filename, diff_args):
        self.filename = filename
        self.abs_filename = abs_filename
        self.old_file = 'UNDEFINED'
        self.diff_args = diff_args
        self.diff_text = ''
        self.hunks = []

    def get_hunks(self):
        if not self.hunks:
            self.parse_diff()
        return self.hunks

    def parse_diff(self):
        if not self.diff_text:
            self.diff_text = git_command(['diff', self.diff_args, '-U0', '--', self.filename])
            hunks = self.HUNK_MATCH.split(self.diff_text)

            # First item is the header - drop it
            hunks.pop(0)
            match_len = 5
            while len(hunks) >= match_len:
                self.hunks.append(HunkDiff(self, hunks[:match_len]))
                hunks = hunks[match_len:]

    def add_regions(self, view):
        view.add_regions(
            ADD_REGION_KEY,
            [h.get_region(view) for h in self.hunks if h.hunk_type == "ADD"],
            "string",
            flags=sublime.DRAW_EMPTY | sublime.HIDE_ON_MINIMAP | sublime.DRAW_EMPTY_AS_OVERWRITE | sublime.DRAW_NO_FILL)
        view.add_regions(
            MOD_REGION_KEY,
            [h.get_region(view) for h in self.hunks if h.hunk_type == "MOD"],
            "comment",
            flags=sublime.DRAW_EMPTY | sublime.HIDE_ON_MINIMAP | sublime.DRAW_EMPTY_AS_OVERWRITE | sublime.DRAW_NO_FILL)
        view.add_regions(
            DEL_REGION_KEY,
            [h.get_region(view) for h in self.hunks if h.hunk_type == "DEL"],
            "invalid",
            flags=sublime.DRAW_EMPTY | sublime.HIDE_ON_MINIMAP | sublime.DRAW_EMPTY_AS_OVERWRITE | sublime.DRAW_NO_FILL)

class HunkDiff(object):
    NEWLINE_MATCH = re.compile('\r?\n')
    LINE_DELIM_MATCH = re.compile('\r?\n~\r?\n')
    ADD_LINE_MATCH = re.compile('^\+(.*)')
    DEL_LINE_MATCH = re.compile('^\-(.*)')

    def __init__(self, file_diff, match):
        self.file_diff = file_diff

        # Maches' meanings are:
        # - 0: start line in old file
        # - 1: num lines removed from old file (0 for ADD, missing if it's a one-line change)
        # - 2: start line in new file
        # - 3: num lines added to new file (0 for DEL, missing if it's a one-line change)
        # - 4: the remainder of the hunk, after the header
        self.old_line_start = int(match[0])
        self.old_hunk_len = 1
        if len(match[1]) > 0:
            self.old_hunk_len = int(match[1])
        self.new_line_start = int(match[2])
        self.new_hunk_len = 1
        if len(match[3]) > 0:
            self.new_hunk_len = int(match[3])
        self.hunk_diff_lines = self.NEWLINE_MATCH.split(match[4])

        if self.old_hunk_len == 0:
            self.hunk_type = "ADD"
        elif self.new_hunk_len == 0:
            self.hunk_type = "DEL"
        else:
            self.hunk_type = "MOD"

        self.description = [
            "{}:{}".format(file_diff.filename, self.new_line_start),
            self.hunk_diff_lines[0],
            self.hunk_type]

    def parse_diff(self):
        # Need to track line number (and character position) as we run through the regions.
        # Multiline adds and deletes are spread over multiple regions.
        old_line_num = self.old_line_start
        new_line_num = self.new_line_start
        for region in self.LINE_DELIM_MATCH.split(self.hunk_diff_text):
            print("$$$$" + region + "&&&&")
            add_match = self.ADD_LINE_MATCH.match(region)
            del_match = self.DEL_LINE_MATCH.match(region)

    def filespec(self):
        return "{}:{}".format(self.file_diff.abs_filename, self.new_line_start)

    def get_region(self, view):
        return sublime.Region(
            view.text_point(self.new_line_start - 1, 0),
            view.text_point(self.new_line_start + self.new_hunk_len - 1, 0))

def git_command(args):
    p = subprocess.Popen(['git'] + args,
                         stdout=subprocess.PIPE,
                         shell=True)
    out, err = p.communicate()
    return out.decode('utf-8')
<|endoftext|>