<commit_before># import logging
import sqlite3
import os
import re
import telebot
import sys
reload(sys)
sys.setdefaultencoding('utf-8')

def db_setup(dbfile='troll.db'):
    db = sqlite3.connect(dbfile)
    cursor = db.cursor()
    cursor.execute('CREATE TABLE IF NOT EXISTS quotes (chatid int, username text, quote text)')
    cursor.execute('DELETE FROM quotes WHERE chatid = 0')
    with open(os.path.expanduser('~/staticquotes.txt'), 'r') as staticquotes:
        for line in staticquotes:
            cursor.execute('''INSERT INTO quotes(chatid,username,quote) VALUES(?,?,?)''', (0, 'nickyabbot', line.strip()))
    db.commit()
    db.close()

if 'TOKEN' not in os.environ:
    print("missing TOKEN.Leaving...")
    os._exit(1)

# logger = telebot.logger
# telebot.logger.setLevel(logging.DEBUG)
bot = telebot.TeleBot(os.environ['TOKEN'])
botname = "@%s" % bot.get_me().username

@bot.message_handler(commands=['start', "start%s" % botname])
def start(message):
    startmessage = ''' *Welcome to the troll bot*

This bot is designed to ease trolling within your group chats

/troll to get a random troll
/troll PATTERN to get matching troll
/trolladd PATTERN to add troll to current group
/trolldelete PATTERN to delete troll from current group
'''
    bot.reply_to(message, startmessage, parse_mode='Markdown')

@bot.message_handler(commands=['trollhelp', 'help', "help%s" % botname])
def help(message):
    helpmessage = '''Those are the commands available:
/troll - Displays random or matching troll
/trolladd - Adds indicated troll
/trolldelete - Deletes given troll
/trolllist - Lists all trolls
/trollhelp - This help
'''
    bot.reply_to(message, helpmessage)

@bot.message_handler(commands=['troll', "troll%s" % botname])
def get(message):
    global trolldb
    quote = re.sub(r"/troll(%s|)" % botname, '', message.text).strip()
    chatid = message.chat.id if message.chat.title is not None else 0
    db = sqlite3.connect(trolldb)
    cursor = db.cursor()
    if quote == '':
        cursor.execute('''SELECT quote FROM quotes WHERE chatid = ? OR chatid = 0 ORDER BY RANDOM() LIMIT 1''', (chatid,))
        fetch = cursor.fetchone()
    else:
        cursor.execute("SELECT quote FROM quotes WHERE quote like ? AND (chatid = ? OR chatid = 0) ORDER BY RANDOM() LIMIT 1", ('%' + quote + '%', chatid,))
        fetch = cursor.fetchone()
        if fetch is None:
            print("No Matching quote found")
            bot.reply_to(message, 'No Matching quote found')
            return
    quote = fetch[0]
    db.close()
    print("Sending troll message to user %s" % message.from_user.username)
    bot.reply_to(message, quote)

@bot.message_handler(commands=['trollall', 'trolllist', "trollall%s" % botname, "trolllist%s" % botname])
def all(message):
    global trolldb
    db = sqlite3.connect(trolldb)
    cursor = db.cursor()
    if 'group' in message.chat.type:
        cursor.execute('SELECT quote FROM quotes where chatid = ? or chatid = 0', (message.chat.id,))
    else:
        cursor.execute('SELECT quote FROM quotes where chatid = 0')
    quotes = '\n'.join([q[0] for q in cursor.fetchall()])
    db.close()
    print("Sending all troll messages to user %s" % message.from_user.username)
    bot.reply_to(message, quotes)

@bot.inline_handler(lambda query: len(query.query) > 0)
def inline_all(query):
    global trolldb
    message = query.query
    db = sqlite3.connect(trolldb)
    cursor = db.cursor()
    cursor.execute("SELECT quote FROM quotes WHERE quote like ? AND ( username = ? OR chatid = 0)", ('%' + message + '%', query.from_user.username))
    fetch = cursor.fetchall()
    if not fetch:
        print("No Matching quote found")
        bot.answer_inline_query(query.id, 'Nothing found')
        return
    results = []
    for index, value in enumerate(fetch):
        value = value[0]
        results.append(telebot.types.InlineQueryResultArticle(str(index), value, telebot.types.InputTextMessageContent(value)))
    print("Sending all troll messages to user %s" % query.from_user.username)
    bot.answer_inline_query(query.id, results)

@bot.message_handler(commands=['trolladd', "trolladd%s" % botname])
def add(message):
    global trolldb
    if 'group' not in message.chat.type:
        bot.reply_to(message, 'Trolls can only be added to groups')
        return
    quote = re.sub(r"/trolladd(%s|)" % botname, '', message.text).strip()
    if quote == '':
        bot.send_message(message.chat.id, "Allright @%s. Give me a troll" % message.from_user.username, reply_markup=telebot.types.ForceReply(selective=True))
        return
    quote = quote.strip()
    db = sqlite3.connect(trolldb)
    cursor = db.cursor()
    cursor.execute('''SELECT quote FROM quotes where chatid = ? AND quote == ?''', (message.chat.id, quote))
    existing = cursor.fetchone()
    if existing is not None:
        bot.reply_to(message, 'Troll allready exists in this group')
        return
    cursor.execute('''INSERT INTO quotes(chatid,username,quote) VALUES(?,?,?)''', (message.chat.id, message.from_user.username, quote))
    print("Adding Troll message to group %s" % message.chat.title)
    bot.reply_to(message, 'Troll added to your group')
    db.commit()
    db.close()

@bot.message_handler(commands=['trolldel', 'trolldelete', "trolldel%s" % botname, "trolldelete%s" % botname])
def delete(message):
    global trolldb
    quote = re.sub(r"/troll(delete|del)(%s|)" % botname, '', message.text).strip()
    if quote == '':
        db = sqlite3.connect(trolldb)
        cursor = db.cursor()
        cursor.execute('''SELECT quote FROM quotes where chatid = ? and username = ?''', (message.chat.id, message.from_user.username))
        existing = cursor.fetchall()
        if not existing:
            bot.reply_to(message, 'No trolls for you to delete')
        else:
            markup = telebot.types.ReplyKeyboardMarkup(row_width=1, selective=True)
            for entry in existing:
                markup.add(telebot.types.KeyboardButton(entry[0]))
            bot.send_message(message.chat.id, "Allright @%s. Delete a troll" % message.from_user.username, reply_markup=markup)
        return
    if 'group' not in message.chat.type:
        bot.reply_to(message, 'Trolls can only be deleted from groups')
        return
    db = sqlite3.connect(trolldb)
    cursor = db.cursor()
    deleted = cursor.execute('''DELETE FROM QUOTES WHERE chatid = ? and  username = ? and quote = ?''', (message.chat.id, message.from_user.username, quote))
    if deleted.rowcount > 0:
        print("Deleted Troll message from group %s" % message.chat.title)
        bot.reply_to(message, 'Troll deleted from your group')
    else:
        print("No Troll message deleted from group %s" % message.chat.title)
        bot.reply_to(message, 'No Troll found for you to delete')
    db.commit()
    db.close()

@bot.message_handler(func=lambda m: True)
def custom(message):
    try:
        if 'transcod' in message.text.lower():
            bot.reply_to(message, 'a chupito for @%s!!!' % message.from_user.username)
        elif 'papichulo' in message.text.lower() and ('minWi' in message.text.lower() or 'eminguez' in message.text.lower()):
            bot.reply_to(message, 'Please stop. @minWi will be a father when he\'s ready')
        elif '$deity' in message.text.lower() or 'dios' in message.text.lower() or 'god' in message.text.lower():
            bot.reply_to(message, '$deity doesn\'t exist @%s. Sorry...' % message.from_user.username)
        elif 'sales' in message.text.lower() and 'spitzer' in message.text.lower():
            bot.reply_to(message, '@cspitzer is not from sales. He told me last time he invited me to dinner')
        elif 'satellite--' in message.text.lower() or 'satellite6--' in message.text.lower() or 'satellite6--' in message.text.lower() or 'sat6--' in message.text.lower():
            bot.reply_to(message, 'Agree with you my lord @%s' % message.from_user.username)
        elif 'subscription-manager--' in message.text.lower():
            bot.reply_to(message, 'Agree with you my lord @%s' % message.from_user.username)
        elif 'dragon' in message.text.lower():
            url = 'https://media.giphy.com/media/9ljido0fjus92/giphy.gif'
            bot.send_document(message.chat.id, url)
        elif message.reply_to_message is not None and message.reply_to_message.text is not None:
            if 'Give me a troll' in message.reply_to_message.text:
                quote = message.text.strip()
                db = sqlite3.connect(trolldb)
                cursor = db.cursor()
                cursor.execute('''SELECT quote FROM quotes where chatid = ? AND quote == ?''', (message.chat.id, quote))
                existing = cursor.fetchone()
                if existing is not None:
                    bot.reply_to(message, 'Troll allready exists in this group')
                    return
                cursor.execute('''INSERT INTO quotes(chatid,username,quote) VALUES(?,?,?)''', (message.chat.id, message.from_user.username, quote))
                print("Adding Troll to group %s" % message.chat.title)
                bot.reply_to(message, 'Troll added to your group')
                db.commit()
                db.close()
            elif 'Delete a troll' in message.reply_to_message.text:
                quote = message.text.strip()
                db = sqlite3.connect(trolldb)
                cursor = db.cursor()
                deleted = cursor.execute('''DELETE FROM quotes WHERE chatid = ? AND username = ? AND quote = ?''', (message.chat.id, message.from_user.username, quote))
                if deleted.rowcount > 0:
                    print("Deleted Troll from group %s" % message.chat.title)
                    markup = telebot.types.ReplyKeyboardHide(selective=True)
                    bot.reply_to(message, 'Troll deleted from your group', reply_markup=markup)
                else:
                    bot.reply_to(message, 'No troll found to delete')
                db.commit()
                db.close()
    except Exception as e:
        print(e)

trolldb = os.path.expanduser("~/troll.db")
db_setup(dbfile=trolldb)
print("Ready for trolling!")
bot.polling()
<commit_msg>caprio troll<commit_after># import logging
import sqlite3
import os
import re
import telebot
import sys
reload(sys)
sys.setdefaultencoding('utf-8')

def db_setup(dbfile='troll.db'):
    db = sqlite3.connect(dbfile)
    cursor = db.cursor()
    cursor.execute('CREATE TABLE IF NOT EXISTS quotes (chatid int, username text, quote text)')
    cursor.execute('DELETE FROM quotes WHERE chatid = 0')
    with open(os.path.expanduser('~/staticquotes.txt'), 'r') as staticquotes:
        for line in staticquotes:
            cursor.execute('''INSERT INTO quotes(chatid,username,quote) VALUES(?,?,?)''', (0, 'nickyabbot', line.strip()))
    db.commit()
    db.close()

if 'TOKEN' not in os.environ:
    print("missing TOKEN.Leaving...")
    os._exit(1)

# logger = telebot.logger
# telebot.logger.setLevel(logging.DEBUG)
bot = telebot.TeleBot(os.environ['TOKEN'])
botname = "@%s" % bot.get_me().username

@bot.message_handler(commands=['start', "start%s" % botname])
def start(message):
    startmessage = ''' *Welcome to the troll bot*

This bot is designed to ease trolling within your group chats

/troll to get a random troll
/troll PATTERN to get matching troll
/trolladd PATTERN to add troll to current group
/trolldelete PATTERN to delete troll from current group
'''
    bot.reply_to(message, startmessage, parse_mode='Markdown')

@bot.message_handler(commands=['trollhelp', 'help', "help%s" % botname])
def help(message):
    helpmessage = '''Those are the commands available:
/troll - Displays random or matching troll
/trolladd - Adds indicated troll
/trolldelete - Deletes given troll
/trolllist - Lists all trolls
/trollhelp - This help
'''
    bot.reply_to(message, helpmessage)

@bot.message_handler(commands=['troll', "troll%s" % botname])
def get(message):
    global trolldb
    quote = re.sub(r"/troll(%s|)" % botname, '', message.text).strip()
    chatid = message.chat.id if message.chat.title is not None else 0
    db = sqlite3.connect(trolldb)
    cursor = db.cursor()
    if quote == '':
        cursor.execute('''SELECT quote FROM quotes WHERE chatid = ? OR chatid = 0 ORDER BY RANDOM() LIMIT 1''', (chatid,))
        fetch = cursor.fetchone()
    else:
        cursor.execute("SELECT quote FROM quotes WHERE quote like ? AND (chatid = ? OR chatid = 0) ORDER BY RANDOM() LIMIT 1", ('%' + quote + '%', chatid,))
        fetch = cursor.fetchone()
        if fetch is None:
            print("No Matching quote found")
            bot.reply_to(message, 'No Matching quote found')
            return
    quote = fetch[0]
    db.close()
    print("Sending troll message to user %s" % message.from_user.username)
    bot.reply_to(message, quote)

@bot.message_handler(commands=['trollall', 'trolllist', "trollall%s" % botname, "trolllist%s" % botname])
def all(message):
    global trolldb
    db = sqlite3.connect(trolldb)
    cursor = db.cursor()
    if 'group' in message.chat.type:
        cursor.execute('SELECT quote FROM quotes where chatid = ? or chatid = 0', (message.chat.id,))
    else:
        cursor.execute('SELECT quote FROM quotes where chatid = 0')
    quotes = '\n'.join([q[0] for q in cursor.fetchall()])
    db.close()
    print("Sending all troll messages to user %s" % message.from_user.username)
    bot.reply_to(message, quotes)

@bot.inline_handler(lambda query: len(query.query) > 0)
def inline_all(query):
    global trolldb
    message = query.query
    db = sqlite3.connect(trolldb)
    cursor = db.cursor()
    cursor.execute("SELECT quote FROM quotes WHERE quote like ? AND ( username = ? OR chatid = 0)", ('%' + message + '%', query.from_user.username))
    fetch = cursor.fetchall()
    if not fetch:
        print("No Matching quote found")
        bot.answer_inline_query(query.id, 'Nothing found')
        return
    results = []
    for index, value in enumerate(fetch):
        value = value[0]
        results.append(telebot.types.InlineQueryResultArticle(str(index), value, telebot.types.InputTextMessageContent(value)))
    print("Sending all troll messages to user %s" % query.from_user.username)
    bot.answer_inline_query(query.id, results)

@bot.message_handler(commands=['trolladd', "trolladd%s" % botname])
def add(message):
    global trolldb
    if 'group' not in message.chat.type:
        bot.reply_to(message, 'Trolls can only be added to groups')
        return
    quote = re.sub(r"/trolladd(%s|)" % botname, '', message.text).strip()
    if quote == '':
        bot.send_message(message.chat.id, "Allright @%s. Give me a troll" % message.from_user.username, reply_markup=telebot.types.ForceReply(selective=True))
        return
    quote = quote.strip()
    db = sqlite3.connect(trolldb)
    cursor = db.cursor()
    cursor.execute('''SELECT quote FROM quotes where chatid = ? AND quote == ?''', (message.chat.id, quote))
    existing = cursor.fetchone()
    if existing is not None:
        bot.reply_to(message, 'Troll allready exists in this group')
        return
    cursor.execute('''INSERT INTO quotes(chatid,username,quote) VALUES(?,?,?)''', (message.chat.id, message.from_user.username, quote))
    print("Adding Troll message to group %s" % message.chat.title)
    bot.reply_to(message, 'Troll added to your group')
    db.commit()
    db.close()

@bot.message_handler(commands=['trolldel', 'trolldelete', "trolldel%s" % botname, "trolldelete%s" % botname])
def delete(message):
    global trolldb
    quote = re.sub(r"/troll(delete|del)(%s|)" % botname, '', message.text).strip()
    if quote == '':
        db = sqlite3.connect(trolldb)
        cursor = db.cursor()
        cursor.execute('''SELECT quote FROM quotes where chatid = ? and username = ?''', (message.chat.id, message.from_user.username))
        existing = cursor.fetchall()
        if not existing:
            bot.reply_to(message, 'No trolls for you to delete')
        else:
            markup = telebot.types.ReplyKeyboardMarkup(row_width=1, selective=True)
            for entry in existing:
                markup.add(telebot.types.KeyboardButton(entry[0]))
            bot.send_message(message.chat.id, "Allright @%s. Delete a troll" % message.from_user.username, reply_markup=markup)
        return
    if 'group' not in message.chat.type:
        bot.reply_to(message, 'Trolls can only be deleted from groups')
        return
    db = sqlite3.connect(trolldb)
    cursor = db.cursor()
    deleted = cursor.execute('''DELETE FROM QUOTES WHERE chatid = ? and  username = ? and quote = ?''', (message.chat.id, message.from_user.username, quote))
    if deleted.rowcount > 0:
        print("Deleted Troll message from group %s" % message.chat.title)
        bot.reply_to(message, 'Troll deleted from your group')
    else:
        print("No Troll message deleted from group %s" % message.chat.title)
        bot.reply_to(message, 'No Troll found for you to delete')
    db.commit()
    db.close()

@bot.message_handler(func=lambda m: True)
def custom(message):
    try:
        if 'transcod' in message.text.lower():
            bot.reply_to(message, 'a chupito for @%s!!!' % message.from_user.username)
        elif 'papichulo' in message.text.lower() and ('minWi' in message.text.lower() or 'eminguez' in message.text.lower()):
            bot.reply_to(message, 'Please stop. @minWi will be a father when he\'s ready')
        elif '$deity' in message.text.lower() or 'dios' in message.text.lower() or 'god' in message.text.lower():
            bot.reply_to(message, '$deity doesn\'t exist @%s. Sorry...' % message.from_user.username)
        elif 'sales' in message.text.lower() and 'spitzer' in message.text.lower():
            bot.reply_to(message, '@cspitzer is not from sales. He told me last time he invited me to dinner')
        elif 'satellite--' in message.text.lower() or 'satellite6--' in message.text.lower() or 'satellite6--' in message.text.lower() or 'sat6--' in message.text.lower():
            bot.reply_to(message, 'Agree with you my lord @%s' % message.from_user.username)
        elif 'subscription-manager--' in message.text.lower():
            bot.reply_to(message, 'Agree with you my lord @%s' % message.from_user.username)
        elif 'dragon' in message.text.lower() or 'daenerys' in message.text.lower():
            url = 'https://media.giphy.com/media/9ljido0fjus92/giphy.gif'
            bot.send_document(message.chat.id, url)
        elif 'fiesta' in message.text.lower() or 'party' in message.text.lower():
            url = 'https://media.giphy.com/media/QMkPpxPDYY0fu/giphy.gif'
            bot.send_document(message.chat.id, url)
        elif message.reply_to_message is not None and message.reply_to_message.text is not None:
            if 'Give me a troll' in message.reply_to_message.text:
                quote = message.text.strip()
                db = sqlite3.connect(trolldb)
                cursor = db.cursor()
                cursor.execute('''SELECT quote FROM quotes where chatid = ? AND quote == ?''', (message.chat.id, quote))
                existing = cursor.fetchone()
                if existing is not None:
                    bot.reply_to(message, 'Troll allready exists in this group')
                    return
                cursor.execute('''INSERT INTO quotes(chatid,username,quote) VALUES(?,?,?)''', (message.chat.id, message.from_user.username, quote))
                print("Adding Troll to group %s" % message.chat.title)
                bot.reply_to(message, 'Troll added to your group')
                db.commit()
                db.close()
            elif 'Delete a troll' in message.reply_to_message.text:
                quote = message.text.strip()
                db = sqlite3.connect(trolldb)
                cursor = db.cursor()
                deleted = cursor.execute('''DELETE FROM quotes WHERE chatid = ? AND username = ? AND quote = ?''', (message.chat.id, message.from_user.username, quote))
                if deleted.rowcount > 0:
                    print("Deleted Troll from group %s" % message.chat.title)
                    markup = telebot.types.ReplyKeyboardHide(selective=True)
                    bot.reply_to(message, 'Troll deleted from your group', reply_markup=markup)
                else:
                    bot.reply_to(message, 'No troll found to delete')
                db.commit()
                db.close()
    except Exception as e:
        print(e)

trolldb = os.path.expanduser("~/troll.db")
db_setup(dbfile=trolldb)
print("Ready for trolling!")
bot.polling()
<|endoftext|>


<commit_before>#!/usr/bin/env python
import subprocess
import sys
import string

if len(sys.argv) != 2:
    print "Usage: ./contributors.py tag-of-previous-release"
    sys.exit(-1)

tag = sys.argv[1]

cmd = "git log %s..HEAD --format=%%aN"
authors = subprocess.check_output((cmd % tag).split()).split('\n')
authors = [a.strip() for a in authors if a.strip()]

def key(author):
    author = [v for v in author.split() if v[0] in string.letters]
    return author[-1]

authors = sorted(set(authors), key=key)

for a in authors:
    print '-', a
<commit_msg>Update contributors script to count by date.<commit_after>#!/usr/bin/env python
import subprocess
import sys
import string
import shlex

if len(sys.argv) != 2:
    print "Usage: ./contributors.py tag-of-previous-release"
    sys.exit(-1)

tag = sys.argv[1]

def call(cmd):
    return subprocess.check_output(shlex.split(cmd)).split('\n')

tag_date = call("git show --format='%%ci' %s" % tag)[0]
print "Release %s was on %s" % (tag, tag_date)
print "\nCommitters since, alphabetical by last name:\n"

authors = call("git log --since='%s' --format=%%aN" % tag_date)
authors = [a.strip() for a in authors if a.strip()]

def key(author):
    author = [v for v in author.split() if v[0] in string.letters]
    return author[-1]

authors = sorted(set(authors), key=key)

for a in authors:
    print '-', a
<|endoftext|>


<commit_before>import nose
import numpy as np
from nengo.simulator_objects import SimModel
from nengo.simulator import Simulator
from nengo.templates.circularconv import CircularConvolution

def test_mini(D=4, neurons_per_product=3):
    # D is dimensionality of semantic pointers

    m = SimModel(.001)
    rng = np.random.RandomState(1234)

    A = m.signal(D, value=rng.randn(D))
    B = m.signal(D, value=rng.randn(D))

    CircularConvolution(m, A, B,
        neurons_per_product=neurons_per_product)

    sim = Simulator(m)
    sim.run_steps(10)

    raise nose.SkipTest()
<commit_msg>Simulator overide for CircularConv TestCase<commit_after>import unittest
import nose
import numpy as np
from nengo.simulator_objects import SimModel
from nengo.simulator import Simulator
from nengo.templates.circularconv import CircularConvolution

class TestCircularConv(unittest.TestCase):
    def Simulator(self, m):
        return Simulator(m)

    def test_small(self, D=4, neurons_per_product=3):
        # D is dimensionality of semantic pointers

        m = SimModel(.001)
        rng = np.random.RandomState(1234)

        A = m.signal(D, value=rng.randn(D))
        B = m.signal(D, value=rng.randn(D))

        CircularConvolution(m, A, B,
            neurons_per_product=neurons_per_product)

        sim = self.Simulator(m)
        sim.run_steps(10)

        raise nose.SkipTest()

    def test_med(self):
        return self.test_small(D=50, neurons_per_product=128)

    def test_large(self):
        return self.test_small(D=512, neurons_per_product=128)

<|endoftext|>


<commit_before>#import unittest
import mt
import queue
import _thread
import time
class DummySocket:

    def __init__(self, ID):
        self.id = ID
        self.lastMsg = ""

    def send(self, msg):
        print(self.id+":",msg)
        self.lastMsg = msg

    def close(self):
        print(self.id+": CLOSE CALLED")
        self.lastMsg = "CLOSED"

    def getpeername(self):
        return self.id, 0

##class TestMt(unittest.TestCase):
##
##    def test_listener_thread(self):
##        data = queue.Queue()
##        s1 = DummySocket("0")
##        s2 = DummySocket("1")
##
##        listenerObj = mt.ListenerThread(data)
##
##        def sender():
##            time.sleep(1)
##            data.put("Hello, World")
##            time.sleep(0.1)
##            listenerObj.add(s2)
##            time.sleep(1)
##            data.put("Hello, World 2")
##            time.sleep(1)
##            listenerObj.removeIP("0")
##            data.put("Hello, World 3")
##
##            data.put("!!INTERNAL=SHUTDOWN!!")
##
##        listenerObj.add(s1)
##        _thread.start_new_thread(sender, ())
##        listenerObj.main()
##
##        self.assertEqual(s1.lastMsg, "CLOSED")
##        self.assertEqual(s2.lastMsg, b"Hello, World 3\r\n")

def test_listener_thread(self):
            data = queue.Queue()
        s1 = DummySocket("0")
        s2 = DummySocket("1")

        listenerObj = mt.ListenerThread(data)

        def sender():
            time.sleep(1)
            data.put("Hello, World")
            time.sleep(0.1)
            listenerObj.add(s2)
            time.sleep(1)
            data.put("Hello, World 2")
            time.sleep(1)
            listenerObj.removeIP("0")
            data.put("Hello, World 3")

            data.put("!!INTERNAL=SHUTDOWN!!")

        listenerObj.add(s1)
        _thread.start_new_thread(sender, ())
        listenerObj.main()

        assert s1.lastMsg == "CLOSED")
        assert s2.lastMsg == b"Hello, World 3\r\n")

<commit_msg>FINAL SOLUTION WHICH SHOULD WORK<commit_after>#import unittest
import mt
import queue
import _thread
import time
class DummySocket:

    def __init__(self, ID):
        self.id = ID
        self.lastMsg = ""

    def send(self, msg):
        print(self.id+":",msg)
        self.lastMsg = msg

    def close(self):
        print(self.id+": CLOSE CALLED")
        self.lastMsg = "CLOSED"

    def getpeername(self):
        return self.id, 0

##class TestMt(unittest.TestCase):
##
##    def test_listener_thread(self):
##        data = queue.Queue()
##        s1 = DummySocket("0")
##        s2 = DummySocket("1")
##
##        listenerObj = mt.ListenerThread(data)
##
##        def sender():
##            time.sleep(1)
##            data.put("Hello, World")
##            time.sleep(0.1)
##            listenerObj.add(s2)
##            time.sleep(1)
##            data.put("Hello, World 2")
##            time.sleep(1)
##            listenerObj.removeIP("0")
##            data.put("Hello, World 3")
##
##            data.put("!!INTERNAL=SHUTDOWN!!")
##
##        listenerObj.add(s1)
##        _thread.start_new_thread(sender, ())
##        listenerObj.main()
##
##        self.assertEqual(s1.lastMsg, "CLOSED")
##        self.assertEqual(s2.lastMsg, b"Hello, World 3\r\n")

def test_listener_thread(self):
        data = queue.Queue()
        s1 = DummySocket("0")
        s2 = DummySocket("1")

        listenerObj = mt.ListenerThread(data)

        def sender():
            time.sleep(1)
            data.put("Hello, World")
            time.sleep(0.1)
            listenerObj.add(s2)
            time.sleep(1)
            data.put("Hello, World 2")
            time.sleep(1)
            listenerObj.removeIP("0")
            data.put("Hello, World 3")

            data.put("!!INTERNAL=SHUTDOWN!!")

        listenerObj.add(s1)
        _thread.start_new_thread(sender, ())
        listenerObj.main()

        assert s1.lastMsg == "CLOSED")
        assert s2.lastMsg == b"Hello, World 3\r\n")
<|endoftext|>


<commit_before>import os
import json

# This is to test imports within actions folder to check
# if we messed up sys.path for actions.
from base import DummyClass

from st2actions.runners.pythonrunner import Action
from st2client.client import Client
from st2client.models import KeyValuePair

__all__ = [
    'DatastoreTestAction'
]

class DatastoreTestAction(Action):

    def run(self):
        t_cls = DummyClass()
        print('Tests begin: %s' % t_cls.now())
        self._test_datastore_actions_via_client()
        self._test_datastore_actions_via_action_service()
        print('Tests end: %s' % t_cls.now())

    def _test_datastore_actions_via_client(self):
        print('Test datastore access via raw client.')
        client = Client(base_url='http://localhost')

        test_name = 'st2tests.putin'
        test_value = 'putout'
        # Put
        kv = KeyValuePair(name=test_name, value=test_value)
        client.keys.update(kv)
        # print('Wrote key: %s value: %s to datastore.' % (test_name, test_value))

        # Get
        val = client.keys.get_by_name(name=test_name)
        if val.value != test_value:
            raise Exception('KeyValue access failed on GET: %s' % test_name)
        # print('Got value: %s from datastore.' % val.value)

        # Delete
        client.keys.delete(val)
        # print('Successfully delete key: %s from datastore.' % test_name)

    def _test_datastore_actions_via_action_service(self):
        print('Test datastore access via action_service')

        # Note: "decrypt" option requires admin access and when we generate service token that
        # token isn't granted admin access so it won't work.
        # To make the tests also pass on st2enteprise with RBAC we use explicit admin token
        admin_token = os.environ.get('ST2_AUTH_TOKEN')
        self.action_service.datastore_service._get_api_client()
        os.environ['ST2_AUTH_TOKEN'] = admin_token

        data = {'somedata': 'foobar'}
        data_json_str = json.dumps(data)

        # Add a value to the datastore
        self.action_service.set_value(name='cache', value=data_json_str)

        # Retrieve a value
        value = self.action_service.get_value('cache')
        retrieved_data = json.loads(value)
        if 'somedata' not in retrieved_data:
            raise Exception('Retrieved incorrect value from datastore: %s. Expected: %s' %
                            (retrieved_data, data))

        if retrieved_data['somedata'] != 'foobar':
            raise Exception('Datastore value corrupted!!!')

        # Delete a value
        self.action_service.delete_value('cache')

        # Add an encrypted value to the datastore
        self.action_service.set_value(name='cache', value='foo', encrypt=True)

        # decrypted value should match
        value = self.action_service.get_value('cache', decrypt=True)
        if not value or value != 'foo':
            raise Exception('Retrieved incorrect value from datastore: %s. Expected: %s' %
                            (value, 'foo'))

        # non-decrypted value should not match
        value = self.action_service.get_value('cache')
        if not value or value == 'foo':
            raise Exception('Retrieved incorrect value from datastore: %s. Did not expect: %s' %
                            (value, 'foo'))

        # Delete a value
        self.action_service.delete_value('cache')
<commit_msg>Update affected test so it works with changes in https://github.com/StackStorm/st2/pull/4259 which made "get_api_client()" method public.<commit_after>import os
import json

# This is to test imports within actions folder to check
# if we messed up sys.path for actions.
from base import DummyClass

from st2actions.runners.pythonrunner import Action
from st2client.client import Client
from st2client.models import KeyValuePair

__all__ = [
    'DatastoreTestAction'
]

class DatastoreTestAction(Action):

    def run(self):
        t_cls = DummyClass()
        print('Tests begin: %s' % t_cls.now())
        self._test_datastore_actions_via_client()
        self._test_datastore_actions_via_action_service()
        print('Tests end: %s' % t_cls.now())

    def _test_datastore_actions_via_client(self):
        print('Test datastore access via raw client.')
        client = Client(base_url='http://localhost')

        test_name = 'st2tests.putin'
        test_value = 'putout'
        # Put
        kv = KeyValuePair(name=test_name, value=test_value)
        client.keys.update(kv)
        # print('Wrote key: %s value: %s to datastore.' % (test_name, test_value))

        # Get
        val = client.keys.get_by_name(name=test_name)
        if val.value != test_value:
            raise Exception('KeyValue access failed on GET: %s' % test_name)
        # print('Got value: %s from datastore.' % val.value)

        # Delete
        client.keys.delete(val)
        # print('Successfully delete key: %s from datastore.' % test_name)

    def _test_datastore_actions_via_action_service(self):
        print('Test datastore access via action_service')

        # Note: "decrypt" option requires admin access and when we generate service token that
        # token isn't granted admin access so it won't work.
        # To make the tests also pass on st2enteprise with RBAC we use explicit admin token
        admin_token = os.environ.get('ST2_AUTH_TOKEN')

        try:
            # Method was made public in v2.9dev
            self.action_service.datastore_service.get_api_client()
        except AttributeError:
            self.action_service.datastore_service._get_api_client()

        os.environ['ST2_AUTH_TOKEN'] = admin_token

        data = {'somedata': 'foobar'}
        data_json_str = json.dumps(data)

        # Add a value to the datastore
        self.action_service.set_value(name='cache', value=data_json_str)

        # Retrieve a value
        value = self.action_service.get_value('cache')
        retrieved_data = json.loads(value)
        if 'somedata' not in retrieved_data:
            raise Exception('Retrieved incorrect value from datastore: %s. Expected: %s' %
                            (retrieved_data, data))

        if retrieved_data['somedata'] != 'foobar':
            raise Exception('Datastore value corrupted!!!')

        # Delete a value
        self.action_service.delete_value('cache')

        # Add an encrypted value to the datastore
        self.action_service.set_value(name='cache', value='foo', encrypt=True)

        # decrypted value should match
        value = self.action_service.get_value('cache', decrypt=True)
        if not value or value != 'foo':
            raise Exception('Retrieved incorrect value from datastore: %s. Expected: %s' %
                            (value, 'foo'))

        # non-decrypted value should not match
        value = self.action_service.get_value('cache')
        if not value or value == 'foo':
            raise Exception('Retrieved incorrect value from datastore: %s. Did not expect: %s' %
                            (value, 'foo'))

        # Delete a value
        self.action_service.delete_value('cache')
<|endoftext|>


<commit_before>####ASCII RESOURCES####
startAscii = """
███████╗████████╗ █████╗ ██████╗ ████████╗███████╗██████╗
██╔════╝╚══██╔══╝██╔══██╗██╔══██╗╚══██╔══╝██╔════╝██╔══██╗
███████╗   ██║   ███████║██████╔╝   ██║   █████╗  ██║  ██║
╚════██║   ██║   ██╔══██║██╔══██╗   ██║   ██╔══╝  ██║  ██║
███████║   ██║   ██║  ██║██║  ██║   ██║   ███████╗██████╔╝
╚══════╝   ╚═╝   ╚═╝  ╚═╝╚═╝  ╚═╝   ╚═╝   ╚══════╝╚═════╝

    ███████╗███████╗██████╗ ██╗   ██╗███████╗██████╗
    ██╔════╝██╔════╝██╔══██╗██║   ██║██╔════╝██╔══██╗
    ███████╗█████╗  ██████╔╝██║   ██║█████╗  ██████╔╝
    ╚════██║██╔══╝  ██╔══██╗╚██╗ ██╔╝██╔══╝  ██╔══██╗
    ███████║███████╗██║  ██║ ╚████╔╝ ███████╗██║  ██║
    ╚══════╝╚══════╝╚═╝  ╚═╝  ╚═══╝  ╚══════╝╚═╝  ╚═╝
"""

allStart = [startAscii]

####MESSAGE RESOURCES####
spamWarning = "Please stop spamming"
joinMsg = "<SERVER>: {0} has joined the chat"
leaveMsg = "<SERVER>: {0} has disconnected from the chat"
unsafeLeaveMsg = "<SERVER>: {0} has unexpectedly disconnected from the chat"
nickChangeMsg = "<SERVER>: {0} has changed their nickname to {1}"
<commit_msg>Updated server resources<commit_after>####ASCII RESOURCES####
startAscii = """
███████╗████████╗ █████╗ ██████╗ ████████╗███████╗██████╗
██╔════╝╚══██╔══╝██╔══██╗██╔══██╗╚══██╔══╝██╔════╝██╔══██╗
███████╗   ██║   ███████║██████╔╝   ██║   █████╗  ██║  ██║
╚════██║   ██║   ██╔══██║██╔══██╗   ██║   ██╔══╝  ██║  ██║
███████║   ██║   ██║  ██║██║  ██║   ██║   ███████╗██████╔╝
╚══════╝   ╚═╝   ╚═╝  ╚═╝╚═╝  ╚═╝   ╚═╝   ╚══════╝╚═════╝

    ███████╗███████╗██████╗ ██╗   ██╗███████╗██████╗
    ██╔════╝██╔════╝██╔══██╗██║   ██║██╔════╝██╔══██╗
    ███████╗█████╗  ██████╔╝██║   ██║█████╗  ██████╔╝
    ╚════██║██╔══╝  ██╔══██╗╚██╗ ██╔╝██╔══╝  ██╔══██╗
    ███████║███████╗██║  ██║ ╚████╔╝ ███████╗██║  ██║
    ╚══════╝╚══════╝╚═╝  ╚═╝  ╚═══╝  ╚══════╝╚═╝  ╚═╝
"""

allStart = [startAscii]

####MESSAGE RESOURCES####
spamWarning = "Please stop spamming"
joinMsg = "server={0} has joined the chat"
leaveMsg = "server={0} has disconnected from the chat"
unsafeLeaveMsg = "server={0} has unexpectedly disconnected from the chat"
nickChangeMsg = "server={0} has changed their nickname to {1}"
<|endoftext|>


<commit_before>import os, sys
import argparse
from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop

def _go(args):
    if args.config is not None:
        path = os.path.abspath(args.config.name)
        os.environ['PAPIKA_SETTINGS'] = path
        args.config.close()

    from . import app
    port = app.app.config['PORT']
    app.setup()

    if args.mode == 'dev':
        print("Starting development server on port %d..." % port)
        app.app.run(port=port)
    elif args.mode == 'prd':
        http_server = HTTPServer(WSGIContainer(app.app))
        http_server.listen(port)
        print("Starting production server on port %d..." % port)
        IOLoop.instance().start()
    else:
        sys.stderr.write("Invalid mode %s, aborting.\n" % args.mode)
        sys.exit(1)

def main():

    _desc = '''
    Run the papika telemetry web server.
    '''
    parser = argparse.ArgumentParser(description=_desc)

    mut = parser.add_mutually_exclusive_group()
    mut.add_argument('-d', '--dev', action='store_const', dest='mode', const='dev', default='dev',
        help="Run in development mode. Will use flask's built-in auto-reloading web server"
    )
    mut.add_argument('-p', '--prd', action='store_const', dest='mode', const='prd',
        help="Run in production mode. Will use an actual web server (gevent)."
    )

    parser.add_argument('config', nargs='?', default=None, type=argparse.FileType('r'),
        help="Config file to use for the server. An alternative is to set the PAPIKA_SETTINGS environment variable."
    )

    _go(parser.parse_args())

<commit_msg>rename environment variable to PAPIKA_CONFIG<commit_after>import os, sys
import argparse
from tornado.wsgi import WSGIContainer
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop

def _go(args):
    if args.config is not None:
        path = os.path.abspath(args.config.name)
        os.environ['PAPIKA_CONFIG'] = path
        args.config.close()

    from . import app
    port = app.app.config['PORT']
    app.setup()

    if args.mode == 'dev':
        print("Starting development server on port %d..." % port)
        app.app.run(port=port)
    elif args.mode == 'prd':
        http_server = HTTPServer(WSGIContainer(app.app))
        http_server.listen(port)
        print("Starting production server on port %d..." % port)
        IOLoop.instance().start()
    else:
        sys.stderr.write("Invalid mode %s, aborting.\n" % args.mode)
        sys.exit(1)

def main():

    _desc = '''
    Run the papika telemetry web server.
    '''
    parser = argparse.ArgumentParser(description=_desc)

    mut = parser.add_mutually_exclusive_group()
    mut.add_argument('-d', '--dev', action='store_const', dest='mode', const='dev', default='dev',
        help="Run in development mode. Will use flask's built-in auto-reloading web server"
    )
    mut.add_argument('-p', '--prd', action='store_const', dest='mode', const='prd',
        help="Run in production mode. Will use an actual web server (gevent)."
    )

    parser.add_argument('config', nargs='?', default=None, type=argparse.FileType('r'),
        help="Config file to use for the server. An alternative is to set the PAPIKA_CONFIG environment variable."
    )

    _go(parser.parse_args())

<|endoftext|>


<commit_before>from numpy import prod
from random import randint

DNA_ALPHABET = "acgt"
# constants which may be used in other scripts/the simulation itself
# vector of mismatch values with index relative to distance from PAM (Hsu et al. from MIT)
mismatch_decay_values = [0.000, 0.000, 0.014, 0.000, 0.000, 0.395, 0.317, 0.000, 0.389, 0.079,
                         0.445, 0.508, 0.613, 0.841, 0.732, 0.828, 0.615, 0.804, 0.685, 0.583]
# average time for SpyCas9 cutting in perfect match case
average_cut_time = 60.0

def prob_concentration(concentration, num_mismatches):
    """Computes the concentration factor of the total cut probability
    Args:
        concentration: intranuclear gRNA-cas9 steady state concentration in nM
        num_mismatches: mismatches between gRNA and target
    Returns:
        float in [0.0, 1.0]
    Notes:
        - the formulas were generated using multiple regression in R with data from Kuscu et al 2014
    """
    if num_mismatches == 0:  # no mismatches case
        return (10**(-0.9935))*(concentration**0.465691)
    else:
        return (10**(0.036775*num_mismatches-1.616842))*(concentration**0.29122)

def prob_cut(grna, target, concentration, dt):
    """Probability of cleavage function
    Args:
        grna: 20 long string of nucleotides
        target: 20 long string of nucleotides
        concentration: intranuclear gRNA-cas9 steady state concentration in nM
        dt: simulation timestep
    Returns:
        float in [0.0, 1.0] representing the probability of cutting within the window of time dt
    Notes:
        - use Hsu et al. (from MIT) data for the decay probability of cutting with mismatches
        - the affect of dt is accounted for by assuming uniform cutting probability over time with a set time to
          bond and cut of 60 s, this is very arbitrary and based on only spurious mentions in a paper of time,
          if anyone has better implementation ideas add them
        - in combining the efficiencies, I assumed independence of cut probability effects, this is once again
          arbitrary, and in fact represents a boundary case to the true interaction, likely the true affects
          are more highly negatively dependent
        - the relationship to concentration was determined using multiple regression on data from at
          2014 paper by Kuscu, here they determined the proportion bonded given different concentrations
          and numbers of matching base pairs, this is a best case, however, as they only did variations on the
          matches furthest from the PAM site, which are very non-specific this function is separated into
          the case without mismatches and the case with n mismatches because the case without mismatches
          is individually important and better fit by a particular level curve
    """
    assert len(grna) == len(target)
    prob_factor_time = dt/average_cut_time  # assume uniform distribution in time - TODO change to poisson
    mismatch_decay_subset = [mismatch_decay_values[idx] for idx in xrange(len(grna)) if grna[idx] != target[idx]]
    prob_factor_mismatch = prod(mismatch_decay_subset)
    prob_factor_concentration = prob_concentration(concentration, len(mismatch_decay_subset))
    return prob_factor_time * prob_factor_mismatch * prob_factor_concentration

def nt_rand(insertion_size):
    """Takes insertion size and produces random insertion string
    """
    insertion = ""
    for x in range(insertion_size):
        insertion += DNA_ALPHABET[randint(0, 3)]

    return insertion

def indel():
    del_left = randint(0, 4)
    del_right = randint(0, 4)
    insert = randint(0, 8)

    return [del_left, del_right, insert]
<commit_msg>minor cleanup for probabilistic.py<commit_after>from numpy import prod
from random import randint

# constants which may be used in other scripts/the simulation itself
DNA_ALPHABET = "acgt"
# vector of mismatch values with index relative to distance from PAM (Hsu et al. from MIT)
mismatch_decay_values = [0.000, 0.000, 0.014, 0.000, 0.000, 0.395, 0.317, 0.000, 0.389, 0.079,
                         0.445, 0.508, 0.613, 0.841, 0.732, 0.828, 0.615, 0.804, 0.685, 0.583]
average_cut_time = 60.0  # average time for SpyCas9 cutting in perfect match case

def prob_concentration(concentration, num_mismatches):
    """Computes the concentration factor of the total cut probability
    Args:
        concentration: intranuclear gRNA-cas9 steady state concentration in nM
        num_mismatches: mismatches between gRNA and target
    Returns:
        float in [0.0, 1.0]
    Notes:
        - the formulas were generated using multiple regression in R with data from Kuscu et al 2014
    """
    if num_mismatches == 0:  # no mismatches case
        return (10**(-0.9935))*(concentration**0.465691)
    else:
        return (10**(0.036775*num_mismatches-1.616842))*(concentration**0.29122)

def prob_cut(grna, target, concentration, dt):
    """Probability of cleavage function
    Args:
        grna: 20 long string of nucleotides
        target: 20 long string of nucleotides
        concentration: intranuclear gRNA-cas9 steady state concentration in nM
        dt: simulation timestep
    Returns:
        float in [0.0, 1.0] representing the probability of cutting within the window of time dt
    Notes:
        - use Hsu et al. (from MIT) data for the decay probability of cutting with mismatches
        - the affect of dt is accounted for by assuming uniform cutting probability over time with a set time to
          bond and cut of 60 s, this is very arbitrary and based on only spurious mentions in a paper of time,
          if anyone has better implementation ideas add them
        - in combining the efficiencies, I assumed independence of cut probability effects, this is once again
          arbitrary, and in fact represents a boundary case to the true interaction, likely the true affects
          are more highly negatively dependent
        - the relationship to concentration was determined using multiple regression on data from at
          2014 paper by Kuscu, here they determined the proportion bonded given different concentrations
          and numbers of matching base pairs, this is a best case, however, as they only did variations on the
          matches furthest from the PAM site, which are very non-specific this function is separated into
          the case without mismatches and the case with n mismatches because the case without mismatches
          is individually important and better fit by a particular level curve
    """
    assert len(grna) == len(target)
    prob_factor_time = dt/average_cut_time  # assume uniform distribution in time - TODO change to poisson
    mismatch_decay_subset = [mismatch_decay_values[idx] for idx in xrange(len(grna)) if grna[idx] != target[idx]]
    prob_factor_mismatch = prod(mismatch_decay_subset)
    prob_factor_concentration = prob_concentration(concentration, len(mismatch_decay_subset))
    return prob_factor_time * prob_factor_mismatch * prob_factor_concentration

def nt_rand(insertion_size):
    """Takes insertion size and produces random insertion string
    """
    insertion = ""
    for x in xrange(insertion_size):
        insertion += DNA_ALPHABET[randint(0, 3)]
    return insertion

def indel():
    """Returns highly random indel sizes
    """
    del_left = randint(0, 4)
    del_right = randint(0, 4)
    insert = randint(0, 8)
    return [del_left, del_right, insert]
<|endoftext|>


<commit_before># -*- test-case-name: vumi.components.tests.test_message_store -*-
# -*- coding: utf-8 -*-

"""Message store."""

from uuid import uuid4

from twisted.internet.defer import returnValue

from vumi.message import TransportEvent, TransportUserMessage
from vumi.persist.model import Model, Manager
from vumi.persist.fields import (VumiMessage, ForeignKey, ListOf, Tag, Dynamic,
                                 Unicode)
from vumi.components.message_store_cache import MessageStoreCache

class Batch(Model):
    # key is batch_id
    tags = ListOf(Tag())
    metadata = Dynamic(Unicode())

class CurrentTag(Model):
    # key is flattened tag
    current_batch = ForeignKey(Batch, null=True)
    tag = Tag()

    @staticmethod
    def _flatten_tag(tag):
        return "%s:%s" % tag

    @staticmethod
    def _split_key(key):
        return tuple(key.split(':', 1))

    @classmethod
    def _tag_and_key(cls, tag_or_key):
        if isinstance(tag_or_key, tuple):
            # key looks like a tag
            tag, key = tag_or_key, cls._flatten_tag(tag_or_key)
        else:
            tag, key = cls._split_key(tag_or_key), tag_or_key
        return tag, key

    def __init__(self, manager, key, _riak_object=None, **kw):
        tag, key = self._tag_and_key(key)
        if _riak_object is None:
            kw['tag'] = tag
        super(CurrentTag, self).__init__(manager, key,
                                         _riak_object=_riak_object, **kw)

    @classmethod
    def load(cls, manager, key, result=None):
        _tag, key = cls._tag_and_key(key)
        return super(CurrentTag, cls).load(manager, key, result)

class OutboundMessage(Model):
    # key is message_id
    msg = VumiMessage(TransportUserMessage)
    batch = ForeignKey(Batch, null=True)

class Event(Model):
    # key is message_id
    event = VumiMessage(TransportEvent)
    message = ForeignKey(OutboundMessage)

class InboundMessage(Model):
    # key is message_id
    msg = VumiMessage(TransportUserMessage)
    batch = ForeignKey(Batch, null=True)

class MessageStore(object):
    """Vumi message store.

    Message batches, inbound messages, outbound messages, events and
    information about which batch a tag is currently associated with is
    stored in Riak.

    A small amount of information about the state of a batch (i.e. number
    of messages in the batch, messages sent, acknowledgements and delivery
    reports received) is stored in Redis.
    """

    def __init__(self, manager, redis):
        self.manager = manager
        self.batches = manager.proxy(Batch)
        self.outbound_messages = manager.proxy(OutboundMessage)
        self.events = manager.proxy(Event)
        self.inbound_messages = manager.proxy(InboundMessage)
        self.current_tags = manager.proxy(CurrentTag)
        self.cache = MessageStoreCache(redis)

    @Manager.calls_manager
    def needs_reconciliation(self, batch_id, delta=0.01):
        """
        Check if a batch_id's cache values need to be reconciled with
        what's stored in the MessageStore.

        :param float delta:
            What an acceptable delta is for the cached values. Defaults to 0.01
            If the cached values are off by the delta then this returns True.
        """
        inbound = float((yield self.batch_inbound_count(batch_id)))
        cached_inbound = yield self.cache.count_inbound_message_keys(
            batch_id)

        if inbound and (abs(cached_inbound - inbound) / inbound) > delta:
            returnValue(True)

        outbound = float((yield self.batch_outbound_count(batch_id)))
        cached_outbound = yield self.cache.count_outbound_message_keys(
            batch_id)

        if outbound and (abs(cached_outbound - outbound) / outbound) > delta:
            returnValue(True)

        returnValue(False)

    @Manager.calls_manager
    def reconcile_cache(self, batch_id):
        yield self.reconcile_inbound_cache(batch_id)
        yield self.reconcile_outbound_cache(batch_id)
        yield self.reconcile_event_cache(batch_id)

    @Manager.calls_manager
    def reconcile_inbound_cache(self, batch_id):
        batch = yield self.batches.load(batch_id)
        inbound_messages = yield batch.backlinks.inboundmessages()
        for inbound in inbound_messages:
            yield self.cache.add_inbound_message(batch_id, inbound.msg)

    @Manager.calls_manager
    def reconcile_outbound_cache(self, batch_id):
        batch = yield self.batches.load(batch_id)
        outbound_messages = yield batch.backlinks.outboundmessages()
        for outbound in outbound_messages:
            yield self.cache.add_outbound_message(batch_id, outbound.msg)
            yield self.reconcile_event_cache(batch_id,
                outbound.msg['message_id'])

    @Manager.calls_manager
    def reconcile_event_cache(self, batch_id, message_id):
        events = yield self.message_events(message_id)
        for event in events:
            yield self.cache.add_event(batch_id, event, reconcile=True)

    @Manager.calls_manager
    def batch_start(self, tags, **metadata):
        batch_id = uuid4().get_hex()
        batch = self.batches(batch_id)
        batch.tags.extend(tags)
        for key, value in metadata.iteritems():
            batch.metadata[key] = value
        yield batch.save()

        for tag in tags:
            tag_record = yield self.current_tags.load(tag)
            if tag_record is None:
                tag_record = self.current_tags(tag)
            tag_record.current_batch.set(batch)
            yield tag_record.save()

        yield self.cache.batch_start(batch_id)
        returnValue(batch_id)

    @Manager.calls_manager
    def batch_done(self, batch_id):
        batch = yield self.batches.load(batch_id)
        tags = yield batch.backlinks.currenttags()
        for tag in tags:
            tag.current_batch.set(None)
            yield tag.save()

    @Manager.calls_manager
    def add_outbound_message(self, msg, tag=None, batch_id=None):
        msg_id = msg['message_id']
        msg_record = self.outbound_messages(msg_id, msg=msg)

        if batch_id is None and tag is not None:
            tag_record = yield self.current_tags.load(tag)
            if tag_record is not None:
                batch_id = tag_record.current_batch.key

        if batch_id is not None:
            msg_record.batch.key = batch_id
            yield self.cache.add_outbound_message(batch_id, msg)

        yield msg_record.save()

    @Manager.calls_manager
    def get_outbound_message(self, msg_id):
        msg = yield self.outbound_messages.load(msg_id)
        returnValue(msg.msg if msg is not None else None)

    @Manager.calls_manager
    def add_event(self, event):
        event_id = event['event_id']
        msg_id = event['user_message_id']
        event_record = self.events(event_id, event=event, message=msg_id)
        yield event_record.save()

        msg_record = yield self.outbound_messages.load(msg_id)
        if msg_record is not None:
            batch_id = msg_record.batch.key
            if batch_id is not None:
                yield self.cache.add_event(batch_id, event)

    @Manager.calls_manager
    def get_event(self, event_id):
        event = yield self.events.load(event_id)
        returnValue(event.event if event is not None else None)

    @Manager.calls_manager
    def add_inbound_message(self, msg, tag=None, batch_id=None):
        msg_id = msg['message_id']
        msg_record = self.inbound_messages(msg_id, msg=msg)

        if batch_id is None and tag is not None:
            tag_record = yield self.current_tags.load(tag)
            if tag_record is not None:
                batch_id = tag_record.current_batch.key

        if batch_id is not None:
            msg_record.batch.key = batch_id
            self.cache.add_inbound_message(batch_id, msg)

        yield msg_record.save()

    @Manager.calls_manager
    def get_inbound_message(self, msg_id):
        msg = yield self.inbound_messages.load(msg_id)
        returnValue(msg.msg if msg is not None else None)

    def get_batch(self, batch_id):
        return self.batches.load(batch_id)

    def get_tag_info(self, tag):
        tagmdl = self.current_tags.load(tag)
        if tagmdl is None:
            tagmdl = self.current_tags(tag)
        return tagmdl

    def batch_status(self, batch_id):
        return self.cache.get_event_status(batch_id)

    @Manager.calls_manager
    def batch_messages(self, batch_id):
        batch = yield self.batches.load(batch_id)
        messages = yield batch.backlinks.outboundmessages()
        returnValue([m.msg for m in messages])

    @Manager.calls_manager
    def batch_replies(self, batch_id):
        batch = yield self.batches.load(batch_id)
        messages = yield batch.backlinks.inboundmessages()
        returnValue([m.msg for m in messages])

    @Manager.calls_manager
    def message_events(self, msg_id):
        message = yield self.outbound_messages.load(msg_id)
        events = yield message.backlinks.events()
        returnValue([e.event for e in events])

    @Manager.calls_manager
    def batch_inbound_count(self, batch_id):
        [count] = yield self.inbound_messages.by_index_count(batch=batch_id)
        returnValue(count)

    @Manager.calls_manager
    def batch_outbound_count(self, batch_id):
        [count] = yield self.outbound_messages.by_index_count(batch=batch_id)
        returnValue(count)
<commit_msg>use the new stuff 406 gives us<commit_after># -*- test-case-name: vumi.components.tests.test_message_store -*-
# -*- coding: utf-8 -*-

"""Message store."""

from uuid import uuid4

from twisted.internet.defer import returnValue

from vumi.message import TransportEvent, TransportUserMessage
from vumi.persist.model import Model, Manager
from vumi.persist.fields import (VumiMessage, ForeignKey, ListOf, Tag, Dynamic,
                                 Unicode)
from vumi.components.message_store_cache import MessageStoreCache

class Batch(Model):
    # key is batch_id
    tags = ListOf(Tag())
    metadata = Dynamic(Unicode())

class CurrentTag(Model):
    # key is flattened tag
    current_batch = ForeignKey(Batch, null=True)
    tag = Tag()

    @staticmethod
    def _flatten_tag(tag):
        return "%s:%s" % tag

    @staticmethod
    def _split_key(key):
        return tuple(key.split(':', 1))

    @classmethod
    def _tag_and_key(cls, tag_or_key):
        if isinstance(tag_or_key, tuple):
            # key looks like a tag
            tag, key = tag_or_key, cls._flatten_tag(tag_or_key)
        else:
            tag, key = cls._split_key(tag_or_key), tag_or_key
        return tag, key

    def __init__(self, manager, key, _riak_object=None, **kw):
        tag, key = self._tag_and_key(key)
        if _riak_object is None:
            kw['tag'] = tag
        super(CurrentTag, self).__init__(manager, key,
                                         _riak_object=_riak_object, **kw)

    @classmethod
    def load(cls, manager, key, result=None):
        _tag, key = cls._tag_and_key(key)
        return super(CurrentTag, cls).load(manager, key, result)

class OutboundMessage(Model):
    # key is message_id
    msg = VumiMessage(TransportUserMessage)
    batch = ForeignKey(Batch, null=True)

class Event(Model):
    # key is message_id
    event = VumiMessage(TransportEvent)
    message = ForeignKey(OutboundMessage)

class InboundMessage(Model):
    # key is message_id
    msg = VumiMessage(TransportUserMessage)
    batch = ForeignKey(Batch, null=True)

class MessageStore(object):
    """Vumi message store.

    Message batches, inbound messages, outbound messages, events and
    information about which batch a tag is currently associated with is
    stored in Riak.

    A small amount of information about the state of a batch (i.e. number
    of messages in the batch, messages sent, acknowledgements and delivery
    reports received) is stored in Redis.
    """

    def __init__(self, manager, redis):
        self.manager = manager
        self.batches = manager.proxy(Batch)
        self.outbound_messages = manager.proxy(OutboundMessage)
        self.events = manager.proxy(Event)
        self.inbound_messages = manager.proxy(InboundMessage)
        self.current_tags = manager.proxy(CurrentTag)
        self.cache = MessageStoreCache(redis)

    @Manager.calls_manager
    def needs_reconciliation(self, batch_id, delta=0.01):
        """
        Check if a batch_id's cache values need to be reconciled with
        what's stored in the MessageStore.

        :param float delta:
            What an acceptable delta is for the cached values. Defaults to 0.01
            If the cached values are off by the delta then this returns True.
        """
        inbound = float((yield self.batch_inbound_count(batch_id)))
        cached_inbound = yield self.cache.count_inbound_message_keys(
            batch_id)

        if inbound and (abs(cached_inbound - inbound) / inbound) > delta:
            returnValue(True)

        outbound = float((yield self.batch_outbound_count(batch_id)))
        cached_outbound = yield self.cache.count_outbound_message_keys(
            batch_id)

        if outbound and (abs(cached_outbound - outbound) / outbound) > delta:
            returnValue(True)

        returnValue(False)

    @Manager.calls_manager
    def reconcile_cache(self, batch_id):
        yield self.reconcile_inbound_cache(batch_id)
        yield self.reconcile_outbound_cache(batch_id)
        yield self.reconcile_event_cache(batch_id)

    @Manager.calls_manager
    def reconcile_inbound_cache(self, batch_id):
        batch = yield self.batches.load(batch_id)
        inbound_messages = yield batch.backlinks.inboundmessages()
        for inbound in inbound_messages:
            yield self.cache.add_inbound_message(batch_id, inbound.msg)

    @Manager.calls_manager
    def reconcile_outbound_cache(self, batch_id):
        batch = yield self.batches.load(batch_id)
        outbound_messages = yield batch.backlinks.outboundmessages()
        for outbound in outbound_messages:
            yield self.cache.add_outbound_message(batch_id, outbound.msg)
            yield self.reconcile_event_cache(batch_id,
                outbound.msg['message_id'])

    @Manager.calls_manager
    def reconcile_event_cache(self, batch_id, message_id):
        events = yield self.message_events(message_id)
        for event in events:
            yield self.cache.add_event(batch_id, event)

    @Manager.calls_manager
    def batch_start(self, tags, **metadata):
        batch_id = uuid4().get_hex()
        batch = self.batches(batch_id)
        batch.tags.extend(tags)
        for key, value in metadata.iteritems():
            batch.metadata[key] = value
        yield batch.save()

        for tag in tags:
            tag_record = yield self.current_tags.load(tag)
            if tag_record is None:
                tag_record = self.current_tags(tag)
            tag_record.current_batch.set(batch)
            yield tag_record.save()

        yield self.cache.batch_start(batch_id)
        returnValue(batch_id)

    @Manager.calls_manager
    def batch_done(self, batch_id):
        batch = yield self.batches.load(batch_id)
        tags = yield batch.backlinks.currenttags()
        for tag in tags:
            tag.current_batch.set(None)
            yield tag.save()

    @Manager.calls_manager
    def add_outbound_message(self, msg, tag=None, batch_id=None):
        msg_id = msg['message_id']
        msg_record = self.outbound_messages(msg_id, msg=msg)

        if batch_id is None and tag is not None:
            tag_record = yield self.current_tags.load(tag)
            if tag_record is not None:
                batch_id = tag_record.current_batch.key

        if batch_id is not None:
            msg_record.batch.key = batch_id
            yield self.cache.add_outbound_message(batch_id, msg)

        yield msg_record.save()

    @Manager.calls_manager
    def get_outbound_message(self, msg_id):
        msg = yield self.outbound_messages.load(msg_id)
        returnValue(msg.msg if msg is not None else None)

    @Manager.calls_manager
    def add_event(self, event):
        event_id = event['event_id']
        msg_id = event['user_message_id']
        event_record = self.events(event_id, event=event, message=msg_id)
        yield event_record.save()

        msg_record = yield self.outbound_messages.load(msg_id)
        if msg_record is not None:
            batch_id = msg_record.batch.key
            if batch_id is not None:
                yield self.cache.add_event(batch_id, event)

    @Manager.calls_manager
    def get_event(self, event_id):
        event = yield self.events.load(event_id)
        returnValue(event.event if event is not None else None)

    @Manager.calls_manager
    def add_inbound_message(self, msg, tag=None, batch_id=None):
        msg_id = msg['message_id']
        msg_record = self.inbound_messages(msg_id, msg=msg)

        if batch_id is None and tag is not None:
            tag_record = yield self.current_tags.load(tag)
            if tag_record is not None:
                batch_id = tag_record.current_batch.key

        if batch_id is not None:
            msg_record.batch.key = batch_id
            self.cache.add_inbound_message(batch_id, msg)

        yield msg_record.save()

    @Manager.calls_manager
    def get_inbound_message(self, msg_id):
        msg = yield self.inbound_messages.load(msg_id)
        returnValue(msg.msg if msg is not None else None)

    def get_batch(self, batch_id):
        return self.batches.load(batch_id)

    def get_tag_info(self, tag):
        tagmdl = self.current_tags.load(tag)
        if tagmdl is None:
            tagmdl = self.current_tags(tag)
        return tagmdl

    def batch_status(self, batch_id):
        return self.cache.get_event_status(batch_id)

    @Manager.calls_manager
    def batch_messages(self, batch_id):
        batch = yield self.batches.load(batch_id)
        messages = yield batch.backlinks.outboundmessages()
        returnValue([m.msg for m in messages])

    @Manager.calls_manager
    def batch_replies(self, batch_id):
        batch = yield self.batches.load(batch_id)
        messages = yield batch.backlinks.inboundmessages()
        returnValue([m.msg for m in messages])

    @Manager.calls_manager
    def message_events(self, msg_id):
        message = yield self.outbound_messages.load(msg_id)
        events = yield message.backlinks.events()
        returnValue([e.event for e in events])

    @Manager.calls_manager
    def batch_inbound_count(self, batch_id):
        [count] = yield self.inbound_messages.by_index_count(batch=batch_id)
        returnValue(count)

    @Manager.calls_manager
    def batch_outbound_count(self, batch_id):
        [count] = yield self.outbound_messages.by_index_count(batch=batch_id)
        returnValue(count)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Test links:
# http://filer.net/get/ivgf5ztw53et3ogd
# http://filer.net/get/hgo14gzcng3scbvv

import pycurl
import re

from urlparse import urljoin

from module.plugins.internal.CaptchaService import ReCaptcha
from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo

class FilerNet(SimpleHoster):
    __name__    = "FilerNet"
    __type__    = "hoster"
    __version__ = "0.16"

    __pattern__ = r'https?://(?:www\.)?filer\.net/get/\w+'

    __description__ = """Filer.net hoster plugin"""
    __license__     = "GPLv3"
    __authors__     = [("stickell", "l.stickell@yahoo.it"),
                       ("Walter Purcaro", "vuolter@gmail.com")]

    INFO_PATTERN    = r'
Free Download (?P<N>\S+) (?P<S>[\w.]+) (?P[\w^_]+)
'
    OFFLINE_PATTERN = r'Nicht gefunden'

    LINK_FREE_PATTERN = LINK_PREMIUM_PATTERN = r'href="([^"]+)">Get download'

    def checkErrors(self):
        # Wait between downloads
        m = re.search(r'musst du (\d+) Sekunden warten', self.html)
        if m:
            errmsg = self.info['error'] = _("Wait between free downloads")
            self.retry(wait_time=int(m.group(1)), reason=errmsg)

        self.info.pop('error', None)

    def handleFree(self, pyfile):
        inputs = self.parseHtmlForm(input_names={'token': re.compile(r'.+')})[1]
        if 'token' not in inputs:
            self.error(_("Unable to detect token"))

        self.html = self.load(pyfile.url, post={'token': inputs['token']}, decode=True)

        inputs = self.parseHtmlForm(input_names={'hash': re.compile(r'.+')})[1]
        if 'hash' not in inputs:
            self.error(_("Unable to detect hash"))

        recaptcha           = ReCaptcha(self)
        response, challenge = recaptcha.challenge()

        #@NOTE: Work-around for v0.4.9 just_header issue
        #@TODO: Check for v0.4.10
        self.req.http.c.setopt(pycurl.FOLLOWLOCATION, 0)
        self.load(pyfile.url, post={'recaptcha_challenge_field': challenge,
                                    'recaptcha_response_field' : response,
                                    'hash'                     : inputs['hash']})
        self.req.http.c.setopt(pycurl.FOLLOWLOCATION, 1)

        if 'location' in self.req.http.header.lower():
            self.link = re.search(r'location: (\S+)', self.req.http.header, re.I).group(1)
            self.correctCaptcha()
        else:
            self.invalidCaptcha()

getInfo = create_getInfo(FilerNet)
<commit_msg>[FilerNet] Fix WAIT_PATTERN<commit_after># -*- coding: utf-8 -*-
#
# Test links:
# http://filer.net/get/ivgf5ztw53et3ogd
# http://filer.net/get/hgo14gzcng3scbvv

import pycurl
import re

from urlparse import urljoin

from module.plugins.internal.CaptchaService import ReCaptcha
from module.plugins.internal.SimpleHoster import SimpleHoster, create_getInfo

class FilerNet(SimpleHoster):
    __name__    = "FilerNet"
    __type__    = "hoster"
    __version__ = "0.17"

    __pattern__ = r'https?://(?:www\.)?filer\.net/get/\w+'

    __description__ = """Filer.net hoster plugin"""
    __license__     = "GPLv3"
    __authors__     = [("stickell", "l.stickell@yahoo.it"),
                       ("Walter Purcaro", "vuolter@gmail.com")]

    INFO_PATTERN    = r'
Free Download (?P<N>\S+) (?P<S>[\w.]+) (?P[\w^_]+)
'
    OFFLINE_PATTERN = r'Nicht gefunden'

    WAIT_PATTERN = r'musst du (\d+)'

    LINK_FREE_PATTERN = LINK_PREMIUM_PATTERN = r'href="([^"]+)">Get download'

    def handleFree(self, pyfile):
        inputs = self.parseHtmlForm(input_names={'token': re.compile(r'.+')})[1]
        if 'token' not in inputs:
            self.error(_("Unable to detect token"))

        self.html = self.load(pyfile.url, post={'token': inputs['token']}, decode=True)

        inputs = self.parseHtmlForm(input_names={'hash': re.compile(r'.+')})[1]
        if 'hash' not in inputs:
            self.error(_("Unable to detect hash"))

        recaptcha           = ReCaptcha(self)
        response, challenge = recaptcha.challenge()

        #@NOTE: Work-around for v0.4.9 just_header issue
        #@TODO: Check for v0.4.10
        self.req.http.c.setopt(pycurl.FOLLOWLOCATION, 0)
        self.load(pyfile.url, post={'recaptcha_challenge_field': challenge,
                                    'recaptcha_response_field' : response,
                                    'hash'                     : inputs['hash']})
        self.req.http.c.setopt(pycurl.FOLLOWLOCATION, 1)

        if 'location' in self.req.http.header.lower():
            self.link = re.search(r'location: (\S+)', self.req.http.header, re.I).group(1)
            self.correctCaptcha()
        else:
            self.invalidCaptcha()

getInfo = create_getInfo(FilerNet)
<|endoftext|>


<commit_before>from copy import deepcopy
import json
from pathlib import Path
import yaml

def generate_bids_skeleton(target_path, bids_config):
    """
    Converts a BIDS directory in dictionary form to a file structure.

    Parameters
    ----------
    target_path : str
        Path to generate BIDS directory at
    bids_config : dict or str
        Configuration on how to create the BIDS directory.
        This can be either a dictionary, JSON, or YAML file.
    """

    if isinstance(bids_config, dict):
        # ensure dictionary remains unaltered
        bids_dict = deepcopy(bids_config)
    elif isinstance(bids_config, str):
        bids_config = Path(bids_config).read_text()
        try:
            bids_dict = json.loads(bids_config)
        except json.JSONDecodeError:
            bids_dict = yaml.load(bids_config, Loader=yaml.Loader)

    root = Path(target_path).absolute()
    root.mkdir(parents=True, exist_ok=True)

    desc = bids_dict.pop("dataset_description", None)
    if desc is None:
        # default description
        desc = {"Name": "Default", "BIDSVersion": "1.6.0"}
    to_json(root / "dataset_description.json", desc)

    cached_subject_data = None
    for subject, sessions in bids_dict.items():
        bids_subject = subject if subject.startswith("sub-") else f"sub-{subject}"
        subj_path = root / bids_subject
        subj_path.mkdir(exist_ok=True)

        if sessions == "*":  # special case to copy previous subject data
            sessions = cached_subject_data.copy()

        if isinstance(sessions, dict):  # single session
            sessions.update({"session": None})
            sessions = [sessions]

        cached_subject_data = deepcopy(sessions)
        for session in sessions:

            ses_name = session.pop("session", None)
            if ses_name is not None:
                bids_session = ses_name if ses_name.startswith("ses-") else f"ses-{ses_name}"
                bids_prefix = f"{bids_subject}_{bids_session}"
                curr_path = subj_path / bids_session
                curr_path.mkdir(exist_ok=True)
            else:
                bids_prefix = bids_subject
                curr_path = subj_path

            # create modalities
            for modality, files in session.items():
                modality_path = curr_path / modality
                modality_path.mkdir(exist_ok=True)

                if isinstance(files, dict):  # single file / metadata combo
                    files = [files]

                for bids_file in files:
                    metadata = bids_file.pop("metadata", None)
                    suffix = bids_file.pop("suffix")
                    entities = combine_entities(**bids_file)
                    nii_file = modality_path / f"{bids_prefix}{entities}_{suffix}.nii.gz"
                    nii_file.touch()

                    if metadata is not None:
                        nii_metadata = nii_file.parent / nii_file.name.replace("nii.gz", "json")
                        to_json(nii_metadata, metadata)

def to_json(filename, data):
    filename = Path(filename)
    filename.write_text(json.dumps(data))
    return filename

def combine_entities(**entities):
    return f"_{'_'.join([f'{lab}-{val}' for lab, val in entities.items()])}" if entities else ""
<commit_msg>RF: Minor change in docstring<commit_after>from copy import deepcopy
import json
from pathlib import Path
import yaml

def generate_bids_skeleton(target_path, bids_config):
    """
    Converts a BIDS directory in dictionary form to a file structure.

    The BIDS configuration can either be a YAML or JSON file, or :obj:dict: object.

    Parameters
    ----------
    target_path : str
        Path to generate BIDS directory at
    bids_config : dict or str
        Configuration on how to create the BIDS directory.
    """

    if isinstance(bids_config, dict):
        # ensure dictionary remains unaltered
        bids_dict = deepcopy(bids_config)
    elif isinstance(bids_config, str):
        bids_config = Path(bids_config).read_text()
        try:
            bids_dict = json.loads(bids_config)
        except json.JSONDecodeError:
            bids_dict = yaml.load(bids_config, Loader=yaml.Loader)

    root = Path(target_path).absolute()
    root.mkdir(parents=True, exist_ok=True)

    desc = bids_dict.pop("dataset_description", None)
    if desc is None:
        # default description
        desc = {"Name": "Default", "BIDSVersion": "1.6.0"}
    to_json(root / "dataset_description.json", desc)

    cached_subject_data = None
    for subject, sessions in bids_dict.items():
        bids_subject = subject if subject.startswith("sub-") else f"sub-{subject}"
        subj_path = root / bids_subject
        subj_path.mkdir(exist_ok=True)

        if sessions == "*":  # special case to copy previous subject data
            sessions = cached_subject_data.copy()

        if isinstance(sessions, dict):  # single session
            sessions.update({"session": None})
            sessions = [sessions]

        cached_subject_data = deepcopy(sessions)
        for session in sessions:

            ses_name = session.pop("session", None)
            if ses_name is not None:
                bids_session = ses_name if ses_name.startswith("ses-") else f"ses-{ses_name}"
                bids_prefix = f"{bids_subject}_{bids_session}"
                curr_path = subj_path / bids_session
                curr_path.mkdir(exist_ok=True)
            else:
                bids_prefix = bids_subject
                curr_path = subj_path

            # create modalities
            for modality, files in session.items():
                modality_path = curr_path / modality
                modality_path.mkdir(exist_ok=True)

                if isinstance(files, dict):  # single file / metadata combo
                    files = [files]

                for bids_file in files:
                    metadata = bids_file.pop("metadata", None)
                    suffix = bids_file.pop("suffix")
                    entities = combine_entities(**bids_file)
                    nii_file = modality_path / f"{bids_prefix}{entities}_{suffix}.nii.gz"
                    nii_file.touch()

                    if metadata is not None:
                        nii_metadata = nii_file.parent / nii_file.name.replace("nii.gz", "json")
                        to_json(nii_metadata, metadata)

def to_json(filename, data):
    filename = Path(filename)
    filename.write_text(json.dumps(data))
    return filename

def combine_entities(**entities):
    return f"_{'_'.join([f'{lab}-{val}' for lab, val in entities.items()])}" if entities else ""
<|endoftext|>


<commit_before>import re  # parsing HTML with regexes LIKE A BOSS.

from django.utils.html import escape
from django.utils.safestring import mark_safe

from wagtail.wagtailcore.whitelist import Whitelister
from wagtail.wagtailcore.models import Page
from wagtail.wagtailcore import hooks

# Define a set of 'embed handlers' and 'link handlers'. These handle the translation
# of 'special' HTML elements in rich text - ones which we do not want to include
# verbatim in the DB representation because they embed information which is stored
# elsewhere in the database and is liable to change - from real HTML representation
# to DB representation and back again.

class PageLinkHandler(object):
    """
    PageLinkHandler will be invoked whenever we encounter an  element in HTML content
    with an attribute of data-linktype="page". The resulting element in the database
    representation will be:
    hello world
    """
    @staticmethod
    def get_db_attributes(tag):
        """
        Given an  tag that we've identified as a page link embed (because it has a
        data-linktype="page" attribute), return a dict of the attributes we should
        have on the resulting  element.
        """
        return {'id': tag['data-id']}

    @staticmethod
    def expand_db_attributes(attrs, for_editor):
        try:
            page = Page.objects.get(id=attrs['id'])

            if for_editor:
                editor_attrs = 'data-linktype="page" data-id="%d" ' % page.id
            else:
                editor_attrs = ''

            return '' % (editor_attrs, escape(page.url))
        except Page.DoesNotExist:
            return ""

EMBED_HANDLERS = {}
LINK_HANDLERS = {
    'page': PageLinkHandler,
}

has_loaded_embed_handlers = False
has_loaded_link_handlers = False

def get_embed_handler(embed_type):
    global EMBED_HANDLERS, has_loaded_embed_handlers

    if not has_loaded_embed_handlers:
        for hook in hooks.get_hooks('register_rich_text_embed_handler'):
            handler_name, handler = hook()
            EMBED_HANDLERS[handler_name] = handler

        has_loaded_embed_handlers = True

    return EMBED_HANDLERS[embed_type]

def get_link_handler(link_type):
    global LINK_HANDLERS, has_loaded_link_handlers

    if not has_loaded_link_handlers:
        for hook in hooks.get_hooks('register_rich_text_link_handler'):
            handler_name, handler = hook()
            LINK_HANDLERS[handler_name] = handler

        has_loaded_link_handlers = True

    return LINK_HANDLERS[link_type]

class DbWhitelister(Whitelister):
    """
    A custom whitelisting engine to convert the HTML as returned by the rich text editor
    into the pseudo-HTML format stored in the database (in which images, documents and other
    linked objects are identified by ID rather than URL):

    * implements a 'construct_whitelister_element_rules' hook so that other apps can modify
      the whitelist ruleset (e.g. to permit additional HTML elements beyond those in the base
      Whitelister module);
    * replaces any element with a 'data-embedtype' attribute with an <embed> element, with
      attributes supplied by the handler for that type as defined in EMBED_HANDLERS;
    * rewrites the attributes of any  element with a 'data-linktype' attribute, as
      determined by the handler for that type defined in LINK_HANDLERS, while keeping the
      element content intact.
    """
    has_loaded_custom_whitelist_rules = False

    @classmethod
    def clean(cls, html):
        if not cls.has_loaded_custom_whitelist_rules:
            for fn in hooks.get_hooks('construct_whitelister_element_rules'):
                cls.element_rules = cls.element_rules.copy()
                cls.element_rules.update(fn())
            cls.has_loaded_custom_whitelist_rules = True

        return super(DbWhitelister, cls).clean(html)

    @classmethod
    def clean_tag_node(cls, doc, tag):
        if 'data-embedtype' in tag.attrs:
            embed_type = tag['data-embedtype']
            # fetch the appropriate embed handler for this embedtype
            embed_handler = get_embed_handler(embed_type)
            embed_attrs = embed_handler.get_db_attributes(tag)
            embed_attrs['embedtype'] = embed_type

            embed_tag = doc.new_tag('embed', **embed_attrs)
            embed_tag.can_be_empty_element = True
            tag.replace_with(embed_tag)
        elif tag.name == 'a' and 'data-linktype' in tag.attrs:
            # first, whitelist the contents of this tag
            for child in tag.contents:
                cls.clean_node(doc, child)

            link_type = tag['data-linktype']
            link_handler = get_link_handler(link_type)
            link_attrs = link_handler.get_db_attributes(tag)
            link_attrs['linktype'] = link_type
            tag.attrs.clear()
            tag.attrs.update(**link_attrs)
        else:
            if tag.name == 'div':
                tag.name = 'p'

            super(DbWhitelister, cls).clean_tag_node(doc, tag)

FIND_A_TAG = re.compile(r']*)>')
FIND_EMBED_TAG = re.compile(r'<embed(\b[^>]*)/>')
FIND_ATTRS = re.compile(r'([\w-]+)\="([^"]*)"')

def extract_attrs(attr_string):
    """
    helper method to extract tag attributes as a dict. Does not escape HTML entities!
    """
    attributes = {}
    for name, val in FIND_ATTRS.findall(attr_string):
        attributes[name] = val
    return attributes

def expand_db_html(html, for_editor=False):
    """
    Expand database-representation HTML into proper HTML usable in either
    templates or the rich text editor
    """
    def replace_a_tag(m):
        attrs = extract_attrs(m.group(1))
        if 'linktype' not in attrs:
            # return unchanged
            return m.group(0)
        handler = get_link_handler(attrs['linktype'])
        return handler.expand_db_attributes(attrs, for_editor)

    def replace_embed_tag(m):
        attrs = extract_attrs(m.group(1))
        handler = get_embed_handler(attrs['embedtype'])
        return handler.expand_db_attributes(attrs, for_editor)

    html = FIND_A_TAG.sub(replace_a_tag, html)
    html = FIND_EMBED_TAG.sub(replace_embed_tag, html)
    return html

class RichText(object):
    """
    A custom object used to represent a renderable rich text value.
    Provides a 'source' property to access the original source code,
    and renders to the front-end HTML rendering.
    Used as the native value of a wagtailcore.blocks.field_block.RichTextBlock.
    """
    def __init__(self, source):
        self.source = (source or '')

    def __str__(self):
        return mark_safe('
' + expand_db_html(self.source) + '
')
<commit_msg>python2 unicode fix<commit_after>from __future__ import unicode_literals  # ensure that RichText.__str__ returns unicode

import re  # parsing HTML with regexes LIKE A BOSS.

from django.utils.encoding import python_2_unicode_compatible
from django.utils.html import escape
from django.utils.safestring import mark_safe

from wagtail.wagtailcore.whitelist import Whitelister
from wagtail.wagtailcore.models import Page
from wagtail.wagtailcore import hooks

# Define a set of 'embed handlers' and 'link handlers'. These handle the translation
# of 'special' HTML elements in rich text - ones which we do not want to include
# verbatim in the DB representation because they embed information which is stored
# elsewhere in the database and is liable to change - from real HTML representation
# to DB representation and back again.

class PageLinkHandler(object):
    """
    PageLinkHandler will be invoked whenever we encounter an  element in HTML content
    with an attribute of data-linktype="page". The resulting element in the database
    representation will be:
    hello world
    """
    @staticmethod
    def get_db_attributes(tag):
        """
        Given an  tag that we've identified as a page link embed (because it has a
        data-linktype="page" attribute), return a dict of the attributes we should
        have on the resulting  element.
        """
        return {'id': tag['data-id']}

    @staticmethod
    def expand_db_attributes(attrs, for_editor):
        try:
            page = Page.objects.get(id=attrs['id'])

            if for_editor:
                editor_attrs = 'data-linktype="page" data-id="%d" ' % page.id
            else:
                editor_attrs = ''

            return '' % (editor_attrs, escape(page.url))
        except Page.DoesNotExist:
            return ""

EMBED_HANDLERS = {}
LINK_HANDLERS = {
    'page': PageLinkHandler,
}

has_loaded_embed_handlers = False
has_loaded_link_handlers = False

def get_embed_handler(embed_type):
    global EMBED_HANDLERS, has_loaded_embed_handlers

    if not has_loaded_embed_handlers:
        for hook in hooks.get_hooks('register_rich_text_embed_handler'):
            handler_name, handler = hook()
            EMBED_HANDLERS[handler_name] = handler

        has_loaded_embed_handlers = True

    return EMBED_HANDLERS[embed_type]

def get_link_handler(link_type):
    global LINK_HANDLERS, has_loaded_link_handlers

    if not has_loaded_link_handlers:
        for hook in hooks.get_hooks('register_rich_text_link_handler'):
            handler_name, handler = hook()
            LINK_HANDLERS[handler_name] = handler

        has_loaded_link_handlers = True

    return LINK_HANDLERS[link_type]

class DbWhitelister(Whitelister):
    """
    A custom whitelisting engine to convert the HTML as returned by the rich text editor
    into the pseudo-HTML format stored in the database (in which images, documents and other
    linked objects are identified by ID rather than URL):

    * implements a 'construct_whitelister_element_rules' hook so that other apps can modify
      the whitelist ruleset (e.g. to permit additional HTML elements beyond those in the base
      Whitelister module);
    * replaces any element with a 'data-embedtype' attribute with an <embed> element, with
      attributes supplied by the handler for that type as defined in EMBED_HANDLERS;
    * rewrites the attributes of any  element with a 'data-linktype' attribute, as
      determined by the handler for that type defined in LINK_HANDLERS, while keeping the
      element content intact.
    """
    has_loaded_custom_whitelist_rules = False

    @classmethod
    def clean(cls, html):
        if not cls.has_loaded_custom_whitelist_rules:
            for fn in hooks.get_hooks('construct_whitelister_element_rules'):
                cls.element_rules = cls.element_rules.copy()
                cls.element_rules.update(fn())
            cls.has_loaded_custom_whitelist_rules = True

        return super(DbWhitelister, cls).clean(html)

    @classmethod
    def clean_tag_node(cls, doc, tag):
        if 'data-embedtype' in tag.attrs:
            embed_type = tag['data-embedtype']
            # fetch the appropriate embed handler for this embedtype
            embed_handler = get_embed_handler(embed_type)
            embed_attrs = embed_handler.get_db_attributes(tag)
            embed_attrs['embedtype'] = embed_type

            embed_tag = doc.new_tag('embed', **embed_attrs)
            embed_tag.can_be_empty_element = True
            tag.replace_with(embed_tag)
        elif tag.name == 'a' and 'data-linktype' in tag.attrs:
            # first, whitelist the contents of this tag
            for child in tag.contents:
                cls.clean_node(doc, child)

            link_type = tag['data-linktype']
            link_handler = get_link_handler(link_type)
            link_attrs = link_handler.get_db_attributes(tag)
            link_attrs['linktype'] = link_type
            tag.attrs.clear()
            tag.attrs.update(**link_attrs)
        else:
            if tag.name == 'div':
                tag.name = 'p'

            super(DbWhitelister, cls).clean_tag_node(doc, tag)

FIND_A_TAG = re.compile(r']*)>')
FIND_EMBED_TAG = re.compile(r'<embed(\b[^>]*)/>')
FIND_ATTRS = re.compile(r'([\w-]+)\="([^"]*)"')

def extract_attrs(attr_string):
    """
    helper method to extract tag attributes as a dict. Does not escape HTML entities!
    """
    attributes = {}
    for name, val in FIND_ATTRS.findall(attr_string):
        attributes[name] = val
    return attributes

def expand_db_html(html, for_editor=False):
    """
    Expand database-representation HTML into proper HTML usable in either
    templates or the rich text editor
    """
    def replace_a_tag(m):
        attrs = extract_attrs(m.group(1))
        if 'linktype' not in attrs:
            # return unchanged
            return m.group(0)
        handler = get_link_handler(attrs['linktype'])
        return handler.expand_db_attributes(attrs, for_editor)

    def replace_embed_tag(m):
        attrs = extract_attrs(m.group(1))
        handler = get_embed_handler(attrs['embedtype'])
        return handler.expand_db_attributes(attrs, for_editor)

    html = FIND_A_TAG.sub(replace_a_tag, html)
    html = FIND_EMBED_TAG.sub(replace_embed_tag, html)
    return html

@python_2_unicode_compatible
class RichText(object):
    """
    A custom object used to represent a renderable rich text value.
    Provides a 'source' property to access the original source code,
    and renders to the front-end HTML rendering.
    Used as the native value of a wagtailcore.blocks.field_block.RichTextBlock.
    """
    def __init__(self, source):
        self.source = (source or '')

    def __str__(self):
        return mark_safe('
' + expand_db_html(self.source) + '
')
<|endoftext|>


<commit_before>from django.conf import settings
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.db import models
from django.utils.translation import ugettext_lazy as _

from wagtail.admin.edit_handlers import FieldPanel
from wagtail.core.fields import RichTextField
from wagtail.core.models import Page
from wagtail.images.models import Image
from wagtail.images import get_image_model

IMAGE_ORDER_TYPES = (
    (1, 'Image title'),
    (2, 'Newest image first'),
)

class SimpleGalleryIndex(Page):
    intro_title = models.CharField(
        verbose_name=_('Intro title'),
        max_length=250,
        blank=True,
        help_text=_('Optional H1 title for the gallery page.')
    )
    intro_text = RichTextField(
        blank=True,
        verbose_name=_('Intro text'),
        help_text=_('Optional text to go with the intro text.')
    )
    collection = models.ForeignKey(
        'wagtailcore.Collection',
        verbose_name=_('Collection'),
        null=True,
        blank=False,
        on_delete=models.SET_NULL,
        related_name='+',
        help_text=_('Show images in this collection in the gallery view.')
    )
    images_per_page = models.IntegerField(
        default=8,
        verbose_name=_('Images per page'),
        help_text=_('How many images there should be on one page.')
    )
    use_lightbox = models.BooleanField(
        verbose_name=_('Use lightbox'),
        default=True,
        help_text=_('Use lightbox to view larger images when clicking the thumbnail.')
    )
    order_images_by = models.IntegerField(choices=IMAGE_ORDER_TYPES, default=1)

    content_panels = Page.content_panels + [
        FieldPanel('intro_title', classname='full title'),
        FieldPanel('intro_text', classname='full title'),
        FieldPanel('collection'),
        FieldPanel('images_per_page', classname='full title'),
        FieldPanel('use_lightbox'),
        FieldPanel('order_images_by'),
    ]

    @property
    def images(self):
        return get_gallery_images(self.collection.name, self)

    def get_context(self, request):
        images = self.images
        page = request.GET.get('page')
        paginator = Paginator(images, self.images_per_page)
        try:
            images = paginator.page(page)
        except PageNotAnInteger:
            images = paginator.page(1)
        except EmptyPage:
            images = paginator.page(paginator.num_pages)
        context = super(SimpleGalleryIndex, self).get_context(request)
        context['gallery_images'] = images
        return context

    class Meta:
        verbose_name = _('Gallery index')
        verbose_name_plural = _('Gallery indices')

    template = getattr(settings, 'SIMPLE_GALLERY_TEMPLATE', 'wagtail_simple_gallery/simple_gallery_index.html')

def get_gallery_images(collection, page=None, tags=None):
    images = None
    try:
        images = get_image_model().objects.filter(collection__name=collection)
        if page:
            if page.order_images_by == 0:
                images = images.order_by('title')
            elif page.order_images_by == 1:
                images = images.order_by('-created_at')
    except Exception as e:
        pass
    if images and tags:
        images = images.filter(tags__name__in=tags).distinct()
    return images
<commit_msg>Fixed small issue with the get_gallery_images conditional using the values 0 and 1, instead of the valid IMAGE_ORDER_TYPES choices, which are 1 and 2<commit_after>from django.conf import settings
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.db import models
from django.utils.translation import ugettext_lazy as _

from wagtail.admin.edit_handlers import FieldPanel
from wagtail.core.fields import RichTextField
from wagtail.core.models import Page
from wagtail.images.models import Image
from wagtail.images import get_image_model

IMAGE_ORDER_TYPES = (
    (1, 'Image title'),
    (2, 'Newest image first'),
)

class SimpleGalleryIndex(Page):
    intro_title = models.CharField(
        verbose_name=_('Intro title'),
        max_length=250,
        blank=True,
        help_text=_('Optional H1 title for the gallery page.')
    )
    intro_text = RichTextField(
        blank=True,
        verbose_name=_('Intro text'),
        help_text=_('Optional text to go with the intro text.')
    )
    collection = models.ForeignKey(
        'wagtailcore.Collection',
        verbose_name=_('Collection'),
        null=True,
        blank=False,
        on_delete=models.SET_NULL,
        related_name='+',
        help_text=_('Show images in this collection in the gallery view.')
    )
    images_per_page = models.IntegerField(
        default=8,
        verbose_name=_('Images per page'),
        help_text=_('How many images there should be on one page.')
    )
    use_lightbox = models.BooleanField(
        verbose_name=_('Use lightbox'),
        default=True,
        help_text=_('Use lightbox to view larger images when clicking the thumbnail.')
    )
    order_images_by = models.IntegerField(choices=IMAGE_ORDER_TYPES, default=1)

    content_panels = Page.content_panels + [
        FieldPanel('intro_title', classname='full title'),
        FieldPanel('intro_text', classname='full title'),
        FieldPanel('collection'),
        FieldPanel('images_per_page', classname='full title'),
        FieldPanel('use_lightbox'),
        FieldPanel('order_images_by'),
    ]

    @property
    def images(self):
        return get_gallery_images(self.collection.name, self)

    def get_context(self, request):
        images = self.images
        page = request.GET.get('page')
        paginator = Paginator(images, self.images_per_page)
        try:
            images = paginator.page(page)
        except PageNotAnInteger:
            images = paginator.page(1)
        except EmptyPage:
            images = paginator.page(paginator.num_pages)
        context = super(SimpleGalleryIndex, self).get_context(request)
        context['gallery_images'] = images
        return context

    class Meta:
        verbose_name = _('Gallery index')
        verbose_name_plural = _('Gallery indices')

    template = getattr(settings, 'SIMPLE_GALLERY_TEMPLATE', 'wagtail_simple_gallery/simple_gallery_index.html')

def get_gallery_images(collection, page=None, tags=None):
    images = None
    try:
        images = get_image_model().objects.filter(collection__name=collection)
        if page:
            if page.order_images_by == 1:
                images = images.order_by('title')
            elif page.order_images_by == 2:
                images = images.order_by('-created_at')
    except Exception as e:
        pass
    if images and tags:
        images = images.filter(tags__name__in=tags).distinct()
    return images
<|endoftext|>