<commit_before>from calendar import month_name, month_abbr
from collections import UserDict
from random import sample
from word2vec import WORD_EMBEDDING_FILE
import sqlite3
import re

UNKNOWN_TOKEN = 'UNKNOWN'
UNKNOWN_TOKEN_INDX = 0

def process_raw(raw):
    # to lower case
    s = raw.lower()
    # replace month name to number
    MONTH_NAME = zip([name.lower() for name in month_name[1:]], [name.lower() for name in month_abbr[1:]])
    for i, (name, abbr) in enumerate(MONTH_NAME):
        s = re.sub(r'\b{}\b|\b{}\b'.format(name, abbr), '%02d' % (i + 1), s)

    # define replace pattern
    GRAMMAR_SYM = r'(\')'
    DATE = r'(([0]?[1-9]|[1][0-2])[\.\/\- ]([0]?[1-9]|[1|2][0-9]|[3][0|1])[\.\/\- ]([0-9]{4}|[0-9]{2}))|' \
           r'(([0]?[1-9]|[1|2][0-9]|[3][0|1])[\.\/\- ]([0]?[1-9]|[1][0-2])[\.\/\- ]([0-9]{4}|[0-9]{2}))'
    TIME = r'[0-2]?[1-9]:[0-5][0-9][ \-]?(am|pm)?'
    MONEY = r'\$[ \-]?\d+(\,\d+)?\.?\d+'
    PRESENT = r'[-+]?\d+(\,\d+)?(\.\d+)?[ \-]?\%'
    NUMBER = r'[-+]?\d+(\,\d+)?(\.\d+)?'
    EMAIL = r'[_a-z0-9-]+(\.[_a-z0-9-]+)*@[a-z0-9-]+' \
            r'(\.[a-z0-9-]+)*\.(([0-9]{1,3})|([a-z]{2,3})|(aero|coop|info|museum|name))'
    SYM = r'(\.|\?|\$|\*|\#|\&)'
    SPACES = r' +'
    # replace all matched phrase to TOKEN name
    RE_SET = [(GRAMMAR_SYM, ' \\1'), (DATE, 'DATE'), (TIME, 'TIME'), (MONEY, 'MONEY'), (PRESENT, 'PRESENT'),
              (NUMBER, 'NUM'), (EMAIL, 'EMAIL'), (SYM, ' \\1 '), (SPACES, ' ')]
    for p, t in RE_SET:
        s = re.sub(p, t, s)
    return s

def word2index(w, voc_dict):
    try:
        return voc_dict[w]
    except KeyError:
        # unseen token
        return UNKNOWN_TOKEN_INDX

# question answer pairs (train) generate from ReVerb corpus
class ReVerbTrainRaw(object):
    def __init__(self):
        self.file = './data/reverb-tuples.db'
        conn = sqlite3.connect(self.file)
        c = conn.cursor()
        self.content = c.execute("select * from tuples")
        # define the pattern
        self.normal_pattern_list = [('who {r} {e2} ?', '{e1} {r} {e2}'),
                                    ('what {r} {e2} ?', '{e1} {r} {e2}'),
                                    ('who does {e1} {r} ?', '{e1} {r} {e2}'),
                                    ('what does {e1} {r} ?', '{e1} {r} {e2}'),
                                    ('what is the {r} of {e2} ?', '{e1} {r} {e2}'),
                                    ('who is the {r} of {e2} ?', '{e1} {r} {e2}'),
                                    ('what is {r} by {e1} ?', '{e1} {r} {e2}'),
                                    ('who is {e2} \'s {r} ?', '{e1} {r} {e2}'),
                                    ('what is {e2} \'s {r} ?', '{e1} {r} {e2}'),
                                    ('who is {r} by {e1} ?', '{e1} {r} {e2}')]
        # shared by *-in, *-on
        self.special_in_pattern_list = [('when did {e1} {r} ?', '{e1} {r} {e2}'),
                                        ('when was {e1} {r} ?', '{e1} {r} {e2}'),
                                        ('where was {e1} {r} ?', '{e1} {r} {e2}'),
                                        ('where did {e1} {r} ?', '{e1} {r} {e2}')]
        self.special_on_pattern_list = [('when did {e1} {r} ?', '{e1} {r} {e2}'),
                                        ('when was {e1} {r} ?', '{e1} {r} {e2}'),

    def __iter__(self):
        for r, e1, e2 in self.content:
            r = r.replace('.r', '')
            e1 = e1.replace('.e', '')
            e2 = e2.replace('.e', '')

            # find the suitable pattern
            # random choose some for training, reduce training size
            if r.endswith('-in'):
                pattern_list = sample(self.special_in_pattern_list, 1)
            elif r.endswith('-on'):
                pattern_list = sample(self.special_on_pattern_list, 1)
            else:
                pattern_list = sample(self.normal_pattern_list, 3)

            # preprocess & replace '-' with space
            r, e1, e2 = [process_raw(w.replace('-', ' ')) for w in [r, e1, e2]]

            # generate the question
            for s, p in pattern_list:
                q = s.format(r=r, e1=e1, e2=e2)
                a = p.format(r=r, e1=e1, e2=e2)

                yield (q, a)

    def __str__(self):
        return "ReVerb train tuples"

# question answer pairs (test) generate from ReVerb corpus
class ReVerbTestRaw(object):
    def __init__(self):
        self.file = './data/labels.txt'
        # load question cluster id
        self.q_id_map = UserDict()
        for line in open('./data/clusterid_questions.txt'):
            id, q = line.strip().split('\t')
            self.q_id_map[q] = id

    def __iter__(self):
        for line in open(self.file, 'r'):
            l, q, a = line.strip().split('\t')
            q_id = self.q_id_map[q]
            # normalize question
            q = re.sub(r'\?', ' ?', q)
            q = re.sub(r'\'s', ' \'s', q)
            q = re.sub(r'\-', ' ', q)
            q = process_raw(q)
            # normalize answer
            r, e1, e2 = [process_raw(re.sub(r'\.(r|e)', '', w.replace('-', ' '))) for w in a.split()]
            a = "{e1} {r} {e2}".format(r=r, e1=e1, e2=e2)

            yield q_id, q, a, l

    def __str__(self):
        return "ReVerb test raw"

# q-a pairs generated by reverb
class ReVerbPairs(object):
    def __init__(self, usage='train', part=None, mode='str', gram=1):
        if mode == 'str':
            suf = 'txt'
        elif mode == 'index':
            suf = 'indx'
        else:
            raise AttributeError("Mode can be only 'str' or 'index'")
        self.mode = mode

        if usage in ['train', 'test']:
            if part is not None:
                assert isinstance(part, int), "must provide a part number"
                self.file = './data/reverb-%s.part%d.%s' % (usage, part, suf)
            else:
                self.file = './data/reverb-%s.full.%s' % (usage, suf)
            self.part = part
            self.usage = usage
        else:
            raise SystemError("usage can be only train/test")
        # index of return data contains sentence
        self.sent_indx = (0, 1)
        # n-gram
        self.gram = gram

    def __iter__(self):
        for line in open(self.file, 'r'):
            if self.usage == 'train':
                # train
                q, a = line.strip().split('\t')
            else:
                # test
                q_id, q, a, l = line.strip().split('\t')
                l = int(l)

            q_tokens, a_tokens = [s.split() for s in [q, a]]

            if self.mode == 'str':
                if self.gram > 1:
                    q_tokens = [w1 + " " + w2 for w1, w2 in zip(*[q_tokens[j:] for j in range(self.gram)])]
                    a_tokens = [w1 + " " + w2 for w1, w2 in zip(*[a_tokens[j:] for j in range(self.gram)])]
            else:
                # mode == 'index
                q_tokens = map(int, q_tokens)
                a_tokens = map(int, a_tokens)

            # produce the token per line
            if self.usage == 'train':
                # train
                yield (q_tokens, a_tokens)
            else:
                # test
                yield (q_tokens, a_tokens, int(q_id), int(l))

    def get_voc_num(self, i):
        if self.gram == 1:
            voc_num = {0:224147, 1:271520}
        elif self.gram == 2:
            voc_num = {0: 0, 1: 0}
        elif self.gram == 3:
            voc_num = {0: 0, 1: 0}
        return voc_num[i]

    def __len__(self):
        if self.usage == 'train':
            # part patterns
            if self.part is None:
                return 35540263
            if self.part == 14:
                return 2369349
            else:
                return 2369351

            # full patterns
            # if self.part is None:
            #     return 117202052
            # 15 parts
            # if self.part == 29:
            #     return 3906708
            # else:
            #     return 3906736
            # 30 parts
            # if self.part == 14:
            #     return 7813458
            # else:
            #     return 7813471

        elif self.usage == 'test':
            return 48910

    def __str__(self):
        return "ReVerb QA pairs"

# Word Embedding
class WordEmbeddingRaw(object):
    def __init__(self):
        self.file = WORD_EMBEDDING_FILE
        self.word_is_index = False

    def __iter__(self):
        line_num = 0
        for line in open(self.file, 'r'):
            # skip the first line
            # embedding files generated by gensim always have a size indicator in the first line
            line_num += 1
            if line_num == 1:
                continue
            w, *emb = line.strip().split()
            yield w.lower(), emb

    def __len__(self):
        return 3000000

    def __str__(self):
        return "ReVerb tuples word embeddings"

# paraphrased sentences
# class PPDB(object):
#     def __init__(self, usage='train', mode='str', voc_dict=None):
#         if usage in ['train', 'test']:
#             self.file = './data/msr_paraphrase_%s.txt' % usage
#             self.usage = usage
#         else:
#             raise SystemError("usage can be only train/test")
#         self.voc_dict = voc_dict
#         if mode == 'index':
#             assert voc_dict is not None, "must take vocabulary-index dictionary."
#         self.mode = mode
#         # index of return data contains sentence
#         self.sent_indx = (0, 1)
#
#     def __iter__(self):
#         for line in open(self.file, 'r'):
#             q1, q2 = line.strip().split('\t')
#             q1_tokens, q2_tokens = [process_raw(s).split() for s in [q1, q2]]
#             # insert sentence start/end symbols
#             q1_tokens.insert(0, "[")
#             q1_tokens.append("]")
#             q2_tokens.insert(0, "[")
#             q2_tokens.append("]")
#
#             # paraphrased sentences similarity
#             similarity_score = 0.0
#
#             if self.mode == 'str':
#                 yield (q1_tokens, q2_tokens, similarity_score)
#             elif self.mode == 'index':
#                 # index each word using hash dictionary
#                 q1_tokens_indx, q2_tokens_indx = [[word2index(w, self.voc_dict) for w in s] for s in
#                                                   [q1_tokens, q2_tokens]]
#                 yield (q1_tokens_indx, q2_tokens_indx, similarity_score)
#             else:
#                 raise AttributeError("Mode can be only 'str' or 'index'")
#
#     def __len__(self):
#         if self.usage == 'train':
#             return 4077
#         elif self.usage == 'test':
#             return 1726
#
#     def __str__(self):
#         return "PPDB"

# question answer pairs in WikiQA corpus
# class WikiQA(object):
#     def __init__(self, usage='train', mode='str', voc_dict=None):
#         if usage in ['train', 'test']:
#             self.file = './data/WikiQA-%s.txt' % usage
#             self.usage = usage
#         else:
#             raise SystemError("usage can be only train/test")
#         self.voc_dict = voc_dict
#         if mode == 'index':
#             assert voc_dict is not None, "must take vocabulary-index dictionary."
#         self.mode = mode
#         # index of return data contains sentence
#         self.sent_indx = (0, 1)
#
#     def __iter__(self):
#         for line in open(self.file, 'r'):
#             q, a, label = line.strip().split('\t')
#             q_tokens, a_tokens = [process_raw(s).split() for s in [q, a]]
#             # insert sentence start/end symbols
#             q_tokens.insert(0, "[")
#             q_tokens.append("]")
#             a_tokens.insert(0, "[")
#             a_tokens.append("]")
#             # convert label as integer, right answer, wrong answer
#             label = int(label)
#
#             if self.mode == 'str':
#                 yield (q_tokens, a_tokens, label)
#             elif self.mode == 'index':
#                 # index each word using hash dictionary
#                 q_tokens_indx, a_tokens_indx = [[word2index(w, self.voc_dict) for w in s] for s in [q_tokens, a_tokens]]
#                 yield (q_tokens_indx, a_tokens_indx, label)
#             else:
#                 raise AttributeError("Mode can be only 'str' or 'index'")
#
#     def __len__(self):
#         if self.usage == 'train':
#             return 1180
#         elif self.usage == 'test':
#             return 6165
#
#     def __str__(self):
#         return "WikiQA"
<commit_msg>fix a bug<commit_after>from calendar import month_name, month_abbr
from collections import UserDict
from random import sample
from word2vec import WORD_EMBEDDING_FILE
import sqlite3
import re

UNKNOWN_TOKEN = 'UNKNOWN'
UNKNOWN_TOKEN_INDX = 0

def process_raw(raw):
    # to lower case
    s = raw.lower()
    # replace month name to number
    MONTH_NAME = zip([name.lower() for name in month_name[1:]], [name.lower() for name in month_abbr[1:]])
    for i, (name, abbr) in enumerate(MONTH_NAME):
        s = re.sub(r'\b{}\b|\b{}\b'.format(name, abbr), '%02d' % (i + 1), s)

    # define replace pattern
    GRAMMAR_SYM = r'(\')'
    DATE = r'(([0]?[1-9]|[1][0-2])[\.\/\- ]([0]?[1-9]|[1|2][0-9]|[3][0|1])[\.\/\- ]([0-9]{4}|[0-9]{2}))|' \
           r'(([0]?[1-9]|[1|2][0-9]|[3][0|1])[\.\/\- ]([0]?[1-9]|[1][0-2])[\.\/\- ]([0-9]{4}|[0-9]{2}))'
    TIME = r'[0-2]?[1-9]:[0-5][0-9][ \-]?(am|pm)?'
    MONEY = r'\$[ \-]?\d+(\,\d+)?\.?\d+'
    PRESENT = r'[-+]?\d+(\,\d+)?(\.\d+)?[ \-]?\%'
    NUMBER = r'[-+]?\d+(\,\d+)?(\.\d+)?'
    EMAIL = r'[_a-z0-9-]+(\.[_a-z0-9-]+)*@[a-z0-9-]+' \
            r'(\.[a-z0-9-]+)*\.(([0-9]{1,3})|([a-z]{2,3})|(aero|coop|info|museum|name))'
    SYM = r'(\.|\?|\$|\*|\#|\&)'
    SPACES = r' +'
    # replace all matched phrase to TOKEN name
    RE_SET = [(GRAMMAR_SYM, ' \\1'), (DATE, 'DATE'), (TIME, 'TIME'), (MONEY, 'MONEY'), (PRESENT, 'PRESENT'),
              (NUMBER, 'NUM'), (EMAIL, 'EMAIL'), (SYM, ' \\1 '), (SPACES, ' ')]
    for p, t in RE_SET:
        s = re.sub(p, t, s)
    return s

def word2index(w, voc_dict):
    try:
        return voc_dict[w]
    except KeyError:
        # unseen token
        return UNKNOWN_TOKEN_INDX

# question answer pairs (train) generate from ReVerb corpus
class ReVerbTrainRaw(object):
    def __init__(self):
        self.file = './data/reverb-tuples.db'
        conn = sqlite3.connect(self.file)
        c = conn.cursor()
        self.content = c.execute("select * from tuples")
        # define the pattern
        self.normal_pattern_list = [('who {r} {e2} ?', '{e1} {r} {e2}'),
                                    ('what {r} {e2} ?', '{e1} {r} {e2}'),
                                    ('who does {e1} {r} ?', '{e1} {r} {e2}'),
                                    ('what does {e1} {r} ?', '{e1} {r} {e2}'),
                                    ('what is the {r} of {e2} ?', '{e1} {r} {e2}'),
                                    ('who is the {r} of {e2} ?', '{e1} {r} {e2}'),
                                    ('what is {r} by {e1} ?', '{e1} {r} {e2}'),
                                    ('who is {e2} \'s {r} ?', '{e1} {r} {e2}'),
                                    ('what is {e2} \'s {r} ?', '{e1} {r} {e2}'),
                                    ('who is {r} by {e1} ?', '{e1} {r} {e2}')]
        # shared by *-in, *-on
        self.special_in_pattern_list = [('when did {e1} {r} ?', '{e1} {r} {e2}'),
                                        ('when was {e1} {r} ?', '{e1} {r} {e2}'),
                                        ('where was {e1} {r} ?', '{e1} {r} {e2}'),
                                        ('where did {e1} {r} ?', '{e1} {r} {e2}')]
        self.special_on_pattern_list = [('when did {e1} {r} ?', '{e1} {r} {e2}'),
                                        ('when was {e1} {r} ?', '{e1} {r} {e2}')]

    def __iter__(self):
        for r, e1, e2 in self.content:
            r = r.replace('.r', '')
            e1 = e1.replace('.e', '')
            e2 = e2.replace('.e', '')

            # find the suitable pattern
            # random choose some for training, reduce training size
            if r.endswith('-in'):
                pattern_list = sample(self.special_in_pattern_list, 1)
            elif r.endswith('-on'):
                pattern_list = sample(self.special_on_pattern_list, 1)
            else:
                pattern_list = sample(self.normal_pattern_list, 3)

            # preprocess & replace '-' with space
            r, e1, e2 = [process_raw(w.replace('-', ' ')) for w in [r, e1, e2]]

            # generate the question
            for s, p in pattern_list:
                q = s.format(r=r, e1=e1, e2=e2)
                a = p.format(r=r, e1=e1, e2=e2)

                yield (q, a)

    def __str__(self):
        return "ReVerb train tuples"

# question answer pairs (test) generate from ReVerb corpus
class ReVerbTestRaw(object):
    def __init__(self):
        self.file = './data/labels.txt'
        # load question cluster id
        self.q_id_map = UserDict()
        for line in open('./data/clusterid_questions.txt'):
            id, q = line.strip().split('\t')
            self.q_id_map[q] = id

    def __iter__(self):
        for line in open(self.file, 'r'):
            l, q, a = line.strip().split('\t')
            q_id = self.q_id_map[q]
            # normalize question
            q = re.sub(r'\?', ' ?', q)
            q = re.sub(r'\'s', ' \'s', q)
            q = re.sub(r'\-', ' ', q)
            q = process_raw(q)
            # normalize answer
            r, e1, e2 = [process_raw(re.sub(r'\.(r|e)', '', w.replace('-', ' '))) for w in a.split()]
            a = "{e1} {r} {e2}".format(r=r, e1=e1, e2=e2)

            yield q_id, q, a, l

    def __str__(self):
        return "ReVerb test raw"

# q-a pairs generated by reverb
class ReVerbPairs(object):
    def __init__(self, usage='train', part=None, mode='str', gram=1):
        if mode == 'str':
            suf = 'txt'
        elif mode == 'index':
            suf = 'indx'
        else:
            raise AttributeError("Mode can be only 'str' or 'index'")
        self.mode = mode

        if usage in ['train', 'test']:
            if part is not None:
                assert isinstance(part, int), "must provide a part number"
                self.file = './data/reverb-%s.part%d.%s' % (usage, part, suf)
            else:
                self.file = './data/reverb-%s.full.%s' % (usage, suf)
            self.part = part
            self.usage = usage
        else:
            raise SystemError("usage can be only train/test")
        # index of return data contains sentence
        self.sent_indx = (0, 1)
        # n-gram
        self.gram = gram

    def __iter__(self):
        for line in open(self.file, 'r'):
            if self.usage == 'train':
                # train
                q, a = line.strip().split('\t')
            else:
                # test
                q_id, q, a, l = line.strip().split('\t')
                l = int(l)

            q_tokens, a_tokens = [s.split() for s in [q, a]]

            if self.mode == 'str':
                if self.gram > 1:
                    q_tokens = [w1 + " " + w2 for w1, w2 in zip(*[q_tokens[j:] for j in range(self.gram)])]
                    a_tokens = [w1 + " " + w2 for w1, w2 in zip(*[a_tokens[j:] for j in range(self.gram)])]
            else:
                # mode == 'index
                q_tokens = map(int, q_tokens)
                a_tokens = map(int, a_tokens)

            # produce the token per line
            if self.usage == 'train':
                # train
                yield (q_tokens, a_tokens)
            else:
                # test
                yield (q_tokens, a_tokens, int(q_id), int(l))

    def get_voc_num(self, i):
        if self.gram == 1:
            voc_num = {0:224147, 1:271520}
        elif self.gram == 2:
            voc_num = {0: 0, 1: 0}
        elif self.gram == 3:
            voc_num = {0: 0, 1: 0}
        return voc_num[i]

    def __len__(self):
        if self.usage == 'train':
            # part patterns
            if self.part is None:
                return 35540263
            if self.part == 14:
                return 2369349
            else:
                return 2369351

            # full patterns
            # if self.part is None:
            #     return 117202052
            # 15 parts
            # if self.part == 29:
            #     return 3906708
            # else:
            #     return 3906736
            # 30 parts
            # if self.part == 14:
            #     return 7813458
            # else:
            #     return 7813471

        elif self.usage == 'test':
            return 48910

    def __str__(self):
        return "ReVerb QA pairs"

# Word Embedding
class WordEmbeddingRaw(object):
    def __init__(self):
        self.file = WORD_EMBEDDING_FILE
        self.word_is_index = False

    def __iter__(self):
        line_num = 0
        for line in open(self.file, 'r'):
            # skip the first line
            # embedding files generated by gensim always have a size indicator in the first line
            line_num += 1
            if line_num == 1:
                continue
            w, *emb = line.strip().split()
            yield w.lower(), emb

    def __len__(self):
        return 3000000

    def __str__(self):
        return "ReVerb tuples word embeddings"

# paraphrased sentences
# class PPDB(object):
#     def __init__(self, usage='train', mode='str', voc_dict=None):
#         if usage in ['train', 'test']:
#             self.file = './data/msr_paraphrase_%s.txt' % usage
#             self.usage = usage
#         else:
#             raise SystemError("usage can be only train/test")
#         self.voc_dict = voc_dict
#         if mode == 'index':
#             assert voc_dict is not None, "must take vocabulary-index dictionary."
#         self.mode = mode
#         # index of return data contains sentence
#         self.sent_indx = (0, 1)
#
#     def __iter__(self):
#         for line in open(self.file, 'r'):
#             q1, q2 = line.strip().split('\t')
#             q1_tokens, q2_tokens = [process_raw(s).split() for s in [q1, q2]]
#             # insert sentence start/end symbols
#             q1_tokens.insert(0, "[")
#             q1_tokens.append("]")
#             q2_tokens.insert(0, "[")
#             q2_tokens.append("]")
#
#             # paraphrased sentences similarity
#             similarity_score = 0.0
#
#             if self.mode == 'str':
#                 yield (q1_tokens, q2_tokens, similarity_score)
#             elif self.mode == 'index':
#                 # index each word using hash dictionary
#                 q1_tokens_indx, q2_tokens_indx = [[word2index(w, self.voc_dict) for w in s] for s in
#                                                   [q1_tokens, q2_tokens]]
#                 yield (q1_tokens_indx, q2_tokens_indx, similarity_score)
#             else:
#                 raise AttributeError("Mode can be only 'str' or 'index'")
#
#     def __len__(self):
#         if self.usage == 'train':
#             return 4077
#         elif self.usage == 'test':
#             return 1726
#
#     def __str__(self):
#         return "PPDB"

# question answer pairs in WikiQA corpus
# class WikiQA(object):
#     def __init__(self, usage='train', mode='str', voc_dict=None):
#         if usage in ['train', 'test']:
#             self.file = './data/WikiQA-%s.txt' % usage
#             self.usage = usage
#         else:
#             raise SystemError("usage can be only train/test")
#         self.voc_dict = voc_dict
#         if mode == 'index':
#             assert voc_dict is not None, "must take vocabulary-index dictionary."
#         self.mode = mode
#         # index of return data contains sentence
#         self.sent_indx = (0, 1)
#
#     def __iter__(self):
#         for line in open(self.file, 'r'):
#             q, a, label = line.strip().split('\t')
#             q_tokens, a_tokens = [process_raw(s).split() for s in [q, a]]
#             # insert sentence start/end symbols
#             q_tokens.insert(0, "[")
#             q_tokens.append("]")
#             a_tokens.insert(0, "[")
#             a_tokens.append("]")
#             # convert label as integer, right answer, wrong answer
#             label = int(label)
#
#             if self.mode == 'str':
#                 yield (q_tokens, a_tokens, label)
#             elif self.mode == 'index':
#                 # index each word using hash dictionary
#                 q_tokens_indx, a_tokens_indx = [[word2index(w, self.voc_dict) for w in s] for s in [q_tokens, a_tokens]]
#                 yield (q_tokens_indx, a_tokens_indx, label)
#             else:
#                 raise AttributeError("Mode can be only 'str' or 'index'")
#
#     def __len__(self):
#         if self.usage == 'train':
#             return 1180
#         elif self.usage == 'test':
#             return 6165
#
#     def __str__(self):
#         return "WikiQA"
<|endoftext|>


<commit_before>import logging
import re
import socket
import threading

from channel import Channel, ChannelList
from message import Message
from server  import Server

class Client:
	def __init__(self, server, nick, username, realname, **conf):
		self.sock = None
		self.nick = None
		self.server = None

		self.conf = {
			"port": 6667,
			"autorejoin": True,
			"autoconn": True,
			"channels": [],
			"prefixes": "&#"
		}

		self.conf.update(conf)
		self.conf.update({
			"server": server,
			"nick": nick,
			"username": username,
			"realname": realname,
		})

		self.channels = ChannelList()
		self.listeners = {}

		self.nickmod = 0

		if self.conf["autoconn"]:
			threading.Thread(name="main", target=self.connect).start()

	def connect(self):
		"""Attempt to connect to the server. Log in if successful."""

		self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

		try:
			self.sock.connect((self.conf["server"], self.conf["port"]))
			self.server = Server(self.conf["server"], self.conf["port"])
			logging.info("Connected to %s", self.server.host)

			self.send("NICK", self.conf["nick"])
			self.send("USER", self.conf["username"], 8, "*", \
				self.conf["realname"])

			threading.Thread(name="listen", target=self.listen).start()
		except socket.error as e:
			logging.error("Cannot connect to %s: %s", self.conf["server"], e)
			self.sock.close()
			self.sock = None

	def send(self, *msg):
		"""Send a raw message to the server."""

		if not self.sock:
			logging.error("Not connected to server")
			return

		message = " ".join(map(str, msg))
		self.sock.sendall(bytes(message + "\r\n", "utf-8"))
		logging.debug("(%s) %s", threading.current_thread().name, message.rstrip())

	def say(self, to, msg):
		"""Send a message to a user/channel."""
		self.send("PRIVMSG", to, ":" + msg)

	def notice(self, to, msg):
		"""Send a notice to a user/channel."""
		self.send("NOTICE", to, ":" + msg)

	def ctcp_say(self, to, text):
		"""Send a CTCP PRIVMSG message."""
		self.say(to, "\x01{0}\x01".format(text))

	def ctcp_notice(self, to, text):
		"""Send a CTCP NOTICE message."""
		self.notice(to, "\x01{0}\x01".format(text))

	def action(self, to, msg):
		self.ctcp_say(to, "ACTION {0}".format(msg))

	def join(self, chan):
		"""Join a channel."""
		self.send("JOIN", chan)

	def part(self, chan, reason=None):
		self.send("PART", chan, ":" + (reason or ""))

	def listen(self):
		"""Listen for incoming messages from the IRC server."""
		if not self.sock:
			logging.error("Not connected to server")
			return

		buf = ""
		while True:
			try:
				buf += str(self.sock.recv(4096), "utf-8")
				*msgs, buf = buf.split("\r\n")
				for msg in msgs:
					m = Message.parse(msg)
					thread = threading.Thread(target=lambda: self.handle(m))
					logging.debug("(%s) handler thread %s [%s]",
						threading.current_thread().name, thread.name, m.raw)
					thread.start()
			except socket.error:
				logging.info("Disconnected from server")
				self.sock.close()
				self.sock = None
				break

	def add_listener(self, event, fn):
		"""Add a function to listen for the specified event."""
		if event not in self.listeners:
			self.listeners[event] = []
		self.listeners[event].append(fn)

	def remove_listener(self, event, fn):
		"""Remove a function as a listener from the specified event."""
		if event not in self.listeners:
			return
		self.listeners[event] = [l for l in self.listeners[event] if l != fn]

	def remove_listeners(self, event):
		"""Remove all functions listening for the specified event."""
		if event not in self.listeners:
			return
		self.listeners.pop(event)

	def emit(self, event, *params):
		"""Emit an event, and call all functions listening for it."""
		if event in self.listeners:
			for listener in self.listeners[event]:
				try:
					thread = threading.Thread(target=lambda: listener(*params))
					logging.debug("(%s) worker thread %s [%s, %s]",
						threading.current_thread().name, thread.name, event,
						listener.__name__)
					thread.start()
				except TypeError as e:
					logging.error("(%s) invalid number of parameters [%s]",
						threading.current_thread().name, listener.__name__)

	def _ctcp(self, fr, to, text, type):
		text = text[1:text.index("\x01")]
		parts = text.split()
		self.emit("ctcp", fr, to, text, type)
		self.emit("ctcp." + type, fr, to, text)

		if type == "privmsg":
			if text == "VERSION":
				self.emit("ctcp.version", fr, to)
			elif len(parts) > 1 and parts[0] == "ACTION":
				self.emit("action", fr, to, " ".join(parts[1:]))
			elif len(parts) > 1 and parts[0] == "PING":
				self.ctcp_notice(fr, "PONG " + " ".join(parts[1:]))

	def handle(self, msg):
		c = msg.command
		if c == "001":
			self.nick = self.conf["nick"] + self.nickmod * "_"
			self.emit("registered", msg.params[0])
		elif c == "004":
			self.server.usermodes = set(msg.params[3])
		elif c == "005":
			for p in msg.params:
				if "=" in p:
					param, value = p.split("=")
					if param == "CHANLIMIT":
						for pair in value.split(","):
							typ, num = pair.split(":")
							self.server.chlimit[typ] = int(num)
					elif param == "CHANMODES":
						modes = value.split(",")
						self.server.chmodes = dict(zip("abcd", map(set, modes)))
					elif param == "CHANTYPES":
						self.server.types = set(value)
					elif param == "CHANNELLEN":
						self.server.chlength = int(value)
					elif param == "IDCHAN":
						for pair in value.split(","):
							typ, num = pair.split(":")
							self.server.idlength[typ] = int(num)
					elif param == "KICKLEN":
						self.server.kicklength = int(value)
					elif param == "NICKLEN":
						self.server.nicklength = int(value)
					elif param == "PREFIX":
						if re.match(r"^\((.*)\)(.*)$", value):
							modes = value.split(")")[0][1:]
							prefixes = value.split(")")[1]

							self.server.prefix_mode = dict(zip(prefixes, modes))
							self.server.mode_prefix = dict(zip(modes, prefixes))

							self.server.chmodes["b"].update(modes)
					elif param == "TARGMAX":
						for pair in value.split(","):
							typ, num = pair.split(":")
							self.server.maxtargets[typ] = int(num) if num else 0
					elif param == "TOPICLEN":
						self.server.topiclength = int(value)
		elif c == "433":
			self.nickmod += 1
			self.send("NICK", self.conf["nick"] + self.nickmod * "_")
		elif c == "PING":
			self.send("PONG", msg.params[0])
			self.emit("ping", msg.params[0])
		elif c == "PONG":
			self.emit("pong", msg.params[0])
		elif c == "NOTICE":
			fr, to = msg.nick, msg.params[0]
			text = msg.params[1] or ""
			if text[0] == "\x01" and "\x01" in text[1:]:
				self._ctcp(fr, to, text, "notice")
			else:
				self.emit("notice", fr, to, text)
		elif c == "MODE":
			pass # TODO
		elif c == "NICK":
			nick = msg.params[0]
			if msg.nick == self.nick:
				self.nick = nick
			chans = list(filter(lambda c: msg.nick in c, self.channels.values()))
			for chan in chans:
				chan.users[nick] = chan.users[msg.nick]
				chan.users.pop(nick)
			self.emit("nick", msg.nick, nick, [c.name for c in chans])
		elif c == "375":
			self.server.motd = msg.params[1] + "\n"
		elif c == "372":
			self.server.motd += msg.params[1] + "\n"
		elif c == "376" or c == "422":
			self.server.motd += msg.params[1] + "\n"
			self.emit("motd", self.server.motd)
		elif c == "353":
			channel = self.channels[msg.params[2][1:]]
			if channel:
				users = msg.params[3].strip().split()
				for user in users:
					if user[0] in self.server.prefix_mode:
						channel.users[user[1:]] = user[0]
					else:
						channel.users[user] = ""
		elif c == "366":
			channel = self.channels[msg.params[1][1:]]
			if channel:
				self.emit("names", msg.params[1], channel.users)
				self.send("MODE", msg.params[1])
		elif c == "332":
			channel = self.channels[msg.params[1][1:]]
			if channel:
				channel.topic = msg.params[2]
		# 301, 311, 312, 313, 317, 318, 319, 330, 321, 322, 323, 333
		elif c == "TOPIC":
			self.emit("topic", msg.params[0], msg.params[1], msg.nick, msg)
			channel = self.channels[msg.params[0][1:]]
			if channel:
				channel.topic = msg.params[1]
				channel.topicby = msg.nick
		elif c == "324":
			channel = self.channels[msg.params[1][1:]]
			if channel:
				channel.mode = msg.params[2]
		elif c == "329":
			channel = self.channels[msg.params[1][1:]]
			if channel:
				channel.created = int(msg.params[2])
		elif c == "JOIN":
			chan = msg.params[0]
			if self.nick == msg.nick:
				self.channels[chan[1:]] = Channel(chan)
			else:
				channel = self.channels[chan[1:]]
				if channel:
					channel.users[msg.nick] = ""
			self.emit("join", chan, msg.nick)
		elif c == "PART":
			chan = msg.params[0]
			self.emit("part", chan, msg.nick)
			if self.nick == msg.nick:
				self.channels.pop(chan[1:])
			else:
				channel = self.channels[chan]
				if channel:
					del channel.users[msg.nick]
		elif c == "KICK":
			chan, who, reason, *rest = msg.params
			self.emit("kick", chan, who, msg.nick, reason)
			if self.nick == msg.nick:
				self.channels.pop(chan[1:])
			else:
				channel = self.channels[chan[1:]]
				if channel:
					channel.users.pop(who)
		elif c == "KILL":
			nick = msg.params[0]
			channels = []
			for chan in self.channels:
				channels.append(chan.name)
				chan.users.pop(nick)
			self.emit("kill", nick, msg.params[1], channels)
		elif c == "PRIVMSG":
			fr, to = msg.nick, msg.params[0]
			text = " ".join(msg.params[1:])
			if text[0] == "\x01" and "\x01" in text[1:]:
				self._ctcp(fr, to, text, "privmsg")
			else:
				self.emit("message", fr, to, text)
			self.emit("pm", fr, text, msg)
		elif c == "INVITE":
			self.emit("invite", msg.params[1], msg.nick)
		elif c == "QUIT":
			if self.nick == msg.nick:
				return
			if msg.nick == self.nick:
				self.nick = nick
			chans = list(filter(lambda c: msg.nick in c, self.channels.values()))
			for chan in chans:
				chan.users.pop(msg.nick)
			self.emit("quit", msg.nick, [c.name for c in chans])
<commit_msg>fix NICK handler maybe<commit_after>import logging
import re
import socket
import threading

from channel import Channel, ChannelList
from message import Message
from server  import Server

class Client:
	def __init__(self, server, nick, username, realname, **conf):
		self.sock = None
		self.nick = None
		self.server = None

		self.conf = {
			"port": 6667,
			"autorejoin": True,
			"autoconn": True,
			"channels": [],
			"prefixes": "&#"
		}

		self.conf.update(conf)
		self.conf.update({
			"server": server,
			"nick": nick,
			"username": username,
			"realname": realname,
		})

		self.channels = ChannelList()
		self.listeners = {}

		self.nickmod = 0

		if self.conf["autoconn"]:
			threading.Thread(name="main", target=self.connect).start()

	def connect(self):
		"""Attempt to connect to the server. Log in if successful."""

		self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

		try:
			self.sock.connect((self.conf["server"], self.conf["port"]))
			self.server = Server(self.conf["server"], self.conf["port"])
			logging.info("Connected to %s", self.server.host)

			self.send("NICK", self.conf["nick"])
			self.send("USER", self.conf["username"], 8, "*", \
				self.conf["realname"])

			threading.Thread(name="listen", target=self.listen).start()
		except socket.error as e:
			logging.error("Cannot connect to %s: %s", self.conf["server"], e)
			self.sock.close()
			self.sock = None

	def send(self, *msg):
		"""Send a raw message to the server."""

		if not self.sock:
			logging.error("Not connected to server")
			return

		message = " ".join(map(str, msg))
		self.sock.sendall(bytes(message + "\r\n", "utf-8"))
		logging.debug("(%s) %s", threading.current_thread().name, message.rstrip())

	def say(self, to, msg):
		"""Send a message to a user/channel."""
		self.send("PRIVMSG", to, ":" + msg)

	def notice(self, to, msg):
		"""Send a notice to a user/channel."""
		self.send("NOTICE", to, ":" + msg)

	def ctcp_say(self, to, text):
		"""Send a CTCP PRIVMSG message."""
		self.say(to, "\x01{0}\x01".format(text))

	def ctcp_notice(self, to, text):
		"""Send a CTCP NOTICE message."""
		self.notice(to, "\x01{0}\x01".format(text))

	def action(self, to, msg):
		self.ctcp_say(to, "ACTION {0}".format(msg))

	def join(self, chan):
		"""Join a channel."""
		self.send("JOIN", chan)

	def part(self, chan, reason=None):
		self.send("PART", chan, ":" + (reason or ""))

	def listen(self):
		"""Listen for incoming messages from the IRC server."""
		if not self.sock:
			logging.error("Not connected to server")
			return

		buf = ""
		while True:
			try:
				buf += str(self.sock.recv(4096), "utf-8")
				*msgs, buf = buf.split("\r\n")
				for msg in msgs:
					m = Message.parse(msg)
					thread = threading.Thread(target=lambda: self.handle(m))
					logging.debug("(%s) handler thread %s [%s]",
						threading.current_thread().name, thread.name, m.raw)
					thread.start()
			except socket.error:
				logging.info("Disconnected from server")
				self.sock.close()
				self.sock = None
				break

	def add_listener(self, event, fn):
		"""Add a function to listen for the specified event."""
		if event not in self.listeners:
			self.listeners[event] = []
		self.listeners[event].append(fn)

	def remove_listener(self, event, fn):
		"""Remove a function as a listener from the specified event."""
		if event not in self.listeners:
			return
		self.listeners[event] = [l for l in self.listeners[event] if l != fn]

	def remove_listeners(self, event):
		"""Remove all functions listening for the specified event."""
		if event not in self.listeners:
			return
		self.listeners.pop(event)

	def emit(self, event, *params):
		"""Emit an event, and call all functions listening for it."""
		if event in self.listeners:
			for listener in self.listeners[event]:
				try:
					thread = threading.Thread(target=lambda: listener(*params))
					logging.debug("(%s) worker thread %s [%s, %s]",
						threading.current_thread().name, thread.name, event,
						listener.__name__)
					thread.start()
				except TypeError as e:
					logging.error("(%s) invalid number of parameters [%s]",
						threading.current_thread().name, listener.__name__)

	def _ctcp(self, fr, to, text, type):
		text = text[1:text.index("\x01")]
		parts = text.split()
		self.emit("ctcp", fr, to, text, type)
		self.emit("ctcp." + type, fr, to, text)

		if type == "privmsg":
			if text == "VERSION":
				self.emit("ctcp.version", fr, to)
			elif len(parts) > 1 and parts[0] == "ACTION":
				self.emit("action", fr, to, " ".join(parts[1:]))
			elif len(parts) > 1 and parts[0] == "PING":
				self.ctcp_notice(fr, "PONG " + " ".join(parts[1:]))

	def handle(self, msg):
		c = msg.command
		if c == "001":
			self.nick = self.conf["nick"] + self.nickmod * "_"
			self.emit("registered", msg.params[0])
		elif c == "004":
			self.server.usermodes = set(msg.params[3])
		elif c == "005":
			for p in msg.params:
				if "=" in p:
					param, value = p.split("=")
					if param == "CHANLIMIT":
						for pair in value.split(","):
							typ, num = pair.split(":")
							self.server.chlimit[typ] = int(num)
					elif param == "CHANMODES":
						modes = value.split(",")
						self.server.chmodes = dict(zip("abcd", map(set, modes)))
					elif param == "CHANTYPES":
						self.server.types = set(value)
					elif param == "CHANNELLEN":
						self.server.chlength = int(value)
					elif param == "IDCHAN":
						for pair in value.split(","):
							typ, num = pair.split(":")
							self.server.idlength[typ] = int(num)
					elif param == "KICKLEN":
						self.server.kicklength = int(value)
					elif param == "NICKLEN":
						self.server.nicklength = int(value)
					elif param == "PREFIX":
						if re.match(r"^\((.*)\)(.*)$", value):
							modes = value.split(")")[0][1:]
							prefixes = value.split(")")[1]

							self.server.prefix_mode = dict(zip(prefixes, modes))
							self.server.mode_prefix = dict(zip(modes, prefixes))

							self.server.chmodes["b"].update(modes)
					elif param == "TARGMAX":
						for pair in value.split(","):
							typ, num = pair.split(":")
							self.server.maxtargets[typ] = int(num) if num else 0
					elif param == "TOPICLEN":
						self.server.topiclength = int(value)
		elif c == "433":
			self.nickmod += 1
			self.send("NICK", self.conf["nick"] + self.nickmod * "_")
		elif c == "PING":
			self.send("PONG", msg.params[0])
			self.emit("ping", msg.params[0])
		elif c == "PONG":
			self.emit("pong", msg.params[0])
		elif c == "NOTICE":
			fr, to = msg.nick, msg.params[0]
			text = msg.params[1] or ""
			if text[0] == "\x01" and "\x01" in text[1:]:
				self._ctcp(fr, to, text, "notice")
			else:
				self.emit("notice", fr, to, text)
		elif c == "MODE":
			pass # TODO
		elif c == "NICK":
			nick = msg.params[0]
			if msg.nick == self.nick:
				self.nick = nick
			chans = list(filter(lambda c: msg.nick in c, self.channels.values()))
			for chan in chans:
				chan.users[nick] = chan.users[msg.nick]
				chan.users.pop(msg.nick)
			self.emit("nick", msg.nick, nick, [c.name for c in chans])
		elif c == "375":
			self.server.motd = msg.params[1] + "\n"
		elif c == "372":
			self.server.motd += msg.params[1] + "\n"
		elif c == "376" or c == "422":
			self.server.motd += msg.params[1] + "\n"
			self.emit("motd", self.server.motd)
		elif c == "353":
			channel = self.channels[msg.params[2][1:]]
			if channel:
				users = msg.params[3].strip().split()
				for user in users:
					if user[0] in self.server.prefix_mode:
						channel.users[user[1:]] = user[0]
					else:
						channel.users[user] = ""
		elif c == "366":
			channel = self.channels[msg.params[1][1:]]
			if channel:
				self.emit("names", msg.params[1], channel.users)
				self.send("MODE", msg.params[1])
		elif c == "332":
			channel = self.channels[msg.params[1][1:]]
			if channel:
				channel.topic = msg.params[2]
		# 301, 311, 312, 313, 317, 318, 319, 330, 321, 322, 323, 333
		elif c == "TOPIC":
			self.emit("topic", msg.params[0], msg.params[1], msg.nick, msg)
			channel = self.channels[msg.params[0][1:]]
			if channel:
				channel.topic = msg.params[1]
				channel.topicby = msg.nick
		elif c == "324":
			channel = self.channels[msg.params[1][1:]]
			if channel:
				channel.mode = msg.params[2]
		elif c == "329":
			channel = self.channels[msg.params[1][1:]]
			if channel:
				channel.created = int(msg.params[2])
		elif c == "JOIN":
			chan = msg.params[0]
			if self.nick == msg.nick:
				self.channels[chan[1:]] = Channel(chan)
			else:
				channel = self.channels[chan[1:]]
				if channel:
					channel.users[msg.nick] = ""
			self.emit("join", chan, msg.nick)
		elif c == "PART":
			chan = msg.params[0]
			self.emit("part", chan, msg.nick)
			if self.nick == msg.nick:
				self.channels.pop(chan[1:])
			else:
				channel = self.channels[chan]
				if channel:
					del channel.users[msg.nick]
		elif c == "KICK":
			chan, who, reason, *rest = msg.params
			self.emit("kick", chan, who, msg.nick, reason)
			if self.nick == msg.nick:
				self.channels.pop(chan[1:])
			else:
				channel = self.channels[chan[1:]]
				if channel:
					channel.users.pop(who)
		elif c == "KILL":
			nick = msg.params[0]
			channels = []
			for chan in self.channels:
				channels.append(chan.name)
				chan.users.pop(nick)
			self.emit("kill", nick, msg.params[1], channels)
		elif c == "PRIVMSG":
			fr, to = msg.nick, msg.params[0]
			text = " ".join(msg.params[1:])
			if text[0] == "\x01" and "\x01" in text[1:]:
				self._ctcp(fr, to, text, "privmsg")
			else:
				self.emit("message", fr, to, text)
			self.emit("pm", fr, text, msg)
		elif c == "INVITE":
			self.emit("invite", msg.params[1], msg.nick)
		elif c == "QUIT":
			if self.nick == msg.nick:
				return
			if msg.nick == self.nick:
				self.nick = nick
			chans = list(filter(lambda c: msg.nick in c, self.channels.values()))
			for chan in chans:
				chan.users.pop(msg.nick)
			self.emit("quit", msg.nick, [c.name for c in chans])
<|endoftext|>


<commit_before>#!/usr/bin/python

import sys
import os
import stat
import re

def checkglobals(libname, exceptsymbols, verbose):
    badglobals = 0
    nmcmd = "nm -g " + libname
    f = os.popen(nmcmd)
    b = f.readline()
    while b != "":
        match = re.match("^([0-9a-f]+)\s(.?)\s(.*)$", b)
        if match == None:
            match = re.match("^\s+(.*)$", b)
            if match == None:
                print "unknown", b
                badglobals = 1
        else:
            type = match.group(2)
            symbol = match.group(3)
            if verbose: print type, symbol
            match = re.match("^toku_|^__toku", symbol)
            if match == None and not exceptsymbols.has_key(symbol):
                print "non toku symbol=", symbol
                badglobals = 1
        b = f.readline()
    f.close()
    return badglobals

def main():
    verbose = 0
    libname = "libdb.so"
    for arg in sys.argv[1:]:
        if arg == "-v":
            verbose += 1
        elif arg[0:3] == "lib":
            libname = arg

    try: st = os.stat(libname)
    except: return 1
    mode = st[stat.ST_MODE]
    if not (mode & stat.S_IREAD): return 1

    exceptsymbols = {}
    for n in [ "_init", "_fini", "_end", "_edata", "__bss_start" ]:
        exceptsymbols[n] = 1
    for n in [ "db_env_create", "db_create", "db_strerror", "db_version", "log_compare",
               "db_env_set_func_fsync",
               "db_env_set_func_free",
               "db_env_set_func_malloc",
               "db_env_set_func_realloc",
               "db_env_set_func_pwrite",
               "db_env_set_func_write",
               "db_env_set_func_loader_fwrite",
               "db_env_set_checkpoint_callback",
               "db_env_set_checkpoint_callback2",
               "db_env_set_recover_callback",
               "db_env_set_recover_callback2",
               "setup_dlmalloc", "dlfree", "dlmalloc", "dlrealloc"
               ]:
        exceptsymbols[n] = 1
    return checkglobals(libname, exceptsymbols, verbose)

sys.exit(main())
<commit_msg>fix the library globals error filter<commit_after>#!/usr/bin/python

import sys
import os
import stat
import re

def checkglobals(libname, exceptsymbols, verbose):
    badglobals = 0
    nmcmd = "nm -g " + libname
    f = os.popen(nmcmd)
    b = f.readline()
    while b != "":
        match = re.match("^([0-9a-f]+)\s(.?)\s(.*)$", b)
        if match == None:
            match = re.match("^\s+(.*)$", b)
            if match == None:
                print "unknown", b
                badglobals = 1
        else:
            type = match.group(2)
            symbol = match.group(3)
            if verbose: print type, symbol
            match = re.match("^toku_|^__toku|^db_env_set", symbol)
            if match == None and not exceptsymbols.has_key(symbol):
                print "non toku symbol=", symbol
                badglobals = 1
        b = f.readline()
    f.close()
    return badglobals

def main():
    verbose = 0
    libname = "libdb.so"
    for arg in sys.argv[1:]:
        if arg == "-v":
            verbose += 1
        elif arg[0:3] == "lib":
            libname = arg

    try: st = os.stat(libname)
    except: return 1
    mode = st[stat.ST_MODE]
    if not (mode & stat.S_IREAD): return 1

    exceptsymbols = {}
    for n in [ "_init", "_fini", "_end", "_edata", "__bss_start" ]:
        exceptsymbols[n] = 1
    for n in [ "db_env_create", "db_create", "db_strerror", "db_version", "log_compare",
               "db_env_set_func_fsync",
               "db_env_set_func_free",
               "db_env_set_func_malloc",
               "db_env_set_func_realloc",
               "db_env_set_func_pwrite",
               "db_env_set_func_write",
               "db_env_set_func_loader_fwrite",
               "db_env_set_checkpoint_callback",
               "db_env_set_checkpoint_callback2",
               "db_env_set_recover_callback",
               "db_env_set_recover_callback2",
               "setup_dlmalloc", "dlfree", "dlmalloc", "dlrealloc"
               ]:
        exceptsymbols[n] = 1
    return checkglobals(libname, exceptsymbols, verbose)

sys.exit(main())
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <gogogo.vm@gmail.com>
"""

from __future__ import absolute_import
import sys

import dataproperty
import logbook
import pyparsing as pp
import six
import subprocrunner

import tcconfig
from .traffic_control import TrafficControl
from .traffic_control import TrafficDirection
from ._argparse_wrapper import ArgparseWrapper
from ._error import ModuleNotFoundError

handler = logbook.StderrHandler()
handler.push_application()

def parse_option():
    parser = ArgparseWrapper(tcconfig.VERSION)

    group = parser.parser.add_mutually_exclusive_group(required=True)
    group.add_argument(
        "--device",
        help="network device name (e.g. eth0)")
    group.add_argument(
        "-f", "--config-file",
        help="""setting traffic controls from a configuration file.
        output file of the tcshow.""")

    group = parser.parser.add_argument_group("Network Interface")
    group.add_argument(
        "--overwrite", action="store_true", default=False,
        help="overwrite existing settings")

    group = parser.parser.add_argument_group("Traffic Control")
    group.add_argument(
        "--direction", choices=tcconfig.traffic_control.TrafficDirection.LIST,
        default=tcconfig.traffic_control.TrafficDirection.OUTGOING,
        help="""the direction of network communication that impose traffic control.
        ``incoming`` requires linux kernel version 2.6.20 or later.
        (default = ``%(default)s``)
        """)
    group.add_argument(
        "--rate", dest="bandwidth_rate",
        help="network bandwidth rate [K|M|G bps]")
    group.add_argument(
        "--delay", dest="network_latency", type=float, default=0,
        help="round trip network delay [ms] (default=%(default)s)")
    group.add_argument(
        "--delay-distro", dest="latency_distro_ms", type=float, default=0,
        help="""
        distribution of network latency becomes X +- Y [ms]
        (normal distribution), with this option.
        (X: value of --delay option, Y: value of --delay-dist option)
        network latency distribution will be uniform without this option.
        """)
    group.add_argument(
        "--loss", dest="packet_loss_rate", type=float, default=0,
        help="round trip packet loss rate [%%] (default=%(default)s)")
    group.add_argument(
        "--corrupt", dest="corruption_rate", type=float, default=0,
        help="""
        packet corruption rate [%%]. packet corruption means single bit error
        at a random offset in the packet. (default=%(default)s)
        """)
    group.add_argument(
        "--network",
        help="IP address/network of traffic control")
    group.add_argument(
        "--port", type=int,
        help="port number of traffic control")

    return parser.parser.parse_args()

def verify_netem_module():
    runner = subprocrunner.SubprocessRunner("lsmod | grep sch_netem")

    if runner.run() != 0:
        raise ModuleNotFoundError("sch_netem module not found")

class TcConfigLoader(object):

    def __init__(self):
        self.__config_table = None
        self.is_overwrite = False

    def load_tcconfig(self, config_file_path):
        import json
        from voluptuous import Schema, Required, Any, ALLOW_EXTRA

        schema = Schema({
            Required(six.text_type): {
                Any(*TrafficDirection.LIST): {
                    six.text_type: {
                        six.text_type: six.text_type,
                    },
                }
            },
        }, extra=ALLOW_EXTRA)

        self.__config_table = json.loads(config_file_path)
        schema(self.__config_table)

    def get_tcconfig_command_list(self):
        command_list = []

        for device, device_table in six.iteritems(self.__config_table):
            if self.is_overwrite:
                command_list.append("tcdel --device " + device)

            for direction, direction_table in six.iteritems(device_table):
                for tc_filter, filter_table in six.iteritems(direction_table):
                    if filter_table == {}:
                        continue

                    option_list = [
                        "--device=" + device,
                        "--direction=" + direction,
                    ] + [
                        "--{:s}={:s}".format(k, v)
                        for k, v in six.iteritems(filter_table)
                    ]

                    try:
                        network = self.__parse_tc_filter_network(tc_filter)
                        if network != "0.0.0.0/0":
                            option_list.append("--network=" + network)
                    except pp.ParseException:
                        pass

                    try:
                        port = self.__parse_tc_filter_port(tc_filter)
                        option_list.append("--port=" + port)
                    except pp.ParseException:
                        pass

                    command_list.append(" ".join(["tcset"] + option_list))

        return command_list

    @staticmethod
    def __parse_tc_filter_network(text):
        network_pattern = (
            pp.SkipTo("network=", include=True) +
            pp.Word(pp.alphanums + "." + "/"))

        return network_pattern.parseString(text)[-1]

    @staticmethod
    def __parse_tc_filter_port(text):
        port_pattern = (
            pp.SkipTo("port=", include=True) +
            pp.Word(pp.nums))

        return port_pattern.parseString(text)[-1]

def main():
    options = parse_option()
    logger = logbook.Logger("tcset")

    subprocrunner.logger.level = options.log_level
    if options.quiet:
        subprocrunner.logger.disable()
    else:
        subprocrunner.logger.enable()

    subprocrunner.Which("tc").verify()
    try:
        verify_netem_module()
    except ModuleNotFoundError as e:
        logger.warning(str(e))
    except subprocrunner.CommandNotFoundError as e:
        logger.error(str(e))

    if dataproperty.is_not_empty_string(options.config_file):
        return_code = 0

        loader = TcConfigLoader()
        loader.is_overwrite = options.overwrite
        loader.load_tcconfig(options.config_file)

        for tcconfig_command in loader.get_tcconfig_command_list():
            return_code |= subprocrunner.SubprocessRunner(
                tcconfig_command).run()

        return return_code

    tc = TrafficControl(options.device)
    tc.direction = options.direction
    tc.bandwidth_rate = options.bandwidth_rate
    tc.latency_ms = options.network_latency
    tc.latency_distro_ms = options.latency_distro_ms
    tc.packet_loss_rate = options.packet_loss_rate
    tc.corruption_rate = options.corruption_rate
    tc.network = options.network
    tc.port = options.port

    tc.validate()

    if options.overwrite:
        tc.delete_tc()

    tc.set_tc()

    return 0

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Suppress excessive an error message<commit_after>#!/usr/bin/env python
# encoding: utf-8

"""
.. codeauthor:: Tsuyoshi Hombashi <gogogo.vm@gmail.com>
"""

from __future__ import absolute_import
import sys

import dataproperty
import logbook
import pyparsing as pp
import six
import subprocrunner

import tcconfig
from .traffic_control import TrafficControl
from .traffic_control import TrafficDirection
from ._argparse_wrapper import ArgparseWrapper
from ._error import ModuleNotFoundError

handler = logbook.StderrHandler()
handler.push_application()

def parse_option():
    parser = ArgparseWrapper(tcconfig.VERSION)

    group = parser.parser.add_mutually_exclusive_group(required=True)
    group.add_argument(
        "--device",
        help="network device name (e.g. eth0)")
    group.add_argument(
        "-f", "--config-file",
        help="""setting traffic controls from a configuration file.
        output file of the tcshow.""")

    group = parser.parser.add_argument_group("Network Interface")
    group.add_argument(
        "--overwrite", action="store_true", default=False,
        help="overwrite existing settings")

    group = parser.parser.add_argument_group("Traffic Control")
    group.add_argument(
        "--direction", choices=tcconfig.traffic_control.TrafficDirection.LIST,
        default=tcconfig.traffic_control.TrafficDirection.OUTGOING,
        help="""the direction of network communication that impose traffic control.
        ``incoming`` requires linux kernel version 2.6.20 or later.
        (default = ``%(default)s``)
        """)
    group.add_argument(
        "--rate", dest="bandwidth_rate",
        help="network bandwidth rate [K|M|G bps]")
    group.add_argument(
        "--delay", dest="network_latency", type=float, default=0,
        help="round trip network delay [ms] (default=%(default)s)")
    group.add_argument(
        "--delay-distro", dest="latency_distro_ms", type=float, default=0,
        help="""
        distribution of network latency becomes X +- Y [ms]
        (normal distribution), with this option.
        (X: value of --delay option, Y: value of --delay-dist option)
        network latency distribution will be uniform without this option.
        """)
    group.add_argument(
        "--loss", dest="packet_loss_rate", type=float, default=0,
        help="round trip packet loss rate [%%] (default=%(default)s)")
    group.add_argument(
        "--corrupt", dest="corruption_rate", type=float, default=0,
        help="""
        packet corruption rate [%%]. packet corruption means single bit error
        at a random offset in the packet. (default=%(default)s)
        """)
    group.add_argument(
        "--network",
        help="IP address/network of traffic control")
    group.add_argument(
        "--port", type=int,
        help="port number of traffic control")

    return parser.parser.parse_args()

def verify_netem_module():
    runner = subprocrunner.SubprocessRunner("lsmod | grep sch_netem")

    if runner.run() != 0:
        raise ModuleNotFoundError("sch_netem module not found")

class TcConfigLoader(object):

    def __init__(self):
        self.__config_table = None
        self.is_overwrite = False

    def load_tcconfig(self, config_file_path):
        import json
        from voluptuous import Schema, Required, Any, ALLOW_EXTRA

        schema = Schema({
            Required(six.text_type): {
                Any(*TrafficDirection.LIST): {
                    six.text_type: {
                        six.text_type: six.text_type,
                    },
                }
            },
        }, extra=ALLOW_EXTRA)

        self.__config_table = json.loads(config_file_path)
        schema(self.__config_table)

    def get_tcconfig_command_list(self):
        command_list = []

        for device, device_table in six.iteritems(self.__config_table):
            if self.is_overwrite:
                command_list.append("tcdel --device " + device)

            for direction, direction_table in six.iteritems(device_table):
                for tc_filter, filter_table in six.iteritems(direction_table):
                    if filter_table == {}:
                        continue

                    option_list = [
                        "--device=" + device,
                        "--direction=" + direction,
                    ] + [
                        "--{:s}={:s}".format(k, v)
                        for k, v in six.iteritems(filter_table)
                    ]

                    try:
                        network = self.__parse_tc_filter_network(tc_filter)
                        if network != "0.0.0.0/0":
                            option_list.append("--network=" + network)
                    except pp.ParseException:
                        pass

                    try:
                        port = self.__parse_tc_filter_port(tc_filter)
                        option_list.append("--port=" + port)
                    except pp.ParseException:
                        pass

                    command_list.append(" ".join(["tcset"] + option_list))

        return command_list

    @staticmethod
    def __parse_tc_filter_network(text):
        network_pattern = (
            pp.SkipTo("network=", include=True) +
            pp.Word(pp.alphanums + "." + "/"))

        return network_pattern.parseString(text)[-1]

    @staticmethod
    def __parse_tc_filter_port(text):
        port_pattern = (
            pp.SkipTo("port=", include=True) +
            pp.Word(pp.nums))

        return port_pattern.parseString(text)[-1]

def main():
    options = parse_option()
    logger = logbook.Logger("tcset")

    subprocrunner.logger.level = options.log_level
    if options.quiet:
        subprocrunner.logger.disable()
    else:
        subprocrunner.logger.enable()

    subprocrunner.Which("tc").verify()
    try:
        verify_netem_module()
    except ModuleNotFoundError as e:
        logger.debug(str(e))
    except subprocrunner.CommandNotFoundError as e:
        logger.error(str(e))

    if dataproperty.is_not_empty_string(options.config_file):
        return_code = 0

        loader = TcConfigLoader()
        loader.is_overwrite = options.overwrite
        loader.load_tcconfig(options.config_file)

        for tcconfig_command in loader.get_tcconfig_command_list():
            return_code |= subprocrunner.SubprocessRunner(
                tcconfig_command).run()

        return return_code

    tc = TrafficControl(options.device)
    tc.direction = options.direction
    tc.bandwidth_rate = options.bandwidth_rate
    tc.latency_ms = options.network_latency
    tc.latency_distro_ms = options.latency_distro_ms
    tc.packet_loss_rate = options.packet_loss_rate
    tc.corruption_rate = options.corruption_rate
    tc.network = options.network
    tc.port = options.port

    tc.validate()

    if options.overwrite:
        tc.delete_tc()

    tc.set_tc()

    return 0

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before># Copyright (c) 2012, Found IT A/S and Piped Project Contributors.
# See LICENSE for details.
from cyclone import web
from zope import interface
from twisted.application import strports
from twisted.python import reflect, filepath

from piped import resource, exceptions, log
from piped_cyclone import handlers

class InvalidHandlerError(exceptions.PipedError):
    pass

class CycloneProvider(object):
    """ Provides support for running `Cyclone <http://cyclone.io>`_ applications within Piped.

    For more in-depth documentation about Cyclone see: `Cyclone on GitHub <http://github.com/fiorix/cyclone>`_,
    and the `Tornado documentation <http://www.tornadoweb.org/documentation/index.html>`_.

    Configuration example:

    .. code-block:: yaml

        cyclone:
            site_name:
                listen: 8888 # (default)
                application_factory: cyclone.web.Application # (default) - a fully qualified name of a subclass
                application:
                    handlers:
                        # a tuple: pattern, handler, kwargs (optional), name (optional)
                        - ['/pattern', name.of.RequestHandler]
                        # a dict
                        - pattern: /another/(?P<pattern>.*)
                          handler: name.of.RequestHandler
                          name: handler_name # optional
                          kwargs: # optional
                            foo: 123

                    debug: true # see the debugging section below.
                    debug_allow: # optional list of ip addresses that are allowed to see tracebacks and the interactive debugger.
                        - 127.0.0.1
                    debug_timeout: 60 # (default), time in seconds to keep debuggers resident in memory before garbage collecting them.
                    debug_template: debugger.html  # name of the debugger template to use. if not set, uses the built-in template.

    The ``listen`` parameter is a Twisted ``strport``, which can be used to declare which interfaces the server should be listening on, SSL
    parameters and more. For more information, see `strports in the Twisted documentation
    <http://twistedmatrix.com/documents/11.0.0/api/twisted.internet.endpoints.html#serverFromString>`_.

    Each ``handler`` may be one of the following:

        * Fully qualified class name of a RequestHandler instance. The class will be invoked with
          cls.configure(runtime_environment) once if the method is defined. This allows for the
          class to perform any necessary setup / bootstrapping into the runtime_environment.

        * A dict with a ``class`` key set: Same as the above.

        * A dict with the ``provider`` key set. The provider is assumed to provide a subclass of
          RequestHandler as the resource.

          If the provider starts with ``pipeline.``, the provided resource will not be used as
          a RequestHandler, but will be called with ``resource.__call__(baton)``. The baton contains
          the following keys:

            * handler: A RequestHandler that is handling the request.
            * args: A tuple of arguments from the url pattern
            * kwargs: A dict of keyword arguments from the url pattern

          The pipeline should take care of calling ``baton['handler'].finish()`` when the request is
          finished.

    If the application setting ``ui_modules`` is a string, it will be loaded to a python object via
    :meth:`twisted.python.reflect.namedAny`. If it is a dict, all values are assumed to be strings
    that are fully qualified name of :class:`cyclone.web.UIModule` classes.

    Application settings that end with ``_path`` and are :class:`twisted.python.filepath.FilePath` instances
    will be converted to absolute paths. This enables the use of the :ref:`\!path <path-constructor>`-constructor
    in the configuration files.

    Global dependencies can be added to the application settings under the ``piped_dependencies`` key, which
    should be a dict. This dict will be converted to a :class:`~piped.dependencies.DependencyMap` before being
    given to the :class:`cyclone.web.Application` instance as a setting.

    In order to enable debugging tracebacks from the web server, configure the cyclone application as in the example
    configuration above and make sure your request handlers subclasses :class:`piped_cyclone.handlers.DebuggableHandler`.

    .. warning:: Enabling debugging enables allowed clients to execute arbitrary code as the user running :program:`piped`.

    To make stack traces keep their frame information, use :option:`piped -D <piped -D>` when launching :program:`piped`.

    """
    interface.classProvides(resource.IResourceProvider)

    configuration_key = 'cyclone'

    def __init__(self):
        self._configured_handler_factory_ids = set()
        self._applications_by_name = dict()

    def configure(self, runtime_environment):
        self.runtime_environment = runtime_environment
        self.dependency_manager = runtime_environment.dependency_manager
        self.resource_manager = runtime_environment.resource_manager

        cm = runtime_environment.configuration_manager

        for name, config in cm.get(self.configuration_key, dict()).items():
            if not config.get('enabled', True):
                log.info('Not starting cyclone site [{0}] because it is disabled.'.format(name))
                continue

            listen = str(config.pop('listen', '8888'))
            application_factory_name = config.pop('application_factory', 'cyclone.web.Application')
            application_factory = reflect.namedAny(application_factory_name)

            application_config = config.pop('application', dict())

            handlers = application_config.pop('handlers', [])

            transforms = self._resolve_transforms(application_config.pop('transforms', list()))

            if 'ui_modules' in application_config:
                ui_modules = application_config['ui_modules']
                if isinstance(ui_modules, dict):
                    for key, value in ui_modules.items():
                        ui_modules[key] = reflect.namedAny(value)
                elif isinstance(ui_modules, basestring):
                    application_config['ui_modules'] = reflect.namedAny(ui_modules)
                else:
                    raise Exception('Unknown ui_modules type: {0}'.format(type(ui_modules)))

            for key, value in application_config.items():
                if key.endswith('_path') and isinstance(value, filepath.FilePath):
                    application_config[key] = value.path

            for i, handler in enumerate(handlers):
                handlers[i] = self._resolve_urlspec(name, handler)

            if 'piped_dependencies' in application_config:
                for key, value in application_config['piped_dependencies'].items():
                    application_config['piped_dependencies'][key] = dict(provider=value) if isinstance(value, basestring) else value
                application_config['piped_dependencies'] = self.dependency_manager.create_dependency_map(self, **application_config['piped_dependencies'])

            application = application_factory(handlers, transforms=transforms, **application_config)

            service = strports.service(listen, application)
            service.setServiceParent(runtime_environment.application)

            self._applications_by_name[name] = application

            self.resource_manager.register('cyclone.application.{0}'.format(name), provider=self)

    def add_consumer(self, resource_dependency):
        name = resource_dependency.provider.rsplit('.', 1)[-1]
        application = self._applications_by_name[name]
        resource_dependency.on_resource_ready(application)

    def _resolve_urlspec(self, site_name, url_spec):
        if isinstance(url_spec, (list, tuple)):
            url_spec = dict(zip(['pattern', 'handler', 'kwargs', 'name'], url_spec))

        if not 'pattern' in url_spec or not 'handler' in url_spec:
            msg = 'Missing attributes in handler configuration in site [{0}]: [{1}]'.format(site_name, url_spec)
            hint = 'Both "pattern" and "handler" must be defined.'
            raise InvalidHandlerError(msg, hint)

        # string -> class specification
        if isinstance(url_spec['handler'], basestring):
            url_spec['handler'] = {'class':url_spec['handler']}

        if 'provider' in url_spec['handler']:
            handler = url_spec.pop('handler')
            provider = handler['provider']
            dependency = self.dependency_manager.add_dependency(self, handler)
            url_spec.setdefault('kwargs', dict()).update(dependency=dependency)

            if provider.startswith('pipeline.'):
                url_spec['handler_class'] = handlers.PipelineRequestHandler
                return web.url(**url_spec)

            url_spec['handler_class'] = handlers.PipedRequestHandlerProxy(dependency)
            return web.url(**url_spec)

        if 'class' in url_spec['handler']:
            handler = url_spec.pop('handler')

            cls = reflect.namedAny(handler['class'])
            if id(cls) not in self._configured_handler_factory_ids:
                self._configured_handler_factory_ids.add(id(cls))

                try:
                    cls.configure(self.runtime_environment)
                except AttributeError as ae:
                    if "has no attribute 'configure'" not in ae.args[0]:
                        raise

            url_spec['handler_class'] = cls
            return web.url(**url_spec)

        msg = 'Invalid handler configuration in site [{0}]: [{1}]'.format(site_name, url_spec)
        raise InvalidHandlerError(msg)

    def _resolve_transforms(self, transforms):
        if not transforms:
            return None

        for i, transform in transforms:
            transforms[i] = reflect.namedAny(transform)

        return transforms
<commit_msg>copy the configuration before modifying it<commit_after># Copyright (c) 2012, Found IT A/S and Piped Project Contributors.
# See LICENSE for details.
import copy

from cyclone import web
from zope import interface
from twisted.application import strports
from twisted.python import reflect, filepath

from piped import resource, exceptions, log
from piped_cyclone import handlers

class InvalidHandlerError(exceptions.PipedError):
    pass

class CycloneProvider(object):
    """ Provides support for running `Cyclone <http://cyclone.io>`_ applications within Piped.

    For more in-depth documentation about Cyclone see: `Cyclone on GitHub <http://github.com/fiorix/cyclone>`_,
    and the `Tornado documentation <http://www.tornadoweb.org/documentation/index.html>`_.

    Configuration example:

    .. code-block:: yaml

        cyclone:
            site_name:
                listen: 8888 # (default)
                application_factory: cyclone.web.Application # (default) - a fully qualified name of a subclass
                application:
                    handlers:
                        # a tuple: pattern, handler, kwargs (optional), name (optional)
                        - ['/pattern', name.of.RequestHandler]
                        # a dict
                        - pattern: /another/(?P<pattern>.*)
                          handler: name.of.RequestHandler
                          name: handler_name # optional
                          kwargs: # optional
                            foo: 123

                    debug: true # see the debugging section below.
                    debug_allow: # optional list of ip addresses that are allowed to see tracebacks and the interactive debugger.
                        - 127.0.0.1
                    debug_timeout: 60 # (default), time in seconds to keep debuggers resident in memory before garbage collecting them.
                    debug_template: debugger.html  # name of the debugger template to use. if not set, uses the built-in template.

    The ``listen`` parameter is a Twisted ``strport``, which can be used to declare which interfaces the server should be listening on, SSL
    parameters and more. For more information, see `strports in the Twisted documentation
    <http://twistedmatrix.com/documents/11.0.0/api/twisted.internet.endpoints.html#serverFromString>`_.

    Each ``handler`` may be one of the following:

        * Fully qualified class name of a RequestHandler instance. The class will be invoked with
          cls.configure(runtime_environment) once if the method is defined. This allows for the
          class to perform any necessary setup / bootstrapping into the runtime_environment.

        * A dict with a ``class`` key set: Same as the above.

        * A dict with the ``provider`` key set. The provider is assumed to provide a subclass of
          RequestHandler as the resource.

          If the provider starts with ``pipeline.``, the provided resource will not be used as
          a RequestHandler, but will be called with ``resource.__call__(baton)``. The baton contains
          the following keys:

            * handler: A RequestHandler that is handling the request.
            * args: A tuple of arguments from the url pattern
            * kwargs: A dict of keyword arguments from the url pattern

          The pipeline should take care of calling ``baton['handler'].finish()`` when the request is
          finished.

    If the application setting ``ui_modules`` is a string, it will be loaded to a python object via
    :meth:`twisted.python.reflect.namedAny`. If it is a dict, all values are assumed to be strings
    that are fully qualified name of :class:`cyclone.web.UIModule` classes.

    Application settings that end with ``_path`` and are :class:`twisted.python.filepath.FilePath` instances
    will be converted to absolute paths. This enables the use of the :ref:`\!path <path-constructor>`-constructor
    in the configuration files.

    Global dependencies can be added to the application settings under the ``piped_dependencies`` key, which
    should be a dict. This dict will be converted to a :class:`~piped.dependencies.DependencyMap` before being
    given to the :class:`cyclone.web.Application` instance as a setting.

    In order to enable debugging tracebacks from the web server, configure the cyclone application as in the example
    configuration above and make sure your request handlers subclasses :class:`piped_cyclone.handlers.DebuggableHandler`.

    .. warning:: Enabling debugging enables allowed clients to execute arbitrary code as the user running :program:`piped`.

    To make stack traces keep their frame information, use :option:`piped -D <piped -D>` when launching :program:`piped`.

    """
    interface.classProvides(resource.IResourceProvider)

    configuration_key = 'cyclone'

    def __init__(self):
        self._configured_handler_factory_ids = set()
        self._applications_by_name = dict()

    def configure(self, runtime_environment):
        self.runtime_environment = runtime_environment
        self.dependency_manager = runtime_environment.dependency_manager
        self.resource_manager = runtime_environment.resource_manager

        cm = runtime_environment.configuration_manager

        for name, config in cm.get(self.configuration_key, dict()).items():
            if not config.get('enabled', True):
                log.info('Not starting cyclone site [{0}] because it is disabled.'.format(name))
                continue

            # take a deep copy of the configurations, since we're going to modify it
            config = copy.deepcopy(config)

            listen = str(config.pop('listen', '8888'))
            application_factory_name = config.pop('application_factory', 'cyclone.web.Application')
            application_factory = reflect.namedAny(application_factory_name)

            application_config = config.pop('application', dict())

            handlers = application_config.pop('handlers', [])

            transforms = self._resolve_transforms(application_config.pop('transforms', list()))

            if 'ui_modules' in application_config:
                ui_modules = application_config['ui_modules']
                if isinstance(ui_modules, dict):
                    for key, value in ui_modules.items():
                        ui_modules[key] = reflect.namedAny(value)
                elif isinstance(ui_modules, basestring):
                    application_config['ui_modules'] = reflect.namedAny(ui_modules)
                else:
                    raise Exception('Unknown ui_modules type: {0}'.format(type(ui_modules)))

            for key, value in application_config.items():
                if key.endswith('_path') and isinstance(value, filepath.FilePath):
                    application_config[key] = value.path

            for i, handler in enumerate(handlers):
                handlers[i] = self._resolve_urlspec(name, handler)

            if 'piped_dependencies' in application_config:
                for key, value in application_config['piped_dependencies'].items():
                    application_config['piped_dependencies'][key] = dict(provider=value) if isinstance(value, basestring) else value
                application_config['piped_dependencies'] = self.dependency_manager.create_dependency_map(self, **application_config['piped_dependencies'])

            application = application_factory(handlers, transforms=transforms, **application_config)

            service = strports.service(listen, application)
            service.setServiceParent(runtime_environment.application)

            self._applications_by_name[name] = application

            self.resource_manager.register('cyclone.application.{0}'.format(name), provider=self)

    def add_consumer(self, resource_dependency):
        name = resource_dependency.provider.rsplit('.', 1)[-1]
        application = self._applications_by_name[name]
        resource_dependency.on_resource_ready(application)

    def _resolve_urlspec(self, site_name, url_spec):
        if isinstance(url_spec, (list, tuple)):
            url_spec = dict(zip(['pattern', 'handler', 'kwargs', 'name'], url_spec))

        if not 'pattern' in url_spec or not 'handler' in url_spec:
            msg = 'Missing attributes in handler configuration in site [{0}]: [{1}]'.format(site_name, url_spec)
            hint = 'Both "pattern" and "handler" must be defined.'
            raise InvalidHandlerError(msg, hint)

        # string -> class specification
        if isinstance(url_spec['handler'], basestring):
            url_spec['handler'] = {'class':url_spec['handler']}

        if 'provider' in url_spec['handler']:
            handler = url_spec.pop('handler')
            provider = handler['provider']
            dependency = self.dependency_manager.add_dependency(self, handler)
            url_spec.setdefault('kwargs', dict()).update(dependency=dependency)

            if provider.startswith('pipeline.'):
                url_spec['handler_class'] = handlers.PipelineRequestHandler
                return web.url(**url_spec)

            url_spec['handler_class'] = handlers.PipedRequestHandlerProxy(dependency)
            return web.url(**url_spec)

        if 'class' in url_spec['handler']:
            handler = url_spec.pop('handler')

            cls = reflect.namedAny(handler['class'])
            if id(cls) not in self._configured_handler_factory_ids:
                self._configured_handler_factory_ids.add(id(cls))

                try:
                    cls.configure(self.runtime_environment)
                except AttributeError as ae:
                    if "has no attribute 'configure'" not in ae.args[0]:
                        raise

            url_spec['handler_class'] = cls
            return web.url(**url_spec)

        msg = 'Invalid handler configuration in site [{0}]: [{1}]'.format(site_name, url_spec)
        raise InvalidHandlerError(msg)

    def _resolve_transforms(self, transforms):
        if not transforms:
            return None

        for i, transform in transforms:
            transforms[i] = reflect.namedAny(transform)

        return transforms
<|endoftext|>


<commit_before>
import mx.DateTime
from pyIEM import iemdb
i = iemdb.iemdb()
postgis = i['postgis']

tests = [
 'LSX.TO.11', 'CAE.TO.2', 'GSP.TO.5', 'PAH.SV.10', 'JKL.TO.4', 'LWX.TO.3',
 'RNK.TO.2', 'IND.TO.7', 'IND.TO.8', 'LMK.TO.12', 'LMK.TO.13', 'LMK.TO.14',
 'PAH.TO.17', 'PAH.TO.18', 'PAH.TO.19', 'PAH.TO.20', 'PAH.TO.21',
 'MEG.TO.20'
]

sql = "SELECT wfo, min(eventid), max(eventid), phenomena from warnings_%s \
       WHERE phenomena IN ('MA','FF','SV','TO') and significance = 'W' \
       GROUP by wfo, phenomena" % (mx.DateTime.now().year, )
rs = postgis.query(sql).dictresult()
for i in range(len(rs)):
  sEvent = int(rs[i]['min'])
  eEvent = int(rs[i]['max'])
  phenomena = rs[i]['phenomena']
  wfo = rs[i]['wfo']

  for eventid in range(sEvent, eEvent):
    sql = "SELECT gtype, count(*) as c from warnings_%s WHERE wfo = '%s' \
           and phenomena = '%s' and eventid = '%s' and significance = 'W' \
           GROUP by gtype" % (mx.DateTime.now().year, wfo,\
           phenomena, eventid)
    rs2 = postgis.query(sql).dictresult()
    polyCount = 0
    cntyCount = 0
    for q in range(len(rs2)):
      if (rs2[q]['gtype'] == "P"): polyCount = int(rs2[q]['c'])
      if (rs2[q]['gtype'] == "C"): cntyCount = int(rs2[q]['c'])

    if (cntyCount == 0 and polyCount == 0):
      lookup = "%s.%s.%s" % (wfo, phenomena, eventid)
      if lookup not in tests:
        print "Warning Missing WFO: %s PHENOMENA: %s EVENTID: %s" % (wfo, phenomena, eventid)
    elif (polyCount == 0):
      print "SBW Missing     WFO: %s PHENOMENA: %s EVENTID: %s" % (wfo, phenomena, eventid)
    elif (cntyCount == 0):
      print "County Missing  WFO: %s PHENOMENA: %s EVENTID: %s" % (wfo, phenomena, eventid)
    elif (polyCount > 1):
      print "Duplicate SBW   WFO: %s PHENOMENA: %s EVENTID: %s" % (wfo, phenomena, eventid)
      sql = "DELETE from warnings_%s WHERE oid IN (\
         SELECT max(oid) as m from warnings_%s WHERE wfo = '%s' \
         and phenomena = '%s' and eventid = '%s' and significance = 'W' \
         and gtype = 'P')" % (mx.DateTime.now().year, mx.DateTime.now().year,\
          wfo, phenomena, eventid)
      postgis.query(sql)
      sql = "DELETE from warnings_%s WHERE oid IN (\
              select m from (\
select ugc,  max(oid) as m, count(oid) as c from warnings_%s WHERE wfo = '%s' \
and phenomena = '%s' and eventid = '%s' and significance = 'W' \
and gtype = 'C' GROUP by ugc) as foo WHERE c > 1)" % (mx.DateTime.now().year, mx.DateTime.now().year,\
          wfo, phenomena, eventid)
      postgis.query(sql)
<commit_msg>add test for Wichita<commit_after>
import mx.DateTime
from pyIEM import iemdb
i = iemdb.iemdb()
postgis = i['postgis']

tests = [
 'LSX.TO.11', 'CAE.TO.2', 'GSP.TO.5', 'PAH.SV.10', 'JKL.TO.4', 'LWX.TO.3',
 'RNK.TO.2', 'IND.TO.7', 'IND.TO.8', 'LMK.TO.12', 'LMK.TO.13', 'LMK.TO.14',
 'PAH.TO.17', 'PAH.TO.18', 'PAH.TO.19', 'PAH.TO.20', 'PAH.TO.21',
 'MEG.TO.20', 'ICT.TO.6'
]

sql = "SELECT wfo, min(eventid), max(eventid), phenomena from warnings_%s \
       WHERE phenomena IN ('MA','FF','SV','TO') and significance = 'W' \
       GROUP by wfo, phenomena" % (mx.DateTime.now().year, )
rs = postgis.query(sql).dictresult()
for i in range(len(rs)):
  sEvent = int(rs[i]['min'])
  eEvent = int(rs[i]['max'])
  phenomena = rs[i]['phenomena']
  wfo = rs[i]['wfo']

  for eventid in range(sEvent, eEvent):
    sql = "SELECT gtype, count(*) as c from warnings_%s WHERE wfo = '%s' \
           and phenomena = '%s' and eventid = '%s' and significance = 'W' \
           GROUP by gtype" % (mx.DateTime.now().year, wfo,\
           phenomena, eventid)
    rs2 = postgis.query(sql).dictresult()
    polyCount = 0
    cntyCount = 0
    for q in range(len(rs2)):
      if (rs2[q]['gtype'] == "P"): polyCount = int(rs2[q]['c'])
      if (rs2[q]['gtype'] == "C"): cntyCount = int(rs2[q]['c'])

    if (cntyCount == 0 and polyCount == 0):
      lookup = "%s.%s.%s" % (wfo, phenomena, eventid)
      if lookup not in tests:
        print "Warning Missing WFO: %s PHENOMENA: %s EVENTID: %s" % (wfo, phenomena, eventid)
    elif (polyCount == 0):
      print "SBW Missing     WFO: %s PHENOMENA: %s EVENTID: %s" % (wfo, phenomena, eventid)
    elif (cntyCount == 0):
      print "County Missing  WFO: %s PHENOMENA: %s EVENTID: %s" % (wfo, phenomena, eventid)
    elif (polyCount > 1):
      print "Duplicate SBW   WFO: %s PHENOMENA: %s EVENTID: %s" % (wfo, phenomena, eventid)
      sql = "DELETE from warnings_%s WHERE oid IN (\
         SELECT max(oid) as m from warnings_%s WHERE wfo = '%s' \
         and phenomena = '%s' and eventid = '%s' and significance = 'W' \
         and gtype = 'P')" % (mx.DateTime.now().year, mx.DateTime.now().year,\
          wfo, phenomena, eventid)
      postgis.query(sql)
      sql = "DELETE from warnings_%s WHERE oid IN (\
              select m from (\
select ugc,  max(oid) as m, count(oid) as c from warnings_%s WHERE wfo = '%s' \
and phenomena = '%s' and eventid = '%s' and significance = 'W' \
and gtype = 'C' GROUP by ugc) as foo WHERE c > 1)" % (mx.DateTime.now().year, mx.DateTime.now().year,\
          wfo, phenomena, eventid)
      postgis.query(sql)
<|endoftext|>


<commit_before># JN 2015-10-03 refactor

"""
cumulative spike time plot
"""

import numpy as np
TEXT_SIZE = 'small'

def spike_cumulative(plot, times, special, ylim=None, text=True):
    """
    draw cumulative spike line
    """
    if ylim is None:
        ylim = (-200, 200)

    duration = (times[-1] - times[0])/1000  # in sec

    if duration < 60:
        dur_str = '{:.0f} s'.format(duration)
    else:
        duration /= 60  # in min
        if duration < 60:
            dur_str = '{:.0f} min'.format(duration)
        else:
            duration /= 60  # in h
            dur_str = '{:.0f} h'.format(duration)

    plot.plot(times,
              np.linspace(ylim[0], ylim[1], len(times)), 'm', lw=1, alpha=.7)
    xlim = plot.get_xlim()
    if text:
        plot.text(xlim[1], ylim[1], dur_str, ha='right',
                  va='top', size=TEXT_SIZE)
    if special:
        nspk_str = '{} spk '.format(len(times))
        plot.text(xlim[1], ylim[0], nspk_str, ha='right',
                  va='bottom', size=TEXT_SIZE)
    plot.set_xticklabels([])
    plot.set_xticks([])
<commit_msg>added options to artifact plot<commit_after># JN 2015-10-03 refactor

"""
cumulative spike time plot
"""

import numpy as np
TEXT_SIZE = 'small'

def spike_cumulative(plot, times, special, ylim=None, text=True,
                     lw=1, alpha=.7):
    """
    draw cumulative spike line
    """
    if ylim is None:
        ylim = (-200, 200)

    duration = (times[-1] - times[0])/1000  # in sec

    if duration < 60:
        dur_str = '{:.0f} s'.format(duration)
    else:
        duration /= 60  # in min
        if duration < 60:
            dur_str = '{:.0f} min'.format(duration)
        else:
            duration /= 60  # in h
            dur_str = '{:.0f} h'.format(duration)

    plot.plot(times,
              np.linspace(ylim[0], ylim[1], len(times)), 'm',
              lw=lw, alpha=alpha)

    xlim = plot.get_xlim()
    if text:
        plot.text(xlim[1], ylim[1], dur_str, ha='right',
                  va='top', size=TEXT_SIZE)
    if special:
        nspk_str = '{} spk '.format(len(times))
        plot.text(xlim[1], ylim[0], nspk_str, ha='right',
                  va='bottom', size=TEXT_SIZE)
    plot.set_xticklabels([])
    plot.set_xticks([])
<|endoftext|>


<commit_before>var=10
<commit_msg>Remove test modules<commit_after><|endoftext|>