<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import sys
import re
import codecs
import treetaggerwrapper

import keywords

from argparse import ArgumentParser

tagdir = os.getenv('TREETAGGER_ROOT')
tagger = treetaggerwrapper.TreeTagger(TAGLANG='en',TAGDIR=tagdir)
options = None

whitelist = []
gene = []
langkeywords = []

def parse_command_line():
    parser = ArgumentParser()
    parser.add_argument(
        '-v',
        '--version',
        action='version',
        version=u'%(prog)s version 0.1'
    )
    parser.add_argument(
        'file',
        metavar='FILE/DIR',
        nargs='+',
        help='source code file/dir'
    )
    options = parser.parse_args()
    return options, parser

r_alnum = re.compile(r'^[a-zA-Z0-9]+$')
def isalnum(s):
    return r_alnum.match(s) is not None

r_alpha = re.compile(r'^[a-zA-Z]+$')
def isalpha(s):
    return r_alpha.match(s) is not None

r_block_comment_begin = re.compile('(.*)/\*.*')
r_block_comment_end = re.compile('.*\*/(.*)')
line_comment = '//'

def checkcomment(text, block_comment):
    if block_comment:
        m = r_block_comment_end.match(text)
        if m:
            block_comment = False
            text = m.group(1)
    if not block_comment:
        m = r_block_comment_begin.match(text)
        if m:
            block_comment = True
            text = m.group(1)
        line_comment_start = text.find(line_comment)
        if line_comment_start != -1:
            text = text[:line_comment_start]
    return text, block_comment

def is_whitelist(word):
    if word in whitelist:
        return True
    if word in langkeywords:
        return True
    # 辞書にあったら除外
    if word in gene:
        return True
    return False

def is_suspicion(word):
    length = len(word)
    # 1文字だけは除外
    if length <= 1:
        return False
    if not isalpha(word):
        return False
    if is_whitelist(word):
        return False
    # 過去形
    if length > 2 and word.endswith('ed'):
        if is_whitelist(word[:-2]):
            return False
        if is_whitelist(word[:-1]):
            return False
    # 進行形
    if length > 3 and word.endswith('ing'):
        if is_whitelist(word[:-3]):
            return False
        if is_whitelist(word[:-3] + 'e'):
            return False
    # 複数形
    if word.endswith('s'):
        if is_whitelist(word[:-1]):
            return False
    if length > 3 and word.endswith('ies'):
        if is_whitelist(word[:-3] + 'y'):
            return False
    return True

r_sign = re.compile('[!-/:-@[-`{-~]')
def text_transform(text):
    text = re.sub("([a-z0-9])([A-Z])(?=[a-z])", lambda x: x.group(1) + "_" + x.group(2), text)
    return re.sub(r_sign, ' ', text)

def checktagger(text, line, words):
    tags_plain = tagger.TagText(text)
    tags = treetaggerwrapper.make_tags(tags_plain)
    for tag in tags:
        if isinstance(tag, treetaggerwrapper.NotTag):
            continue
        word = tag.word.lower()
        if is_suspicion(word):
            if words.has_key(word):
                words[word].append(line)
            else:
                words[word] = [line]

def check(filepath):
    filename = os.path.basename(filepath)
    f = codecs.open(filepath, 'r', encoding='utf-8-sig')
    line_count = 1
    block_comment = False
    words = {}
    for line in f:
        text = line.strip()
        text, block_comment = checkcomment(text, block_comment)
        if not block_comment:
            if len(text) > 0:
                checktagger(text_transform(text), line_count, words)
        line_count += 1
    return words

def printresult(filepath, d):
    for k,v in sorted(d.items(), key=lambda x: len(x[1])):
        for line in v:
            print("{0}({1}): warning: \"{2}\": abbreviation ??".format(filepath, line, k))

def checkfile(f):
    global langkeywords
    langkeywords = keywords.getkeywords(f)
    words = check(f)
    printresult(f, words)

def checkdir(dir):
    for d in os.listdir(dir):
        d = os.path.join(dir, d)
        if os.path.isdir(d):
            checkdir(d)
        else:
            checkfile(d)

def main():
    options, parser = parse_command_line()
    for f in options.file:
        if os.path.isdir(f):
            checkdir(f)
        else:
            checkfile(f)

if __name__ == '__main__':
    main()
<commit_msg>support whitelist<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import os
import sys
import re
import codecs
import treetaggerwrapper

import keywords

from argparse import ArgumentParser

tagdir = os.getenv('TREETAGGER_ROOT')
tagger = treetaggerwrapper.TreeTagger(TAGLANG='en',TAGDIR=tagdir)
options = None

whitelist = []
gene = []
langkeywords = []

def parse_command_line():
    parser = ArgumentParser()
    parser.add_argument(
        '-v',
        '--version',
        action='version',
        version=u'%(prog)s version 0.1'
    )
    parser.add_argument(
        '-g',
        '--gene',
        action='append',
        help='exlude word'
    )
    parser.add_argument(
        '-w',
        '--whitelist',
        help='whitelist file'
    )
    parser.add_argument(
        'file',
        metavar='FILE/DIR',
        nargs='+',
        help='source code file/dir'
    )
    options = parser.parse_args()
    return options, parser

r_alnum = re.compile(r'^[a-zA-Z0-9]+$')
def isalnum(s):
    return r_alnum.match(s) is not None

r_alpha = re.compile(r'^[a-zA-Z]+$')
def isalpha(s):
    return r_alpha.match(s) is not None

r_block_comment_begin = re.compile('(.*)/\*.*')
r_block_comment_end = re.compile('.*\*/(.*)')
line_comment = '//'

def checkcomment(text, block_comment):
    if block_comment:
        m = r_block_comment_end.match(text)
        if m:
            block_comment = False
            text = m.group(1)
    if not block_comment:
        m = r_block_comment_begin.match(text)
        if m:
            block_comment = True
            text = m.group(1)
        line_comment_start = text.find(line_comment)
        if line_comment_start != -1:
            text = text[:line_comment_start]
    return text, block_comment

def is_whitelist(word):
    if word in whitelist:
        return True
    if word in langkeywords:
        return True
    # 辞書にあったら除外
    if word in gene:
        return True
    return False

def is_suspicion(word):
    length = len(word)
    # 1文字だけは除外
    if length <= 1:
        return False
    if not isalpha(word):
        return False
    if is_whitelist(word):
        return False
    # 過去形
    if length > 2 and word.endswith('ed'):
        if is_whitelist(word[:-2]):
            return False
        if is_whitelist(word[:-1]):
            return False
    # 進行形
    if length > 3 and word.endswith('ing'):
        if is_whitelist(word[:-3]):
            return False
        if is_whitelist(word[:-3] + 'e'):
            return False
    # 複数形
    if word.endswith('s'):
        if is_whitelist(word[:-1]):
            return False
    if length > 3 and word.endswith('ies'):
        if is_whitelist(word[:-3] + 'y'):
            return False
    return True

r_sign = re.compile('[!-/:-@[-`{-~]')
def text_transform(text):
    text = re.sub("([a-z0-9])([A-Z])(?=[a-z])", lambda x: x.group(1) + "_" + x.group(2), text)
    return re.sub(r_sign, ' ', text)

def checktagger(text, line, words):
    tags_plain = tagger.TagText(text)
    tags = treetaggerwrapper.make_tags(tags_plain)
    for tag in tags:
        if isinstance(tag, treetaggerwrapper.NotTag):
            continue
        word = tag.word.lower()
        if is_suspicion(word):
            if words.has_key(word):
                words[word].append(line)
            else:
                words[word] = [line]

def check(filepath):
    filename = os.path.basename(filepath)
    f = codecs.open(filepath, 'r', encoding='utf-8-sig')
    line_count = 1
    block_comment = False
    words = {}
    for line in f:
        text = line.strip()
        text, block_comment = checkcomment(text, block_comment)
        if not block_comment:
            if len(text) > 0:
                checktagger(text_transform(text), line_count, words)
        line_count += 1
    return words

def printresult(filepath, d):
    for k,v in sorted(d.items(), key=lambda x: len(x[1])):
        for line in v:
            print("{0}({1}): warning: \"{2}\": abbreviation ??".format(filepath, line, k))

def checkfile(f):
    global langkeywords
    langkeywords = keywords.getkeywords(f)
    words = check(f)
    printresult(f, words)

def checkdir(dir):
    for d in os.listdir(dir):
        d = os.path.join(dir, d)
        if os.path.isdir(d):
            checkdir(d)
        else:
            checkfile(d)

r_geneword = re.compile(r'^[a-zA-Z][a-z]+$')
def isgeneword(s):
    return r_geneword.match(s) is not None

def make_gene(file):
    f = open(file, 'r')
    for line in f:
        word = line.strip()
        if isgeneword(word):
            gene.append(word.lower())

def setup(options):
    global gene
    global whitelist
    if options.gene:
        for g in options.gene:
            gene.extend(make_gene(g))
    if options.whitelist:
        for word in options.whitelist:
            if isalpha(word):
                whitelist.append(word.lower())

def main():
    options, parser = parse_command_line()
    setup(options)
    for f in options.file:
        if os.path.isdir(f):
            checkdir(f)
        else:
            checkfile(f)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># pylint: disable=C1001
class Config:
    # Custom
    DEBUG_FILE = '/tmp/gmw.log'

    # Flask
    PERMANENT_SESSION_LIFETIME = 1209600  # 14 days
    SECRET_KEY = ''
    TESTING = False
    DEBUG = True

    # SQLAlchemy
    SQLALCHEMY_ECHO = False

    # Celery
    CELERY_TIMEZONE = 'Europe/Berlin'
    CELERY_BROKER_URL = ''
    CELERY_RESULT_SERIALIZER = 'json'
    CELERY_TASK_SERIALIZER = 'json'
    CELERY_ACCEPT_CONTENT = ['json']

    # Oauth
    GITHUB_AUTH = (None, None)

class ConfigDevelopment(Config):
    SQLALCHEMY_ECHO = True

class ConfigTesting(Config):
    SECRET_KEY = 'testing'  # noqa
    TESTING = True

class ConfigProduction(Config):
    SQLALCHEMY_POOL_SIZE = 0  # disable pooling
    DEBUG = False
<commit_msg>CELERY_RESULT_ENGINE_OPTIONS added, refs #101<commit_after># pylint: disable=C1001
class Config:
    # Custom
    DEBUG_FILE = '/tmp/gmw.log'

    # Flask
    PERMANENT_SESSION_LIFETIME = 1209600  # 14 days
    SECRET_KEY = ''
    TESTING = False
    DEBUG = True

    # SQLAlchemy
    SQLALCHEMY_ECHO = False

    # Celery
    CELERY_TIMEZONE = 'Europe/Berlin'
    CELERY_BROKER_URL = ''
    CELERY_RESULT_SERIALIZER = 'json'
    CELERY_TASK_SERIALIZER = 'json'
    CELERY_ACCEPT_CONTENT = ['json']
    CELERY_RESULT_ENGINE_OPTIONS = {'echo': True}
    CELERY_RESULT_DB_SHORT_LIVED_SESSIONS = True

    # Oauth
    GITHUB_AUTH = (None, None)

class ConfigDevelopment(Config):
    SQLALCHEMY_ECHO = True

class ConfigTesting(Config):
    SECRET_KEY = 'testing'  # noqa
    TESTING = True

class ConfigProduction(Config):
    SQLALCHEMY_POOL_SIZE = 0  # disable pooling
    DEBUG = False
<|endoftext|>


<commit_before><commit_msg>Fix to use init_scope()<commit_after><|endoftext|>


<commit_before># -*- test-case-name: go.apps.jsbox.tests.test_optout -*-
# -*- coding: utf-8 -*-

"""Resource for accessing and modifying a contact's opt-out/opt-in
   status from the sandbox"""

from twisted.internet.defer import inlineCallbacks, returnValue

from vumi.application.sandbox import SandboxResource

class OptoutException(Exception):
    pass

def optout_authorized(func):
    @inlineCallbacks
    def wrapper(self, api, command):
        if not (yield self.is_allowed(api)):
            returnValue(self.reply(
                command, success=False,
                reason='Account not allowed to manage optouts.'))

        resp = yield func(self, api, command)
        returnValue(resp)
    return wrapper

def ensure_params(*keys):
    def decorator(func):
        def wrapper(self, api, command):
            for key in keys:
                if key not in command:
                    return self.reply(command, success=False,
                                      reason='Missing key: %s' % (key,))

                value = command[key]
                # value is not allowed to be `False`, `None` or an empty
                # string.
                if not value:
                    return self.reply(
                        command, success=False,
                        reason='Invalid value "%s" for "%s"' % (value, key))

            return func(self, api, command)
        return wrapper
    return decorator

class OptoutResource(SandboxResource):

    def get_user_api(self, api):
        return self.app_worker.user_api_for_api(api)

    def optout_store_for_api(self, api):
        return self.get_user_api(api).optout_store

    def is_allowed(self, api):
        user_api = self.get_user_api(api)
        d = user_api.get_user_account()
        d.addCallback(lambda account: account.can_manage_optouts)
        return d

    @ensure_params('address_type', 'address_value')
    @optout_authorized
    @inlineCallbacks
    def handle_status(self, api, command):
        """
        Accepts an address_type and address_value and
        retrieves the opt-out entry for it.

        Returns ``None`` if it doesn't exist.
        """

        address_type = command['address_type']
        address_value = command['address_value']
        optout = yield self.optout_store_for_api(api).get_opt_out(
            address_type, address_value)
        if optout is not None:
            returnValue(self.reply(command, success=True, opted_out=True,
                                   created_at=optout.created_at,
                                   message_id=optout.message))
        else:
            returnValue(self.reply(command, success=True, opted_out=False))

    @optout_authorized
    def handle_count(self, api, command):
        """
        Return a count of however many opt-outs there are
        """
        oos = self.optout_store_for_api(api)
        d = oos.count()
        d.addCallback(
            lambda count: self.reply(command, success=True, count=count))
        return d

    @ensure_params('address_type', 'address_value', 'message_id')
    @optout_authorized
    @inlineCallbacks
    def handle_optout(self, api, command):
        """
        Opt out an address_type, address_value combination
        """
        oos = self.optout_store_for_api(api)
        address_type = command['address_type']
        address_value = command['address_value']
        message_id = command['message_id']
        optout = yield oos.new_opt_out(address_type, address_value, message={
            'message_id': message_id,
        })
        returnValue(self.reply(command, success=True, opted_out=True,
                               created_at=optout.created_at,
                               message_id=optout.message))

    @ensure_params('address_type', 'address_value')
    @optout_authorized
    def handle_cancel_optout(self, api, command):
        """
        Cancel an opt-out, effectively opting an address_type & address_value
        combination back in.
        """
        oos = self.optout_store_for_api(api)
        address_type = command['address_type']
        address_value = command['address_value']
        d = oos.delete_opt_out(address_type, address_value)
        d.addCallback(
            lambda _: self.reply(command, success=True, opted_out=False))
        return d
<commit_msg>now really removing the exception<commit_after># -*- test-case-name: go.apps.jsbox.tests.test_optout -*-
# -*- coding: utf-8 -*-

"""Resource for accessing and modifying a contact's opt-out/opt-in
   status from the sandbox"""

from twisted.internet.defer import inlineCallbacks, returnValue

from vumi.application.sandbox import SandboxResource

def optout_authorized(func):
    @inlineCallbacks
    def wrapper(self, api, command):
        if not (yield self.is_allowed(api)):
            returnValue(self.reply(
                command, success=False,
                reason='Account not allowed to manage optouts.'))

        resp = yield func(self, api, command)
        returnValue(resp)
    return wrapper

def ensure_params(*keys):
    def decorator(func):
        def wrapper(self, api, command):
            for key in keys:
                if key not in command:
                    return self.reply(command, success=False,
                                      reason='Missing key: %s' % (key,))

                value = command[key]
                # value is not allowed to be `False`, `None` or an empty
                # string.
                if not value:
                    return self.reply(
                        command, success=False,
                        reason='Invalid value "%s" for "%s"' % (value, key))

            return func(self, api, command)
        return wrapper
    return decorator

class OptoutResource(SandboxResource):

    def get_user_api(self, api):
        return self.app_worker.user_api_for_api(api)

    def optout_store_for_api(self, api):
        return self.get_user_api(api).optout_store

    def is_allowed(self, api):
        user_api = self.get_user_api(api)
        d = user_api.get_user_account()
        d.addCallback(lambda account: account.can_manage_optouts)
        return d

    @ensure_params('address_type', 'address_value')
    @optout_authorized
    @inlineCallbacks
    def handle_status(self, api, command):
        """
        Accepts an address_type and address_value and
        retrieves the opt-out entry for it.

        Returns ``None`` if it doesn't exist.
        """

        address_type = command['address_type']
        address_value = command['address_value']
        optout = yield self.optout_store_for_api(api).get_opt_out(
            address_type, address_value)
        if optout is not None:
            returnValue(self.reply(command, success=True, opted_out=True,
                                   created_at=optout.created_at,
                                   message_id=optout.message))
        else:
            returnValue(self.reply(command, success=True, opted_out=False))

    @optout_authorized
    def handle_count(self, api, command):
        """
        Return a count of however many opt-outs there are
        """
        oos = self.optout_store_for_api(api)
        d = oos.count()
        d.addCallback(
            lambda count: self.reply(command, success=True, count=count))
        return d

    @ensure_params('address_type', 'address_value', 'message_id')
    @optout_authorized
    @inlineCallbacks
    def handle_optout(self, api, command):
        """
        Opt out an address_type, address_value combination
        """
        oos = self.optout_store_for_api(api)
        address_type = command['address_type']
        address_value = command['address_value']
        message_id = command['message_id']
        optout = yield oos.new_opt_out(address_type, address_value, message={
            'message_id': message_id,
        })
        returnValue(self.reply(command, success=True, opted_out=True,
                               created_at=optout.created_at,
                               message_id=optout.message))

    @ensure_params('address_type', 'address_value')
    @optout_authorized
    def handle_cancel_optout(self, api, command):
        """
        Cancel an opt-out, effectively opting an address_type & address_value
        combination back in.
        """
        oos = self.optout_store_for_api(api)
        address_type = command['address_type']
        address_value = command['address_value']
        d = oos.delete_opt_out(address_type, address_value)
        d.addCallback(
            lambda _: self.reply(command, success=True, opted_out=False))
        return d
<|endoftext|>


<commit_before>import cuttsum.pipeline.jobs as jobs
from cuttsum.sentsim import SentenceLatentVectorsResource
from cuttsum.summarizer.ap import APSummarizer, APSalienceSummarizer
from cuttsum.summarizer.baseline import HACSummarizer
import os
import pandas as pd
import numpy as np
from datetime import datetime, timedelta
from sklearn.metrics.pairwise import cosine_similarity
from cuttsum.pipeline.salience import SaliencePredictionAggregator

class APFilteredSummary(object):
    def __init__(self):
        self.dir_ = os.path.join(
            os.getenv(u'TREC_DATA', u'.'), u'ap-filtered-summaries')
        if not os.path.exists(self.dir_):
            os.makedirs(self.dir_)

    def get_tsv_path(self, event, prefix=None, feature_set=None):
        return os.path.join(self.dir_,
            "ap-{}.tsv".format(event.fs_name()))

    def get_dataframe(self, event):
        tsv = self.get_tsv_path(event)
        if not os.path.exists(tsv):
            return None
        else:
            with open(tsv, u'r') as f:
                df = pd.io.parsers.read_csv(
                    f, sep='\t', quoting=3)
                df.columns = ['query id', 'system id', 'run id', 'stream id',
                              'sentence id', 'timestamp', 'conf', 'text']
            return df

    def make(self, event, min_cluster_size=2, sim_threshold=.2264):
        tsv_path = self.get_tsv_path(event)
        lvecs = SentenceLatentVectorsResource()
        ap = APSummarizer()
        cluster_df = ap.get_dataframe(event)
       #for _, row in cluster_df.iterrows():
       #    print row['hour'], datetime.utcfromtimestamp(row['timestamp'])
        updates = []
        Xcache = None
        timestamps = sorted(list(cluster_df['timestamp'].unique()))
        for timestamp in timestamps:
            hour = datetime.utcfromtimestamp(timestamp) - timedelta(hours=1)
            lvec_df = lvecs.get_dataframe(event, hour)
            lvec_df.drop_duplicates(['stream id', 'sentence id'], inplace=True)
            clusters = cluster_df[cluster_df['timestamp'] == timestamp].copy()
            clusters.sort(['stream id', 'sentence id'], inplace=True)
            for _, row in clusters.iterrows():
                if row['cluster size'] < min_cluster_size:
                    continue
                vec = lvec_df.loc[
                    (lvec_df['stream id'] == row['stream id']) & \
                    (lvec_df['sentence id'] == row['sentence id'])].as_matrix()[:,2:].astype(np.float64)
                if Xcache is None:
                    Xcache = vec
                else:
                    if np.max(cosine_similarity(Xcache, vec)) >= sim_threshold:
                        continue
                    else:
                        Xcache = np.vstack((Xcache, vec))
                updates.append({
                    'query id': event.query_id[5:],
                    'system id': 'cunlp',
                    'run id': 'ap',
                    'stream id': row['stream id'],
                    'sentence id': row['sentence id'],
                    'timestamp': timestamp,
                    'conf': 1.0,
                    'string': row['string']
                })

        df = pd.DataFrame(updates,
            columns=["query id", "system id", "run id",
                     "stream id", "sentence id", "timestamp",
                     "conf", "string"])
        with open(tsv_path, u'w') as f:
            df.to_csv(
                f, sep='\t', index=False, index_label=False, header=False)

       #for timestamp, group in cluster_df.groupby(["timestamp"]):
       #    print timestamp

class APSalienceFilteredSummary(object):
    def __init__(self):
        self.dir_ = os.path.join(
            os.getenv(u'TREC_DATA', u'.'), u'ap-sal-filtered-summaries')
        if not os.path.exists(self.dir_):
            os.makedirs(self.dir_)

    def get_tsv_dir(self, prefix, feature_set):
        return os.path.join(self.dir_, prefix + "." + feature_set.fs_name())

    def get_tsv_path(self, event, prefix, feature_set, cthr):
        tsv_dir = self.get_tsv_dir(prefix, feature_set)
        return os.path.join(tsv_dir,
            "ap-sal-{}-thr{}.tsv".format(event.fs_name(), cthr))

    def get_dataframe(self, event, prefix, feature_set, cthr):
        tsv = self.get_tsv_path(event, prefix, feature_set, cthr)
        if not os.path.exists(tsv):
            return None
        else:
            with open(tsv, u'r') as f:
                df = pd.io.parsers.read_csv(
                    f, sep='\t', quoting=3,
                    )
                df.columns = ['query id', 'system id', 'run id', 'stream id',
                              'sentence id', 'timestamp', 'conf', 'text']
            return df

    def make(self, event, prefix, feature_set,
             min_cluster_size=2, sim_threshold=.2264, center_threshold=1.0):
        tsv_path = self.get_tsv_path(event, prefix, feature_set,
            center_threshold)
        lvecs = SentenceLatentVectorsResource()
        spa = SaliencePredictionAggregator()
        apsal = APSalienceSummarizer()
        cluster_df = apsal.get_dataframe(event, prefix, feature_set)
       #for _, row in cluster_df.iterrows():
       #    print row['hour'], datetime.utcfromtimestamp(row['timestamp'])
        updates = []
        Xcache = None
        timestamps = sorted(list(cluster_df['timestamp'].unique()))
        for timestamp in timestamps:
            hour = datetime.utcfromtimestamp(timestamp) - timedelta(hours=1)
            lvec_df = lvecs.get_dataframe(event, hour)
            lvec_df.drop_duplicates(['stream id', 'sentence id'], inplace=True)
            sal_df = spa.get_dataframe(event, hour, prefix, feature_set)
            sal_df.drop_duplicates(['stream id', 'sentence id'], inplace=True)

            clusters = cluster_df[cluster_df['timestamp'] == timestamp].copy()
            clusters.sort(['stream id', 'sentence id'], inplace=True)

            salience = []
            for _, row in clusters.iterrows():
                sal_pred = sal_df.loc[
                    (sal_df['stream id'] == row['stream id']) & \
                    (sal_df['sentence id'] == row['sentence id'])].as_matrix()[:,2:].astype(np.float64).mean()
                salience.append(sal_pred)
            clusters['salience'] = salience
            clusters.sort(['salience'], ascending=False, inplace=True)

            sal_mean = np.mean(salience)
            sal_std = np.std(salience)

            #print clusters
            for _, row in clusters.iterrows():
                if row['cluster size'] < min_cluster_size:
                    continue
                vec = lvec_df.loc[
                    (lvec_df['stream id'] == row['stream id']) & \
                    (lvec_df['sentence id'] == row['sentence id'])].as_matrix()[:,2:].astype(np.float64)

                sal_norm = (row['salience'] - sal_mean) / sal_std
                if sal_norm < center_threshold:
                    continue

                if Xcache is None:
                    Xcache = vec
                else:
                    if np.max(cosine_similarity(Xcache, vec)) >= sim_threshold:
                        continue
                    else:
                        Xcache = np.vstack((Xcache, vec))
                updates.append({
                    'query id': event.query_id[5:],
                    'system id': 'cunlp',
                    'run id': 'apsal.{}'.format(center_threshold),
                    'stream id': row['stream id'],
                    'sentence id': row['sentence id'],
                    'timestamp': timestamp,
                    'conf': row['salience'],
                    'string': row['string']
                })

        df = pd.DataFrame(updates,
            columns=["query id", "system id", "run id",
                     "stream id", "sentence id", "timestamp",
                     "conf", "string"])
        with open(tsv_path, u'w') as f:
            df.to_csv(
                f, sep='\t', index=False, index_label=False, header=False)

class HACFilteredSummary(object):
    def __init__(self):
        self.dir_ = os.path.join(
            os.getenv(u'TREC_DATA', u'.'), u'hac-filtered-summaries')
        if not os.path.exists(self.dir_):
            os.makedirs(self.dir_)

    def get_tsv_path(self, event, dist_cutoff, cthr):
        return os.path.join(self.dir_,
            "hac-{}-dist{}-cthr{}.tsv".format(
                event.fs_name(), dist_cutoff, cthr))

    def get_dataframe(self, event, dist_cutoff, cthr):
        tsv = self.get_tsv_path(event, dist_cutoff, cthr)
        if not os.path.exists(tsv):
            return None
        else:
            with open(tsv, u'r') as f:
                df = pd.io.parsers.read_csv(
                    f, sep='\t', quoting=3,
                    )
                df.columns = ['query id', 'system id', 'run id', 'stream id',
                              'sentence id', 'timestamp', 'conf', 'text']
            return df

    def make(self, event, prefix, feature_set,
             min_cluster_size=2, sim_threshold=.2264,
             dist_cutoff=1.35, center_threshold=1.0):
        tsv_path = self.get_tsv_path(event, dist_cutoff, center_threshold)
        lvecs = SentenceLatentVectorsResource()
        spa = SaliencePredictionAggregator()
        hac = HACSummarizer()
        cluster_df = hac.get_dataframe(event, dist_cutoff)
       #for _, row in cluster_df.iterrows():
       #    print row['hour'], datetime.utcfromtimestamp(row['timestamp'])
        updates = []
        Xcache = None
        timestamps = sorted(list(cluster_df['timestamp'].unique()))
        for timestamp in timestamps:
            hour = datetime.utcfromtimestamp(timestamp) - timedelta(hours=1)
            lvec_df = lvecs.get_dataframe(event, hour)
            lvec_df.drop_duplicates(['stream id', 'sentence id'], inplace=True)

            sal_df = spa.get_dataframe(event, hour, prefix, feature_set)
            sal_df.drop_duplicates(['stream id', 'sentence id'], inplace=True)

            clusters = cluster_df[cluster_df['timestamp'] == timestamp].copy()
            clusters.sort(['stream id', 'sentence id'], inplace=True)

            salience = []
            for _, row in clusters.iterrows():
                sal_pred = sal_df.loc[
                    (sal_df['stream id'] == row['stream id']) & \
                    (sal_df['sentence id'] == row['sentence id'])
                    ].as_matrix()[:,2:].astype(np.float64).mean()
                salience.append(sal_pred)
            clusters['salience'] = salience
            clusters.sort(['salience'], ascending=False, inplace=True)

            sal_mean = np.mean(salience)
            sal_std = np.std(salience)

            for _, row in clusters.iterrows():
                if row['cluster size'] < min_cluster_size:
                    continue
                vec = lvec_df.loc[
                    (lvec_df['stream id'] == row['stream id']) & \
                    (lvec_df['sentence id'] == row['sentence id'])
                    ].as_matrix()[:,2:].astype(np.float64)

                sal_norm = (row['salience'] - sal_mean) / sal_std
                if sal_norm < center_threshold:
                    continue

                if Xcache is None:
                    Xcache = vec
                else:
                    if np.max(cosine_similarity(Xcache, vec)) >= sim_threshold:
                        continue
                    else:
                        Xcache = np.vstack((Xcache, vec))
                updates.append({
                    'query id': event.query_id[5:],
                    'system id': 'cunlp',
                    'run id': 'hac-d{}-c{}'.format(
                        dist_cutoff, center_threshold),
                    'stream id': row['stream id'],
                    'sentence id': row['sentence id'],
                    'timestamp': timestamp,
                    'conf': row['salience'],
                    'string': row['string']
                })

        df = pd.DataFrame(updates,
            columns=["query id", "system id", "run id",
                     "stream id", "sentence id", "timestamp",
                     "conf", "string"])
        with open(tsv_path, u'w') as f:
            df.to_csv(
                f, sep='\t', index=False, index_label=False, header=False)

<commit_msg>HAC uses centroid not salience for update selection<commit_after>import cuttsum.pipeline.jobs as jobs
from cuttsum.sentsim import SentenceLatentVectorsResource
from cuttsum.summarizer.ap import APSummarizer, APSalienceSummarizer
from cuttsum.summarizer.baseline import HACSummarizer
import os
import pandas as pd
import numpy as np
from datetime import datetime, timedelta
from sklearn.metrics.pairwise import cosine_similarity
from cuttsum.pipeline.salience import SaliencePredictionAggregator

class APFilteredSummary(object):
    def __init__(self):
        self.dir_ = os.path.join(
            os.getenv(u'TREC_DATA', u'.'), u'ap-filtered-summaries')
        if not os.path.exists(self.dir_):
            os.makedirs(self.dir_)

    def get_tsv_path(self, event, prefix=None, feature_set=None):
        return os.path.join(self.dir_,
            "ap-{}.tsv".format(event.fs_name()))

    def get_dataframe(self, event):
        tsv = self.get_tsv_path(event)
        if not os.path.exists(tsv):
            return None
        else:
            with open(tsv, u'r') as f:
                df = pd.io.parsers.read_csv(
                    f, sep='\t', quoting=3)
                df.columns = ['query id', 'system id', 'run id', 'stream id',
                              'sentence id', 'timestamp', 'conf', 'text']
            return df

    def make(self, event, min_cluster_size=2, sim_threshold=.2264):
        tsv_path = self.get_tsv_path(event)
        lvecs = SentenceLatentVectorsResource()
        ap = APSummarizer()
        cluster_df = ap.get_dataframe(event)
       #for _, row in cluster_df.iterrows():
       #    print row['hour'], datetime.utcfromtimestamp(row['timestamp'])
        updates = []
        Xcache = None
        timestamps = sorted(list(cluster_df['timestamp'].unique()))
        for timestamp in timestamps:
            hour = datetime.utcfromtimestamp(timestamp) - timedelta(hours=1)
            lvec_df = lvecs.get_dataframe(event, hour)
            lvec_df.drop_duplicates(['stream id', 'sentence id'], inplace=True)
            clusters = cluster_df[cluster_df['timestamp'] == timestamp].copy()
            clusters.sort(['stream id', 'sentence id'], inplace=True)
            for _, row in clusters.iterrows():
                if row['cluster size'] < min_cluster_size:
                    continue
                vec = lvec_df.loc[
                    (lvec_df['stream id'] == row['stream id']) & \
                    (lvec_df['sentence id'] == row['sentence id'])].as_matrix()[:,2:].astype(np.float64)
                if Xcache is None:
                    Xcache = vec
                else:
                    if np.max(cosine_similarity(Xcache, vec)) >= sim_threshold:
                        continue
                    else:
                        Xcache = np.vstack((Xcache, vec))
                updates.append({
                    'query id': event.query_id[5:],
                    'system id': 'cunlp',
                    'run id': 'ap',
                    'stream id': row['stream id'],
                    'sentence id': row['sentence id'],
                    'timestamp': timestamp,
                    'conf': 1.0,
                    'string': row['string']
                })

        df = pd.DataFrame(updates,
            columns=["query id", "system id", "run id",
                     "stream id", "sentence id", "timestamp",
                     "conf", "string"])
        with open(tsv_path, u'w') as f:
            df.to_csv(
                f, sep='\t', index=False, index_label=False, header=False)

       #for timestamp, group in cluster_df.groupby(["timestamp"]):
       #    print timestamp

class APSalienceFilteredSummary(object):
    def __init__(self):
        self.dir_ = os.path.join(
            os.getenv(u'TREC_DATA', u'.'), u'ap-sal-filtered-summaries')
        if not os.path.exists(self.dir_):
            os.makedirs(self.dir_)

    def get_tsv_dir(self, prefix, feature_set):
        return os.path.join(self.dir_, prefix + "." + feature_set.fs_name())

    def get_tsv_path(self, event, prefix, feature_set, cthr):
        tsv_dir = self.get_tsv_dir(prefix, feature_set)
        return os.path.join(tsv_dir,
            "ap-sal-{}-thr{}.tsv".format(event.fs_name(), cthr))

    def get_dataframe(self, event, prefix, feature_set, cthr):
        tsv = self.get_tsv_path(event, prefix, feature_set, cthr)
        if not os.path.exists(tsv):
            return None
        else:
            with open(tsv, u'r') as f:
                df = pd.io.parsers.read_csv(
                    f, sep='\t', quoting=3,
                    )
                df.columns = ['query id', 'system id', 'run id', 'stream id',
                              'sentence id', 'timestamp', 'conf', 'text']
            return df

    def make(self, event, prefix, feature_set,
             min_cluster_size=2, sim_threshold=.2264, center_threshold=1.0):
        tsv_path = self.get_tsv_path(event, prefix, feature_set,
            center_threshold)
        lvecs = SentenceLatentVectorsResource()
        spa = SaliencePredictionAggregator()
        apsal = APSalienceSummarizer()
        cluster_df = apsal.get_dataframe(event, prefix, feature_set)
       #for _, row in cluster_df.iterrows():
       #    print row['hour'], datetime.utcfromtimestamp(row['timestamp'])
        updates = []
        Xcache = None
        timestamps = sorted(list(cluster_df['timestamp'].unique()))
        for timestamp in timestamps:
            hour = datetime.utcfromtimestamp(timestamp) - timedelta(hours=1)
            lvec_df = lvecs.get_dataframe(event, hour)
            lvec_df.drop_duplicates(['stream id', 'sentence id'], inplace=True)
            sal_df = spa.get_dataframe(event, hour, prefix, feature_set)
            sal_df.drop_duplicates(['stream id', 'sentence id'], inplace=True)

            clusters = cluster_df[cluster_df['timestamp'] == timestamp].copy()
            clusters.sort(['stream id', 'sentence id'], inplace=True)

            salience = []
            for _, row in clusters.iterrows():
                sal_pred = sal_df.loc[
                    (sal_df['stream id'] == row['stream id']) & \
                    (sal_df['sentence id'] == row['sentence id'])].as_matrix()[:,2:].astype(np.float64).mean()
                salience.append(sal_pred)
            clusters['salience'] = salience
            clusters.sort(['salience'], ascending=False, inplace=True)

            sal_mean = np.mean(salience)
            sal_std = np.std(salience)

            #print clusters
            for _, row in clusters.iterrows():
                if row['cluster size'] < min_cluster_size:
                    continue
                vec = lvec_df.loc[
                    (lvec_df['stream id'] == row['stream id']) & \
                    (lvec_df['sentence id'] == row['sentence id'])].as_matrix()[:,2:].astype(np.float64)

                sal_norm = (row['salience'] - sal_mean) / sal_std
                if sal_norm < center_threshold:
                    continue

                if Xcache is None:
                    Xcache = vec
                else:
                    if np.max(cosine_similarity(Xcache, vec)) >= sim_threshold:
                        continue
                    else:
                        Xcache = np.vstack((Xcache, vec))
                updates.append({
                    'query id': event.query_id[5:],
                    'system id': 'cunlp',
                    'run id': 'apsal.{}'.format(center_threshold),
                    'stream id': row['stream id'],
                    'sentence id': row['sentence id'],
                    'timestamp': timestamp,
                    'conf': row['salience'],
                    'string': row['string']
                })

        df = pd.DataFrame(updates,
            columns=["query id", "system id", "run id",
                     "stream id", "sentence id", "timestamp",
                     "conf", "string"])
        with open(tsv_path, u'w') as f:
            df.to_csv(
                f, sep='\t', index=False, index_label=False, header=False)

class HACFilteredSummary(object):
    def __init__(self):
        self.dir_ = os.path.join(
            os.getenv(u'TREC_DATA', u'.'), u'hac-filtered-summaries')
        if not os.path.exists(self.dir_):
            os.makedirs(self.dir_)

    def get_tsv_path(self, event, dist_cutoff):
        return os.path.join(self.dir_,
            "hac-{}-dist{}.tsv".format(
                event.fs_name(), dist_cutoff))

    def get_dataframe(self, event, dist_cutoff):
        tsv = self.get_tsv_path(event, dist_cutoff)
        if not os.path.exists(tsv):
            return None
        else:
            with open(tsv, u'r') as f:
                df = pd.io.parsers.read_csv(
                    f, sep='\t', quoting=3,
                    )
                df.columns = ['query id', 'system id', 'run id', 'stream id',
                              'sentence id', 'timestamp', 'conf', 'text']
            return df

    def make(self, event, prefix, feature_set,
             min_cluster_size=2, sim_threshold=.2264,
             dist_cutoff=1.35):
        tsv_path = self.get_tsv_path(event, dist_cutoff)
        lvecs = SentenceLatentVectorsResource()
        #spa = SaliencePredictionAggregator()
        hac = HACSummarizer()
        cluster_df = hac.get_dataframe(event, dist_cutoff)
       #for _, row in cluster_df.iterrows():
       #    print row['hour'], datetime.utcfromtimestamp(row['timestamp'])
        updates = []
        Xcache = None
        timestamps = sorted(list(cluster_df['timestamp'].unique()))
        for timestamp in timestamps:
            hour = datetime.utcfromtimestamp(timestamp) - timedelta(hours=1)
            lvec_df = lvecs.get_dataframe(event, hour)
            lvec_df.drop_duplicates(['stream id', 'sentence id'], inplace=True)

#            sal_df = spa.get_dataframe(event, hour, prefix, feature_set)
#            sal_df.drop_duplicates(['stream id', 'sentence id'], inplace=True)

            clusters = cluster_df[cluster_df['timestamp'] == timestamp].copy()
            clusters.sort(['stream id', 'sentence id'], inplace=True)

#            salience = []
#            for _, row in clusters.iterrows():
#                sal_pred = sal_df.loc[
#                    (sal_df['stream id'] == row['stream id']) & \
#                    (sal_df['sentence id'] == row['sentence id'])
#                    ].as_matrix()[:,2:].astype(np.float64).mean()
#                salience.append(sal_pred)
#            clusters['salience'] = salience
#            clusters.sort(['salience'], ascending=False, inplace=True)

#            sal_mean = np.mean(salience)
#            sal_std = np.std(salience)

            for _, row in clusters.iterrows():
                if row['cluster size'] < min_cluster_size:
                    continue
                vec = lvec_df.loc[
                    (lvec_df['stream id'] == row['stream id']) & \
                    (lvec_df['sentence id'] == row['sentence id'])
                    ].as_matrix()[:,2:].astype(np.float64)

                #sal_norm = (row['salience'] - sal_mean) / sal_std
                #if sal_norm < center_threshold:
                #    continue

                if Xcache is None:
                    Xcache = vec
                else:
                    if np.max(cosine_similarity(Xcache, vec)) >= sim_threshold:
                        continue
                    else:
                        Xcache = np.vstack((Xcache, vec))
                updates.append({
                    'query id': event.query_id[5:],
                    'system id': 'cunlp',
                    'run id': 'hac-d{}'.format(
                        dist_cutoff),
                    'stream id': row['stream id'],
                    'sentence id': row['sentence id'],
                    'timestamp': timestamp,
                    'conf': 1.0,
                    'string': row['string']
                })

        df = pd.DataFrame(updates,
            columns=["query id", "system id", "run id",
                     "stream id", "sentence id", "timestamp",
                     "conf", "string"])
        with open(tsv_path, u'w') as f:
            df.to_csv(
                f, sep='\t', index=False, index_label=False, header=False)

<|endoftext|>


<commit_before>#!/usr/bin/env python
# A tool to parse the FormatStyle struct from Format.h and update the
# documentation in ../ClangFormatStyleOptions.rst automatically.
# Run from the directory in which this file is located to update the docs.

import collections
import re
import urllib2

FORMAT_STYLE_FILE = '../../include/clang/Format/Format.h'
DOC_FILE = '../ClangFormatStyleOptions.rst'

def substitute(text, tag, contents):
  replacement = '\n.. START_%s\n\n%s\n\n.. END_%s\n' % (tag, contents, tag)
  pattern = r'\n\.\. START_%s\n.*\n\.\. END_%s\n' % (tag, tag)
  return re.sub(pattern, '%s', text, flags=re.S) % replacement

def doxygen2rst(text):
  text = re.sub(r'([^/\*])\*', r'\1\\*', text)
  text = re.sub(r'<tt>\s*(.*?)\s*<\/tt>', r'``\1``', text)
  text = re.sub(r'\\c ([^ ,;\.]+)', r'``\1``', text)
  text = re.sub(r'\\\w+ ', '', text)
  return text

def indent(text, columns):
  indent = ' ' * columns
  s = re.sub(r'\n([^\n])', '\n' + indent + '\\1', text, flags=re.S)
  if s.startswith('\n'):
    return s
  return indent + s

class Option:
  def __init__(self, name, type, comment):
    self.name = name
    self.type = type
    self.comment = comment.strip()
    self.enum = None
    self.nested_struct = None

  def __str__(self):
    s = '**%s** (``%s``)\n%s' % (self.name, self.type,
                                 doxygen2rst(indent(self.comment, 2)))
    if self.enum:
      s += indent('\n\nPossible values:\n\n%s\n' % self.enum, 2)
    if self.nested_struct:
      s += indent('\n\nNested configuration flags:\n\n%s\n' %self.nested_struct,
                  2)
    return s

class NestedStruct:
  def __init__(self, name, comment):
    self.name = name
    self.comment = comment.strip()
    self.values = []

  def __str__(self):
    return '\n'.join(map(str, self.values))

class NestedField:
  def __init__(self, name, comment):
    self.name = name
    self.comment = comment.strip()

  def __str__(self):
    return '* ``%s`` %s' % (self.name, doxygen2rst(self.comment))

class Enum:
  def __init__(self, name, comment):
    self.name = name
    self.comment = comment.strip()
    self.values = []

  def __str__(self):
    return '\n'.join(map(str, self.values))

class EnumValue:
  def __init__(self, name, comment):
    self.name = name
    self.comment = comment.strip()

  def __str__(self):
    return '* ``%s`` (in configuration: ``%s``)\n%s' % (
        self.name,
        re.sub('.*_', '', self.name),
        doxygen2rst(indent(self.comment, 2)))

def clean_comment_line(line):
  if line == '/// \\code':
    return '\n.. code-block:: c++\n\n'
  if line == '/// \\endcode':
    return ''
  return line[4:] + '\n'

def read_options(header):
  class State:
    BeforeStruct, Finished, InStruct, InNestedStruct, InNestedFieldComent, \
    InFieldComment, InEnum, InEnumMemberComment = range(8)
  state = State.BeforeStruct

  options = []
  enums = {}
  nested_structs = {}
  comment = ''
  enum = None
  nested_struct = None

  for line in header:
    line = line.strip()
    if state == State.BeforeStruct:
      if line == 'struct FormatStyle {':
        state = State.InStruct
    elif state == State.InStruct:
      if line.startswith('///'):
        state = State.InFieldComment
        comment = clean_comment_line(line)
      elif line == '};':
        state = State.Finished
        break
    elif state == State.InFieldComment:
      if line.startswith('///'):
        comment += clean_comment_line(line)
      elif line.startswith('enum'):
        state = State.InEnum
        name = re.sub(r'enum\s+(\w+)\s*\{', '\\1', line)
        enum = Enum(name, comment)
      elif line.startswith('struct'):
        state = State.InNestedStruct
        name = re.sub(r'struct\s+(\w+)\s*\{', '\\1', line)
        nested_struct = NestedStruct(name, comment)
      elif line.endswith(';'):
        state = State.InStruct
        field_type, field_name = re.match(r'([<>:\w(,\s)]+)\s+(\w+);',
                                          line).groups()
        option = Option(str(field_name), str(field_type), comment)
        options.append(option)
      else:
        raise Exception('Invalid format, expected comment, field or enum')
    elif state == State.InNestedStruct:
      if line.startswith('///'):
        state = State.InNestedFieldComent
        comment = clean_comment_line(line)
      elif line == '};':
        state = State.InStruct
        nested_structs[nested_struct.name] = nested_struct
    elif state == State.InNestedFieldComent:
      if line.startswith('///'):
        comment += clean_comment_line(line)
      else:
        state = State.InNestedStruct
        nested_struct.values.append(NestedField(line.replace(';', ''), comment))
    elif state == State.InEnum:
      if line.startswith('///'):
        state = State.InEnumMemberComment
        comment = clean_comment_line(line)
      elif line == '};':
        state = State.InStruct
        enums[enum.name] = enum
      else:
        raise Exception('Invalid format, expected enum field comment or };')
    elif state == State.InEnumMemberComment:
      if line.startswith('///'):
        comment += clean_comment_line(line)
      else:
        state = State.InEnum
        enum.values.append(EnumValue(line.replace(',', ''), comment))
  if state != State.Finished:
    raise Exception('Not finished by the end of file')

  for option in options:
    if not option.type in ['bool', 'unsigned', 'int', 'std::string',
                           'std::vector<std::string>',
                           'std::vector<IncludeCategory>']:
      if enums.has_key(option.type):
        option.enum = enums[option.type]
      elif nested_structs.has_key(option.type):
        option.nested_struct = nested_structs[option.type];
      else:
        raise Exception('Unknown type: %s' % option.type)
  return options

options = read_options(open(FORMAT_STYLE_FILE))

options = sorted(options, key=lambda x: x.name)
options_text = '\n\n'.join(map(str, options))

contents = open(DOC_FILE).read()

contents = substitute(contents, 'FORMAT_STYLE_OPTIONS', options_text)

with open(DOC_FILE, 'wb') as output:
  output.write(contents)

<commit_msg>Allow running dump_format_style.py from any directory.<commit_after>#!/usr/bin/env python
# A tool to parse the FormatStyle struct from Format.h and update the
# documentation in ../ClangFormatStyleOptions.rst automatically.
# Run from the directory in which this file is located to update the docs.

import collections
import os
import re
import urllib2

CLANG_DIR = os.path.join(os.path.dirname(__file__), '../..')
FORMAT_STYLE_FILE = os.path.join(CLANG_DIR, 'include/clang/Format/Format.h')
DOC_FILE = os.path.join(CLANG_DIR, 'docs/ClangFormatStyleOptions.rst')

def substitute(text, tag, contents):
  replacement = '\n.. START_%s\n\n%s\n\n.. END_%s\n' % (tag, contents, tag)
  pattern = r'\n\.\. START_%s\n.*\n\.\. END_%s\n' % (tag, tag)
  return re.sub(pattern, '%s', text, flags=re.S) % replacement

def doxygen2rst(text):
  text = re.sub(r'([^/\*])\*', r'\1\\*', text)
  text = re.sub(r'<tt>\s*(.*?)\s*<\/tt>', r'``\1``', text)
  text = re.sub(r'\\c ([^ ,;\.]+)', r'``\1``', text)
  text = re.sub(r'\\\w+ ', '', text)
  return text

def indent(text, columns):
  indent = ' ' * columns
  s = re.sub(r'\n([^\n])', '\n' + indent + '\\1', text, flags=re.S)
  if s.startswith('\n'):
    return s
  return indent + s

class Option:
  def __init__(self, name, type, comment):
    self.name = name
    self.type = type
    self.comment = comment.strip()
    self.enum = None
    self.nested_struct = None

  def __str__(self):
    s = '**%s** (``%s``)\n%s' % (self.name, self.type,
                                 doxygen2rst(indent(self.comment, 2)))
    if self.enum:
      s += indent('\n\nPossible values:\n\n%s\n' % self.enum, 2)
    if self.nested_struct:
      s += indent('\n\nNested configuration flags:\n\n%s\n' %self.nested_struct,
                  2)
    return s

class NestedStruct:
  def __init__(self, name, comment):
    self.name = name
    self.comment = comment.strip()
    self.values = []

  def __str__(self):
    return '\n'.join(map(str, self.values))

class NestedField:
  def __init__(self, name, comment):
    self.name = name
    self.comment = comment.strip()

  def __str__(self):
    return '* ``%s`` %s' % (self.name, doxygen2rst(self.comment))

class Enum:
  def __init__(self, name, comment):
    self.name = name
    self.comment = comment.strip()
    self.values = []

  def __str__(self):
    return '\n'.join(map(str, self.values))

class EnumValue:
  def __init__(self, name, comment):
    self.name = name
    self.comment = comment.strip()

  def __str__(self):
    return '* ``%s`` (in configuration: ``%s``)\n%s' % (
        self.name,
        re.sub('.*_', '', self.name),
        doxygen2rst(indent(self.comment, 2)))

def clean_comment_line(line):
  if line == '/// \\code':
    return '\n.. code-block:: c++\n\n'
  if line == '/// \\endcode':
    return ''
  return line[4:] + '\n'

def read_options(header):
  class State:
    BeforeStruct, Finished, InStruct, InNestedStruct, InNestedFieldComent, \
    InFieldComment, InEnum, InEnumMemberComment = range(8)
  state = State.BeforeStruct

  options = []
  enums = {}
  nested_structs = {}
  comment = ''
  enum = None
  nested_struct = None

  for line in header:
    line = line.strip()
    if state == State.BeforeStruct:
      if line == 'struct FormatStyle {':
        state = State.InStruct
    elif state == State.InStruct:
      if line.startswith('///'):
        state = State.InFieldComment
        comment = clean_comment_line(line)
      elif line == '};':
        state = State.Finished
        break
    elif state == State.InFieldComment:
      if line.startswith('///'):
        comment += clean_comment_line(line)
      elif line.startswith('enum'):
        state = State.InEnum
        name = re.sub(r'enum\s+(\w+)\s*\{', '\\1', line)
        enum = Enum(name, comment)
      elif line.startswith('struct'):
        state = State.InNestedStruct
        name = re.sub(r'struct\s+(\w+)\s*\{', '\\1', line)
        nested_struct = NestedStruct(name, comment)
      elif line.endswith(';'):
        state = State.InStruct
        field_type, field_name = re.match(r'([<>:\w(,\s)]+)\s+(\w+);',
                                          line).groups()
        option = Option(str(field_name), str(field_type), comment)
        options.append(option)
      else:
        raise Exception('Invalid format, expected comment, field or enum')
    elif state == State.InNestedStruct:
      if line.startswith('///'):
        state = State.InNestedFieldComent
        comment = clean_comment_line(line)
      elif line == '};':
        state = State.InStruct
        nested_structs[nested_struct.name] = nested_struct
    elif state == State.InNestedFieldComent:
      if line.startswith('///'):
        comment += clean_comment_line(line)
      else:
        state = State.InNestedStruct
        nested_struct.values.append(NestedField(line.replace(';', ''), comment))
    elif state == State.InEnum:
      if line.startswith('///'):
        state = State.InEnumMemberComment
        comment = clean_comment_line(line)
      elif line == '};':
        state = State.InStruct
        enums[enum.name] = enum
      else:
        raise Exception('Invalid format, expected enum field comment or };')
    elif state == State.InEnumMemberComment:
      if line.startswith('///'):
        comment += clean_comment_line(line)
      else:
        state = State.InEnum
        enum.values.append(EnumValue(line.replace(',', ''), comment))
  if state != State.Finished:
    raise Exception('Not finished by the end of file')

  for option in options:
    if not option.type in ['bool', 'unsigned', 'int', 'std::string',
                           'std::vector<std::string>',
                           'std::vector<IncludeCategory>']:
      if enums.has_key(option.type):
        option.enum = enums[option.type]
      elif nested_structs.has_key(option.type):
        option.nested_struct = nested_structs[option.type];
      else:
        raise Exception('Unknown type: %s' % option.type)
  return options

options = read_options(open(FORMAT_STYLE_FILE))

options = sorted(options, key=lambda x: x.name)
options_text = '\n\n'.join(map(str, options))

contents = open(DOC_FILE).read()

contents = substitute(contents, 'FORMAT_STYLE_OPTIONS', options_text)

with open(DOC_FILE, 'wb') as output:
  output.write(contents)

<|endoftext|>


<commit_before>import os
import spacy

LANG = 'de'
BATCH_SIZE = 10000
N_THREADS = 4

class Cleaner:
    def __init__(self, path):
        self.path = path
        self._nlp = None

    def __iter__(self):
        """Return iterator of cleaned sentences.
        """
        raise NotImplementedError

    def write(self, write_path, delete=False):
        """For a path 'pat/to/corpus' write:
            path/to/corpus/
                corpus.txt
                corpus.tokens.txt
        where 'corpus.txt' is one sent per line, spaces left in place,
        and 'corpus.tokens.txt' is one sent per line, token separated by space.
        """
        folder, file, tokens = self._write_paths(write_path)
        # Create corpus folder
        os.makedirs(folder, exist_ok=True)
        # Write sentence file
        with open(file, 'w') as f:
            for line in self:
                f.write('%s\n' % line)
        # Write tokens file
        with open(tokens, 'w') as f:
            for doc in self.nlp.tokenizer.pipe(self, batch_size=BATCH_SIZE, n_threads=N_THREADS):
                token_line = ' '.join(str(token) for token in doc)
                f.write('%s\n' % token_line)

        # Remove uncleaned data
        if delete:
            if os.path.isfile(self.path):
                os.remove(self.path)
            elif os.path.isdir(self.path):
                os.rmdir(self.path)

    @property
    def nlp(self):
        if not self._nlp:
            self._nlp = spacy.load(LANG)
        return self._nlp

    @staticmethod
    def _write_paths(path):
        """Return write paths for folder, file, and tokens.
        Example for input 'abc/def.txt':
            'abc/def', 'abc/def/def.txt', 'abc/def/def.tokens.txt'
        """
        # Example: abc/def.txt
        name = os.path.splitext(os.path.basename(path))[0]  # def
        folder = os.path.splitext(path)[0]  # abc/def/
        file = '%s.txt' % name  # abc/def.txt
        tokens = '%s.tokens.txt' % name  # abc/def.tokens.txt
        return folder, os.path.join(folder, file), os.path.join(folder, tokens)
<commit_msg>Refactoring<commit_after>import os
import spacy

LANG = 'de'
BATCH_SIZE = 10000
N_THREADS = 4

class Cleaner:
    def __init__(self, path):
        self.path = path
        self._nlp = None

    def __iter__(self):
        """Return iterator of cleaned sentences.
        """
        raise NotImplementedError

    def write(self, write_path, delete=False):
        """For a path 'pat/to/corpus' write:
            path/to/corpus/
                corpus.txt
                corpus.tokens.txt
        where 'corpus.txt' is one sent per line, spaces left in place,
        and 'corpus.tokens.txt' is one sent per line, token separated by space.
        """
        folder, file, tokens = self._write_paths(write_path)
        # Create corpus folder
        os.makedirs(folder, exist_ok=True)

        # Write sentence file
        with open(file, 'w') as f:
            for line in self:
                f.write('%s\n' % line)

        # Write tokens file
        with open(tokens, 'w') as f:
            for doc in self.tokenized_sents():
                token_line = ' '.join(str(token) for token in doc)
                f.write('%s\n' % token_line)

        # Remove uncleaned data
        if delete:
            if os.path.isfile(self.path):
                os.remove(self.path)
            elif os.path.isdir(self.path):
                os.rmdir(self.path)

    def tokenized_sents(self):
        """Return an iterator with SpaCy docs.
        The docs' tokens can be iterated:
        Example:
            [token
             for doc in self.tokenized_sents()
             for token in doc]
        """
        return self.nlp.tokenizer.pipe(self,
                                       batch_size=BATCH_SIZE,
                                       n_threads=N_THREADS)

    @property
    def nlp(self):
        if not self._nlp:
            self._nlp = spacy.load(LANG)
        return self._nlp

    @staticmethod
    def _write_paths(path):
        """Return write paths for folder, file, and tokens.
        Example for input 'abc/def.txt':
            'abc/def', 'abc/def/def.txt', 'abc/def/def.tokens.txt'
        """
        # Example: abc/def.txt
        name = os.path.splitext(os.path.basename(path))[0]  # def
        folder = os.path.splitext(path)[0]  # abc/def/
        file = '%s.txt' % name  # abc/def.txt
        tokens = '%s.tokens.txt' % name  # abc/def.tokens.txt
        return folder, os.path.join(folder, file), os.path.join(folder, tokens)
<|endoftext|>


<commit_before>#!/usr/local/bin/python2.7
# encoding: utf-8
'''
gptwosample.__main__ -- GPTwoSample

gptwosample.__main__ is a package to perform GPTwoSample on given treatment and control experiments

It defines classes_and_methods

@author:     user_name

@copyright:  2013 organization_name. All rights reserved.

@license:    license

@contact:    user_email
@deffield    updated: Updated
'''

import sys
import os

from argparse import ArgumentParser
from argparse import RawDescriptionHelpFormatter

__all__ = []
__version__ = 0.1
__date__ = '2013-04-03'
__updated__ = '2013-04-03'

DEBUG = 1
TESTRUN = 0
PROFILE = 0

class CLIError(Exception):
    '''Generic exception to raise and log different fatal errors.'''
    def __init__(self, msg):
        super(CLIError).__init__(type(self))
        self.msg = "E: %s" % msg
    def __str__(self):
        return self.msg
    def __unicode__(self):
        return self.msg

def main(argv=None): # IGNORE:C0111
    '''Command line options.'''

    if argv is None:
        argv = sys.argv
    else:
        sys.argv.extend(argv)

    program_name = os.path.basename(sys.argv[0])
    program_version = "v%s" % __version__
    program_build_date = str(__updated__)
    program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date)
    program_shortdesc = __import__('__main__').__doc__.split("\n")[1]
    program_license = '''%s

  Created by user_name on %s.
  Copyright 2013 organization_name. All rights reserved.

  Licensed under the Apache License 2.0
  http://www.apache.org/licenses/LICENSE-2.0

  Distributed on an "AS IS" basis without warranties
  or conditions of any kind, either express or implied.

USAGE
''' % (program_shortdesc, str(__date__))

    try:
        # Setup argument parser
        parser = ArgumentParser(description=program_license, formatter_class=RawDescriptionHelpFormatter)
        parser.add_argument("-v", "--verbose", dest="verbose", action="count", help="set verbosity level [default: %(default)s]")
        parser.add_argument('-V', '--version', action='version', version=program_version_message)
        parser.add_argument(dest="paths", help="paths to folder(s) with source file(s) [default: %(default)s]", metavar="path", nargs='+')

        # Process arguments
        args = parser.parse_args()

        paths = args.paths
        verbose = args.verbose

        if verbose > 0:
            print("Verbose mode on")

        for inpath in paths:
            ### do something with inpath ###
            print(inpath)
        return 0
    except KeyboardInterrupt:
        ### handle keyboard interrupt ###
        return 0
    except Exception, e:
        if DEBUG or TESTRUN:
            raise(e)
        indent = len(program_name) * " "
        sys.stderr.write(program_name + ": " + repr(e) + "\n")
        sys.stderr.write(indent + "  for help use --help")
        return 2

if __name__ == "__main__":
    if DEBUG:
        sys.argv.append("-h")
        sys.argv.append("-v")
        sys.argv.append("-r")
    if TESTRUN:
        import doctest
        doctest.testmod()
    if PROFILE:
        import cProfile
        import pstats
        profile_filename = 'gptwosample.__main___profile.txt'
        cProfile.run('main()', profile_filename)
        statsfile = open("profile_stats.txt", "wb")
        p = pstats.Stats(profile_filename, stream=statsfile)
        stats = p.strip_dirs().sort_stats('cumulative')
        stats.print_stats()
        statsfile.close()
        sys.exit(0)
    sys.exit(main())<commit_msg>false main<commit_after><|endoftext|>


<commit_before>from collections import namedtuple
import sys
import traceback

from logconfig import newLogger

log = newLogger(__name__)

# Normal delimiter; separates tokens in messages.
TOKEN_DELIM  = " "

# Used to indicate the start of an unsafe string; must be the first character
# after TOKEN_DELIM.
START_STRING = "|"

# Mapping from command word to Message (sub)classes.
messagesByCommand = {}

def defineMessageType(commandWord, argNamesAndSpecs):
    """
    Define a new message type.

    argNamesAndSpecs should be a list of tuples (name, spec), where name is the
    name of that argument and spec is an ArgumentSpecification object
    describing how it is encoded and decoded when the message is serialized and
    deserialized.
    """

    if commandWord in messagesByCommand:
        raise ValueError("Message command {0!r} is already taken."
                         .format(commandWord))
    assert not any(nameSpec[1].unsafe for nameSpec in argNamesAndSpecs[:-1])

    # TODO: snake_case to BigCamelCase?
    # Ideally we'd choose this name so that it matches the actual message class
    # name. Or just override __str__ in Message.
    NamedTupleType = namedtuple(commandWord + "_message_tuple",
                                [nameSpec[0] for nameSpec in argNamesAndSpecs])

    # Subclass from Message before NamedTupleType, so that we can override some
    # methods of NamedTupleType in Message. (We may want to do this with
    # __str__?)
    class NewMessageType(Message, NamedTupleType):
        command  = commandWord
        argSpecs = [nameSpec[1] for nameSpec in argNamesAndSpecs]

        def __init__(self, *args):
            super(NewMessageType, self).__init__(*args)

    messagesByCommand[commandWord] = NewMessageType
    return NewMessageType

def serializeMessage(message):
    argStrings = []
    assert len(message.argSpecs) == len(message)
    for argSpec, arg in zip(message.argSpecs, message):
        argWords = argSpec.encode(arg)
        if argSpec.count == 1:
            argWords = (argWords,)
        argStrings.extend(argWords)
    lastIsUnsafe = message.argSpecs[-1].unsafe if message.argSpecs else False
    return buildMessage(message.command, argStrings, lastIsUnsafe=lastIsUnsafe)

# Note: errorOnFail might never be passed as False; currently all callers that
# don't want to crash still pass errorOnFail=True and just handle
# InvalidMessageError themselves.
def deserializeMessage(data, errorOnFail=True):
    try:
        cmd, argStrings = tokenize(data)
        if cmd not in messagesByCommand:
            raise InvalidMessageError(data, "Unrecognized message command.")

        messageType = messagesByCommand[cmd]

        args = []
        for argSpec in messageType.argSpecs:
            if argSpec.count > len(argStrings):
                raise InvalidMessageError(data,
                                          "Not enough arguments for command.")
            if argSpec.count == 1:
                args.append(argSpec.decode(argStrings[0]))
            else:
                currWords = tuple(argStrings[:argSpec.count])
                args.append(argSpec.decode(currWords))
            argStrings = argStrings[argSpec.count:]

        if len(argStrings) > 0:
            raise InvalidMessageError(data, "Too many arguments for command.")

        return messageType(*args)
    except StandardError, exc:
        # Log the full traceback, noting where we are, much like what Twisted
        # does for an uncaught exception.
        stackBelow = traceback.format_exception(sys.exc_type, sys.exc_value,
                                                sys.exc_traceback)
        stackAbove = traceback.format_stack()
        # Remove the last stack entry from stackAbove, because that's the call
        # to format_stack() which isn't part of the exception's traceback.
        stackAbove = stackAbove[:-1]
        # Remove the first entry from stackBelow, because that's the "Traceback
        # (most recent call last)" line which we want at the top of the
        # traceback, rather than the middle.
        headerLine = stackBelow[0]
        stackBelow = stackBelow[1:]
        # Add an extra entry in the middle noting that this is where we caught
        # the exception.
        midLine = "--- <exception caught here> ---\n"
        fullStack = [headerLine] + stackAbove + [midLine] + stackBelow
        log.debug("Caught exception in deserializeMessage.\n" +
                  "".join(fullStack))

        if errorOnFail:
            # Reraise the exception, but converted (if necessary) to an
            # InvalidMessageError. This ensures that it'll be handled correctly
            # by the caller and not cause the program to crash unnecessarily.
            if isinstance(exc, InvalidMessageError):
                raise exc
            else:
                raise InvalidMessageError(data, str(exc))
        else:
            return None

class Message(object):
    command  = None
    argSpecs = None

    # Note: this doesn't seem to be necessary, but that might just be because
    # (I think) namedtuple overrides __new__ instead of __init__.
    # def __init__(self, *args):
    #     super(Message, self).__init__(*args)

    # For a message that was just deserialized, maybe we should cache the
    # original string and return it, rather than regenerating it?
    def serialize(self):
        return serializeMessage(self)

    def __str__(self):
        return self.serialize()

class ArgumentSpecification(object):
    """
    Class used to describe how one logical argument to a message is encoded and
    decoded. Note that a single logical argument might be encoded as several
    (space-separated) words in the actual message string -- for example, a
    position is encoded as two words, one for each coordinate.
    """

    def __init__(self, numWords, decodeFunc, encodeFunc=None, unsafe=False):
        """
        Initialize an ArgumentSpecification.
          - numWords is the number of words used to encode this argument in a
            message string.
          - decodeFunc is a function to parse those words into a more useful
            object. It takes in a tuple of strings if numWords > 1, else a
            single string, and returns a parsed object.
          - encodeFunc is similar, but operates in reverse. It returns a tuple
            of strings if numWords > 1, else a single string. If numWords == 1,
            then encodeFunc may be omitted, in which case the argument will
            just be str()ed.
          - unsafe indicates whether the last word of this argument is an
            unsafe string.
        """

        self.count      = numWords
        self.decodeFunc = decodeFunc

        if encodeFunc is None:
            assert numWords == 1
            # TODO: Should we use repr instead?
            self.encodeFunc = str
        else:
            self.encodeFunc = encodeFunc

        self.unsafe = unsafe

    def encode(self, arg):
        """
        Encode an object corresponding to this argument as one or more words.
        Returns either a single string or a tuple of strings, the same as
        the encodeFunct passed to __init__.
        """

        words = self.encodeFunc(arg)
        if self.count == 1:
            assert isinstance(words, str)
        else:
            # Alow encodeFunc to give a list instead of a tuple, because that's
            # close enough.
            assert isinstance(words, (tuple, list))
            assert len(words) == self.count
        return words

    def decode(self, words):
        """
        Parse one or more words into the appropriate type of object. The type
        of 'words' is the same as would be passed to the decodeFunc passed to
        __init__.
        """

        if self.count == 1:
            assert isinstance(words, str)
        else:
            # Since words always comes from deserializeMessage, require that
            # it be a tuple, not a list. There's no real problem with it being
            # a list, but we know that that shouldn't happen, so if it does,
            # then it suggests a bug.
            assert isinstance(words, tuple)
            assert len(words) == self.count
        return self.decodeFunc(words)

# For an arbitrary string command and an arbitrary list of strings args,
#     tokenize(buildMessage(command, args, lastIsUnsafe=<whatever>))
# should either return exactly (command, args) or raise an InvalidMessageError.
def tokenize(message):
    if not message:
        raise InvalidMessageError(message, "Empty message.")
    if message[0] == START_STRING:
        raise InvalidMessageError(message,
                                  "Message starts with unsafe string.")

    index = message.find(TOKEN_DELIM + START_STRING)
    if index == -1:
        tokens = message.split(TOKEN_DELIM)
    else:
        tokens = message[:index].split(TOKEN_DELIM) + [message[index + 2:]]

    if not all(tokens):
        log.warning("Empty token in {}.".format(tokens))

    return (tokens[0], tokens[1:])

# TODO: We can and should unit-test buildMessage.
def buildMessage(command, args, lastIsUnsafe=False):
    """
    Build a message from the given command and arguments. The arguments don't
    have to be strings; if they aren't, then they will be str()ed.
    If lastIsUnsafe, then the last argument is a potentially unsafe string (for
    example, something typed by the user) and will be specially delimited.
    Only the last argument is allowed to be unsafe.
    """

    command = str(command)
    args    = map(str, args)
    lastArg = None

    if lastIsUnsafe:
        if not args:
            raise InvalidMessageError(command, "No arguments.")
        lastArg = args[-1]
        args = args[:-1]

    # Build the message. This has to come before the checking so that we have a
    # message to pass to any InvalidMessageErrors that we raise. Note that if
    # we raise an InvalidMessageError, the message we report with it isn't
    # actually a real message, because in that situation we've failed to build
    # a message. But it doesn't seem worth it to create yet another Exception
    # subclass, and InvalidMessageError seems more or less logically correct
    # for that type of error.
    message = TOKEN_DELIM.join([command] + args)
    if lastIsUnsafe:
        message += TOKEN_DELIM + START_STRING + lastArg

    # Check that (with the exception of the possible unsafe string at the end),
    # the message tokenizes correctly.

    def checkToken(token, tokenDesc):
        if TOKEN_DELIM in token:
            raise InvalidMessageError(message, "{0} may not contain {1!r}"
                                               .format(tokenDesc, TOKEN_DELIM))
        # TODO [#45]: Validate this.
        if command.startswith(START_STRING):
            raise InvalidMessageError(message, "{0} may not start with {0!r}"
                                               .format(tokenDesc,
                                                       START_STRING))

    checkToken(command, "Command")
    for arg in args:
        checkToken(arg, "Argument")

    # Checks passed; this is a valid message.
    return message

# Functions used to report bad messages.

def illFormedEMessage(error, log, clientId=None):
    """
    Log a warning for when a message string originating from an external source
    could not be parsed into a message, for example because it used a
    nonexistent command or passed the wrong number of arguments.
    """

    # TODO: This logic could probably be factored out I guess.
    sender = "external source"
    if clientId is not None:
        sender = "client {}".format(clientId)

    log.warning("Received invalid message from {sender}: {error}"
                .format(sender=sender, error=error))

def badEMessageCommand(message, log, clientId=None):
    """
    Log a warning for a message originating from an external source, where the
    message is well-formed (valid command with the right number of arguments)
    but it was received by a part of the code that doesn't know how to handle
    that type of message.
    """

    sender = "external source"
    if clientId is not None:
        sender = "client {}".format(clientId)

    log.warning("Could not handle message type from {sender}: {command}"
                .format(sender=sender, command=message.command))

def badEMessageArgument(message, log, clientId=None, reason=""):
    """
    Log a warning for a message originating from an external source, where the
    message is well-formed and was received by a part of the code that knows
    how to handle that type of message, but one of the arguments to the message
    is invalid (for example, out of range).
    """

    sender = "external source"
    if clientId is not None:
        sender = "client {}".format(clientId)
    if reason:
        reason = "\n    " + reason

    log.warning("Invalid argument to message from {sender}: {message}{reason}"
                .format(sender=sender, message=message, reason=reason))

# In this case you should just let error propagate up, rather than catching it
# at all.
#
# def illFormedIMessage(error, log, clientId=None):
#     """
#     Give an error for when a message string originating from an internal source
#     could not be parsed into a message, for example because it used a
#     nonexistent command or passed the wrong number of arguments.
#     """
#
#     log.error("Received invalid message: {error}"
#               .format(sender=sender, error=error))
#     raise error

def badIMessageCommand(message, log):
    """
    Give an error for when a message originating from an internal source is
    received by a part of the code that doesn't know how to handle that type of
    message.
    """

    error = "Could not handle message type from internal source: {command}." \
        .format(command=message.command)
    log.error(error)

    raise InvalidMessageError(message.serialize(), error)

def badIMessageArgument(message, log, reason=""):
    """
    Give an error for when a message originating from an internal source is
    received by a part of the code that knows how to handle that type of
    message, but one of the arguments to the message is invalid (for example,
    out of range).
    """

    if reason:
        reason = "\n    " + reason

    log.warning("Invalid argument in internal message: {message}{reason}"
                .format(message=message, reason=reason))

class InvalidMessageError(StandardError):
    def __init__(self, badMessage, errorDesc):
        super(InvalidMessageError, self).__init__()
        self.badMessage = badMessage
        self.errorDesc  = errorDesc

    def __str__(self):
        return "{desc}  (Message is: {msg!r})".format(desc = self.errorDesc,
                                                      msg  = self.badMessage)

<commit_msg>Print right string in error.<commit_after>from collections import namedtuple
import sys
import traceback

from logconfig import newLogger

log = newLogger(__name__)

# Normal delimiter; separates tokens in messages.
TOKEN_DELIM  = " "

# Used to indicate the start of an unsafe string; must be the first character
# after TOKEN_DELIM.
START_STRING = "|"

# Mapping from command word to Message (sub)classes.
messagesByCommand = {}

def defineMessageType(commandWord, argNamesAndSpecs):
    """
    Define a new message type.

    argNamesAndSpecs should be a list of tuples (name, spec), where name is the
    name of that argument and spec is an ArgumentSpecification object
    describing how it is encoded and decoded when the message is serialized and
    deserialized.
    """

    if commandWord in messagesByCommand:
        raise ValueError("Message command {0!r} is already taken."
                         .format(commandWord))
    assert not any(nameSpec[1].unsafe for nameSpec in argNamesAndSpecs[:-1])

    # TODO: snake_case to BigCamelCase?
    # Ideally we'd choose this name so that it matches the actual message class
    # name. Or just override __str__ in Message.
    NamedTupleType = namedtuple(commandWord + "_message_tuple",
                                [nameSpec[0] for nameSpec in argNamesAndSpecs])

    # Subclass from Message before NamedTupleType, so that we can override some
    # methods of NamedTupleType in Message. (We may want to do this with
    # __str__?)
    class NewMessageType(Message, NamedTupleType):
        command  = commandWord
        argSpecs = [nameSpec[1] for nameSpec in argNamesAndSpecs]

        def __init__(self, *args):
            super(NewMessageType, self).__init__(*args)

    messagesByCommand[commandWord] = NewMessageType
    return NewMessageType

def serializeMessage(message):
    argStrings = []
    assert len(message.argSpecs) == len(message)
    for argSpec, arg in zip(message.argSpecs, message):
        argWords = argSpec.encode(arg)
        if argSpec.count == 1:
            argWords = (argWords,)
        argStrings.extend(argWords)
    lastIsUnsafe = message.argSpecs[-1].unsafe if message.argSpecs else False
    return buildMessage(message.command, argStrings, lastIsUnsafe=lastIsUnsafe)

# Note: errorOnFail might never be passed as False; currently all callers that
# don't want to crash still pass errorOnFail=True and just handle
# InvalidMessageError themselves.
def deserializeMessage(data, errorOnFail=True):
    try:
        cmd, argStrings = tokenize(data)
        if cmd not in messagesByCommand:
            raise InvalidMessageError(data, "Unrecognized message command.")

        messageType = messagesByCommand[cmd]

        args = []
        for argSpec in messageType.argSpecs:
            if argSpec.count > len(argStrings):
                raise InvalidMessageError(data,
                                          "Not enough arguments for command.")
            if argSpec.count == 1:
                args.append(argSpec.decode(argStrings[0]))
            else:
                currWords = tuple(argStrings[:argSpec.count])
                args.append(argSpec.decode(currWords))
            argStrings = argStrings[argSpec.count:]

        if len(argStrings) > 0:
            raise InvalidMessageError(data, "Too many arguments for command.")

        return messageType(*args)
    except StandardError, exc:
        # Log the full traceback, noting where we are, much like what Twisted
        # does for an uncaught exception.
        stackBelow = traceback.format_exception(sys.exc_type, sys.exc_value,
                                                sys.exc_traceback)
        stackAbove = traceback.format_stack()
        # Remove the last stack entry from stackAbove, because that's the call
        # to format_stack() which isn't part of the exception's traceback.
        stackAbove = stackAbove[:-1]
        # Remove the first entry from stackBelow, because that's the "Traceback
        # (most recent call last)" line which we want at the top of the
        # traceback, rather than the middle.
        headerLine = stackBelow[0]
        stackBelow = stackBelow[1:]
        # Add an extra entry in the middle noting that this is where we caught
        # the exception.
        midLine = "--- <exception caught here> ---\n"
        fullStack = [headerLine] + stackAbove + [midLine] + stackBelow
        log.debug("Caught exception in deserializeMessage.\n" +
                  "".join(fullStack))

        if errorOnFail:
            # Reraise the exception, but converted (if necessary) to an
            # InvalidMessageError. This ensures that it'll be handled correctly
            # by the caller and not cause the program to crash unnecessarily.
            if isinstance(exc, InvalidMessageError):
                raise exc
            else:
                raise InvalidMessageError(data, str(exc))
        else:
            return None

class Message(object):
    command  = None
    argSpecs = None

    # Note: this doesn't seem to be necessary, but that might just be because
    # (I think) namedtuple overrides __new__ instead of __init__.
    # def __init__(self, *args):
    #     super(Message, self).__init__(*args)

    # For a message that was just deserialized, maybe we should cache the
    # original string and return it, rather than regenerating it?
    def serialize(self):
        return serializeMessage(self)

    def __str__(self):
        return self.serialize()

class ArgumentSpecification(object):
    """
    Class used to describe how one logical argument to a message is encoded and
    decoded. Note that a single logical argument might be encoded as several
    (space-separated) words in the actual message string -- for example, a
    position is encoded as two words, one for each coordinate.
    """

    def __init__(self, numWords, decodeFunc, encodeFunc=None, unsafe=False):
        """
        Initialize an ArgumentSpecification.
          - numWords is the number of words used to encode this argument in a
            message string.
          - decodeFunc is a function to parse those words into a more useful
            object. It takes in a tuple of strings if numWords > 1, else a
            single string, and returns a parsed object.
          - encodeFunc is similar, but operates in reverse. It returns a tuple
            of strings if numWords > 1, else a single string. If numWords == 1,
            then encodeFunc may be omitted, in which case the argument will
            just be str()ed.
          - unsafe indicates whether the last word of this argument is an
            unsafe string.
        """

        self.count      = numWords
        self.decodeFunc = decodeFunc

        if encodeFunc is None:
            assert numWords == 1
            # TODO: Should we use repr instead?
            self.encodeFunc = str
        else:
            self.encodeFunc = encodeFunc

        self.unsafe = unsafe

    def encode(self, arg):
        """
        Encode an object corresponding to this argument as one or more words.
        Returns either a single string or a tuple of strings, the same as
        the encodeFunct passed to __init__.
        """

        words = self.encodeFunc(arg)
        if self.count == 1:
            assert isinstance(words, str)
        else:
            # Alow encodeFunc to give a list instead of a tuple, because that's
            # close enough.
            assert isinstance(words, (tuple, list))
            assert len(words) == self.count
        return words

    def decode(self, words):
        """
        Parse one or more words into the appropriate type of object. The type
        of 'words' is the same as would be passed to the decodeFunc passed to
        __init__.
        """

        if self.count == 1:
            assert isinstance(words, str)
        else:
            # Since words always comes from deserializeMessage, require that
            # it be a tuple, not a list. There's no real problem with it being
            # a list, but we know that that shouldn't happen, so if it does,
            # then it suggests a bug.
            assert isinstance(words, tuple)
            assert len(words) == self.count
        return self.decodeFunc(words)

# For an arbitrary string command and an arbitrary list of strings args,
#     tokenize(buildMessage(command, args, lastIsUnsafe=<whatever>))
# should either return exactly (command, args) or raise an InvalidMessageError.
def tokenize(message):
    if not message:
        raise InvalidMessageError(message, "Empty message.")
    if message[0] == START_STRING:
        raise InvalidMessageError(message,
                                  "Message starts with unsafe string.")

    index = message.find(TOKEN_DELIM + START_STRING)
    if index == -1:
        tokens = message.split(TOKEN_DELIM)
    else:
        tokens = message[:index].split(TOKEN_DELIM) + [message[index + 2:]]

    if not all(tokens):
        log.warning("Empty token in {}.".format(tokens))

    return (tokens[0], tokens[1:])

# TODO: We can and should unit-test buildMessage.
def buildMessage(command, args, lastIsUnsafe=False):
    """
    Build a message from the given command and arguments. The arguments don't
    have to be strings; if they aren't, then they will be str()ed.
    If lastIsUnsafe, then the last argument is a potentially unsafe string (for
    example, something typed by the user) and will be specially delimited.
    Only the last argument is allowed to be unsafe.
    """

    command = str(command)
    args    = map(str, args)
    lastArg = None

    if lastIsUnsafe:
        if not args:
            raise InvalidMessageError(command, "No arguments.")
        lastArg = args[-1]
        args = args[:-1]

    # Build the message. This has to come before the checking so that we have a
    # message to pass to any InvalidMessageErrors that we raise. Note that if
    # we raise an InvalidMessageError, the message we report with it isn't
    # actually a real message, because in that situation we've failed to build
    # a message. But it doesn't seem worth it to create yet another Exception
    # subclass, and InvalidMessageError seems more or less logically correct
    # for that type of error.
    message = TOKEN_DELIM.join([command] + args)
    if lastIsUnsafe:
        message += TOKEN_DELIM + START_STRING + lastArg

    # Check that (with the exception of the possible unsafe string at the end),
    # the message tokenizes correctly.

    def checkToken(token, tokenDesc):
        if TOKEN_DELIM in token:
            raise InvalidMessageError(message, "{0} may not contain {1!r}"
                                               .format(tokenDesc, TOKEN_DELIM))
        # TODO [#45]: Validate this.
        if command.startswith(START_STRING):
            raise InvalidMessageError(message, "{0} may not start with {1!r}"
                                               .format(tokenDesc,
                                                       START_STRING))

    checkToken(command, "Command")
    for arg in args:
        checkToken(arg, "Argument")

    # Checks passed; this is a valid message.
    return message

# Functions used to report bad messages.

def illFormedEMessage(error, log, clientId=None):
    """
    Log a warning for when a message string originating from an external source
    could not be parsed into a message, for example because it used a
    nonexistent command or passed the wrong number of arguments.
    """

    # TODO: This logic could probably be factored out I guess.
    sender = "external source"
    if clientId is not None:
        sender = "client {}".format(clientId)

    log.warning("Received invalid message from {sender}: {error}"
                .format(sender=sender, error=error))

def badEMessageCommand(message, log, clientId=None):
    """
    Log a warning for a message originating from an external source, where the
    message is well-formed (valid command with the right number of arguments)
    but it was received by a part of the code that doesn't know how to handle
    that type of message.
    """

    sender = "external source"
    if clientId is not None:
        sender = "client {}".format(clientId)

    log.warning("Could not handle message type from {sender}: {command}"
                .format(sender=sender, command=message.command))

def badEMessageArgument(message, log, clientId=None, reason=""):
    """
    Log a warning for a message originating from an external source, where the
    message is well-formed and was received by a part of the code that knows
    how to handle that type of message, but one of the arguments to the message
    is invalid (for example, out of range).
    """

    sender = "external source"
    if clientId is not None:
        sender = "client {}".format(clientId)
    if reason:
        reason = "\n    " + reason

    log.warning("Invalid argument to message from {sender}: {message}{reason}"
                .format(sender=sender, message=message, reason=reason))

# In this case you should just let error propagate up, rather than catching it
# at all.
#
# def illFormedIMessage(error, log, clientId=None):
#     """
#     Give an error for when a message string originating from an internal source
#     could not be parsed into a message, for example because it used a
#     nonexistent command or passed the wrong number of arguments.
#     """
#
#     log.error("Received invalid message: {error}"
#               .format(sender=sender, error=error))
#     raise error

def badIMessageCommand(message, log):
    """
    Give an error for when a message originating from an internal source is
    received by a part of the code that doesn't know how to handle that type of
    message.
    """

    error = "Could not handle message type from internal source: {command}." \
        .format(command=message.command)
    log.error(error)

    raise InvalidMessageError(message.serialize(), error)

def badIMessageArgument(message, log, reason=""):
    """
    Give an error for when a message originating from an internal source is
    received by a part of the code that knows how to handle that type of
    message, but one of the arguments to the message is invalid (for example,
    out of range).
    """

    if reason:
        reason = "\n    " + reason

    log.warning("Invalid argument in internal message: {message}{reason}"
                .format(message=message, reason=reason))

class InvalidMessageError(StandardError):
    def __init__(self, badMessage, errorDesc):
        super(InvalidMessageError, self).__init__()
        self.badMessage = badMessage
        self.errorDesc  = errorDesc

    def __str__(self):
        return "{desc}  (Message is: {msg!r})".format(desc = self.errorDesc,
                                                      msg  = self.badMessage)

<|endoftext|>