<commit_before>#!/usr/bin/env python
# encoding=gb18030

import sys
import getopt
import htmlfetcher
from htmlparser import HtmlXPathParser
import colorprint

outputEncoding = 'gb18030'

def usage():
    print '''iciba.py [--daemon|-h|word]'''

def parseOpt(args):
    config = {'daemon': False}
    try:
        opts, args = getopt.getopt(args, 'hd', ['help', 'daemon'])
    except getopt.GetoptError as err:
        print str(err)
        usage()
        sys.exit(2)
    for o, a in opts:
        if o in ('-h', '--help'):
            usage()
            sys.exit()
        elif o in ('-d', '--daemon'):
            config['daemon'] = True
        else:
            assert False, "unhandled option"
    return config, args

def getFirstText(tree, encoding=outputEncoding):
    return tree[0].text.encode(encoding)

def printResult(result):
    tColor = 'green'
    vColor = 'cyan'
    if len(result['prons']) > 0:
        print ''
        for t, v in result['prons']:
            colorprint.colorPrint('  %s' % t, tColor, attr=1)
            colorprint.colorPrint('[%s]' % v, vColor, attr=4)
        print
    if len(result['groupPos']) > 0:
        print ''
        for t, v in result['groupPos']:
            colorprint.colorPrint('  %s' % t, tColor, attr=1)
            colorprint.colorPrint(v, vColor, attr=4)
            print
    if len(result['netContent']) > 0:
        print ''
        colorprint.colorPrint('  %s' % result['netContent'], vColor, attr=1)

def parseHtml(data):
    result = {'prons': [], 'groupPos': [], 'netContent': []}
    parser = HtmlXPathParser()
    parser.feed(data, encode='utf-8')
    dictMainDiv = parser.etlist_xpath('//div[@id="dict_main"]')[0]
    tmpParser = HtmlXPathParser()
    tmpParser.feed_etree(dictMainDiv)
    pronsList = tmpParser.etlist_xpath('.//span[@class="fl"]')
    for prons in pronsList:
        pronsType = prons.text.encode('gbk').strip('\r\n\t ')
        pronsStr = getFirstText(prons.xpath('.//strong[@lang]'))
        result['prons'].append([pronsType, pronsStr])
    groupPosList = tmpParser.etlist_xpath('.//div[@class="group_pos"]/p')
    for groupPos in groupPosList:
        posType = getFirstText(groupPos.xpath('.//strong'))
        posContent = ''
        contentList = groupPos.xpath('.//label/text()')
        for c in contentList:
            posContent += c.encode(outputEncoding)
        result['groupPos'].append([posType, posContent])
    netContentList = tmpParser.etlist_xpath('.//div[@class="net_paraphrase"]/ul/li')
    netContent = ''
    for nc in netContentList:
        netContent += nc.text.encode('gb18030').strip()
    result['netContent'] = netContent

    return result

def translate(word):
    url = 'http://www.iciba.com/' + word
    data = htmlfetcher.http_get(url)
    if data is None:
        print >> sys.stderr, 'Fetch result from iciba.com failed!'
    else:
        result = parseHtml(data)
        printResult(result)

def main():
    config, args = parseOpt(sys.argv[1:])
    if config['daemon']:
        for line in sys.stdin:
            word = line.strip()
            translate(word)
    else:
        if len(args) == 0:
            usage()
            sys.exit()
        word = args[0]
        translate(word)

if __name__ == '__main__':
    main()
<commit_msg>modify iciba.py<commit_after>#!/usr/bin/env python
# encoding=gb18030

import sys
import getopt
import htmlfetcher
from htmlparser import HtmlXPathParser
import colorprint

outputEncoding = 'gb18030'

def usage():
    print '''iciba.py [--daemon|-h|word]'''

def parseOpt(args):
    config = {'daemon': False}
    try:
        opts, args = getopt.getopt(args, 'hd', ['help', 'daemon'])
    except getopt.GetoptError as err:
        print str(err)
        usage()
        sys.exit(2)
    for o, a in opts:
        if o in ('-h', '--help'):
            usage()
            sys.exit()
        elif o in ('-d', '--daemon'):
            config['daemon'] = True
        else:
            assert False, "unhandled option"
    return config, args

def getFirstText(tree, encoding=outputEncoding):
    try:
        if type(tree) is list:
            t = tree[0].text
        else:
            t = tree.text
        if t is not None:
            return t.encode(encoding)
        else:
            return ''
    except IndexError:
        return ''

def printResult(word, result):
    tColor = 'green'
    vColor = 'cyan'
    print word
    if len(result['prons']) > 0:
        print ''
        for t, v in result['prons']:
            colorprint.colorPrint('  %s' % t, tColor, attr=1)
            colorprint.colorPrint('%s' % v, vColor, attr=4)
        print
    if len(result['groupPos']) > 0:
        print ''
        for t, v in result['groupPos']:
            colorprint.colorPrint('  %s' % t, tColor, attr=1)
            colorprint.colorPrint(v, vColor, attr=4)
            print
    if len(result['netContent']) > 0:
        print ''
        colorprint.colorPrint('  %s' % result['netContent'], vColor, attr=1)

def parseHtml(data):
    result = {'prons': [], 'groupPos': [], 'netContent': []}
    parser = HtmlXPathParser()
    parser.feed(data, encode='utf-8')
    dictMainDiv = parser.etlist_xpath('//div[@id="dict_main"]')[0]
    tmpParser = HtmlXPathParser()
    tmpParser.feed_etree(dictMainDiv)
    pronsList = tmpParser.etlist_xpath('.//span[@class="fl"]')
    for prons in pronsList:
        pronsType = getFirstText(prons).strip('\r\n\t')
        pronsStr = getFirstText(prons.xpath('.//strong[@lang]'))
        if pronsStr == '':
            pronsStr = getFirstText(prons.xpath('.//strong'))
        else:
            pronsStr = '[' + ']'
        if len(pronsType) == 0 and len(pronsStr) == 0:
            continue
        result['prons'].append([pronsType, pronsStr])
    groupPosList = tmpParser.etlist_xpath('.//div[@class="group_pos"]/p')
    for groupPos in groupPosList:
        posType = getFirstText(groupPos.xpath('.//strong'))
        if posType is None:
            continue
        posContent = ''
        contentList = groupPos.xpath('.//label/text()')
        for c in contentList:
            posContent += c.encode(outputEncoding)
        if len(posType) == 0 and len(posContent) == 0:
            continue
        result['groupPos'].append([posType, posContent])
    netContentList = tmpParser.etlist_xpath('.//div[@class="net_paraphrase"]/ul/li')
    netContent = ''
    for nc in netContentList:
        netContent += nc.text.encode('gb18030').strip()
    result['netContent'] = netContent

    return result

def translate(word):
    url = 'http://www.iciba.com/' + word.decode('gbk').encode('utf8')
    home = 'http://www.iciba.com/'
    data = htmlfetcher.http_get(url, referer=home)
    if data is None:
        print >> sys.stderr, 'Fetch result from iciba.com failed!'
    else:
        result = parseHtml(data)
        printResult(word, result)

def main():
    config, args = parseOpt(sys.argv[1:])
    if config['daemon']:
        while True:
            line = sys.stdin.readline()
            if not line:
                break
            word = line.strip()
            translate(word)
            sys.stdout.flush()
            sys.stderr.flush()
    else:
        if len(args) == 0:
            usage()
            sys.exit()
        word = args[0]
        translate(word)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
#  kanjidic.py
#  cjktools
#

"""
A nice interface to the Kanjidic dictionary.
"""

import re
from itertools import chain

from cjktools import scripts
from cjktools.common import sopen

import cjkdata

basic_features = set([
    'gloss',
    'stroke_count',
    'jyouyou_grade',
    'skip_code',
    'onyomi',
    'kunyomi',
    'unicode'
])

remappings = {
    'B':    'radical_index',
    'C':    'classical_radical_index',
    'F':    'frequency',
    'G':    'jyouyou_grade',
    'H':    'halpern_index',
    'N':    'nelson_index',
    'V':    'new_nelson_index',
    'D':    'dictionary_code',
    'P':    'skip_code',
    'S':    'stroke_count',
    'U':    'unicode',
    'I':    'spahn_code',
    'Q':    'four_corner_code',
    'M':    'morohashi_index',
    'E':    'henshall_code',
    'K':    'gakken_code',
    'L':    'heisig_code',
    'O':    'oneill_code',
    'W':    'korean_reading',
    'Y':    'pinyin_reading',
    'X':    'cross_references',
    'Z':    'misclassified_as'
}

class KanjidicEntry(object):
    "A single entry in the kanjidic file."

    def __init__(self, **entry_details):
        assert ('on_readings' in entry_details
                and 'kun_readings' in entry_details)
        self.__dict__.update(entry_details)

    def get_all_readings(self):
        "Construct a reading pool for this entry."
        reading_set = set()
        for reading in (self.kun_readings +
                        map(scripts.to_hiragana, self.on_readings)):
            # Ignore suffix/prefix information about readings.
            if '-' in reading:
                reading = reading.replace('-', '')

            # Only use the stem  of okurigana readings.
            if '.' in reading:
                stem, suffix = reading.split('.')
                reading = stem

            reading_set.add(reading)

        return reading_set

    all_readings = property(get_all_readings)

class Kanjidic(dict):
    """
    An interface to the kanjidic dictionary. Create a new instance to parse
    the dictionary. Has a nice getitem interface to get the details of a
    character.
    """

    def __init__(self, kanjidic_files=None):
        dict.__init__(self)

        if kanjidic_files is None:
            kanjidic_files = [
                cjkdata.get_resource('kanjidic'),
                cjkdata.get_resource('kanjd212'),
            ]

        line_stream = reduce(chain, [sopen(f) for f in kanjidic_files])
        self._parse_kanjidic(line_stream)

    @classmethod
    def get_cached(cls):
        if not hasattr(cls, '_cached'):
            cls._cached = cls()

        return cls._cached

    def _parse_kanjidic(self, line_stream):
        """
        Parses the kanjidic file for its contents, updating this class
        with a kanji -> info mapping.

        @param files: The files for kanjidic.
        """
        self._dictionary = {}

        for line in line_stream:
            if line.startswith('#'):
                continue

            entry = self._parse_line(line)
            self[entry.kanji] = entry

    def _parse_line(self, line):
        "Parses a single line in the kanjdic file, returning an entry."
        segment_pattern = re.compile('[^ {]+|{.*?}', re.UNICODE)
        segments = segment_pattern.findall(line.strip())
        segments.reverse()

        kanji = segments.pop()
        jis_code = int(segments.pop(), 16)
        info = {
            'kanji':        kanji,
            'gloss':        [],
            'on_readings':  [],
            'kun_readings': [],
            'jis_code':     jis_code
        }

        while segments:
            s = segments.pop()

            if s.startswith('{'):
                info['gloss'].append(s[1:-1])

            elif (scripts.script_type(s) != scripts.Script.Ascii
                    or s.startswith('-')):
                # It must be a reading.
                char = s[0]
                if char == '-':
                    char = s[1]

                if scripts.script_type(char) == scripts.Script.Katakana:
                    info['on_readings'].append(s)
                elif scripts.script_type(char) == scripts.Script.Hiragana:
                    info['kun_readings'].append(s)
                else:
                    raise Exception("Unknown segment %s" % s)

            elif s in ('T1', 'T2'):
                continue

            else:
                # handle various codes
                code = s[0]
                remainder = s[1:]
                try:
                    remainder = int(remainder)
                except:
                    pass

                info.setdefault(remappings.get(code, code), []).append(
                    remainder)

        info['stroke_count'] = info['stroke_count'][0]
        if 'frequency' in info:
            info['frequency'] = info['frequency'][0]

        info['skip_code'] = tuple(int(i)
                                  for i in info['skip_code'][0].split('-'))

        return KanjidicEntry(**info)

_cached_kanjidic = None
<commit_msg>Added python3 compatibility for kanjidic<commit_after># -*- coding: utf-8 -*-
#
#  kanjidic.py
#  cjktools
#

"""
A nice interface to the Kanjidic dictionary.
"""
from __future__ import unicode_literals

import re
from itertools import chain

from cjktools import scripts
from cjktools.common import sopen

from functools import reduce

import cjkdata

basic_features = set([
    'gloss',
    'stroke_count',
    'jyouyou_grade',
    'skip_code',
    'onyomi',
    'kunyomi',
    'unicode'
])

remappings = {
    'B':    'radical_index',
    'C':    'classical_radical_index',
    'F':    'frequency',
    'G':    'jyouyou_grade',
    'H':    'halpern_index',
    'N':    'nelson_index',
    'V':    'new_nelson_index',
    'D':    'dictionary_code',
    'P':    'skip_code',
    'S':    'stroke_count',
    'U':    'unicode',
    'I':    'spahn_code',
    'Q':    'four_corner_code',
    'M':    'morohashi_index',
    'E':    'henshall_code',
    'K':    'gakken_code',
    'L':    'heisig_code',
    'O':    'oneill_code',
    'W':    'korean_reading',
    'Y':    'pinyin_reading',
    'X':    'cross_references',
    'Z':    'misclassified_as'
}

class KanjidicEntry(object):
    "A single entry in the kanjidic file."

    def __init__(self, **entry_details):
        assert ('on_readings' in entry_details
                and 'kun_readings' in entry_details)
        self.__dict__.update(entry_details)

    def get_all_readings(self):
        "Construct a reading pool for this entry."
        reading_set = set()
        for reading in (self.kun_readings +
                        map(scripts.to_hiragana, self.on_readings)):
            # Ignore suffix/prefix information about readings.
            if '-' in reading:
                reading = reading.replace('-', '')

            # Only use the stem  of okurigana readings.
            if '.' in reading:
                stem, suffix = reading.split('.')
                reading = stem

            reading_set.add(reading)

        return reading_set

    all_readings = property(get_all_readings)

class Kanjidic(dict):
    """
    An interface to the kanjidic dictionary. Create a new instance to parse
    the dictionary. Has a nice getitem interface to get the details of a
    character.
    """

    def __init__(self, kanjidic_files=None):
        dict.__init__(self)

        if kanjidic_files is None:
            kanjidic_files = [
                cjkdata.get_resource('kanjidic'),
                cjkdata.get_resource('kanjd212'),
            ]

        line_stream = reduce(chain, [sopen(f) for f in kanjidic_files])
        self._parse_kanjidic(line_stream)

    @classmethod
    def get_cached(cls):
        if not hasattr(cls, '_cached'):
            cls._cached = cls()

        return cls._cached

    def _parse_kanjidic(self, line_stream):
        """
        Parses the kanjidic file for its contents, updating this class
        with a kanji -> info mapping.

        @param files: The files for kanjidic.
        """
        self._dictionary = {}

        for line in line_stream:
            if line.startswith('#'):
                continue

            entry = self._parse_line(line)
            self[entry.kanji] = entry

    def _parse_line(self, line):
        "Parses a single line in the kanjdic file, returning an entry."
        segment_pattern = re.compile('[^ {]+|{.*?}', re.UNICODE)
        segments = segment_pattern.findall(line.strip())
        segments.reverse()

        kanji = segments.pop()
        jis_code = int(segments.pop(), 16)
        info = {
            'kanji':        kanji,
            'gloss':        [],
            'on_readings':  [],
            'kun_readings': [],
            'jis_code':     jis_code
        }

        while segments:
            s = segments.pop()

            if s.startswith('{'):
                info['gloss'].append(s[1:-1])

            elif (scripts.script_type(s) != scripts.Script.Ascii
                    or s.startswith('-')):
                # It must be a reading.
                char = s[0]
                if char == '-':
                    char = s[1]

                if scripts.script_type(char) == scripts.Script.Katakana:
                    info['on_readings'].append(s)
                elif scripts.script_type(char) == scripts.Script.Hiragana:
                    info['kun_readings'].append(s)
                else:
                    raise Exception("Unknown segment %s" % s)

            elif s in ('T1', 'T2'):
                continue

            else:
                # handle various codes
                code = s[0]
                remainder = s[1:]
                try:
                    remainder = int(remainder)
                except:
                    pass

                info.setdefault(remappings.get(code, code), []).append(
                    remainder)

        info['stroke_count'] = info['stroke_count'][0]
        if 'frequency' in info:
            info['frequency'] = info['frequency'][0]

        info['skip_code'] = tuple(int(i)
                                  for i in info['skip_code'][0].split('-'))

        return KanjidicEntry(**info)

_cached_kanjidic = None
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: UTF-8 -*-

import os
from os import makedirs
from os.path import isdir, dirname
import time
import logging
import boto3
import click
import sqlite3
from watchdog.observers import Observer
from watchdog.events import LoggingEventHandler
from .config import read_config

config = read_config()

class S3Element(LoggingEventHandler):

    def __init__(self, bucket, folder):
        super(LoggingEventHandler, self).__init__()
        self.folder = folder
        self.bucket = bucket
        self.url = ''
        self.format_logging()
        self.valid_ext = ['.png', '.jpg', '.gif', '.jpeg']
        self.db = config.get('vxscreenshots.database')
        if not isdir(dirname(self.db)):
            makedirs(dirname(self.db))
        self.logger.info('Cache dbname: %s' % self.db)
        self.logger.info('Bucket name: %s' % self.bucket)
        self.logger.info('Folder name: %s' % self.folder)

    def format_logging(self, log_level='INFO'):
        root = logging.getLogger()
        if root.handlers:
            for handler in root.handlers:
                root.removeHandler(handler)
                logging.basicConfig(format='%(asctime)s: %(name)s: %(levelname)s: %(message)s',
                                    level=logging.INFO)
        self.logger = logging.getLogger(__name__)
        self.logger.setLevel(logging.INFO)

    def get_conn(self):
        return self.conn.cursor()

    def get_path(self, src_path):
        return '{}/{}'.format(self.folder, src_path)

    def get_url(self, fname):
        return "http://{bucket}/{fname}".format(bucket=self.bucket,
                                                fname=fname)

    def init_db(self):
        self.cursor.execute('''CREATE TABLE stock_images
                                 (path text, url text, synced boolean)
                            ''')

    def db_insert_new(self, images):
        self.cursor.executemany('INSERT INTO stock_images VALUES (?,?,?)',
                                images)
        self.conn.commit()

    def on_modified(self, event):
        what = 'directory' if event.is_directory else 'file'
        self.send_to_s3(what, event)

    def send_to_s3(self, what, event):
        self.logger.info("Screenshot was Modified %s: %s", what, event.src_path)
        name, ext = os.path.splitext(event.src_path)
        if what != 'directory' and \
           os.path.isfile(event.src_path) and \
           ext in self.valid_ext:
            s3 = boto3.resource('s3')
            data = open(event.src_path, 'rb')
            fname = self.get_path(os.path.basename(data.name))
            bucket = s3.Bucket(self.bucket)
            bucket.put_object(Key=fname, Body=data, ACL='public-read',
                              ContentType='image/%s' % ext[1:])
            self.url = self.get_url(fname)
            self.logger.info("Screenshot was pushed to %s" % self.url)
            self.conn = sqlite3.connect(self.db)
            # Asume if running as a main script it is a dev mode
            self.cursor = self.get_conn()
            try:
                self.init_db()
            except Exception, e:
                self.logger.warning(e)
            try:
                self.db_insert_new([(event.src_path, self.url, True)])
                self.logger.warning('Inserted on db %s %s' % (event.src_path,
                                                         self.url))
            except Exception, e:
                self.logger.warning('I could not insert on cache %s' % e)

@click.option('--path', default=config.get('vxscreenshots.supervised'),
              help='Path to be supervised')
@click.option('--bucket', default=config.get('vxscreenshots.bucket_name'),
              help='Bucket where things will be saved to')
@click.option('--folder', default=config.get('vxscreenshots.folder'),
              help='Inside the bucket how the folder will be named')
@click.command()
def cli(path, bucket, folder):
    '''Watch a folder and push images automatically to amazon S3'''
    event_handler = S3Element(bucket, folder)
    msg = 'Sending to this bucket %s %s %s' % (bucket, folder, path)
    event_handler.logger.info(msg)
    observer = Observer()
    observer.schedule(event_handler, path, recursive=True)
    observer.start()
    try:
        while True:
            time.sleep(1)
    except IOError:
        event_handler.logger.info('A crazy file changed')
    except KeyboardInterrupt:
        observer.stop()
    observer.join()
if __name__ == "__main__":
    cli()
<commit_msg>Fix little pylints - rebase<commit_after>#! /usr/bin/env python
# -*- coding: UTF-8 -*-

import os
from os import makedirs
from os.path import isdir, dirname
import time
import logging
import boto3
import click
import sqlite3
from watchdog.observers import Observer
from watchdog.events import LoggingEventHandler
from .config import read_config

config = read_config()

class S3Element(LoggingEventHandler):

    def __init__(self, bucket, folder):
        super(LoggingEventHandler, self).__init__()
        self.folder = folder
        self.bucket = bucket
        self.url = ''
        self.format_logging()
        self.valid_ext = ['.png', '.jpg', '.gif', '.jpeg']
        self.db = config.get('vxscreenshots.database')
        if not isdir(dirname(self.db)):
            makedirs(dirname(self.db))
        self.logger.info('Cache dbname: %s' % self.db)
        self.logger.info('Bucket name: %s' % self.bucket)
        self.logger.info('Folder name: %s' % self.folder)

    def format_logging(self, log_level='INFO'):
        root = logging.getLogger()
        if root.handlers:
            for handler in root.handlers:
                root.removeHandler(handler)
                logging.basicConfig(format='%(asctime)s: %(name)s: '
                                    '%(levelname)s: %(message)s',
                                    level=logging.INFO)
        self.logger = logging.getLogger(__name__)
        self.logger.setLevel(logging.INFO)

    def get_conn(self):
        return self.conn.cursor()

    def get_path(self, src_path):
        return '{}/{}'.format(self.folder, src_path)

    def get_url(self, fname):
        return "http://{bucket}/{fname}".format(bucket=self.bucket,
                                                fname=fname)

    def init_db(self):
        self.cursor.execute('''CREATE TABLE stock_images
                                 (path text, url text, synced boolean)
                            ''')

    def db_insert_new(self, images):
        self.cursor.executemany('INSERT INTO stock_images VALUES (?,?,?)',
                                images)
        self.conn.commit()

    def on_modified(self, event):
        what = 'directory' if event.is_directory else 'file'
        self.send_to_s3(what, event)

    def send_to_s3(self, what, event):
        self.logger.info("Screenshot was Modified %s: %s", what,
                         event.src_path)
        name, ext = os.path.splitext(event.src_path)
        if what != 'directory' and \
           os.path.isfile(event.src_path) and \
           ext in self.valid_ext:
            s3 = boto3.resource('s3')
            data = open(event.src_path, 'rb')
            fname = self.get_path(os.path.basename(data.name))
            bucket = s3.Bucket(self.bucket)
            bucket.put_object(Key=fname, Body=data, ACL='public-read',
                              ContentType='image/%s' % ext[1:])
            self.url = self.get_url(fname)
            self.logger.info("Screenshot was pushed to %s" % self.url)
            self.conn = sqlite3.connect(self.db)
            # Asume if running as a main script it is a dev mode
            self.cursor = self.get_conn()
            try:
                self.init_db()
            except Exception, e:
                self.logger.warning(e)
            try:
                self.db_insert_new([(event.src_path, self.url, True)])
                self.logger.warning('Inserted on db %s %s' % (event.src_path,
                                                         self.url))
            except Exception, e:
                self.logger.warning('I could not insert on cache %s' % e)

@click.option('--path', default=config.get('vxscreenshots.supervised'),
              help='Path to be supervised')
@click.option('--bucket', default=config.get('vxscreenshots.bucket_name'),
              help='Bucket where things will be saved to')
@click.option('--folder', default=config.get('vxscreenshots.folder'),
              help='Inside the bucket how the folder will be named')
@click.command()
def cli(path, bucket, folder):
    '''Watch a folder and push images automatically to amazon S3'''
    event_handler = S3Element(bucket, folder)
    msg = 'Sending to this bucket %s %s %s' % (bucket, folder, path)
    event_handler.logger.info(msg)
    observer = Observer()
    observer.schedule(event_handler, path, recursive=True)
    observer.start()
    try:
        while True:
            time.sleep(1)
    except IOError:
        event_handler.logger.info('A crazy file changed')
    except KeyboardInterrupt:
        observer.stop()
    observer.join()
if __name__ == "__main__":
    cli()
<|endoftext|>


<commit_before>import sentry_sdk
from sentry_sdk.integrations.flask import FlaskIntegration

def sentry_feature(app):
    """
    Sentry feature
    Adds basic integration with Sentry via the raven library
    """

    # get keys
    sentry_public_key = app.config.get('SENTRY_PUBLIC_KEY')
    sentry_project_id = app.config.get('SENTRY_PROJECT_ID')
    sentry_ingest_url = 'o96810.ingest.sentry.io'
    if not sentry_public_key or not sentry_project_id or not sentry_ingest_url:
        return

    # prepare dsn
    dsn = 'https://{key}@{ingest_url}/{project_id}'.format(
        key=sentry_public_key,
        ingest_url=sentry_ingest_url,
        project_id=sentry_project_id
    )

    # init sentry
    sentry_sdk.init(
        dsn=dsn,
        integrations=[FlaskIntegration()]
    )<commit_msg>Reconfigure sentry feature<commit_after>import sentry_sdk
from sentry_sdk.integrations.flask import FlaskIntegration

def sentry_feature(app):
    """
    Sentry feature
    Adds basic integration with Sentry via the raven library
    """

    # get keys
    sentry_key = app.config.get('SENTRY_KEY')
    sentry_project_id = app.config.get('SENTRY_PROJECT_ID')
    sentry_ingest_url = 'SENTRY_INGEST_URL'
    if not sentry_key or not sentry_project_id or not sentry_ingest_url:
        return

    # prepare dsn
    dsn = 'https://{key}@{ingest_url}/{project_id}'.format(
        key=sentry_key,
        ingest_url=sentry_ingest_url,
        project_id=sentry_project_id
    )

    # init sentry
    sentry_sdk.init(
        dsn=dsn,
        integrations=[FlaskIntegration()]
    )<|endoftext|>


<commit_before>#!/usr/bin/env python2.7
"""Wrasse

Usage:
    wrasse
"""

import os
from functools import partial
from os.path import join

import boto
from docopt import docopt

from boto.s3.connection import S3Connection, Location, Key

def entry_console():
    traverse()

def examine_file(candidate, directory=None):
    path = join(directory, candidate)
    print path

    from hashlib import md5
    with open(path) as in_file:
        local_hash = md5(in_file.read()).hexdigest()

    key = bucket.get_key(path)

    # The etag string contains quotes...
    etag = key.etag[1:-1] if key else ""
    if etag != local_hash:
        upload_file(path)

def upload_file(path):
    print "Uploading {0}".format(path)
    key = Key(bucket, path)
    key.set_contents_from_filename(path, reduced_redundancy=True)

def traverse():
    os.chdir('repo')
    for root, dirs, files in os.walk('.'):
        root = root[2:]
        dir_wrapper = partial(examine_file, directory=root)
        map(dir_wrapper, files)

if __name__ == "__main__":
    args = docopt(__doc__)
    conn = S3Connection()
    bucket = conn.get_bucket("com.fishsilo.skipjack")
    entry_console()
<commit_msg>Start expanding functionality of wrasse<commit_after>#!/usr/bin/env python2.7
"""Wrasse

Usage:
    wrasse (pull|push) [options]
    wrasse package (add|remove) [options] <file>

Options:
    -h --help       Show this help
    -v --verbose    Be more verbose
    --debug         Be way more verbose
"""

import os
from functools import partial
from os.path import join
import logging
from hashlib import md5

from docopt import docopt

from boto.s3.connection import S3Connection, Key

logger = logging.getLogger("wrasse")
logger.setLevel(logging.WARNING)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
# create formatter and add it to the handlers
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(ch)

def entry_console():
    if args['--debug']:
        logger.setLevel(logging.DEBUG)
    elif args['--verbose']:
        logger.setLevel(logging.INFO)
    if args['push'] or args['pull']:
        traverse()

def examine_file(candidate, directory=None):
    path = join(directory, candidate)
    logger.debug("Examining file: %s", path)

    with open(path) as in_file:
        local_hash = md5(in_file.read()).hexdigest()

    key = bucket.get_key(path)

    # The etag string contains quotes...
    etag = key.etag[1:-1] if key else ""
    if etag != local_hash:
        if args['push']:
            upload_file(path)
        else:
            logger.info("Local differs from remote: %s", path)

def upload_file(path):
    logger.info("Pushing local to remote: %s", path)
    key = Key(bucket, path)
    key.set_contents_from_filename(path, reduced_redundancy=True)

def traverse():
    os.chdir('repo')
    for root, dirs, files in os.walk('.'):
        root = root[2:]
        dir_wrapper = partial(examine_file, directory=root)
        map(dir_wrapper, files)
    for key in bucket.list():
        logger.debug(key)

if __name__ == "__main__":
    args = docopt(__doc__)
    conn = S3Connection()
    bucket = conn.get_bucket("com.fishsilo.skipjack")
    entry_console()
<|endoftext|>


<commit_before>import re
import platform
from uuid import getnode
import datetime
from objectid import ObjectId

_camel_pat = re.compile(r'([A-Z])')
_under_pat = re.compile(r'_([a-z])')

def camel_to_underscore(name):
    return _camel_pat.sub(lambda x: '_' + x.group(1).lower(), name)[1:]

def underscore_to_camel(name):
    return _under_pat.sub(lambda x: x.group(1).upper(), name.capitalize())

def build_components(conf, key):
    components = conf.get(key, None) or []
    return [build_component(component) for component in components]

def build_component(component):
    if isinstance(component, basestring):
        component = {component:{}}
    for class_, conf in component.items():
        module_class = class_.split(":")
        module_class = class_.split(".")
        route = [camel_to_underscore(c) for c in module_class]
        module_name = route[-1]
        class_name = module_class[-1]
        route = ".".join(route)
        module = __import__(route, globals(), locals(), [module_name])
        class_ =  getattr(module, class_name)
        return class_(conf)

def get_mac():
    mac = hex(getnode())[2:]
    return ("%s:%s:%s:%s:%s:%s" % (
            mac[0:2],
            mac[2:4],
            mac[4:6],
            mac[6:8],
            mac[8:10],
            mac[10:12])).upper()

class HashBuilder:
    def __init__(self, shared, **opts):
        self.shared = {
            "@source_mac" : get_mac(),
            "@source_host" : platform.node()
        }
        self.shared.update(shared)
        self.fake_timestamp = opts.get("fake_timestamp", False)
        self.build()

    def build(self, data=None):
        self.data = self.shared.copy()
        self.data.setdefault("@timestamp", datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S.000Z"))
        if data is not None:
            self.data["@message"] = data

    def log(self, value):
        self.data.setdefault("@message", []).append(unicode(value))

    def dict(self):
        if isinstance(self.data["@message"], list):
            self.data["@message"] = "".join(self.data["@message"])
        # if self.fake_timestamp:
        #     ts = _timestamp_pat.search(self.data["@message"])
        #     if ts:
        #         self.data["@timestamp"] = date_time_parser.parse(ts.group(0))
        # self.data["_id"] = str(ObjectId()._generate(self.data["@timestamp"]))
        # self.data["@timestamp"] = self.data["@timestamp"].strftime("%Y-%m-%dT%H:%M:%S.000Z")#.isoformat()# + "Z" #strftime("T%H:%M:%SZ") #.isoformat()
        return self.data

    def has_data(self):
        return bool(self.data)
<commit_msg>Changed timestamp format<commit_after>import re
import platform
from uuid import getnode
import datetime
from objectid import ObjectId
import time

_camel_pat = re.compile(r'([A-Z])')
_under_pat = re.compile(r'_([a-z])')

def camel_to_underscore(name):
    return _camel_pat.sub(lambda x: '_' + x.group(1).lower(), name)[1:]

def underscore_to_camel(name):
    return _under_pat.sub(lambda x: x.group(1).upper(), name.capitalize())

def build_components(conf, key):
    components = conf.get(key, None) or []
    return [build_component(component) for component in components]

def build_component(component):
    if isinstance(component, basestring):
        component = {component:{}}
    for class_, conf in component.items():
        module_class = class_.split(":")
        module_class = class_.split(".")
        route = [camel_to_underscore(c) for c in module_class]
        module_name = route[-1]
        class_name = module_class[-1]
        route = ".".join(route)
        module = __import__(route, globals(), locals(), [module_name])
        class_ =  getattr(module, class_name)
        return class_(conf)

def get_mac():
    mac = hex(getnode())[2:]
    return ("%s:%s:%s:%s:%s:%s" % (
            mac[0:2],
            mac[2:4],
            mac[4:6],
            mac[6:8],
            mac[8:10],
            mac[10:12])).upper()

class HashBuilder:
    def __init__(self, shared, **opts):
        self.shared = {
            "@source_mac" : get_mac(),
            "@source_host" : platform.node()
        }
        self.shared.update(shared)
        self.fake_timestamp = opts.get("fake_timestamp", False)
        self.build()

    def build(self, data=None):
        self.data = self.shared.copy()
        self.data.setdefault("@timestamp", '{0}{1:+06.2f}'.format(datetime.datetime.now().isoformat(), float(time.timezone) / 3600))
        if data is not None:
            self.data["@message"] = data

    def log(self, value):
        self.data.setdefault("@message", []).append(unicode(value))

    def dict(self):
        if isinstance(self.data["@message"], list):
            self.data["@message"] = "".join(self.data["@message"])
        return self.data

    def has_data(self):
        return bool(self.data)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Paul Nilsson, paul.nilsson@cern.ch

import subprocess
from os import environ, getcwd, setpgrp  #, getpgid  #setsid
from sys import version_info

import logging
logger = logging.getLogger(__name__)

def is_python3():
    """
    Check if we are running on Python 3.

    :return: boolean.
    """

    return version_info >= (3, 0)

def execute(executable, **kwargs):  # noqa: C901
    """
    Execute the command and its options in the provided executable list.
    The function also determines whether the command should be executed within a container.
    TODO: add time-out functionality.

    :param executable: command to be executed (string or list).
    :param kwargs (timeout, usecontainer, returnproc):
    :return: exit code, stdout and stderr (or process if requested via returnproc argument)
    """

    cwd = kwargs.get('cwd', getcwd())
    stdout = kwargs.get('stdout', subprocess.PIPE)
    stderr = kwargs.get('stderr', subprocess.PIPE)
    timeout = kwargs.get('timeout', None)
    usecontainer = kwargs.get('usecontainer', False)
    returnproc = kwargs.get('returnproc', False)
    mute = kwargs.get('mute', False)
    job = kwargs.get('job')
    mode = kwargs.get('mode', 'bash')

    # convert executable to string if it is a list
    if type(executable) is list:
        executable = ' '.join(executable)

    # switch off pilot controlled containers for user defined containers
    if job and job.imagename != "" and "runcontainer" in executable:
        usecontainer = False
        job.usecontainer = usecontainer

    # Import user specific code if necessary (in case the command should be executed in a container)
    # Note: the container.wrapper() function must at least be declared
    if usecontainer:
        user = environ.get('PILOT_USER', 'generic').lower()  # TODO: replace with singleton
        container = __import__('pilot.user.%s.container' % user, globals(), locals(), [user], 0)  # Python 2/3
        if container:
            # should a container really be used?
            do_use_container = job.usecontainer if job else container.do_use_container(**kwargs)

            if do_use_container:
                diagnostics = ""
                try:
                    executable = container.wrapper(executable, **kwargs)
                except Exception as e:
                    diagnostics = 'failed to execute wrapper function: %s' % e
                    logger.fatal(diagnostics)
                else:
                    if executable == "":
                        diagnostics = 'failed to prepare container command'
                        logger.fatal(diagnostics)
                if diagnostics != "":
                    return None if returnproc else -1, "", diagnostics
            else:
                logger.info('pilot user container module has decided to not use a container')
        else:
            logger.warning('container module could not be imported')

    if not mute:
        executable_readable = executable
        executables = executable_readable.split(";")
        for sub_cmd in executables:
            if 'S3_SECRET_KEY=' in sub_cmd:
                secret_key = sub_cmd.split('S3_SECRET_KEY=')[1]
                secret_key = 'S3_SECRET_KEY=' + secret_key
                executable_readable = executable_readable.replace(secret_key, 'S3_SECRET_KEY=********')
        logger.info('executing command: %s' % executable_readable)

    if mode == 'python':
        exe = ['/usr/bin/python', executable]
    else:
        exe = ['/bin/bash', '-c', executable]

    # try: intercept exception such as OSError -> report e.g. error.RESOURCEUNAVAILABLE: "Resource temporarily unavailable"
    process = subprocess.Popen(exe,
                               bufsize=-1,
                               stdout=stdout,
                               stderr=stderr,
                               cwd=cwd,
                               preexec_fn=setpgrp)  #setsid)
    if returnproc:
        return process
    else:
        stdout, stderr = process.communicate()
        exit_code = process.poll()

        # for Python 3, convert from byte-like object to str
        if is_python3():
            stdout = stdout.decode('utf-8')
            stderr = stderr.decode('utf-8')
        # remove any added \n
        if stdout and stdout.endswith('\n'):
            stdout = stdout[:-1]

        return exit_code, stdout, stderr
<commit_msg>Update<commit_after>#!/usr/bin/env python
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
#
# Authors:
# - Paul Nilsson, paul.nilsson@cern.ch

import subprocess
from os import environ, getcwd, setpgrp  #, getpgid  #setsid
from sys import version_info

import logging
logger = logging.getLogger(__name__)

def is_python3():
    """
    Check if we are running on Python 3.

    :return: boolean.
    """

    return version_info >= (3, 0)

def execute(executable, **kwargs):  # noqa: C901
    """
    Execute the command and its options in the provided executable list.
    The function also determines whether the command should be executed within a container.
    TODO: add time-out functionality.

    :param executable: command to be executed (string or list).
    :param kwargs (timeout, usecontainer, returnproc):
    :return: exit code, stdout and stderr (or process if requested via returnproc argument)
    """

    cwd = kwargs.get('cwd', getcwd())
    stdout = kwargs.get('stdout', subprocess.PIPE)
    stderr = kwargs.get('stderr', subprocess.PIPE)
    timeout = kwargs.get('timeout', None)
    usecontainer = kwargs.get('usecontainer', False)
    returnproc = kwargs.get('returnproc', False)
    mute = kwargs.get('mute', False)
    job = kwargs.get('job')
    mode = kwargs.get('mode', 'bash')

    # convert executable to string if it is a list
    if type(executable) is list:
        executable = ' '.join(executable)

    # switch off pilot controlled containers for user defined containers
    if job and job.imagename != "" and "runcontainer" in executable:
        usecontainer = False
        job.usecontainer = usecontainer

    # Import user specific code if necessary (in case the command should be executed in a container)
    # Note: the container.wrapper() function must at least be declared
    if usecontainer:
        user = environ.get('PILOT_USER', 'generic').lower()  # TODO: replace with singleton
        container = __import__('pilot.user.%s.container' % user, globals(), locals(), [user], 0)  # Python 2/3
        if container:
            # should a container really be used?
            do_use_container = job.usecontainer if job else container.do_use_container(**kwargs)

            if do_use_container:
                diagnostics = ""
                try:
                    executable = container.wrapper(executable, **kwargs)
                except Exception as e:
                    diagnostics = 'failed to execute wrapper function: %s' % e
                    logger.fatal(diagnostics)
                else:
                    if executable == "":
                        diagnostics = 'failed to prepare container command'
                        logger.fatal(diagnostics)
                if diagnostics != "":
                    return None if returnproc else -1, "", diagnostics
            else:
                logger.info('pilot user container module has decided to not use a container')
        else:
            logger.warning('container module could not be imported')

    if not mute:
        executable_readable = executable
        executables = executable_readable.split(";")
        for sub_cmd in executables:
            if 'S3_SECRET_KEY=' in sub_cmd:
                secret_key = sub_cmd.split('S3_SECRET_KEY=')[1]
                secret_key = 'S3_SECRET_KEY=' + secret_key
                executable_readable = executable_readable.replace(secret_key, 'S3_SECRET_KEY=********')
        logger.info('executing command: %s' % executable_readable)

    if mode == 'python':
        exe = ['/usr/bin/python'] + executable.split()
    else:
        exe = ['/bin/bash', '-c', executable]

    # try: intercept exception such as OSError -> report e.g. error.RESOURCEUNAVAILABLE: "Resource temporarily unavailable"
    process = subprocess.Popen(exe,
                               bufsize=-1,
                               stdout=stdout,
                               stderr=stderr,
                               cwd=cwd,
                               preexec_fn=setpgrp)  #setsid)
    if returnproc:
        return process
    else:
        stdout, stderr = process.communicate()
        exit_code = process.poll()

        # for Python 3, convert from byte-like object to str
        if is_python3():
            stdout = stdout.decode('utf-8')
            stderr = stderr.decode('utf-8')
        # remove any added \n
        if stdout and stdout.endswith('\n'):
            stdout = stdout[:-1]

        return exit_code, stdout, stderr
<|endoftext|>


<commit_before>#!/usr/bin/python3

"""
Simple helper script generating boilerplate code for a set of functions in order
to make them dynamically loaded and GObject-introspectable.

"""

from collections import namedtuple
import re
import sys
import os

FuncInfo = namedtuple("FuncInfo", ["name", "doc", "rtype", "args"])
FUNC_SPEC_RE = re.compile(r'(?P<rtype>\**\s*\w+\s*\**)'
                          r'\s*'
                          r'(?P<name>\w+)'
                          r'\s*\('
                          r'(?P<args>[\w*\s,]*)'
                          r'\);')

def gather_func_info(line_iter, includes):
    name = doc = rtype = args = ""
    in_doc = False
    in_skip = False
    for line in line_iter:
        if in_skip or line.strip().startswith("/* BpG-skip"):
            in_skip = True
        elif in_skip and line.strip().startswith("/* BpG-skip-end"):
            in_skip = False
        elif line.strip() == "" or line.strip().startswith("//") or line.strip().startswith("/* "):
            # whitespace line or ignored comment, skip it
            continue
        elif line.strip().startswith("#include"):
            includes.append(line.strip()[8:])
        elif line.strip().startswith("/**"):
            doc += line
            in_doc = True
        elif line.strip().endswith("*/") and in_doc:
            doc += line
            in_doc = False
        else:
            match = FUNC_SPEC_RE.match(line.strip())
            if match:
                fields = match.groupdict()
                name = fields["name"]
                rtype = fields["rtype"]
                args = fields["args"]

                # nothing more for this function
                break
            elif in_doc:
                doc += line
            else:
                msg = "Skipping unknown line: '{0}'".format(line.rstrip("\n"))
                print(msg, file=sys.stderr)
    else:
        return None

    return FuncInfo(name, doc, rtype, args)

def process_file(fobj):
    includes = []

    line_iter = iter(fobj)
    fn_infos = list()

    fn_info = gather_func_info(line_iter, includes)
    while fn_info:
        fn_infos.append(fn_info)
        fn_info = gather_func_info(line_iter, includes)

    return includes, fn_infos

def get_arg_names(args):
    if not args.strip():
        return []

    typed_args = args.split(",")
    starred_names = (typed_arg.split()[-1] for typed_arg in typed_args)

    return [starred_name.strip("* ") for starred_name in starred_names]

def get_func_boilerplate(fn_info):
    call_args_str = ", ".join(get_arg_names(fn_info.args))

    if "int" in fn_info.rtype:
        default_ret = "0"
    elif "float" in fn_info.rtype:
        default_ret = "0.0"
    elif "bool" in fn_info.rtype:
        default_ret = "FALSE"
    else:
        default_ret = "NULL"

    # first add the stub function doing nothing and just reporting error
    ret = ("{0.rtype} {0.name}_stub ({0.args}) {{\n" +
           "    g_critical(\"The function '{0.name}' called, but not implemented!\");\n" +
           "    return {1};\n"
           "}}\n\n").format(fn_info, default_ret)

    # then add a variable holding a reference to the dynamically loaded function
    # (if any) initialized to the stub
    ret += "{0.rtype} (*_{0.name}) ({0.args}) = {0.name}_stub;\n\n".format(fn_info)

    # then add a documented function calling the dynamically loaded one via the
    # reference
    ret += ("{0.doc}{0.rtype} {0.name} ({0.args}) {{\n" +
            "    return _{0.name} ({1});\n" +
            "}}\n\n\n").format(fn_info, call_args_str)

    return ret

def get_includes_str(includes):
    if not includes:
        return ""

    ret = ""
    for include in includes:
        ret += "#include%s\n" % include
    ret += "\n"

    return ret

def get_funcs_list(fn_infos, module_name):
    ret = "static gchar* {0}_functions[] = {{\n".format(module_name)
    for info in fn_infos:
        ret += '    "{0.name}",\n'.format(info)
    ret += "    NULL};\n\n"

    return ret

def get_loading_func(fn_infos, module_name):
    # TODO: only error on functions provided by the plugin that fail to load
    # TODO: implement the 'gchar **errors' argument
    ret =  'gboolean load_{0}_from_plugin(gchar *so_name) {{\n'.format(module_name)
    ret += '    void *handle = NULL;\n'
    ret += '    char *error = NULL;\n\n'

    ret += '    handle = dlopen(so_name, RTLD_LAZY);\n'
    ret += '    if (!handle) {\n'
    ret += '        g_warning("failed to load module {0}: %s", dlerror());\n'.format(mod_name)
    ret += '        return FALSE;\n'
    ret += '    }\n\n'

    for info in fn_infos:
        # clear any previous error and load the function
        ret += '    dlerror();\n'
        ret += '    * (void**) (&_{0.name}) = dlsym(handle, "{0.name}");\n'.format(info)
        ret += '    if ((error = dlerror()) != NULL)\n'
        ret += '        g_warning("failed to load {0.name}: %s", error);\n\n'.format(info)

    ret += '    return TRUE;\n'
    ret += '}\n\n'

    return ret

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Needs a file name, exitting.", file=sys.stderr)
        print("Usage: %s FILE_NAME", sys.argv[0])
        sys.exit(1)

    file_name = os.path.basename(sys.argv[1])
    mod_name, dot, _ext = file_name.partition(".")
    if not dot:
        print("Invalid file given, needs to be in MODNAME.[ch] format", file=sys.stderr)
        sys.exit(2)

    includes, fn_infos = process_file(open(sys.argv[1], "r"))
    print(get_funcs_list(fn_infos, mod_name), end='')
    for info in fn_infos:
        print(get_func_boilerplate(info), end='')
    print(get_loading_func(fn_infos, mod_name), end='')
<commit_msg>Swap conditions in the boilerplate generator so that skipping works<commit_after>#!/usr/bin/python3

"""
Simple helper script generating boilerplate code for a set of functions in order
to make them dynamically loaded and GObject-introspectable.

"""

from collections import namedtuple
import re
import sys
import os

FuncInfo = namedtuple("FuncInfo", ["name", "doc", "rtype", "args"])
FUNC_SPEC_RE = re.compile(r'(?P<rtype>\**\s*\w+\s*\**)'
                          r'\s*'
                          r'(?P<name>\w+)'
                          r'\s*\('
                          r'(?P<args>[\w*\s,]*)'
                          r'\);')

def gather_func_info(line_iter, includes):
    name = doc = rtype = args = ""
    in_doc = False
    in_skip = False
    for line in line_iter:
        if in_skip and line.strip().startswith("/* BpG-skip-end"):
            in_skip = False
        elif in_skip or line.strip().startswith("/* BpG-skip"):
            in_skip = True
        elif line.strip() == "" or line.strip().startswith("//") or line.strip().startswith("/* "):
            # whitespace line or ignored comment, skip it
            continue
        elif line.strip().startswith("#include"):
            includes.append(line.strip()[8:])
        elif line.strip().startswith("/**"):
            doc += line
            in_doc = True
        elif line.strip().endswith("*/") and in_doc:
            doc += line
            in_doc = False
        else:
            match = FUNC_SPEC_RE.match(line.strip())
            if match:
                fields = match.groupdict()
                name = fields["name"]
                rtype = fields["rtype"]
                args = fields["args"]

                # nothing more for this function
                break
            elif in_doc:
                doc += line
            else:
                msg = "Skipping unknown line: '{0}'".format(line.rstrip("\n"))
                print(msg, file=sys.stderr)
    else:
        return None

    return FuncInfo(name, doc, rtype, args)

def process_file(fobj):
    includes = []

    line_iter = iter(fobj)
    fn_infos = list()

    fn_info = gather_func_info(line_iter, includes)
    while fn_info:
        fn_infos.append(fn_info)
        fn_info = gather_func_info(line_iter, includes)

    return includes, fn_infos

def get_arg_names(args):
    if not args.strip():
        return []

    typed_args = args.split(",")
    starred_names = (typed_arg.split()[-1] for typed_arg in typed_args)

    return [starred_name.strip("* ") for starred_name in starred_names]

def get_func_boilerplate(fn_info):
    call_args_str = ", ".join(get_arg_names(fn_info.args))

    if "int" in fn_info.rtype:
        default_ret = "0"
    elif "float" in fn_info.rtype:
        default_ret = "0.0"
    elif "bool" in fn_info.rtype:
        default_ret = "FALSE"
    else:
        default_ret = "NULL"

    # first add the stub function doing nothing and just reporting error
    ret = ("{0.rtype} {0.name}_stub ({0.args}) {{\n" +
           "    g_critical(\"The function '{0.name}' called, but not implemented!\");\n" +
           "    return {1};\n"
           "}}\n\n").format(fn_info, default_ret)

    # then add a variable holding a reference to the dynamically loaded function
    # (if any) initialized to the stub
    ret += "{0.rtype} (*_{0.name}) ({0.args}) = {0.name}_stub;\n\n".format(fn_info)

    # then add a documented function calling the dynamically loaded one via the
    # reference
    ret += ("{0.doc}{0.rtype} {0.name} ({0.args}) {{\n" +
            "    return _{0.name} ({1});\n" +
            "}}\n\n\n").format(fn_info, call_args_str)

    return ret

def get_includes_str(includes):
    if not includes:
        return ""

    ret = ""
    for include in includes:
        ret += "#include%s\n" % include
    ret += "\n"

    return ret

def get_funcs_list(fn_infos, module_name):
    ret = "static gchar* {0}_functions[] = {{\n".format(module_name)
    for info in fn_infos:
        ret += '    "{0.name}",\n'.format(info)
    ret += "    NULL};\n\n"

    return ret

def get_loading_func(fn_infos, module_name):
    # TODO: only error on functions provided by the plugin that fail to load
    # TODO: implement the 'gchar **errors' argument
    ret =  'gboolean load_{0}_from_plugin(gchar *so_name) {{\n'.format(module_name)
    ret += '    void *handle = NULL;\n'
    ret += '    char *error = NULL;\n\n'

    ret += '    handle = dlopen(so_name, RTLD_LAZY);\n'
    ret += '    if (!handle) {\n'
    ret += '        g_warning("failed to load module {0}: %s", dlerror());\n'.format(mod_name)
    ret += '        return FALSE;\n'
    ret += '    }\n\n'

    for info in fn_infos:
        # clear any previous error and load the function
        ret += '    dlerror();\n'
        ret += '    * (void**) (&_{0.name}) = dlsym(handle, "{0.name}");\n'.format(info)
        ret += '    if ((error = dlerror()) != NULL)\n'
        ret += '        g_warning("failed to load {0.name}: %s", error);\n\n'.format(info)

    ret += '    return TRUE;\n'
    ret += '}\n\n'

    return ret

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("Needs a file name, exitting.", file=sys.stderr)
        print("Usage: %s FILE_NAME", sys.argv[0])
        sys.exit(1)

    file_name = os.path.basename(sys.argv[1])
    mod_name, dot, _ext = file_name.partition(".")
    if not dot:
        print("Invalid file given, needs to be in MODNAME.[ch] format", file=sys.stderr)
        sys.exit(2)

    includes, fn_infos = process_file(open(sys.argv[1], "r"))
    print(get_funcs_list(fn_infos, mod_name), end='')
    for info in fn_infos:
        print(get_func_boilerplate(info), end='')
    print(get_loading_func(fn_infos, mod_name), end='')
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: latin-1 -*-
#
# Copyright 2009-2013 Ghent University
#
# This file is part of vsc-base,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
# with support of Ghent University (http://ugent.be/hpc),
# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en),
# the Hercules foundation (http://www.herculesstichting.be/in_English)
# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
#
# http://github.com/hpcugent/vsc-base
#
# vsc-base is free software: you can redistribute it and/or modify
# it under the terms of the GNU Library General Public License as
# published by the Free Software Foundation, either version 2 of
# the License, or (at your option) any later version.
#
# vsc-base is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public License
# along with vsc-base. If not, see <http://www.gnu.org/licenses/>.
#
"""
Shared module for vsc-base setup

@author: Stijn De Weirdt (Ghent University)
@author: Andy Georges (Ghent University)
"""
import glob
import os
import shutil
import sys
from distutils import log  # also for setuptools
from distutils.dir_util import remove_tree

# 0 : WARN (default), 1 : INFO, 2 : DEBUG
log.set_verbosity(2)

has_setuptools = None

EXTRA_SDIST_FILES = ['shared_setup.py', 'setup.py']

def find_extra_sdist_files():
    filelist = []
    for fn in EXTRA_SDIST_FILES:
        if os.path.isfile(fn):
            filelist.append(fn)
        else:
            print "sdist add_defaults Failed to find %s" % fn
            sys.exit(1)
    log.info("find_extra_sdist_files: %s" % filelist)
    return filelist

try:
    # setuptools makes copies of the scripts, does not preserve symlinks
    # raise("no setuptools")  # to try distutils, uncomment
    from setuptools import setup
    from setuptools.command.install_scripts import install_scripts
    from setuptools.command.build_py import build_py
    from setuptools.command.sdist import sdist
    # egg_info uses sdist directly through manifest_maker
    from setuptools.command.egg_info import egg_info
    class vsc_egg_info(egg_info):
        def find_sources(self):
            egg_info.find_sources(self)
            self.filelist.extend(find_extra_sdist_files())
    has_setuptools = True
except:
    from distutils.core import setup
    from distutils.command.install_scripts import install_scripts
    from distutils.command.build_py import build_py
    from distutils.command.sdist import sdist

    class vsc_egg_info(object):
        pass  # dummy class for distutils

    has_setuptools = False

# authors
ag = ('Andy Georges', 'andy.georges@ugent.be')
jt = ('Jens Timmermans', 'jens.timmermans@ugent.be')
kh = ('Kenneth Hoste', 'kenneth.hoste@ugent.be')
lm = ('Luis Fernando Munoz Mejias', 'luis.munoz@ugent.be')
sdw = ('Stijn De Weirdt', 'stijn.deweirdt@ugent.be')
wdp = ('Wouter Depypere', 'wouter.depypere@ugent.be')

class vsc_install_scripts(install_scripts):
    """Create the (fake) links for mympirun
        also remove .sh and .py extensions from the scripts
    """
    def __init__(self, *args):
        install_scripts.__init__(self, *args)
        self.original_outfiles = None

    def run(self):
        # old-style class
        install_scripts.run(self)

        self.original_outfiles = self.get_outputs()[:]  # make a copy
        self.outfiles = []  # reset it
        for script in self.original_outfiles:
            # remove suffixes for .py and .sh
            if script.endswith(".py") or script.endswith(".sh"):
                shutil.move(script, script[:-3])
                script = script[:-3]
            self.outfiles.append(script)

class vsc_build_py(build_py):
    def find_package_modules (self, package, package_dir):
        """Extend build_by (not used for now)"""
        result = build_py.find_package_modules(self, package, package_dir)
        return result

class vsc_sdist(sdist):
    def add_defaults(self):
        """Add shared_setup.py"""
        sdist.add_defaults()
        self.filelist.extend(find_extra_sdist_files())

# shared target config
SHARED_TARGET = {
    'url': 'http://hpcugent.github.com/vsc-base',
    'download_url': 'https://github.com/hpcugent/vsc-base',
    'package_dir': {'': 'lib'},
    'cmdclass': {
        "install_scripts": vsc_install_scripts,
        # "build_py":vsc_build_py,
        "sdist":vsc_sdist,
        "egg_info":vsc_egg_info,
    },
}

def cleanup(prefix=''):
    dirs=[prefix+'build']+glob.glob(prefix + 'lib/*.egg-info')
    for d in dirs:
        if os.path.isdir(d):
            log.warn("cleanup %s" % d)
            try:
                remove_tree(d, verbose=False)
            except OSError, _:
                log.error("cleanup failed for %s" % d)

    for fn in ('setup.cfg',):
        ffn = prefix + fn
        if os.path.isfile(ffn):
            os.remove(ffn)

def make_setup(name='base',prefix=''):
    """Create the setup.py
        - default is base
    """
    fn = '%ssetup_%s.py' % (prefix, name)
    if os.path.isfile(fn):
        shutil.copyfile(fn, 'setup.py')
    else:
        log.error("setup file %s for name %s not found" % (fn, name))

def sanitize(v):
    """Transforms v into a sensible string for use in setup.cfg."""
    if isinstance(v, str):
        return v

    if isinstance(v, list):
        return ",".join(v)

def parse_target(target):
    """Add some fields"""
    new_target = {}
    new_target.update(SHARED_TARGET)
    for k, v in target.items():
        if k in ('author', 'maintainer'):
            if not isinstance(v, list):
                log.error("%s of config %s needs to be a list (not tuple or string)" % (k, target['name']))
                sys.exit(1)
            new_target[k] = ";".join([x[0] for x in v])
            new_target["%s_email" % k] = ";".join([x[1] for x in v])
        else:
            if isinstance(v, dict):
                # eg command_class
                if not k in new_target:
                    new_target[k] = type(v)()
                new_target[k].update(v)
            else:
                new_target[k] = type(v)()
                new_target[k] += v
    return new_target

def build_setup_cfg_for_bdist_rpm(target):
    """Generates a setup.cfg on a per-target basis.

    Stores the 'install-requires' in the [bdist_rpm] section

    @type target: dict

    @param target: specifies the options to be passed to setup()
    """

    try:
        setup_cfg = open('setup.cfg', 'w')  # and truncate
    except (IOError, OSError), err:
        print "Cannot create setup.cfg for target %s: %s" % (target['name'], err)
        sys.exit(1)

    s = ["[bdist_rpm]"]
    if 'install_requires' in target:
        s += ["requires = %s" % (sanitize(target['install_requires']))]

    if 'provides' in target:
        s += ["provides = %s" % (sanitize((target['provides'])))]
        target.pop('provides')

    setup_cfg.write("\n".join(s) + "\n")
    setup_cfg.close()

def action_target(target, setupfn=setup, extra_sdist=[]):
    EXTRA_SDIST_FILES.extend(extra_sdist)
    name = '_'.join(target['name'].split('-')[1:])

    cleanup()

    build_setup_cfg_for_bdist_rpm(target)
    x = parse_target(target)
    setupfn(**x)
    cleanup()

if __name__ == '__main__':
    # print all supported packages
    all_setups = [x[len('setup_'):-len('.py')] for x in glob.glob('setup_*.py')]
    all_packages = ['-'.join(['vsc'] + x.split('_')) for x in all_setups]
    print " ".join(all_packages)
<commit_msg>Some patches failed, removing the cleanup to keep the setup.cfg<commit_after>#!/usr/bin/env python
# -*- coding: latin-1 -*-
#
# Copyright 2009-2013 Ghent University
#
# This file is part of vsc-base,
# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
# with support of Ghent University (http://ugent.be/hpc),
# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en),
# the Hercules foundation (http://www.herculesstichting.be/in_English)
# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
#
# http://github.com/hpcugent/vsc-base
#
# vsc-base is free software: you can redistribute it and/or modify
# it under the terms of the GNU Library General Public License as
# published by the Free Software Foundation, either version 2 of
# the License, or (at your option) any later version.
#
# vsc-base is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public License
# along with vsc-base. If not, see <http://www.gnu.org/licenses/>.
#
"""
Shared module for vsc-base setup

@author: Stijn De Weirdt (Ghent University)
@author: Andy Georges (Ghent University)
"""
import glob
import os
import shutil
import sys
from distutils import log  # also for setuptools
from distutils.dir_util import remove_tree

# 0 : WARN (default), 1 : INFO, 2 : DEBUG
log.set_verbosity(2)

has_setuptools = None

EXTRA_SDIST_FILES = ['shared_setup.py', 'setup.py']

def find_extra_sdist_files():
    filelist = []
    for fn in EXTRA_SDIST_FILES:
        if os.path.isfile(fn):
            filelist.append(fn)
        else:
            print "sdist add_defaults Failed to find %s" % fn
            sys.exit(1)
    log.info("find_extra_sdist_files: %s" % filelist)
    return filelist

try:
    # setuptools makes copies of the scripts, does not preserve symlinks
    # raise("no setuptools")  # to try distutils, uncomment
    from setuptools import setup
    from setuptools.command.install_scripts import install_scripts
    from setuptools.command.build_py import build_py
    from setuptools.command.sdist import sdist
    # egg_info uses sdist directly through manifest_maker
    from setuptools.command.egg_info import egg_info
    class vsc_egg_info(egg_info):
        def find_sources(self):
            egg_info.find_sources(self)
            self.filelist.extend(find_extra_sdist_files())
    has_setuptools = True
except:
    from distutils.core import setup
    from distutils.command.install_scripts import install_scripts
    from distutils.command.build_py import build_py
    from distutils.command.sdist import sdist

    class vsc_egg_info(object):
        pass  # dummy class for distutils

    has_setuptools = False

# authors
ag = ('Andy Georges', 'andy.georges@ugent.be')
jt = ('Jens Timmermans', 'jens.timmermans@ugent.be')
kh = ('Kenneth Hoste', 'kenneth.hoste@ugent.be')
lm = ('Luis Fernando Munoz Mejias', 'luis.munoz@ugent.be')
sdw = ('Stijn De Weirdt', 'stijn.deweirdt@ugent.be')
wdp = ('Wouter Depypere', 'wouter.depypere@ugent.be')

class vsc_install_scripts(install_scripts):
    """Create the (fake) links for mympirun
        also remove .sh and .py extensions from the scripts
    """
    def __init__(self, *args):
        install_scripts.__init__(self, *args)
        self.original_outfiles = None

    def run(self):
        # old-style class
        install_scripts.run(self)

        self.original_outfiles = self.get_outputs()[:]  # make a copy
        self.outfiles = []  # reset it
        for script in self.original_outfiles:
            # remove suffixes for .py and .sh
            if script.endswith(".py") or script.endswith(".sh"):
                shutil.move(script, script[:-3])
                script = script[:-3]
            self.outfiles.append(script)

class vsc_build_py(build_py):
    def find_package_modules (self, package, package_dir):
        """Extend build_by (not used for now)"""
        result = build_py.find_package_modules(self, package, package_dir)
        return result

class vsc_sdist(sdist):
    def add_defaults(self):
        """Add shared_setup.py"""
        sdist.add_defaults()
        self.filelist.extend(find_extra_sdist_files())

# shared target config
SHARED_TARGET = {
    'url': 'http://hpcugent.github.com/vsc-base',
    'download_url': 'https://github.com/hpcugent/vsc-base',
    'package_dir': {'': 'lib'},
    'cmdclass': {
        "install_scripts": vsc_install_scripts,
        # "build_py":vsc_build_py,
        "sdist":vsc_sdist,
        "egg_info":vsc_egg_info,
    },
}

def cleanup(prefix=''):
    dirs=[prefix+'build']+glob.glob(prefix + 'lib/*.egg-info')
    for d in dirs:
        if os.path.isdir(d):
            log.warn("cleanup %s" % d)
            try:
                remove_tree(d, verbose=False)
            except OSError, _:
                log.error("cleanup failed for %s" % d)

    for fn in ('setup.cfg',):
        ffn = prefix + fn
        if os.path.isfile(ffn):
            os.remove(ffn)

def make_setup(name='base',prefix=''):
    """Create the setup.py
        - default is base
    """
    fn = '%ssetup_%s.py' % (prefix, name)
    if os.path.isfile(fn):
        shutil.copyfile(fn, 'setup.py')
    else:
        log.error("setup file %s for name %s not found" % (fn, name))

def sanitize(v):
    """Transforms v into a sensible string for use in setup.cfg."""
    if isinstance(v, str):
        return v

    if isinstance(v, list):
        return ",".join(v)

def parse_target(target):
    """Add some fields"""
    new_target = {}
    new_target.update(SHARED_TARGET)
    for k, v in target.items():
        if k in ('author', 'maintainer'):
            if not isinstance(v, list):
                log.error("%s of config %s needs to be a list (not tuple or string)" % (k, target['name']))
                sys.exit(1)
            new_target[k] = ";".join([x[0] for x in v])
            new_target["%s_email" % k] = ";".join([x[1] for x in v])
        else:
            if isinstance(v, dict):
                # eg command_class
                if not k in new_target:
                    new_target[k] = type(v)()
                new_target[k].update(v)
            else:
                new_target[k] = type(v)()
                new_target[k] += v
    return new_target

def build_setup_cfg_for_bdist_rpm(target):
    """Generates a setup.cfg on a per-target basis.

    Stores the 'install-requires' in the [bdist_rpm] section

    @type target: dict

    @param target: specifies the options to be passed to setup()
    """

    try:
        setup_cfg = open('setup.cfg', 'w')  # and truncate
    except (IOError, OSError), err:
        print "Cannot create setup.cfg for target %s: %s" % (target['name'], err)
        sys.exit(1)

    s = ["[bdist_rpm]"]
    if 'install_requires' in target:
        s += ["requires = %s" % (sanitize(target['install_requires']))]

    if 'provides' in target:
        s += ["provides = %s" % (sanitize((target['provides'])))]
        target.pop('provides')

    setup_cfg.write("\n".join(s) + "\n")
    setup_cfg.close()

def action_target(target, setupfn=setup, extra_sdist=[]):
    EXTRA_SDIST_FILES.extend(extra_sdist)
    name = '_'.join(target['name'].split('-')[1:])

    cleanup()

    build_setup_cfg_for_bdist_rpm(target)
    x = parse_target(target)
    setupfn(**x)

if __name__ == '__main__':
    # print all supported packages
    all_setups = [x[len('setup_'):-len('.py')] for x in glob.glob('setup_*.py')]
    all_packages = ['-'.join(['vsc'] + x.split('_')) for x in all_setups]
    print " ".join(all_packages)
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# standard library
import datetime as dt
import json
import logging
import os
import os.path
import re
from random import randint
from sys import path
from time import sleep
# third party
import click
import markovify
import nltk

from hangoutsclient import HangoutsClient

# Inspired by: http://hirelofty.com/blog/how-build-slack-bot-mimics-your-colleague/

# Get absolute path of the dir script is run from
CWD = path[0]  # pylint: disable=C0103

def load_corpus_text(corpus_file):
    """
    Reads in Hangouts chat log data from text file or JSON file.
    Returns a list of all the messages in the file.
    """

    if corpus_file.endswith('.txt'):
        with open(corpus_file, 'r', encoding='utf-8') as f:
            text = f.read()
    elif corpus_file.endswith('.json'):
        with open(corpus_file, 'r') as f:
            messages = json.loads(f.read())
        text = ''.join(messages.values())
    else:
        text = None

    return text

def load_model_json(model_file):
    if os.path.isfile(model_file):
        logging.info('Loading model from file.')
        with open(model_file, 'r') as json_file:
            markov_json = json.load(json_file)
    else:
        # file does not exist
        markov_json = None

    return markov_json

def build_text_model(state_size, use_nltk, corpus_filepath, model_filepath):
    """
    Build a new Markov chain generator model.
    Returns a markovify Text instance.
    """
    markov_json = load_model_json(model_filepath)

    logging.debug('Loading corpus from %s.', corpus_filepath)
    corpus = load_corpus_text(corpus_filepath)
    logging.debug('Creating text model with state size %s', state_size)
    if use_nltk:
        logging.debug('Using nltk')
        nltk.data.path.append(os.path.join(CWD, 'nltk_data'))
        text_model = POSifiedText.from_json(markov_json)
    elif markov_json:
        logging.debug('Using existing chain file from %s.', model_filepath)
        text_model = markovify.Text.from_dict(markov_json)
    else:
        logging.debug('Creating new chain file.')
        text_model = markovify.Text(corpus, state_size=state_size, chain=None)
        # save our newly created Markov chain for the next time script is run
        with open(os.path.join(CWD, 'markov_chain.json'), 'w') as json_file:
            json_file.write(text_model.to_json())

    return text_model

@click.command()
@click.option('--delay', '-d', default=-1, help='delay (in secs) before script enters main subroutine. -1 for random delay.')
@click.option('--chars', '-c', default=140, help='max character length for the generated message.')
@click.option('--state_size', '-s', default=2, help='state size for Markov model.')
@click.option('--natural', '-n', default=0, help='use ntlk (much slower than standard Markov).')
def main(delay, chars, state_size, natural):
    """
    Login to Hangouts, send generated message and disconnect.
    """
    # Path to config file
    config_path = os.path.join(CWD, 'wynbot.ini')
    logging.debug('Using config file: %s', config_path)

    if delay == -1:
        # Sleep random amount of time so messages are sent at a different time everyday
        delay = randint(1, 8 * 60 * 60)  # range of 1s to 8 hours

    delay_date = dt.datetime.now() + dt.timedelta(seconds=delay)
    logging.info('Sleeping for %s seconds, continue at %s', delay, delay_date.strftime("%Y/%m/%d %H:%M:%S"))
    sleep(delay)

    # Build the text model using markovify
    corpus_file = os.path.join(CWD, 'corpus.txt')
    chain_file = os.path.join(CWD, 'markov_chain.json')
    text_model = build_text_model(state_size, natural, corpus_file, chain_file)
    logging.debug('Starting message generation. Max. chars: %s', chars)
    message = text_model.make_short_sentence(chars) or "failed to generate message"
    logging.info('Generated message (%s chars): "%s"', len(message), message)

    # Setup Hangouts bot instance
    hangouts = HangoutsClient(config_path, message)

    # Connect to Hangouts and start processing XMPP stanzas.
    if hangouts.connect(address=('talk.google.com', 5222),
                        reattempt=True, use_tls=True):
        hangouts.process(block=True)
        logging.info("Finished sending today's message.")
    else:
        logging.error('Unable to connect to Hangouts.')

def configure_logging():
    # Configure root logger. Level 5 = verbose to catch mostly everything.
    logger = logging.getLogger()
    logger.setLevel(level=5)

    log_folder = os.path.join(CWD, 'logs')
    if not os.path.exists(log_folder):
        os.makedirs(log_folder, exist_ok=True)

    log_filename = 'wynbot_{0}.log'.format(dt.datetime.now().strftime('%Y%m%d_%Hh%Mm%Ss'))
    log_filepath = os.path.join(log_folder, log_filename)
    log_handler = logging.FileHandler(log_filepath)

    log_format = logging.Formatter(
        fmt='%(asctime)s.%(msecs).03d %(name)-12s %(levelname)-8s %(message)s (%(filename)s:%(lineno)d)',
        datefmt='%Y-%m-%d %H:%M:%S')
    log_handler.setFormatter(log_format)
    logger.addHandler(log_handler)
    # Lower requests module's log level so that OAUTH2 details aren't logged
    logging.getLogger('requests').setLevel(logging.WARNING)
    # Quieten SleekXMPP output
    # logging.getLogger('sleekxmpp.xmlstream.xmlstream').setLevel(logging.INFO)

class POSifiedText(markovify.Text):
    def word_split(self, sentence):
        words = re.split(self.word_split_pattern, sentence)
        words = ['::'.join(tag) for tag in nltk.pos_tag(words)]
        return words

    def word_join(self, words):
        sentence = ' '.join(word.split('::')[0] for word in words)
        return sentence

if __name__ == '__main__':
    configure_logging()
    main()
<commit_msg>readd check for state size when loading model from disk<commit_after>#!/usr/bin/env python3

# standard library
import datetime as dt
import json
import logging
import os
import os.path
import re
from random import randint
from sys import path
from time import sleep
# third party
import click
import markovify
import nltk

from hangoutsclient import HangoutsClient

# Inspired by: http://hirelofty.com/blog/how-build-slack-bot-mimics-your-colleague/

# Get absolute path of the dir script is run from
CWD = path[0]  # pylint: disable=C0103

def load_corpus_text(corpus_file):
    """
    Reads in Hangouts chat log data from text file or JSON file.
    Returns a list of all the messages in the file.
    """

    if corpus_file.endswith('.txt'):
        with open(corpus_file, 'r', encoding='utf-8') as f:
            text = f.read()
    elif corpus_file.endswith('.json'):
        with open(corpus_file, 'r') as f:
            messages = json.loads(f.read())
        text = ''.join(messages.values())
    else:
        text = None

    return text

def load_model_json(model_file):
    if os.path.isfile(model_file):
        logging.info('Loading model from file.')
        with open(model_file, 'r') as json_file:
            markov_json = json.load(json_file)
    else:
        # file does not exist
        markov_json = None

    return markov_json

def build_text_model(state_size, use_nltk, corpus_filepath, model_filepath):
    """
    Build a new Markov chain generator model.
    Returns a markovify Text instance.
    """
    markov_json = load_model_json(model_filepath)

    logging.debug('Loading corpus from %s.', corpus_filepath)
    corpus = load_corpus_text(corpus_filepath)
    logging.debug('Creating text model with state size %s', state_size)
    if use_nltk:
        logging.debug('Using nltk')
        nltk.data.path.append(os.path.join(CWD, 'nltk_data'))
        text_model = POSifiedText.from_json(markov_json)
    elif markov_json and state_size == markov_json["state_s＃＃ize"]:
        logging.debug('Using existing chain file from %s.', model_filepath)
        text_model = markovify.Text.from_dict(markov_json)
    elif markov_json and state_size != markov_json["state_size"]:
        logging.debug('Existing chain file is of state size %s, where as requested is %s', state_size, markov_json["state_size"])
        logging.debug('Creating new chain file.')
        TODO: refactor
        text_model = markovify.Text(corpus, state_size=state_size, chain=None)
        # save our newly created Markov chain for the next time script is run
        with open(os.path.join(CWD, 'markov_chain.json'), 'w') as json_file:
            json_file.write(text_model.to_json())
    else:
        logging.debug('Creating new chain file.')
        text_model = markovify.Text(corpus, state_size=state_size, chain=None)
        # save our newly created Markov chain for the next time script is run
        with open(os.path.join(CWD, 'markov_chain.json'), 'w') as json_file:
            json_file.write(text_model.to_json())

    return text_model

@click.command()
@click.option('--delay', '-d', default=-1, help='delay (in secs) before script enters main subroutine. -1 for random delay.')
@click.option('--chars', '-c', default=140, help='max character length for the generated message.')
@click.option('--state_size', '-s', default=2, help='state size for Markov model.')
@click.option('--natural', '-n', default=0, help='use ntlk (much slower than standard Markov).')
def main(delay, chars, state_size, natural):
    """
    Login to Hangouts, send generated message and disconnect.
    """
    # Path to config file
    config_path = os.path.join(CWD, 'wynbot.ini')
    logging.debug('Using config file: %s', config_path)

    if delay == -1:
        # Sleep random amount of time so messages are sent at a different time everyday
        delay = randint(1, 8 * 60 * 60)  # range of 1s to 8 hours

    delay_date = dt.datetime.now() + dt.timedelta(seconds=delay)
    logging.info('Sleeping for %s seconds, continue at %s', delay, delay_date.strftime("%Y/%m/%d %H:%M:%S"))
    sleep(delay)

    # Build the text model using markovify
    corpus_file = os.path.join(CWD, 'corpus.txt')
    chain_file = os.path.join(CWD, 'markov_chain.json')
    text_model = build_text_model(state_size, natural, corpus_file, chain_file)
    logging.debug('Starting message generation. Max. chars: %s', chars)
    message = text_model.make_short_sentence(chars) or "failed to generate message"
    logging.info('Generated message (%s chars): "%s"', len(message), message)

    # Setup Hangouts bot instance
    hangouts = HangoutsClient(config_path, message)

    # Connect to Hangouts and start processing XMPP stanzas.
    if hangouts.connect(address=('talk.google.com', 5222),
                        reattempt=True, use_tls=True):
        hangouts.process(block=True)
        logging.info("Finished sending today's message.")
    else:
        logging.error('Unable to connect to Hangouts.')

def configure_logging():
    # Configure root logger. Level 5 = verbose to catch mostly everything.
    logger = logging.getLogger()
    logger.setLevel(level=5)

    log_folder = os.path.join(CWD, 'logs')
    if not os.path.exists(log_folder):
        os.makedirs(log_folder, exist_ok=True)

    log_filename = 'wynbot_{0}.log'.format(dt.datetime.now().strftime('%Y%m%d_%Hh%Mm%Ss'))
    log_filepath = os.path.join(log_folder, log_filename)
    log_handler = logging.FileHandler(log_filepath)

    log_format = logging.Formatter(
        fmt='%(asctime)s.%(msecs).03d %(name)-12s %(levelname)-8s %(message)s (%(filename)s:%(lineno)d)',
        datefmt='%Y-%m-%d %H:%M:%S')
    log_handler.setFormatter(log_format)
    logger.addHandler(log_handler)
    # Lower requests module's log level so that OAUTH2 details aren't logged
    logging.getLogger('requests').setLevel(logging.WARNING)
    # Quieten SleekXMPP output
    # logging.getLogger('sleekxmpp.xmlstream.xmlstream').setLevel(logging.INFO)

class POSifiedText(markovify.Text):
    def word_split(self, sentence):
        words = re.split(self.word_split_pattern, sentence)
        words = ['::'.join(tag) for tag in nltk.pos_tag(words)]
        return words

    def word_join(self, words):
        sentence = ' '.join(word.split('::')[0] for word in words)
        return sentence

if __name__ == '__main__':
    configure_logging()
    main()
<|endoftext|>


<commit_before>"""A module containing validation functionality.

Warning:
    It is planned to change from Schema-based to Data-based Codelist validation. As such, this module will change significantly.
"""

from lxml import etree
import iati.core.default

def _correct_codes(dataset, codelist, error_log=False):
    """Determine whether a given Dataset has values from the specified Codelist where expected.

    Args:
        dataset (iati.core.data.Dataset): The Dataset to check Codelist values within.
        codelist (iati.core.codelists.Codelist): The Codelist to check values from.
        error_log (bool): Whether to return a detailed error log, or merely a boolean value. Default False.

    Returns:
        bool: If `error_log` is False. A boolean indicating whether the given Dataset has values from the specified Codelist where they should be.
        list of dict: If `error_log` is True. A list of the errors that occurred.

    """
    errors = []
    mappings = iati.core.default.codelist_mapping()

    if not error_log and not codelist.complete:
        return True

    for mapping in mappings[codelist.name]:
        base_xpath = mapping['xpath']
        condition = mapping['condition']
        split_xpath = base_xpath.split('/')
        parent_el_xpath = '/'.join(split_xpath[:-1])
        attr_name = split_xpath[-1:][0][1:]

        if condition is None:
            parent_el_xpath = parent_el_xpath + '[@' + attr_name + ']'
        else:
            parent_el_xpath = parent_el_xpath + '[' + condition + ' and @' + attr_name + ']'

        parents_to_check = dataset.xml_tree.xpath(parent_el_xpath)

        for parent in parents_to_check:
            code = parent.attrib[attr_name]

            if code not in codelist.codes:
                if error_log:
                    errors.append({
                        'actual_value': code,
                        'category': 'codelist',
                        'element': parent,
                        'expected_value': codelist,
                        'line_number': parent.sourceline,
                        'status': 'error' if codelist.complete else 'warning',
                        'type': 'code not on codelist'
                    })
                else:
                    return False

    if error_log:
        return errors
    else:
        return True

def _correct_codelist_values(dataset, schema, error_log=False):
    """Determine whether a given Dataset has values from Codelists that have been added to a Schema where expected.

    Args:
        dataset (iati.core.data.Dataset): The Dataset to check Codelist values within.
        schema (iati.core.schemas.Schema): The Schema to locate Codelists within.
        error_log (bool): Whether to return a detailed error log, or merely a boolean value. Default False.

    Returns:
        bool: If `error_log` is False. A boolean indicating whether the given Dataset has values from the specified Codelists where they should be.
        list of dict: If `error_log` is True. A list of the errors that occurred.

    """
    errors = []

    for codelist in schema.codelists:
        if error_log:
            errors = errors + _correct_codes(dataset, codelist, error_log)
        else:
            correct_for_codelist = _correct_codes(dataset, codelist)
            if not correct_for_codelist:
                return False

    if error_log:
        return errors
    else:
        return True

def full_validation(dataset, schema):
    """Perform full validation on a Dataset.

    Args:
        dataset (iati.core.Dataset): The Dataset to check validity of.
        schema (iati.core.Schema): The Schema to validate the Dataset against.

    Warning:
        Parameters are likely to change in some manner.

    Returns:
        list of dict: A list of dictionaries containing error output. An empty list indicates that there are no errors.

    Todo:
        Create test against a bad Schema.

    """
    return _correct_codelist_values(dataset, schema, True)

def is_iati_xml(dataset, schema):
    """Determine whether a given Dataset's XML is valid against the specified Schema.

    Args:
        dataset (iati.core.data.Dataset): The Dataset to check validity of.
        schema (iati.core.schemas.Schema): The Schema to validate the Dataset against.

    Warning:
        Parameters are likely to change in some manner.

    Returns:
        bool: A boolean indicating whether the given Dataset is valid XML against the given Schema.

    Raises:
        iati.core.exceptions.SchemaError: An error occurred in the parsing of the Schema.

    Todo:
        Create test against a bad Schema.

    """
    try:
        validator = schema.validator()
    except iati.core.exceptions.SchemaError as err:
        raise err

    try:
        validator.assertValid(dataset.xml_tree)
    except etree.DocumentInvalid:
        return False

    return True

def is_valid(dataset, schema):
    """Determine whether a given Dataset is valid against the specified Schema.

    Args:
        dataset (iati.core.Dataset): The Dataset to check validity of.
        schema (iati.core.Schema): The Schema to validate the Dataset against.

    Warning:
        Parameters are likely to change in some manner.

    Returns:
        bool: A boolean indicating whether the given Dataset is valid against the given Schema.

    Todo:
        Create test against a bad Schema.

    """
    try:
        iati_xml = is_iati_xml(dataset, schema)
        if not iati_xml:
            return False
    except iati.core.exceptions.SchemaError:
        return False

    return _correct_codelist_values(dataset, schema)

def is_xml(maybe_xml):
    """Determine whether a given parameter is XML.

    Args:
        maybe_xml (str): An string that may or may not contain valid XML.

    Returns:
        bool: A boolean indicating whether the given Dataset is valid XML.

    """
    if isinstance(maybe_xml, iati.core.data.Dataset):
        maybe_xml = maybe_xml.xml_str

    try:
        _ = etree.fromstring(maybe_xml.strip())
        return True
    except (etree.XMLSyntaxError, AttributeError, TypeError, ValueError):
        return False

<commit_msg>Provide better example of what error messages may look like<commit_after>"""A module containing validation functionality.

Warning:
    It is planned to change from Schema-based to Data-based Codelist validation. As such, this module will change significantly.
"""

from lxml import etree
import iati.core.default

def _correct_codes(dataset, codelist, error_log=False):
    """Determine whether a given Dataset has values from the specified Codelist where expected.

    Args:
        dataset (iati.core.data.Dataset): The Dataset to check Codelist values within.
        codelist (iati.core.codelists.Codelist): The Codelist to check values from.
        error_log (bool): Whether to return a detailed error log, or merely a boolean value. Default False.

    Returns:
        bool: If `error_log` is False. A boolean indicating whether the given Dataset has values from the specified Codelist where they should be.
        list of dict: If `error_log` is True. A list of the errors that occurred.

    """
    errors = []
    mappings = iati.core.default.codelist_mapping()

    if not error_log and not codelist.complete:
        return True

    for mapping in mappings[codelist.name]:
        base_xpath = mapping['xpath']
        condition = mapping['condition']
        split_xpath = base_xpath.split('/')
        parent_el_xpath = '/'.join(split_xpath[:-1])
        attr_name = split_xpath[-1:][0][1:]

        if condition is None:
            parent_el_xpath = parent_el_xpath + '[@' + attr_name + ']'
        else:
            parent_el_xpath = parent_el_xpath + '[' + condition + ' and @' + attr_name + ']'

        parents_to_check = dataset.xml_tree.xpath(parent_el_xpath)

        for parent in parents_to_check:
            code = parent.attrib[attr_name]

            if code not in codelist.codes:
                if error_log:
                    errors.append({
                        'actual_value': code,
                        'category': 'codelist',
                        'element': parent,
                        'error_info': '{0} is not a valid Code on the {1} Codelist',
                        'error_name': 'code-not-on-codelist',
                        'expected_value': codelist,
                        'help': 'The {0} attribute must have a value on the {1} Codelist. See http://iatistandard.org/202/codelists/{1} for valid values.',
                        'line_number': parent.sourceline,
                        'status': 'error' if codelist.complete else 'warning'
                    })
                else:
                    return False

    if error_log:
        return errors
    else:
        return True

def _correct_codelist_values(dataset, schema, error_log=False):
    """Determine whether a given Dataset has values from Codelists that have been added to a Schema where expected.

    Args:
        dataset (iati.core.data.Dataset): The Dataset to check Codelist values within.
        schema (iati.core.schemas.Schema): The Schema to locate Codelists within.
        error_log (bool): Whether to return a detailed error log, or merely a boolean value. Default False.

    Returns:
        bool: If `error_log` is False. A boolean indicating whether the given Dataset has values from the specified Codelists where they should be.
        list of dict: If `error_log` is True. A list of the errors that occurred.

    """
    errors = []

    for codelist in schema.codelists:
        if error_log:
            errors = errors + _correct_codes(dataset, codelist, error_log)
        else:
            correct_for_codelist = _correct_codes(dataset, codelist)
            if not correct_for_codelist:
                return False

    if error_log:
        return errors
    else:
        return True

def full_validation(dataset, schema):
    """Perform full validation on a Dataset.

    Args:
        dataset (iati.core.Dataset): The Dataset to check validity of.
        schema (iati.core.Schema): The Schema to validate the Dataset against.

    Warning:
        Parameters are likely to change in some manner.

    Returns:
        list of dict: A list of dictionaries containing error output. An empty list indicates that there are no errors.

    Todo:
        Create test against a bad Schema.

    """
    return _correct_codelist_values(dataset, schema, True)

def is_iati_xml(dataset, schema):
    """Determine whether a given Dataset's XML is valid against the specified Schema.

    Args:
        dataset (iati.core.data.Dataset): The Dataset to check validity of.
        schema (iati.core.schemas.Schema): The Schema to validate the Dataset against.

    Warning:
        Parameters are likely to change in some manner.

    Returns:
        bool: A boolean indicating whether the given Dataset is valid XML against the given Schema.

    Raises:
        iati.core.exceptions.SchemaError: An error occurred in the parsing of the Schema.

    Todo:
        Create test against a bad Schema.

    """
    try:
        validator = schema.validator()
    except iati.core.exceptions.SchemaError as err:
        raise err

    try:
        validator.assertValid(dataset.xml_tree)
    except etree.DocumentInvalid:
        return False

    return True

def is_valid(dataset, schema):
    """Determine whether a given Dataset is valid against the specified Schema.

    Args:
        dataset (iati.core.Dataset): The Dataset to check validity of.
        schema (iati.core.Schema): The Schema to validate the Dataset against.

    Warning:
        Parameters are likely to change in some manner.

    Returns:
        bool: A boolean indicating whether the given Dataset is valid against the given Schema.

    Todo:
        Create test against a bad Schema.

    """
    try:
        iati_xml = is_iati_xml(dataset, schema)
        if not iati_xml:
            return False
    except iati.core.exceptions.SchemaError:
        return False

    return _correct_codelist_values(dataset, schema)

def is_xml(maybe_xml):
    """Determine whether a given parameter is XML.

    Args:
        maybe_xml (str): An string that may or may not contain valid XML.

    Returns:
        bool: A boolean indicating whether the given Dataset is valid XML.

    """
    if isinstance(maybe_xml, iati.core.data.Dataset):
        maybe_xml = maybe_xml.xml_str

    try:
        _ = etree.fromstring(maybe_xml.strip())
        return True
    except (etree.XMLSyntaxError, AttributeError, TypeError, ValueError):
        return False

<|endoftext|>