<commit_before>#!/usr/bin/python
#
# A small implementation of a Telepathy AccountManager.

import sys

import dbus
from dbus.bus import NAME_FLAG_DO_NOT_QUEUE, REQUEST_NAME_REPLY_EXISTS
from dbus.mainloop.glib import DBusGMainLoop
from dbus.service import Object, method, signal
from gobject import MainLoop

TP = 'org.freedesktop.Telepathy'

AM_IFACE = TP + '.AccountManager'
AM_BUS_NAME = AM_IFACE
AM_OBJECT_PATH = '/' + AM_IFACE.replace('.', '/')

ACCOUNT_IFACE = TP + '.Account'
ACCOUNT_OBJECT_PATH_BASE = '/' + ACCOUNT_IFACE.replace('.', '/') + '/'

class AccountManager(Object):
    def __init__(self, bus=None):
        #: map from object path to Account
        self._valid_accounts = {}
        #: map from object path to Account
        self._invalid_accounts = {}

        if bus is None:
            bus = dbus.SessionBus()

        ret = bus.request_name(AM_BUS_NAME, NAME_FLAG_DO_NOT_QUEUE)
        if ret == REQUEST_NAME_REPLY_EXISTS:
            raise dbus.NameExistsException(AM_BUS_NAME)

        Object.__init__(self, bus, AM_OBJECT_PATH)

    # overridden from Object, to have the Properties in introspection
    def Introspect(self, **kwargs):
        xml = super(AccountManager, self).Introspect(**kwargs)

        before, _ = xml.rsplit('</node>', 1)

        return before + """
            <property name="Interfaces" type="as" access="read"/>
            <property name="ValidAccounts" type="o" access="read"/>
            <property name="InvalidAccounts" type="o" access="read"/>
        </node>
        """

    def _am_props(self):
        return dbus.Dictionary({
            'Interfaces': dbus.Array([], signature='s'),
            'ValidAccounts': dbus.Array(self._valid_accounts.keys(),
                signature='o'),
            'InvalidAccounts': dbus.Array(self._invalid_accounts.keys(),
                signature='o'),
        }, signature='sv')

    @method(dbus.PROPERTIES_IFACE,
            in_signature='s',
            out_signature='a{sv}')
    def GetAll(self, iface):
        if iface == AM_IFACE:
            return self._am_props()
        else:
            raise ValueError('No such interface')

    @method(dbus.PROPERTIES_IFACE,
            in_signature='ss',
            out_signature='v')
    def Get(self, iface, prop):
        if iface == AM_IFACE:
            props = self._am_props()
        else:
            raise ValueError('No such interface')

        if prop in props:
            return props[prop]
        else:
            raise ValueError('No such property')

    @method(dbus.PROPERTIES_IFACE,
            in_signature='ssv')
    def Set(self, iface, prop, value):
        raise NotImplementedError('No mutable properties')

    @signal(AM_IFACE, signature='ob')
    def AccountValidityChanged(self, path, valid):
        if valid:
            assert path in self._invalid_accounts
            assert path not in self._valid_accounts
            self._valid_accounts[path] = self._invalid_accounts.pop(path)
        else:
            assert path in self._valid_accounts
            assert path not in self._invalid_accounts
            self._invalid_accounts[path] = self._valid_accounts.pop(path)

    @signal(AM_IFACE, signature='o')
    def AccountRemoved(self, path):
        assert path in self._valid_accounts or path in self._invalid_accounts
        self._valid_accounts.pop(path, None)
        self._invalid_accounts.pop(path, None)

    @method(AM_IFACE, in_signature='sssa{sv}', out_signature='o')
    def CreateAccount(self, cm, protocol, display_name, parameters):
        raise NotImplementedError

if __name__ == '__main__':
    DBusGMainLoop(set_as_default=True)

    try:
        am = AccountManager()
    except dbus.NameExistsException:
        print >> sys.stderr, 'AccountManager already running'
        sys.exit(1)

    mainloop = MainLoop()
    mainloop.run()
<commit_msg>account-manager.py: allow creation and deletion of accounts<commit_after>#!/usr/bin/python
#
# A small implementation of a Telepathy AccountManager.

import sys
import re

import dbus
from dbus.bus import NAME_FLAG_DO_NOT_QUEUE, REQUEST_NAME_REPLY_EXISTS
from dbus.mainloop.glib import DBusGMainLoop
from dbus.service import Object, method, signal
from gobject import MainLoop

TP = 'org.freedesktop.Telepathy'

AM_IFACE = TP + '.AccountManager'
AM_BUS_NAME = AM_IFACE
AM_OBJECT_PATH = '/' + AM_IFACE.replace('.', '/')

ACCOUNT_IFACE = TP + '.Account'
ACCOUNT_OBJECT_PATH_BASE = '/' + ACCOUNT_IFACE.replace('.', '/') + '/'

Connection_Status_Disconnected = dbus.UInt32(2)
Connection_Status_Reason_None_Specified = dbus.UInt32(0)
Connection_Presence_Type_Offline = dbus.UInt32(1)
Connection_Presence_Type_Available = dbus.UInt32(2)

VALID_CONNECTION_MANAGER_NAME = re.compile(r'^[A-Za-z0-9][_A-Za-z0-9]+$')
VALID_PROTOCOL_NAME = re.compile(r'^[A-Za-z0-9][-A-Za-z0-9]+$')

class AccountManager(Object):
    def __init__(self, bus=None):
        #: map from object path to Account
        self._valid_accounts = {}
        #: map from object path to Account
        self._invalid_accounts = {}

        if bus is None:
            bus = dbus.SessionBus()

        ret = bus.request_name(AM_BUS_NAME, NAME_FLAG_DO_NOT_QUEUE)
        if ret == REQUEST_NAME_REPLY_EXISTS:
            raise dbus.NameExistsException(AM_BUS_NAME)

        Object.__init__(self, bus, AM_OBJECT_PATH)

    # overridden from Object, to have the Properties in introspection
    def Introspect(self, **kwargs):
        xml = super(AccountManager, self).Introspect(**kwargs)

        before, _ = xml.rsplit('</node>', 1)

        return before + """
            <property name="Interfaces" type="as" access="read"/>
            <property name="ValidAccounts" type="o" access="read"/>
            <property name="InvalidAccounts" type="o" access="read"/>
        </node>
        """

    def _am_props(self):
        return dbus.Dictionary({
            'Interfaces': dbus.Array([], signature='s'),
            'ValidAccounts': dbus.Array(self._valid_accounts.keys(),
                signature='s'),
            'InvalidAccounts': dbus.Array(self._invalid_accounts.keys(),
                signature='s'),
        }, signature='sv')

    @method(dbus.PROPERTIES_IFACE,
            in_signature='s',
            out_signature='a{sv}')
    def GetAll(self, iface):
        if iface == AM_IFACE:
            return self._am_props()
        else:
            raise ValueError('No such interface')

    @method(dbus.PROPERTIES_IFACE,
            in_signature='ss',
            out_signature='v')
    def Get(self, iface, prop):
        if iface == AM_IFACE:
            props = self._am_props()
        else:
            raise ValueError('No such interface')

        if prop in props:
            return props[prop]
        else:
            raise ValueError('No such property')

    @method(dbus.PROPERTIES_IFACE,
            in_signature='ssv')
    def Set(self, iface, prop, value):
        raise NotImplementedError('No mutable properties')

    @signal(AM_IFACE, signature='ob')
    def AccountValidityChanged(self, path, valid):
        if valid:
            assert path in self._invalid_accounts
            assert path not in self._valid_accounts
            self._valid_accounts[path] = self._invalid_accounts.pop(path)
        else:
            assert path in self._valid_accounts
            assert path not in self._invalid_accounts
            self._invalid_accounts[path] = self._valid_accounts.pop(path)

    @signal(AM_IFACE, signature='o')
    def AccountRemoved(self, path):
        assert path in self._valid_accounts or path in self._invalid_accounts
        self._valid_accounts.pop(path, None)
        self._invalid_accounts.pop(path, None)

    @method(AM_IFACE, in_signature='sssa{sv}', out_signature='o')
    def CreateAccount(self, cm, protocol, display_name, parameters):

        if not VALID_CONNECTION_MANAGER_NAME.match(cm):
            raise ValueError('Invalid CM name')

        if not VALID_PROTOCOL_NAME.match(protocol):
            raise ValueError('Invalid protocol name')

        base = ACCOUNT_OBJECT_PATH_BASE + cm + '/' + protocol.replace('-', '_')

        # FIXME: This is a stupid way to generate the paths - we should
        # incorporate the display name somehow. However, it's spec-compliant
        i = 0
        while 1:
            path = '%s/Account%d' % (base, i)

            if (path not in self._valid_accounts and
                path not in self._invalid_accounts):
                account = Account(self, path,
                        '%s (account %d)' % (display_name, i), parameters)

                # put it in the wrong set and move it to the right one -
                # that's probably the simplest implementation
                if account._is_valid():
                    self._invalid_accounts[path] = account
                    self.AccountValidityChanged(path, True)
                    assert path not in self._invalid_accounts
                    assert path in self._valid_accounts
                else:
                    self._valid_accounts[path] = account
                    self.AccountValidityChanged(path, False)
                    assert path not in self._valid_accounts
                    assert path in self._invalid_accounts

                return path

            i += 1

        raise AssertionError('Not reached')

class Account(Object):
    def __init__(self, am, path, display_name, parameters):
        Object.__init__(self, am.connection, path)
        self._am = am

        self._display_name = display_name
        self._icon = u'bob.png'
        self._enabled = True
        self._nickname = u'Bob'
        self._parameters = parameters
        self._connect_automatically = True
        self._normalized_name = u'bob'
        self._automatic_presence = dbus.Struct(
                (Connection_Presence_Type_Available, 'available', ''),
                signature='uss')
        self._current_presence = dbus.Struct(
                (Connection_Presence_Type_Offline, 'offline', ''),
                signature='uss')
        self._requested_presence = dbus.Struct(
                (Connection_Presence_Type_Offline, 'offline', ''),
                signature='uss')

    def _is_valid(self):
        return True

    @method(ACCOUNT_IFACE, in_signature='a{sv}as', out_signature='')
    def UpdateParameters(self, set_, unset):
        for (key, value) in set_.iteritems():
            self._parameters[key] = value
        for key in unset:
            self._parameters.pop(key, None)

        AccountPropertyChanged({'Parameters': self._parameters})

    @signal(ACCOUNT_IFACE, signature='a{sv}')
    def AccountPropertyChanged(self, delta):
        pass

    @method(ACCOUNT_IFACE, in_signature='', out_signature='')
    def Remove(self):
        self.Removed()

    @signal(ACCOUNT_IFACE, signature='')
    def Removed(self):
        self._am.AccountRemoved(self.__dbus_object_path__)
        self.remove_from_connection()

    def _account_props(self):
        return dbus.Dictionary({
            'Interfaces': dbus.Array([], signature='s'),
            'DisplayName': self._display_name,
            'Icon': self._icon,
            'Valid': self._is_valid(),
            'Enabled': self._enabled,
            'Nickname': self._nickname,
            'Parameters': self._parameters,
            'AutomaticPresence': self._automatic_presence,
            'ConnectAutomatically': self._connect_automatically,
            'Connection': dbus.ObjectPath('/'),
            'ConnectionStatus': Connection_Status_Disconnected,
            'ConnectionStatusReason': Connection_Status_Reason_None_Specified,
            'CurrentPresence': self._current_presence,
            'RequestedPresence': self._requested_presence,
            'NormalizedName': self._normalized_name,
        }, signature='sv')

    @method(dbus.PROPERTIES_IFACE,
            in_signature='s',
            out_signature='a{sv}')
    def GetAll(self, iface):
        if iface == ACCOUNT_IFACE:
            return self._account_props()
        else:
            raise ValueError('No such interface')

    @method(dbus.PROPERTIES_IFACE,
            in_signature='ss',
            out_signature='v')
    def Get(self, iface, prop):
        if iface == ACCOUNT_IFACE:
            props = self._am_props()
        else:
            raise ValueError('No such interface')

        if prop in props:
            return props[prop]
        else:
            raise ValueError('No such property')

    @method(dbus.PROPERTIES_IFACE,
            in_signature='ssv')
    def Set(self, iface, prop, value):
        raise NotImplementedError('Not implemented')

if __name__ == '__main__':
    DBusGMainLoop(set_as_default=True)

    try:
        am = AccountManager()
    except dbus.NameExistsException:
        print >> sys.stderr, 'AccountManager already running'
        sys.exit(1)

    mainloop = MainLoop()
    mainloop.run()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#  -*- coding: UTF-8 -*-
"""
Poem classifier
"""
import argparse
import glob
import logging
import pprint
import re
import csv

import pandas
from lxml import etree

import numpy as np
from sklearn.externals import joblib
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.linear_model import SGDClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline

from poem_reader import read_xml_directory, parse_text_lines, block_xpath

argparser = argparse.ArgumentParser(description="Textblock classifier to poems and other text")
argparser.add_argument("job", help="Job to do", choices=['train', 'predict'])
argparser.add_argument("--dir", help="Directory to classify")
argparser.add_argument("--newfile", help="Create new CSV file", dest='newfile', action='store_true')
args = argparser.parse_args()

logging.basicConfig(filename='classifier.log',
                    filemode='a',
                    level=logging.DEBUG,
                    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

log = logging.getLogger(__name__)

def read_training_data(path='./'):
    poem_files = glob.glob(path + "poemblocks/*.txt")
    nonpoem_files = glob.glob(path + "nonpoemblocks/*.txt")

    poems = []
    for poem_file in poem_files:
        with open(poem_file, 'r') as f:
            poems.append(f.read())

    nonpoems = []
    for nonpoem_file in nonpoem_files:
        with open(nonpoem_file, 'r') as f:
            nonpoems.append(f.read())

    return poems, nonpoems

def train():
    poems, nonpoems = read_training_data()

    nonpoems = nonpoems[::5]  # TODO: Use skipped as test data

    print(len(poems))
    print(len(nonpoems))

    # count_vect = CountVectorizer()
    # X_train_counts = count_vect.fit_transform(poems + nonpoems)
    #
    # #print(X_train_counts)
    #
    # #cc = count_vect.vocabulary_.get('on')
    # #print(count_vect.get_feature_names()[cc])
    # #print(np.sum(X_train_counts, axis=0).tolist()[0][cc])
    #
    # tfidf_transformer = TfidfTransformer()
    # X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)
    # print(X_train_tfidf.shape)

    all_train_data = poems + nonpoems
    all_train_target = [1] * len(poems) + [0] * len(nonpoems)

    all_train_data = [d.replace('\n', ' ') for d in all_train_data]

    test_data = all_train_data[::2]  # TODO: Use less test data
    test_target = all_train_target[::2]

    train_data = all_train_data[1::2]
    train_target = all_train_target[1::2]

    text_clf = Pipeline([('vect', CountVectorizer()),
                         ('tfidf', TfidfTransformer()),
                         ('clf', SGDClassifier(loss='hinge', penalty='l2',
                                               alpha=1e-3, n_iter=5, random_state=42)),
                         ])

    text_clf.fit(train_data, train_target)
    predicted = text_clf.predict(test_data)
    acc = np.mean(predicted == test_target)

    print('Cross-validation accuracy %s' % acc)

    parameters = {'vect__ngram_range': [(1, 1), (1, 2), (1, 3)],
                  'tfidf__use_idf': (True, False),
                  'clf__alpha': (1e-2, 1e-3, 1e-4, 1e-5, 1e-6),
                  'clf__penalty': ('l1', 'l2', 'elasticnet'),
                  'clf__loss': ('hinge', 'log'),
                  'clf__n_iter': (3, 4, 5, 6, 7)}

    gs_clf = GridSearchCV(text_clf, parameters, n_jobs=-1)

    gs_clf = gs_clf.fit(train_data, train_target)

    # _ = text_clf.fit(train_data, train_target)
    predicted = gs_clf.predict(test_data)
    acc = np.mean(predicted == test_target)

    print(predicted)
    print(gs_clf.best_params_)

    print('Parameter-tuned cross-validation accuracy %s' % acc)

    # text_clf.fit(all_train_data, all_train_target)
    gs_clf.fit(all_train_data, all_train_target)

    print('Final params: %s' % gs_clf.best_params_)
    print('Best score: %s' % gs_clf.best_score_)

    return gs_clf

def parse_metadata_from_path(path):
    """
    ~/dhh17data/newspapers/newspapers/fin/1820/1457-4888/1457-4888_1820-01-08_1/alto/1457-4888_1820-01-08_1_003.xml
    """

    re_name_split = r'.*/newspapers/newspapers/fin/(.{4})/(.{7,9})/.{7,9}\_.{4}\-(..)\-(..)'

    split = re.search(re_name_split, path)

    year, issn, month, day = split.groups()
    return year, month, day, issn

def get_paper_name_by_issn(issn):
    try:
        paper = issues.loc[issues['issn'] == issn]['paper'].iloc[0]
    except IndexError:
        log.error('ISSN Number not found: %s' % issn)

if args.job == 'train':
    joblib.dump(train(), 'svm.pkl')

elif args.job == 'predict':
    clf = joblib.load('svm.pkl')

    if args.dir[-1] != '/':
        args.dir = args.dir + '/'

    files = glob.glob(args.dir + "**/*.xml", recursive=True)

    if not files:
        log.warning('No files found for %s' % args.dir)
        quit()

    xmls = []
    for xmlfile in files:
        if 'alto' not in xmlfile:
            continue

        with open(xmlfile, 'r') as f:
            try:
                parsed = etree.parse(f)
                xmls.append((parsed, xmlfile))
                log.debug('Read file %s' % xmlfile)
            except etree.XMLSyntaxError:
                log.error('Error in XML: %s' % xmlfile)

    data = []
    metadata = []
    for xml, filename in xmls:
        text_blocks = block_xpath(xml)

        paper_metadata = parse_metadata_from_path(filename)

        # print(paper_metadata)
        # pprint.pprint(list(parse_text_lines(list(block)) for block in text_blocks))
        # quit()

        for block in text_blocks:
            data.append(parse_text_lines(list(block)))
            metadata.append(paper_metadata + (block.get('ID'),))

    data_orig = data
    data = [d.replace('\n', ' ') for d in data]

    # print(len(data))

    predicted = clf.predict(data)
    #print(predicted)

    # print(metadata[100])
    # print(data_orig[100])
    # quit()

    data_trunc = [d for i, d in enumerate(data_orig) if predicted[i] and len(d) >= 94]
    metadata = [d for i, d in enumerate(metadata) if predicted[i] and len(data_orig[i]) >= 94]

    issues = pandas.read_csv('data/issue_numbers.csv', sep=',')
    with open('foundpoems/found_poems.csv'.format(year=metadata[0][0]), 'w' if args.newfile else 'a', newline='') as fp:
        writer = csv.writer(fp, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)

        if args.newfile:
            writer.writerow(('Poem', 'Year', 'Month', 'Day', 'Newspaper name', 'ISSN'))
            log.info('Created new CSV file')

        poemtext = ''
        prev_vector = None
        blockids = []
        for i, d in enumerate(data_trunc):

            # print(metadata[i])

            year, month, day, issn, blockid = metadata[i]

            # if not predicted[i]:
            #     prev_vector = (year, month, day, issn)
            #     continue
            #
            paper = get_paper_name_by_issn(issn)

            if prev_vector == (year, month, day, issn):
                if poemtext:
                    poemtext += "\n"
                poemtext += d
                blockids.append(blockid)
            else:
                if poemtext:
                    year2, month2, day2, issn2 = prev_vector
                    paper2 = get_paper_name_by_issn(issn2)
                    writer.writerow([poemtext.replace('\n', ' '), year2, month2, day2, paper2, issn2])
                    poem_filename = 'foundpoems/{year}_{month}_{day}_{paper} {blocks}'.\
                                    format(year=year2, month=month2, day=day2, paper=paper2, blocks=' '.join(blockids))
                    poem_filename = (poem_filename[:240] + ' TRUNCATED') if len(poem_filename) > 247 else poem_filename
                    poem_filename += '.txt'
                    with open(poem_filename, 'w', newline='') as textp:
                        textp.write(poemtext)

                poemtext = d
                blockids = [blockid]

            prev_vector = (year, month, day, issn)

        writer.writerow([poemtext.replace('\n', ' '), year, month, day, paper, issn])
        log.info('Updated CSV file for year %s' % metadata[0][0])
<commit_msg>Add some logging<commit_after>#!/usr/bin/env python3
#  -*- coding: UTF-8 -*-
"""
Poem classifier
"""
import argparse
import glob
import logging
import pprint
import re
import csv

import pandas
from lxml import etree

import numpy as np
from sklearn.externals import joblib
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from sklearn.linear_model import SGDClassifier
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline

from poem_reader import read_xml_directory, parse_text_lines, block_xpath

argparser = argparse.ArgumentParser(description="Textblock classifier to poems and other text")
argparser.add_argument("job", help="Job to do", choices=['train', 'predict'])
argparser.add_argument("--dir", help="Directory to classify")
argparser.add_argument("--newfile", help="Create new CSV file", dest='newfile', action='store_true')
args = argparser.parse_args()

logging.basicConfig(filename='classifier.log',
                    filemode='a',
                    level=logging.DEBUG,
                    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s')

log = logging.getLogger(__name__)

def read_training_data(path='./'):
    poem_files = glob.glob(path + "poemblocks/*.txt")
    nonpoem_files = glob.glob(path + "nonpoemblocks/*.txt")

    poems = []
    for poem_file in poem_files:
        with open(poem_file, 'r') as f:
            poems.append(f.read())

    nonpoems = []
    for nonpoem_file in nonpoem_files:
        with open(nonpoem_file, 'r') as f:
            nonpoems.append(f.read())

    return poems, nonpoems

def train():
    poems, nonpoems = read_training_data()

    nonpoems = nonpoems[::5]  # TODO: Use skipped as test data

    print(len(poems))
    print(len(nonpoems))

    # count_vect = CountVectorizer()
    # X_train_counts = count_vect.fit_transform(poems + nonpoems)
    #
    # #print(X_train_counts)
    #
    # #cc = count_vect.vocabulary_.get('on')
    # #print(count_vect.get_feature_names()[cc])
    # #print(np.sum(X_train_counts, axis=0).tolist()[0][cc])
    #
    # tfidf_transformer = TfidfTransformer()
    # X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)
    # print(X_train_tfidf.shape)

    all_train_data = poems + nonpoems
    all_train_target = [1] * len(poems) + [0] * len(nonpoems)

    all_train_data = [d.replace('\n', ' ') for d in all_train_data]

    test_data = all_train_data[::2]  # TODO: Use less test data
    test_target = all_train_target[::2]

    train_data = all_train_data[1::2]
    train_target = all_train_target[1::2]

    text_clf = Pipeline([('vect', CountVectorizer()),
                         ('tfidf', TfidfTransformer()),
                         ('clf', SGDClassifier(loss='hinge', penalty='l2',
                                               alpha=1e-3, n_iter=5, random_state=42)),
                         ])

    text_clf.fit(train_data, train_target)
    predicted = text_clf.predict(test_data)
    acc = np.mean(predicted == test_target)

    print('Cross-validation accuracy %s' % acc)

    parameters = {'vect__ngram_range': [(1, 1), (1, 2), (1, 3)],
                  'tfidf__use_idf': (True, False),
                  'clf__alpha': (1e-2, 1e-3, 1e-4, 1e-5, 1e-6),
                  'clf__penalty': ('l1', 'l2', 'elasticnet'),
                  'clf__loss': ('hinge', 'log'),
                  'clf__n_iter': (3, 4, 5, 6, 7)}

    gs_clf = GridSearchCV(text_clf, parameters, n_jobs=-1)

    gs_clf = gs_clf.fit(train_data, train_target)

    # _ = text_clf.fit(train_data, train_target)
    predicted = gs_clf.predict(test_data)
    acc = np.mean(predicted == test_target)

    print(predicted)
    print(gs_clf.best_params_)

    print('Parameter-tuned cross-validation accuracy %s' % acc)

    # text_clf.fit(all_train_data, all_train_target)
    gs_clf.fit(all_train_data, all_train_target)

    print('Final params: %s' % gs_clf.best_params_)
    print('Best score: %s' % gs_clf.best_score_)

    return gs_clf

def parse_metadata_from_path(path):
    """
    ~/dhh17data/newspapers/newspapers/fin/1820/1457-4888/1457-4888_1820-01-08_1/alto/1457-4888_1820-01-08_1_003.xml
    """

    re_name_split = r'.*/newspapers/newspapers/fin/(.{4})/(.{7,9})/.{7,9}\_.{4}\-(..)\-(..)'

    split = re.search(re_name_split, path)

    year, issn, month, day = split.groups()
    return year, month, day, issn

def get_paper_name_by_issn(issn):
    try:
        paper = issues.loc[issues['issn'] == issn]['paper'].iloc[0]
    except IndexError:
        log.error('ISSN Number not found: %s' % issn)

if args.job == 'train':
    joblib.dump(train(), 'svm.pkl')

elif args.job == 'predict':
    log.info('Loading classifier from pickle file')
    clf = joblib.load('svm.pkl')

    log.info('Classifier loaded')

    if args.dir[-1] != '/':
        args.dir = args.dir + '/'

    files = glob.glob(args.dir + "**/*.xml", recursive=True)

    if not files:
        log.warning('No files found for %s' % args.dir)
        quit()

    xmls = []
    for xmlfile in files:
        if 'alto' not in xmlfile:
            continue

        with open(xmlfile, 'r') as f:
            try:
                parsed = etree.parse(f)
                xmls.append((parsed, xmlfile))
                log.debug('Read file %s' % xmlfile)
            except etree.XMLSyntaxError:
                log.error('Error in XML: %s' % xmlfile)

    data = []
    metadata = []
    for xml, filename in xmls:
        text_blocks = block_xpath(xml)

        paper_metadata = parse_metadata_from_path(filename)

        # print(paper_metadata)
        # pprint.pprint(list(parse_text_lines(list(block)) for block in text_blocks))
        # quit()

        for block in text_blocks:
            data.append(parse_text_lines(list(block)))
            metadata.append(paper_metadata + (block.get('ID'),))

    data_orig = data
    data = [d.replace('\n', ' ') for d in data]

    # print(len(data))

    predicted = clf.predict(data)
    #print(predicted)

    # print(metadata[100])
    # print(data_orig[100])
    # quit()

    data_trunc = [d for i, d in enumerate(data_orig) if predicted[i] and len(d) >= 94]
    metadata = [d for i, d in enumerate(metadata) if predicted[i] and len(data_orig[i]) >= 94]

    issues = pandas.read_csv('data/issue_numbers.csv', sep=',')
    with open('foundpoems/found_poems.csv'.format(year=metadata[0][0]), 'w' if args.newfile else 'a', newline='') as fp:
        writer = csv.writer(fp, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)

        if args.newfile:
            writer.writerow(('Poem', 'Year', 'Month', 'Day', 'Newspaper name', 'ISSN'))
            log.info('Created new CSV file')

        poemtext = ''
        prev_vector = None
        blockids = []
        for i, d in enumerate(data_trunc):

            # print(metadata[i])

            year, month, day, issn, blockid = metadata[i]

            # if not predicted[i]:
            #     prev_vector = (year, month, day, issn)
            #     continue
            #
            paper = get_paper_name_by_issn(issn)

            if prev_vector == (year, month, day, issn):
                if poemtext:
                    poemtext += "\n"
                poemtext += d
                blockids.append(blockid)
            else:
                if poemtext:
                    year2, month2, day2, issn2 = prev_vector
                    paper2 = get_paper_name_by_issn(issn2)
                    writer.writerow([poemtext.replace('\n', ' '), year2, month2, day2, paper2, issn2])
                    poem_filename = 'foundpoems/{year}_{month}_{day}_{paper} {blocks}'.\
                                    format(year=year2, month=month2, day=day2, paper=paper2, blocks=' '.join(blockids))
                    poem_filename = (poem_filename[:240] + ' TRUNCATED') if len(poem_filename) > 247 else poem_filename
                    poem_filename += '.txt'
                    with open(poem_filename, 'w', newline='') as textp:
                        textp.write(poemtext)

                poemtext = d
                blockids = [blockid]

            prev_vector = (year, month, day, issn)

        writer.writerow([poemtext.replace('\n', ' '), year, month, day, paper, issn])
        log.info('Updated CSV file for year %s' % metadata[0][0])
<|endoftext|>


<commit_before>from django.core.wsgi import get_wsgi_application
from dj_static import Cling

application = Cling(get_wsgi_application())
<commit_msg>added django_settings_module to setup wsgi<commit_after>from django.core.wsgi import get_wsgi_application
from dj_static import Cling
import os

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'gmmp.settings')
application = Cling(get_wsgi_application())
<|endoftext|>


<commit_before>from datetime import datetime, timedelta

from django.db import models
from django.db.models import F, Q
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.utils.encoding import smart_str

from ella.core.cache import cache_this
from ella.core.cache.invalidate import CACHE_DELETER
from ella.core.conf import core_settings

class RelatedManager(models.Manager):
    def get_related_for_object(self, obj, count, mods=[], only_from_same_site=True):
        from ella.core.models import Publishable

        # manually entered dependencies

        qset = Publishable.objects.filter(
                related__related_ct=ContentType.objects.get_for_model(obj),
                related__related_id=obj.pk
            )
        if mods:
            ct_ids = [ContentType.objects.get_for_model(m).pk for m in mods]
            qset = qset.filter(content_type__in=ct_ids)
        if only_from_same_site:
            qset = qset.filter(category__site__pk=settings.SITE_ID)
        related = list(qset[:count])

        if len(related) >= count:
            return related

        count -= len(related)

        # related objects via tags
        try:
            from tagging.models import TaggedItem
            if TaggedItem._meta.installed:
                # we are only tagging Publishables, not individual content types
                if isinstance(obj, Publishable):
                    obj = obj.publishable_ptr

                qset = Publishable.objects.filter(
                        placement__publish_from__lte=datetime.now(),
                    ).distinct()
                if mods:
                    qset = qset.filter(content_type__in=ct_ids)
                if only_from_same_site:
                    qset = qset.filter(category__site__pk=settings.SITE_ID)

                #print qset
                #print TaggedItem.objects.all()
                to_add = TaggedItem.objects.get_related(obj, qset, num=count+len(related))
                #print to_add
                for rel in to_add:
                    if rel != obj and rel not in related:
                        count -= 1
                        related.append(rel)
                    if count <= 0:
                        return related
        except ImportError, e:
            pass

        # top objects in given category
        if count > 0:
            from ella.core.models import Listing
            cat = obj.category
            listings = Listing.objects.get_listing(category=cat, count=count+len(related), mods=mods)
            for l in listings:
                t = l.target
                if t != obj and t not in related:
                    related.append(t)
                    count -= 1

                if count <= 0:
                    return related

        return related

def invalidate_listing(key, self, *args, **kwargs):
    CACHE_DELETER.register_test(self.model, '', key)

def get_listings_key(func, self, category=None, count=10, offset=1, mods=[], content_types=[], **kwargs):
    c = category and  category.id or ''

    return 'ella.core.managers.ListingManager.get_listing:%s:%d:%d:%s:%s:%s' % (
            c, count, offset,
            ','.join(str(model._meta) for model in mods),
            ','.join(map(str, content_types)),
            ','.join(':'.join((k, smart_str(v))) for k, v in kwargs.items()),
    )

class PlacementManager(models.Manager):
    def get_query_set(self, *args, **kwargs):
        qset = super(PlacementManager, self).get_query_set(*args, **kwargs).select_related('publishable')
        return qset

    def get_static_placements(self, category):
        now = datetime.now()
        return self.filter(models.Q(publish_to__gt=now) | models.Q(publish_to__isnull=True),  publish_from__lt=now, category=category, static=True)

class ListingManager(models.Manager):
    NONE = 0
    IMMEDIATE = 1
    ALL = 2

    def clean_listings(self):
        """
        Method that cleans the Listing model by deleting all listings that are no longer valid.
        Should be run periodicaly to purge the DB from unneeded data.
        """
        self.filter(publish_to__lt=datetime.now()).delete()

    def get_query_set(self, *args, **kwargs):
        # get all the fields you typically need to render listing
        qset = super(ListingManager, self).get_query_set(*args, **kwargs).select_related(
                'placement',
                'placement__category',
                'placement__publishable',
                'placement__publishable__category',
                'placement__publishable__content_type'
            )
        return qset

    def get_listing_queryset(self, category=None, children=NONE, mods=[], content_types=[], now=None, **kwargs):
        if not now:
            now = datetime.now()
        qset = self.filter(publish_from__lte=now, **kwargs)

        if category:
            if children == self.NONE:
                # only this one category
                qset = qset.filter(category=category)
            elif children == self.IMMEDIATE:
                # this category and its children
                qset = qset.filter(models.Q(category__tree_parent=category) | models.Q(category=category))
            elif children == self.ALL:
                # this category and all its descendants
                qset = qset.filter(category__tree_path__startswith=category.tree_path, category__site=category.site_id)

            else:
                raise AttributeError('Invalid children value (%s) - should be one of (%s, %s, %s)' % (children, self.NONE, self.IMMEDIATE, self.ALL))

        # filtering based on Model classes
        if mods or content_types:
            qset = qset.filter(placement__publishable__content_type__in=([ ContentType.objects.get_for_model(m) for m in mods ] + content_types))

        return qset.exclude(publish_to__lt=now)

    @cache_this(get_listings_key, invalidate_listing)
    def get_listing(self, category=None, children=NONE, count=10, offset=1, mods=[], content_types=[], unique=None, **kwargs):
        """
        Get top objects for given category and potentionally also its child categories.

        Params:
            category - Category object to list objects for. None if any category will do
            count - number of objects to output, defaults to 10
            offset - starting with object number... 1-based
            mods - list of Models, if empty, object from all models are included
            [now] - datetime used instead of default datetime.now() value
            [unique] - set of already listed Placement IDs
            **kwargs - rest of the parameter are passed to the queryset unchanged
        """
        def mark_before_output(output):
            for l in output:
                listed_targets.add(l.placement_id)

        def make_items_unique(qset):

            out = []
            listed_targets_now = set()
            for l in qset:
                tgt = l.placement_id
                if tgt in listed_targets or tgt in listed_targets_now:
                    continue
                listed_targets_now.add(tgt)
                out.append(l)
                if len(out) == limit:
                    result = out[offset:limit]
                    mark_before_output(result)
                    return result
            mark_before_output(out)
            return out

        # TODO try to write  SQL (.extra())
        assert offset > 0, "Offset must be a positive integer"
        assert count >= 0, "Count must be a positive integer"

        if not count:
            return []

        now = datetime.now()
        if 'now' in kwargs:
            now = kwargs.pop('now')
        qset = self.get_listing_queryset(category, children, mods, content_types, now, **kwargs)

        # templates are 1-based, compensate
        offset -= 1
        limit = offset + count

        # take out unwanted objects
        if unique:
            listed_targets = unique.copy()
        else:
            listed_targets = set([])

        # only use priorities if somebody wants them
        if not core_settings.USE_PRIORITIES:
            if unique:
                return make_items_unique(qset)
            return qset[offset:limit]

        # listings with active priority override
        active = models.Q(
                    priority_value__isnull=False,
                    priority_from__isnull=False,
                    priority_from__lte=now,
                    priority_to__gte=now
        )

        qsets = (
            # modded-up objects
            qset.filter(active, priority_value__gt=core_settings.DEFAULT_LISTING_PRIORITY).order_by('-priority_value', '-publish_from'),
            # default priority
            qset.exclude(active).order_by('-publish_from'),
            # modded-down priority
            qset.filter(active, priority_value__lt=core_settings.DEFAULT_LISTING_PRIORITY).order_by('-priority_value', '-publish_from'),
        )

        out = []

        # iterate through qsets until we have enough objects
        for q in qsets:
            data = q.iterator()
            for l in data:
                tgt = l.placement_id
                if tgt in listed_targets:
                    continue
                listed_targets.add(tgt)
                out.append(l)
                if len(out) == limit:
                    return out[offset:limit]
        return out[offset:offset + count]

    def get_queryset_wrapper(self, kwargs):
        return ListingQuerySetWrapper(self, kwargs)

class ListingQuerySetWrapper(object):
    def __init__(self, manager, kwargs):
        self.manager = manager
        self._kwargs = kwargs

    def __getitem__(self, k):
        if not isinstance(k, slice) or (k.start is None or k.start < 0) or (k.stop is None  or k.stop < k.start):
            raise TypeError, '%s, %s' % (k.start, k.stop)

        offset = k.start + 1
        count = k.stop - k.start

        return self.manager.get_listing(offset=offset, count=count, **self._kwargs)

    def count(self):
        if not hasattr(self, '_count'):
            self._count = self.manager.get_listing_queryset(**self._kwargs).count()
        return self._count

def get_top_objects_key(func, self, count, days=None, mods=[]):
    return 'ella.core.managers.HitCountManager.get_top_objects_key:%d:%d:%s:%s' % (
            settings.SITE_ID, count, str(days), ','.join('.'.join(str(model._meta) for model in mods))
        )

class HitCountManager(models.Manager):

    def hit(self, placement):
        count = self.filter(placement=placement).update(hits=F('hits')+1)

        if count < 1:
            self.create(placement=placement, hits=1)

    @cache_this(get_top_objects_key)
    def get_top_objects(self, count, days=None, mods=[]):
        """
        Return count top rated objects. Cache this for 10 minutes without any chance of cache invalidation.
        """
        qset = self.filter(placement__category__site=settings.SITE_ID).order_by('-hits')

        if mods:
            qset = qset.filter(placement__publishable__content_type__in = [ ContentType.objects.get_for_model(m) for m in mods ])

        now = datetime.now()
        if days is None:
            qset = qset.filter(placement__publish_from__lte=now)
        else:
            start = now - timedelta(days=days)
            qset = qset.filter(placement__publish_from__range=(start, now,))
        qset = qset.filter(Q(placement__publish_to__gt=now) | Q(placement__publish_to__isnull=True))

        return list(qset[:count])
<commit_msg>Always use unique listings.<commit_after>from datetime import datetime, timedelta

from django.db import models
from django.db.models import F, Q
from django.conf import settings
from django.contrib.contenttypes.models import ContentType
from django.utils.encoding import smart_str

from ella.core.cache import cache_this
from ella.core.cache.invalidate import CACHE_DELETER
from ella.core.conf import core_settings

class RelatedManager(models.Manager):
    def get_related_for_object(self, obj, count, mods=[], only_from_same_site=True):
        from ella.core.models import Publishable

        # manually entered dependencies

        qset = Publishable.objects.filter(
                related__related_ct=ContentType.objects.get_for_model(obj),
                related__related_id=obj.pk
            )
        if mods:
            ct_ids = [ContentType.objects.get_for_model(m).pk for m in mods]
            qset = qset.filter(content_type__in=ct_ids)
        if only_from_same_site:
            qset = qset.filter(category__site__pk=settings.SITE_ID)
        related = list(qset[:count])

        if len(related) >= count:
            return related

        count -= len(related)

        # related objects via tags
        try:
            from tagging.models import TaggedItem
            if TaggedItem._meta.installed:
                # we are only tagging Publishables, not individual content types
                if isinstance(obj, Publishable):
                    obj = obj.publishable_ptr

                qset = Publishable.objects.filter(
                        placement__publish_from__lte=datetime.now(),
                    ).distinct()
                if mods:
                    qset = qset.filter(content_type__in=ct_ids)
                if only_from_same_site:
                    qset = qset.filter(category__site__pk=settings.SITE_ID)

                #print qset
                #print TaggedItem.objects.all()
                to_add = TaggedItem.objects.get_related(obj, qset, num=count+len(related))
                #print to_add
                for rel in to_add:
                    if rel != obj and rel not in related:
                        count -= 1
                        related.append(rel)
                    if count <= 0:
                        return related
        except ImportError, e:
            pass

        # top objects in given category
        if count > 0:
            from ella.core.models import Listing
            cat = obj.category
            listings = Listing.objects.get_listing(category=cat, count=count+len(related), mods=mods)
            for l in listings:
                t = l.target
                if t != obj and t not in related:
                    related.append(t)
                    count -= 1

                if count <= 0:
                    return related

        return related

def invalidate_listing(key, self, *args, **kwargs):
    CACHE_DELETER.register_test(self.model, '', key)

def get_listings_key(func, self, category=None, count=10, offset=1, mods=[], content_types=[], **kwargs):
    c = category and  category.id or ''

    return 'ella.core.managers.ListingManager.get_listing:%s:%d:%d:%s:%s:%s' % (
            c, count, offset,
            ','.join(str(model._meta) for model in mods),
            ','.join(map(str, content_types)),
            ','.join(':'.join((k, smart_str(v))) for k, v in kwargs.items()),
    )

class PlacementManager(models.Manager):
    def get_query_set(self, *args, **kwargs):
        qset = super(PlacementManager, self).get_query_set(*args, **kwargs).select_related('publishable')
        return qset

    def get_static_placements(self, category):
        now = datetime.now()
        return self.filter(models.Q(publish_to__gt=now) | models.Q(publish_to__isnull=True),  publish_from__lt=now, category=category, static=True)

class ListingManager(models.Manager):
    NONE = 0
    IMMEDIATE = 1
    ALL = 2

    def clean_listings(self):
        """
        Method that cleans the Listing model by deleting all listings that are no longer valid.
        Should be run periodicaly to purge the DB from unneeded data.
        """
        self.filter(publish_to__lt=datetime.now()).delete()

    def get_query_set(self, *args, **kwargs):
        # get all the fields you typically need to render listing
        qset = super(ListingManager, self).get_query_set(*args, **kwargs).select_related(
                'placement',
                'placement__category',
                'placement__publishable',
                'placement__publishable__category',
                'placement__publishable__content_type'
            )
        return qset

    def get_listing_queryset(self, category=None, children=NONE, mods=[], content_types=[], now=None, **kwargs):
        if not now:
            now = datetime.now()
        qset = self.filter(publish_from__lte=now, **kwargs)

        if category:
            if children == self.NONE:
                # only this one category
                qset = qset.filter(category=category)
            elif children == self.IMMEDIATE:
                # this category and its children
                qset = qset.filter(models.Q(category__tree_parent=category) | models.Q(category=category))
            elif children == self.ALL:
                # this category and all its descendants
                qset = qset.filter(category__tree_path__startswith=category.tree_path, category__site=category.site_id)

            else:
                raise AttributeError('Invalid children value (%s) - should be one of (%s, %s, %s)' % (children, self.NONE, self.IMMEDIATE, self.ALL))

        # filtering based on Model classes
        if mods or content_types:
            qset = qset.filter(placement__publishable__content_type__in=([ ContentType.objects.get_for_model(m) for m in mods ] + content_types))

        return qset.exclude(publish_to__lt=now)

    @cache_this(get_listings_key, invalidate_listing)
    def get_listing(self, category=None, children=NONE, count=10, offset=1, mods=[], content_types=[], unique=None, **kwargs):
        """
        Get top objects for given category and potentionally also its child categories.

        Params:
            category - Category object to list objects for. None if any category will do
            count - number of objects to output, defaults to 10
            offset - starting with object number... 1-based
            mods - list of Models, if empty, object from all models are included
            [now] - datetime used instead of default datetime.now() value
            [unique] - set of already listed Placement IDs
            **kwargs - rest of the parameter are passed to the queryset unchanged
        """
        def mark_before_output(output):
            for l in output:
                listed_targets.add(l.placement_id)

        def make_items_unique(qset):

            out = []
            listed_targets_now = set()
            for l in qset:
                tgt = l.placement_id
                if tgt in listed_targets or tgt in listed_targets_now:
                    continue
                listed_targets_now.add(tgt)
                out.append(l)
                if len(out) == limit:
                    result = out[offset:limit]
                    mark_before_output(result)
                    return result
            mark_before_output(out)
            return out

        # TODO try to write  SQL (.extra())
        assert offset > 0, "Offset must be a positive integer"
        assert count >= 0, "Count must be a positive integer"

        if not count:
            return []

        now = datetime.now()
        if 'now' in kwargs:
            now = kwargs.pop('now')
        qset = self.get_listing_queryset(category, children, mods, content_types, now, **kwargs)

        # templates are 1-based, compensate
        offset -= 1
        limit = offset + count

        # take out unwanted objects
        if unique:
            listed_targets = unique.copy()
        else:
            listed_targets = set([])

        # only use priorities if somebody wants them
        if not core_settings.USE_PRIORITIES:
            return make_items_unique(qset)[offset:limit]

        # listings with active priority override
        active = models.Q(
                    priority_value__isnull=False,
                    priority_from__isnull=False,
                    priority_from__lte=now,
                    priority_to__gte=now
        )

        qsets = (
            # modded-up objects
            qset.filter(active, priority_value__gt=core_settings.DEFAULT_LISTING_PRIORITY).order_by('-priority_value', '-publish_from'),
            # default priority
            qset.exclude(active).order_by('-publish_from'),
            # modded-down priority
            qset.filter(active, priority_value__lt=core_settings.DEFAULT_LISTING_PRIORITY).order_by('-priority_value', '-publish_from'),
        )

        out = []

        # iterate through qsets until we have enough objects
        for q in qsets:
            data = q.iterator()
            for l in data:
                tgt = l.placement_id
                if tgt in listed_targets:
                    continue
                listed_targets.add(tgt)
                out.append(l)
                if len(out) == limit:
                    return out[offset:limit]
        return out[offset:offset + count]

    def get_queryset_wrapper(self, kwargs):
        return ListingQuerySetWrapper(self, kwargs)

class ListingQuerySetWrapper(object):
    def __init__(self, manager, kwargs):
        self.manager = manager
        self._kwargs = kwargs

    def __getitem__(self, k):
        if not isinstance(k, slice) or (k.start is None or k.start < 0) or (k.stop is None  or k.stop < k.start):
            raise TypeError, '%s, %s' % (k.start, k.stop)

        offset = k.start + 1
        count = k.stop - k.start

        return self.manager.get_listing(offset=offset, count=count, **self._kwargs)

    def count(self):
        if not hasattr(self, '_count'):
            self._count = self.manager.get_listing_queryset(**self._kwargs).count()
        return self._count

def get_top_objects_key(func, self, count, days=None, mods=[]):
    return 'ella.core.managers.HitCountManager.get_top_objects_key:%d:%d:%s:%s' % (
            settings.SITE_ID, count, str(days), ','.join('.'.join(str(model._meta) for model in mods))
        )

class HitCountManager(models.Manager):

    def hit(self, placement):
        count = self.filter(placement=placement).update(hits=F('hits')+1)

        if count < 1:
            self.create(placement=placement, hits=1)

    @cache_this(get_top_objects_key)
    def get_top_objects(self, count, days=None, mods=[]):
        """
        Return count top rated objects. Cache this for 10 minutes without any chance of cache invalidation.
        """
        qset = self.filter(placement__category__site=settings.SITE_ID).order_by('-hits')

        if mods:
            qset = qset.filter(placement__publishable__content_type__in = [ ContentType.objects.get_for_model(m) for m in mods ])

        now = datetime.now()
        if days is None:
            qset = qset.filter(placement__publish_from__lte=now)
        else:
            start = now - timedelta(days=days)
            qset = qset.filter(placement__publish_from__range=(start, now,))
        qset = qset.filter(Q(placement__publish_to__gt=now) | Q(placement__publish_to__isnull=True))

        return list(qset[:count])
<|endoftext|>


<commit_before>#!/usr/bin/python3

import logging
logging.basicConfig(level=logging.DEBUG)

from binascii import a2b_hex, b2a_hex
import bitcoin.txn
import bitcoin.varlen
import jsonrpc
import jsonrpcserver
import merkletree
from struct import pack
import sys
from time import time
from util import RejectedShare

pool = jsonrpc.ServiceProxy(sys.argv[1])

worklog = {}

def MakeWork(username):
	mp = pool.getmemorypool()
	coinbase = a2b_hex(mp['coinbasetxn'])
	cbtxn = bitcoin.txn.Txn(coinbase)
	txnlist = [cbtxn,] + list(map(bitcoin.txn.Txn, map(a2b_hex, mp['transactions'])))
	merkleTree = merkletree.MerkleTree(txnlist)
	merkleRoot = merkleTree.merkleRoot()
	prevBlock = a2b_hex(mp['previousblockhash'])[::-1]
	bits = a2b_hex(mp['bits'])[::-1]
	rollPrevBlk = False

	MRD = (merkleRoot, merkleTree, coinbase, prevBlock, bits, rollPrevBlk)
	timestamp = pack('<L', int(time()))
	hdr = b'\1\0\0\0' + prevBlock + merkleRoot + timestamp + bits + b'ppmg'
	worklog[hdr[4:68]] = (MRD, time())
	return hdr

def SubmitShare(share):
	hdr = share['data'][:80]
	k = hdr[4:68]
	if k not in worklog:
		raise RejectedShare('LOCAL unknown-work')
	(MRD, issueT) = worklog[k]
	(merkleRoot, merkleTree, coinbase, prevBlock, bits, rollPrevBlk) = MRD
	blkdata = bitcoin.varlen.varlenEncode(len(merkleTree.data))
	for txn in merkleTree.data:
		blkdata += txn.data
	data = b2a_hex(hdr + blkdata).decode('utf8')
	if not pool.submitblock(data):
		raise RejectedShare('pool-rejected')

server = jsonrpcserver.JSONRPCServer()
server.getBlockHeader = MakeWork
server.receiveShare = SubmitShare
jsonrpcserver.JSONRPCListener(server, ('::ffff:127.0.0.1', 9332))

server.serve_forever()
<commit_msg>gmp-proxy: Bugfix: submitblock method returns a rejection reason or null, not a boolean<commit_after>#!/usr/bin/python3

import logging
logging.basicConfig(level=logging.DEBUG)

from binascii import a2b_hex, b2a_hex
import bitcoin.txn
import bitcoin.varlen
import jsonrpc
import jsonrpcserver
import merkletree
from struct import pack
import sys
from time import time
from util import RejectedShare

pool = jsonrpc.ServiceProxy(sys.argv[1])

worklog = {}

def MakeWork(username):
	mp = pool.getmemorypool()
	coinbase = a2b_hex(mp['coinbasetxn'])
	cbtxn = bitcoin.txn.Txn(coinbase)
	txnlist = [cbtxn,] + list(map(bitcoin.txn.Txn, map(a2b_hex, mp['transactions'])))
	merkleTree = merkletree.MerkleTree(txnlist)
	merkleRoot = merkleTree.merkleRoot()
	prevBlock = a2b_hex(mp['previousblockhash'])[::-1]
	bits = a2b_hex(mp['bits'])[::-1]
	rollPrevBlk = False

	MRD = (merkleRoot, merkleTree, coinbase, prevBlock, bits, rollPrevBlk)
	timestamp = pack('<L', int(time()))
	hdr = b'\1\0\0\0' + prevBlock + merkleRoot + timestamp + bits + b'ppmg'
	worklog[hdr[4:68]] = (MRD, time())
	return hdr

def SubmitShare(share):
	hdr = share['data'][:80]
	k = hdr[4:68]
	if k not in worklog:
		raise RejectedShare('LOCAL unknown-work')
	(MRD, issueT) = worklog[k]
	(merkleRoot, merkleTree, coinbase, prevBlock, bits, rollPrevBlk) = MRD
	blkdata = bitcoin.varlen.varlenEncode(len(merkleTree.data))
	for txn in merkleTree.data:
		blkdata += txn.data
	data = b2a_hex(hdr + blkdata).decode('utf8')
	rejReason = pool.submitblock(data)
	if not rejReason is None:
		raise RejectedShare('pool-' + rejReason)

server = jsonrpcserver.JSONRPCServer()
server.getBlockHeader = MakeWork
server.receiveShare = SubmitShare
jsonrpcserver.JSONRPCListener(server, ('::ffff:127.0.0.1', 9332))

server.serve_forever()
<|endoftext|>


<commit_before>"""Module for compiling codegen-output, and wrap it for use in python.

What?

This module provides a common interface for different external backends, such
as f2py, fwrap, Cython, SWIG(?) etc.  The goal is to provide access to compiled
binaries of acceptable performance with a one-button user interface, i.e.

    >>> from sympy.abc import x,y
    >>> from sympy.utilities.autowrap import autowrap
    >>> expr = ((x - y)**(35)).expand()
    >>> binary_func = autowrap(expr)
    >>> binary_func.subs((x, 1), (y, 2))
    -1.00000000

Why?

The idea is that a SymPy user will primarily be interested in working with
mathematical expressions, and should not have to learn details about wrapping
tools in order to evaluate expressions numerically, even if they are
computationally expensive.

When is this useful?

    1) For computations on large arrays, Python iterations may be too slow, and
    depending on the mathematical expression, it may be difficult to exploit
    the advanced index operations provided by NumPy.

    2) For *really* long expressions that will be called repeatedly, the
    compiled binary should be significantly faster than SymPy's .evalf()

    3) If you are generating code with the codegen utility in order to use it
    in another project, the automatic python wrappers let you test the binaries
    immediately from within SymPy.

When is this module NOT the best approach?

    1) If you are really concerned about speed or memory optimizations, you
    will probably get better results by working directly with the wrapper tools
    and the low level code.  However, the files generated by this utility may
    provide a useful starting point and reference code. Temporary files will be
    left intact if you supply the keyword filepath="path/to/files/".

    2) If the array computation can be handled easily by numpy, and you don't
    need the binaries for another project.

"""
from sympy.utilities.codegen import codegen, get_code_generator, Routine
import sys, os, tempfile, subprocess

class CodeWrapError(Exception): pass

class CodeWrapper:
    _filename = "codewrapper"
    _module_basename = "autowrapped"
    _module_counter = 0

    @property
    def module_name(self):
        return "%s_%s" % (self._module_basename, CodeWrapper._module_counter)

    def __init__(self, generator, filepath=None):
        """
        generator -- the code generator to use
        """
        self.generator = generator
        self.filepath = filepath

    @property
    def include_header(self):
        return bool(self.filepath)

    @property
    def include_empty(self):
        return bool(self.filepath)

    def _generate_code(self, routine):
        self.generator.write([routine], self._filename, True, self.include_header,
                self.include_empty)

    def wrap_code(self, routine):

        workdir = self.filepath or tempfile.mkdtemp("_sympy_compile")
        oldwork = os.getcwd()
        os.chdir(workdir)
        try:
            self._generate_code(routine)
            self._prepare_files(routine)
            self._process_files(routine)
            sys.path.append(workdir)
            mod = __import__(self.module_name)
            sys.path.pop()
        finally:
            CodeWrapper._module_counter +=1
            os.chdir(oldwork)

        return self._get_wrapped_function(mod)

class F2PyCodeWrapper(CodeWrapper):

    def _process_files(self, routine):
        filename = self._filename + '.' + self.generator.dump_f95.extension
        command = ["f2py", "-m", self.module_name, "-c" , filename]
        null = open(os.devnull, 'w')
        retcode = subprocess.call(command, stdout=null)
        if retcode:
            raise CodeWrapError

    def _prepare_files(self, routine):
        pass

    @classmethod
    def _get_wrapped_function(cls, mod):
        return mod.autofunc

def _get_code_wrapper_class(backend):
    wrappers = { 'F2PY': F2PyCodeWrapper }
    return wrappers[backend.upper()]

def autowrap(expr, language='F95', backend='f2py', tempdir=None):
    """Generates python callable binaries based on the math expression.

    expr  --  the SymPy expression that should be wrapped as a binary routine

    Otional arguments:
    language  --  the programming language to use, currently C or F95
    backend  --  the wrapper backend to use, currently f2py or Cython
    tempdir  --  path to directory for temporary files.  If this argument is
                 supplied, the generated code and the wrapper input files are
                 left intact in the specified path.

    >>> from sympy.abc import x, y, z
    >>> from sympy.utilities.autowrap import autowrap
    >>> expr = ((x - y + z)**(13)).expand()
    >>> binary_func = autowrap(expr)
    >>> binary_func(1, 4, 2)
    -1.0

    """

    code_generator = get_code_generator(language, "autowrap")
    CodeWrapperClass = _get_code_wrapper_class(backend)
    code_wrapper = CodeWrapperClass(code_generator, tempdir)
    routine  = Routine('autofunc',expr)
    return code_wrapper.wrap_code(routine)
<commit_msg>autowrap: new method binary_function()<commit_after>"""Module for compiling codegen-output, and wrap it for use in python.

What?

This module provides a common interface for different external backends, such
as f2py, fwrap, Cython, SWIG(?) etc.  The goal is to provide access to compiled
binaries of acceptable performance with a one-button user interface, i.e.

    >>> from sympy.abc import x,y
    >>> from sympy.utilities.autowrap import autowrap
    >>> expr = ((x - y)**(25)).expand()
    >>> binary_callable = autowrap(expr)
    >>> binary_callable(1, 2)           #doctest: +SKIP
    -1.0

The callable returned from autowrap() is a binary python function, not a
Sympy object.  If it is desired to use the compiled function in symbolic
expressions, it is better to use binary_function() which returns a Sympy
Function object.  The binary callable is attached as the _imp_ attribute and
invoked when a numerical evaluation is requested with evalf(), or with
lambdify().

    >>> from sympy.utilities.autowrap import binary_function
    >>> f = binary_function('f', expr)
    >>> 2*f(x, y) + y
    y + 2*f(x, y)
    >>> (2*f(x, y) + y).evalf(2, subs={x: 1, y:2})    #doctest: +SKIP
    0.0

Why?

The idea is that a SymPy user will primarily be interested in working with
mathematical expressions, and should not have to learn details about wrapping
tools in order to evaluate expressions numerically, even if they are
computationally expensive.

When is this useful?

    1) For computations on large arrays, Python iterations may be too slow, and
    depending on the mathematical expression, it may be difficult to exploit
    the advanced index operations provided by NumPy.

    2) For *really* long expressions that will be called repeatedly, the
    compiled binary should be significantly faster than SymPy's .evalf()

    3) If you are generating code with the codegen utility in order to use it
    in another project, the automatic python wrappers let you test the binaries
    immediately from within SymPy.

When is this module NOT the best approach?

    1) If you are really concerned about speed or memory optimizations, you
    will probably get better results by working directly with the wrapper tools
    and the low level code.  However, the files generated by this utility may
    provide a useful starting point and reference code. Temporary files will be
    left intact if you supply the keyword filepath="path/to/files/".

    2) If the array computation can be handled easily by numpy, and you don't
    need the binaries for another project.

"""
from sympy.utilities.codegen import codegen, get_code_generator, Routine
from sympy.utilities.lambdify import implemented_function
import sys, os, tempfile, subprocess

class CodeWrapError(Exception): pass

class CodeWrapper:
    _filename = "codewrapper"
    _module_basename = "autowrapped"
    _module_counter = 0

    @property
    def module_name(self):
        return "%s_%s" % (self._module_basename, CodeWrapper._module_counter)

    def __init__(self, generator, filepath=None):
        """
        generator -- the code generator to use
        """
        self.generator = generator
        self.filepath = filepath

    @property
    def include_header(self):
        return bool(self.filepath)

    @property
    def include_empty(self):
        return bool(self.filepath)

    def _generate_code(self, routine):
        self.generator.write([routine], self._filename, True, self.include_header,
                self.include_empty)

    def wrap_code(self, routine):

        workdir = self.filepath or tempfile.mkdtemp("_sympy_compile")
        oldwork = os.getcwd()
        os.chdir(workdir)
        try:
            self._generate_code(routine)
            self._prepare_files(routine)
            self._process_files(routine)
            sys.path.append(workdir)
            mod = __import__(self.module_name)
            sys.path.pop()
        finally:
            CodeWrapper._module_counter +=1
            os.chdir(oldwork)

        return self._get_wrapped_function(mod)

class F2PyCodeWrapper(CodeWrapper):

    def _process_files(self, routine):
        filename = self._filename + '.' + self.generator.dump_f95.extension
        command = ["f2py", "-m", self.module_name, "-c" , filename]
        null = open(os.devnull, 'w')
        retcode = subprocess.call(command, stdout=null)
        if retcode:
            raise CodeWrapError

    def _prepare_files(self, routine):
        pass

    @classmethod
    def _get_wrapped_function(cls, mod):
        return mod.autofunc

def _get_code_wrapper_class(backend):
    wrappers = { 'F2PY': F2PyCodeWrapper }
    return wrappers[backend.upper()]

def autowrap(expr, language='F95', backend='f2py', tempdir=None):
    """Generates python callable binaries based on the math expression.

    expr  --  the SymPy expression that should be wrapped as a binary routine

    Otional arguments:
    language  --  the programming language to use, currently C or F95
    backend  --  the wrapper backend to use, currently f2py or Cython
    tempdir  --  path to directory for temporary files.  If this argument is
                 supplied, the generated code and the wrapper input files are
                 left intact in the specified path.

    >>> from sympy.abc import x, y, z
    >>> from sympy.utilities.autowrap import autowrap
    >>> expr = ((x - y + z)**(13)).expand()
    >>> binary_func = autowrap(expr)
    >>> binary_func(1, 4, 2)
    -1.0

    """

    code_generator = get_code_generator(language, "autowrap")
    CodeWrapperClass = _get_code_wrapper_class(backend)
    code_wrapper = CodeWrapperClass(code_generator, tempdir)
    routine  = Routine('autofunc',expr)
    return code_wrapper.wrap_code(routine)

def binary_function(symfunc, expr, **kwargs):
    """Returns a sympy function with expr as binary implementation

    This is a convenience function that relies on autowrap() and implemented_function()

    >>> from sympy.abc import x, y, z
    >>> from sympy.utilities.autowrap import binary_function
    >>> expr = ((x - y)**(25)).expand()
    >>> f = binary_function('f', expr)
    >>> type(f)
    <class 'sympy.core.function.FunctionClass'>
    >>> 2*f(x, y)
    2*f(x, y)
    >>> f(x, y).evalf(2, subs={x: 1, y: 2})
    -1.0
    """
    binary = autowrap(expr, **kwargs)
    return implemented_function(symfunc, binary)
<|endoftext|>


<commit_before>"""
This module is intended to hold default constants.
Constants may be modified via project settings.py.
"""
from django.conf import settings

NEWMAN_URL_PREFIX = 'nm'

# AdminSettings
USER_CONFIG = 'newman_user_config'            # session key for AdminSettings JSON data
CATEGORY_FILTER = 'newman_category_filter'    # user defined category filtering on newman HP
NEWMAN_MARKUP_DEFAULT = getattr(settings, 'NEWMAN_MARKUP_DEFAULT', 'markdown')

# list of recipients for error reporting
ERR_REPORT_RECIPIENTS = getattr(settings, 'NEWMAN_ERR_REPORT_RECIPIENTS', ['ella.errors@gmail.com'])

# conversion functions mapping - from JSON to python (i.e. changing item datatypes etc.)
JSON_CONVERSIONS = (
    (CATEGORY_FILTER, 'decode_category_filter_json'),
)

# JsonResponse status "codes" (used instead of HTTP status codes)
STATUS_OK = "ok"
STATUS_GENERIC_ERROR = 'error'
STATUS_SMTP_ERROR = 'smtp_error'
STATUS_ADDED = 'added'
STATUS_MODIFIED = 'modified'
STATUS_FORM_ERROR = 'form_error'
STATUS_VAR_MISSING = 'variable_missing'
STATUS_OBJECT_NOT_FOUND = 'object_not_found'
STATUS_JSON_REDIRECT = 'redirect'
HTTP_OK = 200
HTTP_ERROR = 405

# Maximum autosave objects hold for bound object:
AUTOSAVE_MAX_AMOUNT = getattr(settings, 'NEWMAN_AUTOSAVE_MAX_AMOUNT', 3)

# List of applicable ContentTypes

NON_PUBLISHABLE_CTS = (
    'photos.photo',
    'polls.survey',
)

# Models that have TaggingInlineAdmin in inlines
TAGGED_MODELS = getattr(settings, 'TAGGED_MODELS', (
    'core.publishable',
    'articles.article',
    'galleries.gallery',
    'interviews.interview',
    'polls.quiz',
    # 'photos.photo',
))

NEWMAN_FAVORITE_ITEMS = getattr(settings, 'NEWMAN_FAVORITE_ITEMS', (
    'publishable',
    'article',
    'photo',
    'gallery',
    'survey',
    'quiz',
    'contest',
    'interview',
    'position',
))

# TODO try to load consts from django.conf.settings
<commit_msg>Newman config updated<commit_after>"""
This module is intended to hold default constants.
Constants may be modified via project settings.py.
"""
from django.conf import settings

NEWMAN_URL_PREFIX = 'nm'

# AdminSettings
USER_CONFIG = 'newman_user_config'            # session key for AdminSettings JSON data
CATEGORY_FILTER = 'newman_category_filter'    # user defined category filtering on newman HP
NEWMAN_MARKUP_DEFAULT = getattr(settings, 'NEWMAN_MARKUP_DEFAULT', 'markdown')

# list of recipients for error reporting
ERR_REPORT_RECIPIENTS = getattr(settings, 'NEWMAN_ERR_REPORT_RECIPIENTS', ['ella.errors@gmail.com'])

# conversion functions mapping - from JSON to python (i.e. changing item datatypes etc.)
JSON_CONVERSIONS = (
    (CATEGORY_FILTER, 'decode_category_filter_json'),
)

# JsonResponse status "codes" (used instead of HTTP status codes)
STATUS_OK = "ok"
STATUS_GENERIC_ERROR = 'error'
STATUS_SMTP_ERROR = 'smtp_error'
STATUS_ADDED = 'added'
STATUS_MODIFIED = 'modified'
STATUS_FORM_ERROR = 'form_error'
STATUS_VAR_MISSING = 'variable_missing'
STATUS_OBJECT_NOT_FOUND = 'object_not_found'
STATUS_JSON_REDIRECT = 'redirect'
HTTP_OK = 200
HTTP_ERROR = 405

# Maximum autosave objects hold for bound object:
AUTOSAVE_MAX_AMOUNT = getattr(settings, 'NEWMAN_AUTOSAVE_MAX_AMOUNT', 3)

# List of applicable ContentTypes
NON_PUBLISHABLE_CTS = (
    'photos.photo',
    'polls.poll',
    'polls.survey',
)

# Models that have TaggingInlineAdmin in inlines
TAGGED_MODELS = getattr(settings, 'NEWMAN_TAGGED_MODELS', (
    'core.publishable',
    'articles.article',
    'galleries.gallery',
    'interviews.interview',
    'polls.quiz',
    'polls.contest',
))

NEWMAN_FAVORITE_ITEMS = getattr(settings, 'NEWMAN_FAVORITE_ITEMS', (
    'publishable',
    'article',
    'photo',
    'gallery',
    'survey',
    'quiz',
    'contest',
    'interview',
    'position',
))

# TODO try to load consts from django.conf.settings
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010-2012, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import unittest

from openquake.engine.db import routers
from openquake.engine.db import models as oq

class OQRouterTestCase(unittest.TestCase):
    '''
    Tests for the :py:class:`openquake.engine.db.routers.OQRouter` class.
    '''
    def setUp(self):
        self.router = routers.OQRouter()

    def _db_for_read_helper(self, classes, expected_db):
        '''
        Common test logic
        '''
        for cls in classes:
            self.assertEqual(expected_db, self.router.db_for_read(cls()))

    def _db_for_write_helper(self, classes, expected_db):
        '''
        Common test logic
        '''
        for cls in classes:
            self.assertEqual(expected_db, self.router.db_for_write(cls()))

    def test_admin_correct_read_db(self):
        '''
        For each model in the 'admin' schema, test for proper db routing
        for read operations.
        '''
        classes = [oq.Organization, oq.OqUser, oq.RevisionInfo]
        expected_db = 'admin'

        self._db_for_read_helper(classes, expected_db)

    def test_admin_correct_write_db(self):
        '''
        For each model in the 'admin' schema, test for proper db routing
        for write operations.
        '''
        classes = [oq.Organization, oq.OqUser, oq.RevisionInfo]
        expected_db = 'admin'

        self._db_for_write_helper(classes, expected_db)

    def test_hzrdi_read_schema(self):
        '''
        For each model in the 'hzrdi' schema, test for proper db routing
        for read operations.
        '''
        classes = [oq.ParsedSource, oq.SiteModel]
        expected_db = 'job_init'

        self._db_for_read_helper(classes, expected_db)

    def test_hzrdi_write_schema(self):
        '''
        For each model in the 'hzrdi' schema, test for proper db routing
        for write operations.
        '''
        classes = [oq.ParsedSource, oq.SiteModel]
        expected_db = 'job_init'

        self._db_for_write_helper(classes, expected_db)

    def test_uiapi_read_schema(self):
        '''
        For each model in the 'uiapi' schema, test for proper db routing
        for read operations.
        '''
        classes = [oq.Input, oq.OqJob, oq.Output, oq.ErrorMsg]
        expected_db = 'job_init'

        self._db_for_read_helper(classes, expected_db)

    def test_uiapi_write_schema(self):
        '''
        For each model in the 'uiapi' schema, test for proper db routing
        for write operations.
        '''
        classes = [oq.Input, oq.OqJob, oq.ErrorMsg]
        expected_db = 'job_init'

        self._db_for_write_helper(classes, expected_db)
        self._db_for_write_helper([oq.Output], 'reslt_writer')

    def test_hzrdr_read_schema(self):
        '''
        For each model in the 'hzrdr' schema, test for proper db routing
        for read operations.
        '''
        classes = [oq.HazardMap, oq.HazardCurve, oq.HazardCurveData,
                   oq.GmfData]
        expected_db = 'reslt_writer'

        self._db_for_read_helper(classes, expected_db)

    def test_hzrdr_write_schema(self):
        '''
        For each model in the 'hzrdr' schema, test for proper db routing
        for write operations.
        '''
        classes = [oq.HazardMap, oq.HazardCurve, oq.HazardCurveData,
                   oq.GmfData]

        expected_db = 'reslt_writer'

        self._db_for_write_helper(classes, expected_db)

    def test_riskr_read_schema(self):
        '''
        For each model in the 'riskr' schema, test for proper db routing
        for read operations.
        '''
        classes = [oq.LossMap, oq.LossMapData, oq.LossCurve, oq.LossCurveData,
                   oq.AggregateLossCurveData, oq.BCRDistribution,
                   oq.BCRDistributionData]
        expected_db = 'reslt_writer'

        self._db_for_read_helper(classes, expected_db)

    def test_riskr_write_schema(self):
        '''
        For each model in the 'riskr' schema, test for proper db routing
        for write operations.
        '''
        classes = [oq.LossMap, oq.LossMapData, oq.LossCurve, oq.LossCurveData,
                   oq.AggregateLossCurveData, oq.BCRDistribution,
                   oq.BCRDistributionData]
        expected_db = 'reslt_writer'

        self._db_for_write_helper(classes, expected_db)

    def test_riski_correct_read_db(self):
        '''
        For each model in the 'riski' schema, test for proper db routing
        for read operations.
        '''
        classes = [oq.ExposureModel, oq.ExposureData]
        expected_db = 'reslt_writer'

        self._db_for_read_helper(classes, expected_db)

    def test_riski_correct_write_db(self):
        '''
        For each model in the 'riski' schema, test for proper db routing
        for write operations.
        '''
        classes = [oq.ExposureModel, oq.ExposureData]
        expected_db = 'job_init'

        self._db_for_write_helper(classes, expected_db)
<commit_msg>tests/db/routers_test:<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4

# Copyright (c) 2010-2012, GEM Foundation.
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.

import unittest

from openquake.engine.db import routers
from openquake.engine.db import models as oq

class OQRouterTestCase(unittest.TestCase):
    '''
    Tests for the :py:class:`openquake.engine.db.routers.OQRouter` class.
    '''
    def setUp(self):
        self.router = routers.OQRouter()

    def _db_for_read_helper(self, classes, expected_db):
        '''
        Common test logic
        '''
        for cls in classes:
            self.assertEqual(expected_db, self.router.db_for_read(cls()))

    def _db_for_write_helper(self, classes, expected_db):
        '''
        Common test logic
        '''
        for cls in classes:
            self.assertEqual(expected_db, self.router.db_for_write(cls()))

    def test_admin_correct_read_db(self):
        '''
        For each model in the 'admin' schema, test for proper db routing
        for read operations.
        '''
        classes = [oq.Organization, oq.OqUser, oq.RevisionInfo]
        expected_db = 'admin'

        self._db_for_read_helper(classes, expected_db)

    def test_admin_correct_write_db(self):
        '''
        For each model in the 'admin' schema, test for proper db routing
        for write operations.
        '''
        classes = [oq.Organization, oq.OqUser, oq.RevisionInfo]
        expected_db = 'admin'

        self._db_for_write_helper(classes, expected_db)

    def test_hzrdi_read_schema(self):
        '''
        For each model in the 'hzrdi' schema, test for proper db routing
        for read operations.
        '''
        classes = [oq.ParsedSource, oq.SiteModel]
        expected_db = 'job_init'

        self._db_for_read_helper(classes, expected_db)

    def test_hzrdi_write_schema(self):
        '''
        For each model in the 'hzrdi' schema, test for proper db routing
        for write operations.
        '''
        classes = [oq.ParsedSource, oq.SiteModel]
        expected_db = 'job_init'

        self._db_for_write_helper(classes, expected_db)

    def test_uiapi_read_schema(self):
        '''
        For each model in the 'uiapi' schema, test for proper db routing
        for read operations.
        '''
        classes = [oq.Input, oq.OqJob, oq.Output, oq.ErrorMsg]
        expected_db = 'job_init'

        self._db_for_read_helper(classes, expected_db)

    def test_uiapi_write_schema(self):
        '''
        For each model in the 'uiapi' schema, test for proper db routing
        for write operations.
        '''
        classes = [oq.Input, oq.OqJob, oq.ErrorMsg]
        expected_db = 'job_init'

        self._db_for_write_helper(classes, expected_db)
        self._db_for_write_helper([oq.Output], 'reslt_writer')

    def test_hzrdr_read_schema(self):
        '''
        For each model in the 'hzrdr' schema, test for proper db routing
        for read operations.
        '''
        classes = [oq.HazardMap, oq.HazardCurve, oq.HazardCurveData,
                   oq.GmfData]
        expected_db = 'reslt_writer'

        self._db_for_read_helper(classes, expected_db)

    def test_hzrdr_write_schema(self):
        '''
        For each model in the 'hzrdr' schema, test for proper db routing
        for write operations.
        '''
        classes = [oq.HazardMap, oq.HazardCurve, oq.HazardCurveData,
                   oq.GmfData]

        expected_db = 'reslt_writer'

        self._db_for_write_helper(classes, expected_db)

    def test_riskr_read_schema(self):
        '''
        For each model in the 'riskr' schema, test for proper db routing
        for read operations.
        '''
        classes = [oq.LossMap, oq.LossMapData, oq.LossCurve, oq.LossCurveData,
                   oq.AggregateLossCurveData, oq.BCRDistribution,
                   oq.BCRDistributionData]
        expected_db = 'reslt_writer'

        self._db_for_read_helper(classes, expected_db)

    def test_riskr_write_schema(self):
        '''
        For each model in the 'riskr' schema, test for proper db routing
        for write operations.
        '''
        classes = [oq.LossMap, oq.LossMapData, oq.LossCurve, oq.LossCurveData,
                   oq.AggregateLossCurveData, oq.BCRDistribution,
                   oq.BCRDistributionData]
        expected_db = 'reslt_writer'

        self._db_for_write_helper(classes, expected_db)

    def test_riski_correct_read_db(self):
        '''
        For each model in the 'riski' schema, test for proper db routing
        for read operations.
        '''
        classes = [oq.ExposureModel, oq.ExposureData, oq.Occupancy]
        expected_db = 'job_init'

        self._db_for_read_helper(classes, expected_db)

    def test_riski_correct_write_db(self):
        '''
        For each model in the 'riski' schema, test for proper db routing
        for write operations.
        '''
        classes = [oq.ExposureModel, oq.ExposureData]
        expected_db = 'job_init'

        self._db_for_write_helper(classes, expected_db)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import model
import IsarParser
import PythonSerializer
from collections import namedtuple

def parse(xml_string):
    return IsarParser.IsarParser().parse_string(xml_string)

def test_includes_parsing():
    xml = """\
<system xmlns:xi="http://www.nsn.com/2008/XInclude">
    <xi:include href="mydlo.xml"/>
    <xi:include href="szydlo.xml"/>
    <xi:include href="powidlo.xml"/>
</system>
"""
    holder = parse(xml)

    assert ["mydlo", "szydlo", "powidlo"] == holder.includes

def test_typedefs_primitive_type_parsing():
    xml = """\
<x>
    <typedef name="a" primitiveType="8 bit integer unsigned"/>
    <typedef name="b" primitiveType="16 bit integer unsigned"/>
    <typedef name="c" primitiveType="32 bit integer unsigned"/>
    <typedef name="d" primitiveType="64 bit integer unsigned"/>
    <typedef name="e" primitiveType="8 bit integer signed"/>
    <typedef name="f" primitiveType="16 bit integer signed"/>
    <typedef name="g" primitiveType="32 bit integer signed"/>
    <typedef name="h" primitiveType="64 bit integer signed"/>
    <typedef name="i" primitiveType="32 bit float"/>
    <typedef name="j" primitiveType="64 bit float"/>
</x>
"""
    holder = parse(xml)

    assert [("a", "u8"),
            ("b", "u16"),
            ("c", "u32"),
            ("d", "u64"),
            ("e", "i8"),
            ("f", "i16"),
            ("g", "i32"),
            ("h", "i64"),
            ("i", "r32"),
            ("j", "r64")] == holder.typedefs

def test_typedefs_parsing():
    xml = """<typedef name="TILoveTypedefs_ALot" type="MyType"/>"""
    holder = parse(xml)

    assert [("TILoveTypedefs_ALot", "MyType")] == holder.typedefs

def test_enums_parsing():
    xml = """\
<enum name="EEnum">
    <enum-member name="EEnum_A" value="0"/>
    <enum-member name="EEnum_B" value="1"/>
    <enum-member name="EEnum_C" value="-1"/>
</enum>
"""
    holder = parse(xml)

    assert 1 == len(holder.enums)
    assert "EEnum" == holder.enums[0][0]
    assert [("EEnum_A", "0"), ("EEnum_B", "1"), (u"EEnum_C", "0xFFFFFFFF")] == holder.enums[0][1]

def test_constants_parsing():
    xml = """\
<x>
    <constant name="CONST_A" value="0"/>
    <constant name="CONST_B" value="31"/>
</x>
"""
    holder = parse(xml)

    assert [("CONST_A", "0"), ("CONST_B", "31")] == holder.constants

def test_constants_parsing_and_sorting():
    xml = """\
<x>
    <constant name="C_A" value="C_B + C_C"/>
    <constant name="C_B" value="1"/>
    <constant name="C_C" value="2"/>
</x>
"""
    holder = parse(xml)

    assert [("C_B", "1"), ("C_C", "2"), ("C_A", "C_B + C_C")] == holder.constants

def test_struct_parsing():
    xml = """\
<struct name="Struct">
    <member name="a" type="u8"/>
    <member name="b" type="i64"/>
    <member name="c" type="r32"/>
    <member name="d" type="TTypeX"/>
</struct>
"""
    holder = parse(xml)

    assert 1 == len(holder.structs)
    assert 4 == len(holder.structs[0].members)
    assert "Struct" == holder.structs[0].name
    assert "a" == holder.structs[0].members[0].name
    assert "u8" == holder.structs[0].members[0].type
    assert None == holder.structs[0].members[0].array
    assert "b" == holder.structs[0].members[1].name
    assert "i64" == holder.structs[0].members[1].type
    assert None == holder.structs[0].members[1].array
    assert "c" == holder.structs[0].members[2].name
    assert "r32" == holder.structs[0].members[2].type
    assert None == holder.structs[0].members[2].array
    assert "d" == holder.structs[0].members[3].name
    assert "TTypeX" == holder.structs[0].members[3].type
    assert None == holder.structs[0].members[3].array

def test_struct_parsing_dynamic_array():
    xml = """\
<struct name="StructWithDynamic">
    <member name="x" type="TTypeX">
        <dimension isVariableSize="true"/>
    </member>
</struct>
"""
    holder = parse(xml)

    assert 1 == len(holder.structs)
    assert 2 == len(holder.structs[0].members)
    assert "StructWithDynamic" == holder.structs[0].name
    assert "x_len" == holder.structs[0].members[0].name
    assert "u32" == holder.structs[0].members[0].type
    assert "x" == holder.structs[0].members[1].name
    assert "TTypeX" == holder.structs[0].members[1].type
    assert True == holder.structs[0].members[1].array
    assert "x_len" == holder.structs[0].members[1].array_bound
    assert None == holder.structs[0].members[1].array_size

def test_struct_parsing_static_array():
    xml = """\
<struct name="StructWithStatic">
    <member name="y" type="TTypeY">
        <dimension size="NUM_OF_Y"/>
    </member>
</struct>
"""
    holder = parse(xml)

    assert 1 == len(holder.structs)
    assert 1 == len(holder.structs[0].members)
    assert "StructWithStatic" == holder.structs[0].name
    assert "y" == holder.structs[0].members[0].name
    assert "TTypeY" == holder.structs[0].members[0].type
    assert True == holder.structs[0].members[0].array
    assert None == holder.structs[0].members[0].array_bound
    assert "NUM_OF_Y" == holder.structs[0].members[0].array_size

def test_struct_parsing_dynamic_array_with_typed_sizer():
    xml = """\
<struct name="StructX">
    <member name="x" type="TTypeX">
        <dimension isVariableSize="true" variableSizeFieldType="TNumberOfItems"/>
    </member>
</struct>
"""
    holder = parse(xml)

    assert 1 == len(holder.structs)
    assert 2 == len(holder.structs[0].members)
    assert "StructX" == holder.structs[0].name
    assert "x_len" == holder.structs[0].members[0].name
    assert "TNumberOfItems" == holder.structs[0].members[0].type
    assert None == holder.structs[0].members[0].array
    assert "x" == holder.structs[0].members[1].name
    assert "TTypeX" == holder.structs[0].members[1].type
    assert True == holder.structs[0].members[1].array
    assert "x_len" == holder.structs[0].members[1].array_bound
    assert None == holder.structs[0].members[1].array_size

def test_struct_parsing_dynamic_array_with_named_sizer():
    xml = """\
<struct name="StructX">
    <member name="x" type="TTypeX">
        <dimension isVariableSize="true" variableSizeFieldName="numOfX"/>
    </member>
</struct>
"""
    holder = parse(xml)

    assert 2 == len(holder.structs[0].members)
    assert "numOfX" == holder.structs[0].members[0].name
    assert "u32" == holder.structs[0].members[0].type
    assert "x" == holder.structs[0].members[1].name
    assert "TTypeX" == holder.structs[0].members[1].type
    assert True == holder.structs[0].members[1].array
    assert "numOfX" == holder.structs[0].members[1].array_bound
    assert None == holder.structs[0].members[1].array_size

def test_struct_parsing_dynamic_array_with_named_and_typed_sizer():
    xml = """\
<struct name="StructX">
    <member name="x" type="TTypeX">
        <dimension isVariableSize="true" variableSizeFieldName="numOfX" variableSizeFieldType="TSize"/>
    </member>
</struct>
"""
    holder = parse(xml)

    assert 2 == len(holder.structs[0].members)
    assert "numOfX" == holder.structs[0].members[0].name
    assert "TSize" == holder.structs[0].members[0].type
    assert "x" == holder.structs[0].members[1].name
    assert "TTypeX" == holder.structs[0].members[1].type
    assert True == holder.structs[0].members[1].array
    assert "numOfX" == holder.structs[0].members[1].array_bound
    assert None == holder.structs[0].members[1].array_size
<commit_msg>next test written<commit_after># -*- coding: utf-8 -*-

import model
import IsarParser
import PythonSerializer
from collections import namedtuple

def parse(xml_string):
    return IsarParser.IsarParser().parse_string(xml_string)

def test_includes_parsing():
    xml = """\
<system xmlns:xi="http://www.nsn.com/2008/XInclude">
    <xi:include href="mydlo.xml"/>
    <xi:include href="szydlo.xml"/>
    <xi:include href="powidlo.xml"/>
</system>
"""
    holder = parse(xml)

    assert ["mydlo", "szydlo", "powidlo"] == holder.includes

def test_typedefs_primitive_type_parsing():
    xml = """\
<x>
    <typedef name="a" primitiveType="8 bit integer unsigned"/>
    <typedef name="b" primitiveType="16 bit integer unsigned"/>
    <typedef name="c" primitiveType="32 bit integer unsigned"/>
    <typedef name="d" primitiveType="64 bit integer unsigned"/>
    <typedef name="e" primitiveType="8 bit integer signed"/>
    <typedef name="f" primitiveType="16 bit integer signed"/>
    <typedef name="g" primitiveType="32 bit integer signed"/>
    <typedef name="h" primitiveType="64 bit integer signed"/>
    <typedef name="i" primitiveType="32 bit float"/>
    <typedef name="j" primitiveType="64 bit float"/>
</x>
"""
    holder = parse(xml)

    assert [("a", "u8"),
            ("b", "u16"),
            ("c", "u32"),
            ("d", "u64"),
            ("e", "i8"),
            ("f", "i16"),
            ("g", "i32"),
            ("h", "i64"),
            ("i", "r32"),
            ("j", "r64")] == holder.typedefs

def test_typedefs_parsing():
    xml = """<typedef name="TILoveTypedefs_ALot" type="MyType"/>"""
    holder = parse(xml)

    assert [("TILoveTypedefs_ALot", "MyType")] == holder.typedefs

def test_enums_parsing():
    xml = """\
<enum name="EEnum">
    <enum-member name="EEnum_A" value="0"/>
    <enum-member name="EEnum_B" value="1"/>
    <enum-member name="EEnum_C" value="-1"/>
</enum>
"""
    holder = parse(xml)

    assert 1 == len(holder.enums)
    assert "EEnum" == holder.enums[0][0]
    assert [("EEnum_A", "0"), ("EEnum_B", "1"), (u"EEnum_C", "0xFFFFFFFF")] == holder.enums[0][1]

def test_constants_parsing():
    xml = """\
<x>
    <constant name="CONST_A" value="0"/>
    <constant name="CONST_B" value="31"/>
</x>
"""
    holder = parse(xml)

    assert [("CONST_A", "0"), ("CONST_B", "31")] == holder.constants

def test_constants_parsing_and_sorting():
    xml = """\
<x>
    <constant name="C_A" value="C_B + C_C"/>
    <constant name="C_B" value="1"/>
    <constant name="C_C" value="2"/>
</x>
"""
    holder = parse(xml)

    assert [("C_B", "1"), ("C_C", "2"), ("C_A", "C_B + C_C")] == holder.constants

def test_struct_parsing():
    xml = """\
<struct name="Struct">
    <member name="a" type="u8"/>
    <member name="b" type="i64"/>
    <member name="c" type="r32"/>
    <member name="d" type="TTypeX"/>
</struct>
"""
    holder = parse(xml)

    assert 1 == len(holder.structs)
    assert 4 == len(holder.structs[0].members)
    assert "Struct" == holder.structs[0].name
    assert "a" == holder.structs[0].members[0].name
    assert "u8" == holder.structs[0].members[0].type
    assert None == holder.structs[0].members[0].array
    assert "b" == holder.structs[0].members[1].name
    assert "i64" == holder.structs[0].members[1].type
    assert None == holder.structs[0].members[1].array
    assert "c" == holder.structs[0].members[2].name
    assert "r32" == holder.structs[0].members[2].type
    assert None == holder.structs[0].members[2].array
    assert "d" == holder.structs[0].members[3].name
    assert "TTypeX" == holder.structs[0].members[3].type
    assert None == holder.structs[0].members[3].array

def test_struct_parsing_dynamic_array():
    xml = """\
<struct name="StructWithDynamic">
    <member name="x" type="TTypeX">
        <dimension isVariableSize="true"/>
    </member>
</struct>
"""
    holder = parse(xml)

    assert 1 == len(holder.structs)
    assert 2 == len(holder.structs[0].members)
    assert "StructWithDynamic" == holder.structs[0].name
    assert "x_len" == holder.structs[0].members[0].name
    assert "u32" == holder.structs[0].members[0].type
    assert "x" == holder.structs[0].members[1].name
    assert "TTypeX" == holder.structs[0].members[1].type
    assert True == holder.structs[0].members[1].array
    assert "x_len" == holder.structs[0].members[1].array_bound
    assert None == holder.structs[0].members[1].array_size

def test_struct_parsing_static_array():
    xml = """\
<struct name="StructWithStatic">
    <member name="y" type="TTypeY">
        <dimension size="NUM_OF_Y"/>
    </member>
</struct>
"""
    holder = parse(xml)

    assert 1 == len(holder.structs)
    assert 1 == len(holder.structs[0].members)
    assert "StructWithStatic" == holder.structs[0].name
    assert "y" == holder.structs[0].members[0].name
    assert "TTypeY" == holder.structs[0].members[0].type
    assert True == holder.structs[0].members[0].array
    assert None == holder.structs[0].members[0].array_bound
    assert "NUM_OF_Y" == holder.structs[0].members[0].array_size

def test_struct_parsing_dynamic_array_with_typed_sizer():
    xml = """\
<struct name="StructX">
    <member name="x" type="TTypeX">
        <dimension isVariableSize="true" variableSizeFieldType="TNumberOfItems"/>
    </member>
</struct>
"""
    holder = parse(xml)

    assert 1 == len(holder.structs)
    assert 2 == len(holder.structs[0].members)
    assert "StructX" == holder.structs[0].name
    assert "x_len" == holder.structs[0].members[0].name
    assert "TNumberOfItems" == holder.structs[0].members[0].type
    assert None == holder.structs[0].members[0].array
    assert "x" == holder.structs[0].members[1].name
    assert "TTypeX" == holder.structs[0].members[1].type
    assert True == holder.structs[0].members[1].array
    assert "x_len" == holder.structs[0].members[1].array_bound
    assert None == holder.structs[0].members[1].array_size

def test_struct_parsing_dynamic_array_with_named_sizer():
    xml = """\
<struct name="StructX">
    <member name="x" type="TTypeX">
        <dimension isVariableSize="true" variableSizeFieldName="numOfX"/>
    </member>
</struct>
"""
    holder = parse(xml)

    assert 2 == len(holder.structs[0].members)
    assert "numOfX" == holder.structs[0].members[0].name
    assert "u32" == holder.structs[0].members[0].type
    assert "x" == holder.structs[0].members[1].name
    assert "TTypeX" == holder.structs[0].members[1].type
    assert True == holder.structs[0].members[1].array
    assert "numOfX" == holder.structs[0].members[1].array_bound
    assert None == holder.structs[0].members[1].array_size

def test_struct_parsing_dynamic_array_with_named_and_typed_sizer():
    xml = """\
<struct name="StructX">
    <member name="x" type="TTypeX">
        <dimension isVariableSize="true" variableSizeFieldName="numOfX" variableSizeFieldType="TSize"/>
    </member>
</struct>
"""
    holder = parse(xml)

    assert 2 == len(holder.structs[0].members)
    assert "numOfX" == holder.structs[0].members[0].name
    assert "TSize" == holder.structs[0].members[0].type
    assert "x" == holder.structs[0].members[1].name
    assert "TTypeX" == holder.structs[0].members[1].type
    assert True == holder.structs[0].members[1].array
    assert "numOfX" == holder.structs[0].members[1].array_bound
    assert None == holder.structs[0].members[1].array_size

def test_message_parsing():
    xml = """\
<message name="MsgX">
    <member name="x" type="TTypeX"/>
</message>
"""
    holder = parse(xml)

    assert 1 == len(holder.structs)
    assert [("x", "TTypeX", None, None, None)] == holder.structs[0].members
<|endoftext|>