<commit_before>#!/usr/bin/env python
# vim: autoindent tabstop=4 expandtab shiftwidth=4 softtabstop=4

"""
Backup borker

Run this inside a git repository with hourly snapshots.
"""

from datetime import datetime, timedelta
import optparse
import os
import re
import subprocess
import sys

MY_ENVIRON = {
    'GIT_AUTHOR_NAME': 'borkbak',
    'GIT_AUTHOR_EMAIL': 'borkbak@feinheit.ch',
    'GIT_COMMITTER_NAME': 'borkbak',
    'GIT_COMMITTER_EMAIL': 'borkbak@feinheit.ch',
    }

for key in ('PWD', 'PATH'):
    MY_ENVIRON[key] = os.environ.get(key, '')

def borkbak():
    parser = optparse.OptionParser()
    parser.add_option('', '--ref', dest='ref',
        help='Set ref to create/update [default: %default]',
        default='refs/heads/borkbak')
    parser.add_option('-q', '--quiet', dest='verbose', action='store_false',
        help='Do not print status messages',
        default=True)
    parser.add_option('', '--prune', dest='prune', action='store_true',
        help='Prune unreachable objects',
        default=False)
    options, args = parser.parse_args()
    if args:
        parser.error('Don\'t try argu(ment)ing with me')

    # keys which have been used (generated using commit timestamps)
    occupied = set()
    # trees which should be preserved and strung using newly created commits
    keep = []

    now = datetime.now()

    for tree_id, timestamp, original_timestamp in get_backups():
        days = (now - timestamp).days

        if days < 7:
            key = timestamp.strftime('original-%Y-%m-%d-%H-%M')
        elif days < 30:
            key = timestamp.strftime('daily-%Y-%m-%d')
        elif days < 180:
            key = timestamp.strftime('weekly-%Y-%W')
        else:
            key = timestamp.strftime('monthly-%Y-%m')

        if key in occupied:
            continue
        occupied.add(key)

        keep.append((tree_id, timestamp, original_timestamp, key))

    if options.verbose:
        print 'Recreating history for selected backups...'

    # first commit has no parent
    commit_id = None
    items = len(keep)

    for idx, (tree_id, timestamp, original_timestamp, key) in enumerate(keep):
        commit_id = create_commit(tree_id, original_timestamp, key, commit_id)

        if options.verbose:
            sys.stdout.write('\r%s/%s' % (idx+1, items))
            sys.stdout.flush()

    p = subprocess.call(['git', 'update-ref', options.ref, commit_id])
    if options.verbose:
        print '\nUpdated ref %s.' % options.ref

    if options.prune:
        args = ['git', 'gc', '--prune']

        if options.verbose:
            print 'Expiring all reflogs and collecting garbage...'
        else:
            args.append('--quiet')

        subprocess.call(['git', 'reflog', 'expire', '--all', '--expire=0', '--expire-unreachable=0'])
        subprocess.call(args)
        subprocess.call(['git', 'prune'])
    elif options.verbose:
        print 'Not removing anything since you did not tell me to do so.'

def create_commit(tree_id, timestamp, key, parent=None):
    args = ['git', 'commit-tree', tree_id]
    if parent:
        args.extend(['-p', parent])
    MY_ENVIRON['GIT_COMMITTER_DATE'] = timestamp
    MY_ENVIRON['GIT_AUTHOR_DATE'] = timestamp
    p = subprocess.Popen(args, env=MY_ENVIRON, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    ret = p.communicate('%s\n' % key)[0]
    return ret.strip()

def get_backups():
    # we only need the commit's tree and timestamp from the old history
    p = subprocess.Popen(['git', 'log', '--format=%T %ct'], stdout=subprocess.PIPE)
    output = p.communicate()[0]

    LOG_RE = re.compile(r'(?P<tree_id>.{40}) (?P<timestamp>\d+)$')

    ret = []
    for line in output.splitlines():
        matches = LOG_RE.search(line)
        if not matches:
            continue

        ret.append((
            matches.group('tree_id'),
            datetime.fromtimestamp(int(matches.group('timestamp'))),
            matches.group('timestamp'),
            ))

    return reversed(ret)

if __name__ == '__main__':
    borkbak()

<commit_msg>Modernized coding style<commit_after>#!/usr/bin/env python
# vim: autoindent tabstop=4 expandtab shiftwidth=4 softtabstop=4

from __future__ import print_function, unicode_literals

"""
Backup borker

Run this inside a git repository with hourly snapshots.
"""

from datetime import datetime
import optparse
import os
import re
import subprocess
import sys

MY_ENVIRON = {
    'GIT_AUTHOR_NAME': 'borkbak',
    'GIT_AUTHOR_EMAIL': 'borkbak@feinheit.ch',
    'GIT_COMMITTER_NAME': 'borkbak',
    'GIT_COMMITTER_EMAIL': 'borkbak@feinheit.ch',
}

for key in ('PWD', 'PATH'):
    MY_ENVIRON[key] = os.environ.get(key, '')

def borkbak():
    parser = optparse.OptionParser()
    parser.add_option(
        '',
        '--ref',
        dest='ref',
        help='Set ref to create/update [default: %default]',
        default='refs/heads/borkbak')
    parser.add_option(
        '-q',
        '--quiet',
        dest='verbose',
        action='store_false',
        help='Do not print status messages',
        default=True)
    parser.add_option(
        '',
        '--prune',
        dest='prune',
        action='store_true',
        help='Prune unreachable objects',
        default=False)
    options, args = parser.parse_args()
    if args:
        parser.error('Don\'t try argu(ment)ing with me')

    # keys which have been used (generated using commit timestamps)
    occupied = set()
    # trees which should be preserved and strung using newly created commits
    keep = []

    now = datetime.now()

    for tree_id, timestamp, original_timestamp in get_backups():
        days = (now - timestamp).days

        if days < 7:
            key = timestamp.strftime('original-%Y-%m-%d-%H-%M')
        elif days < 30:
            key = timestamp.strftime('daily-%Y-%m-%d')
        elif days < 180:
            key = timestamp.strftime('weekly-%Y-%W')
        else:
            key = timestamp.strftime('monthly-%Y-%m')

        if key in occupied:
            continue
        occupied.add(key)

        keep.append((tree_id, timestamp, original_timestamp, key))

    if options.verbose:
        print('Recreating history for selected backups...')

    # first commit has no parent
    commit_id = None
    items = len(keep)

    for idx, (tree_id, timestamp, original_timestamp, key) in enumerate(keep):
        commit_id = create_commit(tree_id, original_timestamp, key, commit_id)

        if options.verbose:
            sys.stdout.write('\r%s/%s' % (idx+1, items))
            sys.stdout.flush()

    subprocess.call(['git', 'update-ref', options.ref, commit_id])
    if options.verbose:
        print('\nUpdated ref %s.' % options.ref)

    if options.prune:
        args = ['git', 'gc', '--prune']

        if options.verbose:
            print('Expiring all reflogs and collecting garbage...')
        else:
            args.append('--quiet')

        subprocess.call([
            'git', 'reflog', 'expire', '--all', '--expire=0',
            '--expire-unreachable=0',
        ])
        subprocess.call(args)
        subprocess.call(['git', 'prune'])
    elif options.verbose:
        print('Not removing anything since you did not tell me to do so.')

def create_commit(tree_id, timestamp, key, parent=None):
    args = ['git', 'commit-tree', tree_id]
    if parent:
        args.extend(['-p', parent])
    MY_ENVIRON['GIT_COMMITTER_DATE'] = timestamp
    MY_ENVIRON['GIT_AUTHOR_DATE'] = timestamp
    p = subprocess.Popen(
        args, env=MY_ENVIRON, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    ret = p.communicate('%s\n' % key)[0]
    return ret.strip()

def get_backups():
    # we only need the commit's tree and timestamp from the old history
    p = subprocess.Popen(
        ['git', 'log', '--format=%T %ct'],
        stdout=subprocess.PIPE)
    output = p.communicate()[0]

    LOG_RE = re.compile(r'(?P<tree_id>.{40}) (?P<timestamp>\d+)$')

    ret = []
    for line in output.splitlines():
        matches = LOG_RE.search(line)
        if not matches:
            continue

        ret.append((
            matches.group('tree_id'),
            datetime.fromtimestamp(int(matches.group('timestamp'))),
            matches.group('timestamp'),
        ))

    return reversed(ret)

if __name__ == '__main__':
    borkbak()
<|endoftext|>


<commit_before>import os, glob, re
from lxml import etree

def parseStyle(independentStyle):
    style = etree.parse (independentStyle)
    styleElement = style.getroot()
    metadata = {}
    try:
        metadata["id"] = styleElement.find(".//{http://purl.org/net/xbiblio/csl}id").text
        metadata["selfLink"] = styleElement.find(".//{http://purl.org/net/xbiblio/csl}link[@rel='self']").attrib.get("href")
        metadata["template"] = styleElement.find(".//{http://purl.org/net/xbiblio/csl}link[@rel='template']").attrib.get("href")
    except:
        pass
    return(metadata)

path = 'C:\Users\Rintze Zelle\Documents\git\styles\\'
metadataList = []
metadata = {}
for independentStyle in glob.glob( os.path.join(path, '*.csl') ):
    fileName = os.path.basename(independentStyle)

    if not(re.match("[a-z0-9](-?[a-z0-9]+)*(.csl)", fileName)):
        print("Non-conforming filename: " + fileName)

    metadata = parseStyle(independentStyle)
    metadata["fileName"] = fileName

    try:
        if not(("http://www.zotero.org/styles/"+fileName) == (metadata["selfLink"]+".csl")):
            print("ID - cs:link[@rel=self] mismatch: " + fileName)
    except:
        print("Missing cs:link[@rel=self] value: " + fileName)
    try:
        if not(("http://www.zotero.org/styles/"+fileName) == (metadata["id"]+".csl")):
            print("ID - filename mismatch: " + fileName)
    except:
        print("Missing cs:id content: " + fileName)

    metadataList.append(metadata)

#for set in metadataList:
#    print(set)
#    print(set["fileName"])

#print(idList)

# Use dict instead? Can use filename as key
# What do I really want? Easy way to compare id, self-link and filename, and compare values to other sets

# Start with independent styles
# Store filename, ID, self and template links
# Report error for missing ID and self link
# Check whether filename conforms to requirements ( ^[a-z0-9][
# Split ID and self link in "http://www.zotero.org/styles/" and filename part
<commit_msg>Update to QC script<commit_after># Python script for additional style validation
# Author: Rintze M. Zelle
# Requires lxml library (http://lxml.de/)
# Checks
# - whether ".csl" files conform to naming scheme [a-z0-9] with optional single
#   hyphen delimiters
# - whether style ID (content of cs:id) matches the URI generated by the Zotero
#   Style Repository:
#   "http://www.zotero.org/styles/" + filename - extension
# - whether "self" link (value of "href" on cs:link with rel="self") matches the
#   Zotero Style Repository
# - for independent styles, whether "template" link (value of "href" on cs:link
#   with rel="template") points to the Zotero Style Repository style URI of an
#   independent style.
# - for dependent styles, whether "independent-parent" link (value of "href" on
#   cs:link with rel="template") points to the Zotero Style Repository style URI
#   of an independent style.

import os, glob, re
from lxml import etree

path = 'C:\Users\Rintze Zelle\Documents\git\styles\\'

def parseStyle(stylePath):
    style = etree.parse(stylePath)
    styleElement = style.getroot()
    metadata = {}
    try:
        metadata["id"] = styleElement.find(".//{http://purl.org/net/xbiblio/csl}id").text
        metadata["selfLink"] = styleElement.find(".//{http://purl.org/net/xbiblio/csl}link[@rel='self']").attrib.get("href")
    except:
        pass
    try:
        metadata["template"] = styleElement.find(".//{http://purl.org/net/xbiblio/csl}link[@rel='template']").attrib.get("href")
    except:
        pass
    try:
        metadata["independentParent"] = styleElement.find(".//{http://purl.org/net/xbiblio/csl}link[@rel='independent-parent']").attrib.get("href")
    except:
        pass
    return(metadata)

def checkFileName(fileName):
    if not(re.match("[a-z0-9](-?[a-z0-9]+)*(.csl)", fileName)):
        print("Non-conforming filename: " + fileName)

metadataList = []
metadata = {}
for independentStyle in glob.glob( os.path.join(path, '*.csl') ):
    fileName = os.path.basename(independentStyle)

    checkFileName(fileName)

    metadata = parseStyle(independentStyle)
    metadata["fileName"] = fileName

    try:
        if not(("http://www.zotero.org/styles/"+fileName) == (metadata["selfLink"]+".csl")):
            print("Name Mismatch - Filename & Style URI (value 'href' on cs:link[@rel=self]): " + fileName)
    except:
        print("Missing Style URI (value 'href' on cs:link[@rel=self]): " + fileName)
    try:
        if not(("http://www.zotero.org/styles/"+fileName) == (metadata["id"]+".csl")):
            print("Name Mismatch - Filename & Style ID (content cs:id): " + fileName)
    except:
        print("Missing Style ID (content cs:id): " + fileName)

    metadataList.append(metadata)

for queryMetadataDict in metadataList:
    match = True
    try:
        if(queryMetadataDict["template"]):
            match = False
            for metadataDict in metadataList:
                if(queryMetadataDict["template"] == metadataDict["selfLink"]):
                    match = True
            if(match == False):
                print("Non-existing Style Template URI (value 'href' on cs:link[@rel=template]): " + queryMetadataDict["fileName"])
    except:
        pass

metadataListDependents = []
metadataDependents = {}
for dependentStyle in glob.glob( os.path.join(path, "dependent", '*.csl') ):
    fileName = os.path.basename(dependentStyle)

    checkFileName(fileName)

    metadataDependents = parseStyle(dependentStyle)
    metadataDependents["fileName"] = fileName

    try:
        if not(("http://www.zotero.org/styles/"+fileName) == (metadataDependents["selfLink"]+".csl")):
            print("Name Mismatch - Filename & Style URI (value 'href' on cs:link[@rel=self]): (dependent) " + fileName)
    except:
        pass
    try:
        if not(("http://www.zotero.org/styles/"+fileName) == (metadataDependents["id"]+".csl")):
            print("Name Mismatch - Filename & Style ID (content cs:id): (dependent) " + fileName)
    except:
        print("Missing Style ID (content cs:id): (dependent) " + fileName)
    try:
        metadataDependents["independentParent"]
    except:
        print("Missing Parent Style ID value 'href' on cs:link[@rel=independent-parent]): (dependent) " + fileName)

    metadataListDependents.append(metadataDependents)

for queryMetadataDict in metadataListDependents:
    match = True
    try:
        if(queryMetadataDict["independentParent"]):
            match = False
            for metadataDict in metadataList:
                if(queryMetadataDict["independentParent"] == metadataDict["selfLink"]):
                    match = True
            if(match == False):
                print("Non-existing Parent Style URI (value 'href' on cs:link[@rel=independent-parent]): (dependent) " + queryMetadataDict["fileName"])
    except:
        pass
<|endoftext|>


<commit_before># Be able to call directly such as `python test_annotators.py`
try:
    from .context import loctext
except SystemError: # Parent module '' not loaded, cannot perform relative import
    pass

from loctext.util import PRO_ID, LOC_ID, REL_PRO_LOC_ID
from nalaf.learning.evaluators import DocumentLevelRelationEvaluator, Evaluations
from nalaf.learning.taggers import StubSameSentenceRelationExtractor, StubRelationExtractor
from loctext.learning.train import read_corpus, evaluate_with_argv
from nalaf import print_verbose, print_debug
from nalaf.preprocessing.edges import SentenceDistanceEdgeGenerator
import math
import sys

# See conftest.py too
TEST_MIN_CORPUS_PERCENTAGE = 0.4

def test_SS_baseline():
    corpus = read_corpus("LocText")

    # Baseline Computation(precision=0.6077586206896551, precision_SE=0.002781236497769688, recall=0.6238938053097345, recall_SE=0.004052746798458239, f_measure=0.6157205240174672, f_measure_SE=0.002826453358117873)
    EXPECTED_F = 0.6157
    EXPECTED_F_SE = 0.0028

    annotator_gen_fun = (lambda _: StubSameSentenceRelationExtractor(PRO_ID, LOC_ID, REL_PRO_LOC_ID).annotate)
    evaluator = DocumentLevelRelationEvaluator(rel_type=REL_PRO_LOC_ID, match_case=False)

    evaluations = Evaluations.cross_validate(annotator_gen_fun, corpus, evaluator, k_num_folds=5, use_validation_set=True)
    rel_evaluation = evaluations(REL_PRO_LOC_ID).compute(strictness="exact")

    assert math.isclose(rel_evaluation.f_measure, EXPECTED_F, abs_tol=EXPECTED_F_SE * 1.1), rel_evaluation.f_measure
    print("SS Baseline", rel_evaluation)

    return rel_evaluation

def _test_LocText(corpus_percentage, model, EXPECTED_F=None, EXPECTED_F_SE=0.001):
    # Note: EXPECTED_F=None will make the test fail for non-yet verified evaluations
    # Note: the real StdErr's are around ~0.0027-0.0095. Decrease them by default to be more strict with tests

    assert corpus_percentage in [TEST_MIN_CORPUS_PERCENTAGE, 1.0], "corpus_percentage must == {} or 1.0. You gave: {}".format(str(TEST_MIN_CORPUS_PERCENTAGE), str(corpus_percentage))

    corpus = read_corpus("LocText", corpus_percentage)

    rel_evaluation = evaluate_with_argv(['--corpus_percentage', str(corpus_percentage), '--model', model])

    print("LocText " + model, rel_evaluation)
    assert math.isclose(rel_evaluation.f_measure, EXPECTED_F, abs_tol=EXPECTED_F_SE * 1.1)

    return rel_evaluation

def test_LocText_SS(corpus_percentage):

    if (corpus_percentage == 1.0):
        # Computation(precision=0.6624365482233503, precision_SE=0.0029261497595035445, recall=0.5787139689578714, recall_SE=0.004036629092741261, f_measure=0.6177514792899409, f_measure_SE=0.0027412422752843557)
        EXPECTED_F = 0.6178
        EXPECTED_F_SE = 0.0027
    else:
        # Computation(precision=0.7426470588235294, precision_SE=0.004447039958950779, recall=0.6121212121212121, recall_SE=0.005972946581336089, f_measure=0.6710963455149502, f_measure_SE=0.0043836182031360155)
        EXPECTED_F = 0.6711
        EXPECTED_F_SE = 0.0043

    _test_LocText(corpus_percentage, model='SS', EXPECTED_F=EXPECTED_F)

def test_DS_baseline():
    corpus = read_corpus("LocText")

    # Computation(precision=0.4149560117302053, precision_SE=0.002457395067634639, recall=0.6206140350877193, recall_SE=0.0037411116863785665, f_measure=0.49736379613356774, f_measure_SE=0.00233567117305811)
    EXPECTED_F = 0.4974
    EXPECTED_F_SE = 0.0023

    edge_generator = SentenceDistanceEdgeGenerator(PRO_ID, LOC_ID, REL_PRO_LOC_ID, distance=1)

    annotator_gen_fun = (lambda _: StubRelationExtractor(edge_generator).annotate)
    evaluator = DocumentLevelRelationEvaluator(rel_type=REL_PRO_LOC_ID, match_case=False)

    evaluations = Evaluations.cross_validate(annotator_gen_fun, corpus, evaluator, k_num_folds=5, use_validation_set=True)
    rel_evaluation = evaluations(REL_PRO_LOC_ID).compute(strictness="exact")

    assert math.isclose(rel_evaluation.f_measure, EXPECTED_F, abs_tol=EXPECTED_F_SE * 1.1), rel_evaluation.f_measure
    print("DS Baseline", rel_evaluation)

    return rel_evaluation

def test_LocText_DS(corpus_percentage):

    if (corpus_percentage == 1.0):
        # Computation(precision=0.41743119266055045, precision_SE=0.0030117270329129047, recall=0.40176600441501104, recall_SE=0.0032167797183620065, f_measure=0.4094488188976378, f_measure_SE=0.0025398561212811644)
        EXPECTED_F = 0.4094
        EXPECTED_F_SE = 0.0024
    else:
        # Computation(precision=0.5254237288135594, precision_SE=0.004919604548886615, recall=0.3712574850299401, recall_SE=0.00506665114212087, f_measure=0.43508771929824563, f_measure_SE=0.004266620575873462)
        EXPECTED_F = 0.4351
        EXPECTED_F_SE = 0.0043

    _test_LocText(corpus_percentage, model='DS', EXPECTED_F=EXPECTED_F)

# # TODO find the way to do the training only once for each submodel
# def test_LocText_Combined(corpus_percentage):
#
#     # Should be now the same as SS results, since the DS model, as of now, _SHOULD_ DO NOTHING
#     if (corpus_percentage == 1.0):
#         EXPECTED_F = 0.6178
#         EXPECTED_F_SE = 0.0027
#     else:
#         EXPECTED_F = 0.6711
#         EXPECTED_F_SE = 0.0043
#
#     _test_LocText(corpus_percentage, model='Combined', EXPECTED_F=EXPECTED_F)

if __name__ == "__main__":

    test_baseline()

    corpus_percentage = float(sys.argv[2]) if len(sys.argv) == 3 else TEST_MIN_CORPUS_PERCENTAGE
    test_LocText_SS(corpus_percentage)
<commit_msg>successfully implement & test the Combined model, now with both SS and DS indeed contributing a good share<commit_after># Be able to call directly such as `python test_annotators.py`
try:
    from .context import loctext
except SystemError: # Parent module '' not loaded, cannot perform relative import
    pass

from loctext.util import PRO_ID, LOC_ID, REL_PRO_LOC_ID
from nalaf.learning.evaluators import DocumentLevelRelationEvaluator, Evaluations
from nalaf.learning.taggers import StubSameSentenceRelationExtractor, StubRelationExtractor
from loctext.learning.train import read_corpus, evaluate_with_argv
from nalaf import print_verbose, print_debug
from nalaf.preprocessing.edges import SentenceDistanceEdgeGenerator
import math
import sys

# See conftest.py too
TEST_MIN_CORPUS_PERCENTAGE = 0.4

def test_SS_baseline():
    corpus = read_corpus("LocText")

    # Baseline Computation(precision=0.6077586206896551, precision_SE=0.002781236497769688, recall=0.6238938053097345, recall_SE=0.004052746798458239, f_measure=0.6157205240174672, f_measure_SE=0.002826453358117873)
    EXPECTED_F = 0.6157
    EXPECTED_F_SE = 0.0028

    annotator_gen_fun = (lambda _: StubSameSentenceRelationExtractor(PRO_ID, LOC_ID, REL_PRO_LOC_ID).annotate)
    evaluator = DocumentLevelRelationEvaluator(rel_type=REL_PRO_LOC_ID, match_case=False)

    evaluations = Evaluations.cross_validate(annotator_gen_fun, corpus, evaluator, k_num_folds=5, use_validation_set=True)
    rel_evaluation = evaluations(REL_PRO_LOC_ID).compute(strictness="exact")

    assert math.isclose(rel_evaluation.f_measure, EXPECTED_F, abs_tol=EXPECTED_F_SE * 1.1), rel_evaluation.f_measure
    print("SS Baseline", rel_evaluation)

    return rel_evaluation

def _test_LocText(corpus_percentage, model, EXPECTED_F=None, EXPECTED_F_SE=0.001):
    # Note: EXPECTED_F=None will make the test fail for non-yet verified evaluations
    # Note: the real StdErr's are around ~0.0027-0.0095. Decrease them by default to be more strict with tests

    assert corpus_percentage in [TEST_MIN_CORPUS_PERCENTAGE, 1.0], "corpus_percentage must == {} or 1.0. You gave: {}".format(str(TEST_MIN_CORPUS_PERCENTAGE), str(corpus_percentage))

    corpus = read_corpus("LocText", corpus_percentage)

    rel_evaluation = evaluate_with_argv(['--corpus_percentage', str(corpus_percentage), '--model', model])

    print("LocText " + model, rel_evaluation)
    assert math.isclose(rel_evaluation.f_measure, EXPECTED_F, abs_tol=EXPECTED_F_SE * 1.1)

    return rel_evaluation

def test_LocText_SS(corpus_percentage):

    if (corpus_percentage == 1.0):
        # Computation(precision=0.6624365482233503, precision_SE=0.0029261497595035445, recall=0.5787139689578714, recall_SE=0.004036629092741261, f_measure=0.6177514792899409, f_measure_SE=0.0027412422752843557)
        EXPECTED_F = 0.6178
        EXPECTED_F_SE = 0.0027
    else:
        # Computation(precision=0.7426470588235294, precision_SE=0.004447039958950779, recall=0.6121212121212121, recall_SE=0.005972946581336089, f_measure=0.6710963455149502, f_measure_SE=0.0043836182031360155)
        EXPECTED_F = 0.6711
        EXPECTED_F_SE = 0.0043

    _test_LocText(corpus_percentage, model='SS', EXPECTED_F=EXPECTED_F)

def test_DS_baseline():
    corpus = read_corpus("LocText")

    # Computation(precision=0.4149560117302053, precision_SE=0.002457395067634639, recall=0.6206140350877193, recall_SE=0.0037411116863785665, f_measure=0.49736379613356774, f_measure_SE=0.00233567117305811)
    EXPECTED_F = 0.4974
    EXPECTED_F_SE = 0.0023

    edge_generator = SentenceDistanceEdgeGenerator(PRO_ID, LOC_ID, REL_PRO_LOC_ID, distance=1)

    annotator_gen_fun = (lambda _: StubRelationExtractor(edge_generator).annotate)
    evaluator = DocumentLevelRelationEvaluator(rel_type=REL_PRO_LOC_ID, match_case=False)

    evaluations = Evaluations.cross_validate(annotator_gen_fun, corpus, evaluator, k_num_folds=5, use_validation_set=True)
    rel_evaluation = evaluations(REL_PRO_LOC_ID).compute(strictness="exact")

    assert math.isclose(rel_evaluation.f_measure, EXPECTED_F, abs_tol=EXPECTED_F_SE * 1.1), rel_evaluation.f_measure
    print("DS Baseline", rel_evaluation)

    return rel_evaluation

def test_LocText_DS(corpus_percentage):

    if (corpus_percentage == 1.0):
        # Computation(precision=0.41743119266055045, precision_SE=0.0030117270329129047, recall=0.40176600441501104, recall_SE=0.0032167797183620065, f_measure=0.4094488188976378, f_measure_SE=0.0025398561212811644)
        EXPECTED_F = 0.4094
        EXPECTED_F_SE = 0.0024
    else:
        # Computation(precision=0.5254237288135594, precision_SE=0.004919604548886615, recall=0.3712574850299401, recall_SE=0.00506665114212087, f_measure=0.43508771929824563, f_measure_SE=0.004266620575873462)
        EXPECTED_F = 0.4351
        EXPECTED_F_SE = 0.0043

    _test_LocText(corpus_percentage, model='DS', EXPECTED_F=EXPECTED_F)

# Note: would be way better to be able to reuse the already trained models in the other tests methods
def test_LocText_Combined(corpus_percentage):

    if (corpus_percentage == 1.0):
        # Computation(precision=0.4774381368267831, precision_SE=0.002812150029056132, recall=0.7177242888402626, recall_SE=0.0038763518483362733, f_measure=0.5734265734265734, f_measure_SE=0.002422393130974523)
        EXPECTED_F = 0.5734
        EXPECTED_F_SE = 0.0024
    else:
        # Computation(precision=0.5844748858447488, precision_SE=0.0032793659586925978, recall=0.7710843373493976, recall_SE=0.004115510298431316, f_measure=0.6649350649350649, f_measure_SE=0.002684996682841366)
        EXPECTED_F = 0.6649
        EXPECTED_F_SE = 0.0027

    _test_LocText(corpus_percentage, model='Combined', EXPECTED_F=EXPECTED_F)

if __name__ == "__main__":

    test_baseline()

    corpus_percentage = float(sys.argv[2]) if len(sys.argv) == 3 else TEST_MIN_CORPUS_PERCENTAGE
    test_LocText_SS(corpus_percentage)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""Runs integration tests."""

import argparse
import os
import platform
import sys
import re

script_dir = os.path.dirname(os.path.realpath(__file__))
docker_dir = os.path.join(script_dir, 'bamboos', 'docker')
sys.path.insert(0, docker_dir)
from environment import docker

def parse_valgrind_log_error_count(log_file):
    """
    Parses valgrind memcheck file and returns the identified error count.
    """
    with open(log_file, 'r') as f:
        regex = re.compile("ERROR SUMMARY:\s(\d+)\serrors")
        for line in f:
            match = re.search(regex, line)
            if match:
                return int(match.groups()[0])
        raise SystemExit("Invalid Valgrind memcheck report file: "+log_file)

parser = argparse.ArgumentParser(
    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    description='Run Common Tests.')

parser.add_argument(
    '--gdb',
    action='store_true',
    default=False,
    help='run tests in GDB')

parser.add_argument(
    '--valgrind',
    action='store_true',
    default=False,
    help='run tests under Valgrind',
    dest='valgrind')

parser.add_argument(
    '--callgrind',
    action='store_true',
    default=False,
    help='run tests under Valgrind callgrind tool',
    dest='callgrind')

parser.add_argument(
    '--image', '-i',
    action='store',
    default='onedata/builder:2102-2',
    help='docker image to use as a test master',
    dest='image')

parser.add_argument(
    '--release',
    action='store',
    default='release',
    help='release directory to run tests from',
    dest='release')

parser.add_argument(
    '--suite',
    action='append',
    default=[],
    help='name of the test suite',
    dest='suites')

[args, pass_args] = parser.parse_known_args()
script_dir = os.path.dirname(os.path.realpath(__file__))
base_test_dir = os.path.join(os.path.realpath(args.release), 'test',
                             'integration')
test_dirs = map(lambda suite: os.path.join(base_test_dir, suite), args.suites)
if args.valgrind:
    if len(test_dirs) != 1:
        raise SystemExit('Valgrind test run requires specification of a single '
                         'test case suite, e.g. \'--suite ceph_helper_test\'')
    if args.gdb:
        raise SystemExit('GDB and Valgrind cannot be used simultanously for '
                         'tests')

if not test_dirs:
    test_dirs = [base_test_dir]

command = '''
import os, subprocess, sys, stat

if {shed_privileges}:
    os.environ['HOME'] = '/tmp'
    docker_gid = os.stat('/var/run/docker.sock').st_gid
    os.chmod('/etc/resolv.conf', 0o666)
    os.setgroups([docker_gid])
    os.setregid({gid}, {gid})
    os.setreuid({uid}, {uid})

if {gdb}:
    command = ['gdb', 'python', '-silent', '-ex', """run -c "
import pytest
pytest.main({args} + ['{test_dirs}'])" """]
elif {valgrind}:
    command = ['valgrind'] \\
            + ['--gen-suppressions=all'] \\
            + ['--suppressions=valgrind.supp'] \\
            + ['--track-origins=yes'] \\
            + ['--log-file=valgrind-{suite}.txt'] \\
            + ['--show-leak-kinds=definite'] \\
            + ['--leak-check=full'] \\
            + ['py.test'] + {args} + ['{test_dirs}']
elif {callgrind}:
    command = ['valgrind'] \\
            + ['--tool=callgrind'] \\
            + ['py.test'] + {args} + ['{test_dirs}']
else:
    command = ['py.test'] + {args} + ['{test_dirs}']

ret = subprocess.call(command)
sys.exit(ret)
'''
command = command.format(
    args=pass_args,
    uid=os.geteuid(),
    gid=os.getegid(),
    test_dirs="', '".join(test_dirs),
    shed_privileges=(platform.system() == 'Linux'),
    gdb=args.gdb,
    valgrind=args.valgrind,
    callgrind=args.callgrind,
    suite=(args.suites[0] if args.valgrind else "', '".join(test_dirs)))

docker.run(tty=True,
           rm=True,
           interactive=True,
           workdir=script_dir,
           reflect=[(script_dir, 'rw'),
                    ('/var/run/docker.sock', 'rw')],
           image=args.image,
           envs={'BASE_TEST_DIR': base_test_dir},
           run_params=['--privileged'] if (args.gdb or args.valgrind) else [],
           command=['python', '-c', command])

# If exit code != 0 then bamboo always fails build.
# If it is 0 then result is based on test report.
ret = 0
# If Valgrind was enabled, parse the memcheck report
# and return error if any errors were identified
if args.valgrind:
    ret = parse_valgrind_log_error_count("valgrind-"+args.suites[0]+".txt")

sys.exit(ret)
<commit_msg>VFS-8073 Updated ctrun builder image<commit_after>#!/usr/bin/env python

"""Runs integration tests."""

import argparse
import os
import platform
import sys
import re

script_dir = os.path.dirname(os.path.realpath(__file__))
docker_dir = os.path.join(script_dir, 'bamboos', 'docker')
sys.path.insert(0, docker_dir)
from environment import docker

def parse_valgrind_log_error_count(log_file):
    """
    Parses valgrind memcheck file and returns the identified error count.
    """
    with open(log_file, 'r') as f:
        regex = re.compile("ERROR SUMMARY:\s(\d+)\serrors")
        for line in f:
            match = re.search(regex, line)
            if match:
                return int(match.groups()[0])
        raise SystemExit("Invalid Valgrind memcheck report file: "+log_file)

parser = argparse.ArgumentParser(
    formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    description='Run Common Tests.')

parser.add_argument(
    '--gdb',
    action='store_true',
    default=False,
    help='run tests in GDB')

parser.add_argument(
    '--valgrind',
    action='store_true',
    default=False,
    help='run tests under Valgrind',
    dest='valgrind')

parser.add_argument(
    '--callgrind',
    action='store_true',
    default=False,
    help='run tests under Valgrind callgrind tool',
    dest='callgrind')

parser.add_argument(
    '--image', '-i',
    action='store',
    default='onedata/builder:2102-3',
    help='docker image to use as a test master',
    dest='image')

parser.add_argument(
    '--release',
    action='store',
    default='release',
    help='release directory to run tests from',
    dest='release')

parser.add_argument(
    '--suite',
    action='append',
    default=[],
    help='name of the test suite',
    dest='suites')

[args, pass_args] = parser.parse_known_args()
script_dir = os.path.dirname(os.path.realpath(__file__))
base_test_dir = os.path.join(os.path.realpath(args.release), 'test',
                             'integration')
test_dirs = map(lambda suite: os.path.join(base_test_dir, suite), args.suites)
if args.valgrind:
    if len(test_dirs) != 1:
        raise SystemExit('Valgrind test run requires specification of a single '
                         'test case suite, e.g. \'--suite ceph_helper_test\'')
    if args.gdb:
        raise SystemExit('GDB and Valgrind cannot be used simultanously for '
                         'tests')

if not test_dirs:
    test_dirs = [base_test_dir]

command = '''
import os, subprocess, sys, stat

if {shed_privileges}:
    os.environ['HOME'] = '/tmp'
    docker_gid = os.stat('/var/run/docker.sock').st_gid
    os.chmod('/etc/resolv.conf', 0o666)
    os.setgroups([docker_gid])
    os.setregid({gid}, {gid})
    os.setreuid({uid}, {uid})

if {gdb}:
    command = ['gdb', 'python', '-silent', '-ex', """run -c "
import pytest
pytest.main({args} + ['{test_dirs}'])" """]
elif {valgrind}:
    command = ['valgrind'] \\
            + ['--gen-suppressions=all'] \\
            + ['--suppressions=valgrind.supp'] \\
            + ['--track-origins=yes'] \\
            + ['--log-file=valgrind-{suite}.txt'] \\
            + ['--show-leak-kinds=definite'] \\
            + ['--leak-check=full'] \\
            + ['py.test'] + {args} + ['{test_dirs}']
elif {callgrind}:
    command = ['valgrind'] \\
            + ['--tool=callgrind'] \\
            + ['py.test'] + {args} + ['{test_dirs}']
else:
    command = ['py.test'] + {args} + ['{test_dirs}']

ret = subprocess.call(command)
sys.exit(ret)
'''
command = command.format(
    args=pass_args,
    uid=os.geteuid(),
    gid=os.getegid(),
    test_dirs="', '".join(test_dirs),
    shed_privileges=(platform.system() == 'Linux'),
    gdb=args.gdb,
    valgrind=args.valgrind,
    callgrind=args.callgrind,
    suite=(args.suites[0] if args.valgrind else "', '".join(test_dirs)))

docker.run(tty=True,
           rm=True,
           interactive=True,
           workdir=script_dir,
           reflect=[(script_dir, 'rw'),
                    ('/var/run/docker.sock', 'rw')],
           image=args.image,
           envs={'BASE_TEST_DIR': base_test_dir},
           run_params=['--privileged'] if (args.gdb or args.valgrind) else [],
           command=['python', '-c', command])

# If exit code != 0 then bamboo always fails build.
# If it is 0 then result is based on test report.
ret = 0
# If Valgrind was enabled, parse the memcheck report
# and return error if any errors were identified
if args.valgrind:
    ret = parse_valgrind_log_error_count("valgrind-"+args.suites[0]+".txt")

sys.exit(ret)
<|endoftext|>


<commit_before># Be able to call directly such as `python test_annotators.py`
try:
    from .context import loctext
except SystemError: # Parent module '' not loaded, cannot perform relative import
    pass

from loctext.util import PRO_ID, LOC_ID, REL_PRO_LOC_ID
from nalaf.learning.evaluators import DocumentLevelRelationEvaluator, Evaluations
from loctext.learning.annotators import LocTextBaselineRelationExtractor
from loctext.learning.train import read_corpus, train
from nalaf import print_verbose, print_debug
import math

k_num_folds = 5
use_validation_set = True

def test_baseline():
    corpus = read_corpus("LocText")
    EXPECTED_F = 0.4234297812279464
    EXPECTED_F_SE = 0.0024623653397242064

    annotator_fun = (lambda _: LocTextBaselineRelationExtractor(PRO_ID, LOC_ID, REL_PRO_LOC_ID))
    evaluator = DocumentLevelRelationEvaluator(rel_type=REL_PRO_LOC_ID, match_case=False)

    evaluations = Evaluations.cross_validate(annotator_fun, corpus, evaluator, k_num_folds, use_validation_set=use_validation_set)
    rel_evaluation = evaluations(REL_PRO_LOC_ID).compute(strictness="exact")

    assert math.isclose(rel_evaluation.f_measure, EXPECTED_F)
    assert math.isclose(rel_evaluation.f_measure_SE, EXPECTED_F_SE, rel_tol=0.1)

    return rel_evaluation

def test_LocText():
    corpus = read_corpus("LocText")
    EXPECTED_F = 0.5095
    EXPECTED_F_SE = 0.0028

    annotator_fun = (lambda train_set: train(train_set, {'use_tk': False}))
    evaluator = DocumentLevelRelationEvaluator(rel_type=REL_PRO_LOC_ID, match_case=False)

    evaluations = Evaluations.cross_validate(annotator_fun, corpus, evaluator, k_num_folds, use_validation_set=use_validation_set)
    rel_evaluation = evaluations(REL_PRO_LOC_ID).compute(strictness="exact")

    assert math.isclose(rel_evaluation.f_measure, EXPECTED_F)
    assert math.isclose(rel_evaluation.f_measure_SE, EXPECTED_F_SE, abs_tol=0.01)

    return rel_evaluation

if __name__ == "__main__":
    test_baseline()
    test_LocText()
<commit_msg>be more flexible<commit_after># Be able to call directly such as `python test_annotators.py`
try:
    from .context import loctext
except SystemError: # Parent module '' not loaded, cannot perform relative import
    pass

from loctext.util import PRO_ID, LOC_ID, REL_PRO_LOC_ID
from nalaf.learning.evaluators import DocumentLevelRelationEvaluator, Evaluations
from loctext.learning.annotators import LocTextBaselineRelationExtractor
from loctext.learning.train import read_corpus, train
from nalaf import print_verbose, print_debug
import math

k_num_folds = 5
use_validation_set = True

def test_baseline():
    corpus = read_corpus("LocText")
    EXPECTED_F = 0.4234297812279464
    EXPECTED_F_SE = 0.0024623653397242064

    annotator_fun = (lambda _: LocTextBaselineRelationExtractor(PRO_ID, LOC_ID, REL_PRO_LOC_ID))
    evaluator = DocumentLevelRelationEvaluator(rel_type=REL_PRO_LOC_ID, match_case=False)

    evaluations = Evaluations.cross_validate(annotator_fun, corpus, evaluator, k_num_folds, use_validation_set=use_validation_set)
    rel_evaluation = evaluations(REL_PRO_LOC_ID).compute(strictness="exact")

    assert math.isclose(rel_evaluation.f_measure, EXPECTED_F)
    assert math.isclose(rel_evaluation.f_measure_SE, EXPECTED_F_SE, rel_tol=0.1)

    return rel_evaluation

def test_LocText():
    corpus = read_corpus("LocText")
    EXPECTED_F = 0.5095
    EXPECTED_F_SE = 0.0028

    annotator_fun = (lambda train_set: train(train_set, {'use_tk': False}))
    evaluator = DocumentLevelRelationEvaluator(rel_type=REL_PRO_LOC_ID, match_case=False)

    evaluations = Evaluations.cross_validate(annotator_fun, corpus, evaluator, k_num_folds, use_validation_set=use_validation_set)
    rel_evaluation = evaluations(REL_PRO_LOC_ID).compute(strictness="exact")

    assert math.isclose(rel_evaluation.f_measure, EXPECTED_F,  abs_tol=0.0001)
    assert math.isclose(rel_evaluation.f_measure_SE, EXPECTED_F_SE, abs_tol=0.01)

    return rel_evaluation

if __name__ == "__main__":
    test_baseline()
    test_LocText()
<|endoftext|>


<commit_before>import time

from bot.action.core.action import Action
from bot.action.core.update import Update
from bot.action.standard.config_status import ConfigStatus
from bot.action.util.textformat import FormattedText
from bot.api.api import Api
from bot.api.telegram import TelegramBotApi
from bot.logger.admin_logger import AdminLogger
from bot.multithreading.scheduler import SchedulerApi
from bot.storage import Config, Cache
from bot.storage import State

CONFIG_DIR = "config"
STATE_DIR = "state"

class Bot:
    def __init__(self):
        self.config = Config(CONFIG_DIR)
        self.state = State(STATE_DIR)
        self.cache = Cache()
        debug = self.config.debug()
        telegram_api = TelegramBotApi(self.config.auth_token, self.config.reuse_connections(), debug)
        self.api = Api(telegram_api, self.state)
        self.logger = AdminLogger(self.api, self.config.admin_chat_id, debug, self.config.send_error_tracebacks())
        self.scheduler = SchedulerApi(self.logger.work_error)
        if self.config.async():
            self.scheduler.setup()
            self.api.enable_async(self.scheduler)
        self.cache.bot_info = self.api.getMe()
        self.action = Action()
        self.update_processor = UpdateProcessor(self.action, self.logger)

    def set_action(self, action: Action):
        action.setup(self.api, self.config, self.state, self.cache, self.scheduler)
        self.action = action
        self.update_processor = UpdateProcessor(self.action, self.logger)

    def run(self):
        self.starting()
        try:
            self.main_loop()
        except KeyboardInterrupt:
            self.logger.info("KeyboardInterrupt")
        except SystemExit as e:
            self.logger.info("SystemExit: " + str(e))
            raise e
        except BaseException as e:
            self.logger.error(e, "Fatal error")
            raise e
        finally:
            self.shutdown()

    def starting(self):
        self.logger.info_formatted_text(
            FormattedText().bold("Starting"),
            *ConfigStatus(self.config, self.state).get_config_status()
        )

    def main_loop(self):
        while True:
            self.process_pending_updates()
            self.process_normal_updates()

    def process_pending_updates(self):
        PendingUpdatesProcessor(self.api.get_pending_updates, self.logger, self.config, self.update_processor).run()

    def process_normal_updates(self):
        NormalUpdatesProcessor(self.api.get_updates, self.logger, self.config, self.update_processor).run()

    def shutdown(self):
        if self.config.async():
            self.scheduler.shutdown()
        self.logger.info("Finished")

class UpdateProcessor:
    def __init__(self, action: Action, logger: AdminLogger):
        self.action = action
        self.logger = logger

    def process_update(self, update: Update):
        try:
            self.action.process(update)
        except Exception as e:
            # As logger errors are probably API failures that the next updates may also get,
            # let them to be propagated so that no more updates are processed before waiting some time
            self.logger.error(e, "process_update")

class UpdatesProcessor:
    def __init__(self, get_updates_func: callable, logger: AdminLogger, config: Config,
                 update_processor: UpdateProcessor):
        self.get_updates_func = get_updates_func
        self.logger = logger
        self.config = config
        self.update_processor = update_processor
        self.last_error = None
        self.number_of_updates_processed = 0

    def run(self):
        self.processing_starting()
        try:
            self.__processing_loop()
        finally:
            self.processing_ended()
        self.processing_ended_successfully()

    def __processing_loop(self):
        while self.should_keep_processing_updates():
            self.__get_and_process_handling_errors()

    def __get_and_process_handling_errors(self):
        try:
            self.__get_and_process()
        except Exception as e:
            self.__handle_error(e)
            # notify there has been an error
            self.processing_error(e)
        else:
            # notify successful processing
            self.processing_successful()

    def __get_and_process(self):
        for update in self.get_updates_func():
            self.update_processor.process_update(update)
            self.number_of_updates_processed += 1

    def __handle_error(self, error: Exception):
        sleep_seconds = self.config.sleep_seconds_on_get_updates_error
        # we do not want to let non-fatal (eg. API) errors to escape from here
        self.safe_log_error(error, "get_and_process", "Sleeping for {seconds} seconds.".format(seconds=sleep_seconds))
        # there has been an error while getting updates, sleep a little to give a chance
        # for the server or the network to recover (if that was the case), and to not to flood the server
        time.sleep(int(sleep_seconds))

    def safe_log_error(self, error: Exception, *info: str):
        """Log error failing silently on error"""
        self.__do_safe(lambda: self.logger.error(error, *info))

    def safe_log_info(self, *info: str):
        """Log info failing silently on error"""
        self.__do_safe(lambda: self.logger.info(*info))

    @staticmethod
    def __do_safe(func: callable):
        try:
            return func()
        except:
            pass

    def should_keep_processing_updates(self):
        raise NotImplementedError()

    def processing_successful(self):
        """Updates were processed successfully"""
        self.last_error = None

    def processing_error(self, error: Exception):
        """There has been an error while processing the last updates"""
        self.last_error = error

    def processing_starting(self):
        """Updates are about to start being processed"""
        pass

    def processing_ended(self):
        """Processing has ended, we don't know if successfully or caused by an error"""
        self.safe_log_info(
            "Ending",
            "Updates processed: {updates_processed_number}"
                .format(updates_processed_number=self.number_of_updates_processed)
        )

    def processing_ended_successfully(self):
        """Processing has ended successfully"""
        pass

class PendingUpdatesProcessor(UpdatesProcessor):
    def __init__(self, get_updates_func: callable, logger: AdminLogger, config: Config,
                 update_processor: UpdateProcessor):
        super().__init__(get_updates_func, logger, config, update_processor)
        # set to some value other than None to let the processing run the first time
        self.last_error = True

    def should_keep_processing_updates(self):
        # if there has been an error not all pending updates were processed
        # so try again until it ends without error
        was_error = self.last_error is not None
        if was_error and self.last_error is not True:
            self.safe_log_info("Restarting", "Recovered from error. Continue processing pending updates...")
        return was_error

    def processing_starting(self):
        self.safe_log_info("Pending", "Processing pending updates...")

    def processing_ended_successfully(self):
        self.safe_log_info("Continuing", "Pending updates successfully processed.")

class NormalUpdatesProcessor(UpdatesProcessor):
    def __init__(self, get_updates_func: callable, logger: AdminLogger, config: Config,
                 update_processor: UpdateProcessor):
        super().__init__(get_updates_func, logger, config, update_processor)
        self.last_successful_processing = time.time()

    def processing_successful(self):
        super().processing_successful()
        self.last_successful_processing = time.time()

    def should_keep_processing_updates(self):
        if self.last_error is None:
            # if last processing ended without error, keep going!
            return True
        error_seconds_in_normal_mode = time.time() - self.last_successful_processing
        max_error_seconds_allowed_in_normal_mode = int(self.config.max_error_seconds_allowed_in_normal_mode)
        if error_seconds_in_normal_mode > max_error_seconds_allowed_in_normal_mode:
            # it has happened too much time since last successful processing
            # although it does not mean no update have been processed, we are
            # having problems and updates are being delayed, so going back to
            # process pending updates mode
            self.safe_log_info(
                "Restarting",
                "Exceeded {max_seconds} maximum seconds with errors (current value: {seconds} seconds)"
                    .format(max_seconds=max_error_seconds_allowed_in_normal_mode,
                            seconds=int(error_seconds_in_normal_mode)),
                "Switching to pending updates mode."
            )
            return False
        else:
            self.safe_log_info(
                "Restarted",
                "Recovered from error (current error burst duration is {seconds} seconds of "
                "{max_seconds} maximum seconds allowed)"
                    .format(seconds=int(error_seconds_in_normal_mode),
                            max_seconds=max_error_seconds_allowed_in_normal_mode),
                "Continuing in normal updates mode."
            )
            return True

    def processing_starting(self):
        self.safe_log_info("Started", "Switched to normal updates mode.")
<commit_msg>Shutdown scheduler unconditionally<commit_after>import time

from bot.action.core.action import Action
from bot.action.core.update import Update
from bot.action.standard.config_status import ConfigStatus
from bot.action.util.textformat import FormattedText
from bot.api.api import Api
from bot.api.telegram import TelegramBotApi
from bot.logger.admin_logger import AdminLogger
from bot.multithreading.scheduler import SchedulerApi
from bot.storage import Config, Cache
from bot.storage import State

CONFIG_DIR = "config"
STATE_DIR = "state"

class Bot:
    def __init__(self):
        self.config = Config(CONFIG_DIR)
        self.state = State(STATE_DIR)
        self.cache = Cache()
        debug = self.config.debug()
        telegram_api = TelegramBotApi(self.config.auth_token, self.config.reuse_connections(), debug)
        self.api = Api(telegram_api, self.state)
        self.logger = AdminLogger(self.api, self.config.admin_chat_id, debug, self.config.send_error_tracebacks())
        self.scheduler = SchedulerApi(self.logger.work_error)
        if self.config.async():
            self.scheduler.setup()
            self.api.enable_async(self.scheduler)
        self.cache.bot_info = self.api.getMe()
        self.action = Action()
        self.update_processor = UpdateProcessor(self.action, self.logger)

    def set_action(self, action: Action):
        action.setup(self.api, self.config, self.state, self.cache, self.scheduler)
        self.action = action
        self.update_processor = UpdateProcessor(self.action, self.logger)

    def run(self):
        self.starting()
        try:
            self.main_loop()
        except KeyboardInterrupt:
            self.logger.info("KeyboardInterrupt")
        except SystemExit as e:
            self.logger.info("SystemExit: " + str(e))
            raise e
        except BaseException as e:
            self.logger.error(e, "Fatal error")
            raise e
        finally:
            self.shutdown()

    def starting(self):
        self.logger.info_formatted_text(
            FormattedText().bold("Starting"),
            *ConfigStatus(self.config, self.state).get_config_status()
        )

    def main_loop(self):
        while True:
            self.process_pending_updates()
            self.process_normal_updates()

    def process_pending_updates(self):
        PendingUpdatesProcessor(self.api.get_pending_updates, self.logger, self.config, self.update_processor).run()

    def process_normal_updates(self):
        NormalUpdatesProcessor(self.api.get_updates, self.logger, self.config, self.update_processor).run()

    def shutdown(self):
        self.scheduler.shutdown()
        self.logger.info("Finished")

class UpdateProcessor:
    def __init__(self, action: Action, logger: AdminLogger):
        self.action = action
        self.logger = logger

    def process_update(self, update: Update):
        try:
            self.action.process(update)
        except Exception as e:
            # As logger errors are probably API failures that the next updates may also get,
            # let them to be propagated so that no more updates are processed before waiting some time
            self.logger.error(e, "process_update")

class UpdatesProcessor:
    def __init__(self, get_updates_func: callable, logger: AdminLogger, config: Config,
                 update_processor: UpdateProcessor):
        self.get_updates_func = get_updates_func
        self.logger = logger
        self.config = config
        self.update_processor = update_processor
        self.last_error = None
        self.number_of_updates_processed = 0

    def run(self):
        self.processing_starting()
        try:
            self.__processing_loop()
        finally:
            self.processing_ended()
        self.processing_ended_successfully()

    def __processing_loop(self):
        while self.should_keep_processing_updates():
            self.__get_and_process_handling_errors()

    def __get_and_process_handling_errors(self):
        try:
            self.__get_and_process()
        except Exception as e:
            self.__handle_error(e)
            # notify there has been an error
            self.processing_error(e)
        else:
            # notify successful processing
            self.processing_successful()

    def __get_and_process(self):
        for update in self.get_updates_func():
            self.update_processor.process_update(update)
            self.number_of_updates_processed += 1

    def __handle_error(self, error: Exception):
        sleep_seconds = self.config.sleep_seconds_on_get_updates_error
        # we do not want to let non-fatal (eg. API) errors to escape from here
        self.safe_log_error(error, "get_and_process", "Sleeping for {seconds} seconds.".format(seconds=sleep_seconds))
        # there has been an error while getting updates, sleep a little to give a chance
        # for the server or the network to recover (if that was the case), and to not to flood the server
        time.sleep(int(sleep_seconds))

    def safe_log_error(self, error: Exception, *info: str):
        """Log error failing silently on error"""
        self.__do_safe(lambda: self.logger.error(error, *info))

    def safe_log_info(self, *info: str):
        """Log info failing silently on error"""
        self.__do_safe(lambda: self.logger.info(*info))

    @staticmethod
    def __do_safe(func: callable):
        try:
            return func()
        except:
            pass

    def should_keep_processing_updates(self):
        raise NotImplementedError()

    def processing_successful(self):
        """Updates were processed successfully"""
        self.last_error = None

    def processing_error(self, error: Exception):
        """There has been an error while processing the last updates"""
        self.last_error = error

    def processing_starting(self):
        """Updates are about to start being processed"""
        pass

    def processing_ended(self):
        """Processing has ended, we don't know if successfully or caused by an error"""
        self.safe_log_info(
            "Ending",
            "Updates processed: {updates_processed_number}"
                .format(updates_processed_number=self.number_of_updates_processed)
        )

    def processing_ended_successfully(self):
        """Processing has ended successfully"""
        pass

class PendingUpdatesProcessor(UpdatesProcessor):
    def __init__(self, get_updates_func: callable, logger: AdminLogger, config: Config,
                 update_processor: UpdateProcessor):
        super().__init__(get_updates_func, logger, config, update_processor)
        # set to some value other than None to let the processing run the first time
        self.last_error = True

    def should_keep_processing_updates(self):
        # if there has been an error not all pending updates were processed
        # so try again until it ends without error
        was_error = self.last_error is not None
        if was_error and self.last_error is not True:
            self.safe_log_info("Restarting", "Recovered from error. Continue processing pending updates...")
        return was_error

    def processing_starting(self):
        self.safe_log_info("Pending", "Processing pending updates...")

    def processing_ended_successfully(self):
        self.safe_log_info("Continuing", "Pending updates successfully processed.")

class NormalUpdatesProcessor(UpdatesProcessor):
    def __init__(self, get_updates_func: callable, logger: AdminLogger, config: Config,
                 update_processor: UpdateProcessor):
        super().__init__(get_updates_func, logger, config, update_processor)
        self.last_successful_processing = time.time()

    def processing_successful(self):
        super().processing_successful()
        self.last_successful_processing = time.time()

    def should_keep_processing_updates(self):
        if self.last_error is None:
            # if last processing ended without error, keep going!
            return True
        error_seconds_in_normal_mode = time.time() - self.last_successful_processing
        max_error_seconds_allowed_in_normal_mode = int(self.config.max_error_seconds_allowed_in_normal_mode)
        if error_seconds_in_normal_mode > max_error_seconds_allowed_in_normal_mode:
            # it has happened too much time since last successful processing
            # although it does not mean no update have been processed, we are
            # having problems and updates are being delayed, so going back to
            # process pending updates mode
            self.safe_log_info(
                "Restarting",
                "Exceeded {max_seconds} maximum seconds with errors (current value: {seconds} seconds)"
                    .format(max_seconds=max_error_seconds_allowed_in_normal_mode,
                            seconds=int(error_seconds_in_normal_mode)),
                "Switching to pending updates mode."
            )
            return False
        else:
            self.safe_log_info(
                "Restarted",
                "Recovered from error (current error burst duration is {seconds} seconds of "
                "{max_seconds} maximum seconds allowed)"
                    .format(seconds=int(error_seconds_in_normal_mode),
                            max_seconds=max_error_seconds_allowed_in_normal_mode),
                "Continuing in normal updates mode."
            )
            return True

    def processing_starting(self):
        self.safe_log_info("Started", "Switched to normal updates mode.")
<|endoftext|>


<commit_before>import pytest

import bonobo
from bonobo.execution import GraphExecutionContext
from mock import patch

@pytest.mark.timeout(2)
def test_run_graph_noop():
    graph = bonobo.Graph(bonobo.noop)
    assert len(graph) == 1

    with patch('bonobo._api._is_interactive_console', side_effect=lambda: False):
        result = bonobo.run(graph)
    assert isinstance(result, GraphExecutionContext)
<commit_msg>mock is in unittest<commit_after>import pytest

import bonobo
from bonobo.execution import GraphExecutionContext
from unittest.mock import patch

@pytest.mark.timeout(2)
def test_run_graph_noop():
    graph = bonobo.Graph(bonobo.noop)
    assert len(graph) == 1

    with patch('bonobo._api._is_interactive_console', side_effect=lambda: False):
        result = bonobo.run(graph)
    assert isinstance(result, GraphExecutionContext)
<|endoftext|>


<commit_before>import time

from bot.action.core.action import Action
from bot.action.core.update import Update
from bot.api.api import Api
from bot.api.telegram import TelegramBotApi
from bot.logger.admin_logger import AdminLogger
from bot.multithreading.scheduler import SchedulerApi
from bot.storage import Config, Cache
from bot.storage import State

CONFIG_DIR = "config"
STATE_DIR = "state"

class Bot:
    def __init__(self):
        self.config = Config(CONFIG_DIR)
        self.state = State(STATE_DIR)
        self.cache = Cache()
        debug = self.config.debug()
        telegram_api = TelegramBotApi(self.config.auth_token, self.config.reuse_connections(), debug)
        self.api = Api(telegram_api, self.state)
        self.logger = AdminLogger(self.api, self.config.admin_chat_id, debug, self.config.send_error_tracebacks())
        self.scheduler = SchedulerApi(self.logger.work_error)
        if self.config.async():
            self.scheduler.setup()
            self.api.enable_async(self.scheduler)
        self.cache.bot_info = self.api.getMe()
        self.action = Action()
        self.update_processor = UpdateProcessor(self.action, self.logger)

    def set_action(self, action: Action):
        action.setup(self.api, self.config, self.state, self.cache, self.scheduler)
        self.action = action
        self.update_processor = UpdateProcessor(self.action, self.logger)

    def run(self):
        self.logger.info(
            "Starting",
            "async: {async}".format(async=self.config.async()),
            "Reusing connections: {reuse_connections}".format(reuse_connections=self.config.reuse_connections()),
            "debug: {debug}".format(debug=self.config.debug()),
            "Error tracebacks: {error_tracebacks}".format(error_tracebacks=self.config.send_error_tracebacks())
        )
        try:
            self.main_loop()
        except KeyboardInterrupt:
            self.logger.info("KeyboardInterrupt")
        except SystemExit as e:
            self.logger.info("SystemExit: " + str(e))
            raise e
        except BaseException as e:
            self.logger.error(e, "Fatal error")
            raise e
        finally:
            self.shutdown()

    def main_loop(self):
        while True:
            self.process_pending_updates()
            self.process_normal_updates()

    def process_pending_updates(self):
        processor = PendingUpdatesProcessor(self.api.get_pending_updates, self.logger, self.config,
                                            self.update_processor)
        processor.run()
        self.logger.info("Started", "All pending updates processed. There were: {pending_updates_number}"
                         .format(pending_updates_number=processor.number_of_updates_processed))

    def process_normal_updates(self):
        NormalUpdatesProcessor(self.api.get_updates, self.logger, self.config, self.update_processor).run()

    def shutdown(self):
        if self.config.async():
            self.scheduler.shutdown()
        self.logger.info("Finished")

class UpdateProcessor:
    def __init__(self, action: Action, logger: AdminLogger):
        self.action = action
        self.logger = logger

    def process_update(self, update: Update):
        try:
            self.action.process(update)
        except Exception as e:
            # As logger errors are probably API failures that the next updates may also get,
            # let them to be propagated so that no more updates are processed before waiting some time
            self.logger.error(e, "process_update")

class UpdatesProcessor:
    def __init__(self, get_updates_func: callable, logger: AdminLogger, config: Config,
                 update_processor: UpdateProcessor):
        self.get_updates_func = get_updates_func
        self.logger = logger
        self.config = config
        self.update_processor = update_processor
        self.last_error = None
        self.number_of_updates_processed = 0

    def run(self):
        self.processing_starting()
        try:
            self.__processing_loop()
        finally:
            self.processing_ended()
        self.processing_ended_successfully()

    def __processing_loop(self):
        while self.should_keep_processing_updates():
            self.__get_and_process_handling_errors()

    def __get_and_process_handling_errors(self):
        try:
            self.__get_and_process()
        except Exception as e:
            self.__handle_error(e)
            # notify there has been an error
            self.processing_error(e)
        else:
            # notify successful processing
            self.processing_successful()

    def __get_and_process(self):
        for update in self.get_updates_func():
            self.update_processor.process_update(update)
            self.number_of_updates_processed += 1

    def __handle_error(self, error: Exception):
        sleep_seconds = self.config.sleep_seconds_on_get_updates_error
        # we do not want to let non-fatal (eg. API) errors to escape from here
        self.safe_log_error(error, "get_and_process", "Sleeping for {seconds} seconds.".format(seconds=sleep_seconds))
        # there has been an error while getting updates, sleep a little to give a chance
        # for the server or the network to recover (if that was the case), and to not to flood the server
        time.sleep(int(sleep_seconds))

    def safe_log_error(self, error: Exception, *info: str):
        """Log error failing silently on error"""
        self.__do_safe(self.logger.error(error, *info))

    def safe_log_info(self, *info: str):
        """Log info failing silently on error"""
        self.__do_safe(lambda: self.logger.info(*info))

    @staticmethod
    def __do_safe(func: callable):
        try:
            return func()
        except:
            pass

    def should_keep_processing_updates(self):
        raise NotImplementedError()

    def processing_successful(self):
        """Updates were processed successfully"""
        self.last_error = None

    def processing_error(self, error: Exception):
        """There has been an error while processing the last updates"""
        self.last_error = error

    def processing_starting(self):
        """Updates are about to start being processed"""
        pass

    def processing_ended(self):
        """Processing has ended, we don't know if successfully or caused by an error"""
        self.safe_log_info(
            "Ending",
            "Updates processed: {updates_processed_number}"
                .format(updates_processed_number=self.number_of_updates_processed)
        )

    def processing_ended_successfully(self):
        """Processing has ended successfully"""
        pass

class PendingUpdatesProcessor(UpdatesProcessor):
    def __init__(self, get_updates_func: callable, logger: AdminLogger, config: Config,
                 update_processor: UpdateProcessor):
        super().__init__(get_updates_func, logger, config, update_processor)
        # set to some value other than None to let the processing run the first time
        self.last_error = True

    def should_keep_processing_updates(self):
        # if there has been an error not all pending updates were processed
        # so try again until it ends without error
        return self.last_error is not None

class NormalUpdatesProcessor(UpdatesProcessor):
    def __init__(self, get_updates_func: callable, logger: AdminLogger, config: Config,
                 update_processor: UpdateProcessor):
        super().__init__(get_updates_func, logger, config, update_processor)
        self.last_successful_processing = time.time()

    def processing_successful(self):
        super().processing_successful()
        self.last_successful_processing = time.time()

    def should_keep_processing_updates(self):
        if self.last_error is None:
            # if last processing ended without error, keep going!
            return True
        error_seconds_in_normal_mode = time.time() - self.last_successful_processing
        max_error_seconds_allowed_in_normal_mode = int(self.config.max_error_seconds_allowed_in_normal_mode)
        if error_seconds_in_normal_mode > max_error_seconds_allowed_in_normal_mode:
            # it has happened too much time since last successful processing
            # although it does not mean no update have been processed, we are
            # having problems and updates are being delayed, so going back to
            # process pending updates mode
            self.safe_log_error(
                MaxErrorSecondsAllowedInNormalModeExceededException(max_error_seconds_allowed_in_normal_mode),
                "normal_updates_processor",
                "Exceeded {max_seconds} max seconds with errors, switching to pending updates mode."
                    .format(max_seconds=max_error_seconds_allowed_in_normal_mode)
            )
            return False
        return True

class MaxErrorSecondsAllowedInNormalModeExceededException(Exception):
    pass
<commit_msg>Send info messages on pending updates start and end processing using the notification methods provided by UpdatesProcessor<commit_after>import time

from bot.action.core.action import Action
from bot.action.core.update import Update
from bot.api.api import Api
from bot.api.telegram import TelegramBotApi
from bot.logger.admin_logger import AdminLogger
from bot.multithreading.scheduler import SchedulerApi
from bot.storage import Config, Cache
from bot.storage import State

CONFIG_DIR = "config"
STATE_DIR = "state"

class Bot:
    def __init__(self):
        self.config = Config(CONFIG_DIR)
        self.state = State(STATE_DIR)
        self.cache = Cache()
        debug = self.config.debug()
        telegram_api = TelegramBotApi(self.config.auth_token, self.config.reuse_connections(), debug)
        self.api = Api(telegram_api, self.state)
        self.logger = AdminLogger(self.api, self.config.admin_chat_id, debug, self.config.send_error_tracebacks())
        self.scheduler = SchedulerApi(self.logger.work_error)
        if self.config.async():
            self.scheduler.setup()
            self.api.enable_async(self.scheduler)
        self.cache.bot_info = self.api.getMe()
        self.action = Action()
        self.update_processor = UpdateProcessor(self.action, self.logger)

    def set_action(self, action: Action):
        action.setup(self.api, self.config, self.state, self.cache, self.scheduler)
        self.action = action
        self.update_processor = UpdateProcessor(self.action, self.logger)

    def run(self):
        self.logger.info(
            "Starting",
            "async: {async}".format(async=self.config.async()),
            "Reusing connections: {reuse_connections}".format(reuse_connections=self.config.reuse_connections()),
            "debug: {debug}".format(debug=self.config.debug()),
            "Error tracebacks: {error_tracebacks}".format(error_tracebacks=self.config.send_error_tracebacks())
        )
        try:
            self.main_loop()
        except KeyboardInterrupt:
            self.logger.info("KeyboardInterrupt")
        except SystemExit as e:
            self.logger.info("SystemExit: " + str(e))
            raise e
        except BaseException as e:
            self.logger.error(e, "Fatal error")
            raise e
        finally:
            self.shutdown()

    def main_loop(self):
        while True:
            self.process_pending_updates()
            self.process_normal_updates()

    def process_pending_updates(self):
        PendingUpdatesProcessor(self.api.get_pending_updates, self.logger, self.config, self.update_processor).run()

    def process_normal_updates(self):
        NormalUpdatesProcessor(self.api.get_updates, self.logger, self.config, self.update_processor).run()

    def shutdown(self):
        if self.config.async():
            self.scheduler.shutdown()
        self.logger.info("Finished")

class UpdateProcessor:
    def __init__(self, action: Action, logger: AdminLogger):
        self.action = action
        self.logger = logger

    def process_update(self, update: Update):
        try:
            self.action.process(update)
        except Exception as e:
            # As logger errors are probably API failures that the next updates may also get,
            # let them to be propagated so that no more updates are processed before waiting some time
            self.logger.error(e, "process_update")

class UpdatesProcessor:
    def __init__(self, get_updates_func: callable, logger: AdminLogger, config: Config,
                 update_processor: UpdateProcessor):
        self.get_updates_func = get_updates_func
        self.logger = logger
        self.config = config
        self.update_processor = update_processor
        self.last_error = None
        self.number_of_updates_processed = 0

    def run(self):
        self.processing_starting()
        try:
            self.__processing_loop()
        finally:
            self.processing_ended()
        self.processing_ended_successfully()

    def __processing_loop(self):
        while self.should_keep_processing_updates():
            self.__get_and_process_handling_errors()

    def __get_and_process_handling_errors(self):
        try:
            self.__get_and_process()
        except Exception as e:
            self.__handle_error(e)
            # notify there has been an error
            self.processing_error(e)
        else:
            # notify successful processing
            self.processing_successful()

    def __get_and_process(self):
        for update in self.get_updates_func():
            self.update_processor.process_update(update)
            self.number_of_updates_processed += 1

    def __handle_error(self, error: Exception):
        sleep_seconds = self.config.sleep_seconds_on_get_updates_error
        # we do not want to let non-fatal (eg. API) errors to escape from here
        self.safe_log_error(error, "get_and_process", "Sleeping for {seconds} seconds.".format(seconds=sleep_seconds))
        # there has been an error while getting updates, sleep a little to give a chance
        # for the server or the network to recover (if that was the case), and to not to flood the server
        time.sleep(int(sleep_seconds))

    def safe_log_error(self, error: Exception, *info: str):
        """Log error failing silently on error"""
        self.__do_safe(self.logger.error(error, *info))

    def safe_log_info(self, *info: str):
        """Log info failing silently on error"""
        self.__do_safe(lambda: self.logger.info(*info))

    @staticmethod
    def __do_safe(func: callable):
        try:
            return func()
        except:
            pass

    def should_keep_processing_updates(self):
        raise NotImplementedError()

    def processing_successful(self):
        """Updates were processed successfully"""
        self.last_error = None

    def processing_error(self, error: Exception):
        """There has been an error while processing the last updates"""
        self.last_error = error

    def processing_starting(self):
        """Updates are about to start being processed"""
        pass

    def processing_ended(self):
        """Processing has ended, we don't know if successfully or caused by an error"""
        self.safe_log_info(
            "Ending",
            "Updates processed: {updates_processed_number}"
                .format(updates_processed_number=self.number_of_updates_processed)
        )

    def processing_ended_successfully(self):
        """Processing has ended successfully"""
        pass

class PendingUpdatesProcessor(UpdatesProcessor):
    def __init__(self, get_updates_func: callable, logger: AdminLogger, config: Config,
                 update_processor: UpdateProcessor):
        super().__init__(get_updates_func, logger, config, update_processor)
        # set to some value other than None to let the processing run the first time
        self.last_error = True

    def should_keep_processing_updates(self):
        # if there has been an error not all pending updates were processed
        # so try again until it ends without error
        return self.last_error is not None

    def processing_starting(self):
        self.safe_log_info("Pending", "Processing pending updates...")

    def processing_ended_successfully(self):
        self.safe_log_info("Continuing", "Pending updates successfully processed.")

class NormalUpdatesProcessor(UpdatesProcessor):
    def __init__(self, get_updates_func: callable, logger: AdminLogger, config: Config,
                 update_processor: UpdateProcessor):
        super().__init__(get_updates_func, logger, config, update_processor)
        self.last_successful_processing = time.time()

    def processing_successful(self):
        super().processing_successful()
        self.last_successful_processing = time.time()

    def should_keep_processing_updates(self):
        if self.last_error is None:
            # if last processing ended without error, keep going!
            return True
        error_seconds_in_normal_mode = time.time() - self.last_successful_processing
        max_error_seconds_allowed_in_normal_mode = int(self.config.max_error_seconds_allowed_in_normal_mode)
        if error_seconds_in_normal_mode > max_error_seconds_allowed_in_normal_mode:
            # it has happened too much time since last successful processing
            # although it does not mean no update have been processed, we are
            # having problems and updates are being delayed, so going back to
            # process pending updates mode
            self.safe_log_error(
                MaxErrorSecondsAllowedInNormalModeExceededException(max_error_seconds_allowed_in_normal_mode),
                "normal_updates_processor",
                "Exceeded {max_seconds} max seconds with errors, switching to pending updates mode."
                    .format(max_seconds=max_error_seconds_allowed_in_normal_mode)
            )
            return False
        return True

class MaxErrorSecondsAllowedInNormalModeExceededException(Exception):
    pass
<|endoftext|>


<commit_before># import unittest

# from . import fixture

# class FindDocumentTestCase(fixture.MongoextTestCase):
#     def setUp(self):
#         self.documents = [{
#             'title': u'Title',
#             'description': u'Description',
#             'content': u'Content',
#             'created_ts': 1,
#         }, {
#             'title': u'Title2',
#             'description': u'Description2',
#             'content': u'Content2',
#             'created_ts': 2,
#         }, {
#             'title': u'Title3',
#             'description': u'Description3',
#             'content': u'Content3',
#             'created_ts': 3,
#         }]

#         fixture.Collection().insert(self.documents)

#     def equal(self, documents1, documents2):
#         documents1 = [d for d in documents1]
#         for document in documents1:
#             document.pop('_id')
#         self.assertEqual(documents1, documents2)

#     def test_find_documents(self):
#         documents = fixture.Collection().find()
#         self.equal(documents, self.documents)

#     def test_find_documents_by_spec(self):
#         spec = {
#             'created_ts': 2
#         }
#         documents = fixture.Collection().find(spec)
#         self.equal(documents, [self.documents[1]])

#     def test_find_documents_by_spec_with_empty_result(self):
#         spec = {
#             'created_ts': -1
#         }
#         documents = fixture.Collection().find(spec)
#         self.equal(documents, [])

#     def test_find_documents_by_spec_with_more_the_one_result(self):
#         spec = {
#             'created_ts': {'$gt': 1},
#         }
#         documents = fixture.Collection().find(spec)
#         self.equal(documents, self.documents[1:])

#     def test_find_documents_with_specified_fields(self):
#         documents = fixture.Collection().find(projection={'created_ts': 1})
#         self.equal(documents, [{'created_ts': d['created_ts']} for d in self.documents])

#     def test_find_documents_with_skip(self):
#         documents = fixture.Collection().find(skip=1)
#         self.equal(documents, self.documents[1:])

#     def test_find_documents_by_spec_with_fields_and_skip(self):
#         spec = {
#             'created_ts': {'$gt': 1},
#         }
#         fields = {'created_ts': 1}
#         skip = 1
#         documents = fixture.Collection().find(spec, fields, skip)
#         self.equal(documents, [{'created_ts': d['created_ts']} for d in self.documents[2:]])

#     def test_find_one_document(self):
#         document = fixture.Collection().find_one({'created_ts': 1})
#         # print [d for d in fixture.Collection().find({'created_ts': 1})]
#         self.equal([document], [self.documents[0]])

#     def test_find_one_document_for_none_results(self):
#         document = fixture.Collection().find_one({'created_ts': -1})
#         self.assertIsNone(document)

#     def test_find_one_document_for_multiple_results(self):
#         document = fixture.Collection().find_one()
#         self.equal([document], [self.documents[0]])

# class CountTestCase(fixture.MongoextTestCase):
#     def setUp(self):
#         pass

#     def test_single_document_in_collection(self):
#         documents = [{
#             'title': u'Title',
#             'description': u'Description',
#             'content': u'Content',
#             'created_ts': 1,
#         }]

#         fixture.Collection().insert(documents)
#         self.assertEqual(fixture.Collection().count(), 1)

#     def test_multiple_document_in_collection(self):
#         documents = [{
#             'title': u'Title',
#             'description': u'Description',
#             'content': u'Content',
#             'created_ts': 1,
#         }, {
#             'title': u'Title2',
#             'description': u'Description2',
#             'content': u'Content2',
#             'created_ts': 2,
#         }, {
#             'title': u'Title3',
#             'description': u'Description3',
#             'content': u'Content3',
#             'created_ts': 3,
#         }]

#         fixture.Collection().insert(documents)
#         self.assertEqual(fixture.Collection().count(), 3)

# class DistinctTestCase(fixture.MongoextTestCase):
#     def setUp(self):
#         self.documents = [{
#             'title': u'Title',
#         }, {
#             'title': u'Title',
#         }, {
#             'title': u'Title1',
#         }]
#         fixture.Collection().insert(self.documents)

#     def test_distinct(self):
#         self.assertEqual(fixture.Collection().distinct('title'), [u'Title', u'Title1'])

# class DropTestCase(fixture.MongoextTestCase):
#     def setUp(self):
#         self.documents = [{
#             'title': u'Title',
#         }, {
#             'title': u'Title',
#         }, {
#             'title': u'Title1',
#         }]
#         fixture.Collection().insert(self.documents)

#     def test_drop(self):
#         self.assertIsNone(fixture.Collection().drop())

#     def test_multiple_drop(self):
#         fixture.Collection().drop()
#         fixture.Collection().drop()
#         self.assertIsNone(fixture.Collection().drop())

# class RemoveTestCase(fixture.MongoextTestCase):
#     def setUp(self):
#         self.documents = [{
#             'title': u'Title',
#         }, {
#             'title': u'Title',
#         }, {
#             'title': u'Title1',
#         }]
#         fixture.Collection().insert(self.documents)

#     def test_complete_remove(self):
#         fixture.Collection().remove()
#         self.assertEqual(fixture.Collection().count(), 0)

#     def test_single_removal(self):
#         fixture.Collection().remove({'title': u'Title1'})
#         self.assertEqual(fixture.Collection().count(), 2)

#     def test_multiple_removal(self):
#         fixture.Collection().remove({'title': u'Title'})
#         self.assertEqual(fixture.Collection().count(), 1)

#     def test_multiple_removal_with_multi_false(self):
#         fixture.Collection().remove({'title': u'Title'}, multi=False)
#         self.assertEqual(fixture.Collection().count(), 2)

# class UpdateTestCase(fixture.MongoextTestCase):
#     def setUp(self):
#         self.documents = [{
#             'title': u'Title',
#         }, {
#             'title': u'Title',
#         }, {
#             'title': u'Title1',
#         }]
#         fixture.Collection().insert(self.documents)

#     def test_update_one(self):
#         fixture.Collection().update({'title': u'Title'}, {'$set': {'title': u'Title2'}})
#         documents = [d for d in fixture.Collection().find({'title': u'Title2'})]
#         self.assertEqual(len(documents), 1)

#     def test_update_multi(self):
#         fixture.Collection().update({'title': u'Title'}, {'$set': {'title': u'Title2'}}, multi=True)
#         documents = [d for d in fixture.Collection().find({'title': u'Title2'})]
#         self.assertEqual(len(documents), 2)

# class SaveDocumentTestCase(fixture.MongoextTestCase):
#     def setUp(self):
#         pass

#     def test_save_new_document(self):
#         document = {
#             'title': u'Title',
#         }
#         fixture.Collection().save(document)
#         self.assertEqual(fixture.Collection().count(), 1)

#     def test_save_existed_document(self):
#         document = {'title': u'Title'}
#         _id = fixture.Collection().save(document)
#         document['_id'] = _id
#         document['title'] = u'Title1'
#         fixture.Collection().save(document)
#         self.assertEqual(fixture.Collection().count(), 1)

# class NoCompressionCollection(fixture.Collection):
#     KEYS_COMPRESSION = None

# class Compression(unittest.TestCase):
#     def setUp(self):
#         pass

#     def tearDown(self):
#         NoCompressionCollection().drop()

#     def test_pack(self):
#         spec = {'field': 1}
#         compressed = NoCompressionCollection().pack_fields(spec)
#         self.assertEqual(spec, compressed)

#     def test_unpack(self):
#         compressed = {'f': 1}
#         spec = NoCompressionCollection().unpack_fields(compressed)
#         self.assertEqual(spec, compressed)

#     def test_database(self):
#         self.assertIsNotNone(NoCompressionCollection.database)

# class FindModelsTestCase(fixture.MongoextTestCase):
#     def setUp(self):
#         self.models = [fixture.Document(**{
#             'title': u'Title',
#             'description': u'Description',
#             'content': u'Content',
#             'created_ts': 1,
#         }), fixture.Document(**{
#             'title': u'Title2',
#             'description': u'Description2',
#             'content': u'Content2',
#             'created_ts': 2,
#         }), fixture.Document(**{
#             'title': u'Title3',
#             'description': u'Description3',
#             'content': u'Content3',
#             'created_ts': 3,
#         })]

#         document_ids = fixture.Collection(fixture.Document).insert(self.models)
#         for document_id, model in zip(document_ids, self.models):
#             model._id = document_id

#     def equal(self, documents1, documents2):
#         self.assertEqual([dict(d) for d in documents1], [dict(d) for d in documents2])

#     def test_find_models(self):
#         models = fixture.Collection(fixture.Document).find()
#         self.equal(models, self.models)

#     def test_find_models_by_spec(self):
#         spec = {
#             'created_ts': 2
#         }
#         models = fixture.Collection(fixture.Document).find(spec)
#         self.equal(models, [self.models[1]])

#     def test_find_models_by_spec_with_empty_result(self):
#         spec = {
#             'created_ts': -1
#         }
#         models = fixture.Collection(fixture.Document).find(spec)
#         self.equal(models, [])

#     def test_find_models_by_spec_with_more_the_one_result(self):
#         spec = {
#             'created_ts': {'$gt': 1},
#         }
#         models = fixture.Collection(fixture.Document).find(spec)
#         self.equal(models, self.models[1:])

#     def test_find_documents_with_skip(self):
#         models = fixture.Collection(fixture.Document).find(skip=1)
#         self.equal(models, self.models[1:])

#     def test_find_documents_by_spec_with_fields_and_skip(self):
#         spec = {
#             'created_ts': {'$gt': 1},
#         }
#         skip = 1
#         models = fixture.Collection(fixture.Document).find(spec, skip=skip)
#         self.equal(models, self.models[2:])

# class SaveModelTestCase(fixture.MongoextTestCase):
#     def setUp(self):
#         pass

#     def test_save_new_model(self):
#         model = fixture.Document(**{
#             'created_ts': 1,
#         })
#         fixture.Collection(fixture.Document).save(model)
#         self.assertEqual(fixture.Collection().count(), 1)

#     def test_save_existed_model(self):
#         model = fixture.Document(**{
#             'created_ts': 1,
#         })
#         _id = fixture.Collection(fixture.Document).save(model)
#         model._id = _id
#         model.created_ts = 2
#         fixture.Collection(fixture.Document).save(model)
#         self.assertEqual(fixture.Collection(fixture.Document).count(), 1)
<commit_msg>uncomment collection tests<commit_after>import unittest

from . import fixture

class FindDocumentTestCase(fixture.MongoextTestCase):
    def setUp(self):
        self.documents = [{
            'title': u'Title',
            'description': u'Description',
            'content': u'Content',
            'created_ts': 1,
        }, {
            'title': u'Title2',
            'description': u'Description2',
            'content': u'Content2',
            'created_ts': 2,
        }, {
            'title': u'Title3',
            'description': u'Description3',
            'content': u'Content3',
            'created_ts': 3,
        }]

        fixture.Collection().insert(self.documents)

    def equal(self, documents1, documents2):
        documents1 = [d for d in documents1]
        for document in documents1:
            document.pop('_id')
        self.assertEqual(documents1, documents2)

    def test_find_documents(self):
        documents = fixture.Collection().find()
        self.equal(documents, self.documents)

    def test_find_documents_by_spec(self):
        spec = {
            'created_ts': 2
        }
        documents = fixture.Collection().find(spec)
        self.equal(documents, [self.documents[1]])

    def test_find_documents_by_spec_with_empty_result(self):
        spec = {
            'created_ts': -1
        }
        documents = fixture.Collection().find(spec)
        self.equal(documents, [])

    def test_find_documents_by_spec_with_more_the_one_result(self):
        spec = {
            'created_ts': {'$gt': 1},
        }
        documents = fixture.Collection().find(spec)
        self.equal(documents, self.documents[1:])

    def test_find_documents_with_specified_fields(self):
        documents = fixture.Collection().find(projection={'created_ts': 1})
        self.equal(documents, [{'created_ts': d['created_ts']} for d in self.documents])

    def test_find_documents_with_skip(self):
        documents = fixture.Collection().find(skip=1)
        self.equal(documents, self.documents[1:])

    def test_find_documents_by_spec_with_fields_and_skip(self):
        spec = {
            'created_ts': {'$gt': 1},
        }
        fields = {'created_ts': 1}
        skip = 1
        documents = fixture.Collection().find(spec, fields, skip)
        self.equal(documents, [{'created_ts': d['created_ts']} for d in self.documents[2:]])

    def test_find_one_document(self):
        document = fixture.Collection().find_one({'created_ts': 1})
        # print [d for d in fixture.Collection().find({'created_ts': 1})]
        self.equal([document], [self.documents[0]])

    def test_find_one_document_for_none_results(self):
        document = fixture.Collection().find_one({'created_ts': -1})
        self.assertIsNone(document)

    def test_find_one_document_for_multiple_results(self):
        document = fixture.Collection().find_one()
        self.equal([document], [self.documents[0]])

class CountTestCase(fixture.MongoextTestCase):
    def setUp(self):
        pass

    def test_single_document_in_collection(self):
        documents = [{
            'title': u'Title',
            'description': u'Description',
            'content': u'Content',
            'created_ts': 1,
        }]

        fixture.Collection().insert(documents)
        self.assertEqual(fixture.Collection().count(), 1)

    def test_multiple_document_in_collection(self):
        documents = [{
            'title': u'Title',
            'description': u'Description',
            'content': u'Content',
            'created_ts': 1,
        }, {
            'title': u'Title2',
            'description': u'Description2',
            'content': u'Content2',
            'created_ts': 2,
        }, {
            'title': u'Title3',
            'description': u'Description3',
            'content': u'Content3',
            'created_ts': 3,
        }]

        fixture.Collection().insert(documents)
        self.assertEqual(fixture.Collection().count(), 3)

class DistinctTestCase(fixture.MongoextTestCase):
    def setUp(self):
        self.documents = [{
            'title': u'Title',
        }, {
            'title': u'Title',
        }, {
            'title': u'Title1',
        }]
        fixture.Collection().insert(self.documents)

    def test_distinct(self):
        self.assertEqual(fixture.Collection().distinct('title'), [u'Title', u'Title1'])

class DropTestCase(fixture.MongoextTestCase):
    def setUp(self):
        self.documents = [{
            'title': u'Title',
        }, {
            'title': u'Title',
        }, {
            'title': u'Title1',
        }]
        fixture.Collection().insert(self.documents)

    def test_drop(self):
        self.assertIsNone(fixture.Collection().drop())

    def test_multiple_drop(self):
        fixture.Collection().drop()
        fixture.Collection().drop()
        self.assertIsNone(fixture.Collection().drop())

class RemoveTestCase(fixture.MongoextTestCase):
    def setUp(self):
        self.documents = [{
            'title': u'Title',
        }, {
            'title': u'Title',
        }, {
            'title': u'Title1',
        }]
        fixture.Collection().insert(self.documents)

    def test_complete_remove(self):
        fixture.Collection().remove()
        self.assertEqual(fixture.Collection().count(), 0)

    def test_single_removal(self):
        fixture.Collection().remove({'title': u'Title1'})
        self.assertEqual(fixture.Collection().count(), 2)

    def test_multiple_removal(self):
        fixture.Collection().remove({'title': u'Title'})
        self.assertEqual(fixture.Collection().count(), 1)

    def test_multiple_removal_with_multi_false(self):
        fixture.Collection().remove({'title': u'Title'}, multi=False)
        self.assertEqual(fixture.Collection().count(), 2)

class UpdateTestCase(fixture.MongoextTestCase):
    def setUp(self):
        self.documents = [{
            'title': u'Title',
        }, {
            'title': u'Title',
        }, {
            'title': u'Title1',
        }]
        fixture.Collection().insert(self.documents)

    def test_update_one(self):
        fixture.Collection().update({'title': u'Title'}, {'$set': {'title': u'Title2'}})
        documents = [d for d in fixture.Collection().find({'title': u'Title2'})]
        self.assertEqual(len(documents), 1)

    def test_update_multi(self):
        fixture.Collection().update({'title': u'Title'}, {'$set': {'title': u'Title2'}}, multi=True)
        documents = [d for d in fixture.Collection().find({'title': u'Title2'})]
        self.assertEqual(len(documents), 2)

class SaveDocumentTestCase(fixture.MongoextTestCase):
    def setUp(self):
        pass

    def test_save_new_document(self):
        document = {
            'title': u'Title',
        }
        fixture.Collection().save(document)
        self.assertEqual(fixture.Collection().count(), 1)

    def test_save_existed_document(self):
        document = {'title': u'Title'}
        _id = fixture.Collection().save(document)
        document['_id'] = _id
        document['title'] = u'Title1'
        fixture.Collection().save(document)
        self.assertEqual(fixture.Collection().count(), 1)

class NoCompressionCollection(fixture.Collection):
    KEYS_COMPRESSION = None

class Compression(unittest.TestCase):
    def setUp(self):
        pass

    def tearDown(self):
        NoCompressionCollection().drop()

    def test_pack(self):
        spec = {'field': 1}
        compressed = NoCompressionCollection().pack_fields(spec)
        self.assertEqual(spec, compressed)

    def test_unpack(self):
        compressed = {'f': 1}
        spec = NoCompressionCollection().unpack_fields(compressed)
        self.assertEqual(spec, compressed)

    def test_database(self):
        self.assertIsNotNone(NoCompressionCollection.database)

class FindModelsTestCase(fixture.MongoextTestCase):
    def setUp(self):
        self.models = [fixture.Document(**{
            'title': u'Title',
            'description': u'Description',
            'content': u'Content',
            'created_ts': 1,
        }), fixture.Document(**{
            'title': u'Title2',
            'description': u'Description2',
            'content': u'Content2',
            'created_ts': 2,
        }), fixture.Document(**{
            'title': u'Title3',
            'description': u'Description3',
            'content': u'Content3',
            'created_ts': 3,
        })]

        document_ids = fixture.Collection(fixture.Document).insert(self.models)
        for document_id, model in zip(document_ids, self.models):
            model._id = document_id

    def equal(self, documents1, documents2):
        self.assertEqual([dict(d) for d in documents1], [dict(d) for d in documents2])

    def test_find_models(self):
        models = fixture.Collection(fixture.Document).find()
        self.equal(models, self.models)

    def test_find_models_by_spec(self):
        spec = {
            'created_ts': 2
        }
        models = fixture.Collection(fixture.Document).find(spec)
        self.equal(models, [self.models[1]])

    def test_find_models_by_spec_with_empty_result(self):
        spec = {
            'created_ts': -1
        }
        models = fixture.Collection(fixture.Document).find(spec)
        self.equal(models, [])

    def test_find_models_by_spec_with_more_the_one_result(self):
        spec = {
            'created_ts': {'$gt': 1},
        }
        models = fixture.Collection(fixture.Document).find(spec)
        self.equal(models, self.models[1:])

    def test_find_documents_with_skip(self):
        models = fixture.Collection(fixture.Document).find(skip=1)
        self.equal(models, self.models[1:])

    def test_find_documents_by_spec_with_fields_and_skip(self):
        spec = {
            'created_ts': {'$gt': 1},
        }
        skip = 1
        models = fixture.Collection(fixture.Document).find(spec, skip=skip)
        self.equal(models, self.models[2:])

class SaveModelTestCase(fixture.MongoextTestCase):
    def setUp(self):
        pass

    def test_save_new_model(self):
        model = fixture.Document(**{
            'created_ts': 1,
        })
        fixture.Collection(fixture.Document).save(model)
        self.assertEqual(fixture.Collection().count(), 1)

    def test_save_existed_model(self):
        model = fixture.Document(**{
            'created_ts': 1,
        })
        _id = fixture.Collection(fixture.Document).save(model)
        model._id = _id
        model.created_ts = 2
        fixture.Collection(fixture.Document).save(model)
        self.assertEqual(fixture.Collection(fixture.Document).count(), 1)
<|endoftext|>