<commit_before>
r"""
Gathers the scores for a set of revisions and
prints a TSV to stdout of the format:
<page_id>\t<title>\n<rev_id>\t<prediction>\t<weighted_sum>

See https://phabricator.wikimedia.org/T135684 for more information.

Usage:
    extract_scores -h | --help
    extract_scores --dump=<dump-file>... --model=<model-file>
                                    [--verbose]
                                    [--rev-scores=<path>]

Options:
    -h --help                   Prints out this documentation.
    --dump=<dump-file>          Path to dump file.
    --model=<model-file>        Path to the model file.
    --verbose                   Prints dots and stuff to stderr
    --rev-scores=<path>         The location to write output to.
                                [default: <stdout>]
"""
from revscoring import ScorerModel
from revscoring.datasources import revision_oriented
from revscoring.dependencies import solve
import logging
import sys

import docopt
import mwxml

import mysqltsv

logger = logging.getLogger(__name__)
r_text = revision_oriented.revision.text

CLASS_WEIGHTS = {
    'Stub': 0,
    'Start': 1,
    'C': 2,
    'B': 3,
    'GA': 4,
    'FA': 5
}

def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)

    logging.basicConfig(
        level=logging.INFO,
        format='%(asctime)s %(levelname)s:%(name)s -- %(message)s'
    )

    dumps = args['--dump']
    with open(args['--model']) as f:
        model = ScorerModel.load(f)

    headers=["page_id", "title", "rev_id", "prediction", "weighted_sum"]
    if args['--rev-scores'] == "<stdout>":
        rev_scores = mysqltsv.Writer(sys.stdout, headers=headers)
    else:
        rev_scores = mysqltsv.Writer(open(args['--rev-scores'], "w"),
                                     headers=headers)

    verbose = args['--verbose']
    run(dumps, model, rev_scores, verbose=verbose)

def run(paths, model, rev_scores, verbose=False):

    def process_dump(dump, path):
        for page in dump:
            if int(page.namespace) != 0 or page.redirect:
                continue
            for revision in page:
                feature_values = list(solve(model.features,
                                       cache={r_text: revision.text}))
                yield (revision.id, model.score(feature_values), page.title, page.id)

    for rev_id, score, title, page_id in mwxml.map(process_dump, paths):
        weighted_sum = sum(CLASS_WEIGHTS[cls] * score['probability'][cls]
                           for cls in score['probability'])
        rev_scores.write(
            [page_id,
            title,
            rev_id,
            CLASS_WEIGHTS[score['prediction']],
            weighted_sum]
        )
        if verbose:
            sys.stderr.write(str(CLASS_WEIGHTS[score['prediction']]))
            sys.stderr.flush()

    if verbose:
        sys.stderr.write("\n")

if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
        sys.stderr.write("\n^C Caught.  Exiting...")

<commit_msg>Extract scores monthly, biannually, annually, every revision or latest<commit_after>r"""
Scores the revisions present in an XML dump file and a TSV to stdout of the
format:
    <page_id>\t<title>\t<rev_id>\t<prediction>\t<weighted_sum>

Usage:
    extract_scores -h | --help
    extract_scores <dump-file>... --model=<model-file> --sunset=<date>
                                 [--score-at=<when>]
                                 [--rev-scores=<path>]
                                 [--extend=<path>]
                                 [--processes=<num>]
                                 [--verbose]
                                 [--debug]

Options:
    -h --help             Prints out this documentation.
    <dump-file>           Path to an XML dump file to gather texts
    --model=<model-file>  Path to the model file to use for scoring.
    --sunset=<date>       The date when the XML dump file was generated
    --score-at=<when>     When should scores be generated?  Options include
                          (revision, monthly, biannually, annually, latest).
                          [default: latest]
    --rev-scores=<path>   The location to write output to.
                          [default: <stdout>]
    --extend=<path>       Path to an old output file for which scores should
                          not be duplicated
    --processes=<num>     The number of parallel processes to start
                          [default: <cpu count>]
    --verbose             Prints dots and stuff to stderr
    --debug               Print debug logging
"""
import logging
import sys
from itertools import chain
from multiprocessing import cpu_count

import docopt
import mwtypes
import mwxml
import mwtypes.files
import mysqltsv
from revscoring import ScorerModel
from revscoring.datasources import revision_oriented
from revscoring.dependencies import solve

logger = logging.getLogger(__name__)
r_text = revision_oriented.revision.text

CLASS_WEIGHTS = {
    'Stub': 0,
    'Start': 1,
    'C': 2,
    'B': 3,
    'GA': 4,
    'FA': 5,
    'e': 0,
    'bd': 1,
    'b': 2,
    'a': 3,
    'ba': 4,
    'adq': 5
}

START_YEAR = 2001
HEADERS = ["page_id", "title", "rev_id", "timestamp", "prediction",
           "weighted_sum"]
MONTHS = ["{0:02d}".format(i) for i in range(1, 13)]
SCORE_ATS = {'revision', 'monthly', 'biannually', 'annually', 'latest'}

def main(argv=None):
    args = docopt.docopt(__doc__, argv=argv)

    logging.basicConfig(
        level=logging.DEBUG if args['--debug'] else logging.INFO,
        format='%(asctime)s %(levelname)s:%(name)s -- %(message)s'
    )

    paths = args['<dump-file>']
    with open(args['--model']) as f:
        model = ScorerModel.load(f)

    sunset = mwtypes.Timestamp(args['--sunset'])

    if args['--score-at'] not in SCORE_ATS:
        raise ValueError("--score-at value {0} not available in {1}"
                         .format(args['--score-at'], SCORE_ATS))
    else:
        score_at = args['--score-at']

    if args['--rev-scores'] == "<stdout>":
        rev_scores = mysqltsv.Writer(sys.stdout, headers=HEADERS)
    else:
        rev_scores = mysqltsv.Writer(
            open(args['--rev-scores'], "w"), headers=HEADERS)

    if args['--extend'] is None:
        skip_scores_before = {}
    else:
        logger.info("Reading in past scores from {0}".format(args['--extend']))
        skip_scores_before = {}
        rows = mysqltsv.read(open(args['--extend']),
                             types=[int, str, int, mwtypes.Timestamp, str, float])
        for row in rows:
            skip_scores_before[row.page_id] = row.timestamp
        logger.info("Completed reading scores from old output.")

    if args['--processes'] == "<cpu count>":
        processes = cpu_count()
    else:
        processes = int(args['--processes'])

    verbose = args['--verbose']
    run(paths, model, sunset, score_at, rev_scores, skip_scores_before, processes, verbose=verbose)

def run(paths, model, sunset, score_at, rev_scores, skip_scores_before, processes,
        verbose=False):

    if score_at == "revision":
        process_dump = revision_scores(model, sunset, skip_scores_before)
    elif score_at == "latest":
        process_dump = latest_scores(model, sunset, skip_scores_before)
    else:
        sunset_year = int(sunset.strftime("%Y"))
        if score_at == "monthly":
            thresholds = [mwtypes.Timestamp(str(year) + month + "01000000")
                          for year, month in chain(
                            *(zip([year] * 12, MONTHS)
                              for year in range(START_YEAR, sunset_year + 1)))]
        elif score_at == "biannually":
            thresholds = [mwtypes.Timestamp(str(year) + month + "01000000")
                          for year, month in chain(
                            *(zip([year] * 2, ["01", "07"])
                              for year in range(START_YEAR, sunset_year + 1)))]
        elif score_at == "annually":
            thresholds = [mwtypes.Timestamp(str(year) + "0101000000")
                          for year in range(START_YEAR, sunset_year + 1)]
        else:
            raise RuntimeError("{0} is not a valid 'score_at' value"
                               .format(score_at))
        process_dump = threshold_scores(model, sunset, skip_scores_before, thresholds)

    results = mwxml.map(process_dump, paths, threads=processes)
    for page_id, title, rev_id, timestamp, (e, score) in results:
        if e is not None:
            logger.error("Error while processing {0}({1}) @ {2}: {3}"
                         .format(title, page_id, rev_id, e))
            continue

        weighted_sum = sum(CLASS_WEIGHTS[cls] * score['probability'][cls]
                           for cls in score['probability'])
        rev_scores.write(
            [page_id, title, rev_id, timestamp.short_format(),
             score['prediction'], weighted_sum])

        if verbose:
            sys.stderr.write(score['prediction'] + " ")
            sys.stderr.flush()

    if verbose:
        sys.stderr.write("\n")

def score_text(model, text):
    try:
        feature_values = list(solve(model.features, cache={r_text: text}))
        return None, model.score(feature_values)
    except Exception as e:
        return e, None

def revision_scores(model, sunset, skip_scores_before):

    def _revision_scores(dump, path):

        for page in dump:
            if int(page.namespace) != 0 or page.redirect:
                continue

            for revision in page:
                if page.id in skip_scores_before and skip_scores_before[page.id] >= revision.timestamp:
                    continue
                error_score = score_text(model, revision.text)
                yield (page.id, page.title, revision.id, revision.timestamp, error_score)

    return _revision_scores

def latest_scores(model, sunset):

    def _latest_scores(dump, path):

        for page in dump:
            if int(page.namespace) != 0 or page.redirect:
                continue

            last_revision = None
            for revision in page:
                last_revision = revision

            if page.id in skip_scores_before and skip_scores_before[page.id] >= sunset:
                continue
            error_score = score_text(model, last_revision.text)
            yield (page.id, page.title, last_revision.id, sunset, error_score)

    return _latest_scores

def threshold_scores(model, sunset, skip_scores_before, thresholds):

    def _threshold_scores(dump, path):

        for page in dump:
            if int(page.namespace) != 0 or page.redirect:
                continue

            page_ts = None
            last_revision = None
            for revision in page:
                # Make a copy of thresholds specific to this page.
                if page_ts is None:
                    page_ts = [ts for ts in thresholds
                               if ts > revision.timestamp and
                               ts <= sunset and (page.id not in skip_scores_before or
                                                 ts > skip_scores_before[page.id])]

                if len(page_ts) > 0 and revision.timestamp > page_ts[0] and \
                   last_revision is not None:
                    error_score = score_text(model, last_revision.text)
                    while len(page_ts) > 0 and revision.timestamp > page_ts[0]:
                        threshold_ts = page_ts.pop(0)
                        yield (page.id, page.title, last_revision.id,
                               threshold_ts, error_score)

                last_revision = revision

            if len(page_ts) > 0:
                error_score = score_text(model, last_revision.text)
                for threshold_ts in page_ts:
                    yield (page.id, page.title, last_revision.id, threshold_ts,
                           error_score)

    return _threshold_scores
<|endoftext|>


<commit_before>#
# This file is part of khmer, http://github.com/ged-lab/khmer/, and is
# Copyright (C) Michigan State University, 2009-2013. It is licensed under
# the three-clause BSD license; see doc/LICENSE.txt.
# Contact: khmer-project@idyll.org
#

# pylint: disable=C0111,C0103

import khmer
import khmer_tst_utils as utils
from nose.plugins.attrib import attr

MAX_COUNT = 255

def test_no_collision():
    kh = khmer.new_hashtable(4, 86)

    kh.count('AAAA')
    assert kh.get('AAAA') == 1

    kh.count('TTTT')                    # reverse complement
    assert kh.get('TTTT') == 2

def test_collision():
    kh = khmer.new_hashtable(4, 85)

    kh.count('AAAA')
    assert kh.get('AAAA') == 1

    kh.count('TTTT')
    assert kh.get('TTTT') == 2

def test_badcount():
    countingtable = khmer.new_hashtable(4, 85)
    try:
        countingtable.count()
        assert 0, "count should require one argument"
    except TypeError, err:
        print str(err)
    try:
        countingtable.count('ABCDE')
        assert 0, "count should require k-mer size to be equal"
    except ValueError, err:
        print str(err)

def test_hashtable_n_entries():
    countingtable = khmer.new_hashtable(4, 4 ** 4)
    try:
        countingtable.n_entries("nope")
        assert 0, "n_entries should accept no arguments"
    except TypeError, err:
        print str(err)

def test_complete_no_collision():
    kh = khmer.new_hashtable(4, 4 ** 4)

    for i in range(0, kh.n_entries()):
        s = khmer.reverse_hash(i, 4)
        kh.count(s)

    n_palindromes = 0
    n_rc_filled = 0
    n_fwd_filled = 0

    for i in range(0, kh.n_entries()):
        s = khmer.reverse_hash(i, 4)
        if kh.get(s):                   # string hashing is rc aware
            n_rc_filled += 1
        if kh.get(s) == 1:              # palindromes are singular
            n_palindromes += 1
        if kh.get(i):                   # int hashing is not rc aware
            n_fwd_filled += 1

    assert n_rc_filled == kh.n_entries(), n_rc_filled
    assert n_palindromes == 16, n_palindromes  # @CTB check this
    assert n_fwd_filled == kh.n_entries() / 2 + n_palindromes / 2, \
        n_fwd_filled

def test_complete_2_collision():
    kh = khmer.new_hashtable(4, 4 ** 4 / 2)

    for i in range(0, kh.n_entries()):
        s = khmer.reverse_hash(i, 4)
        kh.count(s)

    n_rc_filled = 0
    #  n_fwd_filled = 0

    for i in range(0, 128):
        s = khmer.reverse_hash(i, 4)
        if kh.get(s):                   # string hashing is rc aware
            n_rc_filled += 1
    # if kh.get(i):                   # int hashing is not rc aware
    #        n_fwd_filled += 1

    assert n_rc_filled == 128, n_rc_filled
    # @CTB assert n_fwd_filled == 100 # kt.n_entries() / 2, n_fwd_filled

def test_complete_4_collision():
    kh = khmer.new_hashtable(4, 4 ** 4 / 4)

    for i in range(0, kh.n_entries()):
        s = khmer.reverse_hash(i, 4)
        kh.count(s)

    n_rc_filled = 0
    #  n_fwd_filled = 0

    for i in range(0, 64):
        s = khmer.reverse_hash(i, 4)
        if kh.get(s):                   # string hashing is rc aware
            n_rc_filled += 1
    # if kh.get(i):                   # int hashing is not rc aware
    #       n_fwd_filled += 1

    assert n_rc_filled == 64, n_rc_filled
    # @CTB assert n_fwd_filled == kt.n_entries() / 2, n_fwd_filled

def test_maxcount():
    # hashtable should saturate at some point so as not to overflow counter
    kh = khmer.new_hashtable(4, 4 ** 4)

    last_count = None
    for _ in range(0, 10000):
        kh.count('AAAA')
        c = kh.get('AAAA')

        print last_count, c
        if c == last_count:
            break
        last_count = c

    assert c != 10000, "should not be able to count to 10000"
    assert c == MAX_COUNT       # this will depend on HashcountType...

def test_maxcount_with_bigcount():
    # hashtable should not saturate, if use_bigcount is set.
    kh = khmer.new_hashtable(4, 4 ** 4)
    kh.set_use_bigcount(True)

    last_count = None
    for _ in range(0, 10000):
        kh.count('AAAA')
        c = kh.get('AAAA')

        print last_count, c
        if c == last_count:
            break
        last_count = c

    assert c == 10000, "should be able to count to 10000"
    assert c != MAX_COUNT

def test_consume_uniqify_first():
    kh = khmer.new_hashtable(4, 4 ** 4)

    s = "TTTT"
    s_rc = "AAAA"

    kh.consume(s)
    n = kh.get(s_rc)
    assert n == 1

def test_maxcount_consume():
    # hashtable should saturate at some point so as not to overflow counter
    kh = khmer.new_hashtable(4, 4 ** 4)

    s = "A" * 10000
    kh.consume(s)

    c = kh.get('AAAA')
    assert c == MAX_COUNT, c    # this will depend on HashcountType...

def test_maxcount_consume_with_bigcount():
    # use the bigcount hack to avoid saturating the hashtable.
    kh = khmer.new_hashtable(4, 4 ** 4)
    kh.set_use_bigcount(True)

    s = "A" * 10000
    kh.consume(s)

    c = kh.get('AAAA')
    assert c == 10000 - 3, c

def test_get_mincount():
    kh = khmer.new_hashtable(4, 4 ** 4)

    s = "AAAAACGT"
    kh.consume(s)

    x = kh.get_min_count(s)
    assert x == 1

    kh.consume(s)
    x = kh.get_min_count(s)
    assert x == 2

def test_get_maxcount():
    kh = khmer.new_hashtable(4, 4 ** 4)

    s = "AAAAACGT"
    kh.consume(s)

    x = kh.get_max_count(s)
    assert x == 2

    kh.consume(s)
    x = kh.get_max_count(s)
    assert x == 4

def test_get_maxcount_rc():
    kh = khmer.new_hashtable(4, 4 ** 4)

    s = "AAAAACGT"
    src = "ACGTTTTT"
    kh.consume(s)

    x = kh.get_max_count(s)
    assert x == 2

    kh.consume(src)
    x = kh.get_max_count(s)
    assert x == 4

def test_get_mincount_rc():
    kh = khmer.new_hashtable(4, 4 ** 4)

    s = "AAAAACGT"
    src = "ACGTTTTT"

    kh.consume(s)
    x = kh.get_min_count(s)
    assert x == 1

    kh.consume(src)
    x = kh.get_min_count(s)
    assert x == 2

def test_64bitshift():
    kh = khmer.new_hashtable(25, 4 ** 10)
    fullstr = "GTATGCCAGCTCCAACTGGGCCGGTACGAGCAGGCCATTGCCTCTTGCCGCGATGCGTCGGCG"
    substr = "ATGCCAGCTCCAACTGGGCCGGTACGAGCAGGCCATTGCCTCTTGC"

    kh.consume(fullstr)
    assert 0 < kh.get_min_count(substr), kh.get_min_count(substr)

@attr('highmem')
def test_64bitshift_2():
    kh = khmer.new_hashtable(25, 4 ** 10)
    fullstr = "GTATGCCAGCTCCAACTGGGCCGGTACGAGCAGGCCATTGCCTCTTGCCGCGATGCGTCGGCG"

    kh.consume(fullstr)
    for i in range(len(fullstr) - 25 + 1):
        substr = fullstr[i:i + 25]
        assert kh.get(substr) > 0

@attr('highmem')
def test_very_short_read():
    short_filename = utils.get_test_data('test-short.fa')
    kh = khmer.new_hashtable(9, 4 ** 4 + 1)
    n_reads, n_kmers = kh.consume_fasta(short_filename)
    assert n_reads == 1
    assert n_kmers == 0

    kh = khmer.new_hashtable(8, 4 ** 4 + 1)
    n_reads, n_kmers = kh.consume_fasta(short_filename)
    assert n_reads == 1
    assert n_kmers == 1

class Test_ConsumeString(object):

    def setup(self):
        self.kh = khmer.new_hashtable(4, 4 ** 4)

    def test_n_occupied(self):
        assert self.kh.n_occupied() == 0
        self.kh.consume('AAAA')
        assert self.kh.n_occupied() == 1
        self.kh.consume('AACT')
        assert self.kh.n_occupied() == 2
        try:
            self.kh.n_occupied("MU", 1, 3)
            assert 0, "n_occupied shouldn't accept three arguments"
        except TypeError, err:
            print str(err)

    @attr('highmem')
    def test_abundance_by_pos(self):
        kh = self.kh

        for _ in range(0, 300):
            kh.count('ATCG')

        for _ in range(0, 10):
            kh.count('ATGG')

        short_filename = utils.get_test_data('test-short.fa')
        dist = kh.fasta_count_kmers_by_position(short_filename, 6, 10)
        assert dist[4] == 1
        assert sum(dist) == 1

        dist = kh.fasta_count_kmers_by_position(short_filename, 6, MAX_COUNT)
        assert dist[0] == 1, dist[0]
        assert dist[2] == 1
        assert sum(dist) == 2

    @attr('highmem')
    def test_abundance_by_pos_bigcount(self):
        kh = self.kh
        kh.set_use_bigcount(True)       # count past MAX_COUNT

        for _ in range(0, 300):
            kh.count('ATCG')

        for _ in range(0, 10):
            kh.count('ATGG')

        short_filename = utils.get_test_data('test-short.fa')
        dist = kh.fasta_count_kmers_by_position(short_filename, 6, 10)
        assert dist[4] == 1
        assert sum(dist) == 1

        dist = kh.fasta_count_kmers_by_position(short_filename, 6, 300)
        assert dist[0] == 1, dist[0]
        assert dist[2] == 1
        assert sum(dist) == 2

    def test_n_occupied_args(self):
        assert self.kh.n_occupied() == 0
        self.kh.consume('AAAA')
        assert self.kh.n_occupied(0, 1) == 1
        assert self.kh.n_occupied(1, 4 ** 4) == 0

        hashvalue = khmer.forward_hash('AACT', 4)
        self.kh.consume('AACT')
        assert self.kh.n_occupied(0, hashvalue + 1) == 2
        assert self.kh.n_occupied(hashvalue + 1, 4 ** 4) == 0
        assert self.kh.n_occupied(hashvalue, hashvalue + 1) == 1

    def test_simple(self):
        n = self.kh.consume('AAAA')
        assert n == 1
        assert self.kh.get(0) == 1

    def test_simple_2(self):
        n = self.kh.consume('AAAAA')
        assert n == 2
        assert self.kh.get(0) == 2

    def test_simple_rc(self):
        n = self.kh.consume('TTTTT')
        assert n == 2
        assert self.kh.get(0) == 2

    def test_min_count(self):
        self.kh.consume('AAAA')

        count = self.kh.get_min_count('AAAA')
        assert count == 1

    def test_max_count(self):
        self.kh.consume('AAAA')

        count = self.kh.get_max_count('AAAA')
        assert count == 1

class Test_AbundanceDistribution(object):

    def setup(self):
        self.kh = khmer.new_hashtable(4, 4 ** 4)
        A_filename = utils.get_test_data('all-A.fa')
        self.kh.consume_fasta(A_filename)

    @attr('highmem')
    def test_count_A(self):
        A_filename = utils.get_test_data('all-A.fa')

        tracking = khmer.new_hashbits(4, 4 ** 4, 1)
        dist = self.kh.abundance_distribution(A_filename, tracking)

        assert sum(dist) == 1
        assert dist[10] == 1
<commit_msg>Reduced memory requirements for test_counting_single.py<commit_after>#
# This file is part of khmer, http://github.com/ged-lab/khmer/, and is
# Copyright (C) Michigan State University, 2009-2013. It is licensed under
# the three-clause BSD license; see doc/LICENSE.txt.
# Contact: khmer-project@idyll.org
#

# pylint: disable=C0111,C0103

import khmer
import khmer_tst_utils as utils
from nose.plugins.attrib import attr

MAX_COUNT = 255

def test_no_collision():
    kh = khmer.new_hashtable(4, 4)

    kh.count('AAAA')
    assert kh.get('AAAA') == 1

    kh.count('TTTT')                    # reverse complement
    assert kh.get('TTTT') == 2

def test_collision():
    kh = khmer.new_hashtable(4, 4)

    kh.count('AAAA')
    assert kh.get('AAAA') == 1

    kh.count('TTTT')
    assert kh.get('TTTT') == 2

def test_badcount():
    countingtable = khmer.new_hashtable(4, 4)
    try:
        countingtable.count()
        assert 0, "count should require one argument"
    except TypeError, err:
        print str(err)
    try:
        countingtable.count('ABCDE')
        assert 0, "count should require k-mer size to be equal"
    except ValueError, err:
        print str(err)

def test_hashtable_n_entries():
    countingtable = khmer.new_hashtable(4, 4)
    try:
        countingtable.n_entries("nope")
        assert 0, "n_entries should accept no arguments"
    except TypeError, err:
        print str(err)

def test_complete_no_collision():
    kh = khmer.new_hashtable(4, 4)

    for i in range(0, kh.n_entries()):
        s = khmer.reverse_hash(i, 4)
        kh.count(s)

    n_palindromes = 0
    n_rc_filled = 0
    n_fwd_filled = 0

    for i in range(0, kh.n_entries()):
        s = khmer.reverse_hash(i, 4)
        if kh.get(s):                   # string hashing is rc aware
            n_rc_filled += 1
        if kh.get(s) == 1:              # palindromes are singular
            n_palindromes += 1
        if kh.get(i):                   # int hashing is not rc aware
            n_fwd_filled += 1

    assert n_rc_filled == kh.n_entries(), n_rc_filled
    assert n_palindromes == 16, n_palindromes  # @CTB check this
    assert n_fwd_filled == kh.n_entries() / 2 + n_palindromes / 2, \
        n_fwd_filled

def test_complete_2_collision():
    kh = khmer.new_hashtable(4, 4)

    for i in range(0, kh.n_entries()):
        s = khmer.reverse_hash(i, 4)
        kh.count(s)

    n_rc_filled = 0
    #  n_fwd_filled = 0

    for i in range(0, 128):
        s = khmer.reverse_hash(i, 4)
        if kh.get(s):                   # string hashing is rc aware
            n_rc_filled += 1
    # if kh.get(i):                   # int hashing is not rc aware
    #        n_fwd_filled += 1

    assert n_rc_filled == 128, n_rc_filled
    # @CTB assert n_fwd_filled == 100 # kt.n_entries() / 2, n_fwd_filled

def test_complete_4_collision():
    kh = khmer.new_hashtable(4, 4)

    for i in range(0, kh.n_entries()):
        s = khmer.reverse_hash(i, 4)
        kh.count(s)

    n_rc_filled = 0
    #  n_fwd_filled = 0

    for i in range(0, 64):
        s = khmer.reverse_hash(i, 4)
        if kh.get(s):                   # string hashing is rc aware
            n_rc_filled += 1
    # if kh.get(i):                   # int hashing is not rc aware
    #       n_fwd_filled += 1

    assert n_rc_filled == 64, n_rc_filled
    # @CTB assert n_fwd_filled == kt.n_entries() / 2, n_fwd_filled

def test_maxcount():
    # hashtable should saturate at some point so as not to overflow counter
    kh = khmer.new_hashtable(4, 4)

    last_count = None
    for _ in range(0, 10000):
        kh.count('AAAA')
        c = kh.get('AAAA')

        print last_count, c
        if c == last_count:
            break
        last_count = c

    assert c != 10000, "should not be able to count to 10000"
    assert c == MAX_COUNT       # this will depend on HashcountType...

def test_maxcount_with_bigcount():
    # hashtable should not saturate, if use_bigcount is set.
    kh = khmer.new_hashtable(4, 4)
    kh.set_use_bigcount(True)

    last_count = None
    for _ in range(0, 10000):
        kh.count('AAAA')
        c = kh.get('AAAA')

        print last_count, c
        if c == last_count:
            break
        last_count = c

    assert c == 10000, "should be able to count to 10000"
    assert c != MAX_COUNT

def test_consume_uniqify_first():
    kh = khmer.new_hashtable(4, 4)

    s = "TTTT"
    s_rc = "AAAA"

    kh.consume(s)
    n = kh.get(s_rc)
    assert n == 1

def test_maxcount_consume():
    # hashtable should saturate at some point so as not to overflow counter
    kh = khmer.new_hashtable(4, 4)

    s = "A" * 10000
    kh.consume(s)

    c = kh.get('AAAA')
    assert c == MAX_COUNT, c    # this will depend on HashcountType...

def test_maxcount_consume_with_bigcount():
    # use the bigcount hack to avoid saturating the hashtable.
    kh = khmer.new_hashtable(4, 4)
    kh.set_use_bigcount(True)

    s = "A" * 10000
    kh.consume(s)

    c = kh.get('AAAA')
    assert c == 10000 - 3, c

def test_get_mincount():
    kh = khmer.new_hashtable(4, 4)

    s = "AAAAACGT"
    kh.consume(s)

    x = kh.get_min_count(s)
    assert x == 1

    kh.consume(s)
    x = kh.get_min_count(s)
    assert x == 2

def test_get_maxcount():
    kh = khmer.new_hashtable(4, 4)

    s = "AAAAACGT"
    kh.consume(s)

    x = kh.get_max_count(s)
    assert x == 2

    kh.consume(s)
    x = kh.get_max_count(s)
    assert x == 4

def test_get_maxcount_rc():
    kh = khmer.new_hashtable(4, 4)

    s = "AAAAACGT"
    src = "ACGTTTTT"
    kh.consume(s)

    x = kh.get_max_count(s)
    assert x == 2

    kh.consume(src)
    x = kh.get_max_count(s)
    assert x == 4

def test_get_mincount_rc():
    kh = khmer.new_hashtable(4, 4)

    s = "AAAAACGT"
    src = "ACGTTTTT"

    kh.consume(s)
    x = kh.get_min_count(s)
    assert x == 1

    kh.consume(src)
    x = kh.get_min_count(s)
    assert x == 2

def test_64bitshift():
    kh = khmer.new_hashtable(25, 4)
    fullstr = "GTATGCCAGCTCCAACTGGGCCGGTACGAGCAGGCCATTGCCTCTTGCCGCGATGCGTCGGCG"
    substr = "ATGCCAGCTCCAACTGGGCCGGTACGAGCAGGCCATTGCCTCTTGC"

    kh.consume(fullstr)
    assert 0 < kh.get_min_count(substr), kh.get_min_count(substr)

@attr('highmem')
def test_64bitshift_2():
    kh = khmer.new_hashtable(25, 4)
    fullstr = "GTATGCCAGCTCCAACTGGGCCGGTACGAGCAGGCCATTGCCTCTTGCCGCGATGCGTCGGCG"

    kh.consume(fullstr)
    for i in range(len(fullstr) - 25 + 1):
        substr = fullstr[i:i + 25]
        assert kh.get(substr) > 0

@attr('highmem')
def test_very_short_read():
    short_filename = utils.get_test_data('test-short.fa')
    kh = khmer.new_hashtable(9, 4)
    n_reads, n_kmers = kh.consume_fasta(short_filename)
    assert n_reads == 1
    assert n_kmers == 0

    kh = khmer.new_hashtable(8, 4)
    n_reads, n_kmers = kh.consume_fasta(short_filename)
    assert n_reads == 1
    assert n_kmers == 1

class Test_ConsumeString(object):

    def setup(self):
        self.kh = khmer.new_hashtable(4, 4)

    def test_n_occupied(self):
        assert self.kh.n_occupied() == 0
        self.kh.consume('AAAA')
        assert self.kh.n_occupied() == 1
        self.kh.consume('AACT')
        assert self.kh.n_occupied() == 2
        try:
            self.kh.n_occupied("MU", 1, 3)
            assert 0, "n_occupied shouldn't accept three arguments"
        except TypeError, err:
            print str(err)

    @attr('highmem')
    def test_abundance_by_pos(self):
        kh = self.kh

        for _ in range(0, 300):
            kh.count('ATCG')

        for _ in range(0, 10):
            kh.count('ATGG')

        short_filename = utils.get_test_data('test-short.fa')
        dist = kh.fasta_count_kmers_by_position(short_filename, 6, 10)
        assert dist[4] == 1
        assert sum(dist) == 1

        dist = kh.fasta_count_kmers_by_position(short_filename, 6, MAX_COUNT)
        assert dist[0] == 1, dist[0]
        assert dist[2] == 1
        assert sum(dist) == 2

    @attr('highmem')
    def test_abundance_by_pos_bigcount(self):
        kh = self.kh
        kh.set_use_bigcount(True)       # count past MAX_COUNT

        for _ in range(0, 300):
            kh.count('ATCG')

        for _ in range(0, 10):
            kh.count('ATGG')

        short_filename = utils.get_test_data('test-short.fa')
        dist = kh.fasta_count_kmers_by_position(short_filename, 6, 10)
        assert dist[4] == 1
        assert sum(dist) == 1

        dist = kh.fasta_count_kmers_by_position(short_filename, 6, 300)
        assert dist[0] == 1, dist[0]
        assert dist[2] == 1
        assert sum(dist) == 2

    def test_n_occupied_args(self):
        assert self.kh.n_occupied() == 0
        self.kh.consume('AAAA')
        assert self.kh.n_occupied(0, 1) == 1
        assert self.kh.n_occupied(1, 4 ** 4) == 0

        hashvalue = khmer.forward_hash('AACT', 4)
        self.kh.consume('AACT')
        assert self.kh.n_occupied(0, hashvalue + 1) == 2
        assert self.kh.n_occupied(hashvalue + 1, 4 ** 4) == 0
        assert self.kh.n_occupied(hashvalue, hashvalue + 1) == 1

    def test_simple(self):
        n = self.kh.consume('AAAA')
        assert n == 1
        assert self.kh.get(0) == 1

    def test_simple_2(self):
        n = self.kh.consume('AAAAA')
        assert n == 2
        assert self.kh.get(0) == 2

    def test_simple_rc(self):
        n = self.kh.consume('TTTTT')
        assert n == 2
        assert self.kh.get(0) == 2

    def test_min_count(self):
        self.kh.consume('AAAA')

        count = self.kh.get_min_count('AAAA')
        assert count == 1

    def test_max_count(self):
        self.kh.consume('AAAA')

        count = self.kh.get_max_count('AAAA')
        assert count == 1

class Test_AbundanceDistribution(object):

    def setup(self):
        self.kh = khmer.new_hashtable(4, 4)
        A_filename = utils.get_test_data('all-A.fa')
        self.kh.consume_fasta(A_filename)

    @attr('highmem')
    def test_count_A(self):
        A_filename = utils.get_test_data('all-A.fa')

        tracking = khmer.new_hashbits(4, 4, 1)
        dist = self.kh.abundance_distribution(A_filename, tracking)

        assert sum(dist) == 1
        assert dist[10] == 1
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Database hookup for doing text string searches.

Must be run from labs and does these (SQL LIKE style) in
labels, aliases and descriptions of items.
"""
import MySQLdb

class WikidataStringSearch:
    """Run string searches on Wikidata from labs."""

    def __init__(self, verbose=False):
        """
        Set up connection and fetches allowed languages and types.

        @param verbose: if output (errors, queries) should be printed
            (default: False)
        @type verbose: bool
        """
        # Make one d/b connection for all queries - more efficient I believe
        self.conn = MySQLdb.connect(read_default_file="~/replica.my.cnf",
                                    host="wikidatawiki.labsdb",
                                    db="wikidatawiki_p",
                                    charset='utf8'
                                    )
        self.cursor = self.conn.cursor()
        self.term_types = ()
        self.languages = ()
        self.verbose = verbose

        # get types
        self.cursor.execute("SELECT DISTINCT term_type FROM wb_terms;")
        for t in self.cursor.fetchall():
            self.term_types += t
        self._print('found %d term_types' % len(self.term_types))

        # get languages
        self.cursor.execute("SELECT DISTINCT term_language FROM wb_terms;")
        for l in self.cursor.fetchall():
            self.languages += l
        self._print('found %d languages' % len(self.languages))

    def close_connection(self):
        """Close database connection."""
        self.conn.close()

    def testInput(self, text, language=None, term_type=None, entities=None):
        """
        Test that the user input is valid.

        @param text: the text to search for
        @type text: basestring
        @param language: the language to search in, defaults to None=any
        @type language: basestring or None
        @param entities: the language to search in, defaults to None=any
        @type entities: list (of basestring), or None
        @param term_type: field to search in, defaults
            to None=['label', 'alias']
        @type term_type: basestring or None
        @return: if input is valid
        @rtype: bool
        """
        # test text
        if not text.strip():
            self._print('You cannot send stringSearch an empty string')
            return False

        # test language
        if language and language not in self.languages:
            self._print('%s is not a recognised language' % language)
            return False

        # test term_type
        if term_type and term_type not in self.term_types:
            self._print('%s is not a recognised term_type' % term_type)
            return False

        # test list of entities
        if entities:
            if not isinstance(entities, (tuple, list)):
                self._print('Entities must be a non-zero list')
                return False

            # Check each is correctly formatted
            if not all(e.startswith('Q') and
                       isinstance(e, basestring) and
                       WikidataStringSearch.is_int(e[1:])
                       for e in entities):
                self._print('Each entity must be a string like Q<integer>')
                return False

        # nothing flagged
        return True

    def search(self, text, language='sv', term_type=None):
        """
        Search for a given string in a specified language and field.

        Deprecated.
        Calls basic_search with entities=None and language defaulting to 'sv'
        """
        return self.basic_search(text,
                                 language=language,
                                 entities=None,
                                 term_type=term_type)

    def searchInEntities(self, text, entities, language='sv', term_type=None):
        """
        search() but limit results to a provided list of entities.

        Deprecated.
        Calls basic_search with language defaulting to 'sv'
        """
        return self.basic_search(text,
                                 language=language,
                                 entities=entities,
                                 term_type=term_type)

    def basic_search(self, text, language=None, entities=None, term_type=None):
        """
        Search for a given string through exact match or SQL like wild-cards.

        Results can be limited to a specific language, a certain field or
        to within a list of entities.

        @param text: the text to search for
        @type text: basestring
        @param language: the language to search in, defaults to None=any
        @type language: basestring or None
        @param entities: the language to search in, defaults to None=any
        @type entities: list (of basestring), or None
        @param term_type: field to search in, defaults
            to None=['label', 'alias']
        @type term_type: basestring or None
        @return: list of matching Q values
        @rtype: list (of str)
        """
        # validate input
        if not self.testInput(text, language, term_type, entities):
            return None

        # set default term_type
        if term_type is None:
            term_type = ['label', 'alias']
        else:
            term_type = [term_type, ]

        # prepare entities by converting to a list of int strings
        if entities:
            tmp = []
            for e in entities:
                tmp.append(int(e[1:]))
            entities = tmp

        # construct query
        params = []
        query = "SELECT CONCAT('Q',term_entity_id), term_text, term_type " \
                "FROM wb_terms " \
                "WHERE term_entity_type='item' "
        params += term_type
        query += "AND term_type IN %s " % \
                 WikidataStringSearch.sql_in_format(term_type)
        if language:
            params.append(language)
            query += "AND term_language=%s "
        if entities:
            params += entities
            query += "AND term_entity_id IN %s " % \
                     WikidataStringSearch.sql_in_format(entities)
        params.append(text)
        query += "AND term_text LIKE %s " \
                 "LIMIT 100;"

        # execute query
        self.cursor.execute(query, tuple(params))
        self._print(self.cursor._last_executed)

        # handle response
        qs = []
        for q, t, tt in self.cursor.fetchall():
            qs.append(q)

        return qs

    def _print(self, s):
        """
        Print text if verbose.

        @param s: text to print
        @type s: basestring
        """
        if self.verbose:
            print s

    @staticmethod
    def sql_in_format(l):
        """
        Given a list of parameters output an sql compatible list of args.

        @param l: value to test
        @type l: list or tuple
        @return: sql compatible list of args
        @rtype: string
        """
        string_format = ', '.join(['%s'] * len(l))
        return '(%s)' % string_format

    @staticmethod
    def is_int(s):
        """
        Check if value is an int.

        @param s: value to test
        @type s: any
        @return: if value can be interpreted as an integer
        @rtype: bool
        """
        try:
            int(s)
            return True
        except (ValueError, TypeError):
            return False
<commit_msg>Modernize py2 code<commit_after># -*- coding: utf-8 -*-
"""Database hookup for doing text string searches.

Must be run from labs and does these (SQL LIKE style) in
labels, aliases and descriptions of items.
"""
from __future__ import print_function
import MySQLdb

class WikidataStringSearch:
    """Run string searches on Wikidata from labs."""

    def __init__(self, verbose=False):
        """
        Set up connection and fetches allowed languages and types.

        @param verbose: if output (errors, queries) should be printed
            (default: False)
        @type verbose: bool
        """
        # Make one d/b connection for all queries - more efficient I believe
        self.conn = MySQLdb.connect(read_default_file="~/replica.my.cnf",
                                    host="wikidatawiki.labsdb",
                                    db="wikidatawiki_p",
                                    charset='utf8'
                                    )
        self.cursor = self.conn.cursor()
        self.term_types = ()
        self.languages = ()
        self.verbose = verbose

        # get types
        self.cursor.execute("SELECT DISTINCT term_type FROM wb_terms;")
        for t in self.cursor.fetchall():
            self.term_types += t
        self._print('found %d term_types' % len(self.term_types))

        # get languages
        self.cursor.execute("SELECT DISTINCT term_language FROM wb_terms;")
        for l in self.cursor.fetchall():
            self.languages += l
        self._print('found %d languages' % len(self.languages))

    def close_connection(self):
        """Close database connection."""
        self.conn.close()

    def testInput(self, text, language=None, term_type=None, entities=None):
        """
        Test that the user input is valid.

        @param text: the text to search for
        @type text: basestring
        @param language: the language to search in, defaults to None=any
        @type language: basestring or None
        @param entities: the language to search in, defaults to None=any
        @type entities: list (of basestring), or None
        @param term_type: field to search in, defaults
            to None=['label', 'alias']
        @type term_type: basestring or None
        @return: if input is valid
        @rtype: bool
        """
        # test text
        if not text.strip():
            self._print('You cannot send stringSearch an empty string')
            return False

        # test language
        if language and language not in self.languages:
            self._print('%s is not a recognised language' % language)
            return False

        # test term_type
        if term_type and term_type not in self.term_types:
            self._print('%s is not a recognised term_type' % term_type)
            return False

        # test list of entities
        if entities:
            if not isinstance(entities, (tuple, list)):
                self._print('Entities must be a non-zero list')
                return False

            # Check each is correctly formatted
            if not all(e.startswith('Q') and
                       isinstance(e, basestring) and
                       WikidataStringSearch.is_int(e[1:])
                       for e in entities):
                self._print('Each entity must be a string like Q<integer>')
                return False

        # nothing flagged
        return True

    def search(self, text, language='sv', term_type=None):
        """
        Search for a given string in a specified language and field.

        Deprecated.
        Calls basic_search with entities=None and language defaulting to 'sv'
        """
        return self.basic_search(text,
                                 language=language,
                                 entities=None,
                                 term_type=term_type)

    def searchInEntities(self, text, entities, language='sv', term_type=None):
        """
        search() but limit results to a provided list of entities.

        Deprecated.
        Calls basic_search with language defaulting to 'sv'
        """
        return self.basic_search(text,
                                 language=language,
                                 entities=entities,
                                 term_type=term_type)

    def basic_search(self, text, language=None, entities=None, term_type=None):
        """
        Search for a given string through exact match or SQL like wild-cards.

        Results can be limited to a specific language, a certain field or
        to within a list of entities.

        @param text: the text to search for
        @type text: basestring
        @param language: the language to search in, defaults to None=any
        @type language: basestring or None
        @param entities: the language to search in, defaults to None=any
        @type entities: list (of basestring), or None
        @param term_type: field to search in, defaults
            to None=['label', 'alias']
        @type term_type: basestring or None
        @return: list of matching Q values
        @rtype: list (of str)
        """
        # validate input
        if not self.testInput(text, language, term_type, entities):
            return None

        # set default term_type
        if term_type is None:
            term_type = ['label', 'alias']
        else:
            term_type = [term_type, ]

        # prepare entities by converting to a list of int strings
        if entities:
            tmp = []
            for e in entities:
                tmp.append(int(e[1:]))
            entities = tmp

        # construct query
        params = []
        query = "SELECT CONCAT('Q',term_entity_id), term_text, term_type " \
                "FROM wb_terms " \
                "WHERE term_entity_type='item' "
        params += term_type
        query += "AND term_type IN %s " % \
                 WikidataStringSearch.sql_in_format(term_type)
        if language:
            params.append(language)
            query += "AND term_language=%s "
        if entities:
            params += entities
            query += "AND term_entity_id IN %s " % \
                     WikidataStringSearch.sql_in_format(entities)
        params.append(text)
        query += "AND term_text LIKE %s " \
                 "LIMIT 100;"

        # execute query
        self.cursor.execute(query, tuple(params))
        self._print(self.cursor._last_executed)

        # handle response
        qs = []
        for q, t, tt in self.cursor.fetchall():
            qs.append(q)

        return qs

    def _print(self, s):
        """
        Print text if verbose.

        @param s: text to print
        @type s: basestring
        """
        if self.verbose:
            print(s)

    @staticmethod
    def sql_in_format(l):
        """
        Given a list of parameters output an sql compatible list of args.

        @param l: value to test
        @type l: list or tuple
        @return: sql compatible list of args
        @rtype: string
        """
        string_format = ', '.join(['%s'] * len(l))
        return '(%s)' % string_format

    @staticmethod
    def is_int(s):
        """
        Check if value is an int.

        @param s: value to test
        @type s: any
        @return: if value can be interpreted as an integer
        @rtype: bool
        """
        try:
            int(s)
            return True
        except (ValueError, TypeError):
            return False
<|endoftext|>


<commit_before>#! /usr/bin/env python

from unit_timeside import unittest, TestRunner
import timeside
import os
verbose = 0

class TestListCoreProcessors(unittest.TestCase):
    """ test get list of processors """

    def testHasSomeDecoders(self):
        "has some decoders"
        procs = timeside.core.processor.processors(timeside.core.api.IDecoder)
        self.assertNotEquals(len(procs), 0)

    def testHasSomeEncoders(self):
        "has some encoders"
        procs = timeside.core.processor.processors(timeside.core.api.IEncoder)
        self.assertNotEquals(len(procs), 0)

    def testHasSomeAnalyzers(self):
        "has some analyzers"
        procs = timeside.core.processor.processors(timeside.core.api.IAnalyzer)
        self.assertNotEquals(len(procs), 0)

    def testHasSomeGraphers(self):
        "has some graphers"
        procs = timeside.core.processor.processors(timeside.core.api.IGrapher)
        self.assertNotEquals(len(procs), 0)

    def testListProcessorsCSV(self):
        "list processors in .csv file"
        fpath = timeside.core.processor.list_processors_csv()
        path = os.getcwd() + '/list_processors.csv'
        self.assertEqual(fpath, path)
        self.assertTrue(os.path.isfile(fpath))

    def tearDown(self):
        path = os.getcwd() + '/list_processors.csv'
        if os.path.isfile(path):
            os.remove(path)

if __name__ == '__main__':
    unittest.main(testRunner=TestRunner())
<commit_msg>[test] use assertNotEqual in test_list_processors.py, remove trailing space<commit_after>#! /usr/bin/env python

from unit_timeside import unittest, TestRunner
import timeside
import os
verbose = 0

class TestListCoreProcessors(unittest.TestCase):
    """ test get list of processors """

    def testHasSomeDecoders(self):
        "has some decoders"
        procs = timeside.core.processor.processors(timeside.core.api.IDecoder)
        self.assertNotEqual(len(procs), 0)

    def testHasSomeEncoders(self):
        "has some encoders"
        procs = timeside.core.processor.processors(timeside.core.api.IEncoder)
        self.assertNotEqual(len(procs), 0)

    def testHasSomeAnalyzers(self):
        "has some analyzers"
        procs = timeside.core.processor.processors(timeside.core.api.IAnalyzer)
        self.assertNotEqual(len(procs), 0)

    def testHasSomeGraphers(self):
        "has some graphers"
        procs = timeside.core.processor.processors(timeside.core.api.IGrapher)
        self.assertNotEqual(len(procs), 0)

    def testListProcessorsCSV(self):
        "list processors in .csv file"
        fpath = timeside.core.processor.list_processors_csv()
        path = os.getcwd() + '/list_processors.csv'
        self.assertEqual(fpath, path)
        self.assertTrue(os.path.isfile(fpath))

    def tearDown(self):
        path = os.getcwd() + '/list_processors.csv'
        if os.path.isfile(path):
            os.remove(path)

if __name__ == '__main__':
    unittest.main(testRunner=TestRunner())
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
import math
import numpy
import voropy

def prune_nodes(points, cells):
    # Only points/cells that actually used
    uvertices, uidx = numpy.unique(cells, return_inverse=True)
    cells = uidx.reshape(cells.shape)
    points = points[uvertices]
    return points, cells

def compute_volume(points, cells):
    if 'tetra' in cells:
        # 3D: only tetras supported
        mesh = voropy.mesh_tetra.MeshTetra(
                *prune_nodes(points, cells['tetra'])
                )
        vol = math.fsum(mesh.cell_volumes)
    else:
        vol = 0.0
        assert 'triangle' in cells or 'quad' in cells
        if 'triangle' in cells:
            # triangles
            mesh = voropy.mesh_tri.MeshTri(
                    *prune_nodes(points, cells['triangle'])
                    )
            vol += math.fsum(mesh.cell_volumes)
        if 'quad' in cells:
            # quad: treat as two triangles
            quads = cells['quad'].T
            split_cells = numpy.column_stack([
                [quads[0], quads[1], quads[2]],
                [quads[0], quads[2], quads[3]],
                ]).T
            mesh = voropy.mesh_tri.MeshTri(*prune_nodes(points, split_cells))
            vol += math.fsum(mesh.cell_volumes)

    return vol

def plot(filename, points, cells):
    import matplotlib.pyplot as plt
    pts = points[:, :2]
    for e in cells['triangle']:
        for idx in [[0, 1], [1, 2], [2, 0]]:
            X = pts[e[idx]]
            plt.plot(X[:, 0], X[:, 1], '-k')
    plt.gca().set_aspect('equal', 'datalim')
    plt.axis('off')

    # plt.show()
    plt.savefig(filename, transparent=True)
    return
<commit_msg>added perimeter computation in compute volume<commit_after># -*- coding: utf-8 -*-
#
import math
import numpy
import voropy

def prune_nodes(points, cells):
    # Only points/cells that actually used
    uvertices, uidx = numpy.unique(cells, return_inverse=True)
    cells = uidx.reshape(cells.shape)
    points = points[uvertices]
    return points, cells

def compute_volume(points, cells):
    if 'tetra' in cells:
        # 3D: only tetras supported
        mesh = voropy.mesh_tetra.MeshTetra(
                *prune_nodes(points, cells['tetra'])
                )
        vol = math.fsum(mesh.cell_volumes)
    elif 'triangle' in cells or 'quad' in cells:
        vol = 0.0
        if 'triangle' in cells:
            # triangles
            mesh = voropy.mesh_tri.MeshTri(
                    *prune_nodes(points, cells['triangle'])
                    )
            vol += math.fsum(mesh.cell_volumes)
        if 'quad' in cells:
            # quad: treat as two triangles
            quads = cells['quad'].T
            split_cells = numpy.column_stack([
                [quads[0], quads[1], quads[2]],
                [quads[0], quads[2], quads[3]],
                ]).T
            mesh = voropy.mesh_tri.MeshTri(*prune_nodes(points, split_cells))
            vol += math.fsum(mesh.cell_volumes)
    else:
        import numpy as np
        import scipy.linalg as LA
        assert 'line' in cells
        segs = np.diff(points[cells['line']], axis=1).squeeze()
        vol = np.sum(LA.norm(segs, axis=1))

    return vol

def plot(filename, points, cells):
    import matplotlib.pyplot as plt
    pts = points[:, :2]
    for e in cells['triangle']:
        for idx in [[0, 1], [1, 2], [2, 0]]:
            X = pts[e[idx]]
            plt.plot(X[:, 0], X[:, 1], '-k')
    plt.gca().set_aspect('equal', 'datalim')
    plt.axis('off')

    # plt.show()
    plt.savefig(filename, transparent=True)
    return
<|endoftext|>


<commit_before>import multiprocessing
import random
import signal

from sanic import Sanic
from sanic.testing import HOST, PORT

def test_multiprocessing():
    """Tests that the number of children we produce is correct"""
    # Selects a number at random so we can spot check
    num_workers = random.choice(range(2,  multiprocessing.cpu_count() * 2 + 1))
    app = Sanic('test_multiprocessing')
    process_list = set()

    def stop_on_alarm(*args):
        for process in multiprocessing.active_children():
            process_list.add(process.pid)
            process.terminate()

    signal.signal(signal.SIGALRM, stop_on_alarm)
    app.run(HOST, PORT, workers=num_workers)
    signal.alarm(1)

    assert len(process_list) == num_workers
<commit_msg>fix hang build<commit_after>import multiprocessing
import random
import signal

from sanic import Sanic
from sanic.testing import HOST, PORT

def test_multiprocessing():
    """Tests that the number of children we produce is correct"""
    # Selects a number at random so we can spot check
    num_workers = random.choice(range(2,  multiprocessing.cpu_count() * 2 + 1))
    app = Sanic('test_multiprocessing')
    process_list = set()

    def stop_on_alarm(*args):
        for process in multiprocessing.active_children():
            process_list.add(process.pid)
            process.terminate()

    signal.signal(signal.SIGALRM, stop_on_alarm)
    signal.alarm(3)
    app.run(HOST, PORT, workers=num_workers)

    assert len(process_list) == num_workers
<|endoftext|>


<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 13-09-2017 13:10
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
Unit tests for the parallelization module
"""<commit_msg>TestCommandLineRunner unit test placeholder<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 13-09-2017 13:10
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
Unit tests for the parallelization module
"""

import unittest

class TestCommandLineRunner(unittest.TestCase):
    pass
<|endoftext|>


<commit_before>"""Tests for assess_perf_test_simple module."""

from contextlib import contextmanager
from datetime import datetime
import os
from mock import call, patch, Mock, mock_open

import pprof_collector as pc
from tests import TestCase

class TestActiveCollector(TestCase):

    def test_installs_introspection_charm(self):
        client = Mock()
        with patch.object(
                pc, 'install_introspection_charm', autospec=True) as iic:
            pc.ActiveCollector(client, 'test')
        iic.assert_called_once_with(client, 'test')

    def test__collect_profile_uses_correct_profile_url(self):
        client = Mock()
        expected_url = pc.get_profile_url(
            '123.123.123.123', 'test', 'test_profile', '12')
        with patch.object(
                pc, 'install_introspection_charm',
                return_value='123.123.123.123',
                autospec=True):
            with patch.object(pc, 'get_profile_reading', autospec=True) as gpr:
                collector = pc.ActiveCollector(client, 'test')
                collector._collect_profile('test_profile', '/tmp/test', '12')
        gpr.assert_called_once_with(expected_url, '/tmp/test')

    @contextmanager
    def mocked_collector(self, machine_id='test'):
        """Create an ActiveCollector with patched _collect_profile.

        Used to test collect_* methods.
        """
        client = Mock()
        with patch.object(
                pc, 'install_introspection_charm',
                return_value='123.123.123.123',
                autospec=True):
            collector = pc.ActiveCollector(client, machine_id)
            with patch.object(collector, '_collect_profile'):
                yield collector

    def test_collect_profile(self):
        with self.mocked_collector() as mc:
            mc.collect_profile('/path/profile', 42)
            mc._collect_profile.assert_called_once_with(
                'profile', '/path/profile', 42)

    def test_collect_heap(self):
        with self.mocked_collector() as mc:
            mc.collect_heap('/path/heap', 42)
            mc._collect_profile.assert_called_once_with(
                'heap', '/path/heap', 42)

    def test_collect_goroutines(self):
        with self.mocked_collector() as mc:
            mc.collect_goroutines('/path/goroutines', 42)
            mc._collect_profile.assert_called_once_with(
                'goroutines', '/path/goroutines', 42)

class TestHelperFunctions(TestCase):

    def test_get_profile_url(self):
        url = pc.get_profile_url('1.1.1.1', 'test', 'test_profile', 123)
        self.assertEqual(
            url,
            'http://1.1.1.1:19090/agents/machine-test/debug/pprof/test_profile'
            '?seconds=123'
        )

    def test_get_profile_reading(self):
        m = mock_open()
        open_patch = '{}.open'.format(pc.__name__)
        mock_response = Mock()
        mock_response.content = 'This is some content.'
        with patch.object(pc, 'requests', autospect=True) as pr:
            pr.get.return_value = mock_response
            with patch(open_patch, m, create=True):
                pc.get_profile_reading('test/url', '/tmp/testing/url')
                m.assert_called_once_with('/tmp/testing/url', 'wb')
                handle = m()
                handle.write.assert_called_once_with('This is some content.')

    def test_installs_introspection_charm(self):
        client = Mock()
        with patch.object(pc, 'get_unit_ipaddress', autospec=True) as p_pui:
            pc.install_introspection_charm(client, 'test')

        client.deploy.assert_called_once_with(
            'cs:~axwalk/juju-introspection', to='test')
        client.wait_for_started.assert_called_once_with()
        client.wait_for_workloads.assert_called_once_with()
        client.juju.assert_called_once_with('expose', 'juju-introspection')
        p_pui.assert_called_once_with(client, 'juju-introspection/0')

class TestPPROFCollector(TestCase):

    def test_creates_profile_directories(self):
        client = Mock()
        log_dir = '/test/logs/dir'
        with patch.object(pc.os, 'makedirs', autospec=True) as p_md:
            pc.PPROFCollector(client, ['test'], log_dir)

        self.assertListEqual(
            p_md.call_args_list,
            [
                call(os.path.join(log_dir, 'cpu_profile')),
                call(os.path.join(log_dir, 'heap_profile')),
                call(os.path.join(log_dir, 'goroutines_profile'))
            ]
        )

    def test_defaults_to_non_active_collector(self):
        client = Mock()
        log_dir = '/test/logs/dir'
        with patch.object(pc.os, 'makedirs', autospec=True):
            collector = pc.PPROFCollector(client, ['test'], log_dir)

            self.assertEqual(type(collector._collectors[0]), pc.NoopCollector)
            self.assertListEqual(collector._active_collectors, [])

    def test_creates_active_collector(self):
        client = Mock()
        log_dir = '/test/logs/dir'
        with patch.object(pc.os, 'makedirs', autospec=True):
            with patch.object(
                    pc, 'install_introspection_charm',
                    autospec=True):
                collector = pc.PPROFCollector(
                    client, ['test'], log_dir, active=True)
                self.assertEqual(
                    type(collector._collectors[0]),
                    pc.ActiveCollector)
                self.assertListEqual(collector._noop_collectors, [])

    def test_initiates_active_collector_after_creation(self):
        client = Mock()
        log_dir = '/test/logs/dir'
        with patch.object(pc.os, 'makedirs', autospec=True):
            collector = pc.PPROFCollector(client, ['test'], log_dir)
            self.assertEqual(type(collector._collectors[0]), pc.NoopCollector)
            with patch.object(
                    pc, 'install_introspection_charm',
                    autospec=True):
                self.assertListEqual(collector._active_collectors, [])
                collector.set_active()
                self.assertEqual(
                    type(collector._collectors[0]), pc.ActiveCollector)

    def test_collect_profile(self):
        client = Mock()
        log_dir = '/test/logs/dir'
        file_name = 'machine-42-170130-092626.pprof'
        profile_log = os.path.join(log_dir, 'cpu_profile', file_name)
        with patch.object(pc.os, 'makedirs', autospec=True):
            with patch.object(pc, 'NoopCollector', autospec=True) as m_nc:
                mock_noop = Mock()
                mock_noop.machine_id = 42
                m_nc.return_value = mock_noop
                with patch.object(pc, 'datetime') as p_dt:
                    p_dt.utcnow.return_value = datetime(
                        2017, 1, 30, 9, 26, 26, 587930)
                    collector = pc.PPROFCollector(client, ['test'], log_dir)
                    collector.collect_profile()
        collector._collectors[0].collect_profile.assert_called_once_with(
            profile_log, 5
        )

    def test_collect_heap(selfm):
        client = Mock()
        log_dir = '/test/logs/dir'
        file_name = 'machine-42-170130-092626.pprof'
        profile_log = os.path.join(log_dir, 'heap_profile', file_name)
        with patch.object(pc.os, 'makedirs', autospec=True):
            with patch.object(pc, 'NoopCollector', autospec=True) as m_nc:
                mock_noop = Mock()
                mock_noop.machine_id = 42
                m_nc.return_value = mock_noop
                with patch.object(pc, 'datetime') as p_dt:
                    p_dt.utcnow.return_value = datetime(
                        2017, 1, 30, 9, 26, 26, 587930)
                    collector = pc.PPROFCollector(client, ['test'], log_dir)
                    collector.collect_heap()
        collector._collectors[0].collect_heap.assert_called_once_with(
            profile_log, 5
        )

    def test_collect_goroutines(selfm):
        client = Mock()
        log_dir = '/test/logs/dir'
        file_name = 'machine-42-170130-092626.pprof'
        profile_log = os.path.join(log_dir, 'goroutines_profile', file_name)
        with patch.object(pc.os, 'makedirs', autospec=True):
            with patch.object(pc, 'NoopCollector', autospec=True) as m_nc:
                mock_noop = Mock()
                mock_noop.machine_id = 42
                m_nc.return_value = mock_noop
                with patch.object(pc, 'datetime') as p_dt:
                    p_dt.utcnow.return_value = datetime(
                        2017, 1, 30, 9, 26, 26, 587930)
                    collector = pc.PPROFCollector(client, ['test'], log_dir)
                    collector.collect_goroutines()
        collector._collectors[0].collect_goroutines.assert_called_once_with(
            profile_log, 5
        )
<commit_msg>Clarify assert check.<commit_after>"""Tests for assess_perf_test_simple module."""

from contextlib import contextmanager
from datetime import datetime
import os
from mock import call, patch, Mock, mock_open

import pprof_collector as pc
from tests import TestCase

class TestActiveCollector(TestCase):

    def test_installs_introspection_charm(self):
        client = Mock()
        with patch.object(
                pc, 'install_introspection_charm', autospec=True) as iic:
            pc.ActiveCollector(client, 'test')
        iic.assert_called_once_with(client, 'test')

    def test__collect_profile_uses_correct_profile_url(self):
        client = Mock()
        expected_url = pc.get_profile_url(
            '123.123.123.123', 'test', 'test_profile', '12')
        with patch.object(
                pc, 'install_introspection_charm',
                return_value='123.123.123.123',
                autospec=True):
            with patch.object(pc, 'get_profile_reading', autospec=True) as gpr:
                collector = pc.ActiveCollector(client, 'test')
                collector._collect_profile('test_profile', '/tmp/test', '12')
        gpr.assert_called_once_with(expected_url, '/tmp/test')

    @contextmanager
    def mocked_collector(self, machine_id='test'):
        """Create an ActiveCollector with patched _collect_profile.

        Used to test collect_* methods.
        """
        client = Mock()
        with patch.object(
                pc, 'install_introspection_charm',
                return_value='123.123.123.123',
                autospec=True):
            collector = pc.ActiveCollector(client, machine_id)
            with patch.object(collector, '_collect_profile'):
                yield collector

    def test_collect_profile(self):
        with self.mocked_collector() as mc:
            mc.collect_profile('/path/profile', 42)
            mc._collect_profile.assert_called_once_with(
                'profile', '/path/profile', 42)

    def test_collect_heap(self):
        with self.mocked_collector() as mc:
            mc.collect_heap('/path/heap', 42)
            mc._collect_profile.assert_called_once_with(
                'heap', '/path/heap', 42)

    def test_collect_goroutines(self):
        with self.mocked_collector() as mc:
            mc.collect_goroutines('/path/goroutines', 42)
            mc._collect_profile.assert_called_once_with(
                'goroutines', '/path/goroutines', 42)

class TestHelperFunctions(TestCase):

    def test_get_profile_url(self):
        url = pc.get_profile_url('1.1.1.1', 'test', 'test_profile', 123)
        self.assertEqual(
            url,
            'http://1.1.1.1:19090/agents/machine-test/debug/pprof/test_profile'
            '?seconds=123'
        )

    def test_get_profile_reading(self):
        m = mock_open()
        open_patch = '{}.open'.format(pc.__name__)
        mock_response = Mock()
        mock_response.content = 'This is some content.'
        with patch.object(pc, 'requests', autospect=True) as pr:
            pr.get.return_value = mock_response
            with patch(open_patch, m, create=True):
                pc.get_profile_reading('test/url', '/tmp/testing/url')
                m.assert_called_once_with('/tmp/testing/url', 'wb')
                handle = m()
                handle.write.assert_called_once_with('This is some content.')

    def test_installs_introspection_charm(self):
        client = Mock()
        with patch.object(pc, 'get_unit_ipaddress', autospec=True) as p_pui:
            pc.install_introspection_charm(client, 'test')

        client.deploy.assert_called_once_with(
            'cs:~axwalk/juju-introspection', to='test')
        client.wait_for_started.assert_called_once_with()
        client.wait_for_workloads.assert_called_once_with()
        client.juju.assert_called_once_with('expose', 'juju-introspection')
        p_pui.assert_called_once_with(client, 'juju-introspection/0')

class TestPPROFCollector(TestCase):

    def test_creates_profile_directories(self):
        client = Mock()
        log_dir = '/test/logs/dir'
        with patch.object(pc.os, 'makedirs', autospec=True) as p_md:
            pc.PPROFCollector(client, ['test'], log_dir)

        self.assertListEqual(
            p_md.call_args_list,
            [
                call(os.path.join(log_dir, 'cpu_profile')),
                call(os.path.join(log_dir, 'heap_profile')),
                call(os.path.join(log_dir, 'goroutines_profile'))
            ]
        )

    def test_defaults_to_non_active_collector(self):
        client = Mock()
        log_dir = '/test/logs/dir'
        with patch.object(pc.os, 'makedirs', autospec=True):
            collector = pc.PPROFCollector(client, ['test'], log_dir)

            self.assertIs(type(collector._collectors[0]), pc.NoopCollector)
            self.assertListEqual(collector._active_collectors, [])

    def test_creates_active_collector(self):
        client = Mock()
        log_dir = '/test/logs/dir'
        with patch.object(pc.os, 'makedirs', autospec=True):
            with patch.object(
                    pc, 'install_introspection_charm',
                    autospec=True):
                collector = pc.PPROFCollector(
                    client, ['test'], log_dir, active=True)
                self.assertIs(
                    type(collector._collectors[0]),
                    pc.ActiveCollector)
                self.assertListEqual(collector._noop_collectors, [])

    def test_initiates_active_collector_after_creation(self):
        client = Mock()
        log_dir = '/test/logs/dir'
        with patch.object(pc.os, 'makedirs', autospec=True):
            collector = pc.PPROFCollector(client, ['test'], log_dir)
            self.assertIs(type(collector._collectors[0]), pc.NoopCollector)
            with patch.object(
                    pc, 'install_introspection_charm',
                    autospec=True):
                self.assertListEqual(collector._active_collectors, [])
                collector.set_active()
                self.assertIs(
                    type(collector._collectors[0]), pc.ActiveCollector)

    def test_collect_profile(self):
        client = Mock()
        log_dir = '/test/logs/dir'
        file_name = 'machine-42-170130-092626.pprof'
        profile_log = os.path.join(log_dir, 'cpu_profile', file_name)
        with patch.object(pc.os, 'makedirs', autospec=True):
            with patch.object(pc, 'NoopCollector', autospec=True) as m_nc:
                mock_noop = Mock()
                mock_noop.machine_id = 42
                m_nc.return_value = mock_noop
                with patch.object(pc, 'datetime') as p_dt:
                    p_dt.utcnow.return_value = datetime(
                        2017, 1, 30, 9, 26, 26, 587930)
                    collector = pc.PPROFCollector(client, ['test'], log_dir)
                    collector.collect_profile()
        collector._collectors[0].collect_profile.assert_called_once_with(
            profile_log, 5
        )

    def test_collect_heap(selfm):
        client = Mock()
        log_dir = '/test/logs/dir'
        file_name = 'machine-42-170130-092626.pprof'
        profile_log = os.path.join(log_dir, 'heap_profile', file_name)
        with patch.object(pc.os, 'makedirs', autospec=True):
            with patch.object(pc, 'NoopCollector', autospec=True) as m_nc:
                mock_noop = Mock()
                mock_noop.machine_id = 42
                m_nc.return_value = mock_noop
                with patch.object(pc, 'datetime') as p_dt:
                    p_dt.utcnow.return_value = datetime(
                        2017, 1, 30, 9, 26, 26, 587930)
                    collector = pc.PPROFCollector(client, ['test'], log_dir)
                    collector.collect_heap()
        collector._collectors[0].collect_heap.assert_called_once_with(
            profile_log, 5
        )

    def test_collect_goroutines(selfm):
        client = Mock()
        log_dir = '/test/logs/dir'
        file_name = 'machine-42-170130-092626.pprof'
        profile_log = os.path.join(log_dir, 'goroutines_profile', file_name)
        with patch.object(pc.os, 'makedirs', autospec=True):
            with patch.object(pc, 'NoopCollector', autospec=True) as m_nc:
                mock_noop = Mock()
                mock_noop.machine_id = 42
                m_nc.return_value = mock_noop
                with patch.object(pc, 'datetime') as p_dt:
                    p_dt.utcnow.return_value = datetime(
                        2017, 1, 30, 9, 26, 26, 587930)
                    collector = pc.PPROFCollector(client, ['test'], log_dir)
                    collector.collect_goroutines()
        collector._collectors[0].collect_goroutines.assert_called_once_with(
            profile_log, 5
        )
<|endoftext|>


<commit_before># Orca
#
# Copyright 2005-2006 Sun Microsystems Inc.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

import orca.atspi as atspi
import orca.debug as debug
import orca.default as default
import orca.orca as orca
import orca.rolenames as rolenames
import orca.settings as settings
import orca.speech as speech
import orca.speechgenerator as speechgenerator
import orca.util as util

from orca.orca_i18n import _

class SpeechGenerator(speechgenerator.SpeechGenerator):
    """Overrides getSpeechContext to handle Mozilla's unique
    hiearchical representation, such as menus duplicating themselves
    in the hierarchy and tables used for layout and indentation purposes.
    """

    def getSpeechContext(self, obj, stopAncestor=None):
        """Get the speech that describes the names and role of
        the container hierarchy of the object, stopping at and
        not including the stopAncestor.

        Arguments:
        - obj: the object
        - stopAncestor: the anscestor to stop at and not include (None
          means include all ancestors)

        Returns a list of utterances to be spoken.
        """

        utterances = []

        if not obj:
            return utterances

        if obj is stopAncestor:
            return utterances

        parent = obj.parent

        # Just skip cells inside cells - it's kind of a nonsensical
        # hierarchy.
        #
        while True:
            if parent \
                and (obj.role == rolenames.ROLE_TABLE_CELL) \
                and (parent.role == rolenames.ROLE_TABLE_CELL):
                parent = parent.parent
            else:
                break

        # We'll eliminate toolbars if we're in a menu.  The reason for
        # this is that Mozilla puts its menu bar in a couple of nested
        # toolbars.
        #
        inMenuBar = False

        while parent and (parent.parent != parent):
            if parent == stopAncestor:
                break

            if parent.role == rolenames.ROLE_MENU_BAR:
                inMenuBar = True

            # We try to omit things like fillers off the bat...
            #
            if (parent.role != rolenames.ROLE_FILLER) \
                and (parent.role != rolenames.ROLE_SPLIT_PANE) \
                and (parent.role != rolenames.ROLE_UNKNOWN):

                # If it has a label, we typically want to speak it, unless
                # we're on a menu, but the menu is not FOCUSABLE (this
                # is the way to identify the menu that's duplicated in the
                # hierarchy)
                #
                label = util.getLabel(parent)
                if label and len(label):
                    if (parent.role == rolenames.ROLE_MENU) \
                       and not parent.state.count(\
                                   atspi.Accessibility.STATE_FOCUSABLE):
                        pass
                    else:
                        utterances.append(label + " " \
                                      + rolenames.getSpeechForRoleName(parent))
                # Otherwise, we won't speak it if it is a panel with
                # no name,
                elif parent.role == rolenames.ROLE_PANEL:
                    pass
                # Or if we're in a menu and this is a toolbar,
                #
                elif inMenuBar and (parent.role == rolenames.ROLE_TOOL_BAR):
                    pass
                else:
                    utterances.append(rolenames.getSpeechForRoleName(parent))

            parent = parent.parent

        utterances.reverse()

        return utterances

class Script(default.Script):
    """The script for Firefox.

    NOTE: THIS IS UNDER DEVELOPMENT AND DOES NOT PROVIDE ANY COMPELLING
    ACCESS TO FIREFOX AT THIS POINT.
    """

    def __init__(self, app):
        #print "Mozilla.__init__"
        default.Script.__init__(self, app)

    def getSpeechGenerator(self):
        """Returns the speech generator for this script.
        """
        return SpeechGenerator()

    def onCaretMoved(self, event):
        #print "Mozilla.onCaretMoved"
        #print "  source        =", event.source
        #print "  source.parent =", event.source.parent
        #print "  lof           =", orca.locusOfFocus

        # We always automatically go back to focus tracking mode when
        # the caret moves in the focused object.
        #
        if self.flatReviewContext:
            self.toggleFlatReviewMode()

        [string, startOffset, endOffset] = event.source.text.getTextAtOffset(
            event.source.text.caretOffset,
            atspi.Accessibility.TEXT_BOUNDARY_LINE_START)
        #print "  text line at offset", string

        self._presentTextAtNewCaretPosition(event)

    def onNameChanged(self, event):
        """Called whenever a property on an object changes.

        Arguments:
        - event: the Event
        """

        # We ignore these because Mozilla just happily keeps generating
        # name changed events for objects whose name doesn't change.
        #
        return

    # This function is called whenever an object within Mozilla receives
    # focus
    #
    def onFocus(self, event):
        #print "Mozilla.onFocus:", event.type, event.source.toString()

        # We're going to ignore focus events on the frame.  They
        # are often intermingled with menu activity, wreaking havoc
        # on the context.
        #
        if event.source.role == rolenames.ROLE_FRAME:
            return

        # We're also going to ignore menus that are children of menu
        # bars.  They never really get focus themselves - it's always
        # a transient event and one of the menu items or submenus will
        # get focus immediately after the menu gets focus.  So...we
        # compress the events.
        #
        if (event.source.role == rolenames.ROLE_MENU) \
           and event.source.parent \
           and (event.source.parent.role == rolenames.ROLE_MENU_BAR):
            return

        default.Script.onFocus(self, event)

    # This function is called when a hyperlink is selected - This happens
    # when a link is navigated to using tab/shift-tab
    #
    def onLinkSelected(self, event):
        text = event.source.text
        hypertext = event.source.hypertext
        linkIndex = util.getLinkIndex(event.source, text.caretOffset)

        if linkIndex >= 0:
            link = hypertext.getLink(linkIndex)
            linkText = text.getText(link.startIndex, link.endIndex)
            [string, startOffset, endOffset] = text.getTextAtOffset(
                text.caretOffset,
                atspi.Accessibility.TEXT_BOUNDARY_LINE_START)
            #print "onLinkSelected", event.source.role, string,
            #print "  caretOffset:     ", text.caretOffset
            #print "  line startOffset:", startOffset
            #print "  line endOffset:  ", startOffset
            #print "  caret in line:   ", text.caretOffset - startOffset
            speech.speak(linkText, self.voices[settings.HYPERLINK_VOICE])
        else:
            speech.speak(_("link"), self.voices[settings.HYPERLINK_VOICE])

        self.updateBraille(event.source)

    def locusOfFocusChanged(self, event, oldLocusOfFocus, newLocusOfFocus):
        """Called when the visual object with focus changes.

        Arguments:
        - event: if not None, the Event that caused the change
        - oldLocusOfFocus: Accessible that is the old locus of focus
        - newLocusOfFocus: Accessible that is the new locus of focus
        """
        # Don't bother speaking all the information about the HTML
        # container - it's duplicated all over the place.  So, we
        # just speak the role.
        #
        if newLocusOfFocus \
           and newLocusOfFocus.role == rolenames.ROLE_HTML_CONTAINER:
            # We always automatically go back to focus tracking mode when
            # the focus changes.
            #
            if self.flatReviewContext:
                self.toggleFlatReviewMode()
            self.updateBraille(newLocusOfFocus)
            speech.speak(rolenames.getSpeechForRoleName(newLocusOfFocus))
        else:
            default.Script.locusOfFocusChanged(self,
                                               event,
                                               oldLocusOfFocus,
                                               newLocusOfFocus)
<commit_msg>Fix typo in util.getDisplayedLabel<commit_after># Orca
#
# Copyright 2005-2006 Sun Microsystems Inc.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Library General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Library General Public License for more details.
#
# You should have received a copy of the GNU Library General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

import orca.atspi as atspi
import orca.debug as debug
import orca.default as default
import orca.orca as orca
import orca.rolenames as rolenames
import orca.settings as settings
import orca.speech as speech
import orca.speechgenerator as speechgenerator
import orca.util as util

from orca.orca_i18n import _

class SpeechGenerator(speechgenerator.SpeechGenerator):
    """Overrides getSpeechContext to handle Mozilla's unique
    hiearchical representation, such as menus duplicating themselves
    in the hierarchy and tables used for layout and indentation purposes.
    """

    def getSpeechContext(self, obj, stopAncestor=None):
        """Get the speech that describes the names and role of
        the container hierarchy of the object, stopping at and
        not including the stopAncestor.

        Arguments:
        - obj: the object
        - stopAncestor: the anscestor to stop at and not include (None
          means include all ancestors)

        Returns a list of utterances to be spoken.
        """

        utterances = []

        if not obj:
            return utterances

        if obj is stopAncestor:
            return utterances

        parent = obj.parent

        # Just skip cells inside cells - it's kind of a nonsensical
        # hierarchy.
        #
        while True:
            if parent \
                and (obj.role == rolenames.ROLE_TABLE_CELL) \
                and (parent.role == rolenames.ROLE_TABLE_CELL):
                parent = parent.parent
            else:
                break

        # We'll eliminate toolbars if we're in a menu.  The reason for
        # this is that Mozilla puts its menu bar in a couple of nested
        # toolbars.
        #
        inMenuBar = False

        while parent and (parent.parent != parent):
            if parent == stopAncestor:
                break

            if parent.role == rolenames.ROLE_MENU_BAR:
                inMenuBar = True

            # We try to omit things like fillers off the bat...
            #
            if (parent.role != rolenames.ROLE_FILLER) \
                and (parent.role != rolenames.ROLE_SPLIT_PANE) \
                and (parent.role != rolenames.ROLE_UNKNOWN):

                # If it has a label, we typically want to speak it, unless
                # we're on a menu, but the menu is not FOCUSABLE (this
                # is the way to identify the menu that's duplicated in the
                # hierarchy)
                #
                label = util.getDisplayedLabel(parent)
                if label and len(label):
                    if (parent.role == rolenames.ROLE_MENU) \
                       and not parent.state.count(\
                                   atspi.Accessibility.STATE_FOCUSABLE):
                        pass
                    else:
                        utterances.append(label + " " \
                                      + rolenames.getSpeechForRoleName(parent))
                # Otherwise, we won't speak it if it is a panel with
                # no name,
                elif parent.role == rolenames.ROLE_PANEL:
                    pass
                # Or if we're in a menu and this is a toolbar,
                #
                elif inMenuBar and (parent.role == rolenames.ROLE_TOOL_BAR):
                    pass
                else:
                    utterances.append(rolenames.getSpeechForRoleName(parent))

            parent = parent.parent

        utterances.reverse()

        return utterances

class Script(default.Script):
    """The script for Firefox.

    NOTE: THIS IS UNDER DEVELOPMENT AND DOES NOT PROVIDE ANY COMPELLING
    ACCESS TO FIREFOX AT THIS POINT.
    """

    def __init__(self, app):
        #print "Mozilla.__init__"
        default.Script.__init__(self, app)

    def getSpeechGenerator(self):
        """Returns the speech generator for this script.
        """
        return SpeechGenerator()

    def onCaretMoved(self, event):
        #print "Mozilla.onCaretMoved"
        #print "  source        =", event.source
        #print "  source.parent =", event.source.parent
        #print "  lof           =", orca.locusOfFocus

        # We always automatically go back to focus tracking mode when
        # the caret moves in the focused object.
        #
        if self.flatReviewContext:
            self.toggleFlatReviewMode()

        [string, startOffset, endOffset] = event.source.text.getTextAtOffset(
            event.source.text.caretOffset,
            atspi.Accessibility.TEXT_BOUNDARY_LINE_START)
        #print "  text line at offset", string

        self._presentTextAtNewCaretPosition(event)

    def onNameChanged(self, event):
        """Called whenever a property on an object changes.

        Arguments:
        - event: the Event
        """

        # We ignore these because Mozilla just happily keeps generating
        # name changed events for objects whose name doesn't change.
        #
        return

    # This function is called whenever an object within Mozilla receives
    # focus
    #
    def onFocus(self, event):
        #print "Mozilla.onFocus:", event.type, event.source.toString()

        # We're going to ignore focus events on the frame.  They
        # are often intermingled with menu activity, wreaking havoc
        # on the context.
        #
        if event.source.role == rolenames.ROLE_FRAME:
            return

        # We're also going to ignore menus that are children of menu
        # bars.  They never really get focus themselves - it's always
        # a transient event and one of the menu items or submenus will
        # get focus immediately after the menu gets focus.  So...we
        # compress the events.
        #
        if (event.source.role == rolenames.ROLE_MENU) \
           and event.source.parent \
           and (event.source.parent.role == rolenames.ROLE_MENU_BAR):
            return

        default.Script.onFocus(self, event)

    # This function is called when a hyperlink is selected - This happens
    # when a link is navigated to using tab/shift-tab
    #
    def onLinkSelected(self, event):
        text = event.source.text
        hypertext = event.source.hypertext
        linkIndex = util.getLinkIndex(event.source, text.caretOffset)

        if linkIndex >= 0:
            link = hypertext.getLink(linkIndex)
            linkText = text.getText(link.startIndex, link.endIndex)
            [string, startOffset, endOffset] = text.getTextAtOffset(
                text.caretOffset,
                atspi.Accessibility.TEXT_BOUNDARY_LINE_START)
            #print "onLinkSelected", event.source.role, string,
            #print "  caretOffset:     ", text.caretOffset
            #print "  line startOffset:", startOffset
            #print "  line endOffset:  ", startOffset
            #print "  caret in line:   ", text.caretOffset - startOffset
            speech.speak(linkText, self.voices[settings.HYPERLINK_VOICE])
        else:
            speech.speak(_("link"), self.voices[settings.HYPERLINK_VOICE])

        self.updateBraille(event.source)

    def locusOfFocusChanged(self, event, oldLocusOfFocus, newLocusOfFocus):
        """Called when the visual object with focus changes.

        Arguments:
        - event: if not None, the Event that caused the change
        - oldLocusOfFocus: Accessible that is the old locus of focus
        - newLocusOfFocus: Accessible that is the new locus of focus
        """
        # Don't bother speaking all the information about the HTML
        # container - it's duplicated all over the place.  So, we
        # just speak the role.
        #
        if newLocusOfFocus \
           and newLocusOfFocus.role == rolenames.ROLE_HTML_CONTAINER:
            # We always automatically go back to focus tracking mode when
            # the focus changes.
            #
            if self.flatReviewContext:
                self.toggleFlatReviewMode()
            self.updateBraille(newLocusOfFocus)
            speech.speak(rolenames.getSpeechForRoleName(newLocusOfFocus))
        else:
            default.Script.locusOfFocusChanged(self,
                                               event,
                                               oldLocusOfFocus,
                                               newLocusOfFocus)
<|endoftext|>


<commit_before>''' Script to check the correctness of the analysis. The analysis is done on raw data and all results are compared to a recorded analysis.
'''
import unittest
import os

from testbeam_analysis import result_analysis

tests_data_folder = r'tests/test_result_analysis/'

class TestResultAnalysis(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        if os.name != 'nt':
            from xvfbwrapper import Xvfb
            cls.vdisplay = Xvfb()
            cls.vdisplay.start()
        cls.output_folder = tests_data_folder
        cls.pixel_size = (250, 50)  # in um
        cls.z_positions = [0., 19500, 108800, 128300]  # in um

    @classmethod
    def tearDownClass(cls):  # remove created files
        os.remove(cls.output_folder + 'Efficiency.pdf')
        os.remove(cls.output_folder + 'Residuals.pdf')

    def test_residuals_calculation(self):
        residuals = result_analysis.calculate_residuals(tracks_file=tests_data_folder + 'Tracks_result.h5',
                                                        output_pdf=self.output_folder + 'Residuals.pdf',
                                                        z_positions=self.z_positions,
                                                        use_duts=None,
                                                        max_chi2=10000)

        # Only test row residuals, columns are too large (250 um) for meaningfull gaussian residuals distribution
        self.assertAlmostEqual(residuals[1], 22.913569705886552, msg='DUT 0 row residuals do not match')
        self.assertAlmostEqual(residuals[3], 18.73170503500376, msg='DUT 1 row residuals do not match')
        self.assertAlmostEqual(residuals[5], 22.864586411335129, msg='DUT 2 row residuals do not match')
        self.assertAlmostEqual(residuals[7], 27.28165821984965, msg='DUT 3 row residuals do not match')

    def test_efficiency_calculation(self):
        efficiencies = result_analysis.calculate_efficiency(tracks_file=self.output_folder + 'Tracks_result.h5',
                                                            output_pdf=self.output_folder + r'/Efficiency.pdf',
                                                            z_positions=self.z_positions,
                                                            bin_size=(250, 50),
                                                            minimum_track_density=2,
                                                            use_duts=None,
                                                            cut_distance=500,
                                                            max_distance=500,
                                                            col_range=(1250, 17500),
                                                            row_range=(1000, 16000))
        self.assertAlmostEqual(efficiencies[0], 100.0, msg='DUT 0 efficiencies do not match')
        self.assertAlmostEqual(efficiencies[1], 98.701298701298697, msg='DUT 1 efficiencies do not match')
        self.assertAlmostEqual(efficiencies[2], 97.468354430379748, msg='DUT 2 efficiencies do not match')
        self.assertAlmostEqual(efficiencies[3], 100.0, msg='DUT 3 efficiencies do not match')

if __name__ == '__main__':
    tests_data_folder = r'test_result_analysis/'
    suite = unittest.TestLoader().loadTestsFromTestCase(TestResultAnalysis)
    unittest.TextTestRunner(verbosity=2).run(suite)
<commit_msg>ENH: decrease the result comparison<commit_after>''' Script to check the correctness of the analysis. The analysis is done on raw data and all results are compared to a recorded analysis.
'''
import unittest
import os

from testbeam_analysis import result_analysis

tests_data_folder = r'tests/test_result_analysis/'

class TestResultAnalysis(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        if os.name != 'nt':
            from xvfbwrapper import Xvfb
            cls.vdisplay = Xvfb()
            cls.vdisplay.start()
        cls.output_folder = tests_data_folder
        cls.pixel_size = (250, 50)  # in um
        cls.z_positions = [0., 19500, 108800, 128300]  # in um

    @classmethod
    def tearDownClass(cls):  # remove created files
        os.remove(cls.output_folder + 'Efficiency.pdf')
        os.remove(cls.output_folder + 'Residuals.pdf')

    def test_residuals_calculation(self):
        residuals = result_analysis.calculate_residuals(tracks_file=tests_data_folder + 'Tracks_result.h5',
                                                        output_pdf=self.output_folder + 'Residuals.pdf',
                                                        z_positions=self.z_positions,
                                                        use_duts=None,
                                                        max_chi2=10000)
        # Only test row residuals, columns are too large (250 um) for meaningfull gaussian residuals distribution
        self.assertAlmostEqual(residuals[1], 22.9135, msg='DUT 0 row residuals do not match', places=3)
        self.assertAlmostEqual(residuals[3], 18.7317, msg='DUT 1 row residuals do not match', places=3)
        self.assertAlmostEqual(residuals[5], 22.8645, msg='DUT 2 row residuals do not match', places=3)
        self.assertAlmostEqual(residuals[7], 27.2816, msg='DUT 3 row residuals do not match', places=3)

    def test_efficiency_calculation(self):
        efficiencies = result_analysis.calculate_efficiency(tracks_file=self.output_folder + 'Tracks_result.h5',
                                                            output_pdf=self.output_folder + r'/Efficiency.pdf',
                                                            z_positions=self.z_positions,
                                                            bin_size=(250, 50),
                                                            minimum_track_density=2,
                                                            use_duts=None,
                                                            cut_distance=500,
                                                            max_distance=500,
                                                            col_range=(1250, 17500),
                                                            row_range=(1000, 16000))
        self.assertAlmostEqual(efficiencies[0], 100.0, msg='DUT 0 efficiencies do not match', places=3)
        self.assertAlmostEqual(efficiencies[1], 98.701298701298697, msg='DUT 1 efficiencies do not match', places=3)
        self.assertAlmostEqual(efficiencies[2], 97.468354430379748, msg='DUT 2 efficiencies do not match', places=3)
        self.assertAlmostEqual(efficiencies[3], 100.0, msg='DUT 3 efficiencies do not match', places=3)

if __name__ == '__main__':
    tests_data_folder = r'test_result_analysis/'
    suite = unittest.TestLoader().loadTestsFromTestCase(TestResultAnalysis)
    unittest.TextTestRunner(verbosity=2).run(suite)
<|endoftext|>