<commit_before>from __future__ import division, print_function, absolute_import

import os.path
import sys

from perf._formatter import (format_seconds, format_number,
                             format_timedelta, format_datetime)
from perf._metadata import format_metadata as _format_metadata

def empty_line(lines):
    if lines:
        lines.append('')

def format_title(title, level=1, lines=None):
    if lines is None:
        lines = []

    empty_line(lines)

    lines.append(title)
    if level == 1:
        char = '='
    else:
        char = '-'
    lines.append(char * len(title))
    return lines

def display_title(title, level=1):
    for line in format_title(title, level):
        print(line)
    print()

def format_metadata(metadata, prefix='- ', lines=None):
    if lines is None:
        lines = []
    for name, value in sorted(metadata.items()):
        value = _format_metadata(name, value)
        lines.append("%s%s: %s" % (prefix, name, value))
    return lines

def format_run(bench, run_index, run, common_metadata=None, raw=False,
               verbose=0, lines=None):
    if lines is None:
        lines = []

    if run._is_calibration():
        lines.append("Run %s: calibrate" % (run_index,))
        for loops, value in run.warmups:
            lines.append("- %s: %s"
                         % (format_number(loops, 'loop'),
                            format_timedelta(value)))
        return lines

    show_warmup = (verbose >= 0)

    total_loops = run.get_total_loops()
    inner_loops = run._get_inner_loops()

    def format_values(values, percent=True):
        values_str = [bench.format_value(value) for value in values]
        if not percent:
            return values_str

        mean = bench.mean()
        max_delta = mean * 0.05
        for index, value in enumerate(values):
            if raw:
                value = float(value) / total_loops
            delta = float(value) - mean
            if abs(delta) > max_delta:
                values_str[index] += ' (%+.0f%%)' % (delta * 100 / mean)
        return values_str

    values = run.values
    if raw:
        warmups = [('%s (%s)'
                    % (bench.format_value(raw_value),
                       format_number(loops, 'loop')))
                   for loops, raw_value in run.warmups]
        values = [value * total_loops for value in values]
    else:
        warmups = run.warmups
        if warmups:
            warmups = [raw_value / (loops * inner_loops)
                       for loops, raw_value in warmups]
            warmups = format_values(warmups)
    values = format_values(values)

    if raw:
        name = 'raw values'
    else:
        name = 'values'
    text = '%s (%s): %s' % (name, len(values), ', '.join(values))
    if warmups and show_warmup:
        if raw:
            name = 'raw warmup'
        else:
            name = 'warmup'
        text = ('%s (%s): %s; %s'
                % (name, len(warmups), ', '.join(warmups), text))

    text = "Run %s: %s" % (run_index, text)
    lines.append(text)

    if verbose > 0:
        prefix = '  '
        metadata = run.get_metadata()
        for name, value in sorted(metadata.items()):
            if common_metadata and name in common_metadata:
                continue
            value = _format_metadata(name, value)
            lines.append('%s%s: %s' % (prefix, name, value))

    return lines

def _format_runs(bench, quiet=False, verbose=False, raw=False, lines=None):
    runs = bench.get_runs()
    if quiet:
        verbose = -1
    elif verbose:
        verbose = 1
    else:
        verbose = 0

    if lines is None:
        lines = []
    if verbose > 0:
        empty_line(lines)

        # FIXME: display metadata in format_benchmark()
        common_metadata = bench.get_metadata()
        lines.append("Metadata:")
        format_metadata(common_metadata, prefix='  ', lines=lines)
    else:
        common_metadata = None

    empty_line_written = False
    for run_index, run in enumerate(runs, 1):
        if quiet and run._is_calibration():
            continue
        if not empty_line_written:
            empty_line_written = True
            empty_line(lines)
        format_run(bench, run_index, run,
                   common_metadata=common_metadata,
                   verbose=verbose, raw=raw, lines=lines)

    return lines

PERCENTILE_NAMES = {0: 'minimum', 50: 'median', 100: 'maximum'}

def format_stats(bench, lines):
    fmt = bench.format_value
    values = bench.get_values()

    nrun = bench.get_nrun()
    nvalue = len(values)

    empty_line(lines)

    # Total duration
    duration = bench.get_total_duration()
    if duration:
        lines.append("Total duration: %s" % format_seconds(duration))

    # Start/End dates
    dates = bench.get_dates()
    if dates:
        start, end = dates
        lines.append("Start date: %s" % format_datetime(start, microsecond=False))
        lines.append("End date: %s" % format_datetime(end, microsecond=False))

    # Raw value minimize/maximum
    raw_values = bench._get_raw_values()
    lines.append("Raw value minimum: %s" % bench.format_value(min(raw_values)))
    lines.append("Raw value maximum: %s" % bench.format_value(max(raw_values)))
    lines.append('')

    # Number of values
    lines.append("Number of runs: %s" % format_number(nrun))
    lines.append("Total number of values: %s" % format_number(nvalue))

    nvalue_per_run = bench._get_nvalue_per_run()
    text = format_number(nvalue_per_run)
    if isinstance(nvalue_per_run, float):
        text += ' (average)'
    lines.append('Number of values per run: %s' % text)

    nwarmup = bench._get_nwarmup()
    text = format_number(nwarmup)
    if isinstance(nwarmup, float):
        text += ' (average)'
    lines.append('Number of warmups per run: %s' % text)

    # Loop iterations per value
    loops = bench._get_loops()
    inner_loops = bench._get_inner_loops()
    total_loops = loops * inner_loops
    if isinstance(total_loops, int):
        text = format_number(total_loops)
    else:
        text = "%s (average)" % total_loops

    if not(isinstance(inner_loops, int) and inner_loops == 1):
        if isinstance(loops, int):
            loops = format_number(loops, 'outer-loop')
        else:
            loops = '%.1f outer-loops (average)'

        if isinstance(inner_loops, int):
            inner_loops = format_number(inner_loops, 'inner-loop')
        else:
            inner_loops = "%.1f inner-loops (average)" % inner_loops

        text = '%s (%s x %s)' % (text, loops, inner_loops)

    lines.append("Loop iterations per value: %s" % text)
    lines.append('')

    # Minimum
    table = []
    table.append(("Minimum", bench.format_value(min(values))))

    # Median +- MAD
    median = bench.median()
    if len(values) > 2:
        median_abs_dev = bench.median_abs_dev()
        table.append(("Median +- MAD",
                      "%s +- %s"
                      % bench.format_values((median, median_abs_dev))))
    else:
        table.append(("Mean", bench.format_value(median)))

    # Mean +- std dev
    mean = bench.mean()
    if len(values) > 2:
        stdev = bench.stdev()
        table.append(("Mean +- std dev",
                      "%s +- %s" % bench.format_values((mean, stdev))))
    else:
        table.append(("Mean", bench.format_value(mean)))

    table.append(("Maximum", bench.format_value(max(values))))

    # Render table
    width = max(len(row[0]) + 1 for row in table)
    for key, value in table:
        key = (key + ':').ljust(width)
        lines.append("%s %s" % (key, value))
    lines.append('')

    def format_limit(mean, value):
        return ("%s (%+.0f%% of the mean)"
                % (fmt(value), (value - mean) * 100.0 / mean))

    # Percentiles
    for p in (0, 5, 25, 50, 75, 95, 100):
        text = format_limit(mean, bench.percentile(p))
        text = "%3sth percentile: %s" % (p, text)
        name = PERCENTILE_NAMES.get(p)
        if name:
            text = '%s -- %s' % (text, name)
        lines.append(text)
    return lines

def format_histogram(benchmarks, bins=20, extend=False, lines=None,
                     checks=False):
    import collections
    import shutil

    if hasattr(shutil, 'get_terminal_size'):
        columns, nline = shutil.get_terminal_size()
    else:
        columns, nline = (80, 25)

    if not bins:
        bins = max(nline - 3, 3)
        if not extend:
            bins = min(bins, 25)

    all_values = []
    for bench, title in benchmarks:
        all_values.extend(bench.get_values())
    all_min = min(all_values)
    all_max = max(all_values)
    value_k = float(all_max - all_min) / bins
    if not value_k:
        value_k = 1.0

    def value_bucket(value):
        # round towards zero (ROUND_DOWN)
        return int(value / value_k)

    bucket_min = value_bucket(all_min)
    bucket_max = value_bucket(all_max)
    if lines is None:
        lines = []

    for item in benchmarks:
        empty_line(lines)

        bench, title = item
        if title:
            lines.append("[ %s ]" % title)

        values = bench.get_values()

        buckets = [value_bucket(value) for value in values]
        counter = collections.Counter(buckets)
        count_max = max(counter.values())
        count_width = len(str(count_max))

        value_width = max([len(bench.format_value(bucket * value_k))
                           for bucket in range(bucket_min, bucket_max + 1)])
        width = columns - value_width

        line = ': %s #' % count_max
        width = columns - (value_width + len(line))
        if not extend:
            width = min(width, 79)
        width = max(width, 3)
        line_k = float(width) / max(counter.values())
        for bucket in range(bucket_min, bucket_max + 1):
            count = counter.get(bucket, 0)
            linelen = int(round(count * line_k))
            text = bench.format_value(bucket * value_k)
            line = ('#' * linelen) or '|'
            lines.append("{:>{}}: {:>{}} {}".format(text, value_width,
                                                    count, count_width, line))

        if checks:
            format_checks(bench, lines=lines)

    return lines

def format_checks(bench, lines=None):
    if lines is None:
        lines = []
    values = bench.get_values()
    mean = bench.mean()
    warnings = []
    warn = warnings.append

    # Display a warning if the standard deviation is larger than 10%
    if len(values) >= 2:
        stdev = bench.stdev()
        percent = stdev * 100.0 / mean
        if percent >= 10.0:
            warn("the standard deviation (%s) is %.0f%% of the mean (%s)"
                 % (bench.format_value(stdev), percent, bench.format_value(mean)))

    # Minimum and maximum, detect obvious outliers
    for minimum, value in (
        ('minimum', min(values)),
        ('maximum', max(values)),
    ):
        percent = (value - mean) * 100.0 / mean
        if abs(percent) >= 25:
            if percent >= 0:
                text = "%.0f%% greater" % (percent)
            else:
                text = "%.0f%% smaller" % (-percent)
            warn("the %s (%s) is %s than the mean (%s)"
                 % (minimum, bench.format_value(value), text, bench.format_value(mean)))

    # Check that the shortest value took at least 1 ms
    if bench.get_unit() == 'second':
        shortest = min(bench._get_raw_values())
        if shortest < 1e-3:
            warn("the shortest raw value only took %s"
                 % bench.format_value(shortest))

    if warnings:
        empty_line(lines)
        lines.append("WARNING: the benchmark result may be unstable")
        for msg in warnings:
            lines.append("* %s" % msg)
        empty_line(lines)
        lines.append("Try to rerun the benchmark with more runs, values "
                     "and/or loops.")
        lines.append("Run '%s -m perf system tune' command to reduce "
                     "the system jitter."
                     % os.path.basename(sys.executable))
        lines.append("Use perf stats, perf dump and perf hist to analyze results.")
        lines.append("Use --quiet option to hide these warnings.")

    # Warn if nohz_full+intel_pstate combo if found in cpu_config metadata
    for run in bench._runs:
        cpu_config = run._metadata.get('cpu_config')
        if not cpu_config:
            continue
        if 'nohz_full' in cpu_config and 'intel_pstate' in cpu_config:
            empty_line(lines)
            warn("WARNING: nohz_full is enabled on CPUs which use the "
                 "intel_pstate driver, whereas intel_pstate is incompatible "
                 "with nohz_full")
            warn("CPU config: %s" % cpu_config)
            warn("See https://bugzilla.redhat.com/show_bug.cgi?id=1378529")
            break

    return lines

def format_result_value(bench):
    loops = bench._only_calibration()
    if loops is not None:
        return '<calibration: %s>' % format_number(loops, 'loop')

    if bench.get_nvalue() >= 2:
        numbers = [bench.mean()]
        numbers.append(bench.stdev())
        numbers = bench.format_values(numbers)
        text = '%s +- %s' % numbers
    else:
        text = bench.format_value(bench.mean())
    return text

def format_result(bench, prefix=True):
    loops = bench._only_calibration()
    if loops is not None:
        return 'Calibration: %s' % format_number(loops, 'loop')

    text = format_result_value(bench)
    if bench.get_nvalue() >= 2:
        return 'Mean +- std dev: %s' % text
    else:
        return text

def format_benchmark(bench, checks=True, metadata=False,
                     dump=False, stats=False, hist=False, show_name=False,
                     result=True, display_runs_args=None):
    lines = []

    if metadata:
        lines.append("Metadata:")
        format_metadata(bench.get_metadata(), lines=lines)

    if dump:
        if display_runs_args is None:
            display_runs_args = {}
        _format_runs(bench, lines=lines, **display_runs_args)

    if hist:
        format_histogram([(bench, None)], lines=lines)

    if stats:
        format_stats(bench, lines=lines)

    if checks:
        format_checks(bench, lines=lines)

    if result:
        empty_line(lines)

        text = format_result(bench)
        if show_name:
            name = bench.get_name()
            text = "%s: %s" % (name, text)
        lines.append(text)

    return lines

# FIXME: remove this function?
def multiline_output(args):
    return (args.hist or args.stats or args.dump or args.metadata)
<commit_msg>Cleanup format_result_value()<commit_after>from __future__ import division, print_function, absolute_import

import os.path
import sys

from perf._formatter import (format_seconds, format_number,
                             format_timedelta, format_datetime)
from perf._metadata import format_metadata as _format_metadata

def empty_line(lines):
    if lines:
        lines.append('')

def format_title(title, level=1, lines=None):
    if lines is None:
        lines = []

    empty_line(lines)

    lines.append(title)
    if level == 1:
        char = '='
    else:
        char = '-'
    lines.append(char * len(title))
    return lines

def display_title(title, level=1):
    for line in format_title(title, level):
        print(line)
    print()

def format_metadata(metadata, prefix='- ', lines=None):
    if lines is None:
        lines = []
    for name, value in sorted(metadata.items()):
        value = _format_metadata(name, value)
        lines.append("%s%s: %s" % (prefix, name, value))
    return lines

def format_run(bench, run_index, run, common_metadata=None, raw=False,
               verbose=0, lines=None):
    if lines is None:
        lines = []

    if run._is_calibration():
        lines.append("Run %s: calibrate" % (run_index,))
        for loops, value in run.warmups:
            lines.append("- %s: %s"
                         % (format_number(loops, 'loop'),
                            format_timedelta(value)))
        return lines

    show_warmup = (verbose >= 0)

    total_loops = run.get_total_loops()
    inner_loops = run._get_inner_loops()

    def format_values(values, percent=True):
        values_str = [bench.format_value(value) for value in values]
        if not percent:
            return values_str

        mean = bench.mean()
        max_delta = mean * 0.05
        for index, value in enumerate(values):
            if raw:
                value = float(value) / total_loops
            delta = float(value) - mean
            if abs(delta) > max_delta:
                values_str[index] += ' (%+.0f%%)' % (delta * 100 / mean)
        return values_str

    values = run.values
    if raw:
        warmups = [('%s (%s)'
                    % (bench.format_value(raw_value),
                       format_number(loops, 'loop')))
                   for loops, raw_value in run.warmups]
        values = [value * total_loops for value in values]
    else:
        warmups = run.warmups
        if warmups:
            warmups = [raw_value / (loops * inner_loops)
                       for loops, raw_value in warmups]
            warmups = format_values(warmups)
    values = format_values(values)

    if raw:
        name = 'raw values'
    else:
        name = 'values'
    text = '%s (%s): %s' % (name, len(values), ', '.join(values))
    if warmups and show_warmup:
        if raw:
            name = 'raw warmup'
        else:
            name = 'warmup'
        text = ('%s (%s): %s; %s'
                % (name, len(warmups), ', '.join(warmups), text))

    text = "Run %s: %s" % (run_index, text)
    lines.append(text)

    if verbose > 0:
        prefix = '  '
        metadata = run.get_metadata()
        for name, value in sorted(metadata.items()):
            if common_metadata and name in common_metadata:
                continue
            value = _format_metadata(name, value)
            lines.append('%s%s: %s' % (prefix, name, value))

    return lines

def _format_runs(bench, quiet=False, verbose=False, raw=False, lines=None):
    runs = bench.get_runs()
    if quiet:
        verbose = -1
    elif verbose:
        verbose = 1
    else:
        verbose = 0

    if lines is None:
        lines = []
    if verbose > 0:
        empty_line(lines)

        # FIXME: display metadata in format_benchmark()
        common_metadata = bench.get_metadata()
        lines.append("Metadata:")
        format_metadata(common_metadata, prefix='  ', lines=lines)
    else:
        common_metadata = None

    empty_line_written = False
    for run_index, run in enumerate(runs, 1):
        if quiet and run._is_calibration():
            continue
        if not empty_line_written:
            empty_line_written = True
            empty_line(lines)
        format_run(bench, run_index, run,
                   common_metadata=common_metadata,
                   verbose=verbose, raw=raw, lines=lines)

    return lines

PERCENTILE_NAMES = {0: 'minimum', 50: 'median', 100: 'maximum'}

def format_stats(bench, lines):
    fmt = bench.format_value
    values = bench.get_values()

    nrun = bench.get_nrun()
    nvalue = len(values)

    empty_line(lines)

    # Total duration
    duration = bench.get_total_duration()
    if duration:
        lines.append("Total duration: %s" % format_seconds(duration))

    # Start/End dates
    dates = bench.get_dates()
    if dates:
        start, end = dates
        lines.append("Start date: %s" % format_datetime(start, microsecond=False))
        lines.append("End date: %s" % format_datetime(end, microsecond=False))

    # Raw value minimize/maximum
    raw_values = bench._get_raw_values()
    lines.append("Raw value minimum: %s" % bench.format_value(min(raw_values)))
    lines.append("Raw value maximum: %s" % bench.format_value(max(raw_values)))
    lines.append('')

    # Number of values
    lines.append("Number of runs: %s" % format_number(nrun))
    lines.append("Total number of values: %s" % format_number(nvalue))

    nvalue_per_run = bench._get_nvalue_per_run()
    text = format_number(nvalue_per_run)
    if isinstance(nvalue_per_run, float):
        text += ' (average)'
    lines.append('Number of values per run: %s' % text)

    nwarmup = bench._get_nwarmup()
    text = format_number(nwarmup)
    if isinstance(nwarmup, float):
        text += ' (average)'
    lines.append('Number of warmups per run: %s' % text)

    # Loop iterations per value
    loops = bench._get_loops()
    inner_loops = bench._get_inner_loops()
    total_loops = loops * inner_loops
    if isinstance(total_loops, int):
        text = format_number(total_loops)
    else:
        text = "%s (average)" % total_loops

    if not(isinstance(inner_loops, int) and inner_loops == 1):
        if isinstance(loops, int):
            loops = format_number(loops, 'outer-loop')
        else:
            loops = '%.1f outer-loops (average)'

        if isinstance(inner_loops, int):
            inner_loops = format_number(inner_loops, 'inner-loop')
        else:
            inner_loops = "%.1f inner-loops (average)" % inner_loops

        text = '%s (%s x %s)' % (text, loops, inner_loops)

    lines.append("Loop iterations per value: %s" % text)
    lines.append('')

    # Minimum
    table = []
    table.append(("Minimum", bench.format_value(min(values))))

    # Median +- MAD
    median = bench.median()
    if len(values) > 2:
        median_abs_dev = bench.median_abs_dev()
        table.append(("Median +- MAD",
                      "%s +- %s"
                      % bench.format_values((median, median_abs_dev))))
    else:
        table.append(("Mean", bench.format_value(median)))

    # Mean +- std dev
    mean = bench.mean()
    if len(values) > 2:
        stdev = bench.stdev()
        table.append(("Mean +- std dev",
                      "%s +- %s" % bench.format_values((mean, stdev))))
    else:
        table.append(("Mean", bench.format_value(mean)))

    table.append(("Maximum", bench.format_value(max(values))))

    # Render table
    width = max(len(row[0]) + 1 for row in table)
    for key, value in table:
        key = (key + ':').ljust(width)
        lines.append("%s %s" % (key, value))
    lines.append('')

    def format_limit(mean, value):
        return ("%s (%+.0f%% of the mean)"
                % (fmt(value), (value - mean) * 100.0 / mean))

    # Percentiles
    for p in (0, 5, 25, 50, 75, 95, 100):
        text = format_limit(mean, bench.percentile(p))
        text = "%3sth percentile: %s" % (p, text)
        name = PERCENTILE_NAMES.get(p)
        if name:
            text = '%s -- %s' % (text, name)
        lines.append(text)
    return lines

def format_histogram(benchmarks, bins=20, extend=False, lines=None,
                     checks=False):
    import collections
    import shutil

    if hasattr(shutil, 'get_terminal_size'):
        columns, nline = shutil.get_terminal_size()
    else:
        columns, nline = (80, 25)

    if not bins:
        bins = max(nline - 3, 3)
        if not extend:
            bins = min(bins, 25)

    all_values = []
    for bench, title in benchmarks:
        all_values.extend(bench.get_values())
    all_min = min(all_values)
    all_max = max(all_values)
    value_k = float(all_max - all_min) / bins
    if not value_k:
        value_k = 1.0

    def value_bucket(value):
        # round towards zero (ROUND_DOWN)
        return int(value / value_k)

    bucket_min = value_bucket(all_min)
    bucket_max = value_bucket(all_max)
    if lines is None:
        lines = []

    for item in benchmarks:
        empty_line(lines)

        bench, title = item
        if title:
            lines.append("[ %s ]" % title)

        values = bench.get_values()

        buckets = [value_bucket(value) for value in values]
        counter = collections.Counter(buckets)
        count_max = max(counter.values())
        count_width = len(str(count_max))

        value_width = max([len(bench.format_value(bucket * value_k))
                           for bucket in range(bucket_min, bucket_max + 1)])
        width = columns - value_width

        line = ': %s #' % count_max
        width = columns - (value_width + len(line))
        if not extend:
            width = min(width, 79)
        width = max(width, 3)
        line_k = float(width) / max(counter.values())
        for bucket in range(bucket_min, bucket_max + 1):
            count = counter.get(bucket, 0)
            linelen = int(round(count * line_k))
            text = bench.format_value(bucket * value_k)
            line = ('#' * linelen) or '|'
            lines.append("{:>{}}: {:>{}} {}".format(text, value_width,
                                                    count, count_width, line))

        if checks:
            format_checks(bench, lines=lines)

    return lines

def format_checks(bench, lines=None):
    if lines is None:
        lines = []
    values = bench.get_values()
    mean = bench.mean()
    warnings = []
    warn = warnings.append

    # Display a warning if the standard deviation is larger than 10%
    if len(values) >= 2:
        stdev = bench.stdev()
        percent = stdev * 100.0 / mean
        if percent >= 10.0:
            warn("the standard deviation (%s) is %.0f%% of the mean (%s)"
                 % (bench.format_value(stdev), percent, bench.format_value(mean)))

    # Minimum and maximum, detect obvious outliers
    for minimum, value in (
        ('minimum', min(values)),
        ('maximum', max(values)),
    ):
        percent = (value - mean) * 100.0 / mean
        if abs(percent) >= 25:
            if percent >= 0:
                text = "%.0f%% greater" % (percent)
            else:
                text = "%.0f%% smaller" % (-percent)
            warn("the %s (%s) is %s than the mean (%s)"
                 % (minimum, bench.format_value(value), text, bench.format_value(mean)))

    # Check that the shortest value took at least 1 ms
    if bench.get_unit() == 'second':
        shortest = min(bench._get_raw_values())
        if shortest < 1e-3:
            warn("the shortest raw value only took %s"
                 % bench.format_value(shortest))

    if warnings:
        empty_line(lines)
        lines.append("WARNING: the benchmark result may be unstable")
        for msg in warnings:
            lines.append("* %s" % msg)
        empty_line(lines)
        lines.append("Try to rerun the benchmark with more runs, values "
                     "and/or loops.")
        lines.append("Run '%s -m perf system tune' command to reduce "
                     "the system jitter."
                     % os.path.basename(sys.executable))
        lines.append("Use perf stats, perf dump and perf hist to analyze results.")
        lines.append("Use --quiet option to hide these warnings.")

    # Warn if nohz_full+intel_pstate combo if found in cpu_config metadata
    for run in bench._runs:
        cpu_config = run._metadata.get('cpu_config')
        if not cpu_config:
            continue
        if 'nohz_full' in cpu_config and 'intel_pstate' in cpu_config:
            empty_line(lines)
            warn("WARNING: nohz_full is enabled on CPUs which use the "
                 "intel_pstate driver, whereas intel_pstate is incompatible "
                 "with nohz_full")
            warn("CPU config: %s" % cpu_config)
            warn("See https://bugzilla.redhat.com/show_bug.cgi?id=1378529")
            break

    return lines

def format_result_value(bench):
    loops = bench._only_calibration()
    if loops is not None:
        return '<calibration: %s>' % format_number(loops, 'loop')

    mean = bench.mean()
    if bench.get_nvalue() >= 2:
        args = bench.format_values((mean, bench.stdev()))
        return '%s +- %s' % args
    else:
        return bench.format_value(mean)

def format_result(bench, prefix=True):
    loops = bench._only_calibration()
    if loops is not None:
        return 'Calibration: %s' % format_number(loops, 'loop')

    text = format_result_value(bench)
    if bench.get_nvalue() >= 2:
        return 'Mean +- std dev: %s' % text
    else:
        return text

def format_benchmark(bench, checks=True, metadata=False,
                     dump=False, stats=False, hist=False, show_name=False,
                     result=True, display_runs_args=None):
    lines = []

    if metadata:
        lines.append("Metadata:")
        format_metadata(bench.get_metadata(), lines=lines)

    if dump:
        if display_runs_args is None:
            display_runs_args = {}
        _format_runs(bench, lines=lines, **display_runs_args)

    if hist:
        format_histogram([(bench, None)], lines=lines)

    if stats:
        format_stats(bench, lines=lines)

    if checks:
        format_checks(bench, lines=lines)

    if result:
        empty_line(lines)

        text = format_result(bench)
        if show_name:
            name = bench.get_name()
            text = "%s: %s" % (name, text)
        lines.append(text)

    return lines

# FIXME: remove this function?
def multiline_output(args):
    return (args.hist or args.stats or args.dump or args.metadata)
<|endoftext|>


<commit_before>''' Solve the benchmark "differentially heated cavity" natural convection problem using finite elements.

    Solve the natural convection test problem from

        @article
        {danaila2014newton,
          title={A Newton method with adaptive finite elements for solving phase-change problems with natural convection},
          author={Danaila, Ionut and Moglan, Raluca and Hecht, Fr{\'e}d{\'e}ric and Le Masson, St{\'e}phane},
          journal={Journal of Computational Physics},
          volume={274},
          pages={826--840},
          year={2014},
          publisher={Academic Press}
        }

    Match the notation in danaila2014newton as best as possible.

'''

from fenics import \
    UnitSquareMesh, FiniteElement, VectorElement, MixedElement, \
    FunctionSpace, VectorFunctionSpace, \
    Function, TrialFunction, TestFunctions, split, \
    DirichletBC, Constant, Expression, \
    dx, \
    dot, inner, grad, nabla_grad, sym, div, \
    errornorm, norm, \
    File, \
    Progress, set_log_level, PROGRESS, \
    project, interpolate, \
    solve, parameters, info, derivative, \
    LinearVariationalProblem, LinearVariationalSolver, NonlinearVariationalProblem, NonlinearVariationalSolver, \
    AdaptiveLinearVariationalSolver, \
    SubDomain, EdgeFunction, near, adapt

from dolfin import DOLFIN_EPS

import test_phaseflow

def arguments():
    """Returns tuple containing dictionary of calling function's
       named arguments and a list of calling function's unnamed
       positional arguments.
    """
    from inspect import getargvalues, stack
    posname, kwname, args = getargvalues(stack()[1][0])[-3:]
    posargs = args.pop(posname, [])
    args.update(args.pop(kwname, []))
    return args, posargs

Re = 1.

default_Ra = 1.e6

default_Pr = 0.71

class BoundedValue(object):

    def __init__(self, min=0., value=0., max=0.):
        self.min = min
        self.value = value
        self.max = max

    def set(self, value):
        if value > self.max:
            value = self.max
        elif value < self.min:
            value = self.min

class TimeStepSize(BoundedValue):

    def __init__(self, bounded_value):
        super(TimeStepSize, self).__init__(bounded_value.min, bounded_value.value, bounded_value.max)

    def set(self, value):

        old_value = value

        super(TimeStepSize, self).set(value)

        if abs(self.value - old_value) > DOLFIN_EPS:
            print 'Set time step size to dt = ' + str(value)

def run(
    output_dir = "output/natural_convection", \
    Ra = default_Ra, \
    Pr = default_Pr, \
    K = 1., \
    mu = 1., \
    g = (0., -1.), \
    m_B = lambda theta : theta*default_Ra/(default_Pr*Re*Re), \
    dm_B_dtheta = lambda theta : default_Ra/(default_Pr*Re*Re), \
    mesh = UnitSquareMesh(20, 20, "crossed"), \
    s_u = ("0.", "0."), \
    s_p = "0.", \
    s_theta = "0.", \
    initial_values_expression = ( \
        "0.", \
        "0.", \
        "0.", \
        "0.5*near(x[0],  0.) -0.5*near(x[0],  1.)"), \
    bc_expressions = [ \
        [0, ("0.", "0."), 3, "near(x[0],  0.) | near(x[0],  1.) | near(x[1], 0.) | near(x[1],  1.)","topological"], \
        [2, "0.", 2, "near(x[0],  0.)", "topological"], \
        [2, "0.", 2, "near(x[0],  1.)", "topological"]], \
    final_time = 1., \
    time_step_size = BoundedValue(1.e-3, 1.e-3, 1.), \
    adaptive_space = False, \
    gamma = 1.e-7, \
    pressure_degree = 1, \
    temperature_degree = 1, \
    linearize = True, \
    newton_relative_tolerance = 1.e-9, \
    max_newton_iterations = 10, \
    stop_when_steady = True, \
    steady_relative_tolerance = 1.e-8, \
    debug_b_factor = 1., \
    debug_c_factor = 1., \
    exact_solution_expression = [] \
    ):

    #
    print("Running nsb_pcm with the following arguments:")

    print(arguments())

    # Validate inputs
    assert type(time_step_size) == type(BoundedValue())

    # @todo Try 3D
    dim = 2

    '''
    Per Equation 8 from danaila2014newton, we implement an equation scaled with v_ref = nu_liquid/H => t_ref = nu_liquid/H^2 => Re = 1.
    '''
    Re = 1.

    # Compute derived parameters
    velocity_degree = pressure_degree + 1

    # Define function spaces for the system
    VxV = VectorFunctionSpace(mesh, 'P', velocity_degree)

    Q = FunctionSpace(mesh, 'P', pressure_degree) # @todo mixing up test function space

    V = FunctionSpace(mesh, 'P', temperature_degree)

    '''
    MixedFunctionSpace used to be available but is now deprecated.
    The way that fenics separates function spaces and elements is confusing.
    To create the mixed space, I'm using the approach from https://fenicsproject.org/qa/11983/mixedfunctionspace-in-2016-2-0
    '''
    VxV_ele = VectorElement('P', mesh.ufl_cell(), velocity_degree)

    '''
    @todo How can we use the space $Q = \left{q \in L^2(\Omega) | \int{q = 0}\right}$ ?

    All Navier-Stokes FEniCS examples I've found simply use P2P1. danaila2014newton says that
    they are using the "classical Hilbert spaces" for velocity and pressure, but then they write
    down the space Q with less restrictions than H^1_0.

    '''
    Q_ele = FiniteElement('P', mesh.ufl_cell(), pressure_degree)

    V_ele = FiniteElement('P', mesh.ufl_cell(), temperature_degree)

    W_ele = MixedElement([VxV_ele, Q_ele, V_ele])

    W = FunctionSpace(mesh, W_ele)

    # Define function and test functions
    w = Function(W)

    v, q, phi = TestFunctions(W)

    # Split solution function to access variables separately
    u, p, theta = split(w)

    # Set source term expressions
    s_u = Expression(s_u, element=VxV_ele, mu=mu, gamma=gamma)

    s_p = Expression(s_p, element=Q_ele, mu=mu, gamma=gamma)

    s_theta = Expression(s_theta, element=V_ele)

    # Specify the initial values
    w_n = interpolate(Expression(initial_values_expression, element=W_ele), W)

    u_n, p_n, theta_n = split(w_n)

    # Define expressions needed for variational form
    Ra = Constant(Ra)

    Pr = Constant(Pr)

    Re = Constant(Re)

    K = Constant(K)

    mu = Constant(mu)

    g = Constant(g)

    gamma = Constant(gamma)

    # Define variational form
    def a(_mu, _u, _v):

        def D(_u):

            return sym(grad(_u))

        return 2.*_mu*inner(D(_u), D(_v))

    def b(_u, _q):

        return -div(_u)*_q*debug_b_factor

    def c(_w, _z, _v):

        return dot(dot(_w, nabla_grad(_z)), _v)*debug_c_factor # @todo Is this use of nabla_grad correct?

    # Specify boundary conditions
    bc = []

    for subspace, expression, degree, coordinates, method in bc_expressions:

        bc.append(DirichletBC(W.sub(subspace), Expression(expression, degree=degree), coordinates, method=method))

    # Implement the nonlinear variational form
    if not linearize:

        def nonlinear_variational_form(dt):

            dt = Constant(dt)

            F = (\
                    b(u, q) - gamma*p*q - s_p*q \
                    + dot(u, v)/dt + c(u, u, v) + a(mu, u, v) + b(v, p) - dot(u_n, v)/dt + dot(m_B(theta)*g, v) - dot(s_u, v) \
                    + theta*phi/dt - dot(u, grad(phi))*theta + dot(K/Pr*grad(theta), grad(phi)) - theta_n*phi/dt - s_theta*phi \
                    )*dx

            return F
    # Implement the Newton linearized form published in danaila2014newton
    elif linearize:

        w_w = TrialFunction(W)

        u_w, p_w, theta_w = split(w_w)

        w_k = Function(W)

        w_k.assign(w_n)

        u_k, p_k, theta_k = split(w_k)

        def linear_variational_form(dt):

            dt = Constant(dt)

            A = (\
                b(u_w,q) - gamma*p_w*q \
                + dot(u_w, v)/dt + c(u_w, u_k, v) + c(u_k, u_w, v) + a(mu, u_w, v) + b(v, p_w) + dot(theta_w*dm_B_dtheta(theta)*g, v) \
                + theta_w*phi/dt - dot(u_k, grad(phi))*theta_w - dot(u_w, grad(phi))*theta_k + dot(K/Pr*grad(theta_w), grad(phi)) \
                )*dx

            L = (\
                b(u_k,q) - gamma*p_k*q + s_p*q \
                + dot(u_k - u_n, v)/dt + c(u_k, u_k, v) + a(mu, u_k, v) + b(v, p_k) + dot(m_B(theta_k)*g, v) + dot(s_u, v) \
                + (theta_k - theta_n)*phi/dt - dot(u_k, grad(phi))*theta_k + dot(K/Pr*grad(theta_k), grad(phi)) + s_theta*phi \
                )*dx

            return A, L

        if adaptive_space:

            M = theta_w*dx

    # Create progress bar
    progress = Progress('Time-stepping')

    set_log_level(PROGRESS)

    # Define method for writing values, and write initial values# Create VTK file for visualization output
    velocity_file = File(output_dir + '/velocity.pvd')

    pressure_file = File(output_dir + '/pressure.pvd')

    temperature_file = File(output_dir + '/temperature.pvd')

    def write_solution(_w, time):

        _velocity, _pressure, _temperature = _w.split()

        _velocity.rename("u", "velocity")

        _pressure.rename("p", "pressure")

        _temperature.rename("theta", "temperature")

        velocity_file << (_velocity, time)

        pressure_file << (_pressure, time)

        temperature_file << (_temperature, time)

    time = 0.

    w.assign(w_n)

    write_solution(w, time)

    # Solve each time step

    w_w = Function(W) # w_w was previously a TrialFunction, but must be a Function when calling solve()

    time_residual = Function(W)

    def solve_time_step(dt):

        if linearize:

            print '\nIterating Newton method'

            iteration_count = 0

            old_residual = 1e32

            w_k.assign(w_n)

            for k in range(max_newton_iterations):

                A, L = linear_variational_form(dt)

                if not adaptive_space:

                    solve(A == L, w_w, bcs=bc)

                else:

                    problem = LinearVariationalProblem(A, L, w_w, bcs=bc)

                    solver = AdaptiveLinearVariationalSolver(problem, M)

                    solver.parameters["error_control"]["dual_variational_solver"]["linear_solver"] = "cg"

                    solver.parameters["error_control"]["dual_variational_solver"]["symmetric"] = True

                    solver.solve(tol)

                    solver.summary()

                w_k.assign(w_k - w_w)

                norm_residual = norm(w_w, 'L2')/norm(w_k, 'L2')

                print '\nL2 norm of relative residual, || w_w || / || w_k || = ' + str(norm_residual) + '\n'

                if norm_residual > old_residual:

                    diverging = True

                    return diverging

                old_residual = norm_residual

                if norm_residual < newton_relative_tolerance:

                    iteration_count = k + 1

                    print 'Converged after ' + str(k) + ' iterations'

                    break

            w.assign(w_k)

            diverging = False

            return diverging

        else:

            '''  @todo Implement adaptive time for nonlinear version.
            How to get residual from solver.solve() to check if diverging? '''

            F = nonlinear_variational_form(dt)

            problem = NonlinearVariationalProblem(F, w, bc, derivative(F, w))

            solver = NonlinearVariationalSolver(problem)

            iteration_count, converged = solver.solve()

            assert(converged)

    EPSILON = 1.e-12

    time_step_size = TimeStepSize(time_step_size)

    while time < final_time - EPSILON:

        remaining_time = final_time - time

        if time_step_size.value > remaining_time:

            time_step_size.set(remaining_time)

        diverging = True

        while diverging:

            diverging = solve_time_step(time_step_size.value)

            if time_step_size.value <= time_step_size.min + DOLFIN_EPS:

                assert(not diverging)

            if diverging:

                time_step_size.set(time_step_size.value/2.)

        assert(not diverging)

        time += time_step_size.value

        time_step_size.set(2*time_step_size.value)

        print 'Reached time t = ' + str(time)

        # Save solution to files
        write_solution(w, time)

        if stop_when_steady:

            # Check for steady state
            time_residual.assign(w - w_n)

        # Update previous solution
        w_n.assign(w)

        # Show the time progress
        progress.update(time / final_time)

        if stop_when_steady:

            unsteadiness = norm(time_residual, 'L2')/norm(w_n, 'L2')

            print 'Unsteadiness (L2 norm of relative time residual), || w_{n+1} || / || w_n || = ' + str(unsteadiness)

            if (unsteadiness < steady_relative_tolerance):
                print 'Reached steady state at time t = ' + str(time)
                break

    if time >= final_time:

        print 'Reached final time, t = ' + str(final_time)

    if exact_solution_expression:

        w_e = Expression(exact_solution_expression, element=W_ele)

        error = errornorm(w_e, w_n, 'L2')

        print("Error = " + str(error))

    w_n.rename("w", "state")

    fe_field_interpolant = interpolate(w_n, W)

    return fe_field_interpolant

if __name__=='__main__':

    run()<commit_msg>Fixed BoundedValue class, which wasn't actually setting new values. I still haven't made a test for adaptive time stepping<commit_after>''' Solve the benchmark "differentially heated cavity" natural convection problem using finite elements.

    Solve the natural convection test problem from

        @article
        {danaila2014newton,
          title={A Newton method with adaptive finite elements for solving phase-change problems with natural convection},
          author={Danaila, Ionut and Moglan, Raluca and Hecht, Fr{\'e}d{\'e}ric and Le Masson, St{\'e}phane},
          journal={Journal of Computational Physics},
          volume={274},
          pages={826--840},
          year={2014},
          publisher={Academic Press}
        }

    Match the notation in danaila2014newton as best as possible.

'''

from fenics import \
    UnitSquareMesh, FiniteElement, VectorElement, MixedElement, \
    FunctionSpace, VectorFunctionSpace, \
    Function, TrialFunction, TestFunctions, split, \
    DirichletBC, Constant, Expression, \
    dx, \
    dot, inner, grad, nabla_grad, sym, div, \
    errornorm, norm, \
    File, \
    Progress, set_log_level, PROGRESS, \
    project, interpolate, \
    solve, parameters, info, derivative, \
    LinearVariationalProblem, LinearVariationalSolver, NonlinearVariationalProblem, NonlinearVariationalSolver, \
    AdaptiveLinearVariationalSolver, \
    SubDomain, EdgeFunction, near, adapt

from dolfin import DOLFIN_EPS

import test_phaseflow

def arguments():
    """Returns tuple containing dictionary of calling function's
       named arguments and a list of calling function's unnamed
       positional arguments.
    """
    from inspect import getargvalues, stack
    posname, kwname, args = getargvalues(stack()[1][0])[-3:]
    posargs = args.pop(posname, [])
    args.update(args.pop(kwname, []))
    return args, posargs

Re = 1.

default_Ra = 1.e6

default_Pr = 0.71

class BoundedValue(object):

    def __init__(self, min=0., value=0., max=0.):
        self.min = min
        self.value = value
        self.max = max

    def set(self, value):
        if value > self.max:
            value = self.max
        elif value < self.min:
            value = self.min
        self.value = value

class TimeStepSize(BoundedValue):

    def __init__(self, bounded_value):
        super(TimeStepSize, self).__init__(bounded_value.min, bounded_value.value, bounded_value.max)

    def set(self, value):

        old_value = self.value

        super(TimeStepSize, self).set(value)

        if abs(self.value - old_value) > DOLFIN_EPS:
            print 'Set time step size to dt = ' + str(value)

def run(
    output_dir = "output/natural_convection", \
    Ra = default_Ra, \
    Pr = default_Pr, \
    K = 1., \
    mu = 1., \
    g = (0., -1.), \
    m_B = lambda theta : theta*default_Ra/(default_Pr*Re*Re), \
    dm_B_dtheta = lambda theta : default_Ra/(default_Pr*Re*Re), \
    mesh = UnitSquareMesh(20, 20, "crossed"), \
    s_u = ("0.", "0."), \
    s_p = "0.", \
    s_theta = "0.", \
    initial_values_expression = ( \
        "0.", \
        "0.", \
        "0.", \
        "0.5*near(x[0],  0.) -0.5*near(x[0],  1.)"), \
    bc_expressions = [ \
        [0, ("0.", "0."), 3, "near(x[0],  0.) | near(x[0],  1.) | near(x[1], 0.) | near(x[1],  1.)","topological"], \
        [2, "0.", 2, "near(x[0],  0.)", "topological"], \
        [2, "0.", 2, "near(x[0],  1.)", "topological"]], \
    final_time = 1., \
    time_step_size = BoundedValue(1.e-3, 1.e-3, 1.), \
    adaptive_space = False, \
    gamma = 1.e-7, \
    pressure_degree = 1, \
    temperature_degree = 1, \
    linearize = True, \
    newton_relative_tolerance = 1.e-9, \
    max_newton_iterations = 10, \
    stop_when_steady = True, \
    steady_relative_tolerance = 1.e-8, \
    debug_b_factor = 1., \
    debug_c_factor = 1., \
    exact_solution_expression = [] \
    ):

    #
    print("Running nsb_pcm with the following arguments:")

    print(arguments())

    # Validate inputs
    assert type(time_step_size) == type(BoundedValue())

    # @todo Try 3D
    dim = 2

    '''
    Per Equation 8 from danaila2014newton, we implement an equation scaled with v_ref = nu_liquid/H => t_ref = nu_liquid/H^2 => Re = 1.
    '''
    Re = 1.

    # Compute derived parameters
    velocity_degree = pressure_degree + 1

    # Define function spaces for the system
    VxV = VectorFunctionSpace(mesh, 'P', velocity_degree)

    Q = FunctionSpace(mesh, 'P', pressure_degree) # @todo mixing up test function space

    V = FunctionSpace(mesh, 'P', temperature_degree)

    '''
    MixedFunctionSpace used to be available but is now deprecated.
    The way that fenics separates function spaces and elements is confusing.
    To create the mixed space, I'm using the approach from https://fenicsproject.org/qa/11983/mixedfunctionspace-in-2016-2-0
    '''
    VxV_ele = VectorElement('P', mesh.ufl_cell(), velocity_degree)

    '''
    @todo How can we use the space $Q = \left{q \in L^2(\Omega) | \int{q = 0}\right}$ ?

    All Navier-Stokes FEniCS examples I've found simply use P2P1. danaila2014newton says that
    they are using the "classical Hilbert spaces" for velocity and pressure, but then they write
    down the space Q with less restrictions than H^1_0.

    '''
    Q_ele = FiniteElement('P', mesh.ufl_cell(), pressure_degree)

    V_ele = FiniteElement('P', mesh.ufl_cell(), temperature_degree)

    W_ele = MixedElement([VxV_ele, Q_ele, V_ele])

    W = FunctionSpace(mesh, W_ele)

    # Define function and test functions
    w = Function(W)

    v, q, phi = TestFunctions(W)

    # Split solution function to access variables separately
    u, p, theta = split(w)

    # Set source term expressions
    s_u = Expression(s_u, element=VxV_ele, mu=mu, gamma=gamma)

    s_p = Expression(s_p, element=Q_ele, mu=mu, gamma=gamma)

    s_theta = Expression(s_theta, element=V_ele)

    # Specify the initial values
    w_n = interpolate(Expression(initial_values_expression, element=W_ele), W)

    u_n, p_n, theta_n = split(w_n)

    # Define expressions needed for variational form
    Ra = Constant(Ra)

    Pr = Constant(Pr)

    Re = Constant(Re)

    K = Constant(K)

    mu = Constant(mu)

    g = Constant(g)

    gamma = Constant(gamma)

    # Define variational form
    def a(_mu, _u, _v):

        def D(_u):

            return sym(grad(_u))

        return 2.*_mu*inner(D(_u), D(_v))

    def b(_u, _q):

        return -div(_u)*_q*debug_b_factor

    def c(_w, _z, _v):

        return dot(dot(_w, nabla_grad(_z)), _v)*debug_c_factor # @todo Is this use of nabla_grad correct?

    # Specify boundary conditions
    bc = []

    for subspace, expression, degree, coordinates, method in bc_expressions:

        bc.append(DirichletBC(W.sub(subspace), Expression(expression, degree=degree), coordinates, method=method))

    # Implement the nonlinear variational form
    if not linearize:

        def nonlinear_variational_form(dt):

            dt = Constant(dt)

            F = (\
                    b(u, q) - gamma*p*q - s_p*q \
                    + dot(u, v)/dt + c(u, u, v) + a(mu, u, v) + b(v, p) - dot(u_n, v)/dt + dot(m_B(theta)*g, v) - dot(s_u, v) \
                    + theta*phi/dt - dot(u, grad(phi))*theta + dot(K/Pr*grad(theta), grad(phi)) - theta_n*phi/dt - s_theta*phi \
                    )*dx

            return F
    # Implement the Newton linearized form published in danaila2014newton
    elif linearize:

        w_w = TrialFunction(W)

        u_w, p_w, theta_w = split(w_w)

        w_k = Function(W)

        w_k.assign(w_n)

        u_k, p_k, theta_k = split(w_k)

        def linear_variational_form(dt):

            dt = Constant(dt)

            A = (\
                b(u_w,q) - gamma*p_w*q \
                + dot(u_w, v)/dt + c(u_w, u_k, v) + c(u_k, u_w, v) + a(mu, u_w, v) + b(v, p_w) + dot(theta_w*dm_B_dtheta(theta)*g, v) \
                + theta_w*phi/dt - dot(u_k, grad(phi))*theta_w - dot(u_w, grad(phi))*theta_k + dot(K/Pr*grad(theta_w), grad(phi)) \
                )*dx

            L = (\
                b(u_k,q) - gamma*p_k*q + s_p*q \
                + dot(u_k - u_n, v)/dt + c(u_k, u_k, v) + a(mu, u_k, v) + b(v, p_k) + dot(m_B(theta_k)*g, v) + dot(s_u, v) \
                + (theta_k - theta_n)*phi/dt - dot(u_k, grad(phi))*theta_k + dot(K/Pr*grad(theta_k), grad(phi)) + s_theta*phi \
                )*dx

            return A, L

        if adaptive_space:

            M = theta_w*dx

    # Create progress bar
    progress = Progress('Time-stepping')

    set_log_level(PROGRESS)

    # Define method for writing values, and write initial values# Create VTK file for visualization output
    velocity_file = File(output_dir + '/velocity.pvd')

    pressure_file = File(output_dir + '/pressure.pvd')

    temperature_file = File(output_dir + '/temperature.pvd')

    def write_solution(_w, time):

        _velocity, _pressure, _temperature = _w.split()

        _velocity.rename("u", "velocity")

        _pressure.rename("p", "pressure")

        _temperature.rename("theta", "temperature")

        velocity_file << (_velocity, time)

        pressure_file << (_pressure, time)

        temperature_file << (_temperature, time)

    time = 0.

    w.assign(w_n)

    write_solution(w, time)

    # Solve each time step

    w_w = Function(W) # w_w was previously a TrialFunction, but must be a Function when calling solve()

    time_residual = Function(W)

    def solve_time_step(dt):

        if linearize:

            print '\nIterating Newton method'

            iteration_count = 0

            old_residual = 1e32

            w_k.assign(w_n)

            for k in range(max_newton_iterations):

                A, L = linear_variational_form(dt)

                if not adaptive_space:

                    solve(A == L, w_w, bcs=bc)

                else:

                    problem = LinearVariationalProblem(A, L, w_w, bcs=bc)

                    solver = AdaptiveLinearVariationalSolver(problem, M)

                    solver.parameters["error_control"]["dual_variational_solver"]["linear_solver"] = "cg"

                    solver.parameters["error_control"]["dual_variational_solver"]["symmetric"] = True

                    solver.solve(tol)

                    solver.summary()

                w_k.assign(w_k - w_w)

                norm_residual = norm(w_w, 'L2')/norm(w_k, 'L2')

                print '\nL2 norm of relative residual, || w_w || / || w_k || = ' + str(norm_residual) + '\n'

                if norm_residual > old_residual:

                    diverging = True

                    return diverging

                old_residual = norm_residual

                if norm_residual < newton_relative_tolerance:

                    iteration_count = k + 1

                    print 'Converged after ' + str(k) + ' iterations'

                    break

            w.assign(w_k)

            diverging = False

            return diverging

        else:

            '''  @todo Implement adaptive time for nonlinear version.
            How to get residual from solver.solve() to check if diverging? '''

            F = nonlinear_variational_form(dt)

            problem = NonlinearVariationalProblem(F, w, bc, derivative(F, w))

            solver = NonlinearVariationalSolver(problem)

            iteration_count, converged = solver.solve()

            assert(converged)

    EPSILON = 1.e-12

    time_step_size = TimeStepSize(time_step_size)

    while time < final_time - EPSILON:

        remaining_time = final_time - time

        if time_step_size.value > remaining_time:

            time_step_size.set(remaining_time)

        diverging = True

        while diverging:

            diverging = solve_time_step(time_step_size.value)

            if time_step_size.value <= time_step_size.min + DOLFIN_EPS:

                assert(not diverging)

            if diverging:

                time_step_size.set(time_step_size.value/2.)

        assert(not diverging)

        time += time_step_size.value

        time_step_size.set(2*time_step_size.value)

        print 'Reached time t = ' + str(time)

        # Save solution to files
        write_solution(w, time)

        if stop_when_steady:

            # Check for steady state
            time_residual.assign(w - w_n)

        # Update previous solution
        w_n.assign(w)

        # Show the time progress
        progress.update(time / final_time)

        if stop_when_steady:

            unsteadiness = norm(time_residual, 'L2')/norm(w_n, 'L2')

            print 'Unsteadiness (L2 norm of relative time residual), || w_{n+1} || / || w_n || = ' + str(unsteadiness)

            if (unsteadiness < steady_relative_tolerance):
                print 'Reached steady state at time t = ' + str(time)
                break

    if time >= final_time:

        print 'Reached final time, t = ' + str(final_time)

    if exact_solution_expression:

        w_e = Expression(exact_solution_expression, element=W_ele)

        error = errornorm(w_e, w_n, 'L2')

        print("Error = " + str(error))

    w_n.rename("w", "state")

    fe_field_interpolant = interpolate(w_n, W)

    return fe_field_interpolant

if __name__=='__main__':

    run()<|endoftext|>


<commit_before>#!/usr/bin/env python2

import yum
import yumoter
import argparse
import sys
import os

import yumoter as depyumoter

def commonInLists(listOfLists):
    # Input: List of lists
    # Output: Single list of common elements
    # Must take unknown number of lists
    if len(listOfLists) == 0:
        raise Exception('No.')
    elif len(listOfLists) == 1:
        return listOfLists[0]
    elif len(listOfLists) == 2:
        return list(set(listOfLists[0]).intersection(listOfLists[1]))
    elif len(listOfLists) > 2:
        result = set(listOfLists[0]).intersection(listOfLists[1])
        for i in range(2, len(listOfLists)):
            result = result.intersection(listOfLists[i])
        return list(result)
    else:
        raise Exception('How did I get here?')

parser = argparse.ArgumentParser(description='The yumoter promo script will assist you in promoting pkgs and their dependencies through environments.')

subparsers = parser.add_subparsers(help='Use search to promote something, and list to get information about the repos', dest='subprocess_name')

parser_list = subparsers.add_parser('list', help='List repo information')

parser_search = subparsers.add_parser('search', help='Search for pkgs')
parser_search.add_argument('search', help='specify a pkg to search for')
parser_search.add_argument('-c', '--centosversion', help="Specify centos version", default="6.4")
parser_search.add_argument('-e', '--environment', help="Specify enviornment / promotion path", required=True)
parser_search.add_argument('-r', '--repo', help="Repo to search in", required=True)

args = parser.parse_args()

environments = ['wildwest', 'beta', 'live']

yumoter = yumoter.yumoter('config/repos.json', '/home/aarwine/git/yumoter/repos')
#yumoter = yumoter.yumoter('config/repos.json', '/vagrant/yumoter/repos')

if args.subprocess_name == 'list':
    for repo in yumoter.repoConfig:
        print(repo)
        for key in yumoter.repoConfig[repo]:
            if isinstance(yumoter.repoConfig[repo][key], basestring):
                print("\t%s: %s" % (key, yumoter.repoConfig[repo][key]))
            else:
                print("\t%s:" % key)
                for entry in yumoter.repoConfig[repo][key]:
                    print("\t\t%s" % entry)
    sys.exit()

# Load initial repo to do search
if len(yumoter.repoConfig[args.repo]['fullurls']) == 1:
    yumoter._loadRepo(args.repo, yumoter.repoConfig[args.repo]['fullurls'][0])
else:
    yumoter._loadRepo(args.repo, yumoter.repoConfig[args.repo]['fullurls'][yumoter.repoConfig[args.repo]['promotionpath'].index(args.environment)])

#searchPkgList = yumoter._returnNewestByNameArch([args.search])
searchPkgList = yumoter.searchByName(args.search)

# searchByName doesn't return the same format as returnNewestByNameArch
# Let's fix the printing
searchPkgDict = {}

for pkg in searchPkgList:
    # Need to filter out installed pkgs. Not sure where they are coming from.
    if 'rpmdb' not in pkg[0].__dict__:
        if yumoter._urlToRepo(pkg[0].remote_url) not in searchPkgDict:
            searchPkgDict[yumoter._urlToRepo(pkg[0].remote_url)] = []
        searchPkgDict[yumoter._urlToRepo(pkg[0].remote_url)].append(pkg[0])

pkgIdx = []
i = 0
for repo in searchPkgDict:
    print("%s:" % repo)
    for pkg in searchPkgDict[repo]:
        print("%d. %s" % (i, pkg))
        pkgIdx.append(pkg)
        i += 1
print('Please select which pkg to promote:')

#pkgChoice = int(raw_input(": "))

#print pkgIdx[pkgChoice]
#promopkg = pkgIdx[pkgChoice]
print pkgIdx[160]
promopkg = pkgIdx[160]

currEnvIdx = environments.index(args.environment)

# setup depyumoter
depyumoter = depyumoter.yumoter('config/repos.json', '/home/aarwine/git/yumoter/repos')
depyumoter.loadRepos(args.centosversion, args.environment, args.repo)

# get initial deps for our chosen pkg
print 'Getting deps for:', promopkg
neededDeps = depyumoter.getNeededDeps(promopkg)

# Put the chosen pkg in a list with it's deps
depsList = [promopkg]
for dep in neededDeps[promopkg]:
    depsList.append(dep)

depNameDict = {}
# Get full list of dependencies
fullDepResult = depyumoter._getDeps(depsList)

commonPkgDict = {}
for pkg in fullDepResult:
    for dep in fullDepResult[pkg]:
        if dep[0] not in commonPkgDict:
            commonPkgDict[dep[0]] = []
        commonPkgDict[dep[0]].append(fullDepResult[pkg][dep])

for key in commonPkgDict:
    commonPkgDict[key] = commonInLists(commonPkgDict[key])

promoList = [promopkg]

for key in commonPkgDict:
    choicepkg = depyumoter.yb.bestPackagesFromList(commonPkgDict[key])
    for entry in choicepkg:
        if depyumoter._repoIsPromoted(depyumoter._urlToRepo(entry.remote_url)):
            promoList.append(entry)

print ""
print "Promotion list:"
for idx, entry in enumerate(promoList):
    print "%s. %s" % (idx, entry)

goVar = raw_input("Go? (y|n): ")

if goVar.lower() == 'n':
    sys.exit(0)
elif goVar.lower() == 'y':
    print 'lets go'
    for pkg in promoList:
        print "promoting:", pkg
else:
    print 'wut.'

'''
# check for multiple instances of deps
fooDict = {}
for i in fullDepResult:
    print i
    print fullDepResult[i]

'''

'''
# Start parsing for printing
# first let's put it in our own dict format
resultDict = {}
for pkg in fullDepResult:
    for dep in fullDepResult[pkg]:
        deprepo = depyumoter._urlToRepo(dep.remote_url)
        if deprepo not in resultDict:
            resultDict[deprepo] = []
        resultDict[deprepo].append(dep)

# try pretty printing
print "Deps:"
for repo in resultDict:
    print "Repo: %s" % repo
    for dep in resultDict[repo]:
        print "\t%s" % dep
'''

'''
depsList = []

for key in neededDeps:
    for dep in neededDeps[key]:
        print 'Adding initial dep:', dep
        depsList.append(dep)

print 'before shit started:', depsList

notchanged = True
while notchanged:
    olddepsList = depsList
    newDepsDict = depyumoter.getNeededDeps(depsList)
    print 'newDepsDict:', newDepsDict
    for origpkg in newDepsDict:
        for dep in newDepsDict[origpkg]:
            if dep not in depsList:
                print '---'
                print 'from origpkg', origpkg
                print 'Adding recursive dep:', dep
                depsList.append(dep)
    if depsList == olddepsList:
        notchanged = False

depsDict = {}
for dep in depsList:
    depRepo = depyumoter._urlToRepo(dep.remote_url)
    if depRepo not in depsDict:
        depsDict[depRepo] = []
    depsDict[depRepo].append(dep)

print "Deps:"
for repo in depsDict:
    print "Repo: %s" % repo
    for dep in depsDict[repo]:
        print "\t%s" % dep
'''

<commit_msg>cleanup old cruft / notes<commit_after>#!/usr/bin/env python2

import yum
import yumoter
import argparse
import sys
import os

import yumoter as depyumoter

def commonInLists(listOfLists):
    # Input: List of lists
    # Output: Single list of common elements
    # Must take unknown number of lists
    if len(listOfLists) == 0:
        raise Exception('No.')
    elif len(listOfLists) == 1:
        return listOfLists[0]
    elif len(listOfLists) == 2:
        return list(set(listOfLists[0]).intersection(listOfLists[1]))
    elif len(listOfLists) > 2:
        result = set(listOfLists[0]).intersection(listOfLists[1])
        for i in range(2, len(listOfLists)):
            result = result.intersection(listOfLists[i])
        return list(result)
    else:
        raise Exception('How did I get here?')

parser = argparse.ArgumentParser(description='The yumoter promo script will assist you in promoting pkgs and their dependencies through environments.')

subparsers = parser.add_subparsers(help='Use search to promote something, and list to get information about the repos', dest='subprocess_name')

parser_list = subparsers.add_parser('list', help='List repo information')

parser_search = subparsers.add_parser('search', help='Search for pkgs')
parser_search.add_argument('search', help='specify a pkg to search for')
parser_search.add_argument('-c', '--centosversion', help="Specify centos version", default="6.4")
parser_search.add_argument('-e', '--environment', help="Specify enviornment / promotion path", required=True)
parser_search.add_argument('-r', '--repo', help="Repo to search in", required=True)

args = parser.parse_args()

environments = ['wildwest', 'beta', 'live']

yumoter = yumoter.yumoter('config/repos.json', '/home/aarwine/git/yumoter/repos')
#yumoter = yumoter.yumoter('config/repos.json', '/vagrant/yumoter/repos')

if args.subprocess_name == 'list':
    for repo in yumoter.repoConfig:
        print(repo)
        for key in yumoter.repoConfig[repo]:
            if isinstance(yumoter.repoConfig[repo][key], basestring):
                print("\t%s: %s" % (key, yumoter.repoConfig[repo][key]))
            else:
                print("\t%s:" % key)
                for entry in yumoter.repoConfig[repo][key]:
                    print("\t\t%s" % entry)
    sys.exit()

# Load initial repo to do search
if len(yumoter.repoConfig[args.repo]['fullurls']) == 1:
    yumoter._loadRepo(args.repo, yumoter.repoConfig[args.repo]['fullurls'][0])
else:
    yumoter._loadRepo(args.repo, yumoter.repoConfig[args.repo]['fullurls'][yumoter.repoConfig[args.repo]['promotionpath'].index(args.environment)])

#searchPkgList = yumoter._returnNewestByNameArch([args.search])
searchPkgList = yumoter.searchByName(args.search)

# searchByName doesn't return the same format as returnNewestByNameArch
# Let's fix the printing
searchPkgDict = {}

for pkg in searchPkgList:
    # Need to filter out installed pkgs. Not sure where they are coming from.
    if 'rpmdb' not in pkg[0].__dict__:
        if yumoter._urlToRepo(pkg[0].remote_url) not in searchPkgDict:
            searchPkgDict[yumoter._urlToRepo(pkg[0].remote_url)] = []
        searchPkgDict[yumoter._urlToRepo(pkg[0].remote_url)].append(pkg[0])

pkgIdx = []
i = 0
for repo in searchPkgDict:
    print("%s:" % repo)
    for pkg in searchPkgDict[repo]:
        print("%d. %s" % (i, pkg))
        pkgIdx.append(pkg)
        i += 1
print('Please select which pkg to promote:')

#pkgChoice = int(raw_input(": "))

#print pkgIdx[pkgChoice]
#promopkg = pkgIdx[pkgChoice]
print pkgIdx[160]
promopkg = pkgIdx[160]

currEnvIdx = environments.index(args.environment)

# setup depyumoter
depyumoter = depyumoter.yumoter('config/repos.json', '/home/aarwine/git/yumoter/repos')
depyumoter.loadRepos(args.centosversion, args.environment, args.repo)

# get initial deps for our chosen pkg
print 'Getting deps for:', promopkg
neededDeps = depyumoter.getNeededDeps(promopkg)

# Put the chosen pkg in a list with it's deps
depsList = [promopkg]
for dep in neededDeps[promopkg]:
    depsList.append(dep)

depNameDict = {}
# Get full list of dependencies
fullDepResult = depyumoter._getDeps(depsList)

commonPkgDict = {}
for pkg in fullDepResult:
    for dep in fullDepResult[pkg]:
        if dep[0] not in commonPkgDict:
            commonPkgDict[dep[0]] = []
        commonPkgDict[dep[0]].append(fullDepResult[pkg][dep])

for key in commonPkgDict:
    commonPkgDict[key] = commonInLists(commonPkgDict[key])

promoList = [promopkg]

for key in commonPkgDict:
    choicepkg = depyumoter.yb.bestPackagesFromList(commonPkgDict[key])
    for entry in choicepkg:
        if depyumoter._repoIsPromoted(depyumoter._urlToRepo(entry.remote_url)):
            promoList.append(entry)

print ""
print "Promotion list:"
for idx, entry in enumerate(promoList):
    print "%s. %s" % (idx, entry)

goVar = raw_input("Go? (y|n): ")

if goVar.lower() == 'n':
    sys.exit(0)
elif goVar.lower() == 'y':
    print 'lets go'
    for pkg in promoList:
        print "promoting:", pkg
else:
    print 'wut.'
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Predicts article quality and priority with respect to a WikiProject.
Copyright (C) 2015 James Hare
Licensed under MIT License: http://mitlicense.org
"""

import datetime
import gzip
import html
import json
import operator
import os
from math import log  # https://www.youtube.com/watch?v=RTrAVpK9blw
from project_index import WikiProjectTools

def getviewdump(wptools, proj, days=30):
    '''
    Loads the page view dump for the past complete 30 days
    Takes string input (project name/abbreviation as identified in the dump)
    Returns a dict: title => pageviews
    '''

    # Create list of lists; each sub-list is a directory path
    # e.g. ['2015', '2015-06', '20150610-000000']

    filepaths = []
    for i in range(1, days+1):  # day -1 through day -31 (i.e., thirty days in the past, starting with yesterday)
        time = datetime.datetime.now() + datetime.timedelta(-i)
        for j in range(24):  # for each hour
            hourminutesecond = '-' + str(j).zfill(2) + '0000'
            filepaths.append([time.strftime('%Y'), time.strftime('%Y-%m'), time.strftime('%Y%m%d') + hourminutesecond])

    # Generate list of valid article titles (specifically for English Wikipedia) to filter out inane amounts of garbage
    if proj == 'en':
        print("Generating enwiki valid titles list...")
        validtitles = set()
        for row in wptools.query('wiki', 'select page_title from page where page_namespace = 0 and page_is_redirect = 0;', None):
            validtitles.add(row[0].decode('utf-8'))
    else:
        validtitles = None

    # Read through each file, and if it matches with the project, append to output

    counter = 0
    output = {}
    for file in filepaths:
        counter += 1
        print("Processing dump file " + str(counter) + "/" + str(len(filepaths)), end="\r")
        filename = '/public/dumps/pagecounts-raw/{0}/{1}/pagecounts-{2}.gz'.format(file[0], file[1], file[2])

        if os.path.isfile(filename) == False:
            continue

        with gzip.open(filename, mode='rt', encoding='utf-8') as f:
            for line in f:
                entry = line.split(' ')  # It's a space-delimited file, or something
                if entry[0] == proj:
                    entry[1] = html.unescape(entry[1]).replace(' ', '_')
                    if proj == 'en' and entry[1] not in validtitles:  # English Wikipedia specific check
                        continue
                    else:
                        if entry[1] in output:
                            output[entry[1]] += int(entry[2])  # Append to existing record
                        else:
                            output[entry[1]] = int(entry[2])  # Create new record

    print("\nProcessed stats for " + str(len(output)) + " articles.")
    return output

def getpageviews(dump, article):
    '''
    Queries *dump* for the number of page views in the last 30 days
    Takes dict *dump*, string *article* as input, returns view count.
    Does NOT take the logarithm of the view count.
    '''

    if article in dump:
        return dump[article]
    else:
        return 0

def getlinkcount(wptools, package):
    '''
    Gets a list of inbound links for a list of articles
    Takes list *package* as input, returns list of tuples (article, log of linkcount)
    Input MUST be a list. If there is just one article, enter it as such: [article]
    '''

    if len(package) > 1:
        query_builder = 'select pl_title, count(*) from pagelinks where pl_namespace = 0 and pl_title in {0} group by pl_title;'.format(tuple(package))
    else:
        query_builder = 'select pl_title, count(*) from pagelinks where pl_namespace = 0 and pl_title = "{0}" group by pl_title;'.format(package[0])

    output = []
    for row in wptools.query('wiki', query_builder, None):
        output.append((row[0].decode('utf-8'), log(row[1] + 1)))

    if len(output) == 0:
        output = [('', 0)]  # Return a consistent result even if there is nothing

    return output

def getinternalclout(wptools, destination):
    '''
    Gets a list of inbound links from self.articles for a list of articles
    Takes list *destination* as input, returns list of tuples (article, log of internalclout)
    Input MUST be a list. If there is just one article, enter it as such: [article]
    '''

    output = []
    q = "select pl_title, count(*) from pagelinks join page on pl_from = page_id where pl_namespace = 0 and pl_title in {0} and page_title in {1} group by pl_title;"
    for row in wptools.query('wiki', q.format(tuple(destination), tuple(self.articles)), None):
        output.append((row[0].decode('utf-8'), log(row[1])))

    return output

class QualityPredictor:
    def qualitypredictor(self, pagetitle):
        print("Argh! Not ready yet!")
        # chat it up with ORES

class PriorityPredictor:
    def __init__(self, viewdump=None):
        print("Initializing the Priority Predictor")
        self.wptools = WikiProjectTools()

        if viewdump == None:  # If a dumped JSON file of pageviews is not specified
            self.dump = getviewdump(self.wptools, 'en', days=30)
        else:
            with open(viewdump, 'r') as f:
                self.dump = json.load(f)  # Load pageviews from a dumped JSON file

    def loadproject(self, wikiproject, unknownpriority):
        self.projectcat = unknownpriority.replace("Unknown-", "")
        self.project = wikiproject
        self.rank = []  # Sorted list of tuples; allows for ranking
        self.score = {}  # Unsorted dictionary "article: value"; allows for easily looking up scores later
        # We need all the articles for a WikiProject, since the system works by comparing stats for an article to the others.
        print("Preparing Priority Predictor for: " + self.project)
        self.articles = []   # List of strings (article titles)
        pageviews = []  # List of tuples (article title, log of view count)
        linkcount = []  # List of tuples (article title, log of link count)
        for row in self.wptools.query('index', 'select pi_page from projectindex where pi_project = "Wikipedia:{0}";'.format(self.project), None):
            if row[0].startswith("Talk:"):  #
                article = row[0][5:] # Stripping out "Talk:"
                self.articles.append(article)
                pageviews.append((article, log(getpageviews(self.dump, article) + 1)))

        # Inbound link count
        # This is batched, thus broken out of the loop
        print("Getting inbound link count...")
        packages = []
        for i in range(0, len(self.articles), 10000):
            packages.append(self.articles[i:i+10000])

        for package in packages:
                toappend = getlinkcount(self.wptools, package)
                for item in toappend:
                    linkcount.append(item)

        # "Internal Clout"
        # This measures, within a group of articles, the number of links to each other
        # Works amazingly well as a metric

        print("Measuring internal clout...")
        internalclout = getinternalclout(self.wptools, self.articles)

        # Sorting...
        pageviews = sorted(pageviews, key=operator.itemgetter(1), reverse=True)
        linkcount = sorted(linkcount, key=operator.itemgetter(1), reverse=True)
        internalclout = sorted(internalclout, key=operator.itemgetter(1), reverse=True)

        # Computing relative measurements
        # "Relative" means "as a ratio to the highest rank".
        # The most viewed article has a relative pageview score of 1.00. Goes lower from there.

        print("Computing relative measurements...")
        pageviews_relative = {}
        linkcount_relative = {}
        internalclout_relative = {}

        self.mostviews = pageviews[0][1]
        self.mostlinks = linkcount[0][1]
        self.mostinternal = internalclout[0][1]

        for pair in pageviews:
            article = pair[0]
            count = pair[1]
            pageviews_relative[article] = count / self.mostviews

        for pair in linkcount:
            article = pair[0]
            count = pair[1]
            linkcount_relative[article] = count / self.mostlinks

        for pair in internalclout:
            article = pair[0]
            count = pair[1]
            internalclout_relative[article] = count / self.mostinternal

        for article in self.articles:
            weightedscore = (internalclout_relative[article] * 0.5) + (pageviews_relative[article] * 0.375) + (linkcount_relative[article] * 0.125)
            self.rank.append((article, weightedscore))

        self.rank = sorted(self.rank, key=operator.itemgetter(1), reverse=True)

        # Re-scaling scores, such that the highest ranked will always have a score of 1.00.
        self.highestscore = self.rank[0][1]
        self.rank = [(item[0], item[1] / self.highestscore) for item in self.rank]

        # Defining unordered index of scores
        for item in self.rank:
            self.score[item[0]] = item[1]

        # Calculating minimum scores
        # The idea is that there is a minimum score for something to be top, high, or mid-priority
        # The script is fed the category name for the unknown-importance/unknown-priority category
        # Based on this, derive category names for top/high/mid/low, add all the counts together...
        # ...then calculate ratio for top/high/mid as a ratio of the total...
        # ...multiply that ratio by the count of self.rank, convert to an integer
        # ...and then threshold = self.rank[that integer][1]
        # This gives us a general sense of what proportion of articles should be considered top/high/mid/low
        # Far from perfect but it's a start.

        print("Calculating priority thresholds...")
        prioritycount = {}

        q = 'select count(*) from categorylinks where cl_type = "page" and cl_to = "{0}";'
        for priority in ['Top-', 'High-', 'Mid-', 'Low-']:
            prioritycategory = priority + self.projectcat
            prioritycount[priority] = self.wptools.query('wiki', q.format(prioritycategory), None)[0][0]

        total_assessed = sum([x for x in prioritycount.values()])

        top_index = int((prioritycount['Top-'] / total_assessed) * len(self.articles) - 1)
        high_index = top_index + int((prioritycount['High-'] / total_assessed) * len(self.articles) -1)
        mid_index = high_index + int((prioritycount['Mid-'] / total_assessed) * len(self.articles) -1)

        self.threshold_top = self.rank[top_index][1]
        self.threshold_high = self.rank[high_index][1]
        self.threshold_mid = self.rank[mid_index][1]

    def predictpage(self, pagetitle):
        # Pull pagescore if already defined
        # Otherwise, compute it "de novo"
        if pagetitle in self.articles:
            pagescore = self.score[pagetitle]
        else:
            pageviews = log(getpageviews(self.dump, pagetitle) + 1) / self.mostviews
            linkcount = getlinkcount(self.wptools, [pagetitle])[0][1] / self.mostlinks
            internalclout = getinternalclout(self.wptools, [pagetitle])[0][1] / self.mostinternal
            pagescore = ((internalclout * 0.5) + (pageviews * 0.375) + (linkcount * 0.125)) / self.highestscore

        if pagescore >= self.threshold_top:
            return "Top"

        if pagescore >= self.threshold_high:
            return "High"

        if pagescore >= self.threshold_mid:
            return "Mid"

        # If none of these...
        return "Low"

    def audit(self):
        print("Auditing " + self.project)

        for prefix in ['Top', 'High', 'Mid', 'Low']:
            category = prefix + "-" + self.projectcat
            matrix = {'Top': 0, 'High': 0, 'Mid': 0, 'Low': 0}

            q = 'select page_title from categorylinks join page on cl_from = page_id where cl_type = "page" and cl_to = "{0}";'
            for row in self.wptools.query('wiki', q.format(category), None):
                title = row[0].decode('utf-8')
                matrix[self.predictpage(title)] += 1

            total_assessed = sum([x for x in matrix.values()])

            print("Human assessed " + prefix + "-priority:")
            for item in matrix.keys():
                print("Machine assessed " + item + "-priority: " + str(matrix[item]) + " (" + str((matrix[item] / total_assessed) * 100) + "%)")
            print("Total: " + str(total_assessed) + " articles")<commit_msg>Alas, the function is *outside* the class.<commit_after># -*- coding: utf-8 -*-
"""
Predicts article quality and priority with respect to a WikiProject.
Copyright (C) 2015 James Hare
Licensed under MIT License: http://mitlicense.org
"""

import datetime
import gzip
import html
import json
import operator
import os
from math import log  # https://www.youtube.com/watch?v=RTrAVpK9blw
from project_index import WikiProjectTools

def getviewdump(wptools, proj, days=30):
    '''
    Loads the page view dump for the past complete 30 days
    Takes string input (project name/abbreviation as identified in the dump)
    Returns a dict: title => pageviews
    '''

    # Create list of lists; each sub-list is a directory path
    # e.g. ['2015', '2015-06', '20150610-000000']

    filepaths = []
    for i in range(1, days+1):  # day -1 through day -31 (i.e., thirty days in the past, starting with yesterday)
        time = datetime.datetime.now() + datetime.timedelta(-i)
        for j in range(24):  # for each hour
            hourminutesecond = '-' + str(j).zfill(2) + '0000'
            filepaths.append([time.strftime('%Y'), time.strftime('%Y-%m'), time.strftime('%Y%m%d') + hourminutesecond])

    # Generate list of valid article titles (specifically for English Wikipedia) to filter out inane amounts of garbage
    if proj == 'en':
        print("Generating enwiki valid titles list...")
        validtitles = set()
        for row in wptools.query('wiki', 'select page_title from page where page_namespace = 0 and page_is_redirect = 0;', None):
            validtitles.add(row[0].decode('utf-8'))
    else:
        validtitles = None

    # Read through each file, and if it matches with the project, append to output

    counter = 0
    output = {}
    for file in filepaths:
        counter += 1
        print("Processing dump file " + str(counter) + "/" + str(len(filepaths)), end="\r")
        filename = '/public/dumps/pagecounts-raw/{0}/{1}/pagecounts-{2}.gz'.format(file[0], file[1], file[2])

        if os.path.isfile(filename) == False:
            continue

        with gzip.open(filename, mode='rt', encoding='utf-8') as f:
            for line in f:
                entry = line.split(' ')  # It's a space-delimited file, or something
                if entry[0] == proj:
                    entry[1] = html.unescape(entry[1]).replace(' ', '_')
                    if proj == 'en' and entry[1] not in validtitles:  # English Wikipedia specific check
                        continue
                    else:
                        if entry[1] in output:
                            output[entry[1]] += int(entry[2])  # Append to existing record
                        else:
                            output[entry[1]] = int(entry[2])  # Create new record

    print("\nProcessed stats for " + str(len(output)) + " articles.")
    return output

def getpageviews(dump, article):
    '''
    Queries *dump* for the number of page views in the last 30 days
    Takes dict *dump*, string *article* as input, returns view count.
    Does NOT take the logarithm of the view count.
    '''

    if article in dump:
        return dump[article]
    else:
        return 0

def getlinkcount(wptools, package):
    '''
    Gets a list of inbound links for a list of articles
    Takes list *package* as input, returns list of tuples (article, log of linkcount)
    Input MUST be a list. If there is just one article, enter it as such: [article]
    '''

    if len(package) > 1:
        query_builder = 'select pl_title, count(*) from pagelinks where pl_namespace = 0 and pl_title in {0} group by pl_title;'.format(tuple(package))
    else:
        query_builder = 'select pl_title, count(*) from pagelinks where pl_namespace = 0 and pl_title = "{0}" group by pl_title;'.format(package[0])

    output = []
    for row in wptools.query('wiki', query_builder, None):
        output.append((row[0].decode('utf-8'), log(row[1] + 1)))

    if len(output) == 0:
        output = [('', 0)]  # Return a consistent result even if there is nothing

    return output

def getinternalclout(wptools, destination, articlebatch):
    '''
    Gets a list of inbound links from *articlebatch* to *destination*
    Takes lists *articlebatch* and *destination* as input, returns list of tuples (article, log of internalclout)
    Input MUST be a list. If there is just one article, enter it as such: [article]
    '''

    output = []
    q = "select pl_title, count(*) from pagelinks join page on pl_from = page_id where pl_namespace = 0 and pl_title in {0} and page_title in {1} group by pl_title;"
    for row in wptools.query('wiki', q.format(tuple(destination), tuple(articlebatch)), None):
        output.append((row[0].decode('utf-8'), log(row[1])))

    return output

class QualityPredictor:
    def qualitypredictor(self, pagetitle):
        print("Argh! Not ready yet!")
        # chat it up with ORES

class PriorityPredictor:
    def __init__(self, viewdump=None):
        print("Initializing the Priority Predictor")
        self.wptools = WikiProjectTools()

        if viewdump == None:  # If a dumped JSON file of pageviews is not specified
            self.dump = getviewdump(self.wptools, 'en', days=30)
        else:
            with open(viewdump, 'r') as f:
                self.dump = json.load(f)  # Load pageviews from a dumped JSON file

    def loadproject(self, wikiproject, unknownpriority):
        self.projectcat = unknownpriority.replace("Unknown-", "")
        self.project = wikiproject
        self.rank = []  # Sorted list of tuples; allows for ranking
        self.score = {}  # Unsorted dictionary "article: value"; allows for easily looking up scores later
        # We need all the articles for a WikiProject, since the system works by comparing stats for an article to the others.
        print("Preparing Priority Predictor for: " + self.project)
        self.articles = []   # List of strings (article titles)
        pageviews = []  # List of tuples (article title, log of view count)
        linkcount = []  # List of tuples (article title, log of link count)
        for row in self.wptools.query('index', 'select pi_page from projectindex where pi_project = "Wikipedia:{0}";'.format(self.project), None):
            if row[0].startswith("Talk:"):  #
                article = row[0][5:] # Stripping out "Talk:"
                self.articles.append(article)
                pageviews.append((article, log(getpageviews(self.dump, article) + 1)))

        # Inbound link count
        # This is batched, thus broken out of the loop
        print("Getting inbound link count...")
        packages = []
        for i in range(0, len(self.articles), 10000):
            packages.append(self.articles[i:i+10000])

        for package in packages:
                toappend = getlinkcount(self.wptools, package)
                for item in toappend:
                    linkcount.append(item)

        # "Internal Clout"
        # This measures, within a group of articles, the number of links to each other
        # Works amazingly well as a metric

        print("Measuring internal clout...")
        internalclout = getinternalclout(self.wptools, self.articles, self.articles)

        # Sorting...
        pageviews = sorted(pageviews, key=operator.itemgetter(1), reverse=True)
        linkcount = sorted(linkcount, key=operator.itemgetter(1), reverse=True)
        internalclout = sorted(internalclout, key=operator.itemgetter(1), reverse=True)

        # Computing relative measurements
        # "Relative" means "as a ratio to the highest rank".
        # The most viewed article has a relative pageview score of 1.00. Goes lower from there.

        print("Computing relative measurements...")
        pageviews_relative = {}
        linkcount_relative = {}
        internalclout_relative = {}

        self.mostviews = pageviews[0][1]
        self.mostlinks = linkcount[0][1]
        self.mostinternal = internalclout[0][1]

        for pair in pageviews:
            article = pair[0]
            count = pair[1]
            pageviews_relative[article] = count / self.mostviews

        for pair in linkcount:
            article = pair[0]
            count = pair[1]
            linkcount_relative[article] = count / self.mostlinks

        for pair in internalclout:
            article = pair[0]
            count = pair[1]
            internalclout_relative[article] = count / self.mostinternal

        for article in self.articles:
            weightedscore = (internalclout_relative[article] * 0.5) + (pageviews_relative[article] * 0.375) + (linkcount_relative[article] * 0.125)
            self.rank.append((article, weightedscore))

        self.rank = sorted(self.rank, key=operator.itemgetter(1), reverse=True)

        # Re-scaling scores, such that the highest ranked will always have a score of 1.00.
        self.highestscore = self.rank[0][1]
        self.rank = [(item[0], item[1] / self.highestscore) for item in self.rank]

        # Defining unordered index of scores
        for item in self.rank:
            self.score[item[0]] = item[1]

        # Calculating minimum scores
        # The idea is that there is a minimum score for something to be top, high, or mid-priority
        # The script is fed the category name for the unknown-importance/unknown-priority category
        # Based on this, derive category names for top/high/mid/low, add all the counts together...
        # ...then calculate ratio for top/high/mid as a ratio of the total...
        # ...multiply that ratio by the count of self.rank, convert to an integer
        # ...and then threshold = self.rank[that integer][1]
        # This gives us a general sense of what proportion of articles should be considered top/high/mid/low
        # Far from perfect but it's a start.

        print("Calculating priority thresholds...")
        prioritycount = {}

        q = 'select count(*) from categorylinks where cl_type = "page" and cl_to = "{0}";'
        for priority in ['Top-', 'High-', 'Mid-', 'Low-']:
            prioritycategory = priority + self.projectcat
            prioritycount[priority] = self.wptools.query('wiki', q.format(prioritycategory), None)[0][0]

        total_assessed = sum([x for x in prioritycount.values()])

        top_index = int((prioritycount['Top-'] / total_assessed) * len(self.articles) - 1)
        high_index = top_index + int((prioritycount['High-'] / total_assessed) * len(self.articles) -1)
        mid_index = high_index + int((prioritycount['Mid-'] / total_assessed) * len(self.articles) -1)

        self.threshold_top = self.rank[top_index][1]
        self.threshold_high = self.rank[high_index][1]
        self.threshold_mid = self.rank[mid_index][1]

    def predictpage(self, pagetitle):
        # Pull pagescore if already defined
        # Otherwise, compute it "de novo"
        if pagetitle in self.articles:
            pagescore = self.score[pagetitle]
        else:
            pageviews = log(getpageviews(self.dump, pagetitle) + 1) / self.mostviews
            linkcount = getlinkcount(self.wptools, [pagetitle])[0][1] / self.mostlinks
            internalclout = getinternalclout(self.wptools, [pagetitle], self.articles)[0][1] / self.mostinternal
            pagescore = ((internalclout * 0.5) + (pageviews * 0.375) + (linkcount * 0.125)) / self.highestscore

        if pagescore >= self.threshold_top:
            return "Top"

        if pagescore >= self.threshold_high:
            return "High"

        if pagescore >= self.threshold_mid:
            return "Mid"

        # If none of these...
        return "Low"

    def audit(self):
        print("Auditing " + self.project)

        for prefix in ['Top', 'High', 'Mid', 'Low']:
            category = prefix + "-" + self.projectcat
            matrix = {'Top': 0, 'High': 0, 'Mid': 0, 'Low': 0}

            q = 'select page_title from categorylinks join page on cl_from = page_id where cl_type = "page" and cl_to = "{0}";'
            for row in self.wptools.query('wiki', q.format(category), None):
                title = row[0].decode('utf-8')
                matrix[self.predictpage(title)] += 1

            total_assessed = sum([x for x in matrix.values()])

            print("Human assessed " + prefix + "-priority:")
            for item in matrix.keys():
                print("Machine assessed " + item + "-priority: " + str(matrix[item]) + " (" + str((matrix[item] / total_assessed) * 100) + "%)")
            print("Total: " + str(total_assessed) + " articles")<|endoftext|>