<commit_before>import numpy as np
import matplotlib.pyplot as plt
import matplotlib.mlab as mlab
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.collections import PolyCollection
from colors import *
import json
import time
from line import *
from statlib import stats

def cull_outliers(data, n_sigma):
    mean = stats.mean(map(lambda x: x, data))
    sigma  = stats.stdev(map(lambda x: x, data))
    return filter(lambda x: abs(x - mean) < n_sigma * sigma, data)

def clip(data, min, max):
    return filter(lambda x: min<x<max, data)

def normalize(array):
    denom = max(map(lambda x: abs(x), array))
    if denom == 0:
        return array
    else:
        return map(lambda x: float(x) / denom, array)

class TimeSeries(list):
    def __init__(self, units):
        self.units = units

class default_empty_timeseries_dict(dict):
    units_line = line("([A-Za-z_]+)(\[[A-Za-z_]\]+)", [('key', 's'), ('units', 's')])
    def __getitem__(self, key):
        m = self.units_line.parse_line(key)
        if m:
            key = m['key']
            units = m['units']
        else:
            units = ''
        if key in self:
            return self.get(key)
        else:
            return TimeSeries(units)
    def copy(self):
        copy = default_empty_timeseries_dict()
        copy.update(self)
        return copy

class TimeSeriesCollection():
    def __init__(self):
        self.data = default_empty_timeseries_dict()

    def read(self, file_name):
        try:
            data = open(file_name).readlines()
            self.data = self.parse(data)
            self.process()
        except IOError:
            print 'Missing file: %s data from it will not be reported' % file_name
        return self #this just lets you do initialization in one line

    def copy(self):
        copy = self.__class__()
        copy.data = self.data.copy()
        return copy

    def __add__(self, other):
        res = self.copy()
        for val in other.data.iteritems():
            assert not val[0] in res.data
            res.data[val[0]] = val[1]
        return res

#limit the data to just the keys in keys
    def select(self, keys):
        copy = self.copy()
        for key in copy.data.keys():
            if not key in keys:
                copy.data.pop(key)

        return copy

    def drop(self, keys):
        copy = self.copy()
        for key in copy.data.keys():
            if key in keys:
                copy.data.pop(key)

        return copy

    def remap(self, orig_name, new_name):
        copy = self.drop(orig_name)
        copy.data[new_name] = self.data[orig_name]
        return copy

    def parse(self, data):
        pass

#do post processing things on the data (ratios and derivatives and stuff)
    def process(self):
        pass

    def json(self, out_fname, meta_data):
        top_level = {}
        top_level['date'] = time.asctime()
        top_level['meta'] = meta_data
        top_level['data'] = {}
        top_level['data']['rethinkdb'] = {}
        for series in self.data.iteritems():
            top_level['data']['rethinkdb'][series[0]] = {}
            top_level['data']['rethinkdb'][series[0]]['data'] = map(lambda x: list(x), zip(range(len(series[1])), series[1]))
            top_level['data']['rethinkdb'][series[0]]['unit'] = series[1].units

        f = open(out_fname + '.js', 'w')
        print >>f, json.dumps(top_level)
        f.close()

    def histogram(self, out_fname):
        assert self.data

        fig = plt.figure()
        ax = fig.add_subplot(111)

        data = map(lambda x: x[1], self.data.iteritems())
        mean = stats.mean(map(lambda x: x, reduce(lambda x, y: x + y, data)))
        stdev= stats.stdev(map(lambda x: x, reduce(lambda x, y: x + y, data)))
        labels = []
        for series, color in zip(self.data.iteritems(), colors):
            ax.hist(clip(series[1], 0, 6000), 40, histtype='step', facecolor = color, alpha = .4, label = series[0])
            labels.append(series[0])

        ax.set_ylabel('Count')
        ax.set_xlabel('Latency') #simply should not be hardcoded but we want nice pictures now
        ax.grid(True)
        plt.legend(labels, loc=1)
        plt.savefig(out_fname, dpi=300)

    def plot(self, out_fname, normalize = False):
        assert self.data
        fig = plt.figure()
        ax = fig.add_subplot(111)
        labels = []
        color_index = 0
        for series in self.data.iteritems():
            if normalize:
                data_to_use = normalize(series[1])
            else:
                data_to_use = series[1]
            labels.append((ax.plot(range(len(series[1])), data_to_use, colors[color_index]), series[0]))
            color_index += 1

        ax.set_xlabel('Time (seconds)')
        ax.set_xlim(0, len(self.data[self.data.keys()[0]]) - 1)
        if normalize:
            ax.set_ylim(0, 1.0)
        else:
            ax.set_ylim(0, max(self.data[self.data.keys()[0]]))
        ax.grid(True)
        plt.legend(tuple(map(lambda x: x[0], labels)), tuple(map(lambda x: x[1], labels)), loc=1)
        plt.savefig(out_fname, dpi=300)

    def stats(self):
        res = {}
        for val in self.data.iteritems():
            stat_report = {}
            stat_report['mean'] = stats.mean(map(lambda x: x, val[1]))
            stat_report['stdev'] = stats.stdev(map(lambda x: x, val[1]))
            res[val[0]] = stat_report

        return res

#function : (serieses)*len(arg_keys) -> series
#TODO this should return a copy with the changes
    def derive(self, name, arg_keys, function):
        args = []
        for key in arg_keys:
            assert key in self.data
            args.append(self.data[key])

        self.data[name] = function(tuple(args))
        return self

def multi_plot(timeseries, out_fname):
    fig = plt.figure()
    ax = fig.add_subplot(111)
    verts = []

    xs = range(min(map(lambda x: min(map(lambda y: len(y[1]), x.data.iteritems())), timeseries)))

    for z in range(len(timeseries)):
        for series in timeseries[z].data.iteritems():
            verts.append(zip(xs, series[1]))

    poly = PolyCollection(verts, facecolors = colors[0:len(verts)])
    poly.set_alpha(0.7)
    ax.add_collection3d(poly, range(len(timeseries)), zdir='y')

    ax.set_xlabel('X')
    ax.set_xlim3d(0, len(xs))
    ax.set_ylabel('Y')
    ax.set_ylim3d(-1, len(timeseries))
    ax.set_zlabel('Z')
    ax.set_zlim3d(0, max(map(lambda x: max(x), timeseries)))
    plt.savefig(out_fname, dpi=300)

#take discret derivative of a series (shortens series by 1)
def differentiate(series):
#series will be a tuple
    series = series[0]
    res = []
    for f_t, f_t_plus_one in zip(series[:len(series) - 1], series[1:]):
        res.append(f_t_plus_one - f_t)

    return res

def difference(serieses):
    assert serieses[0].units == serieses[1].units
    res = TimeSeries(serieses[0].units)
    for x,y in zip(serieses[0], serieses[1]):
        res.append(x - y)

    return res

class IOStat(TimeSeriesCollection):
    file_hdr_line   = line("Linux.*", [])
    avg_cpu_hdr_line= line("^avg-cpu:  %user   %nice %system %iowait  %steal   %idle$", [])
    avg_cpu_line    = line("^" + "\s+([\d\.]+)" * 6 + "$", [('user', 'f'), ('nice', 'f'), ('system', 'f'), ('iowait', 'f'),  ('steal', 'f'),   ('idle', 'f')])
    dev_hdr_line    = line("^Device:            tps   Blk_read/s   Blk_wrtn/s   Blk_read   Blk_wrtn$", [])
    dev_line        = line("^(\w+)\s+([\d\.]+)\s+([\d\.]+)\s+([\d\.]+)\s+(\d+)\s+(\d+)$", [('device', 's'), ('tps', 'f'), (' Blk_read', 'f'), (' Blk_wrtn', 'f'), (' Blk_read', 'd'), (' Blk_wrtn', 'd')])

    def parse(self, data):
        res = default_empty_timeseries_dict()
        data.reverse()
        m = until(self.file_hdr_line, data)
        assert m != False
        while True:
            m = until(self.avg_cpu_hdr_line, data)
            if m == False:
                break

            m = take(self.avg_cpu_line, data)
            assert m
            for val in m.iteritems():
                res['cpu_' + val[0]] += [val[1]]

            m = until(self.dev_hdr_line, data)
            assert m != False

            m = take_while([self.dev_line], data)
            for device in m:
                dev_name = device.pop('device')
                for val in device.iteritems():
                    res['dev:' + dev_name + '_' + val[0]] += [val[1]]

        return res

class VMStat(TimeSeriesCollection):
    file_hdr_line   = line("^procs -----------memory---------- ---swap-- -----io---- -system-- ----cpu----$", [])
    stats_hdr_line  = line("^ r  b   swpd   free   buff  cache   si   so    bi    bo   in   cs us sy id wa$", [])
    stats_line      = line("\s+(\d+)" * 16, [('r', 'd'),  ('b', 'd'),   ('swpd', 'd'),   ('free', 'd'),   ('buff', 'd'),  ('cache', 'd'),   ('si', 'd'),   ('so', 'd'),    ('bi', 'd'),    ('bo', 'd'),   ('in', 'd'),   ('cs', 'd'), ('us', 'd'), ('sy', 'd'), ('id', 'd'), ('wa', 'd')])

    def parse(self, data):
        res = default_empty_timeseries_dict()
        data.reverse()
        while True:
            m = until(self.file_hdr_line, data)
            if m == False:
                break
            m = take(self.stats_hdr_line, data)
            assert m != False
            m = take_while([self.stats_line], data)
            for stat_line in m:
                for val in stat_line.iteritems():
                    res[val[0]]+= [val[1]]
        return res

class Latency(TimeSeriesCollection):
    line = line("(\d+)\s+([\d.]+)\n", [('tick', 'd'), ('latency', 'f')])

    def parse(self, data):
        res = default_empty_timeseries_dict()
        for line in data:
            res['latency'] += [self.line.parse_line(line)['latency']]
        return res

class QPS(TimeSeriesCollection):
    line = line("(\d+)\s+([\d]+)\n", [('tick', 'd'), ('qps', 'f')])

    def parse(self, data):
        res = default_empty_timeseries_dict()
        for line in data:
            res['qps'] += [self.line.parse_line(line)['qps']]
        return res

class RDBStats(TimeSeriesCollection):
    int_line  = line("STAT\s+(\w+)\s+(\d+)[^\.](?:\s+\(average of \d+\))?", [('name', 's'), ('value', 'd')])
    flt_line  = line("STAT\s+(\w+)\s+([\d.]+)\s+\([\d/]+\)", [('name', 's'), ('value', 'f')])
    end_line  = line("END", [])

    def parse(self, data):
        res = default_empty_timeseries_dict()
        data.reverse()

        while True:
            m = take_while([self.int_line, self.flt_line], data)
            if not m:
                break
            for match in m:
                res[match['name']] += [match['value']]

            m = take(self.end_line, data)
            assert m != False

            if res:
                lens = map(lambda x: len(x[1]), res.iteritems())
                assert max(lens) == min(lens)
        return res

    def process(self):
        differences = [('io_reads_completed', 'io_reads_started'),
                       ('io_writes_started', 'io_writes_completed'),
                       ('transactions_started', 'transactions_ready'),
                       ('transactions_ready', 'transactions_completed'),
                       ('bufs_acquired', 'bufs_ready'),
                       ('bufs_ready', 'bufs_released')]
        keys_to_drop = set()
        for dif in differences:
            self.derive(dif[0] + ' - ' + dif[1], dif, difference)
            keys_to_drop.add(dif[0])
            keys_to_drop.add(dif[1])
        self.drop(keys_to_drop)
<commit_msg>Fix parsing for rdb stats.<commit_after>import numpy as np
import matplotlib.pyplot as plt
import matplotlib.mlab as mlab
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.collections import PolyCollection
from colors import *
import json
import time
from line import *
from statlib import stats

def cull_outliers(data, n_sigma):
    mean = stats.mean(map(lambda x: x, data))
    sigma  = stats.stdev(map(lambda x: x, data))
    return filter(lambda x: abs(x - mean) < n_sigma * sigma, data)

def clip(data, min, max):
    return filter(lambda x: min<x<max, data)

def normalize(array):
    denom = max(map(lambda x: abs(x), array))
    if denom == 0:
        return array
    else:
        return map(lambda x: float(x) / denom, array)

class TimeSeries(list):
    def __init__(self, units):
        self.units = units

class default_empty_timeseries_dict(dict):
    units_line = line("([A-Za-z_]+)(\[[A-Za-z_]\]+)", [('key', 's'), ('units', 's')])
    def __getitem__(self, key):
        m = self.units_line.parse_line(key)
        if m:
            key = m['key']
            units = m['units']
        else:
            units = ''
        if key in self:
            return self.get(key)
        else:
            return TimeSeries(units)
    def copy(self):
        copy = default_empty_timeseries_dict()
        copy.update(self)
        return copy

class TimeSeriesCollection():
    def __init__(self):
        self.data = default_empty_timeseries_dict()

    def read(self, file_name):
        try:
            data = open(file_name).readlines()
            self.data = self.parse(data)
            self.process()
        except IOError:
            print 'Missing file: %s data from it will not be reported' % file_name
        return self #this just lets you do initialization in one line

    def copy(self):
        copy = self.__class__()
        copy.data = self.data.copy()
        return copy

    def __add__(self, other):
        res = self.copy()
        for val in other.data.iteritems():
            assert not val[0] in res.data
            res.data[val[0]] = val[1]
        return res

#limit the data to just the keys in keys
    def select(self, keys):
        copy = self.copy()
        for key in copy.data.keys():
            if not key in keys:
                copy.data.pop(key)

        return copy

    def drop(self, keys):
        copy = self.copy()
        for key in copy.data.keys():
            if key in keys:
                copy.data.pop(key)

        return copy

    def remap(self, orig_name, new_name):
        copy = self.drop(orig_name)
        copy.data[new_name] = self.data[orig_name]
        return copy

    def parse(self, data):
        pass

#do post processing things on the data (ratios and derivatives and stuff)
    def process(self):
        pass

    def json(self, out_fname, meta_data):
        top_level = {}
        top_level['date'] = time.asctime()
        top_level['meta'] = meta_data
        top_level['data'] = {}
        top_level['data']['rethinkdb'] = {}
        for series in self.data.iteritems():
            top_level['data']['rethinkdb'][series[0]] = {}
            top_level['data']['rethinkdb'][series[0]]['data'] = map(lambda x: list(x), zip(range(len(series[1])), series[1]))
            top_level['data']['rethinkdb'][series[0]]['unit'] = series[1].units

        f = open(out_fname + '.js', 'w')
        print >>f, json.dumps(top_level)
        f.close()

    def histogram(self, out_fname):
        assert self.data

        fig = plt.figure()
        ax = fig.add_subplot(111)

        data = map(lambda x: x[1], self.data.iteritems())
        mean = stats.mean(map(lambda x: x, reduce(lambda x, y: x + y, data)))
        stdev= stats.stdev(map(lambda x: x, reduce(lambda x, y: x + y, data)))
        labels = []
        for series, color in zip(self.data.iteritems(), colors):
            ax.hist(clip(series[1], 0, 6000), 40, histtype='step', facecolor = color, alpha = .4, label = series[0])
            labels.append(series[0])

        ax.set_ylabel('Count')
        ax.set_xlabel('Latency') #simply should not be hardcoded but we want nice pictures now
        ax.grid(True)
        plt.legend(labels, loc=1)
        plt.savefig(out_fname, dpi=300)

    def plot(self, out_fname, normalize = False):
        assert self.data
        fig = plt.figure()
        ax = fig.add_subplot(111)
        labels = []
        color_index = 0
        for series in self.data.iteritems():
            if normalize:
                data_to_use = normalize(series[1])
            else:
                data_to_use = series[1]
            labels.append((ax.plot(range(len(series[1])), data_to_use, colors[color_index]), series[0]))
            color_index += 1

        ax.set_xlabel('Time (seconds)')
        ax.set_xlim(0, len(self.data[self.data.keys()[0]]) - 1)
        if normalize:
            ax.set_ylim(0, 1.0)
        else:
            ax.set_ylim(0, max(self.data[self.data.keys()[0]]))
        ax.grid(True)
        plt.legend(tuple(map(lambda x: x[0], labels)), tuple(map(lambda x: x[1], labels)), loc=1)
        plt.savefig(out_fname, dpi=300)

    def stats(self):
        res = {}
        for val in self.data.iteritems():
            stat_report = {}
            stat_report['mean'] = stats.mean(map(lambda x: x, val[1]))
            stat_report['stdev'] = stats.stdev(map(lambda x: x, val[1]))
            res[val[0]] = stat_report

        return res

#function : (serieses)*len(arg_keys) -> series
#TODO this should return a copy with the changes
    def derive(self, name, arg_keys, function):
        args = []
        for key in arg_keys:
            if not key in self.data:
                return self
            args.append(self.data[key])

        self.data[name] = function(tuple(args))
        return self

def multi_plot(timeseries, out_fname):
    fig = plt.figure()
    ax = fig.add_subplot(111)
    verts = []

    xs = range(min(map(lambda x: min(map(lambda y: len(y[1]), x.data.iteritems())), timeseries)))

    for z in range(len(timeseries)):
        for series in timeseries[z].data.iteritems():
            verts.append(zip(xs, series[1]))

    poly = PolyCollection(verts, facecolors = colors[0:len(verts)])
    poly.set_alpha(0.7)
    ax.add_collection3d(poly, range(len(timeseries)), zdir='y')

    ax.set_xlabel('X')
    ax.set_xlim3d(0, len(xs))
    ax.set_ylabel('Y')
    ax.set_ylim3d(-1, len(timeseries))
    ax.set_zlabel('Z')
    ax.set_zlim3d(0, max(map(lambda x: max(x), timeseries)))
    plt.savefig(out_fname, dpi=300)

#take discret derivative of a series (shortens series by 1)
def differentiate(series):
#series will be a tuple
    series = series[0]
    res = []
    for f_t, f_t_plus_one in zip(series[:len(series) - 1], series[1:]):
        res.append(f_t_plus_one - f_t)

    return res

def difference(serieses):
    assert serieses[0].units == serieses[1].units
    res = TimeSeries(serieses[0].units)
    for x,y in zip(serieses[0], serieses[1]):
        res.append(x - y)

    return res

class IOStat(TimeSeriesCollection):
    file_hdr_line   = line("Linux.*", [])
    avg_cpu_hdr_line= line("^avg-cpu:  %user   %nice %system %iowait  %steal   %idle$", [])
    avg_cpu_line    = line("^" + "\s+([\d\.]+)" * 6 + "$", [('user', 'f'), ('nice', 'f'), ('system', 'f'), ('iowait', 'f'),  ('steal', 'f'),   ('idle', 'f')])
    dev_hdr_line    = line("^Device:            tps   Blk_read/s   Blk_wrtn/s   Blk_read   Blk_wrtn$", [])
    dev_line        = line("^(\w+)\s+([\d\.]+)\s+([\d\.]+)\s+([\d\.]+)\s+(\d+)\s+(\d+)$", [('device', 's'), ('tps', 'f'), (' Blk_read', 'f'), (' Blk_wrtn', 'f'), (' Blk_read', 'd'), (' Blk_wrtn', 'd')])

    def parse(self, data):
        res = default_empty_timeseries_dict()
        data.reverse()
        m = until(self.file_hdr_line, data)
        assert m != False
        while True:
            m = until(self.avg_cpu_hdr_line, data)
            if m == False:
                break

            m = take(self.avg_cpu_line, data)
            assert m
            for val in m.iteritems():
                res['cpu_' + val[0]] += [val[1]]

            m = until(self.dev_hdr_line, data)
            assert m != False

            m = take_while([self.dev_line], data)
            for device in m:
                dev_name = device.pop('device')
                for val in device.iteritems():
                    res['dev:' + dev_name + '_' + val[0]] += [val[1]]

        return res

class VMStat(TimeSeriesCollection):
    file_hdr_line   = line("^procs -----------memory---------- ---swap-- -----io---- -system-- ----cpu----$", [])
    stats_hdr_line  = line("^ r  b   swpd   free   buff  cache   si   so    bi    bo   in   cs us sy id wa$", [])
    stats_line      = line("\s+(\d+)" * 16, [('r', 'd'),  ('b', 'd'),   ('swpd', 'd'),   ('free', 'd'),   ('buff', 'd'),  ('cache', 'd'),   ('si', 'd'),   ('so', 'd'),    ('bi', 'd'),    ('bo', 'd'),   ('in', 'd'),   ('cs', 'd'), ('us', 'd'), ('sy', 'd'), ('id', 'd'), ('wa', 'd')])

    def parse(self, data):
        res = default_empty_timeseries_dict()
        data.reverse()
        while True:
            m = until(self.file_hdr_line, data)
            if m == False:
                break
            m = take(self.stats_hdr_line, data)
            assert m != False
            m = take_while([self.stats_line], data)
            for stat_line in m:
                for val in stat_line.iteritems():
                    res[val[0]]+= [val[1]]
        return res

class Latency(TimeSeriesCollection):
    line = line("(\d+)\s+([\d.]+)\n", [('tick', 'd'), ('latency', 'f')])

    def parse(self, data):
        res = default_empty_timeseries_dict()
        for line in data:
            res['latency'] += [self.line.parse_line(line)['latency']]
        return res

class QPS(TimeSeriesCollection):
    line = line("(\d+)\s+([\d]+)\n", [('tick', 'd'), ('qps', 'f')])

    def parse(self, data):
        res = default_empty_timeseries_dict()
        for line in data:
            res['qps'] += [self.line.parse_line(line)['qps']]
        return res

class RDBStats(TimeSeriesCollection):
    stat_line = line("STAT\s+(\w+)(\[\w+\])?\s+(\d+)", [('name', 's'), ('units', 's'), ('value', 'd')])
    int_line  = line("STAT\s+(\w+)\s+(\d+)[^\.](?:\s+\(average of \d+\))?", [('name', 's'), ('value', 'd')])
    flt_line  = line("STAT\s+(\w+)\s+([\d.]+)\s+\([\d/]+\)", [('name', 's'), ('value', 'f')])
    end_line  = line("END", [])

    def parse(self, data):
        res = default_empty_timeseries_dict()
        data.reverse()

        while True:
            m = take_while([self.stat_line], data)
            if not m:
                break
            for match in m:
                res[match['name']] += [match['value']]

            m = take(self.end_line, data)
            assert m != False

            if res:
                lens = map(lambda x: len(x[1]), res.iteritems())
                assert max(lens) == min(lens)
        return res

    def process(self):
        differences = [('io_reads_completed', 'io_reads_started'),
                       ('io_writes_started', 'io_writes_completed'),
                       ('transactions_started', 'transactions_ready'),
                       ('transactions_ready', 'transactions_completed'),
                       ('bufs_acquired', 'bufs_ready'),
                       ('bufs_ready', 'bufs_released')]
        keys_to_drop = set()
        for dif in differences:
            self.derive(dif[0] + ' - ' + dif[1], dif, difference)
            keys_to_drop.add(dif[0])
            keys_to_drop.add(dif[1])
        self.drop(keys_to_drop)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""Combine logs from multiple bitcoin nodes as well as the test_framework log.

This streams the combined log output to stdout. Use combine_logs.py > outputfile
to write to an outputfile."""

import argparse
from collections import defaultdict, namedtuple
import glob
import heapq
import os
import re
import sys

# Matches on the date format at the start of the log event
TIMESTAMP_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6}")

LogEvent = namedtuple('LogEvent', ['timestamp', 'source', 'event'])

def main():
    """Main function. Parses args, reads the log files and renders them as text or html."""

    parser = argparse.ArgumentParser(usage='%(prog)s [options] <test temporary directory>', description=__doc__)
    parser.add_argument('-c', '--color', dest='color', action='store_true', help='outputs the combined log with events colored by source (requires posix terminal colors. Use less -r for viewing)')
    parser.add_argument('--html', dest='html', action='store_true', help='outputs the combined log as html. Requires jinja2. pip install jinja2')
    args, unknown_args = parser.parse_known_args()

    if args.color and os.name != 'posix':
        print("Color output requires posix terminal colors.")
        sys.exit(1)

    if args.html and args.color:
        print("Only one out of --color or --html should be specified")
        sys.exit(1)

    # There should only be one unknown argument - the path of the temporary test directory
    if len(unknown_args) != 1:
        print("Unexpected arguments" + str(unknown_args))
        sys.exit(1)

    log_events = read_logs(unknown_args[0])

    print_logs(log_events, color=args.color, html=args.html)

def read_logs(tmp_dir):
    """Reads log files.

    Delegates to generator function get_log_events() to provide individual log events
    for each of the input log files."""

    files = [("test", "%s/test_framework.log" % tmp_dir)]
    for i, logfile in enumerate(glob.glob("%s/node*/regtest/debug.log" % tmp_dir)):
        files.append(("node%d" % i, logfile))

    return heapq.merge(*[get_log_events(source, f) for source, f in files])

def get_log_events(source, logfile):
    """Generator function that returns individual log events.

    Log events may be split over multiple lines. We use the timestamp
    regex match as the marker for a new log event."""
    try:
        with open(logfile, 'r') as infile:
            event = ''
            timestamp = ''
            for line in infile:
                # skip blank lines
                if line == '\n':
                    continue
                # if this line has a timestamp, it's the start of a new log event.
                time_match = TIMESTAMP_PATTERN.match(line)
                if time_match:
                    if event:
                        yield LogEvent(timestamp=timestamp, source=source, event=event.rstrip())
                    event = line
                    timestamp = time_match.group()
                # if it doesn't have a timestamp, it's a continuation line of the previous log.
                else:
                    event += "\n" + line
            # Flush the final event
            yield LogEvent(timestamp=timestamp, source=source, event=event.rstrip())
    except FileNotFoundError:
        print("File %s could not be opened. Continuing without it." % logfile, file=sys.stderr)

def print_logs(log_events, color=False, html=False):
    """Renders the iterator of log events into text or html."""
    if not html:
        colors = defaultdict(lambda: '')
        if color:
            colors["test"] = "\033[0;36m"   # CYAN
            colors["node0"] = "\033[0;34m"  # BLUE
            colors["node1"] = "\033[0;32m"  # GREEN
            colors["node2"] = "\033[0;31m"  # RED
            colors["node3"] = "\033[0;33m"  # YELLOW
            colors["reset"] = "\033[0m"     # Reset font color

        for event in log_events:
            print("{0} {1: <5} {2} {3}".format(colors[event.source.rstrip()], event.source, event.event, colors["reset"]))

    else:
        try:
            import jinja2
        except ImportError:
            print("jinja2 not found. Try `pip install jinja2`")
            sys.exit(1)
        print(jinja2.Environment(loader=jinja2.FileSystemLoader('./'))
                    .get_template('combined_log_template.html')
                    .render(title="Combined Logs from testcase", log_events=[event._asdict() for event in log_events]))

if __name__ == '__main__':
    main()
<commit_msg>[qa] combine_logs: Use ordered list for logfiles<commit_after>#!/usr/bin/env python3
"""Combine logs from multiple bitcoin nodes as well as the test_framework log.

This streams the combined log output to stdout. Use combine_logs.py > outputfile
to write to an outputfile."""

import argparse
from collections import defaultdict, namedtuple
import heapq
import itertools
import os
import re
import sys

# Matches on the date format at the start of the log event
TIMESTAMP_PATTERN = re.compile(r"^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6}")

LogEvent = namedtuple('LogEvent', ['timestamp', 'source', 'event'])

def main():
    """Main function. Parses args, reads the log files and renders them as text or html."""

    parser = argparse.ArgumentParser(usage='%(prog)s [options] <test temporary directory>', description=__doc__)
    parser.add_argument('-c', '--color', dest='color', action='store_true', help='outputs the combined log with events colored by source (requires posix terminal colors. Use less -r for viewing)')
    parser.add_argument('--html', dest='html', action='store_true', help='outputs the combined log as html. Requires jinja2. pip install jinja2')
    args, unknown_args = parser.parse_known_args()

    if args.color and os.name != 'posix':
        print("Color output requires posix terminal colors.")
        sys.exit(1)

    if args.html and args.color:
        print("Only one out of --color or --html should be specified")
        sys.exit(1)

    # There should only be one unknown argument - the path of the temporary test directory
    if len(unknown_args) != 1:
        print("Unexpected arguments" + str(unknown_args))
        sys.exit(1)

    log_events = read_logs(unknown_args[0])

    print_logs(log_events, color=args.color, html=args.html)

def read_logs(tmp_dir):
    """Reads log files.

    Delegates to generator function get_log_events() to provide individual log events
    for each of the input log files."""

    files = [("test", "%s/test_framework.log" % tmp_dir)]
    for i in itertools.count():
        logfile = "{}/node{}/regtest/debug.log".format(tmp_dir, i)
        if not os.path.isfile(logfile):
            break
        files.append(("node%d" % i, logfile))

    return heapq.merge(*[get_log_events(source, f) for source, f in files])

def get_log_events(source, logfile):
    """Generator function that returns individual log events.

    Log events may be split over multiple lines. We use the timestamp
    regex match as the marker for a new log event."""
    try:
        with open(logfile, 'r') as infile:
            event = ''
            timestamp = ''
            for line in infile:
                # skip blank lines
                if line == '\n':
                    continue
                # if this line has a timestamp, it's the start of a new log event.
                time_match = TIMESTAMP_PATTERN.match(line)
                if time_match:
                    if event:
                        yield LogEvent(timestamp=timestamp, source=source, event=event.rstrip())
                    event = line
                    timestamp = time_match.group()
                # if it doesn't have a timestamp, it's a continuation line of the previous log.
                else:
                    event += "\n" + line
            # Flush the final event
            yield LogEvent(timestamp=timestamp, source=source, event=event.rstrip())
    except FileNotFoundError:
        print("File %s could not be opened. Continuing without it." % logfile, file=sys.stderr)

def print_logs(log_events, color=False, html=False):
    """Renders the iterator of log events into text or html."""
    if not html:
        colors = defaultdict(lambda: '')
        if color:
            colors["test"] = "\033[0;36m"   # CYAN
            colors["node0"] = "\033[0;34m"  # BLUE
            colors["node1"] = "\033[0;32m"  # GREEN
            colors["node2"] = "\033[0;31m"  # RED
            colors["node3"] = "\033[0;33m"  # YELLOW
            colors["reset"] = "\033[0m"     # Reset font color

        for event in log_events:
            print("{0} {1: <5} {2} {3}".format(colors[event.source.rstrip()], event.source, event.event, colors["reset"]))

    else:
        try:
            import jinja2
        except ImportError:
            print("jinja2 not found. Try `pip install jinja2`")
            sys.exit(1)
        print(jinja2.Environment(loader=jinja2.FileSystemLoader('./'))
                    .get_template('combined_log_template.html')
                    .render(title="Combined Logs from testcase", log_events=[event._asdict() for event in log_events]))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin.env python
# coding: utf-8

"""Module for plot functions."""

import matplotlib.pyplot as plt
import sfg2d
from numpy import transpose, where, linspace, all, array

def fit_model(
        x,
        y,
        yerr,
        name,
        kwargs_model={},
        fit=True,
        print_matrix=True,
        kwargs_plot={},
        show_box=True,
        box_coords=None,
):
    """Fit Data in a configurable way.

    **Arguments:**
      - **x**: X data
      - **y**: Y data
      - **yerr**: Y error of the data
      - **name**: Name of the model

    **Keywords:**
      - **kwargs_model**: Keywords for the model
      - **fit**: Boolean to run fit
      - **print_matrix**: Boolean to print Correlation Matrix
      - **show_box**: Show fit result box
      - **box_coords**: Coordinates of the fit result box
    """
    model = getattr(sfg2d.models, name)(x, y, yerr, **kwargs_model)

    if fit:
        sfg2d.fit_model(
            model, print_matrix=print_matrix
        )
    kwargs_plot.setdefault('color', 'red')
    kwargs_plot.setdefault('label', 'Fit')
    plt.plot(model.xsample, model.yfit_sample, **kwargs_plot)

    if show_box:
        model.draw_text_box(box_coords)
    return model

def model_plot(
        model,
        kwargs_errorbar=None,
        kwargs_line_plot=None,
        shift_x=None,
        shift_y=None,
        scale_y=None,
        normalize=False,
        xsample_slice=(0, 1),
        kwargs_textbox=None,
        show_roi=True,
        show_modeled_only=True,
):
    """
    **Kwargs:**
      - **kwargs_errorbar**: Kwargs passed to errorbar plot. Can be used to change e.g. color of the plot.
      - **kwargs_line_plot**: Kwargs passed to line plot of the fit line.
      - **shift_x**: Quick hack to shift the fit plot by x
      - **shift_y**: Qucick hack to shift the fit by y
      - **scale_y**: Qucik hack to scale fit by y
      - **normalize**: Normalize fit height to 1 - 0
      - **xsample_slice**: Tuple with x start and x stop for finding normalization minimun
      - **kwargs_textbox**: If kwargs textbox given, then a textbox with fitresults is drawn
    - **show_roi**: Mark the roi of the fit
    - **show_modeled_only**: Show only the fit, not the data.
    """
    if not kwargs_errorbar:
        kwargs_errorbar = {}
    if not kwargs_line_plot:
        kwargs_line_plot = {}

    kwargs_errorbar.setdefault('marker', 'o')
    kwargs_errorbar.setdefault('linestyle', 'None')

    if show_modeled_only:
        xdata = model.xdata
        ydata = model.ydata
        yerr = model.sigma
    else:
        xdata = model._xdata
        ydata = model._ydata
        yerr = model._sigma

    xsample = linspace(xdata[0], xdata[-1], model._xsample_num)
    ysample = model.fit_res(xsample)

    if shift_x:
        xdata =  xdata + shift_x
        xsample = xsample + shift_x

    if scale_y:
        ydata = ydata * scale_y
        ysample = ysample * scale_y
        yerr = yerr * scale_y

    if normalize:
        x_mask = where((xsample > xsample_slice[0]) & (xsample < xsample_slice[1]))
        factor = 1 - ysample[x_mask].min()
        ydata = (ydata - 1) / factor + 1
        ysample = (ysample - 1) / factor + 1
        yerr = yerr / factor

    if shift_y:
        ydata = ydata + shift_y
        ysample = ysample + shift_y

    plotline, capline, barline = plt.errorbar(
        xdata, ydata, yerr, **kwargs_errorbar
    )
    kwargs_line_plot.setdefault('color', plotline.get_color())
    plt.plot(xsample, ysample, **kwargs_line_plot)

    if show_roi:
        xx = xdata[model.roi]
        yy = ydata[model.roi]
        x = xx[0], xx[-1]
        y = yy[0], yy[-1]
        plt.scatter(x, y, marker='x', color='r')

    if isinstance(kwargs_textbox, dict):
        fig = plt.gcf()
        kwargs_textbox.setdefault('x', .6)
        kwargs_textbox.setdefault('y', .12)
        kwargs_textbox.setdefault('s', model.box_str)
        kwargs_textbox.setdefault('transform', fig.transFigure)
        plt.text(**kwargs_textbox)

def points_modeled(x, y, yerr=None, xline=None, yline=None, kwargs_point={}, kwargs_line={}):
    """Plot points and line."""
    kwargs_point.setdefault('marker', 'o')
    kwargs_point.setdefault('linestyle', 'None')
    if  isinstance(yerr, type(None)):
        lines = plt.plot(x, y, **kwargs_point)
        point = lines[-1]
    else:
        point, capline, barline = plt.errorbar(
            x, y, yerr, **kwargs_point
           )

    if not isinstance(xline, type(None)) and not isinstance(yline, type(None)):
        # For some reason, color must be set explicitly here,
        # otherwise it is not respected. Its a workaround
        kwargs_line.setdefault('color', point.get_color())
        color = kwargs_line.pop('color')
        plt.plot(xline, yline, color=color, **kwargs_line)

def spectrum(
        xdata,
        ydata,
        *args,
        ax=None,
        xlabel=None,
        ylabel=None,
        **kwargs
):
    """
    Plot data with pixel axis of data as x-axis

    xdata: 1d numpy array for x-axis values.
    ydata: 4d numpy array with [delay, frame, spec, pixel]
    ax: axis obj to plot on
    xlabel: Label of the x-axis

    **kwargs are passed to ax.plot

    """
    if not ax:
        ax = plt.gca()

    for delay in ydata:
        for frame in delay:
            for spec in frame:
               # We need a scatter like plot if its just one point
                if all(array(spec.shape) == 1):
                    kwargs.setdefault('marker', 'o')

                if isinstance(xdata, type(None)):
                    ax.plot(spec.T, *args, **kwargs)
                else:
                    ax.plot(xdata, spec.T, *args, **kwargs)

    if xlabel:
        ax.set_xlabel(xlabel)
    if ylabel:
        ax.set_ylabel(ylabel)

def track(
        xdata=None,
        ydata=None,
        ax=None,
        xlabel="RunNumber",
        ylabel='SFG Intensity',
        show_hlines=False,
        **kwargs
):
    """A Track is a Time wise plot of the data.

    **Arguments:**
      - **ydata**: 4d Numpy array to create plot from

    **Keywords:**
      - **show_vlines**: Boolean to show vertical lines after each scan.

    """
    if not ax:
        ax = plt.gca()

    delays, frames, spectra = ydata.shape
    # oder F, because ppelays is the first index and that
    # changes fastest
    ydata = ydata.reshape(delays*frames, spectra, order='F')

    for ispectrum in range(spectra):
        data = ydata[:, ispectrum]
        if isinstance(xdata, type(None)):
            ax.plot(data, **kwargs)
        else:
            ax.plot(xdata, data, **kwargs)
    if show_hlines:
        plt.vlines([delays*frame for frame in range(frames)], ydata.min(), ydata.max())

def trace(
        xdata,
        ydata,
        ax=None,
        yerr=None,
        xlabel='Time in fs',
        ylabel='Bleach in a.u.',
        **kwargs
):
    """
    data is the result of a subselection.

    This plot has delays on its x-axis.

    yerr Error bar for the trace. Must have no frame dimension.
    if yerr is given frame dimension must be 1.
    """
    if not ax:
        ax = plt.gca()

    # Transpose because we want the delay axis to be the last axis
    # of the array.
    kwargs.setdefault('marker', 'o')

    y = ydata.T
    for i in range(len(y)):
        pixel = y[i]
        for j in range(len(pixel)):
            spec = pixel[j]
            if isinstance(yerr, type(None)):
                for frame in spec:
                    ax.plot(xdata, frame.T, **kwargs)
            else:
                plt.errorbar(xdata, spec[0], yerr[:, j, i], axes=ax, **kwargs)

def contour(
        xdata,
        ydata,
        zdata,
        ax=None,
        **kwargs
):
    """
    Makes a contour plot for an SfgRecord.select() return for xdata, ydata and zdata.
    This makes multiple contour plots on top of each other. Normally more then one makes
    no sence but it works this way.
    **Arguments:**
      - **xdata**: Usually pp_delays
      - **ydata**: usually wavenumbers
      - **ydata**: usually bleach
    """
    kwargs.setdefault('extend', 'both')
    num_pp_delays, num_frames, num_spectra, num_pixel = zdata.shape
    for index_spectrum in range(num_spectra):
        for index_frame in range(num_frames):
            zzdata = zdata[:, index_frame, index_spectrum].T
            plt.contourf(xdata, ydata, zzdata, **kwargs)

def multifig(xdata, ydata, fig_axis=0, kwargs_plot=None, titles=None):
    """Create multiple figures for ydata, by using the axis of fig_ax.

    **Argument**:
      - **xdata**: 1D array usually wavenumbers
      - **ydata**: 4D array as usally.

    **kwargs**:
      - **fig_axis**: 0-3 and defines the axis of ydata that will be looped
        over during creation of the figures. Data is then taken from this
        axis per figure. 0  means 1 figure per pp_delay, 1 means 1 figure per
        frame and so on.
      - **kwargs_plot**: kwrges passed to spectrum plot.
      - **titles**: list of titles must have at least same length as number of figures.

    **Returns**:
    list of created figures.
    """
    if not kwargs_plot:
        kwargs_plot = {}

    fig_numbers = ydata.shape[fig_axis]
    figs, axs = [], []

    for i in range(fig_numbers):
        fig, ax = plt.subplots()
        figs.append(fig)
        axs.append(ax)
        try:
            plt.title(titles[i])
        except TypeError:
            pass
        yd = ydata.take([i], fig_axis)
        spectrum(xdata, yd, **kwargs_plot)
    return figs
<commit_msg>Traces with errorbars can not make multiframe plots<commit_after>#!/usr/bin.env python
# coding: utf-8

"""Module for plot functions."""

import matplotlib.pyplot as plt
import sfg2d
from numpy import transpose, where, linspace, all, array

def fit_model(
        x,
        y,
        yerr,
        name,
        kwargs_model={},
        fit=True,
        print_matrix=True,
        kwargs_plot={},
        show_box=True,
        box_coords=None,
):
    """Fit Data in a configurable way.

    **Arguments:**
      - **x**: X data
      - **y**: Y data
      - **yerr**: Y error of the data
      - **name**: Name of the model

    **Keywords:**
      - **kwargs_model**: Keywords for the model
      - **fit**: Boolean to run fit
      - **print_matrix**: Boolean to print Correlation Matrix
      - **show_box**: Show fit result box
      - **box_coords**: Coordinates of the fit result box
    """
    model = getattr(sfg2d.models, name)(x, y, yerr, **kwargs_model)

    if fit:
        sfg2d.fit_model(
            model, print_matrix=print_matrix
        )
    kwargs_plot.setdefault('color', 'red')
    kwargs_plot.setdefault('label', 'Fit')
    plt.plot(model.xsample, model.yfit_sample, **kwargs_plot)

    if show_box:
        model.draw_text_box(box_coords)
    return model

def model_plot(
        model,
        kwargs_errorbar=None,
        kwargs_line_plot=None,
        shift_x=None,
        shift_y=None,
        scale_y=None,
        normalize=False,
        xsample_slice=(0, 1),
        kwargs_textbox=None,
        show_roi=True,
        show_modeled_only=True,
):
    """
    **Kwargs:**
      - **kwargs_errorbar**: Kwargs passed to errorbar plot. Can be used to change e.g. color of the plot.
      - **kwargs_line_plot**: Kwargs passed to line plot of the fit line.
      - **shift_x**: Quick hack to shift the fit plot by x
      - **shift_y**: Qucick hack to shift the fit by y
      - **scale_y**: Qucik hack to scale fit by y
      - **normalize**: Normalize fit height to 1 - 0
      - **xsample_slice**: Tuple with x start and x stop for finding normalization minimun
      - **kwargs_textbox**: If kwargs textbox given, then a textbox with fitresults is drawn
    - **show_roi**: Mark the roi of the fit
    - **show_modeled_only**: Show only the fit, not the data.
    """
    if not kwargs_errorbar:
        kwargs_errorbar = {}
    if not kwargs_line_plot:
        kwargs_line_plot = {}

    kwargs_errorbar.setdefault('marker', 'o')
    kwargs_errorbar.setdefault('linestyle', 'None')

    if show_modeled_only:
        xdata = model.xdata
        ydata = model.ydata
        yerr = model.sigma
    else:
        xdata = model._xdata
        ydata = model._ydata
        yerr = model._sigma

    xsample = linspace(xdata[0], xdata[-1], model._xsample_num)
    ysample = model.fit_res(xsample)

    if shift_x:
        xdata =  xdata + shift_x
        xsample = xsample + shift_x

    if scale_y:
        ydata = ydata * scale_y
        ysample = ysample * scale_y
        yerr = yerr * scale_y

    if normalize:
        x_mask = where((xsample > xsample_slice[0]) & (xsample < xsample_slice[1]))
        factor = 1 - ysample[x_mask].min()
        ydata = (ydata - 1) / factor + 1
        ysample = (ysample - 1) / factor + 1
        yerr = yerr / factor

    if shift_y:
        ydata = ydata + shift_y
        ysample = ysample + shift_y

    plotline, capline, barline = plt.errorbar(
        xdata, ydata, yerr, **kwargs_errorbar
    )
    kwargs_line_plot.setdefault('color', plotline.get_color())
    plt.plot(xsample, ysample, **kwargs_line_plot)

    if show_roi:
        xx = xdata[model.roi]
        yy = ydata[model.roi]
        x = xx[0], xx[-1]
        y = yy[0], yy[-1]
        plt.scatter(x, y, marker='x', color='r')

    if isinstance(kwargs_textbox, dict):
        fig = plt.gcf()
        kwargs_textbox.setdefault('x', .6)
        kwargs_textbox.setdefault('y', .12)
        kwargs_textbox.setdefault('s', model.box_str)
        kwargs_textbox.setdefault('transform', fig.transFigure)
        plt.text(**kwargs_textbox)

def points_modeled(x, y, yerr=None, xline=None, yline=None, kwargs_point={}, kwargs_line={}):
    """Plot points and line."""
    kwargs_point.setdefault('marker', 'o')
    kwargs_point.setdefault('linestyle', 'None')
    if  isinstance(yerr, type(None)):
        lines = plt.plot(x, y, **kwargs_point)
        point = lines[-1]
    else:
        point, capline, barline = plt.errorbar(
            x, y, yerr, **kwargs_point
           )

    if not isinstance(xline, type(None)) and not isinstance(yline, type(None)):
        # For some reason, color must be set explicitly here,
        # otherwise it is not respected. Its a workaround
        kwargs_line.setdefault('color', point.get_color())
        color = kwargs_line.pop('color')
        plt.plot(xline, yline, color=color, **kwargs_line)

def spectrum(
        xdata,
        ydata,
        *args,
        ax=None,
        xlabel=None,
        ylabel=None,
        **kwargs
):
    """
    Plot data with pixel axis of data as x-axis

    xdata: 1d numpy array for x-axis values.
    ydata: 4d numpy array with [delay, frame, spec, pixel]
    ax: axis obj to plot on
    xlabel: Label of the x-axis

    **kwargs are passed to ax.plot

    """
    if not ax:
        ax = plt.gca()

    for delay in ydata:
        for frame in delay:
            for spec in frame:
               # We need a scatter like plot if its just one point
                if all(array(spec.shape) == 1):
                    kwargs.setdefault('marker', 'o')

                if isinstance(xdata, type(None)):
                    ax.plot(spec.T, *args, **kwargs)
                else:
                    ax.plot(xdata, spec.T, *args, **kwargs)

    if xlabel:
        ax.set_xlabel(xlabel)
    if ylabel:
        ax.set_ylabel(ylabel)

def track(
        xdata=None,
        ydata=None,
        ax=None,
        xlabel="RunNumber",
        ylabel='SFG Intensity',
        show_hlines=False,
        **kwargs
):
    """A Track is a Time wise plot of the data.

    **Arguments:**
      - **ydata**: 4d Numpy array to create plot from

    **Keywords:**
      - **show_vlines**: Boolean to show vertical lines after each scan.

    """
    if not ax:
        ax = plt.gca()

    delays, frames, spectra = ydata.shape
    # oder F, because ppelays is the first index and that
    # changes fastest
    ydata = ydata.reshape(delays*frames, spectra, order='F')

    for ispectrum in range(spectra):
        data = ydata[:, ispectrum]
        if isinstance(xdata, type(None)):
            ax.plot(data, **kwargs)
        else:
            ax.plot(xdata, data, **kwargs)
    if show_hlines:
        plt.vlines([delays*frame for frame in range(frames)], ydata.min(), ydata.max())

def trace(
        xdata,
        ydata,
        ax=None,
        yerr=None,
        xlabel='Time in fs',
        ylabel='Bleach in a.u.',
        **kwargs
):
    """
    data is the result of a subselection.

    This plot has delays on its x-axis.

    yerr Error bar for the trace. Must have no frame dimension.
    if yerr is given frame dimension must be 1.
    """
    if not ax:
        ax = plt.gca()

    # Transpose because we want the delay axis to be the last axis
    # of the array.
    kwargs.setdefault('marker', 'o')

    y = ydata.T
    for i in range(len(y)):
        pixel = y[i]
        for j in range(len(pixel)):
            spec = pixel[j]
            for frame in spec:
                if isinstance(yerr, type(None)):
                    ax.plot(xdata, frame.T, **kwargs)
                else:
                    plt.errorbar(xdata, frame, yerr[:, j, i], axes=ax, **kwargs)

def contour(
        xdata,
        ydata,
        zdata,
        ax=None,
        **kwargs
):
    """
    Makes a contour plot for an SfgRecord.select() return for xdata, ydata and zdata.
    This makes multiple contour plots on top of each other. Normally more then one makes
    no sence but it works this way.
    **Arguments:**
      - **xdata**: Usually pp_delays
      - **ydata**: usually wavenumbers
      - **ydata**: usually bleach
    """
    kwargs.setdefault('extend', 'both')
    num_pp_delays, num_frames, num_spectra, num_pixel = zdata.shape
    for index_spectrum in range(num_spectra):
        for index_frame in range(num_frames):
            zzdata = zdata[:, index_frame, index_spectrum].T
            plt.contourf(xdata, ydata, zzdata, **kwargs)

def multifig(xdata, ydata, fig_axis=0, kwargs_plot=None, titles=None):
    """Create multiple figures for ydata, by using the axis of fig_ax.

    **Argument**:
      - **xdata**: 1D array usually wavenumbers
      - **ydata**: 4D array as usally.

    **kwargs**:
      - **fig_axis**: 0-3 and defines the axis of ydata that will be looped
        over during creation of the figures. Data is then taken from this
        axis per figure. 0  means 1 figure per pp_delay, 1 means 1 figure per
        frame and so on.
      - **kwargs_plot**: kwrges passed to spectrum plot.
      - **titles**: list of titles must have at least same length as number of figures.

    **Returns**:
    list of created figures.
    """
    if not kwargs_plot:
        kwargs_plot = {}

    fig_numbers = ydata.shape[fig_axis]
    figs, axs = [], []

    for i in range(fig_numbers):
        fig, ax = plt.subplots()
        figs.append(fig)
        axs.append(ax)
        try:
            plt.title(titles[i])
        except TypeError:
            pass
        yd = ydata.take([i], fig_axis)
        spectrum(xdata, yd, **kwargs_plot)
    return figs
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright (c) 2017-2019 The Syscoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""An example functional test

The module-level docstring should include a high-level description of
what the test is doing. It's the first thing people see when they open
the file and should give the reader information about *what* the test
is testing and *how* it's being tested
"""
# Imports should be in PEP8 ordering (std library first, then third party
# libraries then local imports).
from collections import defaultdict

# Avoid wildcard * imports
from test_framework.blocktools import (create_block, create_coinbase)
from test_framework.messages import CInv
from test_framework.mininode import (
    P2PInterface,
    mininode_lock,
    msg_block,
    msg_getdata,
)
from test_framework.test_framework import SyscoinTestFramework
from test_framework.util import (
    assert_equal,
    connect_nodes,
    wait_until,
)

# P2PInterface is a class containing callbacks to be executed when a P2P
# message is received from the node-under-test. Subclass P2PInterface and
# override the on_*() methods if you need custom behaviour.
class BaseNode(P2PInterface):
    def __init__(self):
        """Initialize the P2PInterface

        Used to initialize custom properties for the Node that aren't
        included by default in the base class. Be aware that the P2PInterface
        base class already stores a counter for each P2P message type and the
        last received message of each type, which should be sufficient for the
        needs of most tests.

        Call super().__init__() first for standard initialization and then
        initialize custom properties."""
        super().__init__()
        # Stores a dictionary of all blocks received
        self.block_receive_map = defaultdict(int)

    def on_block(self, message):
        """Override the standard on_block callback

        Store the hash of a received block in the dictionary."""
        message.block.calc_sha256()
        self.block_receive_map[message.block.sha256] += 1

    def on_inv(self, message):
        """Override the standard on_inv callback"""
        pass

def custom_function():
    """Do some custom behaviour

    If this function is more generally useful for other tests, consider
    moving it to a module in test_framework."""
    # self.log.info("running custom_function")  # Oops! Can't run self.log outside the SyscoinTestFramework
    pass

class ExampleTest(SyscoinTestFramework):
    # Each functional test is a subclass of the SyscoinTestFramework class.

    # Override the set_test_params(), skip_test_if_missing_module(), add_options(), setup_chain(), setup_network()
    # and setup_nodes() methods to customize the test setup as required.

    def set_test_params(self):
        """Override test parameters for your individual test.

        This method must be overridden and num_nodes must be explicitly set."""
        self.setup_clean_chain = True
        self.num_nodes = 3
        # Use self.extra_args to change command-line arguments for the nodes
        self.extra_args = [[], ["-logips"], []]

        # self.log.info("I've finished set_test_params")  # Oops! Can't run self.log before run_test()

    # Use skip_test_if_missing_module() to skip the test if your test requires certain modules to be present.
    # This test uses generate which requires wallet to be compiled
    def skip_test_if_missing_module(self):
        self.skip_if_no_wallet()

    # Use add_options() to add specific command-line options for your test.
    # In practice this is not used very much, since the tests are mostly written
    # to be run in automated environments without command-line options.
    # def add_options()
    #     pass

    # Use setup_chain() to customize the node data directories. In practice
    # this is not used very much since the default behaviour is almost always
    # fine
    # def setup_chain():
    #     pass

    def setup_network(self):
        """Setup the test network topology

        Often you won't need to override this, since the standard network topology
        (linear: node0 <-> node1 <-> node2 <-> ...) is fine for most tests.

        If you do override this method, remember to start the nodes, assign
        them to self.nodes, connect them and then sync."""

        self.setup_nodes()

        # In this test, we're not connecting node2 to node0 or node1. Calls to
        # sync_all() should not include node2, since we're not expecting it to
        # sync.
        connect_nodes(self.nodes[0], 1)
        self.sync_all(self.nodes[0:2])

    # Use setup_nodes() to customize the node start behaviour (for example if
    # you don't want to start all nodes at the start of the test).
    # def setup_nodes():
    #     pass

    def custom_method(self):
        """Do some custom behaviour for this test

        Define it in a method here because you're going to use it repeatedly.
        If you think it's useful in general, consider moving it to the base
        SyscoinTestFramework class so other tests can use it."""

        self.log.info("Running custom_method")

    def run_test(self):
        """Main test logic"""

        # Create P2P connections will wait for a verack to make sure the connection is fully up
        self.nodes[0].add_p2p_connection(BaseNode())

        # Generating a block on one of the nodes will get us out of IBD
        blocks = [int(self.nodes[0].generate(nblocks=1)[0], 16)]
        self.sync_all(self.nodes[0:2])

        # Notice above how we called an RPC by calling a method with the same
        # name on the node object. Notice also how we used a keyword argument
        # to specify a named RPC argument. Neither of those are defined on the
        # node object. Instead there's some __getattr__() magic going on under
        # the covers to dispatch unrecognised attribute calls to the RPC
        # interface.

        # Logs are nice. Do plenty of them. They can be used in place of comments for
        # breaking the test into sub-sections.
        self.log.info("Starting test!")

        self.log.info("Calling a custom function")
        custom_function()

        self.log.info("Calling a custom method")
        self.custom_method()

        self.log.info("Create some blocks")
        self.tip = int(self.nodes[0].getbestblockhash(), 16)
        self.block_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time'] + 1

        height = self.nodes[0].getblockcount()

        for i in range(10):
            # Use the mininode and blocktools functionality to manually build a block
            # Calling the generate() rpc is easier, but this allows us to exactly
            # control the blocks and transactions.
            block = create_block(self.tip, create_coinbase(height+1), self.block_time)
            block.solve()
            block_message = msg_block(block)
            # Send message is used to send a P2P message to the node over our P2PInterface
            self.nodes[0].p2p.send_message(block_message)
            self.tip = block.sha256
            blocks.append(self.tip)
            self.block_time += 1
            height += 1

        self.log.info("Wait for node1 to reach current tip (height 11) using RPC")
        self.nodes[1].waitforblockheight(11)

        self.log.info("Connect node2 and node1")
        connect_nodes(self.nodes[1], 2)

        self.log.info("Add P2P connection to node2")
        self.nodes[0].disconnect_p2ps()

        self.nodes[2].add_p2p_connection(BaseNode())

        self.log.info("Wait for node2 reach current tip. Test that it has propagated all the blocks to us")

        getdata_request = msg_getdata()
        for block in blocks:
            getdata_request.inv.append(CInv(2, block))
        self.nodes[2].p2p.send_message(getdata_request)

        # wait_until() will loop until a predicate condition is met. Use it to test properties of the
        # P2PInterface objects.
        wait_until(lambda: sorted(blocks) == sorted(list(self.nodes[2].p2p.block_receive_map.keys())), timeout=5, lock=mininode_lock)

        self.log.info("Check that each block was received only once")
        # The network thread uses a global lock on data access to the P2PConnection objects when sending and receiving
        # messages. The test thread should acquire the global lock before accessing any P2PConnection data to avoid locking
        # and synchronization issues. Note wait_until() acquires this global lock when testing the predicate.
        with mininode_lock:
            for block in self.nodes[2].p2p.block_receive_map.values():
                assert_equal(block, 1)

if __name__ == '__main__':
    ExampleTest().main()
<commit_msg>[qa] Fix race condition in example_test.py<commit_after>#!/usr/bin/env python3
# Copyright (c) 2017-2019 The Syscoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""An example functional test

The module-level docstring should include a high-level description of
what the test is doing. It's the first thing people see when they open
the file and should give the reader information about *what* the test
is testing and *how* it's being tested
"""
# Imports should be in PEP8 ordering (std library first, then third party
# libraries then local imports).
from collections import defaultdict

# Avoid wildcard * imports
from test_framework.blocktools import (create_block, create_coinbase)
from test_framework.messages import CInv
from test_framework.mininode import (
    P2PInterface,
    mininode_lock,
    msg_block,
    msg_getdata,
)
from test_framework.test_framework import SyscoinTestFramework
from test_framework.util import (
    assert_equal,
    connect_nodes,
    wait_until,
)

# P2PInterface is a class containing callbacks to be executed when a P2P
# message is received from the node-under-test. Subclass P2PInterface and
# override the on_*() methods if you need custom behaviour.
class BaseNode(P2PInterface):
    def __init__(self):
        """Initialize the P2PInterface

        Used to initialize custom properties for the Node that aren't
        included by default in the base class. Be aware that the P2PInterface
        base class already stores a counter for each P2P message type and the
        last received message of each type, which should be sufficient for the
        needs of most tests.

        Call super().__init__() first for standard initialization and then
        initialize custom properties."""
        super().__init__()
        # Stores a dictionary of all blocks received
        self.block_receive_map = defaultdict(int)

    def on_block(self, message):
        """Override the standard on_block callback

        Store the hash of a received block in the dictionary."""
        message.block.calc_sha256()
        self.block_receive_map[message.block.sha256] += 1

    def on_inv(self, message):
        """Override the standard on_inv callback"""
        pass

def custom_function():
    """Do some custom behaviour

    If this function is more generally useful for other tests, consider
    moving it to a module in test_framework."""
    # self.log.info("running custom_function")  # Oops! Can't run self.log outside the SyscoinTestFramework
    pass

class ExampleTest(SyscoinTestFramework):
    # Each functional test is a subclass of the SyscoinTestFramework class.

    # Override the set_test_params(), skip_test_if_missing_module(), add_options(), setup_chain(), setup_network()
    # and setup_nodes() methods to customize the test setup as required.

    def set_test_params(self):
        """Override test parameters for your individual test.

        This method must be overridden and num_nodes must be explicitly set."""
        self.setup_clean_chain = True
        self.num_nodes = 3
        # Use self.extra_args to change command-line arguments for the nodes
        self.extra_args = [[], ["-logips"], []]

        # self.log.info("I've finished set_test_params")  # Oops! Can't run self.log before run_test()

    # Use skip_test_if_missing_module() to skip the test if your test requires certain modules to be present.
    # This test uses generate which requires wallet to be compiled
    def skip_test_if_missing_module(self):
        self.skip_if_no_wallet()

    # Use add_options() to add specific command-line options for your test.
    # In practice this is not used very much, since the tests are mostly written
    # to be run in automated environments without command-line options.
    # def add_options()
    #     pass

    # Use setup_chain() to customize the node data directories. In practice
    # this is not used very much since the default behaviour is almost always
    # fine
    # def setup_chain():
    #     pass

    def setup_network(self):
        """Setup the test network topology

        Often you won't need to override this, since the standard network topology
        (linear: node0 <-> node1 <-> node2 <-> ...) is fine for most tests.

        If you do override this method, remember to start the nodes, assign
        them to self.nodes, connect them and then sync."""

        self.setup_nodes()

        # In this test, we're not connecting node2 to node0 or node1. Calls to
        # sync_all() should not include node2, since we're not expecting it to
        # sync.
        connect_nodes(self.nodes[0], 1)
        self.sync_all(self.nodes[0:2])

    # Use setup_nodes() to customize the node start behaviour (for example if
    # you don't want to start all nodes at the start of the test).
    # def setup_nodes():
    #     pass

    def custom_method(self):
        """Do some custom behaviour for this test

        Define it in a method here because you're going to use it repeatedly.
        If you think it's useful in general, consider moving it to the base
        SyscoinTestFramework class so other tests can use it."""

        self.log.info("Running custom_method")

    def run_test(self):
        """Main test logic"""

        # Create P2P connections will wait for a verack to make sure the connection is fully up
        self.nodes[0].add_p2p_connection(BaseNode())

        # Generating a block on one of the nodes will get us out of IBD
        blocks = [int(self.nodes[0].generate(nblocks=1)[0], 16)]
        self.sync_all(self.nodes[0:2])

        # Notice above how we called an RPC by calling a method with the same
        # name on the node object. Notice also how we used a keyword argument
        # to specify a named RPC argument. Neither of those are defined on the
        # node object. Instead there's some __getattr__() magic going on under
        # the covers to dispatch unrecognised attribute calls to the RPC
        # interface.

        # Logs are nice. Do plenty of them. They can be used in place of comments for
        # breaking the test into sub-sections.
        self.log.info("Starting test!")

        self.log.info("Calling a custom function")
        custom_function()

        self.log.info("Calling a custom method")
        self.custom_method()

        self.log.info("Create some blocks")
        self.tip = int(self.nodes[0].getbestblockhash(), 16)
        self.block_time = self.nodes[0].getblock(self.nodes[0].getbestblockhash())['time'] + 1

        height = self.nodes[0].getblockcount()

        for i in range(10):
            # Use the mininode and blocktools functionality to manually build a block
            # Calling the generate() rpc is easier, but this allows us to exactly
            # control the blocks and transactions.
            block = create_block(self.tip, create_coinbase(height+1), self.block_time)
            block.solve()
            block_message = msg_block(block)
            # Send message is used to send a P2P message to the node over our P2PInterface
            self.nodes[0].p2p.send_message(block_message)
            self.tip = block.sha256
            blocks.append(self.tip)
            self.block_time += 1
            height += 1

        self.log.info("Wait for node1 to reach current tip (height 11) using RPC")
        self.nodes[1].waitforblockheight(11)

        self.log.info("Connect node2 and node1")
        connect_nodes(self.nodes[1], 2)

        self.log.info("Wait for node2 to receive all the blocks from node1")
        self.sync_all()

        self.log.info("Add P2P connection to node2")
        self.nodes[0].disconnect_p2ps()

        self.nodes[2].add_p2p_connection(BaseNode())

        self.log.info("Test that node2 propagates all the blocks to us")

        getdata_request = msg_getdata()
        for block in blocks:
            getdata_request.inv.append(CInv(2, block))
        self.nodes[2].p2p.send_message(getdata_request)

        # wait_until() will loop until a predicate condition is met. Use it to test properties of the
        # P2PInterface objects.
        wait_until(lambda: sorted(blocks) == sorted(list(self.nodes[2].p2p.block_receive_map.keys())), timeout=5, lock=mininode_lock)

        self.log.info("Check that each block was received only once")
        # The network thread uses a global lock on data access to the P2PConnection objects when sending and receiving
        # messages. The test thread should acquire the global lock before accessing any P2PConnection data to avoid locking
        # and synchronization issues. Note wait_until() acquires this global lock when testing the predicate.
        with mininode_lock:
            for block in self.nodes[2].p2p.block_receive_map.values():
                assert_equal(block, 1)

if __name__ == '__main__':
    ExampleTest().main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright (C) 2016 Red Hat, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import os
import sys
import imp
import json
import logging
import requests
import urlparse
import tempfile
import argparse

from rdopkg.repoman import RepoManager
from rdopkg.helpers import cdir
from rdopkg.utils.cmd import git

from sfrdo import config
from sfrdo import msfutils

logging.basicConfig(filename='warns.log', level=logging.DEBUG)

BL = ['aodh',  # clone on SF fails ... !
      'app-catalog-ui',  # same as aodh :/
      'instack-undercloud',  # upstream 2.1.3 tag (used in spec) does not exits
      'tripleo-common',  # upstream 0.1 tag does not exists (0.1.0)
      'cloudkittyclient',  # distgit project not found on pkg.fedoraproject.org
      'tripleoclient',  # distgit project not found on pkg.fedoraproject.org
      'openstacksdk',  # distgit project not found on pkg.fedoraproject.org
      'dracclient',  # distgit project not found on pkg.fedoraproject.org
      'mistralclient',  # distgit project not found on pkg.fedoraproject.org
      ]

class BranchNotFoundException(Exception):
    pass

def fetch_rdoinfo():
    if not os.path.isdir(config.userdir):
        os.mkdir(config.userdir)
    rm = RepoManager(config.userdir, config.rdoinfo, verbose=True)
    rm.init(force_fetch=True)
    file, path, desc = imp.find_module('rdoinfo', [rm.repo_path])
    rdoinfo = imp.load_module('rdoinfo', file, path, desc)
    return rdoinfo.parse_info_file(os.path.join(config.userdir,
                                                'rdoinfo/rdo.yml'))

def fetch_project_infos(rdoinfo, upstream_project_name):
    select = [pkg for pkg in rdoinfo['packages']
              if pkg['project'] == upstream_project_name]
    if not select:
        raise Exception('Project not found in rdoinfo: %s' %
                        upstream_project_name)
    infos = select[0]

    distgit = infos['distgit']
    # Change scheme from ssh to git (avoid the need of being authenticated)
    # For some projects we need it
    # (eg. client project) still hosted fedora side.
    parts = urlparse.urlparse(distgit)
    distgit = urlparse.urlunparse(['git', parts.netloc,
                                   parts.path, '', '', ''])

    mdistgit = infos['master-distgit']
    mirror = infos['patches']
    upstream = infos['upstream']
    name = infos['project']
    maints = infos['maintainers']
    sfdistgit = "%s-distgit" % name
    conf = infos['conf']
    return (name, distgit, mirror, upstream,
            sfdistgit, maints, conf, mdistgit)

def display_details(cmdargs, rdoinfo, workdir=None):
    name, distgit, mirror, upstream, \
        sfdistgit, maintsi, conf, \
        mdistgit = fetch_project_infos(rdoinfo, cmdargs.name)
    print "=== Details ==="
    print "Project name is: %s" % name
    print "Project type is: %s" % conf
    print "Project upstream RDO distgit is: %s" % distgit
    print "Project upstream RDO master-distgit is: %s" % mdistgit
    print "Project upstream RDO mirror is: %s (won't be used)" % mirror

    print "Project upstream is: %s" % upstream

    print "Project distgit name on SF is: %s" % sfdistgit
    print

def fetch_all_project_type(rdoinfo, t):
    select = [pkg for pkg in rdoinfo['packages']]
    return [p['project'] for p in select if
            'conf' in p and p['conf'] == t]

def create_baseproject(msf, name, desc):
    print "Delete previous %s" % name
    msf.deleteProject(name)
    print "Create %s" % name
    opts = {'description': desc}
    msf.createProject(name, opts)
    print "Add %s to core and ptl group for %s" % (
        config.userlogin, name)
    msf.addUsertoProjectGroups(name, config.useremail,
                               "ptl-group core-group")

def sync_and_push_branch(rfrom, rto, branch, tbranch=None):
    if not tbranch:
        tbranch = branch
    print "sync from %s:%s and push to %s:%s" % (
        rfrom, branch, rto, tbranch)
    if tbranch == 'master':
        git('checkout', tbranch)
    else:
        git('checkout', '-b', tbranch)
    git('reset', '--hard', 'remotes/%s/%s' % (rfrom, branch))
    git('push', '-f', rto, tbranch)

def fetch_flat_patches(name):
    patches = set([f for f in os.listdir('.') if
                   f.endswith('.patch') and f[4] == '-'])
    spec = file([s for s in os.listdir('.') if
                 s.endswith('.spec')][0])
    used_patches = [p.split(':') for p in spec.readlines() if
                    p.startswith('Patch00')]
    used_patches = set([p.lstrip().rstrip('\n') for _, p in used_patches])
    if len(patches) != len(used_patches):
        msg = "(%s) %s flat file patches exists but are not used" % \
            (name, len(patches - used_patches))
        logging.warning(msg)
        print msg
    return used_patches

def fetch_upstream_tag_name():
    spec = file([f for f in os.listdir('.') if f.endswith('.spec')][0])
    version = [l for l in spec.readlines() if l.startswith('Version')]
    version = version[0].split()[1]
    return version

def is_branches_exists(expected_remotes_branches):
    remote_branches = git('branch', '-a').split('\n')
    for remote, branch in expected_remotes_branches:
        exists = False
        for b in remote_branches:
            bn = "remotes/%s/%s" % (remote, branch)
            if bn in b:
                exists = True
        if not exists:
            raise BranchNotFoundException("%s does not exist" % bn)

def import_distgit(msf, sfgerrit, sfdistgit, distgit, mdistgit,
                   conf, workdir):
    print "=== Import distgit ==="
    try:
        create_baseproject(msf, sfdistgit,
                           "\"Packaging content for %s (distgit)\"" %
                           sfdistgit.split('-')[0])
    except msfutils.SFManagerException, e:
        print "Unable to create %s: %s" % (sfdistgit, e)
        sys.exit(1)
    with cdir(workdir):
        git('clone', 'http://%s/r/%s' % (config.rpmfactory, sfdistgit),
            sfdistgit)
    with cdir(os.path.join(workdir, sfdistgit)):
        # Set remotes and fetch objects
        git('remote', 'add', 'gerrit', sfgerrit + sfdistgit)
        git('remote', 'add', 'upstream', distgit)
        git('remote', 'add', 'upstream-mdistgit', mdistgit)
        git('fetch', '--all')

        # Behave correctly according to project type and actual upstream layout
        if conf == 'core':
            is_branches_exists([('upstream', 'rdo-liberty'),
                                ('upstream-mdistgit', 'rpm-master')])
            sync_and_push_branch('upstream', 'gerrit', 'rdo-liberty')
            sync_and_push_branch('upstream-mdistgit', 'gerrit',
                                 'rpm-master', 'master')
        elif conf == 'client':
            is_branches_exists([('upstream', 'master'),
                                ('upstream-mdistgit', 'rpm-master')])
            # Assume master targets liberty atm
            sync_and_push_branch('upstream', 'gerrit', 'master', 'rdo-liberty')
            sync_and_push_branch('upstream-mdistgit', 'gerrit',
                                 'rpm-master', 'master')

def import_mirror(msf, sfgerrit, name, mirror, upstream, workdir):
    print "=== Import mirror ==="
    try:
        create_baseproject(msf, name,
                           "\"Mirror of upstream %s (mirror + patches)\"" %
                           name)
    except msfutils.SFManagerException, e:
        print "Unable to create %s: %s" % (name, e)
        sys.exit(1)
    with cdir(workdir):
        git('clone', 'http://%s/r/%s' % (config.rpmfactory, name),
            name)
    with cdir(os.path.join(workdir, name)):
        # Set remotes and fetch objects
        git('remote', 'add', 'gerrit', sfgerrit + name)
        git('remote', 'add', 'upstream', upstream)
        git('fetch', '--all')

        # Assert expected branches exists
        is_branches_exists([('upstream', 'stable/liberty')])

        # sync and push to rpmfactory
        sync_and_push_branch('upstream', 'gerrit', 'master')
        sync_and_push_branch('upstream', 'gerrit', 'stable/liberty')
        git('push', 'gerrit', '--tags')

def set_patches_on_mirror(msf, sfgerrit, name, mirror, sfdistgit,
                          workdir):
    print "=== Compute and create the patches branch on mirror ==="
    with cdir(os.path.join(workdir, sfdistgit)):
        git('checkout', 'rdo-liberty')
        # Fetch flats file patches
        flat_patches = list(fetch_flat_patches(name))
        print "%s owns %s patches" % (sfdistgit, len(flat_patches))

        # Fetch upstream tag based on the spec file
        version = fetch_upstream_tag_name()
        print "%s packaging is based on tag %s" % (sfdistgit, version)
    with cdir(os.path.join(workdir, name)):
        try:
            is_branches_exists([('mirror', 'liberty-patches')])
        except BranchNotFoundException, e:
            msg = "(%s) Upstream layout warn: %s. but we continue" % (name, e)
            logging.warning(msg)
            print msg

        print "Create %s based on tag %s" % ('liberty-patches', version)
        git('checkout', version)
        git('checkout', '-B', 'liberty-patches')
        git('push', '-f', 'gerrit', 'liberty-patches')

        print "Apply detected patches (%s)" % len(flat_patches)
        flat_patches.sort()
        for n, patch in enumerate(flat_patches):
            print "-> Apply patch : %s" % patch
            git('checkout', '-B', 'p%s' % n)
            git('am', os.path.join(workdir, sfdistgit, patch))
            git('review', '-i', '-y', 'liberty-patches')

def project_import(cmdargs, workdir, rdoinfo):
    print "\n=== Start import ==="
    name, distgit, mirror, upstream, \
        sfdistgit, maints, conf, mdistgit = \
        fetch_project_infos(rdoinfo, cmdargs.name)

    r = requests.get('http://%s/r/projects/?d' % config.rpmfactory)
    projects = json.loads(r.text[4:])

    create = True
    if set([name, sfdistgit]).issubset(set(projects)):
        create = False

    if not cmdargs.force and not create:
        print "Project %s and %s already exists" % (name, sfdistgit)
        return

    if cmdargs.force and not create:
        print "Project %s already exists. But force !" % name

    print "Workdir is: %s" % workdir
    msf = msfutils.ManageSfUtils('http://' + config.rpmfactory,
                                 'admin', config.adminpass)
    sfgerrit = config.gerrit_rpmfactory % config.userlogin

    if not cmdargs.only_patches_branch:
        try:
            import_distgit(msf, sfgerrit,
                           sfdistgit, distgit, mdistgit,
                           conf, workdir)
        except BranchNotFoundException, e:
            msg = "(%s) Unable to find a specific branch to import" % name + \
                "distgit: %s" % e
            logging.warning(msg)
            print msg
            return
        try:
            import_mirror(msf, sfgerrit, name, mirror, upstream, workdir)
        except BranchNotFoundException, e:
            msg = "(%s) Unable to find a specific branch to import" % name + \
                " the mirror repo: %s" % e
            logging.warning(msg)
            print msg
            return

    set_patches_on_mirror(msf, sfgerrit, name, mirror, sfdistgit,
                          workdir)

def project_create(cmdargs, workdir, rdoinfo):
    pass

def project_sync_maints(cmdargs, workdir, rdoinfo):
    pass

def main():
    parser = argparse.ArgumentParser(prog='sfrdo')
    parser.add_argument('--workdir', type=str, help='helper option')

    subparsers = parser.add_subparsers(
        title='commands',
        dest='command',
        help='Available commands help')

    parser_import = subparsers.add_parser(
        'import',
        help='Import an existing RDO project (need admin creds)')
    parser_import.add_argument('--name', type=str, help='project name')
    parser_import.add_argument(
        '--type', type=str, default=None,
        help='Import all project of type (core, client, lib)')
    parser_import.add_argument(
        '--only-patches-branch',
        action='store_true', default=False,
        help='Only act on the patches branch (need a workdir)')
    parser_import.add_argument('--force',
                               action='store_true', default=False,
                               help='Overwrite a project if already exists')

    parser_create = subparsers.add_parser(
        'create',
        help='Create a new project template (need admin creds)')
    parser_create.add_argument('name', type=str, help='upstream project name')
    parser_create.add_argument('gituri', type=str, help='upstream project uri')

    parser_sync_maintainer = subparsers.add_parser(
        'sync_maints',
        help='Sync PTL/CORE group with maintainers (rdoinfo)')
    parser_sync_maintainer.add_argument(
        'name', type=str, help='upstream project name')

    args = parser.parse_args()
    rdoinfo = fetch_rdoinfo()
    if not args.workdir:
        workdir = tempfile.mkdtemp()
    else:
        workdir = args.workdir
    kargs = {'cmdargs': args,
             'workdir': workdir,
             'rdoinfo': rdoinfo}

    if args.command == 'import':
        if args.type:
            projects = fetch_all_project_type(rdoinfo, args.type)
        else:
            projects = [args.name]
        print "Import projects : %s" % ", ".join(projects)
        for project in projects:
            if project in BL:
                print "Skip %s as BL" % project
                continue
            kargs['cmdargs'].name = project
            display_details(**kargs)
            project_import(**kargs)
    elif args.command == 'create':
        project_create(**kargs)
    elif args.command == 'sync_maints':
        project_sync_maints(**kargs)
<commit_msg>Remove unused reference to upstream rdo patches repo (mirror)<commit_after># -*- coding: utf-8 -*-
#
# Copyright (C) 2016 Red Hat, Inc
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import os
import sys
import imp
import json
import logging
import requests
import urlparse
import tempfile
import argparse

from rdopkg.repoman import RepoManager
from rdopkg.helpers import cdir
from rdopkg.utils.cmd import git

from sfrdo import config
from sfrdo import msfutils

logging.basicConfig(filename='warns.log', level=logging.DEBUG)

BL = ['aodh',  # clone on SF fails ... !
      'app-catalog-ui',  # same as aodh :/
      'instack-undercloud',  # upstream 2.1.3 tag (used in spec) does not exits
      'tripleo-common',  # upstream 0.1 tag does not exists (0.1.0)
      'cloudkittyclient',  # distgit project not found on pkg.fedoraproject.org
      'tripleoclient',  # distgit project not found on pkg.fedoraproject.org
      'openstacksdk',  # distgit project not found on pkg.fedoraproject.org
      'dracclient',  # distgit project not found on pkg.fedoraproject.org
      'mistralclient',  # distgit project not found on pkg.fedoraproject.org
      ]

class BranchNotFoundException(Exception):
    pass

def fetch_rdoinfo():
    if not os.path.isdir(config.userdir):
        os.mkdir(config.userdir)
    rm = RepoManager(config.userdir, config.rdoinfo, verbose=True)
    rm.init(force_fetch=True)
    file, path, desc = imp.find_module('rdoinfo', [rm.repo_path])
    rdoinfo = imp.load_module('rdoinfo', file, path, desc)
    return rdoinfo.parse_info_file(os.path.join(config.userdir,
                                                'rdoinfo/rdo.yml'))

def fetch_project_infos(rdoinfo, upstream_project_name):
    select = [pkg for pkg in rdoinfo['packages']
              if pkg['project'] == upstream_project_name]
    if not select:
        raise Exception('Project not found in rdoinfo: %s' %
                        upstream_project_name)
    infos = select[0]

    distgit = infos['distgit']
    # Change scheme from ssh to git (avoid the need of being authenticated)
    # For some projects we need it
    # (eg. client project) still hosted fedora side.
    parts = urlparse.urlparse(distgit)
    distgit = urlparse.urlunparse(['git', parts.netloc,
                                   parts.path, '', '', ''])

    mdistgit = infos['master-distgit']
    upstream = infos['upstream']
    name = infos['project']
    maints = infos['maintainers']
    sfdistgit = "%s-distgit" % name
    conf = infos['conf']
    return (name, distgit, upstream,
            sfdistgit, maints, conf, mdistgit)

def display_details(cmdargs, rdoinfo, workdir=None):
    name, distgit, upstream, \
        sfdistgit, maintsi, conf, \
        mdistgit = fetch_project_infos(rdoinfo, cmdargs.name)
    print "=== Details ==="
    print "Project name is: %s" % name
    print "Project type is: %s" % conf
    print "Project upstream RDO distgit is: %s" % distgit
    print "Project upstream RDO master-distgit is: %s" % mdistgit

    print "Project upstream is: %s" % upstream

    print "Project distgit name on SF is: %s" % sfdistgit
    print

def fetch_all_project_type(rdoinfo, t):
    select = [pkg for pkg in rdoinfo['packages']]
    return [p['project'] for p in select if
            'conf' in p and p['conf'] == t]

def create_baseproject(msf, name, desc):
    print "Delete previous %s" % name
    msf.deleteProject(name)
    print "Create %s" % name
    opts = {'description': desc}
    msf.createProject(name, opts)
    print "Add %s to core and ptl group for %s" % (
        config.userlogin, name)
    msf.addUsertoProjectGroups(name, config.useremail,
                               "ptl-group core-group")

def sync_and_push_branch(rfrom, rto, branch, tbranch=None):
    if not tbranch:
        tbranch = branch
    print "sync from %s:%s and push to %s:%s" % (
        rfrom, branch, rto, tbranch)
    if tbranch == 'master':
        git('checkout', tbranch)
    else:
        git('checkout', '-b', tbranch)
    git('reset', '--hard', 'remotes/%s/%s' % (rfrom, branch))
    git('push', '-f', rto, tbranch)

def fetch_flat_patches(name):
    patches = set([f for f in os.listdir('.') if
                   f.endswith('.patch') and f[4] == '-'])
    spec = file([s for s in os.listdir('.') if
                 s.endswith('.spec')][0])
    used_patches = [p.split(':') for p in spec.readlines() if
                    p.startswith('Patch00')]
    used_patches = set([p.lstrip().rstrip('\n') for _, p in used_patches])
    if len(patches) != len(used_patches):
        msg = "(%s) %s flat file patches exists but are not used" % \
            (name, len(patches - used_patches))
        logging.warning(msg)
        print msg
    return used_patches

def fetch_upstream_tag_name():
    spec = file([f for f in os.listdir('.') if f.endswith('.spec')][0])
    version = [l for l in spec.readlines() if l.startswith('Version')]
    version = version[0].split()[1]
    return version

def is_branches_exists(expected_remotes_branches):
    remote_branches = git('branch', '-a').split('\n')
    for remote, branch in expected_remotes_branches:
        exists = False
        for b in remote_branches:
            bn = "remotes/%s/%s" % (remote, branch)
            if bn in b:
                exists = True
        if not exists:
            raise BranchNotFoundException("%s does not exist" % bn)

def import_distgit(msf, sfgerrit, sfdistgit, distgit, mdistgit,
                   conf, workdir):
    print "=== Import distgit ==="
    try:
        create_baseproject(msf, sfdistgit,
                           "\"Packaging content for %s (distgit)\"" %
                           sfdistgit.split('-')[0])
    except msfutils.SFManagerException, e:
        print "Unable to create %s: %s" % (sfdistgit, e)
        sys.exit(1)
    with cdir(workdir):
        git('clone', 'http://%s/r/%s' % (config.rpmfactory, sfdistgit),
            sfdistgit)
    with cdir(os.path.join(workdir, sfdistgit)):
        # Set remotes and fetch objects
        git('remote', 'add', 'gerrit', sfgerrit + sfdistgit)
        git('remote', 'add', 'upstream', distgit)
        git('remote', 'add', 'upstream-mdistgit', mdistgit)
        git('fetch', '--all')

        # Behave correctly according to project type and actual upstream layout
        if conf == 'core':
            is_branches_exists([('upstream', 'rdo-liberty'),
                                ('upstream-mdistgit', 'rpm-master')])
            sync_and_push_branch('upstream', 'gerrit', 'rdo-liberty')
            sync_and_push_branch('upstream-mdistgit', 'gerrit',
                                 'rpm-master', 'master')
        elif conf == 'client':
            is_branches_exists([('upstream', 'master'),
                                ('upstream-mdistgit', 'rpm-master')])
            # Assume master targets liberty atm
            sync_and_push_branch('upstream', 'gerrit', 'master', 'rdo-liberty')
            sync_and_push_branch('upstream-mdistgit', 'gerrit',
                                 'rpm-master', 'master')

def import_mirror(msf, sfgerrit, name, upstream, workdir):
    print "=== Import mirror ==="
    try:
        create_baseproject(msf, name,
                           "\"Mirror of upstream %s (mirror + patches)\"" %
                           name)
    except msfutils.SFManagerException, e:
        print "Unable to create %s: %s" % (name, e)
        sys.exit(1)
    with cdir(workdir):
        git('clone', 'http://%s/r/%s' % (config.rpmfactory, name),
            name)
    with cdir(os.path.join(workdir, name)):
        # Set remotes and fetch objects
        git('remote', 'add', 'gerrit', sfgerrit + name)
        git('remote', 'add', 'upstream', upstream)
        git('fetch', '--all')

        # Assert expected branches exists
        is_branches_exists([('upstream', 'stable/liberty')])

        # sync and push to rpmfactory
        sync_and_push_branch('upstream', 'gerrit', 'master')
        sync_and_push_branch('upstream', 'gerrit', 'stable/liberty')
        git('push', 'gerrit', '--tags')

def set_patches_on_mirror(msf, sfgerrit, name, sfdistgit,
                          workdir):
    print "=== Compute and create the patches branch on mirror ==="
    with cdir(os.path.join(workdir, sfdistgit)):
        git('checkout', 'rdo-liberty')
        # Fetch flats file patches
        flat_patches = list(fetch_flat_patches(name))
        print "%s owns %s patches" % (sfdistgit, len(flat_patches))

        # Fetch upstream tag based on the spec file
        version = fetch_upstream_tag_name()
        print "%s packaging is based on tag %s" % (sfdistgit, version)
    with cdir(os.path.join(workdir, name)):
        print "Create %s based on tag %s" % ('liberty-patches', version)
        git('checkout', version)
        git('checkout', '-B', 'liberty-patches')
        git('push', '-f', 'gerrit', 'liberty-patches')

        print "Apply detected patches (%s)" % len(flat_patches)
        flat_patches.sort()
        for n, patch in enumerate(flat_patches):
            print "-> Apply patch : %s" % patch
            git('checkout', '-B', 'p%s' % n)
            git('am', os.path.join(workdir, sfdistgit, patch))
            git('review', '-i', '-y', 'liberty-patches')

def project_import(cmdargs, workdir, rdoinfo):
    print "\n=== Start import ==="
    name, distgit, upstream, \
        sfdistgit, maints, conf, mdistgit = \
        fetch_project_infos(rdoinfo, cmdargs.name)

    r = requests.get('http://%s/r/projects/?d' % config.rpmfactory)
    projects = json.loads(r.text[4:])

    create = True
    if set([name, sfdistgit]).issubset(set(projects)):
        create = False

    if not cmdargs.force and not create:
        print "Project %s and %s already exists" % (name, sfdistgit)
        return

    if cmdargs.force and not create:
        print "Project %s already exists. But force !" % name

    print "Workdir is: %s" % workdir
    msf = msfutils.ManageSfUtils('http://' + config.rpmfactory,
                                 'admin', config.adminpass)
    sfgerrit = config.gerrit_rpmfactory % config.userlogin

    if not cmdargs.only_patches_branch:
        try:
            import_distgit(msf, sfgerrit,
                           sfdistgit, distgit, mdistgit,
                           conf, workdir)
        except BranchNotFoundException, e:
            msg = "(%s) Unable to find a specific branch to import" % name + \
                "distgit: %s" % e
            logging.warning(msg)
            print msg
            return
        try:
            import_mirror(msf, sfgerrit, name, upstream, workdir)
        except BranchNotFoundException, e:
            msg = "(%s) Unable to find a specific branch to import" % name + \
                " the mirror repo: %s" % e
            logging.warning(msg)
            print msg
            return

    set_patches_on_mirror(msf, sfgerrit, name, sfdistgit,
                          workdir)

def project_create(cmdargs, workdir, rdoinfo):
    pass

def project_sync_maints(cmdargs, workdir, rdoinfo):
    pass

def main():
    parser = argparse.ArgumentParser(prog='sfrdo')
    parser.add_argument('--workdir', type=str, help='helper option')

    subparsers = parser.add_subparsers(
        title='commands',
        dest='command',
        help='Available commands help')

    parser_import = subparsers.add_parser(
        'import',
        help='Import an existing RDO project (need admin creds)')
    parser_import.add_argument('--name', type=str, help='project name')
    parser_import.add_argument(
        '--type', type=str, default=None,
        help='Import all project of type (core, client, lib)')
    parser_import.add_argument(
        '--only-patches-branch',
        action='store_true', default=False,
        help='Only act on the patches branch (need a workdir)')
    parser_import.add_argument('--force',
                               action='store_true', default=False,
                               help='Overwrite a project if already exists')

    parser_create = subparsers.add_parser(
        'create',
        help='Create a new project template (need admin creds)')
    parser_create.add_argument('name', type=str, help='upstream project name')
    parser_create.add_argument('gituri', type=str, help='upstream project uri')

    parser_sync_maintainer = subparsers.add_parser(
        'sync_maints',
        help='Sync PTL/CORE group with maintainers (rdoinfo)')
    parser_sync_maintainer.add_argument(
        'name', type=str, help='upstream project name')

    args = parser.parse_args()
    rdoinfo = fetch_rdoinfo()
    if not args.workdir:
        workdir = tempfile.mkdtemp()
    else:
        workdir = args.workdir
    kargs = {'cmdargs': args,
             'workdir': workdir,
             'rdoinfo': rdoinfo}

    if args.command == 'import':
        if args.type:
            projects = fetch_all_project_type(rdoinfo, args.type)
        else:
            projects = [args.name]
        print "Import projects : %s" % ", ".join(projects)
        for project in projects:
            if project in BL:
                print "Skip %s as BL" % project
                continue
            kargs['cmdargs'].name = project
            display_details(**kargs)
            project_import(**kargs)
    elif args.command == 'create':
        project_create(**kargs)
    elif args.command == 'sync_maints':
        project_sync_maints(**kargs)
<|endoftext|>