<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function

import os
import re

try:
    from urllib.request import urlopen
except ImportError:
    # urllib.request does not exist in Python 2.7
    from urllib2 import urlopen
from collections import defaultdict

def get_program_parameters():
    import argparse
    description = 'Produce tables of VTK classes used in the examples.'
    epilogue = '''
Typical usage:
   To produce tables of classes used in some_path/VTKExamples/src:
      VTKClassesUsedInExamples.py some_path/VTKExamples/src
   To produce tables of classes used and those not used in some_path/VTKExamples/src:
      VTKClassesUsedInExamples.py some_path/VTKExamples/src -u

   The markdown tables are written to some_path/VTKExamples/src/Coverage

   Note:
      To add links to the VTK class documentation on the web, just add -a as a parameter.
      To vary the number of columns for unused classes to say 8, just add -c8 as a parameter.
'''
    parser = argparse.ArgumentParser(description=description, epilog=epilogue,
                                     formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument('vtk_examples', help='The path to the VTK example source files.')
    parser.add_argument('-c', '--columns',
                        help='Specify the number of columns for unused VTK classes output, default is 5.', nargs='?',
                        const=5, type=int, default=5)
    parser.add_argument('-u', '--unused_vtk',
                        help='Select unused VTK classes output in addition to the used VTK classes output.',
                        action='store_true')
    parser.add_argument('-a', '--add_vtk_html', help='Add html paths to the VTK classes.', action='store_true')

    args = parser.parse_args()
    return args.vtk_examples, args.columns, args.unused_vtk, args.add_vtk_html

def print_table(table, filename):
    """
    The table is printed either to the console or to a file.
    :param table: The formatted data to print or save to a file.
    :param filename: File name to use (optional)
    :return:
    """
    if filename:
        path, ext = os.path.splitext(filename)
        if not ext.lower() in ['.md']:
            ext = '.md'
        fn = path + ext
        with open(fn, 'w') as f:
            f.writelines(table)
            f.close()
    else:
        for row in table:
            print(row)

def path_splitter(path):
    """
    Split a path into its constituent parts.
    Might be better written as a recursive function.
    :param path: The path to split.
    :return: A list of the path's constituent parts.
    """
    res = []
    while True:
        p = os.path.split(path)
        if p[0] == path:
            # Were done, this is an absolute path.
            res.insert(0, p[0])
            break
        elif p[1] == path:
            # Were done, this is a relative path.
            res.insert(0, p[0])
            break
        else:
            path = p[0]
            res.insert(0, p[1])
    return res

class VTKClassesInExamples(object):
    """
    Determine what classes are being used or not used in the examples.
    """

    def __init__(self, base_directory, columns, unused_vtk, add_vtk_html):
        """
        :param base_directory: The path to the VTK Examples sources, usually some_path/VTKExamples/src
        :param columns: When generating the classes not used table, the number of columns to use.
        :param unused_vtk: True if the unused VTK class tables are to be generated.
        :param add_vtk_html: True if the Doxygen documentation paths are to be added to the vtk classes in the tables.
        """
        self.example_types = ['CSharp', 'Cxx', 'Java', 'Python']
        # Classes common to most examples.
        self.excluded_classes = ['vtkActor', 'vtkCamera', 'vtkProperty', 'vtkRenderer', 'vtkRenderWindow', 'vtkRenderWindowInteractor', 'vtkNew']
        # Where to get the list of VTK classes from.
        self.vtk_class_url = 'https://www.vtk.org/doc/nightly/html/annotated.html'
        self.vtk_html_fmt = '[{:s}](http://www.vtk.org/doc/nightly/html/class{:s}.html)'

        self.base_directory = base_directory
        self.columns = columns
        self.unused_vtk = unused_vtk
        self.add_vtk_html = add_vtk_html
        self.vtk_classes = set()
        # A dictionary consisting of [example type][directory name][full file paths of each example ...]
        self.example_file_paths = dict()
        # A dictionary consisting of [example type][vtk class][relative example path]{examples, ...}
        self.classes_used = dict()
        # Markdown tables of classes used keyed on [example type]
        self.classes_used_table = dict()
        # Markdown tables of classes not used keyed on [example type]
        self.classes_not_used_table = dict()

        # Used to signal if the tables have been built, see build_tables()
        self.used_tables_built = False
        self.not_used_tables_built = False

        # Build everything so all the user has to do is to call print_tables()
        self.build_tables()

    def get_vtk_classes_from_html(self):
        """
        Parse the html file, getting a list of the classes.
        """
        vtk_class_pattern = re.compile(r'.*>(vtk[A-Za-z0-9]+)<')
        try:
            f = urlopen(self.vtk_class_url)
            for line in f:
                m = vtk_class_pattern.match(line.decode('utf-8'))
                if m:
                    c = m.group(1)
                    self.vtk_classes.add(c)
            f.close()
        except IOError:
            print('Unable to open the URL: {:s}'.format(self.vtk_class_url))

    def get_example_file_paths(self):
        """
        For each example, get the example file paths.
        """
        for eg in self.example_types:
            # Get the paths to the examples in a particular sub directory e.g Cxx.
            file_paths = defaultdict(list)
            directory = os.path.join(self.base_directory, eg)
            # Does the directory exist?
            if not os.path.isdir(directory):
                raise RuntimeError('Non-existent folder: {:s}'.format(directory))
            exclude_dirs = None
            if eg == 'CSharp':
                fn_pattern = re.compile(r'^[0-9a-zA-Z_\-]+\.cs$')
            elif eg == 'Cxx':
                fn_pattern = re.compile(
                    r'^[0-9a-zA-Z_\-]+\.(hxx|HXX|hpp|HPP|[hH]\+\+|[hH]|cpp|CPP|cxx|CXX|[cC]\+\+|txx|TXX)$')
                exclude_dirs = ['CMakeTechniques', 'Untested']
            elif eg == 'Java':
                fn_pattern = re.compile(r'^[0-9a-zA-Z_\-]+\.java$')
            elif eg == 'Python':
                fn_pattern = re.compile(r'^[0-9a-zA-Z_\-]+\.py$')
            else:
                raise RuntimeError('Unknown example type.')

            # Walk the tree.
            for root, directories, files in os.walk(directory):
                sp = path_splitter(root)
                idx = sp.index(eg)
                key = '/'.join(sp[idx:])
                if exclude_dirs:
                    if sp[idx] in exclude_dirs:
                        continue
                for filename in files:
                    m = fn_pattern.match(filename)
                    if m:
                        file_paths[key].append(os.path.join(root, filename))
            self.example_file_paths[eg] = file_paths

    def vtk_classes_in_examples(self):
        """
        Find the vtk classes used in the examples.
        """
        for eg in self.example_types:
            res = defaultdict(lambda: defaultdict(set))
            if eg == 'CSharp':
                class_pattern = re.compile(r'^[A-Za-z0-9=. <>()\t]+(vtk[A-Za-z0-9]+)')
            elif eg == 'Cxx':
                class_pattern = re.compile(
                    r'^[ \t]*#include[ ]+<(vtk[A-Za-z0-9]+)+.h>$|'  # match: #include <vtkClass.h>
                    r'.*[< ]+(vtk[A-Za-z0-9]+)[> ]|'  # match: <vtkClass>
                    r'.*[= ]+(vtk[A-Za-z0-9]+)[ ]*::New'  # match: vtkClass::New()
                )
            elif eg == 'Java':
                class_pattern = re.compile(r'^[A-Za-z0-9=. \t]+new[ ]+(vtk[A-Za-z0-9]+)[ ]*\(')
            elif eg == 'Python':
                class_pattern = re.compile(r'^[A-Za-z0-9=. ()\t]+(vtk[A-Za-z0-9]+)[ ]*\(')
            else:
                raise RuntimeError('Unknown example type.')

            for k, v in self.example_file_paths[eg].items():
                for fn in v:
                    # Open and read the file building a list of classes.
                    with open(fn) as f:
                        for line in f:
                            m = class_pattern.match(line)
                            if m:
                                c = m.group(m.lastindex)
                                if c in self.vtk_classes:
                                    # Just get the file name
                                    sp = path_splitter(fn)
                                    res[c][k].add(sp[-1])
            self.classes_used[eg] = res

    def make_classes_used_table(self):
        """
        Make a table of classes used for each set of examples.
        """
        if self.add_vtk_html:
            vtk_html_fmt = self.vtk_html_fmt
        else:
            vtk_html_fmt = '{:s}'
        eg_fmt = '[{:s}]({:s})'
        h1 = '# VTK Classes used in the Examples\n'
        h2 = '## {:s}\n'
        th1 = '| VTK Class | Examples |\n'
        th2 = '|--------------|----------------------|\n'
        tr = '| {:s} | {:s} |\n'
        for eg in self.example_types:
            if eg == 'Cxx':
                excl_classes = self.excluded_classes + ['vtkSmartPointer']
            else:
                excl_classes = self.excluded_classes
            res = list()
            res.append(h1)
            res.append(h2.format(eg))
            res.append('Out of {:d} available VTK classes, {:d} are demonstrated here.  \n'.format(
                len(self.vtk_classes), len(self.classes_used[eg])))
            # Excluded classes
            res.append('These classes are excluded since they occur in the majority of the examples:  \n')
            for c in list(sorted(excl_classes, key=lambda x: (x.lower(), x.swapcase()))):
                res.append('- ' + vtk_html_fmt.format(c, c) + '\n')
            res.append('\n')
            res.append(th1)
            res.append(th2)
            vtk_keys = list(sorted(list(self.classes_used[eg].keys()), key=lambda x: (x.lower(), x.swapcase())))
            for vtk_class in vtk_keys:
                if vtk_class not in excl_classes:
                    html_class = vtk_html_fmt.format(vtk_class, vtk_class)
                    paths = self.classes_used[eg][vtk_class]
                    f_list = ''
                    # Here we are assuming no two files have the same name.
                    # tmp will be used to generate a sorted list of files.
                    tmp = dict()
                    for path, fn in paths.items():
                        for f in fn:
                            # Remove the extension.
                            f1 = f[:f.rfind('.')]
                            # NOTE: Need leading '/'
                            tmp[f] = eg_fmt.format(f1, '/' + path + '/' + f1) + ' '
                    tmp_keys = list(sorted(list(tmp.keys()), key=lambda x: (x.lower(), x.swapcase())))
                    for k in tmp_keys:
                        f_list += tmp[k]
                    tmp.clear()
                    res.append(tr.format(html_class, f_list))
            self.classes_used_table[eg] = res

    def make_classes_not_used_table(self):
        """
        Make a table of classes that are not used for each set of examples.
        """
        if self.add_vtk_html:
            vtk_html_fmt = self.vtk_html_fmt
        else:
            vtk_html_fmt = '{:s}'
        h1 = '# VTK Classes not used in the Examples\n'
        h2 = '## {:s}\n'
        h1c = ' VTK Class '
        h2c = '-----------'
        r1c = ' {:s} '
        th1 = '|' + h1c
        th2 = '|' + h2c
        tr = '|' + r1c
        for i in range(1, self.columns):
            th1 += '|' + h1c
            th2 += '|' + h2c
            tr += '|' + r1c
        th1 += '|\n'
        th2 += '|\n'
        tr += '|\n'

        for eg in self.example_types:
            res = list()
            unused_classes = list()
            for vtk_class in self.vtk_classes:
                if vtk_class not in self.classes_used[eg]:
                    unused_classes.append(vtk_html_fmt.format(vtk_class, vtk_class))
            unused_classes.sort(key=lambda x: (x.lower(), x.swapcase()))
            res.append(h1)
            res.append(h2.format(eg))
            res.append('Out of {:d} available VTK classes, {:d} have not been used.  \n'.format(
                len(self.vtk_classes), len(unused_classes)))
            res.append('\n')
            res.append(th1)
            res.append(th2)

            for i in range(len(unused_classes) % self.columns):
                unused_classes.append('')
            idx = 0
            tmp = [''] * self.columns
            for i in range(0, len(unused_classes)):
                tmp[idx] = unused_classes[i]
                idx += 1
                if idx % self.columns == 0:
                    idx = 0
                    res.append(tr.format(*tmp))
                    tmp = [''] * self.columns
            res.append(tr.format(*tmp))
            self.classes_not_used_table[eg] = res

    def build_tables(self):
        self.get_vtk_classes_from_html()
        self.get_example_file_paths()
        self.vtk_classes_in_examples()
        self.make_classes_used_table()
        self.used_tables_built = True
        if self.unused_vtk:
            self.make_classes_not_used_table()
            self.not_used_tables_built = True

    def print_tables(self):
        if not self.used_tables_built:
            raise RuntimeError('The classes used tables have not been built.')
        if self.unused_vtk and not self.not_used_tables_built:
            raise RuntimeError('The classes not used tables have not been built. Enable -u.')
        pth = os.path.join(self.base_directory, 'Coverage')
        for eg in self.example_types:
            fn = os.path.join(pth, eg + 'VTKClassesUsed')
            print_table(self.classes_used_table[eg], fn)
            if self.unused_vtk:
                fn = os.path.join(pth, eg + 'VTKClassesNotUsed')
                print_table(self.classes_not_used_table[eg], fn)

def main():
    example_source, columns, unused_vtk, add_vtk_html = get_program_parameters()
    VTKClassesInExamples(example_source, columns, unused_vtk, add_vtk_html).print_tables()

if __name__ == '__main__':
    main()
<commit_msg>Enhancing classes used<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import print_function

import os
import re

try:
    from urllib.request import urlopen
except ImportError:
    # urllib.request does not exist in Python 2.7
    from urllib2 import urlopen
from collections import defaultdict

def get_program_parameters():
    import argparse
    description = 'Produce tables of VTK classes used in the examples.'
    epilogue = '''
Typical usage:
   To produce tables of classes used in some_path/VTKExamples/src:
      VTKClassesUsedInExamples.py some_path/VTKExamples/src
   To produce tables of classes used and those not used in some_path/VTKExamples/src:
      VTKClassesUsedInExamples.py some_path/VTKExamples/src -u

   The markdown tables are written to some_path/VTKExamples/src/Coverage

   Note:
      To add links to the VTK class documentation on the web, just add -a as a parameter.
      To vary the number of columns for unused classes to say 8, just add -c8 as a parameter.
'''
    parser = argparse.ArgumentParser(description=description, epilog=epilogue,
                                     formatter_class=argparse.RawTextHelpFormatter)
    parser.add_argument('vtk_examples', help='The path to the VTK example source files.')
    parser.add_argument('-c', '--columns',
                        help='Specify the number of columns for unused VTK classes output, default is 5.', nargs='?',
                        const=5, type=int, default=5)
    parser.add_argument('-u', '--unused_vtk',
                        help='Select unused VTK classes output in addition to the used VTK classes output.',
                        action='store_true')
    parser.add_argument('-a', '--add_vtk_html', help='Add html paths to the VTK classes.', action='store_true')

    args = parser.parse_args()
    return args.vtk_examples, args.columns, args.unused_vtk, args.add_vtk_html

def print_table(table, filename):
    """
    The table is printed either to the console or to a file.
    :param table: The formatted data to print or save to a file.
    :param filename: File name to use (optional)
    :return:
    """
    if filename:
        path, ext = os.path.splitext(filename)
        if not ext.lower() in ['.md']:
            ext = '.md'
        fn = path + ext
        with open(fn, 'w') as f:
            f.writelines(table)
            f.close()
    else:
        for row in table:
            print(row)

def path_splitter(path):
    """
    Split a path into its constituent parts.
    Might be better written as a recursive function.
    :param path: The path to split.
    :return: A list of the path's constituent parts.
    """
    res = []
    while True:
        p = os.path.split(path)
        if p[0] == path:
            # Were done, this is an absolute path.
            res.insert(0, p[0])
            break
        elif p[1] == path:
            # Were done, this is a relative path.
            res.insert(0, p[0])
            break
        else:
            path = p[0]
            res.insert(0, p[1])
    return res

class VTKClassesInExamples(object):
    """
    Determine what classes are being used or not used in the examples.
    """

    def __init__(self, base_directory, columns, unused_vtk, add_vtk_html):
        """
        :param base_directory: The path to the VTK Examples sources, usually some_path/VTKExamples/src
        :param columns: When generating the classes not used table, the number of columns to use.
        :param unused_vtk: True if the unused VTK class tables are to be generated.
        :param add_vtk_html: True if the Doxygen documentation paths are to be added to the vtk classes in the tables.
        """
        self.example_types = ['CSharp', 'Cxx', 'Java', 'Python']
        # Classes common to most examples.
        self.excluded_classes = ['vtkActor', 'vtkCamera', 'vtkNamedColors', 'vtkPolyDataMapper', 'vtkProperty',
                                 'vtkRenderer', 'vtkRenderWindow', 'vtkRenderWindowInteractor', ]
        # Where to get the list of VTK classes from.
        self.vtk_class_url = 'https://www.vtk.org/doc/nightly/html/annotated.html'
        # This is the path to the details bookmark in the VTK class.
        self.vtk_html_fmt = '[{:s}](http://www.vtk.org/doc/nightly/html/class{:s}.html#details)'

        self.base_directory = base_directory
        self.columns = columns
        self.unused_vtk = unused_vtk
        self.add_vtk_html = add_vtk_html
        self.vtk_classes = set()
        # A dictionary consisting of [example type][directory name][full file paths of each example ...]
        self.example_file_paths = dict()
        # A dictionary consisting of [example type][vtk class][relative example path]{examples, ...}
        self.classes_used = dict()
        # Markdown tables of classes used keyed on [example type]
        self.classes_used_table = dict()
        # Markdown tables of classes not used keyed on [example type]
        self.classes_not_used_table = dict()

        # Used to signal if the tables have been built, see build_tables()
        self.used_tables_built = False
        self.not_used_tables_built = False

        # Build everything so all the user has to do is to call print_tables()
        self.build_tables()

    def get_vtk_classes_from_html(self):
        """
        Parse the html file, getting a list of the classes.
        """
        # We want the first match, hence the use of ?.
        # Adding a ? on a quantifier (?, * or +) makes it non-greedy.
        vtk_class_pattern = re.compile(r'.*?>(vtk[A-Za-z0-9]+)<')
        try:
            f = urlopen(self.vtk_class_url)
            for line in f:
                m = vtk_class_pattern.match(line.decode('utf-8'))
                if m:
                    c = m.group(1)
                    self.vtk_classes.add(c)
            f.close()
        except IOError:
            print('Unable to open the URL: {:s}'.format(self.vtk_class_url))

    def get_example_file_paths(self):
        """
        For each example, get the example file paths.
        """
        for eg in self.example_types:
            # Get the paths to the examples in a particular sub directory e.g Cxx.
            file_paths = defaultdict(list)
            directory = os.path.join(self.base_directory, eg)
            # Does the directory exist?
            if not os.path.isdir(directory):
                raise RuntimeError('Non-existent folder: {:s}'.format(directory))
            exclude_dirs = None
            if eg == 'CSharp':
                fn_pattern = re.compile(r'^[0-9a-zA-Z_\-]+\.cs$')
            elif eg == 'Cxx':
                fn_pattern = re.compile(
                    r'^[0-9a-zA-Z_\-]+\.(hxx|HXX|hpp|HPP|[hH]\+\+|[hH]|cpp|CPP|cxx|CXX|[cC]\+\+|txx|TXX)$')
                exclude_dirs = ['CMakeTechniques', 'Untested']
            elif eg == 'Java':
                fn_pattern = re.compile(r'^[0-9a-zA-Z_\-]+\.java$')
            elif eg == 'Python':
                fn_pattern = re.compile(r'^[0-9a-zA-Z_\-]+\.py$')
            else:
                raise RuntimeError('Unknown example type.')

            # Walk the tree.
            for root, directories, files in os.walk(directory):
                sp = path_splitter(root)
                idx = sp.index(eg)
                key = '/'.join(sp[idx:])
                if exclude_dirs:
                    if sp[idx] in exclude_dirs:
                        continue
                for filename in files:
                    m = fn_pattern.match(filename)
                    if m:
                        file_paths[key].append(os.path.join(root, filename))
            self.example_file_paths[eg] = file_paths

    def vtk_classes_in_examples(self):
        """
        Find the vtk classes used in the examples.
        """
        for eg in self.example_types:
            res = defaultdict(lambda: defaultdict(set))
            if eg == 'CSharp':
                class_pattern = re.compile(r'^[A-Za-z0-9=. <>()\t]+(vtk[A-Za-z0-9]+)')
            elif eg == 'Cxx':
                class_pattern = re.compile(
                    r'^[ \t]*#include[ ]+<(vtk[A-Za-z0-9]+)+.h>$|'  # match: #include <vtkClass.h>
                    r'.*[< ]+(vtk[A-Za-z0-9]+)[> ]|'  # match: <vtkClass>
                    r'.*[= ]+(vtk[A-Za-z0-9]+)[ ]*::New'  # match: vtkClass::New()
                )
            elif eg == 'Java':
                class_pattern = re.compile(r'^[A-Za-z0-9=. \t]+new[ ]+(vtk[A-Za-z0-9]+)[ ]*\(')
            elif eg == 'Python':
                class_pattern = re.compile(r'^[A-Za-z0-9=. ()\t]+(vtk[A-Za-z0-9]+)[ ]*\(')
            else:
                raise RuntimeError('Unknown example type.')

            for k, v in self.example_file_paths[eg].items():
                for fn in v:
                    # Open and read the file building a list of classes.
                    with open(fn) as f:
                        for line in f:
                            m = class_pattern.match(line)
                            if m:
                                c = m.group(m.lastindex)
                                if c in self.vtk_classes:
                                    # Just get the file name
                                    sp = path_splitter(fn)
                                    res[c][k].add(sp[-1])
            self.classes_used[eg] = res

    def make_classes_used_table(self):
        """
        Make a table of classes used for each set of examples.
        """
        if self.add_vtk_html:
            vtk_html_fmt = self.vtk_html_fmt
        else:
            vtk_html_fmt = '{:s}'
        eg_fmt = '[{:s}]({:s})'
        h1 = '# VTK Classes used in the Examples\n\n'
        h2 = '## {:s}\n\n'
        #  Number of columns in the excluded classes table.
        excl_cols = 6
        exth1 = '| VTK Class | VTK Class | VTK Class | VTK Class | VTK Class | VTK Class |\n'
        exth2 = '|--------------|--------------|--------------|--------------|--------------|--------------|\n'
        extr = '| {:s} | {:s} | {:s} | {:s} | {:s} | {:s} |\n'
        th1 = '| VTK Class | Examples |\n'
        th2 = '|--------------|----------------------|\n'
        tr = '| {:s} | {:s} |\n'
        for eg in self.example_types:
            if eg == 'Cxx':
                excl_classes = self.excluded_classes + ['vtkSmartPointer', 'vtkNew']
            else:
                excl_classes = self.excluded_classes
            res = list()
            res.append(h1)
            res.append(h2.format(eg))
            res.append('Out of {:d} available VTK classes, {:d} are demonstrated here.  \n\n'.format(
                len(self.vtk_classes), len(self.classes_used[eg])))
            # Excluded classes
            res.append('These classes are excluded since they occur in the majority of the examples:  \n')
            res.append(exth1)
            res.append(exth2)
            tmp = []
            for c in list(sorted(excl_classes, key=lambda x: (x.lower(), x.swapcase()))):
                tmp.append(vtk_html_fmt.format(c, c))
                if len(tmp) == excl_cols:
                    res.append(extr.format(*tmp))
                    tmp.clear()
            if tmp:
                while len(tmp) < excl_cols:
                    tmp.append('')
                res.append(extr.format(*tmp))
                # res.append('- ' + vtk_html_fmt.format(c, c) + '\n')
            res.append('\n')
            res.append(th1)
            res.append(th2)
            vtk_keys = list(sorted(list(self.classes_used[eg].keys()), key=lambda x: (x.lower(), x.swapcase())))
            for vtk_class in vtk_keys:
                if vtk_class not in excl_classes:
                    html_class = vtk_html_fmt.format(vtk_class, vtk_class)
                    paths = self.classes_used[eg][vtk_class]
                    f_list = ''
                    # Here we are assuming no two files have the same name.
                    # tmp will be used to generate a sorted list of files.
                    tmp = dict()
                    for path, fn in paths.items():
                        for f in fn:
                            # Remove the extension.
                            f1 = f[:f.rfind('.')]
                            # NOTE: Need leading '/'
                            tmp[f] = eg_fmt.format(f1, '/' + path + '/' + f1) + ' '
                    tmp_keys = list(sorted(list(tmp.keys()), key=lambda x: (x.lower(), x.swapcase())))
                    for k in tmp_keys:
                        f_list += tmp[k]
                    tmp.clear()
                    res.append(tr.format(html_class, f_list))
            self.classes_used_table[eg] = res

    def make_classes_not_used_table(self):
        """
        Make a table of classes that are not used for each set of examples.
        """
        if self.add_vtk_html:
            vtk_html_fmt = self.vtk_html_fmt
        else:
            vtk_html_fmt = '{:s}'
        h1 = '# VTK Classes not used in the Examples\n'
        h2 = '## {:s}\n'
        h1c = ' VTK Class '
        h2c = '-----------'
        r1c = ' {:s} '
        th1 = '|' + h1c
        th2 = '|' + h2c
        tr = '|' + r1c
        for i in range(1, self.columns):
            th1 += '|' + h1c
            th2 += '|' + h2c
            tr += '|' + r1c
        th1 += '|\n'
        th2 += '|\n'
        tr += '|\n'

        for eg in self.example_types:
            res = list()
            unused_classes = list()
            for vtk_class in self.vtk_classes:
                if vtk_class not in self.classes_used[eg]:
                    unused_classes.append(vtk_html_fmt.format(vtk_class, vtk_class))
            unused_classes.sort(key=lambda x: (x.lower(), x.swapcase()))
            res.append(h1)
            res.append(h2.format(eg))
            res.append('Out of {:d} available VTK classes, {:d} have not been used.  \n'.format(
                len(self.vtk_classes), len(unused_classes)))
            res.append('\n')
            res.append(th1)
            res.append(th2)

            for i in range(len(unused_classes) % self.columns):
                unused_classes.append('')
            idx = 0
            tmp = [''] * self.columns
            for i in range(0, len(unused_classes)):
                tmp[idx] = unused_classes[i]
                idx += 1
                if idx % self.columns == 0:
                    idx = 0
                    res.append(tr.format(*tmp))
                    tmp = [''] * self.columns
            res.append(tr.format(*tmp))
            self.classes_not_used_table[eg] = res

    def build_tables(self):
        self.get_vtk_classes_from_html()
        self.get_example_file_paths()
        self.vtk_classes_in_examples()
        self.make_classes_used_table()
        self.used_tables_built = True
        if self.unused_vtk:
            self.make_classes_not_used_table()
            self.not_used_tables_built = True

    def print_tables(self):
        if not self.used_tables_built:
            raise RuntimeError('The classes used tables have not been built.')
        if self.unused_vtk and not self.not_used_tables_built:
            raise RuntimeError('The classes not used tables have not been built. Enable -u.')
        pth = os.path.join(self.base_directory, 'Coverage')
        for eg in self.example_types:
            fn = os.path.join(pth, eg + 'VTKClassesUsed')
            print_table(self.classes_used_table[eg], fn)
            if self.unused_vtk:
                fn = os.path.join(pth, eg + 'VTKClassesNotUsed')
                print_table(self.classes_not_used_table[eg], fn)

def main():
    example_source, columns, unused_vtk, add_vtk_html = get_program_parameters()
    VTKClassesInExamples(example_source, columns, unused_vtk, add_vtk_html).print_tables()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
a simple script can run and test your html rendering classes.

Uncomment the steps as you add to your rendering.

"""

from io import StringIO

# importing the html_rendering code with a short name for easy typing.
import html_render as hr
# reloading in case you are running this in iPython
#  -- we want to make sure the latest version is used
import importlib
importlib.reload(hr)

# writing the file out:
def render_page(page, filename):
    """
    render the tree of elements

    This uses StringIO to render to memory, then dump to console and
    write to file -- very handy!
    """

    f = StringIO()
    page.render(f, "    ")

    f.seek(0)

    print(f.read())

    f.seek(0)
    open(filename, 'w').write(f.read())

# Step 1
#########

page = hr.Element()

page.append("Here is a paragraph of text -- there could be more of them, "
            "but this is enough  to show that we can do some text")

page.append("And here is another piece of text -- you should be able to add"
            " any number")

render_page(page, "test_html_output1.html")

# The rest of the steps have been commented out.
#  Uncomment them a you move along with the assignment.

# ## Step 2
# ##########

page = hr.Html()

body = hr.Body()

body.append(hr.P("Here is a paragraph of text -- there could be more of "
                 "them, but this is enough  to show that we can do some text"))

body.append(hr.P("And here is another piece of text -- you should be able to"
                 " add any number"))

page.append(body)

render_page(page, "test_html_output2.html")

# # Step 3
# ##########

# page = hr.Html()

# head = hr.Head()
# head.append(hr.Title("PythonClass = Revision 1087:"))

# page.append(head)

# body = hr.Body()

# body.append(hr.P("Here is a paragraph of text -- there could be more of them, "
#                  "but this is enough  to show that we can do some text"))
# body.append(hr.P("And here is another piece of text -- you should be able to add any number"))

# page.append(body)

# render_page(page, "test_html_output3.html")

# # Step 4
# ##########

# page = hr.Html()

# head = hr.Head()
# head.append(hr.Title("PythonClass = Revision 1087:"))

# page.append(head)

# body = hr.Body()

# body.append(hr.P("Here is a paragraph of text -- there could be more of them, "
#                  "but this is enough  to show that we can do some text",
#               style="text-align: center; font-style: oblique;"))

# page.append(body)

# render_page(page, "test_html_output4.html")

# # Step 5
# #########

# page = hr.Html()

# head = hr.Head()
# head.append(hr.Title("PythonClass = Revision 1087:"))

# page.append(head)

# body = hr.Body()

# body.append(hr.P("Here is a paragraph of text -- there could be more of them, "
#                  "but this is enough  to show that we can do some text",
#               style="text-align: center; font-style: oblique;"))

# body.append(hr.Hr())

# page.append(body)

# render_page(page, "test_html_output5.html")

# # Step 6
# #########

# page = hr.Html()

# head = hr.Head()
# head.append(hr.Title("PythonClass = Revision 1087:"))

# page.append(head)

# body = hr.Body()

# body.append(hr.P("Here is a paragraph of text -- there could be more of them, "
#                  "but this is enough  to show that we can do some text",
#               style="text-align: center; font-style: oblique;"))

# body.append(hr.Hr())

# body.append("And this is a ")
# body.append( hr.A("http://google.com", "link") )
# body.append("to google")

# page.append(body)

# render_page(page, "test_html_output6.html")

# # Step 7
# #########

# page = hr.Html()

# head = hr.Head()
# head.append(hr.Title("PythonClass = Revision 1087:"))

# page.append(head)

# body = hr.Body()

# body.append( hr.H(2, "PythonClass - Class 6 example") )

# body.append(hr.P("Here is a paragraph of text -- there could be more of them, "
#                  "but this is enough  to show that we can do some text",
#               style="text-align: center; font-style: oblique;"))

# body.append(hr.Hr())

# list = hr.Ul(id="TheList", style="line-height:200%")

# list.append( hr.Li("The first item in a list") )
# list.append( hr.Li("This is the second item", style="color: red") )

# item = hr.Li()
# item.append("And this is a ")
# item.append( hr.A("http://google.com", "link") )
# item.append("to google")

# list.append(item)

# body.append(list)

# page.append(body)

# render_page(page, "test_html_output7.html")

# # Step 8
# ########

# page = hr.Html()

# head = hr.Head()
# head.append( hr.Meta(charset="UTF-8") )
# head.append(hr.Title("PythonClass = Revision 1087:"))

# page.append(head)

# body = hr.Body()

# body.append( hr.H(2, "PythonClass - Class 6 example") )

# body.append(hr.P("Here is a paragraph of text -- there could be more of them, "
#                  "but this is enough  to show that we can do some text",
#                  style="text-align: center; font-style: oblique;"))

# body.append(hr.Hr())

# list = hr.Ul(id="TheList", style="line-height:200%")

# list.append( hr.Li("The first item in a list") )
# list.append( hr.Li("This is the second item", style="color: red") )

# item = hr.Li()
# item.append("And this is a ")
# item.append( hr.A("http://google.com", "link") )
# item.append("to google")

# list.append(item)

# body.append(list)

# page.append(body)

# render_page(page, "test_html_output8.html")
<commit_msg>Commented out step1.<commit_after>#!/usr/bin/env python

"""
a simple script can run and test your html rendering classes.

Uncomment the steps as you add to your rendering.

"""

from io import StringIO

# importing the html_rendering code with a short name for easy typing.
import html_render as hr
# reloading in case you are running this in iPython
#  -- we want to make sure the latest version is used
import importlib
importlib.reload(hr)

# writing the file out:
def render_page(page, filename):
    """
    render the tree of elements

    This uses StringIO to render to memory, then dump to console and
    write to file -- very handy!
    """

    f = StringIO()
    page.render(f, "    ")

    f.seek(0)

    print(f.read())

    f.seek(0)
    open(filename, 'w').write(f.read())

# Step 1
#########

# page = hr.Element()

# page.append("Here is a paragraph of text -- there could be more of them, "
#            "but this is enough  to show that we can do some text")

# page.append("And here is another piece of text -- you should be able to add"
#             " any number")

# render_page(page, "test_html_output1.html")

# The rest of the steps have been commented out.
#  Uncomment them a you move along with the assignment.

# ## Step 2
# ##########

page = hr.Html()

body = hr.Body()

body.append(hr.P("Here is a paragraph of text -- there could be more of "
                 "them, but this is enough  to show that we can do some text"))

body.append(hr.P("And here is another piece of text -- you should be able to"
                 " add any number"))

page.append(body)

render_page(page, "test_html_output2.html")

# # Step 3
# ##########

# page = hr.Html()

# head = hr.Head()
# head.append(hr.Title("PythonClass = Revision 1087:"))

# page.append(head)

# body = hr.Body()

# body.append(hr.P("Here is a paragraph of text -- there could be more of them, "
#                  "but this is enough  to show that we can do some text"))
# body.append(hr.P("And here is another piece of text -- you should be able to add any number"))

# page.append(body)

# render_page(page, "test_html_output3.html")

# # Step 4
# ##########

# page = hr.Html()

# head = hr.Head()
# head.append(hr.Title("PythonClass = Revision 1087:"))

# page.append(head)

# body = hr.Body()

# body.append(hr.P("Here is a paragraph of text -- there could be more of them, "
#                  "but this is enough  to show that we can do some text",
#               style="text-align: center; font-style: oblique;"))

# page.append(body)

# render_page(page, "test_html_output4.html")

# # Step 5
# #########

# page = hr.Html()

# head = hr.Head()
# head.append(hr.Title("PythonClass = Revision 1087:"))

# page.append(head)

# body = hr.Body()

# body.append(hr.P("Here is a paragraph of text -- there could be more of them, "
#                  "but this is enough  to show that we can do some text",
#               style="text-align: center; font-style: oblique;"))

# body.append(hr.Hr())

# page.append(body)

# render_page(page, "test_html_output5.html")

# # Step 6
# #########

# page = hr.Html()

# head = hr.Head()
# head.append(hr.Title("PythonClass = Revision 1087:"))

# page.append(head)

# body = hr.Body()

# body.append(hr.P("Here is a paragraph of text -- there could be more of them, "
#                  "but this is enough  to show that we can do some text",
#               style="text-align: center; font-style: oblique;"))

# body.append(hr.Hr())

# body.append("And this is a ")
# body.append( hr.A("http://google.com", "link") )
# body.append("to google")

# page.append(body)

# render_page(page, "test_html_output6.html")

# # Step 7
# #########

# page = hr.Html()

# head = hr.Head()
# head.append(hr.Title("PythonClass = Revision 1087:"))

# page.append(head)

# body = hr.Body()

# body.append( hr.H(2, "PythonClass - Class 6 example") )

# body.append(hr.P("Here is a paragraph of text -- there could be more of them, "
#                  "but this is enough  to show that we can do some text",
#               style="text-align: center; font-style: oblique;"))

# body.append(hr.Hr())

# list = hr.Ul(id="TheList", style="line-height:200%")

# list.append( hr.Li("The first item in a list") )
# list.append( hr.Li("This is the second item", style="color: red") )

# item = hr.Li()
# item.append("And this is a ")
# item.append( hr.A("http://google.com", "link") )
# item.append("to google")

# list.append(item)

# body.append(list)

# page.append(body)

# render_page(page, "test_html_output7.html")

# # Step 8
# ########

# page = hr.Html()

# head = hr.Head()
# head.append( hr.Meta(charset="UTF-8") )
# head.append(hr.Title("PythonClass = Revision 1087:"))

# page.append(head)

# body = hr.Body()

# body.append( hr.H(2, "PythonClass - Class 6 example") )

# body.append(hr.P("Here is a paragraph of text -- there could be more of them, "
#                  "but this is enough  to show that we can do some text",
#                  style="text-align: center; font-style: oblique;"))

# body.append(hr.Hr())

# list = hr.Ul(id="TheList", style="line-height:200%")

# list.append( hr.Li("The first item in a list") )
# list.append( hr.Li("This is the second item", style="color: red") )

# item = hr.Li()
# item.append("And this is a ")
# item.append( hr.A("http://google.com", "link") )
# item.append("to google")

# list.append(item)

# body.append(list)

# page.append(body)

# render_page(page, "test_html_output8.html")
<|endoftext|>


<commit_before>import pymongo
import stackexchange
import time

so = stackexchange.Site(stackexchange.StackOverflow, "WNe2LOp*hdsbD5U7kp0bhg((")

so.be_inclusive()

connection = pymongo.Connection()
db = connection.stack_doc
posts = db.posts

up_to_date = False

while not up_to_date:
    last_in_database = posts.find_one(sort=[("last_activity", pymongo.DESCENDING)])["last_activity"]
    last_in_database_as_unix = int(time.mktime(last_in_database.timetuple()))
    print "Fetching questions active after %s" % last_in_database_as_unix
    rq = so.recent_questions(from_date=last_in_database_as_unix)
    print rq[0]
<commit_msg>Remove API key (it's for the wrong API version)<commit_after>import pymongo
import stackexchange
import time

so = stackexchange.Site(stackexchange.StackOverflow)

so.be_inclusive()

connection = pymongo.Connection()
db = connection.stack_doc
posts = db.posts

up_to_date = False

while not up_to_date:
    last_in_database = posts.find_one(sort=[("last_activity", pymongo.DESCENDING)])["last_activity"]
    last_in_database_as_unix = int(time.mktime(last_in_database.timetuple()))
    print "Fetching questions active after %s" % last_in_database_as_unix
    rq = so.recent_questions(from_date=last_in_database_as_unix)
    for q in rq:
        print q
        time.sleep(0.01) # Do our own throttling as the built in throttling seems broken
<|endoftext|>


<commit_before>#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4

import os, re, traceback
from rapidsms.webui import settings
from django.template import RequestContext
from django.shortcuts import render_to_response as django_r_to_r
from django.core.paginator import Paginator, EmptyPage, InvalidPage

def render_to_response(req, template_name, dictionary=None, **kwargs):
    """Proxies calls to django.shortcuts.render_to_response, to avoid having
       to include the global variables in every request. This is a giant hack,
       and there's probably a much better solution."""

    rs_dict = {
        "apps":  settings.RAPIDSMS_APPS.values(),
        "debug": settings.DEBUG,
        "javascripts": []
    }

    def __js_dir(fs_path, web_prefix):
        """Adds all of the .js files in a given directory to the javascripts array,
           to be included in the <head>. Also checks for a single file js with the
           same name as the directory. (dir_name/*.js and dir_name.js)"""

        if os.path.exists(fs_path):
            rs_dict["javascripts"].extend([
                "%s/%s" % (web_prefix, fn)
                for fn in os.listdir(fs_path)
                if fn[-3:] == ".js"])

        if os.path.exists("%s.js" % (fs_path)):
            rs_dict["javascripts"].append(
                "%s.js" % (web_prefix))

    # add all of the global javascript files for all running
    # apps. this is super handy for packaging functionality
    # which affects the whole webui without hard-coding it
    for app in rs_dict["apps"]:
        __js_dir(
            "%s/static/javascripts/global" % app["path"],
            "/static/%s/javascripts/global" % app["type"])

    # A NEW KIND OF LUNACY: inspect the stack to find out
    # which rapidsms app this function is being called from
    tb = traceback.extract_stack(limit=2)
    m = re.match(r'^.+/(.+?)/views\.py$', tb[-2][0])
    if m is not None:
        app_type = m.group(1)

        # since we're fetching the app conf, add it to the
        # template dict. it wouldn't be a very good idea to
        # use it, but sometimes, when time is short...
        rs_dict["app_conf"] = settings.RAPIDSMS_APPS[app_type]

        # look up this app in the "apps" list that
        # we've already added, to make the tab (or
        # whatever other nav we're using) as active
        for app in rs_dict["apps"]:
            if app["type"] == app_type:
                app["active"] = True

        # find all of the javascript assets for
        # this app, and add them to the <head>
        __js_dir(
            "%s/static/javascripts/app" % rs_dict["app_conf"]["path"],
            "/static/%s/javascripts/app" % rs_dict["app_conf"]["type"])

        # check for a view-specific javascript,
        # to add LAST, after the dependencies
        view_name = tb[-2][2]
        __js_dir(
            "%s/static/javascripts/page/%s" % (rs_dict["app_conf"]["path"], view_name),
            "/static/%s/javascripts/page/%s.js" % (rs_dict["app_conf"]["type"], view_name))

    # allow the dict argument to
    # be omitted without blowing up
    if dictionary is not None:
        rs_dict.update(dictionary)

    # unless a context instance has been provided,
    # default to RequestContext, to get all of
    # the TEMPLATE_CONTEXT_PROCESSORS working
    if "context_instance" not in kwargs:
        kwargs["context_instance"] = RequestContext(req)

    # pass on the combined dicts to the original function
    return django_r_to_r(template_name, rs_dict, **kwargs)

def paginated(req, query_set, per_page=20, wrapper=None):

    # the per_page argument to this function provides
    # a default, but can be overridden per-request. no
    # interface for this yet, so it's... an easter egg?
    if "per-page" in req.GET:
        try:
            per_page = int(req.GET["per-page"])

        # if it was provided, it must be valid. we don't
        # want links containing extra useless junk like
        # invalid GET parameters floating around
        except ValueError:
            raise ValueError("Invalid per-page parameter: %r" % req.GET["per-page"])

    try:
        paginator = Paginator(query_set, per_page)
        page = int(req.GET.get("page", "1"))
        objects = paginator.page(page)

    # have no mercy if the page parameter is not valid. there
    # should be no links to an invalid page, so coercing it to
    # assume "page=xyz" means "page=1" would just mask bugs
    except (ValueError, EmptyPage, InvalidPage):
        raise ValueError("Invalid Page: %r" % req.GET["page"])

    # if a wrapper function was provided, call it for each
    # object on the page, and replace the list with the result
    if wrapper is not None:
        objects.raw_object_list = objects.object_list
        objects.object_list = map(wrapper, objects.object_list)

    return objects
<commit_msg>pass the active_tab directly to the template, to avoid copying app_conf<commit_after>#!/usr/bin/env python
# vim: ai ts=4 sts=4 et sw=4

import os, re, traceback
from rapidsms.webui import settings
from django.template import RequestContext
from django.shortcuts import render_to_response as django_r_to_r
from django.core.paginator import Paginator, EmptyPage, InvalidPage

def render_to_response(req, template_name, dictionary=None, **kwargs):
    """Proxies calls to django.shortcuts.render_to_response, to avoid having
       to include the global variables in every request. This is a giant hack,
       and there's probably a much better solution."""

    rs_dict = {
        "apps":  settings.RAPIDSMS_APPS.values(),
        "debug": settings.DEBUG,
        "javascripts": []
    }

    def __js_dir(fs_path, web_prefix):
        """Adds all of the .js files in a given directory to the javascripts array,
           to be included in the <head>. Also checks for a single file js with the
           same name as the directory. (dir_name/*.js and dir_name.js)"""

        if os.path.exists(fs_path):
            rs_dict["javascripts"].extend([
                "%s/%s" % (web_prefix, fn)
                for fn in os.listdir(fs_path)
                if fn[-3:] == ".js"])

        if os.path.exists("%s.js" % (fs_path)):
            rs_dict["javascripts"].append(
                "%s.js" % (web_prefix))

    # add all of the global javascript files for all running
    # apps. this is super handy for packaging functionality
    # which affects the whole webui without hard-coding it
    for app in rs_dict["apps"]:
        __js_dir(
            "%s/static/javascripts/global" % app["path"],
            "/static/%s/javascripts/global" % app["type"])

    # A NEW KIND OF LUNACY: inspect the stack to find out
    # which rapidsms app this function is being called from
    tb = traceback.extract_stack(limit=2)
    m = re.match(r'^.+/(.+?)/views\.py$', tb[-2][0])
    if m is not None:
        app_type = m.group(1)

        # since we're fetching the app conf, add it to the
        # template dict. it wouldn't be a very good idea to
        # use it, but sometimes, when time is short...
        rs_dict["app_conf"] = settings.RAPIDSMS_APPS[app_type]

        # note which app this func was called from, so the tmpl
        # can mark the tab (or some other type of nav) as active
        rs_dict["active_tab"] = rs_dict["app_conf"]["type"]

        # find all of the javascript assets for
        # this app, and add them to the <head>
        __js_dir(
            "%s/static/javascripts/app" % rs_dict["app_conf"]["path"],
            "/static/%s/javascripts/app" % rs_dict["app_conf"]["type"])

        # check for a view-specific javascript,
        # to add LAST, after the dependencies
        view_name = tb[-2][2]
        __js_dir(
            "%s/static/javascripts/page/%s" % (rs_dict["app_conf"]["path"], view_name),
            "/static/%s/javascripts/page/%s.js" % (rs_dict["app_conf"]["type"], view_name))

    # allow the dict argument to
    # be omitted without blowing up
    if dictionary is not None:
        rs_dict.update(dictionary)

    # unless a context instance has been provided,
    # default to RequestContext, to get all of
    # the TEMPLATE_CONTEXT_PROCESSORS working
    if "context_instance" not in kwargs:
        kwargs["context_instance"] = RequestContext(req)

    # pass on the combined dicts to the original function
    return django_r_to_r(template_name, rs_dict, **kwargs)

def paginated(req, query_set, per_page=20, wrapper=None):

    # the per_page argument to this function provides
    # a default, but can be overridden per-request. no
    # interface for this yet, so it's... an easter egg?
    if "per-page" in req.GET:
        try:
            per_page = int(req.GET["per-page"])

        # if it was provided, it must be valid. we don't
        # want links containing extra useless junk like
        # invalid GET parameters floating around
        except ValueError:
            raise ValueError("Invalid per-page parameter: %r" % req.GET["per-page"])

    try:
        paginator = Paginator(query_set, per_page)
        page = int(req.GET.get("page", "1"))
        objects = paginator.page(page)

    # have no mercy if the page parameter is not valid. there
    # should be no links to an invalid page, so coercing it to
    # assume "page=xyz" means "page=1" would just mask bugs
    except (ValueError, EmptyPage, InvalidPage):
        raise ValueError("Invalid Page: %r" % req.GET["page"])

    # if a wrapper function was provided, call it for each
    # object on the page, and replace the list with the result
    if wrapper is not None:
        objects.raw_object_list = objects.object_list
        objects.object_list = map(wrapper, objects.object_list)

    return objects
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Copyright (c) 2021 The Bitcoin ABC developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Utilities for avalanche tests."""

import struct
from typing import Any, Dict, List, Optional

from .authproxy import JSONRPCException
from .key import ECKey
from .messages import (
    MSG_BLOCK,
    NODE_AVALANCHE,
    NODE_NETWORK,
    AvalancheDelegation,
    AvalancheResponse,
    CInv,
    CTransaction,
    FromHex,
    LegacyAvalancheProof,
    TCPAvalancheResponse,
    ToHex,
    hash256,
    msg_avahello,
    msg_avapoll,
    msg_tcpavaresponse,
)
from .p2p import P2PInterface, p2p_lock
from .test_node import TestNode
from .util import assert_equal, satoshi_round, wait_until_helper
from .wallet_util import bytes_to_wif

def create_coinbase_stakes(
        node: TestNode,
        blockhashes: List[str],
        priv_key: str,
        amount: Optional[str] = None) -> List[Dict[str, Any]]:
    """Returns a list of dictionaries representing stakes, in a format
    compatible with the buildavalancheproof RPC, using only coinbase
    transactions.

    :param node: Test node used to get the block and coinbase data.
    :param blockhashes: List of block hashes, whose coinbase tx will be used
        as a stake.
    :param priv_key: Private key controlling the coinbase UTXO
    :param amount: If specified, this overwrites the amount information
        in the coinbase dicts.
    """
    blocks = [node.getblock(h, 2) for h in blockhashes]
    coinbases = [
        {
            'height': b['height'],
            'txid': b['tx'][0]['txid'],
            'n': 0,
            'value': b['tx'][0]['vout'][0]['value'],
        } for b in blocks
    ]

    return [{
        'txid': coinbase['txid'],
        'vout': coinbase['n'],
        'amount': amount or coinbase['value'],
        'height': coinbase['height'],
        'iscoinbase': True,
        'privatekey': priv_key,
    } for coinbase in coinbases]

def get_utxos_in_blocks(node: TestNode, blockhashes: List[str]) -> List[Dict]:
    """Return all UTXOs in the specified list of blocks.
    """
    utxos = filter(
        lambda u: node.gettransaction(u["txid"])["blockhash"] in blockhashes,
        node.listunspent())
    return list(utxos)

def create_stakes(
        node: TestNode, blockhashes: List[str], count: int
) -> List[Dict[str, Any]]:
    """
    Create a list of stakes by splitting existing UTXOs from a specified list
    of blocks into 10 new coins.

    This function can generate more valid stakes than `get_coinbase_stakes`
    does, because on the regtest chain halving happens every 150 blocks so
    the coinbase amount is below the dust threshold after only 900 blocks.

    :param node: Test node used to generate blocks and send transactions
    :param blockhashes: List of block hashes whose UTXOs will be split.
    :param count: Number of stakes to return.
    """
    assert 10 * len(blockhashes) >= count
    utxos = get_utxos_in_blocks(node, blockhashes)

    addresses = [node.getnewaddress() for _ in range(10)]
    private_keys = {addr: node.dumpprivkey(addr) for addr in addresses}

    for u in utxos:
        inputs = [{"txid": u["txid"], "vout": u["vout"]}]
        outputs = {
            addr: satoshi_round(u['amount'] / 10) for addr in addresses}
        raw_tx = node.createrawtransaction(inputs, outputs)
        ctx = FromHex(CTransaction(), raw_tx)
        ctx.vout[0].nValue -= node.calculate_fee(ctx)
        signed_tx = node.signrawtransactionwithwallet(ToHex(ctx))["hex"]
        node.sendrawtransaction(signed_tx)

    # confirm the transactions
    new_blocks = []
    while node.getmempoolinfo()['size'] > 0:
        new_blocks += node.generate(1)

    utxos = get_utxos_in_blocks(node, new_blocks)
    stakes = []
    # cache block heights
    heights = {}
    for utxo in utxos[:count]:
        blockhash = node.gettransaction(utxo["txid"])["blockhash"]
        if blockhash not in heights:
            heights[blockhash] = node.getblock(blockhash, 1)["height"]
        stakes.append({
            'txid': utxo['txid'],
            'vout': utxo['vout'],
            'amount': utxo['amount'],
            'iscoinbase': utxo['label'] == "coinbase",
            'height': heights[blockhash],
            'privatekey': private_keys[utxo["address"]],
        })

    return stakes

def get_proof_ids(node):
    return [FromHex(LegacyAvalancheProof(), peer['proof']
                    ).proofid for peer in node.getavalanchepeerinfo()]

def wait_for_proof(node, proofid_hex, timeout=60, expect_orphan=None):
    """
    Wait for the proof to be known by the node. If expect_orphan is set, the
    proof should match the orphan state, otherwise it's a don't care parameter.
    """
    def proof_found():
        try:
            wait_for_proof.is_orphan = node.getrawavalancheproof(proofid_hex)[
                "orphan"]
            return True
        except JSONRPCException:
            return False
    wait_until_helper(proof_found, timeout=timeout)

    if expect_orphan is not None:
        assert_equal(expect_orphan, wait_for_proof.is_orphan)

class AvaP2PInterface(P2PInterface):
    """P2PInterface with avalanche capabilities"""

    def __init__(self):
        self.round = 0
        self.avahello = None
        self.avaresponses = []
        self.avapolls = []
        self.nodeid: Optional[int] = None
        super().__init__()

    def peer_connect(self, *args, **kwargs):
        create_conn = super().peer_connect(*args, **kwargs)

        # Save the nonce and extra entropy so they can be reused later.
        self.local_nonce = self.on_connection_send_msg.nNonce
        self.local_extra_entropy = self.on_connection_send_msg.nExtraEntropy

        return create_conn

    def on_version(self, message):
        super().on_version(message)

        # Save the nonce and extra entropy so they can be reused later.
        self.remote_nonce = message.nNonce
        self.remote_extra_entropy = message.nExtraEntropy

    def on_avaresponse(self, message):
        self.avaresponses.append(message.response)

    def on_avapoll(self, message):
        self.avapolls.append(message.poll)

    def on_avahello(self, message):
        assert(self.avahello is None)
        self.avahello = message

    def send_avaresponse(self, round, votes, privkey):
        response = AvalancheResponse(round, 0, votes)
        sig = privkey.sign_schnorr(response.get_hash())
        msg = msg_tcpavaresponse()
        msg.response = TCPAvalancheResponse(response, sig)
        self.send_message(msg)

    def wait_for_avaresponse(self, timeout=5):
        self.wait_until(
            lambda: len(self.avaresponses) > 0,
            timeout=timeout)

        with p2p_lock:
            return self.avaresponses.pop(0)

    def send_poll(self, hashes, type=MSG_BLOCK):
        msg = msg_avapoll()
        msg.poll.round = self.round
        self.round += 1
        for h in hashes:
            msg.poll.invs.append(CInv(type, h))
        self.send_message(msg)

    def get_avapoll_if_available(self):
        with p2p_lock:
            return self.avapolls.pop(0) if len(self.avapolls) > 0 else None

    def wait_for_avahello(self, timeout=5):
        self.wait_until(
            lambda: self.avahello is not None,
            timeout=timeout)

        with p2p_lock:
            return self.avahello

    def send_avahello(self, delegation_hex: str, delegated_privkey: ECKey):
        delegation = FromHex(AvalancheDelegation(), delegation_hex)
        local_sighash = hash256(
            delegation.getid() +
            struct.pack("<QQQQ", self.local_nonce, self.remote_nonce,
                        self.local_extra_entropy, self.remote_extra_entropy))

        msg = msg_avahello()
        msg.hello.delegation = delegation
        msg.hello.sig = delegated_privkey.sign_schnorr(local_sighash)
        self.send_message(msg)

        return delegation.proofid

def get_ava_p2p_interface(
        node: TestNode,
        services=NODE_NETWORK | NODE_AVALANCHE) -> AvaP2PInterface:
    """Build and return an AvaP2PInterface connected to the specified
    TestNode.
    """
    n = AvaP2PInterface()
    node.add_p2p_connection(
        n, services=services)
    n.wait_for_verack()
    n.nodeid = node.getpeerinfo()[-1]['id']

    return n

def gen_proof(node, coinbase_utxos=1):
    blockhashes = node.generate(coinbase_utxos)

    privkey = ECKey()
    privkey.generate()

    stakes = create_coinbase_stakes(
        node, blockhashes, node.get_deterministic_priv_key().key)
    proof_hex = node.buildavalancheproof(
        42, 2000000000, bytes_to_wif(privkey.get_bytes()), stakes)

    return privkey, FromHex(LegacyAvalancheProof(), proof_hex)
<commit_msg>[avalanche] Add send_proof test helper.<commit_after>#!/usr/bin/env python3
# Copyright (c) 2021 The Bitcoin ABC developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
"""Utilities for avalanche tests."""

import struct
from typing import Any, Dict, List, Optional

from .authproxy import JSONRPCException
from .key import ECKey
from .messages import (
    MSG_BLOCK,
    NODE_AVALANCHE,
    NODE_NETWORK,
    AvalancheDelegation,
    AvalancheResponse,
    CInv,
    CTransaction,
    FromHex,
    LegacyAvalancheProof,
    TCPAvalancheResponse,
    ToHex,
    hash256,
    msg_avahello,
    msg_avapoll,
    msg_avaproof,
    msg_tcpavaresponse,
)
from .p2p import P2PInterface, p2p_lock
from .test_node import TestNode
from .util import assert_equal, satoshi_round, wait_until_helper
from .wallet_util import bytes_to_wif

def create_coinbase_stakes(
        node: TestNode,
        blockhashes: List[str],
        priv_key: str,
        amount: Optional[str] = None) -> List[Dict[str, Any]]:
    """Returns a list of dictionaries representing stakes, in a format
    compatible with the buildavalancheproof RPC, using only coinbase
    transactions.

    :param node: Test node used to get the block and coinbase data.
    :param blockhashes: List of block hashes, whose coinbase tx will be used
        as a stake.
    :param priv_key: Private key controlling the coinbase UTXO
    :param amount: If specified, this overwrites the amount information
        in the coinbase dicts.
    """
    blocks = [node.getblock(h, 2) for h in blockhashes]
    coinbases = [
        {
            'height': b['height'],
            'txid': b['tx'][0]['txid'],
            'n': 0,
            'value': b['tx'][0]['vout'][0]['value'],
        } for b in blocks
    ]

    return [{
        'txid': coinbase['txid'],
        'vout': coinbase['n'],
        'amount': amount or coinbase['value'],
        'height': coinbase['height'],
        'iscoinbase': True,
        'privatekey': priv_key,
    } for coinbase in coinbases]

def get_utxos_in_blocks(node: TestNode, blockhashes: List[str]) -> List[Dict]:
    """Return all UTXOs in the specified list of blocks.
    """
    utxos = filter(
        lambda u: node.gettransaction(u["txid"])["blockhash"] in blockhashes,
        node.listunspent())
    return list(utxos)

def create_stakes(
        node: TestNode, blockhashes: List[str], count: int
) -> List[Dict[str, Any]]:
    """
    Create a list of stakes by splitting existing UTXOs from a specified list
    of blocks into 10 new coins.

    This function can generate more valid stakes than `get_coinbase_stakes`
    does, because on the regtest chain halving happens every 150 blocks so
    the coinbase amount is below the dust threshold after only 900 blocks.

    :param node: Test node used to generate blocks and send transactions
    :param blockhashes: List of block hashes whose UTXOs will be split.
    :param count: Number of stakes to return.
    """
    assert 10 * len(blockhashes) >= count
    utxos = get_utxos_in_blocks(node, blockhashes)

    addresses = [node.getnewaddress() for _ in range(10)]
    private_keys = {addr: node.dumpprivkey(addr) for addr in addresses}

    for u in utxos:
        inputs = [{"txid": u["txid"], "vout": u["vout"]}]
        outputs = {
            addr: satoshi_round(u['amount'] / 10) for addr in addresses}
        raw_tx = node.createrawtransaction(inputs, outputs)
        ctx = FromHex(CTransaction(), raw_tx)
        ctx.vout[0].nValue -= node.calculate_fee(ctx)
        signed_tx = node.signrawtransactionwithwallet(ToHex(ctx))["hex"]
        node.sendrawtransaction(signed_tx)

    # confirm the transactions
    new_blocks = []
    while node.getmempoolinfo()['size'] > 0:
        new_blocks += node.generate(1)

    utxos = get_utxos_in_blocks(node, new_blocks)
    stakes = []
    # cache block heights
    heights = {}
    for utxo in utxos[:count]:
        blockhash = node.gettransaction(utxo["txid"])["blockhash"]
        if blockhash not in heights:
            heights[blockhash] = node.getblock(blockhash, 1)["height"]
        stakes.append({
            'txid': utxo['txid'],
            'vout': utxo['vout'],
            'amount': utxo['amount'],
            'iscoinbase': utxo['label'] == "coinbase",
            'height': heights[blockhash],
            'privatekey': private_keys[utxo["address"]],
        })

    return stakes

def get_proof_ids(node):
    return [FromHex(LegacyAvalancheProof(), peer['proof']
                    ).proofid for peer in node.getavalanchepeerinfo()]

def wait_for_proof(node, proofid_hex, timeout=60, expect_orphan=None):
    """
    Wait for the proof to be known by the node. If expect_orphan is set, the
    proof should match the orphan state, otherwise it's a don't care parameter.
    """
    def proof_found():
        try:
            wait_for_proof.is_orphan = node.getrawavalancheproof(proofid_hex)[
                "orphan"]
            return True
        except JSONRPCException:
            return False
    wait_until_helper(proof_found, timeout=timeout)

    if expect_orphan is not None:
        assert_equal(expect_orphan, wait_for_proof.is_orphan)

class AvaP2PInterface(P2PInterface):
    """P2PInterface with avalanche capabilities"""

    def __init__(self):
        self.round = 0
        self.avahello = None
        self.avaresponses = []
        self.avapolls = []
        self.nodeid: Optional[int] = None
        super().__init__()

    def peer_connect(self, *args, **kwargs):
        create_conn = super().peer_connect(*args, **kwargs)

        # Save the nonce and extra entropy so they can be reused later.
        self.local_nonce = self.on_connection_send_msg.nNonce
        self.local_extra_entropy = self.on_connection_send_msg.nExtraEntropy

        return create_conn

    def on_version(self, message):
        super().on_version(message)

        # Save the nonce and extra entropy so they can be reused later.
        self.remote_nonce = message.nNonce
        self.remote_extra_entropy = message.nExtraEntropy

    def on_avaresponse(self, message):
        self.avaresponses.append(message.response)

    def on_avapoll(self, message):
        self.avapolls.append(message.poll)

    def on_avahello(self, message):
        assert(self.avahello is None)
        self.avahello = message

    def send_avaresponse(self, round, votes, privkey):
        response = AvalancheResponse(round, 0, votes)
        sig = privkey.sign_schnorr(response.get_hash())
        msg = msg_tcpavaresponse()
        msg.response = TCPAvalancheResponse(response, sig)
        self.send_message(msg)

    def wait_for_avaresponse(self, timeout=5):
        self.wait_until(
            lambda: len(self.avaresponses) > 0,
            timeout=timeout)

        with p2p_lock:
            return self.avaresponses.pop(0)

    def send_poll(self, hashes, type=MSG_BLOCK):
        msg = msg_avapoll()
        msg.poll.round = self.round
        self.round += 1
        for h in hashes:
            msg.poll.invs.append(CInv(type, h))
        self.send_message(msg)

    def send_proof(self, proof):
        msg = msg_avaproof()
        msg.proof = proof
        self.send_message(msg)

    def get_avapoll_if_available(self):
        with p2p_lock:
            return self.avapolls.pop(0) if len(self.avapolls) > 0 else None

    def wait_for_avahello(self, timeout=5):
        self.wait_until(
            lambda: self.avahello is not None,
            timeout=timeout)

        with p2p_lock:
            return self.avahello

    def send_avahello(self, delegation_hex: str, delegated_privkey: ECKey):
        delegation = FromHex(AvalancheDelegation(), delegation_hex)
        local_sighash = hash256(
            delegation.getid() +
            struct.pack("<QQQQ", self.local_nonce, self.remote_nonce,
                        self.local_extra_entropy, self.remote_extra_entropy))

        msg = msg_avahello()
        msg.hello.delegation = delegation
        msg.hello.sig = delegated_privkey.sign_schnorr(local_sighash)
        self.send_message(msg)

        return delegation.proofid

def get_ava_p2p_interface(
        node: TestNode,
        services=NODE_NETWORK | NODE_AVALANCHE) -> AvaP2PInterface:
    """Build and return an AvaP2PInterface connected to the specified
    TestNode.
    """
    n = AvaP2PInterface()
    node.add_p2p_connection(
        n, services=services)
    n.wait_for_verack()
    n.nodeid = node.getpeerinfo()[-1]['id']

    return n

def gen_proof(node, coinbase_utxos=1):
    blockhashes = node.generate(coinbase_utxos)

    privkey = ECKey()
    privkey.generate()

    stakes = create_coinbase_stakes(
        node, blockhashes, node.get_deterministic_priv_key().key)
    proof_hex = node.buildavalancheproof(
        42, 2000000000, bytes_to_wif(privkey.get_bytes()), stakes)

    return privkey, FromHex(LegacyAvalancheProof(), proof_hex)
<|endoftext|>


<commit_before>import os
import sys
import numpy
import pickle
import zipfile
import urllib2
import subprocess
from matplotlib import pyplot

def pickleload(filename):
    """
    Does what it says.  Loads a pickle (and returns it)
    """
    with open(filename, 'r') as f:
        results_dict = pickle.load(f)

    return results_dict

def compile_dict_results(files):
    """
    Compiles a dict of lists of like results given a list of
    dictionaries containing results

    Inputs
    ------
        files: list of strings
            strings of filenames of input dicts

    Returns
    -------
        compiled_results: dict of lists
            dictionary where keyname is same as input dicts but corresponding
            value is a list containing all results that have the same key
    """
    compiled_results = {}
    for filename in files:
        results_dict = pickleload(filename)
        for k, v in results_dict.items():
            if k in compiled_results.keys():
                compiled_results[k].append(v)
            else:
                compiled_results[k] = [v]

    return compiled_results

def richardson_extrapolation(compiled_results):
    """
    Performs an estimate of the exact solution using
    Richardson extrapolation, given by

    f_ex = (f_1 * f_3 - f_2^2) / (f_3 - 2*f_2+f_1)

    where f_1 is a result from the finest grid and f_3 is from the coarsest.
    The grids f_1, f_2, f_3 should have the same refinement ratio (e.g. 2 -> 4 -> 8)
    """

    esolv = compiled_results['E_solv_kJ']
    f1 = esolv[5] # assuming 6 runs: 1, 2, 4, 8, 12, 16
    f2 = esolv[3]
    f3 = esolv[2]

    return (f1 * f3 - f2**2) / (f3 - 2 * f2 + f1)

def generate_plot(compiled_results, filetype='pdf'):
    """
    Generates a plot with some hard-coded info based on APBS runs
    """
    res = compiled_results
    N_fem = numpy.array([97*65*97, 129*97*129, 161*129*161, 257*161*257, 385*257*385, 449*385*449,513*449*513])
    Vfem = 50.*40.*50.
    Lfem = (Vfem/N_fem)**(1/3.)
    Lfem_aux = numpy.array([[0.521,0.625,0.521],[0.391,0.417,0.391],
                            [0.312,0.312,0.312],[0.195,0.250,0.195],
                            [0.130,0.156,0.130],[0.112,0.104,0.112],
                            [0.098,0.089,0.098]])
    timeAPB = numpy.array([4.3,8.6,17.7,54,161,352,768])
    EsolvAPB = numpy.array([-2237,-2172.9,-2142,-2121.5,-2102.6,-2093.7,-2090.7])
    pyg_ext = -2082.5
    apb_ext = -2070.47

    font = {'family':'serif', 'size':7}
    pyplot.figure(figsize=(3, 2), dpi=80)
    pyplot.rc('font', **font)

    #calc plot extremas for plotting
    xmax = N_fem[-1]*5
    xmin = res['total_elements'][0] / 1.5

    pyplot.semilogx(res['total_elements'], res['E_solv_kJ'],
                    c='k', marker='o', mfc='w', ms=3, ls='-', lw=0.5, label='PyGBe')
    pyplot.semilogx(N_fem, EsolvAPB,
                    c='k', marker='^', mfc='w', ms=3, ls='-', lw=0.5, label='APBS')
    pyplot.semilogx([xmin,xmax],pyg_ext*numpy.ones(2), c='k', marker='', mfc='w', ms=1, ls='dotted',lw=0.2)
    pyplot.semilogx([xmin,xmax],apb_ext*numpy.ones(2), c='k', marker='', mfc='w', ms=1, ls='dotted',lw=0.2)

    pyplot.ylabel('$\Delta G_{solv}$ [kJ/mol]', fontsize=10)
    pyplot.xlabel('N',fontsize=10)
    pyplot.text(5e5, -2101,'PyGBe extrap.',fontsize=6,rotation=0)
    pyplot.text(1e7, -2067,'APBS extrap.',fontsize=6,rotation=0)
    pyplot.subplots_adjust(left=0.22, bottom=0.21, right=0.96, top=0.95)
    pyplot.axis([xmin,xmax,-2450,-2040])
    pyplot.legend(loc='lower right')
    fname = 'Esolv_lys.{}'.format(filetype)
    print('Writing figure to "{}"'.format(fname))
    pyplot.savefig(fname)

    pygbe_err = numpy.abs(numpy.array(res['E_solv_kJ']) - pyg_ext) / numpy.abs(pyg_ext)
    apb_err = numpy.abs(EsolvAPB - apb_ext) / numpy.abs(apb_ext)

    pyplot.figure(figsize=(3, 2), dpi=80)
    pyplot.loglog(pygbe_err, res['total_time'],
                  c='k', marker='o', mfc='w', ms=5,
                  ls='-',lw=0.5, label='PyGBe')
    pyplot.loglog(apb_err, timeAPB,
                  c='k', marker='^', mfc='w', ms=5,
                  ls='-',lw=0.5, label='APBS')
    pyplot.subplots_adjust(left=0.19, bottom=0.21, right=0.96, top=0.95)
    pyplot.ylabel('Time to solution [s]',fontsize=10)
    pyplot.xlabel('Error',fontsize=10)
    pyplot.legend(loc='lower left')
    fname = 'time_lys.{}'.format(filetype)
    print('Writing figure to "{}"'.format(fname))
    pyplot.savefig(fname)

    pyplot.figure(figsize=(3,2), dpi=80)
    pyplot.loglog(res['total_elements'], res['total_time'],
                  c='k', marker='o', mfc='w', ms=5,
                  ls='-',lw=0.5)
    pyplot.xlabel('Number of elements', fontsize=10)
    pyplot.ylabel('Time to solution [s]', fontsize=10)
    pyplot.subplots_adjust(left=0.19, bottom=0.21, right=0.96, top=0.95)
    fname = 'time_v_N_lys.{}'.format(filetype)
    print('Writing figure to "{}"'.format(fname))
    pyplot.savefig(fname)

def check_mesh():
    """
    Check if there is a geometry folder already present in the current location.
    If not, download the mesh files from Zenodo.
    """
    if not os.path.isdir('geometry'):
        dl_check = raw_input('The meshes for the performance check don\'t appear '
                         'to be loaded. Would you like to download them from '
                             'Zenodo? (~11MB) (y/n): ')
        if dl_check == 'y':
            mesh_file = 'https://zenodo.org/record/58308/files/lysozyme_meshes.zip'
            download_zip_with_progress_bar(mesh_file)
            unzip(mesh_file.split('/')[-1])

            print('Done!')

def download_zip_with_progress_bar(url):
    file_name = url.split('/')[-1]
    u = urllib2.urlopen(url)
    f = open(file_name, 'wb')
    meta = u.info()
    file_size = int(meta.getheaders("Content-Length")[0])
    print "Downloading: %s Bytes: %s" % (file_name, file_size)

    file_size_dl = 0
    block_sz = 8192
    while True:
        buffer = u.read(block_sz)
        if not buffer:
            break

        file_size_dl += len(buffer)
        f.write(buffer)
        status = r"%10d  [%3.2f%%]" % (file_size_dl, file_size_dl * 100. / file_size)
        status = status + chr(8)*(len(status)+1)
        print status,

    f.close()

def unzip(meshzip):
    with zipfile.ZipFile(meshzip, 'r') as myzip:
        myzip.extractall(path='.')

    print('Unzipping meshes ...')
    print('Removing zip file...')
    os.remove(meshzip)

def run_check():
    """
    Use subprocess to run through the 6 cases to generate results
    """
    try:
        os.mkdir('output')
    except OSError:
        pass
    if [a for a in os.listdir('output') if 'pickle' in a]:
        run_check_yn = raw_input('\n\n\n'
                              'There are already results in your output directory.  '
                              'Do you want to re-run the tests?  If you select "no" '
                              'then the plotting routine will still run.  If you select '
                              '"yes", note that this script is not smart enough to '
                              'distinguish between "old" and "new" runs and will just '
                              'jam everything together into one figure (yes/no): ')

        if run_check_yn in ['No', 'no', 'n']:
            return
        elif run_check_yn in ['Yes', 'yes', 'y']:
            run_lysozome()
        else:
            print('Didn\'t understand your response, exiting')
            sys.exit()

    else:
        run_lysozome()

def run_lysozome():
    conf_files = ['lys.config', 'lys2.config', 'lys4.config',
                'lys8.config', 'lys12.config', 'lys16.config']
    for conf in conf_files:
        subprocess.call(['pygbe', '-c', conf, '-p', 'lys.param', '.'])

def main():
    run_yn = raw_input('This will run 6 lysozyme cases in order to generate '
                       'results necessary to generate a few figures. It '
                       'takes around 10 minutes to run on a Tesla K40 '
                       'and also time to download meshes from Zenodo (~11MB).  '
                       'Type "y" or some variant of yes to accept this: ')

    if run_yn in ['Yes', 'yes', 'y', 'Y']:
        #check that meshes are present
        check_mesh()
        #run the lysozome problems
        run_check()
        files = [os.path.join('output', a)
                 for a in os.listdir('output') if 'pickle' in a]
        compiled_results = compile_dict_results(files)
        generate_plot(compiled_results, filetype='png')

if __name__ == '__main__':
    main()
<commit_msg>sort scraped files and calculate extrp on the fly<commit_after>import os
import sys
import numpy
import pickle
import zipfile
import urllib2
import subprocess
from matplotlib import pyplot

def pickleload(filename):
    """
    Does what it says.  Loads a pickle (and returns it)
    """
    with open(filename, 'r') as f:
        results_dict = pickle.load(f)

    return results_dict

def compile_dict_results(files):
    """
    Compiles a dict of lists of like results given a list of
    dictionaries containing results

    Inputs
    ------
        files: list of strings
            strings of filenames of input dicts

    Returns
    -------
        compiled_results: dict of lists
            dictionary where keyname is same as input dicts but corresponding
            value is a list containing all results that have the same key
    """
    compiled_results = {}
    for filename in files:
        results_dict = pickleload(filename)
        for k, v in results_dict.items():
            if k in compiled_results.keys():
                compiled_results[k].append(v)
            else:
                compiled_results[k] = [v]

    return compiled_results

def richardson_extrapolation(compiled_results):
    """
    Performs an estimate of the exact solution using
    Richardson extrapolation, given by

    f_ex = (f_1 * f_3 - f_2^2) / (f_3 - 2*f_2+f_1)

    where f_1 is a result from the finest grid and f_3 is from the coarsest.
    The grids f_1, f_2, f_3 should have the same refinement ratio (e.g. 2 -> 4 -> 8)
    """

    esolv = compiled_results['E_solv_kJ']
    f1 = esolv[5] # assuming 6 runs: 1, 2, 4, 8, 12, 16
    f2 = esolv[3]
    f3 = esolv[2]

    return (f1 * f3 - f2**2) / (f3 - 2 * f2 + f1)

def generate_plot(compiled_results, filetype='pdf'):
    """
    Generates a plot with some hard-coded info based on APBS runs
    """
    res = compiled_results
    N_fem = numpy.array([97*65*97, 129*97*129, 161*129*161, 257*161*257, 385*257*385, 449*385*449,513*449*513])
    Vfem = 50.*40.*50.
    Lfem = (Vfem/N_fem)**(1/3.)
    Lfem_aux = numpy.array([[0.521,0.625,0.521],[0.391,0.417,0.391],
                            [0.312,0.312,0.312],[0.195,0.250,0.195],
                            [0.130,0.156,0.130],[0.112,0.104,0.112],
                            [0.098,0.089,0.098]])
    timeAPB = numpy.array([4.3,8.6,17.7,54,161,352,768])
    EsolvAPB = numpy.array([-2237,-2172.9,-2142,-2121.5,-2102.6,-2093.7,-2090.7])
    apb_ext = -2070.47

    pyg_ext = richardson_extrapolation(res)

    font = {'family':'serif', 'size':7}
    pyplot.figure(figsize=(3, 2), dpi=80)
    pyplot.rc('font', **font)

    #calc plot extremas for plotting
    xmax = N_fem[-1]*5
    xmin = res['total_elements'][0] / 1.5

    pyplot.semilogx(res['total_elements'], res['E_solv_kJ'],
                    c='k', marker='o', mfc='w', ms=3, ls='-', lw=0.5, label='PyGBe')
    pyplot.semilogx(N_fem, EsolvAPB,
                    c='k', marker='^', mfc='w', ms=3, ls='-', lw=0.5, label='APBS')
    pyplot.semilogx([xmin,xmax],pyg_ext*numpy.ones(2), c='k', marker='', mfc='w', ms=1, ls='dotted',lw=0.2)
    pyplot.semilogx([xmin,xmax],apb_ext*numpy.ones(2), c='k', marker='', mfc='w', ms=1, ls='dotted',lw=0.2)

    pyplot.ylabel('$\Delta G_{solv}$ [kJ/mol]', fontsize=10)
    pyplot.xlabel('N',fontsize=10)
    pyplot.text(5e5, pyg_ext - 25,'PyGBe extrap.',fontsize=6,rotation=0)
    pyplot.text(1e7, -2067,'APBS extrap.',fontsize=6,rotation=0)
    pyplot.subplots_adjust(left=0.22, bottom=0.21, right=0.96, top=0.95)
    pyplot.axis([xmin,xmax,-2450,-2040])
    pyplot.legend(loc='lower right')
    fname = 'Esolv_lys.{}'.format(filetype)
    print('Writing figure to "{}"'.format(fname))
    pyplot.savefig(fname)

    pygbe_err = numpy.abs(numpy.array(res['E_solv_kJ']) - pyg_ext) / numpy.abs(pyg_ext)
    apb_err = numpy.abs(EsolvAPB - apb_ext) / numpy.abs(apb_ext)

    pyplot.figure(figsize=(3, 2), dpi=80)
    pyplot.loglog(pygbe_err, res['total_time'],
                  c='k', marker='o', mfc='w', ms=5,
                  ls='-',lw=0.5, label='PyGBe')
    pyplot.loglog(apb_err, timeAPB,
                  c='k', marker='^', mfc='w', ms=5,
                  ls='-',lw=0.5, label='APBS')
    pyplot.subplots_adjust(left=0.19, bottom=0.21, right=0.96, top=0.95)
    pyplot.ylabel('Time to solution [s]',fontsize=10)
    pyplot.xlabel('Error',fontsize=10)
    pyplot.legend(loc='lower left')
    fname = 'time_lys.{}'.format(filetype)
    print('Writing figure to "{}"'.format(fname))
    pyplot.savefig(fname)

    pyplot.figure(figsize=(3,2), dpi=80)
    pyplot.loglog(res['total_elements'], res['total_time'],
                  c='k', marker='o', mfc='w', ms=5,
                  ls='-',lw=0.5)
    pyplot.xlabel('Number of elements', fontsize=10)
    pyplot.ylabel('Time to solution [s]', fontsize=10)
    pyplot.subplots_adjust(left=0.19, bottom=0.21, right=0.96, top=0.95)
    fname = 'time_v_N_lys.{}'.format(filetype)
    print('Writing figure to "{}"'.format(fname))
    pyplot.savefig(fname)

def check_mesh():
    """
    Check if there is a geometry folder already present in the current location.
    If not, download the mesh files from Zenodo.
    """
    if not os.path.isdir('geometry'):
        dl_check = raw_input('The meshes for the performance check don\'t appear '
                         'to be loaded. Would you like to download them from '
                             'Zenodo? (~11MB) (y/n): ')
        if dl_check == 'y':
            mesh_file = 'https://zenodo.org/record/58308/files/lysozyme_meshes.zip'
            download_zip_with_progress_bar(mesh_file)
            unzip(mesh_file.split('/')[-1])

            print('Done!')

def download_zip_with_progress_bar(url):
    file_name = url.split('/')[-1]
    u = urllib2.urlopen(url)
    f = open(file_name, 'wb')
    meta = u.info()
    file_size = int(meta.getheaders("Content-Length")[0])
    print "Downloading: %s Bytes: %s" % (file_name, file_size)

    file_size_dl = 0
    block_sz = 8192
    while True:
        buffer = u.read(block_sz)
        if not buffer:
            break

        file_size_dl += len(buffer)
        f.write(buffer)
        status = r"%10d  [%3.2f%%]" % (file_size_dl, file_size_dl * 100. / file_size)
        status = status + chr(8)*(len(status)+1)
        print status,

    f.close()

def unzip(meshzip):
    with zipfile.ZipFile(meshzip, 'r') as myzip:
        myzip.extractall(path='.')

    print('Unzipping meshes ...')
    print('Removing zip file...')
    os.remove(meshzip)

def run_check():
    """
    Use subprocess to run through the 6 cases to generate results
    """
    try:
        os.mkdir('output')
    except OSError:
        pass
    if [a for a in os.listdir('output') if 'pickle' in a]:
        run_check_yn = raw_input('\n\n\n'
                              'There are already results in your output directory.  '
                              'Do you want to re-run the tests?  If you select "no" '
                              'then the plotting routine will still run.  If you select '
                              '"yes", note that this script is not smart enough to '
                              'distinguish between "old" and "new" runs and will just '
                              'jam everything together into one figure (yes/no): ')

        if run_check_yn in ['No', 'no', 'n']:
            return
        elif run_check_yn in ['Yes', 'yes', 'y']:
            run_lysozome()
        else:
            print('Didn\'t understand your response, exiting')
            sys.exit()

    else:
        run_lysozome()

def run_lysozome():
    conf_files = ['lys.config', 'lys2.config', 'lys4.config',
                'lys8.config', 'lys12.config', 'lys16.config']
    for conf in conf_files:
        subprocess.call(['pygbe', '-c', conf, '-p', 'lys.param', '.'])

def main():
    run_yn = raw_input('This will run 6 lysozyme cases in order to generate '
                       'results necessary to generate a few figures. It '
                       'takes around 10 minutes to run on a Tesla K40 '
                       'and also time to download meshes from Zenodo (~11MB).  '
                       'Type "y" or some variant of yes to accept this: ')

    if run_yn in ['Yes', 'yes', 'y', 'Y']:
        #check that meshes are present
        check_mesh()
        #run the lysozome problems
        run_check()
        files = [os.path.join('output', a)
                 for a in os.listdir('output') if 'pickle' in a]
        files.sort()
        compiled_results = compile_dict_results(files)
        generate_plot(compiled_results, filetype='pdf')
        return compiled_results

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import multiprocessing
import rs4
from concurrent.futures import TimeoutError, CancelledError
from rs4.logger import screen_logger
import time
from ..corequest.tasks import Mask
from skitai import was
from aquests.athreads import trigger
import sys

N_CPU = multiprocessing.cpu_count()

class Future:
    def __init__ (self, future, name):
        self.future = future
        self._name = name
        self._started = time.time ()
        self._was = None
        self._fulfilled = None

    def __str__ (self):
        return self._name

    def __getattr__ (self, name):
        return getattr (self.future, name)

    def _settle (self, future):
        expt, result = future.exception (0), None
        if self._fulfilled:
            if not expt:
                result = future.result (0)
            task = Mask (result, expt)
            try:
                self._fulfilled (self._was, task)
            except:
                self._was.traceback ()
                trigger.wakeup (lambda p = self._was.response, d = self._was.app.debug and sys.exc_info () or None: (p.error (500, "Internal Server Error", d), p.done ()) )
            else:
                trigger.wakeup (lambda p = self._was.response: (p.done (),))
            self._fulfilled = None

    def kill (self):
        try: self.future.result (timeout = 0)
        except: pass
        self.future.set_exception (TimeoutError)

    def cancel (self):
        try: self.future.cancel ()
        except: pass
        self.future.set_exception (CancelledError)

    def returning (self, returning):
        return returning

    def then (self, func):
        self._fulfilled = func
        try: self._was = was._clone (True)
        except TypeError: pass
        self.future.add_done_callback (self._settle)
        return self

class Executor:
    MAINTERN_INTERVAL = 30
    def __init__ (self, executor_class, workers = None, zombie_timeout = None, logger = None):
        self._executor_class = executor_class
        self._name = self._executor_class.__name__
        self.logger = logger
        self.workers = workers or N_CPU
        self.zombie_timeout = zombie_timeout
        self.last_maintern = time.time ()
        self.lock = multiprocessing.Lock ()
        self.executor = None
        self.futures = []

    def __len__ (self):
        with self.lock:
            return len (self.futures)

    def create_excutor (self):
        self.executor = self._executor_class (self.workers)

    def maintern (self, now):
        inprogresses = []
        for future in self.futures:
            if future.done ():
                continue
            if self.zombie_timeout and future._started + self.zombie_timeout < now:
                future.kill ()
                self.logger ("zombie {} task is killed: {}".format (self._name, future))
            else:
                inprogresses.append (future)
                continue
        self.futures = inprogresses

    def shutdown (self):
        if not self.executor:
            return
        with self.lock:
            for future in self.futures:
                if future.done ():
                    continue
                if not future.running ():
                    future.cancel ()
                    self.logger ("{} task is canceled: {}".format (self._name, future))
                future.kill ()
                self.logger ("{} task is killed: {}".format (self._name, future))

        self.executor.shutdown (wait = True)
        with self.lock:
            self.maintern (time.time ())
            self.executor = None
            return len (self.futures)

    def __call__ (self, f, *a, **b):
        with self.lock:
            if self.executor is None:
                self.create_excutor ()
            else:
                now = time.time ()
                if now > self.last_maintern + self.MAINTERN_INTERVAL:
                    self.maintern (time.time ())
        future = self.executor.submit (f, *a, **b)
        wrap = Future (future, "{}.{}".format (f.__module__, f.__name__))
        self.logger ("{} task started: {}".format (self._name, wrap))
        with self.lock:
            self.futures.append (wrap)
        return wrap

class Executors:
    def __init__ (self, workers = N_CPU, zombie_timeout = None, logger = None):
        self.logger = logger or screen_logger ()
        self.executors = [
            Executor (rs4.threading, workers, zombie_timeout, self.logger),
            Executor (rs4.processing, workers, zombie_timeout, self.logger)
        ]

    def status (self):
        return dict (
            threads = len (self.executors [0]),
            processes = len (self.executors [1])
        )

    def cleanup (self):
        return [e.shutdown () for e in self.executors]

    def create_thread (self, f, *a, **b):
        return self.executors [0] (f, *a, **b)

    def create_process (self, f, *a, **b):
        return self.executors [1] (f, *a, **b)

        <commit_msg>add Thread.fetch ()<commit_after>import multiprocessing
import rs4
from concurrent.futures import TimeoutError, CancelledError
from rs4.logger import screen_logger
import time
from ..corequest.tasks import Mask
from skitai import was
from aquests.athreads import trigger
import sys

N_CPU = multiprocessing.cpu_count()

class Future:
    def __init__ (self, future, name):
        self.future = future
        self._name = name
        self._started = time.time ()
        self._was = None
        self._fulfilled = None

    def __str__ (self):
        return self._name

    def __getattr__ (self, name):
        return getattr (self.future, name)

    def _settle (self, future):
        expt, result = future.exception (0), None
        if self._fulfilled:
            if not expt:
                result = future.result (0)
            task = Mask (result, expt)
            try:
                self._fulfilled (self._was, task)
            except:
                self._was.traceback ()
                trigger.wakeup (lambda p = self._was.response, d = self._was.app.debug and sys.exc_info () or None: (p.error (500, "Internal Server Error", d), p.done ()) )
            else:
                trigger.wakeup (lambda p = self._was.response: (p.done (),))
            self._fulfilled = None

    def fetch (self, timeout = 10):
        expt = self.future.exception (timeout)
        if expt:
            raise expt
        return self.future.result (0)

    def kill (self):
        try: self.future.result (timeout = 0)
        except: pass
        self.future.set_exception (TimeoutError)

    def cancel (self):
        try: self.future.cancel ()
        except: pass
        self.future.set_exception (CancelledError)

    def returning (self, returning):
        return returning

    def then (self, func):
        self._fulfilled = func
        try: self._was = was._clone (True)
        except TypeError: pass
        self.future.add_done_callback (self._settle)
        return self

class Executor:
    MAINTERN_INTERVAL = 30
    def __init__ (self, executor_class, workers = None, zombie_timeout = None, logger = None):
        self._executor_class = executor_class
        self._name = self._executor_class.__name__
        self.logger = logger
        self.workers = workers or N_CPU
        self.zombie_timeout = zombie_timeout
        self.last_maintern = time.time ()
        self.lock = multiprocessing.Lock ()
        self.executor = None
        self.futures = []

    def __len__ (self):
        with self.lock:
            return len (self.futures)

    def create_excutor (self):
        self.executor = self._executor_class (self.workers)

    def maintern (self, now):
        inprogresses = []
        for future in self.futures:
            if future.done ():
                continue
            if self.zombie_timeout and future._started + self.zombie_timeout < now:
                future.kill ()
                self.logger ("zombie {} task is killed: {}".format (self._name, future))
            else:
                inprogresses.append (future)
                continue
        self.futures = inprogresses

    def shutdown (self):
        if not self.executor:
            return
        with self.lock:
            for future in self.futures:
                if future.done ():
                    continue
                if not future.running ():
                    future.cancel ()
                    self.logger ("{} task is canceled: {}".format (self._name, future))
                future.kill ()
                self.logger ("{} task is killed: {}".format (self._name, future))

        self.executor.shutdown (wait = True)
        with self.lock:
            self.maintern (time.time ())
            self.executor = None
            return len (self.futures)

    def __call__ (self, f, *a, **b):
        with self.lock:
            if self.executor is None:
                self.create_excutor ()
            else:
                now = time.time ()
                if now > self.last_maintern + self.MAINTERN_INTERVAL:
                    self.maintern (time.time ())
        future = self.executor.submit (f, *a, **b)
        wrap = Future (future, "{}.{}".format (f.__module__, f.__name__))
        self.logger ("{} task started: {}".format (self._name, wrap))
        with self.lock:
            self.futures.append (wrap)
        return wrap

class Executors:
    def __init__ (self, workers = N_CPU, zombie_timeout = None, logger = None):
        self.logger = logger or screen_logger ()
        self.executors = [
            Executor (rs4.threading, workers, zombie_timeout, self.logger),
            Executor (rs4.processing, workers, zombie_timeout, self.logger)
        ]

    def status (self):
        return dict (
            threads = len (self.executors [0]),
            processes = len (self.executors [1])
        )

    def cleanup (self):
        return [e.shutdown () for e in self.executors]

    def create_thread (self, f, *a, **b):
        return self.executors [0] (f, *a, **b)

    def create_process (self, f, *a, **b):
        return self.executors [1] (f, *a, **b)

        <|endoftext|>