<commit_before>#!/usr/bin/env python

# Copyright (C) 2013-2014 Steven Myint
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""Checks code blocks in reStructuredText."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import argparse
import io
import json
import locale
import os
import re
import subprocess
import sys
import tempfile

from docutils import core, nodes, utils, writers
from docutils.parsers import rst

__version__ = '0.5'

def check(source, filename='<string>', report_level=1, ignore=None,
          warning_stream=sys.stderr):
    """Yield errors.

    Use lower report_level for noisier error output.

    Each yielded error is a tuple of the form:

        (line_number, message)

    Line numbers are indexed at 1 and are with respect to the full RST file.

    Each code block is checked asynchronously in a subprocess.

    """
    writer = CheckWriter(source, filename, ignore=ignore)

    core.publish_string(source, writer=writer,
                        source_path=filename,
                        settings_overrides={'report_level': report_level,
                                            'warning_stream': warning_stream})

    for checker in writer.checkers:
        for error in checker():
            yield error

def check_file(filename, report_level=1, ignore=None):
    """Yield errors."""
    if filename == '-':
        contents = sys.stdin.read()
    else:
        with open(filename) as input_file:
            contents = input_file.read()

    for error in check(contents,
                       filename=filename,
                       report_level=report_level,
                       ignore=ignore):
        yield error

def check_python(code):
    """Yield errors."""
    try:
        compile(code, '<string>', 'exec')
    except SyntaxError as exception:
        yield (int(exception.lineno), exception.msg)

def check_json(code):
    """Yield errors."""
    try:
        json.loads(code)
    except ValueError as exception:
        message = str(exception)
        line_number = 0

        found = re.search(r': line\s+([0-9]+)[^:]*$', message)
        if found:
            line_number = int(found.group(1))

        yield (int(line_number), message)

def check_rst(code, ignore):
    """Yield errors in nested RST code."""
    string_io = io.StringIO()
    filename = '<string>'

    for result in check(code,
                        filename=filename,
                        ignore=ignore,
                        warning_stream=string_io):
        yield result

    rst_errors = string_io.getvalue().strip()
    if rst_errors:
        for message in rst_errors.splitlines():
            try:
                yield parse_gcc_style_error_message(message,
                                                    filename=filename,
                                                    has_column=False)
            except ValueError:
                continue

class CodeBlockDirective(rst.Directive):

    """Code block directive."""

    has_content = True
    optional_arguments = 1

    def run(self):
        """Run directive."""
        try:
            language = self.arguments[0]
        except IndexError:
            language = ''
        code = '\n'.join(self.content)
        literal = nodes.literal_block(code, code)
        literal['classes'].append('code-block')
        literal['language'] = language
        return [literal]

rst.directives.register_directive('code-block', CodeBlockDirective)
rst.directives.register_directive('sourcecode', CodeBlockDirective)

def bash_checker(code):
    """Return checker."""
    run = run_in_subprocess(code, '.bash', ['bash', '-n'])

    def run_check():
        """Yield errors."""
        result = run()
        if result:
            (output, filename) = result
            prefix = filename + ': line '
            for line in output.splitlines():
                if not line.startswith(prefix):
                    continue
                message = line[len(prefix):]
                split_message = message.split(':', 1)
                yield (int(split_message[0]) - 1,
                       split_message[1].strip())
    return run_check

def c_checker(code):
    """Return checker."""
    return gcc_checker(code, '.c', [os.getenv('CC', 'gcc'), '-std=c99'])

def cpp_checker(code):
    """Return checker."""
    return gcc_checker(code, '.cpp', [os.getenv('CXX', 'g++'), '-std=c++0x'])

def gcc_checker(code, filename_suffix, arguments):
    """Return checker."""
    run = run_in_subprocess(code,
                            filename_suffix,
                            arguments + ['-pedantic', '-fsyntax-only'])

    def run_check():
        """Yield errors."""
        result = run()
        if result:
            (output, filename) = result
            for line in output.splitlines():
                try:
                    yield parse_gcc_style_error_message(line,
                                                        filename=filename)
                except ValueError:
                    continue

    return run_check

def parse_gcc_style_error_message(message, filename, has_column=True):
    """Parse GCC-style error message.

    Return (line_number, message). Raise ValueError if message cannot be
    parsed.

    """
    colons = 2 if has_column else 1
    prefix = filename + ':'
    if not message.startswith(prefix):
        raise ValueError()
    message = message[len(prefix):]
    split_message = message.split(':', colons)
    line_number = int(split_message[0])
    return (line_number,
            split_message[colons].strip())

def run_in_subprocess(code, filename_suffix, arguments):
    """Return None on success."""
    temporary_file = tempfile.NamedTemporaryFile(mode='w',
                                                 suffix=filename_suffix)
    temporary_file.write(code)
    temporary_file.flush()

    process = subprocess.Popen(arguments + [temporary_file.name],
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE)

    def run():
        """Yield errors."""
        raw_result = process.communicate()
        if process.returncode != 0:
            return (raw_result[1].decode(locale.getpreferredencoding()),
                    temporary_file.name)

    return run

class CheckTranslator(nodes.NodeVisitor):

    """Visits code blocks and checks for syntax errors in code."""

    def __init__(self, document, contents, filename, ignore):
        nodes.NodeVisitor.__init__(self, document)
        self.checkers = []
        self.contents = contents
        self.filename = filename
        self.ignore = ignore or []

    def visit_literal_block(self, node):
        """Check syntax of code block."""
        if 'code-block' not in node['classes']:
            return

        language = node.get('language')
        if language in self.ignore:
            return

        checker = {
            'bash': bash_checker,
            'c': c_checker,
            'cpp': cpp_checker,
            'json': lambda source: lambda: check_json(source),
            'python': lambda source: lambda: check_python(source),
            'rst': lambda source: lambda: check_rst(source, ignore=self.ignore)
        }.get(language)

        if checker:
            run = checker(node.rawsource)

            def run_check():
                """Yield errors."""
                all_results = run()
                if all_results is not None:
                    if all_results:
                        for result in all_results:
                            error_offset = result[0] - 1

                            yield (
                                beginning_of_code_block(node, self.contents) +
                                error_offset,
                                '({}) {}'.format(language, result[1]))
                    else:
                        yield (self.filename, 0, 'unknown error')
            self.checkers.append(run_check)

        raise nodes.SkipNode

    def unknown_visit(self, node):
        """Ignore."""

    def unknown_departure(self, node):
        """Ignore."""

def beginning_of_code_block(node, full_contents):
    """Return line number of beginning of code block."""
    lines = full_contents.splitlines()
    line_number = node.line
    code_block_length = len(node.rawsource.splitlines())

    try:
        # Case where there are no extra spaces.
        if lines[line_number - 1].strip():
            return line_number - code_block_length + 1
    except IndexError:
        pass

    # The offsets are wrong if the RST text has multiple blank lines after the
    # code block. This is a workaround.
    for line_number in range(node.line, 1, -1):
        if lines[line_number - 2].strip():
            break

    return line_number - code_block_length

class CheckWriter(writers.Writer):

    """Runs CheckTranslator on code blocks."""

    def __init__(self, contents, filename, ignore):
        writers.Writer.__init__(self)
        self.checkers = []
        self.contents = contents
        self.filename = filename
        self.ignore = ignore

    def translate(self):
        """Run CheckTranslator."""
        visitor = CheckTranslator(self.document,
                                  contents=self.contents,
                                  filename=self.filename,
                                  ignore=self.ignore)
        self.document.walkabout(visitor)
        self.checkers += visitor.checkers

def main():
    """Return 0 on success."""
    parser = argparse.ArgumentParser(description=__doc__, prog='rstcheck')
    parser.add_argument('files', nargs='+',
                        help='files to check')
    parser.add_argument('--report', type=int, metavar='level', default=1,
                        help='report system messages at or higher than level; '
                             '1 info, 2 warning, 3 error, 4 severe, 5 none '
                             '(default: %(default)s)')
    parser.add_argument('--ignore', metavar='language', default='',
                        help='comma-separated list of languages to ignore')
    parser.add_argument('--version', action='version',
                        version='%(prog)s ' + __version__)
    args = parser.parse_args()

    status = 0
    for filename in args.files:
        try:
            for error in check_file(filename,
                                    report_level=args.report,
                                    ignore=args.ignore.split(',')):
                print('{}:{}: (ERROR/3) {}'.format(filename,
                                                   error[0],
                                                   error[1]),
                      file=sys.stderr)
                status = 1
        except utils.SystemMessage:
            # docutils already prints a message to standard error.
            status = 1

    return status

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Be explicit<commit_after>#!/usr/bin/env python

# Copyright (C) 2013-2014 Steven Myint
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""Checks code blocks in reStructuredText."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import argparse
import io
import json
import locale
import os
import re
import subprocess
import sys
import tempfile

import docutils.core
import docutils.nodes
import docutils.utils
import docutils.writers
import docutils.parsers.rst

__version__ = '0.5'

def check(source, filename='<string>', report_level=1, ignore=None,
          warning_stream=sys.stderr):
    """Yield errors.

    Use lower report_level for noisier error output.

    Each yielded error is a tuple of the form:

        (line_number, message)

    Line numbers are indexed at 1 and are with respect to the full RST file.

    Each code block is checked asynchronously in a subprocess.

    """
    writer = CheckWriter(source, filename, ignore=ignore)

    docutils.core.publish_string(
        source, writer=writer,
        source_path=filename,
        settings_overrides={'report_level': report_level,
                            'warning_stream': warning_stream})

    for checker in writer.checkers:
        for error in checker():
            yield error

def check_file(filename, report_level=1, ignore=None):
    """Yield errors."""
    if filename == '-':
        contents = sys.stdin.read()
    else:
        with open(filename) as input_file:
            contents = input_file.read()

    for error in check(contents,
                       filename=filename,
                       report_level=report_level,
                       ignore=ignore):
        yield error

def check_python(code):
    """Yield errors."""
    try:
        compile(code, '<string>', 'exec')
    except SyntaxError as exception:
        yield (int(exception.lineno), exception.msg)

def check_json(code):
    """Yield errors."""
    try:
        json.loads(code)
    except ValueError as exception:
        message = str(exception)
        line_number = 0

        found = re.search(r': line\s+([0-9]+)[^:]*$', message)
        if found:
            line_number = int(found.group(1))

        yield (int(line_number), message)

def check_rst(code, ignore):
    """Yield errors in nested RST code."""
    string_io = io.StringIO()
    filename = '<string>'

    for result in check(code,
                        filename=filename,
                        ignore=ignore,
                        warning_stream=string_io):
        yield result

    rst_errors = string_io.getvalue().strip()
    if rst_errors:
        for message in rst_errors.splitlines():
            try:
                yield parse_gcc_style_error_message(message,
                                                    filename=filename,
                                                    has_column=False)
            except ValueError:
                continue

class CodeBlockDirective(docutils.parsers.rst.Directive):

    """Code block directive."""

    has_content = True
    optional_arguments = 1

    def run(self):
        """Run directive."""
        try:
            language = self.arguments[0]
        except IndexError:
            language = ''
        code = '\n'.join(self.content)
        literal = docutils.nodes.literal_block(code, code)
        literal['classes'].append('code-block')
        literal['language'] = language
        return [literal]

docutils.parsers.rst.directives.register_directive('code-block',
                                                   CodeBlockDirective)
docutils.parsers.rst.directives.register_directive('sourcecode',
                                                   CodeBlockDirective)

def bash_checker(code):
    """Return checker."""
    run = run_in_subprocess(code, '.bash', ['bash', '-n'])

    def run_check():
        """Yield errors."""
        result = run()
        if result:
            (output, filename) = result
            prefix = filename + ': line '
            for line in output.splitlines():
                if not line.startswith(prefix):
                    continue
                message = line[len(prefix):]
                split_message = message.split(':', 1)
                yield (int(split_message[0]) - 1,
                       split_message[1].strip())
    return run_check

def c_checker(code):
    """Return checker."""
    return gcc_checker(code, '.c', [os.getenv('CC', 'gcc'), '-std=c99'])

def cpp_checker(code):
    """Return checker."""
    return gcc_checker(code, '.cpp', [os.getenv('CXX', 'g++'), '-std=c++0x'])

def gcc_checker(code, filename_suffix, arguments):
    """Return checker."""
    run = run_in_subprocess(code,
                            filename_suffix,
                            arguments + ['-pedantic', '-fsyntax-only'])

    def run_check():
        """Yield errors."""
        result = run()
        if result:
            (output, filename) = result
            for line in output.splitlines():
                try:
                    yield parse_gcc_style_error_message(line,
                                                        filename=filename)
                except ValueError:
                    continue

    return run_check

def parse_gcc_style_error_message(message, filename, has_column=True):
    """Parse GCC-style error message.

    Return (line_number, message). Raise ValueError if message cannot be
    parsed.

    """
    colons = 2 if has_column else 1
    prefix = filename + ':'
    if not message.startswith(prefix):
        raise ValueError()
    message = message[len(prefix):]
    split_message = message.split(':', colons)
    line_number = int(split_message[0])
    return (line_number,
            split_message[colons].strip())

def run_in_subprocess(code, filename_suffix, arguments):
    """Return None on success."""
    temporary_file = tempfile.NamedTemporaryFile(mode='w',
                                                 suffix=filename_suffix)
    temporary_file.write(code)
    temporary_file.flush()

    process = subprocess.Popen(arguments + [temporary_file.name],
                               stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE)

    def run():
        """Yield errors."""
        raw_result = process.communicate()
        if process.returncode != 0:
            return (raw_result[1].decode(locale.getpreferredencoding()),
                    temporary_file.name)

    return run

class CheckTranslator(docutils.nodes.NodeVisitor):

    """Visits code blocks and checks for syntax errors in code."""

    def __init__(self, document, contents, filename, ignore):
        docutils.nodes.NodeVisitor.__init__(self, document)
        self.checkers = []
        self.contents = contents
        self.filename = filename
        self.ignore = ignore or []

    def visit_literal_block(self, node):
        """Check syntax of code block."""
        if 'code-block' not in node['classes']:
            return

        language = node.get('language')
        if language in self.ignore:
            return

        checker = {
            'bash': bash_checker,
            'c': c_checker,
            'cpp': cpp_checker,
            'json': lambda source: lambda: check_json(source),
            'python': lambda source: lambda: check_python(source),
            'rst': lambda source: lambda: check_rst(source, ignore=self.ignore)
        }.get(language)

        if checker:
            run = checker(node.rawsource)

            def run_check():
                """Yield errors."""
                all_results = run()
                if all_results is not None:
                    if all_results:
                        for result in all_results:
                            error_offset = result[0] - 1

                            yield (
                                beginning_of_code_block(node, self.contents) +
                                error_offset,
                                '({}) {}'.format(language, result[1]))
                    else:
                        yield (self.filename, 0, 'unknown error')
            self.checkers.append(run_check)

        raise docutils.nodes.SkipNode

    def unknown_visit(self, node):
        """Ignore."""

    def unknown_departure(self, node):
        """Ignore."""

def beginning_of_code_block(node, full_contents):
    """Return line number of beginning of code block."""
    lines = full_contents.splitlines()
    line_number = node.line
    code_block_length = len(node.rawsource.splitlines())

    try:
        # Case where there are no extra spaces.
        if lines[line_number - 1].strip():
            return line_number - code_block_length + 1
    except IndexError:
        pass

    # The offsets are wrong if the RST text has multiple blank lines after the
    # code block. This is a workaround.
    for line_number in range(node.line, 1, -1):
        if lines[line_number - 2].strip():
            break

    return line_number - code_block_length

class CheckWriter(docutils.writers.Writer):

    """Runs CheckTranslator on code blocks."""

    def __init__(self, contents, filename, ignore):
        docutils.writers.Writer.__init__(self)
        self.checkers = []
        self.contents = contents
        self.filename = filename
        self.ignore = ignore

    def translate(self):
        """Run CheckTranslator."""
        visitor = CheckTranslator(self.document,
                                  contents=self.contents,
                                  filename=self.filename,
                                  ignore=self.ignore)
        self.document.walkabout(visitor)
        self.checkers += visitor.checkers

def main():
    """Return 0 on success."""
    parser = argparse.ArgumentParser(description=__doc__, prog='rstcheck')
    parser.add_argument('files', nargs='+',
                        help='files to check')
    parser.add_argument('--report', type=int, metavar='level', default=1,
                        help='report system messages at or higher than level; '
                             '1 info, 2 warning, 3 error, 4 severe, 5 none '
                             '(default: %(default)s)')
    parser.add_argument('--ignore', metavar='language', default='',
                        help='comma-separated list of languages to ignore')
    parser.add_argument('--version', action='version',
                        version='%(prog)s ' + __version__)
    args = parser.parse_args()

    status = 0
    for filename in args.files:
        try:
            for error in check_file(filename,
                                    report_level=args.report,
                                    ignore=args.ignore.split(',')):
                print('{}:{}: (ERROR/3) {}'.format(filename,
                                                   error[0],
                                                   error[1]),
                      file=sys.stderr)
                status = 1
        except docutils.utils.SystemMessage:
            # docutils already prints a message to standard error.
            status = 1

    return status

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before># -*- coding:utf-8 -*-
from util.html import render_progress_bar
from django import template
import logging

register = template.Library()

LOG = logging.getLogger(__name__)

@register.simple_tag
def render_capacity_html(database):
    try:
        message = "%d of %s (MB)" % (
            database.used_size_in_mb, (database.total_size_in_mb) or 'unlimited')
        return render_progress_bar(database.capacity * 100, message=message)
    except:
        # any error show Unkown message and log error. This avoid break page if there is a problem
        # with some database
        LOG.exception('Error getting capacity of database %s', database)
        return "Unknown"
<commit_msg>Create DetailedProgressBar(Not draft)<commit_after># -*- coding:utf-8 -*-
from util.html import render_progress_bar, render_detailed_progress_bar
from decimal import Decimal, getcontext, ROUND_HALF_EVEN
from django import template
from django.utils.safestring import mark_safe
import logging

getcontext().rounding = ROUND_HALF_EVEN
ONE = Decimal(10) ** -1
register = template.Library()

LOG = logging.getLogger(__name__)

@register.simple_tag
def render_capacity_html(database):
    try:
        message = "%d of %s (MB)" % (
            database.used_size_in_mb, (database.total_size_in_mb) or 'unlimited')
        return render_progress_bar(database.capacity * 100, message=message)
    except:
        # any error show Unkown message and log error. This avoid break page if there is a problem
        # with some database
        LOG.exception('Error getting capacity of database %s', database)
        return "Unknown"

@register.simple_tag
def xxrender_detailed_capacity_html(database):
    def render(nfsaas_size_in_gb,
               nfsaas_used_size_in_gb,
               used_size_in_gb,
               used_other_in_gb,
               free_space_in_gb,
               other_percent,
               database_percent):

        return render_detailed_progress_bar(
            used_size_in_gb,
            used_other_in_gb,
            free_space_in_gb,
            other_percent=other_percent,
            database_percent=database_percent,
        )

    databaseinfra = database.databaseinfra
    host_attr = databaseinfra.instances.first().hostname.nfsaas_host_attributes.first()
    nfsaas_size_in_gb = (host_attr.nfsaas_size_kb or 0.0) * (1.0 / 1024.0 / 1024.0)
    nfsaas_used_size_in_gb = databaseinfra.disk_used_size_in_gb
    if nfsaas_used_size_in_gb is None:
        free_space_in_gb = nfsaas_size_in_gb - database.used_size_in_gb
        used_other_in_gb = None
        other_percent = 0
    else:
        used_other_in_gb = nfsaas_used_size_in_gb - database.used_size_in_gb
        free_space_in_gb = nfsaas_size_in_gb - nfsaas_used_size_in_gb
        other_percent = float(used_other_in_gb * 100) / nfsaas_size_in_gb
    database_percent = float(database.used_size_in_gb * 100) / nfsaas_size_in_gb
    params = dict(
        nfsaas_size_in_gb=nfsaas_size_in_gb,
        nfsaas_used_size_in_gb=nfsaas_used_size_in_gb,
        used_size_in_gb=database.used_size_in_gb,
        used_other_in_gb=used_other_in_gb,
        free_space_in_gb=free_space_in_gb,
        other_percent=other_percent,
        database_percent=database_percent,
    )

    return '{}'.format(render(**params))

@register.tag
def render_detailed_capacity_html(parser, token):

    try:
        split_contents = token.split_contents()
        obj_name = split_contents[1]
        bar_type = split_contents[2]
    except IndexError:
        raise template.TemplateSyntaxError(
            "%r tag requires 1 argument." % split_contents
        )

    return DetailedProgressBarNode(obj_name, bar_type)

class DetailedProgressBarNode(template.Node):
    '''
    Static class receive an array with contains dicts.
    Witch index that array means a part of the bar.
    This dicts must have the keys:
        - name: used to name the css class on template;
        - label: will be appers on label;
        - percentage: percentage of the part;
        - used_space: space used;
        - space_metric: Bytes, KB, MB, GB;
        - color: color of bar part;

    Ex. if i wanna make a bar with 2 part ill pass:
        DetailedProgressBar([{
            'name': 'database',
            'label': 'Database',
            'percentage': 10,
            'used_space': 1.0,
            'space_metric': 'GB',
            'color': '#409fdf'},
            {'name': 'free',
             'label': 'Free',
             'percentage': 90.0,
             'used_space': 9.0,
             'space_metric': 'GB',
             'color': '#949398'
             ''}])

        And that class will return the rendered html.
    '''

    COLORS = {
        'blue': '#409fdf',
        'grey': '#949398',
        'green': '#70bf45',
    }

    def __init__(self, obj_name, bar_type):
        self.obj_name = obj_name
        self.bar_type = bar_type
        self.bars = []

    def _init_vars(self, database):
        self.html = ''
        self.disk_parts = []
        self.memory_parts = []
        self.database = database
        self.databaseinfra = self.database.databaseinfra
        self.instance = self.databaseinfra.instances.first()
        self.is_memory = self.database.engine.engine_type.is_in_memory

    @staticmethod
    def normalize_number(n):
        n = Decimal(round(n, 2))
        return n.quantize(ONE)

    def render(self, context):
        obj = context.get(self.obj_name)
        self._init_vars(obj)
        if obj is None:
            return ''

        if self.bar_type == 'disk':
            html = self.render_disk_bar()
        else:
            html = self.render_memory_bar()

        return mark_safe(html)

    def _make_disk_part(self, **kw):
        self.disk_parts.append(kw)

    def _make_memory_part(self, **kw):
        self.memory_parts.append(kw)

    def render_disk_bar(self):
        self.host_attr = self.instance.hostname.nfsaas_host_attributes.first()
        nfsaas_size_in_gb = (self.host_attr.nfsaas_size_kb or 0.0) * (1.0 / 1024.0 / 1024.0)
        nfsaas_used_size_in_gb = self.databaseinfra.disk_used_size_in_gb
        if nfsaas_used_size_in_gb is None:
            free_space_in_gb = nfsaas_size_in_gb - self.database.used_size_in_gb
            used_other_in_gb = None
            other_percent = 0
        else:
            used_other_in_gb = nfsaas_used_size_in_gb - self.database.used_size_in_gb
            free_space_in_gb = nfsaas_size_in_gb - nfsaas_used_size_in_gb
            other_percent = float(used_other_in_gb * 100) / nfsaas_size_in_gb
        database_percent = float(self.database.used_size_in_gb * 100) / nfsaas_size_in_gb
        self._make_disk_part(**{
            'name': 'database',
            'label': 'Database',
            'percentage': self.normalize_number(database_percent or 0),
            'used_space': self.normalize_number(self.database.used_size_in_gb),
            'color': self.COLORS['blue']
        })
        if used_other_in_gb is not None:
            self._make_disk_part(**{
                'name': 'other',
                'label': 'Others',
                'percentage': self.normalize_number(other_percent or 0),
                'used_space': self.normalize_number(used_other_in_gb),
                'color': self.COLORS['grey']
            })
            self._make_disk_part(**{
                'name': 'free',
                'label': 'Free',
                'percentage': self.normalize_number(100 - (database_percent + other_percent)),
                'used_space': self.normalize_number(free_space_in_gb),
                'color': self.COLORS['green']
            })
        else:
            self._make_disk_part(**{
                'name': 'free',
                'label': 'Free',
                'percentage': self.normalize_number(100 - (database_percent + other_percent)),
                'used_space': self.normalize_number(free_space_in_gb),
                'color': self.COLORS['grey']
            })
        return self.render_template(self.disk_parts, 'disk')

    def render_memory_bar(self):
        total_size_in_gb = self.database.total_size_in_gb
        used_size_in_gb = self.database.used_size_in_gb
        used_percent = float(used_size_in_gb * 100) / total_size_in_gb
        free_size_in_gb = total_size_in_gb - used_size_in_gb
        self._make_memory_part(**{
            'name': 'database',
            'label': 'Database',
            'percentage': self.normalize_number(used_percent or 0),
            'used_space': self.normalize_number(used_size_in_gb),
            'color': self.COLORS['blue']
        })
        self._make_memory_part(**{
            'name': 'free',
            'label': 'Free',
            'percentage': self.normalize_number(100 - (used_percent)),
            'used_space': self.normalize_number(free_size_in_gb),
            'color': self.COLORS['grey']
        })

        return self.render_template(self.memory_parts, 'memory')

    def render_template(self, parts, bar_type):
        t = template.loader.get_template('progress_bar/progress_bar.html')
        c = template.Context({
            'bars': parts,
            'bar_type': bar_type
        })
        return t.render(c)
<|endoftext|>


<commit_before>__author__ = ''

# -------------------------------------------------------------------------------------------------------------------- #
# IMPORTS

# Modules:
import os
import platform
from pathlib import Path
import subprocess
from datetime import datetime
import logging

# RiBuild Modules:
from delphin_6_automation.database_interactions import simulation_interactions
from delphin_6_automation.database_interactions import delphin_interactions
from delphin_6_automation.database_interactions import general_interactions
import delphin_6_automation.database_interactions.mongo_setup as mongo_setup
import delphin_6_automation.database_interactions.db_templates.result_raw_entry as result_db
from delphin_6_automation.database_interactions.auth import dtu_byg

# -------------------------------------------------------------------------------------------------------------------- #
# RIBUILD SIMULATION WORKER, DELPHIN SOLVER,
logger = logging.getLogger(__name__)

def worker(id_):
    """
    Simulation worker. Supposed to be used with main simulation loop.
    :param id_: Database entry ID from simulation queue
    :return: True on success otherwise False
    """

    # Find paths
    system = platform.system()
    if system == 'Windows':
        delphin_path = r'C:/ribuild'
        exe_path = r'C:/Program Files/IBK/Delphin 6.0/DelphinSolver.exe'
    elif system == 'Linux':
        home = str(Path.home())
        delphin_path = home + '/ribuild'
        exe_path = ''
    else:
        raise NameError('OS not supported')

    if not os.path.isdir(delphin_path):
        os.mkdir(delphin_path)

    # Download, solve, upload
    time_0 = datetime.now()
    general_interactions.download_full_project_from_database(str(id_), delphin_path)
    solve_delphin(delphin_path + '/' + id_ + '.d6p', delphin_exe=exe_path, verbosity_level=0)
    id_result = delphin_interactions.upload_results_to_database(delphin_path + '/' + id_)
    delta_time = datetime.now() - time_0

    # Check if uploaded:
    test_doc = result_db.Result.objects(id=id_result).first()

    simulation_interactions.set_simulated(id_)

    if test_doc:
        simulation_interactions.clean_simulation_folder(delphin_path)
        print(f'Finished solving {id_}. Simulation duration: {delta_time}\n')
        return True
    else:
        raise FileNotFoundError('Could not find result entry')

def solve_delphin(file, delphin_exe=r'C:/Program Files/IBK/Delphin 6.0/DelphinSolver.exe', verbosity_level=1):
    """Solves a delphin file"""

    print(f'Solves {file}')

    verbosity = "verbosity-level=" + str(verbosity_level)
    command_string = '"' + str(delphin_exe) + '" --close-on-exit --' + verbosity + ' "' + file + '"'
    process = subprocess.run(command_string, shell=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
    logger.info(process.stdout)
    logger.info(process.stderr)
    print(process.stderr)
    return True

def github_updates():
    # Check for github update to code
    pass

def simulation_worker():
    try:
        while True:
            id_ = simulation_interactions.find_next_sim_in_queue()
            if id_:
                worker(id_)
            else:
                pass
    except KeyboardInterrupt:
        pass

if __name__ == "__main__":
    # Setup connection
    mongo_setup.global_init(dtu_byg)

    simulation_worker()
<commit_msg>updated logging in solve_delphin<commit_after>__author__ = ''

# -------------------------------------------------------------------------------------------------------------------- #
# IMPORTS

# Modules:
import os
import platform
from pathlib import Path
import subprocess
from datetime import datetime
import logging

# RiBuild Modules:
from delphin_6_automation.database_interactions import simulation_interactions
from delphin_6_automation.database_interactions import delphin_interactions
from delphin_6_automation.database_interactions import general_interactions
import delphin_6_automation.database_interactions.mongo_setup as mongo_setup
import delphin_6_automation.database_interactions.db_templates.result_raw_entry as result_db
from delphin_6_automation.database_interactions.auth import dtu_byg

# -------------------------------------------------------------------------------------------------------------------- #
# RIBUILD SIMULATION WORKER, DELPHIN SOLVER,
logger = logging.getLogger(__name__)

def worker(id_):
    """
    Simulation worker. Supposed to be used with main simulation loop.
    :param id_: Database entry ID from simulation queue
    :return: True on success otherwise False
    """

    # Find paths
    system = platform.system()
    if system == 'Windows':
        delphin_path = r'C:/ribuild'
        exe_path = r'C:/Program Files/IBK/Delphin 6.0/DelphinSolver.exe'
    elif system == 'Linux':
        home = str(Path.home())
        delphin_path = home + '/ribuild'
        exe_path = ''
    else:
        raise NameError('OS not supported')

    if not os.path.isdir(delphin_path):
        os.mkdir(delphin_path)

    # Download, solve, upload
    time_0 = datetime.now()
    general_interactions.download_full_project_from_database(str(id_), delphin_path)
    solve_delphin(delphin_path + '/' + id_ + '.d6p', delphin_exe=exe_path, verbosity_level=0)
    id_result = delphin_interactions.upload_results_to_database(delphin_path + '/' + id_)
    delta_time = datetime.now() - time_0

    # Check if uploaded:
    test_doc = result_db.Result.objects(id=id_result).first()

    simulation_interactions.set_simulated(id_)

    if test_doc:
        simulation_interactions.clean_simulation_folder(delphin_path)
        print(f'Finished solving {id_}. Simulation duration: {delta_time}\n')
        return True
    else:
        raise FileNotFoundError('Could not find result entry')

def solve_delphin(file, delphin_exe=r'C:/Program Files/IBK/Delphin 6.0/DelphinSolver.exe', verbosity_level=1):
    """Solves a delphin file"""

    print(f'Solves {file}')

    verbosity = "verbosity-level=" + str(verbosity_level)
    command_string = '"' + str(delphin_exe) + '" --close-on-exit --' + verbosity + ' "' + file + '"'
    process = subprocess.run(command_string, shell=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE)
    logger.info(process.stdout.readline().decode())
    logger.info(process.stderr.readline().decode())
    print(process.stderr.readline().decode())
    return True

def github_updates():
    # Check for github update to code
    pass

def simulation_worker():
    try:
        while True:
            id_ = simulation_interactions.find_next_sim_in_queue()
            if id_:
                worker(id_)
            else:
                pass
    except KeyboardInterrupt:
        pass

if __name__ == "__main__":
    # Setup connection
    mongo_setup.global_init(dtu_byg)

    simulation_worker()
<|endoftext|>


<commit_before>import logging
import math
import multiprocessing as mp
from collections import namedtuple
from functools import partial
from statistics import mean
import itertools

import flask
from flask_wtf import Form
from wtforms import validators, ValidationError
from wtforms.fields import TextAreaField, SubmitField, FloatField

from . import flucalc
from . import keys

version = '2017.9.1'
code_address = 'https://github.com/bondarevts/flucalc'
new_issue_address = code_address + '/issues/new'

logging_level = logging.INFO
logging.basicConfig(filename='flucalc.log', level=logging_level,
                    format='%(asctime)s:%(levelname)s:%(name)s:%(message)s')

app = flask.Flask(__name__)
app.secret_key = keys.secret_key

FieldError = namedtuple('FieldError', ['field_name', 'message'])

MediaDescription = namedtuple('MediaDescription', ['c', 'd', 'v'])

Values = namedtuple('Values', ['m', 'mu', 'mu_interval', 'power'])
CalcResult = namedtuple('CalcResult', ['raw', 'corrected', 'mean_frequency'])

CalcStep = namedtuple('CalcStep', ['pos', 'description', 'value', 'img'])

ProcessingResult = namedtuple('ProcessingResult', ['values', 'steps'])

def volume_value(_, field):
    if field.data is not None and float(field.data) <= 0:
        raise ValidationError('Non positive volume')

def clones_numbers(_, field):
    if field.data is not None:
        if any(clone < 0 for clone in field.data):
            raise ValidationError('Negative number of clones')

def int_values(_, field):
    if field.data is not None:
        if not all(value.is_integer() for value in field.data):
            raise ValidationError('Number must be integer')

dilution_validator = validators.NumberRange(min=1, message='Must be ≥ 1')

# noinspection PyAttributeOutsideInit
class MultiFloatAreaField(TextAreaField):
    def _value(self):
        if self.raw_data:
            return self.raw_data[0]
        if self.data:
            return '\n'.join(self.data)
        return ''

    def process_formdata(self, valuelist):
        if not valuelist:
            return
        data = []
        for pos, value in enumerate(valuelist[0].split(), 1):
            try:
                float_value = float(value)
            except ValueError as e:
                self.data = None
                if len(value) > 5:
                    value = value[:5] + '...'
                message = 'Value #{} is not a valid float: "{}"'.format(pos, value)
                raise ValueError(message) from e
            data.append(float_value)
        self.data = data

class FluctuationInputForm(Form):
    v_total = FloatField('Volume of a culture (μl), Vtot',
                         [volume_value], default=200)
    c_selective = MultiFloatAreaField('Observed numbers of clones, Csel',
                                      [clones_numbers, int_values])
    d_selective = FloatField('Dilution factor, Dsel', [dilution_validator])
    v_selective = FloatField('Volume plated (μl), Vsel', [volume_value])

    c_complete = MultiFloatAreaField('Observed numbers of clones, Ccom',
                                     [clones_numbers])
    d_complete = FloatField('Dilution factor, Dcom', [dilution_validator])
    v_complete = FloatField('Volume plated (μl), Vcom', [volume_value])

    submit = SubmitField('Calculate')

    def __repr__(self):
        c_sel = ';'.join(self.c_selective.raw_data[0].split())
        c_com = ';'.join(self.c_complete.raw_data[0].split())
        return 'V_tot={}|C_sel={}|D_sel={}|V_sel={}|C_com={}|D_com={}|V_com={}'.format(
            self.v_total.raw_data[0],
            c_sel, self.d_selective.raw_data[0], self.v_selective.raw_data[0],
            c_com, self.d_complete.raw_data[0], self.v_complete.raw_data[0]
        )

@app.route('/', methods=['GET', 'POST'])
def main_page():
    logging.info('Request from ip %s.', flask.request.remote_addr)
    form = FluctuationInputForm(flask.request.form, csrf_enabled=False)
    render_page_template = partial(flask.render_template,
                                   'form_with_results.html',
                                   format_number_with_power=format_number_with_power,
                                   form=form, version=version)
    if not form.is_submitted():
        return render_page_template()

    logging.info('Form submitted: %s', form)

    if form.validate():
        logging.info('Form validated')
        v_total = form.v_total.data
        selective = MediaDescription(
            c=form.c_selective.data,
            d=form.d_selective.data,
            v=form.v_selective.data
        )
        complete = MediaDescription(
            c=form.c_complete.data,
            d=form.d_complete.data,
            v=form.v_complete.data
        )

        if 'calc-freq' in flask.request.form:
            if len(selective.c) != len(complete.c):
                complete = complete._replace(c=[mean(complete.c)])
                message = 'Frequency was calculated using mean of Ccom'
                flask.flash(flask.Markup(message))
            frequencies = calc_frequencies(selective, complete)
            return render_page_template(frequencies=frequencies)

        complete = complete._replace(c=mean(complete.c))

        try:
            result = run_parallel(solve, args=(v_total, selective, complete), timeout=5)
        except mp.TimeoutError:
            message = 'Calculation failed. Possible Csel too big.'
            logging.info(message)
            flask.flash(flask.Markup(message))
            return render_page_template()
        except Exception:
            message = ('Calculation failed. '
                       'If it is repeated please create an issue.'.format(new_issue_address))
            logging.exception(message)
            flask.flash(flask.Markup(message))
            return render_page_template()
        return render_page_template(results=result.values, process=result.steps)

    for error in get_errors(form):
        message = '{error.field_name}: {error.message}'.format(error=error)
        logging.info('Form error. %s: %s', error.field_name, error.message)
        flask.flash(flask.Markup(message))

    return render_page_template()

def get_errors(form):
    for field in form:
        try:
            # extract field symbol
            field_name = field.label.text.rsplit(',', 1)[1].strip()
        except IndexError:
            field_name = field.label.text
        if field.errors:
            message = '; '.join(error.lower().rstrip('.') for error in field.errors)
            yield FieldError(field_name, message)

def _calc_min_power(*values):
    return int(min(math.floor(math.log10(value)) for value in values))

def format_number_with_power(number, power, precision=4):
    return '{0:.{1}f}e{2:+03d}'.format(number / pow(10, power), precision, power)

def calc_frequencies(selective, complete):
    return [flucalc.frequency(c_sel, c_com * selective.v * complete.d / complete.v / selective.d)
            for c_sel, c_com in zip(selective.c, itertools.cycle(complete.c))]

def apply_function_to_pipe(func, args, kwargs, pipe):
    try:
        pipe.send(func(*args, **kwargs))
    except Exception:
        logging.exception('Processing error')

def run_parallel(func, args=(), kwargs=None, timeout=None):
    if kwargs is None:
        kwargs = {}

    ctx = mp.get_context('spawn')
    out_pipe, in_pipe = ctx.Pipe(duplex=False)
    process = ctx.Process(target=apply_function_to_pipe, args=(func, args, kwargs, in_pipe))
    process.daemon = True
    process.start()
    logging.info('Process started')
    logging.info('Run calculation in process %s', process.pid)
    process.join(timeout=timeout)
    if not process.is_alive():
        logging.info('Calculation finished')
        return out_pipe.recv()

    logging.info('Try to terminate process')
    process.terminate()
    process.terminate()
    logging.info('Calculation terminated by timeout')
    raise mp.TimeoutError

def solve(v_total, selective, complete):
    def calc_raw_results():
        m = flucalc.m_mle_estimation(selective.c)
        add_step(1, 'Number of mutations, m', m, 'm_raw')

        mu = flucalc.calc_mutation_rate(m, n_total)
        add_step(4, 'Mutation rate, μ', mu, 'mu_raw')

        interval = flucalc.mutation_rate_limits(m, len(selective.c), n_total)
        add_step(5, 'Lower limit for mutation rate, μ−', interval.lower, 'mu_raw_lower')
        add_step(6, 'Upper limit for mutation rate, μ+', interval.upper, 'mu_raw_upper')

        return Values(m, mu, interval, _calc_min_power(mu, *interval))

    def calc_corrected_results(raw):
        z_selective = calc_z(selective)
        add_step(7, 'Fraction of a culture plated on selective media, zsel', z_selective, 'z_sel')

        plating_multiplier = flucalc.plating_efficiency_multiplier(z_selective)
        add_step(8, 'Plating efficiency multiplier, ω', plating_multiplier, 'plating_efficiency')

        m = raw.m * plating_multiplier
        add_step(9, 'Corrected number of mutations, mω', m, 'm_corr')

        mu = flucalc.calc_mutation_rate(m, n_total)
        add_step(10, 'Corrected mutation rate, μω', mu, 'mu_corr')

        interval = flucalc.Interval(
            lower=raw.mu_interval.lower * plating_multiplier,
            upper=raw.mu_interval.upper * plating_multiplier,
        )
        add_step(11, 'Corrected lower limit for mutation rate, '
                     'μω'
                     '−',
                 interval.lower, 'mu_corr_lower')
        add_step(12, 'Corrected upper limit for mutation rate, '
                     'μω'
                     '+',
                 interval.upper, 'mu_corr_upper')
        return Values(m, mu, interval, _calc_min_power(mu, *interval))

    def calc_z(media_description):
        return media_description.v / media_description.d / v_total

    def add_step(position, description, value, img=None):
        steps.append(CalcStep(position, description, float(value), img))

    steps = []
    logging.info('Start calculation')

    add_step(2, 'Mean of Ccom', complete.c, 'mean_c_com')

    n_total = complete.c * complete.d * v_total / complete.v
    add_step(3, 'Total number of cells in culture, Ntot', n_total, 'n_total')

    logging.info('Raw results calculation')
    raw_results = calc_raw_results()

    logging.info('Corrected results calculation')
    corrected_results = calc_corrected_results(raw_results)

    logging.info('Frequency calculation')

    c_complete_to_selective = complete.c * selective.v * complete.d / complete.v / selective.d
    add_step(13, 'Ccom adjusted to selective media', c_complete_to_selective, 'c_com_sel')

    frequency = mean(flucalc.frequency(r, c_complete_to_selective) for r in selective.c)
    add_step(14, 'Mean frequency, f', frequency, 'frequency')
    steps.sort()

    result = CalcResult(
        raw=raw_results,
        corrected=corrected_results,
        mean_frequency=frequency
    )
    return ProcessingResult(steps=steps, values=result)
<commit_msg>Extract calculation timeout constant<commit_after>import logging
import math
import multiprocessing as mp
from collections import namedtuple
from functools import partial
from statistics import mean
import itertools

import flask
from flask_wtf import Form
from wtforms import validators, ValidationError
from wtforms.fields import TextAreaField, SubmitField, FloatField

from . import flucalc
from . import keys

CALCULATION_TIMEOUT_SEC = 5

version = '2017.9.1'
code_address = 'https://github.com/bondarevts/flucalc'
new_issue_address = code_address + '/issues/new'

logging_level = logging.INFO
logging.basicConfig(filename='flucalc.log', level=logging_level,
                    format='%(asctime)s:%(levelname)s:%(name)s:%(message)s')

app = flask.Flask(__name__)
app.secret_key = keys.secret_key

FieldError = namedtuple('FieldError', ['field_name', 'message'])

MediaDescription = namedtuple('MediaDescription', ['c', 'd', 'v'])

Values = namedtuple('Values', ['m', 'mu', 'mu_interval', 'power'])
CalcResult = namedtuple('CalcResult', ['raw', 'corrected', 'mean_frequency'])

CalcStep = namedtuple('CalcStep', ['pos', 'description', 'value', 'img'])

ProcessingResult = namedtuple('ProcessingResult', ['values', 'steps'])

def volume_value(_, field):
    if field.data is not None and float(field.data) <= 0:
        raise ValidationError('Non positive volume')

def clones_numbers(_, field):
    if field.data is not None:
        if any(clone < 0 for clone in field.data):
            raise ValidationError('Negative number of clones')

def int_values(_, field):
    if field.data is not None:
        if not all(value.is_integer() for value in field.data):
            raise ValidationError('Number must be integer')

dilution_validator = validators.NumberRange(min=1, message='Must be ≥ 1')

# noinspection PyAttributeOutsideInit
class MultiFloatAreaField(TextAreaField):
    def _value(self):
        if self.raw_data:
            return self.raw_data[0]
        if self.data:
            return '\n'.join(self.data)
        return ''

    def process_formdata(self, valuelist):
        if not valuelist:
            return
        data = []
        for pos, value in enumerate(valuelist[0].split(), 1):
            try:
                float_value = float(value)
            except ValueError as e:
                self.data = None
                if len(value) > 5:
                    value = value[:5] + '...'
                message = 'Value #{} is not a valid float: "{}"'.format(pos, value)
                raise ValueError(message) from e
            data.append(float_value)
        self.data = data

class FluctuationInputForm(Form):
    v_total = FloatField('Volume of a culture (μl), Vtot',
                         [volume_value], default=200)
    c_selective = MultiFloatAreaField('Observed numbers of clones, Csel',
                                      [clones_numbers, int_values])
    d_selective = FloatField('Dilution factor, Dsel', [dilution_validator])
    v_selective = FloatField('Volume plated (μl), Vsel', [volume_value])

    c_complete = MultiFloatAreaField('Observed numbers of clones, Ccom',
                                     [clones_numbers])
    d_complete = FloatField('Dilution factor, Dcom', [dilution_validator])
    v_complete = FloatField('Volume plated (μl), Vcom', [volume_value])

    submit = SubmitField('Calculate')

    def __repr__(self):
        c_sel = ';'.join(self.c_selective.raw_data[0].split())
        c_com = ';'.join(self.c_complete.raw_data[0].split())
        return 'V_tot={}|C_sel={}|D_sel={}|V_sel={}|C_com={}|D_com={}|V_com={}'.format(
            self.v_total.raw_data[0],
            c_sel, self.d_selective.raw_data[0], self.v_selective.raw_data[0],
            c_com, self.d_complete.raw_data[0], self.v_complete.raw_data[0]
        )

@app.route('/', methods=['GET', 'POST'])
def main_page():
    logging.info('Request from ip %s.', flask.request.remote_addr)
    form = FluctuationInputForm(flask.request.form, csrf_enabled=False)
    render_page_template = partial(flask.render_template,
                                   'form_with_results.html',
                                   format_number_with_power=format_number_with_power,
                                   form=form, version=version)
    if not form.is_submitted():
        return render_page_template()

    logging.info('Form submitted: %s', form)

    if form.validate():
        logging.info('Form validated')
        v_total = form.v_total.data
        selective = MediaDescription(
            c=form.c_selective.data,
            d=form.d_selective.data,
            v=form.v_selective.data
        )
        complete = MediaDescription(
            c=form.c_complete.data,
            d=form.d_complete.data,
            v=form.v_complete.data
        )

        if 'calc-freq' in flask.request.form:
            if len(selective.c) != len(complete.c):
                complete = complete._replace(c=[mean(complete.c)])
                message = 'Frequency was calculated using mean of Ccom'
                flask.flash(flask.Markup(message))
            frequencies = calc_frequencies(selective, complete)
            return render_page_template(frequencies=frequencies)

        complete = complete._replace(c=mean(complete.c))

        try:
            result = run_parallel(solve, args=(v_total, selective, complete), timeout=CALCULATION_TIMEOUT_SEC)
        except mp.TimeoutError:
            message = 'Calculation failed. Possible Csel too big.'
            logging.info(message)
            flask.flash(flask.Markup(message))
            return render_page_template()
        except Exception:
            message = ('Calculation failed. '
                       'If it is repeated please create an issue.'.format(new_issue_address))
            logging.exception(message)
            flask.flash(flask.Markup(message))
            return render_page_template()
        return render_page_template(results=result.values, process=result.steps)

    for error in get_errors(form):
        message = '{error.field_name}: {error.message}'.format(error=error)
        logging.info('Form error. %s: %s', error.field_name, error.message)
        flask.flash(flask.Markup(message))

    return render_page_template()

def get_errors(form):
    for field in form:
        try:
            # extract field symbol
            field_name = field.label.text.rsplit(',', 1)[1].strip()
        except IndexError:
            field_name = field.label.text
        if field.errors:
            message = '; '.join(error.lower().rstrip('.') for error in field.errors)
            yield FieldError(field_name, message)

def _calc_min_power(*values):
    return int(min(math.floor(math.log10(value)) for value in values))

def format_number_with_power(number, power, precision=4):
    return '{0:.{1}f}e{2:+03d}'.format(number / pow(10, power), precision, power)

def calc_frequencies(selective, complete):
    return [flucalc.frequency(c_sel, c_com * selective.v * complete.d / complete.v / selective.d)
            for c_sel, c_com in zip(selective.c, itertools.cycle(complete.c))]

def apply_function_to_pipe(func, args, kwargs, pipe):
    try:
        pipe.send(func(*args, **kwargs))
    except Exception:
        logging.exception('Processing error')

def run_parallel(func, args=(), kwargs=None, timeout=None):
    if kwargs is None:
        kwargs = {}

    ctx = mp.get_context('spawn')
    out_pipe, in_pipe = ctx.Pipe(duplex=False)
    process = ctx.Process(target=apply_function_to_pipe, args=(func, args, kwargs, in_pipe))
    process.daemon = True
    process.start()
    logging.info('Process started')
    logging.info('Run calculation in process %s', process.pid)
    process.join(timeout=timeout)
    if not process.is_alive():
        logging.info('Calculation finished')
        return out_pipe.recv()

    logging.info('Try to terminate process')
    process.terminate()
    process.terminate()
    logging.info('Calculation terminated by timeout')
    raise mp.TimeoutError

def solve(v_total, selective, complete):
    def calc_raw_results():
        m = flucalc.m_mle_estimation(selective.c)
        add_step(1, 'Number of mutations, m', m, 'm_raw')

        mu = flucalc.calc_mutation_rate(m, n_total)
        add_step(4, 'Mutation rate, μ', mu, 'mu_raw')

        interval = flucalc.mutation_rate_limits(m, len(selective.c), n_total)
        add_step(5, 'Lower limit for mutation rate, μ−', interval.lower, 'mu_raw_lower')
        add_step(6, 'Upper limit for mutation rate, μ+', interval.upper, 'mu_raw_upper')

        return Values(m, mu, interval, _calc_min_power(mu, *interval))

    def calc_corrected_results(raw):
        z_selective = calc_z(selective)
        add_step(7, 'Fraction of a culture plated on selective media, zsel', z_selective, 'z_sel')

        plating_multiplier = flucalc.plating_efficiency_multiplier(z_selective)
        add_step(8, 'Plating efficiency multiplier, ω', plating_multiplier, 'plating_efficiency')

        m = raw.m * plating_multiplier
        add_step(9, 'Corrected number of mutations, mω', m, 'm_corr')

        mu = flucalc.calc_mutation_rate(m, n_total)
        add_step(10, 'Corrected mutation rate, μω', mu, 'mu_corr')

        interval = flucalc.Interval(
            lower=raw.mu_interval.lower * plating_multiplier,
            upper=raw.mu_interval.upper * plating_multiplier,
        )
        add_step(11, 'Corrected lower limit for mutation rate, '
                     'μω'
                     '−',
                 interval.lower, 'mu_corr_lower')
        add_step(12, 'Corrected upper limit for mutation rate, '
                     'μω'
                     '+',
                 interval.upper, 'mu_corr_upper')
        return Values(m, mu, interval, _calc_min_power(mu, *interval))

    def calc_z(media_description):
        return media_description.v / media_description.d / v_total

    def add_step(position, description, value, img=None):
        steps.append(CalcStep(position, description, float(value), img))

    steps = []
    logging.info('Start calculation')

    add_step(2, 'Mean of Ccom', complete.c, 'mean_c_com')

    n_total = complete.c * complete.d * v_total / complete.v
    add_step(3, 'Total number of cells in culture, Ntot', n_total, 'n_total')

    logging.info('Raw results calculation')
    raw_results = calc_raw_results()

    logging.info('Corrected results calculation')
    corrected_results = calc_corrected_results(raw_results)

    logging.info('Frequency calculation')

    c_complete_to_selective = complete.c * selective.v * complete.d / complete.v / selective.d
    add_step(13, 'Ccom adjusted to selective media', c_complete_to_selective, 'c_com_sel')

    frequency = mean(flucalc.frequency(r, c_complete_to_selective) for r in selective.c)
    add_step(14, 'Mean frequency, f', frequency, 'frequency')
    steps.sort()

    result = CalcResult(
        raw=raw_results,
        corrected=corrected_results,
        mean_frequency=frequency
    )
    return ProcessingResult(steps=steps, values=result)
<|endoftext|>


<commit_before>"""CelebA data loader."""

import os

import numpy as np
import pandas as pd

import torch
from torch.utils.data import Dataset
from torch.utils.data.sampler import WeightedRandomSampler
import torchvision.transforms as transforms

from data import data_util

from util.utils import DEFAULT_MISSING_CONST as DF_M
from util.utils import download as download_from_gcs

DATA_DIRECTORY = 'data/datasets/celeba_dataset/'

class CelebA(object):
    """CelebA data loader."""

    def __init__(self,
                 lab_split=1.0,
                 reweight=False,
                 seed=42,
                 get_dataset_from_lmdb=False,
                 download=False):
        print('Using CelebA dataset!')

        self.root_dir = DATA_DIRECTORY
        self.dataseed = seed
        self.lab_split = lab_split
        self.reweight = reweight
        self.get_dataset_from_lmdb = get_dataset_from_lmdb

        if download and self.get_dataset_from_lmdb:
            self.download()
        else:
            if not os.path.exists(self.root_dir):
                raise ValueError(f'{self.root_dir} does not exist yet.')

        # Read metadata files
        self.metadata = pd.read_csv(os.path.join(self.root_dir, 'list_attr_celeba.csv'))

        # Target values
        self.target = self.metadata['Blond_Hair'].values
        self.target[self.target == -1] = 0
        self.n_targets = len(np.unique(self.target))

        # Control values
        self.environment = self.metadata['Male'].values
        self.environment[self.environment == -1] = 0
        self.n_envs = len(np.unique(self.environment))

        # Generate control groups
        self.n_controls = self.n_targets * self.n_envs # Each target x env counts as one group
        self.control = (self.target*(self.n_controls/2) + self.environment).astype('int')
        assert self.n_controls == len(np.unique(self.control)), "Error in control list"

        # Marginal count from data=0.44,0.41,0.14.0.01

        # Extract filenames and splits
        self.filename = self.metadata['image_id'].values
        split_metadata = pd.read_csv(os.path.join(self.root_dir, 'list_eval_partition.csv'))
        self.split_idx = split_metadata['partition'].values

        # Split train, valid, test
        (fn_train, fn_valid, fn_test,
            y_train, y_valid, y_test,
            self.c_train, c_valid, c_test) = self.generate_splits()

        # Get custom transforms
        train_transform, eval_transform = self.get_transforms()

        # Create Torch Custom Datasets
        self.data_dir = os.path.join(self.root_dir, 'img_align_celeba')
        if self.get_dataset_from_lmdb:
            self.alldata_set = data_util.ImageFromLMDB(filename=self.filename,
                                                       target=self.target,
                                                       control=self.control,
                                                       data_dir=self.data_dir,
                                                       transform=eval_transform)
        else:
            self.alldata_set = data_util.ImageFromDisk(filename=self.filename,
                                                       target=self.target,
                                                       control=self.control,
                                                       data_dir=self.data_dir,
                                                       transform=eval_transform)
        self.train_set = torch.utils.data.Subset(self.alldata_set,
                                                 np.where(self.split_idx==0)[0])
        self.train_set.dataset.transform = train_transform
        self.val_set = torch.utils.data.Subset(self.alldata_set,
                                               np.where(self.split_idx==1)[0])
        self.test_set = torch.utils.data.Subset(self.alldata_set,
                                                np.where(self.split_idx==2)[0])

    def download(self):
        download_data_dir = f'/workdir/xcloud_data/{np.random.randint(99999):05d}'
        download_from_gcs(download_dir='data/datasets/celeba_dataset',
                          gcs_bucket='xcloud_bucket',
                          output_dir=download_data_dir)
        self.root_dir = f'{download_data_dir}/data/datasets/celeba_dataset/'

    def generate_splits(self):
        """Create the splits in filename, targets and controls
        """

        fn_train = self.filename[self.split_idx == 0] # 0 for train
        fn_valid = self.filename[self.split_idx == 1] # 1 for valid
        fn_test = self.filename[self.split_idx == 2] # 2 for test

        y_train = self.target[self.split_idx == 0] # 0 for train
        y_valid = self.target[self.split_idx == 1] # 1 for valid
        y_test = self.target[self.split_idx == 2] # 2 for test

        c_train = self.control[self.split_idx == 0] # 0 for train
        c_valid = self.control[self.split_idx == 1] # 1 for valid
        c_test = self.control[self.split_idx == 2] # 2 for test
        self.c_train_gt = c_train.copy()

        # SSL Setting
        if self.lab_split < 1.0:
            #TODO: check uniform sampling
            np.random.seed(self.dataseed)
            select = np.random.choice([False, True],
                                       size=len(c_train),
                                       replace=True,
                                       p=[self.lab_split, 1-self.lab_split])
            # DF_M denotes that the label is not available for train data.
            c_train[select] = DF_M
            self.control[np.where(self.split_idx == 0)[0][select]] = DF_M

        return (fn_train, fn_valid, fn_test,\
                y_train, y_valid, y_test,\
                c_train, c_valid, c_test)

    def get_transforms(self):
        wd, ht,  = 178, 218
        min_dim = min(wd, ht)

        target_resolution = (224, 224)
        # target_resolution = (wd, ht)

        train_transform = transforms.Compose([
            transforms.CenterCrop(min_dim),
            transforms.Resize(target_resolution),
            transforms.ToTensor(),
            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
        ])

        eval_transform = transforms.Compose([
            transforms.CenterCrop(min_dim),
            transforms.Resize(target_resolution),
            transforms.ToTensor(),
            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
        ])

        return train_transform, eval_transform

    def load_dataset(self,
                     batch_size=64,
                     num_batch_per_epoch=None,
                     num_workers=4,
                     pin_memory=False,
                     **kwargs):
        """Loads dataset.

        Args:
        batch_size: integer for batch size.
        num_batch_per_epoch: integer for number of batch per epoch.
        num_workers: number of workers for data loader.
        **kwargs: for backward compatibility.

        Returns:
        list of data loaders.
        """

        del kwargs

        # Generate DataLoaders
        if self.reweight:
            self.c_train = torch.LongTensor(self.c_train)
            c_counts = (torch.arange(self.n_controls).unsqueeze(1)==self.c_train).sum(1).float()
            c_invprobs = len(self.train_set) / c_counts
            invprobs = c_invprobs[self.c_train_gt] # from uniform sampling

            sampler_train = WeightedRandomSampler(invprobs, len(self.train_set), replacement=True)
            shuffle_train = False
        else:
            sampler_train = None
            shuffle_train = True

        train_loader = torch.utils.data.DataLoader(self.train_set,
                                                   batch_size=batch_size,
                                                   num_workers=num_workers,
                                                   shuffle=shuffle_train,
                                                   sampler=sampler_train, drop_last=True)
        val_loader = torch.utils.data.DataLoader(self.val_set,
                                                 batch_size=batch_size,
                                                 num_workers=num_workers,
                                                 shuffle=False, drop_last=False)
        test_loader = torch.utils.data.DataLoader(self.test_set,
                                                  batch_size=batch_size,
                                                  num_workers=num_workers,
                                                  shuffle=False, drop_last=False)

        return [train_loader, val_loader, test_loader]

if __name__ == '__main__':
    # Compare LMDB-based loader and Disk-based loader
    #db = CelebA(lab_split=0.3, reweight=False, seed=42, get_dataset_from_lmdb=False)
    #loaders_disk = db.load_dataset(batch_size=64)
    db = CelebA(lab_split=0.3, reweight=False, seed=42, get_dataset_from_lmdb=True, download=True)
    loaders_lmdb = db.load_dataset(batch_size=64)
    for batch_disk, batch_lmdb in zip(loaders_disk[1], loaders_lmdb[1]):
        assert torch.norm(batch_disk[0] - batch_lmdb[0]) == 0, 'data are aligned'

<commit_msg>celebA dataset by downloading LMDB from gcs<commit_after>"""CelebA data loader."""

import os

import numpy as np
import pandas as pd

import torch
from torch.utils.data import Dataset
from torch.utils.data.sampler import WeightedRandomSampler
import torchvision.transforms as transforms

from data import data_util

from util.utils import DEFAULT_MISSING_CONST as DF_M
from util.utils import download as download_from_gcs

DATA_DIRECTORY = 'data/datasets/celeba_dataset/'

class CelebA(object):
    """CelebA data loader."""

    def __init__(self,
                 lab_split=1.0,
                 reweight=False,
                 seed=42,
                 get_dataset_from_lmdb=False,
                 download=True):
        print('Using CelebA dataset!')

        self.root_dir = DATA_DIRECTORY
        self.dataseed = seed
        self.lab_split = lab_split
        self.reweight = reweight
        self.get_dataset_from_lmdb = get_dataset_from_lmdb

        download = download and get_dataset_from_lmdb
        if download and self.get_dataset_from_lmdb:
            self.download()
        else:
            if not os.path.exists(self.root_dir):
                raise ValueError(f'{self.root_dir} does not exist yet.')

        # Read metadata files
        self.metadata = pd.read_csv(os.path.join(self.root_dir, 'list_attr_celeba.csv'))

        # Target values
        self.target = self.metadata['Blond_Hair'].values
        self.target[self.target == -1] = 0
        self.n_targets = len(np.unique(self.target))

        # Control values
        self.environment = self.metadata['Male'].values
        self.environment[self.environment == -1] = 0
        self.n_envs = len(np.unique(self.environment))

        # Generate control groups
        self.n_controls = self.n_targets * self.n_envs # Each target x env counts as one group
        self.control = (self.target*(self.n_controls/2) + self.environment).astype('int')
        assert self.n_controls == len(np.unique(self.control)), "Error in control list"

        # Marginal count from data=0.44,0.41,0.14.0.01

        # Extract filenames and splits
        self.filename = self.metadata['image_id'].values
        split_metadata = pd.read_csv(os.path.join(self.root_dir, 'list_eval_partition.csv'))
        self.split_idx = split_metadata['partition'].values

        # Split train, valid, test
        (fn_train, fn_valid, fn_test,
            y_train, y_valid, y_test,
            self.c_train, c_valid, c_test) = self.generate_splits()

        # Get custom transforms
        train_transform, eval_transform = self.get_transforms()

        # Create Torch Custom Datasets
        self.data_dir = os.path.join(self.root_dir, 'img_align_celeba')
        if self.get_dataset_from_lmdb:
            self.alldata_set = data_util.ImageFromLMDB(filename=self.filename,
                                                       target=self.target,
                                                       control=self.control,
                                                       data_dir=self.data_dir,
                                                       transform=eval_transform)
        else:
            self.alldata_set = data_util.ImageFromDisk(filename=self.filename,
                                                       target=self.target,
                                                       control=self.control,
                                                       data_dir=self.data_dir,
                                                       transform=eval_transform)
        self.train_set = torch.utils.data.Subset(self.alldata_set,
                                                 np.where(self.split_idx==0)[0])
        self.train_set.dataset.transform = train_transform
        self.val_set = torch.utils.data.Subset(self.alldata_set,
                                               np.where(self.split_idx==1)[0])
        self.test_set = torch.utils.data.Subset(self.alldata_set,
                                                np.where(self.split_idx==2)[0])

    def download(self):
        download_data_dir = f'/workdir/xcloud_data/{np.random.randint(99999):05d}'
        download_from_gcs(download_dir='data/datasets/celeba_dataset',
                          gcs_bucket='xcloud_bucket',
                          output_dir=download_data_dir)
        self.root_dir = f'{download_data_dir}/data/datasets/celeba_dataset/'

    def generate_splits(self):
        """Create the splits in filename, targets and controls
        """

        fn_train = self.filename[self.split_idx == 0] # 0 for train
        fn_valid = self.filename[self.split_idx == 1] # 1 for valid
        fn_test = self.filename[self.split_idx == 2] # 2 for test

        y_train = self.target[self.split_idx == 0] # 0 for train
        y_valid = self.target[self.split_idx == 1] # 1 for valid
        y_test = self.target[self.split_idx == 2] # 2 for test

        c_train = self.control[self.split_idx == 0] # 0 for train
        c_valid = self.control[self.split_idx == 1] # 1 for valid
        c_test = self.control[self.split_idx == 2] # 2 for test
        self.c_train_gt = c_train.copy()

        # SSL Setting
        if self.lab_split < 1.0:
            #TODO: check uniform sampling
            np.random.seed(self.dataseed)
            select = np.random.choice([False, True],
                                       size=len(c_train),
                                       replace=True,
                                       p=[self.lab_split, 1-self.lab_split])
            # DF_M denotes that the label is not available for train data.
            c_train[select] = DF_M
            self.control[np.where(self.split_idx == 0)[0][select]] = DF_M

        return (fn_train, fn_valid, fn_test,\
                y_train, y_valid, y_test,\
                c_train, c_valid, c_test)

    def get_transforms(self):
        wd, ht,  = 178, 218
        min_dim = min(wd, ht)

        target_resolution = (224, 224)
        # target_resolution = (wd, ht)

        train_transform = transforms.Compose([
            transforms.CenterCrop(min_dim),
            transforms.Resize(target_resolution),
            transforms.ToTensor(),
            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
        ])

        eval_transform = transforms.Compose([
            transforms.CenterCrop(min_dim),
            transforms.Resize(target_resolution),
            transforms.ToTensor(),
            transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
        ])

        return train_transform, eval_transform

    def load_dataset(self,
                     batch_size=64,
                     num_batch_per_epoch=None,
                     num_workers=4,
                     pin_memory=False,
                     **kwargs):
        """Loads dataset.

        Args:
        batch_size: integer for batch size.
        num_batch_per_epoch: integer for number of batch per epoch.
        num_workers: number of workers for data loader.
        **kwargs: for backward compatibility.

        Returns:
        list of data loaders.
        """

        del kwargs

        # Generate DataLoaders
        if self.reweight:
            self.c_train = torch.LongTensor(self.c_train)
            c_counts = (torch.arange(self.n_controls).unsqueeze(1)==self.c_train).sum(1).float()
            c_invprobs = len(self.train_set) / c_counts
            invprobs = c_invprobs[self.c_train_gt] # from uniform sampling

            sampler_train = WeightedRandomSampler(invprobs, len(self.train_set), replacement=True)
            shuffle_train = False
        else:
            sampler_train = None
            shuffle_train = True

        train_loader = torch.utils.data.DataLoader(self.train_set,
                                                   batch_size=batch_size,
                                                   num_workers=num_workers,
                                                   shuffle=shuffle_train,
                                                   sampler=sampler_train, drop_last=True)
        val_loader = torch.utils.data.DataLoader(self.val_set,
                                                 batch_size=batch_size,
                                                 num_workers=num_workers,
                                                 shuffle=False, drop_last=False)
        test_loader = torch.utils.data.DataLoader(self.test_set,
                                                  batch_size=batch_size,
                                                  num_workers=num_workers,
                                                  shuffle=False, drop_last=False)

        return [train_loader, val_loader, test_loader]

if __name__ == '__main__':
    # Compare LMDB-based loader and Disk-based loader
    #db = CelebA(lab_split=0.3, reweight=False, seed=42, get_dataset_from_lmdb=False)
    #loaders_disk = db.load_dataset(batch_size=64)
    db = CelebA(lab_split=0.3, reweight=False, seed=42, get_dataset_from_lmdb=True, download=True)
    loaders_lmdb = db.load_dataset(batch_size=64)
    for batch_disk, batch_lmdb in zip(loaders_disk[1], loaders_lmdb[1]):
        assert torch.norm(batch_disk[0] - batch_lmdb[0]) == 0, 'data are aligned'

<|endoftext|>


<commit_before>"""
Load global configurations, if they exist in a config.ini file
"""

import configparser
import subprocess
from os import path

CONFIG = configparser.ConfigParser()
CONFIGFILE = path.join(path.dirname(__file__), "..", "config.ini")
CONFIG.read(CONFIGFILE)

LIBMACKEOPT = path.expanduser(CONFIG.get("binaries", "libmackeopt"))
LLVMOPT = path.expanduser(CONFIG.get("binaries", "llvmopt", fallback="opt"))
KLEEBIN = path.expanduser(CONFIG.get("binaries", "klee", fallback="klee"))
THREADNUM = CONFIG.getint("runtime", "threadnum", fallback=None)

def __get_output_from(*args, **kwargs):
    """
    Starts a subprocess with the given args and returns its output - no matter
    if the process exits normally or with an error
    """
    # Sadly, some programs return their help pages with non-zero exit code
    try:
        output = subprocess.check_output(*args, **kwargs)
    except subprocess.CalledProcessError as ex:
        output = ex.output
    return output

def check_config():
    """
    Checks all variables in the config. Especially if all given binaries are
    actually executable, have the correct version and support everything needed
    """

    # Check, if LLVMOPT is actually a binary of opt
    if (not path.isfile(LLVMOPT) or b"llvm .bc -> .bc modular optimizer"
            not in __get_output_from([LLVMOPT, "-help"])):
        raise Exception("Config: Invalid opt binary")

    # Check, if LLVMOPT is the correct opt version
    if b"LLVM version 3.4.2" not in __get_output_from([LLVMOPT, "-version"]):
        raise Exception("Config: Invalid opt version")

    # Check, if LIBMACKEOPT actually supports the relevant passes
    if not path.isfile(LIBMACKEOPT):
        raise Exception("Config: Invalid libmackeopt binary")
    mhelp = __get_output_from([LLVMOPT, "-load", LIBMACKEOPT, "-help"])
    if any(t not in mhelp for t in [
            b"-extractcallgraph", b"-listallfuncstopologic",
            b"-encapsulatesymbolic", b"-preprenderror"]):
        raise Exception(
            "Config: limackeopt does not support all required passes")

    # Check, if KLEEBIN
    if not path.isfile(KLEEBIN):
        raise Exception("Config: Invalid KLEE binary")
    kvers = __get_output_from([KLEEBIN, "-version"])
    if b"KLEE" not in kvers or b"LLVM version 3.4.2" not in kvers:
        raise Exception("Config: Invalid klee version")
    khelp = __get_output_from([KLEEBIN, "-help"])
    if any(t not in khelp for t in [b"=ld2t", b"-targeted-function=<string>"]):
        raise Exception("Config: klee does not support targeted search")

    # Check, if a reasonable number of threads is used
    if THREADNUM is not None and not 0 < THREADNUM < 128:
        raise Exception("Config: Invalid Number of threads")

def get_current_git_hash_from(directory):
    """
    Returns the git hash of the currently checked out commit in directory
    """
    try:
        githash = subprocess.check_output(
            ['git', 'rev-parse', 'HEAD'], cwd=directory,
            stderr=subprocess.DEVNULL).decode("utf-8").rstrip()
    except subprocess.CalledProcessError:
        githash = "unknown"

    return githash

def get_current_git_hash():
    """
    Returns the git hash of the currently checked out commit
    """
    return get_current_git_hash_from(path.join(path.dirname(__file__), ".."))

def get_llvm_opt_git_hash():
    """
    Tries to get the git hash of currently checket out version of
    macke-llvm-opt, if the binary is inside the git repository
    """
    return get_current_git_hash_from(path.dirname(LIBMACKEOPT))

def get_klee_git_hash():
    """
    Tries to get the git hash of currently checket out version of klee,
    if the binary is inside the git repository
    """
    return get_current_git_hash_from(path.dirname(KLEEBIN))
<commit_msg>check for sonar search in klee help<commit_after>"""
Load global configurations, if they exist in a config.ini file
"""

import configparser
import subprocess
from os import path

CONFIG = configparser.ConfigParser()
CONFIGFILE = path.join(path.dirname(__file__), "..", "config.ini")
CONFIG.read(CONFIGFILE)

LIBMACKEOPT = path.expanduser(CONFIG.get("binaries", "libmackeopt"))
LLVMOPT = path.expanduser(CONFIG.get("binaries", "llvmopt", fallback="opt"))
KLEEBIN = path.expanduser(CONFIG.get("binaries", "klee", fallback="klee"))
THREADNUM = CONFIG.getint("runtime", "threadnum", fallback=None)

def __get_output_from(*args, **kwargs):
    """
    Starts a subprocess with the given args and returns its output - no matter
    if the process exits normally or with an error
    """
    # Sadly, some programs return their help pages with non-zero exit code
    try:
        output = subprocess.check_output(*args, **kwargs)
    except subprocess.CalledProcessError as ex:
        output = ex.output
    return output

def check_config():
    """
    Checks all variables in the config. Especially if all given binaries are
    actually executable, have the correct version and support everything needed
    """

    # Check, if LLVMOPT is actually a binary of opt
    if (not path.isfile(LLVMOPT) or b"llvm .bc -> .bc modular optimizer"
            not in __get_output_from([LLVMOPT, "-help"])):
        raise Exception("Config: Invalid opt binary")

    # Check, if LLVMOPT is the correct opt version
    if b"LLVM version 3.4.2" not in __get_output_from([LLVMOPT, "-version"]):
        raise Exception("Config: Invalid opt version")

    # Check, if LIBMACKEOPT actually supports the relevant passes
    if not path.isfile(LIBMACKEOPT):
        raise Exception("Config: Invalid libmackeopt binary")
    mhelp = __get_output_from([LLVMOPT, "-load", LIBMACKEOPT, "-help"])
    if any(t not in mhelp for t in [
            b"-extractcallgraph", b"-listallfuncstopologic",
            b"-encapsulatesymbolic", b"-preprenderror"]):
        raise Exception(
            "Config: limackeopt does not support all required passes")

    # Check, if KLEEBIN
    if not path.isfile(KLEEBIN):
        raise Exception("Config: Invalid KLEE binary")
    kvers = __get_output_from([KLEEBIN, "-version"])
    if b"KLEE" not in kvers or b"LLVM version 3.4.2" not in kvers:
        raise Exception("Config: Invalid klee version")
    khelp = __get_output_from([KLEEBIN, "-help"])
    if any(t not in khelp for t in [b"=sonar", b"-sonar-target", b"-sonar-target-info=<string>"]):
        raise Exception("Config: klee does not support sonar search")

    # Check, if a reasonable number of threads is used
    if THREADNUM is not None and not 0 < THREADNUM < 128:
        raise Exception("Config: Invalid Number of threads")

def get_current_git_hash_from(directory):
    """
    Returns the git hash of the currently checked out commit in directory
    """
    try:
        githash = subprocess.check_output(
            ['git', 'rev-parse', 'HEAD'], cwd=directory,
            stderr=subprocess.DEVNULL).decode("utf-8").rstrip()
    except subprocess.CalledProcessError:
        githash = "unknown"

    return githash

def get_current_git_hash():
    """
    Returns the git hash of the currently checked out commit
    """
    return get_current_git_hash_from(path.join(path.dirname(__file__), ".."))

def get_llvm_opt_git_hash():
    """
    Tries to get the git hash of currently checket out version of
    macke-llvm-opt, if the binary is inside the git repository
    """
    return get_current_git_hash_from(path.dirname(LIBMACKEOPT))

def get_klee_git_hash():
    """
    Tries to get the git hash of currently checket out version of klee,
    if the binary is inside the git repository
    """
    return get_current_git_hash_from(path.dirname(KLEEBIN))
<|endoftext|>