<commit_before>#!/usr/bin/env python
# coding=utf8
#
# Copyright 2016 Science & Technology Facilities Council
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Linter for the Pan language"""

import re
import argparse
from glob import glob
from colorama import Fore, Style, init as colorama_init
from sys import stdout, exit as sys_exit
from inspect import getmembers, ismethod
from prettytable import PrettyTable

RS_COMMENT = r'(?:#|@{.*?})'

RE_STRING = re.compile(r'''('.*?'|".*?"(?<!\\"))''')

RE_FIRST_LINE = re.compile(r'^\s*(?:(?:declaration|unique|structure|object)\s+)?template (?:(?:[\S-]+/)+)?[\S-]+;$')
RE_MVN_TEMPLATE = re.compile(r'\$\{\S+\}')
RE_COMMENT = re.compile(RS_COMMENT)
RE_COMMENT_LINE = re.compile(r'^\s*' + RS_COMMENT + '.*$')
RE_ANNOTATION = re.compile(r'@\w*{.*?}', re.S)
RE_OPERATOR = re.compile(r'(.)([>=<!?]=|[+*=/-])(.)')

# Find usage and inclusion of components
RE_COMPONENT_INCLUDE = re.compile(r'^\s*[^#]?\s*include.*components/(?P<name>\w+)/config', re.M)
RE_COMPONENT_USE = re.compile(r'^\s*[^#]?\s*/software/components/(?P<name>\w+)/')
LINE_LENGTH_LIMIT = 120

# Simple regular-expression based checks that will be performed against all non-ignored lines
# Every pattern must provide a single capturing group named "error"
LINE_PATTERNS = {
    "Indentation should be a multiple of four spaces": re.compile(r'^(?!( {4})*(\S|$))(?P<error>\s*)'),
    "Spaces should be used instead of tabs": re.compile(r'(?P<error>\t+)'),
    "Trailing whitespace": re.compile(r'(?P<error>\s+$)'),
    "Use dict() instead of nlist()": re.compile(r'\b(?P<error>nlist)\s*\('),
    "Include statements no longer need curly braces": re.compile(r'''include\s+(?P<error>{[^;]+})'''),
    "Line is longer than %s characters" % LINE_LENGTH_LIMIT: re.compile(r'''^.{0,%s}(?P<error>.*?)$''' % LINE_LENGTH_LIMIT),
    "Commas should be followed by exactly one space": re.compile(r'(?P<error>,(?:\S|\s{2,}))'),
    "Whitespace before semicolon": re.compile(r'(?P<error>\s+;)'),
    "Semicolons should be followed exactly one space or end-of-line": re.compile(r';(?P<error>(?:\S|\s{2,}))'),
}

TAB_ARROW = u'\u2192'

DEBUG = False

class LineChecks:
    """More complex single line checks that require some logic to implement their checks"""

    def whitespace_around_operators(self, line, string_ranges):
        """Check a line of text to ensure that there is whitespace before and after all operators"""
        operators = RE_OPERATOR.finditer(line)

        passed = True
        message = ''
        messages = set()

        diagnosis = ' ' * len(line)

        for operator in operators:
            char_before, _, char_after = operator.groups()
            start, end = operator.span(2)

            if not inside_string(start, end, string_ranges):
                valid = True
                if char_before not in (' ', '\t'):
                    valid = False
                    messages.add('before')
                    start -= 1
                if char_after not in (' ', '\t'):
                    valid = False
                    messages.add('after')
                    end += 1

                if not valid:
                    debug_range(start, end, 'WS Operator', True)
                    diagnosis = diagnosis[:start] + ('^' * (end-start)) + diagnosis[end:]

                passed &= valid

        if not passed:
            messages = list(messages)
            messages.sort(None, None, True)
            message = 'Missing space %s operator' % ' and '.join(messages)
        return (passed, diagnosis.rstrip(), message)

def inside_string(i, j, string_ranges):
    """Returns true if the range described by i and j is contained within a range in the list string_ranges"""
    for s, e in string_ranges:
        if i >= s and j <= e:
            return True
    return False

def print_fileinfo(filename, line_number, message, vi=False):
    """Return a formatted string with filename, line_number and message.

    Keyword arguments:
    vi -- output line numbers in a format suitable for passing to editors such as vi (default False)
    """
    if vi:
        return '%s +%d #%s' % (filename, line_number, message)
    return '%s:%d: %s' % (filename, line_number, message)

def print_line(text):
    """Return a formatted line of text, replacing tabs with a visible character

    If stdout is a tty, a unicode rightwards arrow (u2192) will be used for tabs, otherwise the rarely used negation character (¬).
    This keeps the character counts in line with fixed-width columns while still making tabs distinguishable in output.
    """
    if stdout.isatty():
        text = text.replace('\t', TAB_ARROW)
    else:
        text = text.replace('\t', '¬')

    return ''.join([Fore.GREEN, text.rstrip('\n'), Fore.RESET])

def merge_diagnoses(args):
    """Merge lines of diagnosis produced by diagnose()"""
    if not args:
        return ''

    args = [a.rstrip() for a in args]
    result = [' '] * max(map(len, args))

    for text in args:
        for i, c in enumerate(text):
            if c != ' ':
                result[i] = c
    return ''.join(result).rstrip()

def print_diagnosis(diagnosis):
    """Format a line of diagnosis produced by diagnose() and/or merge_diagnoses()"""
    return ''.join([Fore.BLUE, diagnosis, Fore.RESET])

def debug_line(line, line_number):
    """Print debug information for a processed line of an input file"""
    if DEBUG:
        label = 'DEBUG: %04d %-12s |' % (line_number, '...')
        print ''.join([Style.DIM, Fore.CYAN, label, Style.RESET_ALL, line.replace('\t', TAB_ARROW)])

def debug_ignored_line(line, line_number):
    """Print debug information for an ignored line of an input file"""
    if DEBUG:
        label = 'DEBUG: %04d %-12s |' % (line_number, 'Ignored')
        print ''.join([Fore.CYAN, Style.DIM, label, Fore.RESET, line.replace('\t', TAB_ARROW), Style.RESET_ALL])

def debug_range(start, end, label, problem=False):
    """Print debug information referring to a range of characters in a single line"""
    if DEBUG:
        label = ('DEBUG: ^^^^ %-12s |' % label)
        diagnosis = diagnose(start, end)
        color = Fore.CYAN
        if problem:
            color = Fore.RED
        print ''.join([Style.DIM, Fore.CYAN, label, Style.BRIGHT, color, diagnosis, Style.RESET_ALL])

def diagnose(start, end):
    """Format a line of diagnosis markers from a range of character positions"""
    return (' ' * start) + ('^' * (end - start))

def print_report(filename, line_number, line, diagnosis, message, vi=False):
    """Print a full report of all problems found with a single line of a processed file"""
    print
    print print_fileinfo(filename, line_number, message, vi=vi)
    print print_line(line)
    print print_diagnosis(diagnosis)

def get_string_ranges(line):
    """Find all ranges of strings within a single line of text"""
    string_ranges = []
    strings = RE_STRING.finditer(line)

    if strings:
        for string in strings:
            s, e = string.span()
            debug_range(s, e, 'String Range')
            string_ranges.append((s, e))

    return string_ranges

def filestats_table(problem_stats):
    """Return a formatted table of problem counts per file"""
    t = PrettyTable(['Filename', 'Problems'])
    t.align['Filename'] = 'l'
    t.align['Problems'] = 'r'
    for filename, problem_count in problem_stats.iteritems():
        if problem_count > 0:
            t.add_row([filename, problem_count])
    t.sortby = 'Filename'
    return t

def find_annotation_blocks(text):
    """Find multi-line annotation blocks in a block of text"""
    result = []
    annotations = RE_ANNOTATION.finditer(text)
    for annotation in annotations:
        start_char, end_char = annotation.span()
        start_line = text[:start_char].count('\n') + 1
        end_line = start_line + text[start_char:end_char].count('\n')
        for i in range(start_line, end_line + 1):
            result.append(i)
    return result

def strip_trailing_comments(line, string_ranges):
    """Remove comments from the end of a line, ignoring anything within specified string ranges"""
    for comment in RE_COMMENT.finditer(line):
        # Does the candidate comment start inside a string?
        # If so, it's not really a comment.
        if not inside_string(comment.start(), comment.start()+1, string_ranges):
            debug_range(comment.start(), comment.end(), 'Comment', False)
            line = line[:comment.start()]
    return line.rstrip()

def check_line_component_use(line, components_included):
    """Check a line for usage of a component, flag a problem if any component is not in the list of included components."""
    diagnoses = []
    messages = []
    problem_count = 0

    for m in RE_COMPONENT_USE.finditer(line):
        if m.group('name') not in components_included:
            message = 'Component %s in use, but component config has not been included' % m.group('name')
            diagnoses.append(diagnose(*m.span('name')))
            messages.append(message)
            problem_count += 1

    return diagnoses, messages, problem_count

def check_line_patterns(line, string_ranges):
    """Check line against regular expressions in LINE_PATTERNS, ignoring code within specified string ranges."""
    diagnoses = []
    messages = set()
    problem_count = 0

    for message, pattern in LINE_PATTERNS.iteritems():
        matches = pattern.finditer(line)
        for match in matches:
            if match and match.group('error'):
                start, end = match.span('error')
                debug_range(start, end, 'Match', True)
                if not inside_string(start, end, string_ranges):
                    diagnoses.append(diagnose(start, end))
                    messages.add(message)
                    problem_count += 1

    return diagnoses, messages, problem_count

def check_line_methods(line, string_ranges):
    """Run checks defined as methods of LineChecks against line, ignoring code within specified string ranges."""
    diagnoses = []
    messages = set()
    problem_count = 0

    for _, check_method in getmembers(LineChecks(), predicate=ismethod):
        passed, diagnosis, message = check_method(line, string_ranges)
        if not passed:
            diagnoses.append(diagnosis)
            messages.add(message)
            problem_count += 1

    return diagnoses, messages, problem_count

def lint_line(line, line_number, components_included, first_line=False, allow_mvn_templates=False):
    """Run all lint checks against line and return any problems found."""
    debug_line(line, line_number)

    messages = set()
    diagnoses = []
    problem_count = 0

    if first_line:
        first_line = False
        if not RE_FIRST_LINE.match(line):
            if not (RE_MVN_TEMPLATE.match(line) and allow_mvn_templates):
                messages.add('First non-comment line must be the template type and name')
                diagnoses.append(diagnose(0, len(line)))
                problem_count += 1

    else:
        string_ranges = get_string_ranges(line)
        line = strip_trailing_comments(line, string_ranges)

        d, m, p = check_line_component_use(line, components_included)
        diagnoses += d
        messages.update(m)
        problem_count += p

        d, m, p = check_line_patterns(line, string_ranges)
        diagnoses += d
        messages.update(m)
        problem_count += p

        d, m, p = check_line_methods(line, string_ranges)
        diagnoses += d
        messages.update(m)
        problem_count += p

    return (diagnoses, messages, problem_count, first_line)

def lint_file(filename, allow_mvn_templates=False):
    """Run lint checks against all lines of a file."""
    global DEBUG
    reports = []
    file_problem_count = 0

    raw_text = open(filename).read()

    first_line = True
    ignore_lines = []

    # Identify annotation blocks and exclude them from linting
    # We will need special linting rules for these
    ignore_lines += find_annotation_blocks(raw_text)

    # Get list of all component configs included in template
    components_included = RE_COMPONENT_INCLUDE.findall(raw_text)

    for line_number, line in enumerate(raw_text.splitlines(), start=1):
        line = line.rstrip('\n')

        if line and line_number not in ignore_lines and not RE_COMMENT_LINE.match(line):
            diagnoses, messages, line_problem_count, first_line = lint_line(line, line_number, components_included, first_line, allow_mvn_templates)
            file_problem_count += line_problem_count

            if messages and diagnoses:
                reports.append([filename, line_number, line, merge_diagnoses(diagnoses), ', '.join(messages)])
        else:
            debug_ignored_line(line, line_number)

    return (reports, file_problem_count)

def main():
    """Main function"""
    parser = argparse.ArgumentParser(description='Linter for the pan language')
    parser.add_argument('paths', metavar='PATH', type=str, nargs='+', help='Paths of files to check')
    parser.add_argument('--vi', action='store_true', help='Output line numbers in a vi option style')
    parser.add_argument('--table', action='store_true', help='Display a table of per-file problem stats')
    parser.add_argument('--allow_mvn_templates', action='store_true', help='Allow use of maven templates')
    group_output = parser.add_mutually_exclusive_group()
    group_output.add_argument('--debug', action='store_true', help='Enable debug output')
    group_output.add_argument('--ide', action='store_true', help='Output machine-readable results for use by IDEs')
    args = parser.parse_args()

    # Only output colors sequences if the output is a terminal
    colorama_init(strip=(not stdout.isatty()) or args.ide)
    global DEBUG
    DEBUG = args.debug

    problems_found = 0

    reports = []
    problem_stats = {}

    for path in args.paths:
        for filename in glob(path):
            file_reports, file_problems = lint_file(filename, args.allow_mvn_templates)
            reports += file_reports
            problems_found += file_problems
            problem_stats[filename] = file_problems

    for report in reports:
        print_report(*report, vi=args.vi)

    if args.table:
        print
        print 'Problem count per file:'
        print filestats_table(problem_stats)

    print
    print '%d problems found in total' % problems_found

    if problems_found:
        return(1)

if __name__ == '__main__':
    sys_exit(main())
<commit_msg>Clarify type of match in debug text<commit_after>#!/usr/bin/env python
# coding=utf8
#
# Copyright 2016 Science & Technology Facilities Council
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Linter for the Pan language"""

import re
import argparse
from glob import glob
from colorama import Fore, Style, init as colorama_init
from sys import stdout, exit as sys_exit
from inspect import getmembers, ismethod
from prettytable import PrettyTable

RS_COMMENT = r'(?:#|@{.*?})'

RE_STRING = re.compile(r'''('.*?'|".*?"(?<!\\"))''')

RE_FIRST_LINE = re.compile(r'^\s*(?:(?:declaration|unique|structure|object)\s+)?template (?:(?:[\S-]+/)+)?[\S-]+;$')
RE_MVN_TEMPLATE = re.compile(r'\$\{\S+\}')
RE_COMMENT = re.compile(RS_COMMENT)
RE_COMMENT_LINE = re.compile(r'^\s*' + RS_COMMENT + '.*$')
RE_ANNOTATION = re.compile(r'@\w*{.*?}', re.S)
RE_OPERATOR = re.compile(r'(.)([>=<!?]=|[+*=/-])(.)')

# Find usage and inclusion of components
RE_COMPONENT_INCLUDE = re.compile(r'^\s*[^#]?\s*include.*components/(?P<name>\w+)/config', re.M)
RE_COMPONENT_USE = re.compile(r'^\s*[^#]?\s*/software/components/(?P<name>\w+)/')
LINE_LENGTH_LIMIT = 120

# Simple regular-expression based checks that will be performed against all non-ignored lines
# Every pattern must provide a single capturing group named "error"
LINE_PATTERNS = {
    "Indentation should be a multiple of four spaces": re.compile(r'^(?!( {4})*(\S|$))(?P<error>\s*)'),
    "Spaces should be used instead of tabs": re.compile(r'(?P<error>\t+)'),
    "Trailing whitespace": re.compile(r'(?P<error>\s+$)'),
    "Use dict() instead of nlist()": re.compile(r'\b(?P<error>nlist)\s*\('),
    "Include statements no longer need curly braces": re.compile(r'''include\s+(?P<error>{[^;]+})'''),
    "Line is longer than %s characters" % LINE_LENGTH_LIMIT: re.compile(r'''^.{0,%s}(?P<error>.*?)$''' % LINE_LENGTH_LIMIT),
    "Commas should be followed by exactly one space": re.compile(r'(?P<error>,(?:\S|\s{2,}))'),
    "Whitespace before semicolon": re.compile(r'(?P<error>\s+;)'),
    "Semicolons should be followed exactly one space or end-of-line": re.compile(r';(?P<error>(?:\S|\s{2,}))'),
}

TAB_ARROW = u'\u2192'

DEBUG = False

class LineChecks:
    """More complex single line checks that require some logic to implement their checks"""

    def whitespace_around_operators(self, line, string_ranges):
        """Check a line of text to ensure that there is whitespace before and after all operators"""
        operators = RE_OPERATOR.finditer(line)

        passed = True
        message = ''
        messages = set()

        diagnosis = ' ' * len(line)

        for operator in operators:
            char_before, _, char_after = operator.groups()
            start, end = operator.span(2)

            if not inside_string(start, end, string_ranges):
                valid = True
                if char_before not in (' ', '\t'):
                    valid = False
                    messages.add('before')
                    start -= 1
                if char_after not in (' ', '\t'):
                    valid = False
                    messages.add('after')
                    end += 1

                if not valid:
                    debug_range(start, end, 'WS Operator', True)
                    diagnosis = diagnosis[:start] + ('^' * (end-start)) + diagnosis[end:]

                passed &= valid

        if not passed:
            messages = list(messages)
            messages.sort(None, None, True)
            message = 'Missing space %s operator' % ' and '.join(messages)
        return (passed, diagnosis.rstrip(), message)

def inside_string(i, j, string_ranges):
    """Returns true if the range described by i and j is contained within a range in the list string_ranges"""
    for s, e in string_ranges:
        if i >= s and j <= e:
            return True
    return False

def print_fileinfo(filename, line_number, message, vi=False):
    """Return a formatted string with filename, line_number and message.

    Keyword arguments:
    vi -- output line numbers in a format suitable for passing to editors such as vi (default False)
    """
    if vi:
        return '%s +%d #%s' % (filename, line_number, message)
    return '%s:%d: %s' % (filename, line_number, message)

def print_line(text):
    """Return a formatted line of text, replacing tabs with a visible character

    If stdout is a tty, a unicode rightwards arrow (u2192) will be used for tabs, otherwise the rarely used negation character (¬).
    This keeps the character counts in line with fixed-width columns while still making tabs distinguishable in output.
    """
    if stdout.isatty():
        text = text.replace('\t', TAB_ARROW)
    else:
        text = text.replace('\t', '¬')

    return ''.join([Fore.GREEN, text.rstrip('\n'), Fore.RESET])

def merge_diagnoses(args):
    """Merge lines of diagnosis produced by diagnose()"""
    if not args:
        return ''

    args = [a.rstrip() for a in args]
    result = [' '] * max(map(len, args))

    for text in args:
        for i, c in enumerate(text):
            if c != ' ':
                result[i] = c
    return ''.join(result).rstrip()

def print_diagnosis(diagnosis):
    """Format a line of diagnosis produced by diagnose() and/or merge_diagnoses()"""
    return ''.join([Fore.BLUE, diagnosis, Fore.RESET])

def debug_line(line, line_number):
    """Print debug information for a processed line of an input file"""
    if DEBUG:
        label = 'DEBUG: %04d %-12s |' % (line_number, '...')
        print ''.join([Style.DIM, Fore.CYAN, label, Style.RESET_ALL, line.replace('\t', TAB_ARROW)])

def debug_ignored_line(line, line_number):
    """Print debug information for an ignored line of an input file"""
    if DEBUG:
        label = 'DEBUG: %04d %-12s |' % (line_number, 'Ignored')
        print ''.join([Fore.CYAN, Style.DIM, label, Fore.RESET, line.replace('\t', TAB_ARROW), Style.RESET_ALL])

def debug_range(start, end, label, problem=False):
    """Print debug information referring to a range of characters in a single line"""
    if DEBUG:
        label = ('DEBUG: ^^^^ %-12s |' % label)
        diagnosis = diagnose(start, end)
        color = Fore.CYAN
        if problem:
            color = Fore.RED
        print ''.join([Style.DIM, Fore.CYAN, label, Style.BRIGHT, color, diagnosis, Style.RESET_ALL])

def diagnose(start, end):
    """Format a line of diagnosis markers from a range of character positions"""
    return (' ' * start) + ('^' * (end - start))

def print_report(filename, line_number, line, diagnosis, message, vi=False):
    """Print a full report of all problems found with a single line of a processed file"""
    print
    print print_fileinfo(filename, line_number, message, vi=vi)
    print print_line(line)
    print print_diagnosis(diagnosis)

def get_string_ranges(line):
    """Find all ranges of strings within a single line of text"""
    string_ranges = []
    strings = RE_STRING.finditer(line)

    if strings:
        for string in strings:
            s, e = string.span()
            debug_range(s, e, 'String Range')
            string_ranges.append((s, e))

    return string_ranges

def filestats_table(problem_stats):
    """Return a formatted table of problem counts per file"""
    t = PrettyTable(['Filename', 'Problems'])
    t.align['Filename'] = 'l'
    t.align['Problems'] = 'r'
    for filename, problem_count in problem_stats.iteritems():
        if problem_count > 0:
            t.add_row([filename, problem_count])
    t.sortby = 'Filename'
    return t

def find_annotation_blocks(text):
    """Find multi-line annotation blocks in a block of text"""
    result = []
    annotations = RE_ANNOTATION.finditer(text)
    for annotation in annotations:
        start_char, end_char = annotation.span()
        start_line = text[:start_char].count('\n') + 1
        end_line = start_line + text[start_char:end_char].count('\n')
        for i in range(start_line, end_line + 1):
            result.append(i)
    return result

def strip_trailing_comments(line, string_ranges):
    """Remove comments from the end of a line, ignoring anything within specified string ranges"""
    for comment in RE_COMMENT.finditer(line):
        # Does the candidate comment start inside a string?
        # If so, it's not really a comment.
        if not inside_string(comment.start(), comment.start()+1, string_ranges):
            debug_range(comment.start(), comment.end(), 'Comment', False)
            line = line[:comment.start()]
    return line.rstrip()

def check_line_component_use(line, components_included):
    """Check a line for usage of a component, flag a problem if any component is not in the list of included components."""
    diagnoses = []
    messages = []
    problem_count = 0

    for m in RE_COMPONENT_USE.finditer(line):
        if m.group('name') not in components_included:
            message = 'Component %s in use, but component config has not been included' % m.group('name')
            diagnoses.append(diagnose(*m.span('name')))
            messages.append(message)
            problem_count += 1

    return diagnoses, messages, problem_count

def check_line_patterns(line, string_ranges):
    """Check line against regular expressions in LINE_PATTERNS, ignoring code within specified string ranges."""
    diagnoses = []
    messages = set()
    problem_count = 0

    for message, pattern in LINE_PATTERNS.iteritems():
        matches = pattern.finditer(line)
        for match in matches:
            if match and match.group('error'):
                start, end = match.span('error')
                debug_range(start, end, 'LineRE Match', True)
                if not inside_string(start, end, string_ranges):
                    diagnoses.append(diagnose(start, end))
                    messages.add(message)
                    problem_count += 1

    return diagnoses, messages, problem_count

def check_line_methods(line, string_ranges):
    """Run checks defined as methods of LineChecks against line, ignoring code within specified string ranges."""
    diagnoses = []
    messages = set()
    problem_count = 0

    for _, check_method in getmembers(LineChecks(), predicate=ismethod):
        passed, diagnosis, message = check_method(line, string_ranges)
        if not passed:
            diagnoses.append(diagnosis)
            messages.add(message)
            problem_count += 1

    return diagnoses, messages, problem_count

def lint_line(line, line_number, components_included, first_line=False, allow_mvn_templates=False):
    """Run all lint checks against line and return any problems found."""
    debug_line(line, line_number)

    messages = set()
    diagnoses = []
    problem_count = 0

    if first_line:
        first_line = False
        if not RE_FIRST_LINE.match(line):
            if not (RE_MVN_TEMPLATE.match(line) and allow_mvn_templates):
                messages.add('First non-comment line must be the template type and name')
                diagnoses.append(diagnose(0, len(line)))
                problem_count += 1

    else:
        string_ranges = get_string_ranges(line)
        line = strip_trailing_comments(line, string_ranges)

        d, m, p = check_line_component_use(line, components_included)
        diagnoses += d
        messages.update(m)
        problem_count += p

        d, m, p = check_line_patterns(line, string_ranges)
        diagnoses += d
        messages.update(m)
        problem_count += p

        d, m, p = check_line_methods(line, string_ranges)
        diagnoses += d
        messages.update(m)
        problem_count += p

    return (diagnoses, messages, problem_count, first_line)

def lint_file(filename, allow_mvn_templates=False):
    """Run lint checks against all lines of a file."""
    global DEBUG
    reports = []
    file_problem_count = 0

    raw_text = open(filename).read()

    first_line = True
    ignore_lines = []

    # Identify annotation blocks and exclude them from linting
    # We will need special linting rules for these
    ignore_lines += find_annotation_blocks(raw_text)

    # Get list of all component configs included in template
    components_included = RE_COMPONENT_INCLUDE.findall(raw_text)

    for line_number, line in enumerate(raw_text.splitlines(), start=1):
        line = line.rstrip('\n')

        if line and line_number not in ignore_lines and not RE_COMMENT_LINE.match(line):
            diagnoses, messages, line_problem_count, first_line = lint_line(line, line_number, components_included, first_line, allow_mvn_templates)
            file_problem_count += line_problem_count

            if messages and diagnoses:
                reports.append([filename, line_number, line, merge_diagnoses(diagnoses), ', '.join(messages)])
        else:
            debug_ignored_line(line, line_number)

    return (reports, file_problem_count)

def main():
    """Main function"""
    parser = argparse.ArgumentParser(description='Linter for the pan language')
    parser.add_argument('paths', metavar='PATH', type=str, nargs='+', help='Paths of files to check')
    parser.add_argument('--vi', action='store_true', help='Output line numbers in a vi option style')
    parser.add_argument('--table', action='store_true', help='Display a table of per-file problem stats')
    parser.add_argument('--allow_mvn_templates', action='store_true', help='Allow use of maven templates')
    group_output = parser.add_mutually_exclusive_group()
    group_output.add_argument('--debug', action='store_true', help='Enable debug output')
    group_output.add_argument('--ide', action='store_true', help='Output machine-readable results for use by IDEs')
    args = parser.parse_args()

    # Only output colors sequences if the output is a terminal
    colorama_init(strip=(not stdout.isatty()) or args.ide)
    global DEBUG
    DEBUG = args.debug

    problems_found = 0

    reports = []
    problem_stats = {}

    for path in args.paths:
        for filename in glob(path):
            file_reports, file_problems = lint_file(filename, args.allow_mvn_templates)
            reports += file_reports
            problems_found += file_problems
            problem_stats[filename] = file_problems

    for report in reports:
        print_report(*report, vi=args.vi)

    if args.table:
        print
        print 'Problem count per file:'
        print filestats_table(problem_stats)

    print
    print '%d problems found in total' % problems_found

    if problems_found:
        return(1)

if __name__ == '__main__':
    sys_exit(main())
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright 2014 Nikola Kovacevic <nikolak@outlook.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__author__ = 'Nikola Kovacevic'
__email__ = 'nikolak@outlook.com'
__version__ = '0.2.0dev'

import os
import ConfigParser
import json
import sqlite3
import codecs
import sys
from PyQt4 import QtGui
import webbrowser
import platform
import traceback
import logging
import shutil
from time import strftime
import copy
from collections import OrderedDict

from browsers import Chrome, Firefox

try:
    from RESTool import restoolgui
except ImportError:
    import restoolgui2

DEBUG = os.path.exists("log.txt")

if not DEBUG:
    err = sys.stderr
    out = sys.stdout
    sys.stderr = out  # disable stderr so that py2exe doesn't show that popup message

log = logging.getLogger('RESToolGUI')

if DEBUG:
    log.setLevel(logging.DEBUG)
    fh = logging.FileHandler('log.txt')
    fh.setLevel(logging.DEBUG)

    ch = logging.StreamHandler()
    ch.setLevel(logging.DEBUG)
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    ch.setFormatter(formatter)
    fh.setFormatter(formatter)
    log.addHandler(ch)
    log.addHandler(fh)

class RESToolUI(QtGui.QMainWindow, restoolgui.Ui_MainWindow):

    # noinspection PyUnresolvedReferences
    def __init__(self, parent=None):
        super(RESToolUI, self).__init__(parent)
        self.setupUi(self)
        self.labelMessage.setVisible(False)

        self.choices_first = OrderedDict({"None":None})
        self.choices_second = OrderedDict({"None":None})
        self.profile_choices_first = OrderedDict({"None":None})
        self.profile_choices_second = OrderedDict({"None":None})

        self.first_browser = None
        self.second_browser = None
        self.first_br_profile = None
        self.second_br_profile = None

        self.cboFirstBrowser.currentIndexChanged.connect(self._first_browser_changed)
        self.cboFirstBrowserProfile.currentIndexChanged.connect(self._first_browser_profile_changed)
        self.cboSecondBrowser.currentIndexChanged.connect(self._second_browser_changed)
        self.cboSecondBrowserProfile.currentIndexChanged.connect(self._second_browser_profile_changed)

        self.btnFirstToSecond.clicked.connect(self.migrate_first_to_second)
        self.btnSecondToFirst.clicked.connect(self.migrate_second_to_first)

        self.btnBackupFirst.clicked.connect(self.backup_first)
        self.btnBackupSecond.clicked.connect(self.backup_second)
        self.btnRestoreToFirst.clicked.connect(self.resotre_to_first)
        self.btnBackupSecond.clicked.connect(self.restore_to_second)

        self._set_available_browsers()
        self._set_available_profiles()

    # noinspection PyCallByClass,PyTypeChecker
    def _warn(self, msg, title="Warnning"):
        QtGui.QMessageBox.warning(self, title, msg,
                                  QtGui.QMessageBox.Ok)

    # noinspection PyCallByClass,PyTypeChecker
    def _info(self, msg, title="Info"):
        QtGui.QMessageBox.information(self, title, msg,
                                      QtGui.QMessageBox.Ok)

    # noinspection PyCallByClass,PyTypeChecker
    def _prompt(self, msg, title="Prompt"):
        result = QtGui.QMessageBox.question(self, title, msg,
                                            QtGui.QMessageBox.Cancel,
                                            QtGui.QMessageBox.No,
                                            QtGui.QMessageBox.Yes)

        if result == QtGui.QMessageBox.Cancel:
            return False
        elif result == QtGui.QMessageBox.No:
            return False
        elif result == QtGui.QMessageBox.Yes:
            return True
        else:
            log.debug("Returned unknown value from _prompt, %s" % result)
            return False

    # noinspection PyCallByClass,PyTypeChecker
    def _critical(self, msg, title="Error"):
        QtGui.QMessageBox.critical(self, title, msg, QtGui.QMessageBox.Ok)

    def _show_warning_label(self, msg):
        self.labelMessage.setText(msg)
        self.labelMessage.setVisible(True)

    def _set_available_browsers(self):
        chrome = Chrome()
        firefox = Firefox()

        if chrome.res_exists:
            self.choices_first['Chrome'] = chrome
            self.choices_second['Chrome'] = copy.copy(chrome)

        if firefox.res_exists:
            self.choices_first['Firefox'] = firefox
            self.choices_second['Firefox'] = copy.copy(firefox)

        for browser_name in self.choices_first:
            self.cboFirstBrowser.addItem(browser_name)
            self.cboSecondBrowser.addItem(browser_name)

        if not self.choices_first:
            self._warn("RES could not be found in Firefox or Chrome!")

    def _set_available_profiles(self):
        #TODO: Get real profiles and populate cbo

        for key in self.profile_choices_first:
            self.cboFirstBrowserProfile.addItem(key)

        for key in self.profile_choices_second:
            self.cboSecondBrowserProfile.addItem(key)

    def _first_browser_changed(self):
        self.first_browser = self.choices_first.get(str(self.cboFirstBrowser.currentText()))

        if not self.first_browser:
            self.first_br_profile = None
            self.cboFirstBrowserProfile.setCurrentIndex(0)

        self._update_ui_elements()

    def _first_browser_profile_changed(self):
        pass

    def _second_browser_changed(self):
        self.second_browser = self.choices_second.get(str(self.cboSecondBrowser.currentText()))

        if not self.second_browser:
            self.second_br_profile = None
            self.cboSecondBrowserProfile.setCurrentIndex(0)

        self._update_ui_elements()

    def _second_browser_profile_changed(self):
        pass

    def _update_ui_elements(self):
        if not self.first_browser and not self.second_browser:
            return

        if type(self.first_browser) == type(self.second_browser):
            if self.first_br_profile == self.second_br_profile:
                self._show_warning_label("Pick different browsers and/or profiles!")
                return

        if self.labelMessage.isVisible():
            self.labelMessage.setVisible(False)

        if self.first_browser and not self.second_browser:
            self.btnBackupFirst.setEnabled(True)
            self.btnRestoreToFirst.setEnabled(True)

            self.btnBackupSecond.setEnabled(False)
            self.btnRestoreToSecond.setEnabled(False)
            self.btnFirstToSecond.setEnabled(False)
            self.btnSecondToFirst.setEnabled(False)
            return

        if self.second_browser and not self.first_browser:
            self.btnBackupSecond.setEnabled(True)
            self.btnRestoreToSecond.setEnabled(True)

            self.btnBackupFirst.setEnabled(False)
            self.btnRestoreToFirst.setEnabled(False)
            self.btnFirstToSecond.setEnabled(False)
            self.btnSecondToFirst.setEnabled(False)
            return

        self.btnBackupFirst.setEnabled(True)
        self.btnBackupSecond.setEnabled(True)
        self.btnFirstToSecond.setEnabled(True)
        self.btnSecondToFirst.setEnabled(True)
        self.btnRestoreToFirst.setEnabled(True)
        self.btnRestoreToSecond.setEnabled(True)

    def _update_backups_list(self):
        pass

    def migrate_first_to_second(self):
        pass

    def migrate_second_to_first(self):
        pass

    def backup_first(self):
        pass

    def backup_second(self):
        pass

    def resotre_to_first(self):
        pass

    def restore_to_second(self):
        pass

def main():
    app = QtGui.QApplication(sys.argv)
    form = RESToolUI()
    form.show()
    app.exec_()

def cli():
    pass

if __name__ == '__main__':
    main()<commit_msg>Disable buttons on warnings, update db found label on changes<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

# Copyright 2014 Nikola Kovacevic <nikolak@outlook.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

__author__ = 'Nikola Kovacevic'
__email__ = 'nikolak@outlook.com'
__version__ = '0.2.0dev'

import os
import ConfigParser
import json
import sqlite3
import codecs
import sys
from PyQt4 import QtGui
import webbrowser
import platform
import traceback
import logging
import shutil
from time import strftime
import copy
from collections import OrderedDict

from browsers import Chrome, Firefox

try:
    from RESTool import restoolgui
except ImportError:
    import restoolgui2

DEBUG = os.path.exists("log.txt")

if not DEBUG:
    err = sys.stderr
    out = sys.stdout
    sys.stderr = out  # disable stderr so that py2exe doesn't show that popup message

log = logging.getLogger('RESToolGUI')

if DEBUG:
    log.setLevel(logging.DEBUG)
    fh = logging.FileHandler('log.txt')
    fh.setLevel(logging.DEBUG)

    ch = logging.StreamHandler()
    ch.setLevel(logging.DEBUG)
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    ch.setFormatter(formatter)
    fh.setFormatter(formatter)
    log.addHandler(ch)
    log.addHandler(fh)

class RESToolUI(QtGui.QMainWindow, restoolgui.Ui_MainWindow):

    # noinspection PyUnresolvedReferences
    def __init__(self, parent=None):
        super(RESToolUI, self).__init__(parent)
        self.setupUi(self)
        self.labelMessage.setVisible(False)

        self.choices_first = OrderedDict({"None":None})
        self.choices_second = OrderedDict({"None":None})
        self.profile_choices_first = OrderedDict({"None":None})
        self.profile_choices_second = OrderedDict({"None":None})

        self.first_browser = None
        self.second_browser = None
        self.first_br_profile = None
        self.second_br_profile = None

        self.cboFirstBrowser.currentIndexChanged.connect(self._first_browser_changed)
        self.cboFirstBrowserProfile.currentIndexChanged.connect(self._first_browser_profile_changed)
        self.cboSecondBrowser.currentIndexChanged.connect(self._second_browser_changed)
        self.cboSecondBrowserProfile.currentIndexChanged.connect(self._second_browser_profile_changed)

        self.btnFirstToSecond.clicked.connect(self.migrate_first_to_second)
        self.btnSecondToFirst.clicked.connect(self.migrate_second_to_first)

        self.btnBackupFirst.clicked.connect(self.backup_first)
        self.btnBackupSecond.clicked.connect(self.backup_second)
        self.btnRestoreToFirst.clicked.connect(self.resotre_to_first)
        self.btnBackupSecond.clicked.connect(self.restore_to_second)

        self._set_available_browsers()
        self._set_available_profiles()

    # noinspection PyCallByClass,PyTypeChecker
    def _warn(self, msg, title="Warnning"):
        QtGui.QMessageBox.warning(self, title, msg,
                                  QtGui.QMessageBox.Ok)

    # noinspection PyCallByClass,PyTypeChecker
    def _info(self, msg, title="Info"):
        QtGui.QMessageBox.information(self, title, msg,
                                      QtGui.QMessageBox.Ok)

    # noinspection PyCallByClass,PyTypeChecker
    def _prompt(self, msg, title="Prompt"):
        result = QtGui.QMessageBox.question(self, title, msg,
                                            QtGui.QMessageBox.Cancel,
                                            QtGui.QMessageBox.No,
                                            QtGui.QMessageBox.Yes)

        if result == QtGui.QMessageBox.Cancel:
            return False
        elif result == QtGui.QMessageBox.No:
            return False
        elif result == QtGui.QMessageBox.Yes:
            return True
        else:
            log.debug("Returned unknown value from _prompt, %s" % result)
            return False

    # noinspection PyCallByClass,PyTypeChecker
    def _critical(self, msg, title="Error"):
        QtGui.QMessageBox.critical(self, title, msg, QtGui.QMessageBox.Ok)

    def _show_warning_label(self, msg):
        self.labelMessage.setText(msg)
        self.labelMessage.setVisible(True)

    def _set_available_browsers(self):
        chrome = Chrome()
        firefox = Firefox()

        if chrome.res_exists:
            self.choices_first['Chrome'] = chrome
            self.choices_second['Chrome'] = copy.copy(chrome)

        if firefox.res_exists:
            self.choices_first['Firefox'] = firefox
            self.choices_second['Firefox'] = copy.copy(firefox)

        for browser_name in self.choices_first:
            self.cboFirstBrowser.addItem(browser_name)
            self.cboSecondBrowser.addItem(browser_name)

        if not self.choices_first:
            self._warn("RES could not be found in Firefox or Chrome!")

    def _set_available_profiles(self):
        #TODO: Get real profiles and populate cbo

        for key in self.profile_choices_first:
            self.cboFirstBrowserProfile.addItem(key)

        for key in self.profile_choices_second:
            self.cboSecondBrowserProfile.addItem(key)

    def _first_browser_changed(self):
        self.first_browser = self.choices_first.get(str(self.cboFirstBrowser.currentText()))

        if not self.first_browser:
            self.first_br_profile = None
            self.cboFirstBrowserProfile.setCurrentIndex(0)
        self._update_ui_elements()

    def _first_browser_profile_changed(self):
        pass

    def _second_browser_changed(self):
        self.second_browser = self.choices_second.get(str(self.cboSecondBrowser.currentText()))

        if not self.second_browser:
            self.second_br_profile = None
            self.cboSecondBrowserProfile.setCurrentIndex(0)

        self._update_ui_elements()

    def _second_browser_profile_changed(self):
        pass

    def _update_ui_elements(self):

        if self.first_browser:
            self.FirstBrowserRESLabel.setText("Yes" if
                                              self.first_browser.res_exists
                                              else "No")
        else:
            self.FirstBrowserRESLabel.setText("N/A")

        if self.second_browser:
            self.SecondBrowserRESLabel.setText("Yes" if
                                               self.second_browser.res_exists
                                               else "No")
        else:
            self.SecondBrowserRESLabel.setText("N/A")

        if not self.first_browser and not self.second_browser:
            self.btnBackupFirst.setEnabled(False)
            self.btnBackupSecond.setEnabled(False)
            self.btnFirstToSecond.setEnabled(False)
            self.btnSecondToFirst.setEnabled(False)
            self.btnRestoreToFirst.setEnabled(False)
            self.btnRestoreToSecond.setEnabled(False)
            return

        if type(self.first_browser) == type(self.second_browser):
            if self.first_br_profile == self.second_br_profile:
                self._show_warning_label("Pick different browsers and/or profiles!")
                self.btnBackupFirst.setEnabled(False)
                self.btnBackupSecond.setEnabled(False)
                self.btnFirstToSecond.setEnabled(False)
                self.btnSecondToFirst.setEnabled(False)
                self.btnRestoreToFirst.setEnabled(False)
                self.btnRestoreToSecond.setEnabled(False)
                return

        if self.labelMessage.isVisible():
            self.labelMessage.setVisible(False)

        if self.first_browser and not self.second_browser:
            self.btnBackupFirst.setEnabled(True)
            self.btnRestoreToFirst.setEnabled(True)

            self.btnBackupSecond.setEnabled(False)
            self.btnRestoreToSecond.setEnabled(False)
            self.btnFirstToSecond.setEnabled(False)
            self.btnSecondToFirst.setEnabled(False)
            return

        if self.second_browser and not self.first_browser:
            self.btnBackupSecond.setEnabled(True)
            self.btnRestoreToSecond.setEnabled(True)

            self.btnBackupFirst.setEnabled(False)
            self.btnRestoreToFirst.setEnabled(False)
            self.btnFirstToSecond.setEnabled(False)
            self.btnSecondToFirst.setEnabled(False)
            return

        self.btnBackupFirst.setEnabled(True)
        self.btnBackupSecond.setEnabled(True)
        self.btnFirstToSecond.setEnabled(True)
        self.btnSecondToFirst.setEnabled(True)
        self.btnRestoreToFirst.setEnabled(True)
        self.btnRestoreToSecond.setEnabled(True)

    def _update_backups_list(self):
        pass

    def migrate_first_to_second(self):
        pass

    def migrate_second_to_first(self):
        pass

    def backup_first(self):
        pass

    def backup_second(self):
        pass

    def resotre_to_first(self):
        pass

    def restore_to_second(self):
        pass

def main():
    app = QtGui.QApplication(sys.argv)
    form = RESToolUI()
    form.show()
    app.exec_()

def cli():
    pass

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before># -*- coding: utf8 -*-
import imp
import logging
import os
import re
import shutil
import sys

from lobster import fs, util
from lobster.cmssw import sandbox
from lobster.core.dataset import ProductionDataset
from lobster.core.task import *
from lobster.util import Configurable

logger = logging.getLogger('lobster.workflow')

class Category(Configurable):
    """
    Resource specification for a group of
    :class:`~lobster.core.workflow.Workflow`s.

    This information will be passed on to `WorkQueue`, which will forcibly
    terminate tasks of :class:`Workflow` in the group that exceed the
    specified resources.

    Parameters
    ----------
        name : str
            The name of the resource group.
        cores : int
            The number of cores a task of the group should use.
        runtime : int
            The runtime of the task in seconds.  Lobster will add a grace
            period to this time, and try to adjust the task size such that
            this runtime is achieved.
        memory : int
            How much memory a task is allowed to use, in megabytes.
        disk : int
            How much disk a task is allowed to use, in megabytes.
        tasks : int
            How many tasks should be in the queue (running or waiting) at
            the same time.
    """
    _mutable = {
            'tasks': (None, None, tuple())
    }

    def __init__(self,
            name,
            cores=1,
            runtime=None,
            memory=None,
            disk=None,
            tasks=None
            ):
        self.name = name
        self.cores = cores
        self.runtime = runtime
        self.memory = memory
        self.disk = disk
        self.tasks = tasks

    def __eq__(self, other):
        return self.name == other.name

    def __hash__(self):
        return hash(self.name)

    def wq(self):
        res = {}
        if self.runtime:
            res['wall_time'] = max(30 * 60, int(1.5 * self.runtime)) * int(1e6)
        if self.memory:
            res['memory'] = self.memory
        if self.cores:
            res['cores'] = self.cores
        if self.disk:
            res['disk'] = self.disk
        return res

class Workflow(Configurable):
    """
    A specification for processing a dataset.

    Parameters
    ----------
        label : str
            The shorthand name of the workflow.  This is used as a
            reference throughout Lobster.
        dataset : Dataset
            The specification of data to be processed.  Can be any of the
            dataset related classes.
        category : Category
            The category of resource specification this workflow belongs
            to.
        publish_label : str
            The label to be used for the publication database.
        merge_cleanup : bool
            Delete merged output files.
        merge_size : str
            Activates output file merging when set.  Accepts the suffixes
            *k*, *m*, *g* for kilobyte, megabyte, …
        sandbox : str
            Path to a sandbox to re-use.  This sandbox will be copied over
            to the current working directory.
        sandbox_release : str
            The path to the CMSSW release to be used as a sandbox.
            Defaults to the environment variable `LOCALRT`.
        sandbox_blacklist : list
            A specification of paths to not pack into the sandbox.
        command : str
            Which executable to run (for non-CMSSW workflows)
        extra_inputs : list
            Additional inputs outside the sandbox needed to process the
            workflow.
        arguments : list
            Arguments to pass to the executable.
        unique_arguments : list
            A list of arguments.  Each element of the dataset is processed
            once for each argument in this list.  The unique argument is
            also passed to the executable.

            TODO: should really be in the dataset specification
        outputs : list
            A list of strings which specifies the files produced by the
            workflow.  Will be automatically determined for CMSSW
            workflows.
        output_format : str
            How the output files should be renamed on the storage element.
            This is a new-style format string, allowing for the fields
            `base`, `id`, and `ext`, for the basename of the output file,
            the ID of the task, and the extension of the output file.
        parent : str
            The label of the parent workflow, if any.

            TODO: should really be a dataset specfication
        local : bool
            If set to `True`, Lobster will assume this workflow's input is
            present on the output storage element.
        cmssw_config : str
            The CMSSW configuration to use, if any.
        globaltag : str
            Which GlobalTag this workflow uses.  Needed for publication of
            CMSSW workflows, and can be automatically determined for these.

            TODO: check that the globaltag is determined independently of
            the outputs.
        edm_output : bool
            Autodetermined when outputs are determined automatically.
            Tells Lobster if the output of this workflow is in EDM format.
    """
    _mutable = {}
    def __init__(self,
            label,
            dataset,
            category=None,
            publish_label=None,
            merge_cleanup=True,
            merge_size=-1,
            sandbox=None,
            sandbox_release=None,
            sandbox_blacklist=None,
            command='cmsRun',
            extra_inputs=None,
            arguments=None,
            unique_arguments=None,
            outputs=None,
            output_format="{base}_{id}.{ext}",
            parent=None,
            local=False,
            cmssw_config=None,
            globaltag=None,
            edm_output=True):
        self.label = label
        if not re.match(r'^[A-Za-z][A-Za-z0-9_]*$', label):
            raise ValueError("Workflow label contains illegal characters: {}".format(label))
        self.category = category if category else label
        self.dataset = dataset

        self.publish_label = publish_label if publish_label else label

        self.merge_size = self.__check_merge(merge_size)
        self.merge_cleanup = merge_cleanup

        self.cmd = command
        self.extra_inputs = extra_inputs if extra_inputs else []
        self.args = arguments if arguments else []
        self.unique_args = unique_arguments if unique_arguments else [None]
        self._outputs = outputs
        self.outputformat = output_format

        self.dependents = []
        self.prerequisite = parent

        self.pset = cmssw_config
        self.globaltag = globaltag
        self.local = local or hasattr(dataset, 'files')
        self.edm_output = edm_output

        self.sandbox = sandbox
        if sandbox_release is not None:
            self.sandbox_release = sandbox_release
        else:
            try:
                self.sandbox_release = os.environ['LOCALRT']
            except:
                raise AttributeError("Need to be either in a `cmsenv` or specify a sandbox release!")
        self.sandbox_blacklist = sandbox_blacklist

    def __check_merge(self, size):
        if size <= 0:
            return size

        orig = size
        if isinstance(size, basestring):
            unit = size[-1].lower()
            try:
                size = float(size[:-1])
                if unit == 'k':
                    size *= 1000
                elif unit == 'm':
                    size *= 1e6
                elif unit == 'g':
                    size *= 1e9
                else:
                    size = -1
            except ValueError:
                size = -1
        if size > 0:
            logger.info('merging outputs up to {0} bytes'.format(size))
        else:
            logger.error('merging disabled due to malformed size {0}'.format(orig))
        return size

    def register(self, wflow):
        """Add the workflow `wflow` to the dependents.
        """
        logger.info("marking {0} to be downstream of {1}".format(wflow.label, self.label))
        if len(self._outputs) != 1:
            raise NotImplementedError("dependents for {0} output files not yet supported".format(len(self._outputs)))
        self.dependents.append(wflow)

    def family(self):
        """Returns a flattened hierarchy tree
        """
        yield self
        for d in self.dependents:
            for member in d.family():
                yield member

    def copy_inputs(self, basedirs, overwrite=False):
        """Make a copy of extra input files.

        Includes CMSSW parameter set if specified.  Already present files
        will not be overwritten unless specified.
        """
        if not os.path.exists(self.workdir):
            os.makedirs(self.workdir)

        if self.pset:
            shutil.copy(util.findpath(basedirs, self.pset), os.path.join(self.workdir, os.path.basename(self.pset)))

        if self.extra_inputs is None:
            return []

        def copy_file(fn):
            source = os.path.abspath(util.findpath(basedirs, fn))
            target = os.path.join(self.workdir, os.path.basename(fn))

            if not os.path.exists(target) or overwrite:
                if not os.path.exists(os.path.dirname(target)):
                    os.makedirs(os.path.dirname(target))

                logger.debug("copying '{0}' to '{1}'".format(source, target))
                if os.path.isfile(source):
                    shutil.copy(source, target)
                elif os.path.isdir(source):
                    shutil.copytree(source, target)
                else:
                    raise NotImplementedError

            return target

        files = map(copy_file, self.extra_inputs)
        self.extra_inputs = files

    def determine_outputs(self, basedirs):
        """Determine output files for CMSSW tasks.
        """
        self._outputs = []
        # Save determined outputs to the configuration in the
        # working directory.
        update_config = True

        # To avoid problems loading configs that use the VarParsing module
        sys.argv = [os.path.basename(self.pset)] + self.args
        with open(util.findpath(basedirs, self.pset), 'r') as f:
            source = imp.load_source('cms_config_source', self.pset, f)
            process = source.process
            if hasattr(process, 'GlobalTag') and hasattr(process.GlobalTag.globaltag, 'value'):
                self.global_tag = process.GlobalTag.globaltag.value()
            for label, module in process.outputModules.items():
                self._outputs.append(module.fileName.value())
            if 'TFileService' in process.services:
                self._outputs.append(process.services['TFileService'].fileName.value())
                self.edm_output = False

            logger.info("workflow {0}: adding output file(s) '{1}'".format(self.label, ', '.join(self._outputs)))

    def setup(self, workdir, basedirs):
        self.workdir = os.path.join(workdir, self.label)

        if self.sandbox:
            self.version, self.sandbox = sandbox.recycle(self.sandbox, workdir)
        else:
            self.version, self.sandbox = sandbox.package(
                    self.sandbox_release,
                    workdir,
                    self.sandbox_blacklist)

        self.copy_inputs(basedirs)
        if self.pset and not self._outputs:
            self.determine_outputs(basedirs)

        # Working directory for workflow
        # TODO Should we really check if this already exists?  IMO that
        # constitutes an error, since we really should create the workflow!
        if not os.path.exists(self.workdir):
            os.makedirs(self.workdir)
        # Create the stageout directory
        if not fs.exists(self.label):
            fs.makedirs(self.label)
        else:
            if len(list(fs.ls(self.label))) > 0:
                msg = 'stageout directory is not empty: {0}'
                raise IOError(msg.format(fs.__getattr__('lfn2pfn')(self.label)))

    def handler(self, id_, files, lumis, taskdir, merge=False):
        if merge:
            return MergeTaskHandler(id_, self.label, files, lumis, list(self.outputs(id_)), taskdir)
        elif isinstance(self.dataset, ProductionDataset):
            return ProductionTaskHandler(id_, self.label, lumis, list(self.outputs(id_)), taskdir)
        else:
            return TaskHandler(id_, self.label, files, lumis, list(self.outputs(id_)), taskdir, local=self.local)

    def outputs(self, id):
        for fn in self._outputs:
            base, ext = os.path.splitext(fn)
            outfn = self.outputformat.format(base=base, ext=ext[1:], id=id)
            yield fn, os.path.join(self.label, outfn)

    def adjust(self, params, taskdir, inputs, outputs, merge, reports=None, unique=None):
        cmd = self.cmd
        args = self.args
        pset = self.pset

        inputs.append((self.sandbox, 'sandbox.tar.bz2', True))
        if merge:
            inputs.append((os.path.join(os.path.dirname(__file__), 'data', 'merge_reports.py'), 'merge_reports.py', True))
            inputs.append((os.path.join(os.path.dirname(__file__), 'data', 'task.py'), 'task.py', True))
            inputs.extend((r, "_".join(os.path.normpath(r).split(os.sep)[-3:]), False) for r in reports)

            if self.edm_output:
                args = ['output=' + self._outputs[0]]
                pset = os.path.join(os.path.dirname(__file__), 'data', 'merge_cfg.py')
            else:
                cmd = 'hadd'
                args = ['-n', '0', '-f', self._outputs[0]]
                pset = None
                params['append inputs to args'] = True

            params['prologue'] = None
            params['epilogue'] = ['python', 'merge_reports.py', 'report.json'] \
                    + ["_".join(os.path.normpath(r).split(os.sep)[-3:]) for r in reports]
        else:
            inputs.extend((i, os.path.basename(i), True) for i in self.extra_inputs)

            if unique:
                args.append(unique)
            if pset:
                pset = os.path.join(self.workdir, pset)
            if self.category.runtime:
                # cap task runtime at desired runtime + 10 minutes grace
                # period (CMSSW 7.4 and higher only)
                params['task runtime'] = self.category.runtime + 10 * 60
            params['cores'] = self.category.cores

        if pset:
            inputs.append((pset, os.path.basename(pset), True))
            outputs.append((os.path.join(taskdir, 'report.xml.gz'), 'report.xml.gz'))

            params['pset'] = os.path.basename(pset)

        params['executable'] = cmd
        params['arguments'] = args
        if isinstance(self.dataset, ProductionDataset):
            params['mask']['events'] = self.dataset.events_per_task
            params['mask']['events per lumi'] = self.dataset.events_per_lumi
            params['randomize seeds'] = self.dataset.randomize_seeds
        else:
            params['mask']['events'] = -1
<commit_msg>Remove the grace-period from the CMSSW settings.<commit_after># -*- coding: utf8 -*-
import imp
import logging
import os
import re
import shutil
import sys

from lobster import fs, util
from lobster.cmssw import sandbox
from lobster.core.dataset import ProductionDataset
from lobster.core.task import *
from lobster.util import Configurable

logger = logging.getLogger('lobster.workflow')

class Category(Configurable):
    """
    Resource specification for a group of
    :class:`~lobster.core.workflow.Workflow`s.

    This information will be passed on to `WorkQueue`, which will forcibly
    terminate tasks of :class:`Workflow` in the group that exceed the
    specified resources.

    Parameters
    ----------
        name : str
            The name of the resource group.
        cores : int
            The number of cores a task of the group should use.
        runtime : int
            The runtime of the task in seconds.  Lobster will add a grace
            period to this time, and try to adjust the task size such that
            this runtime is achieved.
        memory : int
            How much memory a task is allowed to use, in megabytes.
        disk : int
            How much disk a task is allowed to use, in megabytes.
        tasks : int
            How many tasks should be in the queue (running or waiting) at
            the same time.
    """
    _mutable = {
            'tasks': (None, None, tuple())
    }

    def __init__(self,
            name,
            cores=1,
            runtime=None,
            memory=None,
            disk=None,
            tasks=None
            ):
        self.name = name
        self.cores = cores
        self.runtime = runtime
        self.memory = memory
        self.disk = disk
        self.tasks = tasks

    def __eq__(self, other):
        return self.name == other.name

    def __hash__(self):
        return hash(self.name)

    def wq(self):
        res = {}
        if self.runtime:
            res['wall_time'] = max(30 * 60, int(1.5 * self.runtime)) * int(1e6)
        if self.memory:
            res['memory'] = self.memory
        if self.cores:
            res['cores'] = self.cores
        if self.disk:
            res['disk'] = self.disk
        return res

class Workflow(Configurable):
    """
    A specification for processing a dataset.

    Parameters
    ----------
        label : str
            The shorthand name of the workflow.  This is used as a
            reference throughout Lobster.
        dataset : Dataset
            The specification of data to be processed.  Can be any of the
            dataset related classes.
        category : Category
            The category of resource specification this workflow belongs
            to.
        publish_label : str
            The label to be used for the publication database.
        merge_cleanup : bool
            Delete merged output files.
        merge_size : str
            Activates output file merging when set.  Accepts the suffixes
            *k*, *m*, *g* for kilobyte, megabyte, …
        sandbox : str
            Path to a sandbox to re-use.  This sandbox will be copied over
            to the current working directory.
        sandbox_release : str
            The path to the CMSSW release to be used as a sandbox.
            Defaults to the environment variable `LOCALRT`.
        sandbox_blacklist : list
            A specification of paths to not pack into the sandbox.
        command : str
            Which executable to run (for non-CMSSW workflows)
        extra_inputs : list
            Additional inputs outside the sandbox needed to process the
            workflow.
        arguments : list
            Arguments to pass to the executable.
        unique_arguments : list
            A list of arguments.  Each element of the dataset is processed
            once for each argument in this list.  The unique argument is
            also passed to the executable.

            TODO: should really be in the dataset specification
        outputs : list
            A list of strings which specifies the files produced by the
            workflow.  Will be automatically determined for CMSSW
            workflows.
        output_format : str
            How the output files should be renamed on the storage element.
            This is a new-style format string, allowing for the fields
            `base`, `id`, and `ext`, for the basename of the output file,
            the ID of the task, and the extension of the output file.
        parent : str
            The label of the parent workflow, if any.

            TODO: should really be a dataset specfication
        local : bool
            If set to `True`, Lobster will assume this workflow's input is
            present on the output storage element.
        cmssw_config : str
            The CMSSW configuration to use, if any.
        globaltag : str
            Which GlobalTag this workflow uses.  Needed for publication of
            CMSSW workflows, and can be automatically determined for these.

            TODO: check that the globaltag is determined independently of
            the outputs.
        edm_output : bool
            Autodetermined when outputs are determined automatically.
            Tells Lobster if the output of this workflow is in EDM format.
    """
    _mutable = {}
    def __init__(self,
            label,
            dataset,
            category=None,
            publish_label=None,
            merge_cleanup=True,
            merge_size=-1,
            sandbox=None,
            sandbox_release=None,
            sandbox_blacklist=None,
            command='cmsRun',
            extra_inputs=None,
            arguments=None,
            unique_arguments=None,
            outputs=None,
            output_format="{base}_{id}.{ext}",
            parent=None,
            local=False,
            cmssw_config=None,
            globaltag=None,
            edm_output=True):
        self.label = label
        if not re.match(r'^[A-Za-z][A-Za-z0-9_]*$', label):
            raise ValueError("Workflow label contains illegal characters: {}".format(label))
        self.category = category if category else label
        self.dataset = dataset

        self.publish_label = publish_label if publish_label else label

        self.merge_size = self.__check_merge(merge_size)
        self.merge_cleanup = merge_cleanup

        self.cmd = command
        self.extra_inputs = extra_inputs if extra_inputs else []
        self.args = arguments if arguments else []
        self.unique_args = unique_arguments if unique_arguments else [None]
        self._outputs = outputs
        self.outputformat = output_format

        self.dependents = []
        self.prerequisite = parent

        self.pset = cmssw_config
        self.globaltag = globaltag
        self.local = local or hasattr(dataset, 'files')
        self.edm_output = edm_output

        self.sandbox = sandbox
        if sandbox_release is not None:
            self.sandbox_release = sandbox_release
        else:
            try:
                self.sandbox_release = os.environ['LOCALRT']
            except:
                raise AttributeError("Need to be either in a `cmsenv` or specify a sandbox release!")
        self.sandbox_blacklist = sandbox_blacklist

    def __check_merge(self, size):
        if size <= 0:
            return size

        orig = size
        if isinstance(size, basestring):
            unit = size[-1].lower()
            try:
                size = float(size[:-1])
                if unit == 'k':
                    size *= 1000
                elif unit == 'm':
                    size *= 1e6
                elif unit == 'g':
                    size *= 1e9
                else:
                    size = -1
            except ValueError:
                size = -1
        if size > 0:
            logger.info('merging outputs up to {0} bytes'.format(size))
        else:
            logger.error('merging disabled due to malformed size {0}'.format(orig))
        return size

    def register(self, wflow):
        """Add the workflow `wflow` to the dependents.
        """
        logger.info("marking {0} to be downstream of {1}".format(wflow.label, self.label))
        if len(self._outputs) != 1:
            raise NotImplementedError("dependents for {0} output files not yet supported".format(len(self._outputs)))
        self.dependents.append(wflow)

    def family(self):
        """Returns a flattened hierarchy tree
        """
        yield self
        for d in self.dependents:
            for member in d.family():
                yield member

    def copy_inputs(self, basedirs, overwrite=False):
        """Make a copy of extra input files.

        Includes CMSSW parameter set if specified.  Already present files
        will not be overwritten unless specified.
        """
        if not os.path.exists(self.workdir):
            os.makedirs(self.workdir)

        if self.pset:
            shutil.copy(util.findpath(basedirs, self.pset), os.path.join(self.workdir, os.path.basename(self.pset)))

        if self.extra_inputs is None:
            return []

        def copy_file(fn):
            source = os.path.abspath(util.findpath(basedirs, fn))
            target = os.path.join(self.workdir, os.path.basename(fn))

            if not os.path.exists(target) or overwrite:
                if not os.path.exists(os.path.dirname(target)):
                    os.makedirs(os.path.dirname(target))

                logger.debug("copying '{0}' to '{1}'".format(source, target))
                if os.path.isfile(source):
                    shutil.copy(source, target)
                elif os.path.isdir(source):
                    shutil.copytree(source, target)
                else:
                    raise NotImplementedError

            return target

        files = map(copy_file, self.extra_inputs)
        self.extra_inputs = files

    def determine_outputs(self, basedirs):
        """Determine output files for CMSSW tasks.
        """
        self._outputs = []
        # Save determined outputs to the configuration in the
        # working directory.
        update_config = True

        # To avoid problems loading configs that use the VarParsing module
        sys.argv = [os.path.basename(self.pset)] + self.args
        with open(util.findpath(basedirs, self.pset), 'r') as f:
            source = imp.load_source('cms_config_source', self.pset, f)
            process = source.process
            if hasattr(process, 'GlobalTag') and hasattr(process.GlobalTag.globaltag, 'value'):
                self.global_tag = process.GlobalTag.globaltag.value()
            for label, module in process.outputModules.items():
                self._outputs.append(module.fileName.value())
            if 'TFileService' in process.services:
                self._outputs.append(process.services['TFileService'].fileName.value())
                self.edm_output = False

            logger.info("workflow {0}: adding output file(s) '{1}'".format(self.label, ', '.join(self._outputs)))

    def setup(self, workdir, basedirs):
        self.workdir = os.path.join(workdir, self.label)

        if self.sandbox:
            self.version, self.sandbox = sandbox.recycle(self.sandbox, workdir)
        else:
            self.version, self.sandbox = sandbox.package(
                    self.sandbox_release,
                    workdir,
                    self.sandbox_blacklist)

        self.copy_inputs(basedirs)
        if self.pset and not self._outputs:
            self.determine_outputs(basedirs)

        # Working directory for workflow
        # TODO Should we really check if this already exists?  IMO that
        # constitutes an error, since we really should create the workflow!
        if not os.path.exists(self.workdir):
            os.makedirs(self.workdir)
        # Create the stageout directory
        if not fs.exists(self.label):
            fs.makedirs(self.label)
        else:
            if len(list(fs.ls(self.label))) > 0:
                msg = 'stageout directory is not empty: {0}'
                raise IOError(msg.format(fs.__getattr__('lfn2pfn')(self.label)))

    def handler(self, id_, files, lumis, taskdir, merge=False):
        if merge:
            return MergeTaskHandler(id_, self.label, files, lumis, list(self.outputs(id_)), taskdir)
        elif isinstance(self.dataset, ProductionDataset):
            return ProductionTaskHandler(id_, self.label, lumis, list(self.outputs(id_)), taskdir)
        else:
            return TaskHandler(id_, self.label, files, lumis, list(self.outputs(id_)), taskdir, local=self.local)

    def outputs(self, id):
        for fn in self._outputs:
            base, ext = os.path.splitext(fn)
            outfn = self.outputformat.format(base=base, ext=ext[1:], id=id)
            yield fn, os.path.join(self.label, outfn)

    def adjust(self, params, taskdir, inputs, outputs, merge, reports=None, unique=None):
        cmd = self.cmd
        args = self.args
        pset = self.pset

        inputs.append((self.sandbox, 'sandbox.tar.bz2', True))
        if merge:
            inputs.append((os.path.join(os.path.dirname(__file__), 'data', 'merge_reports.py'), 'merge_reports.py', True))
            inputs.append((os.path.join(os.path.dirname(__file__), 'data', 'task.py'), 'task.py', True))
            inputs.extend((r, "_".join(os.path.normpath(r).split(os.sep)[-3:]), False) for r in reports)

            if self.edm_output:
                args = ['output=' + self._outputs[0]]
                pset = os.path.join(os.path.dirname(__file__), 'data', 'merge_cfg.py')
            else:
                cmd = 'hadd'
                args = ['-n', '0', '-f', self._outputs[0]]
                pset = None
                params['append inputs to args'] = True

            params['prologue'] = None
            params['epilogue'] = ['python', 'merge_reports.py', 'report.json'] \
                    + ["_".join(os.path.normpath(r).split(os.sep)[-3:]) for r in reports]
        else:
            inputs.extend((i, os.path.basename(i), True) for i in self.extra_inputs)

            if unique:
                args.append(unique)
            if pset:
                pset = os.path.join(self.workdir, pset)
            if self.category.runtime:
                # cap task runtime at desired runtime (CMSSW 7.4 and higher
                # only)
                params['task runtime'] = self.category.runtime
            params['cores'] = self.category.cores

        if pset:
            inputs.append((pset, os.path.basename(pset), True))
            outputs.append((os.path.join(taskdir, 'report.xml.gz'), 'report.xml.gz'))

            params['pset'] = os.path.basename(pset)

        params['executable'] = cmd
        params['arguments'] = args
        if isinstance(self.dataset, ProductionDataset):
            params['mask']['events'] = self.dataset.events_per_task
            params['mask']['events per lumi'] = self.dataset.events_per_lumi
            params['randomize seeds'] = self.dataset.randomize_seeds
        else:
            params['mask']['events'] = -1
<|endoftext|>


<commit_before>import os
import sqlite3
from FWCore.PythonUtilities.LumiList import LumiList

class SQLInterface:
    def __init__(self, config):
        self.config = config
        self.db_path = os.path.join(config['workdir'], "lobster.db")
        self.db = sqlite3.connect(self.db_path) #to do: add handling to check for lost jobs from previous execution, set status to failed
        self.db.execute("create table if not exists jobits(dataset, input_file, run, lumi, status, num_attempts, host, exit_code, run_time, startup_time)")

    def disconnect(self):
        self.db.close()

    def register_jobits(self, das):
        datasets = [x['dataset'] for x in self.config['tasks']]
        for dataset in datasets:
            dataset_info = das[dataset]
            if self.config.has_key('lumi mask'):
                lumi_mask = LumiList(filename=config['lumi mask'])
                for file in dataset_info.files:
                    dataset_info.lumis[file] = lumi_mask.filterLumis(dataset_info.lumis[file])

            for file in dataset_info.files:
                columns = [(dataset_info.dataset, file, run, lumi) for (run, lumi) in dataset_info.lumis[file]]
                self.db.executemany("insert into jobits(dataset, input_file, run, lumi, status, num_attempts) values (?, ?, ?, ?, 'registered', 0)", columns)

        self.db.commit()

    def pop_jobits(self, size, old_status, new_status):
        input_files = []
        lumis = []
        for input_file, run, lumi in self.db.execute("select input_file, run, lumi from jobits where status=? limit ?", (old_status, size)):
            input_files.append(input_file)
            lumis.append((run, lumi))
            self.db.execute("update jobits set status=? where run=? and lumi=?", (new_status, run, lumi))

        self.db.commit()

        job_parameters = [set(input_files), LumiList(lumis=lumis).getVLuminosityBlockRange()]

        return job_parameters

<commit_msg>Adding job id to table.<commit_after>import os
import sqlite3
from FWCore.PythonUtilities.LumiList import LumiList

class SQLInterface:
    def __init__(self, config):
        self.config = config
        self.db_path = os.path.join(config['workdir'], "lobster.db")
        self.db = sqlite3.connect(self.db_path) #to do: add handling to check for lost jobs from previous execution, set status to failed
        self.db.execute("create table if not exists jobits(job_id, dataset, input_file, run, lumi, status, num_attempts, host, exit_code, run_time, startup_time)")
        self.job_id_counter = 0
        self.db.commit()

    def disconnect(self):
        self.db.close()

    def register_jobits(self, das):
        datasets = [x['dataset'] for x in self.config['tasks']]
        for dataset in datasets:
            dataset_info = das[dataset]
            if self.config.has_key('lumi mask'):
                lumi_mask = LumiList(filename=config['lumi mask'])
                for file in dataset_info.files:
                    dataset_info.lumis[file] = lumi_mask.filterLumis(dataset_info.lumis[file])

            for file in dataset_info.files:
                columns = [(dataset_info.dataset, file, run, lumi) for (run, lumi) in dataset_info.lumis[file]]
                self.db.executemany("insert into jobits(dataset, input_file, run, lumi, status, num_attempts) values (?, ?, ?, ?, 'registered', 0)", columns)

        self.db.commit()

    def pop_jobits(self, size, old_status, new_status):
        self.job_id_counter += 1
        input_files = []
        lumis = []
        for input_file, run, lumi in self.db.execute("select input_file, run, lumi from jobits where status=? limit ?", (old_status, size)):
            input_files.append(input_file)
            lumis.append((run, lumi))
            self.db.execute("update jobits set status=?, job_id=? where run=? and lumi=?", (new_status, self.job_id_counter, run, lumi))

        self.db.commit()

        job_parameters = [set(input_files), LumiList(lumis=lumis).getVLuminosityBlockRange()]

        return job_parameters

<|endoftext|>


<commit_before>from __future__ import print_function, division

import hashlib
import os
import json

def generate_file_md5(filename, blocksize=2**20):
    m = hashlib.md5()
    with open(filename, "rb") as f:
        while True:
            buf = f.read(blocksize)
            if not buf:
                break
            m.update(buf)
    return m.hexdigest()

def check_hashes(d):
    counter = 0
    all_good = True
    for k, v in d.items():
        digest = generate_file_md5(k)
        if v == digest:
            counter += 1
            # print("The file {0} has the correct hash.".format(k))
        else:
            print("ERROR: The file {0} has the WRONG hash!".format(k))
            all_good = False
    print("There are " + str(counter) + " correct files.")
    return all_good

d = json.load(open("hashList.txt"))

if __name__ == "__main__":
    check_hashes(d)
<commit_msg>modify path of hash file<commit_after>from __future__ import print_function, division

import hashlib
import os
import json

def generate_file_md5(filename, blocksize=2**20):
    m = hashlib.md5()
    with open(filename, "rb") as f:
        while True:
            buf = f.read(blocksize)
            if not buf:
                break
            m.update(buf)
    return m.hexdigest()

def check_hashes(d):
    counter = 0
    all_good = True
    for k, v in d.items():
        digest = generate_file_md5(k)
        if v == digest:
            counter += 1
            # print("The file {0} has the correct hash.".format(k))
        else:
            print("ERROR: The file {0} has the WRONG hash!".format(k))
            all_good = False
    print("There are " + str(counter) + " correct files.")
    return all_good

d = json.load(open("data/hashList.txt"))

if __name__ == "__main__":
    check_hashes(d)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import numpy as np

from numpy.testing import assert_almost_equal, assert_array_almost_equal
from nose.tools import assert_true, assert_equal
from nibabel.tmpdirs import InTemporaryDirectory
from ..utils import (multiple_mahalanobis, z_score, multiple_fast_inv,
                     matrix_rank, full_rank, pos_recipr)

def test_z_score():
    p = np.random.rand(10)
    z = z_score(p)
    assert_array_almost_equal(norm.sf(z), p)

def test_mahalanobis():
    x = np.random.rand(100) / 100
    A = np.random.rand(100, 100) / 100
    A = np.dot(A.transpose(), A) + np.eye(100)
    mah = np.dot(x, np.dot(np.linalg.inv(A), x))
    assert_almost_equal(mah, multiple_mahalanobis(x, A), decimal=1)

def test_mahalanobis2():
    x = np.random.randn(100, 3)
    Aa = np.zeros([100, 100, 3])
    for i in range(3):
        A = np.random.randn(120, 100)
        A = np.dot(A.T, A)
        Aa[:, :, i] = A
    i = np.random.randint(3)
    mah = np.dot(x[:, i], np.dot(np.linalg.inv(Aa[:, :, i]), x[:, i]))
    f_mah = (multiple_mahalanobis(x, Aa))[i]
    assert_true(np.allclose(mah, f_mah))

def test_multiple_fast_inv():
    shape = (10, 20, 20)
    X = np.random.randn(shape[0], shape[1], shape[2])
    X_inv_ref = np.zeros(shape)
    for i in range(shape[0]):
        X[i] = np.dot(X[i], X[i].T)
        X_inv_ref[i] = np.linalg.inv(X[i])
    X_inv = multiple_fast_inv(X)
    assert_almost_equal(X_inv_ref, X_inv)

def test_full_rank():
    rng = np.random.RandomState(20110831)
    X = rng.standard_normal((40,5))
    # A quick rank check
    assert_equal(matrix_rank(X), 5)
    X[:,0] = X[:,1] + X[:,2]
    assert_equal(matrix_rank(X), 4)
    Y1 = full_rank(X)
    assert_equal(Y1.shape, (40,4))
    Y2 = full_rank(X, r=3)
    assert_equal(Y2.shape, (40,3))
    Y3 = full_rank(X, r=4)
    assert_equal(Y3.shape, (40,4))
    # Windows - there seems to be some randomness in the SVD result; standardize
    # column signs before comparison
    flipper = np.sign(Y1[0]) * np.sign(Y3[0])
    assert_almost_equal(Y1, Y3 * flipper)

def test_pos_recipr():
    X = np.array([2,1,-1,0], dtype=np.int8)
    eX = np.array([0.5,1,0,0])
    Y = pos_recipr(X)
    yield assert_array_almost_equal, Y, eX
    yield assert_equal, Y.dtype.type, np.float64
    X2 = X.reshape((2,2))
    Y2 = pos_recipr(X2)
    yield assert_array_almost_equal, Y2, eX.reshape((2,2))
    # check that lists have arrived
    XL = [0, 1, -1]
    yield assert_array_almost_equal, pos_recipr(XL), [0, 1, 0]
    # scalars
    yield assert_equal, pos_recipr(-1), 0
    yield assert_equal, pos_recipr(0), 0
    yield assert_equal, pos_recipr(2), 0.5

if __name__ == "__main__":
    import nose
    nose.run(argv=['', __file__])
<commit_msg>Some cleaning in test_utils<commit_after>#!/usr/bin/env python

import numpy as np
from scipy.stats import norm
from numpy.testing import assert_almost_equal, assert_array_almost_equal
from nose.tools import assert_true, assert_equal

from ..utils import (multiple_mahalanobis, z_score, multiple_fast_inv,
                     matrix_rank, full_rank, pos_recipr)

def test_z_score():
    p = np.random.rand(10)
    z = z_score(p)
    assert_array_almost_equal(norm.sf(z), p)

def test_mahalanobis():
    x = np.random.rand(100) / 100
    A = np.random.rand(100, 100) / 100
    A = np.dot(A.transpose(), A) + np.eye(100)
    mah = np.dot(x, np.dot(np.linalg.inv(A), x))
    assert_almost_equal(mah, multiple_mahalanobis(x, A), decimal=1)

def test_mahalanobis2():
    x = np.random.randn(100, 3)
    Aa = np.zeros([100, 100, 3])
    for i in range(3):
        A = np.random.randn(120, 100)
        A = np.dot(A.T, A)
        Aa[:, :, i] = A
    i = np.random.randint(3)
    mah = np.dot(x[:, i], np.dot(np.linalg.inv(Aa[:, :, i]), x[:, i]))
    f_mah = (multiple_mahalanobis(x, Aa))[i]
    assert_true(np.allclose(mah, f_mah))

def test_multiple_fast_inv():
    shape = (10, 20, 20)
    X = np.random.randn(shape[0], shape[1], shape[2])
    X_inv_ref = np.zeros(shape)
    for i in range(shape[0]):
        X[i] = np.dot(X[i], X[i].T)
        X_inv_ref[i] = np.linalg.inv(X[i])
    X_inv = multiple_fast_inv(X)
    assert_almost_equal(X_inv_ref, X_inv)

def test_full_rank():
    rng = np.random.RandomState(20110831)
    X = rng.standard_normal((40, 5))
    # A quick rank check
    assert_equal(matrix_rank(X), 5)
    X[:, 0] = X[:, 1] + X[:, 2]
    assert_equal(matrix_rank(X), 4)
    Y1 = full_rank(X)
    assert_equal(Y1.shape, (40, 4))
    Y2 = full_rank(X, r=3)
    assert_equal(Y2.shape, (40, 3))
    Y3 = full_rank(X, r=4)
    assert_equal(Y3.shape, (40, 4))
    # Windows - there seems to be some randomness in the SVD result;
    # standardize column signs before comparison
    flipper = np.sign(Y1[0]) * np.sign(Y3[0])
    assert_almost_equal(Y1, Y3 * flipper)

def test_pos_recipr():
    X = np.array([2, 1, -1, 0], dtype=np.int8)
    eX = np.array([0.5, 1, 0, 0])
    Y = pos_recipr(X)
    yield assert_array_almost_equal, Y, eX
    yield assert_equal, Y.dtype.type, np.float64
    X2 = X.reshape((2, 2))
    Y2 = pos_recipr(X2)
    yield assert_array_almost_equal, Y2, eX.reshape((2, 2))
    # check that lists have arrived
    XL = [0, 1, -1]
    yield assert_array_almost_equal, pos_recipr(XL), [0, 1, 0]
    # scalars
    yield assert_equal, pos_recipr(-1), 0
    yield assert_equal, pos_recipr(0), 0
    yield assert_equal, pos_recipr(2), 0.5
<|endoftext|>