<commit_before>#!/usr/bin/env python3.5

import argparse
import datetime
import itertools
import collections
import re
import sys
import operator

import parsedatetime
import tzlocal
import git

class Commit:
    def __init__(self, timestamp, message):
        self.timestamp = timestamp
        self.message = message

    def __repr__(self):
        return str(self.__dict__)

LOCAL_TIME_ZONE = tzlocal.get_localzone()
ISSUE_MARK_PATTERN = re.compile(r'issue #\d+(?:, issue #\d+)*:', re.IGNORECASE)
SPECIAL_ISSUE = 'прочее'

def parse_timestamp(value):
    try:
        timestamp = datetime.datetime.strptime(value, '%Y-%m-%d')
    except Exception:
        try:
            timestamp = datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S')
        except Exception:
            timestamp, status = parsedatetime.Calendar().parseDT(value)
            if status == 0:
                raise argparse.ArgumentTypeError(
                    'timestamp {} is incorrect'.format(value),
                )

    return LOCAL_TIME_ZONE.localize(timestamp, is_dst=None)

def parse_options():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    parser.add_argument(
        '-r',
        '--repo',
        default='.',
        help='path to the repository',
    )
    parser.add_argument(
        '-R',
        '--revs',
        default='HEAD',
        help='revisions specifier in the git-rev-parse command format',
    )
    parser.add_argument(
        '-s',
        '--start',
        type=parse_timestamp,
        required=True,
        help='a start timestamp of the repository log ' \
            + 'in ISO 8601 or human-readable formats',
    )
    parser.add_argument('-p', '--project', required=True, help='project name')
    parser.add_argument('-o', '--output', help='output path')

    return parser.parse_args()

def read_git_history(repository_path, revisions_specifier, start_timestamp):
    return [
        Commit(
            LOCAL_TIME_ZONE.localize(
                datetime.datetime.fromtimestamp(commit.authored_date),
                is_dst=None,
            ),
            commit.message,
        )
        for commit in git.Repo(repository_path).iter_commits(
            revisions_specifier,
            after=start_timestamp,
        )
    ]

def process_commit_message(message):
    message = message.strip()
    if len(message) == 0 \
        or message.startswith('Merge branch') \
        or message.startswith('Merge the branch'):
        return {}

    issue_mark_match = ISSUE_MARK_PATTERN.match(message)
    if issue_mark_match is None:
        return {SPECIAL_ISSUE: [message[0].lower() + message[1:]]}

    data = collections.defaultdict(list)
    message = message[len(issue_mark_match.group()):].strip()
    for issue_mark in (
        issue_mark.strip()
        for issue_mark in issue_mark_match.group()[:-1].lower().split(',')
    ):
        data[issue_mark].append(message)

    return data

def process_git_history(commits):
    data = collections.defaultdict(lambda: collections.defaultdict(list))
    for commit in commits:
        date = commit.timestamp.date()
        for issue_mark, messages in process_commit_message(
            commit.message,
        ).items():
            data[date][issue_mark].extend(messages)

    return data

def get_dummy_generator(collection):
    return itertools.repeat(' ' * 4, len(collection) - 1)

def format_messages(project_indent, issue_mark, messages):
    return '\n'.join(
        project_indent + issue_indent + message
        for project_indent, issue_indent, message in zip(
            itertools.chain([project_indent], get_dummy_generator(
                messages,
            )),
            itertools.chain(['{}, '.format(issue_mark)], get_dummy_generator(
                messages,
            )),
            messages,
        )
    )

def get_issue_mark_key(pair):
    return int(pair[0][7:]) if pair[0] != SPECIAL_ISSUE else sys.maxsize

def format_issues_marks(project, issues_marks):
    return '\n\n'.join(
        format_messages(project_indent, issue_mark, messages)
        for project_indent, (issue_mark, messages) in zip(
            itertools.chain(['{}, '.format(project)], get_dummy_generator(
                issues_marks,
            )),
            sorted(issues_marks.items(), key=get_issue_mark_key),
        )
    )

def format_git_history(project, data):
    return '# {}\n\n'.format(project) + '\n\n'.join(
        '## {}\n\n```\n{}\n```'.format(
            timestamp.strftime('%Y-%m-%d'),
            format_issues_marks(project, issues_marks),
        )
        for timestamp, issues_marks in sorted(
            data.items(),
            key=operator.itemgetter(0),
        )
    )

def output_git_history(output_path, representation):
    if output_path is None:
        print(representation)
    else:
        with open(output_path + '.md', 'w') as output_file:
            output_file.write(representation + '\n')

if __name__ == '__main__':
    try:
        options = parse_options()
        history = read_git_history(options.repo, options.revs, options.start)
        data = process_git_history(history)
        representation = format_git_history(options.project, data)
        output_git_history(options.output, representation)
    except Exception as exception:
        sys.exit('error: {}'.format(exception))
<commit_msg>Issue #252: add the unique_git_history() function<commit_after>#!/usr/bin/env python3.5

import argparse
import datetime
import itertools
import collections
import re
import sys
import operator

import parsedatetime
import tzlocal
import git

class Commit:
    def __init__(self, timestamp, message):
        self.timestamp = timestamp
        self.message = message

    def __repr__(self):
        return str(self.__dict__)

LOCAL_TIME_ZONE = tzlocal.get_localzone()
ISSUE_MARK_PATTERN = re.compile(r'issue #\d+(?:, issue #\d+)*:', re.IGNORECASE)
SPECIAL_ISSUE = 'прочее'

def parse_timestamp(value):
    try:
        timestamp = datetime.datetime.strptime(value, '%Y-%m-%d')
    except Exception:
        try:
            timestamp = datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S')
        except Exception:
            timestamp, status = parsedatetime.Calendar().parseDT(value)
            if status == 0:
                raise argparse.ArgumentTypeError(
                    'timestamp {} is incorrect'.format(value),
                )

    return LOCAL_TIME_ZONE.localize(timestamp, is_dst=None)

def parse_options():
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    parser.add_argument(
        '-r',
        '--repo',
        default='.',
        help='path to the repository',
    )
    parser.add_argument(
        '-R',
        '--revs',
        default='HEAD',
        help='revisions specifier in the git-rev-parse command format',
    )
    parser.add_argument(
        '-s',
        '--start',
        type=parse_timestamp,
        required=True,
        help='a start timestamp of the repository log ' \
            + 'in ISO 8601 or human-readable formats',
    )
    parser.add_argument('-p', '--project', required=True, help='project name')
    parser.add_argument('-o', '--output', help='output path')

    return parser.parse_args()

def read_git_history(repository_path, revisions_specifier, start_timestamp):
    return [
        Commit(
            LOCAL_TIME_ZONE.localize(
                datetime.datetime.fromtimestamp(commit.authored_date),
                is_dst=None,
            ),
            commit.message,
        )
        for commit in git.Repo(repository_path).iter_commits(
            revisions_specifier,
            after=start_timestamp,
        )
    ]

def process_commit_message(message):
    message = message.strip()
    if len(message) == 0 \
        or message.startswith('Merge branch') \
        or message.startswith('Merge the branch'):
        return {}

    issue_mark_match = ISSUE_MARK_PATTERN.match(message)
    if issue_mark_match is None:
        return {SPECIAL_ISSUE: [message[0].lower() + message[1:]]}

    data = collections.defaultdict(list)
    message = message[len(issue_mark_match.group()):].strip()
    for issue_mark in (
        issue_mark.strip()
        for issue_mark in issue_mark_match.group()[:-1].lower().split(',')
    ):
        data[issue_mark].append(message)

    return data

def process_git_history(commits):
    data = collections.defaultdict(lambda: collections.defaultdict(list))
    for commit in commits:
        date = commit.timestamp.date()
        for issue_mark, messages in process_commit_message(
            commit.message,
        ).items():
            data[date][issue_mark].extend(messages)

    return data

# https://docs.python.org/3/library/itertools.html#itertools-recipes
def unique_everseen(iterable):
    seen = set()
    for element in itertools.filterfalse(seen.__contains__, iterable):
        seen.add(element)
        yield element

def unique_git_history(data):
    unique_data = collections.defaultdict(dict)
    for date, issues_marks in data.items():
        for issue_mark, messages in issues_marks.items():
            unique_data[date][issue_mark] = list(unique_everseen(messages))

    return unique_data

def get_dummy_generator(collection):
    return itertools.repeat(' ' * 4, len(collection) - 1)

def format_messages(project_indent, issue_mark, messages):
    return '\n'.join(
        project_indent + issue_indent + message
        for project_indent, issue_indent, message in zip(
            itertools.chain([project_indent], get_dummy_generator(
                messages,
            )),
            itertools.chain(['{}, '.format(issue_mark)], get_dummy_generator(
                messages,
            )),
            messages,
        )
    )

def get_issue_mark_key(pair):
    return int(pair[0][7:]) if pair[0] != SPECIAL_ISSUE else sys.maxsize

def format_issues_marks(project, issues_marks):
    return '\n\n'.join(
        format_messages(project_indent, issue_mark, messages)
        for project_indent, (issue_mark, messages) in zip(
            itertools.chain(['{}, '.format(project)], get_dummy_generator(
                issues_marks,
            )),
            sorted(issues_marks.items(), key=get_issue_mark_key),
        )
    )

def format_git_history(project, data):
    return '# {}\n\n'.format(project) + '\n\n'.join(
        '## {}\n\n```\n{}\n```'.format(
            date.strftime('%Y-%m-%d'),
            format_issues_marks(project, issues_marks),
        )
        for date, issues_marks in sorted(
            data.items(),
            key=operator.itemgetter(0),
        )
    )

def output_git_history(output_path, representation):
    if output_path is None:
        print(representation)
    else:
        with open(output_path + '.md', 'w') as output_file:
            output_file.write(representation + '\n')

if __name__ == '__main__':
    try:
        options = parse_options()
        history = read_git_history(options.repo, options.revs, options.start)
        data = process_git_history(history)
        unique_data = unique_git_history(data)
        representation = format_git_history(options.project, unique_data)
        output_git_history(options.output, representation)
    except Exception as exception:
        sys.exit('error: {}'.format(exception))
<|endoftext|>


<commit_before>#
# Copyright (C) 2011 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#         * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#         * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#         * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

{
    'variables': {
      'blink_web_output_dir': '<(SHARED_INTERMEDIATE_DIR)/blink/web',
    },
    'includes': [
        '../bindings/bindings.gypi',
        '../core/core.gypi',
        '../build/features.gypi',
        '../build/scripts/scripts.gypi',
        '../build/win/precompile.gypi',
        '../modules/modules.gypi',
        '../platform/blink_platform.gypi',
        '../wtf/wtf.gypi',
        'web.gypi',
    ],
    'target_defaults': {
        'variables': {
            'clang_warning_flags': ['-Wglobal-constructors'],
        },
    },
    'targets': [
        {
            'target_name': 'blink_web',
            'type': '<(component)',
            'variables': { 'enable_wexit_time_destructors': 1, },
            'dependencies': [
                '../config.gyp:config',
                '../platform/blink_platform.gyp:blink_common',
                '../core/core.gyp:webcore',
                '../modules/modules.gyp:modules',
                '<(DEPTH)/skia/skia.gyp:skia',
                '<(angle_path)/src/angle.gyp:translator',
                '<(DEPTH)/third_party/icu/icu.gyp:icuuc',
                '<(DEPTH)/third_party/npapi/npapi.gyp:npapi',
                '<(DEPTH)/v8/tools/gyp/v8.gyp:v8',
            ],
            'export_dependent_settings': [
                '<(DEPTH)/skia/skia.gyp:skia',
                '<(DEPTH)/third_party/icu/icu.gyp:icuuc',
                '<(DEPTH)/third_party/npapi/npapi.gyp:npapi',
                '<(DEPTH)/v8/tools/gyp/v8.gyp:v8',
            ],
            'include_dirs': [
                '<(angle_path)/include',
                '<(DEPTH)/third_party/skia/include/utils',
            ],
            'defines': [
                'BLINK_IMPLEMENTATION=1',
                'INSIDE_BLINK',
            ],
            'sources': [
                '<@(web_files)',
            ],
            'conditions': [
                ['component=="shared_library"', {
                    'dependencies': [
                        '../core/core.gyp:webcore_generated',
                        '../core/core.gyp:webcore_testing',
                        '../modules/modules.gyp:modules_testing',
                        '../wtf/wtf_tests.gyp:wtf_unittest_helpers',
                        '<(DEPTH)/base/base.gyp:test_support_base',
                        '<(DEPTH)/testing/gmock.gyp:gmock',
                        '<(DEPTH)/testing/gtest.gyp:gtest',
                        '<(DEPTH)/third_party/icu/icu.gyp:icuuc',
                        '<(DEPTH)/third_party/icu/icu.gyp:icui18n',
                        '<(DEPTH)/third_party/libpng/libpng.gyp:libpng',
                        '<(DEPTH)/third_party/libwebp/libwebp.gyp:libwebp',
                        '<(DEPTH)/third_party/libxml/libxml.gyp:libxml',
                        '<(DEPTH)/third_party/libxslt/libxslt.gyp:libxslt',
                        '<(DEPTH)/third_party/modp_b64/modp_b64.gyp:modp_b64',
                        '<(DEPTH)/third_party/ots/ots.gyp:ots',
                        '<(DEPTH)/third_party/zlib/zlib.gyp:zlib',
                        '<(DEPTH)/url/url.gyp:url_lib',
                        '<(DEPTH)/v8/tools/gyp/v8.gyp:v8',
                        '<(libjpeg_gyp_path):libjpeg',
                        # We must not add webkit_support here because of cyclic dependency.
                    ],
                    'export_dependent_settings': [
                        '<(DEPTH)/url/url.gyp:url_lib',
                        '<(DEPTH)/v8/tools/gyp/v8.gyp:v8',
                    ],
                    'variables': {
                      'clang_warning_flags_unset': [
                        # FIXME: It would be nice to enable this in shared builds too,
                        # but the test files have global constructors from the GTEST macro
                        # and we pull in the test files into the blink_web target in the
                        # shared build.
                        '-Wglobal-constructors',
                      ],
                    },
                    'sources': [
                        # Compile Blink unittest files into blink_web.dll in component build mode
                        '<@(bindings_unittest_files)',
                        '<@(core_unittest_files)',
                        '<@(modules_unittest_files)',
                        # FIXME: the next line should not be needed. We prefer to run these unit tests outside blink_web.dll.
                        '<@(platform_web_unittest_files)',
                        '<@(web_unittest_files)',
                        'WebTestingSupport.cpp',
                    ],
                    'conditions': [
                        ['OS=="win" or OS=="mac"', {
                            'dependencies': [
                                '<(DEPTH)/third_party/nss/nss.gyp:*',
                            ],
                        }],
                    ],
                    'msvs_settings': {
                      'VCLinkerTool': {
                        'conditions': [
                          ['incremental_chrome_dll==1', {
                            'UseLibraryDependencyInputs': "true",
                          }],
                        ],
                      },
                    },
                }],
                ['OS == "linux"', {
                    'dependencies': [
                        '<(DEPTH)/build/linux/system.gyp:fontconfig',
                    ],
                }, {
                    'sources/': [
                        ['exclude', 'linux/'],
                    ],
                }],
                ['use_x11 == 1', {
                    'dependencies': [
                        '<(DEPTH)/build/linux/system.gyp:x11',
                    ],
                }, {
                    'sources/': [
                        ['exclude', 'x11/'],
                    ]
                }],
                ['OS!="android"', {
                    'sources/': [
                        ['exclude', 'WebInputEventFactoryAndroid.cpp$'],
                    ],
                }],
                ['OS=="mac"', {
                    'link_settings': {
                        'libraries': [
                            '$(SDKROOT)/System/Library/Frameworks/Accelerate.framework',
                            '$(SDKROOT)/System/Library/Frameworks/OpenGL.framework',
                        ],
                    },
                }, { # else: OS!="mac"
                    'sources/': [
                        ['exclude', 'WebInputEventFactoryMac.mm$'],
                        ['exclude', 'mac/WebScrollbarTheme.cpp$'],
                    ],
                }],
                ['OS!="win"', {
                    'sources/': [
                        ['exclude', 'WebInputEventFactoryWin.cpp$'],
                    ],
                }],
                ['use_default_render_theme==0', {
                    'sources/': [
                        ['exclude', 'default/WebRenderTheme.cpp'],
                    ],
                }],
            ],
            'direct_dependent_settings': {
                'include_dirs': [
                    '../../',
                ],
            },
            'target_conditions': [
                ['OS=="android"', {
                    'sources/': [
                        ['include', '^linux/WebFontRendering\\.cpp$'],
                    ],
                }],
            ],
        },
        {
            # GN version: //third_party/WebKit/Source/web:test_support
            'target_name': 'blink_web_test_support',
            'conditions': [
                ['component=="shared_library"', {
                    'type': 'none',
                }, {
                    'type': 'static_library',
                    'dependencies': [
                        '../config.gyp:config',
                        '../core/core.gyp:webcore_testing',
                        '../modules/modules.gyp:modules_testing',
                        '../wtf/wtf.gyp:wtf',
                        '<(DEPTH)/skia/skia.gyp:skia',
                        '<(DEPTH)/v8/tools/gyp/v8.gyp:v8',
                    ],
                    'include_dirs': [
                        '../../',
                        '<(SHARED_INTERMEDIATE_DIR)/blink',  # gen/blink
                    ],
                    'sources': [
                        'WebTestingSupport.cpp',
                    ],
                }],
            ],
        },
    ], # targets
    'conditions': [
        ['gcc_version>=46', {
            'target_defaults': {
                # Disable warnings about c++0x compatibility, as some names (such
                # as nullptr) conflict with upcoming c++0x types.
                'cflags_cc': ['-Wno-c++0x-compat'],
            },
        }],
        ['OS=="mac"', {
            'targets': [
                {
                    'target_name': 'copy_mesa',
                    'type': 'none',
                    'dependencies': ['<(DEPTH)/third_party/mesa/mesa.gyp:osmesa'],
                    'copies': [{
                        'destination': '<(PRODUCT_DIR)/DumpRenderTree.app/Contents/MacOS/',
                        'files': ['<(PRODUCT_DIR)/osmesa.so'],
                    }],
                },
            ],
        }],
    ], # conditions
}
<commit_msg>Added dependencies to blink_web_test_support.<commit_after>#
# Copyright (C) 2011 Google Inc. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#         * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
#         * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
#         * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

{
    'variables': {
      'blink_web_output_dir': '<(SHARED_INTERMEDIATE_DIR)/blink/web',
    },
    'includes': [
        '../bindings/bindings.gypi',
        '../core/core.gypi',
        '../build/features.gypi',
        '../build/scripts/scripts.gypi',
        '../build/win/precompile.gypi',
        '../modules/modules.gypi',
        '../platform/blink_platform.gypi',
        '../wtf/wtf.gypi',
        'web.gypi',
    ],
    'target_defaults': {
        'variables': {
            'clang_warning_flags': ['-Wglobal-constructors'],
        },
    },
    'targets': [
        {
            'target_name': 'blink_web',
            'type': '<(component)',
            'variables': { 'enable_wexit_time_destructors': 1, },
            'dependencies': [
                '../config.gyp:config',
                '../platform/blink_platform.gyp:blink_common',
                '../core/core.gyp:webcore',
                '../modules/modules.gyp:modules',
                '<(DEPTH)/skia/skia.gyp:skia',
                '<(angle_path)/src/angle.gyp:translator',
                '<(DEPTH)/third_party/icu/icu.gyp:icuuc',
                '<(DEPTH)/third_party/npapi/npapi.gyp:npapi',
                '<(DEPTH)/v8/tools/gyp/v8.gyp:v8',
            ],
            'export_dependent_settings': [
                '<(DEPTH)/skia/skia.gyp:skia',
                '<(DEPTH)/third_party/icu/icu.gyp:icuuc',
                '<(DEPTH)/third_party/npapi/npapi.gyp:npapi',
                '<(DEPTH)/v8/tools/gyp/v8.gyp:v8',
            ],
            'include_dirs': [
                '<(angle_path)/include',
                '<(DEPTH)/third_party/skia/include/utils',
            ],
            'defines': [
                'BLINK_IMPLEMENTATION=1',
                'INSIDE_BLINK',
            ],
            'sources': [
                '<@(web_files)',
            ],
            'conditions': [
                ['component=="shared_library"', {
                    'dependencies': [
                        '../core/core.gyp:webcore_generated',
                        '../core/core.gyp:webcore_testing',
                        '../modules/modules.gyp:modules_testing',
                        '../wtf/wtf_tests.gyp:wtf_unittest_helpers',
                        '<(DEPTH)/base/base.gyp:test_support_base',
                        '<(DEPTH)/testing/gmock.gyp:gmock',
                        '<(DEPTH)/testing/gtest.gyp:gtest',
                        '<(DEPTH)/third_party/icu/icu.gyp:icuuc',
                        '<(DEPTH)/third_party/icu/icu.gyp:icui18n',
                        '<(DEPTH)/third_party/libpng/libpng.gyp:libpng',
                        '<(DEPTH)/third_party/libwebp/libwebp.gyp:libwebp',
                        '<(DEPTH)/third_party/libxml/libxml.gyp:libxml',
                        '<(DEPTH)/third_party/libxslt/libxslt.gyp:libxslt',
                        '<(DEPTH)/third_party/modp_b64/modp_b64.gyp:modp_b64',
                        '<(DEPTH)/third_party/ots/ots.gyp:ots',
                        '<(DEPTH)/third_party/zlib/zlib.gyp:zlib',
                        '<(DEPTH)/url/url.gyp:url_lib',
                        '<(DEPTH)/v8/tools/gyp/v8.gyp:v8',
                        '<(libjpeg_gyp_path):libjpeg',
                        # We must not add webkit_support here because of cyclic dependency.
                    ],
                    'export_dependent_settings': [
                        '<(DEPTH)/url/url.gyp:url_lib',
                        '<(DEPTH)/v8/tools/gyp/v8.gyp:v8',
                    ],
                    'variables': {
                      'clang_warning_flags_unset': [
                        # FIXME: It would be nice to enable this in shared builds too,
                        # but the test files have global constructors from the GTEST macro
                        # and we pull in the test files into the blink_web target in the
                        # shared build.
                        '-Wglobal-constructors',
                      ],
                    },
                    'sources': [
                        # Compile Blink unittest files into blink_web.dll in component build mode
                        '<@(bindings_unittest_files)',
                        '<@(core_unittest_files)',
                        '<@(modules_unittest_files)',
                        # FIXME: the next line should not be needed. We prefer to run these unit tests outside blink_web.dll.
                        '<@(platform_web_unittest_files)',
                        '<@(web_unittest_files)',
                        'WebTestingSupport.cpp',
                    ],
                    'conditions': [
                        ['OS=="win" or OS=="mac"', {
                            'dependencies': [
                                '<(DEPTH)/third_party/nss/nss.gyp:*',
                            ],
                        }],
                    ],
                    'msvs_settings': {
                      'VCLinkerTool': {
                        'conditions': [
                          ['incremental_chrome_dll==1', {
                            'UseLibraryDependencyInputs': "true",
                          }],
                        ],
                      },
                    },
                }],
                ['OS == "linux"', {
                    'dependencies': [
                        '<(DEPTH)/build/linux/system.gyp:fontconfig',
                    ],
                }, {
                    'sources/': [
                        ['exclude', 'linux/'],
                    ],
                }],
                ['use_x11 == 1', {
                    'dependencies': [
                        '<(DEPTH)/build/linux/system.gyp:x11',
                    ],
                }, {
                    'sources/': [
                        ['exclude', 'x11/'],
                    ]
                }],
                ['OS!="android"', {
                    'sources/': [
                        ['exclude', 'WebInputEventFactoryAndroid.cpp$'],
                    ],
                }],
                ['OS=="mac"', {
                    'link_settings': {
                        'libraries': [
                            '$(SDKROOT)/System/Library/Frameworks/Accelerate.framework',
                            '$(SDKROOT)/System/Library/Frameworks/OpenGL.framework',
                        ],
                    },
                }, { # else: OS!="mac"
                    'sources/': [
                        ['exclude', 'WebInputEventFactoryMac.mm$'],
                        ['exclude', 'mac/WebScrollbarTheme.cpp$'],
                    ],
                }],
                ['OS!="win"', {
                    'sources/': [
                        ['exclude', 'WebInputEventFactoryWin.cpp$'],
                    ],
                }],
                ['use_default_render_theme==0', {
                    'sources/': [
                        ['exclude', 'default/WebRenderTheme.cpp'],
                    ],
                }],
            ],
            'direct_dependent_settings': {
                'include_dirs': [
                    '../../',
                ],
            },
            'target_conditions': [
                ['OS=="android"', {
                    'sources/': [
                        ['include', '^linux/WebFontRendering\\.cpp$'],
                    ],
                }],
            ],
        },
        {
            # GN version: //third_party/WebKit/Source/web:test_support
            'target_name': 'blink_web_test_support',
            'conditions': [
                ['component=="shared_library"', {
                    'type': 'none',
                }, {
                    'type': 'static_library',
                    'dependencies': [
                        '../config.gyp:config',
                        '../core/core.gyp:webcore_generated',
                        '../core/core.gyp:webcore_testing',
                        '../modules/modules.gyp:modules_testing',
                        '../wtf/wtf.gyp:wtf',
                        '<(DEPTH)/skia/skia.gyp:skia',
                        '<(DEPTH)/v8/tools/gyp/v8.gyp:v8',
                    ],
                    'include_dirs': [
                        '../../',
                        '<(SHARED_INTERMEDIATE_DIR)/blink',  # gen/blink
                    ],
                    'sources': [
                        'WebTestingSupport.cpp',
                    ],
                }],
            ],
        },
    ], # targets
    'conditions': [
        ['gcc_version>=46', {
            'target_defaults': {
                # Disable warnings about c++0x compatibility, as some names (such
                # as nullptr) conflict with upcoming c++0x types.
                'cflags_cc': ['-Wno-c++0x-compat'],
            },
        }],
        ['OS=="mac"', {
            'targets': [
                {
                    'target_name': 'copy_mesa',
                    'type': 'none',
                    'dependencies': ['<(DEPTH)/third_party/mesa/mesa.gyp:osmesa'],
                    'copies': [{
                        'destination': '<(PRODUCT_DIR)/DumpRenderTree.app/Contents/MacOS/',
                        'files': ['<(PRODUCT_DIR)/osmesa.so'],
                    }],
                },
            ],
        }],
    ], # conditions
}
<|endoftext|>


<commit_before>selnode = hou.selectedNodes()
for node in hou.selectedNodes():
    selPath = node.path()

sel = selPath.split("/")[-2]

hou.node('obj/'+sel).createNode("python", "ExportScript" )
hou.node('obj/'+sel+'/ExportScript/').setParms({"python": '''
import tempfile, os, random, sys

filePath = tempfile.gettempdir() + os.sep + ".." + os.sep + "ODVertexData.txt"

node = hou.pwd()
geo = node.geometry()

if len(geo.points()) > 0:
    f = open(filePath, "w")

    f.write("VERTICES:"+str(len(geo.points())) + "\\n")
    for point in geo.points():
        pos = point.position()
        f.write(str(pos[0]) + " " + str(pos[1]) + " " + str(pos[2]*-1) + "\\n")
    polyPrims = [prim for prim in geo.prims() if prim.type() == hou.primType.Polygon]

    uvs = []
    check = geo.findVertexAttrib("uv")
    f.write("POLYGONS:"+str(len(polyPrims)) + "\\n")
    count = 0
    for (fid, prim) in enumerate(polyPrims):
        ppoint = ""
        for point in prim.vertices():
             ppoint += "," + str(geo.points().index(hou.Vertex.point(point)))
             if check != None:
                uvs.append(str(point.attribValue("uv")[0]) + " " + str(point.attribValue("uv")[1]) + ":PLY:" + str(count) + ":PNT:" + str(geo.points().index(hou.Vertex.point(point))) + "\\n")
        surf = "Default"
        polytype = "FACE"
        #print ppoint
        f.write(ppoint[1:] + ";;" + surf + ";;" + polytype + "\\n")
        count += 1

    attribs = geo.pointAttribs()
    weights = []
    for attrib in attribs:
        if attrib.type() == hou.attribType.Point:
            if attrib.dataType() == hou.attribData.Float and attrib.size() == 1:
                weights.append(attrib.name())

    if len(weights) > 0:
        for wmap in weights:
            attrib = wmap
            f.write("WEIGHT:" + wmap + "\\n")
            for p in geo.points():
                f.write(str(p.attribValue(wmap))+ "\\n")

    if len(uvs) > 0:
        f.write("UV:UVMap:"+ str(len(uvs)) + "\\n")
        for uv in uvs:
            f.write(uv)
    f.close()
    '''})

hou.node('obj/'+sel+'/ExportScript/').setInput(0, hou.node(selPath))
hou.node('obj/'+sel+'/ExportScript/').setDisplayFlag(True)
hou.node('obj/'+sel+'/ExportScript/').setRenderFlag(True)
hou.node('obj/'+sel+'/ExportScript/').setCurrent(1, False)
hou.node('obj/'+sel+'/ExportScript/').cook(force=True, frame_range=(int(hou.frame()),int(hou.frame())))
hou.node('obj/'+sel+'/ExportScript/').destroy()<commit_msg>Added ConvertGeo Node on Export<commit_after>selnode = hou.selectedNodes()
for node in hou.selectedNodes():
    selPath = node.path()

sel = selPath.split("/")[-2]

hou.node('obj/'+sel).createNode("python", "ExportScript" )
hou.node('obj/'+sel+'/ExportScript/').setParms({"python": '''
import tempfile, os, random, sys

filePath = tempfile.gettempdir() + os.sep + ".." + os.sep + "ODVertexData.txt"

node = hou.pwd()
geo = node.geometry()

if len(geo.points()) > 0:
    f = open(filePath, "w")

    f.write("VERTICES:"+str(len(geo.points())) + "\\n")
    for point in geo.points():
        pos = point.position()
        f.write(str(pos[0]) + " " + str(pos[1]) + " " + str(pos[2]*-1) + "\\n")
    polyPrims = [prim for prim in geo.prims() if prim.type() == hou.primType.Polygon]

    uvs = []
    check = geo.findVertexAttrib("uv")
    f.write("POLYGONS:"+str(len(polyPrims)) + "\\n")
    count = 0
    for (fid, prim) in enumerate(polyPrims):
        ppoint = ""
        for point in prim.vertices():
             ppoint += "," + str(geo.points().index(hou.Vertex.point(point)))
             if check != None:
                uvs.append(str(point.attribValue("uv")[0]) + " " + str(point.attribValue("uv")[1]) + ":PLY:" + str(count) + ":PNT:" + str(geo.points().index(hou.Vertex.point(point))) + "\\n")
        surf = "Default"
        polytype = "FACE"
        #print ppoint
        f.write(ppoint[1:] + ";;" + surf + ";;" + polytype + "\\n")
        count += 1

    attribs = geo.pointAttribs()
    weights = []
    for attrib in attribs:
        if attrib.type() == hou.attribType.Point:
            if attrib.dataType() == hou.attribData.Float and attrib.size() == 1:
                weights.append(attrib.name())

    if len(weights) > 0:
        for wmap in weights:
            attrib = wmap
            f.write("WEIGHT:" + wmap + "\\n")
            for p in geo.points():
                f.write(str(p.attribValue(wmap))+ "\\n")

    if len(uvs) > 0:
        f.write("UV:UVMap:"+ str(len(uvs)) + "\\n")
        for uv in uvs:
            f.write(uv)
    f.close()
    '''})

hou.node('obj/'+sel).createNode("convert", "ODConvertToPolygon" )
hou.node('obj/'+sel+'/ODConvertToPolygon/').setInput(0, hou.node(selPath))
hou.node('obj/'+sel+'/ExportScript/').setInput(0, hou.node('obj/'+sel+'/ODConvertToPolygon/'))

#hou.node('obj/'+sel+'/ExportScript/').setInput(0, hou.node(selPath))
hou.node('obj/'+sel+'/ExportScript/').setDisplayFlag(True)
hou.node('obj/'+sel+'/ExportScript/').setRenderFlag(True)
hou.node('obj/'+sel+'/ExportScript/').setCurrent(1, False)
hou.node('obj/'+sel+'/ExportScript/').cook(force=True, frame_range=(int(hou.frame()),int(hou.frame())))
hou.node('obj/'+sel+'/ExportScript/').destroy()
hou.node('obj/'+sel+'/ODConvertToPolygon/').destroy()
<|endoftext|>


<commit_before>#!/usr/bin/python

#  This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

"""This file contains various locale lists consumed by other tools"""

# Sorted list of locales that ship in release builds of Focus/Klar.
#
# Other builds might include more locales.
#
# Note that there are differences in the locale codes used by Pontoon
# and by Android (e.g. Hebrew: he (Pontoon) vs. iw (Android)).
# This list uses the Android notation. A valid entry can be a
# language code (de) or a language plus country (de-DE). Do not use
# the name of the Androidresource folder (de-rDE) here.
#
# Releases should contain all locales that are at 100% or have been
# shipping in previous releases. Ping :delphine in case you want
# to add or remove locales from releases.
RELEASE_LOCALES = [
	"am",
	"an",
	"anp",
	"ar",
	"ast",
	"az",
	"bg",
	"bn-BD",
	"bn-IN",
	"bs",
	"ca",
	"cak",
	"cs",
	"cy",
	"da",
	"de",
	"dsb",
	"el",
	"eo",
	"es-AR",
	"es-CL",
	"es-ES",
	"es-MX",
	"eu",
	"fa",
	"fi",
	"fr",
	"fy-NL",
	"ga-IE",
	"gu-IN",
	"hi-IN",
	"hr",
	"hsb",
	"hu",
	"hy-AM",
	"ia",
	"in",
	"it",
	"iw",
	"ja",
	"ka",
	"kab",
	"kk",
	"ko",
	"kw",
	"lo",
	"meh",
	"mix",
	"ms",
	"my",
	"nb-NO",
	"ne-NP",
	"nl",
	"nn-NO",
	"oc",
	"pl",
	"pt-BR",
	"ro",
	"ru",
	"sk",
	"sl",
	"sq",
	"sr",
	"sv-SE",
	"ta",
	"te",
	"th",
	"tr",
	"trs",
	"tsz",
	"tt",
	"uk",
	"ur",
	"vi",
	"wo",
	"zam",
	"zh-CN",
	"zh-TW"
]

# This is the list of locales that we want to take automated screenshots of
# in addition to the list of release locales. We want to take screenshots
# of other locales so that translators of not yet completed locales can
# verify their work in progress.
ADDITIONAL_SCREENSHOT_LOCALES = [
	"ixl",
	"lt"
]

# Those are locales that we take automated screenshots of.
SCREENSHOT_LOCALES = sorted(RELEASE_LOCALES + ADDITIONAL_SCREENSHOT_LOCALES)
<commit_msg>Closes #2605: Add "en-CA" to screenshot locales.<commit_after>#!/usr/bin/python

#  This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

"""This file contains various locale lists consumed by other tools"""

# Sorted list of locales that ship in release builds of Focus/Klar.
#
# Other builds might include more locales.
#
# Note that there are differences in the locale codes used by Pontoon
# and by Android (e.g. Hebrew: he (Pontoon) vs. iw (Android)).
# This list uses the Android notation. A valid entry can be a
# language code (de) or a language plus country (de-DE). Do not use
# the name of the Androidresource folder (de-rDE) here.
#
# Releases should contain all locales that are at 100% or have been
# shipping in previous releases. Ping :delphine in case you want
# to add or remove locales from releases.
RELEASE_LOCALES = [
	"am",
	"an",
	"anp",
	"ar",
	"ast",
	"az",
	"bg",
	"bn-BD",
	"bn-IN",
	"bs",
	"ca",
	"cak",
	"cs",
	"cy",
	"da",
	"de",
	"dsb",
	"el",
	"eo",
	"es-AR",
	"es-CL",
	"es-ES",
	"es-MX",
	"eu",
	"fa",
	"fi",
	"fr",
	"fy-NL",
	"ga-IE",
	"gu-IN",
	"hi-IN",
	"hr",
	"hsb",
	"hu",
	"hy-AM",
	"ia",
	"in",
	"it",
	"iw",
	"ja",
	"ka",
	"kab",
	"kk",
	"ko",
	"kw",
	"lo",
	"meh",
	"mix",
	"ms",
	"my",
	"nb-NO",
	"ne-NP",
	"nl",
	"nn-NO",
	"oc",
	"pl",
	"pt-BR",
	"ro",
	"ru",
	"sk",
	"sl",
	"sq",
	"sr",
	"sv-SE",
	"ta",
	"te",
	"th",
	"tr",
	"trs",
	"tsz",
	"tt",
	"uk",
	"ur",
	"vi",
	"wo",
	"zam",
	"zh-CN",
	"zh-TW"
]

# This is the list of locales that we want to take automated screenshots of
# in addition to the list of release locales. We want to take screenshots
# of other locales so that translators of not yet completed locales can
# verify their work in progress.
ADDITIONAL_SCREENSHOT_LOCALES = [
	"ixl",
	"lt",
	"en-CA"
]

# Those are locales that we take automated screenshots of.
SCREENSHOT_LOCALES = sorted(RELEASE_LOCALES + ADDITIONAL_SCREENSHOT_LOCALES)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import argparse

def cmd_printer(exec_line, *args, **kwargs):
    """
    Creates string to be printed to bash in order to run executable.

    Parameters
    ----------
    exec_line : str (execution line including path/to/exec)
    extra args : str (option to be passed, e.g. "-v 1")
    extra kwargs : may include "output" kwarg if redirection to "output" file
    is demanded.

    Returns
    ----------
    cmd : str (block of text to be printed to a bash script to run exec_line
    [with options and with output redirected if specified in *args and *kwargs]
    when the bash script is ran.)
    """
    comment = False
    cmd = "%s " % exec_line
    for arg in args:
        cmd += "%s " % arg
    if kwargs is not None:
        for key, value in kwargs.iteritems():
            if key in ["out", "output", "output_file"]:
                cmd += "> %s" % value
            if key == 'comment' and value == True:
                comment=True
    cmd += "\n"

    if comment:
        cmd = '#' + cmd

    return cmd

parser = argparse.ArgumentParser(description="creates bash script"
                                "to test assemblers on a dataset")
parser.add_argument("reads", help="path to reads file (fasta or fastq)")
parser.add_argument("-p", "--prefix", required=True, help="prefix for output")
parser.add_argument("-d", "--dir",required=True, help="directory for output")
parser.add_argument("--minimap", required=True,
                    help="path/to/minimap (executable)."
                    "Required for spectrassembler.")
parser.add_argument("--minimap_opts", default="-Sw5 -L100 -m0",
                    help="options for minimap")
parser.add_argument("--miniasm", help="path/to/miniasm (executable)")
parser.add_argument("--miniasm_opts", default="",
                    help="options for miniasm")
parser.add_argument("--racon", help="path/to/racon (executable)")
parser.add_argument("--canu", help="path/to/canu (executable)")
parser.add_argument("--canu_opts", default="",
                    help="options for canu. Read file needs to be in it through"
                    "e.g. -nanopore-raw reads.fasta")
parser.add_argument("--spectrassembler_opts", default="",
                    help="options for spectrassembler")
parser.add_argument("--spoa", help="path/to/spoa/")

args = parser.parse_args()
tools_path = os.path.realpath(__file__)
spectrassembler = tools_path.split('/tools')[0]

# In order not to recompute old things
do_comment = False

# Create main script
runall_script = "%s_runall.sh" % (args.prefix)
runall_fh = open(runall_script, 'wb')
runall_fh.write("#!/bin/bash\n")

# Create preprocess script file
pp_script = "%s_preprocess.sh" % (args.prefix)
pp_fh = open(pp_script, 'wb')

# Prelude
pp_fh.write("#!/bin/bash\n")
pp_fh.write("reads=\"%s\"\n" % args.reads)
pp_fh.write("if [ ! -d \"%s\" ]; then mkdir %s; fi\n" % (args.dir, args.dir))

# Check read ids are unique
pp_fh.write("# Check that read ids are unique\n")
pp_fh.write("python %s/tools/check_reads.py %s\n" % (spectrassembler, args.reads))
readspp = '.'.join(args.reads.split('.')[:-1]) + '.pp.' + args.reads.split('.')[-1]
pp_fh.write("if [ -e \"%s\" ]; then reads=\"%s\"; fi\n" % (readspp, readspp))

# make fastq file for racon
if args.racon and args.minimap:
    # Racon + spectral
    pp_fh.write("# Check that reads are .fastq for racon\n")
    mycmd = "if [ \"${reads:(-1)}\" == \"q\" ]; then\
    \n  readsq=\"$reads\"\
    \nelse\
    \n  echo \"Reads are in fasta format. Convert to fastq before calling racon...\"\
    \n  readsq=\"${reads/.f*a/.fastq}\"\
    \n  python %s/tools/fasta2fastq.py \"$reads\" \"$readsq\"\
    \nfi\n" % (spectrassembler)
    pp_fh.write(mycmd)

mycmd = "/usr/bin/time -f \"%sE time %sK avg."\
" mem %sM max mem\" /bin/bash %s 2> %s.log 1> %s.out \n" % \
('%', '%', '%', pp_script, pp_script.split('.sh')[0], pp_script.split('.sh')[0])
runall_fh.write(mycmd)

# Create minimap script
minimap_script = "%s_minimap.sh" % (args.prefix)
minimap_fh = open(minimap_script, 'wb')
minimap_fh.write("#!/bin/bash\n")
minimap_fh.write("reads=\"%s\"\n" % args.reads)
minimap_fh.write("if [ -e \"%s\" ]; then reads=\"%s\"; fi\n" % (readspp, readspp))
minimap_fh.write("# Run minimap\n")
minimap_out = "%s/%s.mini.paf" % (args.dir, args.prefix)
mycmd = cmd_printer(args.minimap, args.minimap_opts, "$reads",
"$reads", output=minimap_out, comment=do_comment)
minimap_fh.write(mycmd)

mycmd = "/usr/bin/time -f \"%sE time %sK avg."\
" mem %sM max mem\" /bin/bash %s 2> %s.log 1> %s.out \n" % \
('%', '%', '%', minimap_script, minimap_script.split('.sh')[0], minimap_script, minimap_script.split('.sh')[0])
runall_fh.write(mycmd)

# Create miniasm script
if args.miniasm:
    miniasm_script = "%s_miniasm.sh" % (args.prefix)
    miniasm_dir = "%s/miniasm" % (args.dir)
    miniasm_fh = open(miniasm_script, 'wb')
    miniasm_fh.write("#!/bin/bash\n")
    miniasm_fh.write("if [ ! -d \"%s\" ]; then mkdir %s; fi\n" % (miniasm_dir, miniasm_dir))
    miniasm_fh.write("reads=\"%s\"\n" % args.reads)
    miniasm_fh.write("if [ -e \"%s\" ]; then reads=\"%s\"; fi\n" % (readspp, readspp))
    miniasm_fh.write("# Run miniasm\n")
    miniasm_out = "%s/%s.miniasm.gfa" % (miniasm_dir, args.prefix)
    mycmd = cmd_printer(args.miniasm, args.miniasm_opts, "-f $reads",
    minimap_out, output=miniasm_out, comment=do_comment)
    miniasm_fh.write(mycmd)
    miniasm_fasta = "%s/%s.miniasm.fasta" % (miniasm_dir, args.prefix)
    mycmd = "#awk '/^S/{print \">\"$2\"\\n\"$3}' %s | fold > %s\n" % (miniasm_out, miniasm_fasta)
    miniasm_fh.write(mycmd)

mycmd = "/usr/bin/time -f \"%sE time %sK avg."\
" mem %sM max mem\" /bin/bash %s 2> %s.log 1> %s.out \n" % \
('%', '%', '%', miniasm_script, miniasm_script.split('.sh')[0], miniasm_script.split('.sh')[0])
runall_fh.write(mycmd)

# Run spectrassembler
spectral_script = "%s_spectral.sh" % (args.prefix)
spectral_dir = "%s/spectral" % (args.dir)
spectral_fh = open(spectral_script, 'wb')
spectral_fh.write("#!/bin/bash\n")
spectral_fh.write("if [ ! -d \"%s\" ]; then mkdir %s; fi\n" % (spectral_dir, spectral_dir))
spectral_fh.write("reads=\"%s\"\n" % args.reads)
spectral_fh.write("if [ -e \"%s\" ]; then reads=\"%s\"; fi\n" % (readspp, readspp))
spectral_fh.write("# Run spectral\n")
spectral_out = "%s/%s.spectral.fasta" % (spectral_dir, args.prefix)
mycmd = cmd_printer("python %s/spectrassembler.py" % (spectrassembler), "-f $reads",
            "-m", minimap_out, "-r", spectral_dir,
            args.spectrassembler_opts, output=spectral_out, comment=do_comment)
spectral_fh.write(mycmd)

mycmd = "/usr/bin/time -f \"%sE time %sK avg."\
" mem %sM max mem\" /bin/bash %s 2> %s.log 1> %s.out \n" % \
('%', '%', '%', spectral_script, spectral_script.split('.sh')[0], spectral_script.split('.sh')[0])
runall_fh.write(mycmd)

# Run Racon if provided
if args.racon and args.minimap:
    raconspectral_script = "%s_raconspectral.sh" % (args.prefix)
    raconspectral_dir = "%s/raconspectral" % (args.dir)
    raconspectral_fh = open(raconspectral_script, 'wb')
    raconspectral_fh.write("#!/bin/bash\n")
    raconspectral_fh.write("if [ ! -d \"%s\" ]; then mkdir %s; fi\n" % (raconspectral_dir, raconspectral_dir))
    raconspectral_fh.write("# Check that reads are .fastq for racon\n")
    mycmd = "if [ \"${reads:(-1)}\" == \"q\" ]; then\
    \n  readsq=\"$reads\"\
    \nelse\
    \n  readsq=\"${reads/.f*a/.fastq}\"\
    \nfi\n"
    raconspectral_fh.write(mycmd)
    raconspectral_fh.write("# Run racon + spectral\n")
    # Map reads to miniasm assembly with minimap
    spectral_mappings_out = "%s/%s.spectral_mappings.paf" % (raconspectral_dir, args.prefix)
    mycmd = cmd_printer(args.minimap, spectral_out, "$readsq",
    output=spectral_mappings_out, comment=do_comment)
    raconspectral_fh.write(mycmd)

    # Run Racon
    spectral_racon_out = "%s/%s.spectral_racon.fasta" % (raconspectral_dir, args.prefix)
    mycmd = cmd_printer(args.racon, "$readsq", spectral_mappings_out,
    spectral_out, spectral_racon_out, comment=do_comment)
    raconspectral_fh.write(mycmd)

    mycmd = "/usr/bin/time -f \"%sE time %sK avg."\
    " mem %sM max mem\" /bin/bash %s 2> %s.log 1> %s.out \n" % \
    ('%', '%', '%', raconspectral_script, raconspectral_script.split('.sh')[0],
    raconspectral_script.split('.sh')[0])
    runall_fh.write(mycmd)

    # Racon + miniasm
    if args.miniasm:
        raconminiasm_script = "%s_raconminiasm.sh" % (args.prefix)
        raconminiasm_dir = "%s/raconminiasm" % (args.dir)
        raconminiasm_fh = open(raconminiasm_script, 'wb')
        raconminiasm_fh.write("#!/bin/bash\n")
        raconminiasm_fh.write("if [ ! -d \"%s\" ]; then mkdir %s; fi\n" % (raconminiasm_dir, raconminiasm_dir))
        raconminiasm_fh.write("# Check that reads are .fastq for racon\n")
        mycmd = "if [ \"${reads:(-1)}\" == \"q\" ]; then\
        \n  readsq=\"$reads\"\
        \nelse\
        \n  readsq=\"${reads/.f*a/.fastq}\"\
        \nfi\n"
        raconminiasm_fh.write(mycmd)
        raconminiasm_fh.write("# Run racon + miniasm\n")
        # Map reads to miniasm assembly with minimap
        mappings_out = "%s/%s.miniasm_mappings.paf" % (raconminiasm_dir, args.prefix)
        mycmd = cmd_printer(args.minimap, miniasm_fasta, "$readsq", output=mappings_out, comment=do_comment)
        raconminiasm_fh.write(mycmd)

        # Run Racon
        racon_out = "%s/%s.miniasm_racon.fasta" % (raconminiasm_dir, args.prefix)
        mycmd = cmd_printer(args.racon, "$readsq", mappings_out, miniasm_out, racon_out, comment=do_comment)
        raconminiasm_fh.write(mycmd)

        # Round 2 of Racon + miniasm
        raconminiasm_fh.write("# Run racon + miniasm 2nd iteration\n")
        # Map reads to miniasm assembly with minimap
        mappings2_out = "%s/%s.racon2_mappings.paf" % (raconminiasm_dir, args.prefix)
        mycmd = cmd_printer(args.minimap, racon_out, "$readsq", output=mappings2_out)
        raconminiasm_fh.write(mycmd)

        # Run Racon
        racon2_out = "%s/%s.miniasm_racon2.fasta" % (raconminiasm_dir, args.prefix)
        mycmd = cmd_printer(args.racon, "$readsq", mappings2_out, racon_out, racon2_out)
        raconminiasm_fh.write(mycmd)

        mycmd = "/usr/bin/time -f \"%sE time %sK avg."\
        " mem %sM max mem\" /bin/bash %s 2> %s.log 1> %s.out \n" % \
        ('%', '%', '%', raconminiasm_script, raconminiasm_script.split('.sh')[0],
        raconminiasm_script.split('.sh')[0])
        runall_fh.write(mycmd)

# Run Canu if provided
if args.canu:
    canu_script = "%s_canu.sh" % (args.prefix)
    canu_dir = "%s/canu" % (args.dir)
    canu_fh = open(canu_script, 'wb')
    canu_fh.write("#!/bin/bash\n")
    canu_fh.write("if [ ! -d \"%s\" ]; then mkdir %s; fi\n" % (canu_dir, canu_dir))
    canu_fh.write("reads=\"%s\"\n" % args.reads)
    canu_fh.write("if [ -e \"%s\" ]; then reads=\"%s\"; fi\n" % (readspp, readspp))
    canu_fh.write("# Run canu\n")
    mycmd = cmd_printer(args.canu, "-p", args.prefix, "-d", canu_dir,
    args.canu_opts, comment=do_comment)
    canu_fh.write(mycmd)

    mycmd = "/usr/bin/time -f \"%sE time %sK avg."\
    " mem %sM max mem\" /bin/bash %s 2> %s.log 1> %s.out \n" % \
    ('%', '%', '%', canu_script, canu_script.split('.sh')[0],
    canu_script.split('.sh')[0])
    runall_fh.write(mycmd)
<commit_msg>automatic script to start jobs<commit_after>#!/usr/bin/env python

import os
import argparse

def cmd_printer(exec_line, *args, **kwargs):
    """
    Creates string to be printed to bash in order to run executable.

    Parameters
    ----------
    exec_line : str (execution line including path/to/exec)
    extra args : str (option to be passed, e.g. "-v 1")
    extra kwargs : may include "output" kwarg if redirection to "output" file
    is demanded.

    Returns
    ----------
    cmd : str (block of text to be printed to a bash script to run exec_line
    [with options and with output redirected if specified in *args and *kwargs]
    when the bash script is ran.)
    """
    comment = False
    cmd = "%s " % exec_line
    for arg in args:
        cmd += "%s " % arg
    if kwargs is not None:
        for key, value in kwargs.iteritems():
            if key in ["out", "output", "output_file"]:
                cmd += "> %s" % value
            if key == 'comment' and value == True:
                comment=True
    cmd += "\n"

    if comment:
        cmd = '#' + cmd

    return cmd

parser = argparse.ArgumentParser(description="creates bash script"
                                "to test assemblers on a dataset")
parser.add_argument("reads", help="path to reads file (fasta or fastq)")
parser.add_argument("-p", "--prefix", required=True, help="prefix for output")
parser.add_argument("-d", "--dir",required=True, help="directory for output")
parser.add_argument("--minimap", required=True,
                    help="path/to/minimap (executable)."
                    "Required for spectrassembler.")
parser.add_argument("--minimap_opts", default="-Sw5 -L100 -m0",
                    help="options for minimap")
parser.add_argument("--miniasm", help="path/to/miniasm (executable)")
parser.add_argument("--miniasm_opts", default="",
                    help="options for miniasm")
parser.add_argument("--racon", help="path/to/racon (executable)")
parser.add_argument("--canu", help="path/to/canu (executable)")
parser.add_argument("--canu_opts", default="",
                    help="options for canu. Read file needs to be in it through"
                    "e.g. -nanopore-raw reads.fasta")
parser.add_argument("--spectrassembler_opts", default="",
                    help="options for spectrassembler")
parser.add_argument("--spoa", help="path/to/spoa/")

args = parser.parse_args()
tools_path = os.path.realpath(__file__)
spectrassembler = tools_path.split('/tools')[0]

# In order not to recompute old things
do_comment = False

# Create main script
runall_script = "%s_runall.sh" % (args.prefix)
runall_fh = open(runall_script, 'wb')
runall_fh.write("#!/bin/bash\n")

# Create preprocess script file
pp_script = "%s_preprocess.sh" % (args.prefix)
pp_fh = open(pp_script, 'wb')

# Prelude
pp_fh.write("#!/bin/bash\n")
pp_fh.write("reads=\"%s\"\n" % args.reads)
pp_fh.write("if [ ! -d \"%s\" ]; then mkdir %s; fi\n" % (args.dir, args.dir))

# Check read ids are unique
pp_fh.write("# Check that read ids are unique\n")
pp_fh.write("python %s/tools/check_reads.py %s\n" % (spectrassembler, args.reads))
readspp = '.'.join(args.reads.split('.')[:-1]) + '.pp.' + args.reads.split('.')[-1]
pp_fh.write("if [ -e \"%s\" ]; then reads=\"%s\"; fi\n" % (readspp, readspp))

# make fastq file for racon
if args.racon and args.minimap:
    # Racon + spectral
    pp_fh.write("# Check that reads are .fastq for racon\n")
    mycmd = "if [ \"${reads:(-1)}\" == \"q\" ]; then\
    \n  readsq=\"$reads\"\
    \nelse\
    \n  echo \"Reads are in fasta format. Convert to fastq before calling racon...\"\
    \n  readsq=\"${reads/.f*a/.fastq}\"\
    \n  python %s/tools/fasta2fastq.py \"$reads\" \"$readsq\"\
    \nfi\n" % (spectrassembler)
    pp_fh.write(mycmd)

mycmd = "/usr/bin/time -f \"%sE time %sK avg."\
" mem %sM max mem\" /bin/bash %s 2> %s.log 1> %s.out \n" % \
('%', '%', '%', pp_script, pp_script.split('.sh')[0], pp_script.split('.sh')[0])
runall_fh.write(mycmd)

# Create minimap script
minimap_script = "%s_minimap.sh" % (args.prefix)
minimap_fh = open(minimap_script, 'wb')
minimap_fh.write("#!/bin/bash\n")
minimap_fh.write("reads=\"%s\"\n" % args.reads)
minimap_fh.write("if [ -e \"%s\" ]; then reads=\"%s\"; fi\n" % (readspp, readspp))
minimap_fh.write("# Run minimap\n")
minimap_out = "%s/%s.mini.paf" % (args.dir, args.prefix)
mycmd = cmd_printer(args.minimap, args.minimap_opts, "$reads",
"$reads", output=minimap_out, comment=do_comment)
minimap_fh.write(mycmd)

mycmd = "/usr/bin/time -f \"%sE time %sK avg."\
" mem %sM max mem\" /bin/bash %s 2> %s.log 1> %s.out \n" % \
('%', '%', '%', minimap_script, minimap_script.split('.sh')[0],
 minimap_script.split('.sh')[0])
runall_fh.write(mycmd)

# Create miniasm script
if args.miniasm:
    miniasm_script = "%s_miniasm.sh" % (args.prefix)
    miniasm_dir = "%s/miniasm" % (args.dir)
    miniasm_fh = open(miniasm_script, 'wb')
    miniasm_fh.write("#!/bin/bash\n")
    miniasm_fh.write("if [ ! -d \"%s\" ]; then mkdir %s; fi\n" % (miniasm_dir, miniasm_dir))
    miniasm_fh.write("reads=\"%s\"\n" % args.reads)
    miniasm_fh.write("if [ -e \"%s\" ]; then reads=\"%s\"; fi\n" % (readspp, readspp))
    miniasm_fh.write("# Run miniasm\n")
    miniasm_out = "%s/%s.miniasm.gfa" % (miniasm_dir, args.prefix)
    mycmd = cmd_printer(args.miniasm, args.miniasm_opts, "-f $reads",
    minimap_out, output=miniasm_out, comment=do_comment)
    miniasm_fh.write(mycmd)
    miniasm_fasta = "%s/%s.miniasm.fasta" % (miniasm_dir, args.prefix)
    mycmd = "#awk '/^S/{print \">\"$2\"\\n\"$3}' %s | fold > %s\n" % (miniasm_out, miniasm_fasta)
    miniasm_fh.write(mycmd)

mycmd = "/usr/bin/time -f \"%sE time %sK avg."\
" mem %sM max mem\" /bin/bash %s 2> %s.log 1> %s.out \n" % \
('%', '%', '%', miniasm_script, miniasm_script.split('.sh')[0], miniasm_script.split('.sh')[0])
runall_fh.write(mycmd)

# Run spectrassembler
spectral_script = "%s_spectral.sh" % (args.prefix)
spectral_dir = "%s/spectral" % (args.dir)
spectral_fh = open(spectral_script, 'wb')
spectral_fh.write("#!/bin/bash\n")
spectral_fh.write("if [ ! -d \"%s\" ]; then mkdir %s; fi\n" % (spectral_dir, spectral_dir))
spectral_fh.write("reads=\"%s\"\n" % args.reads)
spectral_fh.write("if [ -e \"%s\" ]; then reads=\"%s\"; fi\n" % (readspp, readspp))
spectral_fh.write("# Run spectral\n")
spectral_out = "%s/%s.spectral.fasta" % (spectral_dir, args.prefix)
mycmd = cmd_printer("python %s/spectrassembler.py" % (spectrassembler), "-f $reads",
            "-m", minimap_out, "-r", spectral_dir,
            args.spectrassembler_opts, output=spectral_out, comment=do_comment)
spectral_fh.write(mycmd)

mycmd = "/usr/bin/time -f \"%sE time %sK avg."\
" mem %sM max mem\" /bin/bash %s 2> %s.log 1> %s.out \n" % \
('%', '%', '%', spectral_script, spectral_script.split('.sh')[0], spectral_script.split('.sh')[0])
runall_fh.write(mycmd)

# Run Racon if provided
if args.racon and args.minimap:
    raconspectral_script = "%s_raconspectral.sh" % (args.prefix)
    raconspectral_dir = "%s/raconspectral" % (args.dir)
    raconspectral_fh = open(raconspectral_script, 'wb')
    raconspectral_fh.write("#!/bin/bash\n")
    raconspectral_fh.write("if [ ! -d \"%s\" ]; then mkdir %s; fi\n" % (raconspectral_dir, raconspectral_dir))
    raconspectral_fh.write("# Check that reads are .fastq for racon\n")
    mycmd = "if [ \"${reads:(-1)}\" == \"q\" ]; then\
    \n  readsq=\"$reads\"\
    \nelse\
    \n  readsq=\"${reads/.f*a/.fastq}\"\
    \nfi\n"
    raconspectral_fh.write(mycmd)
    raconspectral_fh.write("# Run racon + spectral\n")
    # Map reads to miniasm assembly with minimap
    spectral_mappings_out = "%s/%s.spectral_mappings.paf" % (raconspectral_dir, args.prefix)
    mycmd = cmd_printer(args.minimap, spectral_out, "$readsq",
    output=spectral_mappings_out, comment=do_comment)
    raconspectral_fh.write(mycmd)

    # Run Racon
    spectral_racon_out = "%s/%s.spectral_racon.fasta" % (raconspectral_dir, args.prefix)
    mycmd = cmd_printer(args.racon, "$readsq", spectral_mappings_out,
    spectral_out, spectral_racon_out, comment=do_comment)
    raconspectral_fh.write(mycmd)

    mycmd = "/usr/bin/time -f \"%sE time %sK avg."\
    " mem %sM max mem\" /bin/bash %s 2> %s.log 1> %s.out \n" % \
    ('%', '%', '%', raconspectral_script, raconspectral_script.split('.sh')[0],
    raconspectral_script.split('.sh')[0])
    runall_fh.write(mycmd)

    # Racon + miniasm
    if args.miniasm:
        raconminiasm_script = "%s_raconminiasm.sh" % (args.prefix)
        raconminiasm_dir = "%s/raconminiasm" % (args.dir)
        raconminiasm_fh = open(raconminiasm_script, 'wb')
        raconminiasm_fh.write("#!/bin/bash\n")
        raconminiasm_fh.write("if [ ! -d \"%s\" ]; then mkdir %s; fi\n" % (raconminiasm_dir, raconminiasm_dir))
        raconminiasm_fh.write("# Check that reads are .fastq for racon\n")
        mycmd = "if [ \"${reads:(-1)}\" == \"q\" ]; then\
        \n  readsq=\"$reads\"\
        \nelse\
        \n  readsq=\"${reads/.f*a/.fastq}\"\
        \nfi\n"
        raconminiasm_fh.write(mycmd)
        raconminiasm_fh.write("# Run racon + miniasm\n")
        # Map reads to miniasm assembly with minimap
        mappings_out = "%s/%s.miniasm_mappings.paf" % (raconminiasm_dir, args.prefix)
        mycmd = cmd_printer(args.minimap, miniasm_fasta, "$readsq", output=mappings_out, comment=do_comment)
        raconminiasm_fh.write(mycmd)

        # Run Racon
        racon_out = "%s/%s.miniasm_racon.fasta" % (raconminiasm_dir, args.prefix)
        mycmd = cmd_printer(args.racon, "$readsq", mappings_out, miniasm_out, racon_out, comment=do_comment)
        raconminiasm_fh.write(mycmd)

        # Round 2 of Racon + miniasm
        raconminiasm_fh.write("# Run racon + miniasm 2nd iteration\n")
        # Map reads to miniasm assembly with minimap
        mappings2_out = "%s/%s.racon2_mappings.paf" % (raconminiasm_dir, args.prefix)
        mycmd = cmd_printer(args.minimap, racon_out, "$readsq", output=mappings2_out)
        raconminiasm_fh.write(mycmd)

        # Run Racon
        racon2_out = "%s/%s.miniasm_racon2.fasta" % (raconminiasm_dir, args.prefix)
        mycmd = cmd_printer(args.racon, "$readsq", mappings2_out, racon_out, racon2_out)
        raconminiasm_fh.write(mycmd)

        mycmd = "/usr/bin/time -f \"%sE time %sK avg."\
        " mem %sM max mem\" /bin/bash %s 2> %s.log 1> %s.out \n" % \
        ('%', '%', '%', raconminiasm_script, raconminiasm_script.split('.sh')[0],
        raconminiasm_script.split('.sh')[0])
        runall_fh.write(mycmd)

# Run Canu if provided
if args.canu:
    canu_script = "%s_canu.sh" % (args.prefix)
    canu_dir = "%s/canu" % (args.dir)
    canu_fh = open(canu_script, 'wb')
    canu_fh.write("#!/bin/bash\n")
    canu_fh.write("if [ ! -d \"%s\" ]; then mkdir %s; fi\n" % (canu_dir, canu_dir))
    canu_fh.write("reads=\"%s\"\n" % args.reads)
    canu_fh.write("if [ -e \"%s\" ]; then reads=\"%s\"; fi\n" % (readspp, readspp))
    canu_fh.write("# Run canu\n")
    mycmd = cmd_printer(args.canu, "-p", args.prefix, "-d", canu_dir,
    args.canu_opts, comment=do_comment)
    canu_fh.write(mycmd)

    mycmd = "/usr/bin/time -f \"%sE time %sK avg."\
    " mem %sM max mem\" /bin/bash %s 2> %s.log 1> %s.out \n" % \
    ('%', '%', '%', canu_script, canu_script.split('.sh')[0],
    canu_script.split('.sh')[0])
    runall_fh.write(mycmd)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html)
"""
Data about OCA Projects, with a few helper functions.

OCA_PROJECTS: dictionary of OCA Projects mapped to the list of related
repository names, based on
https://odoo-community.org/page/List

OCA_REPOSITORY_NAMES: list of OCA repository names

"""
from __future__ import print_function
from contextlib import contextmanager
import os
import shutil
import subprocess
import tempfile

import appdirs
from .config import MAIN_BRANCHES, NOT_ADDONS
from .github_login import login

ALL = ['OCA_PROJECTS', 'OCA_REPOSITORY_NAMES', 'url']

OCA_PROJECTS = {
    'accounting': ['account-analytic',
                   'account-budgeting',
                   'account-closing',
                   'account-consolidation',
                   'account-financial-tools',
                   'account-financial-reporting',
                   'account-invoice-reporting',
                   'account-invoicing',
                   'account-fiscal-rule',
                   'operating-unit',
                   'intrastat',
                   'mis-builder',
                   'currency',
                   'credit-control',
                   'data-protection',
                   ],
    # 'backport': ['OCB',
    #              ],
    'Apps Store': ['apps-store'],

    'banking': ['bank-payment',
                'account-reconcile',
                'bank-statement-import',
                'account-payment',
                ],
    'community': ['maintainer-tools',
                  'maintainer-quality-tools',
                  'runbot-addons',
                  ],
    'connector': ['connector',
                  'connector-ecommerce',
                  'queue',
                  ],
    'connector AccountEdge': ['connector-accountedge'],
    'connector CMIS': ['connector-cmis'],
    'connector Infor': ['connector-infor'],
    'connector Lengow': ['connector-lengow'],
    'connector LIMS': ['connector-lims'],
    'connector Magento': ['connector-magento'],
    'connector Prestashop': ['connector-prestashop'],
    'connector Sage': ['connector-sage'],
    'connector Salesforce': ['connector-salesforce'],
    'connector SPSCommerce': ['connector-spscommerce'],
    'connector WooCommerce': ['connector-woocommerce'],
    'crm sales marketing': ['sale-workflow',
                            'crm',
                            'partner-contact',
                            'sale-financial',
                            'sale-reporting',
                            'commission',
                            'event',
                            'survey',
                            ],
    'document': ['knowledge'],
    'ecommerce': ['e-commerce'],
    'edi': ['edi'],
    'field-service': ['field-service'],
    'financial control': ['margin-analysis'],
    'Infrastructure': ['infrastructure-dns'],
    'geospatial': ['geospatial'],
    'hr': ['hr-timesheet',
           'hr',
           'department',
           ],
    'connector-odoo2odoo': ['connector-odoo2odoo'],
    'multi-company': ['multi-company'],
    'l10n-argentina': ['l10n-argentina'],
    'l10n-belarus': ['l10n-belarus'],
    'l10n-belgium': ['l10n-belgium'],
    'l10n-brazil': ['l10n-brazil'],
    'l10n-cambodia': ['l10n-cambodia'],
    'l10n-canada': ['l10n-canada'],
    'l10n-chile': ['l10n-chile'],
    'l10n-china': ['l10n-china'],
    'l10n-colombia': ['l10n-colombia'],
    'l10n-costa-rica': ['l10n-costa-rica'],
    'l10n-croatia': ['l10n-croatia'],
    'l10n-ecuador': ['l10n-ecuador'],
    'l10n-estonia': ['l10n-estonia'],
    'l10n-ethiopia': ['l10n-ethiopia'],
    'l10n-finland': ['l10n-finland'],
    'l10n-france': ['l10n-france'],
    'l10n-germany': ['l10n-germany'],
    'l10n-greece': ['l10n-greece'],
    'l10n-india': ['l10n-india'],
    'l10n-indonesia': ['l10n-indonesia'],
    'l10n-iran': ['l10n-iran'],
    'l10n-ireland': ['l10n-ireland'],
    'l10n-italy': ['l10n-italy'],
    'l10n-japan': ['l10n-japan'],
    'l10n-luxemburg': ['l10n-luxemburg'],
    'l10n-mexico': ['l10n-mexico'],
    'l10n-morocco': ['l10n-morocco'],
    'l10n-netherlands': ['l10n-netherlands'],
    'l10n-norway': ['l10n-norway'],
    'l10n-peru': ['l10n-peru'],
    'l10n-poland': ['l10n-poland'],
    'l10n-portugal': ['l10n-portugal'],
    'l10n-romania': ['l10n-romania'],
    'l10n-russia': ['l10n-russia'],
    'l10n-slovenia': ['l10n-slovenia'],
    'l10n-spain': ['l10n-spain'],
    'l10n-switzerland': ['l10n-switzerland'],
    'l10n-taiwan': ['l10n-taiwan'],
    'l10n-thailand': ['l10n-thailand'],
    'l10n-turkey': ['l10n-turkey'],
    'l10n-usa': ['l10n-usa'],
    'l10n-united-kingdom': ['l10n-united-kingdom'],
    'l10n-venezuela': ['l10n-venezuela'],
    'l10n-vietnam': ['l10n-vietnam'],
    'logistics': ['carrier-delivery',
                  'stock-logistics-barcode',
                  'stock-logistics-workflow',
                  'stock-logistics-tracking',
                  'stock-logistics-warehouse',
                  'stock-logistics-reporting',
                  'rma',
                  'ddmrp',
                  ],
    'manufacturing': ['manufacture',
                      'manufacture-reporting',
                      ],
    'management system': ['management-system'],
    'purchase': ['purchase-workflow',
                 'purchase-reporting',
                 ],
    'product': ['product-attribute',
                'product-kitting',
                'product-variant',
                ],
    'project / services': ['project-reporting',
                           'project-service',
                           'project-agile',
                           'contract',
                           'program',
                           'business-requirement',
                           'connector-redmine',
                           'connector-jira',
                           ],
    'social': ['social'],
    'tools': ['reporting-engine',
              'report-print-send',
              'webkit-tools',
              'server-tools',
              'server-auth',
              'server-env',
              'server-backend',
              'server-brand',
              'server-ux',
              'community-data-files',
              'webhook',
              'interface-github',
              'iot',
              'rest-framework',
              ],
    'vertical association': ['vertical-association'],
    'vertical hotel': ['vertical-hotel'],
    'vertical ISP': ['vertical-isp'],
    'vertical edition': ['vertical-edition'],
    'vertical education': ['vertical-education'],
    'vertical medical': ['vertical-medical'],
    'vertical NGO': ['vertical-ngo',
                     # XXX
                     ],
    'vertical construction': ['vertical-construction'],
    'vertical real state': ['vertical-realstate'],
    'vertical travel': ['vertical-travel'],
    'web': ['web'],
    'website': ['website',
                'website-cms',
                ],
}

def get_repositories():
    gh = login()
    all_repos = [repo.name for repo in gh.repositories_by('OCA')
                 if repo.name not in NOT_ADDONS]
    return all_repos

def get_repositories_and_branches(repos=(), branches=MAIN_BRANCHES):
    gh = login()
    for repo in gh.repositories_by('OCA'):
        if repos and repo.name not in repos:
            continue
        if repo.name in NOT_ADDONS:
            continue
        for branch in repo.iter_branches():
            if branches and branch.name not in branches:
                continue
            yield repo.name, branch.name

try:
    OCA_REPOSITORY_NAMES = get_repositories()
except Exception as exc:
    print(exc)
    OCA_REPOSITORY_NAMES = []
    for repos in OCA_PROJECTS.values():
        OCA_REPOSITORY_NAMES += repos

OCA_REPOSITORY_NAMES.sort()

_OCA_REPOSITORY_NAMES = set(OCA_REPOSITORY_NAMES)

_URL_MAPPINGS = {'git': 'git@github.com:%s/%s.git',
                 'https': 'https://github.com/%s/%s.git',
                 }

def url(project_name, protocol='git', org_name='OCA'):
    """get the URL for an OCA project repository"""
    if project_name not in _OCA_REPOSITORY_NAMES:
        raise ValueError('Unknown project', project_name)
    return _URL_MAPPINGS[protocol] % (org_name, project_name)

class BranchNotFoundError(RuntimeError):
    pass

@contextmanager
def temporary_clone(project_name, branch, protocol='git', org_name='OCA'):
    """ context manager that clones a git branch and cd to it, with cache """
    # init cache directory
    cache_dir = appdirs.user_cache_dir('oca-mqt')
    repo_cache_dir = os.path.join(
        cache_dir, 'github.com', org_name.lower(), project_name.lower())
    if not os.path.isdir(repo_cache_dir):
        os.makedirs(repo_cache_dir)
        subprocess.check_call(['git', 'init', '--bare'], cwd=repo_cache_dir)
    repo_url = url(project_name, protocol, org_name)
    # fetch all branches into cache
    fetch_cmd = [
        'git', 'fetch', '--quiet', '--force',
        repo_url,
        'refs/heads/*:refs/heads/*',
    ]
    subprocess.check_call(fetch_cmd, cwd=repo_cache_dir)
    # check if branch exist
    branches = subprocess.check_output(
        ['git', 'branch'], universal_newlines=True, cwd=repo_cache_dir)
    branches = [b.strip() for b in branches.split()]
    if branch not in branches:
        raise BranchNotFoundError()
    # clone to temp dir, with --reference to cache
    tempdir = tempfile.mkdtemp()
    try:
        clone_cmd = [
            'git', 'clone', '--quiet',
            '--reference', repo_cache_dir,
            '--branch', branch,
            '--',
            repo_url,
            tempdir,
        ]
        subprocess.check_call(clone_cmd)
        cwd = os.getcwd()
        os.chdir(tempdir)
        try:
            yield
        finally:
            os.chdir(cwd)
    finally:
        shutil.rmtree(tempdir)
<commit_msg>fix github3.py iter_branches<commit_after># -*- coding: utf-8 -*-
# License AGPLv3 (http://www.gnu.org/licenses/agpl-3.0-standalone.html)
"""
Data about OCA Projects, with a few helper functions.

OCA_PROJECTS: dictionary of OCA Projects mapped to the list of related
repository names, based on
https://odoo-community.org/page/List

OCA_REPOSITORY_NAMES: list of OCA repository names

"""
from __future__ import print_function
from contextlib import contextmanager
import os
import shutil
import subprocess
import tempfile

import appdirs
from .config import MAIN_BRANCHES, NOT_ADDONS
from .github_login import login

ALL = ['OCA_PROJECTS', 'OCA_REPOSITORY_NAMES', 'url']

OCA_PROJECTS = {
    'accounting': ['account-analytic',
                   'account-budgeting',
                   'account-closing',
                   'account-consolidation',
                   'account-financial-tools',
                   'account-financial-reporting',
                   'account-invoice-reporting',
                   'account-invoicing',
                   'account-fiscal-rule',
                   'operating-unit',
                   'intrastat',
                   'mis-builder',
                   'currency',
                   'credit-control',
                   'data-protection',
                   ],
    # 'backport': ['OCB',
    #              ],
    'Apps Store': ['apps-store'],

    'banking': ['bank-payment',
                'account-reconcile',
                'bank-statement-import',
                'account-payment',
                ],
    'community': ['maintainer-tools',
                  'maintainer-quality-tools',
                  'runbot-addons',
                  ],
    'connector': ['connector',
                  'connector-ecommerce',
                  'queue',
                  ],
    'connector AccountEdge': ['connector-accountedge'],
    'connector CMIS': ['connector-cmis'],
    'connector Infor': ['connector-infor'],
    'connector Lengow': ['connector-lengow'],
    'connector LIMS': ['connector-lims'],
    'connector Magento': ['connector-magento'],
    'connector Prestashop': ['connector-prestashop'],
    'connector Sage': ['connector-sage'],
    'connector Salesforce': ['connector-salesforce'],
    'connector SPSCommerce': ['connector-spscommerce'],
    'connector WooCommerce': ['connector-woocommerce'],
    'crm sales marketing': ['sale-workflow',
                            'crm',
                            'partner-contact',
                            'sale-financial',
                            'sale-reporting',
                            'commission',
                            'event',
                            'survey',
                            ],
    'document': ['knowledge'],
    'ecommerce': ['e-commerce'],
    'edi': ['edi'],
    'field-service': ['field-service'],
    'financial control': ['margin-analysis'],
    'Infrastructure': ['infrastructure-dns'],
    'geospatial': ['geospatial'],
    'hr': ['hr-timesheet',
           'hr',
           'department',
           ],
    'connector-odoo2odoo': ['connector-odoo2odoo'],
    'multi-company': ['multi-company'],
    'l10n-argentina': ['l10n-argentina'],
    'l10n-belarus': ['l10n-belarus'],
    'l10n-belgium': ['l10n-belgium'],
    'l10n-brazil': ['l10n-brazil'],
    'l10n-cambodia': ['l10n-cambodia'],
    'l10n-canada': ['l10n-canada'],
    'l10n-chile': ['l10n-chile'],
    'l10n-china': ['l10n-china'],
    'l10n-colombia': ['l10n-colombia'],
    'l10n-costa-rica': ['l10n-costa-rica'],
    'l10n-croatia': ['l10n-croatia'],
    'l10n-ecuador': ['l10n-ecuador'],
    'l10n-estonia': ['l10n-estonia'],
    'l10n-ethiopia': ['l10n-ethiopia'],
    'l10n-finland': ['l10n-finland'],
    'l10n-france': ['l10n-france'],
    'l10n-germany': ['l10n-germany'],
    'l10n-greece': ['l10n-greece'],
    'l10n-india': ['l10n-india'],
    'l10n-indonesia': ['l10n-indonesia'],
    'l10n-iran': ['l10n-iran'],
    'l10n-ireland': ['l10n-ireland'],
    'l10n-italy': ['l10n-italy'],
    'l10n-japan': ['l10n-japan'],
    'l10n-luxemburg': ['l10n-luxemburg'],
    'l10n-mexico': ['l10n-mexico'],
    'l10n-morocco': ['l10n-morocco'],
    'l10n-netherlands': ['l10n-netherlands'],
    'l10n-norway': ['l10n-norway'],
    'l10n-peru': ['l10n-peru'],
    'l10n-poland': ['l10n-poland'],
    'l10n-portugal': ['l10n-portugal'],
    'l10n-romania': ['l10n-romania'],
    'l10n-russia': ['l10n-russia'],
    'l10n-slovenia': ['l10n-slovenia'],
    'l10n-spain': ['l10n-spain'],
    'l10n-switzerland': ['l10n-switzerland'],
    'l10n-taiwan': ['l10n-taiwan'],
    'l10n-thailand': ['l10n-thailand'],
    'l10n-turkey': ['l10n-turkey'],
    'l10n-usa': ['l10n-usa'],
    'l10n-united-kingdom': ['l10n-united-kingdom'],
    'l10n-venezuela': ['l10n-venezuela'],
    'l10n-vietnam': ['l10n-vietnam'],
    'logistics': ['carrier-delivery',
                  'stock-logistics-barcode',
                  'stock-logistics-workflow',
                  'stock-logistics-tracking',
                  'stock-logistics-warehouse',
                  'stock-logistics-reporting',
                  'rma',
                  'ddmrp',
                  ],
    'manufacturing': ['manufacture',
                      'manufacture-reporting',
                      ],
    'management system': ['management-system'],
    'purchase': ['purchase-workflow',
                 'purchase-reporting',
                 ],
    'product': ['product-attribute',
                'product-kitting',
                'product-variant',
                ],
    'project / services': ['project-reporting',
                           'project-service',
                           'project-agile',
                           'contract',
                           'program',
                           'business-requirement',
                           'connector-redmine',
                           'connector-jira',
                           ],
    'social': ['social'],
    'tools': ['reporting-engine',
              'report-print-send',
              'webkit-tools',
              'server-tools',
              'server-auth',
              'server-env',
              'server-backend',
              'server-brand',
              'server-ux',
              'community-data-files',
              'webhook',
              'interface-github',
              'iot',
              'rest-framework',
              ],
    'vertical association': ['vertical-association'],
    'vertical hotel': ['vertical-hotel'],
    'vertical ISP': ['vertical-isp'],
    'vertical edition': ['vertical-edition'],
    'vertical education': ['vertical-education'],
    'vertical medical': ['vertical-medical'],
    'vertical NGO': ['vertical-ngo',
                     # XXX
                     ],
    'vertical construction': ['vertical-construction'],
    'vertical real state': ['vertical-realstate'],
    'vertical travel': ['vertical-travel'],
    'web': ['web'],
    'website': ['website',
                'website-cms',
                ],
}

def get_repositories():
    gh = login()
    all_repos = [repo.name for repo in gh.repositories_by('OCA')
                 if repo.name not in NOT_ADDONS]
    return all_repos

def get_repositories_and_branches(repos=(), branches=MAIN_BRANCHES):
    gh = login()
    for repo in gh.repositories_by('OCA'):
        if repos and repo.name not in repos:
            continue
        if repo.name in NOT_ADDONS:
            continue
        for branch in repo.branches():
            if branches and branch.name not in branches:
                continue
            yield repo.name, branch.name

try:
    OCA_REPOSITORY_NAMES = get_repositories()
except Exception as exc:
    print(exc)
    OCA_REPOSITORY_NAMES = []
    for repos in OCA_PROJECTS.values():
        OCA_REPOSITORY_NAMES += repos

OCA_REPOSITORY_NAMES.sort()

_OCA_REPOSITORY_NAMES = set(OCA_REPOSITORY_NAMES)

_URL_MAPPINGS = {'git': 'git@github.com:%s/%s.git',
                 'https': 'https://github.com/%s/%s.git',
                 }

def url(project_name, protocol='git', org_name='OCA'):
    """get the URL for an OCA project repository"""
    if project_name not in _OCA_REPOSITORY_NAMES:
        raise ValueError('Unknown project', project_name)
    return _URL_MAPPINGS[protocol] % (org_name, project_name)

class BranchNotFoundError(RuntimeError):
    pass

@contextmanager
def temporary_clone(project_name, branch, protocol='git', org_name='OCA'):
    """ context manager that clones a git branch and cd to it, with cache """
    # init cache directory
    cache_dir = appdirs.user_cache_dir('oca-mqt')
    repo_cache_dir = os.path.join(
        cache_dir, 'github.com', org_name.lower(), project_name.lower())
    if not os.path.isdir(repo_cache_dir):
        os.makedirs(repo_cache_dir)
        subprocess.check_call(['git', 'init', '--bare'], cwd=repo_cache_dir)
    repo_url = url(project_name, protocol, org_name)
    # fetch all branches into cache
    fetch_cmd = [
        'git', 'fetch', '--quiet', '--force',
        repo_url,
        'refs/heads/*:refs/heads/*',
    ]
    subprocess.check_call(fetch_cmd, cwd=repo_cache_dir)
    # check if branch exist
    branches = subprocess.check_output(
        ['git', 'branch'], universal_newlines=True, cwd=repo_cache_dir)
    branches = [b.strip() for b in branches.split()]
    if branch not in branches:
        raise BranchNotFoundError()
    # clone to temp dir, with --reference to cache
    tempdir = tempfile.mkdtemp()
    try:
        clone_cmd = [
            'git', 'clone', '--quiet',
            '--reference', repo_cache_dir,
            '--branch', branch,
            '--',
            repo_url,
            tempdir,
        ]
        subprocess.check_call(clone_cmd)
        cwd = os.getcwd()
        os.chdir(tempdir)
        try:
            yield
        finally:
            os.chdir(cwd)
    finally:
        shutil.rmtree(tempdir)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright JS Foundation and other contributors, http://js.foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function
import signal
import subprocess
import sys

TERM_NORMAL = '\033[0m'
TERM_RED = '\033[1;31m'
TERM_GREEN = '\033[1;32m'

def set_timezone(timezone):
    assert sys.platform == 'win32', "set_timezone is Windows only function"
    subprocess.call(['cmd', '/S', '/C', 'tzutil', '/s', timezone])

def set_timezone_and_exit(timezone):
    assert sys.platform == 'win32', "set_timezone_and_exit is Windows only function"
    set_timezone(timezone)
    sys.exit(1)

def get_timezone():
    assert sys.platform == 'win32', "get_timezone is Windows only function"
    return subprocess.check_output(['cmd', '/S', '/C', 'tzutil', '/g'])

def set_sighdl_to_reset_timezone(timezone):
    assert sys.platform == 'win32', "install_signal_handler_to_restore_timezone is Windows only function"
    signal.signal(signal.SIGINT, lambda signal, frame: set_timezone_and_exit(timezone))

def print_test_summary(summary_string, total, passed, failed):
    print("\n[summary] %s\n" % summary_string)
    print("TOTAL: %d" % total)
    print("%sPASS: %d%s" % (TERM_GREEN, passed, TERM_NORMAL))
    print("%sFAIL: %d%s\n" % (TERM_RED, failed, TERM_NORMAL))

    success_color = TERM_GREEN if passed == total else TERM_RED
    print("%sSuccess: %d%%%s" % (success_color, passed*100/total, TERM_NORMAL))

def print_test_result(tested, total, is_passed, passed_string, test_path, is_snapshot_generation=None):
    if is_snapshot_generation is None:
        snapshot_string = ''
    elif is_snapshot_generation:
        snapshot_string = ' (generate snapshot)'
    else:
        snapshot_string = ' (execute snapshot)'

    color = TERM_GREEN if is_passed else TERM_RED
    print("[%4d/%4d] %s%s: %s%s%s" % (tested, total, color, passed_string, test_path, snapshot_string, TERM_NORMAL))
<commit_msg>Fixes timezone get on win32 (#4494)<commit_after>#!/usr/bin/env python

# Copyright JS Foundation and other contributors, http://js.foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function
import signal
import subprocess
import sys

TERM_NORMAL = '\033[0m'
TERM_RED = '\033[1;31m'
TERM_GREEN = '\033[1;32m'

def set_timezone(timezone):
    assert sys.platform == 'win32', "set_timezone is Windows only function"
    subprocess.call(['cmd', '/S', '/C', 'tzutil', '/s', timezone])

def set_timezone_and_exit(timezone):
    assert sys.platform == 'win32', "set_timezone_and_exit is Windows only function"
    set_timezone(timezone)
    sys.exit(1)

def get_timezone():
    assert sys.platform == 'win32', "get_timezone is Windows only function"
    return subprocess.check_output(['cmd', '/S', '/C', 'tzutil', '/g'], universal_newlines=True)

def set_sighdl_to_reset_timezone(timezone):
    assert sys.platform == 'win32', "install_signal_handler_to_restore_timezone is Windows only function"
    signal.signal(signal.SIGINT, lambda signal, frame: set_timezone_and_exit(timezone))

def print_test_summary(summary_string, total, passed, failed):
    print("\n[summary] %s\n" % summary_string)
    print("TOTAL: %d" % total)
    print("%sPASS: %d%s" % (TERM_GREEN, passed, TERM_NORMAL))
    print("%sFAIL: %d%s\n" % (TERM_RED, failed, TERM_NORMAL))

    success_color = TERM_GREEN if passed == total else TERM_RED
    print("%sSuccess: %d%%%s" % (success_color, passed*100/total, TERM_NORMAL))

def print_test_result(tested, total, is_passed, passed_string, test_path, is_snapshot_generation=None):
    if is_snapshot_generation is None:
        snapshot_string = ''
    elif is_snapshot_generation:
        snapshot_string = ' (generate snapshot)'
    else:
        snapshot_string = ' (execute snapshot)'

    color = TERM_GREEN if is_passed else TERM_RED
    print("[%4d/%4d] %s%s: %s%s%s" % (tested, total, color, passed_string, test_path, snapshot_string, TERM_NORMAL))
<|endoftext|>