<commit_before>from __future__ import (unicode_literals, absolute_import, division,
                        print_function)
import six

import os
from collections import OrderedDict

def read_log_from_script(path_to_log):
    """
    Parse the log that is output from the `dev-build` script

    Parameters
    ----------
    path_to_log : str
        The path to the log file that results from `bash dev-build > log 2>&1`

    Yields
    ------
    package : str
        The name of the package that is being built
    output : list
        The lines that were output for the build/test/upload of `package`
    """
    BUILD_START_LINE = '/tmp/staged-recipes'
    CONDA_BUILD_START_LINE = "BUILD START: "
    PACKAGE_NAME_LINE = '# $ anaconda upload '
    full_path = os.path.abspath(path_to_log)
    output = []
    package_name = ''
    built_name = ''
    with open(full_path, 'r') as f:
        for line in f.readlines():
            # remove white space and newline characters
            line = line.strip()
            if line.startswith(PACKAGE_NAME_LINE):
                # split the line on the whitespace that looks something like:
                # "# $ anaconda upload /tmp/root/ramdisk/mc/conda-bld/linux-64/album-v0.0.2_py35.tar.bz2"
                built_package_path = line.split()[-1]
                # remove the folder path
                built_package_name = os.path.split(built_package_path)[-1]
                # trim the '.tar.bz2'
                built_name = built_package_name[:-8]
            if line.startswith(CONDA_BUILD_START_LINE) and built_name == '':
                # this is the start of the build section in the conda-build CLI
                built_name = line.split(CONDA_BUILD_START_LINE)[1].strip()
                package_name = built_name.split('-')[0]
            if line.startswith(BUILD_START_LINE):
                # this is the start of *my* cron job script
                # always have to treat the first package differently...
                if package_name != '':
                    yield package_name, built_name, output
                package_name = os.path.split(line)[1]
                built_name = '%s-build-name-not-found' % package_name
                output = []
            else:
                output.append(line)

    yield package_name, built_name, output

def parse_conda_build(lines_iterable):
    """
    Group the output from conda-build into
    - 'build_init'
    - 'build'
    - 'test'
    - 'upload'
    """
    bundle = []
    gen = (line for line in lines_iterable)
    ret = OrderedDict()
    # parse the init section
    for line in gen:
        bundle.append(line)
        if line.startswith("BUILD START"):
            # remove the build start line from the init section
            line = bundle.pop()
            ret['init'] = bundle
            bundle = [line]
            break
    else:
        ret['init'] = bundle
        return ret
    # parse the build section
    for line in gen:
        bundle.append(line)
        if line.startswith("BUILD END"):
            ret['build'] = bundle
            bundle = []
            break
    else:
        ret['build'] = bundle
        return ret
    # parse the test section
    line = next(gen)
    bundle.append(line)
    if line.startswith("TEST START"):
        # we are in the test section
        for line in gen:
            bundle.append(line)
            if line.startswith("TEST END"):
                ret['test'] = bundle
                bundle = []
                break
        else:
            # we have exhausted the generator without finding
            # the end of the test section.
            ret['test'] = bundle
            return ret
    elif 'test' in line:
        # This line will appear if there is no test section
        ret['test'] = [line]
        bundle = []
    # the rest is the upload section
    ret['upload'] = list(gen)
    return ret

def check_for_errors(line, gen):
    """Check for errors in the generator `gen` that is passed in.

    Specifically look at `line` and see if it starts with "Error: " or
    "Traceback". If so, start iterating over `gen` until there is a blank line
    or the generator ends.  Return the last line and any lines that are errors

    Parameters
    ----------
    line : str
        The last line that was taken from `gen`
    gen : generator
        The generator that should be iterated over if `line` is the start of
        an error message

    Returns
    -------
    line : str
        The last line that was taken from `gen`. Might be the same line that
        was passed in
    err : list
        The list of lines that make up the error message. If `err` is an empty
        list, that signifies that the input `line` is not an error message.

    TODO: Join the error on a new line. Need to verify that tests still pass
          after I do this
    """
    ERROR = "Error: "
    TRACEBACK = 'Traceback (most recent call last):'
    err = []
    if line.startswith(ERROR) or line == TRACEBACK:
        if line.startswith(ERROR) or line == TRACEBACK:
            try:
                while line != '':
                    err.append(line)
                    line = next(gen)
            except StopIteration:
                # this is thrown when the error goes to the end of the
                # log section
                pass
    return line, '\n'.join(err)

def parse_init(init_section):
    """Parse the init section from the output of conda build

    Specifically look for the build_command and any errors that may have
    occurred

    Parameters
    ----------
    init_section : iterable
        Iterable of the lines that occur before conda-build spits out the line
        that reads: "BUILD START"

    Returns
    -------
    dict
        Keyed on
        - 'build_command': something like this: conda-build /tmp/staged-recipes/recipes/tifffile --python=3.5
        - 'err': Might be an empty list. Otherwise it will be a list of lists,
                 where each top level list will be the output of
                 `check_for_errors()`
    """
    ret = {}
    gen = (line for line in init_section)
    ret['err'] = []
    for line in gen:
        if 'CONDA_CMD' in line:
            ret['build_command'] = line.split('-->')[1].strip()
        line, err = check_for_errors(line, gen)
        if err:
            ret['err'].append(err)
    return ret

def parse_build(build_section):
    """Parse the build section output from `conda-build`. This is everything
    between the lines that start with "BUILD START" and "BUILD FINISH".

    Parameters
    ----------
    build_section : iterable
        Iterable of all lines between "BUILD START" and "BUILD END"

    Returns
    -------
    dict
        Keyed on
        - 'built_name': Will either be 'failed' or the full name of the
                        package that will look something like
                        suitcase-0.2.1-py35_0
        - 'err': Might be an empty list. Otherwise it will be a list of lists,
                 where each top level list will be the output of
                 `check_for_errors()`
    """
    gen = (line for line in build_section)
    PACKAGE_NAME = 'Package: '
    ret = {'err': []}
    error = False
    traceback = False
    lines = []
    for line in gen:
        if PACKAGE_NAME in line:
            # format the package name
            ret['built_name'] = line[len(PACKAGE_NAME):]
        line, err = check_for_errors(line, gen)
        if err:
            ret['err'].append(err)
            ret['built_name'] = 'failed'
    return ret

def parse_test(test_section):
    gen = (line for line in test_section)
    ret = {'err': [], 'nothing_to_test': False}
    NOTHING_TO_TEST = 'Nothing to test for: '
    NO_PKGS_FOUND = 'Error: No packages found in'
    for line in gen:
        if line.startswith(NOTHING_TO_TEST):
            ret['nothing_to_test'] = True
        line, err = check_for_errors(line, gen)
        if err:
            ret['err'].append(err)

    return ret

def parse_upload(upload_section):
    ret = {'auto_upload': True}
    AUTO_UPLOAD_NOT_SET = '# $ anaconda upload'
    FILE_ALREADY_EXISTS = '[Conflict]'
    ret['err'] = []
    gen = (line for line in upload_section)
    for line in gen:
        line, err = check_for_errors(line, gen)
        if line.startswith(AUTO_UPLOAD_NOT_SET):
            ret['auto_upload'] = False
        if line.startswith(FILE_ALREADY_EXISTS):
            ret['err'].append(line)
        if err:
            ret['err'].append(err)
    return ret

def simple_parse(path_to_log):
    """Do the whole parsing in a single function call

    Parameters
    ----------
    path_to_log : str
        The path to the log that you want to parse

    Returns
    -------
    dict
        Top level keys are the package name. For each top level key there will
        be up to four keys, {'init', 'build', 'test', and 'upload'}
        package_name:
          init:
            build_command: string
            err: list of strings
          build:
            built_name: string
            err: list of strings
          test:
            nothing_to_test: bool
            err: list of strings
          upload:

    """
    parsed = {}
    for name, built_name, lines in read_log_from_script(path_to_log):
        grouped = parse_conda_build(lines)
        parsed[name] = {}
        for section, group in grouped.items():
            func = globals()['parse_%s' % section]
            parsed[name][section] = func(group)

    return parsed

if __name__ == "__main__":
    log = 'build.log'
    gen = list(read_log_from_script(log))
    parsed = {built_name: parse_conda_build(lines)
              for name, built_name, lines in gen}
    width = max([len(name) for name in parsed.keys()])
    for name, groups in parsed.items():
        print(('{:%ds} -- {}' % width).format(
            name, [key for key, bundle in groups]))<commit_msg>ENH: Add simple printout of build results<commit_after>from __future__ import (unicode_literals, absolute_import, division,
                        print_function)
import six

from prettytable import PrettyTable
import os
from collections import OrderedDict

def read_log_from_script(path_to_log):
    """
    Parse the log that is output from the `dev-build` script

    Parameters
    ----------
    path_to_log : str
        The path to the log file that results from `bash dev-build > log 2>&1`

    Yields
    ------
    package : str
        The name of the package that is being built
    output : list
        The lines that were output for the build/test/upload of `package`
    """
    BUILD_START_LINE = '/tmp/staged-recipes'
    CONDA_BUILD_START_LINE = "BUILD START: "
    PACKAGE_NAME_LINE = '# $ anaconda upload '
    full_path = os.path.abspath(path_to_log)
    output = []
    package_name = ''
    built_name = ''
    with open(full_path, 'r') as f:
        for line in f.readlines():
            # remove white space and newline characters
            line = line.strip()
            if line.startswith(PACKAGE_NAME_LINE):
                # split the line on the whitespace that looks something like:
                # "# $ anaconda upload /tmp/root/ramdisk/mc/conda-bld/linux-64/album-v0.0.2_py35.tar.bz2"
                built_package_path = line.split()[-1]
                # remove the folder path
                built_package_name = os.path.split(built_package_path)[-1]
                # trim the '.tar.bz2'
                built_name = built_package_name[:-8]
            if line.startswith(CONDA_BUILD_START_LINE) and built_name == '':
                # this is the start of the build section in the conda-build CLI
                built_name = line.split(CONDA_BUILD_START_LINE)[1].strip()
                package_name = built_name.split('-')[0]
            if line.startswith(BUILD_START_LINE):
                # this is the start of *my* cron job script
                # always have to treat the first package differently...
                if package_name != '':
                    yield package_name, built_name, output
                package_name = os.path.split(line)[1]
                built_name = '%s-build-name-not-found' % package_name
                output = []
            else:
                output.append(line)

    yield package_name, built_name, output

def parse_conda_build(lines_iterable):
    """
    Group the output from conda-build into
    - 'build_init'
    - 'build'
    - 'test'
    - 'upload'
    """
    bundle = []
    gen = (line for line in lines_iterable)
    ret = OrderedDict()
    # parse the init section
    for line in gen:
        bundle.append(line)
        if line.startswith("BUILD START"):
            # remove the build start line from the init section
            line = bundle.pop()
            ret['init'] = bundle
            bundle = [line]
            break
    else:
        ret['init'] = bundle
        return ret
    # parse the build section
    for line in gen:
        bundle.append(line)
        if line.startswith("BUILD END"):
            ret['build'] = bundle
            bundle = []
            break
    else:
        ret['build'] = bundle
        return ret
    # parse the test section
    line = next(gen)
    bundle.append(line)
    if line.startswith("TEST START"):
        # we are in the test section
        for line in gen:
            bundle.append(line)
            if line.startswith("TEST END"):
                ret['test'] = bundle
                bundle = []
                break
        else:
            # we have exhausted the generator without finding
            # the end of the test section.
            ret['test'] = bundle
            return ret
    elif 'test' in line:
        # This line will appear if there is no test section
        ret['test'] = [line]
        bundle = []
    # the rest is the upload section
    ret['upload'] = list(gen)
    return ret

def check_for_errors(line, gen):
    """Check for errors in the generator `gen` that is passed in.

    Specifically look at `line` and see if it starts with "Error: " or
    "Traceback". If so, start iterating over `gen` until there is a blank line
    or the generator ends.  Return the last line and any lines that are errors

    Parameters
    ----------
    line : str
        The last line that was taken from `gen`
    gen : generator
        The generator that should be iterated over if `line` is the start of
        an error message

    Returns
    -------
    line : str
        The last line that was taken from `gen`. Might be the same line that
        was passed in
    err : list
        The list of lines that make up the error message. If `err` is an empty
        list, that signifies that the input `line` is not an error message.

    TODO: Join the error on a new line. Need to verify that tests still pass
          after I do this
    """
    ERROR = "Error: "
    TRACEBACK = 'Traceback (most recent call last):'
    err = []
    if line.startswith(ERROR) or line == TRACEBACK:
        if line.startswith(ERROR) or line == TRACEBACK:
            try:
                while line != '':
                    err.append(line)
                    line = next(gen)
            except StopIteration:
                # this is thrown when the error goes to the end of the
                # log section
                pass
    return line, '\n'.join(err)

def parse_init(init_section):
    """Parse the init section from the output of conda build

    Specifically look for the build_command and any errors that may have
    occurred

    Parameters
    ----------
    init_section : iterable
        Iterable of the lines that occur before conda-build spits out the line
        that reads: "BUILD START"

    Returns
    -------
    dict
        Keyed on
        - 'build_command': something like this: conda-build /tmp/staged-recipes/recipes/tifffile --python=3.5
        - 'err': Might be an empty list. Otherwise it will be a list of lists,
                 where each top level list will be the output of
                 `check_for_errors()`
    """
    ret = {}
    gen = (line for line in init_section)
    ret['err'] = []
    for line in gen:
        if 'CONDA_CMD' in line:
            ret['build_command'] = line.split('-->')[1].strip()
        line, err = check_for_errors(line, gen)
        if err:
            ret['err'].append(err)
    return ret

def parse_build(build_section):
    """Parse the build section output from `conda-build`. This is everything
    between the lines that start with "BUILD START" and "BUILD FINISH".

    Parameters
    ----------
    build_section : iterable
        Iterable of all lines between "BUILD START" and "BUILD END"

    Returns
    -------
    dict
        Keyed on
        - 'built_name': Will either be 'failed' or the full name of the
                        package that will look something like
                        suitcase-0.2.1-py35_0
        - 'err': Might be an empty list. Otherwise it will be a list of lists,
                 where each top level list will be the output of
                 `check_for_errors()`
    """
    gen = (line for line in build_section)
    PACKAGE_NAME = 'Package: '
    ret = {'err': []}
    error = False
    traceback = False
    lines = []
    for line in gen:
        if PACKAGE_NAME in line:
            # format the package name
            ret['built_name'] = line[len(PACKAGE_NAME):]
        line, err = check_for_errors(line, gen)
        if err:
            ret['err'].append(err)
            ret['built_name'] = 'failed'
    return ret

def parse_test(test_section):
    gen = (line for line in test_section)
    ret = {'err': [], 'nothing_to_test': False}
    NOTHING_TO_TEST = 'Nothing to test for: '
    NO_PKGS_FOUND = 'Error: No packages found in'
    for line in gen:
        if line.startswith(NOTHING_TO_TEST):
            ret['nothing_to_test'] = True
        line, err = check_for_errors(line, gen)
        if err:
            ret['err'].append(err)

    return ret

def parse_upload(upload_section):
    ret = {'auto_upload': True}
    AUTO_UPLOAD_NOT_SET = '# $ anaconda upload'
    FILE_ALREADY_EXISTS = '[Conflict]'
    ret['err'] = []
    gen = (line for line in upload_section)
    for line in gen:
        line, err = check_for_errors(line, gen)
        if line.startswith(AUTO_UPLOAD_NOT_SET):
            ret['auto_upload'] = False
        if line.startswith(FILE_ALREADY_EXISTS):
            ret['err'].append(line)
        if err:
            ret['err'].append(err)
    return ret

def simple_parse(path_to_log):
    """Do the whole parsing in a single function call

    Parameters
    ----------
    path_to_log : str
        The path to the log that you want to parse

    Returns
    -------
    dict
        Top level keys are the package name. For each top level key there will
        be up to four keys, {'init', 'build', 'test', and 'upload'}
        package_name:
          init:
            build_command: string
            err: list of strings
          build:
            built_name: string
            err: list of strings
          test:
            nothing_to_test: bool
            err: list of strings
          upload:

    """
    parsed = {}
    for name, built_name, lines in read_log_from_script(path_to_log):
        grouped = parse_conda_build(lines)
        parsed[built_name] = {}
        for section, group in grouped.items():
            func = globals()['parse_%s' % section]
            parsed[built_name][section] = func(group)
            parsed[built_name]['package_name'] = name

    return parsed

def summarize(parsed_log):
    """Summarize the parsed log into a table

    Parameters
    ----------
    parsed_log : output of simple_parse()
    """
    sections = ['init', 'build', 'test', 'upload']
    table = PrettyTable(['library name', 'built package name'] + sections)
    table.align['library name'] = 'l'
    table.align['built package name'] = 'l'
    for pkg_name, parsed_groups in sorted(parsed_log.items()):
        lib_name = parsed_groups['package_name']
        if len(table._rows) > 0 and table._rows[-1][0] == lib_name:
            lib_name = ''
        row = [lib_name, pkg_name]

        for section in sections:
            if section in parsed_groups and parsed_groups[section]['err'] == []:
                msg = 'pass'
            else:
                msg = 'fail'
            row.append(msg)
        table.add_row(row)
    print(table)

if __name__ == "__main__":
    log = os.path.join('test_data', 'build.log')
    summarize(simple_parse(log))
<|endoftext|>


<commit_before>import os
from verifier import Verifier
import verifier
import unittest
# Verification tests
import json
import codecs

TASK_FILE = '201606231548.json'
with codecs.open(TASK_FILE, mode='r', encoding='utf-8') as file:
    run_info = json.load(file)

v = Verifier()

class TestVerifer(unittest.TestCase):
    def test_handle_errors(self):
        l1 = verifier.send_to_retry
        verifier.handle_errors()
        l2 = verifier.send_to_retry
        self.assertGreater(len(l1), len(l2))
        self.assertEqual(len(l2), len(l1) + len(run_info['error_list']))

    def test_get_path_from_url(self):
        path = v.get_path_from_url('http://staging.osf.io/mst3k')
        self.assertTrue(os.path.exists(path))

    def test_generate_page_dictionary(self):
        d1 = v.generate_page_dictionary('wiki/')
        self.assertGreater(len(d1), 0)

if __name__ == '__main__':
    unittest.main()<commit_msg>hey guys<commit_after>import os
from verifier import Verifier
import verifier
import unittest
# Verification tests
import json
import codecs

TASK_FILE = '201606231548.json'
with codecs.open(TASK_FILE, mode='r', encoding='utf-8') as file:
    run_info = json.load(file)

v = Verifier()

class TestVerifer(unittest.TestCase):
    def test_handle_errors(self):
        l1 = verifier.send_to_retry
        verifier.handle_errors()
        l2 = verifier.send_to_retry
        self.assertGreater(len(l1), len(l2))
        self.assertEqual(len(l2), len(l1) + len(run_info['error_list']))

    def test_get_path_from_url(self):
        path = v.get_path_from_url('http://staging.osf.io/mst3k')
        self.assertTrue(os.path.exists(path))

    def test_generate_page_dictionary(self):
        d1 = v.generate_page_dictionary('wiki/')
        self.assertGreater(len(d1), 0)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>"""
Module that defines Tool functions and test runners/result for use with
the unittest library.
"""
import sys
if sys.version_info < (2, 7):
    import unittest2 as unittest
else:
    import unittest
import os
import subprocess
import posixpath

def get_posix_path(path):
    """translates path to a posix path"""
    heads = []
    tail = path
    while tail != '':
        tail, head = os.path.split(tail)
        heads.append(head)
    return posixpath.join(*heads[::-1])

def run_with_stdlib(file_path, file_name=None):
    """Creates a test that runs a js file with the stdlib."""
    file_name = file_name if file_name else file_path

    class TestStdLib(unittest.TestCase):
        """Tests js code with the stdlib"""
        templ = {
            "js_path": file_path,
            "js_unix_path": get_posix_path(file_path),
            "js_out_path": file_path + ".out",
            "js_error": file_path + ".err",
            "name": file_name,
        }
        def reportProgres(self):
            """Should be overloaded by the test result class."""

        def runTest(self):
            """The actual test goes here."""
            cmd = (
                  'js -f "py-builtins.js" '
                  '-f "%(js_path)s" > "%(js_out_path)s" 2> "%(js_error)s"'
                  )% self.templ
            self.assertEqual(0, subprocess.call([cmd], shell = True))
            self.reportProgres()
        def __str__(self):
            return "%(js_unix_path)s [1]: " % self.templ

    return TestStdLib

def compile_file_test(file_path, file_name=None):
    """Creates a test that tests if a file can be compiled by python"""
    file_name = file_name if file_name else file_path

    class CompileFile(unittest.TestCase):
        """Test if a file can be compiled by python."""

        templ = {
            "py_path": file_path,
            "py_unix_path": get_posix_path(file_path),
            "py_out_path": file_path + ".out",
            "py_error": file_path + ".err",
            "name": file_name,
        }
        def reportProgres(self):
            """Should be overloaded by the test result class"""

        def runTest(self):
            """The actual test goes here."""
            commands = (
                (
                'python "%(py_path)s" > '
                '"%(py_out_path)s" 2> "%(py_error)s"'
                ) % self.templ,
              )
            for cmd in commands:
                self.assertEqual(0, subprocess.call([cmd], shell = True))
                self.reportProgres()
        def __str__(self):
            return "%(py_unix_path)s [1]: " % self.templ
    return CompileFile

def compile_and_run_file_test(file_path, file_name=None):
    """Creates a test that compiles and runs the python file as js"""
    file_name = file_name if file_name else file_path

    class CompileAndRunFile(unittest.TestCase):
        """Tests that a file can be compiled and run as js"""
        templ = {
        "py_path": file_path,
        "py_unix_path": get_posix_path(file_path),
        "py_out_path": file_path + ".out",
        "js_path": file_path + ".js",
        "js_out_path": file_path + ".js.out",
        "py_error": file_path + ".err",
        "js_error": file_path + ".js.err",
        "compiler_error": file_path + ".comp.err",
        "name": file_name,
        }
        def reportProgres(self):
            """Should be overloaded by the test result class"""

        def runTest(self):
            """The actual test goes here."""
            mtime_src = os.path.getmtime(self.templ['py_path'])
            try:
                mtime_py_res = os.path.getmtime(self.templ['py_out_path'])
            except OSError:
                mtime_py_res = 0
            python_command = (
                'python "%(py_path)s" > "%(py_out_path)s" 2> '
                '"%(py_error)s"'
                ) % self.templ

            try:
                mtime_js_res = os.path.getmtime(self.templ['js_path'])
            except OSError:
                mtime_js_res = 0
            compile_command = (
                'python pyjs.py --import-builtins '
                '"%(py_path)s" > "%(js_path)s" 2> '
                '"%(compiler_error)s"'
                ) % self.templ

            javascript_command = (
                'js -f "%(js_path)s" > "%(js_out_path)s" 2> '
                '"%(js_error)s"'
                ) % self.templ

            commands = []
            if mtime_py_res < mtime_src:
                commands.append(python_command)
            if mtime_js_res < mtime_src:
                commands.append(compile_command)
            commands.append(javascript_command)

            for cmd in commands:
                self.assertEqual(0, subprocess.call([cmd], shell = True))
                self.reportProgres()
            self.assertEqual(
                file(self.templ["py_out_path"]).readlines(),
                file(self.templ["js_out_path"]).readlines()
                )
            self.reportProgres()

        def __str__(self):
            return "%(py_unix_path)s [4]: " % self.templ

    return CompileAndRunFile

def compile_and_run_file_failing_test(*a, **k):
    """Turn a test to a failing test"""
    _class = compile_and_run_file_test(*a, **k)

    class FailingTest(_class):
        """Failing test"""
        @unittest.expectedFailure
        def runTest(self):
            return super(FailingTest, self).runTest()

    return FailingTest

<commit_msg> * Updated testtools/util.py to work with new pyjs.py<commit_after>"""
Module that defines Tool functions and test runners/result for use with
the unittest library.
"""
import sys
if sys.version_info < (2, 7):
    import unittest2 as unittest
else:
    import unittest
import os
import subprocess
import posixpath

def get_posix_path(path):
    """translates path to a posix path"""
    heads = []
    tail = path
    while tail != '':
        tail, head = os.path.split(tail)
        heads.append(head)
    return posixpath.join(*heads[::-1])

def run_with_stdlib(file_path, file_name=None):
    """Creates a test that runs a js file with the stdlib."""
    file_name = file_name if file_name else file_path

    class TestStdLib(unittest.TestCase):
        """Tests js code with the stdlib"""
        templ = {
            "js_path": file_path,
            "js_unix_path": get_posix_path(file_path),
            "js_out_path": file_path + ".out",
            "js_error": file_path + ".err",
            "name": file_name,
        }
        def reportProgres(self):
            """Should be overloaded by the test result class."""

        def runTest(self):
            """The actual test goes here."""
            cmd = (
                  'js -f "py-builtins.js" '
                  '-f "%(js_path)s" > "%(js_out_path)s" 2> "%(js_error)s"'
                  )% self.templ
            self.assertEqual(0, subprocess.call([cmd], shell = True))
            self.reportProgres()
        def __str__(self):
            return "%(js_unix_path)s [1]: " % self.templ

    return TestStdLib

def compile_file_test(file_path, file_name=None):
    """Creates a test that tests if a file can be compiled by python"""
    file_name = file_name if file_name else file_path

    class CompileFile(unittest.TestCase):
        """Test if a file can be compiled by python."""

        templ = {
            "py_path": file_path,
            "py_unix_path": get_posix_path(file_path),
            "py_out_path": file_path + ".out",
            "py_error": file_path + ".err",
            "name": file_name,
        }
        def reportProgres(self):
            """Should be overloaded by the test result class"""

        def runTest(self):
            """The actual test goes here."""
            commands = (
                (
                'python "%(py_path)s" > '
                '"%(py_out_path)s" 2> "%(py_error)s"'
                ) % self.templ,
              )
            for cmd in commands:
                self.assertEqual(0, subprocess.call([cmd], shell = True))
                self.reportProgres()
        def __str__(self):
            return "%(py_unix_path)s [1]: " % self.templ
    return CompileFile

def compile_and_run_file_test(file_path, file_name=None):
    """Creates a test that compiles and runs the python file as js"""
    file_name = file_name if file_name else file_path

    class CompileAndRunFile(unittest.TestCase):
        """Tests that a file can be compiled and run as js"""
        templ = {
        "py_path": file_path,
        "py_unix_path": get_posix_path(file_path),
        "py_out_path": file_path + ".out",
        "js_path": file_path + ".js",
        "js_out_path": file_path + ".js.out",
        "py_error": file_path + ".err",
        "js_error": file_path + ".js.err",
        "compiler_error": file_path + ".comp.err",
        "name": file_name,
        }
        def reportProgres(self):
            """Should be overloaded by the test result class"""

        def runTest(self):
            """The actual test goes here."""
            mtime_src = os.path.getmtime(self.templ['py_path'])
            try:
                mtime_py_res = os.path.getmtime(self.templ['py_out_path'])
            except OSError:
                mtime_py_res = 0
            python_command = (
                'python "%(py_path)s" > "%(py_out_path)s" 2> '
                '"%(py_error)s"'
                ) % self.templ

            try:
                mtime_js_res = os.path.getmtime(self.templ['js_path'])
            except OSError:
                mtime_js_res = 0
            compile_command = (
                'python pyjs.py -I '
                '"%(py_path)s" > "%(js_path)s" 2> '
                '"%(compiler_error)s"'
                ) % self.templ

            javascript_command = (
                'js -f "%(js_path)s" > "%(js_out_path)s" 2> '
                '"%(js_error)s"'
                ) % self.templ

            commands = []
            if mtime_py_res < mtime_src:
                commands.append(python_command)
            if mtime_js_res < mtime_src:
                commands.append(compile_command)
            commands.append(javascript_command)

            for cmd in commands:
                self.assertEqual(0, subprocess.call([cmd], shell = True))
                self.reportProgres()
            self.assertEqual(
                file(self.templ["py_out_path"]).readlines(),
                file(self.templ["js_out_path"]).readlines()
                )
            self.reportProgres()

        def __str__(self):
            return "%(py_unix_path)s [4]: " % self.templ

    return CompileAndRunFile

def compile_and_run_file_failing_test(*a, **k):
    """Turn a test to a failing test"""
    _class = compile_and_run_file_test(*a, **k)

    class FailingTest(_class):
        """Failing test"""
        @unittest.expectedFailure
        def runTest(self):
            return super(FailingTest, self).runTest()

    return FailingTest

<|endoftext|>


<commit_before>import logging
from enum import Enum
from functools import partial
from . import cgi
from .info import URL, DEFAULT_MASTERCODE

class Param(str, Enum):
    wifi_timeout = "APPAUTOTIME"  # set wifi timeout
    app_info = "APPINFO"  # set "application unique info"
    wifi_mode = "APPMODE"  # set WLAN mode (see Mode and ModeOnBoot)
    wifi_key = "APPNETWORKKEY"  # set network security key
    wifi_ssid = "APPSSID"  # set SSID
    passthrough_key = "BRGNETWORKKEY"  # set internet passthrough sec. key
    passthrough_ssid = "BRGSSID"  # set internet passthrough SSID
    bootscreen_path = "CIPATH"  # set wireless LAN bootscreen path
    mastercode = "MASTERCODE"  # set mastercode (required!)
    clear_mastercode = "CLEARCODE"  # clear mastercode
    timezone = "TIMEZONE"  # set timezone (e.g. 36)
    drive_mode = "WEBDAV"  # set FlashAir drive (WebDAV)

class _ModeValue(int, Enum): ...

class WifiMode(_ModeValue):
    """Wireless modes effective IMMEDIATELY"""
    access_point = 0
    station = 2
    passthrough = 3  # for wireless pass through (FW 2.00.02+)

class WifiModeOnBoot(_ModeValue):
    """Wireless modes effective upon REBOOT of the device"""
    access_point = 4
    station = 5
    passthrough = 6

class DriveMode(_ModeValue):
    disable = 0  # FlashAir Drive disabled
    enable = 1  # enabled, only read is allowed
    # NOTE: for uploads to work, you also need UPLOAD=1 in the config file
    upload = 2  # enabled, read AND write allowed

def config(param_map, mastercode=DEFAULT_MASTERCODE):
    """Takes a dictionary of {Param.key: value} and
    returns a dictionary of processed keys and values to be used in the
    construction of a POST request to FlashAir's config.cgi"""
    pmap = {Param.mastercode: mastercode}
    pmap.update(param_map)
    processed_params = dict(_process_params(pmap))
    return processed_params

def _process_params(params):
    for param, value in params.items():
        assert param in Param, "Invalid param: {}".format(param)
        yield param.value, value_validators[param](value)

def post(param_map, url=URL):
    """Posts a `param_map` created with `config` to
    the FlashAir config.cgi entrypoint"""
    prepped_request = _prep_post(url=url, **param_map)
    return cgi.send(prepped_request)

_prep_post = partial(cgi.prep_post, cgi.Entrypoint.config)

######################################################
# Functions for creating config POST parameter values

value_validators = {}

def _validator(parameter):
    def wrapper(fn):
        value_validators[parameter] = fn
        return fn
    return wrapper

@_validator(Param.wifi_timeout)
def _validate_timeout(seconds: float):
    """Creates an int from 60000 to 4294967294 that represents a
    valid millisecond wireless LAN timeout"""
    param_value = int(seconds * 1000)
    assert 60000 <= param_value <= 4294967294
    return param_value

@_validator(Param.app_info)
def _validate_app_info(info: str):
    assert 1 <= len(info) <= 16
    return info

@_validator(Param.wifi_mode)
def _validate_wifi_mode(mode: _ModeValue):
    assert mode in WifiMode or mode in WifiModeOnBoot
    return int(mode)

@_validator(Param.wifi_key)
def _validate_wifi_key(network_key: str):
    assert 0 <= len(network_key) <= 63
    return network_key

@_validator(Param.wifi_ssid)
def _validate_wifi_ssid(network_ssid: str):
    assert 1 <= len(network_ssid) <= 32
    return network_ssid

@_validator(Param.passthrough_key)
def _validate_passthrough_key(network_key: str):
    assert 0 <= len(network_key) <= 63
    return network_key

@_validator(Param.passthrough_ssid)
def _validate_passthroughssid(network_ssid: str):
    assert 1 <= len(network_ssid) <= 32
    return network_ssid

@_validator(Param.mastercode)
def _validate_mastercode(code: str):
    assert len(code) == 12
    code = code.upper()
    valid_chars = "0123456789ABCDEF"
    assert all(c in valid_chars for c in code)
    return code

@_validator(Param.bootscreen_path)
def _validate_bootscreen_path(path: str):
    return path

@_validator(Param.clear_mastercode)
def _validate_clear_mastercode(_):
    return 1

@_validator(Param.timezone)
def _validate_timezone(hours_offset: int):
    """Creates an int from -48 to 36 that represents the timezone
    offset in 15-minute increments as per the FlashAir docs"""
    param_value = int(hours_offset * 4)
    assert -48 <= param_value <= 36
    return param_value

@_validator(Param.drive_mode)
def _validate_drive_mode(mode: DriveMode):
    assert mode in DriveMode
    return int(mode)

<commit_msg>added more info to timeout validation exception<commit_after>import logging
from enum import Enum
from functools import partial
from . import cgi
from .info import URL, DEFAULT_MASTERCODE

class Param(str, Enum):
    wifi_timeout = "APPAUTOTIME"  # set wifi timeout
    app_info = "APPINFO"  # set "application unique info"
    wifi_mode = "APPMODE"  # set WLAN mode (see Mode and ModeOnBoot)
    wifi_key = "APPNETWORKKEY"  # set network security key
    wifi_ssid = "APPSSID"  # set SSID
    passthrough_key = "BRGNETWORKKEY"  # set internet passthrough sec. key
    passthrough_ssid = "BRGSSID"  # set internet passthrough SSID
    bootscreen_path = "CIPATH"  # set wireless LAN bootscreen path
    mastercode = "MASTERCODE"  # set mastercode (required!)
    clear_mastercode = "CLEARCODE"  # clear mastercode
    timezone = "TIMEZONE"  # set timezone (e.g. 36)
    drive_mode = "WEBDAV"  # set FlashAir drive (WebDAV)

class _ModeValue(int, Enum): ...

class WifiMode(_ModeValue):
    """Wireless modes effective IMMEDIATELY"""
    access_point = 0
    station = 2
    passthrough = 3  # for wireless pass through (FW 2.00.02+)

class WifiModeOnBoot(_ModeValue):
    """Wireless modes effective upon REBOOT of the device"""
    access_point = 4
    station = 5
    passthrough = 6

class DriveMode(_ModeValue):
    disable = 0  # FlashAir Drive disabled
    enable = 1  # enabled, only read is allowed
    # NOTE: for uploads to work, you also need UPLOAD=1 in the config file
    upload = 2  # enabled, read AND write allowed

def config(param_map, mastercode=DEFAULT_MASTERCODE):
    """Takes a dictionary of {Param.key: value} and
    returns a dictionary of processed keys and values to be used in the
    construction of a POST request to FlashAir's config.cgi"""
    pmap = {Param.mastercode: mastercode}
    pmap.update(param_map)
    processed_params = dict(_process_params(pmap))
    return processed_params

def _process_params(params):
    for param, value in params.items():
        assert param in Param, "Invalid param: {}".format(param)
        yield param.value, value_validators[param](value)

def post(param_map, url=URL):
    """Posts a `param_map` created with `config` to
    the FlashAir config.cgi entrypoint"""
    prepped_request = _prep_post(url=url, **param_map)
    return cgi.send(prepped_request)

_prep_post = partial(cgi.prep_post, cgi.Entrypoint.config)

######################################################
# Functions for creating config POST parameter values

value_validators = {}

def _validator(parameter):
    def wrapper(fn):
        value_validators[parameter] = fn
        return fn
    return wrapper

@_validator(Param.wifi_timeout)
def _validate_timeout(seconds: float):
    """Creates an int from 60000 to 4294967294 that represents a
    valid millisecond wireless LAN timeout"""
    val = int(seconds * 1000)
    assert 60000 <= val <= 4294967294, "Bad value: {}".format(val)
    return val

@_validator(Param.app_info)
def _validate_app_info(info: str):
    assert 1 <= len(info) <= 16
    return info

@_validator(Param.wifi_mode)
def _validate_wifi_mode(mode: _ModeValue):
    assert mode in WifiMode or mode in WifiModeOnBoot
    return int(mode)

@_validator(Param.wifi_key)
def _validate_wifi_key(network_key: str):
    assert 0 <= len(network_key) <= 63
    return network_key

@_validator(Param.wifi_ssid)
def _validate_wifi_ssid(network_ssid: str):
    assert 1 <= len(network_ssid) <= 32
    return network_ssid

@_validator(Param.passthrough_key)
def _validate_passthrough_key(network_key: str):
    assert 0 <= len(network_key) <= 63
    return network_key

@_validator(Param.passthrough_ssid)
def _validate_passthroughssid(network_ssid: str):
    assert 1 <= len(network_ssid) <= 32
    return network_ssid

@_validator(Param.mastercode)
def _validate_mastercode(code: str):
    assert len(code) == 12
    code = code.upper()
    valid_chars = "0123456789ABCDEF"
    assert all(c in valid_chars for c in code)
    return code

@_validator(Param.bootscreen_path)
def _validate_bootscreen_path(path: str):
    return path

@_validator(Param.clear_mastercode)
def _validate_clear_mastercode(_):
    return 1

@_validator(Param.timezone)
def _validate_timezone(hours_offset: int):
    """Creates an int from -48 to 36 that represents the timezone
    offset in 15-minute increments as per the FlashAir docs"""
    param_value = int(hours_offset * 4)
    assert -48 <= param_value <= 36
    return param_value

@_validator(Param.drive_mode)
def _validate_drive_mode(mode: DriveMode):
    assert mode in DriveMode
    return int(mode)

<|endoftext|>


<commit_before>import os
import re
import logging

import requests
from bs4 import BeautifulSoup
from PIL import Image
from django.conf import settings
from django.utils.text import slugify

from games.models import Company
from platforms.models import Platform

LOGGER = logging.getLogger(__name__)
API_URL = "http://thegamesdb.net/api/"
UNSUPPORTED_PLATFORMS = (
    'Microsoft Xbox',
    'Microsoft Xbox 360',
    'Microsoft Xbox One',
    'Sony Playstation 3',
    'Sony Playstation 4',
    'Sony Playstation Vita',
    'Android',
    'iOS',
    'Nintendo 3DS',
    'N-Gage',
)

PLATFORM_MAP = {
    'Arcade': 'arcade',
    'Amiga': 'amiga',
    'Atari 2600': 'atari-2600',
    'Atari 5200': 'atari-5200',
    'Atari ST': 'atari-st',
    'PC': 'windows',
    'Sega Genesis': 'sega-genesis',
    'Sega Mega Drive': 'sega-genesis',
    'Sega Master System': 'sega-master-system',
    'Sega Saturn': 'sega-saturn',
    'Sega Game Gear': 'sega-game-gear',
    'Sega Dreamcast': 'sega-dreamcast',
    'Atari Jaguar': 'atari-jaguar',
    'Sony Playstation': 'sony-playstation',
    'Sony Playstation 2': 'sony-playstation-2',
    'Sony PSP': 'sony-psp',
    'Nintendo Entertainment System (NES)': 'nes',
    'Super Nintendo (SNES)': 'super-nintendo',
    'Neo Geo': 'neo-geo',
    'Neo Geo CD': 'neo-geo-cd',
    'Nintendo Game Boy': 'game-boy',
    'Nintendo Game Boy Color': 'nintendo-game-boy-color',
    'Nintendo 64': 'nintendo-64',
    'Nintendo GameCube': 'gamecube',
    'Nintendo DS': 'nintendo-ds',
    'Nintendo Wii': 'nintendo-wii',
    'NeoGeo': 'neo-geo',
    'TurboGrafx 16': 'turbografx-16',
    'Vectrex': 'vectrex',
}

def download_image(media_type, url):
    """Download an image from TheGamesDB (is necessary) and return its local path"""
    media_type_map = {
        'clearlogo': settings.TGD_CLEAR_LOGO_PATH,
        'banner': settings.TGD_BANNER_PATH,
        'screenshot': settings.TGD_SCREENSHOT_PATH,
        'fanart': settings.TGD_FANART_PATH
    }
    dest_path = os.path.join(media_type_map[media_type], os.path.basename(url))
    if not os.path.exists(dest_path):
        response = requests.get(url)
        with open(dest_path, 'w') as dest_file:
            dest_file.write(response.content)
    return dest_path

def convert_clearlogo_to_banner(logo_path):
    base_width, base_height = settings.BANNER_SIZE.split('x')
    base_ratio = float(base_width) / float(base_height)

    logo = Image.open(logo_path)
    logo_width = float(logo.width)
    logo_height = float(logo.height)
    logo_ratio = logo_width / logo_height

    margin_factor = 0.1
    if logo_ratio < base_ratio:
        new_h = logo_height + (logo_height * margin_factor)
        new_w = new_h * base_ratio
    else:
        new_w = logo_width + (logo_width * margin_factor)
        new_h = new_w / base_ratio

    new_image = Image.new('RGBA', (int(new_w), int(new_h)), (0, 0, 0, 255))
    if logo.mode == 'RGBA':
        mask = logo
    else:
        mask = None
    offset = (int((new_w - logo_width) / 2), int((new_h - logo_height) / 2))
    new_image.paste(logo, offset, mask)
    dest_path = os.path.join(settings.TGD_LUTRIS_BANNER_PATH,
                             os.path.basename(logo_path))
    new_image.save(dest_path)
    return dest_path

def api_request(url):
    response = requests.get(API_URL + url)
    return BeautifulSoup(response.content, 'xml')

def get_value(soup, field):
    if not soup:
        LOGGER.warn("No soup provided")
        return
    tag = soup.find(field)
    if tag:
        return tag.text

def get_similar(game_data):
    similar = game_data.find('Similar')
    if similar:
        return [
            {
                'id': get_value(game, 'id'),
                'platform_id': get_value(game, 'PlatformId')
            } for game in similar.find_all('Game')
        ]

def get_pics(game_data, tag_name):
    fanarts = game_data.find_all(tag_name)
    results = []
    for fanart in fanarts:
        original = fanart.find('original')
        original_meta = original.attrs
        original_meta['url'] = original.get_text()
        results.append({
            'original': original_meta,
            'thumb': get_value(fanart, 'thumb'),
        })
    return results

def get_tags_with_attrs(soup, tag_name, value_name='value'):
    tags = soup.find_all(tag_name)
    results = []
    for tag in tags:
        attrs = tag.attrs
        if attrs:
            attrs[value_name] = tag.get_text()
        else:
            attrs = tag.get_text()
        results.append(attrs)
    if len(results) == 1:
        return results[0]
    else:
        return results

def get_games_list(query, remove_unsupported=True):
    soup = api_request('GetGamesList.php?name=' + query)
    game_list = []
    games = soup.find_all('Game')
    for game in games:
        platform = get_value(game, 'Platform')
        if remove_unsupported and platform in UNSUPPORTED_PLATFORMS:
            continue
        game_list.append({
            'id': get_value(game, 'id'),
            'game_title': get_value(game, 'GameTitle'),
            'release_date': get_value(game, 'ReleaseDate'),
            'platform': get_value(game, 'Platform')
        })
    return game_list

def get_game(game_id):
    soup = api_request('GetGame.php?id={}'.format(game_id))
    game_data = soup.find('Game')
    game_info = {
        'id': get_value(game_data, 'id'),
        'base_img_url': get_value(soup, 'baseImgUrl'),
        'game_title': get_value(game_data, 'GameTitle'),
        'platform': get_value(game_data, 'Platform'),
        'release_date': get_value(game_data, 'ReleaseDate'),
        'overview': get_value(game_data, 'Overview'),
        'esrb': get_value(game_data, 'ESRB'),
        'publisher': get_value(game_data, 'Publisher'),
        'developer': get_value(game_data, 'Developer'),
        'rating': get_value(game_data, 'Rating'),
        'similar': get_similar(game_data),
        'fanarts': get_pics(game_data, 'fanart'),
        'boxarts': get_tags_with_attrs(game_data, 'boxart'),
        'banners': get_tags_with_attrs(game_data, 'banner'),
        'screenshots': get_pics(game_data, 'screenshot'),
        'clearlogo': get_tags_with_attrs(game_data, 'clearlogo'),
    }
    return game_info

def get_lutris_platform(platform_name):
    try:
        slug = PLATFORM_MAP[platform_name]
    except KeyError:
        LOGGER.error('No map for %s', platform_name)
        return
    return Platform.objects.get(slug=slug)

def to_lutris(game):
    lutris_game = {}
    lutris_game['name'] = game['game_title']
    if game['release_date']:
        match = re.search(r'\d{4}', game['release_date'])
        lutris_game['year'] = match.group(0)

    publisher = get_or_create_company(game['publisher'])
    if publisher:
        lutris_game['publisher'] = publisher.id

    if game['developer'] == game['publisher']:
        developer = publisher
    else:
        developer = get_or_create_company(game['developer'])
    if developer:
        lutris_game['developer'] = developer.id

    platform = get_lutris_platform(game['platform'])
    if platform:
        lutris_game['platforms'] = [platform.id]
    if game['clearlogo']:
        clearlogo_url = game['base_img_url'] + game['clearlogo']['value']
        logo_path = download_image('clearlogo', clearlogo_url)
        banner_path = convert_clearlogo_to_banner(logo_path)
        banner_url = banner_path[len(settings.MEDIA_ROOT):].strip('/')
        lutris_game['banner'] = settings.MEDIA_URL + banner_url
    return lutris_game

def get_or_create_company(name):
    if not name:
        return
    slug = slugify(name)[:50]
    try:
        company = Company.objects.get(slug=slug)
    except Company.DoesNotExist:
        company = Company.objects.create(
            slug=slug,
            name=name
        )
    return company
<commit_msg>Add mapping for GBA<commit_after>import os
import re
import logging

import requests
from bs4 import BeautifulSoup
from PIL import Image
from django.conf import settings
from django.utils.text import slugify

from games.models import Company
from platforms.models import Platform

LOGGER = logging.getLogger(__name__)
API_URL = "http://thegamesdb.net/api/"
UNSUPPORTED_PLATFORMS = (
    'Microsoft Xbox',
    'Microsoft Xbox 360',
    'Microsoft Xbox One',
    'Sony Playstation 3',
    'Sony Playstation 4',
    'Sony Playstation Vita',
    'Android',
    'iOS',
    'Nintendo 3DS',
    'N-Gage',
)

PLATFORM_MAP = {
    'Arcade': 'arcade',
    'Amiga': 'amiga',
    'Atari 2600': 'atari-2600',
    'Atari 5200': 'atari-5200',
    'Atari ST': 'atari-st',
    'PC': 'windows',
    'Sega Genesis': 'sega-genesis',
    'Sega Mega Drive': 'sega-genesis',
    'Sega Master System': 'sega-master-system',
    'Sega Saturn': 'sega-saturn',
    'Sega Game Gear': 'sega-game-gear',
    'Sega Dreamcast': 'sega-dreamcast',
    'Atari Jaguar': 'atari-jaguar',
    'Sony Playstation': 'sony-playstation',
    'Sony Playstation 2': 'sony-playstation-2',
    'Sony PSP': 'sony-psp',
    'Nintendo Entertainment System (NES)': 'nes',
    'Super Nintendo (SNES)': 'super-nintendo',
    'Neo Geo': 'neo-geo',
    'Neo Geo CD': 'neo-geo-cd',
    'Nintendo Game Boy': 'game-boy',
    'Nintendo Game Boy Color': 'nintendo-game-boy-color',
    'Nintendo Game Boy Advance': 'nintendo-game-boy-advance',
    'Nintendo 64': 'nintendo-64',
    'Nintendo GameCube': 'gamecube',
    'Nintendo DS': 'nintendo-ds',
    'Nintendo Wii': 'nintendo-wii',
    'NeoGeo': 'neo-geo',
    'TurboGrafx 16': 'turbografx-16',
    'Vectrex': 'vectrex',
}

def download_image(media_type, url):
    """Download an image from TheGamesDB (is necessary) and return its local path"""
    media_type_map = {
        'clearlogo': settings.TGD_CLEAR_LOGO_PATH,
        'banner': settings.TGD_BANNER_PATH,
        'screenshot': settings.TGD_SCREENSHOT_PATH,
        'fanart': settings.TGD_FANART_PATH
    }
    dest_path = os.path.join(media_type_map[media_type], os.path.basename(url))
    if not os.path.exists(dest_path):
        response = requests.get(url)
        with open(dest_path, 'w') as dest_file:
            dest_file.write(response.content)
    return dest_path

def convert_clearlogo_to_banner(logo_path):
    base_width, base_height = settings.BANNER_SIZE.split('x')
    base_ratio = float(base_width) / float(base_height)

    logo = Image.open(logo_path)
    logo_width = float(logo.width)
    logo_height = float(logo.height)
    logo_ratio = logo_width / logo_height

    margin_factor = 0.1
    if logo_ratio < base_ratio:
        new_h = logo_height + (logo_height * margin_factor)
        new_w = new_h * base_ratio
    else:
        new_w = logo_width + (logo_width * margin_factor)
        new_h = new_w / base_ratio

    new_image = Image.new('RGBA', (int(new_w), int(new_h)), (0, 0, 0, 255))
    if logo.mode == 'RGBA':
        mask = logo
    else:
        mask = None
    offset = (int((new_w - logo_width) / 2), int((new_h - logo_height) / 2))
    new_image.paste(logo, offset, mask)
    dest_path = os.path.join(settings.TGD_LUTRIS_BANNER_PATH,
                             os.path.basename(logo_path))
    new_image.save(dest_path)
    return dest_path

def api_request(url):
    response = requests.get(API_URL + url)
    return BeautifulSoup(response.content, 'xml')

def get_value(soup, field):
    if not soup:
        LOGGER.warn("No soup provided")
        return
    tag = soup.find(field)
    if tag:
        return tag.text

def get_similar(game_data):
    similar = game_data.find('Similar')
    if similar:
        return [
            {
                'id': get_value(game, 'id'),
                'platform_id': get_value(game, 'PlatformId')
            } for game in similar.find_all('Game')
        ]

def get_pics(game_data, tag_name):
    fanarts = game_data.find_all(tag_name)
    results = []
    for fanart in fanarts:
        original = fanart.find('original')
        original_meta = original.attrs
        original_meta['url'] = original.get_text()
        results.append({
            'original': original_meta,
            'thumb': get_value(fanart, 'thumb'),
        })
    return results

def get_tags_with_attrs(soup, tag_name, value_name='value'):
    tags = soup.find_all(tag_name)
    results = []
    for tag in tags:
        attrs = tag.attrs
        if attrs:
            attrs[value_name] = tag.get_text()
        else:
            attrs = tag.get_text()
        results.append(attrs)
    if len(results) == 1:
        return results[0]
    else:
        return results

def get_games_list(query, remove_unsupported=True):
    soup = api_request('GetGamesList.php?name=' + query)
    game_list = []
    games = soup.find_all('Game')
    for game in games:
        platform = get_value(game, 'Platform')
        if remove_unsupported and platform in UNSUPPORTED_PLATFORMS:
            continue
        game_list.append({
            'id': get_value(game, 'id'),
            'game_title': get_value(game, 'GameTitle'),
            'release_date': get_value(game, 'ReleaseDate'),
            'platform': get_value(game, 'Platform')
        })
    return game_list

def get_game(game_id):
    soup = api_request('GetGame.php?id={}'.format(game_id))
    game_data = soup.find('Game')
    game_info = {
        'id': get_value(game_data, 'id'),
        'base_img_url': get_value(soup, 'baseImgUrl'),
        'game_title': get_value(game_data, 'GameTitle'),
        'platform': get_value(game_data, 'Platform'),
        'release_date': get_value(game_data, 'ReleaseDate'),
        'overview': get_value(game_data, 'Overview'),
        'esrb': get_value(game_data, 'ESRB'),
        'publisher': get_value(game_data, 'Publisher'),
        'developer': get_value(game_data, 'Developer'),
        'rating': get_value(game_data, 'Rating'),
        'similar': get_similar(game_data),
        'fanarts': get_pics(game_data, 'fanart'),
        'boxarts': get_tags_with_attrs(game_data, 'boxart'),
        'banners': get_tags_with_attrs(game_data, 'banner'),
        'screenshots': get_pics(game_data, 'screenshot'),
        'clearlogo': get_tags_with_attrs(game_data, 'clearlogo'),
    }
    return game_info

def get_lutris_platform(platform_name):
    try:
        slug = PLATFORM_MAP[platform_name]
    except KeyError:
        LOGGER.error('No map for %s', platform_name)
        return
    return Platform.objects.get(slug=slug)

def to_lutris(game):
    lutris_game = {}
    lutris_game['name'] = game['game_title']
    if game['release_date']:
        match = re.search(r'\d{4}', game['release_date'])
        lutris_game['year'] = match.group(0)

    publisher = get_or_create_company(game['publisher'])
    if publisher:
        lutris_game['publisher'] = publisher.id

    if game['developer'] == game['publisher']:
        developer = publisher
    else:
        developer = get_or_create_company(game['developer'])
    if developer:
        lutris_game['developer'] = developer.id

    platform = get_lutris_platform(game['platform'])
    if platform:
        lutris_game['platforms'] = [platform.id]
    if game['clearlogo']:
        clearlogo_url = game['base_img_url'] + game['clearlogo']['value']
        logo_path = download_image('clearlogo', clearlogo_url)
        banner_path = convert_clearlogo_to_banner(logo_path)
        banner_url = banner_path[len(settings.MEDIA_ROOT):].strip('/')
        lutris_game['banner'] = settings.MEDIA_URL + banner_url
    return lutris_game

def get_or_create_company(name):
    if not name:
        return
    slug = slugify(name)[:50]
    try:
        company = Company.objects.get(slug=slug)
    except Company.DoesNotExist:
        company = Company.objects.create(
            slug=slug,
            name=name
        )
    return company
<|endoftext|>


<commit_before># Copyright (c) 2015 Ultimaker B.V.
# Uranium is released under the terms of the AGPLv3 or higher.

import pep8
import re
import os
import sys
import token as token_type

CLASS_NAME_MATCH = re.compile(r'^_*[A-Z][a-zA-Z0-9]*_*$')
FUNCTION_NAME_MATCH = re.compile(r'^(_*|test_)[a-z][a-zA-Z0-9]*_*$')
MEMBER_NAME_MATCH = re.compile(r'^[a-z_][a-z0-9_]*$')
VARIABLE_NAME_MATCH = re.compile(r'^[a-z][a-z0-9_]*$')

# Check the logical line to see if there is a class or function definition. When there is, check if this definition matches
# The coding style set at Ultimaker.
def checkNames(logical_line, tokens):
    idx = 0
    while tokens[idx].type in [token_type.INDENT, token_type.DEDENT]:
        idx += 1
    if tokens[idx].string == "def":
        idx += 1
        if not FUNCTION_NAME_MATCH.match(tokens[idx].string):
            yield tokens[idx].start[1], "U101 Function name not properly formatted with lower camel case"
        idx += 1
        if tokens[idx].string != "(":
            yield tokens[idx].start[1], "U901 Parsing error"
            return
        idx += 1
        if tokens[idx].string != ")":
            while True:
                if tokens[idx].string == "*":
                    idx += 1
                if tokens[idx].string == "**":
                    idx += 1
                if not VARIABLE_NAME_MATCH.match(tokens[idx].string):
                    yield tokens[idx].start[1], "U201 Function parameter not in lower_case_underscore_format"
                idx += 1
                if tokens[idx].string == "=":
                    while tokens[idx].string not in [",", ")"]:
                        idx += 1
                if tokens[idx].string == ")":
                    break
                if tokens[idx].string != ",":
                    yield tokens[idx].start[1], "U902 Parsing error: %s: %s" % (tokens[idx].string, logical_line)
                    return
                idx += 1
    elif tokens[idx].string == "class":
        idx += 1
        if not CLASS_NAME_MATCH.match(tokens[idx].string) and tokens[idx].string != "i18nCatalog":
            yield tokens[idx].start[1], "U103 Class name not properly formatted with upper camel case"
    elif len(tokens) > idx + 3 and tokens[idx].string == "self" and tokens[idx + 1].string == "." and tokens[idx + 3].string == "=":
        if not MEMBER_NAME_MATCH.match(tokens[idx + 2].string):
            yield tokens[idx + 2].start[1], "U202 Member name not in lower_case_underscore_format"
    # elif len(tokens) > idx + 2 and tokens[idx + 1].string == "=":
    #     if not VARIABLE_NAME_MATCH.match(tokens[idx].string):
    #         yield tokens[idx].start[1], "U203 Variable name not in lower_case_underscore_format"

# Check the string definitions in the line. All strings should be double quotes.
def checkStrings(logical_line, tokens):
    for token in tokens:
        if token.type == token_type.STRING:
            if token.string.startswith("'") and not token.string.startswith("'''"): # ''' indicates multiline comment so should be ignored.
                yield token.start, "U110 Strings should use double quotes, not single quotes."

def blankLines(logical_line, blank_lines, indent_level, line_number, blank_before, previous_logical):
    if line_number < 3 and not previous_logical:
        return  # Don't expect blank lines before the first line
    if logical_line.startswith(("def ", "class ", "@")) and not previous_logical.startswith("@"):
        if not indent_level and blank_before < 1:
            yield 0, "U302 expected 2 blank lines, found 0"

def main(paths=["."]):
    pep8.register_check(checkNames)
    pep8.register_check(blankLines)
    pep8.register_check(checkStrings)

    ignore = []
    ignore.append("E501")  # Ignore line length violations.
    ignore.append("E226")  # Ignore too many leading # in comment block.

    critical = []
    critical.append("E301")  # expected 1 blank line, found 0
    critical.append("U302")  # expected 2 blank lines, found 0
    critical.append("E304")  # blank lines found after function decorator
    critical.append("E401")  # multiple imports on one line
    critical.append("E402")  # module level import not at top of file
    critical.append("E713")  # test for membership should be ‘not in’
    critical.append("W191")  # indentation contains tabs
    critical.append("U9")    # Parsing errors (error in this script, or error in code that we're trying to parse)
    critical.append("U")     # All Ultimaker specific stuff.

    pep8style = pep8.StyleGuide(quiet=False, select=critical, ignore=ignore, show_source=True)
    for path in paths:
        if path.endswith(".py") and "_pb2.py" not in path:
            pep8style.paths.append(path)
        for base_path, _, filenames in os.walk(path):
            for filename in filenames:
                if filename.endswith(".py") and "_pb2.py" not in filename:
                    pep8style.paths.append(os.path.join(base_path, filename))
    result = pep8style.check_files()
    print("----------------------------------")
    result.print_statistics()
    print("Total: %d" % (result.get_count()))

if __name__ == "__main__":
    if len(sys.argv) > 1:
        main(sys.argv[1:])
    else:
        main()
<commit_msg>Some more checks for the coding style. Now properly check variable names as well as a limited check for class defined members.<commit_after># Copyright (c) 2015 Ultimaker B.V.
# Uranium is released under the terms of the AGPLv3 or higher.

import pep8
import re
import os
import sys
import token as token_type

CLASS_NAME_MATCH = re.compile(r'^_*[A-Z][a-zA-Z0-9]*_*$')
FUNCTION_NAME_MATCH = re.compile(r'^(_*|test_)[a-z][a-zA-Z0-9]*_*$')
PUBLIC_MEMBER_NAME_MATCH = re.compile(r'^[A-Z][a-zA-Z0-9]*_*$')
MEMBER_NAME_MATCH = re.compile(r'^[a-z_][a-z0-9_]*$')
VARIABLE_NAME_MATCH = re.compile(r'^[a-z][a-z0-9_]*$')

indent_stack = []

# Check the logical line to see if there is a class or function definition. When there is, check if this definition matches
# The coding style set at Ultimaker.
def checkNames(logical_line, tokens, indent_level):
    global indent_stack
    if logical_line != '':
        while indent_stack and indent_stack[-1][0] >= indent_level:
            indent_stack.pop()
        if not indent_stack or indent_stack[-1][0] < indent_level:
            indent_stack.append((indent_level, logical_line))

    idx = 0
    while tokens[idx].type in [token_type.INDENT, token_type.DEDENT]:
        idx += 1
    if tokens[idx].string == "def":
        idx += 1
        if not FUNCTION_NAME_MATCH.match(tokens[idx].string):
            yield tokens[idx].start, "U101 Function name not properly formatted with lower camel case"
        idx += 1
        if tokens[idx].string != "(":
            yield tokens[idx].start, "U901 Parsing error"
            return
        idx += 1
        if tokens[idx].string != ")":
            while True:
                if tokens[idx].string == "*":
                    idx += 1
                if tokens[idx].string == "**":
                    idx += 1
                if not VARIABLE_NAME_MATCH.match(tokens[idx].string):
                    yield tokens[idx].start, "U201 Function parameter not in lower_case_underscore_format"
                idx += 1
                if tokens[idx].string == "=":
                    while tokens[idx].string not in [",", ")"]:
                        idx += 1
                if tokens[idx].string == ")":
                    break
                if tokens[idx].string != ",":
                    yield tokens[idx].start, "U902 Parsing error: %s: %s" % (tokens[idx].string, logical_line)
                    return
                idx += 1
    elif tokens[idx].string == "class":
        idx += 1
        if not CLASS_NAME_MATCH.match(tokens[idx].string) and tokens[idx].string != "i18nCatalog":
            yield tokens[idx].start, "U103 Class name not properly formatted with upper camel case"
    elif len(tokens) > idx + 3 and tokens[idx].string == "self" and tokens[idx + 1].string == "." and tokens[idx + 3].string == "=":
        if not MEMBER_NAME_MATCH.match(tokens[idx + 2].string):
            yield tokens[idx + 2].start, "U202 Member name not in lower_case_underscore_format"
    elif len(tokens) > idx + 2 and tokens[idx + 1].string == "=":
        if len(indent_stack) > 1 and indent_stack[-2][1].startswith('class '):
            # definition is a class member
            if tokens[idx].string.startswith('_'):
                # class member is a private, match the member name style.
                if not MEMBER_NAME_MATCH.match(tokens[idx].string):
                    yield tokens[idx].start, "U202 Member name not in lower_case_underscore_format"
            else:
                # TODO, the conding standard for this is not defined yet. And there are a few variations used in the code.
                # if not PUBLIC_MEMBER_NAME_MATCH.match(tokens[idx].string):
                #     yield tokens[idx].start, "U102 Public member name not in UpperCamelCase format"
                pass
        else:
            # definition is a variable
            if not VARIABLE_NAME_MATCH.match(tokens[idx].string):
                yield tokens[idx].start, "U203 Variable name not in lower_case_underscore_format"

# Check the string definitions in the line. All strings should be double quotes.
def checkStrings(logical_line, tokens):
    for token in tokens:
        if token.type == token_type.STRING:
            if token.string.startswith("'") and not token.string.startswith("'''"): # ''' indicates multiline comment so should be ignored.
                yield token.start, "U110 Strings should use double quotes, not single quotes."

def blankLines(logical_line, blank_lines, indent_level, line_number, blank_before, previous_logical):
    if line_number < 3 and not previous_logical:
        return  # Don't expect blank lines before the first line
    if logical_line.startswith(("def ", "class ", "@")) and not previous_logical.startswith("@"):
        if not indent_level and blank_before < 1:
            yield 0, "U302 expected 2 blank lines, found 0"

def main(paths=["."]):
    pep8.register_check(checkNames)
    pep8.register_check(blankLines)
    pep8.register_check(checkStrings)

    ignore = []
    ignore.append("E501")  # Ignore line length violations.
    ignore.append("E226")  # Ignore too many leading # in comment block.

    critical = []
    critical.append("E301")  # expected 1 blank line, found 0
    critical.append("U302")  # expected 2 blank lines, found 0
    critical.append("E304")  # blank lines found after function decorator
    critical.append("E401")  # multiple imports on one line
    critical.append("E402")  # module level import not at top of file
    critical.append("E713")  # test for membership should be ‘not in’
    critical.append("W191")  # indentation contains tabs
    critical.append("U9")    # Parsing errors (error in this script, or error in code that we're trying to parse)
    critical.append("U")     # All Ultimaker specific stuff.

    pep8style = pep8.StyleGuide(quiet=False, select=critical, ignore=ignore, show_source=True)
    for path in paths:
        if path.endswith(".py") and "_pb2.py" not in path:
            pep8style.paths.append(path)
        for base_path, _, filenames in os.walk(path):
            for filename in filenames:
                if filename.endswith(".py") and "_pb2.py" not in filename:
                    pep8style.paths.append(os.path.join(base_path, filename))
    result = pep8style.check_files()
    print("----------------------------------")
    result.print_statistics()
    print("Total: %d" % (result.get_count()))

if __name__ == "__main__":
    if len(sys.argv) > 1:
        main(sys.argv[1:])
    else:
        main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

import argparse, json, os
import numpy as np
import h5py
import codecs

parser = argparse.ArgumentParser()
parser.add_argument('--input_txt', default='data/tiny-shakespeare.txt')
parser.add_argument('--output_h5', default='data/tiny-shakespeare.h5')
parser.add_argument('--output_json', default='data/tiny-shakespeare.json')
parser.add_argument('--val_frac', type=float, default=0.1)
parser.add_argument('--test_frac', type=float, default=0.1)
parser.add_argument('--quiet', action='store_true')
parser.add_argument('--encoding', default='utf-8')
args = parser.parse_args()

if __name__ == '__main__':
  if args.encoding == 'bytes': args.encoding = None

  # First go the file once to see how big it is and to build the vocab
  token_to_idx = {}
  total_size = 0
  with codecs.open(args.input_txt, 'r', args.encoding) as f:
    for line in f:
      total_size += len(line)
      for char in line:
        if char not in token_to_idx:
          token_to_idx[char] = len(token_to_idx) + 1

  # Now we can figure out the split sizes
  val_size = int(args.val_frac * total_size)
  test_size = int(args.test_frac * total_size)
  train_size = total_size - val_size - test_size

  if not args.quiet:
    print 'Total vocabulary size: %d' % len(token_to_idx)
    print 'Total tokens in file: %d' % total_size
    print '  Training size: %d' % train_size
    print '  Val size: %d' % val_size
    print '  Test size: %d' % test_size

  # Choose the datatype based on the vocabulary size
  dtype = np.uint8
  if len(token_to_idx) > 255:
    dtype = np.uint32
  if not args.quiet:
    print 'Using dtype ', dtype

  # Just load data into memory ... we'll have to do something more clever
  # for huge datasets but this should be fine for now
  train = np.zeros(train_size, dtype=dtype)
  val = np.zeros(val_size, dtype=dtype)
  test = np.zeros(test_size, dtype=dtype)
  splits = [train, val, test]

  # Go through the file again and write data to numpy arrays
  split_idx, cur_idx = 0, 0
  with codecs.open(args.input_txt, 'r', args.encoding) as f:
    for line in f:
      for char in line:
        splits[split_idx][cur_idx] = token_to_idx[char]
        cur_idx += 1
        if cur_idx == splits[split_idx].size:
          split_idx += 1
          cur_idx = 0

  # Write data to HDF5 file
  with h5py.File(args.output_h5, 'w') as f:
    f.create_dataset('train', data=train)
    f.create_dataset('val', data=val)
    f.create_dataset('test', data=test)

  # For 'bytes' encoding, replace non-ascii characters so the json dump
  # doesn't crash
  if args.encoding is None:
    new_token_to_idx = {}
    for token, idx in token_to_idx.iteritems():
      if ord(token) > 127:
        new_token_to_idx['[%d]' % ord(token)] = idx
      else:
        new_token_to_idx[token] = idx
    token_to_idx = new_token_to_idx

  # Dump a JSON file for the vocab
  json_data = {
    'token_to_idx': token_to_idx,
    'idx_to_token': {v: k for k, v in token_to_idx.iteritems()},
  }
  with open(args.output_json, 'w') as f:
    json.dump(json_data, f)
<commit_msg>Modified dreavjr's preprocess.py for syllabic prediction based on Pyphen library<commit_after># -*- coding: utf-8 -*-

import argparse, json, os
import numpy as np
import h5py
import codecs
import sys

parser = argparse.ArgumentParser()
parser.add_argument('--input_txt', default='data/tiny-shakespeare.txt')
parser.add_argument('--output_h5', default='data/tiny-shakespeare.h5')
parser.add_argument('--output_json', default='data/tiny-shakespeare.json')
parser.add_argument('--val_frac', type=float, default=0.1)
parser.add_argument('--test_frac', type=float, default=0.1)
parser.add_argument('--quiet', action='store_true')
parser.add_argument('--syllabic', default='none')
parser.add_argument('--encoding', default='utf-8')
args = parser.parse_args()

if __name__ == '__main__':
  if args.encoding == 'bytes': args.encoding = None

  # First go the file once to see how big it is and to build the vocab
  if args.syllabic == 'none' :
    syllabic = False
    token_to_idx = {}
    total_size = 0
    with codecs.open(args.input_txt, 'r', args.encoding) as f:
      for line in f:
        total_size += len(line)
        for char in line:
          if char not in token_to_idx:
            token_to_idx[char] = len(token_to_idx) + 1
  else :
      syllabic = True

      import unicodedata
      import pyphen
      if not (args.syllabic in pyphen.LANGUAGES) :
	fallback = pyphen.language_fallback(args.syllabic)
	if fallback is None:
          print 'Syllabic dictionary', args.syllabic, 'is unavailable'
          sys.exit(0)
        else:
	  print 'Warning: dictionary', args.syllabic, 'is unavailable, using', fallback, 'as fallback'
	  args.syllabic = fallback
      primary = pyphen.Pyphen(lang=args.syllabic)
      if args.syllabic == 'en_US':
	secondary = None
      else:
	secondary = pyphen.Pyphen(lang='en_US')

      def scanSyllables(stream, encoding, processing) :
        word = ''
        space = False
        with codecs.open(stream, 'r', encoding) as f:
          for line in f:
            for char in line:
              cat = unicodedata.category(char)
              if cat[0]=='L' :
                  word = word + char
                  space = False
                  continue
              if len(word)>0 :
		  splits = primary.positions(word.lower())
		  if secondary:
		    secsplits = secondary.positions(word.lower())
		  else:
		    secsplits = []
		  if len(secsplits) > len(splits):
		    splits = secsplits
		  syls = splitWord(word.lower(), splits)
                  word = ''
              else :
                  syls = [ ]
              if cat[0]=='Z' :
                if not space : syls.append( u' ' )
                space = True
              elif cat[0]=='N' or cat[0]=='P' :
                syls.append( char )
                space = False
              elif char == u'\n' :
                syls.append( char )
                space = False
              for syl in syls :
                  processing(syl)

      def createVocab(syl) :
        global token_to_idx
        global total_size
        total_size += 1
        if syl not in token_to_idx:
          token_to_idx[syl] = len(token_to_idx) + 1

      def splitWord(word, splits) :
	if len(splits) == 0:
	  return [word]
	else:
	  firstidx = 0
	  lastidx = 0
	  syls = []
	  while splits:
	    firstidx = lastidx
	    lastidx = splits.pop(0)
	    syls.append(word[firstidx:lastidx])
	  syls.append(word[lastidx:])
	  return syls

      token_to_idx = { u'\n' : 1 }
      total_size = 0
      scanSyllables(args.input_txt, args.encoding, createVocab)

  # Now we can figure out the split sizes
  val_size = int(args.val_frac * total_size)
  test_size = int(args.test_frac * total_size)
  train_size = total_size - val_size - test_size

  if not args.quiet:
    print 'Total vocabulary size: %d' % len(token_to_idx)
    print 'Total tokens in file: %d' % total_size
    print '  Training size: %d' % train_size
    print '  Val size: %d' % val_size
    print '  Test size: %d' % test_size

  # Choose the datatype based on the vocabulary size
  dtype = np.uint8
  if len(token_to_idx) > 255:
    dtype = np.uint32
  if not args.quiet:
    print 'Using dtype ', dtype

  # Just load data into memory ... we'll have to do something more clever
  # for huge datasets but this should be fine for now
  train = np.zeros(train_size, dtype=dtype)
  val = np.zeros(val_size, dtype=dtype)
  test = np.zeros(test_size, dtype=dtype)
  splits = [train, val, test]

  # Go through the file again and write data to numpy arrays
  split_idx, cur_idx = 0, 0
  if not syllabic :
    with codecs.open(args.input_txt, 'r', args.encoding) as f:
      for line in f:
        for char in line:
          splits[split_idx][cur_idx] = token_to_idx[char]
          cur_idx += 1
          if cur_idx == splits[split_idx].size:
            split_idx += 1
            cur_idx = 0
  else :

    def convertInput(syl) :
      global check_size
      global splits
      global split_idx
      global cur_idx
      global token_to_idx
      check_size += 1
      # print check_size, syl
      splits[split_idx][cur_idx] = token_to_idx[syl]
      cur_idx += 1
      if cur_idx == splits[split_idx].size:
        split_idx += 1
        cur_idx = 0

    check_size = 0
    scanSyllables(args.input_txt, args.encoding, convertInput)

    if total_size != check_size :
      print 'WARNING : File sizes mismatched between vocabulary building (', total_size, ') and token conversion (', check_size, ')'
    if cur_idx!=0  :
      print 'ERROR : File size mismatched between splits. cur_idx =', cur_idx
      sys.exit(1)

  # Write data to HDF5 file
  with h5py.File(args.output_h5, 'w') as f:
    f.create_dataset('train', data=train)
    f.create_dataset('val', data=val)
    f.create_dataset('test', data=test)

  # For 'bytes' encoding, replace non-ascii characters so the json dump
  # doesn't crash
  if args.encoding is None:
    new_token_to_idx = {}
    for token, idx in token_to_idx.iteritems():
      if ord(token) > 127:
        new_token_to_idx['[%d]' % ord(token)] = idx
      else:
        new_token_to_idx[token] = idx
    token_to_idx = new_token_to_idx

  # Dump a JSON file for the vocab
  json_data = {
    'token_to_idx': token_to_idx,
    'idx_to_token': {v: k for k, v in token_to_idx.iteritems()},
  }
  with open(args.output_json, 'w') as f:
    json.dump(json_data, f)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
import argparse
from metasv.main import run_metasv
from metasv.defaults import *
from metasv._version import __version__

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Merge SVs from multiple tools for accurate SV calling",
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    input_parser = parser.add_argument_group("Input data options")
    input_parser.add_argument("--sample", metavar="Sample", help="Sample name", required=True)
    input_parser.add_argument("--pindel_vcf", nargs="+", metavar="pindel_vcf", help="VCF file or dir for Pindel VCFs",
                        required=False, default=[])
    input_parser.add_argument("--pindel_native", nargs="+", metavar="File list", help="Pindel native files", required=False,
                        default=[])
    input_parser.add_argument("--breakdancer_vcf", nargs="+", metavar="breakdancer_vcf",
                        help="VCF file or dir for BreakDancer VCFs", required=False, default=[])
    input_parser.add_argument("--breakdancer_native", nargs="+", metavar="File list", help="BreakDancer native files",
                        required=False, default=[])
    input_parser.add_argument("--breakseq_vcf", nargs="+", metavar="breakseq_vcf", help="VCF file or dir for BreakSeq VCFs",
                        required=False, default=[])
    input_parser.add_argument("--breakseq_native", nargs="+", metavar="breakseq_native", help="BreakSeq native GFF files",
                        required=False, default=[])
    input_parser.add_argument("--cnvnator_vcf", nargs="+", metavar="cnvnator_vcf", help="VCF file or dir for CNVnator VCFs",
                        required=False, default=[])
    input_parser.add_argument("--cnvnator_native", nargs="+", metavar="File list", help="CNVnator native files",
                        required=False,
                        default=[])
    input_parser.add_argument("--gatk_vcf", nargs="+", metavar="file", help="VCF file or dir for gatk VCFs", required=False,
                        default=[])

    reference_parser = parser.add_argument_group("Reference options")
    reference_parser.add_argument("--reference", metavar="reference", help="Reference file", required=True)
    reference_parser.add_argument("--chromosomes",
                        help="Chromosome list to process. If unspecified, then all chromosomes will be considered.",
                        nargs="+", default=[])
    reference_parser.add_argument("--gaps", metavar="gaps", help="Gap bed file", required=False, default=None)
    reference_parser.add_argument("--filter_gaps", help="Filter out gaps", action="store_true", required=False)
    reference_parser.add_argument("--keep_standard_contigs", action="store_true", help="Keep only the major contigs + MT")

    bam_parser = parser.add_argument_group("Input BAM options")
    bam_parser.add_argument("--bam", help="BAM", type=file)
    bam_parser.add_argument("--isize_mean", type=float, default=ISIZE_MEAN, help="Insert size mean")
    bam_parser.add_argument("--isize_sd", type=float, default=ISIZE_SD, help="Insert size standard deviation")

    merging_parser = parser.add_argument_group("Tool output merging options")
    merging_parser.add_argument("--wiggle", help="Wiggle for interval overlap", default=WIGGLE, type=int, required=False)
    merging_parser.add_argument("--inswiggle", help="Wiggle for insertions, overides wiggle", default=INS_WIGGLE, type=int,
                        required=False)
    merging_parser.add_argument("--minsvlen", help="Minimum length acceptable to be an SV", default=MIN_SV_LENGTH, type=int,
                        required=False)
    merging_parser.add_argument("--overlap_ratio", help="Reciprocal overlap ratio", default=OVERLAP_RATIO, type=float,
                        required=False)

    insertion_parser = parser.add_argument_group("Insertion detection options")
    insertion_parser.add_argument("--boost_ins", help="Use soft-clips for improving insertion detection", action="store_true")
    insertion_parser.add_argument("--min_ins_support", help="Minimum read support for calling insertions using soft-clips",
                        type=int, default=MIN_SUPPORT)
    insertion_parser.add_argument("--min_ins_support_frac",
                        help="Minimum fraction of reads supporting insertion using soft-clips", type=float,
                        default=MIN_SUPPORT_FRAC)
    insertion_parser.add_argument("--max_ins_intervals", help="Maximum number of insertion intervals to generate", type=int,
                        default=MAX_INTERVALS)

    as_parser = parser.add_argument_group("Assembly options")
    as_parser.add_argument("--spades", help="Path to SPAdes executable", required=False)
    as_parser.add_argument("--disable_assembly", action="store_true", help="Disable assembly")
    as_parser.add_argument("--disable_deletion_assembly", action="store_true", help="Disable assembly for deletions")
    as_parser.add_argument("--extraction_max_read_pairs", type=int, default=EXTRACTION_MAX_READ_PAIRS, help="Maximum number of pairs to extract for assembly")
    as_parser.add_argument("--stop_spades_on_fail", action="store_true", help="Abort on SPAdes failure")
    as_parser.add_argument("--age", help="Path to AGE executable", required=False)

    gt_parser = parser.add_argument_group("Genotyping options")
    gt_parser.add_argument("--gt_window", type=int, default=GT_WINDOW, help="Window for genotyping")
    gt_parser.add_argument("--gt_normal_frac", type=float, default=GT_NORMAL_FRAC, help="Min. fraction of reads supporting reference for genotyping")

    out_parser = parser.add_argument_group("Output options")
    out_parser.add_argument("--svs_to_report", nargs="+", help="SV types to report", default=SVS_SUPPORTED)
    out_parser.add_argument("--enable_per_tool_output", action="store_true",
                        help="Enable output of merged SVs for individual tools")

    work_parser = parser.add_argument_group("Running environment options")
    work_parser.add_argument("--workdir", help="Scratch directory for working", default="work", required=False)
    work_parser.add_argument("--num_threads", help="Number of threads to use", type=int, default=1)
    work_parser.add_argument("--outdir", help="Output directory", required=True)

    other_parser = parser.add_argument_group("Other options")
    other_parser.add_argument('--version', action='version', version='%(prog)s ' + __version__)

    args = parser.parse_args()

    sys.exit(run_metasv(args.sample, args.reference, pindel_vcf=args.pindel_vcf, pindel_native=args.pindel_native,
                        breakdancer_vcf=args.breakdancer_vcf, breakdancer_native=args.breakdancer_native,
                        breakseq_vcf=args.breakseq_vcf, breakseq_native=args.breakseq_native,
                        cnvnator_vcf=args.cnvnator_vcf,
                        cnvnator_native=args.cnvnator_native, gatk_vcf=args.gatk_vcf,
                        gaps=args.gaps, filter_gaps=args.filter_gaps, keep_standard_contigs=args.keep_standard_contigs,
                        wiggle=args.wiggle, overlap_ratio=args.overlap_ratio,
                        workdir=args.workdir, outdir=args.outdir, boost_ins=args.boost_ins, bam=args.bam,
                        chromosomes=args.chromosomes, num_threads=args.num_threads, spades=args.spades, age=args.age,
                        disable_assembly=args.disable_assembly, minsvlen=args.minsvlen, inswiggle=args.inswiggle,
                        enable_per_tool_output=args.enable_per_tool_output, min_support=args.min_ins_support,
                        min_support_frac=args.min_ins_support_frac, max_intervals=args.max_ins_intervals,
                        disable_deletion_assembly=args.disable_deletion_assembly,
                        stop_spades_on_fail=args.stop_spades_on_fail, gt_window=args.gt_window, gt_normal_frac=args.gt_normal_frac, isize_mean=args.isize_mean, isize_sd=args.isize_sd, extraction_max_read_pairs=args.extraction_max_read_pairs, svs_to_report=args.svs_to_report))

<commit_msg>formatting<commit_after>#!/usr/bin/env python

import sys
import argparse
from metasv.main import run_metasv
from metasv.defaults import *
from metasv._version import __version__

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Merge SVs from multiple tools for accurate SV calling",
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)

    input_parser = parser.add_argument_group("Input data options")
    input_parser.add_argument("--sample", metavar="Sample", help="Sample name", required=True)
    input_parser.add_argument("--pindel_vcf", nargs="+", metavar="pindel_vcf", help="VCF file or dir for Pindel VCFs",
                              required=False, default=[])
    input_parser.add_argument("--pindel_native", nargs="+", metavar="File list", help="Pindel native files",
                              required=False,
                              default=[])
    input_parser.add_argument("--breakdancer_vcf", nargs="+", metavar="breakdancer_vcf",
                              help="VCF file or dir for BreakDancer VCFs", required=False, default=[])
    input_parser.add_argument("--breakdancer_native", nargs="+", metavar="File list", help="BreakDancer native files",
                              required=False, default=[])
    input_parser.add_argument("--breakseq_vcf", nargs="+", metavar="breakseq_vcf",
                              help="VCF file or dir for BreakSeq VCFs",
                              required=False, default=[])
    input_parser.add_argument("--breakseq_native", nargs="+", metavar="breakseq_native",
                              help="BreakSeq native GFF files",
                              required=False, default=[])
    input_parser.add_argument("--cnvnator_vcf", nargs="+", metavar="cnvnator_vcf",
                              help="VCF file or dir for CNVnator VCFs",
                              required=False, default=[])
    input_parser.add_argument("--cnvnator_native", nargs="+", metavar="File list", help="CNVnator native files",
                              required=False,
                              default=[])
    input_parser.add_argument("--gatk_vcf", nargs="+", metavar="file", help="VCF file or dir for gatk VCFs",
                              required=False,
                              default=[])

    reference_parser = parser.add_argument_group("Reference options")
    reference_parser.add_argument("--reference", metavar="reference", help="Reference file", required=True)
    reference_parser.add_argument("--chromosomes",
                                  help="Chromosome list to process. If unspecified, then all chromosomes will be considered.",
                                  nargs="+", default=[])
    reference_parser.add_argument("--gaps", metavar="gaps", help="Gap bed file", required=False, default=None)
    reference_parser.add_argument("--filter_gaps", help="Filter out gaps", action="store_true", required=False)
    reference_parser.add_argument("--keep_standard_contigs", action="store_true",
                                  help="Keep only the major contigs + MT")

    bam_parser = parser.add_argument_group("Input BAM options")
    bam_parser.add_argument("--bam", help="BAM", type=file)
    bam_parser.add_argument("--isize_mean", type=float, default=ISIZE_MEAN, help="Insert size mean")
    bam_parser.add_argument("--isize_sd", type=float, default=ISIZE_SD, help="Insert size standard deviation")

    merging_parser = parser.add_argument_group("Tool output merging options")
    merging_parser.add_argument("--wiggle", help="Wiggle for interval overlap", default=WIGGLE, type=int,
                                required=False)
    merging_parser.add_argument("--inswiggle", help="Wiggle for insertions, overides wiggle", default=INS_WIGGLE,
                                type=int,
                                required=False)
    merging_parser.add_argument("--minsvlen", help="Minimum length acceptable to be an SV", default=MIN_SV_LENGTH,
                                type=int,
                                required=False)
    merging_parser.add_argument("--overlap_ratio", help="Reciprocal overlap ratio", default=OVERLAP_RATIO, type=float,
                                required=False)

    insertion_parser = parser.add_argument_group("Insertion detection options")
    insertion_parser.add_argument("--boost_ins", help="Use soft-clips for improving insertion detection",
                                  action="store_true")
    insertion_parser.add_argument("--min_ins_support",
                                  help="Minimum read support for calling insertions using soft-clips",
                                  type=int, default=MIN_SUPPORT)
    insertion_parser.add_argument("--min_ins_support_frac",
                                  help="Minimum fraction of reads supporting insertion using soft-clips", type=float,
                                  default=MIN_SUPPORT_FRAC)
    insertion_parser.add_argument("--max_ins_intervals", help="Maximum number of insertion intervals to generate",
                                  type=int,
                                  default=MAX_INTERVALS)

    as_parser = parser.add_argument_group("Assembly options")
    as_parser.add_argument("--spades", help="Path to SPAdes executable", required=False)
    as_parser.add_argument("--disable_assembly", action="store_true", help="Disable assembly")
    as_parser.add_argument("--disable_deletion_assembly", action="store_true", help="Disable assembly for deletions")
    as_parser.add_argument("--extraction_max_read_pairs", type=int, default=EXTRACTION_MAX_READ_PAIRS,
                           help="Maximum number of pairs to extract for assembly")
    as_parser.add_argument("--stop_spades_on_fail", action="store_true", help="Abort on SPAdes failure")
    as_parser.add_argument("--age", help="Path to AGE executable", required=False)

    gt_parser = parser.add_argument_group("Genotyping options")
    gt_parser.add_argument("--gt_window", type=int, default=GT_WINDOW, help="Window for genotyping")
    gt_parser.add_argument("--gt_normal_frac", type=float, default=GT_NORMAL_FRAC,
                           help="Min. fraction of reads supporting reference for genotyping")

    out_parser = parser.add_argument_group("Output options")
    out_parser.add_argument("--svs_to_report", nargs="+", help="SV types to report", default=SVS_SUPPORTED)
    out_parser.add_argument("--enable_per_tool_output", action="store_true",
                            help="Enable output of merged SVs for individual tools")

    work_parser = parser.add_argument_group("Running environment options")
    work_parser.add_argument("--workdir", help="Scratch directory for working", default="work", required=False)
    work_parser.add_argument("--num_threads", help="Number of threads to use", type=int, default=1)
    work_parser.add_argument("--outdir", help="Output directory", required=True)

    other_parser = parser.add_argument_group("Other options")
    other_parser.add_argument('--version', action='version', version='%(prog)s ' + __version__)

    args = parser.parse_args()

    sys.exit(run_metasv(args.sample, args.reference, pindel_vcf=args.pindel_vcf, pindel_native=args.pindel_native,
                        breakdancer_vcf=args.breakdancer_vcf, breakdancer_native=args.breakdancer_native,
                        breakseq_vcf=args.breakseq_vcf, breakseq_native=args.breakseq_native,
                        cnvnator_vcf=args.cnvnator_vcf,
                        cnvnator_native=args.cnvnator_native, gatk_vcf=args.gatk_vcf,
                        gaps=args.gaps, filter_gaps=args.filter_gaps, keep_standard_contigs=args.keep_standard_contigs,
                        wiggle=args.wiggle, overlap_ratio=args.overlap_ratio,
                        workdir=args.workdir, outdir=args.outdir, boost_ins=args.boost_ins, bam=args.bam,
                        chromosomes=args.chromosomes, num_threads=args.num_threads, spades=args.spades, age=args.age,
                        disable_assembly=args.disable_assembly, minsvlen=args.minsvlen, inswiggle=args.inswiggle,
                        enable_per_tool_output=args.enable_per_tool_output, min_support=args.min_ins_support,
                        min_support_frac=args.min_ins_support_frac, max_intervals=args.max_ins_intervals,
                        disable_deletion_assembly=args.disable_deletion_assembly,
                        stop_spades_on_fail=args.stop_spades_on_fail, gt_window=args.gt_window,
                        gt_normal_frac=args.gt_normal_frac, isize_mean=args.isize_mean, isize_sd=args.isize_sd,
                        extraction_max_read_pairs=args.extraction_max_read_pairs, svs_to_report=args.svs_to_report))
<|endoftext|>


<commit_before>__version__ = '0.8.0'
<commit_msg>Bump to version 0.9.0<commit_after>__version__ = '0.9.0'
<|endoftext|>