<commit_before>"""Defines the command line interface for darglint."""
import argparse
import ast
import pathlib
import sys

from .function_description import (
    read_program,
    get_function_descriptions,
)
from .integrity_checker import IntegrityChecker
from .config import (
    Configuration,
    get_config,
    Strictness,
)
from .docstring.base import DocstringStyle

# ---------------------- ARGUMENT PARSER -----------------------------

parser = argparse.ArgumentParser(description='Check docstring validity.')
parser.add_argument(
    '--message-template',
    '-m',
    type=str,
    help=(
        'Specify a message template.  This is a Python format string '
        'describing errors, which can access the following attributes:\n'
        '    line: The line number,\n'
        '    msg: The error message,\n'
        '    msg_id: The error code,\n'
        '    obj: The function/method name,\n'
        '    path: The relative file path.\n'
    ),
)
parser.add_argument(
    '--raise-syntax',
    action='store_true',
    help=(
        'When a docstring is incorrectly formatted, raise an exception '
        'rather than storing the error.  Useful for debugging darglint.'
    ),
)
parser.add_argument(
    '--verbosity',
    '-v',
    default=1,
    type=int,
    choices=[1, 2],
    help='The level of verbosity.',
)
parser.add_argument(
    '--version',
    action='store_true',
    help=(
        'Return the current version number of darglint.'
    ),
)
parser.add_argument(
    'files',
    nargs='*',
    help=(
        'The python source files to check. If "-" is given, then stdin will '
        'be read.'
    ),
)
parser.add_argument(
    '--no-exit-code',
    '-x',
    action='store_true',
    help=(
        'Exit with status 0, even on errors.  By default, darglint '
        'exits with status 1 when errors are encountered.  Giving '
        'this flag prevents that.  Useful when invocating with xargs '
        'and you want to see all errors.  '
        'Ex: `find . -name "*.py" | xargs darglint -x`'
    ),
)
parser.add_argument(
    '--list-errors',
    action='store_true',
    help=(
        'Print a list of error codes and what they represent.'
    )
)
parser.add_argument(
    '-s',
    '--docstring-style',
    default=None,
    choices=['google', 'sphinx'],
    help=(
        'The docstring style used in the given project. Currently, '
        'only google or sphinx styles are supported.'
    )
)
parser.add_argument(
    '-z',
    '--strictness',
    default=None,
    choices=[
        'short',
        'long',
        'full',
    ],
    help=(
        'The minimum strictness when checking docstrings. '
        '`short`, for example, will result in one-line '
        'docstrings always being accepted.  Anything more than one line '
        'would go through the full check.'
    ),
)

# ---------------------- MAIN SCRIPT ---------------------------------

def get_error_report(filename,
                     verbosity,
                     config,
                     raise_errors_for_syntax,
                     message_template=None,
                     ):
    # type: (str, int, Configuration, bool, str) -> str
    """Get the error report for the given file.

    Args:
        filename: The name of the module to check.
        verbosity: The level of verbosity, in the range [1, 3].
        config: The configuration for the error report.
        raise_errors_for_syntax: True if we want parser errors
            to propagate up (crashing darglint.)  This is useful
            if we are developing on darglint -- we can get the stack
            trace and know exactly where darglint failed.
        message_template: A python format string for specifying
            how the message should appear to the user.

    Returns:
        An error report for the file.

    """
    program = read_program(filename)
    tree = ast.parse(program)
    functions = get_function_descriptions(tree)
    checker = IntegrityChecker(
        config,
        raise_errors=raise_errors_for_syntax,
    )
    for function in functions:
        checker.run_checks(function)
    return checker.get_error_report_string(
        verbosity,
        filename,
        message_template=message_template,
    )

def print_error_list():
    print('\n'.join([
        'I101: The docstring is missing a parameter in the definition.',
        'I102: The docstring contains a parameter not in function.',
        'I103: The docstring parameter type doesn\'t match function.',
        'I201: The docstring is missing a return from definition.',
        'I202: The docstring has a return not in definition.',
        'I203: The docstring parameter type doesn\'t match function.',
        'I301: The docstring is missing a yield present in definition.',
        'I302: The docstring has a yield not in definition.',
        'I401: The docstring is missing an exception raised.',
        'I402: The docstring describes an exception not explicitly raised.',
        'S001: Describes that something went wrong in parsing the docstring.',
        'S002: An argument/exception lacks a description.',
    ]))

def print_version():
    print('0.6.1')

def main():
    # type: () -> None
    """Run darglint.

    Called as a script when setup.py is installed.

    """
    args = parser.parse_args()
    exit_code = not args.no_exit_code
    encountered_errors = False

    if args.list_errors:
        print_error_list()
        sys.exit(0)

    if args.version:
        print_version()

    # Expand directories.
    files = []
    for f in args.files:
        p = pathlib.Path(f)
        if not p.is_dir():
            files.append(f)
        # Convert back to strings to not require modifications of any
        # subsequent code.
        files.extend(str(i) for i in p.glob('**/*.py'))

    try:
        config = get_config()

        if args.docstring_style == 'sphinx':
            config.style = DocstringStyle.SPHINX
        elif args.docstring_style == 'google':
            config.style = DocstringStyle.GOOGLE

        if args.strictness == 'short':
            config.strictness = Strictness.SHORT_DESCRIPTION
        elif args.strictness == 'long':
            config.strictness = Strictness.LONG_DESCRIPTION
        elif args.strictness == 'full':
            config.strictness = Strictness.FULL_DESCRIPTION

        raise_errors_for_syntax = args.raise_syntax or False
        for filename in files:
            error_report = get_error_report(
                filename,
                args.verbosity,
                config,
                raise_errors_for_syntax,
                message_template=args.message_template,
            )
            if error_report:
                print(error_report + '\n')
                encountered_errors = True
    except Exception as exc:
        # Exit with status 129 regardless of whether user wants a
        # exit code or not -- darglint failed, and it should
        # look like it failed.
        print(exc, file=sys.stderr)
        sys.exit(129)
    if encountered_errors and exit_code:
        sys.exit(1)
    else:
        sys.exit(0)

if __name__ == '__main__':
    main()
<commit_msg>Filter out all files passed to `files` that does not have *.py suffix<commit_after>"""Defines the command line interface for darglint."""
import argparse
import ast
import pathlib
import sys

from .function_description import (
    read_program,
    get_function_descriptions,
)
from .integrity_checker import IntegrityChecker
from .config import (
    Configuration,
    get_config,
    Strictness,
)
from .docstring.base import DocstringStyle

# ---------------------- ARGUMENT PARSER -----------------------------

parser = argparse.ArgumentParser(description='Check docstring validity.')
parser.add_argument(
    '--message-template',
    '-m',
    type=str,
    help=(
        'Specify a message template.  This is a Python format string '
        'describing errors, which can access the following attributes:\n'
        '    line: The line number,\n'
        '    msg: The error message,\n'
        '    msg_id: The error code,\n'
        '    obj: The function/method name,\n'
        '    path: The relative file path.\n'
    ),
)
parser.add_argument(
    '--raise-syntax',
    action='store_true',
    help=(
        'When a docstring is incorrectly formatted, raise an exception '
        'rather than storing the error.  Useful for debugging darglint.'
    ),
)
parser.add_argument(
    '--verbosity',
    '-v',
    default=1,
    type=int,
    choices=[1, 2],
    help='The level of verbosity.',
)
parser.add_argument(
    '--version',
    action='store_true',
    help=(
        'Return the current version number of darglint.'
    ),
)
parser.add_argument(
    'files',
    nargs='*',
    help=(
        'The python source files to check. If "-" is given, then stdin will '
        'be read.'
    ),
)
parser.add_argument(
    '--no-exit-code',
    '-x',
    action='store_true',
    help=(
        'Exit with status 0, even on errors.  By default, darglint '
        'exits with status 1 when errors are encountered.  Giving '
        'this flag prevents that.  Useful when invocating with xargs '
        'and you want to see all errors.  '
        'Ex: `find . -name "*.py" | xargs darglint -x`'
    ),
)
parser.add_argument(
    '--list-errors',
    action='store_true',
    help=(
        'Print a list of error codes and what they represent.'
    )
)
parser.add_argument(
    '-s',
    '--docstring-style',
    default=None,
    choices=['google', 'sphinx'],
    help=(
        'The docstring style used in the given project. Currently, '
        'only google or sphinx styles are supported.'
    )
)
parser.add_argument(
    '-z',
    '--strictness',
    default=None,
    choices=[
        'short',
        'long',
        'full',
    ],
    help=(
        'The minimum strictness when checking docstrings. '
        '`short`, for example, will result in one-line '
        'docstrings always being accepted.  Anything more than one line '
        'would go through the full check.'
    ),
)

# ---------------------- MAIN SCRIPT ---------------------------------

def get_error_report(filename,
                     verbosity,
                     config,
                     raise_errors_for_syntax,
                     message_template=None,
                     ):
    # type: (str, int, Configuration, bool, str) -> str
    """Get the error report for the given file.

    Args:
        filename: The name of the module to check.
        verbosity: The level of verbosity, in the range [1, 3].
        config: The configuration for the error report.
        raise_errors_for_syntax: True if we want parser errors
            to propagate up (crashing darglint.)  This is useful
            if we are developing on darglint -- we can get the stack
            trace and know exactly where darglint failed.
        message_template: A python format string for specifying
            how the message should appear to the user.

    Returns:
        An error report for the file.

    """
    program = read_program(filename)
    tree = ast.parse(program)
    functions = get_function_descriptions(tree)
    checker = IntegrityChecker(
        config,
        raise_errors=raise_errors_for_syntax,
    )
    for function in functions:
        checker.run_checks(function)
    return checker.get_error_report_string(
        verbosity,
        filename,
        message_template=message_template,
    )

def print_error_list():
    print('\n'.join([
        'I101: The docstring is missing a parameter in the definition.',
        'I102: The docstring contains a parameter not in function.',
        'I103: The docstring parameter type doesn\'t match function.',
        'I201: The docstring is missing a return from definition.',
        'I202: The docstring has a return not in definition.',
        'I203: The docstring parameter type doesn\'t match function.',
        'I301: The docstring is missing a yield present in definition.',
        'I302: The docstring has a yield not in definition.',
        'I401: The docstring is missing an exception raised.',
        'I402: The docstring describes an exception not explicitly raised.',
        'S001: Describes that something went wrong in parsing the docstring.',
        'S002: An argument/exception lacks a description.',
    ]))

def print_version():
    print('0.6.1')

def main():
    # type: () -> None
    """Run darglint.

    Called as a script when setup.py is installed.

    """
    args = parser.parse_args()
    exit_code = not args.no_exit_code
    encountered_errors = False

    if args.list_errors:
        print_error_list()
        sys.exit(0)

    if args.version:
        print_version()

    # Expand directories.
    files = []
    for f in args.files:
        p = pathlib.Path(f)
        if not p.is_dir() and p.suffix == '.py':
            files.append(f)
        # Convert back to strings to not require modifications of any
        # subsequent code.
        files.extend(str(i) for i in p.glob('**/*.py'))

    try:
        config = get_config()

        if args.docstring_style == 'sphinx':
            config.style = DocstringStyle.SPHINX
        elif args.docstring_style == 'google':
            config.style = DocstringStyle.GOOGLE

        if args.strictness == 'short':
            config.strictness = Strictness.SHORT_DESCRIPTION
        elif args.strictness == 'long':
            config.strictness = Strictness.LONG_DESCRIPTION
        elif args.strictness == 'full':
            config.strictness = Strictness.FULL_DESCRIPTION

        raise_errors_for_syntax = args.raise_syntax or False
        for filename in files:
            error_report = get_error_report(
                filename,
                args.verbosity,
                config,
                raise_errors_for_syntax,
                message_template=args.message_template,
            )
            if error_report:
                print(error_report + '\n')
                encountered_errors = True
    except Exception as exc:
        # Exit with status 129 regardless of whether user wants a
        # exit code or not -- darglint failed, and it should
        # look like it failed.
        print(exc, file=sys.stderr)
        sys.exit(129)
    if encountered_errors and exit_code:
        sys.exit(1)
    else:
        sys.exit(0)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from django.shortcuts import render
from django.core.exceptions import ObjectDoesNotExist

from dashboard.models import Sessions
from api.views import get_platforms, get_queue, get_sessions

def dashboard(request):
    try:
        session_name = request.GET["q"]
    except:
        session_name = ''

    if not session_name:
        sessions = Sessions.objects.order_by('-time')
    else:
        # try:
        sessions = Sessions.objects.filter(name__contains=session_name)
        # except ObjectDoesNotExist:
        #     sessions = []

    print sessions
    platforms = get_platforms()
    active_sessions = get_sessions()
    queue = get_queue()

    context = {
        'sessions': sessions,
        'platforms': platforms,
        'active_sessions': active_sessions,
        'queue': queue
    }
    return render(request, 'dashboard/dashboard.html', context)<commit_msg>fix contains query to icontains<commit_after>from django.shortcuts import render

from dashboard.models import Sessions
from api.views import get_platforms, get_queue, get_sessions

def dashboard(request):
    try:
        session_name = request.GET["q"]
    except:
        session_name = ''

    if not session_name:
        sessions = Sessions.objects.order_by('-time')
    else:
        sessions = Sessions.objects.filter(name__icontains=session_name)

    platforms = get_platforms()
    active_sessions = get_sessions()
    queue = get_queue()

    context = {
        'sessions': sessions,
        'platforms': platforms,
        'active_sessions': active_sessions,
        'queue': queue
    }
    return render(request, 'dashboard/dashboard.html', context)<|endoftext|>


<commit_before>
from django.views.generic import TemplateView
from BeautifulSoup import BeautifulSoup
import re
import urllib
import time

ci_url = "http://ci.apps-system.com/"
base_url = "%s/job" % ci_url

builds = [
    'django-admin-ext',
    'django-dynamic-rules',
    'django-dynamic-validation',
    'django-forms-ext',
    'django-dynamic-manipulation',
    'django-wizard',
    'django-pretty-times',
    'django-response-helpers',
    'django-attachments',
]

class Status(TemplateView):
    template_name = 'dashboard/status.html'

    def get_context_data(self, **kwargs):
        f = urllib.urlopen(ci_url)
        data = f.read()
        f.close()
        page = BeautifulSoup(data)

        status_table = page.findAll(**{'id': 'projectstatus'})[0]
        statuses = []
        for cell in status_table.findAll('tr')[1:]:
            status_icon = cell.find('img', **{'class': 'icon32x32'})
            if status_icon:
                statuses.append(dict(status_icon.attrs)['alt'])

        ss = [s not in ('Success', 'Disabled', 'In progress') for s in statuses]
        fail = any(ss)
        if fail:
            for i in range(20):
                print "\a"
                time.sleep(.2)

        return {'success': not fail}

class Index(TemplateView):
    template_name = 'dashboard/index.html'
    current_build = -1

    def get_context_data(self, **kwargs):
        Index.current_build += 1
        if Index.current_build >= len(builds):
            Index.current_build = 0

        build = builds[Index.current_build]
        return {
            'params': kwargs,
            'status':self.get_status(),
            'build': build,
            'test_section_title':'Tests',
            'test_section_content':self.get_test_graph(build),
            'coverage_section_title':'Coverage',
            'coverage_section_content':self.get_cobertura_graph(build),
            'violations_section_title':'Violations',
            'violations_section_content':self.get_violations_graph(build),
            'pylint_section_title':'Pylint',
            'pylint_section_content':self.get_pylint_graph(build),
            'cpd_section_title':'CPD',
            'cpd_section_content':self.get_cpd_graph(build),
            'pep8_section_title':'Pep8',
            'pep8_section_content':self.get_pep8_graph(build),
        }

    def get_status(self):
        page = get_build_page(builds[Index.current_build] + "/lastBuild")
        status = get_status(page)
        return status.replace(" ", '')

    def get_test_graph(self, build):
        return "%s/%s/test/trend" % (base_url, build)

    def get_cobertura_graph(self, build):
        return "%s/%s/cobertura/graph" % (base_url, build)

    def get_violations_graph(self, build):
        return "%s/%s/violations/graph" % (base_url, build)

    def get_pylint_graph(self, build):
        return "%s/%s/violations/graph?type=pylint" % (base_url, build)

    def get_cpd_graph(self, build):
        return "%s/%s/violations/graph?type=cpd" % (base_url, build)

    def get_pep8_graph(self, build):
        return "%s/%s/violations/graph?type=pep8" % (base_url, build)

def get_build_page(build):
    f = urllib.urlopen("%s/%s/" % (base_url, build))
    page = f.read()
    f.close()
    return BeautifulSoup(page)

def get_coverage(page):
    return page.findAll(text=re.compile("Cobertura Coverage"))[0]

def get_test_results(page):
    return page.findAll(text=re.compile("Test Result:"))[0]

def get_status(page):
    return page.findAll(src="buildStatus")[0]['alt']

#def get_violation_results(violation_type, page):
#    violation = page.find('a', {'href':'#' + violation_type})
#    violation_count = violation.parent.nextSibling
#    return (violation_count.text, violation_count.nextSibling.text)

#def get_image_html(src):
#    return r"" % src

#def get_section(title, content):
#    return """
#

#

#
%s

#                %s
#

#

#    """ % (title, content)
#
#def build_stats(build):
#
#    #build, status
#
##    print get_section("CPD", get_image_html(get_cpd_graph(build)))
##    print get_section("Pep8", get_image_html(get_pep8_graph(build)))
##    print """
"""
#
#
#def x():
#    for build in builds:
#        build_stats(build)
<commit_msg>added pending status as passing build<commit_after>
from django.views.generic import TemplateView
from BeautifulSoup import BeautifulSoup
import re
import urllib
import time

ci_url = "http://ci.apps-system.com/"
base_url = "%s/job" % ci_url

builds = [
    'django-admin-ext',
    'django-dynamic-rules',
    'django-dynamic-validation',
    'django-forms-ext',
    'django-dynamic-manipulation',
    'django-wizard',
    'django-pretty-times',
    'django-response-helpers',
    'django-attachments',
]

class Status(TemplateView):
    template_name = 'dashboard/status.html'

    def get_context_data(self, **kwargs):
        f = urllib.urlopen(ci_url)
        data = f.read()
        f.close()
        page = BeautifulSoup(data)

        status_table = page.findAll(**{'id': 'projectstatus'})[0]
        statuses = []
        for cell in status_table.findAll('tr')[1:]:
            status_icon = cell.find('img', **{'class': 'icon32x32'})
            if status_icon:
                statuses.append(dict(status_icon.attrs)['alt'])

        ss = [s not in ('Success', 'Disabled', 'In progress', 'Pending') for s in statuses]
        fail = any(ss)
        if fail:
            for i in range(20):
                print "\a"
                time.sleep(.2)

        return {'success': not fail}

class Index(TemplateView):
    template_name = 'dashboard/index.html'
    current_build = -1

    def get_context_data(self, **kwargs):
        Index.current_build += 1
        if Index.current_build >= len(builds):
            Index.current_build = 0

        build = builds[Index.current_build]
        return {
            'params': kwargs,
            'status':self.get_status(),
            'build': build,
            'test_section_title':'Tests',
            'test_section_content':self.get_test_graph(build),
            'coverage_section_title':'Coverage',
            'coverage_section_content':self.get_cobertura_graph(build),
            'violations_section_title':'Violations',
            'violations_section_content':self.get_violations_graph(build),
            'pylint_section_title':'Pylint',
            'pylint_section_content':self.get_pylint_graph(build),
            'cpd_section_title':'CPD',
            'cpd_section_content':self.get_cpd_graph(build),
            'pep8_section_title':'Pep8',
            'pep8_section_content':self.get_pep8_graph(build),
        }

    def get_status(self):
        page = get_build_page(builds[Index.current_build] + "/lastBuild")
        status = get_status(page)
        return status.replace(" ", '')

    def get_test_graph(self, build):
        return "%s/%s/test/trend" % (base_url, build)

    def get_cobertura_graph(self, build):
        return "%s/%s/cobertura/graph" % (base_url, build)

    def get_violations_graph(self, build):
        return "%s/%s/violations/graph" % (base_url, build)

    def get_pylint_graph(self, build):
        return "%s/%s/violations/graph?type=pylint" % (base_url, build)

    def get_cpd_graph(self, build):
        return "%s/%s/violations/graph?type=cpd" % (base_url, build)

    def get_pep8_graph(self, build):
        return "%s/%s/violations/graph?type=pep8" % (base_url, build)

def get_build_page(build):
    f = urllib.urlopen("%s/%s/" % (base_url, build))
    page = f.read()
    f.close()
    return BeautifulSoup(page)

def get_coverage(page):
    return page.findAll(text=re.compile("Cobertura Coverage"))[0]

def get_test_results(page):
    return page.findAll(text=re.compile("Test Result:"))[0]

def get_status(page):
    return page.findAll(src="buildStatus")[0]['alt']

#def get_violation_results(violation_type, page):
#    violation = page.find('a', {'href':'#' + violation_type})
#    violation_count = violation.parent.nextSibling
#    return (violation_count.text, violation_count.nextSibling.text)

#def get_image_html(src):
#    return r"" % src

#def get_section(title, content):
#    return """
#

#

#
%s

#                %s
#

#

#    """ % (title, content)
#
#def build_stats(build):
#
#    #build, status
#
##    print get_section("CPD", get_image_html(get_cpd_graph(build)))
##    print get_section("Pep8", get_image_html(get_pep8_graph(build)))
##    print """
"""
#
#
#def x():
#    for build in builds:
#        build_stats(build)
<|endoftext|>


<commit_before>#coding: utf8
from sweet.utils import *
from sweet.utils.inflection import *
from sweet.orm.relations import *
from sweet.orm.method_missing import *

class ModelMetaClass(type):

    def __init__(cls, name, bases, attr):
        # print ("^"*5, " meta class init ", "^"*5, cls, name, bases, attr)

        if name != 'Model':
            # set __tablename__ to model Class
            if not hasattr(cls, '__tablename__'):
                setattr(cls, '__tablename__', tableize(cls.__name__))

            if not hasattr(cls, '__field_define_dict__'):
                setattr(cls, '__field_define_dict__', {})
                cls._init_fields()

            if not hasattr(cls, '__pk__'):
                setattr(cls, '__pk__', 'id')

            if not hasattr(cls, '__relations__'):
                setattr(cls, '__relations__', {})

            # if cls.__pk__ not in cls.__field_define_dict__:
            #     raise Exception('%s field of %s does not exist' % (cls.__pk__, cls.__name__))

            if getattr(cls, '__timestamp__', True):
                setattr(cls, 'created_at', None)
                setattr(cls, 'updated_at', None)

            while True:
                r = relation_q.get()
                if r:
                    r.set_owner(cls)
                else:
                    break

        return type.__init__(cls, name, bases, attr)

class Model(metaclass=ModelMetaClass):

    class HasNotBeenPersisted(Exception):
        pass

    def __init__(self, **attrs):
        for k, v in attrs.items():
            setattr(self, k, v)
        self._init_field_default_value() # !Very Important

    def __getattribute__(self, name):
        try:
            return super().__getattribute__(name)
        except AttributeError as ex:
            if name in self.__relations__: # it does means a relation
                cache_key = self._build_relation_cache_key(name)
                if not hasattr(self, cache_key): # can not get cache of relation
                    r = self.__relations__[name]
                    real_value = r.get_real_value(self)
                    setattr(self, cache_key, real_value) # set cache
                return getattr(self, cache_key)
            else:
                method = MethodMissing.match(self, name)
                if method:
                    return method(self, name)
            raise ex

    def __setattr__(self, name, value):
        cls = self.__class__
        relt = super().__setattr__(name, value)

        if name in cls.__relations__:
            relation = cls.__relations__[name]
            relation.inject(self, value)
        return relt

    def _delete_relation_cache(self, name):
        key = self._build_relation_cache_key(name)
        if hasattr(self, key):
            delattr(self, key)
        return self

    def _build_relation_cache_key(self, name):
        return '_cached_relation_which_named_%s' % name

    def _init_field_default_value(self):
        """ set default value of field which does not init
        """
        cls = self.__class__
        for name, field in cls.__field_define_dict__.items():
            if not hasattr(self, name):
                setattr(self, name, field.default)
        return self

    def column_dict(self):
        d = {
            name:getattr(self, name, field.default) \
                for name, field in self.__field_define_dict__.items()
        }
        return d

    def save(self):
        """ save object.
        will update the record if it has been persisted else create one.
        """
        if self.persisted():
            self.update()
        else:
            _id = self.objects.insert_getid(**self.column_dict())
            self.set_pk(_id)
        return self

    @classmethod
    def create(cls, **attrs):
        """ create a record in db.
        should raise ModelHasBeenPersisted if it has been persisted
        """
        model = cls(**attrs)
        model.save()
        return model

    @classmethod
    def create_all(cls, *items):
        """ batch create and return how many insert successful
        """
        return cls.objects.insert(items)

    def update(self, **attrs):
        """ update a record in db,
        should raise HasNotBeenPersisted if it has not been persisted
        """
        if not self.persisted():
            raise HasNotBeenPersisted()
        if attrs:
            # old_values = {
            #     k: getattr(self, k) for k, v in attrs.items() if hasattr(self, k)
            # }
            for k, v in attrs.items():
                setattr(self, k, v)
        self.objects.where(**{self.__pk__: self.get_pk()}).update(**self.column_dict())
        return self

    @classmethod
    def update_all(cls, **attrs):
        """ delete all record
        """
        cls.objects.update(**attrs)
        return cls

    def delete(self):
        """ delete a record in db and set id is None.
        would do nothing if it has not been persisted
        """
        self.__class__._delete_relations([self])

        if self.persisted():
            pk = self.__pk__
            self.objects.where(**{self.__pk__: self.get_pk()}).delete()
            self.set_pk(None)
        return self

    @classmethod
    def delete_all(cls, **attrs):
        """ delete all record
        """
        if attrs:
            objs = cls.objects.where(**attrs)
        else:
            objs = cls.objects

        rs = objs.all() if cls.__relations__ else []
        cls._delete_relations(rs)

        objs.delete()
        return cls

    @classmethod
    def _delete_relations(cls, owner_models):
        for name, r in cls.__relations__.items():
            r.delete_all_real_value(owner_models)

    def set_pk(self, value):
        return setattr(self, self.__pk__, value)

    def get_pk(self):
        return getattr(self, self.__pk__, None)

    def persisted(self):
        return True if self.get_pk() else False

    @classmethod
    def count(cls, column=None, distinct=False):
        return cls.objects.count(column, distinct)

    @classmethod
    def truncate(cls):
        return cls.objects.truncate()

    @classmethod
    def exists(cls):
        return cls.objects.exists()

    @classmethod
    def max(cls, column, distinct=False):
        return cls.objects.max(column, distinct)

    @classmethod
    def min(cls, column, distinct=False):
        return cls.objects.min(column, distinct)

    @classmethod
    def avg(cls, column, distinct=False):
        return cls.objects.avg(column, distinct)

    @classmethod
    def sum(cls, column, distinct=False):
        return cls.objects.sum(column, distinct)

    @classmethod
    def all(cls):
        return cls.objects.all()

    @classmethod
    def first(cls):
        return cls.objects.first()

    @classmethod
    def last(self):
        return self.objects.last()

    @classmethod
    def find(cls, *ids):
        """ find by ids. there are 2 cases.
        case 1: one id，should return model instance if found else None
        case 2: multiple ids, should return the model list
        """
        if len(ids) == 1:
            return cls.objects.where(id=ids).first()
        else:
            return cls.objects.where(id=ids).all()

    @classmethod
    def find_or_create(cls, **kwargs):
        model = cls.objects.where(**kwargs).limit(1).first()
        if not model:
            model = cls.create(**kwargs)
        return model

    @classmethod
    def _new_db(cls):
        return cls.db_manager.new_db()

    @classproperty
    def objects(cls):
        db = cls._new_db()
        rs = db.records(cls.__tablename__)
        rs.model_class = cls
        return rs

    @classmethod
    def _init_fields(cls):
        for c in cls.db_manager.new_db().get_columns(cls.__tablename__):
            cls.__field_define_dict__[c.name] = c
        return cls

    @classmethod
    def _register_relation(cls, name, r):
        cls.__relations__[name] = r

    @classmethod
    def select(cls, *columns):
        return cls.objects.select(*columns)

    @classmethod
    def where(cls, **filters):
        return cls.objects.where(**filters)
<commit_msg>fix bug and refactoring<commit_after>#coding: utf8
from sweet.utils import *
from sweet.utils.inflection import *
from sweet.orm.relations import *
from sweet.orm.method_missing import *

class ModelMetaClass(type):

    def __init__(cls, name, bases, attr):
        # print ("^"*5, " meta class init ", "^"*5, cls, name, bases, attr)

        if name != 'Model':
            # set __tablename__ to model Class
            if not hasattr(cls, '__tablename__'):
                setattr(cls, '__tablename__', tableize(cls.__name__))

            if not hasattr(cls, '__field_define_dict__'):
                setattr(cls, '__field_define_dict__', {})
                cls._init_fields()

            if not hasattr(cls, '__pk__'):
                setattr(cls, '__pk__', 'id')

            if not hasattr(cls, '__relations__'):
                setattr(cls, '__relations__', {})

            # if cls.__pk__ not in cls.__field_define_dict__:
            #     raise Exception('%s field of %s does not exist' % (cls.__pk__, cls.__name__))

            if getattr(cls, '__timestamp__', True):
                setattr(cls, 'created_at', None)
                setattr(cls, 'updated_at', None)

            while True:
                r = relation_q.get()
                if r:
                    r.set_owner(cls)
                else:
                    break

        return type.__init__(cls, name, bases, attr)

class Model(metaclass=ModelMetaClass):

    class HasNotBeenPersisted(Exception):
        pass

    def __init__(self, **attrs):
        for k, v in attrs.items():
            setattr(self, k, v)
        self._init_field_default_value() # !Very Important

    def __getattribute__(self, name):
        try:
            return super().__getattribute__(name)
        except AttributeError as ex:
            if name in self.__relations__: # it does means a relation
                cache_key = self._build_relation_cache_key(name)
                if not hasattr(self, cache_key): # can not get cache of relation
                    r = self.__relations__[name]
                    real_value = r.get_real_value(self)
                    self._set_relation_cache(cache_key, real_value, False) # set cache
                return getattr(self, cache_key)
            else:
                method = MethodMissing.match(self, name)
                if method:
                    return method(self, name)
            raise ex

    def __setattr__(self, name, value):
        cls = self.__class__
        if name in cls.__relations__:
            relation = cls.__relations__[name]
            relation.inject(self, value)
            self._set_relation_cache(self, name, value)
        super().__setattr__(name, value)

    def _set_relation_cache(self, name, value, should_build_key=True):
        cache_key = self._build_relation_cache_key(name) if should_build_key else name
        setattr(self, cache_key, value)
        return self

    def _delete_relation_cache(self, name):
        key = self._build_relation_cache_key(name)
        if hasattr(self, key):
            delattr(self, key)
        return self

    def _build_relation_cache_key(self, name):
        return '_cached_relation_which_named_%s' % name

    def _init_field_default_value(self):
        """ set default value of field which does not init
        """
        cls = self.__class__
        for name, field in cls.__field_define_dict__.items():
            if not hasattr(self, name):
                setattr(self, name, field.default)
        return self

    def column_dict(self):
        d = {
            name:getattr(self, name, field.default) \
                for name, field in self.__field_define_dict__.items()
        }
        return d

    def save(self):
        """ save object.
        will update the record if it has been persisted else create one.
        """
        if self.persisted():
            self.update()
        else:
            _id = self.objects.insert_getid(**self.column_dict())
            self.set_pk(_id)
        return self

    @classmethod
    def create(cls, **attrs):
        """ create a record in db.
        should raise ModelHasBeenPersisted if it has been persisted
        """
        model = cls(**attrs)
        model.save()
        return model

    @classmethod
    def create_all(cls, *items):
        """ batch create and return how many insert successful
        """
        return cls.objects.insert(items)

    def update(self, **attrs):
        """ update a record in db,
        should raise HasNotBeenPersisted if it has not been persisted
        """
        if not self.persisted():
            raise HasNotBeenPersisted()
        if attrs:
            # old_values = {
            #     k: getattr(self, k) for k, v in attrs.items() if hasattr(self, k)
            # }
            for k, v in attrs.items():
                setattr(self, k, v)
        self.objects.where(**{self.__pk__: self.get_pk()}).update(**self.column_dict())
        return self

    @classmethod
    def update_all(cls, **attrs):
        """ delete all record
        """
        cls.objects.update(**attrs)
        return cls

    def delete(self):
        """ delete a record in db and set id is None.
        would do nothing if it has not been persisted
        """
        self.__class__._delete_relations([self])

        if self.persisted():
            pk = self.__pk__
            self.objects.where(**{self.__pk__: self.get_pk()}).delete()
            self.set_pk(None)
        return self

    @classmethod
    def delete_all(cls, **attrs):
        """ delete all record
        """
        if attrs:
            objs = cls.objects.where(**attrs)
        else:
            objs = cls.objects

        rs = objs.all() if cls.__relations__ else []
        cls._delete_relations(rs)

        objs.delete()
        return cls

    @classmethod
    def _delete_relations(cls, owner_models):
        for name, r in cls.__relations__.items():
            r.delete_all_real_value(owner_models)

    def set_pk(self, value):
        return setattr(self, self.__pk__, value)

    def get_pk(self):
        return getattr(self, self.__pk__, None)

    def persisted(self):
        return True if self.get_pk() else False

    @classmethod
    def count(cls, column=None, distinct=False):
        return cls.objects.count(column, distinct)

    @classmethod
    def truncate(cls):
        return cls.objects.truncate()

    @classmethod
    def exists(cls):
        return cls.objects.exists()

    @classmethod
    def max(cls, column, distinct=False):
        return cls.objects.max(column, distinct)

    @classmethod
    def min(cls, column, distinct=False):
        return cls.objects.min(column, distinct)

    @classmethod
    def avg(cls, column, distinct=False):
        return cls.objects.avg(column, distinct)

    @classmethod
    def sum(cls, column, distinct=False):
        return cls.objects.sum(column, distinct)

    @classmethod
    def all(cls):
        return cls.objects.all()

    @classmethod
    def first(cls):
        return cls.objects.first()

    @classmethod
    def last(self):
        return self.objects.last()

    @classmethod
    def find(cls, *ids):
        """ find by ids. there are 2 cases.
        case 1: one id，should return model instance if found else None
        case 2: multiple ids, should return the model list
        """
        if len(ids) == 1:
            return cls.objects.where(id=ids).first()
        else:
            return cls.objects.where(id=ids).all()

    @classmethod
    def find_or_create(cls, **kwargs):
        model = cls.objects.where(**kwargs).limit(1).first()
        if not model:
            model = cls.create(**kwargs)
        return model

    @classmethod
    def _new_db(cls):
        return cls.db_manager.new_db()

    @classproperty
    def objects(cls):
        db = cls._new_db()
        rs = db.records(cls.__tablename__)
        rs.model_class = cls
        return rs

    @classmethod
    def _init_fields(cls):
        for c in cls.db_manager.new_db().get_columns(cls.__tablename__):
            cls.__field_define_dict__[c.name] = c
        return cls

    @classmethod
    def _register_relation(cls, name, r):
        cls.__relations__[name] = r

    @classmethod
    def select(cls, *columns):
        return cls.objects.select(*columns)

    @classmethod
    def where(cls, **filters):
        return cls.objects.where(**filters)
<|endoftext|>


<commit_before>__author__ = 'joe.snyder'
import argparse
import os
import zipfile
import shutil
import re
import subprocess
import sys

OTJ_SURVEY_EXPRESSIONS = (("Do not [Re]*distribute",[]),
  ("Copyright [0-9\-]*",[]),
  ("(Released|Licensed) ",[]),
  ("All rights reserved",[]),
  ("Deriv[atived]+",[])
)

OTJ_FILE_EXCLUDES = ("dat","dll", "DAT","exe", "EXE", "mdf", "pdb", "class", "jpeg",
    "jpg","png","gif","bmp","docx","pdf","vsd","doc", "xls", "xlsx", "ppt", "pptx",
    ".db","xml","XML")
OTJ_UNZIP_EXCLUDES= ("EAP","dll","docx")
OTJ_SURVEY_HISTORY = []
global seven_zip

def remove_readonly(func, path, excinfo):
    os.chmod(path, stat.S_IWRITE)
    func(path)

def searchArchive(filepath,unzipPath, tmpDir):
    if os.path.isdir(unzipPath):
        unzipPath += "_1"
    if not (filepath.endswith(OTJ_UNZIP_EXCLUDES)):
        cmd=[seven_zip,"x","-bd", "-y", "-o"+unzipPath, filepath]
        ret = subprocess.call(cmd)
        if os.path.isdir(unzipPath):
            OTJ_SURVEY_HISTORY.append(filepath)
            parseSubmission(unzipPath.replace("\\\\?\\",""),tmpDir)
            OTJ_SURVEY_HISTORY.pop()
            shutil.rmtree(unzipPath.replace("\\\\?\\",""))

def parseSubmission(inputPath,tmpDir):
    # Read in file type
    for fileName in os.listdir(inputPath):
          filepath=''
          unzipPath=''
          if os.name == 'nt':
            filepath = "\\\\?\\"
            unzipPath= "\\\\?\\"
          filepath  += os.path.join(os.path.relpath(inputPath), fileName)
          splitVal = fileName.rfind('.')
          if splitVal:
            fileName = fileName[:splitVal]
          unzipPath += os.path.join(os.path.abspath(tmpDir),fileName.replace(" ","_"))
          print "Processing file: %s" % filepath
          if (zipfile.is_zipfile(filepath)) or filepath.endswith("7z"):
            searchArchive(filepath, unzipPath, tmpDir)
          elif os.path.isdir(filepath):
            parseSubmission(filepath,tmpDir)
          else:
            if not (filepath.endswith(OTJ_FILE_EXCLUDES)):
              outline= filepath
              if len(OTJ_SURVEY_HISTORY):
                 outline += "  from (" + OTJ_SURVEY_HISTORY[-1]+")"
              for line in open(filepath, "r"):
                for entry in OTJ_SURVEY_EXPRESSIONS:
                  if re.search(entry[0], line, re.I):
                    entry[1].append((outline,line))

def main():
    global seven_zip
    parser = argparse.ArgumentParser(description='OTJ Submission Analyzer')
    parser.add_argument('-p', '--paths', required=True, action='append',
        help='input file path to the directory that contains the OTJ submission to check')
    parser.add_argument('-d', '--dir', required=True,
        help='a file path where the script will attempt to unzip any archive files and will write any output files')
    parser.add_argument('-z', '--zip', required=True,
        help='a file path to the 7zip executable')
    result = parser.parse_args()
    seven_zip = result.zip
    print seven_zip
    try:
      for path in result.paths:
        parseSubmission(path, result.dir)
    finally:
        with open(os.path.join(result.dir, "SurveyResult.txt"),"w") as outfile:
          outfile.write("Begin Report")
          for entry in OTJ_SURVEY_EXPRESSIONS:
            if len(entry[1]):
              outfile.write("**********************\nFound results for\n")
              outfile.write(entry[0]+"\n")
              outfile.write("**********************\n")
              for item in entry[1]:
                outfile.write(item[0]+"\n     "+ item[1]+"\n")
            outfile.write("\n\n")
          outfile.write("End Report")
          outfile.close()

if __name__ == '__main__':
  main()
<commit_msg>Remove parsing of "Public_Report" directory<commit_after>__author__ = 'joe.snyder'
import argparse
import os
import zipfile
import shutil
import re
import subprocess
import sys

OTJ_SURVEY_EXPRESSIONS = (("Do not [Re]*distribute",[]),
  ("Copyright [0-9\-]*",[]),
  ("(Released|Licensed) ",[]),
  ("All rights reserved",[]),
  ("Deriv[atived]+",[])
)

OTJ_FILE_EXCLUDES = ("dat","dll", "DAT","exe", "EXE", "mdf", "pdb", "class", "jpeg",
    "jpg","png","gif","bmp","docx","pdf","vsd","doc", "xls", "xlsx", "ppt", "pptx",
    ".db","xml","XML")
OTJ_UNZIP_EXCLUDES= ("EAP","dll","docx")
OTJ_SURVEY_HISTORY = []
global seven_zip

def remove_readonly(func, path, excinfo):
    os.chmod(path, stat.S_IWRITE)
    func(path)

def searchArchive(filepath,unzipPath, tmpDir):
    if os.path.isdir(unzipPath):
        unzipPath += "_1"
    if not (filepath.endswith(OTJ_UNZIP_EXCLUDES)):
        cmd=[seven_zip,"x","-bd", "-y", "-o"+unzipPath, filepath]
        ret = subprocess.call(cmd)
        if os.path.isdir(unzipPath):
            OTJ_SURVEY_HISTORY.append(filepath)
            parseSubmission(unzipPath.replace("\\\\?\\",""),tmpDir)
            OTJ_SURVEY_HISTORY.pop()
            shutil.rmtree(unzipPath.replace("\\\\?\\",""))

def parseSubmission(inputPath,tmpDir):
    # Read in file type
    for fileName in os.listdir(inputPath):
          filepath=''
          unzipPath=''
          if os.name == 'nt':
            filepath = "\\\\?\\"
            unzipPath= "\\\\?\\"
          filepath  += os.path.join(os.path.relpath(inputPath), fileName)
          splitVal = fileName.rfind('.')
          if splitVal:
            fileName = fileName[:splitVal]
          unzipPath += os.path.join(os.path.abspath(tmpDir),fileName.replace(" ","_"))
          print "Processing file: %s" % filepath
          if (zipfile.is_zipfile(filepath)) or filepath.endswith("7z"):
            searchArchive(filepath, unzipPath, tmpDir)
          elif (os.path.isdir(filepath)):
            if (not ("Public_Report" in filepath)):
              parseSubmission(filepath,tmpDir)
            else:
              continue
          else:
            if not (filepath.endswith(OTJ_FILE_EXCLUDES)):
              outline= filepath
              if len(OTJ_SURVEY_HISTORY):
                 outline += "  from (" + OTJ_SURVEY_HISTORY[-1]+")"
              for line in open(filepath, "r"):
                for entry in OTJ_SURVEY_EXPRESSIONS:
                  if re.search(entry[0], line, re.I):
                    entry[1].append((outline,line))

def main():
    global seven_zip
    parser = argparse.ArgumentParser(description='OTJ Submission Analyzer')
    parser.add_argument('-p', '--paths', required=True, action='append',
        help='input file path to the directory that contains the OTJ submission to check')
    parser.add_argument('-d', '--dir', required=True,
        help='a file path where the script will attempt to unzip any archive files and will write any output files')
    parser.add_argument('-z', '--zip', required=True,
        help='a file path to the 7zip executable')
    result = parser.parse_args()
    seven_zip = result.zip
    print seven_zip
    try:
      for path in result.paths:
        parseSubmission(path, result.dir)
    finally:
        with open(os.path.join(result.dir, "SurveyResult.txt"),"w") as outfile:
          outfile.write("Begin Report")
          for entry in OTJ_SURVEY_EXPRESSIONS:
            if len(entry[1]):
              outfile.write("**********************\nFound results for\n")
              outfile.write(entry[0]+"\n")
              outfile.write("**********************\n")
              for item in entry[1]:
                outfile.write(item[0]+"\n     "+ item[1]+"\n")
            outfile.write("\n\n")
          outfile.write("End Report")
          outfile.close()

if __name__ == '__main__':
  main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#Script to build a scriptorium paper in a cross-platform friendly fashion

import argparse
import shutil
import sys

import scriptorium

def make(args):
    """Creates PDF from paper in the requested location."""
    pdf = scriptorium.to_pdf(args.paper, use_shell_escape=args.shell_escape)

    if args.output and pdf != args.output:
        shutil.move(pdf, args.output)

def info(args):
    """Function to attempt to extract useful information from a specified paper."""
    fname = scriptorium.paper_root(args.paper)

    if not fname:
        raise IOError('{0} does not contain a valid root document.'.format(args.paper))

    if not fname:
        print('Could not find the root of the paper.')
        sys.exit(1)

    if args.template:
        template = scriptorium.get_template(os.path.join(args.paper, fname))
        if not template:
            print('Could not find footer indicating template name.')
            sys.exit(2)
        print(template)

def list_cmd(args):
    """Prints out all installed templates."""
    templates = scriptorium.all_templates(args.template_dir)
    for template in templates:
        print('{0}'.format(template))

def create(args):
    """Creates a new paper given flags."""
    if not scriptorium.create(args.output, args.template, force=args.force, config=args.config):
        sys.exit(3)

def main():
    parser = argparse.ArgumentParser()

    subparsers = parser.add_subparsers()

    # Make command
    make_parser = subparsers.add_parser("make")
    make_parser.add_argument("paper", default=".", help="Directory containing paper to make")
    make_parser.add_argument('-o', '--output', help='Filename to store resulting PDF as.')
    make_parser.add_argument('-s', '--shell-escape', action='store_true', help='Flag to indicate shell-escape should be used')
    make_parser.set_defaults(func=make)

    info_parser = subparsers.add_parser("info")
    info_parser.add_argument("paper", default=".", help="Directory containing paper to make")
    info_parser.add_argument('-t', '--template', action="store_true", help="Flag to extract template")
    info_parser.set_defaults(func=info)

    new_parser = subparsers.add_parser("new")
    new_parser.add_argument("output", help="Directory to create paper in.")
    new_parser.add_argument("-f", "--force", action="store_true", help="Overwrite files in paper creation.")
    new_parser.add_argument("-t", "--template", help="Template to use in paper.")
    new_parser.add_argument("-c", "--config", nargs=2, action='append', default=[],
                            help='Flag to provide options for filling out variables in new papers, in the form key value')
    new_parser.set_defaults(func=create)

    list_parser = subparsers.add_parser("list")
    list_parser.add_argument("-t", "--template_dir", default=scriptorium.TEMPLATE_DIR, help="Overrides template directory used for listing templates")
    list_parser.set_defaults(func=list_cmd)

    args = parser.parse_args()

    if 'func' in args:
        args.func(args)
    else:
        parser.print_help()

if __name__ == '__main__':
    main()<commit_msg>Make make command default to current working directory if no argument is passed<commit_after>#!/usr/bin/env python
#Script to build a scriptorium paper in a cross-platform friendly fashion

import argparse
import shutil
import sys

import scriptorium

def make(args):
    """Creates PDF from paper in the requested location."""
    pdf = scriptorium.to_pdf(args.paper, use_shell_escape=args.shell_escape)

    if args.output and pdf != args.output:
        shutil.move(pdf, args.output)

def info(args):
    """Function to attempt to extract useful information from a specified paper."""
    fname = scriptorium.paper_root(args.paper)

    if not fname:
        raise IOError('{0} does not contain a valid root document.'.format(args.paper))

    if not fname:
        print('Could not find the root of the paper.')
        sys.exit(1)

    if args.template:
        template = scriptorium.get_template(os.path.join(args.paper, fname))
        if not template:
            print('Could not find footer indicating template name.')
            sys.exit(2)
        print(template)

def list_cmd(args):
    """Prints out all installed templates."""
    templates = scriptorium.all_templates(args.template_dir)
    for template in templates:
        print('{0}'.format(template))

def create(args):
    """Creates a new paper given flags."""
    if not scriptorium.create(args.output, args.template, force=args.force, config=args.config):
        sys.exit(3)

def main():
    parser = argparse.ArgumentParser()

    subparsers = parser.add_subparsers()

    # Make command
    make_parser = subparsers.add_parser("make")
    make_parser.add_argument("paper", default=".", nargs='?', help="Directory containing paper to make")
    make_parser.add_argument('-o', '--output', help='Filename to store resulting PDF as.')
    make_parser.add_argument('-s', '--shell-escape', action='store_true', help='Flag to indicate shell-escape should be used')
    make_parser.set_defaults(func=make)

    info_parser = subparsers.add_parser("info")
    info_parser.add_argument("paper", default=".", help="Directory containing paper to make")
    info_parser.add_argument('-t', '--template', action="store_true", help="Flag to extract template")
    info_parser.set_defaults(func=info)

    new_parser = subparsers.add_parser("new")
    new_parser.add_argument("output", help="Directory to create paper in.")
    new_parser.add_argument("-f", "--force", action="store_true", help="Overwrite files in paper creation.")
    new_parser.add_argument("-t", "--template", help="Template to use in paper.")
    new_parser.add_argument("-c", "--config", nargs=2, action='append', default=[],
                            help='Flag to provide options for filling out variables in new papers, in the form key value')
    new_parser.set_defaults(func=create)

    list_parser = subparsers.add_parser("list")
    list_parser.add_argument("-t", "--template_dir", default=scriptorium.TEMPLATE_DIR, help="Overrides template directory used for listing templates")
    list_parser.set_defaults(func=list_cmd)

    args = parser.parse_args()

    if 'func' in args:
        args.func(args)
    else:
        parser.print_help()

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before># Copyright (c) 2009 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

{
  'includes': [
    'plugin.gypi',
  ],
  'target_defaults': {
    'variables': {
      'target_base': 'none',
    },
    'target_conditions': [
      ['target_base=="npNaClPlugin"', {
        'sources': [
          '<@(common_sources)',
          '<@(npapi_sources)',
          'npapi/video.cc',
        ],
        'xcode_settings': {
          'WARNING_CFLAGS!': [
            # TODO(bradnelson): remove -pedantic when --std=c++98 in common.gypi
            '-pedantic',
          ],
          'WARNING_CFLAGS': [
            '-Wno-deprecated',
            '-Wno-deprecated-declarations',
          ],
        },
        'conditions': [
          ['OS=="win"', {
            'sources': [
              'win/nacl_plugin.rc',
            ],
            'msvs_settings': {
              'VCCLCompilerTool': {
                'ExceptionHandling': '2',  # /EHsc
              },
              'VCLinkerTool': {
                'AdditionalLibraryDirectories': [
                   '$(OutDir)/lib',
                ],
              },
            },
          }],
        ],
      }],
      ['target_base=="ppNaClPlugin"', {
        'sources': [
          '<@(common_sources)',
          '<@(ppapi_sources)',
        ],
        'defines': [
          'NACL_PPAPI',
        ],
        'xcode_settings': {
          'WARNING_CFLAGS!': [
            # TODO(bradnelson): remove -pedantic when --std=c++98 in common.gypi
            '-pedantic',
          ],
          'WARNING_CFLAGS': [
            '-Wno-deprecated',
            '-Wno-deprecated-declarations',
          ],
        },
        'conditions': [
          ['OS=="win"', {
            'sources': [
              'win/nacl_plugin.rc',
            ],
            'msvs_settings': {
              'VCCLCompilerTool': {
                'ExceptionHandling': '2',  # /EHsc
              },
              'VCLinkerTool': {
                'AdditionalLibraryDirectories': [
                   '$(OutDir)/lib',
                ],
              },
            },
          }],
        ],
      }],
    ],
  },
  'targets': [],
  'conditions': [
    ['target_arch!="x64"', {
      'targets': [
      # ----------------------------------------------------------------------
        {
          'target_name': 'npGoogleNaClPlugin',
          'type': 'shared_library',
          'variables': {
            'target_base': 'npNaClPlugin',
          },
          'dependencies': [
            '<(DEPTH)/native_client/src/shared/gio/gio.gyp:gio',
            '<(DEPTH)/native_client/src/shared/imc/imc.gyp:google_nacl_imc_c',
            '<(DEPTH)/native_client/src/shared/npruntime/npruntime.gyp:google_nacl_npruntime',
            '<(DEPTH)/native_client/src/shared/platform/platform.gyp:platform',
            '<(DEPTH)/native_client/src/shared/srpc/srpc.gyp:nonnacl_srpc',
            '<(DEPTH)/native_client/src/trusted/desc/desc.gyp:nrd_xfer',
            '<(DEPTH)/native_client/src/trusted/expiration/expiration.gyp:expiration',
            '<(DEPTH)/native_client/src/trusted/nonnacl_util/nonnacl_util.gyp:nonnacl_util',
            '<(DEPTH)/native_client/src/trusted/platform_qualify/platform_qualify.gyp:platform_qual_lib',
            '<(DEPTH)/native_client/src/trusted/gio/gio_wrapped_desc.gyp:gio_wrapped_desc',
          ],
          'conditions': [
            ['OS=="win"', {
              'sources': [
                'win/nacl_plugin.def',
              ],
            }],
          ],
        },
        {
          'target_name': 'ppGoogleNaClPlugin',
          'type': 'shared_library',
          'variables': {
            'target_base': 'ppNaClPlugin',
          },
          'dependencies': [
            '<(DEPTH)/native_client/src/shared/gio/gio.gyp:gio',
            '<(DEPTH)/native_client/src/shared/imc/imc.gyp:google_nacl_imc_c',
            '<(DEPTH)/native_client/src/shared/platform/platform.gyp:platform',
            '<(DEPTH)/native_client/src/shared/ppapi/ppapi.gyp:ppapi_cpp',
            '<(DEPTH)/native_client/src/shared/ppapi_proxy/ppapi_proxy.gyp:nacl_ppapi_browser',
            '<(DEPTH)/native_client/src/shared/srpc/srpc.gyp:nonnacl_srpc',
            '<(DEPTH)/native_client/src/shared/srpc/srpc.gyp:nonnacl_srpc',
            '<(DEPTH)/native_client/src/trusted/desc/desc.gyp:nrd_xfer',
            '<(DEPTH)/native_client/src/trusted/expiration/expiration.gyp:expiration',
            '<(DEPTH)/native_client/src/trusted/nonnacl_util/nonnacl_util.gyp:nonnacl_util',
            '<(DEPTH)/native_client/src/trusted/platform_qualify/platform_qualify.gyp:platform_qual_lib',
            '<(DEPTH)/native_client/src/trusted/gio/gio_wrapped_desc.gyp:gio_wrapped_desc',
          ],
# TODO(noelallen) We will need to put this back in with a new .def file once we need to export symbols
# to support the plugin as a sandboxed DLL.
#          'conditions': [
#            ['OS=="win"', {
#              'sources': [
#                'win/nacl_plugin.def',
#              ],
#            }],
#          ],
        },
      ],
    }],
    ['OS=="win"', {
      'targets': [
        # ---------------------------------------------------------------------
        {
          'target_name': 'npGoogleNaClPlugin64',
          'type': 'shared_library',
          'variables': {
            'target_base': 'npNaClPlugin',
            'win_target': 'x64',
          },
          'sources': [
            'win/nacl_plugin64.def',
          ],
          'dependencies': [
            '<(DEPTH)/native_client/src/shared/gio/gio.gyp:gio64',
            '<(DEPTH)/native_client/src/shared/imc/imc.gyp:google_nacl_imc_c64',
            '<(DEPTH)/native_client/src/shared/npruntime/npruntime.gyp:google_nacl_npruntime64',
            '<(DEPTH)/native_client/src/shared/platform/platform.gyp:platform64',
            '<(DEPTH)/native_client/src/shared/srpc/srpc.gyp:nonnacl_srpc64',
            '<(DEPTH)/native_client/src/trusted/desc/desc.gyp:nrd_xfer64',
            '<(DEPTH)/native_client/src/trusted/expiration/expiration.gyp:expiration64',
            '<(DEPTH)/native_client/src/trusted/nonnacl_util/nonnacl_util.gyp:nonnacl_util64',
            '<(DEPTH)/native_client/src/trusted/platform_qualify/platform_qualify.gyp:platform_qual_lib64',
            '<(DEPTH)/native_client/src/trusted/gio/gio_wrapped_desc.gyp:gio_wrapped_desc64',
          ],
          'configurations': {
            'Common_Base': {
              'msvs_target_platform': 'x64',
            },
          },
        },
      ],
    }],
    ['nacl_standalone==0', {
      'targets': [
        {
          # Static library for linking with Chrome.
          'target_name': 'npGoogleNaClPluginChrome',
          'type': 'static_library',
          'dependencies': [
            '<(DEPTH)/native_client/src/shared/gio/gio.gyp:gio',
            '<(DEPTH)/native_client/src/shared/imc/imc.gyp:google_nacl_imc_c',
            '<(DEPTH)/native_client/src/shared/npruntime/npruntime.gyp:google_nacl_npruntime',
            '<(DEPTH)/native_client/src/shared/platform/platform.gyp:platform',
            '<(DEPTH)/native_client/src/trusted/desc/desc.gyp:nrd_xfer',
            '<(DEPTH)/native_client/src/trusted/expiration/expiration.gyp:expiration',
            '<(DEPTH)/native_client/src/trusted/nonnacl_util/nonnacl_util.gyp:nonnacl_util_chrome',
            '<(DEPTH)/native_client/src/trusted/gio/gio_wrapped_desc.gyp:gio_wrapped_desc',
            '<(DEPTH)/third_party/npapi/npapi.gyp:npapi',
          ],
          'sources': [
            '<@(common_sources)',
            '<@(npapi_sources)',
            'nacl_entry_points.cc',
            'npapi/video_chrome.cc',
          ],
        },
        {
          'target_name': 'ppGoogleNaClPluginChrome',
          'type': 'loadable_module',
          'sources': [
            '<@(common_sources)',
            '<@(ppapi_sources)',
            'browser_utils.cc',
          ],
          'defines': [
            'NACL_PPAPI',
          ],
          'dependencies': [
            '<(DEPTH)/native_client/src/shared/gio/gio.gyp:gio',
            '<(DEPTH)/native_client/src/shared/imc/imc.gyp:google_nacl_imc_c',
            '<(DEPTH)/native_client/src/shared/platform/platform.gyp:platform',
            '<(DEPTH)/native_client/src/shared/ppapi/ppapi.gyp:ppapi_cpp',
            '<(DEPTH)/native_client/src/shared/ppapi_proxy/ppapi_proxy.gyp:nacl_ppapi_browser',
            '<(DEPTH)/native_client/src/shared/srpc/srpc.gyp:nonnacl_srpc',
            '<(DEPTH)/native_client/src/trusted/desc/desc.gyp:nrd_xfer',
            '<(DEPTH)/native_client/src/trusted/expiration/expiration.gyp:expiration',
            '<(DEPTH)/native_client/src/trusted/handle_pass/handle_pass.gyp:browserhandle',
            '<(DEPTH)/native_client/src/trusted/handle_pass/handle_pass.gyp:handle_lookup',
            '<(DEPTH)/native_client/src/trusted/handle_pass/handle_pass.gyp:ldrhandle',
            '<(DEPTH)/native_client/src/trusted/nonnacl_util/nonnacl_util.gyp:nonnacl_util',
            '<(DEPTH)/native_client/src/trusted/platform_qualify/platform_qualify.gyp:platform_qual_lib',
            '<(DEPTH)/native_client/src/trusted/gio/gio_wrapped_desc.gyp:gio_wrapped_desc',
          ],
          'conditions': [
            ['OS=="mac"', {
              'mac_bundle': 1,
              'product_name': 'ppGoogleNaClPluginChrome',
              'product_extension': 'plugin',
            }],
          ],
        },
      ],
    }],
    ['nacl_standalone==0 and OS=="win"', {
      'targets': [
        {
          'target_name': 'npGoogleNaClPluginChrome64',
          'type': 'static_library',
          'variables': {
            'win_target': 'x64',
          },
          'sources': [
            '<@(common_sources)',
            '<@(npapi_sources)',
            'nacl_entry_points.cc',
            'npapi/video_chrome.cc',
          ],
          'dependencies': [
            '<(DEPTH)/native_client/src/shared/gio/gio.gyp:gio64',
            '<(DEPTH)/native_client/src/shared/imc/imc.gyp:google_nacl_imc_c64',
            '<(DEPTH)/native_client/src/shared/npruntime/npruntime.gyp:google_nacl_npruntime64',
            '<(DEPTH)/native_client/src/shared/platform/platform.gyp:platform64',
            '<(DEPTH)/native_client/src/trusted/desc/desc.gyp:nrd_xfer64',
            '<(DEPTH)/native_client/src/trusted/expiration/expiration.gyp:expiration64',
            '<(DEPTH)/native_client/src/trusted/nonnacl_util/nonnacl_util.gyp:nonnacl_util_chrome64',
            '<(DEPTH)/native_client/src/trusted/gio/gio_wrapped_desc.gyp:gio_wrapped_desc64',
            '<(DEPTH)/third_party/npapi/npapi.gyp:npapi',
          ],
          'configurations': {
            'Common_Base': {
              'msvs_target_platform': 'x64',
            },
          },
        }
      ],
    }],
  ],
}
<commit_msg>Add a condition to browser handle libraries to only be included for windows. This is to support window's flavor of process handle passing.<commit_after># Copyright (c) 2009 The Native Client Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

{
  'includes': [
    'plugin.gypi',
  ],
  'target_defaults': {
    'variables': {
      'target_base': 'none',
    },
    'target_conditions': [
      ['target_base=="npNaClPlugin"', {
        'sources': [
          '<@(common_sources)',
          '<@(npapi_sources)',
          'npapi/video.cc',
        ],
        'xcode_settings': {
          'WARNING_CFLAGS!': [
            # TODO(bradnelson): remove -pedantic when --std=c++98 in common.gypi
            '-pedantic',
          ],
          'WARNING_CFLAGS': [
            '-Wno-deprecated',
            '-Wno-deprecated-declarations',
          ],
        },
        'conditions': [
          ['OS=="win"', {
            'sources': [
              'win/nacl_plugin.rc',
            ],
            'msvs_settings': {
              'VCCLCompilerTool': {
                'ExceptionHandling': '2',  # /EHsc
              },
              'VCLinkerTool': {
                'AdditionalLibraryDirectories': [
                   '$(OutDir)/lib',
                ],
              },
            },
          }],
        ],
      }],
      ['target_base=="ppNaClPlugin"', {
        'sources': [
          '<@(common_sources)',
          '<@(ppapi_sources)',
        ],
        'defines': [
          'NACL_PPAPI',
        ],
        'xcode_settings': {
          'WARNING_CFLAGS!': [
            # TODO(bradnelson): remove -pedantic when --std=c++98 in common.gypi
            '-pedantic',
          ],
          'WARNING_CFLAGS': [
            '-Wno-deprecated',
            '-Wno-deprecated-declarations',
          ],
        },
        'conditions': [
          ['OS=="win"', {
            'sources': [
              'win/nacl_plugin.rc',
            ],
            'msvs_settings': {
              'VCCLCompilerTool': {
                'ExceptionHandling': '2',  # /EHsc
              },
              'VCLinkerTool': {
                'AdditionalLibraryDirectories': [
                   '$(OutDir)/lib',
                ],
              },
            },
          }],
        ],
      }],
    ],
  },
  'targets': [],
  'conditions': [
    ['target_arch!="x64"', {
      'targets': [
      # ----------------------------------------------------------------------
        {
          'target_name': 'npGoogleNaClPlugin',
          'type': 'shared_library',
          'variables': {
            'target_base': 'npNaClPlugin',
          },
          'dependencies': [
            '<(DEPTH)/native_client/src/shared/gio/gio.gyp:gio',
            '<(DEPTH)/native_client/src/shared/imc/imc.gyp:google_nacl_imc_c',
            '<(DEPTH)/native_client/src/shared/npruntime/npruntime.gyp:google_nacl_npruntime',
            '<(DEPTH)/native_client/src/shared/platform/platform.gyp:platform',
            '<(DEPTH)/native_client/src/shared/srpc/srpc.gyp:nonnacl_srpc',
            '<(DEPTH)/native_client/src/trusted/desc/desc.gyp:nrd_xfer',
            '<(DEPTH)/native_client/src/trusted/expiration/expiration.gyp:expiration',
            '<(DEPTH)/native_client/src/trusted/nonnacl_util/nonnacl_util.gyp:nonnacl_util',
            '<(DEPTH)/native_client/src/trusted/platform_qualify/platform_qualify.gyp:platform_qual_lib',
            '<(DEPTH)/native_client/src/trusted/gio/gio_wrapped_desc.gyp:gio_wrapped_desc',
          ],
          'conditions': [
            ['OS=="win"', {
              'sources': [
                'win/nacl_plugin.def',
              ],
            }],
          ],
        },
        {
          'target_name': 'ppGoogleNaClPlugin',
          'type': 'shared_library',
          'variables': {
            'target_base': 'ppNaClPlugin',
          },
          'dependencies': [
            '<(DEPTH)/native_client/src/shared/gio/gio.gyp:gio',
            '<(DEPTH)/native_client/src/shared/imc/imc.gyp:google_nacl_imc_c',
            '<(DEPTH)/native_client/src/shared/platform/platform.gyp:platform',
            '<(DEPTH)/native_client/src/shared/ppapi/ppapi.gyp:ppapi_cpp',
            '<(DEPTH)/native_client/src/shared/ppapi_proxy/ppapi_proxy.gyp:nacl_ppapi_browser',
            '<(DEPTH)/native_client/src/shared/srpc/srpc.gyp:nonnacl_srpc',
            '<(DEPTH)/native_client/src/shared/srpc/srpc.gyp:nonnacl_srpc',
            '<(DEPTH)/native_client/src/trusted/desc/desc.gyp:nrd_xfer',
            '<(DEPTH)/native_client/src/trusted/expiration/expiration.gyp:expiration',
            '<(DEPTH)/native_client/src/trusted/nonnacl_util/nonnacl_util.gyp:nonnacl_util',
            '<(DEPTH)/native_client/src/trusted/platform_qualify/platform_qualify.gyp:platform_qual_lib',
            '<(DEPTH)/native_client/src/trusted/gio/gio_wrapped_desc.gyp:gio_wrapped_desc',
          ],
# TODO(noelallen) We will need to put this back in with a new .def file once we need to export symbols
# to support the plugin as a sandboxed DLL.
#          'conditions': [
#            ['OS=="win"', {
#              'sources': [
#                'win/nacl_plugin.def',
#              ],
#            }],
#          ],
        },
      ],
    }],
    ['OS=="win"', {
      'targets': [
        # ---------------------------------------------------------------------
        {
          'target_name': 'npGoogleNaClPlugin64',
          'type': 'shared_library',
          'variables': {
            'target_base': 'npNaClPlugin',
            'win_target': 'x64',
          },
          'sources': [
            'win/nacl_plugin64.def',
          ],
          'dependencies': [
            '<(DEPTH)/native_client/src/shared/gio/gio.gyp:gio64',
            '<(DEPTH)/native_client/src/shared/imc/imc.gyp:google_nacl_imc_c64',
            '<(DEPTH)/native_client/src/shared/npruntime/npruntime.gyp:google_nacl_npruntime64',
            '<(DEPTH)/native_client/src/shared/platform/platform.gyp:platform64',
            '<(DEPTH)/native_client/src/shared/srpc/srpc.gyp:nonnacl_srpc64',
            '<(DEPTH)/native_client/src/trusted/desc/desc.gyp:nrd_xfer64',
            '<(DEPTH)/native_client/src/trusted/expiration/expiration.gyp:expiration64',
            '<(DEPTH)/native_client/src/trusted/nonnacl_util/nonnacl_util.gyp:nonnacl_util64',
            '<(DEPTH)/native_client/src/trusted/platform_qualify/platform_qualify.gyp:platform_qual_lib64',
            '<(DEPTH)/native_client/src/trusted/gio/gio_wrapped_desc.gyp:gio_wrapped_desc64',
          ],
          'configurations': {
            'Common_Base': {
              'msvs_target_platform': 'x64',
            },
          },
        },
      ],
    }],
    ['nacl_standalone==0', {
      'targets': [
        {
          # Static library for linking with Chrome.
          'target_name': 'npGoogleNaClPluginChrome',
          'type': 'static_library',
          'dependencies': [
            '<(DEPTH)/native_client/src/shared/gio/gio.gyp:gio',
            '<(DEPTH)/native_client/src/shared/imc/imc.gyp:google_nacl_imc_c',
            '<(DEPTH)/native_client/src/shared/npruntime/npruntime.gyp:google_nacl_npruntime',
            '<(DEPTH)/native_client/src/shared/platform/platform.gyp:platform',
            '<(DEPTH)/native_client/src/trusted/desc/desc.gyp:nrd_xfer',
            '<(DEPTH)/native_client/src/trusted/expiration/expiration.gyp:expiration',
            '<(DEPTH)/native_client/src/trusted/nonnacl_util/nonnacl_util.gyp:nonnacl_util_chrome',
            '<(DEPTH)/native_client/src/trusted/gio/gio_wrapped_desc.gyp:gio_wrapped_desc',
            '<(DEPTH)/third_party/npapi/npapi.gyp:npapi',
          ],
          'sources': [
            '<@(common_sources)',
            '<@(npapi_sources)',
            'nacl_entry_points.cc',
            'npapi/video_chrome.cc',
          ],
        },
        {
          'target_name': 'ppGoogleNaClPluginChrome',
          'type': 'loadable_module',
          'sources': [
            '<@(common_sources)',
            '<@(ppapi_sources)',
            'browser_utils.cc',
          ],
          'defines': [
            'NACL_PPAPI',
          ],
          'dependencies': [
            '<(DEPTH)/native_client/src/shared/gio/gio.gyp:gio',
            '<(DEPTH)/native_client/src/shared/imc/imc.gyp:google_nacl_imc_c',
            '<(DEPTH)/native_client/src/shared/platform/platform.gyp:platform',
            '<(DEPTH)/native_client/src/shared/ppapi/ppapi.gyp:ppapi_cpp',
            '<(DEPTH)/native_client/src/shared/ppapi_proxy/ppapi_proxy.gyp:nacl_ppapi_browser',
            '<(DEPTH)/native_client/src/shared/srpc/srpc.gyp:nonnacl_srpc',
            '<(DEPTH)/native_client/src/trusted/desc/desc.gyp:nrd_xfer',
            '<(DEPTH)/native_client/src/trusted/expiration/expiration.gyp:expiration',
            '<(DEPTH)/native_client/src/trusted/nonnacl_util/nonnacl_util.gyp:nonnacl_util',
            '<(DEPTH)/native_client/src/trusted/platform_qualify/platform_qualify.gyp:platform_qual_lib',
            '<(DEPTH)/native_client/src/trusted/gio/gio_wrapped_desc.gyp:gio_wrapped_desc',
          ],
          'conditions': [
            ['OS=="mac"', {
              'mac_bundle': 1,
              'product_name': 'ppGoogleNaClPluginChrome',
              'product_extension': 'plugin',
            }],
            ['OS=="win"', {
              'dependencies': [
                '<(DEPTH)/native_client/src/trusted/handle_pass/handle_pass.gyp:browserhandle',
                '<(DEPTH)/native_client/src/trusted/handle_pass/handle_pass.gyp:handle_lookup',
                '<(DEPTH)/native_client/src/trusted/handle_pass/handle_pass.gyp:ldrhandle',
              ],
            }],
          ],
        },
      ],
    }],
    ['nacl_standalone==0 and OS=="win"', {
      'targets': [
        {
          'target_name': 'npGoogleNaClPluginChrome64',
          'type': 'static_library',
          'variables': {
            'win_target': 'x64',
          },
          'sources': [
            '<@(common_sources)',
            '<@(npapi_sources)',
            'nacl_entry_points.cc',
            'npapi/video_chrome.cc',
          ],
          'dependencies': [
            '<(DEPTH)/native_client/src/shared/gio/gio.gyp:gio64',
            '<(DEPTH)/native_client/src/shared/imc/imc.gyp:google_nacl_imc_c64',
            '<(DEPTH)/native_client/src/shared/npruntime/npruntime.gyp:google_nacl_npruntime64',
            '<(DEPTH)/native_client/src/shared/platform/platform.gyp:platform64',
            '<(DEPTH)/native_client/src/trusted/desc/desc.gyp:nrd_xfer64',
            '<(DEPTH)/native_client/src/trusted/expiration/expiration.gyp:expiration64',
            '<(DEPTH)/native_client/src/trusted/nonnacl_util/nonnacl_util.gyp:nonnacl_util_chrome64',
            '<(DEPTH)/native_client/src/trusted/gio/gio_wrapped_desc.gyp:gio_wrapped_desc64',
            '<(DEPTH)/third_party/npapi/npapi.gyp:npapi',
          ],
          'configurations': {
            'Common_Base': {
              'msvs_target_platform': 'x64',
            },
          },
        }
      ],
    }],
  ],
}
<|endoftext|>


<commit_before>import logging
import shlex
import shutil
import subprocess
import tempfile
import textwrap
import filecmp
import vcf
from contextlib import closing
from unittest import TestCase, skip
from urlparse import urlparse
from uuid import uuid4

import os
import posixpath
from bd2k.util.iterables import concat
from boto.s3.connection import S3Connection, Bucket
from boto.s3.key import Key
from toil_vg.iostore import IOStore

log = logging.getLogger(__name__)

class VGCGLTest(TestCase):
    """
    These tests *can* be parameterized with the following optional environment variables:

    TOIL_SCRIPTS_TEST_TOIL_OPTIONS - a space-separated list of additional command line arguments to pass to Toil via
    the script entry point. Default is the empty string.

    TOIL_SCRIPTS_TEST_JOBSTORE - the job store locator to use for the tests. The default is a file: locator pointing
    at a local temporary directory.

    TOIL_SCRIPTS_TEST_NUM_SAMPLES - the number of sample lines to generate in the input manifest
    """

    @classmethod
    def setUpClass(cls):
        super(VGCGLTest, cls).setUpClass()
        # FIXME: pull up into common base class
        logging.basicConfig(level=logging.INFO)

    def _download_input(self, filename, local_path = None):
        tgt = os.path.join(self.workdir, filename) if local_path is None else local_path
        with open(tgt, 'w') as f:
            self.bucket.get_key('/vg_cgl/ci/{}'.format(filename)).get_contents_to_file(f)

    def setUp(self):
        self.workdir = tempfile.mkdtemp()
        self.jobStoreAWS = 'aws:us-west-2:testvg-{}'.format(uuid4())
        self.jobStoreLocal = '{}/local-testvg-{}'.format(self.workdir, uuid4())
        self.connection = S3Connection()
        self.bucket = self.connection.get_bucket('cgl-pipeline-inputs')
        self.base_command = concat('toil-vg', 'run',
                                   '--realTimeLogging', '--logInfo', '--reads_per_chunk', '8000',
                                   '--call_chunk_size', '20000',
                                   '--index_mode', 'gcsa-mem', '--gcsa_index_cores', '7', '--kmers_cores', '7',
                                   '--alignment_cores', '4',
                                   '--calling_cores', '4', '--vcfeval_cores', '4')

        # default output store
        self.outstore = 'aws:us-west-2:toilvg-jenkinstest-outstore-{}'.format(uuid4())
        self.local_outstore = os.path.join(self.workdir, 'toilvg-jenkinstest-outstore-{}'.format(uuid4()))

    def test_1_sim_small(self):
        '''
        This test uses simulated reads from the small dataset from vg, created as follows:
        vg construct -r test/small/x.fa -v test/small/x.vcf.gz > small.vg
        vg index -x small.xg small.vg
        vg sim -x small.xg  -l 100 -n 10000 -s 0 -e 0.001 -i 0.0001 > small_sim_reads
        # Reads converted to small_sim_reads.fq with script setting qualities to B
        (small.vcf.gz and small.fa.gz below are just x.vcf.gz and x.fa from input)
        '''
        self.sample_reads = 's3://cgl-pipeline-inputs/vg_cgl/ci/small_sim_reads.fq.gz'
        self.test_vg_graph = 's3://cgl-pipeline-inputs/vg_cgl/ci/small.vg'
        self.baseline = 's3://cgl-pipeline-inputs/vg_cgl/ci/small.vcf.gz'
        self.chrom_fa = 's3://cgl-pipeline-inputs/vg_cgl/ci/small.fa.gz'

        self._run(self.base_command, self.jobStoreLocal, 'sample',
                  self.local_outstore,  '--fastq', self.sample_reads,
                  '--graphs', self.test_vg_graph,
                  '--chroms', 'x', '--vcfeval_baseline', self.baseline,
                  '--vcfeval_fasta', self.chrom_fa, '--vcfeval_opts', ' --squash-ploidy')

        self._assertOutput('sample', self.local_outstore, f1_threshold=0.95)

    def test_2_sim_small_standalone(self):
        '''
        Same as above, but chain standalone tools instead of toil-vg run
        '''
        self.sample_reads = 's3://cgl-pipeline-inputs/vg_cgl/ci/small_sim_reads.fq.gz'
        self.test_vg_graph = 's3://cgl-pipeline-inputs/vg_cgl/ci/small.vg'
        self.baseline = 's3://cgl-pipeline-inputs/vg_cgl/ci/small.vcf.gz'
        self.chrom_fa = 's3://cgl-pipeline-inputs/vg_cgl/ci/small.fa.gz'

        self._run('toil-vg', 'index', self.jobStoreLocal, self.local_outstore,
                  '--graphs', self.test_vg_graph, '--chroms', 'x',
                   '--gcsa_index_cores', '7', '--kmers_cores', '7',
                  '--realTimeLogging', '--logInfo', '--index_name', 'small')

        self._run('toil-vg', 'map', self.jobStoreLocal, 'sample',
                  os.path.join(self.local_outstore, 'small.xg'),
                  os.path.join(self.local_outstore, 'small.gcsa'),
                  self.local_outstore,  '--fastq', self.sample_reads,
                  '--id_ranges', os.path.join(self.local_outstore, 'small_id_ranges.tsv'),
                  '--alignment_cores', '7', '--reads_per_chunk', '8000',
                  '--realTimeLogging', '--logInfo')

        self._run('toil-vg', 'call', self.jobStoreLocal,
                  os.path.join(self.local_outstore, 'small.xg'), 'sample',
                  self.local_outstore, '--gams', os.path.join(self.local_outstore, 'sample_x.gam'),
                  '--chroms', 'x', '--call_chunk_size', '20000', '--calling_cores', '4',
                  '--realTimeLogging', '--logInfo')

        self._run('toil-vg', 'vcfeval', self.jobStoreLocal,
                  os.path.join(self.local_outstore, 'sample.vcf.gz'), self.baseline,
                  self.chrom_fa, self.local_outstore,
                  '--vcfeval_opts', ' --squash-ploidy',
                  '--realTimeLogging', '--logInfo')

        self._assertOutput(None, self.local_outstore, f1_threshold=0.95)

    def test_3_BRCA1_NA12877(self):
        ''' Test sample BRCA1 output, graph construction and use, and local file processing
        '''
        self._download_input('NA12877.brca1.bam_1.fq.gz')
        self._download_input('NA12877.brca1.bam_2.fq.gz')
        self._download_input('snp1kg-brca1.vg')
        self._download_input('platinum_NA12877_BRCA1.vcf.gz.tbi')
        self._download_input('platinum_NA12877_BRCA1.vcf.gz')
        self._download_input('BRCA1.fa.gz')

        self.sample_reads = os.path.join(self.workdir, 'NA12877.brca1.bam_1.fq.gz')
        self.sample_reads2 = os.path.join(self.workdir, 'NA12877.brca1.bam_2.fq.gz')
        self.test_vg_graph = os.path.join(self.workdir, 'snp1kg-brca1.vg')
        self.baseline = os.path.join(self.workdir, 'platinum_NA12877_BRCA1.vcf.gz')
        self.chrom_fa = os.path.join(self.workdir, 'BRCA1.fa.gz')

        self._run(self.base_command, self.jobStoreLocal, 'NA12877',
                  self.local_outstore, '--fastq', self.sample_reads, self.sample_reads2, '--graphs',
                  self.test_vg_graph, '--chroms', '17',
                  '--call_opts', '--offset 43044293',
                  '--vcfeval_baseline', self.baseline, '--vcfeval_fasta', self.chrom_fa)

        self._assertOutput('NA12877', self.local_outstore, f1_threshold=0.48)

    def _run(self, *args):
        args = list(concat(*args))
        log.info('Running %r', args)
        subprocess.check_call(args)

    def _assertOutput(self, sample_name, outstore, f1_threshold=0.90):

        # grab the f1.txt file
        local_f1 = os.path.join(self.workdir, 'f1.txt')
        io_store = IOStore.get(outstore)
        f1_path = 'vcfeval_output_f1.txt'
        if sample_name:
            f1_path = sample_name + '_' + f1_path
        io_store.read_input_file(f1_path, local_f1)

        with open(local_f1) as f1_file:
            f1_score = float(f1_file.readline().strip())
        self.assertTrue(f1_score >= f1_threshold)

    def tearDown(self):
        shutil.rmtree(self.workdir)
        subprocess.check_call(['toil', 'clean', self.jobStoreAWS])
        subprocess.check_call(['toil', 'clean', self.jobStoreLocal])

<commit_msg>lower f1 bar even more<commit_after>import logging
import shlex
import shutil
import subprocess
import tempfile
import textwrap
import filecmp
import vcf
from contextlib import closing
from unittest import TestCase, skip
from urlparse import urlparse
from uuid import uuid4

import os
import posixpath
from bd2k.util.iterables import concat
from boto.s3.connection import S3Connection, Bucket
from boto.s3.key import Key
from toil_vg.iostore import IOStore

log = logging.getLogger(__name__)

class VGCGLTest(TestCase):
    """
    These tests *can* be parameterized with the following optional environment variables:

    TOIL_SCRIPTS_TEST_TOIL_OPTIONS - a space-separated list of additional command line arguments to pass to Toil via
    the script entry point. Default is the empty string.

    TOIL_SCRIPTS_TEST_JOBSTORE - the job store locator to use for the tests. The default is a file: locator pointing
    at a local temporary directory.

    TOIL_SCRIPTS_TEST_NUM_SAMPLES - the number of sample lines to generate in the input manifest
    """

    @classmethod
    def setUpClass(cls):
        super(VGCGLTest, cls).setUpClass()
        # FIXME: pull up into common base class
        logging.basicConfig(level=logging.INFO)

    def _download_input(self, filename, local_path = None):
        tgt = os.path.join(self.workdir, filename) if local_path is None else local_path
        with open(tgt, 'w') as f:
            self.bucket.get_key('/vg_cgl/ci/{}'.format(filename)).get_contents_to_file(f)

    def setUp(self):
        self.workdir = tempfile.mkdtemp()
        self.jobStoreAWS = 'aws:us-west-2:testvg-{}'.format(uuid4())
        self.jobStoreLocal = '{}/local-testvg-{}'.format(self.workdir, uuid4())
        self.connection = S3Connection()
        self.bucket = self.connection.get_bucket('cgl-pipeline-inputs')
        self.base_command = concat('toil-vg', 'run',
                                   '--realTimeLogging', '--logInfo', '--reads_per_chunk', '8000',
                                   '--call_chunk_size', '20000',
                                   '--index_mode', 'gcsa-mem', '--gcsa_index_cores', '7', '--kmers_cores', '7',
                                   '--alignment_cores', '4',
                                   '--calling_cores', '4', '--vcfeval_cores', '4')

        # default output store
        self.outstore = 'aws:us-west-2:toilvg-jenkinstest-outstore-{}'.format(uuid4())
        self.local_outstore = os.path.join(self.workdir, 'toilvg-jenkinstest-outstore-{}'.format(uuid4()))

    def test_1_sim_small(self):
        '''
        This test uses simulated reads from the small dataset from vg, created as follows:
        vg construct -r test/small/x.fa -v test/small/x.vcf.gz > small.vg
        vg index -x small.xg small.vg
        vg sim -x small.xg  -l 100 -n 10000 -s 0 -e 0.001 -i 0.0001 > small_sim_reads
        # Reads converted to small_sim_reads.fq with script setting qualities to B
        (small.vcf.gz and small.fa.gz below are just x.vcf.gz and x.fa from input)
        '''
        self.sample_reads = 's3://cgl-pipeline-inputs/vg_cgl/ci/small_sim_reads.fq.gz'
        self.test_vg_graph = 's3://cgl-pipeline-inputs/vg_cgl/ci/small.vg'
        self.baseline = 's3://cgl-pipeline-inputs/vg_cgl/ci/small.vcf.gz'
        self.chrom_fa = 's3://cgl-pipeline-inputs/vg_cgl/ci/small.fa.gz'

        self._run(self.base_command, self.jobStoreLocal, 'sample',
                  self.local_outstore,  '--fastq', self.sample_reads,
                  '--graphs', self.test_vg_graph,
                  '--chroms', 'x', '--vcfeval_baseline', self.baseline,
                  '--vcfeval_fasta', self.chrom_fa, '--vcfeval_opts', ' --squash-ploidy')

        self._assertOutput('sample', self.local_outstore, f1_threshold=0.95)

    def test_2_sim_small_standalone(self):
        '''
        Same as above, but chain standalone tools instead of toil-vg run
        '''
        self.sample_reads = 's3://cgl-pipeline-inputs/vg_cgl/ci/small_sim_reads.fq.gz'
        self.test_vg_graph = 's3://cgl-pipeline-inputs/vg_cgl/ci/small.vg'
        self.baseline = 's3://cgl-pipeline-inputs/vg_cgl/ci/small.vcf.gz'
        self.chrom_fa = 's3://cgl-pipeline-inputs/vg_cgl/ci/small.fa.gz'

        self._run('toil-vg', 'index', self.jobStoreLocal, self.local_outstore,
                  '--graphs', self.test_vg_graph, '--chroms', 'x',
                   '--gcsa_index_cores', '7', '--kmers_cores', '7',
                  '--realTimeLogging', '--logInfo', '--index_name', 'small')

        self._run('toil-vg', 'map', self.jobStoreLocal, 'sample',
                  os.path.join(self.local_outstore, 'small.xg'),
                  os.path.join(self.local_outstore, 'small.gcsa'),
                  self.local_outstore,  '--fastq', self.sample_reads,
                  '--id_ranges', os.path.join(self.local_outstore, 'small_id_ranges.tsv'),
                  '--alignment_cores', '7', '--reads_per_chunk', '8000',
                  '--realTimeLogging', '--logInfo')

        self._run('toil-vg', 'call', self.jobStoreLocal,
                  os.path.join(self.local_outstore, 'small.xg'), 'sample',
                  self.local_outstore, '--gams', os.path.join(self.local_outstore, 'sample_x.gam'),
                  '--chroms', 'x', '--call_chunk_size', '20000', '--calling_cores', '4',
                  '--realTimeLogging', '--logInfo')

        self._run('toil-vg', 'vcfeval', self.jobStoreLocal,
                  os.path.join(self.local_outstore, 'sample.vcf.gz'), self.baseline,
                  self.chrom_fa, self.local_outstore,
                  '--vcfeval_opts', ' --squash-ploidy',
                  '--realTimeLogging', '--logInfo')

        self._assertOutput(None, self.local_outstore, f1_threshold=0.95)

    def test_3_BRCA1_NA12877(self):
        ''' Test sample BRCA1 output, graph construction and use, and local file processing
        '''
        self._download_input('NA12877.brca1.bam_1.fq.gz')
        self._download_input('NA12877.brca1.bam_2.fq.gz')
        self._download_input('snp1kg-brca1.vg')
        self._download_input('platinum_NA12877_BRCA1.vcf.gz.tbi')
        self._download_input('platinum_NA12877_BRCA1.vcf.gz')
        self._download_input('BRCA1.fa.gz')

        self.sample_reads = os.path.join(self.workdir, 'NA12877.brca1.bam_1.fq.gz')
        self.sample_reads2 = os.path.join(self.workdir, 'NA12877.brca1.bam_2.fq.gz')
        self.test_vg_graph = os.path.join(self.workdir, 'snp1kg-brca1.vg')
        self.baseline = os.path.join(self.workdir, 'platinum_NA12877_BRCA1.vcf.gz')
        self.chrom_fa = os.path.join(self.workdir, 'BRCA1.fa.gz')

        self._run(self.base_command, self.jobStoreLocal, 'NA12877',
                  self.local_outstore, '--fastq', self.sample_reads, self.sample_reads2, '--graphs',
                  self.test_vg_graph, '--chroms', '17',
                  '--call_opts', '--offset 43044293',
                  '--vcfeval_baseline', self.baseline, '--vcfeval_fasta', self.chrom_fa)

        self._assertOutput('NA12877', self.local_outstore, f1_threshold=0.45)

    def _run(self, *args):
        args = list(concat(*args))
        log.info('Running %r', args)
        subprocess.check_call(args)

    def _assertOutput(self, sample_name, outstore, f1_threshold=0.90):

        # grab the f1.txt file
        local_f1 = os.path.join(self.workdir, 'f1.txt')
        io_store = IOStore.get(outstore)
        f1_path = 'vcfeval_output_f1.txt'
        if sample_name:
            f1_path = sample_name + '_' + f1_path
        io_store.read_input_file(f1_path, local_f1)

        with open(local_f1) as f1_file:
            f1_score = float(f1_file.readline().strip())
        print f1_score
        self.assertTrue(f1_score >= f1_threshold)

    def tearDown(self):
        shutil.rmtree(self.workdir)
        subprocess.check_call(['toil', 'clean', self.jobStoreAWS])
        subprocess.check_call(['toil', 'clean', self.jobStoreLocal])

<|endoftext|>