<commit_before>#!/usr/bin/env python

import argparse
import re
import sys
import os
import shutil
from subprocess import call

def to_lower_case(string):
    string = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', string)
    return re.sub('([a-z0-9])([A-Z])', r'\1_\2', string).lower()

def read_proto(path):
    package = None
    imports = []
    content = []
    with open(path, 'r') as file:
        is_header = True
        for line in file.readlines():
            line = line.rstrip()
            if is_header:
                if line.startswith('package'):
                    match = re.match(r'package (\w+(?:\.\w+)*);', line)
                    package = match.group(1)
                elif line.startswith('import'):
                    match = re.match(
                        r'import (public )?"(\w+(?:\/\w+)*)\.proto";', line)
                    import_path = match.group(2)
                    import_is_public = False if match.group(
                        1) is None else True
                    imports.append((import_path, import_is_public))
                elif line.startswith('message') or line.startswith('enum'):
                    is_header = False
            if not is_header and line:
                content.append(line)
        content.append('')
    return (package, imports, content)

def read_protos(path):
    protos = dict()
    for folder_path, _, file_names in os.walk(path):
        proto_folder = dict()
        for file_name in file_names:
            if file_name.endswith('.proto'):
                file_path = os.path.join(folder_path, file_name)
                key, _ = os.path.splitext(file_name)
                proto_folder[key] = read_proto(file_path)
        if proto_folder:
            key = '/'.join(
                os.path.normpath(
                    os.path.relpath(
                        folder_path,
                        path)).split(
                    os.sep))
            protos[key] = proto_folder
    return protos

def write_proto(path, name, proto):
    package, imports, content = proto

    if not os.path.exists(path):
        os.makedirs(path)

    proto_file = os.path.abspath(os.path.join(path, name + '.proto'))

    with open(proto_file, 'w') as file:
        file.write('syntax = "proto3";\n\n')
        file.write('package ')
        file.write(package)
        file.write(';\n\n')

        options = False

        if args.language == 'java' or args.language == 'javanano':
            options = True
            package_split = package.split('.')
            java_package = '.'.join(package_split[:-1])
            file.write('option java_package = "')
            file.write(java_package)
            file.write('";\n')
            java_outer_classname = name
            file.write('option java_outer_classname = "')
            file.write(java_outer_classname)
            file.write('";\n')
            if args.java_generate_equals_and_hash:
                file.write('option java_generate_equals_and_hash = true;\n')

        if args.language == 'cpp' and args.cc_enable_arenas:
            options = True
            file.write('option cc_enable_arenas = true;\n')

        if args.language == 'objc':
            options = True
            file.write('option objc_class_prefix = "GPB";\n')

        if args.language == 'go':
            options = True
            go_package = package.split('.')[-1]
            file.write('option go_package = "')
            file.write(go_package)
            file.write('";\n')

        if options:
            file.write('\n')

        for import_path, import_is_public in imports:
            file.write('import "')
            if import_is_public:
                file.write('public ')
            file.write(import_path)
            file.write('.proto";\n')

        if imports:
            file.write('\n')

        for line in content:
            file.write(line)
            file.write('\n')

    return proto_file

def write_protos(path, protos):
    proto_files = []
    for proto_path in protos:
        proto_files_folder = []
        proto_folder = protos[proto_path]
        proto_path = os.path.join(*proto_path.split('/'))
        for proto_file_name in proto_folder:
            proto = proto_folder[proto_file_name]
            proto_files_folder.append(
                write_proto(
                    os.path.join(
                        path,
                        proto_path),
                    proto_file_name,
                    proto))
        proto_files.append(proto_files_folder)
    return proto_files

def merge_protos(protos):
    new_protos = dict()
    for proto_path in protos:
        proto_folder = list(protos[proto_path].values())

        package, _, _ = proto_folder[0]
        new_package = package
        new_imports_path = []
        new_imports_is_public = []
        new_content = []
        for proto in proto_folder:
            package, imports, content = proto
            if package != new_package:
                raise Exception(
                    'All .proto files in "' +
                    proto_path +
                    '" must be in the same package.')
            for import_path, import_is_public in imports:
                if import_path.startswith('POGOProtos'):
                    import_path = '/'.join(import_path.split('/')[:-1])
                if import_path in new_imports_path:
                    if import_is_public:
                        new_imports_is_public[
                            new_imports_path.index(import_path)] = True
                elif import_path != proto_path:
                    new_imports_path.append(import_path)
                    new_imports_is_public.append(import_is_public)
            new_content.extend(content)
        new_imports = []
        for i in range(len(new_imports_path)):
            import_path = new_imports_path[i]
            import_is_public = new_imports_is_public[i]
            new_imports.append((import_path, import_is_public))

        proto_path_split = proto_path.split('/')
        new_proto_path = '/'.join(proto_path_split[:-1])
        new_proto_file_name = proto_path_split[-1]
        if new_proto_path not in new_protos:
            new_protos[new_proto_path] = dict()
        new_protos[new_proto_path][new_proto_file_name] = (
            new_package, new_imports, new_content)

    return new_protos

def format_protos(
        protos,
        path,
        namespace,
        path_lower,
        file_lower,
        package_lower):
    packages = []
    for proto_path in protos:
        proto_folder = protos[proto_path]
        for proto_file_name in proto_folder:
            package, _, _ = proto_folder[proto_file_name]
            packages.append('.'.join(package.split('.')[1:]))
    new_protos = dict()
    for proto_path in protos:
        proto_folder = protos[proto_path]
        new_proto_folder = dict()
        new_proto_path = path + '/' + '/'.join(proto_path.split('/')[1:])
        if path_lower:
            new_proto_path = new_proto_path.lower()
        for proto_file_name in proto_folder:
            new_proto_file_name = to_lower_case(
                proto_file_name) if file_lower else proto_file_name
            package, imports, content = proto_folder[proto_file_name]
            new_package = namespace + '.' + '.'.join(package.split('.')[1:])
            if package_lower:
                new_package = new_package.lower()
            new_imports = []
            for import_path, import_is_public in imports:
                import_path_split = import_path.split('/')
                import_path_path = '/'.join(import_path_split[:-1])
                if import_path.startswith('POGOProtos/'):
                    if import_path_split[1:-1]:
                        import_path_path = path + '/' + \
                            '/'.join(import_path_split[1:-1])
                    else:
                        import_path_path = path
                if path_lower:
                    import_path_path = import_path_path.lower()
                import_path_file = import_path_split[-1]
                if file_lower:
                    import_path_file = to_lower_case(import_path_file)
                import_path = import_path_path + '/' + import_path_file
                new_imports.append((import_path, import_is_public))
            new_content = []

            def sub(match):
                type = match.group(1)
                found_package = ''
                for package in packages:
                    if type.startswith(
                            package + '.') and len(package) > len(found_package):
                        found_package = package
                type = type[len(found_package):]
                found_package = namespace + '.' + found_package
                if package_lower:
                    found_package = found_package.lower()
                return found_package + type
            for line in content:
                new_content.append(
                    re.sub(
                        '\.?POGOProtos.(\w+(?:\.\w+)*)',
                        sub,
                        line))
            new_proto_folder[new_proto_file_name] = (
                new_package, new_imports, new_content)
        new_protos[new_proto_path] = new_proto_folder
    return new_protos

parser = argparse.ArgumentParser()
parser.add_argument(
    'language',
    choices=[
        'cpp',
        'csharp',
        'go',
        'java',
        'javanano',
        'js',
        'objc',
        'python',
        'ruby'],
    help='language to pass to protoc')
parser.add_argument(
    '-p', '--protoc_path',
    default='protoc',
    help='path to protoc')
parser.add_argument('-o', '--out_path', default='out',
                    help='output path for protoc')
parser.add_argument(
    '--java_generate_equals_and_hash',
    action='store_true',
    help='generate Java\'s .equals() and .hashCode()')
parser.add_argument(
    '--cc_enable_arenas',
    action='store_true',
    help='enable C++ arena allocation')
parser.add_argument(
    '--include_imports',
    action='store_true',
    help='include imports in .desc file')
parser.add_argument(
    '--include_source_info',
    action='store_true',
    help='do not strip source code info from .desc file')
args = parser.parse_args()

protoc_path = args.protoc_path
out_path = args.out_path

src_path = os.path.abspath('src')
out_path = os.path.abspath(args.out_path)

if out_path == os.path.abspath('out') and os.path.exists(out_path):
    shutil.rmtree(out_path)

if not os.path.exists(out_path):
    os.makedirs(out_path)

namespace = 'POGOProtos'
path = 'POGOProtos'
merge = False
path_lower = False
file_lower = False
package_lower = False

if args.language == 'js':
    namespace = 'com.github.aeonlucid.pogoprotos'
    path = 'com/github/aeonlucid/pogoprotos'
    merge = False
    path_lower = True
    file_lower = True
    package_lower = True
elif args.language == 'csharp' or args.language == 'cpp' or args.language == 'objc':
    namespace = 'POGOProtos'
    path = 'POGOProtos'
    merge = False
    path_lower = False
    file_lower = False
    package_lower = False
elif args.language == 'go':
    namespace = 'com.github.aeonlucid.pogoprotos'
    path = 'github.com/aeonlucid/pogoprotos'
    merge = False
    path_lower = True
    file_lower = True
    package_lower = True
elif args.language == 'java' or args.language == 'javanano':
    namespace = 'com.github.aeonlucid.pogoprotos'
    path = 'com/github/aeonlucid/pogoprotos'
    merge = True
    path_lower = True
    file_lower = False
    package_lower = True
elif args.language == 'python':
    namespace = 'pogoprotos'
    path = 'pogoprotos'
    merge = False
    path_lower = True
    file_lower = True
    package_lower = True
elif args.language == 'ruby':
    namespace = 'POGOProtos'
    path = 'pogoprotos'
    merge = False
    path_lower = True
    file_lower = True
    package_lower = False

protos = read_protos(src_path)

if merge:
    protos = merge_protos(protos)

protos = format_protos(
    protos,
    path,
    namespace,
    path_lower,
    file_lower,
    package_lower)

if args.language == 'go':
    new_protos = dict()
    for proto_path in protos:
        proto_folder = protos[proto_path]
        new_proto_path = re.sub('/map', r'/maps', proto_path)
        new_proto_folder = dict()
        for proto_file_name in proto_folder:
            package, imports, content = proto_folder[proto_file_name]
            new_package = re.sub('\.map', r'.maps', package)
            new_imports = []
            for import_path, import_is_public in imports:
                new_imports.append(
                    (re.sub(
                        '/map/',
                        r'/maps/',
                        import_path),
                        import_is_public))
            new_content = []
            for line in content:
                new_content.append(re.sub('\.map\.', r'.maps.', line))
            new_proto_folder[proto_file_name] = (
                new_package, new_imports, new_content)
        new_protos[new_proto_path] = new_proto_folder
    protos = new_protos

proto_folders = write_protos(out_path, protos)
proto_files = sum(proto_folders, [])

commands = []

desc_path = os.path.join(out_path, *path.split('/')) + '.desc'
desc_arguments = []
if args.include_imports:
    desc_arguments.append('--include_imports')
if args.include_source_info:
    desc_arguments.append('--include_source_info')

commands.append(
    """"{0}" --proto_path="{1}" --descriptor_set_out="{2}" {3} {4}""".format(
        args.protoc_path,
        out_path,
        desc_path,
        ' '.join(desc_arguments),
        '"' +
        '" "'.join(proto_files) +
        '"'))

arguments = ''
options = ''
all_at_once = True

if args.language == 'js':
    options = 'library=pogoprotos,binary,error_on_name_conflict'
elif args.language == 'csharp':
    arguments = '--csharp_opt=file_extension=.g.cs --csharp_opt=base_namespace'
elif args.language == 'go':
    options = 'plugins=grpc'
    all_at_once = False

if all_at_once:
    proto_folders = [proto_files]

for proto_files in proto_folders:
    commands.append(
        """"{0}" --proto_path="{1}" --{2}_out={3}:"{4}" {5} {6}""".format(
            args.protoc_path,
            out_path,
            args.language,
            options,
            out_path,
            arguments,
            '"' +
            '" "'.join(proto_files) +
            '"'))

for command in commands:
    call(command, shell=(os.name != 'nt'))

if args.language == 'python':
    for path in protos:
        open(
            os.path.join(
                os.path.join(
                    out_path,
                    *path.split('/')),
                '__init__.py'),
            'w').close()
elif args.language == 'ruby':
    for path in protos:
        with open(os.path.join(out_path, *path.split('/')) + '.rb', 'w') as file:
            for name in protos[path]:
                file.write('require "' + path + '/' + name + '"\n')
<commit_msg>Made removing .proto files from compilation output default.<commit_after>#!/usr/bin/env python

import argparse
import glob
import re
import sys
import os
import shutil
from subprocess import call

def to_lower_case(string):
    string = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', string)
    return re.sub('([a-z0-9])([A-Z])', r'\1_\2', string).lower()

def read_proto(path):
    package = None
    imports = []
    content = []
    with open(path, 'r') as file:
        is_header = True
        for line in file.readlines():
            line = line.rstrip()
            if is_header:
                if line.startswith('package'):
                    match = re.match(r'package (\w+(?:\.\w+)*);', line)
                    package = match.group(1)
                elif line.startswith('import'):
                    match = re.match(
                        r'import (public )?"(\w+(?:\/\w+)*)\.proto";', line)
                    import_path = match.group(2)
                    import_is_public = False if match.group(
                        1) is None else True
                    imports.append((import_path, import_is_public))
                elif line.startswith('message') or line.startswith('enum'):
                    is_header = False
            if not is_header and line:
                content.append(line)
        content.append('')
    return (package, imports, content)

def read_protos(path):
    protos = dict()
    for folder_path, _, file_names in os.walk(path):
        proto_folder = dict()
        for file_name in file_names:
            if file_name.endswith('.proto'):
                file_path = os.path.join(folder_path, file_name)
                key, _ = os.path.splitext(file_name)
                proto_folder[key] = read_proto(file_path)
        if proto_folder:
            key = '/'.join(
                os.path.normpath(
                    os.path.relpath(
                        folder_path,
                        path)).split(
                    os.sep))
            protos[key] = proto_folder
    return protos

def write_proto(path, name, proto):
    package, imports, content = proto

    if not os.path.exists(path):
        os.makedirs(path)

    proto_file = os.path.abspath(os.path.join(path, name + '.proto'))

    with open(proto_file, 'w') as file:
        file.write('syntax = "proto3";\n\n')
        file.write('package ')
        file.write(package)
        file.write(';\n\n')

        options = False

        if args.language == 'java' or args.language == 'javanano':
            options = True
            package_split = package.split('.')
            java_package = '.'.join(package_split[:-1])
            file.write('option java_package = "')
            file.write(java_package)
            file.write('";\n')
            java_outer_classname = name
            file.write('option java_outer_classname = "')
            file.write(java_outer_classname)
            file.write('";\n')
            if args.java_generate_equals_and_hash:
                file.write('option java_generate_equals_and_hash = true;\n')

        if args.language == 'cpp' and args.cc_enable_arenas:
            options = True
            file.write('option cc_enable_arenas = true;\n')

        if args.language == 'objc':
            options = True
            file.write('option objc_class_prefix = "GPB";\n')

        if args.language == 'go':
            options = True
            go_package = package.split('.')[-1]
            file.write('option go_package = "')
            file.write(go_package)
            file.write('";\n')

        if options:
            file.write('\n')

        for import_path, import_is_public in imports:
            file.write('import "')
            if import_is_public:
                file.write('public ')
            file.write(import_path)
            file.write('.proto";\n')

        if imports:
            file.write('\n')

        for line in content:
            file.write(line)
            file.write('\n')

    return proto_file

def write_protos(path, protos):
    proto_files = []
    for proto_path in protos:
        proto_files_folder = []
        proto_folder = protos[proto_path]
        proto_path = os.path.join(*proto_path.split('/'))
        for proto_file_name in proto_folder:
            proto = proto_folder[proto_file_name]
            proto_files_folder.append(
                write_proto(
                    os.path.join(
                        path,
                        proto_path),
                    proto_file_name,
                    proto))
        proto_files.append(proto_files_folder)
    return proto_files

def merge_protos(protos):
    new_protos = dict()
    for proto_path in protos:
        proto_folder = list(protos[proto_path].values())

        package, _, _ = proto_folder[0]
        new_package = package
        new_imports_path = []
        new_imports_is_public = []
        new_content = []
        for proto in proto_folder:
            package, imports, content = proto
            if package != new_package:
                raise Exception(
                    'All .proto files in "' +
                    proto_path +
                    '" must be in the same package.')
            for import_path, import_is_public in imports:
                if import_path.startswith('POGOProtos'):
                    import_path = '/'.join(import_path.split('/')[:-1])
                if import_path in new_imports_path:
                    if import_is_public:
                        new_imports_is_public[
                            new_imports_path.index(import_path)] = True
                elif import_path != proto_path:
                    new_imports_path.append(import_path)
                    new_imports_is_public.append(import_is_public)
            new_content.extend(content)
        new_imports = []
        for i in range(len(new_imports_path)):
            import_path = new_imports_path[i]
            import_is_public = new_imports_is_public[i]
            new_imports.append((import_path, import_is_public))

        proto_path_split = proto_path.split('/')
        new_proto_path = '/'.join(proto_path_split[:-1])
        new_proto_file_name = proto_path_split[-1]
        if new_proto_path not in new_protos:
            new_protos[new_proto_path] = dict()
        new_protos[new_proto_path][new_proto_file_name] = (
            new_package, new_imports, new_content)

    return new_protos

def format_protos(
        protos,
        path,
        namespace,
        path_lower,
        file_lower,
        package_lower):
    packages = []
    for proto_path in protos:
        proto_folder = protos[proto_path]
        for proto_file_name in proto_folder:
            package, _, _ = proto_folder[proto_file_name]
            packages.append('.'.join(package.split('.')[1:]))
    new_protos = dict()
    for proto_path in protos:
        proto_folder = protos[proto_path]
        new_proto_folder = dict()
        new_proto_path = path + '/' + '/'.join(proto_path.split('/')[1:])
        if path_lower:
            new_proto_path = new_proto_path.lower()
        for proto_file_name in proto_folder:
            new_proto_file_name = to_lower_case(
                proto_file_name) if file_lower else proto_file_name
            package, imports, content = proto_folder[proto_file_name]
            new_package = namespace + '.' + '.'.join(package.split('.')[1:])
            if package_lower:
                new_package = new_package.lower()
            new_imports = []
            for import_path, import_is_public in imports:
                import_path_split = import_path.split('/')
                import_path_path = '/'.join(import_path_split[:-1])
                if import_path.startswith('POGOProtos/'):
                    if import_path_split[1:-1]:
                        import_path_path = path + '/' + \
                            '/'.join(import_path_split[1:-1])
                    else:
                        import_path_path = path
                if path_lower:
                    import_path_path = import_path_path.lower()
                import_path_file = import_path_split[-1]
                if file_lower:
                    import_path_file = to_lower_case(import_path_file)
                import_path = import_path_path + '/' + import_path_file
                new_imports.append((import_path, import_is_public))
            new_content = []

            def sub(match):
                type = match.group(1)
                found_package = ''
                for package in packages:
                    if type.startswith(
                            package + '.') and len(package) > len(found_package):
                        found_package = package
                type = type[len(found_package):]
                found_package = namespace + '.' + found_package
                if package_lower:
                    found_package = found_package.lower()
                return found_package + type
            for line in content:
                new_content.append(
                    re.sub(
                        '\.?POGOProtos.(\w+(?:\.\w+)*)',
                        sub,
                        line))
            new_proto_folder[new_proto_file_name] = (
                new_package, new_imports, new_content)
        new_protos[new_proto_path] = new_proto_folder
    return new_protos

parser = argparse.ArgumentParser()
parser.add_argument(
    'language',
    choices=[
        'cpp',
        'csharp',
        'go',
        'java',
        'javanano',
        'js',
        'objc',
        'python',
        'ruby'],
    help='language to pass to protoc')
parser.add_argument(
    '-p', '--protoc_path',
    default='protoc',
    help='path to protoc')
parser.add_argument('-o', '--out_path', default='out',
                    help='output path for protoc')
parser.add_argument(
    '--java_generate_equals_and_hash',
    action='store_true',
    help='generate Java\'s .equals() and .hashCode()')
parser.add_argument(
    '--cc_enable_arenas',
    action='store_true',
    help='enable C++ arena allocation')
parser.add_argument(
    '--include_imports',
    action='store_true',
    help='include imports in .desc file')
parser.add_argument(
    '--include_source_info',
    action='store_true',
    help='do not strip source code info from .desc file')
parser.add_argument(
    '--keep_proto_files',
    action='store_true',
    help='do not remove .proto files after compiling')
args = parser.parse_args()

protoc_path = args.protoc_path
src_path = os.path.abspath('src')
out_path = os.path.abspath(args.out_path)

if out_path == os.path.abspath('out') and os.path.exists(out_path):
    shutil.rmtree(out_path)

if not os.path.exists(out_path):
    os.makedirs(out_path)

namespace = 'POGOProtos'
path = 'POGOProtos'
merge = False
path_lower = False
file_lower = False
package_lower = False

if args.language == 'js':
    namespace = 'com.github.aeonlucid.pogoprotos'
    path = 'com/github/aeonlucid/pogoprotos'
    merge = False
    path_lower = True
    file_lower = True
    package_lower = True
elif args.language == 'csharp' or args.language == 'cpp' or args.language == 'objc':
    namespace = 'POGOProtos'
    path = 'POGOProtos'
    merge = False
    path_lower = False
    file_lower = False
    package_lower = False
elif args.language == 'go':
    namespace = 'com.github.aeonlucid.pogoprotos'
    path = 'github.com/aeonlucid/pogoprotos'
    merge = False
    path_lower = True
    file_lower = True
    package_lower = True
elif args.language == 'java' or args.language == 'javanano':
    namespace = 'com.github.aeonlucid.pogoprotos'
    path = 'com/github/aeonlucid/pogoprotos'
    merge = True
    path_lower = True
    file_lower = False
    package_lower = True
elif args.language == 'python':
    namespace = 'pogoprotos'
    path = 'pogoprotos'
    merge = False
    path_lower = True
    file_lower = True
    package_lower = True
elif args.language == 'ruby':
    namespace = 'POGOProtos'
    path = 'pogoprotos'
    merge = False
    path_lower = True
    file_lower = True
    package_lower = False

protos = read_protos(src_path)

if merge:
    protos = merge_protos(protos)

protos = format_protos(
    protos,
    path,
    namespace,
    path_lower,
    file_lower,
    package_lower)

if args.language == 'go':
    new_protos = dict()
    for proto_path in protos:
        proto_folder = protos[proto_path]
        new_proto_path = re.sub('/map', r'/maps', proto_path)
        new_proto_folder = dict()
        for proto_file_name in proto_folder:
            package, imports, content = proto_folder[proto_file_name]
            new_package = re.sub('\.map', r'.maps', package)
            new_imports = []
            for import_path, import_is_public in imports:
                new_imports.append(
                    (re.sub(
                        '/map/',
                        r'/maps/',
                        import_path),
                        import_is_public))
            new_content = []
            for line in content:
                new_content.append(re.sub('\.map\.', r'.maps.', line))
            new_proto_folder[proto_file_name] = (
                new_package, new_imports, new_content)
        new_protos[new_proto_path] = new_proto_folder
    protos = new_protos

proto_folders = write_protos(out_path, protos)
proto_files = sum(proto_folders, [])

commands = []

desc_path = os.path.join(out_path, *path.split('/')) + '.desc'
desc_arguments = []
if args.include_imports:
    desc_arguments.append('--include_imports')
if args.include_source_info:
    desc_arguments.append('--include_source_info')

commands.append(
    """"{0}" --proto_path="{1}" --descriptor_set_out="{2}" {3} {4}""".format(
        args.protoc_path,
        out_path,
        desc_path,
        ' '.join(desc_arguments),
        '"' +
        '" "'.join(proto_files) +
        '"'))

arguments = ''
options = ''
all_at_once = True

if args.language == 'js':
    options = 'library=pogoprotos,binary,error_on_name_conflict'
elif args.language == 'csharp':
    arguments = '--csharp_opt=file_extension=.g.cs --csharp_opt=base_namespace'
elif args.language == 'go':
    options = 'plugins=grpc'
    all_at_once = False

if all_at_once:
    proto_folders = [proto_files]

for proto_files in proto_folders:
    commands.append(
        """"{0}" --proto_path="{1}" --{2}_out={3}:"{4}" {5} {6}""".format(
            args.protoc_path,
            out_path,
            args.language,
            options,
            out_path,
            arguments,
            '"' +
            '" "'.join(proto_files) +
            '"'))

for command in commands:
    call(command, shell=(os.name != 'nt'))

if args.language == 'python':
    for path in protos:
        open(
            os.path.join(
                os.path.join(
                    out_path,
                    *path.split('/')),
                '__init__.py'),
            'w').close()
elif args.language == 'ruby':
    for path in protos:
        with open(os.path.join(out_path, *path.split('/')) + '.rb', 'w') as file:
            for name in protos[path]:
                file.write('require "' + path + '/' + name + '"\n')

# Remove all .proto files
if not args.keep_proto_files:
    for folder_path, _, file_names in os.walk(out_path):
        for file_name in file_names:
            if file_name.endswith(".proto"):
                os.unlink(os.path.join(folder_path, file_name))
<|endoftext|>


<commit_before>ENCL_NAME = "Katniss"
BAY_NUM = "7"
DEPL_PLAN = "Docker CentOS 7.3"
PROFILE_NAME = "docker1"
PROFILE_DEFINITION = "profile_docker.py"
SERVER_FQDN = "synergy-docker1.haib.org"
CONSUL_FQDN = "hpedemo.haib.org"
DATADOG_TAG = "docker"
NET_NAME = "CentOS Net Set"
VLAN_ID = "907"
<commit_msg>modified config_docker.py<commit_after>ENCL_NAME = ""
BAY_NUM = ""
DEPL_PLAN = "Docker CentOS 7.3"
PROFILE_NAME = ""
PROFILE_DEFINITION = "profile_docker.py"
SERVER_FQDN = ""
CONSUL_FQDN = ""
DATADOG_TAG = ""
NET_NAME = ""
VLAN_ID = ""
<|endoftext|>


<commit_before>#!/usr/bin/env python
import sys
import pprint

contexts = []
context_stack = []

def current_context():
    return context_stack[-1] if context_stack else None

class Context:
    def __init__(self, function, parent=None):
        self.function = function
        self.parent = parent
        self.specs = []

    def __str__(self):
        return self.name

    def __repr__(self):
        return 'Context({0}, {1})'.format(repr(self.function.__name__),
                                          repr(self.parent))

    def add_spec(self, spec):
        self.specs.append(spec)

def describe(context_fn):
    parent = current_context()
    context = Context(context_fn, parent)

    contexts.append(context)
    context_stack.append(context)
    context_fn()
    context_stack.pop()

def it(spec):
    context = current_context()
    if not context:
        raise Exception("Adding an 'it' without telling me what you're "
                        "describing seems a bit silly.")

    context.add_spec(spec)

@describe
def outermost():

    @describe
    def innermost():

        @it
        def can_fail():
            assert 1 == 0

@describe
def another_outermost():

    @describe
    def another_innermost():
        pass

def main():
    pp = pprint.PrettyPrinter()
    pp.pprint(contexts)

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Added fixture class<commit_after>#!/usr/bin/env python
import sys
import pprint
import functools

pprint = pprint.PrettyPrinter().pprint

contexts = []
context_stack = []

def current_context():
    return context_stack[-1] if context_stack else None

class Context:
    def __init__(self, function, parent=None):
        self.function = function
        self.parent = parent
        self.specs = []
        self.before_eaches = []

    def __str__(self):
        return self.name

    def __repr__(self):
        return 'Context({0}, {1})'.format(repr(self.function.__name__),
                                          repr(self.parent))

    def add_spec(self, spec):
        self.specs.append(spec)

    def add_before_each(self, before_each):
        self.before_eaches.append(before_each)

    def run(self):
        for spec in self.specs:
            self.before_each()
            spec()

    def before_each(self):
        if self.parent:
            self.parent.before_each()

        for be in self.before_eaches:
            be()

def describe(context_fn):
    parent = current_context()
    context = Context(context_fn, parent)

    contexts.append(context)
    context_stack.append(context)
    context_fn()
    context_stack.pop()

def it(spec):
    context = current_context()
    if not context:
        raise Exception("Adding an 'it' without telling me what you're "
                        "describing seems a bit silly.")

    context.add_spec(spec)

def before_each(setup):
    context = current_context()
    if not context:
        raise Exception("Adding a 'before_each' without describing something "
                        "seems a bit silly.")

    context.add_before_each(setup)

def run_contexts():
    for c in contexts:
        c.run()

class Fixture():
    pass

@describe
def outermost():

    fix = Fixture()

    @before_each
    def setup():
        fix.a = 2
        fix.b = 23

    @describe
    def innermost():

        @before_each
        def setup():
            fix.b = 2

        @it
        def can_fail():
            assert fix.a == 2
            assert fix.b == 2

    @it
    def check_again():
        pprint(fix.__dict__)

@describe
def another_outermost():

    @describe
    def another_innermost():
        pass

def main():
    pprint(contexts)

    run_contexts()

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# Based on AboutClassMethods in the Ruby Koans
#

from runner.koan import *

class AboutClassAttributes(Koan):
    class Dog:
        pass

    def test_objects_are_objects(self):
        fido = self.Dog()
        self.assertEqual(__, isinstance(fido, object))

    def test_classes_are_types(self):
        self.assertEqual(__, self.Dog.__class__ == type)

    def test_classes_are_objects_too(self):
        self.assertEqual(__, issubclass(self.Dog, object))

    def test_objects_have_methods(self):
        fido = self.Dog()
        self.assertEqual(__, len(dir(fido)))

    def test_classes_have_methods(self):
        self.assertEqual(__, len(dir(self.Dog)))

    def test_creating_objects_without_defining_a_class(self):
        singularity = object()
        self.assertEqual(__, len(dir(singularity)))

    def test_defining_attributes_on_individual_objects(self):
        fido = self.Dog()
        fido.legs = 4

        self.assertEqual(__, fido.legs)

    def test_defining_functions_on_individual_objects(self):
        fido = self.Dog()
        fido.wag = lambda : 'fidos wag'

        self.assertEqual(__, fido.wag())

    def test_other_objects_are_not_affected_by_functions_added_to_an_instance(self):
        fido = self.Dog()
        rover = self.Dog()

        def wag():
            return 'fidos wag'
        fido.wag = wag

        with self.assertRaises(___): rover.wag()

    # ------------------------------------------------------------------

    class Dog2:
        def wag(self):
            return 'instance wag'

        def bark(self):
            return "instance bark"

        def growl(self):
            return "instance growl"

        @staticmethod
        def bark():
            return "staticmethod bark, arg: None"

        @classmethod
        def growl(cls):
            return "classmethod growl, arg: cls=" + cls.__name__

    def test_since_classes_are_objects_you_can_define_methods_on_them_too(self):
        self.assertRegexpMatches(self.Dog2.growl(), __)

    def test_classmethods_are_not_independent_of_instance_methods(self):
        fido = self.Dog2()
        self.assertRegexpMatches(fido.growl(), __)
        self.assertRegexpMatches(self.Dog2.growl(), __)

    def test_staticmethods_are_unbound_functions_housed_in_a_class(self):
        self.assertRegexpMatches(self.Dog2.bark(), __)

    def test_staticmethods_also_overshadow_instance_methods(self):
        fido = self.Dog2()
        self.assertRegexpMatches(fido.bark(), __)

    # ------------------------------------------------------------------

    class Dog3:
        def __init__(self):
            self._name = None

        def get_name_from_instance(self):
            return self._name

        def set_name_from_instance(self, name):
            self._name = name

        @classmethod
        def get_name(cls):
            return cls._name

        @classmethod
        def set_name(cls, name):
            cls._name = name

        name = property(get_name, set_name)
        name_from_instance = property(get_name_from_instance, set_name_from_instance)

    def test_classmethods_can_not_be_used_as_properties(self):
        fido = self.Dog3()
        with self.assertRaises(___): fido.name = "Fido"

    def test_classes_and_instances_do_not_share_instance_attributes(self):
        fido = self.Dog3()
        fido.set_name_from_instance("Fido")
        fido.set_name("Rover")
        self.assertEqual(__, fido.get_name_from_instance())
        self.assertEqual(__, self.Dog3.get_name())

    def test_classes_and_instances_do_share_class_attributes(self):
        fido = self.Dog3()
        fido.set_name("Fido")
        self.assertEqual(__, fido.get_name())
        self.assertEqual(__, self.Dog3.get_name())

    # ------------------------------------------------------------------

    class Dog4:
        def a_class_method(cls):
            return 'dogs class method'

        def a_static_method():
            return 'dogs static method'

        a_class_method = classmethod(a_class_method)
        a_static_method = staticmethod(a_static_method)

    def test_you_can_define_class_methods_without_using_a_decorator(self):
        self.assertEqual(__, self.Dog4.a_class_method())

    def test_you_can_define_static_methods_without_using_a_decorator(self):
        self.assertEqual(__, self.Dog4.a_static_method())

    # ------------------------------------------------------------------

    def test_heres_an_easy_way_to_explicitly_call_class_methods_from_instance_methods(self):
        fido = self.Dog4()
        self.assertEqual(__, fido.__class__.a_class_method())
<commit_msg>Removes an unnecessary assertion<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#
# Based on AboutClassMethods in the Ruby Koans
#

from runner.koan import *

class AboutClassAttributes(Koan):
    class Dog:
        pass

    def test_objects_are_objects(self):
        fido = self.Dog()
        self.assertEqual(__, isinstance(fido, object))

    def test_classes_are_types(self):
        self.assertEqual(__, self.Dog.__class__ == type)

    def test_classes_are_objects_too(self):
        self.assertEqual(__, issubclass(self.Dog, object))

    def test_objects_have_methods(self):
        fido = self.Dog()
        self.assertEqual(__, len(dir(fido)))

    def test_classes_have_methods(self):
        self.assertEqual(__, len(dir(self.Dog)))

    def test_creating_objects_without_defining_a_class(self):
        singularity = object()
        self.assertEqual(__, len(dir(singularity)))

    def test_defining_attributes_on_individual_objects(self):
        fido = self.Dog()
        fido.legs = 4

        self.assertEqual(__, fido.legs)

    def test_defining_functions_on_individual_objects(self):
        fido = self.Dog()
        fido.wag = lambda : 'fidos wag'

        self.assertEqual(__, fido.wag())

    def test_other_objects_are_not_affected_by_functions_added_to_an_instance(self):
        fido = self.Dog()
        rover = self.Dog()

        def wag():
            return 'fidos wag'
        fido.wag = wag

        with self.assertRaises(___): rover.wag()

    # ------------------------------------------------------------------

    class Dog2:
        def wag(self):
            return 'instance wag'

        def bark(self):
            return "instance bark"

        def growl(self):
            return "instance growl"

        @staticmethod
        def bark():
            return "staticmethod bark, arg: None"

        @classmethod
        def growl(cls):
            return "classmethod growl, arg: cls=" + cls.__name__

    def test_since_classes_are_objects_you_can_define_methods_on_them_too(self):
        self.assertRegexpMatches(self.Dog2.growl(), __)

    def test_classmethods_are_not_independent_of_instance_methods(self):
        fido = self.Dog2()
        self.assertRegexpMatches(fido.growl(), __)

    def test_staticmethods_are_unbound_functions_housed_in_a_class(self):
        self.assertRegexpMatches(self.Dog2.bark(), __)

    def test_staticmethods_also_overshadow_instance_methods(self):
        fido = self.Dog2()
        self.assertRegexpMatches(fido.bark(), __)

    # ------------------------------------------------------------------

    class Dog3:
        def __init__(self):
            self._name = None

        def get_name_from_instance(self):
            return self._name

        def set_name_from_instance(self, name):
            self._name = name

        @classmethod
        def get_name(cls):
            return cls._name

        @classmethod
        def set_name(cls, name):
            cls._name = name

        name = property(get_name, set_name)
        name_from_instance = property(get_name_from_instance, set_name_from_instance)

    def test_classmethods_can_not_be_used_as_properties(self):
        fido = self.Dog3()
        with self.assertRaises(___): fido.name = "Fido"

    def test_classes_and_instances_do_not_share_instance_attributes(self):
        fido = self.Dog3()
        fido.set_name_from_instance("Fido")
        fido.set_name("Rover")
        self.assertEqual(__, fido.get_name_from_instance())
        self.assertEqual(__, self.Dog3.get_name())

    def test_classes_and_instances_do_share_class_attributes(self):
        fido = self.Dog3()
        fido.set_name("Fido")
        self.assertEqual(__, fido.get_name())
        self.assertEqual(__, self.Dog3.get_name())

    # ------------------------------------------------------------------

    class Dog4:
        def a_class_method(cls):
            return 'dogs class method'

        def a_static_method():
            return 'dogs static method'

        a_class_method = classmethod(a_class_method)
        a_static_method = staticmethod(a_static_method)

    def test_you_can_define_class_methods_without_using_a_decorator(self):
        self.assertEqual(__, self.Dog4.a_class_method())

    def test_you_can_define_static_methods_without_using_a_decorator(self):
        self.assertEqual(__, self.Dog4.a_static_method())

    # ------------------------------------------------------------------

    def test_heres_an_easy_way_to_explicitly_call_class_methods_from_instance_methods(self):
        fido = self.Dog4()
        self.assertEqual(__, fido.__class__.a_class_method())
<|endoftext|>


<commit_before>"""Endpoints for listing Projects, Versions, Builds, etc."""

import json
import logging

from allauth.socialaccount.models import SocialAccount
from django.conf import settings
from django.core.files.storage import get_storage_class
from django.db.models import BooleanField, Value, When, Case
from django.shortcuts import get_object_or_404
from django.template.loader import render_to_string
from rest_framework import decorators, permissions, status, viewsets
from rest_framework.parsers import JSONParser, MultiPartParser
from rest_framework.renderers import BaseRenderer, JSONRenderer
from rest_framework.response import Response

from readthedocs.builds.constants import (
    BRANCH,
    INTERNAL,
    TAG,
)
from readthedocs.builds.models import Build, BuildCommandResult, Version
from readthedocs.core.utils import trigger_build
from readthedocs.oauth.models import RemoteOrganization, RemoteRepository
from readthedocs.oauth.services import GitHubService, registry
from readthedocs.projects.models import Domain, Project
from readthedocs.projects.version_handling import determine_stable_version

from ..permissions import (
    APIPermission,
    APIRestrictedPermission,
    IsOwner,
)
from ..serializers import (
    BuildAdminSerializer,
    BuildCommandSerializer,
    BuildSerializer,
    DomainSerializer,
    ProjectAdminSerializer,
    ProjectSerializer,
    RemoteOrganizationSerializer,
    RemoteRepositorySerializer,
    SocialAccountSerializer,
    VersionAdminSerializer,
    VersionSerializer,
)
from ..utils import (
    ProjectPagination,
    RemoteOrganizationPagination,
    RemoteProjectPagination,
    delete_versions_from_db,
    get_deleted_active_versions,
    run_automation_rules,
    sync_versions_to_db,
)

log = logging.getLogger(__name__)

class PlainTextBuildRenderer(BaseRenderer):

    """
    Custom renderer for text/plain format.

    charset is 'utf-8' by default.
    """

    media_type = 'text/plain'
    format = 'txt'

    def render(self, data, accepted_media_type=None, renderer_context=None):
        renderer_context = renderer_context or {}
        response = renderer_context.get('response')
        if not response or response.exception:
            return data.get('detail', '').encode(self.charset)
        data = render_to_string(
            'restapi/log.txt',
            {'build': data},
        )
        return data.encode(self.charset)

class UserSelectViewSet(viewsets.ModelViewSet):

    """
    View set that varies serializer class based on request user credentials.

    Viewsets using this class should have an attribute `admin_serializer_class`,
    which is a serializer that might have more fields that only admin/staff
    users require. If the user is staff, this class will be returned instead.
    """

    def get_serializer_class(self):
        try:
            if (
                self.request.user.is_staff and
                self.admin_serializer_class is not None
            ):
                return self.admin_serializer_class
        except AttributeError:
            pass
        return self.serializer_class

    def get_queryset(self):
        """Use our API manager method to determine authorization on queryset."""
        return self.model.objects.api(self.request.user)

class ProjectViewSet(UserSelectViewSet):

    """List, filter, etc, Projects."""

    permission_classes = [APIPermission]
    renderer_classes = (JSONRenderer,)
    serializer_class = ProjectSerializer
    admin_serializer_class = ProjectAdminSerializer
    model = Project
    pagination_class = ProjectPagination
    filterset_fields = ('slug',)

    @decorators.action(detail=True)
    def translations(self, *_, **__):
        translations = self.get_object().translations.all()
        return Response({
            'translations': ProjectSerializer(translations, many=True).data,
        })

    @decorators.action(detail=True)
    def subprojects(self, request, **kwargs):
        project = get_object_or_404(
            Project.objects.api(request.user),
            pk=kwargs['pk'],
        )
        rels = project.subprojects.all()
        children = [rel.child for rel in rels]
        return Response({
            'subprojects': ProjectSerializer(children, many=True).data,
        })

    @decorators.action(detail=True)
    def active_versions(self, request, **kwargs):
        project = get_object_or_404(
            Project.objects.api(request.user),
            pk=kwargs['pk'],
        )
        versions = project.versions(manager=INTERNAL).filter(active=True)
        return Response({
            'versions': VersionSerializer(versions, many=True).data,
        })

    @decorators.action(
        detail=True,
        permission_classes=[permissions.IsAdminUser],
    )
    def token(self, request, **kwargs):
        project = get_object_or_404(
            Project.objects.api(request.user),
            pk=kwargs['pk'],
        )
        token = GitHubService.get_token_for_project(project, force_local=True)
        return Response({
            'token': token,
        })

    @decorators.action(detail=True)
    def canonical_url(self, request, **kwargs):
        project = get_object_or_404(
            Project.objects.api(request.user),
            pk=kwargs['pk'],
        )
        return Response({
            'url': project.get_docs_url(),
        })

    @decorators.action(
        detail=True,
        permission_classes=[permissions.IsAdminUser],
        methods=['post'],
    )
    def sync_versions(self, request, **kwargs):  # noqa: D205
        """
        Sync the version data in the repo (on the build server).

        Version data in the repo is synced with what we have in the database.

        :returns: the identifiers for the versions that have been deleted.
        """
        project = get_object_or_404(
            Project.objects.api(request.user),
            pk=kwargs['pk'],
        )

        # If the currently highest non-prerelease version is active, then make
        # the new latest version active as well.
        old_highest_version = determine_stable_version(project.versions.all())
        if old_highest_version is not None:
            activate_new_stable = old_highest_version.active
        else:
            activate_new_stable = False

        try:
            # Update All Versions
            data = request.data
            added_versions = set()
            if 'tags' in data:
                ret_set = sync_versions_to_db(
                    project=project,
                    versions=data['tags'],
                    type=TAG,
                )
                added_versions.update(ret_set)
            if 'branches' in data:
                ret_set = sync_versions_to_db(
                    project=project,
                    versions=data['branches'],
                    type=BRANCH,
                )
                added_versions.update(ret_set)
            deleted_versions = delete_versions_from_db(project, data)
            deleted_active_versions = get_deleted_active_versions(project, data)
        except Exception as e:
            log.exception('Sync Versions Error')
            return Response(
                {
                    'error': str(e),
                },
                status=status.HTTP_400_BAD_REQUEST,
            )

        try:
            # The order of added_versions isn't deterministic.
            # We don't track the commit time or any other metadata.
            # We usually have one version added per webhook.
            run_automation_rules(project, added_versions, deleted_active_versions)
        except Exception:
            # Don't interrupt the request if something goes wrong
            # in the automation rules.
            log.exception(
                'Failed to execute automation rules for [%s]: %s',
                project.slug, added_versions
            )

        # TODO: move this to an automation rule
        promoted_version = project.update_stable_version()
        new_stable = project.get_stable_version()
        if promoted_version and new_stable and new_stable.active:
            log.info(
                'Triggering new stable build: %(project)s:%(version)s',
                {
                    'project': project.slug,
                    'version': new_stable.identifier,
                }
            )
            trigger_build(project=project, version=new_stable)

            # Marking the tag that is considered the new stable version as
            # active and building it if it was just added.
            if (
                activate_new_stable and
                promoted_version.slug in added_versions
            ):
                promoted_version.active = True
                promoted_version.save()
                trigger_build(project=project, version=promoted_version)

        return Response({
            'added_versions': added_versions,
            'deleted_versions': deleted_versions,
        })

class VersionViewSet(UserSelectViewSet):

    permission_classes = [APIRestrictedPermission]
    renderer_classes = (JSONRenderer,)
    serializer_class = VersionSerializer
    admin_serializer_class = VersionAdminSerializer
    model = Version
    filterset_fields = (
        'active',
        'project__slug',
    )

class BuildViewSet(UserSelectViewSet):
    permission_classes = [APIRestrictedPermission]
    renderer_classes = (JSONRenderer, PlainTextBuildRenderer)
    serializer_class = BuildSerializer
    admin_serializer_class = BuildAdminSerializer
    model = Build
    filterset_fields = ('project__slug', 'commit')

    @decorators.action(
        detail=False,
        permission_classes=[permissions.IsAdminUser],
        methods=['get'],
    )
    def concurrent(self, request, **kwargs):
        project_slug = request.GET.get('project__slug')
        project = get_object_or_404(Project, slug=project_slug)
        limit_reached, concurrent, max_concurrent = Build.objects.concurrent(project)
        data = {
            'limit_reached': limit_reached,
            'concurrent': concurrent,
            'max_concurrent': max_concurrent,
        }
        return Response(data)

    def retrieve(self, *args, **kwargs):
        """
        Retrieves command data from storage.

        This uses files from storage to get the JSON,
        and replaces the ``commands`` part of the response data.
        """
        if not settings.RTD_SAVE_BUILD_COMMANDS_TO_STORAGE:
            return super().retrieve(*args, **kwargs)

        instance = self.get_object()
        serializer = self.get_serializer(instance)
        data = serializer.data
        if instance.cold_storage:
            storage = get_storage_class(settings.RTD_BUILD_COMMANDS_STORAGE)()
            storage_path = '{date}/{id}.json'.format(
                date=str(instance.date.date()),
                id=instance.id,
            )
            if storage.exists(storage_path):
                try:
                    json_resp = storage.open(storage_path).read()
                    data['commands'] = json.loads(json_resp)
                except Exception:
                    log.exception(
                        'Failed to read build data from storage. path=%s.',
                        storage_path,
                    )
        return Response(data)

class BuildCommandViewSet(UserSelectViewSet):
    parser_classes = [JSONParser, MultiPartParser]
    permission_classes = [APIRestrictedPermission]
    renderer_classes = (JSONRenderer,)
    serializer_class = BuildCommandSerializer
    model = BuildCommandResult

class DomainViewSet(UserSelectViewSet):
    permission_classes = [APIRestrictedPermission]
    renderer_classes = (JSONRenderer,)
    serializer_class = DomainSerializer
    model = Domain

class RemoteOrganizationViewSet(viewsets.ReadOnlyModelViewSet):
    permission_classes = [IsOwner]
    renderer_classes = (JSONRenderer,)
    serializer_class = RemoteOrganizationSerializer
    model = RemoteOrganization
    pagination_class = RemoteOrganizationPagination

    def get_queryset(self):
        return (
            self.model.objects.api(self.request.user).filter(
                account__provider__in=[
                    service.adapter.provider_id for service in registry
                ],
            )
        )

class RemoteRepositoryViewSet(viewsets.ReadOnlyModelViewSet):
    permission_classes = [IsOwner]
    renderer_classes = (JSONRenderer,)
    serializer_class = RemoteRepositorySerializer
    model = RemoteRepository
    pagination_class = RemoteProjectPagination

    def get_queryset(self):
        query = self.model.objects.api(self.request.user).annotate(
            admin=Case(
                When(
                    remote_repository_relations__user=self.request.user,
                    remote_repository_relations__admin=True,
                    then=Value(True)
                ),
                When(
                    remote_repository_relations__user=self.request.user,
                    remote_repository_relations__admin=False,
                    then=Value(False)
                ),
                default=Value(False),
                output_field=BooleanField(),
            )
        )
        full_name = self.request.query_params.get('full_name')
        if full_name is not None:
            query = query.filter(full_name__icontains=full_name)
        org = self.request.query_params.get('org', None)
        if org is not None:
            query = query.filter(organization__pk=org)

        own = self.request.query_params.get('own', None)
        if own is not None:
            query = query.filter(
                remote_repository_relations__account__provider=own,
                organization=None,
            )

        query = query.filter(
            remote_repository_relations__account__provider__in=[
                service.adapter.provider_id for service in registry
            ],
        ).distinct()

        # optimizes for the RemoteOrganizationSerializer
        query = query.select_related('organization')

        return query

class SocialAccountViewSet(viewsets.ReadOnlyModelViewSet):
    permission_classes = [IsOwner]
    renderer_classes = (JSONRenderer,)
    serializer_class = SocialAccountSerializer
    model = SocialAccount

    def get_queryset(self):
        return self.model.objects.filter(user=self.request.user.pk)
<commit_msg>Fix unauthenticated user issue<commit_after>"""Endpoints for listing Projects, Versions, Builds, etc."""

import json
import logging

from allauth.socialaccount.models import SocialAccount
from django.conf import settings
from django.core.files.storage import get_storage_class
from django.db.models import BooleanField, Value, When, Case
from django.shortcuts import get_object_or_404
from django.template.loader import render_to_string
from rest_framework import decorators, permissions, status, viewsets
from rest_framework.parsers import JSONParser, MultiPartParser
from rest_framework.renderers import BaseRenderer, JSONRenderer
from rest_framework.response import Response

from readthedocs.builds.constants import (
    BRANCH,
    INTERNAL,
    TAG,
)
from readthedocs.builds.models import Build, BuildCommandResult, Version
from readthedocs.core.utils import trigger_build
from readthedocs.oauth.models import RemoteOrganization, RemoteRepository
from readthedocs.oauth.services import GitHubService, registry
from readthedocs.projects.models import Domain, Project
from readthedocs.projects.version_handling import determine_stable_version

from ..permissions import (
    APIPermission,
    APIRestrictedPermission,
    IsOwner,
)
from ..serializers import (
    BuildAdminSerializer,
    BuildCommandSerializer,
    BuildSerializer,
    DomainSerializer,
    ProjectAdminSerializer,
    ProjectSerializer,
    RemoteOrganizationSerializer,
    RemoteRepositorySerializer,
    SocialAccountSerializer,
    VersionAdminSerializer,
    VersionSerializer,
)
from ..utils import (
    ProjectPagination,
    RemoteOrganizationPagination,
    RemoteProjectPagination,
    delete_versions_from_db,
    get_deleted_active_versions,
    run_automation_rules,
    sync_versions_to_db,
)

log = logging.getLogger(__name__)

class PlainTextBuildRenderer(BaseRenderer):

    """
    Custom renderer for text/plain format.

    charset is 'utf-8' by default.
    """

    media_type = 'text/plain'
    format = 'txt'

    def render(self, data, accepted_media_type=None, renderer_context=None):
        renderer_context = renderer_context or {}
        response = renderer_context.get('response')
        if not response or response.exception:
            return data.get('detail', '').encode(self.charset)
        data = render_to_string(
            'restapi/log.txt',
            {'build': data},
        )
        return data.encode(self.charset)

class UserSelectViewSet(viewsets.ModelViewSet):

    """
    View set that varies serializer class based on request user credentials.

    Viewsets using this class should have an attribute `admin_serializer_class`,
    which is a serializer that might have more fields that only admin/staff
    users require. If the user is staff, this class will be returned instead.
    """

    def get_serializer_class(self):
        try:
            if (
                self.request.user.is_staff and
                self.admin_serializer_class is not None
            ):
                return self.admin_serializer_class
        except AttributeError:
            pass
        return self.serializer_class

    def get_queryset(self):
        """Use our API manager method to determine authorization on queryset."""
        return self.model.objects.api(self.request.user)

class ProjectViewSet(UserSelectViewSet):

    """List, filter, etc, Projects."""

    permission_classes = [APIPermission]
    renderer_classes = (JSONRenderer,)
    serializer_class = ProjectSerializer
    admin_serializer_class = ProjectAdminSerializer
    model = Project
    pagination_class = ProjectPagination
    filterset_fields = ('slug',)

    @decorators.action(detail=True)
    def translations(self, *_, **__):
        translations = self.get_object().translations.all()
        return Response({
            'translations': ProjectSerializer(translations, many=True).data,
        })

    @decorators.action(detail=True)
    def subprojects(self, request, **kwargs):
        project = get_object_or_404(
            Project.objects.api(request.user),
            pk=kwargs['pk'],
        )
        rels = project.subprojects.all()
        children = [rel.child for rel in rels]
        return Response({
            'subprojects': ProjectSerializer(children, many=True).data,
        })

    @decorators.action(detail=True)
    def active_versions(self, request, **kwargs):
        project = get_object_or_404(
            Project.objects.api(request.user),
            pk=kwargs['pk'],
        )
        versions = project.versions(manager=INTERNAL).filter(active=True)
        return Response({
            'versions': VersionSerializer(versions, many=True).data,
        })

    @decorators.action(
        detail=True,
        permission_classes=[permissions.IsAdminUser],
    )
    def token(self, request, **kwargs):
        project = get_object_or_404(
            Project.objects.api(request.user),
            pk=kwargs['pk'],
        )
        token = GitHubService.get_token_for_project(project, force_local=True)
        return Response({
            'token': token,
        })

    @decorators.action(detail=True)
    def canonical_url(self, request, **kwargs):
        project = get_object_or_404(
            Project.objects.api(request.user),
            pk=kwargs['pk'],
        )
        return Response({
            'url': project.get_docs_url(),
        })

    @decorators.action(
        detail=True,
        permission_classes=[permissions.IsAdminUser],
        methods=['post'],
    )
    def sync_versions(self, request, **kwargs):  # noqa: D205
        """
        Sync the version data in the repo (on the build server).

        Version data in the repo is synced with what we have in the database.

        :returns: the identifiers for the versions that have been deleted.
        """
        project = get_object_or_404(
            Project.objects.api(request.user),
            pk=kwargs['pk'],
        )

        # If the currently highest non-prerelease version is active, then make
        # the new latest version active as well.
        old_highest_version = determine_stable_version(project.versions.all())
        if old_highest_version is not None:
            activate_new_stable = old_highest_version.active
        else:
            activate_new_stable = False

        try:
            # Update All Versions
            data = request.data
            added_versions = set()
            if 'tags' in data:
                ret_set = sync_versions_to_db(
                    project=project,
                    versions=data['tags'],
                    type=TAG,
                )
                added_versions.update(ret_set)
            if 'branches' in data:
                ret_set = sync_versions_to_db(
                    project=project,
                    versions=data['branches'],
                    type=BRANCH,
                )
                added_versions.update(ret_set)
            deleted_versions = delete_versions_from_db(project, data)
            deleted_active_versions = get_deleted_active_versions(project, data)
        except Exception as e:
            log.exception('Sync Versions Error')
            return Response(
                {
                    'error': str(e),
                },
                status=status.HTTP_400_BAD_REQUEST,
            )

        try:
            # The order of added_versions isn't deterministic.
            # We don't track the commit time or any other metadata.
            # We usually have one version added per webhook.
            run_automation_rules(project, added_versions, deleted_active_versions)
        except Exception:
            # Don't interrupt the request if something goes wrong
            # in the automation rules.
            log.exception(
                'Failed to execute automation rules for [%s]: %s',
                project.slug, added_versions
            )

        # TODO: move this to an automation rule
        promoted_version = project.update_stable_version()
        new_stable = project.get_stable_version()
        if promoted_version and new_stable and new_stable.active:
            log.info(
                'Triggering new stable build: %(project)s:%(version)s',
                {
                    'project': project.slug,
                    'version': new_stable.identifier,
                }
            )
            trigger_build(project=project, version=new_stable)

            # Marking the tag that is considered the new stable version as
            # active and building it if it was just added.
            if (
                activate_new_stable and
                promoted_version.slug in added_versions
            ):
                promoted_version.active = True
                promoted_version.save()
                trigger_build(project=project, version=promoted_version)

        return Response({
            'added_versions': added_versions,
            'deleted_versions': deleted_versions,
        })

class VersionViewSet(UserSelectViewSet):

    permission_classes = [APIRestrictedPermission]
    renderer_classes = (JSONRenderer,)
    serializer_class = VersionSerializer
    admin_serializer_class = VersionAdminSerializer
    model = Version
    filterset_fields = (
        'active',
        'project__slug',
    )

class BuildViewSet(UserSelectViewSet):
    permission_classes = [APIRestrictedPermission]
    renderer_classes = (JSONRenderer, PlainTextBuildRenderer)
    serializer_class = BuildSerializer
    admin_serializer_class = BuildAdminSerializer
    model = Build
    filterset_fields = ('project__slug', 'commit')

    @decorators.action(
        detail=False,
        permission_classes=[permissions.IsAdminUser],
        methods=['get'],
    )
    def concurrent(self, request, **kwargs):
        project_slug = request.GET.get('project__slug')
        project = get_object_or_404(Project, slug=project_slug)
        limit_reached, concurrent, max_concurrent = Build.objects.concurrent(project)
        data = {
            'limit_reached': limit_reached,
            'concurrent': concurrent,
            'max_concurrent': max_concurrent,
        }
        return Response(data)

    def retrieve(self, *args, **kwargs):
        """
        Retrieves command data from storage.

        This uses files from storage to get the JSON,
        and replaces the ``commands`` part of the response data.
        """
        if not settings.RTD_SAVE_BUILD_COMMANDS_TO_STORAGE:
            return super().retrieve(*args, **kwargs)

        instance = self.get_object()
        serializer = self.get_serializer(instance)
        data = serializer.data
        if instance.cold_storage:
            storage = get_storage_class(settings.RTD_BUILD_COMMANDS_STORAGE)()
            storage_path = '{date}/{id}.json'.format(
                date=str(instance.date.date()),
                id=instance.id,
            )
            if storage.exists(storage_path):
                try:
                    json_resp = storage.open(storage_path).read()
                    data['commands'] = json.loads(json_resp)
                except Exception:
                    log.exception(
                        'Failed to read build data from storage. path=%s.',
                        storage_path,
                    )
        return Response(data)

class BuildCommandViewSet(UserSelectViewSet):
    parser_classes = [JSONParser, MultiPartParser]
    permission_classes = [APIRestrictedPermission]
    renderer_classes = (JSONRenderer,)
    serializer_class = BuildCommandSerializer
    model = BuildCommandResult

class DomainViewSet(UserSelectViewSet):
    permission_classes = [APIRestrictedPermission]
    renderer_classes = (JSONRenderer,)
    serializer_class = DomainSerializer
    model = Domain

class RemoteOrganizationViewSet(viewsets.ReadOnlyModelViewSet):
    permission_classes = [IsOwner]
    renderer_classes = (JSONRenderer,)
    serializer_class = RemoteOrganizationSerializer
    model = RemoteOrganization
    pagination_class = RemoteOrganizationPagination

    def get_queryset(self):
        return (
            self.model.objects.api(self.request.user).filter(
                account__provider__in=[
                    service.adapter.provider_id for service in registry
                ],
            )
        )

class RemoteRepositoryViewSet(viewsets.ReadOnlyModelViewSet):
    permission_classes = [IsOwner]
    renderer_classes = (JSONRenderer,)
    serializer_class = RemoteRepositorySerializer
    model = RemoteRepository
    pagination_class = RemoteProjectPagination

    def get_queryset(self):
        user = (
            self.request.user
            if self.request.user.is_authenticated
            else None
        )
        query = self.model.objects.api(self.request.user).annotate(
            admin=Case(
                When(
                    remote_repository_relations__user=user,
                    remote_repository_relations__admin=True,
                    then=Value(True)
                ),
                When(
                    remote_repository_relations__user=user,
                    remote_repository_relations__admin=False,
                    then=Value(False)
                ),
                default=Value(False),
                output_field=BooleanField(),
            )
        )
        full_name = self.request.query_params.get('full_name')
        if full_name is not None:
            query = query.filter(full_name__icontains=full_name)
        org = self.request.query_params.get('org', None)
        if org is not None:
            query = query.filter(organization__pk=org)

        own = self.request.query_params.get('own', None)
        if own is not None:
            query = query.filter(
                remote_repository_relations__account__provider=own,
                organization=None,
            )

        query = query.filter(
            remote_repository_relations__account__provider__in=[
                service.adapter.provider_id for service in registry
            ],
        ).distinct()

        # optimizes for the RemoteOrganizationSerializer
        query = query.select_related('organization')

        return query

class SocialAccountViewSet(viewsets.ReadOnlyModelViewSet):
    permission_classes = [IsOwner]
    renderer_classes = (JSONRenderer,)
    serializer_class = SocialAccountSerializer
    model = SocialAccount

    def get_queryset(self):
        return self.model.objects.filter(user=self.request.user.pk)
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Program to align rnaseq transcriptome reads to the genome using STAR aligner

Requirement:
    STAR - https://github.com/alexdobin/STAR/releases
    pysam - http://code.google.com/p/pysam/
    mmr - https://github.com/ratschlab/mmr
"""

import os
import re
import sys
import subprocess
from collections import defaultdict

def run_mmr(org_name, read_map_dir, threads=3):
    """
    a pythonic wrapper for multiple mapper resolution program

    @args org_name: Organism name, example case A_thaliana
    @type org_name: str
    @args read_map_dir: directory where the STAR bam (aligned reads) file located
    @type read_map_dir: str
    @args threads: number of threads to use for the run (default: 3)
    @type threads: int

    expect mmr to be available under the PATH env variable
    """
    import pysam

    ## mmr works well with bam file sorted by read id
    bam_file = "%s/%s_Aligned.sortedByName.out.bam" % (read_map_dir, org_name)
    if not os.path.isfile(bam_file):
        print "warning: failed to fetch read id sorted BAM file for organism: %s, trying to get the raw alignment file..." % org_name

        bam_file = "%s/%s_Aligned.out.bam" % (read_map_dir, org_name) ## unsorted bam file from star output
        if not os.path.isfile(bam_file):
            print "error: failed to fetch star alignment file for %s %s" % (org_name, bam_file)
            sys.exit(-1)

        ## sorting bam file
        sorted_bam = "%s/%s_Aligned.sortedByName.out" % (read_map_dir, org_name)
        print "trying to sort based by read id with output prefix as: %s" % sorted_bam
        if not os.path.isfile("%s.bam" % sorted_bam):
            print 'sorting...'
            pysam.sort("-n", bam_file, sorted_bam)

        bam_file = "%s.bam" % sorted_bam

    print 'using bam file from %s' % bam_file
    outFile = "%s/%s_Aligned_mmr.bam" % (read_map_dir, org_name)

    iterations = 3
    ## provide a bam file sorted by read id
    cli_mmr = "module load gcc; mmr -b -p -V -t %d -I %d -o %s %s" % (threads, iterations, outFile, bam_file)

    sys.stdout.write('\trun MMR as: %s \n' % cli_mmr)
    try:
        ## changing the working dir to run mmr
        os.chdir(read_map_dir)

        process = subprocess.Popen(cli_mmr, shell=True)
        process.wait()

        print 'mmr run completed. result file stored at %s' outFile
    except Exception, e:
        print 'Error running MMR.\n%s' %  str( e )
        sys.exit(0)

def run_star_alignment(org_db, read_type='PE', max_mates_gap_length=100000, num_cpus=1):
    """
    wrapper for running STAR program

    @args org_db: a python dictionary with all details about a single organism
    @type org_db: defaultdict
    @args read_type: library type - paired-end or single-end (default: PE)
    @type read_type: str
    @args max_mates_gap_length: maximum insert size from the sample (default: 10000)
    @type max_mates_gap_length: int
    @args num_cpus: number of threads to use for the run (default: 1)
    @type num_cpus: int
    """
    from gfftools import helper, GFFParser
    ## genome indices and annotation file
    genome_dir = org_db['genome_index_dir']
    gtf_db = org_db['gtf']

    ## check for the file type
    gff_hand = helper.open_file(gtf_db)
    for rec in gff_hand:
        rec = rec.strip('\n\r')
        # skip empty line fasta identifier and commented line
        if not rec or rec[0] in  ['#', '>']:
            continue
        # skip the genome sequence
        if not re.search('\t', rec):
            continue
        parts = rec.split('\t')
        assert len(parts) >= 8, rec
        ftype, tags = GFFParser.attribute_tags(parts[-1])
        break
    gff_hand.close()

    ## library type
    if read_type == 'PE':
        read_file = "%s/%s %s/%s" % (org_db['fastq_path'], org_db['fastq'][0], org_db['fastq_path'], org_db['fastq'][1])
    else:
        read_file = "%s/%s" % (org_db['fastq_path'], org_db['fastq'][0])

    ## getting the command to uncompress the read file
    zip_type = {".gz" : "zcat", ".bz2" : "bzcat"}
    file_prefx, ext = os.path.splitext(org_db['fastq'][0])

    ## genomic feature information
    max_lenth_intron = org_db['max_intron_len']

    out_prefix = '%s/%s_' % (org_db['read_map_dir'], org_db['short_name'])

    ## according to the file type
    if ftype:
        make_star_run = "STAR \
        --genomeDir %s \
        --readFilesIn %s \
        --readFilesCommand %s \
        --outFileNamePrefix %s \
        --runThreadN %d \
        --outFilterMultimapScoreRange 2 \
        --outFilterMultimapNmax 30 \
        --outFilterMismatchNmax 4 \
        --alignIntronMax %d \
        --sjdbGTFfile %s \
        --sjdbGTFtagExonParentTranscript Parent \
        --sjdbScore 1 \
        --sjdbOverhang 5 \
        --alignMatesGapMax %d\
        --outSAMstrandField intronMotif \
        --outFilterIntronMotifs RemoveNoncanonical \
        --outSAMtype BAM Unsorted \
        --genomeLoad LoadAndRemove" % (genome_dir, read_file,
            zip_type[ext], out_prefix, num_cpus, max_lenth_intron, gtf_db, max_mates_gap_length)
    else:
        make_star_run = "STAR \
        --genomeDir %s \
        --readFilesIn %s \
        --readFilesCommand %s \
        --outFileNamePrefix %s \
        --runThreadN %d \
        --outFilterMultimapScoreRange 2 \
        --outFilterMultimapNmax 30 \
        --outFilterMismatchNmax 4 \
        --alignIntronMax %d \
        --sjdbGTFfile %s \
        --sjdbGTFfeatureExon exon \
        --sjdbScore 1 \
        --sjdbOverhang 5 \
        --alignMatesGapMax %d\
        --outSAMstrandField intronMotif \
        --outFilterIntronMotifs RemoveNoncanonical \
        --outSAMtype BAM Unsorted \
        --genomeLoad LoadAndRemove" % (genome_dir, read_file,
            zip_type[ext], out_prefix, num_cpus, max_lenth_intron, gtf_db, max_mates_gap_length)

    sys.stdout.write('\trunning STAR program as: %s \n' % make_star_run)
    try:
        process = subprocess.Popen(make_star_run, shell=True)
        process.wait()

        print "star run completed. result file stored at %sAligned.out.bam" % out_prefix
    except Exception, e:
        print 'Error running STAR.\n%s' %  str( e )
        sys.exit(0)

def uniq_mapped_reads(bam_file, multi_map=1):
    """
    Filter a BAM file to extract only the uniquely mapped reads

    @args bam_file: binary file for storing the sequencing reads.
    @type bam_file: str
    @args multi_map: number of hits of a read (default 1)
    @type multi_map: integer
    """
    import pysam

    ## indexing the in bam file
    if not os.path.exists(bam_file + ".bai"):
        pysam.index(bam_file)

    #import time
    #t_start = time.time()

    sam_file = pysam.Samfile(bam_file, "rb")

    ## getting the abs path of in bam file
    bam_file_path = os.path.abspath(bam_file)
    bam_file_dir = os.path.dirname(bam_file_path)

    ## defining the out bam file
    uniq_map_bam_file = '%s/unique_map_reads.bam' % bam_file_dir
    out_bam_fh = pysam.Samfile(uniq_map_bam_file, "wb", template=sam_file)

    ## filtering the alignment based on the read hits
    for read in sam_file.fetch():
        for tag in read.tags:
            if tag[0] == 'NH':
                if int(tag[1]) <= multi_map:
                    out_bam_fh.write(read)

    sam_file.close()
    out_bam_fh.close()

    ## indexing the out bam file
    pysam.index(uniq_map_bam_file)

    #time_taken = time.time() - t_start
    #print time_taken

def read_directions_count(bam_file):
    """
    get the reads directions count from a bam file

    @args bam_file: binary file formt for storing sequencing reads information
    @type bam_file: str
    """

    import pysam

    ## indexing the in bam file
    if not os.path.exists(bam_file + ".bai"):
        pysam.index(bam_file)

    reverse_cnt = 0
    forward_cnt = 0

    bam_fh = pysam.Samfile(bam_file, "rb")

    for read in bam_fh.fetch():
         if read.is_proper_pair and read.is_read1:
            if read.is_reverse:
                reverse_cnt += 1
            else:
                forward_cnt += 1

    bam_fh.close()
    return {'forward_reads_count': forward_cnt, 'reverse_reads_count': reverse_cnt}

def calculate_insert_size_from_bam(bam_file):
    """
    estimate the insert size from uniquely mapped reads in a BAM file
    """

    import pysam

def calculate_insert_size(org_db):
    """
    calculate the insert-size from raw read sequence file
    """

    import re
    from glob import glob

    #FIXME
    """
    $ python ~/tmp/7281991/estimate-insert-sizes Salmonella_enterica_subsp_enterica_serovar_typhimurium_str_lt2.GCA_000006945.1.21.dna.toplevel.fa SRR863221_1.fastq.bz2 SRR863221_2.fastq.bz2
    Processing:
     SRR863221_1.fastq.bz2
      SRR863221_2.fastq.bz2
      [M::mem_pestat] # candidate unique pairs for (FF, FR, RF, RR): (0, 0, 0, 0)
      [M::mem_pestat] skip orientation FF as there are not enough pairs
      [M::mem_pestat] skip orientation FR as there are not enough pairs
      [M::mem_pestat] skip orientation RF as there are not enough pairs
      [M::mem_pestat] skip orientation RR as there are not enough pairs
      Traceback (most recent call last):
        File "/home/vipin/tmp/7281991/estimate-insert-sizes", line 100, in <module>
            mean = most_likely[1]['mean']
            KeyError: 'mean'

    """
<commit_msg>ran a functional test on run_mmr module with G_gallus genome data<commit_after>#!/usr/bin/env python
"""
Program to align rnaseq transcriptome reads to the genome using STAR aligner

Requirement:
    STAR - https://github.com/alexdobin/STAR/releases
    pysam - http://code.google.com/p/pysam/
    mmr - https://github.com/ratschlab/mmr
"""

import os
import re
import sys
import subprocess
from collections import defaultdict

def run_mmr(org_name, read_map_dir, threads=3):
    """
    a pythonic wrapper for multiple mapper resolution program

    @args org_name: Organism name, example case A_thaliana
    @type org_name: str
    @args read_map_dir: directory where the STAR bam (aligned reads) file located
    @type read_map_dir: str
    @args threads: number of threads to use for the run (default: 3)
    @type threads: int
    """
    import pysam

    try:
        subprocess.call(["mmr"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    except:
        exit("Please make sure that the `mmr` binary is in your $PATH")

    ## mmr works well with bam file sorted by read id
    bam_file = "%s/%s_Aligned.sortedByName.out.bam" % (read_map_dir, org_name)
    if not os.path.isfile(bam_file):
        sys.stdout.write("warning: failed to fetch read id sorted BAM file for organism: %s, trying to get the raw alignment file\n" % org_name)

        bam_file = "%s/%s_Aligned.out.bam" % (read_map_dir, org_name) ## unsorted bam file from STAR output
        if not os.path.isfile(bam_file):
            exit("error: failed to fetch STAR read alignment file for %s %s\n" % (org_name, bam_file))

        ## sorting bam file
        sorted_bam = "%s/%s_Aligned.sortedByName.out" % (read_map_dir, org_name)
        if not os.path.isfile("%s.bam" % sorted_bam):
            sys.stdout.write("trying to sort based by read id with output prefix as: %s\n" % sorted_bam)
            pysam.sort("-n", bam_file, sorted_bam)

        bam_file = "%s.bam" % sorted_bam

    sys.stdout.write("using bam file from %s\n" % bam_file)
    outFile = "%s/%s_Aligned_mmr.bam" % (read_map_dir, org_name)

    iterations = 3
    ## provide a bam file sorted by read id
    cli_mmr = "module load gcc; mmr -b -p -V -t %d -I %d -o %s %s" % (threads, iterations, outFile, bam_file)

    try:
        sys.stdout.write('\trun MMR as: %s \n' % cli_mmr)
        ## changing the working dir to run mmr
        os.chdir(read_map_dir)

        process = subprocess.Popen(cli_mmr, shell=True)
        returncode = process.wait()

        if returncode !=0:
            raise Exception, "Exit status return code = %i" % returncode

        sys.stdout.write('MMR run finished. result file stored at %s\n' outFile)
    except Exception, e:
        exit('Error running MMR.\n%s' %  str( e ))

def run_star_alignment(org_db, read_type='PE', max_mates_gap_length=100000, num_cpus=1):
    """
    wrapper for running STAR program

    @args org_db: a python dictionary with all details about a single organism
    @type org_db: defaultdict
    @args read_type: library type - paired-end or single-end (default: PE)
    @type read_type: str
    @args max_mates_gap_length: maximum insert size from the sample (default: 10000)
    @type max_mates_gap_length: int
    @args num_cpus: number of threads to use for the run (default: 1)
    @type num_cpus: int
    """
    from gfftools import helper, GFFParser
    ## genome indices and annotation file
    genome_dir = org_db['genome_index_dir']
    gtf_db = org_db['gtf']

    ## check for the file type
    gff_hand = helper.open_file(gtf_db)
    for rec in gff_hand:
        rec = rec.strip('\n\r')
        # skip empty line fasta identifier and commented line
        if not rec or rec[0] in  ['#', '>']:
            continue
        # skip the genome sequence
        if not re.search('\t', rec):
            continue
        parts = rec.split('\t')
        assert len(parts) >= 8, rec
        ftype, tags = GFFParser.attribute_tags(parts[-1])
        break
    gff_hand.close()

    ## library type
    if read_type == 'PE':
        read_file = "%s/%s %s/%s" % (org_db['fastq_path'], org_db['fastq'][0], org_db['fastq_path'], org_db['fastq'][1])
    else:
        read_file = "%s/%s" % (org_db['fastq_path'], org_db['fastq'][0])

    ## getting the command to uncompress the read file
    zip_type = {".gz" : "zcat", ".bz2" : "bzcat"}
    file_prefx, ext = os.path.splitext(org_db['fastq'][0])

    ## genomic feature information
    max_lenth_intron = org_db['max_intron_len']

    out_prefix = '%s/%s_' % (org_db['read_map_dir'], org_db['short_name'])

    ## according to the file type
    if ftype:
        make_star_run = "STAR \
        --genomeDir %s \
        --readFilesIn %s \
        --readFilesCommand %s \
        --outFileNamePrefix %s \
        --runThreadN %d \
        --outFilterMultimapScoreRange 2 \
        --outFilterMultimapNmax 30 \
        --outFilterMismatchNmax 4 \
        --alignIntronMax %d \
        --sjdbGTFfile %s \
        --sjdbGTFtagExonParentTranscript Parent \
        --sjdbScore 1 \
        --sjdbOverhang 5 \
        --alignMatesGapMax %d\
        --outSAMstrandField intronMotif \
        --outFilterIntronMotifs RemoveNoncanonical \
        --outSAMtype BAM Unsorted \
        --genomeLoad LoadAndRemove" % (genome_dir, read_file,
            zip_type[ext], out_prefix, num_cpus, max_lenth_intron, gtf_db, max_mates_gap_length)
    else:
        make_star_run = "STAR \
        --genomeDir %s \
        --readFilesIn %s \
        --readFilesCommand %s \
        --outFileNamePrefix %s \
        --runThreadN %d \
        --outFilterMultimapScoreRange 2 \
        --outFilterMultimapNmax 30 \
        --outFilterMismatchNmax 4 \
        --alignIntronMax %d \
        --sjdbGTFfile %s \
        --sjdbGTFfeatureExon exon \
        --sjdbScore 1 \
        --sjdbOverhang 5 \
        --alignMatesGapMax %d\
        --outSAMstrandField intronMotif \
        --outFilterIntronMotifs RemoveNoncanonical \
        --outSAMtype BAM Unsorted \
        --genomeLoad LoadAndRemove" % (genome_dir, read_file,
            zip_type[ext], out_prefix, num_cpus, max_lenth_intron, gtf_db, max_mates_gap_length)

    sys.stdout.write('\trunning STAR program as: %s \n' % make_star_run)
    try:
        process = subprocess.Popen(make_star_run, shell=True)
        process.wait()

        print "star run completed. result file stored at %sAligned.out.bam" % out_prefix
    except Exception, e:
        print 'Error running STAR.\n%s' %  str( e )
        sys.exit(0)

def uniq_mapped_reads(bam_file, multi_map=1):
    """
    Filter a BAM file to extract only the uniquely mapped reads

    @args bam_file: binary file for storing the sequencing reads.
    @type bam_file: str
    @args multi_map: number of hits of a read (default 1)
    @type multi_map: integer
    """
    import pysam

    ## indexing the in bam file
    if not os.path.exists(bam_file + ".bai"):
        pysam.index(bam_file)

    #import time
    #t_start = time.time()

    sam_file = pysam.Samfile(bam_file, "rb")

    ## getting the abs path of in bam file
    bam_file_path = os.path.abspath(bam_file)
    bam_file_dir = os.path.dirname(bam_file_path)

    ## defining the out bam file
    uniq_map_bam_file = '%s/unique_map_reads.bam' % bam_file_dir
    out_bam_fh = pysam.Samfile(uniq_map_bam_file, "wb", template=sam_file)

    ## filtering the alignment based on the read hits
    for read in sam_file.fetch():
        for tag in read.tags:
            if tag[0] == 'NH':
                if int(tag[1]) <= multi_map:
                    out_bam_fh.write(read)

    sam_file.close()
    out_bam_fh.close()

    ## indexing the out bam file
    pysam.index(uniq_map_bam_file)

    #time_taken = time.time() - t_start
    #print time_taken

def read_directions_count(bam_file):
    """
    get the reads directions count from a bam file

    @args bam_file: binary file formt for storing sequencing reads information
    @type bam_file: str
    """

    import pysam

    ## indexing the in bam file
    if not os.path.exists(bam_file + ".bai"):
        pysam.index(bam_file)

    reverse_cnt = 0
    forward_cnt = 0

    bam_fh = pysam.Samfile(bam_file, "rb")

    for read in bam_fh.fetch():
         if read.is_proper_pair and read.is_read1:
            if read.is_reverse:
                reverse_cnt += 1
            else:
                forward_cnt += 1

    bam_fh.close()
    return {'forward_reads_count': forward_cnt, 'reverse_reads_count': reverse_cnt}

def calculate_insert_size_from_bam(bam_file):
    """
    estimate the insert size from uniquely mapped reads in a BAM file
    """

    import pysam

def calculate_insert_size(org_db):
    """
    calculate the insert-size from raw read sequence file
    """

    import re
    from glob import glob

    #FIXME
    """
    $ python ~/tmp/7281991/estimate-insert-sizes Salmonella_enterica_subsp_enterica_serovar_typhimurium_str_lt2.GCA_000006945.1.21.dna.toplevel.fa SRR863221_1.fastq.bz2 SRR863221_2.fastq.bz2
    Processing:
     SRR863221_1.fastq.bz2
      SRR863221_2.fastq.bz2
      [M::mem_pestat] # candidate unique pairs for (FF, FR, RF, RR): (0, 0, 0, 0)
      [M::mem_pestat] skip orientation FF as there are not enough pairs
      [M::mem_pestat] skip orientation FR as there are not enough pairs
      [M::mem_pestat] skip orientation RF as there are not enough pairs
      [M::mem_pestat] skip orientation RR as there are not enough pairs
      Traceback (most recent call last):
        File "/home/vipin/tmp/7281991/estimate-insert-sizes", line 100, in <module>
            mean = most_likely[1]['mean']
            KeyError: 'mean'

    """
<|endoftext|>


<commit_before># -*- coding: utf-8; -*-
#
# This file is part of Superdesk.
#
# Copyright 2013, 2014 Sourcefabric z.u. and contributors.
#
# For the full copyright and license information, please see the
# AUTHORS and LICENSE files distributed with this source code, or
# at https://www.sourcefabric.org/superdesk/license
from eve.versioning import resolve_document_version

import superdesk

from flask import request
from apps.tasks import send_to

from superdesk import get_resource_service
from superdesk.errors import SuperdeskApiError, InvalidStateTransitionError
from superdesk.resource import Resource
from superdesk.services import BaseService
from apps.archive.common import item_url, insert_into_versions
from apps.archive.archive import SOURCE as ARCHIVE
from superdesk.workflow import is_workflow_state_transition_valid
from eve.utils import config

class MoveResource(Resource):
    endpoint_name = 'move'
    resource_title = endpoint_name

    schema = {
        'desk': Resource.rel('desks', False, required=True),
        'stage': Resource.rel('stages', False, required=True)
    }

    url = 'archive/<{0}:guid>/move'.format(item_url)

    resource_methods = ['POST']
    item_methods = []

    privileges = {'POST': 'move'}

class MoveService(BaseService):
    def create(self, docs, **kwargs):
        guid_of_item_to_be_moved = request.view_args['guid']

        guid_of_moved_items = []

        for doc in docs:
            archive_service = get_resource_service(ARCHIVE)

            archived_doc = archive_service.find_one(req=None, _id=guid_of_item_to_be_moved)
            if not archived_doc:
                raise SuperdeskApiError.notFoundError('Fail to found item with guid: %s' %
                                                      guid_of_item_to_be_moved)

            current_desk_of_item = archived_doc.get('task', {}).get('desk')
            if current_desk_of_item and str(current_desk_of_item) == str(doc.get('desk')):
                raise SuperdeskApiError.preconditionFailedError(message='Move is not allowed within the same desk.')

            if not is_workflow_state_transition_valid('submit_to_desk', archived_doc[config.CONTENT_STATE]):
                raise InvalidStateTransitionError()

            original = dict(archived_doc)

            send_to(archived_doc, doc.get('desk'), doc.get('stage'))
            archived_doc[config.CONTENT_STATE] = 'submitted'
            resolve_document_version(archived_doc, ARCHIVE, 'PATCH', original)

            del archived_doc['_id']
            archive_service.update(original['_id'], archived_doc, original)

            insert_into_versions(doc=archived_doc)

            guid_of_moved_items.append(archived_doc['guid'])

        return guid_of_moved_items

superdesk.workflow_action(
    name='submit_to_desk',
    include_states=['draft', 'fetched', 'routed', 'submitted', 'in_progress'],
    privileges=['archive', 'move']
)
<commit_msg>[SD-1884] Refactor Fetch Endpoints ::: Trying to fix travis build error<commit_after># -*- coding: utf-8; -*-
#
# This file is part of Superdesk.
#
# Copyright 2013, 2014 Sourcefabric z.u. and contributors.
#
# For the full copyright and license information, please see the
# AUTHORS and LICENSE files distributed with this source code, or
# at https://www.sourcefabric.org/superdesk/license
from eve.versioning import resolve_document_version

import superdesk

from flask import request
from apps.tasks import send_to

from superdesk import get_resource_service
from superdesk.errors import SuperdeskApiError, InvalidStateTransitionError
from superdesk.resource import Resource
from superdesk.services import BaseService
from apps.archive.common import item_url, insert_into_versions
from apps.archive.archive import SOURCE as ARCHIVE
from superdesk.workflow import is_workflow_state_transition_valid
from eve.utils import config

class MoveResource(Resource):
    endpoint_name = 'move'
    resource_title = endpoint_name

    schema = {
        'desk': Resource.rel('desks', False, required=True),
        'stage': Resource.rel('stages', False, required=True)
    }

    url = 'archive/<{0}:guid>/move'.format(item_url)

    resource_methods = ['POST']
    item_methods = []

    privileges = {'POST': 'move'}

class MoveService(BaseService):
    def create(self, docs, **kwargs):
        guid_of_item_to_be_moved = request.view_args['guid']

        guid_of_moved_items = []

        for doc in docs:
            archive_service = get_resource_service(ARCHIVE)

            archived_doc = archive_service.find_one(req=None, _id=guid_of_item_to_be_moved)
            if not archived_doc:
                raise SuperdeskApiError.notFoundError('Fail to found item with guid: %s' %
                                                      guid_of_item_to_be_moved)

            current_desk_of_item = archived_doc.get('task', {}).get('desk')
            if current_desk_of_item and str(current_desk_of_item) == str(doc.get('desk')):
                raise SuperdeskApiError.preconditionFailedError(message='Move is not allowed within the same desk.')

            if not is_workflow_state_transition_valid('submit_to_desk', archived_doc[config.CONTENT_STATE]):
                raise InvalidStateTransitionError()

            original = dict(archived_doc)

            send_to(archived_doc, doc.get('desk'), doc.get('stage'))
            archived_doc[config.CONTENT_STATE] = 'submitted'
            resolve_document_version(archived_doc, ARCHIVE, 'PATCH', original)

            del archived_doc['_id']
            archive_service.update(original['_id'], archived_doc, original)

            insert_into_versions(guid=original['_id'])

            guid_of_moved_items.append(archived_doc['guid'])

        return guid_of_moved_items

superdesk.workflow_action(
    name='submit_to_desk',
    include_states=['draft', 'fetched', 'routed', 'submitted', 'in_progress'],
    privileges=['archive', 'move']
)
<|endoftext|>