<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" ForwardingRuleMigration class: It is the handler to migrate
a forwarding rule based on the type of it.

"""
import google.auth
from googleapiclient import discovery
from vm_network_migration.handler_helper.selfLink_executor import SelfLinkExecutor
from vm_network_migration.module_helpers.forwarding_rule_helper import ForwardingRuleHelper
from vm_network_migration.modules.external_regional_forwarding_rule import ExternalRegionalForwardingRule
from vm_network_migration.modules.global_forwarding_rule import GlobalForwardingRule
from vm_network_migration.modules.internal_regional_forwarding_rule import InternalRegionalForwardingRule

class ForwardingRuleMigration(object):
    def __init__(self, project, forwarding_rule_name,
                 network_name, subnetwork_name,
                 preserve_instance_external_ip, region=None):
        """ Initialize a InstanceNetworkMigration object

        Args:
            project: project ID
            forwarding_rule_name: name of the forwarding rule
            network: target network
            subnetwork: target subnet
            preserve_instance_external_ip: whether preserve the external IP
            of the instances which serves this load balancer
            region: region of the internal load balancer
        """
        self.compute = self.set_compute_engine()
        self.project = project
        self.network_name = network_name
        self.subnetwork_name = subnetwork_name
        self.preserve_instance_external_ip = preserve_instance_external_ip
        self.region = region
        self.forwarding_rule_name = forwarding_rule_name
        self.forwarding_rule = self.build_forwarding_rule()
        self.backends_migration_handlers = []

    def set_compute_engine(self):
        """ Credential setup

        Returns:google compute engine

        """
        credentials, default_project = google.auth.default()
        return discovery.build('compute', 'v1', credentials=credentials)

    def build_forwarding_rule(self):
        """ Use a helper class to create a ForwardingRule object

        Returns: a ForwardingRule object

        """
        forwarding_rule_helper = ForwardingRuleHelper(self.compute,
                                                      self.project,
                                                      self.forwarding_rule_name,
                                                      self.network_name,
                                                      self.subnetwork_name,
                                                      self.region)
        return forwarding_rule_helper.build_a_forwarding_rule()

    def migrate_an_external_regional_forwarding_rule(self):
        """ Network migration for an external regional forwarding rule.
        The external regional forwarding rule is in use by a target pool.
        So the tool will run the target pool migration.

        Returns:

        """
        target_pool_selfLink = self.forwarding_rule.target_pool_selfLink
        if target_pool_selfLink == None:
            print('No backends need to be migrated. Terminating the migration.')
            return
        selfLink_executor = SelfLinkExecutor(target_pool_selfLink,
                                             self.network_name,
                                             self.subnetwork_name,
                                             self.preserve_instance_external_ip)
        backends_migration_handler = selfLink_executor.build_target_pool_migration_handler()
        self.backends_migration_handlers.append(backends_migration_handler)
        print('Migrating the target pool.')
        backends_migration_handler.network_migration()

    def migrate_an_internal_regional_forwarding_rule(self):
        """ Network migration for an internal regional forwarding rule.
        The internal regional forwarding rule will be in use by a single
        backend service. The forwarding rule will be deleted first.
        Then, the tool will migrate the backend service.
        Finally, recreate the forwarding rule in the target subnet.

        Returns:

        """
        backend_service_selfLink = self.forwarding_rule.backend_service_selfLink
        if backend_service_selfLink != None:

            selfLink_executor = SelfLinkExecutor(backend_service_selfLink,
                                                 self.network_name,
                                                 self.subnetwork_name,
                                                 self.preserve_instance_external_ip)
            backends_migration_handler = selfLink_executor.build_backend_service_migration_handler()
            self.backends_migration_handlers.append(backends_migration_handler)
            backend_service = backends_migration_handler.backend_service

            if backend_service.count_forwarding_rules() > 1:
                print(
                    'The backend service is associated with two or more forwarding rules, so it can not be migrated.')
                print(
                    'Unable to handle the one backend service to many forwarding rule case. Terminating. ')
                return
        else:
            print('Deleting the forwarding rule.')
            self.forwarding_rule.delete_forwarding_rule()
            for backend_service_migration_handler in self.backends_migration_handlers:
                print('Migrating the backend service.')
                backend_service_migration_handler.network_migration()
            print('Recreating the forwarding rule in the target subnet.')
            self.forwarding_rule.insert_forwarding_rule(
                self.forwarding_rule.new_forwarding_rule_configs)

    def migrate_a_global_forwarding_rule(self):
        """ Network migration for a global forwarding rule. The global
        forwarding rule points to a target proxy. Through the target proxy,
        the tool can find the backend services information.
        The tool will migrate these backends one by one without deleting
        and recreating.

        Returns:

        """
        backend_service_selfLinks = self.forwarding_rule.backend_service_selfLinks
        if backend_service_selfLinks == []:
            print('No backends need to be migrated. Terminating the migration.')
            return
        for selfLink in backend_service_selfLinks:
            selfLink_executor = SelfLinkExecutor(selfLink, self.network_name,
                                                 self.subnetwork_name,
                                                 self.preserve_instance_external_ip)
            backend_service_migration_handler = selfLink_executor.build_backend_service_migration_handler()
            # Save handlers for rollback purpose
            self.backends_migration_handlers.append(
                backend_service_migration_handler)
            backend_service_migration_handler.network_migration()

    def rollback_a_global_forwarding_rule(self):
        """ Rollback a global forwarding rule

        Returns:

        """
        for backend_service_migration_handler in self.backends_migration_handlers:
            backend_service_migration_handler.rollback()

    def rollback_an_internal_regional_forwarding_rule(self):
        """ Rollback an internal regional forwarding rule

        Returns:

        """
        if not self.forwarding_rule.migrated and \
                self.forwarding_rule.check_forwarding_rule_exists():
            return
        if self.forwarding_rule.migrated:
            print('Deleting the migrated forwarding rule.')
            self.forwarding_rule.delete_forwarding_rule()
            for backend_service_migration_handler in self.backends_migration_handlers:
                backend_service_migration_handler.rollback()
            self.forwarding_rule.insert_forwarding_rule(
                 self.forwarding_rule.forwarding_rule_configs)
        else:
            for backend_service_migration_handler in self.backends_migration_handlers:
                backend_service_migration_handler.rollback()
            self.forwarding_rule.insert_forwarding_rule(
                self.forwarding_rule.forwarding_rule_configs)

    def rollback_an_external_regional_forwarding_rule(self):
        """ Rollback an external regional forwarding rule

        Returns:

        """
        for target_pool_migration_handler in self.backends_migration_handlers:
            target_pool_migration_handler.rollback()

    def network_migration(self):
        """ Select correct network migration functions based on the type of the
        forwarding rule.

        Returns:

        """
        # try:
        if isinstance(self.forwarding_rule, ExternalRegionalForwardingRule):
            self.migrate_an_external_regional_forwarding_rule()
        elif isinstance(self.forwarding_rule,
                        InternalRegionalForwardingRule):
            self.migrate_an_internal_regional_forwarding_rule()
        elif isinstance(self.forwarding_rule, GlobalForwardingRule):
            self.migrate_a_global_forwarding_rule()
        # except Exception as e:
        #     warnings.warn(e, Warning)
        #     self.rollback()

    def rollback(self):
        """ Error happens. Rollback to the original status.

        """
        print(
            'The migration is failed. Rolling back to the original forwarding rule settings.')
        if isinstance(self.forwarding_rule, ExternalRegionalForwardingRule):
            self.rollback_an_external_regional_forwarding_rule()
        elif isinstance(self.forwarding_rule, InternalRegionalForwardingRule):
            self.rollback_an_internal_regional_forwarding_rule()
        elif isinstance(self.forwarding_rule, GlobalForwardingRule):
            self.rollback_a_global_forwarding_rule()
        print('Rollback has been finished.')
<commit_msg>uncomment rollback part in network_migration func<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

""" ForwardingRuleMigration class: It is the handler to migrate
a forwarding rule based on the type of it.

"""
import google.auth
from googleapiclient import discovery
from vm_network_migration.handler_helper.selfLink_executor import SelfLinkExecutor
from vm_network_migration.module_helpers.forwarding_rule_helper import ForwardingRuleHelper
from vm_network_migration.modules.external_regional_forwarding_rule import ExternalRegionalForwardingRule
from vm_network_migration.modules.global_forwarding_rule import GlobalForwardingRule
from vm_network_migration.modules.internal_regional_forwarding_rule import InternalRegionalForwardingRule
import warnings
from vm_network_migration.errors import *

class ForwardingRuleMigration(object):
    def __init__(self, project, forwarding_rule_name,
                 network_name, subnetwork_name,
                 preserve_instance_external_ip, region=None):
        """ Initialize a InstanceNetworkMigration object

        Args:
            project: project ID
            forwarding_rule_name: name of the forwarding rule
            network: target network
            subnetwork: target subnet
            preserve_instance_external_ip: whether preserve the external IP
            of the instances which serves this load balancer
            region: region of the internal load balancer
        """
        self.compute = self.set_compute_engine()
        self.project = project
        self.network_name = network_name
        self.subnetwork_name = subnetwork_name
        self.preserve_instance_external_ip = preserve_instance_external_ip
        self.region = region
        self.forwarding_rule_name = forwarding_rule_name
        self.forwarding_rule = self.build_forwarding_rule()
        self.backends_migration_handlers = []

    def set_compute_engine(self):
        """ Credential setup

        Returns:google compute engine

        """
        credentials, default_project = google.auth.default()
        return discovery.build('compute', 'v1', credentials=credentials)

    def build_forwarding_rule(self):
        """ Use a helper class to create a ForwardingRule object

        Returns: a ForwardingRule object

        """
        forwarding_rule_helper = ForwardingRuleHelper(self.compute,
                                                      self.project,
                                                      self.forwarding_rule_name,
                                                      self.network_name,
                                                      self.subnetwork_name,
                                                      self.region)
        return forwarding_rule_helper.build_a_forwarding_rule()

    def migrate_an_external_regional_forwarding_rule(self):
        """ Network migration for an external regional forwarding rule.
        The external regional forwarding rule is in use by a target pool.
        So the tool will run the target pool migration.

        Returns:

        """
        target_pool_selfLink = self.forwarding_rule.target_pool_selfLink
        if target_pool_selfLink == None:
            print('No backends need to be migrated. Terminating the migration.')
            return
        selfLink_executor = SelfLinkExecutor(target_pool_selfLink,
                                             self.network_name,
                                             self.subnetwork_name,
                                             self.preserve_instance_external_ip)
        backends_migration_handler = selfLink_executor.build_target_pool_migration_handler()
        self.backends_migration_handlers.append(backends_migration_handler)
        print('Migrating the target pool.')
        backends_migration_handler.network_migration()

    def migrate_an_internal_regional_forwarding_rule(self):
        """ Network migration for an internal regional forwarding rule.
        The internal regional forwarding rule will be in use by a single
        backend service. The forwarding rule will be deleted first.
        Then, the tool will migrate the backend service.
        Finally, recreate the forwarding rule in the target subnet.

        Returns:

        """
        backend_service_selfLink = self.forwarding_rule.backend_service_selfLink
        if backend_service_selfLink != None:

            selfLink_executor = SelfLinkExecutor(backend_service_selfLink,
                                                 self.network_name,
                                                 self.subnetwork_name,
                                                 self.preserve_instance_external_ip)
            backends_migration_handler = selfLink_executor.build_backend_service_migration_handler()
            self.backends_migration_handlers.append(backends_migration_handler)
            backend_service = backends_migration_handler.backend_service

            if backend_service.count_forwarding_rules() > 1:
                print(
                    'The backend service is associated with two or more forwarding rules, so it can not be migrated.')
                print(
                    'Unable to handle the one backend service to many forwarding rule case. Terminating. ')
                return
        else:
            print('Deleting the forwarding rule.')
            self.forwarding_rule.delete_forwarding_rule()
            for backend_service_migration_handler in self.backends_migration_handlers:
                print('Migrating the backend service.')
                backend_service_migration_handler.network_migration()
            print('Recreating the forwarding rule in the target subnet.')
            self.forwarding_rule.insert_forwarding_rule(
                self.forwarding_rule.new_forwarding_rule_configs)

    def migrate_a_global_forwarding_rule(self):
        """ Network migration for a global forwarding rule. The global
        forwarding rule points to a target proxy. Through the target proxy,
        the tool can find the backend services information.
        The tool will migrate these backends one by one without deleting
        and recreating.

        Returns:

        """
        backend_service_selfLinks = self.forwarding_rule.backend_service_selfLinks
        if backend_service_selfLinks == []:
            print('No backends need to be migrated. Terminating the migration.')
            return
        for selfLink in backend_service_selfLinks:
            selfLink_executor = SelfLinkExecutor(selfLink, self.network_name,
                                                 self.subnetwork_name,
                                                 self.preserve_instance_external_ip)
            backend_service_migration_handler = selfLink_executor.build_backend_service_migration_handler()
            # Save handlers for rollback purpose
            self.backends_migration_handlers.append(
                backend_service_migration_handler)
            backend_service_migration_handler.network_migration()

    def rollback_a_global_forwarding_rule(self):
        """ Rollback a global forwarding rule

        Returns:

        """
        for backend_service_migration_handler in self.backends_migration_handlers:
            backend_service_migration_handler.rollback()

    def rollback_an_internal_regional_forwarding_rule(self):
        """ Rollback an internal regional forwarding rule

        Returns:

        """
        if not self.forwarding_rule.migrated and \
                self.forwarding_rule.check_forwarding_rule_exists():
            return
        if self.forwarding_rule.migrated:
            print('Deleting the migrated forwarding rule.')
            self.forwarding_rule.delete_forwarding_rule()
            for backend_service_migration_handler in self.backends_migration_handlers:
                backend_service_migration_handler.rollback()
            self.forwarding_rule.insert_forwarding_rule(
                 self.forwarding_rule.forwarding_rule_configs)
        else:
            for backend_service_migration_handler in self.backends_migration_handlers:
                backend_service_migration_handler.rollback()
            self.forwarding_rule.insert_forwarding_rule(
                self.forwarding_rule.forwarding_rule_configs)

    def rollback_an_external_regional_forwarding_rule(self):
        """ Rollback an external regional forwarding rule

        Returns:

        """
        for target_pool_migration_handler in self.backends_migration_handlers:
            target_pool_migration_handler.rollback()

    def network_migration(self):
        """ Select correct network migration functions based on the type of the
        forwarding rule.

        Returns:

        """
        try:
            if isinstance(self.forwarding_rule, ExternalRegionalForwardingRule):
                self.migrate_an_external_regional_forwarding_rule()
            elif isinstance(self.forwarding_rule,
                            InternalRegionalForwardingRule):
                self.migrate_an_internal_regional_forwarding_rule()
            elif isinstance(self.forwarding_rule, GlobalForwardingRule):
                self.migrate_a_global_forwarding_rule()
        except Exception as e:
            warnings.warn(e, Warning)
            print(
                'The migration is failed. Rolling back to the original instance group.')
            self.rollback()
            raise MigrationFailed('Rollback has been finished.')

    def rollback(self):
        """ Error happens. Rollback to the original status.

        """
        print(
            'The migration was failed. Rolling back to the original forwarding rule settings.')
        if isinstance(self.forwarding_rule, ExternalRegionalForwardingRule):
            self.rollback_an_external_regional_forwarding_rule()
        elif isinstance(self.forwarding_rule, InternalRegionalForwardingRule):
            self.rollback_an_internal_regional_forwarding_rule()
        elif isinstance(self.forwarding_rule, GlobalForwardingRule):
            self.rollback_a_global_forwarding_rule()
        print('Rollback has been finished.')
<|endoftext|>


<commit_before>"""SCons.Tool.ifl

Tool-specific initialization for the Intel Fortran compiler.

There normally shouldn't be any need to import this module directly.
It will usually be imported through the generic SCons.Tool.Tool()
selection method.

"""

#
# __COPYRIGHT__
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#

__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"

import SCons.Defaults
from SCons.Scanner.Fortran import FortranScan
from FortranCommon import add_all_to_env

def generate(env):
    """Add Builders and construction variables for ifl to an Environment."""
    fscan = FortranScan("FORTRANPATH")
    SCons.Tool.SourceFileScanner.add_scanner('.i', fscan)
    SCons.Tool.SourceFileScanner.add_scanner('.i90', fscan)
    fortran.FortranSuffixes.extend(['.i90'])
    fortran.generate(env)

    if not env.has_key('FORTRANFILESUFFIXES'):
        env['FORTRANFILESUFFIXES'] = ['.i']
    else:
        env['FORTRANFILESUFFIXES'].append('.i')

    if not env.has_key('F90FILESUFFIXES'):
        env['F90FILESUFFIXES'] = ['.i90']
    else:
        env['F90FILESUFFIXES'].append('.i90')

    add_all_to_env(env)

    env['FORTRAN']        = 'ifl'
    env['SHFORTRAN']      = '$FORTRAN'
    env['FORTRANCOM']     = '$FORTRAN $FORTRANFLAGS $_FORTRANINCFLAGS /c $SOURCES /Fo$TARGET'
    env['FORTRANPPCOM']   = '$FORTRAN $FORTRANFLAGS $CPPFLAGS $_CPPDEFFLAGS $_FORTRANINCFLAGS /c $SOURCES /Fo$TARGET'
    env['SHFORTRANCOM']   = '$SHFORTRAN $SHFORTRANFLAGS $_FORTRANINCFLAGS /c $SOURCES /Fo$TARGET'
    env['SHFORTRANPPCOM'] = '$SHFORTRAN $SHFORTRANFLAGS $CPPFLAGS $_CPPDEFFLAGS $_FORTRANINCFLAGS /c $SOURCES /Fo$TARGET'

def exists(env):
    return env.Detect('ifl')
<commit_msg>Remove left-over fortran.* lines.<commit_after>"""SCons.Tool.ifl

Tool-specific initialization for the Intel Fortran compiler.

There normally shouldn't be any need to import this module directly.
It will usually be imported through the generic SCons.Tool.Tool()
selection method.

"""

#
# __COPYRIGHT__
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#

__revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__"

import SCons.Defaults
from SCons.Scanner.Fortran import FortranScan
from FortranCommon import add_all_to_env

def generate(env):
    """Add Builders and construction variables for ifl to an Environment."""
    fscan = FortranScan("FORTRANPATH")
    SCons.Tool.SourceFileScanner.add_scanner('.i', fscan)
    SCons.Tool.SourceFileScanner.add_scanner('.i90', fscan)

    if not env.has_key('FORTRANFILESUFFIXES'):
        env['FORTRANFILESUFFIXES'] = ['.i']
    else:
        env['FORTRANFILESUFFIXES'].append('.i')

    if not env.has_key('F90FILESUFFIXES'):
        env['F90FILESUFFIXES'] = ['.i90']
    else:
        env['F90FILESUFFIXES'].append('.i90')

    add_all_to_env(env)

    env['FORTRAN']        = 'ifl'
    env['SHFORTRAN']      = '$FORTRAN'
    env['FORTRANCOM']     = '$FORTRAN $FORTRANFLAGS $_FORTRANINCFLAGS /c $SOURCES /Fo$TARGET'
    env['FORTRANPPCOM']   = '$FORTRAN $FORTRANFLAGS $CPPFLAGS $_CPPDEFFLAGS $_FORTRANINCFLAGS /c $SOURCES /Fo$TARGET'
    env['SHFORTRANCOM']   = '$SHFORTRAN $SHFORTRANFLAGS $_FORTRANINCFLAGS /c $SOURCES /Fo$TARGET'
    env['SHFORTRANPPCOM'] = '$SHFORTRAN $SHFORTRANFLAGS $CPPFLAGS $_CPPDEFFLAGS $_FORTRANINCFLAGS /c $SOURCES /Fo$TARGET'

def exists(env):
    return env.Detect('ifl')
<|endoftext|>


<commit_before>#!/usr/bin/env python2
# -*- coding: utf-8 -*-

import re
import os
import sys
import argparse
import codecs
from collections import namedtuple

try:
    from typing import Iterable, Tuple, Match, Any, TypeVar
    T = TypeVar("T")
except ImportError:
    pass

# Utilities
# ---------

FST_DIRECTIVE_RE = re.compile("^ *.. fst::")
INDENTATION_RE = re.compile("^ *")

def measure_indent(line): # type: (str,) -> int
    return INDENTATION_RE.match(line).end()

Line = namedtuple('Line', 'raw marker_pos clean')

def empty(line):
    return line.clean == ""

def mkLine(raw, marker): # type: (str, str) -> Line
    if marker:
        return Line(raw, raw.find(marker), raw.replace(marker, "", 1))
    else:
        return Line(raw, -1, raw)

def strip_prefix(line, marker, prefix_len): # type: (Line, str, int) -> str
    if line.marker_pos >= 0 and line.marker_pos < prefix_len:
        return marker + line.raw[prefix_len + len(marker):]
    else:
        return line.raw[prefix_len:]

# reStructuredText → F*
# ---------------------

def rst2fst(rawlines, marker): # type: (Iterable[str], str) -> Iterable[str]
    idx, lines = 0, [mkLine(raw.rstrip(), marker) for raw in rawlines]

    # Each iteration processes a sequence of text + optional header + code
    first_round = True
    while idx < len(lines):
        output = [] # type: List[Tuple[Line, str]]
        fst_directive = None # type: Match[str]
        prev_indentation, indentation = None, None # type: Tuple[int, int]

        # Skip until start of code block
        while idx < len(lines):
            line = lines[idx]
            if not empty(line):
                prev_indentation = indentation
                indentation = measure_indent(line.clean)
            fst_directive = FST_DIRECTIVE_RE.match(line.clean)
            if fst_directive:
                break
            output.append((line, "///" + (" " + line.raw if line.raw else "")))
            idx += 1

        if idx >= len(lines):
            for _, o in output:
                yield o
            break

        # Skip until actual code
        dropped_lines = []
        has_direct_opts = fst_directive.end() < len(lines[idx].clean)
        has_further_opts = idx + 1 < len(lines) and not empty(lines[idx + 1])
        is_top_of_file = first_round and prev_indentation is None
        has_distinct_indentation = indentation != prev_indentation and not is_top_of_file
        keep_header = has_direct_opts or has_further_opts or has_distinct_indentation
        while idx < len(lines):
            line = lines[idx]
            if empty(line):
                break
            if keep_header:
                output.append((line, "/// " + line.raw))
            elif line.marker_pos >= 0:
                dropped_lines.append(marker)
            idx += 1

        # Get rid of extra empty lines
        while output and empty(output[-1][0]):
            dropped_lines.append(output.pop()[0].raw)
        while idx < len(lines):
            line = lines[idx]
            if not empty(line):
                break
            dropped_lines.append(line.raw)
            idx += 1
        dropped_markers = "".join(dropped_lines)

        # Stream partial results
        for _, o in output:
            yield o

        # Dump markers found in dropped lines (whitespace and ‘.. fst::’)
        if not is_top_of_file:
            yield dropped_markers
            dropped_markers = ""

        # Dump actual code, dedented
        code_block_offset = indentation + 3
        while idx < len(lines):
            line = lines[idx]
            indentation = measure_indent(line.clean)
            if indentation < code_block_offset and not empty(line):
                break
            yield dropped_markers + strip_prefix(line, marker, code_block_offset)
            dropped_markers = ""
            idx += 1

        if dropped_markers:
            yield dropped_markers
        first_round = False

# F* → reStructuredText
# ---------------------

RST_LINE_MARKER_RE = re.compile("^///( |$)")
F2R_LITERATE_COMMENT, F2R_FST_DIRECTIVE, F2R_CODE = range(3)

def fst2rst_classify(line, is_rst): # type: (Line, bool) -> int
    if is_rst:
        if FST_DIRECTIVE_RE.match(line.clean):
            return F2R_FST_DIRECTIVE
        return F2R_LITERATE_COMMENT
    return F2R_CODE

def fst2rst_annotate(raw, marker): # type: (str, str) -> Tuple[int, Line]
    line = mkLine(raw.rstrip(), marker)
    rst_prefix = RST_LINE_MARKER_RE.match(line.clean)
    if rst_prefix:
        clean = line.clean[rst_prefix.end():]
        raw = strip_prefix(line, marker, rst_prefix.end())
        line = Line(raw, line.marker_pos, clean)
    kind = fst2rst_classify(line, bool(rst_prefix))
    return kind, line

def fst2rst_linums(rawlines, marker): # type: Iterable[str] -> Iterable[Tuple[int, str]]
    if not rawlines:
        return

    idx = 0
    kinds, lines = zip(*(fst2rst_annotate(raw, marker) for raw in rawlines))

    # Each iteration processes a sequence of text + optional header + code
    while idx < len(lines):
        rst_indentation = 0
        existing_header_indentation = None # type: int

        # Uncomment literate comments
        while idx < len(lines):
            kind, line = kinds[idx], lines[idx]
            if kind == F2R_CODE:
                break
            indentation = measure_indent(line.clean)
            if kind == F2R_FST_DIRECTIVE:
                existing_header_indentation = indentation
            if not empty(line):
                rst_indentation = indentation
            yield idx, line.raw
            idx += 1

        # Skip blank lines
        while idx < len(lines) and empty(lines[idx]):
            yield idx, lines[idx].raw
            idx += 1

        if idx >= len(lines):
            break

        # Was this an empty code section?
        if kinds[idx] != F2R_CODE:
            continue

        # Emit code header
        prev_line_empty = idx == 0 or empty(lines[idx - 1])
        if existing_header_indentation is None:
            if not prev_line_empty:
                yield idx, "" # Empty line before ‘.. fst::’
            yield idx, " " * rst_indentation + ".. fst::"
            prev_line_empty = False
            existing_header_indentation = rst_indentation
        if not empty(lines[idx]) and not prev_line_empty:
            yield idx, "" # Empty line after ‘.. fst::’

        # Emit actual code
        while idx < len(lines):
            kind, line = kinds[idx], lines[idx]
            if kind != F2R_CODE:
                break
            indent = "" if empty(line) else " " * (3 + existing_header_indentation)
            yield idx, indent + line.raw
            idx += 1

def fst2rst(rawlines, markers): # type: Iterable[str] -> Iterable[str]
    for _, line in fst2rst_linums(rawlines, markers):
        yield line

# Command-line interface
# ----------------------

def parse_args():
    parser = argparse.ArgumentParser(description="Convert a literate F* file to reStructuredText, and vice-versa.")
    group = parser.add_mutually_exclusive_group()
    group.add_argument("--fst2rst", dest="fn",
                       action="store_const", const=fst2rst,
                       help="Convert from F* to reStructuredText.")
    group.add_argument("--rst2fst", dest="fn",
                       action="store_const", const=rst2fst,
                       help="Convert from reStructuredText to F*.")
    parser.add_argument("--marker",
                        help="Special string to ignore while analyzing INPUT.")
    parser.add_argument("input", nargs="?", default="-")

    args = parser.parse_args()
    if args.input == "-":
        if not args.fn:
            parser.error("Reading from standard input requires one of --fst2rst, --rst2fst.")
    else:
        _, ext = os.path.splitext(args.input)
        args.fn = {".fst": fst2rst, ".rst": rst2fst}.get(ext)
        if not args.fn:
            parser.error("Unexpected file extension: "
                         "expected '.rst' or '.fst', got '{}'.".format(ext))

    return args

def writeout(lines):
    stdout = codecs.getwriter('utf-8')(sys.stdout)
    for line in lines:
        stdout.write(line)
        stdout.write("\n")

def run(translator, fname, marker):
    if fname == "-":
        writeout(translator(codecs.getreader('utf-8')(sys.stdin), marker))
    else:
        with codecs.open(fname, encoding="utf-8") as fstream:
            writeout(translator(fstream, marker))

def main():
    args = parse_args()
    run(args.fn, args.input, args.marker)

if __name__ == '__main__':
    main()
<commit_msg>fslit: Ensure that code blocks are followed by at least one newline<commit_after>#!/usr/bin/env python2
# -*- coding: utf-8 -*-

import re
import os
import sys
import argparse
import codecs
from collections import namedtuple

try:
    from typing import Iterable, Tuple, Match, Any, TypeVar
    T = TypeVar("T")
except ImportError:
    pass

# Utilities
# ---------

FST_DIRECTIVE_RE = re.compile("^ *.. fst::")
INDENTATION_RE = re.compile("^ *")

def measure_indent(line): # type: (str,) -> int
    return INDENTATION_RE.match(line).end()

Line = namedtuple('Line', 'raw marker_pos clean')

def empty(line):
    return line.clean == ""

def mkLine(raw, marker): # type: (str, str) -> Line
    if marker:
        return Line(raw, raw.find(marker), raw.replace(marker, "", 1))
    else:
        return Line(raw, -1, raw)

def strip_prefix(line, marker, prefix_len): # type: (Line, str, int) -> str
    if line.marker_pos >= 0 and line.marker_pos < prefix_len:
        return marker + line.raw[prefix_len + len(marker):]
    else:
        return line.raw[prefix_len:]

# reStructuredText → F*
# ---------------------

def rst2fst(rawlines, marker): # type: (Iterable[str], str) -> Iterable[str]
    idx, lines = 0, [mkLine(raw.rstrip(), marker) for raw in rawlines]

    # Each iteration processes a sequence of text + optional header + code
    first_round = True
    while idx < len(lines):
        output = [] # type: List[Tuple[Line, str]]
        fst_directive = None # type: Match[str]
        prev_indentation, indentation = None, None # type: Tuple[int, int]

        # Skip until start of code block
        while idx < len(lines):
            line = lines[idx]
            if not empty(line):
                prev_indentation = indentation
                indentation = measure_indent(line.clean)
            fst_directive = FST_DIRECTIVE_RE.match(line.clean)
            if fst_directive:
                break
            output.append((line, "///" + (" " + line.raw if line.raw else "")))
            idx += 1

        if idx >= len(lines):
            for _, o in output:
                yield o
            break

        # Skip until actual code
        dropped_lines = []
        has_direct_opts = fst_directive.end() < len(lines[idx].clean)
        has_further_opts = idx + 1 < len(lines) and not empty(lines[idx + 1])
        is_top_of_file = first_round and prev_indentation is None
        has_distinct_indentation = indentation != prev_indentation and not is_top_of_file
        keep_header = has_direct_opts or has_further_opts or has_distinct_indentation
        while idx < len(lines):
            line = lines[idx]
            if empty(line):
                break
            if keep_header:
                output.append((line, "/// " + line.raw))
            elif line.marker_pos >= 0:
                dropped_lines.append(marker)
            idx += 1

        # Get rid of extra empty lines
        while output and empty(output[-1][0]):
            dropped_lines.append(output.pop()[0].raw)
        while idx < len(lines):
            line = lines[idx]
            if not empty(line):
                break
            dropped_lines.append(line.raw)
            idx += 1
        dropped_markers = "".join(dropped_lines)

        # Stream partial results
        for _, o in output:
            yield o

        # Dump markers found in dropped lines (whitespace and ‘.. fst::’)
        if not is_top_of_file:
            yield dropped_markers
            dropped_markers = ""

        # Dump actual code, dedented
        code_block_offset = indentation + 3
        while idx < len(lines):
            line = lines[idx]
            indentation = measure_indent(line.clean)
            if indentation < code_block_offset and not empty(line):
                break
            yield dropped_markers + strip_prefix(line, marker, code_block_offset)
            dropped_markers = ""
            idx += 1

        if dropped_markers:
            yield dropped_markers
        first_round = False

# F* → reStructuredText
# ---------------------

RST_LINE_MARKER_RE = re.compile("^///( |$)")
F2R_LITERATE_COMMENT, F2R_FST_DIRECTIVE, F2R_CODE = range(3)

def fst2rst_classify(line, is_rst): # type: (Line, bool) -> int
    if is_rst:
        if FST_DIRECTIVE_RE.match(line.clean):
            return F2R_FST_DIRECTIVE
        return F2R_LITERATE_COMMENT
    return F2R_CODE

def fst2rst_annotate(raw, marker): # type: (str, str) -> Tuple[int, Line]
    line = mkLine(raw.rstrip(), marker)
    rst_prefix = RST_LINE_MARKER_RE.match(line.clean)
    if rst_prefix:
        clean = line.clean[rst_prefix.end():]
        raw = strip_prefix(line, marker, rst_prefix.end())
        line = Line(raw, line.marker_pos, clean)
    kind = fst2rst_classify(line, bool(rst_prefix))
    return kind, line

def fst2rst_linums(rawlines, marker): # type: Iterable[str] -> Iterable[Tuple[int, str]]
    if not rawlines:
        return

    idx = 0
    kinds, lines = zip(*(fst2rst_annotate(raw, marker) for raw in rawlines))

    # Each iteration processes a sequence of text + optional header + code
    while idx < len(lines):
        rst_indentation = 0
        existing_header_indentation = None # type: int

        # Uncomment literate comments
        while idx < len(lines):
            kind, line = kinds[idx], lines[idx]
            if kind == F2R_CODE:
                break
            indentation = measure_indent(line.clean)
            if kind == F2R_FST_DIRECTIVE:
                existing_header_indentation = indentation
            if not empty(line):
                rst_indentation = indentation
            yield idx, line.raw
            idx += 1

        # Skip blank lines
        while idx < len(lines) and empty(lines[idx]):
            yield idx, lines[idx].raw
            idx += 1

        if idx >= len(lines):
            break

        # Was this an empty code section?
        if kinds[idx] != F2R_CODE:
            continue

        # Emit code header
        prev_line_empty = idx == 0 or empty(lines[idx - 1])
        if existing_header_indentation is None:
            if not prev_line_empty:
                yield idx, "" # Empty line before ‘.. fst::’
            yield idx, " " * rst_indentation + ".. fst::"
            prev_line_empty = False
            existing_header_indentation = rst_indentation
        if not empty(lines[idx]) and not prev_line_empty:
            yield idx, "" # Empty line after ‘.. fst::’

        # Emit actual code
        while idx < len(lines):
            kind, line = kinds[idx], lines[idx]
            if kind != F2R_CODE:
                break
            indent = "" if empty(line) else " " * (3 + existing_header_indentation)
            yield idx, indent + line.raw
            idx += 1

        if not empty(lines[idx - 1]):
            yield idx - 1, "" # Empty line after the code block

def fst2rst(rawlines, markers): # type: Iterable[str] -> Iterable[str]
    for _, line in fst2rst_linums(rawlines, markers):
        yield line

# Command-line interface
# ----------------------

def parse_args():
    parser = argparse.ArgumentParser(description="Convert a literate F* file to reStructuredText, and vice-versa.")
    group = parser.add_mutually_exclusive_group()
    group.add_argument("--fst2rst", dest="fn",
                       action="store_const", const=fst2rst,
                       help="Convert from F* to reStructuredText.")
    group.add_argument("--rst2fst", dest="fn",
                       action="store_const", const=rst2fst,
                       help="Convert from reStructuredText to F*.")
    parser.add_argument("--marker",
                        help="Special string to ignore while analyzing INPUT.")
    parser.add_argument("input", nargs="?", default="-")

    args = parser.parse_args()
    if args.input == "-":
        if not args.fn:
            parser.error("Reading from standard input requires one of --fst2rst, --rst2fst.")
    else:
        _, ext = os.path.splitext(args.input)
        args.fn = {".fst": fst2rst, ".rst": rst2fst}.get(ext)
        if not args.fn:
            parser.error("Unexpected file extension: "
                         "expected '.rst' or '.fst', got '{}'.".format(ext))

    return args

def writeout(lines):
    stdout = codecs.getwriter('utf-8')(sys.stdout)
    for line in lines:
        stdout.write(line)
        stdout.write("\n")

def run(translator, fname, marker):
    if fname == "-":
        writeout(translator(codecs.getreader('utf-8')(sys.stdin), marker))
    else:
        with codecs.open(fname, encoding="utf-8") as fstream:
            writeout(translator(fstream, marker))

def main():
    args = parse_args()
    run(args.fn, args.input, args.marker)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# checkupdates.py - part of the FDroid server tools
# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys
import os
import re
import urllib2
import time
import subprocess
from optparse import OptionParser
import traceback
import HTMLParser
import common
from common import BuildException
from common import VCSException

# Check for a new version by looking at the tags in the source repo.
# Whether this can be used reliably or not depends on
# the development procedures used by the project's developers. Use it with
# caution, because it's inappropriate for many projects.
# Returns (None, "a message") if this didn't work, or (version, vercode) for
# the details of the current version.
def check_tags(app, sdk_path):

    try:

        if app['Repo Type'] == 'srclib':
            build_dir = os.path.join('build', 'srclib')
            repotype = common.getsrclibvcs(app['Repo'])
        else:
            build_dir = os.path.join('build/', app['id'])
            repotype = app['Repo Type']

        if repotype not in ('git', 'git-svn'):
            return (None, 'Tags update mode only works for git and git-svn repositories currently')

        # Set up vcs interface and make sure we have the latest code...
        vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path)

        if app['Repo Type'] == 'srclib':
            build_dir = os.path.join(build_dir, app['Repo'])

        vcs.gotorevision(None)

        if len(app['builds']) > 0:
            if 'subdir' in app['builds'][-1]:
                build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])

        hver = None
        hcode = "0"

        for tag in vcs.gettags():
            vcs.gotorevision(tag)

            # Only process tags where the manifest exists...
            if os.path.exists(os.path.join(build_dir, 'AndroidManifest.xml')):
                version, vercode, package = common.parse_androidmanifest(build_dir)
                if package and package == app['id'] and version and vercode:
                    if int(vercode) > int(hcode):
                        hcode = str(int(vercode))
                        hver = version

        if hver:
            return (hver, hcode)
        return (None, "Couldn't find any version information")

    except BuildException as be:
        msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
        return (None, msg)
    except VCSException as vcse:
        msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
        return (None, msg)
    except Exception:
        msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
        return (None, msg)

# Check for a new version by looking at the AndroidManifest.xml at the HEAD
# of the source repo. Whether this can be used reliably or not depends on
# the development procedures used by the project's developers. Use it with
# caution, because it's inappropriate for many projects.
# Returns (None, "a message") if this didn't work, or (version, vercode) for
# the details of the current version.
def check_repomanifest(app, sdk_path, branch=None):

    try:

        if app['Repo Type'] == 'srclib':
            build_dir = os.path.join('build', 'srclib')
            repotype = common.getsrclibvcs(app['Repo'])
        else:
            build_dir = os.path.join('build/', app['id'])
            repotype = app['Repo Type']

        # Set up vcs interface and make sure we have the latest code...
        vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path)
        if app['Repo Type'] == 'srclib':
            build_dir = os.path.join(build_dir, app['Repo'])

        if vcs.repotype() == 'git':
            if branch:
                vcs.gotorevision('origin/'+branch)
            else:
                vcs.gotorevision('origin/master')
                pass
        elif vcs.repotype() == 'git-svn':
            if branch:
                vcs.gotorevision(branch)
            else:
                vcs.gotorevision(None)
        elif vcs.repotype() == 'svn':
            vcs.gotorevision(None)
        elif vcs.repotype() == 'hg':
            if branch:
                vcs.gotorevision(branch)
            else:
                vcs.gotorevision('default')
        elif vcs.repotype() == 'bzr':
            vcs.gotorevision(None)

        if len(app['builds']) > 0:
            if 'subdir' in app['builds'][-1]:
                build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])

        if not os.path.isdir(build_dir):
            return (None, "Subdir '" + app['builds'][-1]['subdir'] + "'is not a valid directory")

        version, vercode, package = common.parse_androidmanifest(build_dir)
        if not package:
            return (None, "Couldn't find package ID")
        if package != app['id']:
            return (None, "Package ID mismatch")
        if not version:
            return (None,"Couldn't find latest version name")
        if not vercode:
            return (None,"Couldn't find latest version code")

        return (version, str(int(vercode)))

    except BuildException as be:
        msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
        return (None, msg)
    except VCSException as vcse:
        msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
        return (None, msg)
    except Exception:
        msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
        return (None, msg)

# Check for a new version by looking at the Google market.
# Returns (None, "a message") if this didn't work, or (version, vercode) for
# the details of the current version.
def check_market(app):
    time.sleep(15)
    url = 'https://play.google.com/store/apps/details?id=' + app['id']
    headers = {'User-Agent' : 'Mozilla/5.0 (X11; Linux i686; rv:18.0) Gecko/20100101 Firefox/18.0'}
    req = urllib2.Request(url, None, headers)
    try:
        resp = urllib2.urlopen(req, None, 20)
        page = resp.read()
    except urllib2.HTTPError, e:
        if e.code == 404:
            return (None, 'Not in market')
        else:
            return (None, 'Failed with HTTP status' + str(req.getcode()))
    except Exception, e:
        return (None, 'Failed:' + str(e))

    version = None
    vercode = None

    m = re.search('itemprop="softwareVersion">[ ]*([^<]+)[ ]*
', page)
    if m:
        html_parser = HTMLParser.HTMLParser()
        version = html_parser.unescape(m.group(1))

    if version == 'Varies with device':
        return (None, 'Device-variable version, cannot use this method')

    m = re.search('data-paramValue="(\d+)">
Latest Version<', page)
    if m:
        vercode = m.group(1)

    if not vercode:
        return (None, "Couldn't find version code")
    if not version:
        return (None, "Couldn't find version")
    return (version, str(int(vercode)))

def main():

    #Read configuration...
    execfile('config.py', globals())

    # Parse command line...
    parser = OptionParser()
    parser.add_option("-v", "--verbose", action="store_true", default=False,
                      help="Spew out even more information than normal")
    parser.add_option("-p", "--package", default=None,
                      help="Check only the specified package")
    parser.add_option("--auto", action="store_true", default=False,
                      help="Process auto-updates")
    parser.add_option("--autoonly", action="store_true", default=False,
                      help="Only process apps with auto-updates")
    parser.add_option("--commit", action="store_true", default=False,
                      help="Commit changes")
    (options, args) = parser.parse_args()

    # Get all apps...
    apps = common.read_metadata(options.verbose)

    # Filter apps according to command-line options
    if options.package:
        apps = [app for app in apps if app['id'] == options.package]
        if len(apps) == 0:
            print "No such package"
            sys.exit(1)

    for app in apps:

        process = True

        if options.autoonly and app['Auto Update Mode'] == 'None':
            process = False

        if process:

            print "Processing " + app['id'] + '...'

            writeit = False
            logmsg = None

            mode = app['Update Check Mode']
            if mode == 'Market':
                (version, vercode) = check_market(app)
            elif mode == 'Tags':
                (version, vercode) = check_tags(app, sdk_path)
            elif mode == 'RepoManifest':
                (version, vercode) = check_repomanifest(app, sdk_path)
            elif mode.startswith('RepoManifest/'):
                (version, vercode) = check_repomanifest(app, sdk_path, mode[13:])
            elif mode == 'Static':
                version = None
                vercode = 'Checking disabled'
            elif mode == 'None':
                version = None
                vercode = 'Checking disabled'
            else:
                version = None
                vercode = 'Invalid update check method'

            if not version:
                print "..." + vercode
            elif vercode == app['Current Version Code'] and version == app['Current Version']:
                print "...up to date"
            else:
                print '...updating to version:' + version + ' vercode:' + vercode
                app['Current Version'] = version
                app['Current Version Code'] = str(int(vercode))
                writeit = True
                logmsg = "Update current version of " + app['id'] + " to " + version

            # Do the Auto Name thing...
            if len(app["Repo Type"]) > 0:

                try:

                    if app['Repo Type'] == 'srclib':
                        app_dir = os.path.join('build', 'srclib', app['Repo'])
                    else:
                        app_dir = os.path.join('build/', app['id'])

                    vcs = common.getvcs(app["Repo Type"], app["Repo"], app_dir, sdk_path)
                    vcs.gotorevision(None)

                    if len(app['builds']) > 0:
                        if 'subdir' in app['builds'][-1]:
                            app_dir = os.path.join(app_dir, app['builds'][-1]['subdir'])

                    new_name = common.fetch_real_name(app_dir)
                    if new_name != app['Auto Name']:
                        app['Auto Name'] = new_name
                        if not writeit:
                            writeit = True
                except Exception:
                    msg = "Auto Name failed for  %s due to exception: %s" % (app['id'], traceback.format_exc())

            if options.auto:
                mode = app['Auto Update Mode']
                if mode == 'None':
                    pass
                elif mode.startswith('Version '):
                    pattern = mode[8:]
                    if pattern.startswith('+'):
                        o = pattern.find(' ')
                        suffix = pattern[1:o]
                        pattern = pattern[o + 1:]
                    else:
                        suffix = ''
                    gotcur = False
                    latest = None
                    for build in app['builds']:
                        if build['vercode'] == app['Current Version Code']:
                            gotcur = True
                        if not latest or int(build['vercode']) > int(latest['vercode']):
                            latest = build
                    if not gotcur:
                        newbuild = latest.copy()
                        del newbuild['origlines']
                        newbuild['vercode'] = app['Current Version Code']
                        newbuild['version'] = app['Current Version'] + suffix
                        print "...auto-generating build for " + newbuild['version']
                        commit = pattern.replace('%v', newbuild['version'])
                        commit = commit.replace('%c', newbuild['vercode'])
                        newbuild['commit'] = commit
                        app['builds'].append(newbuild)
                        writeit = True
                        logmsg = "Update " + app['id'] + " to " + newbuild['version']
                else:
                    print 'Invalid auto update mode'

            if writeit:
                metafile = os.path.join('metadata', app['id'] + '.txt')
                common.write_metadata(metafile, app)
                if options.commit and logmsg:
                    if subprocess.call("git add " + metafile, shell=True) != 0:
                        print "Git add failed"
                        sys.exit(1)
                    if subprocess.call("git commit -m '" + logmsg.replace("'", "\\'") +  "'", shell=True) != 0:
                        print "Git commit failed"
                        sys.exit(1)

    print "Finished."

if __name__ == "__main__":
    main()

<commit_msg>Use new manifest path function in checkupdates<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# checkupdates.py - part of the FDroid server tools
# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

import sys
import os
import re
import urllib2
import time
import subprocess
from optparse import OptionParser
import traceback
import HTMLParser
import common
from common import BuildException
from common import VCSException

# Check for a new version by looking at the tags in the source repo.
# Whether this can be used reliably or not depends on
# the development procedures used by the project's developers. Use it with
# caution, because it's inappropriate for many projects.
# Returns (None, "a message") if this didn't work, or (version, vercode) for
# the details of the current version.
def check_tags(app, sdk_path):

    try:

        if app['Repo Type'] == 'srclib':
            build_dir = os.path.join('build', 'srclib')
            repotype = common.getsrclibvcs(app['Repo'])
        else:
            build_dir = os.path.join('build/', app['id'])
            repotype = app['Repo Type']

        if repotype not in ('git', 'git-svn'):
            return (None, 'Tags update mode only works for git and git-svn repositories currently')

        # Set up vcs interface and make sure we have the latest code...
        vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path)

        if app['Repo Type'] == 'srclib':
            build_dir = os.path.join(build_dir, app['Repo'])

        vcs.gotorevision(None)

        if len(app['builds']) > 0:
            if 'subdir' in app['builds'][-1]:
                build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])

        hver = None
        hcode = "0"

        for tag in vcs.gettags():
            vcs.gotorevision(tag)

            # Only process tags where the manifest exists...
            if os.path.exists(common.manifest_path(build_dir)):
                version, vercode, package = common.parse_androidmanifest(build_dir)
                if package and package == app['id'] and version and vercode:
                    if int(vercode) > int(hcode):
                        hcode = str(int(vercode))
                        hver = version

        if hver:
            return (hver, hcode)
        return (None, "Couldn't find any version information")

    except BuildException as be:
        msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
        return (None, msg)
    except VCSException as vcse:
        msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
        return (None, msg)
    except Exception:
        msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
        return (None, msg)

# Check for a new version by looking at the AndroidManifest.xml at the HEAD
# of the source repo. Whether this can be used reliably or not depends on
# the development procedures used by the project's developers. Use it with
# caution, because it's inappropriate for many projects.
# Returns (None, "a message") if this didn't work, or (version, vercode) for
# the details of the current version.
def check_repomanifest(app, sdk_path, branch=None):

    try:

        if app['Repo Type'] == 'srclib':
            build_dir = os.path.join('build', 'srclib')
            repotype = common.getsrclibvcs(app['Repo'])
        else:
            build_dir = os.path.join('build/', app['id'])
            repotype = app['Repo Type']

        # Set up vcs interface and make sure we have the latest code...
        vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path)
        if app['Repo Type'] == 'srclib':
            build_dir = os.path.join(build_dir, app['Repo'])

        if vcs.repotype() == 'git':
            if branch:
                vcs.gotorevision('origin/'+branch)
            else:
                vcs.gotorevision('origin/master')
                pass
        elif vcs.repotype() == 'git-svn':
            if branch:
                vcs.gotorevision(branch)
            else:
                vcs.gotorevision(None)
        elif vcs.repotype() == 'svn':
            vcs.gotorevision(None)
        elif vcs.repotype() == 'hg':
            if branch:
                vcs.gotorevision(branch)
            else:
                vcs.gotorevision('default')
        elif vcs.repotype() == 'bzr':
            vcs.gotorevision(None)

        if len(app['builds']) > 0:
            if 'subdir' in app['builds'][-1]:
                build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])

        if not os.path.isdir(build_dir):
            return (None, "Subdir '" + app['builds'][-1]['subdir'] + "'is not a valid directory")

        version, vercode, package = common.parse_androidmanifest(build_dir)
        if not package:
            return (None, "Couldn't find package ID")
        if package != app['id']:
            return (None, "Package ID mismatch")
        if not version:
            return (None,"Couldn't find latest version name")
        if not vercode:
            return (None,"Couldn't find latest version code")

        return (version, str(int(vercode)))

    except BuildException as be:
        msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
        return (None, msg)
    except VCSException as vcse:
        msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
        return (None, msg)
    except Exception:
        msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
        return (None, msg)

# Check for a new version by looking at the Google market.
# Returns (None, "a message") if this didn't work, or (version, vercode) for
# the details of the current version.
def check_market(app):
    time.sleep(15)
    url = 'https://play.google.com/store/apps/details?id=' + app['id']
    headers = {'User-Agent' : 'Mozilla/5.0 (X11; Linux i686; rv:18.0) Gecko/20100101 Firefox/18.0'}
    req = urllib2.Request(url, None, headers)
    try:
        resp = urllib2.urlopen(req, None, 20)
        page = resp.read()
    except urllib2.HTTPError, e:
        if e.code == 404:
            return (None, 'Not in market')
        else:
            return (None, 'Failed with HTTP status' + str(req.getcode()))
    except Exception, e:
        return (None, 'Failed:' + str(e))

    version = None
    vercode = None

    m = re.search('itemprop="softwareVersion">[ ]*([^<]+)[ ]*
', page)
    if m:
        html_parser = HTMLParser.HTMLParser()
        version = html_parser.unescape(m.group(1))

    if version == 'Varies with device':
        return (None, 'Device-variable version, cannot use this method')

    m = re.search('data-paramValue="(\d+)">
Latest Version<', page)
    if m:
        vercode = m.group(1)

    if not vercode:
        return (None, "Couldn't find version code")
    if not version:
        return (None, "Couldn't find version")
    return (version, str(int(vercode)))

def main():

    #Read configuration...
    execfile('config.py', globals())

    # Parse command line...
    parser = OptionParser()
    parser.add_option("-v", "--verbose", action="store_true", default=False,
                      help="Spew out even more information than normal")
    parser.add_option("-p", "--package", default=None,
                      help="Check only the specified package")
    parser.add_option("--auto", action="store_true", default=False,
                      help="Process auto-updates")
    parser.add_option("--autoonly", action="store_true", default=False,
                      help="Only process apps with auto-updates")
    parser.add_option("--commit", action="store_true", default=False,
                      help="Commit changes")
    (options, args) = parser.parse_args()

    # Get all apps...
    apps = common.read_metadata(options.verbose)

    # Filter apps according to command-line options
    if options.package:
        apps = [app for app in apps if app['id'] == options.package]
        if len(apps) == 0:
            print "No such package"
            sys.exit(1)

    for app in apps:

        process = True

        if options.autoonly and app['Auto Update Mode'] == 'None':
            process = False

        if process:

            print "Processing " + app['id'] + '...'

            writeit = False
            logmsg = None

            mode = app['Update Check Mode']
            if mode == 'Market':
                (version, vercode) = check_market(app)
            elif mode == 'Tags':
                (version, vercode) = check_tags(app, sdk_path)
            elif mode == 'RepoManifest':
                (version, vercode) = check_repomanifest(app, sdk_path)
            elif mode.startswith('RepoManifest/'):
                (version, vercode) = check_repomanifest(app, sdk_path, mode[13:])
            elif mode == 'Static':
                version = None
                vercode = 'Checking disabled'
            elif mode == 'None':
                version = None
                vercode = 'Checking disabled'
            else:
                version = None
                vercode = 'Invalid update check method'

            if not version:
                print "..." + vercode
            elif vercode == app['Current Version Code'] and version == app['Current Version']:
                print "...up to date"
            else:
                print '...updating to version:' + version + ' vercode:' + vercode
                app['Current Version'] = version
                app['Current Version Code'] = str(int(vercode))
                writeit = True
                logmsg = "Update current version of " + app['id'] + " to " + version

            # Do the Auto Name thing...
            if len(app["Repo Type"]) > 0:

                try:

                    if app['Repo Type'] == 'srclib':
                        app_dir = os.path.join('build', 'srclib', app['Repo'])
                    else:
                        app_dir = os.path.join('build/', app['id'])

                    vcs = common.getvcs(app["Repo Type"], app["Repo"], app_dir, sdk_path)
                    vcs.gotorevision(None)

                    if len(app['builds']) > 0:
                        if 'subdir' in app['builds'][-1]:
                            app_dir = os.path.join(app_dir, app['builds'][-1]['subdir'])

                    new_name = common.fetch_real_name(app_dir)
                    if new_name != app['Auto Name']:
                        app['Auto Name'] = new_name
                        if not writeit:
                            writeit = True
                except Exception:
                    msg = "Auto Name failed for  %s due to exception: %s" % (app['id'], traceback.format_exc())

            if options.auto:
                mode = app['Auto Update Mode']
                if mode == 'None':
                    pass
                elif mode.startswith('Version '):
                    pattern = mode[8:]
                    if pattern.startswith('+'):
                        o = pattern.find(' ')
                        suffix = pattern[1:o]
                        pattern = pattern[o + 1:]
                    else:
                        suffix = ''
                    gotcur = False
                    latest = None
                    for build in app['builds']:
                        if build['vercode'] == app['Current Version Code']:
                            gotcur = True
                        if not latest or int(build['vercode']) > int(latest['vercode']):
                            latest = build
                    if not gotcur:
                        newbuild = latest.copy()
                        del newbuild['origlines']
                        newbuild['vercode'] = app['Current Version Code']
                        newbuild['version'] = app['Current Version'] + suffix
                        print "...auto-generating build for " + newbuild['version']
                        commit = pattern.replace('%v', newbuild['version'])
                        commit = commit.replace('%c', newbuild['vercode'])
                        newbuild['commit'] = commit
                        app['builds'].append(newbuild)
                        writeit = True
                        logmsg = "Update " + app['id'] + " to " + newbuild['version']
                else:
                    print 'Invalid auto update mode'

            if writeit:
                metafile = os.path.join('metadata', app['id'] + '.txt')
                common.write_metadata(metafile, app)
                if options.commit and logmsg:
                    if subprocess.call("git add " + metafile, shell=True) != 0:
                        print "Git add failed"
                        sys.exit(1)
                    if subprocess.call("git commit -m '" + logmsg.replace("'", "\\'") +  "'", shell=True) != 0:
                        print "Git commit failed"
                        sys.exit(1)

    print "Finished."

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before>#!/usr/bin/python -tt
# Copyright 2010 Google Inc.
# Licensed under the Apache License, Version 2.0
# http://www.apache.org/licenses/LICENSE-2.0

# Google's Python Class
# http://code.google.com/edu/languages/google-python-class/

# Basic list exercises
# Fill in the code for the functions below. main() is already set up
# to call the functions with a few different inputs,
# printing 'OK' when each function is correct.
# The starter code for each function includes a 'return'
# which is just a placeholder for your code.
# It's ok if you do not complete all the functions, and there
# are some additional functions to try in list2.py.

# A. match_ends
# Given a list of strings, return the count of the number of
# strings where the string length is 2 or more and the first
# and last chars of the string are the same.
# Note: python does not have a ++ operator, but += works.
def match_ends(words):
    # +++your code here+++
    count = 0
    for w in words:
        if len(w) >= 2 and w[0] == w[-1]:
                count += 1
    return count

# B. front_x
# Given a list of strings, return a list with the strings
# in sorted order, except group all the strings that begin with 'x' first.
# e.g. ['mix', 'xyz', 'apple', 'xanadu', 'aardvark'] yields
# ['xanadu', 'xyz', 'aardvark', 'apple', 'mix']
# Hint: this can be done by making 2 lists and sorting each of them
# before combining them.
def front_x(words):
    # +++your code here+++
    x_list = [w for w in words if w[0] == 'x']
    o_list = [w for w in words if w[0] != 'x']
    return sorted(x_list) + sorted(o_list)

# C. sort_last
# Given a list of non-empty tuples, return a list sorted in increasing
# order by the last element in each tuple.
# e.g. [(1, 7), (1, 3), (3, 4, 5), (2, 2)] yields
# [(2, 2), (1, 3), (3, 4, 5), (1, 7)]
# Hint: use a custom key= function to extract the last element form each tuple.
def sort_last(tuples):
    # +++your code here+++
    return sorted(tuples, key=lambda x: x[-1])

# Simple provided test() function used in main() to print
# what each function returns vs. what it's supposed to return.
def test(got, expected):
    if got == expected:
        prefix = ' OK '
    else:
        prefix = '  X '
    print('%s got: %s expected: %s' % (prefix, repr(got), repr(expected)))

# Calls the above functions with interesting inputs.
def main():
    print('match_ends')
    test(match_ends(['aba', 'xyz', 'aa', 'x', 'bbb']), 3)
    test(match_ends(['', 'x', 'xy', 'xyx', 'xx']), 2)
    test(match_ends(['aaa', 'be', 'abc', 'hello']), 1)

    print()
    print('front_x')
    test(front_x(['bbb', 'ccc', 'axx', 'xzz', 'xaa']),
         ['xaa', 'xzz', 'axx', 'bbb', 'ccc'])
    test(front_x(['ccc', 'bbb', 'aaa', 'xcc', 'xaa']),
         ['xaa', 'xcc', 'aaa', 'bbb', 'ccc'])
    test(front_x(['mix', 'xyz', 'apple', 'xanadu', 'aardvark']),
         ['xanadu', 'xyz', 'aardvark', 'apple', 'mix'])

    print()
    print('sort_last')
    test(sort_last([(1, 3), (3, 2), (2, 1)]),
         [(2, 1), (3, 2), (1, 3)])
    test(sort_last([(2, 3), (1, 2), (3, 1)]),
         [(3, 1), (1, 2), (2, 3)])
    test(sort_last([(1, 7), (1, 3), (3, 4, 5), (2, 2)]),
         [(2, 2), (1, 3), (3, 4, 5), (1, 7)])

if __name__ == '__main__':
    main()
<commit_msg>more improvements on solution<commit_after>#!/usr/bin/python -tt
# Copyright 2010 Google Inc.
# Licensed under the Apache License, Version 2.0
# http://www.apache.org/licenses/LICENSE-2.0

# Google's Python Class
# http://code.google.com/edu/languages/google-python-class/

# Basic list exercises
# Fill in the code for the functions below. main() is already set up
# to call the functions with a few different inputs,
# printing 'OK' when each function is correct.
# The starter code for each function includes a 'return'
# which is just a placeholder for your code.
# It's ok if you do not complete all the functions, and there
# are some additional functions to try in list2.py.

# A. match_ends
# Given a list of strings, return the count of the number of
# strings where the string length is 2 or more and the first
# and last chars of the string are the same.
# Note: python does not have a ++ operator, but += works.
def match_ends(words):
    # +++your code here+++
    count = 0
    for w in words:
        if len(w) >= 2 and w[0] == w[-1]:
                count += 1
    return count

# B. front_x
# Given a list of strings, return a list with the strings
# in sorted order, except group all the strings that begin with 'x' first.
# e.g. ['mix', 'xyz', 'apple', 'xanadu', 'aardvark'] yields
# ['xanadu', 'xyz', 'aardvark', 'apple', 'mix']
# Hint: this can be done by making 2 lists and sorting each of them
# before combining them.
def front_x(words):
    # +++your code here+++
    words_cp = words[:]
    xlist = list()
    for w in words_cp:
        if w[0] == 'x':
            xlist.append(w)
            words_cp.remove(w)
    return sorted(xlist) + sorted(words_cp)

# C. sort_last
# Given a list of non-empty tuples, return a list sorted in increasing
# order by the last element in each tuple.
# e.g. [(1, 7), (1, 3), (3, 4, 5), (2, 2)] yields
# [(2, 2), (1, 3), (3, 4, 5), (1, 7)]
# Hint: use a custom key= function to extract the last element form each tuple.
def sort_last(tuples):
    # +++your code here+++
    return sorted(tuples, key=lambda x: x[-1])

# Simple provided test() function used in main() to print
# what each function returns vs. what it's supposed to return.
def test(got, expected):
    if got == expected:
        prefix = ' OK '
    else:
        prefix = '  X '
    print('%s got: %s expected: %s' % (prefix, repr(got), repr(expected)))

# Calls the above functions with interesting inputs.
def main():
    print('match_ends')
    test(match_ends(['aba', 'xyz', 'aa', 'x', 'bbb']), 3)
    test(match_ends(['', 'x', 'xy', 'xyx', 'xx']), 2)
    test(match_ends(['aaa', 'be', 'abc', 'hello']), 1)

    print()
    print('front_x')
    test(front_x(['bbb', 'ccc', 'axx', 'xzz', 'xaa']),
         ['xaa', 'xzz', 'axx', 'bbb', 'ccc'])
    test(front_x(['ccc', 'bbb', 'aaa', 'xcc', 'xaa']),
         ['xaa', 'xcc', 'aaa', 'bbb', 'ccc'])
    test(front_x(['mix', 'xyz', 'apple', 'xanadu', 'aardvark']),
         ['xanadu', 'xyz', 'aardvark', 'apple', 'mix'])

    print()
    print('sort_last')
    test(sort_last([(1, 3), (3, 2), (2, 1)]),
         [(2, 1), (3, 2), (1, 3)])
    test(sort_last([(2, 3), (1, 2), (3, 1)]),
         [(3, 1), (1, 2), (2, 3)])
    test(sort_last([(1, 7), (1, 3), (3, 4, 5), (2, 2)]),
         [(2, 2), (1, 3), (3, 4, 5), (1, 7)])

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8  -*-
import sys
import ast
import warnings

import click

import numpy as np

from . import _dedup, _io, _pairsam_format, _headerops, cli

UTIL_NAME = 'pairsam_dedup'

# you don't need to load more than 10k lines at a time b/c you get out of the
# CPU cache, so this parameter is not adjustable
MAX_LEN = 10000

@cli.command()
@click.argument(
    'pairsam_path',
    type=str,
    required=False)
@click.option(
    "-o", "--output",
    type=str,
    default="",
    help='output file for pairs after duplicate removal.'
        ' If the path ends with .gz, the output is bgzip-compressed.'
        ' By default, the output is printed into stdout.')
@click.option(
    "--output-dups",
    type=str,
    default="",
    help='output file for duplicates. '
        ' If the path ends with .gz, the output is bgzip-compressed.'
        ' By default, duplicates are dropped.')
@click.option(
    "--stats-file",
    type=str,
    default="",
    help='output file for duplicate statistics. '
        ' If file exists, it will be open in the append mode.'
        ' If the path ends with .gz, the output is bgzip-compressed.'
        ' By default, statistics are not printed.')
@click.option(
    "--max-mismatch",
    type=int,
    default=3,
    help='Pairs with both sides mapped within this distance (bp) from each '
         'other are considered duplicates.')
@click.option(
    '--method',
    type=click.Choice(['max', 'sum']),
    default="max",
    help='define the mismatch as either the max or the sum of the mismatches of'
        'the genomic locations of the both sides of the two compared molecules',
    show_default=True,)
@click.option(
    "--sep",
    type=str,
    default=_pairsam_format.PAIRSAM_SEP_ESCAPE,
    help=r"Separator (\t, \v, etc. characters are "
          "supported, pass them in quotes) ")
@click.option(
    "--comment-char",
    type=str,
    default="#",
    help="The first character of comment lines")
@click.option(
    "--send-header-to",
    type=click.Choice(['dups', 'dedup', 'both', 'none']),
    default="both",
    help="Which of the outputs should receive header and comment lines")
@click.option(
    "--c1",
    type=int,
    default=_pairsam_format.COL_C1,
    help='Chrom 1 column; default {}'.format(_pairsam_format.COL_C1))
@click.option(
    "--c2",
    type=int,
    default=_pairsam_format.COL_C2,
    help='Chrom 2 column; default {}'.format(_pairsam_format.COL_C2))
@click.option(
    "--p1",
    type=int,
    default=_pairsam_format.COL_P1,
    help='Position 1 column; default {}'.format(_pairsam_format.COL_P1))
@click.option(
    "--p2",
    type=int,
    default=_pairsam_format.COL_P2,
    help='Position 2 column; default {}'.format(_pairsam_format.COL_P2))
@click.option(
    "--s1",
    type=int,
    default=_pairsam_format.COL_S1,
    help='Strand 1 column; default {}'.format(_pairsam_format.COL_S1))
@click.option(
    "--s2",
    type=int,
    default=_pairsam_format.COL_S2,
    help='Strand 2 column; default {}'.format(_pairsam_format.COL_S2))
@click.option(
    "--unmapped-chrom",
    type=str,
    default=_pairsam_format.UNMAPPED_CHROM,
    help='Placeholder for a chromosome on an unmapped side; default {}'.format(_pairsam_format.UNMAPPED_CHROM))

def dedup(pairsam_path, output, output_dups,
    stats_file,
    max_mismatch, method,
    sep, comment_char, send_header_to,
    c1, c2, p1, p2, s1, s2, unmapped_chrom
    ):
    '''find and remove PCR duplicates.

    Find PCR duplicates in an upper-triangular flipped sorted pairs/pairsam
    file. Allow for a +/-N bp mismatch at each side of duplicated molecules.

    PAIRSAM_PATH : input triu-flipped sorted .pairs or .pairsam file.  If the
    path ends with .gz, the input is gzip-decompressed. By default, the input
    is read from stdin.
    '''
    dedup_py(pairsam_path, output, output_dups,
        stats_file,
        max_mismatch, method,
        sep, comment_char, send_header_to,
        c1, c2, p1, p2, s1, s2, unmapped_chrom
        )

def dedup_py(pairsam_path, output, output_dups,
    stats_file,
    max_mismatch, method,
    sep, comment_char, send_header_to,
    c1, c2, p1, p2, s1, s2, unmapped_chrom
    ):
    sep = ast.literal_eval('"""' + sep + '"""')
    send_header_to_dedup = send_header_to in ['both', 'dedup']
    send_header_to_dup = send_header_to in ['both', 'dups']

    instream = (_io.open_bgzip(pairsam_path, mode='r')
                if pairsam_path else sys.stdin)
    outstream = (_io.open_bgzip(output, mode='w')
                 if output else sys.stdout)
    outstream_dups = (_io.open_bgzip(output_dups, mode='w')
                      if output_dups else None)

    header, body_stream = _headerops.get_header(instream)
    header = _headerops.append_new_pg(header, ID=UTIL_NAME, PN=UTIL_NAME)

    if send_header_to_dedup:
        outstream.writelines((l+'\n' for l in header))
    if send_header_to_dup and outstream_dups:
        outstream_dups.writelines((l+'\n' for l in header))

    n_unmapped, n_dups, n_nodups = streaming_dedup(
        method, max_mismatch, sep,
        c1, c2, p1, p2, s1, s2, unmapped_chrom,
        body_stream, outstream, outstream_dups)

    if stats_file:
        stat_f = _io.open_bgzip(stats_file, mode='a')
        stat_f.write('{}\t{}\n'.format('n_unmapped', n_unmapped))
        stat_f.write('{}\t{}\n'.format('n_dups', n_dups))
        stat_f.write('{}\t{}\n'.format('n_nodups', n_nodups))
        stat_f.close()

    if instream != sys.stdin:
        instream.close()

    if outstream != sys.stdout:
        outstream.close()

    if outstream_dups:
        outstream_dups.close()

def fetchadd(key, mydict):
    key = key.strip()
    if key not in mydict:
        mydict[key] = len(mydict)
    return mydict[key]

def ar(mylist, val):
    return np.array(mylist, dtype={8: np.int8, 16: np.int16, 32: np.int32}[val])

def streaming_dedup(
        method, max_mismatch, sep,
        c1ind, c2ind, p1ind, p2ind, s1ind, s2ind,
        unmapped_chrom,
        instream, outstream, outstream_dups):

    maxind = max(c1ind, c2ind, p1ind, p2ind, s1ind, s2ind)

    dd = _dedup.OnlineDuplicateDetector(method, max_mismatch, returnData=False)

    c1 = []; c2 = []; p1 = []; p2 = []; s1 = []; s2 = []
    lines = []
    chromDict = {}
    strandDict = {}
    n_unmapped = 0
    n_dups = 0
    n_nodups = 0

    while True:
        line = next(instream, None)
        stripline = line.strip() if line else None

        if line:
            if not stripline:
                warnings.warn("Empty line detected not at the end of the file")
                continue

            words = line.split(sep)
            if len(words) <= maxind:
                raise ValueError(
                    "Error parsing line {}: ".format(line)
                    + " expected {} words, got {}".format(maxind, len(words)))

            if ((words[c1ind] == unmapped_chrom)
                or (words[c1ind] == unmapped_chrom)):

                outstream.write(line)
                n_unmapped += 1

            else:
                lines.append(line)
                c1.append(fetchadd(words[c1ind], chromDict))
                c2.append(fetchadd(words[c2ind], chromDict))
                p1.append(int(words[p1ind]))
                p2.append(int(words[p2ind]))
                s1.append(fetchadd(words[s1ind], strandDict))
                s2.append(fetchadd(words[s2ind], strandDict))

        if (not line) or (len(c1) == MAX_LEN):
            res = dd.push(ar(c1, 8),
                          ar(c2, 8),
                          ar(p1, 32),
                          ar(p2, 32),
                          ar(s1, 8),
                          ar(s2, 8))
            if not line:
                res = np.concatenate([res, dd.finish()])

            for newline, remove in zip(lines[:len(res)], res):
                if not remove:
                    outstream.write(newline)
                    n_nodups += 1
                else:
                    n_dups += 1
                    if outstream_dups:
                        outstream_dups.write(newline)

            c1 = []; c2 = []; p1 = []; p2 = []; s1 = []; s2 = []
            lines = lines[len(res):]
            if not line:
                if(len(lines) != 0):
                    raise ValueError(
                        "{} lines left in the buffer, ".format(len(lines))
                        + "should be none;"
                        + "something went terribly wrong")
                break

    return n_unmapped, n_dups, n_nodups

if __name__ == '__main__':
    dedup()
<commit_msg>dedup: a major bugfix in the recently introduced bypassing of unmapped reads<commit_after>#!/usr/bin/env python
# -*- coding: utf-8  -*-
import sys
import ast
import warnings

import click

import numpy as np

from . import _dedup, _io, _pairsam_format, _headerops, cli

UTIL_NAME = 'pairsam_dedup'

# you don't need to load more than 10k lines at a time b/c you get out of the
# CPU cache, so this parameter is not adjustable
MAX_LEN = 10000

@cli.command()
@click.argument(
    'pairsam_path',
    type=str,
    required=False)
@click.option(
    "-o", "--output",
    type=str,
    default="",
    help='output file for pairs after duplicate removal.'
        ' If the path ends with .gz, the output is bgzip-compressed.'
        ' By default, the output is printed into stdout.')
@click.option(
    "--output-dups",
    type=str,
    default="",
    help='output file for duplicates. '
        ' If the path ends with .gz, the output is bgzip-compressed.'
        ' By default, duplicates are dropped.')
@click.option(
    "--stats-file",
    type=str,
    default="",
    help='output file for duplicate statistics. '
        ' If file exists, it will be open in the append mode.'
        ' If the path ends with .gz, the output is bgzip-compressed.'
        ' By default, statistics are not printed.')
@click.option(
    "--max-mismatch",
    type=int,
    default=3,
    help='Pairs with both sides mapped within this distance (bp) from each '
         'other are considered duplicates.')
@click.option(
    '--method',
    type=click.Choice(['max', 'sum']),
    default="max",
    help='define the mismatch as either the max or the sum of the mismatches of'
        'the genomic locations of the both sides of the two compared molecules',
    show_default=True,)
@click.option(
    "--sep",
    type=str,
    default=_pairsam_format.PAIRSAM_SEP_ESCAPE,
    help=r"Separator (\t, \v, etc. characters are "
          "supported, pass them in quotes) ")
@click.option(
    "--comment-char",
    type=str,
    default="#",
    help="The first character of comment lines")
@click.option(
    "--send-header-to",
    type=click.Choice(['dups', 'dedup', 'both', 'none']),
    default="both",
    help="Which of the outputs should receive header and comment lines")
@click.option(
    "--c1",
    type=int,
    default=_pairsam_format.COL_C1,
    help='Chrom 1 column; default {}'.format(_pairsam_format.COL_C1))
@click.option(
    "--c2",
    type=int,
    default=_pairsam_format.COL_C2,
    help='Chrom 2 column; default {}'.format(_pairsam_format.COL_C2))
@click.option(
    "--p1",
    type=int,
    default=_pairsam_format.COL_P1,
    help='Position 1 column; default {}'.format(_pairsam_format.COL_P1))
@click.option(
    "--p2",
    type=int,
    default=_pairsam_format.COL_P2,
    help='Position 2 column; default {}'.format(_pairsam_format.COL_P2))
@click.option(
    "--s1",
    type=int,
    default=_pairsam_format.COL_S1,
    help='Strand 1 column; default {}'.format(_pairsam_format.COL_S1))
@click.option(
    "--s2",
    type=int,
    default=_pairsam_format.COL_S2,
    help='Strand 2 column; default {}'.format(_pairsam_format.COL_S2))
@click.option(
    "--unmapped-chrom",
    type=str,
    default=_pairsam_format.UNMAPPED_CHROM,
    help='Placeholder for a chromosome on an unmapped side; default {}'.format(_pairsam_format.UNMAPPED_CHROM))

def dedup(pairsam_path, output, output_dups,
    stats_file,
    max_mismatch, method,
    sep, comment_char, send_header_to,
    c1, c2, p1, p2, s1, s2, unmapped_chrom
    ):
    '''find and remove PCR duplicates.

    Find PCR duplicates in an upper-triangular flipped sorted pairs/pairsam
    file. Allow for a +/-N bp mismatch at each side of duplicated molecules.

    PAIRSAM_PATH : input triu-flipped sorted .pairs or .pairsam file.  If the
    path ends with .gz, the input is gzip-decompressed. By default, the input
    is read from stdin.
    '''
    dedup_py(pairsam_path, output, output_dups,
        stats_file,
        max_mismatch, method,
        sep, comment_char, send_header_to,
        c1, c2, p1, p2, s1, s2, unmapped_chrom
        )

def dedup_py(pairsam_path, output, output_dups,
    stats_file,
    max_mismatch, method,
    sep, comment_char, send_header_to,
    c1, c2, p1, p2, s1, s2, unmapped_chrom
    ):
    sep = ast.literal_eval('"""' + sep + '"""')
    send_header_to_dedup = send_header_to in ['both', 'dedup']
    send_header_to_dup = send_header_to in ['both', 'dups']

    instream = (_io.open_bgzip(pairsam_path, mode='r')
                if pairsam_path else sys.stdin)
    outstream = (_io.open_bgzip(output, mode='w')
                 if output else sys.stdout)
    outstream_dups = (_io.open_bgzip(output_dups, mode='w')
                      if output_dups else None)

    header, body_stream = _headerops.get_header(instream)
    header = _headerops.append_new_pg(header, ID=UTIL_NAME, PN=UTIL_NAME)

    if send_header_to_dedup:
        outstream.writelines((l+'\n' for l in header))
    if send_header_to_dup and outstream_dups:
        outstream_dups.writelines((l+'\n' for l in header))

    n_unmapped, n_dups, n_nodups = streaming_dedup(
        method, max_mismatch, sep,
        c1, c2, p1, p2, s1, s2, unmapped_chrom,
        body_stream, outstream, outstream_dups)

    if stats_file:
        stat_f = _io.open_bgzip(stats_file, mode='a')
        stat_f.write('{}\t{}\n'.format('n_unmapped', n_unmapped))
        stat_f.write('{}\t{}\n'.format('n_dups', n_dups))
        stat_f.write('{}\t{}\n'.format('n_nodups', n_nodups))
        stat_f.close()

    if instream != sys.stdin:
        instream.close()

    if outstream != sys.stdout:
        outstream.close()

    if outstream_dups:
        outstream_dups.close()

def fetchadd(key, mydict):
    key = key.strip()
    if key not in mydict:
        mydict[key] = len(mydict)
    return mydict[key]

def ar(mylist, val):
    return np.array(mylist, dtype={8: np.int8, 16: np.int16, 32: np.int32}[val])

def streaming_dedup(
        method, max_mismatch, sep,
        c1ind, c2ind, p1ind, p2ind, s1ind, s2ind,
        unmapped_chrom,
        instream, outstream, outstream_dups):

    maxind = max(c1ind, c2ind, p1ind, p2ind, s1ind, s2ind)

    dd = _dedup.OnlineDuplicateDetector(method, max_mismatch, returnData=False)

    c1 = []; c2 = []; p1 = []; p2 = []; s1 = []; s2 = []
    lines = []
    chromDict = {}
    strandDict = {}
    n_unmapped = 0
    n_dups = 0
    n_nodups = 0

    while True:
        line = next(instream, None)
        stripline = line.strip() if line else None

        if line:
            if not stripline:
                warnings.warn("Empty line detected not at the end of the file")
                continue

            words = line.split(sep)
            if len(words) <= maxind:
                raise ValueError(
                    "Error parsing line {}: ".format(line)
                    + " expected {} words, got {}".format(maxind, len(words)))

            if ((words[c1ind] == unmapped_chrom)
                or (words[c2ind] == unmapped_chrom)):

                outstream.write(line)
                n_unmapped += 1

            else:
                lines.append(line)
                c1.append(fetchadd(words[c1ind], chromDict))
                c2.append(fetchadd(words[c2ind], chromDict))
                p1.append(int(words[p1ind]))
                p2.append(int(words[p2ind]))
                s1.append(fetchadd(words[s1ind], strandDict))
                s2.append(fetchadd(words[s2ind], strandDict))

        if (not line) or (len(c1) == MAX_LEN):
            res = dd.push(ar(c1, 8),
                          ar(c2, 8),
                          ar(p1, 32),
                          ar(p2, 32),
                          ar(s1, 8),
                          ar(s2, 8))
            if not line:
                res = np.concatenate([res, dd.finish()])

            for newline, remove in zip(lines[:len(res)], res):
                if not remove:
                    outstream.write(newline)
                    n_nodups += 1
                else:
                    n_dups += 1
                    if outstream_dups:
                        outstream_dups.write(newline)

            c1 = []; c2 = []; p1 = []; p2 = []; s1 = []; s2 = []
            lines = lines[len(res):]
            if not line:
                if(len(lines) != 0):
                    raise ValueError(
                        "{} lines left in the buffer, ".format(len(lines))
                        + "should be none;"
                        + "something went terribly wrong")
                break

    return n_unmapped, n_dups, n_nodups

if __name__ == '__main__':
    dedup()
<|endoftext|>