<commit_before>#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
"""This module contains Google BigQuery to Google Cloud Storage operator."""
from __future__ import annotations

from typing import TYPE_CHECKING, Any, Sequence

from google.api_core.exceptions import Conflict
from google.api_core.retry import Retry
from google.cloud.bigquery import DEFAULT_RETRY, ExtractJob

from airflow import AirflowException
from airflow.models import BaseOperator
from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook, BigQueryJob
from airflow.providers.google.cloud.links.bigquery import BigQueryTableLink
from airflow.providers.google.cloud.triggers.bigquery import BigQueryInsertJobTrigger

if TYPE_CHECKING:
    from airflow.utils.context import Context

class BigQueryToGCSOperator(BaseOperator):
    """
    Transfers a BigQuery table to a Google Cloud Storage bucket.

    .. seealso::
        For more details about these parameters:
        https://cloud.google.com/bigquery/docs/reference/v2/jobs

    :param source_project_dataset_table: The dotted
        ``(<project>.|<project>:)<dataset>.<table>`` BigQuery table to use as the
        source data. If ``<project>`` is not included, project will be the project
        defined in the connection json. (templated)
    :param destination_cloud_storage_uris: The destination Google Cloud
        Storage URI (e.g. gs://some-bucket/some-file.txt). (templated) Follows
        convention defined here:
        https://cloud.google.com/bigquery/exporting-data-from-bigquery#exportingmultiple
    :param project_id: Google Cloud Project where the job is running
    :param compression: Type of compression to use.
    :param export_format: File format to export.
    :param field_delimiter: The delimiter to use when extracting to a CSV.
    :param print_header: Whether to print a header for a CSV file extract.
    :param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud.
    :param delegate_to: The account to impersonate using domain-wide delegation of authority,
        if any. For this to work, the service account making the request must have
        domain-wide delegation enabled.
    :param labels: a dictionary containing labels for the job/query,
        passed to BigQuery
    :param location: The location used for the operation.
    :param impersonation_chain: Optional service account to impersonate using short-term
        credentials, or chained list of accounts required to get the access_token
        of the last account in the list, which will be impersonated in the request.
        If set as a string, the account must grant the originating account
        the Service Account Token Creator IAM role.
        If set as a sequence, the identities from the list must grant
        Service Account Token Creator IAM role to the directly preceding identity, with first
        account from the list granting this role to the originating account (templated).
    :param result_retry: How to retry the `result` call that retrieves rows
    :param result_timeout: The number of seconds to wait for `result` method before using `result_retry`
    :param job_id: The ID of the job. It will be suffixed with hash of job configuration
        unless ``force_rerun`` is True.
        The ID must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), or
        dashes (-). The maximum length is 1,024 characters. If not provided then uuid will
        be generated.
    :param force_rerun: If True then operator will use hash of uuid as job id suffix
    :param reattach_states: Set of BigQuery job's states in case of which we should reattach
        to the job. Should be other than final states.
    :param deferrable: Run operator in the deferrable mode
    """

    template_fields: Sequence[str] = (
        "source_project_dataset_table",
        "destination_cloud_storage_uris",
        "labels",
        "impersonation_chain",
    )
    template_ext: Sequence[str] = ()
    ui_color = "#e4e6f0"
    operator_extra_links = (BigQueryTableLink(),)

    def __init__(
        self,
        *,
        source_project_dataset_table: str,
        destination_cloud_storage_uris: list[str],
        project_id: str | None = None,
        compression: str = "NONE",
        export_format: str = "CSV",
        field_delimiter: str = ",",
        print_header: bool = True,
        gcp_conn_id: str = "google_cloud_default",
        delegate_to: str | None = None,
        labels: dict | None = None,
        location: str | None = None,
        impersonation_chain: str | Sequence[str] | None = None,
        result_retry: Retry = DEFAULT_RETRY,
        result_timeout: float | None = None,
        job_id: str | None = None,
        force_rerun: bool = False,
        reattach_states: set[str] | None = None,
        deferrable: bool = False,
        **kwargs,
    ) -> None:
        super().__init__(**kwargs)
        self.project_id = project_id
        self.source_project_dataset_table = source_project_dataset_table
        self.destination_cloud_storage_uris = destination_cloud_storage_uris
        self.compression = compression
        self.export_format = export_format
        self.field_delimiter = field_delimiter
        self.print_header = print_header
        self.gcp_conn_id = gcp_conn_id
        self.delegate_to = delegate_to
        self.labels = labels
        self.location = location
        self.impersonation_chain = impersonation_chain
        self.result_retry = result_retry
        self.result_timeout = result_timeout
        self.job_id = job_id
        self.force_rerun = force_rerun
        self.reattach_states: set[str] = reattach_states or set()
        self.hook: BigQueryHook | None = None
        self.deferrable = deferrable

    @staticmethod
    def _handle_job_error(job: ExtractJob) -> None:
        if job.error_result:
            raise AirflowException(f"BigQuery job {job.job_id} failed: {job.error_result}")

    def _prepare_configuration(self):
        source_project, source_dataset, source_table = self.hook.split_tablename(
            table_input=self.source_project_dataset_table,
            default_project_id=self.project_id or self.hook.project_id,
            var_name="source_project_dataset_table",
        )

        configuration: dict[str, Any] = {
            "extract": {
                "sourceTable": {
                    "projectId": source_project,
                    "datasetId": source_dataset,
                    "tableId": source_table,
                },
                "compression": self.compression,
                "destinationUris": self.destination_cloud_storage_uris,
                "destinationFormat": self.export_format,
            }
        }

        if self.labels:
            configuration["labels"] = self.labels

        if self.export_format == "CSV":
            # Only set fieldDelimiter and printHeader fields if using CSV.
            # Google does not like it if you set these fields for other export
            # formats.
            configuration["extract"]["fieldDelimiter"] = self.field_delimiter
            configuration["extract"]["printHeader"] = self.print_header
        return configuration

    def _submit_job(
        self,
        hook: BigQueryHook,
        job_id: str,
        configuration: dict,
    ) -> BigQueryJob:
        # Submit a new job without waiting for it to complete.

        return hook.insert_job(
            configuration=configuration,
            project_id=hook.project_id,
            location=self.location,
            job_id=job_id,
            timeout=self.result_timeout,
            retry=self.result_retry,
            nowait=True,
        )

    def execute(self, context: Context):
        self.log.info(
            "Executing extract of %s into: %s",
            self.source_project_dataset_table,
            self.destination_cloud_storage_uris,
        )
        hook = BigQueryHook(
            gcp_conn_id=self.gcp_conn_id,
            delegate_to=self.delegate_to,
            location=self.location,
            impersonation_chain=self.impersonation_chain,
        )
        self.hook = hook

        configuration = self._prepare_configuration()
        job_id = hook.generate_job_id(
            job_id=self.job_id,
            dag_id=self.dag_id,
            task_id=self.task_id,
            logical_date=context["logical_date"],
            configuration=configuration,
            force_rerun=self.force_rerun,
        )

        try:
            self.log.info("Executing: %s", configuration)
            job: ExtractJob = self._submit_job(hook=hook, job_id=job_id, configuration=configuration)
        except Conflict:
            # If the job already exists retrieve it
            job = hook.get_job(
                project_id=self.project_id,
                location=self.location,
                job_id=job_id,
            )
            if job.state in self.reattach_states:
                # We are reattaching to a job
                job.result(timeout=self.result_timeout, retry=self.result_retry)
                self._handle_job_error(job)
            else:
                # Same job configuration so we need force_rerun
                raise AirflowException(
                    f"Job with id: {job_id} already exists and is in {job.state} state. If you "
                    f"want to force rerun it consider setting `force_rerun=True`."
                    f"Or, if you want to reattach in this scenario add {job.state} to `reattach_states`"
                )

        conf = job.to_api_repr()["configuration"]["extract"]["sourceTable"]
        dataset_id, project_id, table_id = conf["datasetId"], conf["projectId"], conf["tableId"]
        BigQueryTableLink.persist(
            context=context,
            task_instance=self,
            dataset_id=dataset_id,
            project_id=project_id,
            table_id=table_id,
        )

        if self.deferrable:
            self.defer(
                timeout=self.execution_timeout,
                trigger=BigQueryInsertJobTrigger(
                    conn_id=self.gcp_conn_id,
                    job_id=job_id,
                    project_id=self.hook.project_id,
                ),
                method_name="execute_complete",
            )
        else:
            job.result(timeout=self.result_timeout, retry=self.result_retry)

    def execute_complete(self, context: Context, event: dict[str, Any]):
        """
        Callback for when the trigger fires - returns immediately.
        Relies on trigger to throw an exception, otherwise it assumes execution was
        successful.
        """
        if event["status"] == "error":
            raise AirflowException(event["message"])
        self.log.info(
            "%s completed with response %s ",
            self.task_id,
            event["message"],
        )
<commit_msg>Add Export Format to Template Fields in BigQueryToGCSOperator (#27910)<commit_after>#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
"""This module contains Google BigQuery to Google Cloud Storage operator."""
from __future__ import annotations

from typing import TYPE_CHECKING, Any, Sequence

from google.api_core.exceptions import Conflict
from google.api_core.retry import Retry
from google.cloud.bigquery import DEFAULT_RETRY, ExtractJob

from airflow import AirflowException
from airflow.models import BaseOperator
from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook, BigQueryJob
from airflow.providers.google.cloud.links.bigquery import BigQueryTableLink
from airflow.providers.google.cloud.triggers.bigquery import BigQueryInsertJobTrigger

if TYPE_CHECKING:
    from airflow.utils.context import Context

class BigQueryToGCSOperator(BaseOperator):
    """
    Transfers a BigQuery table to a Google Cloud Storage bucket.

    .. seealso::
        For more details about these parameters:
        https://cloud.google.com/bigquery/docs/reference/v2/jobs

    :param source_project_dataset_table: The dotted
        ``(<project>.|<project>:)<dataset>.<table>`` BigQuery table to use as the
        source data. If ``<project>`` is not included, project will be the project
        defined in the connection json. (templated)
    :param destination_cloud_storage_uris: The destination Google Cloud
        Storage URI (e.g. gs://some-bucket/some-file.txt). (templated) Follows
        convention defined here:
        https://cloud.google.com/bigquery/exporting-data-from-bigquery#exportingmultiple
    :param project_id: Google Cloud Project where the job is running
    :param compression: Type of compression to use.
    :param export_format: File format to export.
    :param field_delimiter: The delimiter to use when extracting to a CSV.
    :param print_header: Whether to print a header for a CSV file extract.
    :param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud.
    :param delegate_to: The account to impersonate using domain-wide delegation of authority,
        if any. For this to work, the service account making the request must have
        domain-wide delegation enabled.
    :param labels: a dictionary containing labels for the job/query,
        passed to BigQuery
    :param location: The location used for the operation.
    :param impersonation_chain: Optional service account to impersonate using short-term
        credentials, or chained list of accounts required to get the access_token
        of the last account in the list, which will be impersonated in the request.
        If set as a string, the account must grant the originating account
        the Service Account Token Creator IAM role.
        If set as a sequence, the identities from the list must grant
        Service Account Token Creator IAM role to the directly preceding identity, with first
        account from the list granting this role to the originating account (templated).
    :param result_retry: How to retry the `result` call that retrieves rows
    :param result_timeout: The number of seconds to wait for `result` method before using `result_retry`
    :param job_id: The ID of the job. It will be suffixed with hash of job configuration
        unless ``force_rerun`` is True.
        The ID must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), or
        dashes (-). The maximum length is 1,024 characters. If not provided then uuid will
        be generated.
    :param force_rerun: If True then operator will use hash of uuid as job id suffix
    :param reattach_states: Set of BigQuery job's states in case of which we should reattach
        to the job. Should be other than final states.
    :param deferrable: Run operator in the deferrable mode
    """

    template_fields: Sequence[str] = (
        "source_project_dataset_table",
        "destination_cloud_storage_uris",
        "export_format",
        "labels",
        "impersonation_chain",
    )
    template_ext: Sequence[str] = ()
    ui_color = "#e4e6f0"
    operator_extra_links = (BigQueryTableLink(),)

    def __init__(
        self,
        *,
        source_project_dataset_table: str,
        destination_cloud_storage_uris: list[str],
        project_id: str | None = None,
        compression: str = "NONE",
        export_format: str = "CSV",
        field_delimiter: str = ",",
        print_header: bool = True,
        gcp_conn_id: str = "google_cloud_default",
        delegate_to: str | None = None,
        labels: dict | None = None,
        location: str | None = None,
        impersonation_chain: str | Sequence[str] | None = None,
        result_retry: Retry = DEFAULT_RETRY,
        result_timeout: float | None = None,
        job_id: str | None = None,
        force_rerun: bool = False,
        reattach_states: set[str] | None = None,
        deferrable: bool = False,
        **kwargs,
    ) -> None:
        super().__init__(**kwargs)
        self.project_id = project_id
        self.source_project_dataset_table = source_project_dataset_table
        self.destination_cloud_storage_uris = destination_cloud_storage_uris
        self.compression = compression
        self.export_format = export_format
        self.field_delimiter = field_delimiter
        self.print_header = print_header
        self.gcp_conn_id = gcp_conn_id
        self.delegate_to = delegate_to
        self.labels = labels
        self.location = location
        self.impersonation_chain = impersonation_chain
        self.result_retry = result_retry
        self.result_timeout = result_timeout
        self.job_id = job_id
        self.force_rerun = force_rerun
        self.reattach_states: set[str] = reattach_states or set()
        self.hook: BigQueryHook | None = None
        self.deferrable = deferrable

    @staticmethod
    def _handle_job_error(job: ExtractJob) -> None:
        if job.error_result:
            raise AirflowException(f"BigQuery job {job.job_id} failed: {job.error_result}")

    def _prepare_configuration(self):
        source_project, source_dataset, source_table = self.hook.split_tablename(
            table_input=self.source_project_dataset_table,
            default_project_id=self.project_id or self.hook.project_id,
            var_name="source_project_dataset_table",
        )

        configuration: dict[str, Any] = {
            "extract": {
                "sourceTable": {
                    "projectId": source_project,
                    "datasetId": source_dataset,
                    "tableId": source_table,
                },
                "compression": self.compression,
                "destinationUris": self.destination_cloud_storage_uris,
                "destinationFormat": self.export_format,
            }
        }

        if self.labels:
            configuration["labels"] = self.labels

        if self.export_format == "CSV":
            # Only set fieldDelimiter and printHeader fields if using CSV.
            # Google does not like it if you set these fields for other export
            # formats.
            configuration["extract"]["fieldDelimiter"] = self.field_delimiter
            configuration["extract"]["printHeader"] = self.print_header
        return configuration

    def _submit_job(
        self,
        hook: BigQueryHook,
        job_id: str,
        configuration: dict,
    ) -> BigQueryJob:
        # Submit a new job without waiting for it to complete.

        return hook.insert_job(
            configuration=configuration,
            project_id=hook.project_id,
            location=self.location,
            job_id=job_id,
            timeout=self.result_timeout,
            retry=self.result_retry,
            nowait=True,
        )

    def execute(self, context: Context):
        self.log.info(
            "Executing extract of %s into: %s",
            self.source_project_dataset_table,
            self.destination_cloud_storage_uris,
        )
        hook = BigQueryHook(
            gcp_conn_id=self.gcp_conn_id,
            delegate_to=self.delegate_to,
            location=self.location,
            impersonation_chain=self.impersonation_chain,
        )
        self.hook = hook

        configuration = self._prepare_configuration()
        job_id = hook.generate_job_id(
            job_id=self.job_id,
            dag_id=self.dag_id,
            task_id=self.task_id,
            logical_date=context["logical_date"],
            configuration=configuration,
            force_rerun=self.force_rerun,
        )

        try:
            self.log.info("Executing: %s", configuration)
            job: ExtractJob = self._submit_job(hook=hook, job_id=job_id, configuration=configuration)
        except Conflict:
            # If the job already exists retrieve it
            job = hook.get_job(
                project_id=self.project_id,
                location=self.location,
                job_id=job_id,
            )
            if job.state in self.reattach_states:
                # We are reattaching to a job
                job.result(timeout=self.result_timeout, retry=self.result_retry)
                self._handle_job_error(job)
            else:
                # Same job configuration so we need force_rerun
                raise AirflowException(
                    f"Job with id: {job_id} already exists and is in {job.state} state. If you "
                    f"want to force rerun it consider setting `force_rerun=True`."
                    f"Or, if you want to reattach in this scenario add {job.state} to `reattach_states`"
                )

        conf = job.to_api_repr()["configuration"]["extract"]["sourceTable"]
        dataset_id, project_id, table_id = conf["datasetId"], conf["projectId"], conf["tableId"]
        BigQueryTableLink.persist(
            context=context,
            task_instance=self,
            dataset_id=dataset_id,
            project_id=project_id,
            table_id=table_id,
        )

        if self.deferrable:
            self.defer(
                timeout=self.execution_timeout,
                trigger=BigQueryInsertJobTrigger(
                    conn_id=self.gcp_conn_id,
                    job_id=job_id,
                    project_id=self.hook.project_id,
                ),
                method_name="execute_complete",
            )
        else:
            job.result(timeout=self.result_timeout, retry=self.result_retry)

    def execute_complete(self, context: Context, event: dict[str, Any]):
        """
        Callback for when the trigger fires - returns immediately.
        Relies on trigger to throw an exception, otherwise it assumes execution was
        successful.
        """
        if event["status"] == "error":
            raise AirflowException(event["message"])
        self.log.info(
            "%s completed with response %s ",
            self.task_id,
            event["message"],
        )
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright (c) 2014 Intel Corporation. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import optparse
import os
import shutil
import sys

from bridge_generator import BridgeGenerator
from interface_generator import InterfaceGenerator
from java_class import JavaClassLoader
from string import Template
from wrapper_generator import WrapperGenerator

GYP_ANDROID_DIR = os.path.join(os.path.dirname(__file__),
                               os.pardir, os.pardir, os.pardir,
                               'build',
                               'android',
                               'gyp')
sys.path.append(GYP_ANDROID_DIR)

from util import build_utils

# Classes list that have to generate bridge and wrap code.
CLASSES_TO_BE_PROCESS = [
    'ClientCertRequestHandlerInternal',
    'ClientCertRequestInternal',
    'XWalkCookieManagerInternal',
    'XWalkDownloadListenerInternal',
    'XWalkExtensionInternal',
    'XWalkExternalExtensionManagerInternal',
    'XWalkFindListenerInternal',
    'XWalkGetBitmapCallbackInternal',
    'XWalkHttpAuthHandlerInternal',
    'XWalkHttpAuthInternal',
    'XWalkJavascriptResultHandlerInternal',
    'XWalkJavascriptResultInternal',
    'XWalkNativeExtensionLoaderInternal',
    'XWalkNavigationHistoryInternal',
    'XWalkNavigationItemInternal',
    'XWalkPreferencesInternal',
    'XWalkResourceClientInternal',
    'XWalkSettingsInternal',
    'XWalkUIClientInternal',
    'XWalkViewInternal',
    'XWalkWebResourceRequestHandlerInternal',
    'XWalkWebResourceRequestInternal',
    'XWalkWebResourceResponseInternal',
    "XWalkHitTestResultInternal",
]

REFLECTION_HERLPER = [
    'ReflectMethod.java',
    'ReflectField.java',
    'ReflectConstructor.java',
]

WRAPPER_PACKAGE = 'org.xwalk.core'
BRIDGE_PACKAGE = 'org.xwalk.core.internal'

bridge_path = ''
wrapper_path = ''

def PerformSerialize(output_path, generator):
  file_name = generator.GetGeneratedClassFileName()
  with open(os.path.join(output_path, file_name), 'w') as f:
    f.write(generator.GetGeneratedCode())
  print('%s has been generated!' % file_name)

def GenerateJavaBindingClass(input_dir):
  class_loader = JavaClassLoader(input_dir, CLASSES_TO_BE_PROCESS)
  for input_class in CLASSES_TO_BE_PROCESS:
    print('Generate bridge and wrapper code for %s' % input_class)
    java_data = class_loader.GetJavaData(input_class)
    if java_data.class_type == 'interface':
      # Generate Interface code.
      interface_generator = InterfaceGenerator(java_data, class_loader)
      interface_generator.RunTask()
      PerformSerialize(wrapper_path, interface_generator)
    else:
      # Generate Bridge code.
      bridge_generator = BridgeGenerator(java_data, class_loader)
      bridge_generator.RunTask()
      PerformSerialize(bridge_path, bridge_generator)
      # Generate Wrapper code.
      wrapper_generator = WrapperGenerator(java_data, class_loader)
      wrapper_generator.RunTask()
      PerformSerialize(wrapper_path, wrapper_generator)

def GenerateJavaReflectClass(input_dir):
  for helper in REFLECTION_HERLPER:
    with open(os.path.join(wrapper_path, helper), 'w') as f:
      for line in open(os.path.join(input_dir, helper), 'r'):
        if line.startswith('package '):
          f.write('package ' + WRAPPER_PACKAGE + ';\n')
        else:
          f.write(line)

def GenerateJavaTemplateClass(template_dir, xwalk_build_version,
    api_version, min_api_version, verify_xwalk_apk):
  template_file = os.path.join(template_dir, 'XWalkCoreVersion.template')
  template = Template(open(template_file, 'r').read())
  value = {'API_VERSION': api_version,
           'MIN_API_VERSION': min_api_version,
           'XWALK_BUILD_VERSION': xwalk_build_version}
  output_file = os.path.join(bridge_path, "XWalkCoreVersion.java")
  with open(output_file, 'w') as f:
    f.write(template.substitute(value))

  template_file = os.path.join(template_dir, 'XWalkAppVersion.template')
  template = Template(open(template_file, 'r').read())
  value = {'API_VERSION': api_version,
           'VERIFY_XWALK_APK': 'true' if verify_xwalk_apk == 1 else 'false',
           'XWALK_BUILD_VERSION': xwalk_build_version}
  output_file = os.path.join(wrapper_path, "XWalkAppVersion.java")
  with open(output_file, 'w') as f:
    f.write(template.substitute(value))

def main(argv):
  usage = """Usage: %prog [OPTIONS]
This script can generate bridge and wrap source files for given directory.
\'input_dir\' is provided as directory containing source files.
  """
  option_parser = optparse.OptionParser(usage=usage)
  option_parser.add_option('--input-dir',
                           help=('Input source file directory which contains '
                                 'input files'))
  option_parser.add_option('--template-dir',
                           help=('Templates directory to generate java source '
                                 'file'))
  option_parser.add_option('--bridge-output',
                           help=('Output directory where the bridge code is '
                                 'placed.'))
  option_parser.add_option('--wrapper-output',
                           help=('Output directory where the wrap code is '
                                 'placed.'))
  option_parser.add_option('--stamp', help='the file to touch on success.')
  option_parser.add_option('--api-version', help='API Version')
  option_parser.add_option('--min-api-version', help='Min API Version')
  option_parser.add_option('--verify-xwalk-apk', default=0, type='int',
      help='Verify Crosswalk library APK before loading')
  option_parser.add_option('--xwalk-build-version', help='XWalk Build Version')

  options, _ = option_parser.parse_args(argv)
  if (not options.input_dir or
      not options.bridge_output or
      not options.wrapper_output):
    print('Error: Must specify input and output.')
    return 1

  if os.path.isdir(options.bridge_output):
    shutil.rmtree(options.bridge_output)
  if os.path.isdir(options.wrapper_output):
    shutil.rmtree(options.wrapper_output)

  global bridge_path
  bridge_path = os.path.join(options.bridge_output,
                             os.path.sep.join(BRIDGE_PACKAGE.split('.')))
  os.makedirs(bridge_path)

  global wrapper_path
  wrapper_path = os.path.join(options.wrapper_output,
                              os.path.sep.join(WRAPPER_PACKAGE.split('.')))
  os.makedirs(wrapper_path)

  if options.input_dir:
    GenerateJavaBindingClass(options.input_dir)
    GenerateJavaReflectClass(options.input_dir)

  if options.template_dir:
    GenerateJavaTemplateClass(options.template_dir, options.xwalk_build_version,
        options.api_version, options.min_api_version, options.verify_xwalk_apk)

  if options.stamp:
    build_utils.Touch(options.stamp)

if __name__ == '__main__':
  sys.exit(main(sys.argv))
<commit_msg>[Android] Refine embedding API getHitTestResult<commit_after>#!/usr/bin/env python

# Copyright (c) 2014 Intel Corporation. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import optparse
import os
import shutil
import sys

from bridge_generator import BridgeGenerator
from interface_generator import InterfaceGenerator
from java_class import JavaClassLoader
from string import Template
from wrapper_generator import WrapperGenerator

GYP_ANDROID_DIR = os.path.join(os.path.dirname(__file__),
                               os.pardir, os.pardir, os.pardir,
                               'build',
                               'android',
                               'gyp')
sys.path.append(GYP_ANDROID_DIR)

from util import build_utils

# Classes list that have to generate bridge and wrap code.
CLASSES_TO_BE_PROCESS = [
    'ClientCertRequestHandlerInternal',
    'ClientCertRequestInternal',
    'XWalkCookieManagerInternal',
    'XWalkDownloadListenerInternal',
    'XWalkExtensionInternal',
    'XWalkExternalExtensionManagerInternal',
    'XWalkFindListenerInternal',
    'XWalkGetBitmapCallbackInternal',
    'XWalkHttpAuthHandlerInternal',
    'XWalkHttpAuthInternal',
    'XWalkJavascriptResultHandlerInternal',
    'XWalkJavascriptResultInternal',
    'XWalkNativeExtensionLoaderInternal',
    'XWalkNavigationHistoryInternal',
    'XWalkNavigationItemInternal',
    'XWalkPreferencesInternal',
    'XWalkResourceClientInternal',
    'XWalkSettingsInternal',
    'XWalkUIClientInternal',
    "XWalkHitTestResultInternal",
    'XWalkViewInternal',
    'XWalkWebResourceRequestHandlerInternal',
    'XWalkWebResourceRequestInternal',
    'XWalkWebResourceResponseInternal',
]

REFLECTION_HERLPER = [
    'ReflectMethod.java',
    'ReflectField.java',
    'ReflectConstructor.java',
]

WRAPPER_PACKAGE = 'org.xwalk.core'
BRIDGE_PACKAGE = 'org.xwalk.core.internal'

bridge_path = ''
wrapper_path = ''

def PerformSerialize(output_path, generator):
  file_name = generator.GetGeneratedClassFileName()
  with open(os.path.join(output_path, file_name), 'w') as f:
    f.write(generator.GetGeneratedCode())
  print('%s has been generated!' % file_name)

def GenerateJavaBindingClass(input_dir):
  class_loader = JavaClassLoader(input_dir, CLASSES_TO_BE_PROCESS)
  for input_class in CLASSES_TO_BE_PROCESS:
    print('Generate bridge and wrapper code for %s' % input_class)
    java_data = class_loader.GetJavaData(input_class)
    if java_data.class_type == 'interface':
      # Generate Interface code.
      interface_generator = InterfaceGenerator(java_data, class_loader)
      interface_generator.RunTask()
      PerformSerialize(wrapper_path, interface_generator)
    else:
      # Generate Bridge code.
      bridge_generator = BridgeGenerator(java_data, class_loader)
      bridge_generator.RunTask()
      PerformSerialize(bridge_path, bridge_generator)
      # Generate Wrapper code.
      wrapper_generator = WrapperGenerator(java_data, class_loader)
      wrapper_generator.RunTask()
      PerformSerialize(wrapper_path, wrapper_generator)

def GenerateJavaReflectClass(input_dir):
  for helper in REFLECTION_HERLPER:
    with open(os.path.join(wrapper_path, helper), 'w') as f:
      for line in open(os.path.join(input_dir, helper), 'r'):
        if line.startswith('package '):
          f.write('package ' + WRAPPER_PACKAGE + ';\n')
        else:
          f.write(line)

def GenerateJavaTemplateClass(template_dir, xwalk_build_version,
    api_version, min_api_version, verify_xwalk_apk):
  template_file = os.path.join(template_dir, 'XWalkCoreVersion.template')
  template = Template(open(template_file, 'r').read())
  value = {'API_VERSION': api_version,
           'MIN_API_VERSION': min_api_version,
           'XWALK_BUILD_VERSION': xwalk_build_version}
  output_file = os.path.join(bridge_path, "XWalkCoreVersion.java")
  with open(output_file, 'w') as f:
    f.write(template.substitute(value))

  template_file = os.path.join(template_dir, 'XWalkAppVersion.template')
  template = Template(open(template_file, 'r').read())
  value = {'API_VERSION': api_version,
           'VERIFY_XWALK_APK': 'true' if verify_xwalk_apk == 1 else 'false',
           'XWALK_BUILD_VERSION': xwalk_build_version}
  output_file = os.path.join(wrapper_path, "XWalkAppVersion.java")
  with open(output_file, 'w') as f:
    f.write(template.substitute(value))

def main(argv):
  usage = """Usage: %prog [OPTIONS]
This script can generate bridge and wrap source files for given directory.
\'input_dir\' is provided as directory containing source files.
  """
  option_parser = optparse.OptionParser(usage=usage)
  option_parser.add_option('--input-dir',
                           help=('Input source file directory which contains '
                                 'input files'))
  option_parser.add_option('--template-dir',
                           help=('Templates directory to generate java source '
                                 'file'))
  option_parser.add_option('--bridge-output',
                           help=('Output directory where the bridge code is '
                                 'placed.'))
  option_parser.add_option('--wrapper-output',
                           help=('Output directory where the wrap code is '
                                 'placed.'))
  option_parser.add_option('--stamp', help='the file to touch on success.')
  option_parser.add_option('--api-version', help='API Version')
  option_parser.add_option('--min-api-version', help='Min API Version')
  option_parser.add_option('--verify-xwalk-apk', default=0, type='int',
      help='Verify Crosswalk library APK before loading')
  option_parser.add_option('--xwalk-build-version', help='XWalk Build Version')

  options, _ = option_parser.parse_args(argv)
  if (not options.input_dir or
      not options.bridge_output or
      not options.wrapper_output):
    print('Error: Must specify input and output.')
    return 1

  if os.path.isdir(options.bridge_output):
    shutil.rmtree(options.bridge_output)
  if os.path.isdir(options.wrapper_output):
    shutil.rmtree(options.wrapper_output)

  global bridge_path
  bridge_path = os.path.join(options.bridge_output,
                             os.path.sep.join(BRIDGE_PACKAGE.split('.')))
  os.makedirs(bridge_path)

  global wrapper_path
  wrapper_path = os.path.join(options.wrapper_output,
                              os.path.sep.join(WRAPPER_PACKAGE.split('.')))
  os.makedirs(wrapper_path)

  if options.input_dir:
    GenerateJavaBindingClass(options.input_dir)
    GenerateJavaReflectClass(options.input_dir)

  if options.template_dir:
    GenerateJavaTemplateClass(options.template_dir, options.xwalk_build_version,
        options.api_version, options.min_api_version, options.verify_xwalk_apk)

  if options.stamp:
    build_utils.Touch(options.stamp)

if __name__ == '__main__':
  sys.exit(main(sys.argv))
<|endoftext|>


<commit_before># TODO
#
# @author    Oktay Acikalin <oktay.acikalin@gmail.com>
# @copyright Oktay Acikalin
# @license   MIT (LICENSE.txt)

from diamond import pyglet
from diamond.vault import Vault

class Sprite(pyglet.sprite.Sprite):

    def __init__(self, vault, parent_node=None, position=None, name=None, usage='dynamic'):
        self._vault = vault
        self._name = name
        self._action = 'none'
        self._parent_node = parent_node
        self._my_visibility = True
        if parent_node:
            parent_node._child_sprites.append(self)
            group = parent_node._sprite_group
            try:
                batch = parent_node.window._batch
            except AttributeError:
                batch = None
        else:
            batch, group = None, None
        # if position:
        #     self._x_rel, self._y_rel = position
        # else:
        #     self._x_rel, self._y_rel = 0, 0
        self._anim = vault.actions[self._action].animation
        # self._update_real_position()
        x, y = position if position is not None else (0, 0)
        super(Sprite, self).__init__(
            img=self._anim,
            # x=self._x,
            # y=self._y,
            x=x,
            y=y,
            batch=batch,
            group=group,
            usage=usage,
        )

    # def _update_real_position(self):
    #     if self._parent_node:
    #         self._x = self._parent_node._x_real + self._x_rel
    #         self._y = self._parent_node._y_real + self._y_rel
    #     else:
    #         self._x = self._x_rel
    #         self._y = self._y_rel
    #     self._x = self._x_rel
    #     self._y = self._y_rel
    #     if self._vertex_list:
    #         self._update_position()

    # def _set_x(self, x):
    #     if x != self._x_rel:
    #         self._x_rel = x
    #         self._update_real_position()

    # x = property(lambda self: self._x_rel, _set_x)

    # def _set_y(self, y):
    #     if y != self._y_rel:
    #         self._y_rel = y
    #         self._update_real_position()

    # y = property(lambda self: self._y_rel, _set_y)

    # def set_position(self, x, y):
    #     self._x_rel, self._y_rel = x, y
    #     self._update_real_position()

    # position = property(lambda self: (self._x_rel, self._y_rel),
    #                     lambda self, pos: self.set_position(*pos))

    def _update_inherited_visibility(self):
        self._visible = self._parent_node._inherited_visibility

    def _set_visible(self, visible):
        self._my_visibility = visible
        self._update_inherited_visibility()
        self._update_position()

    visible = property(lambda self: self._visible, _set_visible)

    def set_action(self, name):
        if name != self._action:
            self._action = name
            self._anim = self._vault.actions[name].animation
            self.image = self._anim

    action = property(lambda self: self._action, set_action)

    @classmethod
    def make(cls, vault_module, sprite_name=None, **kwargs):
        '''Factory which creates a vault and returns a proper Sprite object.'''
        if not isinstance(vault_module, Vault):
            vault = Vault.get_instance(vault_module)
        else:
            vault = vault_module
        if sprite_name is not None:
            vault = vault.get_sprite(str(sprite_name))
        else:
            vault = vault.get_sprites().values()[0]
        return cls(vault=vault, **kwargs)

    @classmethod
    def make_many(cls, vault_module, sprite_name=None, amount=1, **kwargs):
        '''Factory which creates a vault and returns x Sprite objects within a list.'''
        if not isinstance(vault_module, Vault):
            vault = Vault.get_instance(vault_module)
        else:
            vault = vault_module
        if sprite_name is not None:
            vault = vault.get_sprite(sprite_name)
        else:
            vault = vault.get_sprites().values()[0]
        return [cls(vault=vault, **kwargs) for count in xrange(0, amount)]

    def add_to(self, node):
        node.add_sprite(self)

    def remove_from_parent(self):
        self._parent_node.remove_sprite(self)
<commit_msg>Improved visibility code in Sprite class.<commit_after># TODO
#
# @author    Oktay Acikalin <oktay.acikalin@gmail.com>
# @copyright Oktay Acikalin
# @license   MIT (LICENSE.txt)

from diamond import pyglet
from diamond.vault import Vault

class Sprite(pyglet.sprite.Sprite):

    def __init__(self, vault, parent_node=None, position=None, name=None, usage='dynamic'):
        self._vault = vault
        self._name = name
        self._action = 'none'
        self._parent_node = parent_node
        self._my_visibility = True
        if parent_node:
            parent_node._child_sprites.append(self)
            group = parent_node._sprite_group
            try:
                batch = parent_node.window._batch
            except AttributeError:
                batch = None
        else:
            batch, group = None, None
        # if position:
        #     self._x_rel, self._y_rel = position
        # else:
        #     self._x_rel, self._y_rel = 0, 0
        self._anim = vault.actions[self._action].animation
        # self._update_real_position()
        x, y = position if position is not None else (0, 0)
        super(Sprite, self).__init__(
            img=self._anim,
            # x=self._x,
            # y=self._y,
            x=x,
            y=y,
            batch=batch,
            group=group,
            usage=usage,
        )

    # def _update_real_position(self):
    #     if self._parent_node:
    #         self._x = self._parent_node._x_real + self._x_rel
    #         self._y = self._parent_node._y_real + self._y_rel
    #     else:
    #         self._x = self._x_rel
    #         self._y = self._y_rel
    #     self._x = self._x_rel
    #     self._y = self._y_rel
    #     if self._vertex_list:
    #         self._update_position()

    # def _set_x(self, x):
    #     if x != self._x_rel:
    #         self._x_rel = x
    #         self._update_real_position()

    # x = property(lambda self: self._x_rel, _set_x)

    # def _set_y(self, y):
    #     if y != self._y_rel:
    #         self._y_rel = y
    #         self._update_real_position()

    # y = property(lambda self: self._y_rel, _set_y)

    # def set_position(self, x, y):
    #     self._x_rel, self._y_rel = x, y
    #     self._update_real_position()

    # position = property(lambda self: (self._x_rel, self._y_rel),
    #                     lambda self, pos: self.set_position(*pos))

    def _update_inherited_visibility(self):
        new_visibility = self._my_visibility and self._parent_node._inherited_visibility
        if new_visibility != self._visible:
            # We reuse pyglets sprite code here.
            self._visible = new_visibility
            self._update_position()

    def _set_visible(self, visible):
        self._my_visibility = visible
        self._update_inherited_visibility()

    visible = property(lambda self: self._visible, _set_visible)

    def set_action(self, name):
        if name != self._action:
            self._action = name
            self._anim = self._vault.actions[name].animation
            self.image = self._anim

    action = property(lambda self: self._action, set_action)

    @classmethod
    def make(cls, vault_module, sprite_name=None, **kwargs):
        '''Factory which creates a vault and returns a proper Sprite object.'''
        if not isinstance(vault_module, Vault):
            vault = Vault.get_instance(vault_module)
        else:
            vault = vault_module
        if sprite_name is not None:
            vault = vault.get_sprite(str(sprite_name))
        else:
            vault = vault.get_sprites().values()[0]
        return cls(vault=vault, **kwargs)

    @classmethod
    def make_many(cls, vault_module, sprite_name=None, amount=1, **kwargs):
        '''Factory which creates a vault and returns x Sprite objects within a list.'''
        if not isinstance(vault_module, Vault):
            vault = Vault.get_instance(vault_module)
        else:
            vault = vault_module
        if sprite_name is not None:
            vault = vault.get_sprite(sprite_name)
        else:
            vault = vault.get_sprites().values()[0]
        return [cls(vault=vault, **kwargs) for count in xrange(0, amount)]

    def add_to(self, node):
        node.add_sprite(self)

    def remove_from_parent(self):
        self._parent_node.remove_sprite(self)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Uploads RBE results to BigQuery"""

import argparse
import os
import json
import sys
import urllib2
import uuid

gcp_utils_dir = os.path.abspath(
    os.path.join(os.path.dirname(__file__), '../../gcp/utils'))
sys.path.append(gcp_utils_dir)
import big_query_utils

_DATASET_ID = 'jenkins_test_results'
_DESCRIPTION = 'Test results from master RBE builds on Kokoro'
# 90 days in milliseconds
_EXPIRATION_MS = 90 * 24 * 60 * 60 * 1000
_PARTITION_TYPE = 'DAY'
_PROJECT_ID = 'grpc-testing'
_RESULTS_SCHEMA = [
    ('job_name', 'STRING', 'Name of Kokoro job'),
    ('build_id', 'INTEGER', 'Build ID of Kokoro job'),
    ('build_url', 'STRING', 'URL of Kokoro build'),
    ('test_target', 'STRING', 'Bazel target path'),
    ('test_case', 'STRING', 'Name of test case'),
    ('result', 'STRING', 'Test or build result'),
    ('timestamp', 'TIMESTAMP', 'Timestamp of test run'),
    ('duration', 'FLOAT', 'Duration of the test run'),
]
_TABLE_ID = 'rbe_test_results'

def _get_api_key():
    """Returns string with API key to access ResultStore.
	Intended to be used in Kokoro environment."""
    api_key_directory = os.getenv('KOKORO_GFILE_DIR')
    api_key_file = os.path.join(api_key_directory, 'resultstore_api_key')
    assert os.path.isfile(api_key_file), 'Must add --api_key arg if not on ' \
     'Kokoro or Kokoro envrionment is not set up properly.'
    with open(api_key_file, 'r') as f:
        return f.read().replace('\n', '')

def _get_invocation_id():
    """Returns String of Bazel invocation ID. Intended to be used in
	Kokoro environment."""
    bazel_id_directory = os.getenv('KOKORO_ARTIFACTS_DIR')
    bazel_id_file = os.path.join(bazel_id_directory, 'bazel_invocation_ids')
    assert os.path.isfile(bazel_id_file), 'bazel_invocation_ids file, written ' \
     'by RBE initialization script, expected but not found.'
    with open(bazel_id_file, 'r') as f:
        return f.read().replace('\n', '')

def _parse_test_duration(duration_str):
    """Parse test duration string in '123.567s' format"""
    try:
        if duration_str.endswith('s'):
            duration_str = duration_str[:-1]
        return float(duration_str)
    except:
        return None

def _upload_results_to_bq(rows):
    """Upload test results to a BQ table.

  Args:
      rows: A list of dictionaries containing data for each row to insert
  """
    bq = big_query_utils.create_big_query()
    big_query_utils.create_partitioned_table(
        bq,
        _PROJECT_ID,
        _DATASET_ID,
        _TABLE_ID,
        _RESULTS_SCHEMA,
        _DESCRIPTION,
        partition_type=_PARTITION_TYPE,
        expiration_ms=_EXPIRATION_MS)

    max_retries = 3
    for attempt in range(max_retries):
        if big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, _TABLE_ID,
                                       rows):
            break
        else:
            if attempt < max_retries - 1:
                print('Error uploading result to bigquery, will retry.')
            else:
                print(
                    'Error uploading result to bigquery, all attempts failed.')
                sys.exit(1)

def _get_resultstore_data(api_key, invocation_id):
    """Returns dictionary of test results by querying ResultStore API.
  Args:
      api_key: String of ResultStore API key
      invocation_id: String of ResultStore invocation ID to results from
  """
    all_actions = []
    page_token = ''
    # ResultStore's API returns data on a limited number of tests. When we exceed
    # that limit, the 'nextPageToken' field is included in the request to get
    # subsequent data, so keep requesting until 'nextPageToken' field is omitted.
    while True:
        req = urllib2.Request(
            url=
            'https://resultstore.googleapis.com/v2/invocations/%s/targets/-/configuredTargets/-/actions?key=%s&pageToken=%s&fields=next_page_token,actions.id,actions.status_attributes'
            % (invocation_id, api_key, page_token),
            headers={
                'Content-Type': 'application/json'
            })
        results = json.loads(urllib2.urlopen(req).read())
        all_actions.extend(results['actions'])
        if 'nextPageToken' not in results:
            break
        page_token = results['nextPageToken']
    return all_actions

if __name__ == "__main__":
    # Arguments are necessary if running in a non-Kokoro environment.
    argp = argparse.ArgumentParser(description='Upload RBE results.')
    argp.add_argument('--api_key', default='', type=str)
    argp.add_argument('--invocation_id', default='', type=str)
    args = argp.parse_args()

    api_key = args.api_key or _get_api_key()
    invocation_id = args.invocation_id or _get_invocation_id()
    resultstore_actions = _get_resultstore_data(api_key, invocation_id)

    bq_rows = []
    for index, action in enumerate(resultstore_actions):
        # Filter out non-test related data, such as build results.
        if 'testAction' not in action:
            continue
        # Some test results contain the fileProcessingErrors field, which indicates
        # an issue with parsing results individual test cases.
        if 'fileProcessingErrors' in action:
            test_cases = [{
                'testCase': {
                    'caseName': str(action['id']['actionId']),
                }
            }]
        # Test timeouts have a different dictionary structure compared to pass and
        # fail results.
        elif action['statusAttributes']['status'] == 'TIMED_OUT':
            test_cases = [{
                'testCase': {
                    'caseName': str(action['id']['actionId']),
                    'timedOut': True
                }
            }]
        # When RBE believes its infrastructure is failing, it will abort and
        # mark running tests as UNKNOWN. These infrastructure failures may be
        # related to our tests, so we should investigate if specific tests are
        # repeatedly being marked as UNKNOWN.
        elif action['statusAttributes']['status'] == 'UNKNOWN':
            test_cases = [{
                'testCase': {
                    'caseName': str(action['id']['actionId']),
                    'unknown': True
                }
            }]
            # Take the timestamp from the previous action, which should be
            # a close approximation.
            action['timing'] = {
                'startTime':
                resultstore_actions[index - 1]['timing']['startTime']
            }
        else:
            test_cases = action['testAction']['testSuite']['tests'][0][
                'testSuite']['tests']
        for test_case in test_cases:
            if any(s in test_case['testCase'] for s in ['errors', 'failures']):
                result = 'FAILED'
            elif 'timedOut' in test_case['testCase']:
                result = 'TIMEOUT'
            elif 'unknown' in test_case['testCase']:
                result = 'UNKNOWN'
            else:
                result = 'PASSED'
            try:
                bq_rows.append({
                    'insertId': str(uuid.uuid4()),
                    'json': {
                        'job_name':
                        os.getenv('KOKORO_JOB_NAME'),
                        'build_id':
                        os.getenv('KOKORO_BUILD_NUMBER'),
                        'build_url':
                        'https://source.cloud.google.com/results/invocations/%s'
                        % invocation_id,
                        'test_target':
                        action['id']['targetId'],
                        'test_case':
                        test_case['testCase']['caseName'],
                        'result':
                        result,
                        'timestamp':
                        action['timing']['startTime'],
                        'duration':
                        _parse_test_duration(action['timing']['duration']),
                    }
                })
            except Exception as e:
                print('Failed to parse test result. Error: %s' % str(e))
                print(json.dumps(test_case, indent=4))
                bq_rows.append({
                    'insertId': str(uuid.uuid4()),
                    'json': {
                        'job_name':
                        os.getenv('KOKORO_JOB_NAME'),
                        'build_id':
                        os.getenv('KOKORO_BUILD_NUMBER'),
                        'build_url':
                        'https://source.cloud.google.com/results/invocations/%s'
                        % invocation_id,
                        'test_target':
                        action['id']['targetId'],
                        'test_case':
                        'N/A',
                        'result':
                        'UNPARSEABLE',
                        'timestamp':
                        'N/A',
                    }
                })

    # BigQuery sometimes fails with large uploads, so batch 1,000 rows at a time.
    for i in range((len(bq_rows) / 1000) + 1):
        _upload_results_to_bq(bq_rows[i * 1000:(i + 1) * 1000])
<commit_msg>debug printouts<commit_after>#!/usr/bin/env python
# Copyright 2017 gRPC authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Uploads RBE results to BigQuery"""

import argparse
import os
import json
import sys
import urllib2
import uuid

gcp_utils_dir = os.path.abspath(
    os.path.join(os.path.dirname(__file__), '../../gcp/utils'))
sys.path.append(gcp_utils_dir)
import big_query_utils

_DATASET_ID = 'jenkins_test_results'
_DESCRIPTION = 'Test results from master RBE builds on Kokoro'
# 90 days in milliseconds
_EXPIRATION_MS = 90 * 24 * 60 * 60 * 1000
_PARTITION_TYPE = 'DAY'
_PROJECT_ID = 'grpc-testing'
_RESULTS_SCHEMA = [
    ('job_name', 'STRING', 'Name of Kokoro job'),
    ('build_id', 'INTEGER', 'Build ID of Kokoro job'),
    ('build_url', 'STRING', 'URL of Kokoro build'),
    ('test_target', 'STRING', 'Bazel target path'),
    ('test_case', 'STRING', 'Name of test case'),
    ('result', 'STRING', 'Test or build result'),
    ('timestamp', 'TIMESTAMP', 'Timestamp of test run'),
    ('duration', 'FLOAT', 'Duration of the test run'),
]
_TABLE_ID = 'rbe_test_results'

def _get_api_key():
    """Returns string with API key to access ResultStore.
	Intended to be used in Kokoro environment."""
    api_key_directory = os.getenv('KOKORO_GFILE_DIR')
    api_key_file = os.path.join(api_key_directory, 'resultstore_api_key')
    assert os.path.isfile(api_key_file), 'Must add --api_key arg if not on ' \
     'Kokoro or Kokoro envrionment is not set up properly.'
    with open(api_key_file, 'r') as f:
        return f.read().replace('\n', '')

def _get_invocation_id():
    """Returns String of Bazel invocation ID. Intended to be used in
	Kokoro environment."""
    bazel_id_directory = os.getenv('KOKORO_ARTIFACTS_DIR')
    bazel_id_file = os.path.join(bazel_id_directory, 'bazel_invocation_ids')
    assert os.path.isfile(bazel_id_file), 'bazel_invocation_ids file, written ' \
     'by RBE initialization script, expected but not found.'
    with open(bazel_id_file, 'r') as f:
        return f.read().replace('\n', '')

def _parse_test_duration(duration_str):
    """Parse test duration string in '123.567s' format"""
    try:
        if duration_str.endswith('s'):
            duration_str = duration_str[:-1]
        return float(duration_str)
    except:
        return None

def _upload_results_to_bq(rows):
    """Upload test results to a BQ table.

  Args:
      rows: A list of dictionaries containing data for each row to insert
  """
    bq = big_query_utils.create_big_query()
    big_query_utils.create_partitioned_table(
        bq,
        _PROJECT_ID,
        _DATASET_ID,
        _TABLE_ID,
        _RESULTS_SCHEMA,
        _DESCRIPTION,
        partition_type=_PARTITION_TYPE,
        expiration_ms=_EXPIRATION_MS)

    max_retries = 3
    for attempt in range(max_retries):
        if big_query_utils.insert_rows(bq, _PROJECT_ID, _DATASET_ID, _TABLE_ID,
                                       rows):
            break
        else:
            if attempt < max_retries - 1:
                print('Error uploading result to bigquery, will retry.')
            else:
                print(
                    'Error uploading result to bigquery, all attempts failed.')
                sys.exit(1)

def _get_resultstore_data(api_key, invocation_id):
    """Returns dictionary of test results by querying ResultStore API.
  Args:
      api_key: String of ResultStore API key
      invocation_id: String of ResultStore invocation ID to results from
  """
    all_actions = []
    page_token = ''
    # ResultStore's API returns data on a limited number of tests. When we exceed
    # that limit, the 'nextPageToken' field is included in the request to get
    # subsequent data, so keep requesting until 'nextPageToken' field is omitted.
    while True:
        req = urllib2.Request(
            url=
            'https://resultstore.googleapis.com/v2/invocations/%s/targets/-/configuredTargets/-/actions?key=%s&pageToken=%s&fields=next_page_token,actions.id,actions.status_attributes'
            % (invocation_id, api_key, page_token),
            headers={
                'Content-Type': 'application/json'
            })
        results = json.loads(urllib2.urlopen(req).read())
        all_actions.extend(results['actions'])
        if 'nextPageToken' not in results:
            break
        page_token = results['nextPageToken']
    return all_actions

if __name__ == "__main__":
    # Arguments are necessary if running in a non-Kokoro environment.
    argp = argparse.ArgumentParser(description='Upload RBE results.')
    argp.add_argument('--api_key', default='', type=str)
    argp.add_argument('--invocation_id', default='', type=str)
    args = argp.parse_args()

    api_key = args.api_key or _get_api_key()
    invocation_id = args.invocation_id or _get_invocation_id()
    resultstore_actions = _get_resultstore_data(api_key, invocation_id)
    print(resultstore_actions)

    bq_rows = []
    for index, action in enumerate(resultstore_actions):
        # Filter out non-test related data, such as build results.
        if 'testAction' not in action:
            continue
        # Some test results contain the fileProcessingErrors field, which indicates
        # an issue with parsing results individual test cases.
        if 'fileProcessingErrors' in action:
            test_cases = [{
                'testCase': {
                    'caseName': str(action['id']['actionId']),
                }
            }]
        # Test timeouts have a different dictionary structure compared to pass and
        # fail results.
        elif action['statusAttributes']['status'] == 'TIMED_OUT':
            test_cases = [{
                'testCase': {
                    'caseName': str(action['id']['actionId']),
                    'timedOut': True
                }
            }]
        # When RBE believes its infrastructure is failing, it will abort and
        # mark running tests as UNKNOWN. These infrastructure failures may be
        # related to our tests, so we should investigate if specific tests are
        # repeatedly being marked as UNKNOWN.
        elif action['statusAttributes']['status'] == 'UNKNOWN':
            test_cases = [{
                'testCase': {
                    'caseName': str(action['id']['actionId']),
                    'unknown': True
                }
            }]
            # Take the timestamp from the previous action, which should be
            # a close approximation.
            action['timing'] = {
                'startTime':
                resultstore_actions[index - 1]['timing']['startTime']
            }
        else:
            test_cases = action['testAction']['testSuite']['tests'][0][
                'testSuite']['tests']
        for test_case in test_cases:
            if any(s in test_case['testCase'] for s in ['errors', 'failures']):
                result = 'FAILED'
            elif 'timedOut' in test_case['testCase']:
                result = 'TIMEOUT'
            elif 'unknown' in test_case['testCase']:
                result = 'UNKNOWN'
            else:
                result = 'PASSED'
            try:
                bq_rows.append({
                    'insertId': str(uuid.uuid4()),
                    'json': {
                        'job_name':
                        os.getenv('KOKORO_JOB_NAME'),
                        'build_id':
                        os.getenv('KOKORO_BUILD_NUMBER'),
                        'build_url':
                        'https://source.cloud.google.com/results/invocations/%s'
                        % invocation_id,
                        'test_target':
                        action['id']['targetId'],
                        'test_case':
                        test_case['testCase']['caseName'],
                        'result':
                        result,
                        'timestamp':
                        action['timing']['startTime'],
                        'duration':
                        _parse_test_duration(action['timing']['duration']),
                    }
                })
            except Exception as e:
                print('Failed to parse test result. Error: %s' % str(e))
                print(json.dumps(test_case, indent=4))
                bq_rows.append({
                    'insertId': str(uuid.uuid4()),
                    'json': {
                        'job_name':
                        os.getenv('KOKORO_JOB_NAME'),
                        'build_id':
                        os.getenv('KOKORO_BUILD_NUMBER'),
                        'build_url':
                        'https://source.cloud.google.com/results/invocations/%s'
                        % invocation_id,
                        'test_target':
                        action['id']['targetId'],
                        'test_case':
                        'N/A',
                        'result':
                        'UNPARSEABLE',
                        'timestamp':
                        'N/A',
                    }
                })

    # BigQuery sometimes fails with large uploads, so batch 1,000 rows at a time.
    for i in range((len(bq_rows) / 1000) + 1):
        _upload_results_to_bq(bq_rows[i * 1000:(i + 1) * 1000])
<|endoftext|>


<commit_before>#!/usr/local/bin/python
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

SVNLOOK="/usr/local/svn-install/current/bin/svnlook"
#SVNLOOK="/usr/local/bin/svnlook"

HOST="127.0.0.1"
PORT=2069

import sys
try:
    import simplejson as json
except ImportError:
    import json

import urllib2

import svnpubsub.util

def svnlook(cmd, **kwargs):
    args = [SVNLOOK] + cmd
    return svnpubsub.util.check_output(args, **kwargs)

def svnlook_uuid(repo):
    cmd = ["uuid", "--", repo]
    return svnlook(cmd).strip()

def svnlook_revprop(repo, revision, propname):
    cmd = ["propget", "-r", revision, "--revprop", "--", repo, propname]
    data = svnlook(cmd)
    #print data
    return data

def do_put(body):
    opener = urllib2.build_opener(urllib2.HTTPHandler)
    request = urllib2.Request("http://%s:%d/metadata" %(HOST, PORT), data=body)
    request.add_header('Content-Type', 'application/json')
    request.get_method = lambda: 'PUT'
    url = opener.open(request)

def main(repo, revision, author, propname, action):
    revision = revision.lstrip('r')
    if action in ('A', 'M'):
        new_value = svnlook_revprop(repo, revision, propname)
    elif action == 'D':
        new_value = None
    else:
        sys.stderr.write('Unknown revprop change action "%s"\n' % action)
        return
    if action in ('D', 'M'):
        old_value = sys.stdin.read()
    else:
        old_value = None
    data = {'type': 'svn',
            'format': 1,
            'id': int(revision),
            'repository': svnlook_uuid(repo),
            'revprop': {
                'name': propname,
                'committer': author,
                'value': new_value,
                'old_value': old_value,
                }
            }
    body = json.dumps(data)
    do_put(body)

if __name__ == "__main__":
    if len(sys.argv) != 6:
        sys.stderr.write("invalid args\n")
        sys.exit(1)

    main(*sys.argv[1:6])
<commit_msg>* tools/server-side/svnpubsub/revprop-change-hook.py (main):    Exit with an error code if the action parameter is not valid.<commit_after>#!/usr/local/bin/python
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

SVNLOOK="/usr/local/svn-install/current/bin/svnlook"
#SVNLOOK="/usr/local/bin/svnlook"

HOST="127.0.0.1"
PORT=2069

import sys
try:
    import simplejson as json
except ImportError:
    import json

import urllib2

import svnpubsub.util

def svnlook(cmd, **kwargs):
    args = [SVNLOOK] + cmd
    return svnpubsub.util.check_output(args, **kwargs)

def svnlook_uuid(repo):
    cmd = ["uuid", "--", repo]
    return svnlook(cmd).strip()

def svnlook_revprop(repo, revision, propname):
    cmd = ["propget", "-r", revision, "--revprop", "--", repo, propname]
    data = svnlook(cmd)
    #print data
    return data

def do_put(body):
    opener = urllib2.build_opener(urllib2.HTTPHandler)
    request = urllib2.Request("http://%s:%d/metadata" %(HOST, PORT), data=body)
    request.add_header('Content-Type', 'application/json')
    request.get_method = lambda: 'PUT'
    url = opener.open(request)

def main(repo, revision, author, propname, action):
    revision = revision.lstrip('r')
    if action in ('A', 'M'):
        new_value = svnlook_revprop(repo, revision, propname)
    elif action == 'D':
        new_value = None
    else:
        sys.stderr.write('Unknown revprop change action "%s"\n' % action)
        sys.exit(1)
    if action in ('D', 'M'):
        old_value = sys.stdin.read()
    else:
        old_value = None
    data = {'type': 'svn',
            'format': 1,
            'id': int(revision),
            'repository': svnlook_uuid(repo),
            'revprop': {
                'name': propname,
                'committer': author,
                'value': new_value,
                'old_value': old_value,
                }
            }
    body = json.dumps(data)
    do_put(body)

if __name__ == "__main__":
    if len(sys.argv) != 6:
        sys.stderr.write("invalid args\n")
        sys.exit(1)

    main(*sys.argv[1:6])
<|endoftext|>


<commit_before>#!/usr/bin/env python
#coding=utf8

"""
Usage:
  mov create [options] DIRECTORY ...
  mov update [options] DIRECTORY ...
  mov destroy [options]
  mov ls [options] [PATTERN ...]
  mov play [options] [PATTERN ...]
  mov -h | --help
  mov --version

Commands:
  create               Create a new database with information about the films
                       in the specified directory or directories.
  update               Update an existing database. Old items that are not
                       found are deleted.
  destroy              Destroy a database.
  ls                   List movies and relevant metadata optionally only
                       showing those that match a specified pattern.
  play                 Open the first matching movie with a media player.

Options:
  --database=DATABASE  Database to save film metadata [default: ~/.mov.db].
  --force              Do not prompt for a confirmation upon database
                       destruction.
  -s, --strict         Only show exact matches.
  -n, --name           Only show the name of the movie.
  -l, --location       Only show the location of the movie.
  -S, --size           Only show the size of the movie.
  -f, --files          Only show the files of the movie.
  --prefix=PREFIX      Size prefix, one of "", k, M, G or T. If this option is
                       not used the best prefix in each case is selected.
  --player=PLAYER      Media player to open movies with [default: vlc].
  -h, --help           Show this help message and exit.
  --version            Show version.
"""

__version__ = '0.0.1'

import os
import sqlite3

try:
    from docopt import docopt
except ImportError, error:
    exit('Error: {0}. Install it with "pip install docopt".'.format(error))

def get_size(path):
    '''Return the size of path in bytes if it exists and can be determined.'''
    size = os.path.getsize(path)
    for item in os.walk(path):
        for file in item[2]:
            size += os.path.getsize(os.path.join(item[0], file))
    return size

def local_data(path):
    """Return tuples of names, directories, total sizes and files. Each
       directory represents a single film and the files are the files contained
       in the directory, such as video, audio and subtitle files."""
    dirs = [os.path.join(path, item) for item in os.listdir(path)]
    names, sizes, files = zip(*[(dir.split('/')[-1], str(get_size(dir)),
                                 '##'.join([file for file in os.listdir(dir)]))
                                for dir in dirs])
    return zip(names, dirs, sizes, files)

def prefix_size(size, base=1024):
    '''Return size in B (bytes), kB, MB, GB or TB.'''
    if ARGS.prefix == 'None':
        for i, prefix in enumerate(['', 'ki', 'Mi', 'Gi', 'Ti']):
            if size < pow(base, i + 1):
                return '{0} {1}B'.format(round(float(size) / pow(base, i), 1),
                                         prefix)
    else:
        try:
            prefix = {'': 0, 'k': 1, 'M': 2, 'G': 3, 'T': 4}[ARGS.prefix]
        except KeyError:
            exit('Invalid prefix.')
        return '{0} {1}B'.format(round(float(size) / pow(base, prefix), 1),
                                 ARGS.prefix)

def create():
    """Create a new database with information about the films in the specified
       directory or directories."""
    if not all(map(os.path.isdir, ARGS.directory)):
        exit('Error: One or more of the specified directories does not exist.')
    with sqlite3.connect(ARGS.database) as connection:
        connection.text_factory = str
        cursor = connection.cursor()
        cursor.execute('DROP TABLE IF EXISTS Movies')
        cursor.execute('''CREATE TABLE Movies(name TEXT, path TEXT, size TEXT,
                          files BLOB)''')
        for dir in ARGS.directory:
            cursor.executemany('INSERT INTO Movies VALUES(?, ?, ?, ?)',
                               local_data(dir))

def update():
    """Update an existing database. Old items that are not found are
       deleted."""
    # Temporarily use the same method to create and update a database.
    return create

def destroy():
    """Destroy a database."""
    if not os.path.exists(ARGS.database):
        exit('Error: The database does not exist; you must create it first.')
    if ARGS.force:
        os.remove(ARGS.database)
    elif raw_input('Destroy {0} [y/n]? '.format(ARGS.database)) in ('y', 'Y'):
        os.remove(ARGS.database)

def ls():
    """List all items in the database in a predefined format."""
    if not os.path.exists(ARGS.database):
        exit('Error: The database does not exist; you must create it first.')
    with sqlite3.connect(ARGS.database) as connection:
        connection.text_factory = str
        cursor = connection.cursor()
        if ARGS.pattern:
            if not ARGS.strict:
                ARGS.pattern = '%{0}%'.format(ARGS.pattern)
            cursor.execute('SELECT * FROM Movies WHERE Name LIKE (?)',
                      [ARGS.pattern])
        else:
            cursor.execute('SELECT * FROM Movies')
        movies = sorted([row for row in cursor])
    if ARGS.name:
        print '\n'.join([movie[0] for movie in movies])
    elif ARGS.location:
        print '\n'.join([movie[1] for movie in movies])
    elif ARGS.size:
        print '\n'.join([prefix_size(int(movie[2])) for movie in movies])
    elif ARGS.files:
        for movie in movies:
            print ', '.join(movie[3].split('##'))
    else:
        for i, movie in enumerate(movies):
            print 'Name:\t\t{0}'.format(movie[0])
            print 'Location:\t{0}'.format(movie[1])
            print 'Size:\t\t{0}'.format(prefix_size(int(movie[2])))
            print 'Files:\t\t{0}'.format(', '.join(movie[3].split('##')))
            if not i == len(movies) - 1:
                print

def play():
    """Open the matched movie with a media player."""
    with sqlite3.connect(ARGS.database) as connection:
        connection.text_factory = str
        cursor = connection.cursor()
        if ARGS.pattern:
            if not ARGS.strict:
                ARGS.pattern = '%{0}%'.format(ARGS.pattern)
            cursor.execute('SELECT * FROM Movies WHERE Name LIKE (?)',
                           [ARGS.pattern])
            try:
                path = sorted([row for row in cursor])[0][1]
                replace_map = {' ': '\\ ', '"': '\\"', "'": "\\'"}
                for key, val in replace_map.iteritems():
                    path = path.replace(key, val)
                os.system('{0} {1} &'.format(ARGS.player, path))
            except IndexError:
                exit('Error: Movie not found.')

def main():
    # If the arguments are not valid (as defined in __doc__) the program exits.
    args_dict = docopt(__doc__, version='mov {0}'.format(__version__))
    global ARGS
    # Clean key names for later namespace use.
    ARGS = {arg.lower().replace('-', ''): args_dict[arg] for arg in args_dict}
    # Make arguments available in a namespace.
    ARGS = type('Namespace', (object,), ARGS)
    # Resolve ~ to the user's home directory if applicable.
    if ARGS.database[0] == '~':
        ARGS.database = os.path.expanduser('~') + ARGS.database[1:]
    if len(ARGS.pattern) > 1:
        ARGS.pattern = ' '.join(ARGS.pattern)
    elif len(ARGS.pattern):
        ARGS.pattern = ARGS.pattern[0]
    commands = ('create', 'update', 'destroy', 'ls', 'play')
    # Call the respective function for the command entered.
    globals()[[command for command in commands if args_dict[command]][0]]()

if __name__ == '__main__':
    main()
<commit_msg>Add default prefix (listing movies would fail without it)<commit_after>#!/usr/bin/env python
#coding=utf8

"""
Usage:
  mov create [options] DIRECTORY ...
  mov update [options] DIRECTORY ...
  mov destroy [options]
  mov ls [options] [PATTERN ...]
  mov play [options] [PATTERN ...]
  mov -h | --help
  mov --version

Commands:
  create               Create a new database with information about the films
                       in the specified directory or directories.
  update               Update an existing database. Old items that are not
                       found are deleted.
  destroy              Destroy a database.
  ls                   List movies and relevant metadata optionally only
                       showing those that match a specified pattern.
  play                 Open the first matching movie with a media player.

Options:
  --database=DATABASE  Database to save film metadata [default: ~/.mov.db].
  --force              Do not prompt for a confirmation upon database
                       destruction.
  -s, --strict         Only show exact matches.
  -n, --name           Only show the name of the movie.
  -l, --location       Only show the location of the movie.
  -S, --size           Only show the size of the movie.
  -f, --files          Only show the files of the movie.
  --prefix=PREFIX      Size prefix, one of "", k, M, G or T. If this option is
                       not used the best prefix in each case is selected
                       [default: G].
  --player=PLAYER      Media player to open movies with [default: vlc].
  -h, --help           Show this help message and exit.
  --version            Show version.
"""

__version__ = '0.0.1'

import os
import sqlite3

try:
    from docopt import docopt
except ImportError, error:
    exit('Error: {0}. Install it with "pip install docopt".'.format(error))

def get_size(path):
    '''Return the size of path in bytes if it exists and can be determined.'''
    size = os.path.getsize(path)
    for item in os.walk(path):
        for file in item[2]:
            size += os.path.getsize(os.path.join(item[0], file))
    return size

def local_data(path):
    """Return tuples of names, directories, total sizes and files. Each
       directory represents a single film and the files are the files contained
       in the directory, such as video, audio and subtitle files."""
    dirs = [os.path.join(path, item) for item in os.listdir(path)]
    names, sizes, files = zip(*[(dir.split('/')[-1], str(get_size(dir)),
                                 '##'.join([file for file in os.listdir(dir)]))
                                for dir in dirs])
    return zip(names, dirs, sizes, files)

def prefix_size(size, base=1024):
    '''Return size in B (bytes), kB, MB, GB or TB.'''
    if ARGS.prefix == 'None':
        for i, prefix in enumerate(['', 'ki', 'Mi', 'Gi', 'Ti']):
            if size < pow(base, i + 1):
                return '{0} {1}B'.format(round(float(size) / pow(base, i), 1),
                                         prefix)
    else:
        try:
            prefix = {'': 0, 'k': 1, 'M': 2, 'G': 3, 'T': 4}[ARGS.prefix]
        except KeyError:
            exit('Invalid prefix.')
        return '{0} {1}B'.format(round(float(size) / pow(base, prefix), 1),
                                 ARGS.prefix)

def create():
    """Create a new database with information about the films in the specified
       directory or directories."""
    if not all(map(os.path.isdir, ARGS.directory)):
        exit('Error: One or more of the specified directories does not exist.')
    with sqlite3.connect(ARGS.database) as connection:
        connection.text_factory = str
        cursor = connection.cursor()
        cursor.execute('DROP TABLE IF EXISTS Movies')
        cursor.execute('''CREATE TABLE Movies(name TEXT, path TEXT, size TEXT,
                          files BLOB)''')
        for dir in ARGS.directory:
            cursor.executemany('INSERT INTO Movies VALUES(?, ?, ?, ?)',
                               local_data(dir))

def update():
    """Update an existing database. Old items that are not found are
       deleted."""
    # Temporarily use the same method to create and update a database.
    return create

def destroy():
    """Destroy a database."""
    if not os.path.exists(ARGS.database):
        exit('Error: The database does not exist; you must create it first.')
    if ARGS.force:
        os.remove(ARGS.database)
    elif raw_input('Destroy {0} [y/n]? '.format(ARGS.database)) in ('y', 'Y'):
        os.remove(ARGS.database)

def ls():
    """List all items in the database in a predefined format."""
    if not os.path.exists(ARGS.database):
        exit('Error: The database does not exist; you must create it first.')
    with sqlite3.connect(ARGS.database) as connection:
        connection.text_factory = str
        cursor = connection.cursor()
        if ARGS.pattern:
            if not ARGS.strict:
                ARGS.pattern = '%{0}%'.format(ARGS.pattern)
            cursor.execute('SELECT * FROM Movies WHERE Name LIKE (?)',
                           [ARGS.pattern])
        else:
            cursor.execute('SELECT * FROM Movies')
        movies = sorted([row for row in cursor])
    if ARGS.name:
        print '\n'.join([movie[0] for movie in movies])
    elif ARGS.location:
        print '\n'.join([movie[1] for movie in movies])
    elif ARGS.size:
        print '\n'.join([prefix_size(int(movie[2])) for movie in movies])
    elif ARGS.files:
        for movie in movies:
            print ', '.join(movie[3].split('##'))
    else:
        for i, movie in enumerate(movies):
            print 'Name:\t\t{0}'.format(movie[0])
            print 'Location:\t{0}'.format(movie[1])
            print 'Size:\t\t{0}'.format(prefix_size(int(movie[2])))
            print 'Files:\t\t{0}'.format(', '.join(movie[3].split('##')))
            if not i == len(movies) - 1:
                print

def play():
    """Open the matched movie with a media player."""
    with sqlite3.connect(ARGS.database) as connection:
        connection.text_factory = str
        cursor = connection.cursor()
        if ARGS.pattern:
            if not ARGS.strict:
                ARGS.pattern = '%{0}%'.format(ARGS.pattern)
            cursor.execute('SELECT * FROM Movies WHERE Name LIKE (?)',
                           [ARGS.pattern])
            try:
                path = sorted([row for row in cursor])[0][1]
                replace_map = {' ': '\\ ', '"': '\\"', "'": "\\'"}
                for key, val in replace_map.iteritems():
                    path = path.replace(key, val)
                os.system('{0} {1} &'.format(ARGS.player, path))
            except IndexError:
                exit('Error: Movie not found.')

def main():
    # If the arguments are not valid (as defined in __doc__) the program exits.
    args_dict = docopt(__doc__, version='mov {0}'.format(__version__))
    global ARGS
    # Clean key names for later namespace use.
    ARGS = {arg.lower().replace('-', ''): args_dict[arg] for arg in args_dict}
    # Make arguments available in a namespace.
    ARGS = type('Namespace', (object,), ARGS)
    # Resolve ~ to the user's home directory if applicable.
    if ARGS.database[0] == '~':
        ARGS.database = os.path.expanduser('~') + ARGS.database[1:]
    if len(ARGS.pattern) > 1:
        ARGS.pattern = ' '.join(ARGS.pattern)
    elif len(ARGS.pattern):
        ARGS.pattern = ARGS.pattern[0]
    commands = ('create', 'update', 'destroy', 'ls', 'play')
    # Call the respective function for the command entered.
    globals()[[command for command in commands if args_dict[command]][0]]()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import sys

import numpy as np

"""
    Just out of interest, here is a version, based on a version
    written in Ruby that I found in the forums, and translated
    into Python. It is much slower, but I find it fascinating.
    For some reason, examples of Dynamic Programming always tend
    to turn my brain inside out, and this example is no different
    in that respect.

    def matchings(rna):
        A = np.zeros((len(rna), len(rna)), dtype = int)

        for i in xrange(len(rna)):
            for j in xrange(i - 1, -1, -1):
                if rna[i] == COMPLEMENT[rna[j]]:
                    if i - 1 >= j + 1:
                        A[i, j] = A[i - 1, j + 1]
                    else:
                        A[i, j] = 1

            for j in xrange(i):
                for k in xrange(j):
                    A[i, k] += A[i, j] * A[j - 1, k]
                    A[i, k] %= 1000000

        return A

"""

COMPLEMENT = {'A': 'U', 'U': 'A', 'C': 'G', 'G': 'C'}

def read_fasta(file_path):
    dna = ''

    with open(file_path) as file:
        for line in file:
            line = line.strip()
            if not line.startswith('>'):
                dna += line

    return dna

def check_occurences(rna):
    return rna.count('A') == rna.count('U') and rna.count('C') == rna.count('G')

def matchings(rna, start, end, A):
    if end < start:
        return 1
    elif A[start, end] == -1:
        A[start, end] = 0

        if check_occurences(rna[start:end + 1]):
            for i in xrange(start + 1, end + 1, 2):
                if rna[start] == COMPLEMENT[rna[i]]:
                    A[start, end] += matchings(rna, start + 1, i - 1, A) * matchings(rna, i + 1, end, A)
                    A[start, end] %= 1000000

    return A[start, end]

def main(argv):
    rna = read_fasta(argv[0])

    A = np.empty((len(rna), len(rna)), dtype = int)
    A.fill(-1)

    print matchings(rna, 0, len(rna) - 1, A)

if __name__ == "__main__":
    main(sys.argv[1:])
<commit_msg>slight cleanup - introduce N, the length of the rna string<commit_after>import sys

import numpy as np

"""
    Just out of interest, here is a version, based on a version
    written in Ruby that I found in the forums, and translated
    into Python. It is much slower, but I find it fascinating.
    For some reason, examples of Dynamic Programming always tend
    to turn my brain inside out, and this example is no different
    in that respect.

    def matchings(rna):
        A = np.zeros((len(rna), len(rna)), dtype = int)

        for i in xrange(len(rna)):
            for j in xrange(i - 1, -1, -1):
                if rna[i] == COMPLEMENT[rna[j]]:
                    if i - 1 >= j + 1:
                        A[i, j] = A[i - 1, j + 1]
                    else:
                        A[i, j] = 1

            for j in xrange(i):
                for k in xrange(j):
                    A[i, k] += A[i, j] * A[j - 1, k]
                    A[i, k] %= 1000000

        return A

"""

COMPLEMENT = {'A': 'U', 'U': 'A', 'C': 'G', 'G': 'C'}

def read_fasta(file_path):
    dna = ''

    with open(file_path) as file:
        for line in file:
            line = line.strip()
            if not line.startswith('>'):
                dna += line

    return dna

def check_occurences(rna):
    return rna.count('A') == rna.count('U') and rna.count('C') == rna.count('G')

def matchings(rna, start, end, A):
    if end < start:
        return 1
    elif A[start, end] == -1:
        A[start, end] = 0

        if check_occurences(rna[start:end + 1]):
            for i in xrange(start + 1, end + 1, 2):
                if rna[start] == COMPLEMENT[rna[i]]:
                    A[start, end] += matchings(rna, start + 1, i - 1, A) * matchings(rna, i + 1, end, A)
                    A[start, end] %= 1000000

    return A[start, end]

def main(argv):
    rna = read_fasta(argv[0])
    N   = len(rna)

    A = np.empty((N, N), dtype = int)
    A.fill(-1)

    print matchings(rna, 0, N - 1, A)

if __name__ == "__main__":
    main(sys.argv[1:])
<|endoftext|>