<commit_before>from datetime import datetime
import logging

from django.core.cache import cache
from django.core.exceptions import ObjectDoesNotExist
from django.utils.translation import ugettext as _
from edx_rest_api_client.client import EdxRestApiClient
from requests import RequestException
from slumber.exceptions import HttpClientError, HttpNotFoundError

from bridge_lti.models import LtiConsumer

log = logging.getLogger(__name__)

class OpenEdxApiClient(EdxRestApiClient):
    """API client to interact with OpenEdx Course API."""

    API_URLS = {
        "get_token": "/oauth2/access_token",
        "base_url": "/api/courses/v1/",
    }

    def __init__(self, content_source, url=None, jwt=None, **kwargs):
        log.debug("Creating new OpenEdx API client...")
        self.content_source = content_source

        if not url:
            url = '{}{}'.format(content_source.host_url, self.API_URLS['base_url'])
        if not jwt:
            api_client_id = self.content_source.o_auth_client.client_id
            token_cache_key = "api:{}:token".format(api_client_id)

            access_token = cache.get(token_cache_key)
            if not access_token:
                access_token, expires_at = self.get_oauth_access_token()
                ttl = expires_at - datetime.now()
                cache.set(token_cache_key, access_token, ttl.seconds)

        super(OpenEdxApiClient, self).__init__(url, jwt=access_token, **kwargs)

    def get_oauth_access_token(self):
        """
        Request OpenEdx API OAuth2 token.

        Token type: JWT (reference: https://jwt.io/).
        :return: access_token, expires_at
        """
        url = "{host_url}{token_url}".format(
            host_url=self.content_source.host_url,
            token_url=self.API_URLS['get_token']
        )
        log.debug("Requesting oauth token: (url={})".format(url))
        try:
            oauth_client = self.content_source.o_auth_client
            access_token, expires_at = super(OpenEdxApiClient, self).get_oauth_access_token(
                url=url,
                client_id=oauth_client.client_id,
                client_secret=oauth_client.client_secret,
                token_type='jwt',
            )
        except ObjectDoesNotExist:
            raise HttpClientError(
                "OAuth token request failure. Please, configure OAuth client in order to be able make API requests."
            )
        except ValueError:
            log.exception(
                "You may want to check your OAuth registration on LTI Provider."
                "LTI Provider may be disabled (to enable: LMS config > FEATURES > ENABLE_OAUTH2_PROVIDER: true"
            )
            raise HttpClientError(
                "OAuth token request failure. You may want to check your OAuth registration on LTI Provider or"
                "enable OAuth Provider."
            )
        except RequestException:
            log.exception('OAuth2 token request to the OpenEdx LTI Provider failed.')
            raise HttpClientError(
                "OAuth token request failure. You may want to check your LTI Provider's HOST_URL(https)."
            )
        return access_token, expires_at

    def get_course_blocks(self, course_id, all_blocks=True, depth='all', type_filter=None):
        """
        Provide API GET request to OpenEdx Course Blocks endpoint.

        Optional block type filtering available.
        Endpoint: /api/courses/v1/blocks/?course_id
        API query parameters:
        - block_types_filter=['sequential', 'vertical', 'html', 'problem', 'video', 'discussion']
        - block_counts=['video', 'problem'...]
        -
        see: http://edx.readthedocs.io/projects/edx-platform-api/en/latest/courses/blocks.html#query-parameters

        :param filter_function: partially applied function `apply_data_filter` with filter parameters.
        :return: (list)
        """
        resource = self.blocks.get(
            course_id=course_id,
            all_blocks=all_blocks,
            depth=depth,
            requested_fields='lti_url',
            return_type='list',
            block_types_filter=type_filter or []
        )
        return resource

    def get_provider_courses(self, username=None, org=None, mobile=None):
        """
        Provide API GET request to OpenEdx Courses Resource endpoint.

        Gets a List of Courses.
        Endpoint: /api/courses/v1/courses/
        see: http://edx.readthedocs.io/projects/edx-platform-api/en/latest/courses/courses.html#query-parameters
        :return: (dict)
        """
        resource = self.courses.get(
            username=username,
            org=org,
            mobile=mobile
        )
        return resource.get('results')

def get_active_content_sources(source_id=None, not_allow_empty_source_id=True):
    """
    Check that passed source_id parameter is valid.

    If there's only one active source provider - source_id parameter is not required, it will get first active.
    :param source_id: LtiConsumer object id
    :param not_allow_empty_source_id: if True - it will not allow empty source_id, if False - source_id could be None
    :return: queryset of content_sources
    :raise HttpClientError: if provided parameters are not valid
    """
    if not_allow_empty_source_id and not source_id:
        # if source_id is not provided and more than one active content_provider
        raise HttpClientError(_("Parameter source_id is mandatory if there's more than one active content source."))

    content_sources = get_content_providers(source_id=source_id)

    if not content_sources:
        # if no active content sources
        raise HttpClientError(_("No active Content Provider"))

    return content_sources

def get_available_blocks(source_id, course_id=''):
    """
    Content Source API requester.

    Fetches all source blocks from the course with the given ID.
    Blocks data is filtered by `apply_data_filter`.
    :param course_id: Course id
    :param source_id: LtiConsumer id
    :return: (list) blocks data
    """
    content_source = get_active_content_sources(source_id).first()
    all_blocks = []

    # Get API client instance:
    api = OpenEdxApiClient(content_source=content_source)
    try:
        # filter function will be applied to api response
        all_blocks.extend(
            apply_data_filter(
                api.get_course_blocks(
                    course_id, type_filter=['html', 'problem', 'video']
                ),
                filters=['id', 'block_id', 'display_name', 'lti_url', 'type', 'content_source_id'],
                context_id=course_id,
                content_source_id=content_source.id
            )
        )
    except HttpNotFoundError:
        raise HttpClientError(_("Requested course not found. Check `course_id` url encoding."))
    except HttpClientError as exc:
        raise HttpClientError(_("Not valid query: {}").format(exc.message))

    return all_blocks

def get_available_courses(source_id=None):
    """
    Fetch all available courses from all sources of from source with provided ID.

    :param source_id: content provider's ID
    :return: (list) course_ids
    """
    content_sources = get_active_content_sources(source_id, not_allow_empty_source_id=False)

    all_courses = []

    for content_source in content_sources:
        # Get API client instance:
        api = OpenEdxApiClient(content_source=content_source)
        try:
            all_courses.extend(
                apply_data_filter(
                    api.get_provider_courses(),
                    filters=['id', 'course_id', 'name', 'org', 'content_source_id'],
                    content_source_id=content_source.id
                )
            )
        except HttpClientError as exc:
            raise HttpClientError(_("Not valid query: {}").format(exc.message))

    return all_courses

def add_to_dict(data, **kwargs):
    """Add key and value to dict only if this pair not exist yet in data dict."""
    for k, v in kwargs.items():
        if k not in data:
            data[k] = v
    return data

def apply_data_filter(data, filters=None, **kwargs):
    """
    Filter for `blocks` OpenEdx Course API response.

    Picks data which is listed in `filters` only.
    :param data: (list)
    :param filters: list of desired resource keys
    :param kwargs: params to append to every object if it's not present in the object yet
    :return: list of resources which keys were filtered
    """
    if filters is None:
        return data

    filtered_data = []
    for resource in data:
        filtered_resource = {k: v for k, v in resource.items() if k in filters}
        filtered_data.append(add_to_dict(filtered_resource, **kwargs))
    return filtered_data

def get_content_providers(source_id=None):
    """
    Pick active (enabled) content Sources (aka Providers).

    LTI Providers which expose courses content blocks to use in adaptive collections.
    :param source_id: LtiConsumer ID or None
    :return: content_sources queryset
    """
    q_kw = {
        'is_active': True
    }
    if source_id:
        q_kw['id'] = source_id
    content_source = LtiConsumer.objects.filter(**q_kw)
    log.debug('Picked content Source(s){}: {}'.format(
        " with content_source_id={}".format(source_id),
        ", ".join(content_source.values_list('name', flat=True))))
    return content_source
<commit_msg>Fix response's page size for the course list request (#117)<commit_after>from datetime import datetime
import logging

from django.core.cache import cache
from django.core.exceptions import ObjectDoesNotExist
from django.utils.translation import ugettext as _
from edx_rest_api_client.client import EdxRestApiClient
from requests import RequestException
from slumber.exceptions import HttpClientError, HttpNotFoundError

from bridge_lti.models import LtiConsumer

log = logging.getLogger(__name__)

class OpenEdxApiClient(EdxRestApiClient):
    """API client to interact with OpenEdx Course API."""

    API_URLS = {
        "get_token": "/oauth2/access_token",
        "base_url": "/api/courses/v1/",
    }

    def __init__(self, content_source, url=None, jwt=None, **kwargs):
        log.debug("Creating new OpenEdx API client...")
        self.content_source = content_source

        if not url:
            url = '{}{}'.format(content_source.host_url, self.API_URLS['base_url'])
        if not jwt:
            api_client_id = self.content_source.o_auth_client.client_id
            token_cache_key = "api:{}:token".format(api_client_id)

            access_token = cache.get(token_cache_key)
            if not access_token:
                access_token, expires_at = self.get_oauth_access_token()
                ttl = expires_at - datetime.now()
                cache.set(token_cache_key, access_token, ttl.seconds)

        super(OpenEdxApiClient, self).__init__(url, jwt=access_token, **kwargs)

    def get_oauth_access_token(self):
        """
        Request OpenEdx API OAuth2 token.

        Token type: JWT (reference: https://jwt.io/).
        :return: access_token, expires_at
        """
        url = "{host_url}{token_url}".format(
            host_url=self.content_source.host_url,
            token_url=self.API_URLS['get_token']
        )
        log.debug("Requesting oauth token: (url={})".format(url))
        try:
            oauth_client = self.content_source.o_auth_client
            access_token, expires_at = super(OpenEdxApiClient, self).get_oauth_access_token(
                url=url,
                client_id=oauth_client.client_id,
                client_secret=oauth_client.client_secret,
                token_type='jwt',
            )
        except ObjectDoesNotExist:
            raise HttpClientError(
                "OAuth token request failure. Please, configure OAuth client in order to be able make API requests."
            )
        except ValueError:
            log.exception(
                "You may want to check your OAuth registration on LTI Provider."
                "LTI Provider may be disabled (to enable: LMS config > FEATURES > ENABLE_OAUTH2_PROVIDER: true"
            )
            raise HttpClientError(
                "OAuth token request failure. You may want to check your OAuth registration on LTI Provider or"
                "enable OAuth Provider."
            )
        except RequestException:
            log.exception('OAuth2 token request to the OpenEdx LTI Provider failed.')
            raise HttpClientError(
                "OAuth token request failure. You may want to check your LTI Provider's HOST_URL(https)."
            )
        return access_token, expires_at

    def get_course_blocks(self, course_id, all_blocks=True, depth='all', type_filter=None):
        """
        Provide API GET request to OpenEdx Course Blocks endpoint.

        Optional block type filtering available.
        Endpoint: /api/courses/v1/blocks/?course_id
        API query parameters:
        - block_types_filter=['sequential', 'vertical', 'html', 'problem', 'video', 'discussion']
        - block_counts=['video', 'problem'...]
        -
        see: http://edx.readthedocs.io/projects/edx-platform-api/en/latest/courses/blocks.html#query-parameters

        :param filter_function: partially applied function `apply_data_filter` with filter parameters.
        :return: (list)
        """
        resource = self.blocks.get(
            course_id=course_id,
            all_blocks=all_blocks,
            depth=depth,
            requested_fields='lti_url',
            return_type='list',
            block_types_filter=type_filter or []
        )
        return resource

    def get_provider_courses(self, username=None, org=None, mobile=None):
        """
        Provide API GET request to OpenEdx Courses Resource endpoint.

        Gets a List of Courses.
        Endpoint: /api/courses/v1/courses/
        see: http://edx.readthedocs.io/projects/edx-platform-api/en/latest/courses/courses.html#query-parameters
        :return: (dict)
        """
        resource = self.courses.get(
            username=username,
            org=org,
            mobile=mobile,
            page_size=1000,
        )
        return resource.get('results')

def get_active_content_sources(source_id=None, not_allow_empty_source_id=True):
    """
    Check that passed source_id parameter is valid.

    If there's only one active source provider - source_id parameter is not required, it will get first active.
    :param source_id: LtiConsumer object id
    :param not_allow_empty_source_id: if True - it will not allow empty source_id, if False - source_id could be None
    :return: queryset of content_sources
    :raise HttpClientError: if provided parameters are not valid
    """
    if not_allow_empty_source_id and not source_id:
        # if source_id is not provided and more than one active content_provider
        raise HttpClientError(_("Parameter source_id is mandatory if there's more than one active content source."))

    content_sources = get_content_providers(source_id=source_id)

    if not content_sources:
        # if no active content sources
        raise HttpClientError(_("No active Content Provider"))

    return content_sources

def get_available_blocks(source_id, course_id=''):
    """
    Content Source API requester.

    Fetches all source blocks from the course with the given ID.
    Blocks data is filtered by `apply_data_filter`.
    :param course_id: Course id
    :param source_id: LtiConsumer id
    :return: (list) blocks data
    """
    content_source = get_active_content_sources(source_id).first()
    all_blocks = []

    # Get API client instance:
    api = OpenEdxApiClient(content_source=content_source)
    try:
        # filter function will be applied to api response
        all_blocks.extend(
            apply_data_filter(
                api.get_course_blocks(
                    course_id, type_filter=['html', 'problem', 'video']
                ),
                filters=['id', 'block_id', 'display_name', 'lti_url', 'type', 'content_source_id'],
                context_id=course_id,
                content_source_id=content_source.id
            )
        )
    except HttpNotFoundError:
        raise HttpClientError(_("Requested course not found. Check `course_id` url encoding."))
    except HttpClientError as exc:
        raise HttpClientError(_("Not valid query: {}").format(exc.message))

    return all_blocks

def get_available_courses(source_id=None):
    """
    Fetch all available courses from all sources of from source with provided ID.

    :param source_id: content provider's ID
    :return: (list) course_ids
    """
    content_sources = get_active_content_sources(source_id, not_allow_empty_source_id=False)

    all_courses = []

    for content_source in content_sources:
        # Get API client instance:
        api = OpenEdxApiClient(content_source=content_source)
        try:
            all_courses.extend(
                apply_data_filter(
                    api.get_provider_courses(),
                    filters=['id', 'course_id', 'name', 'org', 'content_source_id'],
                    content_source_id=content_source.id
                )
            )
        except HttpClientError as exc:
            raise HttpClientError(_("Not valid query: {}").format(exc.message))

    return all_courses

def add_to_dict(data, **kwargs):
    """Add key and value to dict only if this pair not exist yet in data dict."""
    for k, v in kwargs.items():
        if k not in data:
            data[k] = v
    return data

def apply_data_filter(data, filters=None, **kwargs):
    """
    Filter for `blocks` OpenEdx Course API response.

    Picks data which is listed in `filters` only.
    :param data: (list)
    :param filters: list of desired resource keys
    :param kwargs: params to append to every object if it's not present in the object yet
    :return: list of resources which keys were filtered
    """
    if filters is None:
        return data

    filtered_data = []
    for resource in data:
        filtered_resource = {k: v for k, v in resource.items() if k in filters}
        filtered_data.append(add_to_dict(filtered_resource, **kwargs))
    return filtered_data

def get_content_providers(source_id=None):
    """
    Pick active (enabled) content Sources (aka Providers).

    LTI Providers which expose courses content blocks to use in adaptive collections.
    :param source_id: LtiConsumer ID or None
    :return: content_sources queryset
    """
    q_kw = {
        'is_active': True
    }
    if source_id:
        q_kw['id'] = source_id
    content_source = LtiConsumer.objects.filter(**q_kw)
    log.debug('Picked content Source(s){}: {}'.format(
        " with content_source_id={}".format(source_id),
        ", ".join(content_source.values_list('name', flat=True))))
    return content_source
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import collections
import glob
import multiprocessing
import os
import shutil
import sys

import bb_utils
import bb_annotations

sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
import provision_devices
from pylib import android_commands
from pylib import constants
from pylib.gtest import gtest_config

sys.path.append(os.path.join(
    constants.DIR_SOURCE_ROOT, 'third_party', 'android_testrunner'))
import errors

CHROME_SRC = constants.DIR_SOURCE_ROOT
LOGCAT_DIR = os.path.join(CHROME_SRC, 'out', 'logcat')

# Describes an instrumation test suite:
#   test: Name of test we're running.
#   apk: apk to be installed.
#   apk_package: package for the apk to be installed.
#   test_apk: apk to run tests on.
#   test_data: data folder in format destination:source.
#   host_driven_root: The host-driven test root directory.
#   annotation: Annotation of the tests to include.
#   exclude_annotation: The annotation of the tests to exclude.
I_TEST = collections.namedtuple('InstrumentationTest', [
    'name', 'apk', 'apk_package', 'test_apk', 'test_data', 'host_driven_root',
    'annotation', 'exclude_annotation', 'extra_flags'])

def I(name, apk, apk_package, test_apk, test_data, host_driven_root=None,
      annotation=None, exclude_annotation=None, extra_flags=None):
  return I_TEST(name, apk, apk_package, test_apk, test_data, host_driven_root,
                annotation, exclude_annotation, extra_flags)

INSTRUMENTATION_TESTS = dict((suite.name, suite) for suite in [
    I('ContentShell',
      'ContentShell.apk',
      'org.chromium.content_shell_apk',
      'ContentShellTest',
      'content:content/test/data/android/device_files'),
    I('ChromiumTestShell',
      'ChromiumTestShell.apk',
      'org.chromium.chrome.testshell',
      'ChromiumTestShellTest',
      'chrome:chrome/test/data/android/device_files',
      constants.CHROMIUM_TEST_SHELL_HOST_DRIVEN_DIR),
    I('AndroidWebView',
      'AndroidWebView.apk',
      'org.chromium.android_webview.shell',
      'AndroidWebViewTest',
      'webview:android_webview/test/data/device_files'),
    ])

VALID_TESTS = set(['chromedriver', 'ui', 'unit', 'webkit', 'webkit_layout',
                   'webrtc'])

RunCmd = bb_utils.RunCmd

# multiprocessing map_async requires a top-level function for pickle library.
def RebootDeviceSafe(device):
  """Reboot a device, wait for it to start, and squelch timeout exceptions."""
  try:
    android_commands.AndroidCommands(device).Reboot(True)
  except errors.DeviceUnresponsiveError as e:
    return e

def RebootDevices():
  """Reboot all attached and online devices."""
  # Early return here to avoid presubmit dependence on adb,
  # which might not exist in this checkout.
  if bb_utils.TESTING:
    return
  devices = android_commands.GetAttachedDevices(emulator=False)
  print 'Rebooting: %s' % devices
  if devices:
    pool = multiprocessing.Pool(len(devices))
    results = pool.map_async(RebootDeviceSafe, devices).get(99999)

    for device, result in zip(devices, results):
      if result:
        print '%s failed to startup.' % device

    if any(results):
      bb_annotations.PrintWarning()
    else:
      print 'Reboots complete.'

def RunTestSuites(options, suites):
  """Manages an invocation of test_runner.py for gtests.

  Args:
    options: options object.
    suites: List of suite names to run.
  """
  args = ['--verbose']
  if options.target == 'Release':
    args.append('--release')
  if options.asan:
    args.append('--tool=asan')
  for suite in suites:
    bb_annotations.PrintNamedStep(suite)
    cmd = ['build/android/test_runner.py', 'gtest', '-s', suite] + args
    if suite == 'content_browsertests':
      cmd.append('--num_retries=1')
    RunCmd(cmd)

def RunChromeDriverTests(_):
  """Run all the steps for running chromedriver tests."""
  bb_annotations.PrintNamedStep('chromedriver_annotation')
  RunCmd(['chrome/test/chromedriver/run_buildbot_steps.py',
          '--android-package=%s' % constants.CHROMIUM_TEST_SHELL_PACKAGE])

def InstallApk(options, test, print_step=False):
  """Install an apk to all phones.

  Args:
    options: options object
    test: An I_TEST namedtuple
    print_step: Print a buildbot step
  """
  if print_step:
    bb_annotations.PrintNamedStep('install_%s' % test.name.lower())
  args = ['--apk', test.apk, '--apk_package', test.apk_package]
  if options.target == 'Release':
    args.append('--release')

  RunCmd(['build/android/adb_install_apk.py'] + args, halt_on_failure=True)

def RunInstrumentationSuite(options, test, flunk_on_failure=True,
                            python_only=False):
  """Manages an invocation of test_runner.py for instrumentation tests.

  Args:
    options: options object
    test: An I_TEST namedtuple
    flunk_on_failure: Flunk the step if tests fail.
    Python: Run only host driven Python tests.
  """
  bb_annotations.PrintNamedStep('%s_instrumentation_tests' % test.name.lower())

  InstallApk(options, test)
  args = ['--test-apk', test.test_apk, '--test_data', test.test_data,
          '--verbose']
  if options.target == 'Release':
    args.append('--release')
  if options.asan:
    args.append('--tool=asan')
  if options.flakiness_server:
    args.append('--flakiness-dashboard-server=%s' %
                options.flakiness_server)
  if test.host_driven_root:
    args.append('--host-driven-root=%s' % test.host_driven_root)
  if test.annotation:
    args.extend(['-A', test.annotation])
  if test.exclude_annotation:
    args.extend(['-E', test.exclude_annotation])
  if test.extra_flags:
    args.extend(test.extra_flags)
  if python_only:
    args.append('-p')

  RunCmd(['build/android/test_runner.py', 'instrumentation'] + args,
         flunk_on_failure=flunk_on_failure)

def RunWebkitLint(target):
  """Lint WebKit's TestExpectation files."""
  bb_annotations.PrintNamedStep('webkit_lint')
  RunCmd(['webkit/tools/layout_tests/run_webkit_tests.py',
          '--lint-test-files',
          '--chromium',
          '--target', target])

def RunWebkitLayoutTests(options):
  """Run layout tests on an actual device."""
  bb_annotations.PrintNamedStep('webkit_tests')
  cmd_args = [
        '--no-show-results',
        '--no-new-test-results',
        '--full-results-html',
        '--clobber-old-results',
        '--exit-after-n-failures', '5000',
        '--exit-after-n-crashes-or-timeouts', '100',
        '--debug-rwt-logging',
        '--results-directory', '..layout-test-results',
        '--target', options.target,
        '--builder-name', options.build_properties.get('buildername', ''),
        '--build-number', str(options.build_properties.get('buildnumber', '')),
        '--master-name', options.build_properties.get('mastername', ''),
        '--build-name', options.build_properties.get('buildername', ''),
        '--platform=android']

  for flag in 'test_results_server', 'driver_name', 'additional_drt_flag':
    if flag in options.factory_properties:
      cmd_args.extend(['--%s' % flag.replace('_', '-'),
                       options.factory_properties.get(flag)])

  for f in options.factory_properties.get('additional_expectations', []):
    cmd_args.extend(
        ['--additional-expectations=%s' % os.path.join(CHROME_SRC, *f)])

  # TODO(dpranke): Remove this block after
  # https://codereview.chromium.org/12927002/ lands.
  for f in options.factory_properties.get('additional_expectations_files', []):
    cmd_args.extend(
        ['--additional-expectations=%s' % os.path.join(CHROME_SRC, *f)])

  RunCmd(['webkit/tools/layout_tests/run_webkit_tests.py'] + cmd_args,
         flunk_on_failure=False)

def SpawnLogcatMonitor():
  shutil.rmtree(LOGCAT_DIR, ignore_errors=True)
  bb_utils.SpawnCmd([
      os.path.join(CHROME_SRC, 'build', 'android', 'adb_logcat_monitor.py'),
      LOGCAT_DIR])

  # Wait for logcat_monitor to pull existing logcat
  RunCmd(['sleep', '5'])

def ProvisionDevices(options):
  bb_annotations.PrintNamedStep('provision_devices')

  if not bb_utils.TESTING:
    # Restart adb to work around bugs, sleep to wait for usb discovery.
    adb = android_commands.AndroidCommands()
    adb.RestartAdbServer()
    RunCmd(['sleep', '1'])

  if options.reboot:
    RebootDevices()
  provision_cmd = ['build/android/provision_devices.py', '-t', options.target]
  if options.auto_reconnect:
    provision_cmd.append('--auto-reconnect')
  RunCmd(provision_cmd)

def DeviceStatusCheck(_):
  bb_annotations.PrintNamedStep('device_status_check')
  RunCmd(['build/android/buildbot/bb_device_status_check.py'],
         halt_on_failure=True)

def GetDeviceSetupStepCmds():
  return [
    ('provision_devices', ProvisionDevices),
    ('device_status_check', DeviceStatusCheck)
  ]

def RunUnitTests(options):
  RunTestSuites(options, gtest_config.STABLE_TEST_SUITES)

def RunInstrumentationTests(options):
  for test in INSTRUMENTATION_TESTS.itervalues():
    RunInstrumentationSuite(options, test)

def RunWebkitTests(options):
  RunTestSuites(options, ['webkit_unit_tests'])
  RunWebkitLint(options.target)

def RunWebRTCTests(options):
  RunTestSuites(options, gtest_config.WEBRTC_TEST_SUITES)

def GetTestStepCmds():
  return [
      ('chromedriver', RunChromeDriverTests),
      ('unit', RunUnitTests),
      ('ui', RunInstrumentationTests),
      ('webkit', RunWebkitTests),
      ('webkit_layout', RunWebkitLayoutTests),
      ('webrtc', RunWebRTCTests),
  ]

def LogcatDump(options):
  # Print logcat, kill logcat monitor
  bb_annotations.PrintNamedStep('logcat_dump')
  logcat_file = os.path.join(CHROME_SRC, 'out', options.target, 'full_log')
  with open(logcat_file, 'w') as f:
    RunCmd([
        os.path.join(CHROME_SRC, 'build', 'android', 'adb_logcat_printer.py'),
        LOGCAT_DIR], stdout=f)
  RunCmd(['cat', logcat_file])

def GenerateTestReport(options):
  bb_annotations.PrintNamedStep('test_report')
  for report in glob.glob(
      os.path.join(CHROME_SRC, 'out', options.target, 'test_logs', '*.log')):
    RunCmd(['cat', report])
    os.remove(report)

def MainTestWrapper(options):
  try:
    # Spawn logcat monitor
    SpawnLogcatMonitor()

    # Run all device setup steps
    for _, cmd in GetDeviceSetupStepCmds():
      cmd(options)

    if options.install:
      test_obj = INSTRUMENTATION_TESTS[options.install]
      InstallApk(options, test_obj, print_step=True)

    if options.test_filter:
      bb_utils.RunSteps(options.test_filter, GetTestStepCmds(), options)

    if options.experimental:
      RunTestSuites(options, gtest_config.EXPERIMENTAL_TEST_SUITES)

  finally:
    # Run all post test steps
    LogcatDump(options)
    GenerateTestReport(options)
    # KillHostHeartbeat() has logic to check if heartbeat process is running,
    # and kills only if it finds the process is running on the host.
    provision_devices.KillHostHeartbeat()

def GetDeviceStepsOptParser():
  parser = bb_utils.GetParser()
  parser.add_option('--experimental', action='store_true',
                    help='Run experiemental tests')
  parser.add_option('-f', '--test-filter', metavar='<filter>', default=[],
                    action='append',
                    help=('Run a test suite. Test suites: "%s"' %
                          '", "'.join(VALID_TESTS)))
  parser.add_option('--asan', action='store_true', help='Run tests with asan.')
  parser.add_option('--install', metavar='<apk name>',
                    help='Install an apk by name')
  parser.add_option('--reboot', action='store_true',
                    help='Reboot devices before running tests')
  parser.add_option(
      '--flakiness-server',
      help='The flakiness dashboard server to which the results should be '
           'uploaded.')
  parser.add_option(
      '--auto-reconnect', action='store_true',
      help='Push script to device which restarts adbd on disconnections.')
  parser.add_option(
      '--logcat-dump-output',
      help='The logcat dump output will be "tee"-ed into this file')

  return parser

def main(argv):
  parser = GetDeviceStepsOptParser()
  options, args = parser.parse_args(argv[1:])

  if args:
    return sys.exit('Unused args %s' % args)

  unknown_tests = set(options.test_filter) - VALID_TESTS
  if unknown_tests:
    return sys.exit('Unknown tests %s' % list(unknown_tests))

  setattr(options, 'target', options.factory_properties.get('target', 'Debug'))

  MainTestWrapper(options)

if __name__ == '__main__':
  sys.exit(main(sys.argv))
<commit_msg>Make failing webkit_tests fail android build.<commit_after>#!/usr/bin/env python
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import collections
import glob
import multiprocessing
import os
import shutil
import sys

import bb_utils
import bb_annotations

sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
import provision_devices
from pylib import android_commands
from pylib import constants
from pylib.gtest import gtest_config

sys.path.append(os.path.join(
    constants.DIR_SOURCE_ROOT, 'third_party', 'android_testrunner'))
import errors

CHROME_SRC = constants.DIR_SOURCE_ROOT
LOGCAT_DIR = os.path.join(CHROME_SRC, 'out', 'logcat')

# Describes an instrumation test suite:
#   test: Name of test we're running.
#   apk: apk to be installed.
#   apk_package: package for the apk to be installed.
#   test_apk: apk to run tests on.
#   test_data: data folder in format destination:source.
#   host_driven_root: The host-driven test root directory.
#   annotation: Annotation of the tests to include.
#   exclude_annotation: The annotation of the tests to exclude.
I_TEST = collections.namedtuple('InstrumentationTest', [
    'name', 'apk', 'apk_package', 'test_apk', 'test_data', 'host_driven_root',
    'annotation', 'exclude_annotation', 'extra_flags'])

def I(name, apk, apk_package, test_apk, test_data, host_driven_root=None,
      annotation=None, exclude_annotation=None, extra_flags=None):
  return I_TEST(name, apk, apk_package, test_apk, test_data, host_driven_root,
                annotation, exclude_annotation, extra_flags)

INSTRUMENTATION_TESTS = dict((suite.name, suite) for suite in [
    I('ContentShell',
      'ContentShell.apk',
      'org.chromium.content_shell_apk',
      'ContentShellTest',
      'content:content/test/data/android/device_files'),
    I('ChromiumTestShell',
      'ChromiumTestShell.apk',
      'org.chromium.chrome.testshell',
      'ChromiumTestShellTest',
      'chrome:chrome/test/data/android/device_files',
      constants.CHROMIUM_TEST_SHELL_HOST_DRIVEN_DIR),
    I('AndroidWebView',
      'AndroidWebView.apk',
      'org.chromium.android_webview.shell',
      'AndroidWebViewTest',
      'webview:android_webview/test/data/device_files'),
    ])

VALID_TESTS = set(['chromedriver', 'ui', 'unit', 'webkit', 'webkit_layout',
                   'webrtc'])

RunCmd = bb_utils.RunCmd

# multiprocessing map_async requires a top-level function for pickle library.
def RebootDeviceSafe(device):
  """Reboot a device, wait for it to start, and squelch timeout exceptions."""
  try:
    android_commands.AndroidCommands(device).Reboot(True)
  except errors.DeviceUnresponsiveError as e:
    return e

def RebootDevices():
  """Reboot all attached and online devices."""
  # Early return here to avoid presubmit dependence on adb,
  # which might not exist in this checkout.
  if bb_utils.TESTING:
    return
  devices = android_commands.GetAttachedDevices(emulator=False)
  print 'Rebooting: %s' % devices
  if devices:
    pool = multiprocessing.Pool(len(devices))
    results = pool.map_async(RebootDeviceSafe, devices).get(99999)

    for device, result in zip(devices, results):
      if result:
        print '%s failed to startup.' % device

    if any(results):
      bb_annotations.PrintWarning()
    else:
      print 'Reboots complete.'

def RunTestSuites(options, suites):
  """Manages an invocation of test_runner.py for gtests.

  Args:
    options: options object.
    suites: List of suite names to run.
  """
  args = ['--verbose']
  if options.target == 'Release':
    args.append('--release')
  if options.asan:
    args.append('--tool=asan')
  for suite in suites:
    bb_annotations.PrintNamedStep(suite)
    cmd = ['build/android/test_runner.py', 'gtest', '-s', suite] + args
    if suite == 'content_browsertests':
      cmd.append('--num_retries=1')
    RunCmd(cmd)

def RunChromeDriverTests(_):
  """Run all the steps for running chromedriver tests."""
  bb_annotations.PrintNamedStep('chromedriver_annotation')
  RunCmd(['chrome/test/chromedriver/run_buildbot_steps.py',
          '--android-package=%s' % constants.CHROMIUM_TEST_SHELL_PACKAGE])

def InstallApk(options, test, print_step=False):
  """Install an apk to all phones.

  Args:
    options: options object
    test: An I_TEST namedtuple
    print_step: Print a buildbot step
  """
  if print_step:
    bb_annotations.PrintNamedStep('install_%s' % test.name.lower())
  args = ['--apk', test.apk, '--apk_package', test.apk_package]
  if options.target == 'Release':
    args.append('--release')

  RunCmd(['build/android/adb_install_apk.py'] + args, halt_on_failure=True)

def RunInstrumentationSuite(options, test, flunk_on_failure=True,
                            python_only=False):
  """Manages an invocation of test_runner.py for instrumentation tests.

  Args:
    options: options object
    test: An I_TEST namedtuple
    flunk_on_failure: Flunk the step if tests fail.
    Python: Run only host driven Python tests.
  """
  bb_annotations.PrintNamedStep('%s_instrumentation_tests' % test.name.lower())

  InstallApk(options, test)
  args = ['--test-apk', test.test_apk, '--test_data', test.test_data,
          '--verbose']
  if options.target == 'Release':
    args.append('--release')
  if options.asan:
    args.append('--tool=asan')
  if options.flakiness_server:
    args.append('--flakiness-dashboard-server=%s' %
                options.flakiness_server)
  if test.host_driven_root:
    args.append('--host-driven-root=%s' % test.host_driven_root)
  if test.annotation:
    args.extend(['-A', test.annotation])
  if test.exclude_annotation:
    args.extend(['-E', test.exclude_annotation])
  if test.extra_flags:
    args.extend(test.extra_flags)
  if python_only:
    args.append('-p')

  RunCmd(['build/android/test_runner.py', 'instrumentation'] + args,
         flunk_on_failure=flunk_on_failure)

def RunWebkitLint(target):
  """Lint WebKit's TestExpectation files."""
  bb_annotations.PrintNamedStep('webkit_lint')
  RunCmd(['webkit/tools/layout_tests/run_webkit_tests.py',
          '--lint-test-files',
          '--chromium',
          '--target', target])

def RunWebkitLayoutTests(options):
  """Run layout tests on an actual device."""
  bb_annotations.PrintNamedStep('webkit_tests')
  cmd_args = [
        '--no-show-results',
        '--no-new-test-results',
        '--full-results-html',
        '--clobber-old-results',
        '--exit-after-n-failures', '5000',
        '--exit-after-n-crashes-or-timeouts', '100',
        '--debug-rwt-logging',
        '--results-directory', '..layout-test-results',
        '--target', options.target,
        '--builder-name', options.build_properties.get('buildername', ''),
        '--build-number', str(options.build_properties.get('buildnumber', '')),
        '--master-name', options.build_properties.get('mastername', ''),
        '--build-name', options.build_properties.get('buildername', ''),
        '--platform=android']

  for flag in 'test_results_server', 'driver_name', 'additional_drt_flag':
    if flag in options.factory_properties:
      cmd_args.extend(['--%s' % flag.replace('_', '-'),
                       options.factory_properties.get(flag)])

  for f in options.factory_properties.get('additional_expectations', []):
    cmd_args.extend(
        ['--additional-expectations=%s' % os.path.join(CHROME_SRC, *f)])

  # TODO(dpranke): Remove this block after
  # https://codereview.chromium.org/12927002/ lands.
  for f in options.factory_properties.get('additional_expectations_files', []):
    cmd_args.extend(
        ['--additional-expectations=%s' % os.path.join(CHROME_SRC, *f)])

  RunCmd(['webkit/tools/layout_tests/run_webkit_tests.py'] + cmd_args)

def SpawnLogcatMonitor():
  shutil.rmtree(LOGCAT_DIR, ignore_errors=True)
  bb_utils.SpawnCmd([
      os.path.join(CHROME_SRC, 'build', 'android', 'adb_logcat_monitor.py'),
      LOGCAT_DIR])

  # Wait for logcat_monitor to pull existing logcat
  RunCmd(['sleep', '5'])

def ProvisionDevices(options):
  bb_annotations.PrintNamedStep('provision_devices')

  if not bb_utils.TESTING:
    # Restart adb to work around bugs, sleep to wait for usb discovery.
    adb = android_commands.AndroidCommands()
    adb.RestartAdbServer()
    RunCmd(['sleep', '1'])

  if options.reboot:
    RebootDevices()
  provision_cmd = ['build/android/provision_devices.py', '-t', options.target]
  if options.auto_reconnect:
    provision_cmd.append('--auto-reconnect')
  RunCmd(provision_cmd)

def DeviceStatusCheck(_):
  bb_annotations.PrintNamedStep('device_status_check')
  RunCmd(['build/android/buildbot/bb_device_status_check.py'],
         halt_on_failure=True)

def GetDeviceSetupStepCmds():
  return [
    ('provision_devices', ProvisionDevices),
    ('device_status_check', DeviceStatusCheck)
  ]

def RunUnitTests(options):
  RunTestSuites(options, gtest_config.STABLE_TEST_SUITES)

def RunInstrumentationTests(options):
  for test in INSTRUMENTATION_TESTS.itervalues():
    RunInstrumentationSuite(options, test)

def RunWebkitTests(options):
  RunTestSuites(options, ['webkit_unit_tests'])
  RunWebkitLint(options.target)

def RunWebRTCTests(options):
  RunTestSuites(options, gtest_config.WEBRTC_TEST_SUITES)

def GetTestStepCmds():
  return [
      ('chromedriver', RunChromeDriverTests),
      ('unit', RunUnitTests),
      ('ui', RunInstrumentationTests),
      ('webkit', RunWebkitTests),
      ('webkit_layout', RunWebkitLayoutTests),
      ('webrtc', RunWebRTCTests),
  ]

def LogcatDump(options):
  # Print logcat, kill logcat monitor
  bb_annotations.PrintNamedStep('logcat_dump')
  logcat_file = os.path.join(CHROME_SRC, 'out', options.target, 'full_log')
  with open(logcat_file, 'w') as f:
    RunCmd([
        os.path.join(CHROME_SRC, 'build', 'android', 'adb_logcat_printer.py'),
        LOGCAT_DIR], stdout=f)
  RunCmd(['cat', logcat_file])

def GenerateTestReport(options):
  bb_annotations.PrintNamedStep('test_report')
  for report in glob.glob(
      os.path.join(CHROME_SRC, 'out', options.target, 'test_logs', '*.log')):
    RunCmd(['cat', report])
    os.remove(report)

def MainTestWrapper(options):
  try:
    # Spawn logcat monitor
    SpawnLogcatMonitor()

    # Run all device setup steps
    for _, cmd in GetDeviceSetupStepCmds():
      cmd(options)

    if options.install:
      test_obj = INSTRUMENTATION_TESTS[options.install]
      InstallApk(options, test_obj, print_step=True)

    if options.test_filter:
      bb_utils.RunSteps(options.test_filter, GetTestStepCmds(), options)

    if options.experimental:
      RunTestSuites(options, gtest_config.EXPERIMENTAL_TEST_SUITES)

  finally:
    # Run all post test steps
    LogcatDump(options)
    GenerateTestReport(options)
    # KillHostHeartbeat() has logic to check if heartbeat process is running,
    # and kills only if it finds the process is running on the host.
    provision_devices.KillHostHeartbeat()

def GetDeviceStepsOptParser():
  parser = bb_utils.GetParser()
  parser.add_option('--experimental', action='store_true',
                    help='Run experiemental tests')
  parser.add_option('-f', '--test-filter', metavar='<filter>', default=[],
                    action='append',
                    help=('Run a test suite. Test suites: "%s"' %
                          '", "'.join(VALID_TESTS)))
  parser.add_option('--asan', action='store_true', help='Run tests with asan.')
  parser.add_option('--install', metavar='<apk name>',
                    help='Install an apk by name')
  parser.add_option('--reboot', action='store_true',
                    help='Reboot devices before running tests')
  parser.add_option(
      '--flakiness-server',
      help='The flakiness dashboard server to which the results should be '
           'uploaded.')
  parser.add_option(
      '--auto-reconnect', action='store_true',
      help='Push script to device which restarts adbd on disconnections.')
  parser.add_option(
      '--logcat-dump-output',
      help='The logcat dump output will be "tee"-ed into this file')

  return parser

def main(argv):
  parser = GetDeviceStepsOptParser()
  options, args = parser.parse_args(argv[1:])

  if args:
    return sys.exit('Unused args %s' % args)

  unknown_tests = set(options.test_filter) - VALID_TESTS
  if unknown_tests:
    return sys.exit('Unknown tests %s' % list(unknown_tests))

  setattr(options, 'target', options.factory_properties.get('target', 'Debug'))

  MainTestWrapper(options)

if __name__ == '__main__':
  sys.exit(main(sys.argv))
<|endoftext|>


<commit_before># Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

from __future__ import absolute_import, division, print_function, unicode_literals

import errno
import glob
import ntpath
import os
import subprocess
import sys
import tempfile

from .copytree import containing_repo_type
from .envfuncs import Env, add_path_entry
from .fetcher import get_fbsource_repo_data
from .manifest import ContextGenerator
from .platform import HostType, is_windows

try:
    import typing  # noqa: F401
except ImportError:
    pass

def detect_project(path):
    repo_type, repo_root = containing_repo_type(path)
    if repo_type is None:
        return None, None

    # Look for a .projectid file.  If it exists, read the project name from it.
    project_id_path = os.path.join(repo_root, ".projectid")
    try:
        with open(project_id_path, "r") as f:
            project_name = f.read().strip()
            return repo_root, project_name
    except EnvironmentError as ex:
        if ex.errno != errno.ENOENT:
            raise

    return repo_root, None

class BuildOptions(object):
    def __init__(
        self,
        fbcode_builder_dir,
        scratch_dir,
        host_type,
        install_dir=None,
        num_jobs=0,
        use_shipit=False,
        vcvars_path=None,
        allow_system_packages=False,
    ):
        """ fbcode_builder_dir - the path to either the in-fbsource fbcode_builder dir,
                                 or for shipit-transformed repos, the build dir that
                                 has been mapped into that dir.
            scratch_dir - a place where we can store repos and build bits.
                          This path should be stable across runs and ideally
                          should not be in the repo of the project being built,
                          but that is ultimately where we generally fall back
                          for builds outside of FB
            install_dir - where the project will ultimately be installed
            num_jobs - the level of concurrency to use while building
            use_shipit - use real shipit instead of the simple shipit transformer
            vcvars_path - Path to external VS toolchain's vsvarsall.bat
        """
        if not num_jobs:
            import multiprocessing

            num_jobs = multiprocessing.cpu_count()
            if is_windows():
                # On Windows the cpu count tends to be the HT count.
                # Running with that level of concurrency tends to
                # swamp the system and make hard to perform other
                # light work.  Let's halve the number of cores here
                # to win that back. The user can still specify a
                # larger number if desired.
                num_jobs = int(num_jobs / 2)

        if not install_dir:
            install_dir = os.path.join(scratch_dir, "installed")

        self.project_hashes = None
        for p in ["../deps/github_hashes", "../project_hashes"]:
            hashes = os.path.join(fbcode_builder_dir, p)
            if os.path.exists(hashes):
                self.project_hashes = hashes
                break

        # Detect what repository and project we are being run from.
        self.repo_root, self.repo_project = detect_project(os.getcwd())

        # If we are running from an fbsource repository, set self.fbsource_dir
        # to allow the ShipIt-based fetchers to use it.
        if self.repo_project == "fbsource":
            self.fbsource_dir = self.repo_root
        else:
            self.fbsource_dir = None

        self.num_jobs = num_jobs
        self.scratch_dir = scratch_dir
        self.install_dir = install_dir
        self.fbcode_builder_dir = fbcode_builder_dir
        self.host_type = host_type
        self.use_shipit = use_shipit
        self.allow_system_packages = allow_system_packages
        if vcvars_path is None and is_windows():

            # On Windows, the compiler is not available in the PATH by
            # default so we need to run the vcvarsall script to populate the
            # environment. We use a glob to find some version of this script
            # as deployed with Visual Studio 2017.  This logic can also
            # locate Visual Studio 2019 but note that at the time of writing
            # the version of boost in our manifest cannot be built with
            # VS 2019, so we're effectively tied to VS 2017 until we upgrade
            # the boost dependency.
            vcvarsall = []
            for year in ["2017", "2019"]:
                vcvarsall += glob.glob(
                    os.path.join(
                        os.environ["ProgramFiles(x86)"],
                        "Microsoft Visual Studio",
                        year,
                        "*",
                        "VC",
                        "Auxiliary",
                        "Build",
                        "vcvarsall.bat",
                    )
                )
            vcvars_path = vcvarsall[0]

        self.vcvars_path = vcvars_path

    @property
    def manifests_dir(self):
        return os.path.join(self.fbcode_builder_dir, "manifests")

    def is_darwin(self):
        return self.host_type.is_darwin()

    def is_windows(self):
        return self.host_type.is_windows()

    def get_vcvars_path(self):
        return self.vcvars_path

    def is_linux(self):
        return self.host_type.is_linux()

    def get_context_generator(self, host_tuple=None, facebook_internal=None):
        """ Create a manifest ContextGenerator for the specified target platform. """
        if host_tuple is None:
            host_type = self.host_type
        elif isinstance(host_tuple, HostType):
            host_type = host_tuple
        else:
            host_type = HostType.from_tuple_string(host_tuple)

        # facebook_internal is an Optional[bool]
        # If it is None, default to assuming this is a Facebook-internal build if
        # we are running in an fbsource repository.
        if facebook_internal is None:
            facebook_internal = self.fbsource_dir is not None

        return ContextGenerator(
            {
                "os": host_type.ostype,
                "distro": host_type.distro,
                "distro_vers": host_type.distrovers,
                "fb": "on" if facebook_internal else "off",
                "test": "off",
            }
        )

    def compute_env_for_install_dirs(self, install_dirs, env=None):
        if env is not None:
            env = env.copy()
        else:
            env = Env()

        if self.fbsource_dir:
            env["YARN_YARN_OFFLINE_MIRROR"] = os.path.join(
                self.fbsource_dir, "xplat/third-party/yarn/offline-mirror"
            )
            yarn_exe = "yarn.bat" if self.is_windows() else "yarn"
            env["YARN_PATH"] = os.path.join(
                self.fbsource_dir, "xplat/third-party/yarn/", yarn_exe
            )
            node_exe = "node-win-x64.exe" if self.is_windows() else "node"
            env["NODE_BIN"] = os.path.join(
                self.fbsource_dir, "xplat/third-party/node/bin/", node_exe
            )
            env["RUST_VENDORED_CRATES_DIR"] = os.path.join(
                self.fbsource_dir, "third-party/rust/vendor"
            )
            hash_data = get_fbsource_repo_data(self)
            env["FBSOURCE_HASH"] = hash_data.hash
            env["FBSOURCE_DATE"] = hash_data.date

        lib_path = None
        if self.is_darwin():
            lib_path = "DYLD_LIBRARY_PATH"
        elif self.is_linux():
            lib_path = "LD_LIBRARY_PATH"
        elif self.is_windows():
            lib_path = "PATH"
        else:
            lib_path = None

        for d in install_dirs:
            add_path_entry(env, "CMAKE_PREFIX_PATH", d)

            pkgconfig = os.path.join(d, "lib/pkgconfig")
            if os.path.exists(pkgconfig):
                add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig)

            pkgconfig = os.path.join(d, "lib64/pkgconfig")
            if os.path.exists(pkgconfig):
                add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig)

            # Allow resolving shared objects built earlier (eg: zstd
            # doesn't include the full path to the dylib in its linkage
            # so we need to give it an assist)
            if lib_path:
                for lib in ["lib", "lib64"]:
                    libdir = os.path.join(d, lib)
                    if os.path.exists(libdir):
                        add_path_entry(env, lib_path, libdir)

            # Allow resolving binaries (eg: cmake, ninja) and dlls
            # built by earlier steps
            bindir = os.path.join(d, "bin")
            if os.path.exists(bindir):
                add_path_entry(env, "PATH", bindir, append=False)

            # If rustc is present in the `bin` directory, set RUSTC to prevent
            # cargo uses the rustc installed in the system.
            if self.is_windows():
                rustc_path = os.path.join(bindir, "rustc.bat")
                rustdoc_path = os.path.join(bindir, "rustdoc.bat")
            else:
                rustc_path = os.path.join(bindir, "rustc")
                rustdoc_path = os.path.join(bindir, "rustdoc")

            if os.path.isfile(rustc_path):
                env["RUSTC"] = rustc_path
                env["RUSTDOC"] = rustdoc_path

        return env

def list_win32_subst_letters():
    output = subprocess.check_output(["subst"]).decode("utf-8")
    # The output is a set of lines like: `F:\: => C:\open\some\where`
    lines = output.strip().split("\r\n")
    mapping = {}
    for line in lines:
        fields = line.split(": => ")
        if len(fields) != 2:
            continue
        letter = fields[0]
        path = fields[1]
        mapping[letter] = path

    return mapping

def find_existing_win32_subst_for_path(
    path,  # type: str
    subst_mapping,  # type: typing.Mapping[str, str]
):
    # type: (...) -> typing.Optional[str]
    path = ntpath.normcase(ntpath.normpath(path))
    for letter, target in subst_mapping.items():
        if ntpath.normcase(target) == path:
            return letter
    return None

def find_unused_drive_letter():
    import ctypes

    buffer_len = 256
    blen = ctypes.c_uint(buffer_len)
    rv = ctypes.c_uint()
    bufs = ctypes.create_string_buffer(buffer_len)
    rv = ctypes.windll.kernel32.GetLogicalDriveStringsA(blen, bufs)
    if rv > buffer_len:
        raise Exception("GetLogicalDriveStringsA result too large for buffer")
    nul = "\x00".encode("ascii")

    used = [drive.decode("ascii")[0] for drive in bufs.raw.strip(nul).split(nul)]
    possible = [c for c in "ABCDEFGHIJKLMNOPQRSTUVWXYZ"]
    available = sorted(list(set(possible) - set(used)))
    if len(available) == 0:
        return None
    # Prefer to assign later letters rather than earlier letters
    return available[-1]

def create_subst_path(path):
    for _attempt in range(0, 24):
        drive = find_existing_win32_subst_for_path(
            path, subst_mapping=list_win32_subst_letters()
        )
        if drive:
            return drive
        available = find_unused_drive_letter()
        if available is None:
            raise Exception(
                (
                    "unable to make shorter subst mapping for %s; "
                    "no available drive letters"
                )
                % path
            )

        # Try to set up a subst mapping; note that we may be racing with
        # other processes on the same host, so this may not succeed.
        try:
            subprocess.check_call(["subst", "%s:" % available, path])
            return "%s:\\" % available
        except Exception:
            print("Failed to map %s -> %s" % (available, path))

    raise Exception("failed to set up a subst path for %s" % path)

def _check_host_type(args, host_type):
    if host_type is None:
        host_tuple_string = getattr(args, "host_type", None)
        if host_tuple_string:
            host_type = HostType.from_tuple_string(host_tuple_string)
        else:
            host_type = HostType()

    assert isinstance(host_type, HostType)
    return host_type

def setup_build_options(args, host_type=None):
    """ Create a BuildOptions object based on the arguments """

    fbcode_builder_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    scratch_dir = args.scratch_path
    if not scratch_dir:
        # TODO: `mkscratch` doesn't currently know how best to place things on
        # sandcastle, so whip up something reasonable-ish
        if "SANDCASTLE" in os.environ:
            if "DISK_TEMP" not in os.environ:
                raise Exception(
                    (
                        "I need DISK_TEMP to be set in the sandcastle environment "
                        "so that I can store build products somewhere sane"
                    )
                )
            scratch_dir = os.path.join(
                os.environ["DISK_TEMP"], "fbcode_builder_getdeps"
            )
        if not scratch_dir:
            try:
                scratch_dir = (
                    subprocess.check_output(
                        ["mkscratch", "path", "--subdir", "fbcode_builder_getdeps"]
                    )
                    .strip()
                    .decode("utf-8")
                )
            except OSError as exc:
                if exc.errno != errno.ENOENT:
                    # A legit failure; don't fall back, surface the error
                    raise
                # This system doesn't have mkscratch so we fall back to
                # something local.
                munged = fbcode_builder_dir.replace("Z", "zZ")
                for s in ["/", "\\", ":"]:
                    munged = munged.replace(s, "Z")

                if is_windows() and os.path.isdir("c:/open"):
                    temp = "c:/open/scratch"
                else:
                    temp = tempfile.gettempdir()

                scratch_dir = os.path.join(temp, "fbcode_builder_getdeps-%s" % munged)
                if not is_windows() and os.geteuid() == 0:
                    # Running as root; in the case where someone runs
                    # sudo getdeps.py install-system-deps
                    # and then runs as build without privs, we want to avoid creating
                    # a scratch dir that the second stage cannot write to.
                    # So we generate a different path if we are root.
                    scratch_dir += "-root"

        if not os.path.exists(scratch_dir):
            os.makedirs(scratch_dir)

        if is_windows():
            subst = create_subst_path(scratch_dir)
            print(
                "Mapping scratch dir %s -> %s" % (scratch_dir, subst), file=sys.stderr
            )
            scratch_dir = subst
    else:
        if not os.path.exists(scratch_dir):
            os.makedirs(scratch_dir)

    # Make sure we normalize the scratch path.  This path is used as part of the hash
    # computation for detecting if projects have been updated, so we need to always
    # use the exact same string to refer to a given directory.
    scratch_dir = os.path.realpath(scratch_dir)

    host_type = _check_host_type(args, host_type)

    return BuildOptions(
        fbcode_builder_dir,
        scratch_dir,
        host_type,
        install_dir=args.install_prefix,
        num_jobs=args.num_jobs,
        use_shipit=args.use_shipit,
        vcvars_path=args.vcvars_path,
        allow_system_packages=args.allow_system_packages,
    )
<commit_msg>getdeps: windows: avoid expanding drive letter substitutions<commit_after># Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

from __future__ import absolute_import, division, print_function, unicode_literals

import errno
import glob
import ntpath
import os
import subprocess
import sys
import tempfile

from .copytree import containing_repo_type
from .envfuncs import Env, add_path_entry
from .fetcher import get_fbsource_repo_data
from .manifest import ContextGenerator
from .platform import HostType, is_windows

try:
    import typing  # noqa: F401
except ImportError:
    pass

def detect_project(path):
    repo_type, repo_root = containing_repo_type(path)
    if repo_type is None:
        return None, None

    # Look for a .projectid file.  If it exists, read the project name from it.
    project_id_path = os.path.join(repo_root, ".projectid")
    try:
        with open(project_id_path, "r") as f:
            project_name = f.read().strip()
            return repo_root, project_name
    except EnvironmentError as ex:
        if ex.errno != errno.ENOENT:
            raise

    return repo_root, None

class BuildOptions(object):
    def __init__(
        self,
        fbcode_builder_dir,
        scratch_dir,
        host_type,
        install_dir=None,
        num_jobs=0,
        use_shipit=False,
        vcvars_path=None,
        allow_system_packages=False,
    ):
        """ fbcode_builder_dir - the path to either the in-fbsource fbcode_builder dir,
                                 or for shipit-transformed repos, the build dir that
                                 has been mapped into that dir.
            scratch_dir - a place where we can store repos and build bits.
                          This path should be stable across runs and ideally
                          should not be in the repo of the project being built,
                          but that is ultimately where we generally fall back
                          for builds outside of FB
            install_dir - where the project will ultimately be installed
            num_jobs - the level of concurrency to use while building
            use_shipit - use real shipit instead of the simple shipit transformer
            vcvars_path - Path to external VS toolchain's vsvarsall.bat
        """
        if not num_jobs:
            import multiprocessing

            num_jobs = multiprocessing.cpu_count()
            if is_windows():
                # On Windows the cpu count tends to be the HT count.
                # Running with that level of concurrency tends to
                # swamp the system and make hard to perform other
                # light work.  Let's halve the number of cores here
                # to win that back. The user can still specify a
                # larger number if desired.
                num_jobs = int(num_jobs / 2)

        if not install_dir:
            install_dir = os.path.join(scratch_dir, "installed")

        self.project_hashes = None
        for p in ["../deps/github_hashes", "../project_hashes"]:
            hashes = os.path.join(fbcode_builder_dir, p)
            if os.path.exists(hashes):
                self.project_hashes = hashes
                break

        # Detect what repository and project we are being run from.
        self.repo_root, self.repo_project = detect_project(os.getcwd())

        # If we are running from an fbsource repository, set self.fbsource_dir
        # to allow the ShipIt-based fetchers to use it.
        if self.repo_project == "fbsource":
            self.fbsource_dir = self.repo_root
        else:
            self.fbsource_dir = None

        self.num_jobs = num_jobs
        self.scratch_dir = scratch_dir
        self.install_dir = install_dir
        self.fbcode_builder_dir = fbcode_builder_dir
        self.host_type = host_type
        self.use_shipit = use_shipit
        self.allow_system_packages = allow_system_packages
        if vcvars_path is None and is_windows():

            # On Windows, the compiler is not available in the PATH by
            # default so we need to run the vcvarsall script to populate the
            # environment. We use a glob to find some version of this script
            # as deployed with Visual Studio 2017.  This logic can also
            # locate Visual Studio 2019 but note that at the time of writing
            # the version of boost in our manifest cannot be built with
            # VS 2019, so we're effectively tied to VS 2017 until we upgrade
            # the boost dependency.
            vcvarsall = []
            for year in ["2017", "2019"]:
                vcvarsall += glob.glob(
                    os.path.join(
                        os.environ["ProgramFiles(x86)"],
                        "Microsoft Visual Studio",
                        year,
                        "*",
                        "VC",
                        "Auxiliary",
                        "Build",
                        "vcvarsall.bat",
                    )
                )
            vcvars_path = vcvarsall[0]

        self.vcvars_path = vcvars_path

    @property
    def manifests_dir(self):
        return os.path.join(self.fbcode_builder_dir, "manifests")

    def is_darwin(self):
        return self.host_type.is_darwin()

    def is_windows(self):
        return self.host_type.is_windows()

    def get_vcvars_path(self):
        return self.vcvars_path

    def is_linux(self):
        return self.host_type.is_linux()

    def get_context_generator(self, host_tuple=None, facebook_internal=None):
        """ Create a manifest ContextGenerator for the specified target platform. """
        if host_tuple is None:
            host_type = self.host_type
        elif isinstance(host_tuple, HostType):
            host_type = host_tuple
        else:
            host_type = HostType.from_tuple_string(host_tuple)

        # facebook_internal is an Optional[bool]
        # If it is None, default to assuming this is a Facebook-internal build if
        # we are running in an fbsource repository.
        if facebook_internal is None:
            facebook_internal = self.fbsource_dir is not None

        return ContextGenerator(
            {
                "os": host_type.ostype,
                "distro": host_type.distro,
                "distro_vers": host_type.distrovers,
                "fb": "on" if facebook_internal else "off",
                "test": "off",
            }
        )

    def compute_env_for_install_dirs(self, install_dirs, env=None):
        if env is not None:
            env = env.copy()
        else:
            env = Env()

        if self.fbsource_dir:
            env["YARN_YARN_OFFLINE_MIRROR"] = os.path.join(
                self.fbsource_dir, "xplat/third-party/yarn/offline-mirror"
            )
            yarn_exe = "yarn.bat" if self.is_windows() else "yarn"
            env["YARN_PATH"] = os.path.join(
                self.fbsource_dir, "xplat/third-party/yarn/", yarn_exe
            )
            node_exe = "node-win-x64.exe" if self.is_windows() else "node"
            env["NODE_BIN"] = os.path.join(
                self.fbsource_dir, "xplat/third-party/node/bin/", node_exe
            )
            env["RUST_VENDORED_CRATES_DIR"] = os.path.join(
                self.fbsource_dir, "third-party/rust/vendor"
            )
            hash_data = get_fbsource_repo_data(self)
            env["FBSOURCE_HASH"] = hash_data.hash
            env["FBSOURCE_DATE"] = hash_data.date

        lib_path = None
        if self.is_darwin():
            lib_path = "DYLD_LIBRARY_PATH"
        elif self.is_linux():
            lib_path = "LD_LIBRARY_PATH"
        elif self.is_windows():
            lib_path = "PATH"
        else:
            lib_path = None

        for d in install_dirs:
            add_path_entry(env, "CMAKE_PREFIX_PATH", d)

            pkgconfig = os.path.join(d, "lib/pkgconfig")
            if os.path.exists(pkgconfig):
                add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig)

            pkgconfig = os.path.join(d, "lib64/pkgconfig")
            if os.path.exists(pkgconfig):
                add_path_entry(env, "PKG_CONFIG_PATH", pkgconfig)

            # Allow resolving shared objects built earlier (eg: zstd
            # doesn't include the full path to the dylib in its linkage
            # so we need to give it an assist)
            if lib_path:
                for lib in ["lib", "lib64"]:
                    libdir = os.path.join(d, lib)
                    if os.path.exists(libdir):
                        add_path_entry(env, lib_path, libdir)

            # Allow resolving binaries (eg: cmake, ninja) and dlls
            # built by earlier steps
            bindir = os.path.join(d, "bin")
            if os.path.exists(bindir):
                add_path_entry(env, "PATH", bindir, append=False)

            # If rustc is present in the `bin` directory, set RUSTC to prevent
            # cargo uses the rustc installed in the system.
            if self.is_windows():
                rustc_path = os.path.join(bindir, "rustc.bat")
                rustdoc_path = os.path.join(bindir, "rustdoc.bat")
            else:
                rustc_path = os.path.join(bindir, "rustc")
                rustdoc_path = os.path.join(bindir, "rustdoc")

            if os.path.isfile(rustc_path):
                env["RUSTC"] = rustc_path
                env["RUSTDOC"] = rustdoc_path

        return env

def list_win32_subst_letters():
    output = subprocess.check_output(["subst"]).decode("utf-8")
    # The output is a set of lines like: `F:\: => C:\open\some\where`
    lines = output.strip().split("\r\n")
    mapping = {}
    for line in lines:
        fields = line.split(": => ")
        if len(fields) != 2:
            continue
        letter = fields[0]
        path = fields[1]
        mapping[letter] = path

    return mapping

def find_existing_win32_subst_for_path(
    path,  # type: str
    subst_mapping,  # type: typing.Mapping[str, str]
):
    # type: (...) -> typing.Optional[str]
    path = ntpath.normcase(ntpath.normpath(path))
    for letter, target in subst_mapping.items():
        if ntpath.normcase(target) == path:
            return letter
    return None

def find_unused_drive_letter():
    import ctypes

    buffer_len = 256
    blen = ctypes.c_uint(buffer_len)
    rv = ctypes.c_uint()
    bufs = ctypes.create_string_buffer(buffer_len)
    rv = ctypes.windll.kernel32.GetLogicalDriveStringsA(blen, bufs)
    if rv > buffer_len:
        raise Exception("GetLogicalDriveStringsA result too large for buffer")
    nul = "\x00".encode("ascii")

    used = [drive.decode("ascii")[0] for drive in bufs.raw.strip(nul).split(nul)]
    possible = [c for c in "ABCDEFGHIJKLMNOPQRSTUVWXYZ"]
    available = sorted(list(set(possible) - set(used)))
    if len(available) == 0:
        return None
    # Prefer to assign later letters rather than earlier letters
    return available[-1]

def create_subst_path(path):
    for _attempt in range(0, 24):
        drive = find_existing_win32_subst_for_path(
            path, subst_mapping=list_win32_subst_letters()
        )
        if drive:
            return drive
        available = find_unused_drive_letter()
        if available is None:
            raise Exception(
                (
                    "unable to make shorter subst mapping for %s; "
                    "no available drive letters"
                )
                % path
            )

        # Try to set up a subst mapping; note that we may be racing with
        # other processes on the same host, so this may not succeed.
        try:
            subprocess.check_call(["subst", "%s:" % available, path])
            return "%s:\\" % available
        except Exception:
            print("Failed to map %s -> %s" % (available, path))

    raise Exception("failed to set up a subst path for %s" % path)

def _check_host_type(args, host_type):
    if host_type is None:
        host_tuple_string = getattr(args, "host_type", None)
        if host_tuple_string:
            host_type = HostType.from_tuple_string(host_tuple_string)
        else:
            host_type = HostType()

    assert isinstance(host_type, HostType)
    return host_type

def setup_build_options(args, host_type=None):
    """ Create a BuildOptions object based on the arguments """

    fbcode_builder_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    scratch_dir = args.scratch_path
    if not scratch_dir:
        # TODO: `mkscratch` doesn't currently know how best to place things on
        # sandcastle, so whip up something reasonable-ish
        if "SANDCASTLE" in os.environ:
            if "DISK_TEMP" not in os.environ:
                raise Exception(
                    (
                        "I need DISK_TEMP to be set in the sandcastle environment "
                        "so that I can store build products somewhere sane"
                    )
                )
            scratch_dir = os.path.join(
                os.environ["DISK_TEMP"], "fbcode_builder_getdeps"
            )
        if not scratch_dir:
            try:
                scratch_dir = (
                    subprocess.check_output(
                        ["mkscratch", "path", "--subdir", "fbcode_builder_getdeps"]
                    )
                    .strip()
                    .decode("utf-8")
                )
            except OSError as exc:
                if exc.errno != errno.ENOENT:
                    # A legit failure; don't fall back, surface the error
                    raise
                # This system doesn't have mkscratch so we fall back to
                # something local.
                munged = fbcode_builder_dir.replace("Z", "zZ")
                for s in ["/", "\\", ":"]:
                    munged = munged.replace(s, "Z")

                if is_windows() and os.path.isdir("c:/open"):
                    temp = "c:/open/scratch"
                else:
                    temp = tempfile.gettempdir()

                scratch_dir = os.path.join(temp, "fbcode_builder_getdeps-%s" % munged)
                if not is_windows() and os.geteuid() == 0:
                    # Running as root; in the case where someone runs
                    # sudo getdeps.py install-system-deps
                    # and then runs as build without privs, we want to avoid creating
                    # a scratch dir that the second stage cannot write to.
                    # So we generate a different path if we are root.
                    scratch_dir += "-root"

        if not os.path.exists(scratch_dir):
            os.makedirs(scratch_dir)

        if is_windows():
            subst = create_subst_path(scratch_dir)
            print(
                "Mapping scratch dir %s -> %s" % (scratch_dir, subst), file=sys.stderr
            )
            scratch_dir = subst
    else:
        if not os.path.exists(scratch_dir):
            os.makedirs(scratch_dir)

    # Make sure we normalize the scratch path.  This path is used as part of the hash
    # computation for detecting if projects have been updated, so we need to always
    # use the exact same string to refer to a given directory.
    # But! realpath in some combinations of Windows/Python3 versions can expand the
    # drive substitutions on Windows, so avoid that!
    if not is_windows():
        scratch_dir = os.path.realpath(scratch_dir)

    host_type = _check_host_type(args, host_type)

    return BuildOptions(
        fbcode_builder_dir,
        scratch_dir,
        host_type,
        install_dir=args.install_prefix,
        num_jobs=args.num_jobs,
        use_shipit=args.use_shipit,
        vcvars_path=args.vcvars_path,
        allow_system_packages=args.allow_system_packages,
    )
<|endoftext|>


<commit_before>from test.unit_tests.providers import common
from test.unit_tests.providers.common import ProviderTestCase
from test.utils import http
from totalimpact.providers.provider import Provider, ProviderContentMalformedError

import os
import collections
from nose.tools import assert_equals, raises, nottest

datadir = os.path.join(os.path.split(__file__)[0], "../../../extras/sample_provider_pages/pubmed")
SAMPLE_EXTRACT_METRICS_PAGE = os.path.join(datadir, "metrics")
SAMPLE_EXTRACT_ALIASES_FROM_DOI_PAGE = os.path.join(datadir, "aliases_from_doi")
SAMPLE_EXTRACT_ALIASES_FROM_PMID_PAGE = os.path.join(datadir, "aliases_from_pmid")
SAMPLE_EXTRACT_BIBLIO_PAGE = os.path.join(datadir, "biblio")
SAMPLE_EXTRACT_PROVENANCE_URL_PAGE = SAMPLE_EXTRACT_METRICS_PAGE

TEST_DOI = "10.1371/journal.pcbi.1000361"
TEST_PMID = "16060722"
TEST_DOI_HAS_NO_PMID = "10.1016/j.artint.2005.10.007"

class TestPubmed(ProviderTestCase):

    provider_name = "pubmed"

    testitem_aliases = ("pmid", TEST_PMID)
    testitem_biblio = ("pmid", TEST_PMID)
    testitem_metrics = ("pmid", TEST_PMID)
    testitem_members = "123\n456\n789"

    def setUp(self):
        ProviderTestCase.setUp(self)

    def test_is_relevant_alias(self):
        # ensure that it matches an appropriate ids
        assert_equals(self.provider.is_relevant_alias(self.testitem_aliases), True)

        assert_equals(self.provider.is_relevant_alias(("github", "egonw,cdk")), False)

    def test_extract_aliases_from_doi(self):
        # ensure that the dryad reader can interpret an xml doc appropriately
        f = open(SAMPLE_EXTRACT_ALIASES_FROM_DOI_PAGE, "r")
        aliases = self.provider._extract_aliases_from_doi(f.read(), "10.1371/journal.pcbi.1000361")
        print aliases
        assert_equals(aliases, [('pmid', '19381256')])

    def test_extract_aliases_from_pmid(self):
        # ensure that the dryad reader can interpret an xml doc appropriately
        f = open(SAMPLE_EXTRACT_ALIASES_FROM_PMID_PAGE, "r")
        aliases = self.provider._extract_aliases_from_pmid(f.read(), "17593900")
        print aliases
        expected = [('doi', u'10.1371/journal.pmed.0040215'), ('url', u'http://dx.doi.org/10.1371/journal.pmed.0040215')]
        assert_equals(aliases, expected)

    # override default because returns url even if pmid api page is empty
    def test_provider_aliases_empty(self):
        Provider.http_get = common.get_empty
        aliases = self.provider.aliases([self.testitem_aliases])
        print aliases
        assert_equals(aliases, [("url", 'http://www.ncbi.nlm.nih.gov/pubmed/16060722')])

    def test_extract_citing_pmcids(self):
        f = open(SAMPLE_EXTRACT_METRICS_PAGE, "r")
        pmcids = self.provider._extract_citing_pmcids(f.read())
        assert_equals(len(pmcids), 149)

    def test_extract_biblio(self):
        f = open(SAMPLE_EXTRACT_BIBLIO_PAGE, "r")
        ret = self.provider._extract_biblio(f.read())
        expected = {'authors': u'Ioannidis', 'date': '2005-08-30T00:00:00', 'journal': u'PLoS medicine', 'year': "2005", 'title': u'Why most published research findings are false.'}
        assert_equals(ret, expected)

    def test_member_items(self):
        ret = self.provider.member_items("123\n456\n678")
        expected = [('pmid', '123'), ('pmid', '456'), ('pmid', '678')]
        assert_equals(ret, expected)

    def test_provider_member_items_400(self):
        pass
    def test_provider_member_items_500(self):
        pass
    def test_provider_member_items_empty(self):
        pass
    def test_provider_member_items_nonsense_txt(self):
        pass
    def test_provider_member_items_nonsense_xml(self):
        pass

    @http
    def test_biblio(self):
        biblio_dict = self.provider.biblio([self.testitem_biblio])
        expected = {'authors': u'Ioannidis', 'date': '2005-08-30T00:00:00', 'journal': u'PLoS medicine', 'year': "2005", 'title': u'Why most published research findings are false.'}
        assert_equals(biblio_dict, expected)

    @http
    def test_aliases_from_pmid(self):
        aliases = self.provider.aliases([self.testitem_aliases])
        print aliases
        expected = [('biblio', {'title': u'Why most published research findings are false.', 'journal': u'PLoS medicine', 'year': "2005", 'authors': u'Ioannidis', 'date': '2005-08-30T00:00:00'}), ('doi', u'10.1371/journal.pmed.0020124'), ('url', u'http://dx.doi.org/10.1371/journal.pmed.0020124'), ('url', 'http://www.ncbi.nlm.nih.gov/pubmed/16060722')]
        assert_equals(aliases, expected)

    @http
    def test_aliases_from_pmid_different_date_format(self):
        aliases = self.provider.aliases([("pmid", "11457506")])
        print aliases
        expected = [('biblio', {'title': u'Radiation hybrid mapping of 11 alpha and beta nicotinic acetylcholine receptor genes in Rattus norvegicus.', 'journal': u'Brain research. Molecular brain research', 'year': "2001", 'month': u'Jul', 'authors': u'Tseng, Kwitek-Black, Erbe, Popper, Jacob, Wackym', 'day': 13}), ('url', 'http://www.ncbi.nlm.nih.gov/pubmed/11457506')]
        assert_equals(aliases, expected)

    @http
    def test_aliases_from_pmid_when_doi_fragment(self):
        #this pmid has a partial doi in its doi field.  Make sure we don't include it in our doi field.
        aliases = self.provider.aliases([("pmid", "11244366")])
        print aliases
        expected = [('biblio', {'journal': u'ORL; journal for oto-rhino-laryngology and its related specialties', 'authors': u'Oshima, Ikeda, Furukawa, Suzuki, Takasaka', 'title': u'Expression of the voltage-dependent chloride channel ClC-3 in human nasal tissue.'}), ('url', 'http://www.ncbi.nlm.nih.gov/pubmed/11244366')]
        assert_equals(aliases, expected)

    @http
    def test_aliases_from_pmid_when_doi_in_different_part_of_xml(self):
        aliases = self.provider.aliases([("pmid", "23682040")])
        print aliases
        expected = [('biblio', {'title': u'Influenza: marketing vaccine by marketing disease.', 'journal': u'BMJ (Clinical research ed.)', 'year': "2013", 'authors': u'Doshi', 'date': '2013-05-16T00:00:00'}), ('doi', u'10.1136/bmj.f3037'), ('url', u'http://dx.doi.org/10.1136/bmj.f3037'), ('url', 'http://www.ncbi.nlm.nih.gov/pubmed/23682040')]
        assert_equals(aliases, expected)

    @http
    def test_aliases_from_doi(self):
        aliases_dict = self.provider.aliases([("doi", TEST_DOI)])
        assert_equals(set(aliases_dict), set([('pmid', '19381256')]))

        aliases_dict = self.provider.aliases([("doi", "TEST_DOI_HAS_NO_PMID")])
        assert_equals(aliases_dict, [])

    @http
    def test_metrics(self):
        metrics_dict = self.provider.metrics([self.testitem_metrics])
        print metrics_dict
        expected = {'pubmed:pmc_citations': (149, 'http://www.ncbi.nlm.nih.gov/pubmed?linkname=pubmed_pubmed_citedin&from_uid=16060722'), 'pubmed:f1000': (True, 'http://f1000.com/pubmed/16060722'), 'pubmed:pmc_citations_reviews': (20, 'http://www.ncbi.nlm.nih.gov/pubmed?term=22182676%20OR%2022065657%20OR%2021998558%20OR%2021890791%20OR%2021788505%20OR%2021407270%20OR%2020967426%20OR%2020637084%20OR%2020571517%20OR%2020420659%20OR%2020382258%20OR%2020307281%20OR%2019956635%20OR%2019860651%20OR%2019207020%20OR%2018834308%20OR%2018612135%20OR%2018603647%20OR%2017705840%20OR%2017587446&cmd=DetailsSearch'), 'pubmed:pmc_citations_editorials': (11, 'http://www.ncbi.nlm.nih.gov/pubmed?term=22515987%20OR%2022285994%20OR%2021693091%20OR%2021153562%20OR%2020876290%20OR%2020596038%20OR%2020420659%20OR%2020040241%20OR%2019967369%20OR%2019949717%20OR%2017880356&cmd=DetailsSearch')}
        for key in expected:
            assert metrics_dict[key][0] >= expected[key][0], [key, metrics_dict[key], expected[key]]

            # the drilldown url changes with the metrics for some pubmed metrics, so don't check those
            if (key != "pubmed:pmc_citations_editorials") and (key != "pubmed:pmc_citations_reviews"):
                assert metrics_dict[key][1] == expected[key][1], [key, metrics_dict[key], expected[key]]

<commit_msg>use canned test item<commit_after>from test.unit_tests.providers import common
from test.unit_tests.providers.common import ProviderTestCase
from test.utils import http
from totalimpact.providers.provider import Provider, ProviderContentMalformedError

import os
import collections
from nose.tools import assert_equals, raises, nottest

datadir = os.path.join(os.path.split(__file__)[0], "../../../extras/sample_provider_pages/pubmed")
SAMPLE_EXTRACT_METRICS_PAGE = os.path.join(datadir, "metrics")
SAMPLE_EXTRACT_ALIASES_FROM_DOI_PAGE = os.path.join(datadir, "aliases_from_doi")
SAMPLE_EXTRACT_ALIASES_FROM_PMID_PAGE = os.path.join(datadir, "aliases_from_pmid")
SAMPLE_EXTRACT_BIBLIO_PAGE = os.path.join(datadir, "biblio")
SAMPLE_EXTRACT_PROVENANCE_URL_PAGE = SAMPLE_EXTRACT_METRICS_PAGE

TEST_DOI = "10.1371/journal.pcbi.1000361"
TEST_PMID = "16060722"
TEST_DOI_HAS_NO_PMID = "10.1016/j.artint.2005.10.007"

class TestPubmed(ProviderTestCase):

    provider_name = "pubmed"

    testitem_aliases = ("pmid", TEST_PMID)
    testitem_biblio = ("pmid", TEST_PMID)
    testitem_metrics = ("pmid", TEST_PMID)
    testitem_members = "123\n456\n789"

    def setUp(self):
        ProviderTestCase.setUp(self)

    def test_is_relevant_alias(self):
        # ensure that it matches an appropriate ids
        assert_equals(self.provider.is_relevant_alias(self.testitem_aliases), True)

        assert_equals(self.provider.is_relevant_alias(("github", "egonw,cdk")), False)

    def test_extract_aliases_from_doi(self):
        # ensure that the dryad reader can interpret an xml doc appropriately
        f = open(SAMPLE_EXTRACT_ALIASES_FROM_DOI_PAGE, "r")
        aliases = self.provider._extract_aliases_from_doi(f.read(), "10.1371/journal.pcbi.1000361")
        print aliases
        assert_equals(aliases, [('pmid', '19381256')])

    def test_extract_aliases_from_pmid(self):
        # ensure that the dryad reader can interpret an xml doc appropriately
        f = open(SAMPLE_EXTRACT_ALIASES_FROM_PMID_PAGE, "r")
        aliases = self.provider._extract_aliases_from_pmid(f.read(), "17593900")
        print aliases
        expected = [('doi', u'10.1371/journal.pmed.0040215'), ('url', u'http://dx.doi.org/10.1371/journal.pmed.0040215')]
        assert_equals(aliases, expected)

    # override default because returns url even if pmid api page is empty
    def test_provider_aliases_empty(self):
        Provider.http_get = common.get_empty
        aliases = self.provider.aliases([self.testitem_aliases])
        print aliases
        assert_equals(aliases, [("url", 'http://www.ncbi.nlm.nih.gov/pubmed/16060722')])

    def test_extract_citing_pmcids(self):
        f = open(SAMPLE_EXTRACT_METRICS_PAGE, "r")
        pmcids = self.provider._extract_citing_pmcids(f.read())
        assert_equals(len(pmcids), 149)

    def test_extract_biblio(self):
        f = open(SAMPLE_EXTRACT_BIBLIO_PAGE, "r")
        ret = self.provider._extract_biblio(f.read())
        expected = {'authors': u'Ioannidis', 'date': '2005-08-30T00:00:00', 'journal': u'PLoS medicine', 'year': "2005", 'title': u'Why most published research findings are false.'}
        assert_equals(ret, expected)

    def test_member_items(self):
        ret = self.provider.member_items(self.testitem_members)
        expected = [('pmid', '123'), ('pmid', '456'), ('pmid', '789')]
        assert_equals(ret, expected)

    def test_provider_member_items_400(self):
        pass
    def test_provider_member_items_500(self):
        pass
    def test_provider_member_items_empty(self):
        pass
    def test_provider_member_items_nonsense_txt(self):
        pass
    def test_provider_member_items_nonsense_xml(self):
        pass

    @http
    def test_biblio(self):
        biblio_dict = self.provider.biblio([self.testitem_biblio])
        expected = {'authors': u'Ioannidis', 'date': '2005-08-30T00:00:00', 'journal': u'PLoS medicine', 'year': "2005", 'title': u'Why most published research findings are false.'}
        assert_equals(biblio_dict, expected)

    @http
    def test_aliases_from_pmid(self):
        aliases = self.provider.aliases([self.testitem_aliases])
        print aliases
        expected = [('biblio', {'title': u'Why most published research findings are false.', 'journal': u'PLoS medicine', 'year': "2005", 'authors': u'Ioannidis', 'date': '2005-08-30T00:00:00'}), ('doi', u'10.1371/journal.pmed.0020124'), ('url', u'http://dx.doi.org/10.1371/journal.pmed.0020124'), ('url', 'http://www.ncbi.nlm.nih.gov/pubmed/16060722')]
        assert_equals(aliases, expected)

    @http
    def test_aliases_from_pmid_different_date_format(self):
        aliases = self.provider.aliases([("pmid", "11457506")])
        print aliases
        expected = [('biblio', {'title': u'Radiation hybrid mapping of 11 alpha and beta nicotinic acetylcholine receptor genes in Rattus norvegicus.', 'journal': u'Brain research. Molecular brain research', 'year': "2001", 'month': u'Jul', 'authors': u'Tseng, Kwitek-Black, Erbe, Popper, Jacob, Wackym', 'day': 13}), ('url', 'http://www.ncbi.nlm.nih.gov/pubmed/11457506')]
        assert_equals(aliases, expected)

    @http
    def test_aliases_from_pmid_when_doi_fragment(self):
        #this pmid has a partial doi in its doi field.  Make sure we don't include it in our doi field.
        aliases = self.provider.aliases([("pmid", "11244366")])
        print aliases
        expected = [('biblio', {'journal': u'ORL; journal for oto-rhino-laryngology and its related specialties', 'authors': u'Oshima, Ikeda, Furukawa, Suzuki, Takasaka', 'title': u'Expression of the voltage-dependent chloride channel ClC-3 in human nasal tissue.'}), ('url', 'http://www.ncbi.nlm.nih.gov/pubmed/11244366')]
        assert_equals(aliases, expected)

    @http
    def test_aliases_from_pmid_when_doi_in_different_part_of_xml(self):
        aliases = self.provider.aliases([("pmid", "23682040")])
        print aliases
        expected = [('biblio', {'title': u'Influenza: marketing vaccine by marketing disease.', 'journal': u'BMJ (Clinical research ed.)', 'year': "2013", 'authors': u'Doshi', 'date': '2013-05-16T00:00:00'}), ('doi', u'10.1136/bmj.f3037'), ('url', u'http://dx.doi.org/10.1136/bmj.f3037'), ('url', 'http://www.ncbi.nlm.nih.gov/pubmed/23682040')]
        assert_equals(aliases, expected)

    @http
    def test_aliases_from_doi(self):
        aliases_dict = self.provider.aliases([("doi", TEST_DOI)])
        assert_equals(set(aliases_dict), set([('pmid', '19381256')]))

        aliases_dict = self.provider.aliases([("doi", "TEST_DOI_HAS_NO_PMID")])
        assert_equals(aliases_dict, [])

    @http
    def test_metrics(self):
        metrics_dict = self.provider.metrics([self.testitem_metrics])
        print metrics_dict
        expected = {'pubmed:pmc_citations': (149, 'http://www.ncbi.nlm.nih.gov/pubmed?linkname=pubmed_pubmed_citedin&from_uid=16060722'), 'pubmed:f1000': (True, 'http://f1000.com/pubmed/16060722'), 'pubmed:pmc_citations_reviews': (20, 'http://www.ncbi.nlm.nih.gov/pubmed?term=22182676%20OR%2022065657%20OR%2021998558%20OR%2021890791%20OR%2021788505%20OR%2021407270%20OR%2020967426%20OR%2020637084%20OR%2020571517%20OR%2020420659%20OR%2020382258%20OR%2020307281%20OR%2019956635%20OR%2019860651%20OR%2019207020%20OR%2018834308%20OR%2018612135%20OR%2018603647%20OR%2017705840%20OR%2017587446&cmd=DetailsSearch'), 'pubmed:pmc_citations_editorials': (11, 'http://www.ncbi.nlm.nih.gov/pubmed?term=22515987%20OR%2022285994%20OR%2021693091%20OR%2021153562%20OR%2020876290%20OR%2020596038%20OR%2020420659%20OR%2020040241%20OR%2019967369%20OR%2019949717%20OR%2017880356&cmd=DetailsSearch')}
        for key in expected:
            assert metrics_dict[key][0] >= expected[key][0], [key, metrics_dict[key], expected[key]]

            # the drilldown url changes with the metrics for some pubmed metrics, so don't check those
            if (key != "pubmed:pmc_citations_editorials") and (key != "pubmed:pmc_citations_reviews"):
                assert metrics_dict[key][1] == expected[key][1], [key, metrics_dict[key], expected[key]]

<|endoftext|>