<commit_before># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import codecs
import logging
import os
import time
import traceback
import urlparse
import random

from telemetry.core import util
from telemetry.core import wpr_modes
from telemetry.core import exceptions
from telemetry.page import page_filter as page_filter_module
from telemetry.page import page_test

class PageState(object):
  def __init__(self):
    self.did_login = False

class _RunState(object):
  def __init__(self):
    self.first_browser = True
    self.browser = None
    self.tab = None
    self.is_tracing = False

  def Close(self):
    self.is_tracing = False

    if self.tab:
      self.tab.Disconnect()
      self.tab = None

    if self.browser:
      self.browser.Close()
      self.browser = None

def _ShuffleAndFilterPageSet(page_set, options):
  if options.pageset_shuffle_order_file and not options.pageset_shuffle:
    raise Exception('--pageset-shuffle-order-file requires --pageset-shuffle.')

  if options.pageset_shuffle_order_file:
    return page_set.ReorderPageSet(options.pageset_shuffle_order_file)

  page_filter = page_filter_module.PageFilter(options)
  pages = [page for page in page_set.pages[:]
           if not page.disabled and page_filter.IsSelected(page)]

  if options.pageset_shuffle:
    random.Random().shuffle(pages)
  return [page
      for _ in xrange(int(options.pageset_repeat))
      for page in pages
      for _ in xrange(int(options.page_repeat))]

class PageRunner(object):
  """Runs a given test against a given test."""
  def __init__(self, page_set):
    self.page_set = page_set

  def __enter__(self):
    return self

  def __exit__(self, *args):
    self.Close()

  def Run(self, options, possible_browser, test, results):
    # Check if we can run against WPR.
    for page in self.page_set.pages:
      parsed_url = urlparse.urlparse(page.url)
      if parsed_url.scheme == 'file':
        continue
      if not page.archive_path:
        logging.warning("""
  No page set archive provided for the page %s. Benchmarking against live sites!
  Results won't be repeatable or comparable.
""", page.url)
      elif options.wpr_mode != wpr_modes.WPR_RECORD:
        # The page has an archive, and we're not recording.
        if not os.path.isfile(page.archive_path):
          logging.warning("""
  The page set archive %s for page %s does not exist, benchmarking against live
  sites! Results won't be repeatable or comparable.

  To fix this, either add svn-internal to your .gclient using
  http://goto/read-src-internal, or create a new archive using record_wpr.
  """, os.path.relpath(page.archive_path), page.url)

    # Verify credentials path.
    credentials_path = None
    if self.page_set.credentials_path:
      credentials_path = os.path.join(os.path.dirname(self.page_set.file_path),
                                      self.page_set.credentials_path)
      if not os.path.exists(credentials_path):
        credentials_path = None

    # Set up user agent.
    if self.page_set.user_agent_type:
      options.browser_user_agent_type = self.page_set.user_agent_type

    for page in self.page_set:
      test.CustomizeBrowserOptionsForPage(page, possible_browser.options)

    # Check tracing directory.
    if options.trace_dir:
      if not os.path.exists(options.trace_dir):
        os.mkdir(options.trace_dir)
      if not os.path.isdir(options.trace_dir):
        raise Exception('--trace-dir isn\'t a directory: %s' %
                        options.trace_dir)
      elif os.listdir(options.trace_dir):
        raise Exception('Trace directory isn\'t empty: %s' % options.trace_dir)

    # Reorder page set based on options.
    pages = _ShuffleAndFilterPageSet(self.page_set, options)

    state = _RunState()
    last_archive_path = None
    try:
      for page in pages:
        if options.wpr_mode != wpr_modes.WPR_RECORD:
          if page.archive_path and os.path.isfile(page.archive_path):
            possible_browser.options.wpr_mode = wpr_modes.WPR_REPLAY
          else:
            possible_browser.options.wpr_mode = wpr_modes.WPR_OFF
        if last_archive_path != page.archive_path:
          state.Close()
          state = _RunState()
          last_archive_path = page.archive_path
        tries = 3
        while tries:
          try:
            if not state.browser:
              self._SetupBrowser(state, test, possible_browser,
                                 credentials_path, page.archive_path)
            if not state.tab:
              if len(state.browser.tabs) == 0:
                state.browser.tabs.New()
              state.tab = state.browser.tabs[0]
            if options.trace_dir:
              self._SetupTracingTab(state)

            self._WaitForThermalThrottlingIfNeeded(state.browser.platform)

            try:
              self._RunPage(options, page, state.tab, test, results)
              self._CheckThermalThrottling(state.browser.platform)
            except exceptions.TabCrashException:
              stdout = ''
              if not options.show_stdout:
                stdout = state.browser.GetStandardOutput()
                stdout = (('\nStandard Output:\n') +
                          ('*' * 80) +
                          '\n\t' + stdout.replace('\n', '\n\t') + '\n' +
                          ('*' * 80))
              logging.warning('Tab crashed: %s%s', page.url, stdout)
              state.Close()

            if options.trace_dir:
              self._EndTracing(state, options, page)

            if test.needs_browser_restart_after_each_run:
              state.Close()

            break
          except exceptions.BrowserGoneException:
            logging.warning('Lost connection to browser. Retrying.')
            state.Close()
            tries -= 1
            if not tries:
              logging.error('Lost connection to browser 3 times. Failing.')
              raise
    finally:
      state.Close()

  def _RunPage(self, options, page, tab, test, results):
    if not test.CanRunForPage(page):
      logging.warning('Skiping test: it cannot run for %s', page.url)
      results.AddSkippedPage(page, 'Test cannot run', '')
      return

    logging.info('Running %s' % page.url)

    page_state = PageState()
    try:
      did_prepare = self._PreparePage(page, tab, page_state, test, results)
    except util.TimeoutException, ex:
      logging.warning('Timed out waiting for reply on %s. This is unusual.',
                      page.url)
      results.AddFailure(page, ex, traceback.format_exc())
      return
    except exceptions.TabCrashException, ex:
      results.AddFailure(page, ex, traceback.format_exc())
      raise
    except exceptions.BrowserGoneException:
      raise
    except Exception, ex:
      logging.error('Unexpected failure while running %s: %s',
                    page.url, traceback.format_exc())
      self._CleanUpPage(page, tab, page_state)
      raise

    if not did_prepare:
      self._CleanUpPage(page, tab, page_state)
      return

    try:
      test.Run(options, page, tab, results)
    except page_test.Failure, ex:
      logging.info('%s: %s', ex, page.url)
      results.AddFailure(page, ex, traceback.format_exc())
      return
    except util.TimeoutException, ex:
      logging.warning('Timed out while running %s', page.url)
      results.AddFailure(page, ex, traceback.format_exc())
      return
    except exceptions.TabCrashException, ex:
      results.AddFailure(page, ex, traceback.format_exc())
      raise
    except exceptions.BrowserGoneException:
      raise
    except Exception, ex:
      logging.error('Unexpected failure while running %s: %s',
                    page.url, traceback.format_exc())
      raise
    finally:
      self._CleanUpPage(page, tab, page_state)

    results.AddSuccess(page)

  def Close(self):
    pass

  def _SetupBrowser(self, state, test, possible_browser, credentials_path,
                    archive_path):
    assert not state.tab
    state.browser = possible_browser.Create()
    state.browser.credentials.credentials_path = credentials_path
    test.SetUpBrowser(state.browser)

    if state.first_browser:
      state.browser.credentials.WarnIfMissingCredentials(self.page_set)
      state.first_browser = False

    state.browser.SetReplayArchivePath(archive_path)

  def _SetupTracingTab(self, state):
    if state.browser.supports_tracing:
      state.is_tracing = True
      state.browser.StartTracing()

  def _EndTracing(self, state, options, page):
    if state.is_tracing:
      assert state.browser
      state.is_tracing = False
      state.browser.StopTracing()
      trace_result = state.browser.GetTraceResultAndReset()
      logging.info('Processing trace...')

      trace_file_base = os.path.join(
          options.trace_dir, page.url_as_file_safe_name)

      if options.page_repeat != 1 or options.pageset_repeat != 1:
        trace_file_index = 0

        while True:
          trace_file = '%s_%03d.json' % (trace_file_base, trace_file_index)
          if not os.path.exists(trace_file):
            break
          trace_file_index = trace_file_index + 1
      else:
        trace_file = '%s.json' % trace_file_base
      with codecs.open(trace_file, 'w',
                       encoding='utf-8') as trace_file:
        trace_result.Serialize(trace_file)
      logging.info('Trace saved.')

  def _PreparePage(self, page, tab, page_state, test, results):
    parsed_url = urlparse.urlparse(page.url)
    if parsed_url[0] == 'file':
      dirname, filename = page.url_base_dir_and_file
      tab.browser.SetHTTPServerDirectory(dirname)
      target_side_url = tab.browser.http_server.UrlOf(filename)
    else:
      target_side_url = page.url

    if page.credentials:
      page_state.did_login = tab.browser.credentials.LoginNeeded(
        tab, page.credentials)
      if not page_state.did_login:
        msg = 'Could not login to %s on %s' % (page.credentials,
                                               target_side_url)
        logging.info(msg)
        results.AddFailure(page, msg, "")
        return False

    test.WillNavigateToPage(page, tab)
    tab.Navigate(target_side_url)
    test.DidNavigateToPage(page, tab)

    # Wait for unpredictable redirects.
    if page.wait_time_after_navigate:
      time.sleep(page.wait_time_after_navigate)
    page.WaitToLoad(tab, 60)
    tab.WaitForDocumentReadyStateToBeInteractiveOrBetter()

    return True

  def _CleanUpPage(self, page, tab, page_state): # pylint: disable=R0201
    if page.credentials and page_state.did_login:
      tab.browser.credentials.LoginNoLongerNeeded(tab, page.credentials)
    try:
      tab.EvaluateJavaScript("""window.chrome && chrome.benchmarking &&
                              chrome.benchmarking.closeConnections()""")
    except Exception:
      pass

  def _WaitForThermalThrottlingIfNeeded(self, platform):
    if not platform.CanMonitorThermalThrottling():
      return
    thermal_throttling_retry = 0
    while (platform.IsThermallyThrottled() and
           thermal_throttling_retry < 3):
      logging.warning('Thermally throttled, waiting (%d)...',
                      thermal_throttling_retry)
      thermal_throttling_retry += 1
      time.sleep(thermal_throttling_retry * 2)

    if platform.IsThermallyThrottled():
      logging.error('Device is thermally throttled before running '
                    'performance tests, results will vary.')

  def _CheckThermalThrottling(self, platform):
    if not platform.CanMonitorThermalThrottling():
      return
    if platform.HasBeenThermallyThrottled():
      logging.error('Device has been thermally throttled during '
                    'performance tests, results will vary.')

  @staticmethod
  def AddCommandLineOptions(parser):
    page_filter_module.PageFilter.AddCommandLineOptions(parser)
<commit_msg>Telemetry: only whine about missing archives for pages which are really ran.<commit_after># Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import codecs
import logging
import os
import time
import traceback
import urlparse
import random

from telemetry.core import util
from telemetry.core import wpr_modes
from telemetry.core import exceptions
from telemetry.page import page_filter as page_filter_module
from telemetry.page import page_test

class PageState(object):
  def __init__(self):
    self.did_login = False

class _RunState(object):
  def __init__(self):
    self.first_browser = True
    self.browser = None
    self.tab = None
    self.is_tracing = False

  def Close(self):
    self.is_tracing = False

    if self.tab:
      self.tab.Disconnect()
      self.tab = None

    if self.browser:
      self.browser.Close()
      self.browser = None

def _ShuffleAndFilterPageSet(page_set, options):
  if options.pageset_shuffle_order_file and not options.pageset_shuffle:
    raise Exception('--pageset-shuffle-order-file requires --pageset-shuffle.')

  if options.pageset_shuffle_order_file:
    return page_set.ReorderPageSet(options.pageset_shuffle_order_file)

  page_filter = page_filter_module.PageFilter(options)
  pages = [page for page in page_set.pages[:]
           if not page.disabled and page_filter.IsSelected(page)]

  if options.pageset_shuffle:
    random.Random().shuffle(pages)
  return [page
      for _ in xrange(int(options.pageset_repeat))
      for page in pages
      for _ in xrange(int(options.page_repeat))]

class PageRunner(object):
  """Runs a given test against a given test."""
  def __init__(self, page_set):
    self.page_set = page_set

  def __enter__(self):
    return self

  def __exit__(self, *args):
    self.Close()

  def Run(self, options, possible_browser, test, results):
    # Reorder page set based on options.
    pages = _ShuffleAndFilterPageSet(self.page_set, options)

    # Check if we can run against WPR.
    for page in pages:
      parsed_url = urlparse.urlparse(page.url)
      if parsed_url.scheme == 'file':
        continue
      if not page.archive_path:
        logging.warning("""
  No page set archive provided for the page %s. Benchmarking against live sites!
  Results won't be repeatable or comparable.
""", page.url)
      elif options.wpr_mode != wpr_modes.WPR_RECORD:
        # The page has an archive, and we're not recording.
        if not os.path.isfile(page.archive_path):
          logging.warning("""
  The page set archive %s for page %s does not exist, benchmarking against live
  sites! Results won't be repeatable or comparable.

  To fix this, either add svn-internal to your .gclient using
  http://goto/read-src-internal, or create a new archive using record_wpr.
  """, os.path.relpath(page.archive_path), page.url)

    # Verify credentials path.
    credentials_path = None
    if self.page_set.credentials_path:
      credentials_path = os.path.join(os.path.dirname(self.page_set.file_path),
                                      self.page_set.credentials_path)
      if not os.path.exists(credentials_path):
        credentials_path = None

    # Set up user agent.
    if self.page_set.user_agent_type:
      options.browser_user_agent_type = self.page_set.user_agent_type

    for page in pages:
      test.CustomizeBrowserOptionsForPage(page, possible_browser.options)

    # Check tracing directory.
    if options.trace_dir:
      if not os.path.exists(options.trace_dir):
        os.mkdir(options.trace_dir)
      if not os.path.isdir(options.trace_dir):
        raise Exception('--trace-dir isn\'t a directory: %s' %
                        options.trace_dir)
      elif os.listdir(options.trace_dir):
        raise Exception('Trace directory isn\'t empty: %s' % options.trace_dir)

    state = _RunState()
    last_archive_path = None
    try:
      for page in pages:
        if options.wpr_mode != wpr_modes.WPR_RECORD:
          if page.archive_path and os.path.isfile(page.archive_path):
            possible_browser.options.wpr_mode = wpr_modes.WPR_REPLAY
          else:
            possible_browser.options.wpr_mode = wpr_modes.WPR_OFF
        if last_archive_path != page.archive_path:
          state.Close()
          state = _RunState()
          last_archive_path = page.archive_path
        tries = 3
        while tries:
          try:
            if not state.browser:
              self._SetupBrowser(state, test, possible_browser,
                                 credentials_path, page.archive_path)
            if not state.tab:
              if len(state.browser.tabs) == 0:
                state.browser.tabs.New()
              state.tab = state.browser.tabs[0]
            if options.trace_dir:
              self._SetupTracingTab(state)

            self._WaitForThermalThrottlingIfNeeded(state.browser.platform)

            try:
              self._RunPage(options, page, state.tab, test, results)
              self._CheckThermalThrottling(state.browser.platform)
            except exceptions.TabCrashException:
              stdout = ''
              if not options.show_stdout:
                stdout = state.browser.GetStandardOutput()
                stdout = (('\nStandard Output:\n') +
                          ('*' * 80) +
                          '\n\t' + stdout.replace('\n', '\n\t') + '\n' +
                          ('*' * 80))
              logging.warning('Tab crashed: %s%s', page.url, stdout)
              state.Close()

            if options.trace_dir:
              self._EndTracing(state, options, page)

            if test.needs_browser_restart_after_each_run:
              state.Close()

            break
          except exceptions.BrowserGoneException:
            logging.warning('Lost connection to browser. Retrying.')
            state.Close()
            tries -= 1
            if not tries:
              logging.error('Lost connection to browser 3 times. Failing.')
              raise
    finally:
      state.Close()

  def _RunPage(self, options, page, tab, test, results):
    if not test.CanRunForPage(page):
      logging.warning('Skiping test: it cannot run for %s', page.url)
      results.AddSkippedPage(page, 'Test cannot run', '')
      return

    logging.info('Running %s' % page.url)

    page_state = PageState()
    try:
      did_prepare = self._PreparePage(page, tab, page_state, test, results)
    except util.TimeoutException, ex:
      logging.warning('Timed out waiting for reply on %s. This is unusual.',
                      page.url)
      results.AddFailure(page, ex, traceback.format_exc())
      return
    except exceptions.TabCrashException, ex:
      results.AddFailure(page, ex, traceback.format_exc())
      raise
    except exceptions.BrowserGoneException:
      raise
    except Exception, ex:
      logging.error('Unexpected failure while running %s: %s',
                    page.url, traceback.format_exc())
      self._CleanUpPage(page, tab, page_state)
      raise

    if not did_prepare:
      self._CleanUpPage(page, tab, page_state)
      return

    try:
      test.Run(options, page, tab, results)
    except page_test.Failure, ex:
      logging.info('%s: %s', ex, page.url)
      results.AddFailure(page, ex, traceback.format_exc())
      return
    except util.TimeoutException, ex:
      logging.warning('Timed out while running %s', page.url)
      results.AddFailure(page, ex, traceback.format_exc())
      return
    except exceptions.TabCrashException, ex:
      results.AddFailure(page, ex, traceback.format_exc())
      raise
    except exceptions.BrowserGoneException:
      raise
    except Exception, ex:
      logging.error('Unexpected failure while running %s: %s',
                    page.url, traceback.format_exc())
      raise
    finally:
      self._CleanUpPage(page, tab, page_state)

    results.AddSuccess(page)

  def Close(self):
    pass

  def _SetupBrowser(self, state, test, possible_browser, credentials_path,
                    archive_path):
    assert not state.tab
    state.browser = possible_browser.Create()
    state.browser.credentials.credentials_path = credentials_path
    test.SetUpBrowser(state.browser)

    if state.first_browser:
      state.browser.credentials.WarnIfMissingCredentials(self.page_set)
      state.first_browser = False

    state.browser.SetReplayArchivePath(archive_path)

  def _SetupTracingTab(self, state):
    if state.browser.supports_tracing:
      state.is_tracing = True
      state.browser.StartTracing()

  def _EndTracing(self, state, options, page):
    if state.is_tracing:
      assert state.browser
      state.is_tracing = False
      state.browser.StopTracing()
      trace_result = state.browser.GetTraceResultAndReset()
      logging.info('Processing trace...')

      trace_file_base = os.path.join(
          options.trace_dir, page.url_as_file_safe_name)

      if options.page_repeat != 1 or options.pageset_repeat != 1:
        trace_file_index = 0

        while True:
          trace_file = '%s_%03d.json' % (trace_file_base, trace_file_index)
          if not os.path.exists(trace_file):
            break
          trace_file_index = trace_file_index + 1
      else:
        trace_file = '%s.json' % trace_file_base
      with codecs.open(trace_file, 'w',
                       encoding='utf-8') as trace_file:
        trace_result.Serialize(trace_file)
      logging.info('Trace saved.')

  def _PreparePage(self, page, tab, page_state, test, results):
    parsed_url = urlparse.urlparse(page.url)
    if parsed_url[0] == 'file':
      dirname, filename = page.url_base_dir_and_file
      tab.browser.SetHTTPServerDirectory(dirname)
      target_side_url = tab.browser.http_server.UrlOf(filename)
    else:
      target_side_url = page.url

    if page.credentials:
      page_state.did_login = tab.browser.credentials.LoginNeeded(
        tab, page.credentials)
      if not page_state.did_login:
        msg = 'Could not login to %s on %s' % (page.credentials,
                                               target_side_url)
        logging.info(msg)
        results.AddFailure(page, msg, "")
        return False

    test.WillNavigateToPage(page, tab)
    tab.Navigate(target_side_url)
    test.DidNavigateToPage(page, tab)

    # Wait for unpredictable redirects.
    if page.wait_time_after_navigate:
      time.sleep(page.wait_time_after_navigate)
    page.WaitToLoad(tab, 60)
    tab.WaitForDocumentReadyStateToBeInteractiveOrBetter()

    return True

  def _CleanUpPage(self, page, tab, page_state): # pylint: disable=R0201
    if page.credentials and page_state.did_login:
      tab.browser.credentials.LoginNoLongerNeeded(tab, page.credentials)
    try:
      tab.EvaluateJavaScript("""window.chrome && chrome.benchmarking &&
                              chrome.benchmarking.closeConnections()""")
    except Exception:
      pass

  def _WaitForThermalThrottlingIfNeeded(self, platform):
    if not platform.CanMonitorThermalThrottling():
      return
    thermal_throttling_retry = 0
    while (platform.IsThermallyThrottled() and
           thermal_throttling_retry < 3):
      logging.warning('Thermally throttled, waiting (%d)...',
                      thermal_throttling_retry)
      thermal_throttling_retry += 1
      time.sleep(thermal_throttling_retry * 2)

    if platform.IsThermallyThrottled():
      logging.error('Device is thermally throttled before running '
                    'performance tests, results will vary.')

  def _CheckThermalThrottling(self, platform):
    if not platform.CanMonitorThermalThrottling():
      return
    if platform.HasBeenThermallyThrottled():
      logging.error('Device has been thermally throttled during '
                    'performance tests, results will vary.')

  @staticmethod
  def AddCommandLineOptions(parser):
    page_filter_module.PageFilter.AddCommandLineOptions(parser)
<|endoftext|>


<commit_before>""" Routines to copy / relink library dependencies in trees and wheels
"""

from __future__ import division, print_function

import os
from os.path import (join as pjoin, dirname, basename, exists, isdir, abspath,
                     relpath)
import shutil
import warnings
import hashlib
import csv
import glob

from wheel.util import urlsafe_b64encode, open_for_csv, native

from .libsana import tree_libs
from .tools import (add_rpath, set_install_name, zip2dir, dir2zip,
                    find_package_dirs)
from .tmpdirs import InTemporaryDirectory, InGivenDirectory

class DelocationError(Exception):
    pass

def delocate_tree_libs(lib_dict, lib_path, root_path):
    """ Move needed libraries in `lib_dict` into `lib_path`

    `lib_dict` has keys naming libraries required by the files in the
    corresponding value.  Call the keys, "required libs".  Call the values
    "requiring objects".

    Copy all the required libs to `lib_path`.  Fix up the rpaths and install
    names in the requiring objects to point to these new copies.

    Exception: required libs within the directory tree pointed to by
    `root_path` stay where they are, but we modify requiring objects to use
    relative paths to these libraries.

    Parameters
    ----------
    lib_dict : dict
        Dictionary with (key, value) pairs of (required library, set of files
        in tree depending on that library)
    lib_path : str
        Path in which to store copies of libs referred to in keys of
        `lib_dict`.  Assumed to exist
    root_path : str, optional
        Root directory of tree analyzed in `lib_dict`.  Any required
        library within the subtrees of `root_path` does not get copied, but
        libraries linking to it have links adjusted to use relative path to
        this library.

    Returns
    -------
    copied_libs : dict
        Dict with (key, value) pairs of (name of library copied into
        `lib_path`, sequence of libraries depending on copied library). The
        names are the original names from the keys of `lib_dict`
    """
    copied_libs = {}
    delocated_libs = set()
    copied_basenames = set()
    # Test for errors first to avoid getting half-way through changing the tree
    for required, requirings in lib_dict.items():
        if required.startswith('@'): # assume @rpath etc are correct
            # But warn, because likely they are not
            warnings.warn('Not processing required path ' + required)
            continue
        r_ed_base = basename(required)
        if relpath(required, root_path).startswith('..'):
            # Not local, plan to copy
            if r_ed_base in copied_basenames:
                raise DelocationError('Already planning to copy library with '
                                      'same basename as: ' + r_ed_base)
            if not exists(required):
                raise DelocationError('library "{0}" does not exist'.format(
                    required))
            copied_libs[required] = requirings
            copied_basenames.add(r_ed_base)
        else: # Is local, plan to set relative loader_path
            delocated_libs.add(required)
    # Modify in place now that we've checked for errors
    for required in copied_libs:
        shutil.copy2(required, lib_path)
        # Set rpath and install names for this copied library
        for requiring in lib_dict[required]:
            req_rel = relpath(lib_path, dirname(requiring))
            set_install_name(requiring, required,
                             '@loader_path/{0}/{1}'.format(
                                 req_rel, basename(required)))
    for required in delocated_libs:
        # Set relative path for local library
        for requiring in lib_dict[required]:
            req_rel = relpath(required, dirname(requiring))
            set_install_name(requiring, required,
                             '@loader_path/' + req_rel)
    return copied_libs

def copy_recurse(lib_path, copy_filt_func = None, copied_libs = None):
    """ Analyze `lib_path` for library dependencies and copy libraries

    `lib_path` is a directory containing libraries.  The libraries might
    themselves have dependencies.  This function analyzes the dependencies and
    copies library dependencies that match the filter `copy_filt_func`. It alos
    adjusts the depending libraries to use the copy. It keeps iterating over
    `lib_path` until all matching dependencies (of dependencies of
    dependencies ...) have been copied.

    Parameters
    ----------
    lib_path : str
        Directory containing libraries
    copy_filt_func : None or callable, optional
        If None, copy any library that found libraries depend on.  If callable,
        called on each depended library name; copy where
        ``copy_filt_func(libname)`` is True, don't copy otherwise
    copied_libs : dict
        Dict with (key, value) pairs of (name of library copied into
        `lib_path`, sequence of libraries depending on copied library). The
        names are the original names from the keys of `lib_dict`

    Returns
    -------
    copied_libs : dict
        Input `copied_libs` dict with any extra libraries and / or dependencies
        added.
    """
    if copied_libs is None:
        copied_libs = {}
    else:
        copied_libs = dict(copied_libs)
    done = False
    while not done:
        in_len = len(copied_libs)
        _copy_required(lib_path, copy_filt_func, copied_libs)
        done = len(copied_libs) == in_len
    return copied_libs

def _copy_required(lib_path, copy_filt_func, copied_libs):
    """ Copy libraries required for files in `lib_path` to `lib_path`

    Augment `copied_libs` dictionary with any newly copied libraries, modifying
    `copied_libs` in-place.

    This is one pass of ``copy_recurse``

    Parameters
    ----------
    lib_path : str
        Directory containing libraries
    copy_filt_func : None or callable, optional
        If None, copy any library that found libraries depend on.  If callable,
        called on each library name; copy where ``copy_filt_func(libname)`` is
        True, don't copy otherwise
    copied_libs : dict
        Dict with (key, value) pairs of (name of library copied into
        `lib_path`, sequence of libraries depending on copied library). The
        names are the original names from the keys of `lib_dict`
    """
    # Paths will be prepended with `lib_path`
    lib_dict = tree_libs(lib_path)
    # Map library paths after copy to path before copy
    copied2orig = dict((pjoin(lib_path, basename(c)), c) for c in copied_libs)
    for required, requirings in lib_dict.items():
        if not copy_filt_func is None and not copy_filt_func(required):
            continue
        if required.startswith('@'):
            # May have been processed by us, or have some rpath, loader_path of
            # its own. Either way, leave alone
            continue
        required_base = basename(required)
        # Will we need to make a local copy or do we have this one already?
        needs_copy = required not in copied_libs
        # Set requiring lib install names to point to local copy
        for requiring in requirings:
            set_install_name(requiring,
                             required,
                             '@loader_path/' + required_base)
        # Make requiring names point to original libraries
        procd_requirings = set(copied2orig.get(r, r) for r in requirings)
        if needs_copy: # Haven't see this one before, add entry to copied_libs
            shutil.copy2(required, lib_path)
            copied2orig[pjoin(lib_path, required_base)] = required
            copied_libs[required] = procd_requirings
        else: # Have seen this before, add any new requirements
            copied_libs[required] = copied_libs[required] | procd_requirings

def _dylibs_only(filename):
    return (filename.endswith('.so') or
            filename.endswith('.dylib'))

def filter_system_libs(libname):
    return not (libname.startswith('/usr/lib') or
                libname.startswith('/System'))

def delocate_path(tree_path, lib_path,
                  lib_filt_func = _dylibs_only,
                  copy_filt_func = filter_system_libs):
    """ Copy required libraries for files in `tree_path` into `lib_path`

    Parameters
    ----------
    tree_path : str
        Root path of tree to search for required libraries
    lib_path : str
        Directory into which we copy required libraries
    lib_filt_func : None or callable, optional
        If None, inspect all files for dependencies on dynamic libraries. If
        callable, accepts filename as argument, returns True if we should
        inspect the file, False otherwise. Default is callable rejecting all
        but files ending in ``.so`` or ``.dylib``.
    copy_filt_func : None or callable, optional
        If callable, called on each library name detected as a dependency; copy
        where ``copy_filt_func(libname)`` is True, don't copy otherwise.
        Default is callable rejecting only libraries beginning with
        ``/usr/lib`` or ``/System``.  None means copy all libraries. This will
        usually end up copying large parts of the system run-time.

    Returns
    -------
    copied_libs : dict
        Dict with (key, value) pairs of (name of library copied into
        `lib_path`, sequence of libraries depending on copied library). The
        names are the original names from the keys of `lib_dict`
    """
    if not exists(lib_path):
        os.makedirs(lib_path)
    lib_dict = tree_libs(tree_path, lib_filt_func)
    if not copy_filt_func is None:
        lib_dict = dict((key, value) for key, value in lib_dict.items()
                        if copy_filt_func(key))
    copied = delocate_tree_libs(lib_dict, lib_path, tree_path)
    return copy_recurse(lib_path, copy_filt_func, copied)

def rewrite_record(bdist_dir):
    """ Rewrite RECORD file with hashes for all files in `wheel_sdir`

    Copied from :method:`wheel.bdist_wheel.bdist_wheel.write_record`

    Will also unsign wheel

    Parameters
    ----------
    bdist_dir : str
        Path of unpacked wheel file
    """
    info_dirs = glob.glob(pjoin(bdist_dir, '*.dist-info'))
    if len(info_dirs) != 1:
        raise DelocationError("Should be exactly one `*.dist_info` directory")
    record_path = os.path.join(info_dirs[0], 'RECORD')
    record_relpath = os.path.relpath(record_path, bdist_dir)
    # Unsign wheel - because we're invalidating the record hash
    sig_path = os.path.join(info_dirs[0], 'RECORD.jws')
    if exists(sig_path):
        os.unlink(sig_path)

    def walk():
        for dir, dirs, files in os.walk(bdist_dir):
            for f in files:
                yield os.path.join(dir, f)

    def skip(path):
        """Wheel hashes every possible file."""
        return (path == record_relpath)

    with open_for_csv(record_path, 'w+') as record_file:
        writer = csv.writer(record_file)
        for path in walk():
            relpath = os.path.relpath(path, bdist_dir)
            if skip(relpath):
                hash = ''
                size = ''
            else:
                with open(path, 'rb') as f:
                    data = f.read()
                digest = hashlib.sha256(data).digest()
                hash = 'sha256=' + native(urlsafe_b64encode(digest))
                size = len(data)
            record_path = os.path.relpath(
                path, bdist_dir).replace(os.path.sep, '/')
            writer.writerow((record_path, hash, size))

def _merge_lib_dict(d1, d2):
    for required, requirings in d2.items():
        if required in d1:
            d1[required] = d1[required] | requirings
        else:
            d1[required] = requirings
    return

def delocate_wheel(in_wheel,
                   out_wheel = None,
                   lib_sdir = '.dylibs',
                   lib_filt_func = _dylibs_only,
                   copy_filt_func = filter_system_libs):
    """ Update wheel by copying required libraries to `lib_sdir` in wheel

    Create `lib_sdir` in wheel tree only if we are copying one or more
    libraries.

    Overwrite the wheel `wheel_fname` in-place.

    Parameters
    ----------
    in_wheel : str
        Filename of wheel to process
    out_wheel : None or str
        Filename of processed wheel to write.  If None, overwrite `in_wheel`
    lib_sdir : str, optional
        Subdirectory name in wheel package directory (or directories) to store
        needed libraries.
    lib_filt_func : None or callable, optional
        If None, inspect all files for dependencies on dynamic libraries. If
        callable, accepts filename as argument, returns True if we should
        inspect the file, False otherwise. Default is callable rejecting all
        but files ending in ``.so`` or ``.dylib``.
    copy_filt_func : None or callable, optional
        If callable, called on each library name detected as a dependency; copy
        where ``copy_filt_func(libname)`` is True, don't copy otherwise.
        Default is callable rejecting only libraries beginning with
        ``/usr/lib`` or ``/System``.  None means copy all libraries. This will
        usually end up copying large parts of the system run-time.

    Returns
    -------
    copied_libs : dict
        Dict with (key, value) pairs of (name of library copied into
        `lib_path`, sequence of libraries depending on copied library). The
        names are the original names from the keys of `lib_dict`
    """
    in_wheel = abspath(in_wheel)
    if out_wheel is None:
        out_wheel = in_wheel
    else:
        out_wheel = abspath(out_wheel)
    in_place = in_wheel == out_wheel
    with InTemporaryDirectory():
        all_copied = {}
        zip2dir(in_wheel, 'wheel')
        with InGivenDirectory('wheel'):
            for package_path in find_package_dirs('.'):
                lib_path = pjoin(package_path, lib_sdir)
                if exists(lib_path):
                    raise DelocationError(
                        '{0} already exists in wheel'.format(lib_path))
                copied_libs = delocate_path(package_path, lib_path,
                                            lib_filt_func, copy_filt_func)
                if len(os.listdir(lib_path)) == 0:
                    shutil.rmtree(lib_path)
                _merge_lib_dict(all_copied, copied_libs)
        if len(all_copied):
            rewrite_record('wheel')
        if len(all_copied) or not in_place:
            dir2zip('wheel', out_wheel)
    return all_copied
<commit_msg>don't raise on already-delocated wheel<commit_after>""" Routines to copy / relink library dependencies in trees and wheels
"""

from __future__ import division, print_function

import os
from os.path import (join as pjoin, dirname, basename, exists, isdir, abspath,
                     relpath)
import shutil
import warnings
import hashlib
import csv
import glob

from wheel.util import urlsafe_b64encode, open_for_csv, native

from .libsana import tree_libs
from .tools import (add_rpath, set_install_name, zip2dir, dir2zip,
                    find_package_dirs)
from .tmpdirs import InTemporaryDirectory, InGivenDirectory

class DelocationError(Exception):
    pass

def delocate_tree_libs(lib_dict, lib_path, root_path):
    """ Move needed libraries in `lib_dict` into `lib_path`

    `lib_dict` has keys naming libraries required by the files in the
    corresponding value.  Call the keys, "required libs".  Call the values
    "requiring objects".

    Copy all the required libs to `lib_path`.  Fix up the rpaths and install
    names in the requiring objects to point to these new copies.

    Exception: required libs within the directory tree pointed to by
    `root_path` stay where they are, but we modify requiring objects to use
    relative paths to these libraries.

    Parameters
    ----------
    lib_dict : dict
        Dictionary with (key, value) pairs of (required library, set of files
        in tree depending on that library)
    lib_path : str
        Path in which to store copies of libs referred to in keys of
        `lib_dict`.  Assumed to exist
    root_path : str, optional
        Root directory of tree analyzed in `lib_dict`.  Any required
        library within the subtrees of `root_path` does not get copied, but
        libraries linking to it have links adjusted to use relative path to
        this library.

    Returns
    -------
    copied_libs : dict
        Dict with (key, value) pairs of (name of library copied into
        `lib_path`, sequence of libraries depending on copied library). The
        names are the original names from the keys of `lib_dict`
    """
    copied_libs = {}
    delocated_libs = set()
    copied_basenames = set()
    # Test for errors first to avoid getting half-way through changing the tree
    for required, requirings in lib_dict.items():
        if required.startswith('@'): # assume @rpath etc are correct
            # But warn, because likely they are not
            warnings.warn('Not processing required path ' + required)
            continue
        r_ed_base = basename(required)
        if relpath(required, root_path).startswith('..'):
            # Not local, plan to copy
            if r_ed_base in copied_basenames:
                raise DelocationError('Already planning to copy library with '
                                      'same basename as: ' + r_ed_base)
            if not exists(required):
                raise DelocationError('library "{0}" does not exist'.format(
                    required))
            copied_libs[required] = requirings
            copied_basenames.add(r_ed_base)
        else: # Is local, plan to set relative loader_path
            delocated_libs.add(required)
    # Modify in place now that we've checked for errors
    for required in copied_libs:
        shutil.copy2(required, lib_path)
        # Set rpath and install names for this copied library
        for requiring in lib_dict[required]:
            req_rel = relpath(lib_path, dirname(requiring))
            set_install_name(requiring, required,
                             '@loader_path/{0}/{1}'.format(
                                 req_rel, basename(required)))
    for required in delocated_libs:
        # Set relative path for local library
        for requiring in lib_dict[required]:
            req_rel = relpath(required, dirname(requiring))
            set_install_name(requiring, required,
                             '@loader_path/' + req_rel)
    return copied_libs

def copy_recurse(lib_path, copy_filt_func = None, copied_libs = None):
    """ Analyze `lib_path` for library dependencies and copy libraries

    `lib_path` is a directory containing libraries.  The libraries might
    themselves have dependencies.  This function analyzes the dependencies and
    copies library dependencies that match the filter `copy_filt_func`. It alos
    adjusts the depending libraries to use the copy. It keeps iterating over
    `lib_path` until all matching dependencies (of dependencies of
    dependencies ...) have been copied.

    Parameters
    ----------
    lib_path : str
        Directory containing libraries
    copy_filt_func : None or callable, optional
        If None, copy any library that found libraries depend on.  If callable,
        called on each depended library name; copy where
        ``copy_filt_func(libname)`` is True, don't copy otherwise
    copied_libs : dict
        Dict with (key, value) pairs of (name of library copied into
        `lib_path`, sequence of libraries depending on copied library). The
        names are the original names from the keys of `lib_dict`

    Returns
    -------
    copied_libs : dict
        Input `copied_libs` dict with any extra libraries and / or dependencies
        added.
    """
    if copied_libs is None:
        copied_libs = {}
    else:
        copied_libs = dict(copied_libs)
    done = False
    while not done:
        in_len = len(copied_libs)
        _copy_required(lib_path, copy_filt_func, copied_libs)
        done = len(copied_libs) == in_len
    return copied_libs

def _copy_required(lib_path, copy_filt_func, copied_libs):
    """ Copy libraries required for files in `lib_path` to `lib_path`

    Augment `copied_libs` dictionary with any newly copied libraries, modifying
    `copied_libs` in-place.

    This is one pass of ``copy_recurse``

    Parameters
    ----------
    lib_path : str
        Directory containing libraries
    copy_filt_func : None or callable, optional
        If None, copy any library that found libraries depend on.  If callable,
        called on each library name; copy where ``copy_filt_func(libname)`` is
        True, don't copy otherwise
    copied_libs : dict
        Dict with (key, value) pairs of (name of library copied into
        `lib_path`, sequence of libraries depending on copied library). The
        names are the original names from the keys of `lib_dict`
    """
    # Paths will be prepended with `lib_path`
    lib_dict = tree_libs(lib_path)
    # Map library paths after copy to path before copy
    copied2orig = dict((pjoin(lib_path, basename(c)), c) for c in copied_libs)
    for required, requirings in lib_dict.items():
        if not copy_filt_func is None and not copy_filt_func(required):
            continue
        if required.startswith('@'):
            # May have been processed by us, or have some rpath, loader_path of
            # its own. Either way, leave alone
            continue
        required_base = basename(required)
        # Will we need to make a local copy or do we have this one already?
        needs_copy = required not in copied_libs
        # Set requiring lib install names to point to local copy
        for requiring in requirings:
            set_install_name(requiring,
                             required,
                             '@loader_path/' + required_base)
        # Make requiring names point to original libraries
        procd_requirings = set(copied2orig.get(r, r) for r in requirings)
        if needs_copy: # Haven't see this one before, add entry to copied_libs
            shutil.copy2(required, lib_path)
            copied2orig[pjoin(lib_path, required_base)] = required
            copied_libs[required] = procd_requirings
        else: # Have seen this before, add any new requirements
            copied_libs[required] = copied_libs[required] | procd_requirings

def _dylibs_only(filename):
    return (filename.endswith('.so') or
            filename.endswith('.dylib'))

def filter_system_libs(libname):
    return not (libname.startswith('/usr/lib') or
                libname.startswith('/System'))

def delocate_path(tree_path, lib_path,
                  lib_filt_func = _dylibs_only,
                  copy_filt_func = filter_system_libs):
    """ Copy required libraries for files in `tree_path` into `lib_path`

    Parameters
    ----------
    tree_path : str
        Root path of tree to search for required libraries
    lib_path : str
        Directory into which we copy required libraries
    lib_filt_func : None or callable, optional
        If None, inspect all files for dependencies on dynamic libraries. If
        callable, accepts filename as argument, returns True if we should
        inspect the file, False otherwise. Default is callable rejecting all
        but files ending in ``.so`` or ``.dylib``.
    copy_filt_func : None or callable, optional
        If callable, called on each library name detected as a dependency; copy
        where ``copy_filt_func(libname)`` is True, don't copy otherwise.
        Default is callable rejecting only libraries beginning with
        ``/usr/lib`` or ``/System``.  None means copy all libraries. This will
        usually end up copying large parts of the system run-time.

    Returns
    -------
    copied_libs : dict
        Dict with (key, value) pairs of (name of library copied into
        `lib_path`, sequence of libraries depending on copied library). The
        names are the original names from the keys of `lib_dict`
    """
    if not exists(lib_path):
        os.makedirs(lib_path)
    lib_dict = tree_libs(tree_path, lib_filt_func)
    if not copy_filt_func is None:
        lib_dict = dict((key, value) for key, value in lib_dict.items()
                        if copy_filt_func(key))
    copied = delocate_tree_libs(lib_dict, lib_path, tree_path)
    return copy_recurse(lib_path, copy_filt_func, copied)

def rewrite_record(bdist_dir):
    """ Rewrite RECORD file with hashes for all files in `wheel_sdir`

    Copied from :method:`wheel.bdist_wheel.bdist_wheel.write_record`

    Will also unsign wheel

    Parameters
    ----------
    bdist_dir : str
        Path of unpacked wheel file
    """
    info_dirs = glob.glob(pjoin(bdist_dir, '*.dist-info'))
    if len(info_dirs) != 1:
        raise DelocationError("Should be exactly one `*.dist_info` directory")
    record_path = os.path.join(info_dirs[0], 'RECORD')
    record_relpath = os.path.relpath(record_path, bdist_dir)
    # Unsign wheel - because we're invalidating the record hash
    sig_path = os.path.join(info_dirs[0], 'RECORD.jws')
    if exists(sig_path):
        os.unlink(sig_path)

    def walk():
        for dir, dirs, files in os.walk(bdist_dir):
            for f in files:
                yield os.path.join(dir, f)

    def skip(path):
        """Wheel hashes every possible file."""
        return (path == record_relpath)

    with open_for_csv(record_path, 'w+') as record_file:
        writer = csv.writer(record_file)
        for path in walk():
            relpath = os.path.relpath(path, bdist_dir)
            if skip(relpath):
                hash = ''
                size = ''
            else:
                with open(path, 'rb') as f:
                    data = f.read()
                digest = hashlib.sha256(data).digest()
                hash = 'sha256=' + native(urlsafe_b64encode(digest))
                size = len(data)
            record_path = os.path.relpath(
                path, bdist_dir).replace(os.path.sep, '/')
            writer.writerow((record_path, hash, size))

def _merge_lib_dict(d1, d2):
    for required, requirings in d2.items():
        if required in d1:
            d1[required] = d1[required] | requirings
        else:
            d1[required] = requirings
    return

def delocate_wheel(in_wheel,
                   out_wheel = None,
                   lib_sdir = '.dylibs',
                   lib_filt_func = _dylibs_only,
                   copy_filt_func = filter_system_libs):
    """ Update wheel by copying required libraries to `lib_sdir` in wheel

    Create `lib_sdir` in wheel tree only if we are copying one or more
    libraries.

    Overwrite the wheel `wheel_fname` in-place.

    Parameters
    ----------
    in_wheel : str
        Filename of wheel to process
    out_wheel : None or str
        Filename of processed wheel to write.  If None, overwrite `in_wheel`
    lib_sdir : str, optional
        Subdirectory name in wheel package directory (or directories) to store
        needed libraries.
    lib_filt_func : None or callable, optional
        If None, inspect all files for dependencies on dynamic libraries. If
        callable, accepts filename as argument, returns True if we should
        inspect the file, False otherwise. Default is callable rejecting all
        but files ending in ``.so`` or ``.dylib``.
    copy_filt_func : None or callable, optional
        If callable, called on each library name detected as a dependency; copy
        where ``copy_filt_func(libname)`` is True, don't copy otherwise.
        Default is callable rejecting only libraries beginning with
        ``/usr/lib`` or ``/System``.  None means copy all libraries. This will
        usually end up copying large parts of the system run-time.

    Returns
    -------
    copied_libs : dict
        Dict with (key, value) pairs of (name of library copied into
        `lib_path`, sequence of libraries depending on copied library). The
        names are the original names from the keys of `lib_dict`
    """
    in_wheel = abspath(in_wheel)
    if out_wheel is None:
        out_wheel = in_wheel
    else:
        out_wheel = abspath(out_wheel)
    in_place = in_wheel == out_wheel
    with InTemporaryDirectory():
        all_copied = {}
        zip2dir(in_wheel, 'wheel')
        with InGivenDirectory('wheel'):
            for package_path in find_package_dirs('.'):
                lib_path = pjoin(package_path, lib_sdir)
                lib_path_exists = exists(lib_path)
                copied_libs = delocate_path(package_path, lib_path,
                                            lib_filt_func, copy_filt_func)
                if copied_libs and lib_path_exists:
                    raise DelocationError(
                        '{0} already exists in wheel'.format(lib_path))
                if len(os.listdir(lib_path)) == 0:
                    shutil.rmtree(lib_path)
                _merge_lib_dict(all_copied, copied_libs)
        if len(all_copied):
            rewrite_record('wheel')
        if len(all_copied) or not in_place:
            dir2zip('wheel', out_wheel)
    return all_copied
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
    pygments.lexers.praat
    ~~~~~~~~~~~~~~~~~~~~

    Lexer for Praat

    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import re

from pygments.lexer import RegexLexer, words, bygroups, default, include
from pygments.token import *

__all__ = ['PraatLexer']

class PraatLexer(RegexLexer):
    """
    For `Praat <http://www.praat.org>`_ scripts.
    """

    name = 'Praat'
    aliases = ['praat']
    filenames = ['*.praat', '*.proc', '*.psc']

    keywords = [
        'if', 'then', 'else', 'elsif', 'elif', 'endif', 'fi', 'for', 'from', 'to', 'endfor', 'endproc',
        'while', 'endwhile', 'repeat', 'until', 'select', 'plus', 'minus', 'demo', 'assert', 'stopwatch',
        'nocheck', 'nowarn', 'noprogress', 'editor', 'endeditor', 'clearinfo'
    ]

    functions_string = [
        'backslashTrigraphsToUnicode', 'chooseDirectory', 'chooseReadFile',
        'chooseWriteFile', 'date', 'demoKey', 'do', 'environment', 'extractLine', 'extractWord',
        'fixed', 'info', 'left', 'mid', 'percent', 'readFile', 'replace', 'replace_regex', 'right',
        'selected', 'string', 'unicodeToBackslashTrigraphs',
    ]

    functions_numeric = [
        'abs', 'appendFile', 'appendFileLine', 'appendInfo', 'appendInfoLine', 'arccos', 'arccosh',
        'arcsin', 'arcsinh', 'arctan', 'arctan2', 'arctanh', 'barkToHertz', 'beginPause',
        'beginSendPraat', 'besselI', 'besselK', 'beta', 'beta2', 'binomialP', 'binomialQ', 'boolean',
        'ceiling', 'chiSquareP', 'chiSquareQ', 'choice', 'comment', 'cos', 'cosh', 'createDirectory',
        'deleteFile', 'demoClicked', 'demoClickedIn', 'demoCommandKeyPressed',
        'demoExtraControlKeyPressed', 'demoInput', 'demoKeyPressed',
        'demoOptionKeyPressed', 'demoShiftKeyPressed', 'demoShow', 'demoWaitForInput',
        'demoWindowTitle', 'demoX', 'demoY', 'differenceLimensToPhon', 'do', 'editor', 'endPause',
        'endSendPraat', 'endsWith', 'erb', 'erbToHertz', 'erf', 'erfc', 'exitScript', 'exp',
        'extractNumber', 'fileReadable', 'fisherP', 'fisherQ', 'floor', 'gaussP', 'gaussQ',
        'hertzToBark', 'hertzToErb', 'hertzToMel', 'hertzToSemitones', 'imax', 'imin',
        'incompleteBeta', 'incompleteGammaP', 'index', 'index_regex', 'invBinomialP',
        'invBinomialQ', 'invChiSquareQ', 'invFisherQ', 'invGaussQ', 'invSigmoid', 'invStudentQ',
        'length', 'ln', 'lnBeta', 'lnGamma', 'log10', 'log2', 'max', 'melToHertz', 'min', 'minusObject',
        'natural', 'number', 'numberOfColumns', 'numberOfRows', 'numberOfSelected',
        'objectsAreIdentical', 'option', 'optionMenu', 'pauseScript',
        'phonToDifferenceLimens', 'plusObject', 'positive', 'randomBinomial', 'randomGauss',
        'randomInteger', 'randomPoisson', 'randomUniform', 'real', 'readFile', 'removeObject',
        'rindex', 'rindex_regex', 'round', 'runScript', 'runSystem', 'runSystem_nocheck',
        'selectObject', 'selected', 'semitonesToHertz', 'sentencetext', 'sigmoid', 'sin', 'sinc',
        'sincpi', 'sinh', 'soundPressureToPhon', 'sqrt', 'startsWith', 'studentP', 'studentQ', 'tan',
        'tanh', 'variableExists', 'word', 'writeFile', 'writeFileLine', 'writeInfo',
        'writeInfoLine',
    ]

    functions_array = [
        'linear', 'randomGauss', 'randomInteger', 'randomUniform', 'zero',
    ]

    objects = [
        'Activation', 'AffineTransform', 'AmplitudeTier', 'Art', 'Artword', 'Autosegment',
        'BarkFilter', 'BarkSpectrogram', 'CCA', 'Categories', 'Cepstrogram', 'Cepstrum',
        'Cepstrumc', 'ChebyshevSeries', 'ClassificationTable', 'Cochleagram', 'Collection',
        'ComplexSpectrogram', 'Configuration', 'Confusion', 'ContingencyTable', 'Corpus',
        'Correlation', 'Covariance', 'CrossCorrelationTable', 'CrossCorrelationTables', 'DTW',
        'DataModeler', 'Diagonalizer', 'Discriminant', 'Dissimilarity', 'Distance',
        'Distributions', 'DurationTier', 'EEG', 'ERP', 'ERPTier', 'EditCostsTable',
        'EditDistanceTable', 'Eigen', 'Excitation', 'Excitations', 'ExperimentMFC', 'FFNet',
        'FeatureWeights', 'FileInMemory', 'FilesInMemory', 'Formant', 'FormantFilter',
        'FormantGrid', 'FormantModeler', 'FormantPoint', 'FormantTier', 'GaussianMixture', 'HMM',
        'HMM_Observation', 'HMM_ObservationSequence', 'HMM_State', 'HMM_StateSequence',
        'Harmonicity', 'ISpline', 'Index', 'Intensity', 'IntensityTier', 'IntervalTier', 'KNN',
        'KlattGrid', 'KlattTable', 'LFCC', 'LPC', 'Label', 'LegendreSeries', 'LinearRegression',
        'LogisticRegression', 'LongSound', 'Ltas', 'MFCC', 'MSpline', 'ManPages', 'Manipulation',
        'Matrix', 'MelFilter', 'MelSpectrogram', 'MixingMatrix', 'Movie', 'Network', 'OTGrammar',
        'OTHistory', 'OTMulti', 'PCA', 'PairDistribution', 'ParamCurve', 'Pattern', 'Permutation',
        'Photo', 'Pitch', 'PitchModeler', 'PitchTier', 'PointProcess', 'Polygon', 'Polynomial',
        'PowerCepstrogram', 'PowerCepstrum', 'Procrustes', 'RealPoint', 'RealTier', 'ResultsMFC',
        'Roots', 'SPINET', 'SSCP', 'SVD', 'Salience', 'ScalarProduct', 'Similarity', 'SimpleString',
        'SortedSetOfString', 'Sound', 'Speaker', 'Spectrogram', 'Spectrum', 'SpectrumTier',
        'SpeechSynthesizer', 'SpellingChecker', 'Strings', 'StringsIndex', 'Table',
        'TableOfReal', 'TextGrid', 'TextInterval', 'TextPoint', 'TextTier', 'Tier', 'Transition',
        'VocalTract', 'VocalTractTier', 'Weight', 'WordList',
    ]

    variables_numeric = [
        'macintosh', 'windows', 'unix', 'praatVersion', 'pi', 'e', 'undefined',
    ]

    variables_string = [
        'praatVersion', 'tab', 'shellDirectory', 'homeDirectory',
        'preferencesDirectory', 'newline', 'temporaryDirectory',
        'defaultDirectory',
    ]

    tokens = {
        'root': [
            (r'(\s+)(#.*?$)',  bygroups(Text, Comment.Single)),
            (r'^#.*?$',        Comment.Single),
            (r';[^\n]*',       Comment.Single),
            (r'\s+',           Text),

            (r'\bprocedure\b', Keyword,       'procedure_definition'),
            (r'\bcall\b',      Keyword,       'procedure_call'),
            (r'@',             Name.Function, 'procedure_call'),

            include('function_call'),

            (words(keywords, suffix=r'\b'), Keyword),

            (r'(\bform\b)(\s+)([^\n]+)',
                bygroups(Keyword, Text, String), 'old_form'),

            (r'(print(?:line|tab)?|echo|exit|asserterror|pause|send(?:praat|socket)|include|execute|system(?:_nocheck)?)(\s+)',
                bygroups(Keyword, Text), 'string_unquoted'),

            (r'(goto|label)(\s+)(\w+)', bygroups(Keyword, Text, Name.Label)),

            include('variable_name'),
            include('number'),

            (r'"', String, 'string'),

            (words((objects), suffix=r'(?=\s+\S+\n)'), Name.Class, 'string_unquoted'),

            (r'\b[A-Z]', Keyword, 'command'),
            (r'(\.{3}|[)(,])', Punctuation),
        ],
        'command': [
            (r'( ?[\w()-]+ ?)', Keyword),
            (r"'(?=.*')", String.Interpol, 'string_interpolated'),
            (r'\.{3}', Keyword, ('#pop', 'old_arguments')),
            (r':', Keyword, ('#pop', 'comma_list')),
            (r'[\s\n]', Text, '#pop'),
        ],
        'procedure_call': [
            (r'\s+', Text),
            (r'([\w.]+)(:|\s*\()',
                bygroups(Name.Function, Text), '#pop'),
            (r'([\w.]+)', Name.Function, ('#pop', 'old_arguments')),
        ],
        'procedure_definition': [
            (r'\s', Text),
            (r'([\w.]+)(\s*?[(:])',
                bygroups(Name.Function, Text), '#pop'),
            (r'([\w.]+)([^\n]*)',
                bygroups(Name.Function, Text), '#pop'),
        ],
        'function_call': [
            (words(functions_string , suffix=r'\$(?=\s*[:(])'), Name.Function, 'function'),
            (words(functions_array  , suffix=r'#(?=\s*[:(])'),  Name.Function, 'function'),
            (words(functions_numeric, suffix=r'(?=\s*[:(])'),   Name.Function, 'function'),
        ],
        'function': [
            (r'\s+',   Text),
            (r':',     Punctuation, ('#pop', 'comma_list')),
            (r'\s*\(', Punctuation, ('#pop', 'comma_list')),
        ],
        'comma_list': [
            (r'(\s*\n\s*)(\.{3})', bygroups(Text, Punctuation)),

            (r'(\s*[])\n])', Text, '#pop'),

            (r'\s+', Text),
            (r'"',   String, 'string'),
            (r'\b(if|then|else|fi|endif)\b', Keyword),

            include('function_call'),
            include('variable_name'),
            include('operator'),
            include('number'),

            (r',', Punctuation),
        ],
        'old_arguments': [
            (r'\n', Text, '#pop'),

            include('variable_name'),
            include('operator'),
            include('number'),

            (r'"', String, 'string'),
            (r'[^\n]', Text),
        ],
        'number': [
            (r'\b\d+(\.\d*)?([eE][-+]?\d+)?%?', Number),
        ],
        'object_attributes': [
            (r'\.?(n(col|row)|[xy]min|[xy]max|[nd][xy])\b', Name.Builtin, '#pop'),
            (r'(\.?(?:col|row)\$)(\[)',
                bygroups(Name.Builtin, Text), 'variable_name'),
            (r'(\$?)(\[)',
                bygroups(Name.Builtin, Text), ('#pop', 'comma_list')),
        ],
        'variable_name': [
            include('operator'),
            include('number'),

            (words(variables_string,  suffix=r'\$'), Name.Variable.Global),
            (words(variables_numeric, suffix=r'\b'), Name.Variable.Global),

            (r'\bObject_\w+', Name.Builtin, 'object_attributes'),
            (words(objects, prefix=r'\b', suffix=r'_\w+'), Name.Builtin, 'object_attributes'),

            (r"\b(Object_)(')",
                bygroups(Name.Builtin, String.Interpol),
                ('object_attributes', 'string_interpolated')),
            (words(objects, prefix=r'\b', suffix=r"(_)(')"),
                bygroups(Name.Builtin, Name.Builtin, String.Interpol),
                ('object_attributes', 'string_interpolated')),

            (r'\.?[a-z][a-zA-Z0-9_.]*(\$|#)?', Text),
            (r'[\[\]]', Punctuation, 'comma_list'),
            (r"'(?=.*')", String.Interpol, 'string_interpolated'),
        ],
        'operator': [
            (r'([+\/*<>=!-]=?|[&*|][&*|]?|\^|<>)', Operator),
            (r'\b(and|or|not|div|mod)\b',          Operator.Word),
        ],
        'string_interpolated': [
            (r'\.?[_a-z][a-zA-Z0-9_.]*[\$#]?(?:\[[a-zA-Z0-9,]+\])?(:[0-9]+)?', String.Interpol),
            (r"'",          String.Interpol, '#pop'),
        ],
        'string_unquoted': [
            (r'(\n\s*)(\.{3})', bygroups(Text, Punctuation)),

            (r'\n',       Text,            '#pop'),
            (r'\s',       Text),
            (r"'(?=.*')", String.Interpol, 'string_interpolated'),
            (r"'",        String),
            (r"[^'\n]+",  String),
        ],
        'string': [
            (r'(\n\s*)(\.{3})', bygroups(Text, Punctuation)),

            (r'"',          String,          '#pop'),
            (r"'(?=.*')",   String.Interpol, 'string_interpolated'),
            (r"'",          String),
            (r'[^\'"\n]+',  String),
        ],
        'old_form': [
            (r'\s+', Text),

            (r'(optionmenu|choice)([ \t]+\S+:[ \t]+)',
                bygroups(Keyword, Text), 'number'),

            (r'(option|button)([ \t]+)',
                bygroups(Keyword, Text), 'number'),

            (r'(option|button)([ \t]+)',
                bygroups(Keyword, Text), 'string_unquoted'),

            (r'(sentence|text)([ \t]+\S+)',
                bygroups(Keyword, Text), 'string_unquoted'),

            (r'(word)([ \t]+\S+[ \t]*)(\S+)?([ \t]+.*)?',
                bygroups(Keyword, Text, String, Error)),

            (r'(boolean)(\s+\S+\s*)(0|1|"?(?:yes|no)"?)',
                bygroups(Keyword, Text, Name.Variable)),

            # Ideally processing of the number would happend in the 'number'
            # but that doesn't seem to work
            (r'(real|natural|positive|integer)([ \t]+\S+[ \t]*)([+-]?)(\d+(?:\.\d*)?(?:[eE][-+]?\d+)?%?)',
                bygroups(Keyword, Text, Operator, Number)),

            (r'(comment)(\s+)',
                bygroups(Keyword, Text), 'string_unquoted'),

            (r'\bendform\b', Keyword, '#pop'),
        ]
    }
<commit_msg>Praat: removed error token from `word` in forms and added underscore _variables<commit_after># -*- coding: utf-8 -*-
"""
    pygments.lexers.praat
    ~~~~~~~~~~~~~~~~~~~~

    Lexer for Praat

    :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

import re

from pygments.lexer import RegexLexer, words, bygroups, default, include
from pygments.token import *

__all__ = ['PraatLexer']

class PraatLexer(RegexLexer):
    """
    For `Praat <http://www.praat.org>`_ scripts.
    """

    name = 'Praat'
    aliases = ['praat']
    filenames = ['*.praat', '*.proc', '*.psc']

    keywords = [
        'if', 'then', 'else', 'elsif', 'elif', 'endif', 'fi', 'for', 'from', 'to', 'endfor', 'endproc',
        'while', 'endwhile', 'repeat', 'until', 'select', 'plus', 'minus', 'demo', 'assert', 'stopwatch',
        'nocheck', 'nowarn', 'noprogress', 'editor', 'endeditor', 'clearinfo'
    ]

    functions_string = [
        'backslashTrigraphsToUnicode', 'chooseDirectory', 'chooseReadFile',
        'chooseWriteFile', 'date', 'demoKey', 'do', 'environment', 'extractLine', 'extractWord',
        'fixed', 'info', 'left', 'mid', 'percent', 'readFile', 'replace', 'replace_regex', 'right',
        'selected', 'string', 'unicodeToBackslashTrigraphs',
    ]

    functions_numeric = [
        'abs', 'appendFile', 'appendFileLine', 'appendInfo', 'appendInfoLine', 'arccos', 'arccosh',
        'arcsin', 'arcsinh', 'arctan', 'arctan2', 'arctanh', 'barkToHertz', 'beginPause',
        'beginSendPraat', 'besselI', 'besselK', 'beta', 'beta2', 'binomialP', 'binomialQ', 'boolean',
        'ceiling', 'chiSquareP', 'chiSquareQ', 'choice', 'comment', 'cos', 'cosh', 'createDirectory',
        'deleteFile', 'demoClicked', 'demoClickedIn', 'demoCommandKeyPressed',
        'demoExtraControlKeyPressed', 'demoInput', 'demoKeyPressed',
        'demoOptionKeyPressed', 'demoShiftKeyPressed', 'demoShow', 'demoWaitForInput',
        'demoWindowTitle', 'demoX', 'demoY', 'differenceLimensToPhon', 'do', 'editor', 'endPause',
        'endSendPraat', 'endsWith', 'erb', 'erbToHertz', 'erf', 'erfc', 'exitScript', 'exp',
        'extractNumber', 'fileReadable', 'fisherP', 'fisherQ', 'floor', 'gaussP', 'gaussQ',
        'hertzToBark', 'hertzToErb', 'hertzToMel', 'hertzToSemitones', 'imax', 'imin',
        'incompleteBeta', 'incompleteGammaP', 'index', 'index_regex', 'invBinomialP',
        'invBinomialQ', 'invChiSquareQ', 'invFisherQ', 'invGaussQ', 'invSigmoid', 'invStudentQ',
        'length', 'ln', 'lnBeta', 'lnGamma', 'log10', 'log2', 'max', 'melToHertz', 'min', 'minusObject',
        'natural', 'number', 'numberOfColumns', 'numberOfRows', 'numberOfSelected',
        'objectsAreIdentical', 'option', 'optionMenu', 'pauseScript',
        'phonToDifferenceLimens', 'plusObject', 'positive', 'randomBinomial', 'randomGauss',
        'randomInteger', 'randomPoisson', 'randomUniform', 'real', 'readFile', 'removeObject',
        'rindex', 'rindex_regex', 'round', 'runScript', 'runSystem', 'runSystem_nocheck',
        'selectObject', 'selected', 'semitonesToHertz', 'sentencetext', 'sigmoid', 'sin', 'sinc',
        'sincpi', 'sinh', 'soundPressureToPhon', 'sqrt', 'startsWith', 'studentP', 'studentQ', 'tan',
        'tanh', 'variableExists', 'word', 'writeFile', 'writeFileLine', 'writeInfo',
        'writeInfoLine',
    ]

    functions_array = [
        'linear', 'randomGauss', 'randomInteger', 'randomUniform', 'zero',
    ]

    objects = [
        'Activation', 'AffineTransform', 'AmplitudeTier', 'Art', 'Artword', 'Autosegment',
        'BarkFilter', 'BarkSpectrogram', 'CCA', 'Categories', 'Cepstrogram', 'Cepstrum',
        'Cepstrumc', 'ChebyshevSeries', 'ClassificationTable', 'Cochleagram', 'Collection',
        'ComplexSpectrogram', 'Configuration', 'Confusion', 'ContingencyTable', 'Corpus',
        'Correlation', 'Covariance', 'CrossCorrelationTable', 'CrossCorrelationTables', 'DTW',
        'DataModeler', 'Diagonalizer', 'Discriminant', 'Dissimilarity', 'Distance',
        'Distributions', 'DurationTier', 'EEG', 'ERP', 'ERPTier', 'EditCostsTable',
        'EditDistanceTable', 'Eigen', 'Excitation', 'Excitations', 'ExperimentMFC', 'FFNet',
        'FeatureWeights', 'FileInMemory', 'FilesInMemory', 'Formant', 'FormantFilter',
        'FormantGrid', 'FormantModeler', 'FormantPoint', 'FormantTier', 'GaussianMixture', 'HMM',
        'HMM_Observation', 'HMM_ObservationSequence', 'HMM_State', 'HMM_StateSequence',
        'Harmonicity', 'ISpline', 'Index', 'Intensity', 'IntensityTier', 'IntervalTier', 'KNN',
        'KlattGrid', 'KlattTable', 'LFCC', 'LPC', 'Label', 'LegendreSeries', 'LinearRegression',
        'LogisticRegression', 'LongSound', 'Ltas', 'MFCC', 'MSpline', 'ManPages', 'Manipulation',
        'Matrix', 'MelFilter', 'MelSpectrogram', 'MixingMatrix', 'Movie', 'Network', 'OTGrammar',
        'OTHistory', 'OTMulti', 'PCA', 'PairDistribution', 'ParamCurve', 'Pattern', 'Permutation',
        'Photo', 'Pitch', 'PitchModeler', 'PitchTier', 'PointProcess', 'Polygon', 'Polynomial',
        'PowerCepstrogram', 'PowerCepstrum', 'Procrustes', 'RealPoint', 'RealTier', 'ResultsMFC',
        'Roots', 'SPINET', 'SSCP', 'SVD', 'Salience', 'ScalarProduct', 'Similarity', 'SimpleString',
        'SortedSetOfString', 'Sound', 'Speaker', 'Spectrogram', 'Spectrum', 'SpectrumTier',
        'SpeechSynthesizer', 'SpellingChecker', 'Strings', 'StringsIndex', 'Table',
        'TableOfReal', 'TextGrid', 'TextInterval', 'TextPoint', 'TextTier', 'Tier', 'Transition',
        'VocalTract', 'VocalTractTier', 'Weight', 'WordList',
    ]

    variables_numeric = [
        'macintosh', 'windows', 'unix', 'praatVersion', 'pi', 'e', 'undefined',
    ]

    variables_string = [
        'praatVersion', 'tab', 'shellDirectory', 'homeDirectory',
        'preferencesDirectory', 'newline', 'temporaryDirectory',
        'defaultDirectory',
    ]

    tokens = {
        'root': [
            (r'(\s+)(#.*?$)',  bygroups(Text, Comment.Single)),
            (r'^#.*?$',        Comment.Single),
            (r';[^\n]*',       Comment.Single),
            (r'\s+',           Text),

            (r'\bprocedure\b', Keyword,       'procedure_definition'),
            (r'\bcall\b',      Keyword,       'procedure_call'),
            (r'@',             Name.Function, 'procedure_call'),

            include('function_call'),

            (words(keywords, suffix=r'\b'), Keyword),

            (r'(\bform\b)(\s+)([^\n]+)',
                bygroups(Keyword, Text, String), 'old_form'),

            (r'(print(?:line|tab)?|echo|exit|asserterror|pause|send(?:praat|socket)|include|execute|system(?:_nocheck)?)(\s+)',
                bygroups(Keyword, Text), 'string_unquoted'),

            (r'(goto|label)(\s+)(\w+)', bygroups(Keyword, Text, Name.Label)),

            include('variable_name'),
            include('number'),

            (r'"', String, 'string'),

            (words((objects), suffix=r'(?=\s+\S+\n)'), Name.Class, 'string_unquoted'),

            (r'\b[A-Z]', Keyword, 'command'),
            (r'(\.{3}|[)(,])', Punctuation),
        ],
        'command': [
            (r'( ?[\w()-]+ ?)', Keyword),
            (r"'(?=.*')", String.Interpol, 'string_interpolated'),
            (r'\.{3}', Keyword, ('#pop', 'old_arguments')),
            (r':', Keyword, ('#pop', 'comma_list')),
            (r'[\s\n]', Text, '#pop'),
        ],
        'procedure_call': [
            (r'\s+', Text),
            (r'([\w.]+)(:|\s*\()',
                bygroups(Name.Function, Text), '#pop'),
            (r'([\w.]+)', Name.Function, ('#pop', 'old_arguments')),
        ],
        'procedure_definition': [
            (r'\s', Text),
            (r'([\w.]+)(\s*?[(:])',
                bygroups(Name.Function, Text), '#pop'),
            (r'([\w.]+)([^\n]*)',
                bygroups(Name.Function, Text), '#pop'),
        ],
        'function_call': [
            (words(functions_string , suffix=r'\$(?=\s*[:(])'), Name.Function, 'function'),
            (words(functions_array  , suffix=r'#(?=\s*[:(])'),  Name.Function, 'function'),
            (words(functions_numeric, suffix=r'(?=\s*[:(])'),   Name.Function, 'function'),
        ],
        'function': [
            (r'\s+',   Text),
            (r':',     Punctuation, ('#pop', 'comma_list')),
            (r'\s*\(', Punctuation, ('#pop', 'comma_list')),
        ],
        'comma_list': [
            (r'(\s*\n\s*)(\.{3})', bygroups(Text, Punctuation)),

            (r'(\s*[])\n])', Text, '#pop'),

            (r'\s+', Text),
            (r'"',   String, 'string'),
            (r'\b(if|then|else|fi|endif)\b', Keyword),

            include('function_call'),
            include('variable_name'),
            include('operator'),
            include('number'),

            (r',', Punctuation),
        ],
        'old_arguments': [
            (r'\n', Text, '#pop'),

            include('variable_name'),
            include('operator'),
            include('number'),

            (r'"', String, 'string'),
            (r'[^\n]', Text),
        ],
        'number': [
            (r'\b\d+(\.\d*)?([eE][-+]?\d+)?%?', Number),
        ],
        'object_attributes': [
            (r'\.?(n(col|row)|[xy]min|[xy]max|[nd][xy])\b', Name.Builtin, '#pop'),
            (r'(\.?(?:col|row)\$)(\[)',
                bygroups(Name.Builtin, Text), 'variable_name'),
            (r'(\$?)(\[)',
                bygroups(Name.Builtin, Text), ('#pop', 'comma_list')),
        ],
        'variable_name': [
            include('operator'),
            include('number'),

            (words(variables_string,  suffix=r'\$'), Name.Variable.Global),
            (words(variables_numeric, suffix=r'\b'), Name.Variable.Global),

            (r'\bObject_\w+', Name.Builtin, 'object_attributes'),
            (words(objects, prefix=r'\b', suffix=r'_\w+'), Name.Builtin, 'object_attributes'),

            (r"\b(Object_)(')",
                bygroups(Name.Builtin, String.Interpol),
                ('object_attributes', 'string_interpolated')),
            (words(objects, prefix=r'\b', suffix=r"(_)(')"),
                bygroups(Name.Builtin, Name.Builtin, String.Interpol),
                ('object_attributes', 'string_interpolated')),

            (r'\.?_?[a-z][a-zA-Z0-9_.]*(\$|#)?', Text),
            (r'[\[\]]', Punctuation, 'comma_list'),
            (r"'(?=.*')", String.Interpol, 'string_interpolated'),
        ],
        'operator': [
            (r'([+\/*<>=!-]=?|[&*|][&*|]?|\^|<>)', Operator),
            (r'\b(and|or|not|div|mod)\b',          Operator.Word),
        ],
        'string_interpolated': [
            (r'\.?[_a-z][a-zA-Z0-9_.]*[\$#]?(?:\[[a-zA-Z0-9,]+\])?(:[0-9]+)?', String.Interpol),
            (r"'",          String.Interpol, '#pop'),
        ],
        'string_unquoted': [
            (r'(\n\s*)(\.{3})', bygroups(Text, Punctuation)),

            (r'\n',       Text,            '#pop'),
            (r'\s',       Text),
            (r"'(?=.*')", String.Interpol, 'string_interpolated'),
            (r"'",        String),
            (r"[^'\n]+",  String),
        ],
        'string': [
            (r'(\n\s*)(\.{3})', bygroups(Text, Punctuation)),

            (r'"',          String,          '#pop'),
            (r"'(?=.*')",   String.Interpol, 'string_interpolated'),
            (r"'",          String),
            (r'[^\'"\n]+',  String),
        ],
        'old_form': [
            (r'\s+', Text),

            (r'(optionmenu|choice)([ \t]+\S+:[ \t]+)',
                bygroups(Keyword, Text), 'number'),

            (r'(option|button)([ \t]+)',
                bygroups(Keyword, Text), 'number'),

            (r'(option|button)([ \t]+)',
                bygroups(Keyword, Text), 'string_unquoted'),

            (r'(sentence|text)([ \t]+\S+)',
                bygroups(Keyword, Text), 'string_unquoted'),

            (r'(word)([ \t]+\S+[ \t]*)(\S+)?([ \t]+.*)?',
                bygroups(Keyword, Text, String, Text)),

            (r'(boolean)(\s+\S+\s*)(0|1|"?(?:yes|no)"?)',
                bygroups(Keyword, Text, Name.Variable)),

            # Ideally processing of the number would happend in the 'number'
            # but that doesn't seem to work
            (r'(real|natural|positive|integer)([ \t]+\S+[ \t]*)([+-]?)(\d+(?:\.\d*)?(?:[eE][-+]?\d+)?%?)',
                bygroups(Keyword, Text, Operator, Number)),

            (r'(comment)(\s+)',
                bygroups(Keyword, Text), 'string_unquoted'),

            (r'\bendform\b', Keyword, '#pop'),
        ]
    }
<|endoftext|>


<commit_before>from __future__ import unicode_literals

import re

from django.contrib.contenttypes.models import ContentType
from django.db.models import Q
import django_filters
from rest_framework import settings, filters
from rest_framework.serializers import ValidationError

from nodeconductor.core import serializers as core_serializers, filters as core_filters
from nodeconductor.logging import models, utils
from nodeconductor.logging.log import event_logger

def _convert(name):
    """ Converts CamelCase to underscore """
    s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
    return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()

class EventFilterBackend(filters.BaseFilterBackend):

    def filter_queryset(self, request, queryset, view):
        search_text = request.query_params.get(settings.api_settings.SEARCH_PARAM, '')
        must_terms = {}
        should_terms = {}
        if 'event_type' in request.query_params:
            must_terms['event_type'] = request.query_params.getlist('event_type')

        if 'scope' in request.query_params:
            field = core_serializers.GenericRelatedField(related_models=utils.get_loggable_models())
            obj = field.to_internal_value(request.query_params['scope'])
            must_terms[_convert(obj.__class__.__name__ + '_uuid')] = [obj.uuid.hex]
        elif 'scope_type' in request.query_params:
            choices = {_convert(m.__name__): m for m in utils.get_loggable_models()}
            try:
                scope_type = choices[request.query_params['scope_type']]
            except KeyError:
                raise ValidationError(
                    'Scope type "{}" is not valid. Has to be one from list: {}'.format(
                        request.query_params['scope_type'], ', '.join(choices.keys()))
                )
            else:
                must_terms.update(scope_type.get_permitted_objects_uuids(request.user))
        else:
            should_terms.update(event_logger.get_permitted_objects_uuids(request.user))

        queryset = queryset.filter(search_text=search_text, should_terms=should_terms, must_terms=must_terms)

        order_by = request.query_params.get('o', '-@timestamp')
        queryset = queryset.order_by(order_by)

        return queryset

class AlertFilter(django_filters.FilterSet):
    """ Basic filters for alerts """

    acknowledged = django_filters.BooleanFilter(name='acknowledged', distinct=True)
    closed_from = core_filters.TimestampFilter(name='closed', lookup_type='gte')
    closed_to = core_filters.TimestampFilter(name='closed', lookup_type='lt')
    created_from = core_filters.TimestampFilter(name='created', lookup_type='gte')
    created_to = core_filters.TimestampFilter(name='created', lookup_type='lt')
    content_type = core_filters.ContentTypeFilter()

    class Meta:
        model = models.Alert
        fields = [
            'acknowledged',
            'closed_from',
            'closed_to',
            'created_from',
            'created_to',
            'content_type',
        ]
        order_by = [
            'severity',
            '-severity',
            'created',
            '-created',
        ]

class AlertScopeFilterBackend(core_filters.GenericKeyFilterBackend):

    def get_related_models(self):
        return utils.get_loggable_models()

    def get_field_name(self):
        return 'scope'

class AdditionalAlertFilterBackend(filters.BaseFilterBackend):
    """
    Additional filters for alerts.

    Support for filters that are related to more than one field or provides unusual query.
    """

    def filter_queryset(self, request, queryset, view):
        mapped = {
            'start': request.query_params.get('from'),
            'end': request.query_params.get('to'),
        }
        timestamp_interval_serializer = core_serializers.TimestampIntervalSerializer(
            data={k: v for k, v in mapped.items() if v})
        timestamp_interval_serializer.is_valid(raise_exception=True)
        filter_data = timestamp_interval_serializer.get_filter_data()
        if 'start' in filter_data:
            queryset = queryset.filter(
                Q(closed__gte=filter_data['start']) | Q(closed__isnull=True))
        if 'end' in filter_data:
            queryset = queryset.filter(created__lte=filter_data['end'])

        if 'opened' in request.query_params:
            queryset = queryset.filter(closed__isnull=True)

        if 'severity' in request.query_params:
            severity_codes = {v: k for k, v in models.Alert.SeverityChoices.CHOICES}
            severities = [
                severity_codes.get(severity_name) for severity_name in request.query_params.getlist('severity')]
            queryset = queryset.filter(severity__in=severities)

        # XXX: this filter is wrong and deprecated, need to be removed after replacement in Portal
        if 'scope_type' in request.query_params:
            choices = {_convert(m.__name__): m for m in utils.get_loggable_models()}
            try:
                scope_type = choices[request.query_params['scope_type']]
            except KeyError:
                raise ValidationError(
                    'Scope type "{}" is not valid. Has to be one from list: {}'.format(
                        request.query_params['scope_type'], ', '.join(choices.keys()))
                )
            else:
                ct = ContentType.objects.get_for_model(scope_type)
                queryset = queryset.filter(content_type=ct)

        if 'alert_type' in request.query_params:
            queryset = queryset.filter(alert_type__in=request.query_params.getlist('alert_type'))

        return queryset

class BaseExternalFilter(object):
    """ Interface for external alert filter """
    def filter(self, request, queryset, view):
        raise NotImplementedError

class ExternalAlertFilterBackend(filters.BaseFilterBackend):
    """
    Support external filters registered in other apps
    """

    @classmethod
    def get_registered_filters(cls):
        return getattr(cls, '_filters', [])

    @classmethod
    def register(cls, external_filter):
        assert isinstance(external_filter, BaseExternalFilter), 'Registered filter has to inherit BaseExternalFilter'
        if hasattr(cls, '_filters'):
            cls._filters.append(external_filter)
        else:
            cls._filters = [external_filter]

    def filter_queryset(self, request, queryset, view):
        for filt in self.__class__.get_registered_filters():
            queryset = filt.filter(request, queryset, view)
        return queryset
<commit_msg>Add missing request to the context<commit_after>from __future__ import unicode_literals

import re

from django.contrib.contenttypes.models import ContentType
from django.db.models import Q
import django_filters
from rest_framework import settings, filters
from rest_framework.serializers import ValidationError

from nodeconductor.core import serializers as core_serializers, filters as core_filters
from nodeconductor.logging import models, utils
from nodeconductor.logging.log import event_logger

def _convert(name):
    """ Converts CamelCase to underscore """
    s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
    return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()

class EventFilterBackend(filters.BaseFilterBackend):

    def filter_queryset(self, request, queryset, view):
        search_text = request.query_params.get(settings.api_settings.SEARCH_PARAM, '')
        must_terms = {}
        should_terms = {}
        if 'event_type' in request.query_params:
            must_terms['event_type'] = request.query_params.getlist('event_type')

        if 'scope' in request.query_params:
            field = core_serializers.GenericRelatedField(related_models=utils.get_loggable_models())
            field._context = {'request': request}
            obj = field.to_internal_value(request.query_params['scope'])
            must_terms[_convert(obj.__class__.__name__ + '_uuid')] = [obj.uuid.hex]
        elif 'scope_type' in request.query_params:
            choices = {_convert(m.__name__): m for m in utils.get_loggable_models()}
            try:
                scope_type = choices[request.query_params['scope_type']]
            except KeyError:
                raise ValidationError(
                    'Scope type "{}" is not valid. Has to be one from list: {}'.format(
                        request.query_params['scope_type'], ', '.join(choices.keys()))
                )
            else:
                must_terms.update(scope_type.get_permitted_objects_uuids(request.user))
        else:
            should_terms.update(event_logger.get_permitted_objects_uuids(request.user))

        queryset = queryset.filter(search_text=search_text, should_terms=should_terms, must_terms=must_terms)

        order_by = request.query_params.get('o', '-@timestamp')
        queryset = queryset.order_by(order_by)

        return queryset

class AlertFilter(django_filters.FilterSet):
    """ Basic filters for alerts """

    acknowledged = django_filters.BooleanFilter(name='acknowledged', distinct=True)
    closed_from = core_filters.TimestampFilter(name='closed', lookup_type='gte')
    closed_to = core_filters.TimestampFilter(name='closed', lookup_type='lt')
    created_from = core_filters.TimestampFilter(name='created', lookup_type='gte')
    created_to = core_filters.TimestampFilter(name='created', lookup_type='lt')
    content_type = core_filters.ContentTypeFilter()

    class Meta:
        model = models.Alert
        fields = [
            'acknowledged',
            'closed_from',
            'closed_to',
            'created_from',
            'created_to',
            'content_type',
        ]
        order_by = [
            'severity',
            '-severity',
            'created',
            '-created',
        ]

class AlertScopeFilterBackend(core_filters.GenericKeyFilterBackend):

    def get_related_models(self):
        return utils.get_loggable_models()

    def get_field_name(self):
        return 'scope'

class AdditionalAlertFilterBackend(filters.BaseFilterBackend):
    """
    Additional filters for alerts.

    Support for filters that are related to more than one field or provides unusual query.
    """

    def filter_queryset(self, request, queryset, view):
        mapped = {
            'start': request.query_params.get('from'),
            'end': request.query_params.get('to'),
        }
        timestamp_interval_serializer = core_serializers.TimestampIntervalSerializer(
            data={k: v for k, v in mapped.items() if v})
        timestamp_interval_serializer.is_valid(raise_exception=True)
        filter_data = timestamp_interval_serializer.get_filter_data()
        if 'start' in filter_data:
            queryset = queryset.filter(
                Q(closed__gte=filter_data['start']) | Q(closed__isnull=True))
        if 'end' in filter_data:
            queryset = queryset.filter(created__lte=filter_data['end'])

        if 'opened' in request.query_params:
            queryset = queryset.filter(closed__isnull=True)

        if 'severity' in request.query_params:
            severity_codes = {v: k for k, v in models.Alert.SeverityChoices.CHOICES}
            severities = [
                severity_codes.get(severity_name) for severity_name in request.query_params.getlist('severity')]
            queryset = queryset.filter(severity__in=severities)

        # XXX: this filter is wrong and deprecated, need to be removed after replacement in Portal
        if 'scope_type' in request.query_params:
            choices = {_convert(m.__name__): m for m in utils.get_loggable_models()}
            try:
                scope_type = choices[request.query_params['scope_type']]
            except KeyError:
                raise ValidationError(
                    'Scope type "{}" is not valid. Has to be one from list: {}'.format(
                        request.query_params['scope_type'], ', '.join(choices.keys()))
                )
            else:
                ct = ContentType.objects.get_for_model(scope_type)
                queryset = queryset.filter(content_type=ct)

        if 'alert_type' in request.query_params:
            queryset = queryset.filter(alert_type__in=request.query_params.getlist('alert_type'))

        return queryset

class BaseExternalFilter(object):
    """ Interface for external alert filter """
    def filter(self, request, queryset, view):
        raise NotImplementedError

class ExternalAlertFilterBackend(filters.BaseFilterBackend):
    """
    Support external filters registered in other apps
    """

    @classmethod
    def get_registered_filters(cls):
        return getattr(cls, '_filters', [])

    @classmethod
    def register(cls, external_filter):
        assert isinstance(external_filter, BaseExternalFilter), 'Registered filter has to inherit BaseExternalFilter'
        if hasattr(cls, '_filters'):
            cls._filters.append(external_filter)
        else:
            cls._filters = [external_filter]

    def filter_queryset(self, request, queryset, view):
        for filt in self.__class__.get_registered_filters():
            queryset = filt.filter(request, queryset, view)
        return queryset
<|endoftext|>