<commit_before># Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Finds android browsers that can be controlled by telemetry."""

import logging
import os
import subprocess
import sys

from py_utils import dependency_util
from devil import base_error
from devil.android import apk_helper

from telemetry.core import exceptions
from telemetry.core import platform
from telemetry.core import util
from telemetry import decorators
from telemetry.internal.backends import android_browser_backend_settings
from telemetry.internal.backends.chrome import android_browser_backend
from telemetry.internal.browser import browser
from telemetry.internal.browser import possible_browser
from telemetry.internal.platform import android_device
from telemetry.internal.util import binary_manager

CHROME_PACKAGE_NAMES = {
  'android-content-shell':
      ['org.chromium.content_shell_apk',
       android_browser_backend_settings.ContentShellBackendSettings,
       'ContentShell.apk'],
  'android-webview':
      ['org.chromium.webview_shell',
       android_browser_backend_settings.WebviewBackendSettings,
       None],
  'android-webview-shell':
      ['org.chromium.android_webview.shell',
       android_browser_backend_settings.WebviewShellBackendSettings,
       'AndroidWebView.apk'],
  'android-chromium':
      ['org.chromium.chrome',
       android_browser_backend_settings.ChromeBackendSettings,
       'ChromePublic.apk'],
  'android-chrome':
      ['com.google.android.apps.chrome',
       android_browser_backend_settings.ChromeBackendSettings,
       'Chrome.apk'],
  'android-chrome-work':
      ['com.chrome.work',
       android_browser_backend_settings.ChromeBackendSettings,
       None],
  'android-chrome-beta':
      ['com.chrome.beta',
       android_browser_backend_settings.ChromeBackendSettings,
       None],
  'android-chrome-dev':
      ['com.chrome.dev',
       android_browser_backend_settings.ChromeBackendSettings,
       None],
  'android-chrome-canary':
      ['com.chrome.canary',
       android_browser_backend_settings.ChromeBackendSettings,
       None],
  'android-system-chrome':
      ['com.android.chrome',
       android_browser_backend_settings.ChromeBackendSettings,
       None],
}

class PossibleAndroidBrowser(possible_browser.PossibleBrowser):
  """A launchable android browser instance."""
  def __init__(self, browser_type, finder_options, android_platform,
               backend_settings, apk_name):
    super(PossibleAndroidBrowser, self).__init__(
        browser_type, 'android', backend_settings.supports_tab_control)
    assert browser_type in FindAllBrowserTypes(finder_options), (
        'Please add %s to android_browser_finder.FindAllBrowserTypes' %
         browser_type)
    self._platform = android_platform
    self._platform_backend = (
        android_platform._platform_backend)  # pylint: disable=protected-access
    self._backend_settings = backend_settings
    self._local_apk = None

    if browser_type == 'exact':
      if not os.path.exists(apk_name):
        raise exceptions.PathMissingError(
            'Unable to find exact apk %s specified by --browser-executable' %
            apk_name)
      self._local_apk = apk_name
    elif browser_type == 'reference':
      if not os.path.exists(apk_name):
        raise exceptions.PathMissingError(
            'Unable to find reference apk at expected location %s.' % apk_name)
      self._local_apk = apk_name
    elif apk_name:
      assert finder_options.chrome_root, (
          'Must specify Chromium source to use apk_name')
      chrome_root = finder_options.chrome_root
      candidate_apks = []
      for build_path in util.GetBuildDirectories(chrome_root):
        apk_full_name = os.path.join(build_path, 'apks', apk_name)
        if os.path.exists(apk_full_name):
          last_changed = os.path.getmtime(apk_full_name)
          candidate_apks.append((last_changed, apk_full_name))

      if candidate_apks:
        # Find the candidate .apk with the latest modification time.
        newest_apk_path = sorted(candidate_apks)[-1][1]
        self._local_apk = newest_apk_path

  def __repr__(self):
    return 'PossibleAndroidBrowser(browser_type=%s)' % self.browser_type

  def _InitPlatformIfNeeded(self):
    pass

  def Create(self, finder_options):
    self._InitPlatformIfNeeded()
    browser_backend = android_browser_backend.AndroidBrowserBackend(
        self._platform_backend,
        finder_options.browser_options, self._backend_settings)
    try:
      return browser.Browser(
          browser_backend, self._platform_backend, self._credentials_path)
    except Exception:
      logging.exception('Failure while creating Android browser.')
      original_exception = sys.exc_info()
      try:
        browser_backend.Close()
      except Exception:
        logging.exception('Secondary failure while closing browser backend.')

      raise original_exception[0], original_exception[1], original_exception[2]

  def SupportsOptions(self, browser_options):
    if len(browser_options.extensions_to_load) != 0:
      return False
    return True

  def HaveLocalAPK(self):
    return self._local_apk and os.path.exists(self._local_apk)

  @decorators.Cache
  def UpdateExecutableIfNeeded(self):
    if self.HaveLocalAPK():
      logging.warn('Installing %s on device if needed.' % self._local_apk)
      self.platform.InstallApplication(self._local_apk)

  def last_modification_time(self):
    if self.HaveLocalAPK():
      return os.path.getmtime(self._local_apk)
    return -1

def SelectDefaultBrowser(possible_browsers):
  """Return the newest possible browser."""
  if not possible_browsers:
    return None
  return max(possible_browsers, key=lambda b: b.last_modification_time())

def CanFindAvailableBrowsers():
  return android_device.CanDiscoverDevices()

def CanPossiblyHandlePath(target_path):
  return os.path.splitext(target_path.lower())[1] == '.apk'

def FindAllBrowserTypes(options):
  del options  # unused
  return CHROME_PACKAGE_NAMES.keys() + ['exact', 'reference']

def _FindAllPossibleBrowsers(finder_options, android_platform):
  """Testable version of FindAllAvailableBrowsers."""
  if not android_platform:
    return []
  possible_browsers = []

  # Add the exact APK if given.
  if (finder_options.browser_executable and
      CanPossiblyHandlePath(finder_options.browser_executable)):
    apk_name = os.path.basename(finder_options.browser_executable)
    normalized_path = os.path.expanduser(finder_options.browser_executable)
    exact_package = apk_helper.GetPackageName(normalized_path)
    package_info = next(
        (info for info in CHROME_PACKAGE_NAMES.itervalues()
         if info[0] == exact_package or info[2] == apk_name), None)

    # It is okay if the APK name or package doesn't match any of known chrome
    # browser APKs, since it may be of a different browser.
    if package_info:
      if not exact_package:
        raise exceptions.PackageDetectionError(
            'Unable to find package for %s specified by --browser-executable' %
            normalized_path)

      [package, backend_settings, _] = package_info
      if package == exact_package:
        possible_browsers.append(PossibleAndroidBrowser(
            'exact',
            finder_options,
            android_platform,
            backend_settings(package),
            normalized_path))
      else:
        raise exceptions.UnknownPackageError(
            '%s specified by --browser-executable has an unknown package: %s' %
            (normalized_path, exact_package))

  # Add the reference build if found.
  os_version = dependency_util.GetChromeApkOsVersion(
      android_platform.GetOSVersionName())
  arch = android_platform.GetArchName()
  try:
    reference_build = binary_manager.FetchPath(
        'chrome_stable', arch, 'android', os_version)
  except (binary_manager.NoPathFoundError,
          binary_manager.CloudStorageError):
    reference_build = None

  if reference_build and os.path.exists(reference_build):
    # TODO(aiolos): how do we stably map the android chrome_stable apk to the
    # correct package name?
    package, backend_settings, _ = CHROME_PACKAGE_NAMES['android-chrome']
    possible_browsers.append(PossibleAndroidBrowser(
        'reference',
        finder_options,
        android_platform,
        backend_settings(package),
        reference_build))

  # Add any known local versions.
  for name, package_info in CHROME_PACKAGE_NAMES.iteritems():
    package, backend_settings, apk_name = package_info
    if apk_name and not finder_options.chrome_root:
      continue
    b = PossibleAndroidBrowser(name,
                               finder_options,
                               android_platform,
                               backend_settings(package),
                               apk_name)
    if b.platform.CanLaunchApplication(package) or b.HaveLocalAPK():
      possible_browsers.append(b)
  return possible_browsers

def FindAllAvailableBrowsers(finder_options, device):
  """Finds all the possible browsers on one device.

  The device is either the only device on the host platform,
  or |finder_options| specifies a particular device.
  """
  if not isinstance(device, android_device.AndroidDevice):
    return []

  try:
    android_platform = platform.GetPlatformForDevice(device, finder_options)
    return _FindAllPossibleBrowsers(finder_options, android_platform)
  except base_error.BaseError as e:
    logging.error('Unable to find browsers on %s: %s', device.device_id, str(e))
    ps_output = subprocess.check_output(['ps', '-e'])
    logging.error('Ongoing processes:\n%s', ps_output)
  return []
<commit_msg>[Telemetry] Make add more details for ongoing processes logging<commit_after># Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Finds android browsers that can be controlled by telemetry."""

import logging
import os
import subprocess
import sys

from py_utils import dependency_util
from devil import base_error
from devil.android import apk_helper

from telemetry.core import exceptions
from telemetry.core import platform
from telemetry.core import util
from telemetry import decorators
from telemetry.internal.backends import android_browser_backend_settings
from telemetry.internal.backends.chrome import android_browser_backend
from telemetry.internal.browser import browser
from telemetry.internal.browser import possible_browser
from telemetry.internal.platform import android_device
from telemetry.internal.util import binary_manager

CHROME_PACKAGE_NAMES = {
  'android-content-shell':
      ['org.chromium.content_shell_apk',
       android_browser_backend_settings.ContentShellBackendSettings,
       'ContentShell.apk'],
  'android-webview':
      ['org.chromium.webview_shell',
       android_browser_backend_settings.WebviewBackendSettings,
       None],
  'android-webview-shell':
      ['org.chromium.android_webview.shell',
       android_browser_backend_settings.WebviewShellBackendSettings,
       'AndroidWebView.apk'],
  'android-chromium':
      ['org.chromium.chrome',
       android_browser_backend_settings.ChromeBackendSettings,
       'ChromePublic.apk'],
  'android-chrome':
      ['com.google.android.apps.chrome',
       android_browser_backend_settings.ChromeBackendSettings,
       'Chrome.apk'],
  'android-chrome-work':
      ['com.chrome.work',
       android_browser_backend_settings.ChromeBackendSettings,
       None],
  'android-chrome-beta':
      ['com.chrome.beta',
       android_browser_backend_settings.ChromeBackendSettings,
       None],
  'android-chrome-dev':
      ['com.chrome.dev',
       android_browser_backend_settings.ChromeBackendSettings,
       None],
  'android-chrome-canary':
      ['com.chrome.canary',
       android_browser_backend_settings.ChromeBackendSettings,
       None],
  'android-system-chrome':
      ['com.android.chrome',
       android_browser_backend_settings.ChromeBackendSettings,
       None],
}

class PossibleAndroidBrowser(possible_browser.PossibleBrowser):
  """A launchable android browser instance."""
  def __init__(self, browser_type, finder_options, android_platform,
               backend_settings, apk_name):
    super(PossibleAndroidBrowser, self).__init__(
        browser_type, 'android', backend_settings.supports_tab_control)
    assert browser_type in FindAllBrowserTypes(finder_options), (
        'Please add %s to android_browser_finder.FindAllBrowserTypes' %
         browser_type)
    self._platform = android_platform
    self._platform_backend = (
        android_platform._platform_backend)  # pylint: disable=protected-access
    self._backend_settings = backend_settings
    self._local_apk = None

    if browser_type == 'exact':
      if not os.path.exists(apk_name):
        raise exceptions.PathMissingError(
            'Unable to find exact apk %s specified by --browser-executable' %
            apk_name)
      self._local_apk = apk_name
    elif browser_type == 'reference':
      if not os.path.exists(apk_name):
        raise exceptions.PathMissingError(
            'Unable to find reference apk at expected location %s.' % apk_name)
      self._local_apk = apk_name
    elif apk_name:
      assert finder_options.chrome_root, (
          'Must specify Chromium source to use apk_name')
      chrome_root = finder_options.chrome_root
      candidate_apks = []
      for build_path in util.GetBuildDirectories(chrome_root):
        apk_full_name = os.path.join(build_path, 'apks', apk_name)
        if os.path.exists(apk_full_name):
          last_changed = os.path.getmtime(apk_full_name)
          candidate_apks.append((last_changed, apk_full_name))

      if candidate_apks:
        # Find the candidate .apk with the latest modification time.
        newest_apk_path = sorted(candidate_apks)[-1][1]
        self._local_apk = newest_apk_path

  def __repr__(self):
    return 'PossibleAndroidBrowser(browser_type=%s)' % self.browser_type

  def _InitPlatformIfNeeded(self):
    pass

  def Create(self, finder_options):
    self._InitPlatformIfNeeded()
    browser_backend = android_browser_backend.AndroidBrowserBackend(
        self._platform_backend,
        finder_options.browser_options, self._backend_settings)
    try:
      return browser.Browser(
          browser_backend, self._platform_backend, self._credentials_path)
    except Exception:
      logging.exception('Failure while creating Android browser.')
      original_exception = sys.exc_info()
      try:
        browser_backend.Close()
      except Exception:
        logging.exception('Secondary failure while closing browser backend.')

      raise original_exception[0], original_exception[1], original_exception[2]

  def SupportsOptions(self, browser_options):
    if len(browser_options.extensions_to_load) != 0:
      return False
    return True

  def HaveLocalAPK(self):
    return self._local_apk and os.path.exists(self._local_apk)

  @decorators.Cache
  def UpdateExecutableIfNeeded(self):
    if self.HaveLocalAPK():
      logging.warn('Installing %s on device if needed.' % self._local_apk)
      self.platform.InstallApplication(self._local_apk)

  def last_modification_time(self):
    if self.HaveLocalAPK():
      return os.path.getmtime(self._local_apk)
    return -1

def SelectDefaultBrowser(possible_browsers):
  """Return the newest possible browser."""
  if not possible_browsers:
    return None
  return max(possible_browsers, key=lambda b: b.last_modification_time())

def CanFindAvailableBrowsers():
  return android_device.CanDiscoverDevices()

def CanPossiblyHandlePath(target_path):
  return os.path.splitext(target_path.lower())[1] == '.apk'

def FindAllBrowserTypes(options):
  del options  # unused
  return CHROME_PACKAGE_NAMES.keys() + ['exact', 'reference']

def _FindAllPossibleBrowsers(finder_options, android_platform):
  """Testable version of FindAllAvailableBrowsers."""
  if not android_platform:
    return []
  possible_browsers = []

  # Add the exact APK if given.
  if (finder_options.browser_executable and
      CanPossiblyHandlePath(finder_options.browser_executable)):
    apk_name = os.path.basename(finder_options.browser_executable)
    normalized_path = os.path.expanduser(finder_options.browser_executable)
    exact_package = apk_helper.GetPackageName(normalized_path)
    package_info = next(
        (info for info in CHROME_PACKAGE_NAMES.itervalues()
         if info[0] == exact_package or info[2] == apk_name), None)

    # It is okay if the APK name or package doesn't match any of known chrome
    # browser APKs, since it may be of a different browser.
    if package_info:
      if not exact_package:
        raise exceptions.PackageDetectionError(
            'Unable to find package for %s specified by --browser-executable' %
            normalized_path)

      [package, backend_settings, _] = package_info
      if package == exact_package:
        possible_browsers.append(PossibleAndroidBrowser(
            'exact',
            finder_options,
            android_platform,
            backend_settings(package),
            normalized_path))
      else:
        raise exceptions.UnknownPackageError(
            '%s specified by --browser-executable has an unknown package: %s' %
            (normalized_path, exact_package))

  # Add the reference build if found.
  os_version = dependency_util.GetChromeApkOsVersion(
      android_platform.GetOSVersionName())
  arch = android_platform.GetArchName()
  try:
    reference_build = binary_manager.FetchPath(
        'chrome_stable', arch, 'android', os_version)
  except (binary_manager.NoPathFoundError,
          binary_manager.CloudStorageError):
    reference_build = None

  if reference_build and os.path.exists(reference_build):
    # TODO(aiolos): how do we stably map the android chrome_stable apk to the
    # correct package name?
    package, backend_settings, _ = CHROME_PACKAGE_NAMES['android-chrome']
    possible_browsers.append(PossibleAndroidBrowser(
        'reference',
        finder_options,
        android_platform,
        backend_settings(package),
        reference_build))

  # Add any known local versions.
  for name, package_info in CHROME_PACKAGE_NAMES.iteritems():
    package, backend_settings, apk_name = package_info
    if apk_name and not finder_options.chrome_root:
      continue
    b = PossibleAndroidBrowser(name,
                               finder_options,
                               android_platform,
                               backend_settings(package),
                               apk_name)
    if b.platform.CanLaunchApplication(package) or b.HaveLocalAPK():
      possible_browsers.append(b)
  return possible_browsers

def FindAllAvailableBrowsers(finder_options, device):
  """Finds all the possible browsers on one device.

  The device is either the only device on the host platform,
  or |finder_options| specifies a particular device.
  """
  if not isinstance(device, android_device.AndroidDevice):
    return []

  try:
    android_platform = platform.GetPlatformForDevice(device, finder_options)
    return _FindAllPossibleBrowsers(finder_options, android_platform)
  except base_error.BaseError as e:
    logging.error('Unable to find browsers on %s: %s', device.device_id, str(e))
    ps_output = subprocess.check_output(['ps', '-ef'])
    logging.error('Ongoing processes:\n%s', ps_output)
  return []
<|endoftext|>


<commit_before># Copyright 2014 Cloudbase Solutions Srl
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import argparse
import base64
import importlib
import logging
import pkgutil
import subprocess
import sys

import six

from argus import config
from argus import remote_client

__all__ = (
    'WinRemoteClient',
    'decrypt_password',
    'run_once',
    'get_resource',
    'cached_property',
    'load_qualified_object',
)

DEFAULT_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'

class WinRemoteClient(remote_client.WinRemoteClient):
    def run_command_verbose(self, cmd):
        """Run the given command and log anything it returns."""

        LOG.info("Running command %s...", cmd)
        stdout, stderr, exit_code = self.run_remote_cmd(cmd)

        LOG.info("The command returned the output: %s", stdout)
        LOG.info("The stderr of the command was: %s", stderr)
        LOG.info("The exit code of the command was: %s", exit_code)
        return stdout

def decrypt_password(private_key, password):
    """Decode password and unencrypts it with private key.

    Requires openssl binary available in the path.
    """
    unencoded = base64.b64decode(password)
    cmd = ['openssl', 'rsautl', '-decrypt', '-inkey', private_key]
    proc = subprocess.Popen(cmd, stdin=subprocess.PIPE,
                            stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE)
    out, err = proc.communicate(unencoded)
    proc.stdin.close()
    if proc.returncode:
        raise Exception("Failed calling openssl with error: {!r}"
                        .format(err))
    return out

# pylint: disable=dangerous-default-value
def run_once(func, state={}, exceptions={}):
    """A memoization decorator, whose purpose is to cache calls."""
    @six.wraps(func)
    def wrapper(*args, **kwargs):
        if func in exceptions:
            # Deliberate use of LBYL.
            six.reraise(*exceptions[func])

        try:
            return state[func]
        except KeyError:
            try:
                state[func] = result = func(*args, **kwargs)
                return result
            except Exception:
                exceptions[func] = sys.exc_info()
                raise
    return wrapper

def trap_failure(func):
    """Call pdb.set_trace when an exception occurs."""
    @six.wraps(func)
    def wrapper(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except BaseException:
            # TODO(cpopa): Save the original exception, since pdb will happily
            # overwrite it. This makes flake8 scream, though.
            # pylint: disable=unused-variable
            exc = sys.exc_info()  # NOQA

            LOG.exception("Exception occurred for func %s.", func)
            import pdb
            pdb.set_trace()
    return wrapper

def get_resource(resource):
    """Get the given resource from the list of known resources."""
    return pkgutil.get_data('argus.resources', resource)

class cached_property(object):  # pylint: disable=invalid-name
    """A property which caches the result on access."""

    def __init__(self, func, name=None):
        self.func = func
        self.name = name or func.__name__

    def __get__(self, instance, klass=None):
        instance.__dict__[self.name] = result = self.func(instance)
        return result

def parse_cli():
    """Parse the command line and return an object with the given options."""
    parser = argparse.ArgumentParser()
    parser.add_argument('--failfast', action='store_true',
                        default=False,
                        help='Fail the tests on the first failure.')
    parser.add_argument('--conf', type=str, required=True,
                        help="Give a path to the argus conf. "
                             "It should be an .ini file format "
                             "with a section called [argus].")
    parser.add_argument("--git-command", type=str, default=None,
                        help="Pass a git command which should be interpreted "
                             "by a recipe.")
    parser.add_argument("-p", "--pause", action="store_true",
                        help="Pause argus before doing any test.")
    parser.add_argument("--logging-format",
                        type=str, default=DEFAULT_FORMAT,
                        help="The logging format argus should use.")
    parser.add_argument("--logging-file",
                        type=str, default="argus.log",
                        help="The logging file argus should use.")
    parser.add_argument("--test-os-types",
                        type=str, nargs="*",
                        help="Test only those scenarios with these OS types. "
                             "By default, all scenarios are executed. "
                             "For instance, to run only the Windows and "
                             "FreeBSD scenarios, use "
                             "`--test-os-types Windows,FreeBSD`")
    parser.add_argument("--test-scenario-type",
                        type=str,
                        help="Test only the scenarios with this type. "
                             "The type can be `smoke` or `deep`. By default, "
                             "all scenarios types are executed.")
    parser.add_argument("-o", "--instance-output",
                        metavar="DIRECTORY",
                        help="Save the instance console output "
                             "content in this path.")
    opts = parser.parse_args()
    return opts

@run_once
def get_config():
    """Get the argus config object."""
    opts = parse_cli()
    return config.parse_config(opts.conf)

def get_logger(name="argus", format_string=None):
    """Obtain a new logger object.

    The `name` parameter will be the name of the logger
    and `format_string` will be the format it will
    use for logging.
    If it is not given, the the one given at command
    line will be used, otherwise the default format.
    """
    logger = logging.getLogger(name)
    opts = parse_cli()
    formatter = logging.Formatter(
        format_string or opts.logging_format or DEFAULT_FORMAT)

    if not logger.handlers:
        # If the logger wasn't obtained another time,
        # then it shouldn't have any loggers

        if opts.logging_file:
            file_handler = logging.FileHandler(opts.logging_file)
            file_handler.setFormatter(formatter)
            logger.addHandler(file_handler)

        stdout_handler = logging.StreamHandler(sys.stdout)
        stdout_handler.setFormatter(formatter)
        logger.addHandler(stdout_handler)

    logger.setLevel(logging.DEBUG)
    return logger

def load_qualified_object(obj):
    """Load a qualified object name.

    The name must be in the format module:qualname syntax.
    """
    mod_name, has_attrs, attrs = obj.partition(":")
    obj = module = importlib.import_module(mod_name)

    if has_attrs:
        parts = attrs.split(".")
        obj = module
        for part in parts:
            obj = getattr(obj, part)
    return obj

LOG = get_logger()
<commit_msg>Use lazy logging in util<commit_after># Copyright 2014 Cloudbase Solutions Srl
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import argparse
import base64
import importlib
import logging
import pkgutil
import subprocess
import sys

import six

from argus import config
from argus import remote_client

__all__ = (
    'WinRemoteClient',
    'decrypt_password',
    'run_once',
    'get_resource',
    'cached_property',
    'load_qualified_object',
)

DEFAULT_FORMAT = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'

class WinRemoteClient(remote_client.WinRemoteClient):
    def run_command_verbose(self, cmd):
        """Run the given command and log anything it returns."""

        LOG.info("Running command %s...", cmd)
        stdout, stderr, exit_code = self.run_remote_cmd(cmd)

        LOG.info("The command returned the output: %s", stdout)
        LOG.info("The stderr of the command was: %s", stderr)
        LOG.info("The exit code of the command was: %s", exit_code)
        return stdout

def decrypt_password(private_key, password):
    """Decode password and unencrypts it with private key.

    Requires openssl binary available in the path.
    """
    unencoded = base64.b64decode(password)
    cmd = ['openssl', 'rsautl', '-decrypt', '-inkey', private_key]
    proc = subprocess.Popen(cmd, stdin=subprocess.PIPE,
                            stdout=subprocess.PIPE,
                            stderr=subprocess.PIPE)
    out, err = proc.communicate(unencoded)
    proc.stdin.close()
    if proc.returncode:
        raise Exception("Failed calling openssl with error: {!r}"
                        .format(err))
    return out

# pylint: disable=dangerous-default-value
def run_once(func, state={}, exceptions={}):
    """A memoization decorator, whose purpose is to cache calls."""
    @six.wraps(func)
    def wrapper(*args, **kwargs):
        if func in exceptions:
            # Deliberate use of LBYL.
            six.reraise(*exceptions[func])

        try:
            return state[func]
        except KeyError:
            try:
                state[func] = result = func(*args, **kwargs)
                return result
            except Exception:
                exceptions[func] = sys.exc_info()
                raise
    return wrapper

def trap_failure(func):
    """Call pdb.set_trace when an exception occurs."""
    @six.wraps(func)
    def wrapper(*args, **kwargs):
        try:
            return func(*args, **kwargs)
        except BaseException:
            # TODO(cpopa): Save the original exception, since pdb will happily
            # overwrite it. This makes flake8 scream, though.
            # pylint: disable=unused-variable
            exc = sys.exc_info()  # NOQA

            LOG.exception("Exception occurred for func %s.", func)
            import pdb
            pdb.set_trace()
    return wrapper

def get_resource(resource):
    """Get the given resource from the list of known resources."""
    return pkgutil.get_data('argus.resources', resource)

class cached_property(object):  # pylint: disable=invalid-name
    """A property which caches the result on access."""

    def __init__(self, func, name=None):
        self.func = func
        self.name = name or func.__name__

    def __get__(self, instance, klass=None):
        instance.__dict__[self.name] = result = self.func(instance)
        return result

def parse_cli():
    """Parse the command line and return an object with the given options."""
    parser = argparse.ArgumentParser()
    parser.add_argument('--failfast', action='store_true',
                        default=False,
                        help='Fail the tests on the first failure.')
    parser.add_argument('--conf', type=str, required=True,
                        help="Give a path to the argus conf. "
                             "It should be an .ini file format "
                             "with a section called [argus].")
    parser.add_argument("--git-command", type=str, default=None,
                        help="Pass a git command which should be interpreted "
                             "by a recipe.")
    parser.add_argument("-p", "--pause", action="store_true",
                        help="Pause argus before doing any test.")
    parser.add_argument("--logging-format",
                        type=str, default=DEFAULT_FORMAT,
                        help="The logging format argus should use.")
    parser.add_argument("--logging-file",
                        type=str, default="argus.log",
                        help="The logging file argus should use.")
    parser.add_argument("--test-os-types",
                        type=str, nargs="*",
                        help="Test only those scenarios with these OS types. "
                             "By default, all scenarios are executed. "
                             "For instance, to run only the Windows and "
                             "FreeBSD scenarios, use "
                             "`--test-os-types Windows,FreeBSD`")
    parser.add_argument("--test-scenario-type",
                        type=str,
                        help="Test only the scenarios with this type. "
                             "The type can be `smoke` or `deep`. By default, "
                             "all scenarios types are executed.")
    parser.add_argument("-o", "--instance-output",
                        metavar="DIRECTORY",
                        help="Save the instance console output "
                             "content in this path.")
    opts = parser.parse_args()
    return opts

@run_once
def get_config():
    """Get the argus config object."""
    opts = parse_cli()
    return config.parse_config(opts.conf)

def get_logger(name="argus", format_string=None):
    """Obtain a new logger object.

    The `name` parameter will be the name of the logger
    and `format_string` will be the format it will
    use for logging.
    If it is not given, the the one given at command
    line will be used, otherwise the default format.
    """
    logger = logging.getLogger(name)
    opts = parse_cli()
    formatter = logging.Formatter(
        format_string or opts.logging_format or DEFAULT_FORMAT)

    if not logger.handlers:
        # If the logger wasn't obtained another time,
        # then it shouldn't have any loggers

        if opts.logging_file:
            file_handler = logging.FileHandler(opts.logging_file)
            file_handler.setFormatter(formatter)
            logger.addHandler(file_handler)

        stdout_handler = logging.StreamHandler(sys.stdout)
        stdout_handler.setFormatter(formatter)
        logger.addHandler(stdout_handler)

    logger.setLevel(logging.DEBUG)
    return logger

def load_qualified_object(obj):
    """Load a qualified object name.

    The name must be in the format module:qualname syntax.
    """
    mod_name, has_attrs, attrs = obj.partition(":")
    obj = module = importlib.import_module(mod_name)

    if has_attrs:
        parts = attrs.split(".")
        obj = module
        for part in parts:
            obj = getattr(obj, part)
    return obj

class ProxyLogger(object):

    """Proxy class for the logging object.

    This comes in hand when using argus as a library,
    so there is no need to provide required
    CLI arguments necessary by the argparse.
    """

    def __init__(self):
        self._logger = None

    def __getattr__(self, attr):
        # single instantiation on access only
        if not self._logger:
            self._logger = get_logger()
        obj = getattr(self._logger, attr)
        self.__dict__[attr] = obj
        return obj

LOG = ProxyLogger()
<|endoftext|>


<commit_before>from tornado.ncss import ncssbook_log as log
from tornado.ncss import Server
import sqlite3

from .home import home
from .game import game, Game
from .player import player, profiles, profile, Player
from .murder import murder, lodge, murder_list, murder_map, Murder
from .stats import stats
from .admin import admin, login_page, login
from .achievement import achievements, Achievement, AchievementProgress

def init_db(database=None):
	tables = [Game, Player, Murder, Achievement, AchievementProgress]
	for table in tables:
		# Set the database connection
		if database:
			table._conn = sqlite3.connect(database, isolation_level=None)

		# Initialise the database tables
		try:
			table.init_db()
		except sqlite3.OperationalError as e:
			log.info("Error creating tables: {}".format(e))

def init_server():
	server = Server()

	# API pages
	server.register('/game', game)
	server.register('/player', player, post=player)
	server.register('/murder', murder, post=murder)
	server.register('/login', login_page, post=login)

	# HTML pages
	game_id = '/([0-9a-zA-Z-]+)'
	player_id = '([a-zA-Z+-]+)'
	server.register('/admin/?', admin)
	server.register('{}/admin/?'.format(game_id), admin)
	server.register('{}/admin/lodge/?'.format(game_id), lodge)
	server.register('{}/profiles/?'.format(game_id), profiles)
	server.register('{}/profiles/{}/?'.format(game_id, player_id), profile)
	server.register('{}/map/?'.format(game_id), murder_map)
	server.register('{}/murders/?'.format(game_id), murder_list)
	server.register('{}/stats/?'.format(game_id), stats)
	server.register('{}/achievements/?'.format(game_id), achievements)
	server.register('{}?/?'.format(game_id), home)

	return server
<commit_msg>Fixed database connection for custom databases<commit_after>from tornado.ncss import ncssbook_log as log
from tornado.ncss import Server
import sqlite3

from .home import home
from .game import game, Game
from .player import player, profiles, profile, Player
from .murder import murder, lodge, murder_list, murder_map, Murder
from .stats import stats
from .admin import admin, login_page, login
from .achievement import achievements, Achievement, AchievementProgress

def init_db(database=None):
	# Create a custom database connection
	if database:
		conn = sqlite3.connect(database, isolation_level=None)

	tables = [Game, Player, Murder, Achievement, AchievementProgress]
	for table in tables:
		# Set the database connection
		if database:
			table._conn = conn

		# Initialise the database tables
		try:
			table.init_db()
		except sqlite3.OperationalError as e:
			log.info("Error creating tables: {}".format(e))

def init_server():
	server = Server()

	# API pages
	server.register('/game', game)
	server.register('/player', player, post=player)
	server.register('/murder', murder, post=murder)
	server.register('/login', login_page, post=login)

	# HTML pages
	game_id = '/([0-9a-zA-Z-]+)'
	player_id = '([a-zA-Z+-]+)'
	server.register('/admin/?', admin)
	server.register('{}/admin/?'.format(game_id), admin)
	server.register('{}/admin/lodge/?'.format(game_id), lodge)
	server.register('{}/profiles/?'.format(game_id), profiles)
	server.register('{}/profiles/{}/?'.format(game_id, player_id), profile)
	server.register('{}/map/?'.format(game_id), murder_map)
	server.register('{}/murders/?'.format(game_id), murder_list)
	server.register('{}/stats/?'.format(game_id), stats)
	server.register('{}/achievements/?'.format(game_id), achievements)
	server.register('{}?/?'.format(game_id), home)

	return server
<|endoftext|>


<commit_before>import os
import time
import asyncio
import audioop
import traceback

from enum import Enum
from array import array
from collections import deque
from shutil import get_terminal_size

from .lib.event_emitter import EventEmitter

class PatchedBuff:
    """
        PatchedBuff monkey patches a readable object, allowing you to vary what the volume is as the song is playing.
    """

    def __init__(self, player, buff):
        self.player = player
        self.buff = buff
        self.frame_count = 0

        self.draw = False
        self.use_audioop = True
        self.frame_skip = 2
        self.rmss = deque([2048], maxlen=90)

    def __del__(self):
        print(' ' * (get_terminal_size().columns-1), end='\r')

    def read(self, frame_size):
        self.frame_count += 1

        frame = self.buff.read(frame_size)
        volume = self.player.volume
        frame = self._frame_vol(frame, volume, maxv=2)

        rms = audioop.rms(frame, 2)
        self.rmss.append(rms)

        if self.draw and not self.frame_count % self.frame_skip:
            max_rms = sorted(self.rmss)[-1]
            meter_text = 'avg rms: {:.2f}, max rms: {:.2f} '.format(self._avg(self.rmss), max_rms)
            self._pprint_meter(rms / max(1, max_rms), text=meter_text, shift=True)

        return frame

    def _frame_vol(self, frame, mult, *, maxv=2, use_audioop=True):
        if use_audioop:
            return audioop.mul(frame, 2, min(mult, maxv))
        else:
            # ffmpeg returns s16le pcm frames.
            frame_array = array('h', frame)

            for i in range(len(frame_array)):
                frame_array[i] = int(frame_array[i] * min(mult, min(1, maxv)))

            return frame_array.tobytes()

    def _avg(self, i):
        return sum(i) / len(i)

    def _pprint_meter(self, perc, *, char='#', text='', shift=True):
        tx, ty = get_terminal_size()

        if shift:
            outstr = text + "{}".format(char * (int((tx - len(text)) * perc) - 1))
        else:
            outstr = text + "{}".format(char * (int(tx * perc) - 1))[len(text):]

        print(outstr.ljust(tx - 1), end='\r')

class MusicPlayerState(Enum):
    STOPPED = 0  # When the player isn't playing anything
    PLAYING = 1  # The player is actively playing music.
    PAUSED = 2  # The player is paused on a song.

    def __str__(self):
        return self.name

class MusicPlayer(EventEmitter):
    def __init__(self, bot, voice_client, playlist):
        super().__init__()
        self.bot = bot
        self.loop = bot.loop
        self.voice_client = voice_client
        self.playlist = playlist
        self.playlist.on('entry-added', self.on_entry_added)
        self.volume = bot.config.default_volume

        self._play_lock = asyncio.Lock()
        self._current_player = None
        self._current_entry = None
        self.state = MusicPlayerState.STOPPED

    def on_entry_added(self, playlist, entry):
        if self.is_stopped:
            self.loop.call_later(2, self.play)

    def skip(self):
        self._kill_current_player()

    def stop(self):
        self.state = MusicPlayerState.STOPPED
        self._kill_current_player()

        self.emit('stop', player=self)

    def resume(self):
        if self.is_paused and self._current_player:
            self._current_player.resume()
            self.state = MusicPlayerState.PLAYING
            self.emit('resume', player=self, entry=self.current_entry)
            return

        raise ValueError('Cannot resume playback from state %s' % self.state)

    def pause(self):
        if self.is_playing:
            self.state = MusicPlayerState.PAUSED

            if self._current_player:
                self._current_player.pause()

            self.emit('pause', player=self, entry=self.current_entry)
            return

        elif self.is_paused:
            return

        raise ValueError('Cannot pause a MusicPlayer in state %s' % self.state)

    def _playback_finished(self):
        entry = self._current_entry

        if self._current_player:
            self._current_player.after = None
            try:
                self._current_player.stop()
            except OSError:
                pass

        self._current_entry = None
        self._current_player = None

        if not self.is_stopped:
            self.play(_continue=True)

        if not self.bot.config.save_videos and entry:
            if any([entry.filename == e.filename for e in self.playlist.entries]):
                print("[Config:SaveVideos] Skipping deletion, found song in queue")

            else:
                # print("[Config:SaveVideos] Deleting file: %s" % os.path.relpath(entry.filename))
                asyncio.ensure_future(self._delete_file(entry.filename))

        self.emit('finished-playing', player=self, entry=entry)

    def _kill_current_player(self):
        if self._current_player:
            if self.is_paused:
                self.resume()

            try:
                self._current_player.stop()
            except OSError:
                pass
            self._current_player = None
            return True

        return False

    async def _delete_file(self, filename):
        for x in range(30):
            try:
                os.unlink(filename)
                break

            except PermissionError as e:
                if e.winerror == 32:  # File is in use
                    await asyncio.sleep(0.25)

            except Exception as e:
                traceback.print_exc()
                print("Error trying to delete " + filename)
                break
        else:
            print("[Config:SaveVideos] Could not delete file {}, giving up and moving on".format(
                os.path.relpath(filename)))

    def play(self, _continue=False):
        self.loop.create_task(self._play(_continue=_continue))

    async def _play(self, _continue=False):
        """
            Plays the next entry from the playlist, or resumes playback of the current entry if paused.
        """
        if self.is_paused:
            return self.resume()

        with await self._play_lock:
            if self.is_stopped or _continue:
                try:
                    entry = await self.playlist.get_next_entry()

                except Exception as e:
                    print("Failed to get entry.")
                    traceback.print_exc()
                    # Retry playing the next entry in a sec.
                    self.loop.call_later(0.1, self.play)
                    return

                # If nothing left to play, transition to the stopped state.
                if not entry:
                    self.stop()
                    return

                # In-case there was a player, kill it. RIP.
                self._kill_current_player()

                self._current_player = self._monkeypatch_player(self.voice_client.create_ffmpeg_player(
                    entry.filename,
                    before_options="-nostdin",
                    # Threadsafe call soon, b/c after will be called from the voice playback thread.
                    after=lambda: self.loop.call_soon_threadsafe(self._playback_finished)
                ))

                # I need to add ytdl hooks
                self.state = MusicPlayerState.PLAYING
                self._current_entry = entry

                self._current_player.start()
                self.emit('play', player=self, entry=entry)

    def _monkeypatch_player(self, player):
        original_buff = player.buff
        player.buff = PatchedBuff(self, original_buff)
        return player

    @property
    def current_entry(self):
        return self._current_entry

    @property
    def is_playing(self):
        return self.state == MusicPlayerState.PLAYING

    @property
    def is_paused(self):
        return self.state == MusicPlayerState.PAUSED

    @property
    def is_stopped(self):
        return self.state == MusicPlayerState.STOPPED

    @property
    def progress(self):
        return round(self._current_player.buff.frame_count * 0.02)
        # TODO: Properly implement this
        #       Correct calculation should be bytes_read/192k
        #       192k AKA sampleRate * (bitDepth / 8) * channelCount
        #       Change frame_count to bytes_read in the PatchedBuff

# if redistributing ffmpeg is an issue, it can be downloaded from here:
#  - http://ffmpeg.zeranoe.com/builds/win32/static/ffmpeg-latest-win32-static.7z
#  - http://ffmpeg.zeranoe.com/builds/win64/static/ffmpeg-latest-win64-static.7z
#
# Extracting bin/ffmpeg.exe, bin/ffplay.exe, and bin/ffprobe.exe should be fine
# However, the files are in 7z format so meh
# I don't know if we can even do this for the user, at most we open it in the browser
# I can't imagine the user is so incompetent that they can't pull 3 files out of it...
# ...
# ...right?

# ffprobe.exe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 -sexagesimal filename.mp3

# -af dynaudnorm
<commit_msg>Daemonize player threads<commit_after>import os
import time
import asyncio
import audioop
import traceback

from enum import Enum
from array import array
from collections import deque
from shutil import get_terminal_size

from .lib.event_emitter import EventEmitter

class PatchedBuff:
    """
        PatchedBuff monkey patches a readable object, allowing you to vary what the volume is as the song is playing.
    """

    def __init__(self, player, buff):
        self.player = player
        self.buff = buff
        self.frame_count = 0

        self.draw = False
        self.use_audioop = True
        self.frame_skip = 2
        self.rmss = deque([2048], maxlen=90)

    def __del__(self):
        print(' ' * (get_terminal_size().columns-1), end='\r')

    def read(self, frame_size):
        self.frame_count += 1

        frame = self.buff.read(frame_size)
        volume = self.player.volume
        frame = self._frame_vol(frame, volume, maxv=2)

        rms = audioop.rms(frame, 2)
        self.rmss.append(rms)

        if self.draw and not self.frame_count % self.frame_skip:
            max_rms = sorted(self.rmss)[-1]
            meter_text = 'avg rms: {:.2f}, max rms: {:.2f} '.format(self._avg(self.rmss), max_rms)
            self._pprint_meter(rms / max(1, max_rms), text=meter_text, shift=True)

        return frame

    def _frame_vol(self, frame, mult, *, maxv=2, use_audioop=True):
        if use_audioop:
            return audioop.mul(frame, 2, min(mult, maxv))
        else:
            # ffmpeg returns s16le pcm frames.
            frame_array = array('h', frame)

            for i in range(len(frame_array)):
                frame_array[i] = int(frame_array[i] * min(mult, min(1, maxv)))

            return frame_array.tobytes()

    def _avg(self, i):
        return sum(i) / len(i)

    def _pprint_meter(self, perc, *, char='#', text='', shift=True):
        tx, ty = get_terminal_size()

        if shift:
            outstr = text + "{}".format(char * (int((tx - len(text)) * perc) - 1))
        else:
            outstr = text + "{}".format(char * (int(tx * perc) - 1))[len(text):]

        print(outstr.ljust(tx - 1), end='\r')

class MusicPlayerState(Enum):
    STOPPED = 0  # When the player isn't playing anything
    PLAYING = 1  # The player is actively playing music.
    PAUSED = 2  # The player is paused on a song.

    def __str__(self):
        return self.name

class MusicPlayer(EventEmitter):
    def __init__(self, bot, voice_client, playlist):
        super().__init__()
        self.bot = bot
        self.loop = bot.loop
        self.voice_client = voice_client
        self.playlist = playlist
        self.playlist.on('entry-added', self.on_entry_added)
        self.volume = bot.config.default_volume

        self._play_lock = asyncio.Lock()
        self._current_player = None
        self._current_entry = None
        self.state = MusicPlayerState.STOPPED

    def on_entry_added(self, playlist, entry):
        if self.is_stopped:
            self.loop.call_later(2, self.play)

    def skip(self):
        self._kill_current_player()

    def stop(self):
        self.state = MusicPlayerState.STOPPED
        self._kill_current_player()

        self.emit('stop', player=self)

    def resume(self):
        if self.is_paused and self._current_player:
            self._current_player.resume()
            self.state = MusicPlayerState.PLAYING
            self.emit('resume', player=self, entry=self.current_entry)
            return

        raise ValueError('Cannot resume playback from state %s' % self.state)

    def pause(self):
        if self.is_playing:
            self.state = MusicPlayerState.PAUSED

            if self._current_player:
                self._current_player.pause()

            self.emit('pause', player=self, entry=self.current_entry)
            return

        elif self.is_paused:
            return

        raise ValueError('Cannot pause a MusicPlayer in state %s' % self.state)

    def _playback_finished(self):
        entry = self._current_entry

        if self._current_player:
            self._current_player.after = None
            try:
                self._current_player.stop()
            except OSError:
                pass

        self._current_entry = None
        self._current_player = None

        if not self.is_stopped:
            self.play(_continue=True)

        if not self.bot.config.save_videos and entry:
            if any([entry.filename == e.filename for e in self.playlist.entries]):
                print("[Config:SaveVideos] Skipping deletion, found song in queue")

            else:
                # print("[Config:SaveVideos] Deleting file: %s" % os.path.relpath(entry.filename))
                asyncio.ensure_future(self._delete_file(entry.filename))

        self.emit('finished-playing', player=self, entry=entry)

    def _kill_current_player(self):
        if self._current_player:
            if self.is_paused:
                self.resume()

            try:
                self._current_player.stop()
            except OSError:
                pass
            self._current_player = None
            return True

        return False

    async def _delete_file(self, filename):
        for x in range(30):
            try:
                os.unlink(filename)
                break

            except PermissionError as e:
                if e.winerror == 32:  # File is in use
                    await asyncio.sleep(0.25)

            except Exception as e:
                traceback.print_exc()
                print("Error trying to delete " + filename)
                break
        else:
            print("[Config:SaveVideos] Could not delete file {}, giving up and moving on".format(
                os.path.relpath(filename)))

    def play(self, _continue=False):
        self.loop.create_task(self._play(_continue=_continue))

    async def _play(self, _continue=False):
        """
            Plays the next entry from the playlist, or resumes playback of the current entry if paused.
        """
        if self.is_paused:
            return self.resume()

        with await self._play_lock:
            if self.is_stopped or _continue:
                try:
                    entry = await self.playlist.get_next_entry()

                except Exception as e:
                    print("Failed to get entry.")
                    traceback.print_exc()
                    # Retry playing the next entry in a sec.
                    self.loop.call_later(0.1, self.play)
                    return

                # If nothing left to play, transition to the stopped state.
                if not entry:
                    self.stop()
                    return

                # In-case there was a player, kill it. RIP.
                self._kill_current_player()

                self._current_player = self._monkeypatch_player(self.voice_client.create_ffmpeg_player(
                    entry.filename,
                    before_options="-nostdin",
                    # Threadsafe call soon, b/c after will be called from the voice playback thread.
                    after=lambda: self.loop.call_soon_threadsafe(self._playback_finished)
                ))
                self._current_player.setDaemon(True)

                # I need to add ytdl hooks
                self.state = MusicPlayerState.PLAYING
                self._current_entry = entry

                self._current_player.start()
                self.emit('play', player=self, entry=entry)

    def _monkeypatch_player(self, player):
        original_buff = player.buff
        player.buff = PatchedBuff(self, original_buff)
        return player

    @property
    def current_entry(self):
        return self._current_entry

    @property
    def is_playing(self):
        return self.state == MusicPlayerState.PLAYING

    @property
    def is_paused(self):
        return self.state == MusicPlayerState.PAUSED

    @property
    def is_stopped(self):
        return self.state == MusicPlayerState.STOPPED

    @property
    def progress(self):
        return round(self._current_player.buff.frame_count * 0.02)
        # TODO: Properly implement this
        #       Correct calculation should be bytes_read/192k
        #       192k AKA sampleRate * (bitDepth / 8) * channelCount
        #       Change frame_count to bytes_read in the PatchedBuff

# if redistributing ffmpeg is an issue, it can be downloaded from here:
#  - http://ffmpeg.zeranoe.com/builds/win32/static/ffmpeg-latest-win32-static.7z
#  - http://ffmpeg.zeranoe.com/builds/win64/static/ffmpeg-latest-win64-static.7z
#
# Extracting bin/ffmpeg.exe, bin/ffplay.exe, and bin/ffprobe.exe should be fine
# However, the files are in 7z format so meh
# I don't know if we can even do this for the user, at most we open it in the browser
# I can't imagine the user is so incompetent that they can't pull 3 files out of it...
# ...
# ...right?

# ffprobe.exe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 -sexagesimal filename.mp3

# -af dynaudnorm
<|endoftext|>


<commit_before>#!/usr/bin/env python
'''
Iterate over all cgroup mountpoints and output global cgroup statistics
'''

import os
import sys
import stat
import pwd
import time
import psutil

from collections import defaultdict

try:
    import curses, _curses
except ImportError:
    print >> sys.stderr, "Curse is not available on this system. Exiting."
    sys.exit(0)

HIDE_EMPTY_CGROUP = True
UPDATE_INTERVAL = 1.0 # seconds
CGROUP_MOUNTPOINTS={}
CONFIGURATION = {
        'sort_by': 'cpu_total',
        'sort_asc': False,
}

# TODO:
# - react to keyborad/mouse events
# - select display colums
# - select refresh rate
# - select sort column
# - detect container technology
# - visual CPU/memory usage
# - block-io
# - auto-color
# - adapt name / commands to underlying container system
# - hiereachical view

## Utils

def to_human(num, suffix='B'):
    num = int(num)
    for unit in [' ','K','M','G','T','P','E','Z']:
        if abs(num) < 1024.0:
            return "{:.1f}{}{}".format(num, unit, suffix)
        num /= 1024.0
    return "{:5.1d}{}{}" % (num, 'Y', suffix)

class Cgroup(object):
    def __init__(self, path, base_path):
        self.path = path
        self.base_path = base_path

    @property
    def name(self):
        return self.path[len(self.base_path):] or '/'

    @property
    def owner(self):
        path = os.path.join(self.base_path, self.path, 'tasks')
        uid = os.stat(path).st_uid
        try:
            return pwd.getpwuid(uid).pw_name
        except:
            return uid

    def _coerce(self, value):
        try:
            return int(value)
        except:
            pass

        try:
            return float(value)
        except:
            pass

        return value

    def __getitem__(self, name):
        path = os.path.join(self.base_path, self.path, name)

        with open(path) as f:
            content = f.read().strip()

        if name == 'tasks' or '\n' in content:
            content = content.split('\n')

            if ' ' in content[0]:
                content = dict((l.split(' ') for l in content))
                for k, v in content.iteritems():
                    content[k] = self._coerce(v)
            else:
                content = [self._coerce(v) for v in content]

        else:
            content = self._coerce(content)

        return content

def cgroups(base_path):
    '''
    Generator of cgroups under path ``name``
    '''
    for cgroup_path, dirs, files in os.walk(base_path):
        yield Cgroup(cgroup_path, base_path)

## Grab cgroup data

def init():
    # Get all cgroup subsystems avalaible on this system
    with open("/proc/cgroups") as f:
        cgroups = f.read().strip()

    subsystems = []
    for cgroup in cgroups.split('\n'):
        if cgroup[0] == '#': continue
        subsystems.append(cgroup.split()[0])

    # Match cgroup mountpoints to susbsytems. Always take the first matching
    with open("/proc/mounts") as f:
        mounts = f.read().strip()

    for mount in mounts.split('\n'):
        mount = mount.split(' ')

        if mount[2] != "cgroup":
            continue

        for arg in mount[3].split(','):
            if arg in subsystems and arg not in CGROUP_MOUNTPOINTS:
                CGROUP_MOUNTPOINTS[arg] = mount[1]

def collect(measures):
    cur = defaultdict(dict)
    prev = measures['data']

    # Collect global data
    if 'cpuacct' in CGROUP_MOUNTPOINTS:
        # list all "folders" under mountpoint
        for cgroup in cgroups(CGROUP_MOUNTPOINTS['cpuacct']):
            # Collect tasks
            cur[cgroup.name]['tasks'] = cgroup['tasks']

            # Collect user
            cur[cgroup.name]['owner'] = cgroup.owner

    # Collect memory statistics
    if 'memory' in CGROUP_MOUNTPOINTS:
        # list all "folders" under mountpoint
        for cgroup in cgroups(CGROUP_MOUNTPOINTS['memory']):
            cur[cgroup.name]['memory.usage_in_bytes'] = cgroup['memory.usage_in_bytes']
            cur[cgroup.name]['memory.max_usage_in_bytes'] = cgroup['memory.usage_in_bytes']
            cur[cgroup.name]['memory.limit_in_bytes'] = min(int(cgroup['memory.limit_in_bytes']), measures['global']['total_memory'])

    # Collect CPU statistics
    if 'cpuacct' in CGROUP_MOUNTPOINTS:
        # list all "folders" under mountpoint
        for cgroup in cgroups(CGROUP_MOUNTPOINTS['cpuacct']):
            # Collect CPU stats
            cur[cgroup.name]['cpuacct.stat'] = cgroup['cpuacct.stat']
            cur[cgroup.name]['cpuacct.stat.diff'] = {'user':0, 'system':0}

            # Collect CPU increase on run > 1
            if cgroup.name in prev:
                for key, value in cur[cgroup.name]['cpuacct.stat'].iteritems():
                    cur[cgroup.name]['cpuacct.stat.diff'][key] = value - prev[cgroup.name]['cpuacct.stat'][key]

    # Apply
    measures['data'] = cur

def display(scr, measures, conf):
    # Time
    prev_time = measures['global'].get('time', -1)
    cur_time = time.time()
    time_delta = cur_time - prev_time
    measures['global']['time'] = cur_time
    cpu_to_percent = measures['global']['scheduler_frequency'] * measures['global']['total_cpu'] * time_delta

    # Build data lines
    results = []
    for cgroup, data in measures['data'].iteritems():
        cpu_usage = data.get('cpuacct.stat.diff', {})
        line = {
            'owner': data.get('owner', 'nobody'),
            'tasks': len(data['tasks']),
            'memory_cur': "{: >7}/{: <7}".format(to_human(data.get('memory.usage_in_bytes', 0)), to_human(data.get('memory.limit_in_bytes', measures['global']['total_memory']))),
            'memory_peak': to_human(data.get('memory.max_usage_in_bytes', 0)),
            'cpu_syst': cpu_usage.get('system', 0) / cpu_to_percent,
            'cpu_user': cpu_usage.get('user', 0) / cpu_to_percent,
            'cgroup': cgroup,
        }
        line['cpu_total'] = line['cpu_syst'] + line['cpu_user']
        results.append(line)

    # Sort
    results = sorted(results, key=lambda line: line.get(conf['sort_by'], 0), reverse=not conf['sort_asc'])

    # Display statistics
    curses.endwin()
    height, width = scr.getmaxyx()
    LINE_TMPL = "{:"+str(width)+"s}"
    scr.addstr(0, 0, LINE_TMPL.format('OWNER      PROC     CURRENT       PEAK  SYSTEM USER CGROUP'), curses.color_pair(1))
    RES_TMPL = "{owner:10s} {tasks:4d} {memory_cur:15s} {memory_peak:>7s} {cpu_syst: >5.1%} {cpu_user: >5.1%} {cgroup}"

    lineno = 1
    for line in results:
        try:
            line = RES_TMPL.format(**line)
            scr.addstr(lineno, 0, LINE_TMPL.format(line))
        except:
            break
        lineno += 1

    scr.refresh()

def set_sort_col(sort_by):
    if CONFIGURATION['sort_by'] == sort_by:
        CONFIGURATION['sort_asc'] = not CONFIGURATION['sort_asc']
    else:
        CONFIGURATION['sort_by'] = sort_by

def on_keyboard(c):
    '''Handle keyborad shortcuts'''
    if c == ord('q'):
        raise KeyboardInterrupt()
    return 1

def on_mouse():
    '''Update selected line / sort'''
    _, x, y, z, bstate =  curses.getmouse()

    # Left button click ?
    if bstate & curses.BUTTON1_CLICKED:
        # Is it title line ?
        if y == 0:
            # "Guess" sort colum based on X
            # TODO: make it "discoverable"
            RES_TMPL = "{owner:10s} {tasks:4d} {memory_cur:15s} {memory_peak:>7s} {cpu_syst: >5.1%} {cpu_user: >5.1%} {cgroup}"
            if   x < 11: set_sort_col('owner')
            elif x < 16: set_sort_col('tasks')
            elif x < 32: set_sort_col('memory_cur')
            elif x < 40: set_sort_col('memory_peak')
            elif x < 47: set_sort_col('cpu_syst')
            elif x < 54: set_sort_col('cpu_user')
            else:        set_sort_col('cgroup')
            return 2
    return 1

def on_resize():
    '''Redraw screen, do not refresh'''
    return 1

def event_listener(scr, timeout):
    '''
    Wait for curses events on screen ``scr`` at mot ``timeout`` ms

    return
     - 1 OK
     - 2 redraw
     - 0 error
    '''
    try:
        scr.timeout(timeout)
        c = scr.getch()
        if c == -1:
            return 1
        elif c == curses.KEY_MOUSE:
            return on_mouse()
        elif c == curses.KEY_RESIZE:
            return on_resize()
        elif c < 256:
            return on_keyboard(c)
    except _curses.error:
        return 0

def main():
    # Initialization, global system data
    measures = {
        'data': defaultdict(dict),
        'global': {
            'total_cpu': psutil.cpu_count(),
            'total_memory': psutil.virtual_memory().total,
            'scheduler_frequency': os.sysconf('SC_CLK_TCK'),
        }
    }

    init()

    try:
        # Curse initialization
        stdscr = curses.initscr()
        curses.start_color() # load colors
        curses.use_default_colors()
        curses.noecho()      # do not echo text
        curses.cbreak()      # do not wait for "enter"
        curses.curs_set(0)   # hide cursor
        stdscr.keypad(1)     # parse keypad controll sequences
        curses.mousemask(curses.ALL_MOUSE_EVENTS)

        # Curses colors
        curses.init_pair(1, -1, curses.COLOR_GREEN)

        # Main loop
        while True:
            collect(measures)
            display(stdscr, measures, CONFIGURATION)
            sleep_start = time.time()
            while time.time() < sleep_start + UPDATE_INTERVAL:
                to_sleep = sleep_start + UPDATE_INTERVAL - time.time()
                ret = event_listener(stdscr, int(to_sleep*1000))
                if ret == 2:
                    display(stdscr, measures, CONFIGURATION)

    except KeyboardInterrupt:
        pass
    finally:
        curses.nocbreak()
        stdscr.keypad(0)
        curses.echo()
        curses.endwin()

if __name__ == "__main__":
    main()

<commit_msg>fix: derivative on screen resfresh<commit_after>#!/usr/bin/env python
'''
Iterate over all cgroup mountpoints and output global cgroup statistics
'''

import os
import sys
import stat
import pwd
import time
import psutil

from collections import defaultdict

try:
    import curses, _curses
except ImportError:
    print >> sys.stderr, "Curse is not available on this system. Exiting."
    sys.exit(0)

HIDE_EMPTY_CGROUP = True
UPDATE_INTERVAL = 1.0 # seconds
CGROUP_MOUNTPOINTS={}
CONFIGURATION = {
        'sort_by': 'cpu_total',
        'sort_asc': False,
}

# TODO:
# - select display colums
# - select refresh rate
# - detect container technology
# - visual CPU/memory usage
# - block-io
# - auto-color
# - adapt name / commands to underlying container system
# - hiereachical view
# - fix sort human column

## Utils

def to_human(num, suffix='B'):
    num = int(num)
    for unit in [' ','K','M','G','T','P','E','Z']:
        if abs(num) < 1024.0:
            return "{:.1f}{}{}".format(num, unit, suffix)
        num /= 1024.0
    return "{:5.1d}{}{}" % (num, 'Y', suffix)

class Cgroup(object):
    def __init__(self, path, base_path):
        self.path = path
        self.base_path = base_path

    @property
    def name(self):
        return self.path[len(self.base_path):] or '/'

    @property
    def owner(self):
        path = os.path.join(self.base_path, self.path, 'tasks')
        uid = os.stat(path).st_uid
        try:
            return pwd.getpwuid(uid).pw_name
        except:
            return uid

    def _coerce(self, value):
        try:
            return int(value)
        except:
            pass

        try:
            return float(value)
        except:
            pass

        return value

    def __getitem__(self, name):
        path = os.path.join(self.base_path, self.path, name)

        with open(path) as f:
            content = f.read().strip()

        if name == 'tasks' or '\n' in content:
            content = content.split('\n')

            if ' ' in content[0]:
                content = dict((l.split(' ') for l in content))
                for k, v in content.iteritems():
                    content[k] = self._coerce(v)
            else:
                content = [self._coerce(v) for v in content]

        else:
            content = self._coerce(content)

        return content

def cgroups(base_path):
    '''
    Generator of cgroups under path ``name``
    '''
    for cgroup_path, dirs, files in os.walk(base_path):
        yield Cgroup(cgroup_path, base_path)

## Grab cgroup data

def init():
    # Get all cgroup subsystems avalaible on this system
    with open("/proc/cgroups") as f:
        cgroups = f.read().strip()

    subsystems = []
    for cgroup in cgroups.split('\n'):
        if cgroup[0] == '#': continue
        subsystems.append(cgroup.split()[0])

    # Match cgroup mountpoints to susbsytems. Always take the first matching
    with open("/proc/mounts") as f:
        mounts = f.read().strip()

    for mount in mounts.split('\n'):
        mount = mount.split(' ')

        if mount[2] != "cgroup":
            continue

        for arg in mount[3].split(','):
            if arg in subsystems and arg not in CGROUP_MOUNTPOINTS:
                CGROUP_MOUNTPOINTS[arg] = mount[1]

def collect(measures):
    cur = defaultdict(dict)
    prev = measures['data']

    # Collect global data
    if 'cpuacct' in CGROUP_MOUNTPOINTS:
        # list all "folders" under mountpoint
        for cgroup in cgroups(CGROUP_MOUNTPOINTS['cpuacct']):
            # Collect tasks
            cur[cgroup.name]['tasks'] = cgroup['tasks']

            # Collect user
            cur[cgroup.name]['owner'] = cgroup.owner

    # Collect memory statistics
    if 'memory' in CGROUP_MOUNTPOINTS:
        # list all "folders" under mountpoint
        for cgroup in cgroups(CGROUP_MOUNTPOINTS['memory']):
            cur[cgroup.name]['memory.usage_in_bytes'] = cgroup['memory.usage_in_bytes']
            cur[cgroup.name]['memory.max_usage_in_bytes'] = cgroup['memory.usage_in_bytes']
            cur[cgroup.name]['memory.limit_in_bytes'] = min(int(cgroup['memory.limit_in_bytes']), measures['global']['total_memory'])

    # Collect CPU statistics
    if 'cpuacct' in CGROUP_MOUNTPOINTS:
        # list all "folders" under mountpoint
        for cgroup in cgroups(CGROUP_MOUNTPOINTS['cpuacct']):
            # Collect CPU stats
            cur[cgroup.name]['cpuacct.stat'] = cgroup['cpuacct.stat']
            cur[cgroup.name]['cpuacct.stat.diff'] = {'user':0, 'system':0}

            # Collect CPU increase on run > 1
            if cgroup.name in prev:
                for key, value in cur[cgroup.name]['cpuacct.stat'].iteritems():
                    cur[cgroup.name]['cpuacct.stat.diff'][key] = value - prev[cgroup.name]['cpuacct.stat'][key]

    # Apply
    measures['data'] = cur

def built_statistics(measures, conf):
    # Time
    prev_time = measures['global'].get('time', -1)
    cur_time = time.time()
    time_delta = cur_time - prev_time
    measures['global']['time'] = cur_time
    cpu_to_percent = measures['global']['scheduler_frequency'] * measures['global']['total_cpu'] * time_delta

    # Build data lines
    results = []
    for cgroup, data in measures['data'].iteritems():
        cpu_usage = data.get('cpuacct.stat.diff', {})
        line = {
            'owner': data.get('owner', 'nobody'),
            'tasks': len(data['tasks']),
            'memory_cur': "{: >7}/{: <7}".format(to_human(data.get('memory.usage_in_bytes', 0)), to_human(data.get('memory.limit_in_bytes', measures['global']['total_memory']))),
            'memory_peak': to_human(data.get('memory.max_usage_in_bytes', 0)),
            'cpu_syst': cpu_usage.get('system', 0) / cpu_to_percent,
            'cpu_user': cpu_usage.get('user', 0) / cpu_to_percent,
            'cgroup': cgroup,
        }
        line['cpu_total'] = line['cpu_syst'] + line['cpu_user']
        results.append(line)

    return results

def display(scr, results, conf):
    # Sort
    results = sorted(results, key=lambda line: line.get(conf['sort_by'], 0), reverse=not conf['sort_asc'])

    # Display statistics
    curses.endwin()
    height, width = scr.getmaxyx()
    LINE_TMPL = "{:"+str(width)+"s}"
    scr.addstr(0, 0, LINE_TMPL.format('OWNER      PROC     CURRENT       PEAK  SYSTEM USER CGROUP'), curses.color_pair(1))
    RES_TMPL = "{owner:10s} {tasks:4d} {memory_cur:15s} {memory_peak:>7s} {cpu_syst: >5.1%} {cpu_user: >5.1%} {cgroup}"

    lineno = 1
    for line in results:
        try:
            line = RES_TMPL.format(**line)
            scr.addstr(lineno, 0, LINE_TMPL.format(line))
        except:
            break
        lineno += 1

    scr.refresh()

def set_sort_col(sort_by):
    if CONFIGURATION['sort_by'] == sort_by:
        CONFIGURATION['sort_asc'] = not CONFIGURATION['sort_asc']
    else:
        CONFIGURATION['sort_by'] = sort_by

def on_keyboard(c):
    '''Handle keyborad shortcuts'''
    if c == ord('q'):
        raise KeyboardInterrupt()
    return 1

def on_mouse():
    '''Update selected line / sort'''
    _, x, y, z, bstate =  curses.getmouse()

    # Left button click ?
    if bstate & curses.BUTTON1_CLICKED:
        # Is it title line ?
        if y == 0:
            # "Guess" sort colum based on X
            # TODO: make it "discoverable"
            RES_TMPL = "{owner:10s} {tasks:4d} {memory_cur:15s} {memory_peak:>7s} {cpu_syst: >5.1%} {cpu_user: >5.1%} {cgroup}"
            if   x < 11: set_sort_col('owner')
            elif x < 16: set_sort_col('tasks')
            elif x < 32: set_sort_col('memory_cur')
            elif x < 40: set_sort_col('memory_peak')
            elif x < 47: set_sort_col('cpu_syst')
            elif x < 54: set_sort_col('cpu_user')
            else:        set_sort_col('cgroup')
            return 2
    return 1

def on_resize():
    '''Redraw screen, do not refresh'''
    return 1

def event_listener(scr, timeout):
    '''
    Wait for curses events on screen ``scr`` at mot ``timeout`` ms

    return
     - 1 OK
     - 2 redraw
     - 0 error
    '''
    try:
        scr.timeout(timeout)
        c = scr.getch()
        if c == -1:
            return 1
        elif c == curses.KEY_MOUSE:
            return on_mouse()
        elif c == curses.KEY_RESIZE:
            return on_resize()
        elif c < 256:
            return on_keyboard(c)
    except _curses.error:
        return 0

def main():
    # Initialization, global system data
    measures = {
        'data': defaultdict(dict),
        'global': {
            'total_cpu': psutil.cpu_count(),
            'total_memory': psutil.virtual_memory().total,
            'scheduler_frequency': os.sysconf('SC_CLK_TCK'),
        }
    }

    init()

    try:
        # Curse initialization
        stdscr = curses.initscr()
        curses.start_color() # load colors
        curses.use_default_colors()
        curses.noecho()      # do not echo text
        curses.cbreak()      # do not wait for "enter"
        curses.curs_set(0)   # hide cursor
        stdscr.keypad(1)     # parse keypad controll sequences
        curses.mousemask(curses.ALL_MOUSE_EVENTS)

        # Curses colors
        curses.init_pair(1, -1, curses.COLOR_GREEN)

        # Main loop
        while True:
            collect(measures)
            results = built_statistics(measures, CONFIGURATION)
            display(stdscr, results, CONFIGURATION)
            sleep_start = time.time()
            while time.time() < sleep_start + UPDATE_INTERVAL:
                to_sleep = sleep_start + UPDATE_INTERVAL - time.time()
                ret = event_listener(stdscr, int(to_sleep*1000))
                if ret == 2:
                    display(stdscr, results, CONFIGURATION)

    except KeyboardInterrupt:
        pass
    finally:
        curses.nocbreak()
        stdscr.keypad(0)
        curses.echo()
        curses.endwin()

if __name__ == "__main__":
    main()

<|endoftext|>


<commit_before>from marshmallow import fields

from microcosm_pubsub.codecs import PubSubMessageSchema

class URIMessageSchema(PubSubMessageSchema):
    """
    Define a baseline message schema that points to (the URI of) a resource that experienced a lifecycle change.

    By convention, pubsub messages should be *references* to something that happened within some other
    source-of-truth (e.g. a CRUD microservice). Because there are inherent race conditions between publishing
    a message and commit a persistent transaction, message consumers are expected to call back to source-of-truth
    (e.g. via HTTP) and fetch the current resource value. In the event that message has not been committed yet,
    the consumer can retry a few times before giving up (e.g. via SQS dead-lettering).

    """
    uri = fields.String(required=True)

class ChangedURIMessageSchema(URIMessageSchema):
    """
    Define a baseline message schema that points to the URI of a updated resource, with the updated value.

    By convention, pubsub messages are a reference to something which happened, but changed messages can
    be published (and handled) before the changes are committed to DB. Keeping the changed field on the message
    allows us to retry handling the message in case value doesn't match the one in the message.

    """
    field_name = fields.String()
    new_value = fields.String()

class IdentityMessageSchema(PubSubMessageSchema):
    """
    Define a baseline message schema that points to (the identity of) a resource that experienced a lifecycle change.

    In general, these conventions prefer URI-based messages, but for some lifecycle changes (e.g. deletion),
    a URI may not be available.

    """
    id = fields.String(required=True)
<commit_msg>Backward-compatibility for message schemas (#191)<commit_after>from marshmallow import fields

from microcosm_pubsub.codecs import PubSubMessageSchema

class URIMessageSchema(PubSubMessageSchema):
    """
    Define a baseline message schema that points to (the URI of) a resource that experienced a lifecycle change.

    By convention, pubsub messages should be *references* to something that happened within some other
    source-of-truth (e.g. a CRUD microservice). Because there are inherent race conditions between publishing
    a message and commit a persistent transaction, message consumers are expected to call back to source-of-truth
    (e.g. via HTTP) and fetch the current resource value. In the event that message has not been committed yet,
    the consumer can retry a few times before giving up (e.g. via SQS dead-lettering).

    """
    def __init__(self, media_type=None, **kwargs):
        super().__init__(**kwargs)
        # Left as an optional argument for backwards-compatibility
        if media_type is not None:
            self.MEDIA_TYPE = media_type

    uri = fields.String(required=True)

class ChangedURIMessageSchema(URIMessageSchema):
    """
    Define a baseline message schema that points to the URI of a updated resource, with the updated value.

    By convention, pubsub messages are a reference to something which happened, but changed messages can
    be published (and handled) before the changes are committed to DB. Keeping the changed field on the message
    allows us to retry handling the message in case value doesn't match the one in the message.

    """
    field_name = fields.String()
    new_value = fields.String()

class IdentityMessageSchema(PubSubMessageSchema):
    """
    Define a baseline message schema that points to (the identity of) a resource that experienced a lifecycle change.

    In general, these conventions prefer URI-based messages, but for some lifecycle changes (e.g. deletion),
    a URI may not be available.

    """
    def __init__(self, media_type=None, **kwargs):
        super().__init__(**kwargs)
        # Left as an optional argument for backwards-compatibility
        if media_type is not None:
            self.MEDIA_TYPE = media_type

    id = fields.String(required=True)
<|endoftext|>


<commit_before>import json

import pytest
from hamcrest import assert_that, contains_inanyorder as contains_in_any_order

@pytest.mark.django_db
def test_0019_forward(migrate, user):
    extraction_configuration_with_formats = {'gis_formats': ['txt', 'foobar'], 'gis_options': {'detail_level': 'standard'}}

    old_apps = migrate('excerptexport', '0017_auto_20160310_1102')
    ExtractionOrder = old_apps.get_model('excerptexport', 'ExtractionOrder')  # noqa
    extraction_order = ExtractionOrder.objects.create(orderer_id=user.id)
    extraction_order._extraction_configuration = json.dumps(extraction_configuration_with_formats)
    extraction_order.save()
    extraction_order_id = extraction_order.id

    new_apps = migrate('excerptexport', '0019_one_export_per_file_format')
    ExtractionOrder = new_apps.get_model('excerptexport', 'ExtractionOrder')  # noqa
    extraction_order_after_migration = ExtractionOrder.objects.get(id=extraction_order_id)
    assert_that(
        extraction_order_after_migration.exports.values_list('file_format', flat=True), contains_in_any_order(
            'txt',
            'foobar',
        )
    )

@pytest.mark.django_db
def test_0019_backward(migrate, user):
    extraction_configuration_with_formats = {'gis_formats': ['txt', 'foobar'], 'gis_options': {'detail_level': 'standard'}}

    old_apps = migrate('excerptexport', '0019_one_export_per_file_format')
    ExtractionOrder = old_apps.get_model('excerptexport', 'ExtractionOrder')  # noqa
    extraction_order = ExtractionOrder.objects.create(orderer_id=user.id)
    extraction_order._extraction_configuration = json.dumps({'gis_options': {'detail_level': 'standard'}})
    extraction_order.exports.create(file_format='txt')
    extraction_order.exports.create(file_format='foobar')
    extraction_order.save()
    extraction_order_id = extraction_order.id

    new_apps = migrate('excerptexport', '0017_auto_20160310_1102')
    ExtractionOrder = new_apps.get_model('excerptexport', 'ExtractionOrder')  # noqa
    extraction_order_after_migration = ExtractionOrder.objects.get(id=extraction_order_id)
    assert json.loads(extraction_order_after_migration._extraction_configuration) == extraction_configuration_with_formats
<commit_msg>refactoring: extract fixture for example extraction_configuration with formats<commit_after>import json

import pytest
from hamcrest import assert_that, contains_inanyorder as contains_in_any_order

@pytest.fixture
def extraction_configuration_with_formats():
    return {'gis_formats': ['txt', 'foobar'], 'gis_options': {'detail_level': 'standard'}}

@pytest.mark.django_db
def test_0019_forward(migrate, user, extraction_configuration_with_formats):
    old_apps = migrate('excerptexport', '0017_auto_20160310_1102')
    ExtractionOrder = old_apps.get_model('excerptexport', 'ExtractionOrder')  # noqa
    extraction_order = ExtractionOrder.objects.create(orderer_id=user.id)
    extraction_order._extraction_configuration = json.dumps(extraction_configuration_with_formats)
    extraction_order.save()
    extraction_order_id = extraction_order.id

    new_apps = migrate('excerptexport', '0019_one_export_per_file_format')
    ExtractionOrder = new_apps.get_model('excerptexport', 'ExtractionOrder')  # noqa
    extraction_order_after_migration = ExtractionOrder.objects.get(id=extraction_order_id)
    assert_that(
        extraction_order_after_migration.exports.values_list('file_format', flat=True), contains_in_any_order(
            'txt',
            'foobar',
        )
    )

@pytest.mark.django_db
def test_0019_backward(migrate, user, extraction_configuration_with_formats):
    old_apps = migrate('excerptexport', '0019_one_export_per_file_format')
    ExtractionOrder = old_apps.get_model('excerptexport', 'ExtractionOrder')  # noqa
    extraction_order = ExtractionOrder.objects.create(orderer_id=user.id)
    extraction_order._extraction_configuration = json.dumps({'gis_options': {'detail_level': 'standard'}})
    extraction_order.exports.create(file_format='txt')
    extraction_order.exports.create(file_format='foobar')
    extraction_order.save()
    extraction_order_id = extraction_order.id

    new_apps = migrate('excerptexport', '0017_auto_20160310_1102')
    ExtractionOrder = new_apps.get_model('excerptexport', 'ExtractionOrder')  # noqa
    extraction_order_after_migration = ExtractionOrder.objects.get(id=extraction_order_id)
    assert json.loads(extraction_order_after_migration._extraction_configuration) == extraction_configuration_with_formats
<|endoftext|>