<commit_before># Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import glob
import heapq
import logging
import os
import subprocess as subprocess
import shutil
import sys
import tempfile
import time

from telemetry.core import exceptions
from telemetry.core import util
from telemetry.core.backends import browser_backend
from telemetry.core.backends.chrome import chrome_browser_backend

class DesktopBrowserBackend(chrome_browser_backend.ChromeBrowserBackend):
  """The backend for controlling a locally-executed browser instance, on Linux,
  Mac or Windows.
  """
  def __init__(self, browser_options, executable, flash_path, is_content_shell,
               browser_directory, output_profile_path, extensions_to_load):
    super(DesktopBrowserBackend, self).__init__(
        is_content_shell=is_content_shell,
        supports_extensions=not is_content_shell,
        browser_options=browser_options,
        output_profile_path=output_profile_path,
        extensions_to_load=extensions_to_load)

    # Initialize fields so that an explosion during init doesn't break in Close.
    self._proc = None
    self._tmp_profile_dir = None
    self._tmp_output_file = None

    self._executable = executable
    if not self._executable:
      raise Exception('Cannot create browser, no executable found!')

    self._flash_path = flash_path
    if (browser_options.warn_if_no_flash
        and self._flash_path and not os.path.exists(self._flash_path)):
      logging.warning(('Could not find flash at %s. Running without flash.\n\n'
                       'To fix this see http://go/read-src-internal') %
                      self._flash_path)
      self._flash_path = None

    if len(extensions_to_load) > 0 and is_content_shell:
      raise browser_backend.ExtensionsNotSupportedException(
          'Content shell does not support extensions.')

    self._browser_directory = browser_directory
    self._port = None
    self._profile_dir = None
    self._tmp_minidump_dir = tempfile.mkdtemp()

    self._SetupProfile()

  def _SetupProfile(self):
    if not self.browser_options.dont_override_profile:
      if self._output_profile_path:
        # If both |_output_profile_path| and |profile_dir| are specified then
        # the calling code will throw an exception, so we don't need to worry
        # about that case here.
        self._tmp_profile_dir = self._output_profile_path
      else:
        self._tmp_profile_dir = tempfile.mkdtemp()
      profile_dir = self._profile_dir or self.browser_options.profile_dir
      if profile_dir:
        if self.is_content_shell:
          logging.critical('Profiles cannot be used with content shell')
          sys.exit(1)
        logging.info("Using profile directory:'%s'." % profile_dir)
        shutil.rmtree(self._tmp_profile_dir)
        shutil.copytree(profile_dir, self._tmp_profile_dir)

  def _LaunchBrowser(self):
    args = [self._executable]
    args.extend(self.GetBrowserStartupArgs())
    if self.browser_options.startup_url:
      args.append(self.browser_options.startup_url)
    env = os.environ.copy()
    env['CHROME_HEADLESS'] = '1'  # Don't upload minidumps.
    env['BREAKPAD_DUMP_LOCATION'] = self._tmp_minidump_dir
    logging.debug('Starting Chrome %s', args)
    if not self.browser_options.show_stdout:
      self._tmp_output_file = tempfile.NamedTemporaryFile('w', 0)
      self._proc = subprocess.Popen(
          args, stdout=self._tmp_output_file, stderr=subprocess.STDOUT, env=env)
    else:
      self._proc = subprocess.Popen(args, env=env)

    try:
      self._WaitForBrowserToComeUp()
      self._PostBrowserStartupInitialization()
    except:
      self.Close()
      raise

  def HasBrowserFinishedLaunching(self):
    # In addition to the functional check performed by the base class, quickly
    # check if the browser process is still alive.
    if not self.IsBrowserRunning():
      raise exceptions.ProcessGoneException(
          "Return code: %d" % self._proc.returncode)
    return super(DesktopBrowserBackend, self).HasBrowserFinishedLaunching()

  def GetBrowserStartupArgs(self):
    args = super(DesktopBrowserBackend, self).GetBrowserStartupArgs()
    self._port = util.GetUnreservedAvailableLocalPort()
    args.append('--remote-debugging-port=%i' % self._port)
    args.append('--enable-crash-reporter-for-testing')
    args.append('--use-mock-keychain')
    if not self.is_content_shell:
      args.append('--window-size=1280,1024')
      if self._flash_path:
        args.append('--ppapi-flash-path=%s' % self._flash_path)
      if not self.browser_options.dont_override_profile:
        args.append('--user-data-dir=%s' % self._tmp_profile_dir)
    return args

  def SetProfileDirectory(self, profile_dir):
    # Make sure _profile_dir hasn't already been set.
    assert self._profile_dir is None

    if self.is_content_shell:
      logging.critical('Profile creation cannot be used with content shell')
      sys.exit(1)

    self._profile_dir = profile_dir

  def Start(self):
    self._LaunchBrowser()

  @property
  def pid(self):
    if self._proc:
      return self._proc.pid
    return None

  @property
  def browser_directory(self):
    return self._browser_directory

  @property
  def profile_directory(self):
    return self._tmp_profile_dir

  def IsBrowserRunning(self):
    return self._proc and self._proc.poll() == None

  def GetStandardOutput(self):
    if not self._tmp_output_file:
      if self.browser_options.show_stdout:
        # This can happen in the case that loading the Chrome binary fails.
        # We print rather than using logging here, because that makes a
        # recursive call to this function.
        print >> sys.stderr, "Can't get standard output with --show_stdout"
      return ''
    self._tmp_output_file.flush()
    try:
      with open(self._tmp_output_file.name) as f:
        return f.read()
    except IOError:
      return ''

  def GetStackTrace(self):
    stackwalk = util.FindSupportBinary('minidump_stackwalk')
    if not stackwalk:
      logging.warning('minidump_stackwalk binary not found. Must build it to '
                      'symbolize crash dumps. Returning browser stdout.')
      return self.GetStandardOutput()

    dumps = glob.glob(os.path.join(self._tmp_minidump_dir, '*.dmp'))
    if not dumps:
      logging.warning('No crash dump found. Returning browser stdout.')
      return self.GetStandardOutput()
    most_recent_dump = heapq.nlargest(1, dumps, os.path.getmtime)[0]
    if os.path.getmtime(most_recent_dump) < (time.time() - (5 * 60)):
      logging.warning('Crash dump is older than 5 minutes. May not be correct.')

    symbols = glob.glob(os.path.join(self._browser_directory, '*.breakpad*'))
    if not symbols:
      logging.warning('No breakpad symbols found. Returning browser stdout.')
      return self.GetStandardOutput()

    minidump = most_recent_dump + '.stripped'
    with open(most_recent_dump, 'rb') as infile:
      with open(minidump, 'wb') as outfile:
        outfile.write(''.join(infile.read().partition('MDMP')[1:]))

    symbols_path = os.path.join(self._tmp_minidump_dir, 'symbols')
    for symbol in sorted(symbols, key=os.path.getmtime, reverse=True):
      if not os.path.isfile(symbol):
        continue
      with open(symbol, 'r') as f:
        fields = f.readline().split()
        if not fields:
          continue
        sha = fields[3]
        binary = ' '.join(fields[4:])
      symbol_path = os.path.join(symbols_path, binary, sha)
      if os.path.exists(symbol_path):
        continue
      os.makedirs(symbol_path)
      shutil.copyfile(symbol, os.path.join(symbol_path, binary + '.sym'))

    error = tempfile.NamedTemporaryFile('w', 0)
    return subprocess.Popen(
        [stackwalk, minidump, symbols_path],
        stdout=subprocess.PIPE, stderr=error).communicate()[0]

  def __del__(self):
    self.Close()

  def Close(self):
    super(DesktopBrowserBackend, self).Close()

    # First, try to politely shutdown.
    if self.IsBrowserRunning():
      self._proc.terminate()
      try:
        util.WaitFor(lambda: not self.IsBrowserRunning(), timeout=5)
        self._proc = None
      except util.TimeoutException:
        logging.warning('Failed to gracefully shutdown. Proceeding to kill.')

    # If it didn't comply, get more aggressive.
    if self.IsBrowserRunning():
      self._proc.kill()

    try:
      util.WaitFor(lambda: not self.IsBrowserRunning(), timeout=10)
    except util.TimeoutException:
      raise Exception('Could not shutdown the browser.')
    finally:
      self._proc = None

    if self._output_profile_path:
      # If we need the output then double check that it exists.
      if not (self._tmp_profile_dir and os.path.exists(self._tmp_profile_dir)):
        raise Exception("No profile directory generated by Chrome: '%s'." %
            self._tmp_profile_dir)
    else:
      # If we don't need the profile after the run then cleanup.
      if self._tmp_profile_dir and os.path.exists(self._tmp_profile_dir):
        shutil.rmtree(self._tmp_profile_dir, ignore_errors=True)
        self._tmp_profile_dir = None

    if self._tmp_output_file:
      self._tmp_output_file.close()
      self._tmp_output_file = None
<commit_msg>[Telemetry] Assert that browser is properly closed.<commit_after># Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import glob
import heapq
import logging
import os
import subprocess as subprocess
import shutil
import sys
import tempfile
import time

from telemetry.core import exceptions
from telemetry.core import util
from telemetry.core.backends import browser_backend
from telemetry.core.backends.chrome import chrome_browser_backend

class DesktopBrowserBackend(chrome_browser_backend.ChromeBrowserBackend):
  """The backend for controlling a locally-executed browser instance, on Linux,
  Mac or Windows.
  """
  def __init__(self, browser_options, executable, flash_path, is_content_shell,
               browser_directory, output_profile_path, extensions_to_load):
    super(DesktopBrowserBackend, self).__init__(
        is_content_shell=is_content_shell,
        supports_extensions=not is_content_shell,
        browser_options=browser_options,
        output_profile_path=output_profile_path,
        extensions_to_load=extensions_to_load)

    # Initialize fields so that an explosion during init doesn't break in Close.
    self._proc = None
    self._tmp_profile_dir = None
    self._tmp_output_file = None

    self._executable = executable
    if not self._executable:
      raise Exception('Cannot create browser, no executable found!')

    self._flash_path = flash_path
    if (browser_options.warn_if_no_flash
        and self._flash_path and not os.path.exists(self._flash_path)):
      logging.warning(('Could not find flash at %s. Running without flash.\n\n'
                       'To fix this see http://go/read-src-internal') %
                      self._flash_path)
      self._flash_path = None

    if len(extensions_to_load) > 0 and is_content_shell:
      raise browser_backend.ExtensionsNotSupportedException(
          'Content shell does not support extensions.')

    self._browser_directory = browser_directory
    self._port = None
    self._profile_dir = None
    self._tmp_minidump_dir = tempfile.mkdtemp()

    self._SetupProfile()

  def _SetupProfile(self):
    if not self.browser_options.dont_override_profile:
      if self._output_profile_path:
        # If both |_output_profile_path| and |profile_dir| are specified then
        # the calling code will throw an exception, so we don't need to worry
        # about that case here.
        self._tmp_profile_dir = self._output_profile_path
      else:
        self._tmp_profile_dir = tempfile.mkdtemp()
      profile_dir = self._profile_dir or self.browser_options.profile_dir
      if profile_dir:
        if self.is_content_shell:
          logging.critical('Profiles cannot be used with content shell')
          sys.exit(1)
        logging.info("Using profile directory:'%s'." % profile_dir)
        shutil.rmtree(self._tmp_profile_dir)
        shutil.copytree(profile_dir, self._tmp_profile_dir)

  def HasBrowserFinishedLaunching(self):
    # In addition to the functional check performed by the base class, quickly
    # check if the browser process is still alive.
    if not self.IsBrowserRunning():
      raise exceptions.ProcessGoneException(
          "Return code: %d" % self._proc.returncode)
    return super(DesktopBrowserBackend, self).HasBrowserFinishedLaunching()

  def GetBrowserStartupArgs(self):
    args = super(DesktopBrowserBackend, self).GetBrowserStartupArgs()
    self._port = util.GetUnreservedAvailableLocalPort()
    args.append('--remote-debugging-port=%i' % self._port)
    args.append('--enable-crash-reporter-for-testing')
    args.append('--use-mock-keychain')
    if not self.is_content_shell:
      args.append('--window-size=1280,1024')
      if self._flash_path:
        args.append('--ppapi-flash-path=%s' % self._flash_path)
      if not self.browser_options.dont_override_profile:
        args.append('--user-data-dir=%s' % self._tmp_profile_dir)
    return args

  def SetProfileDirectory(self, profile_dir):
    # Make sure _profile_dir hasn't already been set.
    assert self._profile_dir is None

    if self.is_content_shell:
      logging.critical('Profile creation cannot be used with content shell')
      sys.exit(1)

    self._profile_dir = profile_dir

  def Start(self):
    assert not self._proc, 'Must call Close() before Start()'

    args = [self._executable]
    args.extend(self.GetBrowserStartupArgs())
    if self.browser_options.startup_url:
      args.append(self.browser_options.startup_url)
    env = os.environ.copy()
    env['CHROME_HEADLESS'] = '1'  # Don't upload minidumps.
    env['BREAKPAD_DUMP_LOCATION'] = self._tmp_minidump_dir
    logging.debug('Starting Chrome %s', args)
    if not self.browser_options.show_stdout:
      self._tmp_output_file = tempfile.NamedTemporaryFile('w', 0)
      self._proc = subprocess.Popen(
          args, stdout=self._tmp_output_file, stderr=subprocess.STDOUT, env=env)
    else:
      self._proc = subprocess.Popen(args, env=env)

    try:
      self._WaitForBrowserToComeUp()
      self._PostBrowserStartupInitialization()
    except:
      self.Close()
      raise

  @property
  def pid(self):
    if self._proc:
      return self._proc.pid
    return None

  @property
  def browser_directory(self):
    return self._browser_directory

  @property
  def profile_directory(self):
    return self._tmp_profile_dir

  def IsBrowserRunning(self):
    return self._proc and self._proc.poll() == None

  def GetStandardOutput(self):
    if not self._tmp_output_file:
      if self.browser_options.show_stdout:
        # This can happen in the case that loading the Chrome binary fails.
        # We print rather than using logging here, because that makes a
        # recursive call to this function.
        print >> sys.stderr, "Can't get standard output with --show_stdout"
      return ''
    self._tmp_output_file.flush()
    try:
      with open(self._tmp_output_file.name) as f:
        return f.read()
    except IOError:
      return ''

  def GetStackTrace(self):
    stackwalk = util.FindSupportBinary('minidump_stackwalk')
    if not stackwalk:
      logging.warning('minidump_stackwalk binary not found. Must build it to '
                      'symbolize crash dumps. Returning browser stdout.')
      return self.GetStandardOutput()

    dumps = glob.glob(os.path.join(self._tmp_minidump_dir, '*.dmp'))
    if not dumps:
      logging.warning('No crash dump found. Returning browser stdout.')
      return self.GetStandardOutput()
    most_recent_dump = heapq.nlargest(1, dumps, os.path.getmtime)[0]
    if os.path.getmtime(most_recent_dump) < (time.time() - (5 * 60)):
      logging.warning('Crash dump is older than 5 minutes. May not be correct.')

    symbols = glob.glob(os.path.join(self._browser_directory, '*.breakpad*'))
    if not symbols:
      logging.warning('No breakpad symbols found. Returning browser stdout.')
      return self.GetStandardOutput()

    minidump = most_recent_dump + '.stripped'
    with open(most_recent_dump, 'rb') as infile:
      with open(minidump, 'wb') as outfile:
        outfile.write(''.join(infile.read().partition('MDMP')[1:]))

    symbols_path = os.path.join(self._tmp_minidump_dir, 'symbols')
    for symbol in sorted(symbols, key=os.path.getmtime, reverse=True):
      if not os.path.isfile(symbol):
        continue
      with open(symbol, 'r') as f:
        fields = f.readline().split()
        if not fields:
          continue
        sha = fields[3]
        binary = ' '.join(fields[4:])
      symbol_path = os.path.join(symbols_path, binary, sha)
      if os.path.exists(symbol_path):
        continue
      os.makedirs(symbol_path)
      shutil.copyfile(symbol, os.path.join(symbol_path, binary + '.sym'))

    error = tempfile.NamedTemporaryFile('w', 0)
    return subprocess.Popen(
        [stackwalk, minidump, symbols_path],
        stdout=subprocess.PIPE, stderr=error).communicate()[0]

  def __del__(self):
    self.Close()

  def Close(self):
    super(DesktopBrowserBackend, self).Close()

    # First, try to politely shutdown.
    if self.IsBrowserRunning():
      self._proc.terminate()
      try:
        util.WaitFor(lambda: not self.IsBrowserRunning(), timeout=5)
        self._proc = None
      except util.TimeoutException:
        logging.warning('Failed to gracefully shutdown. Proceeding to kill.')

    # If it didn't comply, get more aggressive.
    if self.IsBrowserRunning():
      self._proc.kill()

    try:
      util.WaitFor(lambda: not self.IsBrowserRunning(), timeout=10)
    except util.TimeoutException:
      raise Exception('Could not shutdown the browser.')
    finally:
      self._proc = None

    if self._output_profile_path:
      # If we need the output then double check that it exists.
      if not (self._tmp_profile_dir and os.path.exists(self._tmp_profile_dir)):
        raise Exception("No profile directory generated by Chrome: '%s'." %
            self._tmp_profile_dir)
    else:
      # If we don't need the profile after the run then cleanup.
      if self._tmp_profile_dir and os.path.exists(self._tmp_profile_dir):
        shutil.rmtree(self._tmp_profile_dir, ignore_errors=True)
        self._tmp_profile_dir = None

    if self._tmp_output_file:
      self._tmp_output_file.close()
      self._tmp_output_file = None
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
"""
Interfaces to specific cloud providers, using `Apache LibCloud <http://libcloud.apache.org>`
"""
# Copyright (C) 2011, 2012 ETH Zurich and University of Zurich. All rights reserved.
#
# Authors:
#   Riccardo Murri <riccardo.murri@gmail.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
__docformat__ = 'reStructuredText'
__version__ = '$Revision$'

# stdlib imports
from abc import abstractmethod
from copy import copy

# libcloud imports
import libcloud.compute.types
import libcloud.compute.providers

# local imports
from vmmad import log
from orchestrator import VmInfo

class Cloud:
    """
    Abstract base class describing the interface that a cloud provider
    should implement.
    """

    def __init__(self, image, kind):
        """
        Initialize a cloud provider instance.

        The `image` and `kind` arguments specify the features of the
        VM instances that are later created by `start_vm`.
        """
        pass

    @abstractmethod
    def start_vm(self, vm):
        """
        Start a new VM.

        Return a `VmInfo` object describing the started virtual
        machine, which can be passed to the `stop_vm` method to stop
        it later.
        """
        pass

    @abstractmethod
    def update_vm_status(self, vms):
        """
        Query cloud providers and update each `VmInfo` object in list
        `vms` *in place* with the current VM node status.
        """
        pass

    @abstractmethod
    def stop_vm(self, vm):
        """
        Stop a running VM.

        After this method has successfully completed, the VM must no
        longer be running, all of its resources have been freed, and
        -most importantly- nothing is being charged to the account
        used for initialization.

        The `vm` argument is a `VmInfo` object, on which previous
        `start_vm` call should have recorded instance information.
        """
        pass

    @staticmethod
    def _vminfo_state_from_libcloud_status(status):
        """
        Return the `orchestrator.VmInfo` state word corresponding to
        LibCloud's `NodeState`.
        """
        return {
            libcloud.compute.types.NodeState.RUNNING:    VmInfo.UP,
            libcloud.compute.types.NodeState.REBOOTING:  VmInfo.STARTING,
            libcloud.compute.types.NodeState.TERMINATED: VmInfo.DOWN,
            libcloud.compute.types.NodeState.PENDING:    VmInfo.STARTING,
            libcloud.compute.types.NodeState.UNKNOWN:    VmInfo.OTHER,
            }[status]

class DummyCloud(Cloud):
    """
    Interface `Apache LibCloud <http://libcloud.apache.org/>` "dummy" cloud provider.
    """

    def __init__(self, image='1', kind='1'):

        self.image = image
        self.kind = kind

        log.debug("Creating LibCloud's 'Dummy' provider ...")
        driver = libcloud.compute.providers.get_driver(libcloud.compute.types.Provider.DUMMY)
        self.provider = driver(0)
        # LibCloud's "dummy" provider always starts two instances; remove them
        for node in copy(self.provider.list_nodes()):
            node.destroy()
        log.info("Using cloud provider '%s'.", self.provider.__class__.__name__)

        log.info("Listing available images ...")
        self._images = dict((img.id, img) for img in self.provider.list_images())
        log.debug("Available images: %s", self._images.keys())
        if image not in self._images:
            raise RuntimeError("Image '%s' not available on %s"
                               % (image, self.provider.__class__.__name__))
        log.info("... done: %d images available.", len(self._images))

        log.info("Listing available kinds ...")
        self._kinds = dict((kind.id, kind) for kind in self.provider.list_sizes())
        log.debug("Available kinds: %s", self._kinds.keys())
        if kind not in self._kinds:
            raise RuntimeError("Kind '%s' not available on %s"
                               % (kind, self.provider.__class__.__name__))
        log.info("... done: %d kinds available.", len(self._kinds))

        log.info("VMs will use image '%s' (%s) on hardware kind '%s' (%s)",
                 self.image, self._images[self.image].name,
                 self.kind, self._kinds[self.kind].name)

        # associate the Node ID we get from the cloud provider with
        # the VM object we get from the orchestrator
        self._instance_to_vm_map = { }

    def start_vm(self, vm):
        vm.instance = self.provider.create_node(
            name=str(vm.vmid), image=self._images[self.image], size=self._kinds[self.kind])
        vm.cloud = self.provider
        self._instance_to_vm_map[vm.instance.uuid] = vm
        vm.state = VmInfo.UP

    def stop_vm(self, vm):
        id = vm.instance.uuid
        self.provider.destroy_node(vm.instance)
        del self._instance_to_vm_map[id]
        vm.state = VmInfo.DOWN

    def update_vm_status(self, vms):
        nodes = [ node for node in self.provider.list_nodes()
                  if node.id in self._instance_to_vm_map ]
        for node in nodes:
            vm = self._instance_to_vm_map[node.uuid]
            vm.instance = node
            vm.state = self._vminfo_state_from_libcloud_status(node.state)

class EC2Cloud(Cloud):
    """
    Interface to Amazon EC2 on top of `Apache LibCloud <http://libcloud.apache.org/>`.
    """

    def __init__(self, image, kind, access_id=None, secret_key=None):

        self.image = image
        self.kind = kind
        if access_id is not None:
            self.access_id = access_id
            self.secret_key = secret_key
        else:
            # use same environment variables as Boto
            self.access_id = os.environ['AWS_ACCESS_KEY_ID']
            self.secret_key = os.environ['AWS_SECRET_ACCESS_KEY']

        log.debug("Creating EC2 cloud provider with access ID '%s' ...", access_id)
        driver = libcloud.compute.providers.get_driver(libcloud.compute.types.Provider.EC2)
        self.provider = driver(self.access_id, self.secret_key)
        log.info("Using cloud provider '%s'.", self.provider.friendly_name)

        log.info("Listing available images ...")
        self._images = dict((img.id, img) for img in self.provider.list_images())
        if image not in self._images:
            raise RuntimeError("Image '%s' not available on %s"
                               % (image, self.provider.friendly_name))
        log.info("... done: %d images available.", len(self._images))

        log.info("Listing available sizes ...")
        self._kinds = dict((kind.id, kind) for kind in self.provider.list_sizes())
        if kind not in self._kinds:
            raise RuntimeError("Kind '%s' not available on %s"
                               % (kind, self.provider.friendly_name))
        log.info("... done: %d kinds available.", len(self._kinds))

        log.info("VMs will use image '%s' (%s) on hardware kind '%s' (%s)",
                 self.image, self._images[self.image].name,
                 self.kind, self._kinds[self.kind].name)

        # log.info("Getting list of running instances ...")
        # self.instances = dict((node.uuid, node) for node in self.provider.list_nodes())
        # log.info("... Done: %d instances available.", len(self._sizes))

        # associate the Node ID we get from the cloud provider with
        # the VM object we get from the orchestrator
        self._instance_to_vm_map = { }

    def start_vm(self, vm):
        vm.instance = self.provider.create_node(
            name=str(vm.vmid), image=self._images[self.image], size=self._kinds[self.kind])
        vm.cloud = self.provider
        self._instance_to_vm_map[vm.instance.uuid] = vm

    def stop_vm(self, vm):
        # XXX: this is tricky: we must:
        #   1. gracefully shutdown the node, and (after a timeout) proceed to:
        #   2. destroy the node
        # In addition this should not block the main Orchestrator thread.
        id = vm.instance.uuid
        self.provider.destroy_node(vm.instance)
        del self._instance_to_vm_map[id]

    def update_vm_status(self, vms):
        nodes = self.provider.list_nodes(ex_node_ids=[vm.instance.id for vm in vms])
        for node in nodes:
            vm = self._instance_to_vm_map[node.uuid]
            vm.instance = node
            vm.state = self._vminfo_state_from_libcloud_status(node.state)

<commit_msg>Use `vm.vmid` as a unique key, as it appears that libcloud.DummyCloud does not issue unique UUIDs...<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
"""
Interfaces to specific cloud providers, using `Apache LibCloud <http://libcloud.apache.org>`
"""
# Copyright (C) 2011, 2012 ETH Zurich and University of Zurich. All rights reserved.
#
# Authors:
#   Riccardo Murri <riccardo.murri@gmail.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
__docformat__ = 'reStructuredText'
__version__ = '$Revision$'

# stdlib imports
from abc import abstractmethod
from copy import copy

# libcloud imports
import libcloud.compute.types
import libcloud.compute.providers

# local imports
from vmmad import log
from orchestrator import VmInfo

class Cloud:
    """
    Abstract base class describing the interface that a cloud provider
    should implement.
    """

    def __init__(self, image, kind):
        """
        Initialize a cloud provider instance.

        The `image` and `kind` arguments specify the features of the
        VM instances that are later created by `start_vm`.
        """
        pass

    @abstractmethod
    def start_vm(self, vm):
        """
        Start a new VM.

        Return a `VmInfo` object describing the started virtual
        machine, which can be passed to the `stop_vm` method to stop
        it later.
        """
        pass

    @abstractmethod
    def update_vm_status(self, vms):
        """
        Query cloud providers and update each `VmInfo` object in list
        `vms` *in place* with the current VM node status.
        """
        pass

    @abstractmethod
    def stop_vm(self, vm):
        """
        Stop a running VM.

        After this method has successfully completed, the VM must no
        longer be running, all of its resources have been freed, and
        -most importantly- nothing is being charged to the account
        used for initialization.

        The `vm` argument is a `VmInfo` object, on which previous
        `start_vm` call should have recorded instance information.
        """
        pass

    @staticmethod
    def _vminfo_state_from_libcloud_status(status):
        """
        Return the `orchestrator.VmInfo` state word corresponding to
        LibCloud's `NodeState`.
        """
        return {
            libcloud.compute.types.NodeState.RUNNING:    VmInfo.UP,
            libcloud.compute.types.NodeState.REBOOTING:  VmInfo.STARTING,
            libcloud.compute.types.NodeState.TERMINATED: VmInfo.DOWN,
            libcloud.compute.types.NodeState.PENDING:    VmInfo.STARTING,
            libcloud.compute.types.NodeState.UNKNOWN:    VmInfo.OTHER,
            }[status]

class DummyCloud(Cloud):
    """
    Interface `Apache LibCloud <http://libcloud.apache.org/>` "dummy" cloud provider.
    """

    def __init__(self, image='1', kind='1'):

        self.image = image
        self.kind = kind

        log.debug("Creating LibCloud's 'Dummy' provider ...")
        driver = libcloud.compute.providers.get_driver(libcloud.compute.types.Provider.DUMMY)
        self.provider = driver(0)
        # LibCloud's "dummy" provider always starts two instances; remove them
        for node in copy(self.provider.list_nodes()):
            node.destroy()
        log.info("Using cloud provider '%s'.", self.provider.__class__.__name__)

        log.info("Listing available images ...")
        self._images = dict((img.id, img) for img in self.provider.list_images())
        log.debug("Available images: %s", self._images.keys())
        if image not in self._images:
            raise RuntimeError("Image '%s' not available on %s"
                               % (image, self.provider.__class__.__name__))
        log.info("... done: %d images available.", len(self._images))

        log.info("Listing available kinds ...")
        self._kinds = dict((kind.id, kind) for kind in self.provider.list_sizes())
        log.debug("Available kinds: %s", self._kinds.keys())
        if kind not in self._kinds:
            raise RuntimeError("Kind '%s' not available on %s"
                               % (kind, self.provider.__class__.__name__))
        log.info("... done: %d kinds available.", len(self._kinds))

        log.info("VMs will use image '%s' (%s) on hardware kind '%s' (%s)",
                 self.image, self._images[self.image].name,
                 self.kind, self._kinds[self.kind].name)

        # associate the Node ID we get from the cloud provider with
        # the VM object we get from the orchestrator
        self._instance_to_vm_map = { }

    def start_vm(self, vm):
        vm.instance = self.provider.create_node(
            name=str(vm.vmid), image=self._images[self.image], size=self._kinds[self.kind])
        assert vm.vmid not in self._instance_to_vm_map
        vm.cloud = self.provider
        self._instance_to_vm_map[vm.vmid] = vm
        vm.state = VmInfo.UP

    def stop_vm(self, vm):
        assert vm.vmid in self._instance_to_vm_map
        id = vm.vmid
        self.provider.destroy_node(vm.instance)
        del self._instance_to_vm_map[id]
        vm.state = VmInfo.DOWN

    def update_vm_status(self, vms):
        nodes = [ node for node in self.provider.list_nodes()
                  if node.id in self._instance_to_vm_map ]
        for node in nodes:
            vm = self._instance_to_vm_map[node.uuid]
            vm.instance = node
            vm.state = self._vminfo_state_from_libcloud_status(node.state)

class EC2Cloud(Cloud):
    """
    Interface to Amazon EC2 on top of `Apache LibCloud <http://libcloud.apache.org/>`.
    """

    def __init__(self, image, kind, access_id=None, secret_key=None):

        self.image = image
        self.kind = kind
        if access_id is not None:
            self.access_id = access_id
            self.secret_key = secret_key
        else:
            # use same environment variables as Boto
            self.access_id = os.environ['AWS_ACCESS_KEY_ID']
            self.secret_key = os.environ['AWS_SECRET_ACCESS_KEY']

        log.debug("Creating EC2 cloud provider with access ID '%s' ...", access_id)
        driver = libcloud.compute.providers.get_driver(libcloud.compute.types.Provider.EC2)
        self.provider = driver(self.access_id, self.secret_key)
        log.info("Using cloud provider '%s'.", self.provider.friendly_name)

        log.info("Listing available images ...")
        self._images = dict((img.id, img) for img in self.provider.list_images())
        if image not in self._images:
            raise RuntimeError("Image '%s' not available on %s"
                               % (image, self.provider.friendly_name))
        log.info("... done: %d images available.", len(self._images))

        log.info("Listing available sizes ...")
        self._kinds = dict((kind.id, kind) for kind in self.provider.list_sizes())
        if kind not in self._kinds:
            raise RuntimeError("Kind '%s' not available on %s"
                               % (kind, self.provider.friendly_name))
        log.info("... done: %d kinds available.", len(self._kinds))

        log.info("VMs will use image '%s' (%s) on hardware kind '%s' (%s)",
                 self.image, self._images[self.image].name,
                 self.kind, self._kinds[self.kind].name)

        # log.info("Getting list of running instances ...")
        # self.instances = dict((node.uuid, node) for node in self.provider.list_nodes())
        # log.info("... Done: %d instances available.", len(self._sizes))

        # associate the Node ID we get from the cloud provider with
        # the VM object we get from the orchestrator
        self._instance_to_vm_map = { }

    def start_vm(self, vm):
        vm.instance = self.provider.create_node(
            name=str(vm.vmid), image=self._images[self.image], size=self._kinds[self.kind])
        vm.cloud = self.provider
        self._instance_to_vm_map[vm.instance.uuid] = vm

    def stop_vm(self, vm):
        # XXX: this is tricky: we must:
        #   1. gracefully shutdown the node, and (after a timeout) proceed to:
        #   2. destroy the node
        # In addition this should not block the main Orchestrator thread.
        id = vm.instance.uuid
        self.provider.destroy_node(vm.instance)
        del self._instance_to_vm_map[id]

    def update_vm_status(self, vms):
        nodes = self.provider.list_nodes(ex_node_ids=[vm.instance.id for vm in vms])
        for node in nodes:
            vm = self._instance_to_vm_map[node.uuid]
            vm.instance = node
            vm.state = self._vminfo_state_from_libcloud_status(node.state)

<|endoftext|>


<commit_before># Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import glob
import heapq
import logging
import os
import subprocess as subprocess
import shutil
import sys
import tempfile
import time

from telemetry.core import util
from telemetry.core.backends import browser_backend
from telemetry.core.backends.chrome import chrome_browser_backend

class DesktopBrowserBackend(chrome_browser_backend.ChromeBrowserBackend):
  """The backend for controlling a locally-executed browser instance, on Linux,
  Mac or Windows.
  """
  def __init__(self, finder_options, executable, flash_path, is_content_shell,
               browser_directory, output_profile_path=None):
    super(DesktopBrowserBackend, self).__init__(
        is_content_shell=is_content_shell,
        supports_extensions=not is_content_shell,
        finder_options=finder_options)

    # Initialize fields so that an explosion during init doesn't break in Close.
    self._proc = None
    self._tmp_profile_dir = None
    self._tmp_output_file = None

    self._executable = executable
    if not self._executable:
      raise Exception('Cannot create browser, no executable found!')

    self._flash_path = flash_path
    if self._flash_path and not os.path.exists(self._flash_path):
      logging.warning(('Could not find flash at %s. Running without flash.\n\n'
                       'To fix this see http://go/read-src-internal') %
                      self._flash_path)
      self._flash_path = None

    if len(finder_options.extensions_to_load) > 0 and is_content_shell:
      raise browser_backend.ExtensionsNotSupportedException(
          'Content shell does not support extensions.')

    self._browser_directory = browser_directory
    self._port = util.GetAvailableLocalPort()
    self._profile_dir = None
    self._supports_net_benchmarking = True
    self._output_profile_path = output_profile_path
    self._tmp_minidump_dir = tempfile.mkdtemp()

    self._SetupProfile()

  def _SetupProfile(self):
    if not self.finder_options.dont_override_profile:
      if self._output_profile_path:
        # If both |_output_profile_path| and |profile_dir| are specified then
        # the calling code will throw an exception, so we don't need to worry
        # about that case here.
        self._tmp_profile_dir = self._output_profile_path
      else:
        self._tmp_profile_dir = tempfile.mkdtemp()
      profile_dir = self._profile_dir or self.finder_options.profile_dir
      if profile_dir:
        if self.is_content_shell:
          logging.critical('Profiles cannot be used with content shell')
          sys.exit(1)
        logging.info("Using profile directory:'%s'." % profile_dir)
        shutil.rmtree(self._tmp_profile_dir)
        shutil.copytree(profile_dir, self._tmp_profile_dir)

  def _LaunchBrowser(self):
    args = [self._executable]
    args.extend(self.GetBrowserStartupArgs())
    env = os.environ.copy()
    env['CHROME_HEADLESS'] = '1'  # Don't upload minidumps.
    env['BREAKPAD_DUMP_LOCATION'] = self._tmp_minidump_dir
    if not self.finder_options.show_stdout:
      self._tmp_output_file = tempfile.NamedTemporaryFile('w', 0)
      self._proc = subprocess.Popen(
          args, stdout=self._tmp_output_file, stderr=subprocess.STDOUT, env=env)
    else:
      self._proc = subprocess.Popen(args, env=env)

    try:
      self._WaitForBrowserToComeUp()
      self._PostBrowserStartupInitialization()
    except:
      self.Close()
      raise

  def GetBrowserStartupArgs(self):
    args = super(DesktopBrowserBackend, self).GetBrowserStartupArgs()
    args.append('--remote-debugging-port=%i' % self._port)
    args.append('--enable-crash-reporter-for-testing')
    # Temporarily disable compositing to see the effect on performance results.
    # http://crbug.com/281709
    args.append('--disable-force-compositing-mode')
    if not self.is_content_shell:
      args.append('--window-size=1280,1024')
      if self._flash_path:
        args.append('--ppapi-flash-path=%s' % self._flash_path)
      if self._supports_net_benchmarking:
        args.append('--enable-net-benchmarking')
      else:
        args.append('--enable-benchmarking')
      if not self.finder_options.dont_override_profile:
        args.append('--user-data-dir=%s' % self._tmp_profile_dir)
    return args

  def SetProfileDirectory(self, profile_dir):
    # Make sure _profile_dir hasn't already been set.
    assert self._profile_dir is None

    if self.is_content_shell:
      logging.critical('Profile creation cannot be used with content shell')
      sys.exit(1)

    self._profile_dir = profile_dir

  def Start(self):
    self._LaunchBrowser()

    # For old chrome versions, might have to relaunch to have the
    # correct net_benchmarking switch.
    if self.chrome_branch_number < 1418:
      self.Close()
      self._supports_net_benchmarking = False
      self._LaunchBrowser()

  @property
  def pid(self):
    if self._proc:
      return self._proc.pid
    return None

  @property
  def browser_directory(self):
    return self._browser_directory

  @property
  def profile_directory(self):
    return self._tmp_profile_dir

  def IsBrowserRunning(self):
    return self._proc.poll() == None

  def GetStandardOutput(self):
    assert self._tmp_output_file, "Can't get standard output with show_stdout"
    self._tmp_output_file.flush()
    try:
      with open(self._tmp_output_file.name) as f:
        return f.read()
    except IOError:
      return ''

  def GetStackTrace(self):
    stackwalk = util.FindSupportBinary('minidump_stackwalk')
    if not stackwalk:
      logging.warning('minidump_stackwalk binary not found. Must build it to '
                      'symbolize crash dumps. Returning browser stdout.')
      return self.GetStandardOutput()

    dumps = glob.glob(os.path.join(self._tmp_minidump_dir, '*.dmp'))
    if not dumps:
      logging.warning('No crash dump found. Returning browser stdout.')
      return self.GetStandardOutput()
    most_recent_dump = heapq.nlargest(1, dumps, os.path.getmtime)[0]
    if os.path.getmtime(most_recent_dump) < (time.time() - (5 * 60)):
      logging.warn('Crash dump is older than 5 minutes. May not be correct.')

    minidump = most_recent_dump + '.stripped'
    with open(most_recent_dump, 'rb') as infile:
      with open(minidump, 'wb') as outfile:
        outfile.write(''.join(infile.read().partition('MDMP')[1:]))

    symbols = glob.glob(os.path.join(os.path.dirname(stackwalk), '*.breakpad*'))
    if not symbols:
      logging.warning('No breakpad symbols found. Returning browser stdout.')
      return self.GetStandardOutput()

    symbols_path = os.path.join(self._tmp_minidump_dir, 'symbols')
    for symbol in symbols:
      with open(symbol, 'r') as f:
        fields = f.readline().split()
        if not fields:
          continue
        sha = fields[3]
        binary = ' '.join(fields[4:])
      symbol_path = os.path.join(symbols_path, binary, sha)
      os.makedirs(symbol_path)
      shutil.copyfile(symbol, os.path.join(symbol_path, binary + '.sym'))

    error = tempfile.NamedTemporaryFile('w', 0)
    return subprocess.Popen(
        [stackwalk, minidump, symbols_path],
        stdout=subprocess.PIPE, stderr=error).communicate()[0]

  def __del__(self):
    self.Close()

  def Close(self):
    super(DesktopBrowserBackend, self).Close()

    if self._proc:

      def IsClosed():
        if not self._proc:
          return True
        return self._proc.poll() != None

      # Try to politely shutdown, first.
      self._proc.terminate()
      try:
        util.WaitFor(IsClosed, timeout=1)
        self._proc = None
      except util.TimeoutException:
        pass

      # Kill it.
      if not IsClosed():
        self._proc.kill()
        try:
          util.WaitFor(IsClosed, timeout=5)
          self._proc = None
        except util.TimeoutException:
          self._proc = None
          raise Exception('Could not shutdown the browser.')

    if self._output_profile_path:
      # If we need the output then double check that it exists.
      if not (self._tmp_profile_dir and os.path.exists(self._tmp_profile_dir)):
        raise Exception("No profile directory generated by Chrome: '%s'." %
            self._tmp_profile_dir)
    else:
      # If we don't need the profile after the run then cleanup.
      if self._tmp_profile_dir and os.path.exists(self._tmp_profile_dir):
        shutil.rmtree(self._tmp_profile_dir, ignore_errors=True)
        self._tmp_profile_dir = None

    if self._tmp_output_file:
      self._tmp_output_file.close()
      self._tmp_output_file = None

  def CreateForwarder(self, *port_pairs):
    return browser_backend.DoNothingForwarder(*port_pairs)
<commit_msg>Revert 220704 "Run the perf bots with FCM disabled to see what t..."<commit_after># Copyright 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import glob
import heapq
import logging
import os
import subprocess as subprocess
import shutil
import sys
import tempfile
import time

from telemetry.core import util
from telemetry.core.backends import browser_backend
from telemetry.core.backends.chrome import chrome_browser_backend

class DesktopBrowserBackend(chrome_browser_backend.ChromeBrowserBackend):
  """The backend for controlling a locally-executed browser instance, on Linux,
  Mac or Windows.
  """
  def __init__(self, finder_options, executable, flash_path, is_content_shell,
               browser_directory, output_profile_path=None):
    super(DesktopBrowserBackend, self).__init__(
        is_content_shell=is_content_shell,
        supports_extensions=not is_content_shell,
        finder_options=finder_options)

    # Initialize fields so that an explosion during init doesn't break in Close.
    self._proc = None
    self._tmp_profile_dir = None
    self._tmp_output_file = None

    self._executable = executable
    if not self._executable:
      raise Exception('Cannot create browser, no executable found!')

    self._flash_path = flash_path
    if self._flash_path and not os.path.exists(self._flash_path):
      logging.warning(('Could not find flash at %s. Running without flash.\n\n'
                       'To fix this see http://go/read-src-internal') %
                      self._flash_path)
      self._flash_path = None

    if len(finder_options.extensions_to_load) > 0 and is_content_shell:
      raise browser_backend.ExtensionsNotSupportedException(
          'Content shell does not support extensions.')

    self._browser_directory = browser_directory
    self._port = util.GetAvailableLocalPort()
    self._profile_dir = None
    self._supports_net_benchmarking = True
    self._output_profile_path = output_profile_path
    self._tmp_minidump_dir = tempfile.mkdtemp()

    self._SetupProfile()

  def _SetupProfile(self):
    if not self.finder_options.dont_override_profile:
      if self._output_profile_path:
        # If both |_output_profile_path| and |profile_dir| are specified then
        # the calling code will throw an exception, so we don't need to worry
        # about that case here.
        self._tmp_profile_dir = self._output_profile_path
      else:
        self._tmp_profile_dir = tempfile.mkdtemp()
      profile_dir = self._profile_dir or self.finder_options.profile_dir
      if profile_dir:
        if self.is_content_shell:
          logging.critical('Profiles cannot be used with content shell')
          sys.exit(1)
        logging.info("Using profile directory:'%s'." % profile_dir)
        shutil.rmtree(self._tmp_profile_dir)
        shutil.copytree(profile_dir, self._tmp_profile_dir)

  def _LaunchBrowser(self):
    args = [self._executable]
    args.extend(self.GetBrowserStartupArgs())
    env = os.environ.copy()
    env['CHROME_HEADLESS'] = '1'  # Don't upload minidumps.
    env['BREAKPAD_DUMP_LOCATION'] = self._tmp_minidump_dir
    if not self.finder_options.show_stdout:
      self._tmp_output_file = tempfile.NamedTemporaryFile('w', 0)
      self._proc = subprocess.Popen(
          args, stdout=self._tmp_output_file, stderr=subprocess.STDOUT, env=env)
    else:
      self._proc = subprocess.Popen(args, env=env)

    try:
      self._WaitForBrowserToComeUp()
      self._PostBrowserStartupInitialization()
    except:
      self.Close()
      raise

  def GetBrowserStartupArgs(self):
    args = super(DesktopBrowserBackend, self).GetBrowserStartupArgs()
    args.append('--remote-debugging-port=%i' % self._port)
    args.append('--enable-crash-reporter-for-testing')
    if not self.is_content_shell:
      args.append('--window-size=1280,1024')
      if self._flash_path:
        args.append('--ppapi-flash-path=%s' % self._flash_path)
      if self._supports_net_benchmarking:
        args.append('--enable-net-benchmarking')
      else:
        args.append('--enable-benchmarking')
      if not self.finder_options.dont_override_profile:
        args.append('--user-data-dir=%s' % self._tmp_profile_dir)
    return args

  def SetProfileDirectory(self, profile_dir):
    # Make sure _profile_dir hasn't already been set.
    assert self._profile_dir is None

    if self.is_content_shell:
      logging.critical('Profile creation cannot be used with content shell')
      sys.exit(1)

    self._profile_dir = profile_dir

  def Start(self):
    self._LaunchBrowser()

    # For old chrome versions, might have to relaunch to have the
    # correct net_benchmarking switch.
    if self.chrome_branch_number < 1418:
      self.Close()
      self._supports_net_benchmarking = False
      self._LaunchBrowser()

  @property
  def pid(self):
    if self._proc:
      return self._proc.pid
    return None

  @property
  def browser_directory(self):
    return self._browser_directory

  @property
  def profile_directory(self):
    return self._tmp_profile_dir

  def IsBrowserRunning(self):
    return self._proc.poll() == None

  def GetStandardOutput(self):
    assert self._tmp_output_file, "Can't get standard output with show_stdout"
    self._tmp_output_file.flush()
    try:
      with open(self._tmp_output_file.name) as f:
        return f.read()
    except IOError:
      return ''

  def GetStackTrace(self):
    stackwalk = util.FindSupportBinary('minidump_stackwalk')
    if not stackwalk:
      logging.warning('minidump_stackwalk binary not found. Must build it to '
                      'symbolize crash dumps. Returning browser stdout.')
      return self.GetStandardOutput()

    dumps = glob.glob(os.path.join(self._tmp_minidump_dir, '*.dmp'))
    if not dumps:
      logging.warning('No crash dump found. Returning browser stdout.')
      return self.GetStandardOutput()
    most_recent_dump = heapq.nlargest(1, dumps, os.path.getmtime)[0]
    if os.path.getmtime(most_recent_dump) < (time.time() - (5 * 60)):
      logging.warn('Crash dump is older than 5 minutes. May not be correct.')

    minidump = most_recent_dump + '.stripped'
    with open(most_recent_dump, 'rb') as infile:
      with open(minidump, 'wb') as outfile:
        outfile.write(''.join(infile.read().partition('MDMP')[1:]))

    symbols = glob.glob(os.path.join(os.path.dirname(stackwalk), '*.breakpad*'))
    if not symbols:
      logging.warning('No breakpad symbols found. Returning browser stdout.')
      return self.GetStandardOutput()

    symbols_path = os.path.join(self._tmp_minidump_dir, 'symbols')
    for symbol in symbols:
      with open(symbol, 'r') as f:
        fields = f.readline().split()
        if not fields:
          continue
        sha = fields[3]
        binary = ' '.join(fields[4:])
      symbol_path = os.path.join(symbols_path, binary, sha)
      os.makedirs(symbol_path)
      shutil.copyfile(symbol, os.path.join(symbol_path, binary + '.sym'))

    error = tempfile.NamedTemporaryFile('w', 0)
    return subprocess.Popen(
        [stackwalk, minidump, symbols_path],
        stdout=subprocess.PIPE, stderr=error).communicate()[0]

  def __del__(self):
    self.Close()

  def Close(self):
    super(DesktopBrowserBackend, self).Close()

    if self._proc:

      def IsClosed():
        if not self._proc:
          return True
        return self._proc.poll() != None

      # Try to politely shutdown, first.
      self._proc.terminate()
      try:
        util.WaitFor(IsClosed, timeout=1)
        self._proc = None
      except util.TimeoutException:
        pass

      # Kill it.
      if not IsClosed():
        self._proc.kill()
        try:
          util.WaitFor(IsClosed, timeout=5)
          self._proc = None
        except util.TimeoutException:
          self._proc = None
          raise Exception('Could not shutdown the browser.')

    if self._output_profile_path:
      # If we need the output then double check that it exists.
      if not (self._tmp_profile_dir and os.path.exists(self._tmp_profile_dir)):
        raise Exception("No profile directory generated by Chrome: '%s'." %
            self._tmp_profile_dir)
    else:
      # If we don't need the profile after the run then cleanup.
      if self._tmp_profile_dir and os.path.exists(self._tmp_profile_dir):
        shutil.rmtree(self._tmp_profile_dir, ignore_errors=True)
        self._tmp_profile_dir = None

    if self._tmp_output_file:
      self._tmp_output_file.close()
      self._tmp_output_file = None

  def CreateForwarder(self, *port_pairs):
    return browser_backend.DoNothingForwarder(*port_pairs)
<|endoftext|>


<commit_before>#! /usr/bin/env python
#
"""
Simulate an `Orchestrator` run given some parameters.
"""
# Copyright (C) 2011, 2012 ETH Zurich and University of Zurich. All rights reserved.
#
# Authors:
#   Christian Panse <cp@fgcz.ethz.ch>
#   Riccardo Murri <riccardo.murri@gmail.com>
#   Tyanko Aleksiev <tyanko.alexiev@gmail.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from __future__ import absolute_import

__docformat__ = 'reStructuredText'
__version__ = "1.0dev (SVN $Revision$)"

# stdlib imports
from copy import copy
import random
import os
import sys
import argparse
import csv
import time
from time import mktime
from datetime import datetime

# local imports
from vmmad import log
from vmmad.provider.libcloud import DummyCloud
from orchestrator import Orchestrator, JobInfo, VmInfo

class OrchestratorSimulation(Orchestrator, DummyCloud):

    def __init__(self, max_vms, max_delta, max_idle, startup_delay,
                 job_number, min_duration, max_duration,
                 output_file, csv_file, start_time, time_interval, cluster_size):
        # implement the `Cloud` interface to simulate a cloud provider
        DummyCloud.__init__(self, '1', '1')

        # init the Orchestrator part, using `self` as cloud provider and batch system interface
        Orchestrator.__init__(self, cloud=self, batchsys=self,
                              max_vms=(max_vms+cluster_size),
                              max_delta=max_delta)

        # no jobs are running at the onset, all are pending
        self._running = [ ]

        #Random generated pending jobs
        #self._pending = [ JobInfo(jobid=random.randint(1,job_number*10),
        #                          state=JobInfo.PENDING,
        #                          duration=random.randint(min_duration, max_duration))
        #                  for _ in range(0,job_number) ]

        self._pending = [ JobInfo(jobid=0, state=JobInfo.PENDING, duration=1) ]

        # Set simulation settings
        self.max_idle = max_idle
        self.startup_delay = startup_delay
        self.cluster_size = cluster_size

        self.output_file = open(output_file, "wb")
        self.writer = csv.writer(self.output_file, delimiter=',')
        self.writer.writerow(
            ['#TimeStamp'] + ['Pending Jobs'] + ['Running Jobs'] + ['Started VMs'] + ['Idle VMS'])

        self.time_interval = int(time_interval)
        self._next_row = None

        # info about running VMs
        self._vmid = 0
        self._idle_vm_count = 0

        self._running = [ ]
        self._pending = [ JobInfo(jobid=1, state=JobInfo.PENDING, duration=0) ]

        # Convert starting time to UNIX time (may read the first CSV file line)
        if start_time is not None:
            self.starting_time = mktime(time.strptime(start_time, "%Y-%m-%dT%H:%M:%S" ))
        else:
            # use an impossible value that makes us accept all jobs
            # from the CSV file; will correct this later on
            self.starting_time = -1

        # auto-detect CSV delimiter, etc.
        with open(csv_file, 'r') as input_file:
            sample = input_file.read(1024)
            input_file.seek(0)
            rows = csv.DictReader(input_file,
                                  dialect = csv.Sniffer().sniff(sample))
            # load all jobs into memory, ordered by descending
            # submission time, so that the last item is the next job
            # to be submitted and we can just .pop() it
            self.__jobs = list(
                sorted((JobInfo(jobid=row['JOBID'],
                                state=JobInfo.PENDING,
                                submitted_at=float(row['SUBMITTED_AT']),
                                duration=float(row['RUN_DURATION']))
                        for row in rows if float(row['SUBMITTED_AT']) > self.starting_time),
                    # sort list of jobs by submission time
                    cmp=(lambda x,y: cmp(x.submitted_at, y.submitted_at)),
                    reverse=True))
            assert self.__jobs[0].submitted_at >= self.__jobs[1].submitted_at
        log.info("Loaded %d jobs from file '%s'", len(self.__jobs), csv_file)

        # if `starting_time` has not been set, then use earliest job
        # submission time as starting point
        if self.starting_time == -1:
            self.starting_time = self.__jobs[-1].submitted_at - self.time_interval
            log.info("Starting simulation at %s",
                     time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime(self.starting_time)))

    def update_job_status(self):
        # do regular work
        Orchestrator.update_job_status(self)
        # simulate job run time passing and stop finished jobs
        for job in copy(self._running):
            job.duration -= self.time_interval
            if job.duration <= 0:
                vm = job.vm
                vm.jobs.remove(job.jobid)
                self._running.remove(job)
                if not vm.ever_running:
                    self._idle_vm_count += 1
                    log.info("Job %s just finished; VM %s is now idle.",
                        job.jobid, vm.vmid)
        # simulate SGE scheduler starting a new job
        for vm in self._started_vms.values():
            if vm.last_idle > 0:
                if not self._pending:
                    break
                job = self._pending.pop()
                vm.jobs.add(job.jobid)
                job.vm = vm
                vm.last_idle = 0
                if not vm.ever_running:
                    if self._idle_vm_count > 0:
                        self._idle_vm_count -= 1
                self._running.append(job)
                log.info("Job %s just started running on VM %s.", job.jobid, vm.vmid)

    def before(self):
        if len(self._running) == 0 and len(self._pending) == 0 and len(self.__jobs) == 0:
            log.info("No more jobs, stopping here")
            self.output_file.close()
            sys.exit(0)

        self.writer.writerow(
            #  timestamp,         pending jobs,       running jobs,            started VMs,            idle VMs,
            [self.starting_time + (self.time_interval*self.cycle),
                                  len(self._pending), len(self._running),      len(self._started_vms), self._idle_vm_count])

        log.info("At time %d: pending jobs %d, running jobs %d, started VMs %d, idle VMs %d",
                     (self.starting_time + self.time_interval*self.cycle), len(self._pending), len(self._running), len(self._started_vms), self._idle_vm_count)

    ##
    ## Interface to the CSV file format
    ##
    def get_sched_info(self):
        now = self.starting_time + self.cycle * self.time_interval
        for job in self.next_jobs(now, now + self.time_interval):
            self._pending.append(job)
        return (self._running + self._pending)

    def next_jobs(self, since, until):
        while len(self.__jobs) > 0:
            job = self.__jobs.pop()
            if since > job.submitted_at:
                if job.submitted_at <= until:
                    yield job
                else:
                    # job is in the past (relative to the Simulator's conecpt of time) so ignore it
                    pass
            else:
                # job is in the future: put it back for next round
                self.__jobs.append(job)
                raise StopIteration

    ##
    ## policy implementation interface
    ##
    def is_cloud_candidate(self, job):
        # every job is a candidate in this simulation
        return True

    def is_new_vm_needed(self):
        if len(self._pending) > 2 * len(self._running):
            return True

    def can_vm_be_stopped(self, vm):
        if not vm.ever_running and (vm.last_idle > self.max_idle):
            return True
        else:
            return False

    ##
    ## (fake) cloud provider interface
    ##

    def start_vm(self, vm):
        DummyCloud.start_vm(self, vm)
        vm.been_running=0
        vm.total_idle=0

        # Setting the first "cluster_size" machines to be ever_running.
        if vm.vmid <= self.cluster_size:
            vm.ever_running = True
            vm.last_idle = 1
        else:
            vm.ever_running = False
            vm.last_idle=(-self.startup_delay)
        log.info("Started VM %d (%s).", vm.vmid, vm.instance.uuid)

    def update_vm_status(self, vms):
        DummyCloud.update_vm_status(self, vms)
        for vm in self._started_vms.values():
            if not vm.ever_running:
                vm.been_running += self.time_interval
                if not vm.jobs:
                    vm.total_idle += self.time_interval
                    vm.last_idle += self.time_interval
            else:
                if not vm.jobs:
                    vm.last_idle = 1

    def stop_vm(self, vm):
        log.info("Stopping VM %s (%s): has run for %d steps, been idle for %d of them",
                     vm.vmid, vm.instance.uuid, vm.been_running, vm.total_idle)
        DummyCloud.stop_vm(self, vm)
        if vm.last_idle > 0 and self._idle_vm_count > 0 and not vm.ever_running:
            self._idle_vm_count -= 1
        return True

if "__main__" == __name__:
    parser = argparse.ArgumentParser(description='Simulates a cloud orchestrator')
    parser.add_argument('--max-vms', '-mv', metavar='N', dest="max_vms", default=10, type=int, help="Maximum number of VMs to be started, default is %(default)s")
    parser.add_argument('--max-delta', '-md', metavar='N', dest="max_delta", default=1, type=int, help="Cap the number of VMs that can be started or stopped in a single orchestration cycle. Default is %(default)d.")
    parser.add_argument('--max-idle', '-mi', metavar='NUM_SECS', dest="max_idle", default=7200, type=int, help="Maximum idle time (in seconds) before swithing off a VM, default is %(default)s")
    parser.add_argument('--startup-delay', '-s', metavar='NUM_SECS', dest="startup_delay", default=3600, type=int, help="Time (in seconds) delay before staring up a VM, default is %(default)s")
    parser.add_argument('--job-number', '-jn', metavar='N', dest="job_number", default=50, type=int, help="Number of job to be started, default is %(default)s")
    parser.add_argument('--min-duration', '-mind', metavar='NUM_SECS', dest="min_duration", default=30, type=int, help="Lower bound for job's time (in seconds) execution, default is %(default)s")
    parser.add_argument('--max-duration', '-maxd', metavar='NUM_SECS', dest="max_duration", default=120, type=int, help="Upper bound for job's time (in seconds)  execution, default is %(default)s")
    parser.add_argument('--csv-file', '-csvf',  metavar='String', dest="csv_file", default="accounting.csv", help="File containing the CSV information, %(default)s")
    parser.add_argument('--output-file', '-o',  metavar='String', dest="output_file", default="main_sim.txt", help="File name where the output of the simulation will be stored, %(default)s")
    parser.add_argument('--cluster-size', '-cs',  metavar='NUM_CPUS', dest="cluster_size", default="20", type=int, help="Number of VMs, used for the simulation of real available cluster: %(default)s")
    parser.add_argument('--start-time', '-stime',  metavar='String', dest="start_time", default=None, help="Start time for the simulation, default: %(default)s")
    parser.add_argument('--time-interval', '-timei',  metavar='NUM_SECS', dest="time_interval", default="3600", help="UNIX interval in seconds used as parsing interval for the jobs in the CSV file, default: %(default)s")
    parser.add_argument('--version', '-V', action='version',
                        version=("%(prog)s version " + __version__))
    args = parser.parse_args()
    OrchestratorSimulation(args.max_vms, args.max_delta, args.max_idle, args.startup_delay, args.job_number, args.min_duration, args.max_duration, args.output_file, args.csv_file, args.start_time, args.time_interval, args.cluster_size).run(0)
<commit_msg>simul.py: Check that there are at least 2 jobs before verifying that ordering is correct.<commit_after>#! /usr/bin/env python
#
"""
Simulate an `Orchestrator` run given some parameters.
"""
# Copyright (C) 2011, 2012 ETH Zurich and University of Zurich. All rights reserved.
#
# Authors:
#   Christian Panse <cp@fgcz.ethz.ch>
#   Riccardo Murri <riccardo.murri@gmail.com>
#   Tyanko Aleksiev <tyanko.alexiev@gmail.com>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from __future__ import absolute_import

__docformat__ = 'reStructuredText'
__version__ = "1.0dev (SVN $Revision$)"

# stdlib imports
from copy import copy
import random
import os
import sys
import argparse
import csv
import time
from time import mktime
from datetime import datetime

# local imports
from vmmad import log
from vmmad.provider.libcloud import DummyCloud
from orchestrator import Orchestrator, JobInfo, VmInfo

class OrchestratorSimulation(Orchestrator, DummyCloud):

    def __init__(self, max_vms, max_delta, max_idle, startup_delay,
                 job_number, min_duration, max_duration,
                 output_file, csv_file, start_time, time_interval, cluster_size):
        # implement the `Cloud` interface to simulate a cloud provider
        DummyCloud.__init__(self, '1', '1')

        # init the Orchestrator part, using `self` as cloud provider and batch system interface
        Orchestrator.__init__(self, cloud=self, batchsys=self,
                              max_vms=(max_vms+cluster_size),
                              max_delta=max_delta)

        # no jobs are running at the onset, all are pending
        self._running = [ ]

        #Random generated pending jobs
        #self._pending = [ JobInfo(jobid=random.randint(1,job_number*10),
        #                          state=JobInfo.PENDING,
        #                          duration=random.randint(min_duration, max_duration))
        #                  for _ in range(0,job_number) ]

        self._pending = [ JobInfo(jobid=0, state=JobInfo.PENDING, duration=1) ]

        # Set simulation settings
        self.max_idle = max_idle
        self.startup_delay = startup_delay
        self.cluster_size = cluster_size

        self.output_file = open(output_file, "wb")
        self.writer = csv.writer(self.output_file, delimiter=',')
        self.writer.writerow(
            ['#TimeStamp'] + ['Pending Jobs'] + ['Running Jobs'] + ['Started VMs'] + ['Idle VMS'])

        self.time_interval = int(time_interval)
        self._next_row = None

        # info about running VMs
        self._vmid = 0
        self._idle_vm_count = 0

        self._running = [ ]
        self._pending = [ JobInfo(jobid=1, state=JobInfo.PENDING, duration=0) ]

        # Convert starting time to UNIX time (may read the first CSV file line)
        if start_time is not None:
            self.starting_time = mktime(time.strptime(start_time, "%Y-%m-%dT%H:%M:%S" ))
        else:
            # use an impossible value that makes us accept all jobs
            # from the CSV file; will correct this later on
            self.starting_time = -1

        # auto-detect CSV delimiter, etc.
        with open(csv_file, 'r') as input_file:
            sample = input_file.read(1024)
            input_file.seek(0)
            rows = csv.DictReader(input_file,
                                  dialect = csv.Sniffer().sniff(sample))
            # load all jobs into memory, ordered by descending
            # submission time, so that the last item is the next job
            # to be submitted and we can just .pop() it
            self.__jobs = list(
                sorted((JobInfo(jobid=row['JOBID'],
                                state=JobInfo.PENDING,
                                submitted_at=float(row['SUBMITTED_AT']),
                                duration=float(row['RUN_DURATION']))
                        for row in rows if float(row['SUBMITTED_AT']) > self.starting_time),
                    # sort list of jobs by submission time
                    cmp=(lambda x,y: cmp(x.submitted_at, y.submitted_at)),
                    reverse=True))
        log.info("Loaded %d jobs from file '%s'", len(self.__jobs), csv_file)
        assert len(self.__jobs) < 2 or (self.__jobs[0].submitted_at >= self.__jobs[1].submitted_at)

        # if `starting_time` has not been set, then use earliest job
        # submission time as starting point
        if self.starting_time == -1:
            self.starting_time = self.__jobs[-1].submitted_at - self.time_interval
            log.info("Starting simulation at %s",
                     time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime(self.starting_time)))

    def update_job_status(self):
        # do regular work
        Orchestrator.update_job_status(self)
        # simulate job run time passing and stop finished jobs
        for job in copy(self._running):
            job.duration -= self.time_interval
            if job.duration <= 0:
                vm = job.vm
                vm.jobs.remove(job.jobid)
                self._running.remove(job)
                if not vm.ever_running:
                    self._idle_vm_count += 1
                    log.info("Job %s just finished; VM %s is now idle.",
                        job.jobid, vm.vmid)
        # simulate SGE scheduler starting a new job
        for vm in self._started_vms.values():
            if vm.last_idle > 0:
                if not self._pending:
                    break
                job = self._pending.pop()
                vm.jobs.add(job.jobid)
                job.vm = vm
                vm.last_idle = 0
                if not vm.ever_running:
                    if self._idle_vm_count > 0:
                        self._idle_vm_count -= 1
                self._running.append(job)
                log.info("Job %s just started running on VM %s.", job.jobid, vm.vmid)

    def before(self):
        if len(self._running) == 0 and len(self._pending) == 0 and len(self.__jobs) == 0:
            log.info("No more jobs, stopping here")
            self.output_file.close()
            sys.exit(0)

        self.writer.writerow(
            #  timestamp,         pending jobs,       running jobs,            started VMs,            idle VMs,
            [self.starting_time + (self.time_interval*self.cycle),
                                  len(self._pending), len(self._running),      len(self._started_vms), self._idle_vm_count])

        log.info("At time %d: pending jobs %d, running jobs %d, started VMs %d, idle VMs %d",
                     (self.starting_time + self.time_interval*self.cycle), len(self._pending), len(self._running), len(self._started_vms), self._idle_vm_count)

    ##
    ## Interface to the CSV file format
    ##
    def get_sched_info(self):
        now = self.starting_time + self.cycle * self.time_interval
        for job in self.next_jobs(now, now + self.time_interval):
            self._pending.append(job)
        return (self._running + self._pending)

    def next_jobs(self, since, until):
        while len(self.__jobs) > 0:
            job = self.__jobs.pop()
            if since > job.submitted_at:
                if job.submitted_at <= until:
                    yield job
                else:
                    # job is in the past (relative to the Simulator's conecpt of time) so ignore it
                    pass
            else:
                # job is in the future: put it back for next round
                self.__jobs.append(job)
                raise StopIteration

    ##
    ## policy implementation interface
    ##
    def is_cloud_candidate(self, job):
        # every job is a candidate in this simulation
        return True

    def is_new_vm_needed(self):
        if len(self._pending) > 2 * len(self._running):
            return True

    def can_vm_be_stopped(self, vm):
        if not vm.ever_running and (vm.last_idle > self.max_idle):
            return True
        else:
            return False

    ##
    ## (fake) cloud provider interface
    ##

    def start_vm(self, vm):
        DummyCloud.start_vm(self, vm)
        vm.been_running=0
        vm.total_idle=0

        # Setting the first "cluster_size" machines to be ever_running.
        if vm.vmid <= self.cluster_size:
            vm.ever_running = True
            vm.last_idle = 1
        else:
            vm.ever_running = False
            vm.last_idle=(-self.startup_delay)
        log.info("Started VM %d (%s).", vm.vmid, vm.instance.uuid)

    def update_vm_status(self, vms):
        DummyCloud.update_vm_status(self, vms)
        for vm in self._started_vms.values():
            if not vm.ever_running:
                vm.been_running += self.time_interval
                if not vm.jobs:
                    vm.total_idle += self.time_interval
                    vm.last_idle += self.time_interval
            else:
                if not vm.jobs:
                    vm.last_idle = 1

    def stop_vm(self, vm):
        log.info("Stopping VM %s (%s): has run for %d steps, been idle for %d of them",
                     vm.vmid, vm.instance.uuid, vm.been_running, vm.total_idle)
        DummyCloud.stop_vm(self, vm)
        if vm.last_idle > 0 and self._idle_vm_count > 0 and not vm.ever_running:
            self._idle_vm_count -= 1
        return True

if "__main__" == __name__:
    parser = argparse.ArgumentParser(description='Simulates a cloud orchestrator')
    parser.add_argument('--max-vms', '-mv', metavar='N', dest="max_vms", default=10, type=int, help="Maximum number of VMs to be started, default is %(default)s")
    parser.add_argument('--max-delta', '-md', metavar='N', dest="max_delta", default=1, type=int, help="Cap the number of VMs that can be started or stopped in a single orchestration cycle. Default is %(default)d.")
    parser.add_argument('--max-idle', '-mi', metavar='NUM_SECS', dest="max_idle", default=7200, type=int, help="Maximum idle time (in seconds) before swithing off a VM, default is %(default)s")
    parser.add_argument('--startup-delay', '-s', metavar='NUM_SECS', dest="startup_delay", default=3600, type=int, help="Time (in seconds) delay before staring up a VM, default is %(default)s")
    parser.add_argument('--job-number', '-jn', metavar='N', dest="job_number", default=50, type=int, help="Number of job to be started, default is %(default)s")
    parser.add_argument('--min-duration', '-mind', metavar='NUM_SECS', dest="min_duration", default=30, type=int, help="Lower bound for job's time (in seconds) execution, default is %(default)s")
    parser.add_argument('--max-duration', '-maxd', metavar='NUM_SECS', dest="max_duration", default=120, type=int, help="Upper bound for job's time (in seconds)  execution, default is %(default)s")
    parser.add_argument('--csv-file', '-csvf',  metavar='String', dest="csv_file", default="accounting.csv", help="File containing the CSV information, %(default)s")
    parser.add_argument('--output-file', '-o',  metavar='String', dest="output_file", default="main_sim.txt", help="File name where the output of the simulation will be stored, %(default)s")
    parser.add_argument('--cluster-size', '-cs',  metavar='NUM_CPUS', dest="cluster_size", default="20", type=int, help="Number of VMs, used for the simulation of real available cluster: %(default)s")
    parser.add_argument('--start-time', '-stime',  metavar='String', dest="start_time", default=None, help="Start time for the simulation, default: %(default)s")
    parser.add_argument('--time-interval', '-timei',  metavar='NUM_SECS', dest="time_interval", default="3600", help="UNIX interval in seconds used as parsing interval for the jobs in the CSV file, default: %(default)s")
    parser.add_argument('--version', '-V', action='version',
                        version=("%(prog)s version " + __version__))
    args = parser.parse_args()
    OrchestratorSimulation(args.max_vms, args.max_delta, args.max_idle, args.startup_delay, args.job_number, args.min_duration, args.max_duration, args.output_file, args.csv_file, args.start_time, args.time_interval, args.cluster_size).run(0)
<|endoftext|>


<commit_before>from __future__ import division
import numpy as np
from mayavi import mlab #gateway to VTK
"""
trivial demo of Mayavi VTK visualiztion
using ionosphere-like space
Michael Hirsch

prereqs: conda install numpy mayavi
requires Python 2 due to mayavi and vtk
"""

def gsim(z0=200, h0=50, n0=1e12, zlim=(100,500),dz=2.,
                                 xlim=(-50,50),   dx=1.,
                                 ylim=(-100,100),  dy=1.,
                                 xsc=1/100, ysc=1/150):

    # keep the axes in x,y,z order instead of z,x,y.
    # must be mgrid, NOT ogrid
    x,y,z = np.mgrid[xlim[0]:xlim[1]+dx:dx,
                     ylim[0]:ylim[1]+dy:dy,
                     zlim[0]:zlim[1]+dz:dz]
    ne = chapman(z0,h0,n0,z)
    ne = modu(x,xsc,ne)
    ne = modu(y,ysc,ne)

    return ne,x,y,z

def chapman(z0,h0,n0,z):
    z1 = (z-z0)/h0
    return n0*np.exp(0.5*(1-z1-np.exp(-z1)))

def modu(p,mp,ne):
    return ne  *(0.5*np.cos(2*np.pi*mp*p)+1)

def plotsim(s, x,y,z):
    p1090 = np.percentile(s,(10,90)) #for axes limit
#%% transparent volume example
    """
    http://docs.enthought.com/mayavi/mayavi/auto/mlab_pipeline_other_functions.html
    can manipulate opacity and combine with mutable slices
    """

    scf = mlab.pipeline.scalar_field(x,y,z,s)
    fig1 = mlab.gcf()
    makevol(scf,p1090,fig1)
    figlbl(fig1)
#%% mutable slice
    fig2=mlab.figure()
    scf = mlab.pipeline.scalar_field(x,y,z,s)
    makeslice(scf,fig2)
    figlbl(fig2)
    mlab.show()

def makevol(scf,p1090,figh=None):
    """    transparent, pretty but hard to quantify    """
    mlab.pipeline.volume(scf,figure=figh,vmin=p1090[0],vmax=p1090[1])

def makeslice(scf,figh=None):
    """
     we can put multiple slices in one figure
     each slice can be flipped, twisted, and slid on the fly wuth mouse or touchscreen
    http://docs.enthought.com/mayavi/mayavi/auto/mlab_pipeline_other_functions.html
    """
    mlab.pipeline.image_plane_widget(scf, figure=figh,
                                     colormap="jet",
                                     plane_orientation='x_axes',
                                     slice_index=10,
                                     )

    mlab.pipeline.image_plane_widget(scf, figure=figh,
                                     plane_orientation="z_axes",
                                     slice_index=50)

def figlbl(figh=None):
    figh.scene.anti_aliasing_frames=0
    mlab.outline(figure=figh) #box around data axes
    mlab.orientation_axes(figure=figh)
    mlab.scalarbar()
    mlab.axes(figure=figh,xlabel="x (km)", ylabel="y (km)", zlabel="z (km)")

if __name__ == '__main__':
    ne, x,y,z = gsim()
    plotsim(ne, x,y,z)<commit_msg>cleanup' git push<commit_after>from __future__ import division
import numpy as np
from mayavi import mlab #gateway to VTK
"""
trivial demo of Mayavi VTK visualiztion
using ionosphere-like space
Michael Hirsch

prereqs: conda install numpy mayavi
requires Python 2 due to mayavi and vtk
"""

def gsim(z0=200, h0=50, n0=1e12, zlim=(100,500), dz=2,
                                 xlim=(-50,50),  dx=1,
                                 ylim=(-100,100),dy=1,
                                 xsc=1/100, ysc=1/150):

    # keep the axes in x,y,z order instead of z,x,y.
    # must be mgrid, NOT ogrid
    x,y,z = np.mgrid[xlim[0]:xlim[1]+dx:dx,
                     ylim[0]:ylim[1]+dy:dy,
                     zlim[0]:zlim[1]+dz:dz]
    ne = chapman(z0,h0,n0,z)
    ne = modu(x,xsc,ne)
    ne = modu(y,ysc,ne)

    return ne,x,y,z

def chapman(z0,h0,n0,z):
    z1 = (z-z0)/h0
    return n0*np.exp(0.5*(1-z1-np.exp(-z1)))

def modu(p,mp,ne):
    return ne  *(0.5*np.cos(2*np.pi*mp*p)+1)

def plotsim(s, x,y,z):
    p1090 = np.percentile(s, (10,90)) #for axes limit
#%% transparent volume example
    """
    http://docs.enthought.com/mayavi/mayavi/auto/mlab_pipeline_other_functions.html
    can manipulate opacity and combine with mutable slices
    """

    scf = mlab.pipeline.scalar_field(x,y,z,s)
    fig1 = mlab.gcf()
    makevol(scf,p1090,fig1)
    figlbl(fig1)
#%% mutable slice
    fig2=mlab.figure()
    scf = mlab.pipeline.scalar_field(x,y,z,s)
    makeslice(scf,fig2)
    figlbl(fig2)
    mlab.show()

def makevol(scf,p1090,figh=None):
    """    transparent, pretty but hard to quantify    """
    mlab.pipeline.volume(scf, figure=figh, vmin=p1090[0], vmax=p1090[1])

def makeslice(scf,figh=None):
    """
     we can put multiple slices in one figure
     each slice can be flipped, twisted, and slid on the fly wuth mouse or touchscreen
    http://docs.enthought.com/mayavi/mayavi/auto/mlab_pipeline_other_functions.html
    """
    mlab.pipeline.image_plane_widget(scf, figure=figh,
                                     colormap="jet",
                                     plane_orientation='x_axes',
                                     slice_index=10,
                                     )

    mlab.pipeline.image_plane_widget(scf, figure=figh,
                                     plane_orientation="z_axes",
                                     slice_index=50)

def figlbl(figh=None):
    figh.scene.anti_aliasing_frames=0
    mlab.outline(figure=figh) #box around data axes
    mlab.orientation_axes(figure=figh)
    mlab.scalarbar()
    mlab.axes(figure=figh,xlabel="x (km)", ylabel="y (km)", zlabel="z (km)")

if __name__ == '__main__':
    ne, x,y,z = gsim()
    plotsim(ne, x,y,z)<|endoftext|>


<commit_before>#class Transgression:
from configurator.configurator import Configurator
import os.path

def main():
  # Check to make sure we have a config file, or create one
  config = Configurator(os.path.join('.transgression', 'config.xml'), aDebugOutput=True)
  config.addSectionToConfig('GlobalOptions')
  config.addSectionToConfig('Targets')
  config.addSectionToConfig('Targets')

  globalOptionsSection = config.getTopLevelSection('GlobalOptions')
  globalOptionsSection.setOption('debug', 'on')
  globalOptionsSection.addSubSection('CoolGlobals')

  subSect = config.getSectionByPath('GlobalOptions.CoolGlobals')
  subSect.setOption('myName', 'Scott')

  print("Value of GlobalOptions.CoolGlobals.myName:" + config.getOptionByPath('GlobalOptions.CoolGlobals.myName').getValue())

  config.createSectionByPath('GlobalOptions.SomeSection')
  config.createOptionByPath('Something.Someone.Was.here', 'ohyeah')

if __name__ == '__main__':
  main()
<commit_msg>Issue #1: Added basic configuration reading capability to transgression, with the program feature that it searches for a configuration section called 'Binaries', and does nothing with it, aside from alert the user.<commit_after>#class Transgression:
from configurator.configurator import Configurator
import os.path

def promptUserForProgram():
  response = raw_input("transgression didn't find any binaries configured. Would you like to configure one now? ")
  if response.lower() == 'y':
    pass
  else:
    return

def main():
  # Check to make sure we have a config file, or create one
  config = Configurator(os.path.join('.transgression', 'config.xml'), aDebugOutput=True)

  # Configuration should include at least one binary
  try:
    section = config.getSectionByPath('Binaries')
  except:
    # We don't have any binaries listed, so perhaps we should prompt the user
    # for one?
    promptUserForProgram()

if __name__ == '__main__':
  main()
<|endoftext|>