<commit_before># coding=utf-8
# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import absolute_import, division, print_function, unicode_literals

import datetime
import os
import signal
import sys
import termios
import time
from builtins import str, zip
from contextlib import contextmanager

from setproctitle import setproctitle as set_process_title

from pants.base.build_environment import get_buildroot
from pants.base.exiter import Exiter
from pants.bin.local_pants_runner import LocalPantsRunner
from pants.init.util import clean_global_runtime_state
from pants.java.nailgun_io import NailgunStreamStdinReader, NailgunStreamWriter
from pants.java.nailgun_protocol import ChunkType, NailgunProtocol
from pants.pantsd.process_manager import ProcessManager
from pants.util.contextutil import HardSystemExit, hermetic_environment_as, stdio_as
from pants.util.socket import teardown_socket

class DaemonExiter(Exiter):
  """An Exiter that emits unhandled tracebacks and exit codes via the Nailgun protocol."""

  def __init__(self, socket):
    super(DaemonExiter, self).__init__()
    self._socket = socket
    self._finalizer = None

  def set_finalizer(self, finalizer):
    """Sets a finalizer that will be called before exiting."""
    self._finalizer = finalizer

  def exit(self, result=0, msg=None):
    """Exit the runtime."""
    if self._finalizer:
      try:
        self._finalizer()
      except Exception as e:
        try:
          NailgunProtocol.send_stderr(
            self._socket,
            '\nUnexpected exception in finalizer: {!r}\n'.format(e)
          )
        except Exception:
          pass

    try:
      # Write a final message to stderr if present.
      if msg:
        NailgunProtocol.send_stderr(self._socket, msg)

      # Send an Exit chunk with the result.
      NailgunProtocol.send_exit(self._socket, str(result).encode('ascii'))

      # Shutdown the connected socket.
      teardown_socket(self._socket)
    finally:
      # N.B. Assuming a fork()'d child, cause os._exit to be called here to avoid the routine
      # sys.exit behavior (via `pants.util.contextutil.hard_exit_handler()`).
      raise HardSystemExit()

class DaemonPantsRunner(ProcessManager):
  """A daemonizing PantsRunner that speaks the nailgun protocol to a remote client.

  N.B. this class is primarily used by the PailgunService in pantsd.
  """

  @classmethod
  def create(cls, sock, args, env, fork_lock, scheduler_service):
    try:
      # N.B. This will temporarily redirect stdio in the daemon's pre-fork context
      # to the nailgun session. We'll later do this a second time post-fork, because
      # threads.
      with cls.nailgunned_stdio(sock, env, handle_stdin=False):
        options, build_config, options_bootstrapper = LocalPantsRunner.parse_options(args, env)
        subprocess_dir = options.for_global_scope().pants_subprocessdir
        graph_helper, target_roots = scheduler_service.prefork(options, build_config)
        deferred_exc = None
    except Exception:
      deferred_exc = sys.exc_info()
      graph_helper = None
      target_roots = None
      options_bootstrapper = None
      # N.B. This will be overridden with the correct value if options
      # parsing is successful - otherwise it permits us to run just far
      # enough to raise the deferred exception.
      subprocess_dir = os.path.join(get_buildroot(), '.pids')

    return cls(
      sock,
      args,
      env,
      fork_lock,
      graph_helper,
      target_roots,
      subprocess_dir,
      options_bootstrapper,
      deferred_exc
    )

  def __init__(self, socket, args, env, fork_lock, graph_helper, target_roots,
               metadata_base_dir, options_bootstrapper, deferred_exc):
    """
    :param socket socket: A connected socket capable of speaking the nailgun protocol.
    :param list args: The arguments (i.e. sys.argv) for this run.
    :param dict env: The environment (i.e. os.environ) for this run.
    :param threading.RLock fork_lock: A lock to use during forking for thread safety.
    :param LegacyGraphSession graph_helper: The LegacyGraphSession instance to use for BuildGraph
                                            construction. In the event of an exception, this will be
                                            None.
    :param TargetRoots target_roots: The `TargetRoots` for this run.
    :param str metadata_base_dir: The ProcessManager metadata_base_dir from options.
    :param OptionsBootstrapper options_bootstrapper: An OptionsBootstrapper to reuse.
    :param Exception deferred_exception: A deferred exception from the daemon's pre-fork context.
                                         If present, this will be re-raised in the client context.
    """
    super(DaemonPantsRunner, self).__init__(
      name=self._make_identity(),
      metadata_base_dir=metadata_base_dir
    )
    self._socket = socket
    self._args = args
    self._env = env
    self._fork_lock = fork_lock
    self._graph_helper = graph_helper
    self._target_roots = target_roots
    self._options_bootstrapper = options_bootstrapper
    self._deferred_exception = deferred_exc

    self._exiter = DaemonExiter(socket)

  def _make_identity(self):
    """Generate a ProcessManager identity for a given pants run.

    This provides for a reasonably unique name e.g. 'pantsd-run-2015-09-16T23_17_56_581899'.
    """
    return 'pantsd-run-{}'.format(datetime.datetime.now().strftime('%Y-%m-%dT%H_%M_%S_%f'))

  @classmethod
  @contextmanager
  def _tty_stdio(cls, env):
    """Handles stdio redirection in the case of all stdio descriptors being the same tty."""
    # If all stdio is a tty, there's only one logical I/O device (the tty device). This happens to
    # be addressable as a file in OSX and Linux, so we take advantage of that and directly open the
    # character device for output redirection - eliminating the need to directly marshall any
    # interactive stdio back/forth across the socket and permitting full, correct tty control with
    # no middle-man.
    stdin_ttyname, stdout_ttyname, stderr_ttyname = NailgunProtocol.ttynames_from_env(env)
    assert stdin_ttyname == stdout_ttyname == stderr_ttyname, (
      'expected all stdio ttys to be the same, but instead got: {}\n'
      'please file a bug at http://github.com/pantsbuild/pants'
      .format([stdin_ttyname, stdout_ttyname, stderr_ttyname])
    )
    with open(stdin_ttyname, 'rb+wb', 0) as tty:
      tty_fileno = tty.fileno()
      with stdio_as(stdin_fd=tty_fileno, stdout_fd=tty_fileno, stderr_fd=tty_fileno):
        def finalizer():
          termios.tcdrain(tty_fileno)
        yield finalizer

  @classmethod
  @contextmanager
  def _pipe_stdio(cls, sock, stdin_isatty, stdout_isatty, stderr_isatty, handle_stdin):
    """Handles stdio redirection in the case of pipes and/or mixed pipes and ttys."""
    stdio_writers = (
      (ChunkType.STDOUT, stdout_isatty),
      (ChunkType.STDERR, stderr_isatty)
    )
    types, ttys = zip(*(stdio_writers))

    @contextmanager
    def maybe_handle_stdin(want):
      if want:
        # TODO: Launching this thread pre-fork to handle @rule input currently results
        # in an unhandled SIGILL in `src/python/pants/engine/scheduler.py, line 313 in pre_fork`.
        # More work to be done here in https://github.com/pantsbuild/pants/issues/6005
        with NailgunStreamStdinReader.open(sock, stdin_isatty) as fd:
          yield fd
      else:
        with open('/dev/null', 'rb') as fh:
          yield fh.fileno()

    with maybe_handle_stdin(handle_stdin) as stdin_fd,\
         NailgunStreamWriter.open_multi(sock, types, ttys) as ((stdout_fd, stderr_fd), writer),\
         stdio_as(stdout_fd=stdout_fd, stderr_fd=stderr_fd, stdin_fd=stdin_fd):
      # N.B. This will be passed to and called by the `DaemonExiter` prior to sending an
      # exit chunk, to avoid any socket shutdown vs write races.
      stdout, stderr = sys.stdout, sys.stderr
      def finalizer():
        try:
          stdout.flush()
          stderr.flush()
        finally:
          time.sleep(.001)  # HACK: Sleep 1ms in the main thread to free the GIL.
          writer.stop()
          writer.join()
          stdout.close()
          stderr.close()
      yield finalizer

  @classmethod
  @contextmanager
  def nailgunned_stdio(cls, sock, env, handle_stdin=True):
    """Redirects stdio to the connected socket speaking the nailgun protocol."""
    # Determine output tty capabilities from the environment.
    stdin_isatty, stdout_isatty, stderr_isatty = NailgunProtocol.isatty_from_env(env)
    is_tty_capable = all((stdin_isatty, stdout_isatty, stderr_isatty))

    if is_tty_capable:
      with cls._tty_stdio(env) as finalizer:
        yield finalizer
    else:
      with cls._pipe_stdio(
        sock,
        stdin_isatty,
        stdout_isatty,
        stderr_isatty,
        handle_stdin
      ) as finalizer:
        yield finalizer

  def _setup_sigint_handler(self):
    """Sets up a control-c signal handler for the daemon runner context."""
    def handle_control_c(signum, frame):
      raise KeyboardInterrupt('remote client sent control-c!')
    signal.signal(signal.SIGINT, handle_control_c)

  def _raise_deferred_exc(self):
    """Raises deferred exceptions from the daemon's synchronous path in the post-fork client."""
    if self._deferred_exception:
      try:
        # Expect `_deferred_exception` to be a 3-item tuple of the values returned by sys.exc_info().
        # This permits use the 3-arg form of the `raise` statement to preserve the original traceback.
        exc_type, exc_value, exc_traceback = self._deferred_exception
        raise exc_type, exc_value, exc_traceback
      except ValueError:
        # If `_deferred_exception` isn't a 3-item tuple, treat it like a bare exception.
        raise self._deferred_exception

  def _maybe_get_client_start_time_from_env(self, env):
    client_start_time = env.pop('PANTSD_RUNTRACKER_CLIENT_START_TIME', None)
    return None if client_start_time is None else float(client_start_time)

  def run(self):
    """Fork, daemonize and invoke self.post_fork_child() (via ProcessManager)."""
    with self._fork_lock:
      self.daemonize(write_pid=False)

  def pre_fork(self):
    """Pre-fork callback executed via ProcessManager.daemonize().

    The scheduler has thread pools which need to be re-initialized after a fork: this ensures that
    when the pantsd-runner forks from pantsd, there is a working pool for any work that happens
    in that child process.
    """
    if self._graph_helper:
      self._graph_helper.scheduler_session.pre_fork()

  def post_fork_child(self):
    """Post-fork child process callback executed via ProcessManager.daemonize()."""
    # Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception
    # hook with socket-specific behavior. Note that this intentionally points the faulthandler
    # trace stream to sys.stderr, which at this point is still a _LoggerStream object writing to
    # the `pantsd.log`. This ensures that in the event of e.g. a hung but detached pantsd-runner
    # process that the stacktrace output lands deterministically in a known place vs to a stray
    # terminal window.
    self._exiter.set_except_hook(sys.stderr)

    # Ensure anything referencing sys.argv inherits the Pailgun'd args.
    sys.argv = self._args

    # Set context in the process title.
    set_process_title('pantsd-runner [{}]'.format(' '.join(self._args)))

    # Setup a SIGINT signal handler.
    self._setup_sigint_handler()

    # Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
    # they can send signals (e.g. SIGINT) to all processes in the runners process group.
    pid = str(os.getpgrp() * -1).encode('ascii')
    NailgunProtocol.send_pid(self._socket, pid)

    # Invoke a Pants run with stdio redirected and a proxied environment.
    with self.nailgunned_stdio(self._socket, self._env) as finalizer,\
         hermetic_environment_as(**self._env):
      try:
        # Setup the Exiter's finalizer.
        self._exiter.set_finalizer(finalizer)

        # Clean global state.
        clean_global_runtime_state(reset_subsystem=True)

        # Re-raise any deferred exceptions, if present.
        self._raise_deferred_exc()

        # Otherwise, conduct a normal run.
        runner = LocalPantsRunner.create(
          self._exiter,
          self._args,
          self._env,
          self._target_roots,
          self._graph_helper,
          self._options_bootstrapper
        )
        runner.set_start_time(self._maybe_get_client_start_time_from_env(self._env))
        runner.run()
      except KeyboardInterrupt:
        self._exiter.exit(1, msg='Interrupted by user.\n')
      except Exception:
        self._exiter.handle_unhandled_exception(add_newline=True)
      else:
        self._exiter.exit(0)
<commit_msg>Fix syntax issue with raising error (#6245)<commit_after># coding=utf-8
# Copyright 2015 Pants project contributors (see CONTRIBUTORS.md).
# Licensed under the Apache License, Version 2.0 (see LICENSE).

from __future__ import absolute_import, division, print_function, unicode_literals

import datetime
import os
import signal
import sys
import termios
import time
from builtins import str, zip
from contextlib import contextmanager

from future.utils import raise_with_traceback
from setproctitle import setproctitle as set_process_title

from pants.base.build_environment import get_buildroot
from pants.base.exiter import Exiter
from pants.bin.local_pants_runner import LocalPantsRunner
from pants.init.util import clean_global_runtime_state
from pants.java.nailgun_io import NailgunStreamStdinReader, NailgunStreamWriter
from pants.java.nailgun_protocol import ChunkType, NailgunProtocol
from pants.pantsd.process_manager import ProcessManager
from pants.util.contextutil import HardSystemExit, hermetic_environment_as, stdio_as
from pants.util.socket import teardown_socket

class DaemonExiter(Exiter):
  """An Exiter that emits unhandled tracebacks and exit codes via the Nailgun protocol."""

  def __init__(self, socket):
    super(DaemonExiter, self).__init__()
    self._socket = socket
    self._finalizer = None

  def set_finalizer(self, finalizer):
    """Sets a finalizer that will be called before exiting."""
    self._finalizer = finalizer

  def exit(self, result=0, msg=None):
    """Exit the runtime."""
    if self._finalizer:
      try:
        self._finalizer()
      except Exception as e:
        try:
          NailgunProtocol.send_stderr(
            self._socket,
            '\nUnexpected exception in finalizer: {!r}\n'.format(e)
          )
        except Exception:
          pass

    try:
      # Write a final message to stderr if present.
      if msg:
        NailgunProtocol.send_stderr(self._socket, msg)

      # Send an Exit chunk with the result.
      NailgunProtocol.send_exit(self._socket, str(result).encode('ascii'))

      # Shutdown the connected socket.
      teardown_socket(self._socket)
    finally:
      # N.B. Assuming a fork()'d child, cause os._exit to be called here to avoid the routine
      # sys.exit behavior (via `pants.util.contextutil.hard_exit_handler()`).
      raise HardSystemExit()

class DaemonPantsRunner(ProcessManager):
  """A daemonizing PantsRunner that speaks the nailgun protocol to a remote client.

  N.B. this class is primarily used by the PailgunService in pantsd.
  """

  @classmethod
  def create(cls, sock, args, env, fork_lock, scheduler_service):
    try:
      # N.B. This will temporarily redirect stdio in the daemon's pre-fork context
      # to the nailgun session. We'll later do this a second time post-fork, because
      # threads.
      with cls.nailgunned_stdio(sock, env, handle_stdin=False):
        options, build_config, options_bootstrapper = LocalPantsRunner.parse_options(args, env)
        subprocess_dir = options.for_global_scope().pants_subprocessdir
        graph_helper, target_roots = scheduler_service.prefork(options, build_config)
        deferred_exc = None
    except Exception:
      deferred_exc = sys.exc_info()
      graph_helper = None
      target_roots = None
      options_bootstrapper = None
      # N.B. This will be overridden with the correct value if options
      # parsing is successful - otherwise it permits us to run just far
      # enough to raise the deferred exception.
      subprocess_dir = os.path.join(get_buildroot(), '.pids')

    return cls(
      sock,
      args,
      env,
      fork_lock,
      graph_helper,
      target_roots,
      subprocess_dir,
      options_bootstrapper,
      deferred_exc
    )

  def __init__(self, socket, args, env, fork_lock, graph_helper, target_roots,
               metadata_base_dir, options_bootstrapper, deferred_exc):
    """
    :param socket socket: A connected socket capable of speaking the nailgun protocol.
    :param list args: The arguments (i.e. sys.argv) for this run.
    :param dict env: The environment (i.e. os.environ) for this run.
    :param threading.RLock fork_lock: A lock to use during forking for thread safety.
    :param LegacyGraphSession graph_helper: The LegacyGraphSession instance to use for BuildGraph
                                            construction. In the event of an exception, this will be
                                            None.
    :param TargetRoots target_roots: The `TargetRoots` for this run.
    :param str metadata_base_dir: The ProcessManager metadata_base_dir from options.
    :param OptionsBootstrapper options_bootstrapper: An OptionsBootstrapper to reuse.
    :param Exception deferred_exception: A deferred exception from the daemon's pre-fork context.
                                         If present, this will be re-raised in the client context.
    """
    super(DaemonPantsRunner, self).__init__(
      name=self._make_identity(),
      metadata_base_dir=metadata_base_dir
    )
    self._socket = socket
    self._args = args
    self._env = env
    self._fork_lock = fork_lock
    self._graph_helper = graph_helper
    self._target_roots = target_roots
    self._options_bootstrapper = options_bootstrapper
    self._deferred_exception = deferred_exc

    self._exiter = DaemonExiter(socket)

  def _make_identity(self):
    """Generate a ProcessManager identity for a given pants run.

    This provides for a reasonably unique name e.g. 'pantsd-run-2015-09-16T23_17_56_581899'.
    """
    return 'pantsd-run-{}'.format(datetime.datetime.now().strftime('%Y-%m-%dT%H_%M_%S_%f'))

  @classmethod
  @contextmanager
  def _tty_stdio(cls, env):
    """Handles stdio redirection in the case of all stdio descriptors being the same tty."""
    # If all stdio is a tty, there's only one logical I/O device (the tty device). This happens to
    # be addressable as a file in OSX and Linux, so we take advantage of that and directly open the
    # character device for output redirection - eliminating the need to directly marshall any
    # interactive stdio back/forth across the socket and permitting full, correct tty control with
    # no middle-man.
    stdin_ttyname, stdout_ttyname, stderr_ttyname = NailgunProtocol.ttynames_from_env(env)
    assert stdin_ttyname == stdout_ttyname == stderr_ttyname, (
      'expected all stdio ttys to be the same, but instead got: {}\n'
      'please file a bug at http://github.com/pantsbuild/pants'
      .format([stdin_ttyname, stdout_ttyname, stderr_ttyname])
    )
    with open(stdin_ttyname, 'rb+wb', 0) as tty:
      tty_fileno = tty.fileno()
      with stdio_as(stdin_fd=tty_fileno, stdout_fd=tty_fileno, stderr_fd=tty_fileno):
        def finalizer():
          termios.tcdrain(tty_fileno)
        yield finalizer

  @classmethod
  @contextmanager
  def _pipe_stdio(cls, sock, stdin_isatty, stdout_isatty, stderr_isatty, handle_stdin):
    """Handles stdio redirection in the case of pipes and/or mixed pipes and ttys."""
    stdio_writers = (
      (ChunkType.STDOUT, stdout_isatty),
      (ChunkType.STDERR, stderr_isatty)
    )
    types, ttys = zip(*(stdio_writers))

    @contextmanager
    def maybe_handle_stdin(want):
      if want:
        # TODO: Launching this thread pre-fork to handle @rule input currently results
        # in an unhandled SIGILL in `src/python/pants/engine/scheduler.py, line 313 in pre_fork`.
        # More work to be done here in https://github.com/pantsbuild/pants/issues/6005
        with NailgunStreamStdinReader.open(sock, stdin_isatty) as fd:
          yield fd
      else:
        with open('/dev/null', 'rb') as fh:
          yield fh.fileno()

    with maybe_handle_stdin(handle_stdin) as stdin_fd,\
         NailgunStreamWriter.open_multi(sock, types, ttys) as ((stdout_fd, stderr_fd), writer),\
         stdio_as(stdout_fd=stdout_fd, stderr_fd=stderr_fd, stdin_fd=stdin_fd):
      # N.B. This will be passed to and called by the `DaemonExiter` prior to sending an
      # exit chunk, to avoid any socket shutdown vs write races.
      stdout, stderr = sys.stdout, sys.stderr
      def finalizer():
        try:
          stdout.flush()
          stderr.flush()
        finally:
          time.sleep(.001)  # HACK: Sleep 1ms in the main thread to free the GIL.
          writer.stop()
          writer.join()
          stdout.close()
          stderr.close()
      yield finalizer

  @classmethod
  @contextmanager
  def nailgunned_stdio(cls, sock, env, handle_stdin=True):
    """Redirects stdio to the connected socket speaking the nailgun protocol."""
    # Determine output tty capabilities from the environment.
    stdin_isatty, stdout_isatty, stderr_isatty = NailgunProtocol.isatty_from_env(env)
    is_tty_capable = all((stdin_isatty, stdout_isatty, stderr_isatty))

    if is_tty_capable:
      with cls._tty_stdio(env) as finalizer:
        yield finalizer
    else:
      with cls._pipe_stdio(
        sock,
        stdin_isatty,
        stdout_isatty,
        stderr_isatty,
        handle_stdin
      ) as finalizer:
        yield finalizer

  def _setup_sigint_handler(self):
    """Sets up a control-c signal handler for the daemon runner context."""
    def handle_control_c(signum, frame):
      raise KeyboardInterrupt('remote client sent control-c!')
    signal.signal(signal.SIGINT, handle_control_c)

  def _raise_deferred_exc(self):
    """Raises deferred exceptions from the daemon's synchronous path in the post-fork client."""
    if self._deferred_exception:
      try:
        # Expect `_deferred_exception` to be a 3-item tuple of the values returned by sys.exc_info().
        # This permits use the 3-arg form of the `raise` statement to preserve the original traceback.
        exc_type, exc_value, exc_traceback = self._deferred_exception
        raise_with_traceback(exc_type(exc_value), exc_traceback)
      except ValueError:
        # If `_deferred_exception` isn't a 3-item tuple, treat it like a bare exception.
        raise self._deferred_exception

  def _maybe_get_client_start_time_from_env(self, env):
    client_start_time = env.pop('PANTSD_RUNTRACKER_CLIENT_START_TIME', None)
    return None if client_start_time is None else float(client_start_time)

  def run(self):
    """Fork, daemonize and invoke self.post_fork_child() (via ProcessManager)."""
    with self._fork_lock:
      self.daemonize(write_pid=False)

  def pre_fork(self):
    """Pre-fork callback executed via ProcessManager.daemonize().

    The scheduler has thread pools which need to be re-initialized after a fork: this ensures that
    when the pantsd-runner forks from pantsd, there is a working pool for any work that happens
    in that child process.
    """
    if self._graph_helper:
      self._graph_helper.scheduler_session.pre_fork()

  def post_fork_child(self):
    """Post-fork child process callback executed via ProcessManager.daemonize()."""
    # Set the Exiter exception hook post-fork so as not to affect the pantsd processes exception
    # hook with socket-specific behavior. Note that this intentionally points the faulthandler
    # trace stream to sys.stderr, which at this point is still a _LoggerStream object writing to
    # the `pantsd.log`. This ensures that in the event of e.g. a hung but detached pantsd-runner
    # process that the stacktrace output lands deterministically in a known place vs to a stray
    # terminal window.
    self._exiter.set_except_hook(sys.stderr)

    # Ensure anything referencing sys.argv inherits the Pailgun'd args.
    sys.argv = self._args

    # Set context in the process title.
    set_process_title('pantsd-runner [{}]'.format(' '.join(self._args)))

    # Setup a SIGINT signal handler.
    self._setup_sigint_handler()

    # Broadcast our process group ID (in PID form - i.e. negated) to the remote client so
    # they can send signals (e.g. SIGINT) to all processes in the runners process group.
    pid = str(os.getpgrp() * -1).encode('ascii')
    NailgunProtocol.send_pid(self._socket, pid)

    # Invoke a Pants run with stdio redirected and a proxied environment.
    with self.nailgunned_stdio(self._socket, self._env) as finalizer,\
         hermetic_environment_as(**self._env):
      try:
        # Setup the Exiter's finalizer.
        self._exiter.set_finalizer(finalizer)

        # Clean global state.
        clean_global_runtime_state(reset_subsystem=True)

        # Re-raise any deferred exceptions, if present.
        self._raise_deferred_exc()

        # Otherwise, conduct a normal run.
        runner = LocalPantsRunner.create(
          self._exiter,
          self._args,
          self._env,
          self._target_roots,
          self._graph_helper,
          self._options_bootstrapper
        )
        runner.set_start_time(self._maybe_get_client_start_time_from_env(self._env))
        runner.run()
      except KeyboardInterrupt:
        self._exiter.exit(1, msg='Interrupted by user.\n')
      except Exception:
        self._exiter.handle_unhandled_exception(add_newline=True)
      else:
        self._exiter.exit(0)
<|endoftext|>


<commit_before>from __future__ import print_function

from collections import defaultdict
import getpass
import hashlib
import sys
import tempfile
import threading
import time

from twitter.common.contextutil import open_zip
from twitter.common.quantity import Amount, Time, Data
from twitter.mesos.client.config import populate_namespaces
from twitter.mesos.common.aurora_job_key import AuroraJobKey
from twitter.mesos.config.schema import (
    Announcer,
    Constraint,
    Job,
    Packer,
    Process,
    Resources,
    SequentialTask)
from twitter.mesos.config import AuroraConfig
from twitter.mesos.packer import sd_packer_client

from gen.twitter.mesos.ttypes import ResponseCode, ScheduleStatus

from .job_monitor import JobMonitor

class Quickrun(object):
  QUERY_INTERVAL = Amount(5, Time.SECONDS)
  WAIT_STATES = frozenset([
      ScheduleStatus.PENDING,
      ScheduleStatus.ASSIGNED,
      ScheduleStatus.STARTING])
  ACTIVE_STATES = frozenset([
      ScheduleStatus.RUNNING])
  FINISHED_STATES = frozenset([
      ScheduleStatus.FAILED,
      ScheduleStatus.FINISHED,
      ScheduleStatus.KILLED])
  PACKAGE_NAME = '__quickrun'

  def __init__(self,
               cluster_name,
               command,
               name,
               role=None,
               instances=1,
               cpu=1,
               ram=Amount(1, Data.GB),
               disk=Amount(1, Data.GB),
               announce=False,
               packages=None,
               additional_files=None):
    self._jobname = name
    self._role = role or getpass.getuser()
    self._stop = threading.Event()
    self._instances = instances
    self._cluster_name = cluster_name
    processes = [Process(name=name, cmdline=command)]
    if packages:
      if not isinstance(packages, list):
        raise ValueError('Expect packages to be a list of 3-tuples!')
      for package in packages:
        if not hasattr(package, '__iter__') or len(package) != 3:
          raise ValueError('Expect each package to be a 3-tuple of role, name, version')
        package_role, package_name, package_version = package
        processes.insert(0, Packer.copy(package_name, role=package_role, version=package_version))
    if additional_files:
      processes.insert(0, self.additional_files_process(cluster_name, self._role, additional_files))
    task = SequentialTask(
        name=name,
        processes=processes,
        resources=Resources(cpu=cpu, ram=ram.as_(Data.BYTES), disk=disk.as_(Data.BYTES)))
    job = Job(task=task, instances=instances, role=self._role, environment='test', cluster=cluster_name)
    if announce:
      job = job(announce=Announcer(), daemon=True)
    self._config = self.populate_namespaces(AuroraConfig(job))

  def populate_namespaces(self, config):
    return populate_namespaces(config)

  @classmethod
  def generate_package_file(cls, filemap):
    filename = tempfile.mktemp(suffix='.zip')
    with open_zip(filename, 'w') as zf:
      for an, fn in filemap.items():
        zf.write(fn, arcname=an)
    return filename

  @classmethod
  def get_md5(cls, filename):
    with open(filename) as fp:
      return hashlib.md5(fp.read()).hexdigest()

  @classmethod
  def get_package_file(cls, cluster, role, filename):
    md5 = cls.get_md5(filename)
    packer = sd_packer_client.create_packer(cluster)
    if cls.PACKAGE_NAME in packer.list_packages(role):
      for package in packer.list_versions(role, cls.PACKAGE_NAME):
        if package['md5sum'] == md5:
          return (role, cls.PACKAGE_NAME, package['id'])
    # packer miss, upload new copy
    package = packer.add(role, cls.PACKAGE_NAME, filename, None, digest=md5)
    return (role, cls.PACKAGE_NAME, package['id'])

  @classmethod
  def additional_files_process(cls, cluster, role, filemap):
    # filemap is of the form local => remote filename
    package_role, package_name, package_version = cls.get_package_file(cluster, role,
        cls.generate_package_file(filemap))
    return Packer.install(package_name, role=package_role, version=package_version)

  def _terminal(self, statuses):
    terminals = sum(status in self.FINISHED_STATES for status in statuses.values())
    return terminals == self._instances

  def _write_line(self, line):
    whitespace = max(0, 80 - len(line))
    sys.stderr.write(line + ' ' * whitespace)
    sys.stderr.write('\r')
    sys.stderr.flush()

  def run(self, client):
    monitor = JobMonitor(client, self._config.role(), self._config.environment(), self._config.name())
    response = client.create_job(self._config)
    if response.responseCode != ResponseCode.OK:
      print('Failed to create job: %s' % response.message)
      return False
    try:
      while True:
        statuses = monitor.states()
        statuses_count = defaultdict(int)
        for status in statuses.values():
          statuses_count[status] += 1
        self._write_line(' :: '.join('%s %2d' % (ScheduleStatus._VALUES_TO_NAMES[status], count)
            for (status, count) in statuses_count.items()))
        if self._terminal(statuses):
          print('\nTask finished.')
          return True
        time.sleep(self.QUERY_INTERVAL.as_(Time.SECONDS))
    except KeyboardInterrupt:
      print('\nKilling job...')
      job_key = AuroraJobKey(self._cluster_name, self._config.role(), self._config.environment(),
          self._config.name())
      response = client.kill_job(job_key)
      if response.responseCode != ResponseCode.OK:
        print('Failed to kill task: %s' % response.message)

  def stop(self):
    self._stop.set()

  def __str__(self):
    return str(self._config._job)
<commit_msg>Add state listeners to kazoo group since watches do not reliably fire SESSION_EXPIRED events.<commit_after>from __future__ import print_function

from collections import defaultdict
import getpass
import hashlib
import sys
import tempfile
import threading
import time

from twitter.common.contextutil import open_zip
from twitter.common.quantity import Amount, Time, Data
from twitter.mesos.client.config import populate_namespaces
from twitter.mesos.common.aurora_job_key import AuroraJobKey
from twitter.mesos.config.schema import (
    Announcer,
    Constraint,
    Job,
    Packer,
    Process,
    Resources,
    SequentialTask)
from twitter.mesos.config import AuroraConfig
from twitter.mesos.packer import sd_packer_client

from gen.twitter.mesos.ttypes import ResponseCode, ScheduleStatus

from .job_monitor import JobMonitor

class Quickrun(object):
  QUERY_INTERVAL = Amount(30, Time.SECONDS)
  WAIT_STATES = frozenset([
      ScheduleStatus.PENDING,
      ScheduleStatus.ASSIGNED,
      ScheduleStatus.STARTING])
  ACTIVE_STATES = frozenset([
      ScheduleStatus.RUNNING])
  FINISHED_STATES = frozenset([
      ScheduleStatus.FAILED,
      ScheduleStatus.FINISHED,
      ScheduleStatus.KILLED])
  PACKAGE_NAME = '__quickrun'

  def __init__(self,
               cluster_name,
               command,
               name,
               role=None,
               instances=1,
               cpu=1,
               ram=Amount(1, Data.GB),
               disk=Amount(1, Data.GB),
               announce=False,
               packages=None,
               additional_files=None):
    self._jobname = name
    self._role = role or getpass.getuser()
    self._stop = threading.Event()
    self._instances = instances
    self._cluster_name = cluster_name
    processes = [Process(name=name, cmdline=command)]
    if packages:
      if not isinstance(packages, list):
        raise ValueError('Expect packages to be a list of 3-tuples!')
      for package in packages:
        if not hasattr(package, '__iter__') or len(package) != 3:
          raise ValueError('Expect each package to be a 3-tuple of role, name, version')
        package_role, package_name, package_version = package
        processes.insert(0, Packer.copy(package_name, role=package_role, version=package_version))
    if additional_files:
      processes.insert(0, self.additional_files_process(cluster_name, self._role, additional_files))
    task = SequentialTask(
        name=name,
        processes=processes,
        resources=Resources(cpu=cpu, ram=ram.as_(Data.BYTES), disk=disk.as_(Data.BYTES)))
    job = Job(task=task, instances=instances, role=self._role, environment='test', cluster=cluster_name)
    if announce:
      job = job(announce=Announcer(), daemon=True)
    self._config = self.populate_namespaces(AuroraConfig(job))

  def populate_namespaces(self, config):
    return populate_namespaces(config)

  @classmethod
  def generate_package_file(cls, filemap):
    filename = tempfile.mktemp(suffix='.zip')
    with open_zip(filename, 'w') as zf:
      for an, fn in filemap.items():
        zf.write(fn, arcname=an)
    return filename

  @classmethod
  def get_md5(cls, filename):
    with open(filename) as fp:
      return hashlib.md5(fp.read()).hexdigest()

  @classmethod
  def get_package_file(cls, cluster, role, filename):
    md5 = cls.get_md5(filename)
    packer = sd_packer_client.create_packer(cluster)
    if cls.PACKAGE_NAME in packer.list_packages(role):
      for package in packer.list_versions(role, cls.PACKAGE_NAME):
        if package['md5sum'] == md5:
          return (role, cls.PACKAGE_NAME, package['id'])
    # packer miss, upload new copy
    package = packer.add(role, cls.PACKAGE_NAME, filename, None, digest=md5)
    return (role, cls.PACKAGE_NAME, package['id'])

  @classmethod
  def additional_files_process(cls, cluster, role, filemap):
    # filemap is of the form local => remote filename
    package_role, package_name, package_version = cls.get_package_file(cluster, role,
        cls.generate_package_file(filemap))
    return Packer.install(package_name, role=package_role, version=package_version)

  def _terminal(self, statuses):
    terminals = sum(status in self.FINISHED_STATES for status in statuses.values())
    return terminals == self._instances

  def _write_line(self, line):
    whitespace = max(0, 80 - len(line))
    sys.stderr.write(line + ' ' * whitespace)
    sys.stderr.write('\r')
    sys.stderr.flush()

  def run(self, client):
    monitor = JobMonitor(client, self._config.role(), self._config.environment(), self._config.name())
    response = client.create_job(self._config)
    if response.responseCode != ResponseCode.OK:
      print('Failed to create job: %s' % response.message)
      return False
    try:
      while True:
        statuses = monitor.states()
        statuses_count = defaultdict(int)
        for status in statuses.values():
          statuses_count[status] += 1
        self._write_line(' :: '.join('%s %2d' % (ScheduleStatus._VALUES_TO_NAMES[status], count)
            for (status, count) in statuses_count.items()))
        if self._terminal(statuses):
          print('\nTask finished.')
          return True
        time.sleep(self.QUERY_INTERVAL.as_(Time.SECONDS))
    except KeyboardInterrupt:
      print('\nKilling job...')
      job_key = AuroraJobKey(self._cluster_name, self._config.role(), self._config.environment(),
          self._config.name())
      response = client.kill_job(job_key)
      if response.responseCode != ResponseCode.OK:
        print('Failed to kill task: %s' % response.message)

  def stop(self):
    self._stop.set()

  def __str__(self):
    return str(self._config._job)
<|endoftext|>


<commit_before>"""Finds a directory or file inside the current project."""
from . import DodoCommand
from dodo_commands.framework.config import CommandPath
import os
import sys

script_src = """# noqa
from dodo_commands.system_commands import DodoCommand

class Command(DodoCommand):  # noqa
    help = ""
    decorators = []

    def add_arguments_imp(self, parser):  # noqa
        parser.add_argument('foo')
        parser.add_argument(
            '--bar',
            required=True,
            help=''
        )

    def handle_imp(self, foo, bar, **kwargs):  # noqa
        pass
"""

class Command(DodoCommand):  # noqa
    help = "Creates a new Dodo command."
    safe = False

    def add_arguments_imp(self, parser):  # noqa
        """
        Entry point for subclassed commands to add custom arguments.
        """
        parser.add_argument('name')
        parser.add_argument(
            '--next-to',
            required=True,
            help='Create the new command at the location of this command'
        )

    def handle_imp(self, name, next_to, **kwargs):  # noqa
        dest_path = None
        command_path = CommandPath(self.config)
        for item in command_path.items:
            script_path = os.path.join(
                item.full_path, next_to + ".py"
            )
            if os.path.exists(script_path):
                dest_path = os.path.join(
                    item.full_path, name + ".py"
                )

        if not dest_path:
            sys.stderr.write("Script not found: %s\n" % next_to)
            return

        with open(dest_path, "w") as f:
            f.write(script_src)

        print(dest_path)
<commit_msg>Add an example of calling self.runcmd to new-command<commit_after>"""Finds a directory or file inside the current project."""
from . import DodoCommand
from dodo_commands.framework.config import CommandPath
import os
import sys

script_src = """# noqa
from dodo_commands.system_commands import DodoCommand

class Command(DodoCommand):  # noqa
    help = ""
    decorators = []

    def add_arguments_imp(self, parser):  # noqa
        parser.add_argument('foo')
        parser.add_argument(
            '--bar',
            required=True,
            help=''
        )

    def handle_imp(self, foo, bar, **kwargs):  # noqa
        self.runcmd(["echo"], cwd=".")
"""

class Command(DodoCommand):  # noqa
    help = "Creates a new Dodo command."
    safe = False

    def add_arguments_imp(self, parser):  # noqa
        """
        Entry point for subclassed commands to add custom arguments.
        """
        parser.add_argument('name')
        parser.add_argument(
            '--next-to',
            required=True,
            help='Create the new command at the location of this command'
        )

    def handle_imp(self, name, next_to, **kwargs):  # noqa
        dest_path = None
        command_path = CommandPath(self.config)
        for item in command_path.items:
            script_path = os.path.join(
                item.full_path, next_to + ".py"
            )
            if os.path.exists(script_path):
                dest_path = os.path.join(
                    item.full_path, name + ".py"
                )

        if not dest_path:
            sys.stderr.write("Script not found: %s\n" % next_to)
            return

        with open(dest_path, "w") as f:
            f.write(script_src)

        print(dest_path)
<|endoftext|>


<commit_before>#!/usr/bin/env python2
# Build the documentation.

from __future__ import print_function
import os, pip, shutil, tempfile
from subprocess import check_call, check_output, CalledProcessError, Popen, PIPE
from distutils.version import LooseVersion

def pip_install(package, commit=None, **kwargs):
  "Install package using pip."
  if commit:
    check_version = kwargs.get('check_version', '')
    #output = check_output(['pip', 'show', package.split('/')[1]])
    #if check_version in output:
    #  print('{} already installed'.format(package))
    #  return
    package = 'git+git://github.com/{0}.git@{1}'.format(package, commit)
  print('Installing {}'.format(package))
  check_call(['pip', 'install', '--upgrade', package])

def build_docs():
  # Create virtualenv.
  doc_dir = os.path.dirname(os.path.realpath(__file__))
  virtualenv_dir = 'virtualenv'
  check_call(['virtualenv', virtualenv_dir])
  activate_this_file = os.path.join(virtualenv_dir, 'bin', 'activate_this.py')
  execfile(activate_this_file, dict(__file__=activate_this_file))
  # Upgrade pip because installation of sphinx with pip 1.1 available on Travis
  # is broken (see #207) and it doesn't support the show command.
  if LooseVersion(pip.__version__) < LooseVersion('1.5.4'):
    print("Updating pip")
    check_call(['pip', 'install', '--upgrade', 'pip==1.5.4'])
  # Install Sphinx and Breathe.
  pip_install('sphinx-doc/sphinx',
              '4d2c17e043d9e8197fa5cd0db34212af3bb17069',
              check_version='1.4a0.dev-20151013')
  pip_install('michaeljones/breathe',
              '511b0887293e7c6b12310bb61b3659068f48f0f4')
  print(check_output(['pip', '--version']))
  print(check_output(['sphinx-build', '--version']))
  print('PATH:', os.environ['PATH'])
  print(check_output(['which', 'sphinx-build']))
  import sphinx
  print(sphinx.__version__)
  # Build docs.
  cmd = ['doxygen', '-']
  p = Popen(cmd, stdin=PIPE)
  p.communicate(input=r'''
      PROJECT_NAME      = C++ Format
      GENERATE_LATEX    = NO
      GENERATE_MAN      = NO
      GENERATE_RTF      = NO
      CASE_SENSE_NAMES  = NO
      INPUT             = {0}/format.h
      QUIET             = YES
      JAVADOC_AUTOBRIEF = YES
      AUTOLINK_SUPPORT  = NO
      GENERATE_HTML     = NO
      GENERATE_XML      = YES
      XML_OUTPUT        = doxyxml
      ALIASES           = "rst=\verbatim embed:rst"
      ALIASES          += "endrst=\endverbatim"
      PREDEFINED        = _WIN32=1 \
                          FMT_USE_VARIADIC_TEMPLATES=1 \
                          FMT_USE_RVALUE_REFERENCES=1 \
                          FMT_USE_USER_DEFINED_LITERALS=1
      EXCLUDE_SYMBOLS   = fmt::internal::* StringValue write_str
    '''.format(os.path.dirname(doc_dir)))
  if p.returncode != 0:
    raise CalledProcessError(p.returncode, cmd)
  check_call(['sphinx-build', '-D',
              'breathe_projects.format=' + os.path.join(os.getcwd(), 'doxyxml'),
              '-b', 'html', doc_dir, 'html'])
  check_call(['lessc', '--clean-css',
              '--include-path=' + os.path.join(doc_dir, 'bootstrap'),
              os.path.join(doc_dir, 'cppformat.less'),
              'html/_static/cppformat.css'])
  return 'html'

if __name__ == '__main__':
  build_docs()
<commit_msg>More diagnostics<commit_after>#!/usr/bin/env python2
# Build the documentation.

from __future__ import print_function
import os, pip, shutil, tempfile
from subprocess import check_call, check_output, CalledProcessError, Popen, PIPE
from distutils.version import LooseVersion

def pip_install(package, commit=None, **kwargs):
  "Install package using pip."
  if commit:
    check_version = kwargs.get('check_version', '')
    #output = check_output(['pip', 'show', package.split('/')[1]])
    #if check_version in output:
    #  print('{} already installed'.format(package))
    #  return
    package = 'git+git://github.com/{0}.git@{1}'.format(package, commit)
  print('Installing {}'.format(package))
  check_call(['pip', 'install', '--upgrade', package])

def build_docs():
  # Create virtualenv.
  doc_dir = os.path.dirname(os.path.realpath(__file__))
  virtualenv_dir = 'virtualenv'
  check_call(['virtualenv', virtualenv_dir])
  activate_this_file = os.path.join(virtualenv_dir, 'bin', 'activate_this.py')
  execfile(activate_this_file, dict(__file__=activate_this_file))
  # Upgrade pip because installation of sphinx with pip 1.1 available on Travis
  # is broken (see #207) and it doesn't support the show command.
  print('pip version: {} {}'.format(pip.__version__, LooseVersion(pip.__version__) < LooseVersion('1.5.4')))
  if LooseVersion(pip.__version__) < LooseVersion('1.5.4'):
    print("Updating pip")
    check_call(['pip', 'install', '--upgrade', 'pip==1.5.4'])
  # Install Sphinx and Breathe.
  pip_install('sphinx-doc/sphinx',
              '4d2c17e043d9e8197fa5cd0db34212af3bb17069',
              check_version='1.4a0.dev-20151013')
  pip_install('michaeljones/breathe',
              '511b0887293e7c6b12310bb61b3659068f48f0f4')
  print(check_output(['pip', '--version']))
  print(check_output(['sphinx-build', '--version']))
  print('PATH:', os.environ['PATH'])
  print(check_output(['which', 'sphinx-build']))
  import sphinx
  print(sphinx.__version__)
  # Build docs.
  cmd = ['doxygen', '-']
  p = Popen(cmd, stdin=PIPE)
  p.communicate(input=r'''
      PROJECT_NAME      = C++ Format
      GENERATE_LATEX    = NO
      GENERATE_MAN      = NO
      GENERATE_RTF      = NO
      CASE_SENSE_NAMES  = NO
      INPUT             = {0}/format.h
      QUIET             = YES
      JAVADOC_AUTOBRIEF = YES
      AUTOLINK_SUPPORT  = NO
      GENERATE_HTML     = NO
      GENERATE_XML      = YES
      XML_OUTPUT        = doxyxml
      ALIASES           = "rst=\verbatim embed:rst"
      ALIASES          += "endrst=\endverbatim"
      PREDEFINED        = _WIN32=1 \
                          FMT_USE_VARIADIC_TEMPLATES=1 \
                          FMT_USE_RVALUE_REFERENCES=1 \
                          FMT_USE_USER_DEFINED_LITERALS=1
      EXCLUDE_SYMBOLS   = fmt::internal::* StringValue write_str
    '''.format(os.path.dirname(doc_dir)))
  if p.returncode != 0:
    raise CalledProcessError(p.returncode, cmd)
  check_call(['sphinx-build', '-D',
              'breathe_projects.format=' + os.path.join(os.getcwd(), 'doxyxml'),
              '-b', 'html', doc_dir, 'html'])
  check_call(['lessc', '--clean-css',
              '--include-path=' + os.path.join(doc_dir, 'bootstrap'),
              os.path.join(doc_dir, 'cppformat.less'),
              'html/_static/cppformat.css'])
  return 'html'

if __name__ == '__main__':
  build_docs()
<|endoftext|>


<commit_before>######################################################################################
# Date: 2016/July/11
#
# Module: module_hist.py
#
# VERSION: 0.9
#
# AUTHOR: Matt Thoburn (mthoburn@ufl.edu)
#
# DESCRIPTION: This module contains methods to plot a histogram in matplotlib
#
#######################################################################################
import matplotlib.pyplot as plt
import pandas as pd

def quickHist(ax,dat,orientation="vertical",color="#0000FF"):
    """
    draws a histogram with minimal formatting

    :Arguments:
        :type ax: matplotlib axis
        :param ax: axis to plot data on

        :type dat: pandas dataframe
        :param dat: data to be plotted

        :type orientation: string
        :param orientation: vertical or horizontal
    """
    dat.hist(ax=ax,orientation=orientation,color=color)
    return ax

def serHist(ax,dat,color,range=None,bins=10,normed=False,orientation="vertical",label=""):
    """
    draws a histogram with formatting options and coloring for groups

    :Arguments:
        :type ax: matplotlib axis
        :param ax: axis to plot data on

        :type dat: pandas Series
        :param dat: data to be plotted

        :type colors: list of colors
        :param colors: colors for groups

        :type range: tuple of numbers or None
        :param range: lower and upper range of bins

        :type bins: int
        :param bins: number of bins for histogram

        :type normed: boolean or None
        :param normed: normalize integral of histogram to 1 if true

        :type orientation: string
        :param orientation: vertical or horizontal
    """
    ax.hist(dat,color=color,alpha=.7,orientation=orientation,range=range,bins=bins,label=label)

def hist(ax,dat,colors,range=None,bins=10,normed=False,orientation="vertical",label=""):
    """
    draws a histogram with formatting options and coloring for groups

    :Arguments:
        :type ax: matplotlib axis
        :param ax: axis to plot data on

        :type dat: pandas dataframeO
        :param dat: data to be plotted

        :type colors: list of colors
        :param colors: colors for groups

        :type range: tuple of numbers or None
        :param range: lower and upper range of bins

        :type bins: int
        :param bins: number of bins for histogram

        :type normed: boolean or None
        :param normed: normalize integral of histogram to 1 if true

        :type orientation: string
        :param orientation: vertical or horizontal
    """
    headers = dat.wide.columns.values
    groupList = dat.design[dat.group].values

    for group,color in zip(dat.levels,colors):
        valPerGroup =  dat.wide[dat.design.index[dat.design[dat.group]==group]].values
        lists = sum(list(map(list,valPerGroup)),[])
        ax.hist(lists,color=color,alpha=.7,orientation=orientation,range=range,bins=bins,normed=normed,label=label)
<commit_msg>Reformat module_hist.py<commit_after>######################################################################################
#
# Module: module_hist.py
#
# AUTHOR: Matt Thoburn <mthoburn@ufl.edu>
#
# DESCRIPTION: This module contains methods to plot a histogram in matplotlib
#
#######################################################################################

import matplotlib.pyplot as plt
import pandas as pd

def quickHist(ax, dat, orientation="vertical", color="#0000FF"):
    """
    draws a histogram with minimal formatting

    :Arguments:
        :type ax: matplotlib axis
        :param ax: axis to plot data on

        :type dat: pandas dataframe
        :param dat: data to be plotted

        :type orientation: string
        :param orientation: vertical or horizontal
    """
    dat.hist(ax=ax, orientation=orientation, color=color)
    return ax

def serHist(
    ax, dat, color, range=None, bins=10, normed=False, orientation="vertical", label=""
):
    """
    draws a histogram with formatting options and coloring for groups

    :Arguments:
        :type ax: matplotlib axis
        :param ax: axis to plot data on

        :type dat: pandas Series
        :param dat: data to be plotted

        :type colors: list of colors
        :param colors: colors for groups

        :type range: tuple of numbers or None
        :param range: lower and upper range of bins

        :type bins: int
        :param bins: number of bins for histogram

        :type normed: boolean or None
        :param normed: normalize integral of histogram to 1 if true

        :type orientation: string
        :param orientation: vertical or horizontal
    """
    ax.hist(
        dat,
        color=color,
        alpha=0.7,
        orientation=orientation,
        range=range,
        bins=bins,
        label=label,
    )

def hist(
    ax, dat, colors, range=None, bins=10, normed=False, orientation="vertical", label=""
):
    """
    draws a histogram with formatting options and coloring for groups

    :Arguments:
        :type ax: matplotlib axis
        :param ax: axis to plot data on

        :type dat: pandas dataframeO
        :param dat: data to be plotted

        :type colors: list of colors
        :param colors: colors for groups

        :type range: tuple of numbers or None
        :param range: lower and upper range of bins

        :type bins: int
        :param bins: number of bins for histogram

        :type normed: boolean or None
        :param normed: normalize integral of histogram to 1 if true

        :type orientation: string
        :param orientation: vertical or horizontal
    """
    headers = dat.wide.columns.values
    groupList = dat.design[dat.group].values

    for group, color in zip(dat.levels, colors):
        valPerGroup = dat.wide[dat.design.index[dat.design[dat.group] == group]].values
        lists = sum(list(map(list, valPerGroup)), [])
        ax.hist(
            lists,
            color=color,
            alpha=0.7,
            orientation=orientation,
            range=range,
            bins=bins,
            normed=normed,
            label=label,
        )
<|endoftext|>


<commit_before># Copyright 2016 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Classes to implement dbus interface.
"""

import abc
import enum
import types

import dbus

from into_dbus_python import xformers

_FALSE = lambda n: lambda x: x

def _option_to_tuple(value, default):
    """
    Converts a Python "option" type, None or a value, to a tuple.

    :param object value: any value
    :param object default: a valid default to use if value is None
    :returns: a tuple encoding the meaning of value
    :rtypes: bool * object
    """
    return (False, default) if value is None else (True, value)

def _info_to_xformer(names, inxform, sig):
    """
    Function that yields a xformer function.

    :param names: the names of the parameters for this function
    :type names: list of str
    :param inxform: a function that yields transformation functions
    :type inform: str -> (object -> object)
    :param str sig: a D-Bus signature
    :return: a transformation function
    :rtype: (list of object) -> (list of object)
    """
    inxforms = [inxform(name) for name in names]
    outxforms = [f for (f, _) in xformers(sig)]
    expected_length = len(names)

    if len(outxforms) != expected_length:
        raise ValueError("xformation functions do not match")

    def xformer(objects):
        """
        Transforms a list of objects to dbus-python types.

        :param objects: list of objects
        :type objects: list of object
        :returns: a transformed list of objects
        :rtype: list of object
        """
        if len(objects) != expected_length:
            raise ValueError("wrong number of objects")

        return \
           [x for (x, _) in (f(g(a)) for \
           ((f, g), a) in zip(zip(outxforms, inxforms), objects))]

    return xformer

def _xformers(key_to_sig):
    """
    Get a map from keys to functions from a map of names to signatures.

    :param key_to_sig: a map from keys to signatures
    :type key_to_sig: dict of object * str
    :returns: a map from keys to functions
    :rtype: dict of object * xformation function
    """
    return dict((method, (names, _info_to_xformer(names, inxform, sig))) for \
       (method, (names, inxform, sig)) in key_to_sig.items())

class InterfaceSpec(abc.ABC):
    """
    Parent class for an interface hierarchy.
    """
    # pylint: disable=too-few-public-methods

    INTERFACE_NAME = abc.abstractproperty(doc="interface name")
    INPUT_SIGS = \
       abc.abstractproperty(doc="map from method name to input signatures")
    OUTPUT_SIGS = \
       abc.abstractproperty(doc="map from method name to output signatures")
    XFORMERS = abc.abstractproperty(doc="map from method name to xformer")
    PROPERTY_NAMES = abc.abstractproperty(doc="list of property names")

class FilesystemSpec(InterfaceSpec):
    """
    Filesystem interface.
    """
    # pylint: disable=too-few-public-methods

    class MethodNames(enum.Enum):
        """
        Names of the methods of the Filesystem class.
        """
        CreateSnapshot = "CreateSnapshot"
        Rename = "Rename"
        SetMountpoint = "SetMountpoint"
        SetQuota = "SetQuota"

    class PropertyNames(enum.Enum):
        """
        Names of the properties of the Filesystem interface.
        """
        pass

    INTERFACE_NAME = 'org.storage.stratis1.filesystem'

    INPUT_SIGS = {
       MethodNames.CreateSnapshot: (("name", ), _FALSE, "s"),
       MethodNames.Rename: (("name", ), _FALSE, "s"),
       MethodNames.SetMountpoint: ((), _FALSE, ""),
       MethodNames.SetQuota: (("quota", ), _FALSE, "s")
    }
    OUTPUT_SIGS = {
       MethodNames.CreateSnapshot: "oqs",
       MethodNames.Rename: "oqs",
       MethodNames.SetMountpoint: "oqs",
       MethodNames.SetQuota: "oqs"
    }
    XFORMERS = _xformers(INPUT_SIGS)

class ManagerSpec(InterfaceSpec):
    """
    Manager interface.
    """
    # pylint: disable=too-few-public-methods

    class MethodNames(enum.Enum):
        """
        The method names of the manager interface.
        """
        ConfigureSimulator = "ConfigureSimulator"
        CreatePool = "CreatePool"
        DestroyPool = "DestroyPool"
        GetCacheObjectPath = "GetCacheObjectPath"
        GetDevObjectPath = "GetDevObjectPath"
        GetDevTypes = "GetDevTypes"
        GetErrorCodes = "GetErrorCodes"
        GetFilesystemObjectPath = "GetFilesystemObjectPath"
        GetPoolObjectPath = "GetPoolObjectPath"
        GetRaidLevels = "GetRaidLevels"
        ListPools = "ListPools"

    class PropertyNames(enum.Enum):
        """
        Names of the properties of the manager interface.
        """
        pass

    INTERFACE_NAME = 'org.storage.stratis1.Manager'

    INPUT_SIGS = {
        MethodNames.ConfigureSimulator : (("denominator", ), _FALSE, "u"),
        MethodNames.CreatePool :
           (("name", "redundancy", "force", "devices"), _FALSE, "sqbas"),
        MethodNames.DestroyPool : (("name", ), _FALSE, "s"),
        MethodNames.GetCacheObjectPath : (("name", ), _FALSE, "s"),
        MethodNames.GetDevObjectPath : (("name", ), _FALSE, "s"),
        MethodNames.GetDevTypes : ((), _FALSE, ""),
        MethodNames.GetErrorCodes : ((), _FALSE, ""),
        MethodNames.GetFilesystemObjectPath :
           (("pool_name", "filesystem_name"), _FALSE, "ss"),
        MethodNames.GetPoolObjectPath : (("name", ), _FALSE, "s"),
        MethodNames.GetRaidLevels : ((), _FALSE, ""),
        MethodNames.ListPools : ((), _FALSE, ""),
    }
    OUTPUT_SIGS = {
        MethodNames.ConfigureSimulator : "qs",
        MethodNames.CreatePool : "oqs",
        MethodNames.DestroyPool : "bqs",
        MethodNames.GetCacheObjectPath : "oqs",
        MethodNames.GetDevObjectPath : "oqs",
        MethodNames.GetDevTypes : "",
        MethodNames.GetErrorCodes : "a(sqs)",
        MethodNames.GetFilesystemObjectPath : "oqs",
        MethodNames.GetPoolObjectPath : "oqs",
        MethodNames.GetRaidLevels : "a(sqs)",
        MethodNames.ListPools : "asqs",
    }
    XFORMERS = _xformers(INPUT_SIGS)

class PoolSpec(InterfaceSpec):
    """
    Pool interface.
    """
    # pylint: disable=too-few-public-methods

    class MethodNames(enum.Enum):
        """
        Names of the methods of the Pool class.
        """
        AddCacheDevs = "AddCacheDevs"
        AddDevs = "AddDevs"
        CreateFilesystems = "CreateFilesystems"
        DestroyFilesystems = "DestroyFilesystems"
        ListCacheDevs = "ListCacheDevs"
        ListDevs = "ListDevs"
        ListFilesystems = "ListFilesystems"
        RemoveCacheDevs = "RemoveCacheDevs"
        RemoveDevs = "RemoveDevs"
        Rename = "Rename"

    class PropertyNames(enum.Enum):
        """
        Names of the properties of the manager interface.
        """
        pass

    INTERFACE_NAME = 'org.storage.stratis1.pool'

    INPUT_SIGS = { # pragma: no cover
       MethodNames.AddCacheDevs: (("force", "devices", ), _FALSE, "bas"),
       MethodNames.AddDevs: (("force", "devices", ), _FALSE, "bas"),
       MethodNames.CreateFilesystems: (
          ("specs", ),
          (lambda n: \
             lambda x: \
                [(x, y, _option_to_tuple(quota, 0)) for (x, y, quota) in x]),
          "a(ss(bt))"
       ),
       MethodNames.DestroyFilesystems: (("names", ), _FALSE, "as"),
       MethodNames.ListCacheDevs: ((), _FALSE, ""),
       MethodNames.ListDevs: ((), _FALSE, ""),
       MethodNames.ListFilesystems: ((), _FALSE, ""),
       MethodNames.RemoveCacheDevs: (("names", ), _FALSE, "as"),
       MethodNames.RemoveDevs: (("names", ), _FALSE, "as"),
       MethodNames.Rename: (("new_name", ), _FALSE, "s")
    }
    OUTPUT_SIGS = {
       MethodNames.AddCacheDevs: "a(oqs)qs",
       MethodNames.AddDevs: "a(oqs)qs",
       MethodNames.CreateFilesystems: "a(oqs)qs",
       MethodNames.DestroyFilesystems: "a(qs)qs",
       MethodNames.ListCacheDevs: "asqs",
       MethodNames.ListDevs: "asqs",
       MethodNames.ListFilesystems: "asqs",
       MethodNames.RemoveCacheDevs: "a(qs)qs",
       MethodNames.RemoveDevs: "a(qs)qs",
       MethodNames.Rename: "bqs"
    }
    XFORMERS = _xformers(INPUT_SIGS)

def _prop_builder(spec):
    """
    Returns a function that builds a property interface based on 'spec'.

    :param spec: the interface specification
    :type spec: type, a subtype of InterfaceSpec
    """

    def builder(namespace):
        """
        The property class's namespace.

        :param namespace: the class's namespace
        """

        def build_property(prop):
            """
            Build a single property getter for this class.

            :param prop: the property
            """

            def dbus_func(proxy_object): # pragma: no cover
                """
                The property getter.
                """
                return proxy_object.Get(
                   spec.INTERFACE_NAME,
                   prop.name,
                   dbus_interface=dbus.PROPERTIES_IFACE
                )

            return dbus_func

        for prop in spec.PropertyNames:
            namespace[prop.name] = staticmethod(build_property(prop))

    return builder

def _iface_builder(spec):
    """
    Returns a function that builds a method interface based on 'spec'.

    :param spec: the interface specification
    :type spec: type, a subtype of InterfaceSpec
    """

    def builder(namespace):
        """
        Builds the class.

        :param namespace: the class's namespace
        """

        def build_method(method):
            """
            Build a single method for this class.

            :param method: the method
            """
            (names, func) = spec.XFORMERS[method]

            def dbus_func(proxy_object, **kwargs): # pragma: no cover
                """
                The function method spec.
                """
                if frozenset(names) != frozenset(kwargs.keys()):
                    raise ValueError("Bad keys")
                args = \
                   [v for (k, v) in \
                   sorted(kwargs.items(), key=lambda x: names.index(x[0]))]
                xformed_args = func(args)
                dbus_method = getattr(proxy_object, method.name)
                return dbus_method(
                   *xformed_args,
                   dbus_interface=spec.INTERFACE_NAME
                )

            return dbus_func

        for method in spec.MethodNames:
            namespace[method.name] = staticmethod(build_method(method))

        namespace['Properties'] = \
           types.new_class(
              "Properties",
              bases=(object,),
              exec_body=_prop_builder(spec)
           )

    return builder

Filesystem = types.new_class(
   "Filesystem",
   bases=(object,),
   exec_body=_iface_builder(FilesystemSpec)
)
Manager = types.new_class(
   "Manager",
   bases=(object,),
   exec_body=_iface_builder(ManagerSpec)
)
Pool = \
   types.new_class("Pool", bases=(object,), exec_body=_iface_builder(PoolSpec))
<commit_msg>Temporarily exclude property making methods from coverage.<commit_after># Copyright 2016 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Classes to implement dbus interface.
"""

import abc
import enum
import types

import dbus

from into_dbus_python import xformers

_FALSE = lambda n: lambda x: x

def _option_to_tuple(value, default):
    """
    Converts a Python "option" type, None or a value, to a tuple.

    :param object value: any value
    :param object default: a valid default to use if value is None
    :returns: a tuple encoding the meaning of value
    :rtypes: bool * object
    """
    return (False, default) if value is None else (True, value)

def _info_to_xformer(names, inxform, sig):
    """
    Function that yields a xformer function.

    :param names: the names of the parameters for this function
    :type names: list of str
    :param inxform: a function that yields transformation functions
    :type inform: str -> (object -> object)
    :param str sig: a D-Bus signature
    :return: a transformation function
    :rtype: (list of object) -> (list of object)
    """
    inxforms = [inxform(name) for name in names]
    outxforms = [f for (f, _) in xformers(sig)]
    expected_length = len(names)

    if len(outxforms) != expected_length:
        raise ValueError("xformation functions do not match")

    def xformer(objects):
        """
        Transforms a list of objects to dbus-python types.

        :param objects: list of objects
        :type objects: list of object
        :returns: a transformed list of objects
        :rtype: list of object
        """
        if len(objects) != expected_length:
            raise ValueError("wrong number of objects")

        return \
           [x for (x, _) in (f(g(a)) for \
           ((f, g), a) in zip(zip(outxforms, inxforms), objects))]

    return xformer

def _xformers(key_to_sig):
    """
    Get a map from keys to functions from a map of names to signatures.

    :param key_to_sig: a map from keys to signatures
    :type key_to_sig: dict of object * str
    :returns: a map from keys to functions
    :rtype: dict of object * xformation function
    """
    return dict((method, (names, _info_to_xformer(names, inxform, sig))) for \
       (method, (names, inxform, sig)) in key_to_sig.items())

class InterfaceSpec(abc.ABC):
    """
    Parent class for an interface hierarchy.
    """
    # pylint: disable=too-few-public-methods

    INTERFACE_NAME = abc.abstractproperty(doc="interface name")
    INPUT_SIGS = \
       abc.abstractproperty(doc="map from method name to input signatures")
    OUTPUT_SIGS = \
       abc.abstractproperty(doc="map from method name to output signatures")
    XFORMERS = abc.abstractproperty(doc="map from method name to xformer")
    PROPERTY_NAMES = abc.abstractproperty(doc="list of property names")

class FilesystemSpec(InterfaceSpec):
    """
    Filesystem interface.
    """
    # pylint: disable=too-few-public-methods

    class MethodNames(enum.Enum):
        """
        Names of the methods of the Filesystem class.
        """
        CreateSnapshot = "CreateSnapshot"
        Rename = "Rename"
        SetMountpoint = "SetMountpoint"
        SetQuota = "SetQuota"

    class PropertyNames(enum.Enum):
        """
        Names of the properties of the Filesystem interface.
        """
        pass

    INTERFACE_NAME = 'org.storage.stratis1.filesystem'

    INPUT_SIGS = {
       MethodNames.CreateSnapshot: (("name", ), _FALSE, "s"),
       MethodNames.Rename: (("name", ), _FALSE, "s"),
       MethodNames.SetMountpoint: ((), _FALSE, ""),
       MethodNames.SetQuota: (("quota", ), _FALSE, "s")
    }
    OUTPUT_SIGS = {
       MethodNames.CreateSnapshot: "oqs",
       MethodNames.Rename: "oqs",
       MethodNames.SetMountpoint: "oqs",
       MethodNames.SetQuota: "oqs"
    }
    XFORMERS = _xformers(INPUT_SIGS)

class ManagerSpec(InterfaceSpec):
    """
    Manager interface.
    """
    # pylint: disable=too-few-public-methods

    class MethodNames(enum.Enum):
        """
        The method names of the manager interface.
        """
        ConfigureSimulator = "ConfigureSimulator"
        CreatePool = "CreatePool"
        DestroyPool = "DestroyPool"
        GetCacheObjectPath = "GetCacheObjectPath"
        GetDevObjectPath = "GetDevObjectPath"
        GetDevTypes = "GetDevTypes"
        GetErrorCodes = "GetErrorCodes"
        GetFilesystemObjectPath = "GetFilesystemObjectPath"
        GetPoolObjectPath = "GetPoolObjectPath"
        GetRaidLevels = "GetRaidLevels"
        ListPools = "ListPools"

    class PropertyNames(enum.Enum):
        """
        Names of the properties of the manager interface.
        """
        pass

    INTERFACE_NAME = 'org.storage.stratis1.Manager'

    INPUT_SIGS = {
        MethodNames.ConfigureSimulator : (("denominator", ), _FALSE, "u"),
        MethodNames.CreatePool :
           (("name", "redundancy", "force", "devices"), _FALSE, "sqbas"),
        MethodNames.DestroyPool : (("name", ), _FALSE, "s"),
        MethodNames.GetCacheObjectPath : (("name", ), _FALSE, "s"),
        MethodNames.GetDevObjectPath : (("name", ), _FALSE, "s"),
        MethodNames.GetDevTypes : ((), _FALSE, ""),
        MethodNames.GetErrorCodes : ((), _FALSE, ""),
        MethodNames.GetFilesystemObjectPath :
           (("pool_name", "filesystem_name"), _FALSE, "ss"),
        MethodNames.GetPoolObjectPath : (("name", ), _FALSE, "s"),
        MethodNames.GetRaidLevels : ((), _FALSE, ""),
        MethodNames.ListPools : ((), _FALSE, ""),
    }
    OUTPUT_SIGS = {
        MethodNames.ConfigureSimulator : "qs",
        MethodNames.CreatePool : "oqs",
        MethodNames.DestroyPool : "bqs",
        MethodNames.GetCacheObjectPath : "oqs",
        MethodNames.GetDevObjectPath : "oqs",
        MethodNames.GetDevTypes : "",
        MethodNames.GetErrorCodes : "a(sqs)",
        MethodNames.GetFilesystemObjectPath : "oqs",
        MethodNames.GetPoolObjectPath : "oqs",
        MethodNames.GetRaidLevels : "a(sqs)",
        MethodNames.ListPools : "asqs",
    }
    XFORMERS = _xformers(INPUT_SIGS)

class PoolSpec(InterfaceSpec):
    """
    Pool interface.
    """
    # pylint: disable=too-few-public-methods

    class MethodNames(enum.Enum):
        """
        Names of the methods of the Pool class.
        """
        AddCacheDevs = "AddCacheDevs"
        AddDevs = "AddDevs"
        CreateFilesystems = "CreateFilesystems"
        DestroyFilesystems = "DestroyFilesystems"
        ListCacheDevs = "ListCacheDevs"
        ListDevs = "ListDevs"
        ListFilesystems = "ListFilesystems"
        RemoveCacheDevs = "RemoveCacheDevs"
        RemoveDevs = "RemoveDevs"
        Rename = "Rename"

    class PropertyNames(enum.Enum):
        """
        Names of the properties of the manager interface.
        """
        pass

    INTERFACE_NAME = 'org.storage.stratis1.pool'

    INPUT_SIGS = { # pragma: no cover
       MethodNames.AddCacheDevs: (("force", "devices", ), _FALSE, "bas"),
       MethodNames.AddDevs: (("force", "devices", ), _FALSE, "bas"),
       MethodNames.CreateFilesystems: (
          ("specs", ),
          (lambda n: \
             lambda x: \
                [(x, y, _option_to_tuple(quota, 0)) for (x, y, quota) in x]),
          "a(ss(bt))"
       ),
       MethodNames.DestroyFilesystems: (("names", ), _FALSE, "as"),
       MethodNames.ListCacheDevs: ((), _FALSE, ""),
       MethodNames.ListDevs: ((), _FALSE, ""),
       MethodNames.ListFilesystems: ((), _FALSE, ""),
       MethodNames.RemoveCacheDevs: (("names", ), _FALSE, "as"),
       MethodNames.RemoveDevs: (("names", ), _FALSE, "as"),
       MethodNames.Rename: (("new_name", ), _FALSE, "s")
    }
    OUTPUT_SIGS = {
       MethodNames.AddCacheDevs: "a(oqs)qs",
       MethodNames.AddDevs: "a(oqs)qs",
       MethodNames.CreateFilesystems: "a(oqs)qs",
       MethodNames.DestroyFilesystems: "a(qs)qs",
       MethodNames.ListCacheDevs: "asqs",
       MethodNames.ListDevs: "asqs",
       MethodNames.ListFilesystems: "asqs",
       MethodNames.RemoveCacheDevs: "a(qs)qs",
       MethodNames.RemoveDevs: "a(qs)qs",
       MethodNames.Rename: "bqs"
    }
    XFORMERS = _xformers(INPUT_SIGS)

def _prop_builder(spec):
    """
    Returns a function that builds a property interface based on 'spec'.

    :param spec: the interface specification
    :type spec: type, a subtype of InterfaceSpec
    """

    def builder(namespace):
        """
        The property class's namespace.

        :param namespace: the class's namespace
        """

        def build_property(prop): # pragma: no cover
            """
            Build a single property getter for this class.

            :param prop: the property
            """

            def dbus_func(proxy_object): # pragma: no cover
                """
                The property getter.
                """
                return proxy_object.Get(
                   spec.INTERFACE_NAME,
                   prop.name,
                   dbus_interface=dbus.PROPERTIES_IFACE
                )

            return dbus_func

        for prop in spec.PropertyNames:
            namespace[prop.name] = staticmethod(build_property(prop))

    return builder

def _iface_builder(spec):
    """
    Returns a function that builds a method interface based on 'spec'.

    :param spec: the interface specification
    :type spec: type, a subtype of InterfaceSpec
    """

    def builder(namespace):
        """
        Builds the class.

        :param namespace: the class's namespace
        """

        def build_method(method):
            """
            Build a single method for this class.

            :param method: the method
            """
            (names, func) = spec.XFORMERS[method]

            def dbus_func(proxy_object, **kwargs): # pragma: no cover
                """
                The function method spec.
                """
                if frozenset(names) != frozenset(kwargs.keys()):
                    raise ValueError("Bad keys")
                args = \
                   [v for (k, v) in \
                   sorted(kwargs.items(), key=lambda x: names.index(x[0]))]
                xformed_args = func(args)
                dbus_method = getattr(proxy_object, method.name)
                return dbus_method(
                   *xformed_args,
                   dbus_interface=spec.INTERFACE_NAME
                )

            return dbus_func

        for method in spec.MethodNames:
            namespace[method.name] = staticmethod(build_method(method))

        namespace['Properties'] = \
           types.new_class(
              "Properties",
              bases=(object,),
              exec_body=_prop_builder(spec)
           )

    return builder

Filesystem = types.new_class(
   "Filesystem",
   bases=(object,),
   exec_body=_iface_builder(FilesystemSpec)
)
Manager = types.new_class(
   "Manager",
   bases=(object,),
   exec_body=_iface_builder(ManagerSpec)
)
Pool = \
   types.new_class("Pool", bases=(object,), exec_body=_iface_builder(PoolSpec))
<|endoftext|>


<commit_before>#!/usr/bin/env python
import os
from igf_data.utils.tools.samtools_utils import convert_bam_to_cram
from ehive.runnable.IGFBaseProcess import IGFBaseProcess
from igf_data.utils.analysis_collection_utils import Analysis_collection_utils
from igf_data.utils.tools.reference_genome_utils import Reference_genome_utils
from igf_data.utils.fileutils import get_temp_dir,remove_dir

class ConvertBamToCram(IGFBaseProcess):
  '''
  A ehive process class for converting bam files to cram files
  '''
  def param_defaults(self):
    params_dict=super(ConvertBamToCram,self).param_defaults()
    params_dict.update({
        'force_overwrite':True,
        'reference_type':None,
        'collection_name':None,
        'analysis_name':None,
        'tag_name':None,
        'collection_type':'ANALYSIS_CRAM',
        'reference_type':'GENOME_FASTA',
        'collection_table':'experiment',
      })
    return params_dict

  def run(self):
    '''
    A method for running bam to cram conversion

    :param project_igf_id: A project igf id
    :param sample_igf_id: A sample igf id
    :param igf_session_class: A database session class
    :param species_name: Reference genome collection name
    :param analysis_name: Name of the analysis for source bam
    :param tag_name: Additional tag for renaming file
    :param base_result_dir: Base work directory path
    :param collection_name: A database collection name for output file, default None
    :param collection_type: A database collection type for output file, default ANALYSIS_CRAM
    :param collection_table: A database collection table for output file, default experiment
    :param force_overwrite: A toggle for retiring old collection group, default True
    :param reference_type: Reference genome collection type, default GENOME_FASTA
    '''
    try:
      project_igf_id=self.param_required('project_igf_id')
      sample_igf_id=self.param_required('sample_igf_id')
      igf_session_class=self.param_required('igf_session_class')
      species_name=self.param_required('species_name')
      bam_file=self.param_required('bam_file')
      base_result_dir=self.param_required('base_result_dir')
      collection_name=self.param_required('collection_name')
      collection_type=self.param_required('collection_type')
      collection_table=self.param_required('collection_table')
      analysis_name=self.param_required('analysis_name')
      tag_name=self.param_required('tag_name')
      force_overwrite=self.param_required('force_overwrite')
      reference_type=self.param('reference_type')

      if collection_type is None or \
         collection_name is None or \
         collection_table is None:
        raise ValueError('Missing collection information for loading cram file')

      ref_genome=Reference_genome_utils(genome_tag=species_name,
                                        dbsession_class=igf_session_class,
                                        genome_fasta_type=reference_type)
      genome_fasta=ref_genome.get_genome_fasta()                                # get genome fasta
      temp_work_dir=get_temp_dir()                                              # get temp dir
      cram_file=os.path.basename(bam_file).replace('.bam','.cram')              # get base cram file name
      cram_file=os.path.join(temp_work_dir,cram_file)                           # get cram file path in work dir
      convert_bam_to_cram(bam_file=bam_file,
                          reference_file=genome_fasta,
                          cram_path=cram_file)                                  # create new cramfile
      au=Analysis_collection_utils(dbsession_class=igf_session_class,
                                   analysis_name=analysis_name,
                                   tag_name=tag_name,
                                   collection_name=collection_name,
                                   collection_type=collection_type,
                                   collection_table=collection_table,
                                   base_path=base_result_dir)
      output_cram_list=au.load_file_to_disk_and_db(input_file_list=[cram_file],
                                   withdraw_exisitng_collection=force_overwrite) # load file to db and disk
      self.param('dataflow_params',{'bam_file':bam_file,
                                    'output_cram_list':output_cram_list})       # pass on bam output path
    except Exception as e:
      message='project: {2}, sample:{3}, Error in {0}: {1}'.format(self.__class__.__name__, \
                                                      e, \
                                                      project_igf_id,
                                                      sample_igf_id)
      self.warning(message)
      self.post_message_to_slack(message,reaction='fail')                       # post msg to slack for failed jobs
      raise<commit_msg>adding multi thread support and slack log for bam to cram analysis<commit_after>#!/usr/bin/env python
import os
from igf_data.utils.tools.samtools_utils import convert_bam_to_cram
from ehive.runnable.IGFBaseProcess import IGFBaseProcess
from igf_data.utils.analysis_collection_utils import Analysis_collection_utils
from igf_data.utils.tools.reference_genome_utils import Reference_genome_utils
from igf_data.utils.fileutils import get_temp_dir,remove_dir

class ConvertBamToCram(IGFBaseProcess):
  '''
  A ehive process class for converting bam files to cram files
  '''
  def param_defaults(self):
    params_dict=super(ConvertBamToCram,self).param_defaults()
    params_dict.update({
        'force_overwrite':True,
        'reference_type':None,
        'collection_name':None,
        'analysis_name':None,
        'tag_name':None,
        'collection_type':'ANALYSIS_CRAM',
        'reference_type':'GENOME_FASTA',
        'collection_table':'experiment',
        'threads':4,
      })
    return params_dict

  def run(self):
    '''
    A method for running bam to cram conversion

    :param project_igf_id: A project igf id
    :param sample_igf_id: A sample igf id
    :param igf_session_class: A database session class
    :param species_name: Reference genome collection name
    :param analysis_name: Name of the analysis for source bam
    :param tag_name: Additional tag for renaming file
    :param base_result_dir: Base work directory path
    :param collection_name: A database collection name for output file, default None
    :param collection_type: A database collection type for output file, default ANALYSIS_CRAM
    :param collection_table: A database collection table for output file, default experiment
    :param force_overwrite: A toggle for retiring old collection group, default True
    :param reference_type: Reference genome collection type, default GENOME_FASTA
    :param threads: Number of threads to use for Bam to Cram conversion, default 4
    '''
    try:
      project_igf_id=self.param_required('project_igf_id')
      sample_igf_id=self.param_required('sample_igf_id')
      experiment_igf_id=self.param_required('experiment_igf_id')
      igf_session_class=self.param_required('igf_session_class')
      species_name=self.param_required('species_name')
      bam_file=self.param_required('bam_file')
      base_result_dir=self.param_required('base_result_dir')
      collection_name=self.param_required('collection_name')
      collection_type=self.param_required('collection_type')
      collection_table=self.param_required('collection_table')
      analysis_name=self.param_required('analysis_name')
      tag_name=self.param_required('tag_name')
      force_overwrite=self.param_required('force_overwrite')
      reference_type=self.param('reference_type')
      threads=self.param('threads')

      if collection_type is None or \
         collection_name is None or \
         collection_table is None:
        raise ValueError('Missing collection information for loading cram file')

      ref_genome=Reference_genome_utils(genome_tag=species_name,
                                        dbsession_class=igf_session_class,
                                        genome_fasta_type=reference_type)
      genome_fasta=ref_genome.get_genome_fasta()                                # get genome fasta
      temp_work_dir=get_temp_dir()                                              # get temp dir
      cram_file=os.path.basename(bam_file).replace('.bam','.cram')              # get base cram file name
      cram_file=os.path.join(temp_work_dir,cram_file)                           # get cram file path in work dir
      convert_bam_to_cram(bam_file=bam_file,
                          reference_file=genome_fasta,
                          cram_path=cram_file,
                          threads=threads)                                      # create new cramfile
      au=Analysis_collection_utils(dbsession_class=igf_session_class,
                                   analysis_name=analysis_name,
                                   tag_name=tag_name,
                                   collection_name=collection_name,
                                   collection_type=collection_type,
                                   collection_table=collection_table,
                                   base_path=base_result_dir)
      output_cram_list=au.load_file_to_disk_and_db(input_file_list=[cram_file],
                                   withdraw_exisitng_collection=force_overwrite) # load file to db and disk
      self.param('dataflow_params',{'bam_file':bam_file,
                                    'output_cram_list':output_cram_list})       # pass on bam output path
      message='finished bam to cram conversion for {0}, {1} {2}'.\
              format(project_igf_id,
                     sample_igf_id,
                     experiment_igf_id)
      self.post_message_to_slack(message,reaction='pass')                       # send log to slack
      self.comment_asana_task(task_name=project_igf_id, comment=message)        # send comment to Asana
    except Exception as e:
      message='project: {2}, sample:{3}, Error in {0}: {1}'.format(self.__class__.__name__, \
                                                      e, \
                                                      project_igf_id,
                                                      sample_igf_id)
      self.warning(message)
      self.post_message_to_slack(message,reaction='fail')                       # post msg to slack for failed jobs
      raise<|endoftext|>


<commit_before>import nltk

class PosExtractor(object):

    def __init__(self, debug = False):
        pass

    def extract_entities(self, token_sent, np_bounds, rel_bounds):
        pass

<commit_msg>update relation extractor wording and comments<commit_after>import nltk

class PosExtractor(object):

    def __init__(self, debug = False):
        pass

    def extract_relationships(self, token_sent, np_bounds, rel_bounds):
        """ Extract relationships based upon entity's found in Noun Phrases and
            Relational Phrases - A simple approach - if we find Relational Phrases
            that are surrounded by Noun Phrases, then we'll call this a relationship
        """

        pass

<|endoftext|>