<commit_before>#
#

# Copyright (C) 2014 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.

"""QA utility functions for testing jobs

"""

import re
import sys
import threading
import time

from ganeti import constants
from ganeti import locking
from ganeti import utils
from ganeti.utils import retry

import qa_config
import qa_error

from qa_utils import AssertCommand, GetCommandOutput, GetObjectInfo

AVAILABLE_LOCKS = [locking.LEVEL_NODE, ]

def _GetOutputFromMaster(cmd, use_multiplexer=True, log_cmd=True):
  """ Gets the output of a command executed on master.

  """
  if isinstance(cmd, basestring):
    cmdstr = cmd
  else:
    cmdstr = utils.ShellQuoteArgs(cmd)

  # Necessary due to the stderr stream not being captured properly on the
  # buildbot
  cmdstr += " 2>&1"

  return GetCommandOutput(qa_config.GetMasterNode().primary, cmdstr,
                          use_multiplexer=use_multiplexer, log_cmd=log_cmd)

def ExecuteJobProducingCommand(cmd):
  """ Executes a command that contains the --submit flag, and returns a job id.

  @type cmd: list of string
  @param cmd: The command to execute, broken into constituent components.

  """
  job_id_output = GetOutputFromMaster(cmd)

  # Usually, the output contains "JobID: <job_id>", but for instance related
  # commands, the output is of the form "<job_id>: <instance_name>"
  possible_job_ids = re.findall("JobID: ([0-9]+)", job_id_output) or \
                     re.findall("([0-9]+): .+", job_id_output)
  if len(possible_job_ids) != 1:
    raise qa_error.Error("Cannot parse command output to find job id: output "
                         "is %s" % job_id_output)

  return int(possible_job_ids[0])

def _RetrieveTerminationInfo(job_id):
  """ Retrieves the termination info from a job caused by gnt-debug delay.

  @rtype: dict or None
  @return: The termination log entry, or None if no entry was found

  """
  job_info = GetObjectInfo(["gnt-job", "info", str(job_id)])

  opcodes = job_info[0]["Opcodes"]
  if not opcodes:
    raise qa_error.Error("Cannot retrieve a list of opcodes")

  execution_logs = opcodes[0]["Execution log"]
  if not execution_logs:
    return None

  is_termination_info_fn = \
    lambda e: e["Content"][1] == constants.ELOG_DELAY_TEST

  filtered_logs = filter(is_termination_info_fn, execution_logs)

  no_logs = len(filtered_logs)
  if no_logs > 1:
    raise qa_error.Error("Too many interruption information entries found!")
  elif no_logs == 1:
    return filtered_logs[0]
  else:
    return None

def _StartDelayFunction(locks, timeout):
  """ Starts the gnt-debug delay option with the given locks and timeout.

  """
  # The interruptible switch must be used
  cmd = ["gnt-debug", "delay", "-i", "--submit", "--no-master"]

  for node in locks.get(locking.LEVEL_NODE, []):
    cmd.append("-n%s" % node)
  cmd.append(str(timeout))

  job_id = ExecuteJobProducingCommand(cmd)

  # Waits until a non-empty result is returned from the function
  log_entry = retry.SimpleRetry(lambda x: x, _RetrieveTerminationInfo, 2.0,
                                10.0, args=[job_id])

  if not log_entry:
    raise qa_error.Error("Failure when trying to retrieve delay termination "
                         "information")

  _, _, (socket_path, ) = log_entry["Content"]

  return socket_path

def _TerminateDelayFunction(termination_socket):
  """ Terminates the delay function by communicating with the domain socket.

  """
  AssertCommand("echo a | socat -u stdin UNIX-CLIENT:%s" % termination_socket)

def _GetNodeUUIDMap(nodes):
  """ Given a list of nodes, retrieves a mapping of their names to UUIDs.

  @type nodes: list of string
  @param nodes: The nodes to retrieve a map for. If empty, returns information
                for all the nodes.

  """
  cmd = ["gnt-node", "list", "--no-header", "-o", "name,uuid"]
  cmd.extend(nodes)
  output = GetOutputFromMaster(cmd)
  return dict(map(lambda x: x.split(), output.splitlines()))

def _FindLockNames(locks):
  """ Finds the ids and descriptions of locks that given locks can block.

  @type locks: dict of locking level to list
  @param locks: The locks that gnt-debug delay is holding.

  @rtype: dict of string to string
  @return: The lock name to entity name map.

  For a given set of locks, some internal locks (e.g. ALL_SET locks) can be
  blocked even though they were not listed explicitly. This function has to take
  care and list all locks that can be blocked by the locks given as parameters.

  """
  lock_map = {}

  if locking.LEVEL_NODE in locks:
    node_locks = locks[locking.LEVEL_NODE]
    if node_locks == locking.ALL_SET:
      # Empty list retrieves all info
      name_uuid_map = _GetNodeUUIDMap([])
    else:
      name_uuid_map = _GetNodeUUIDMap(node_locks)

    for name in name_uuid_map:
      lock_map["node/%s" % name_uuid_map[name]] = name

    # If ALL_SET was requested explicitly, or there is at least one lock
    # Note that locking.ALL_SET is None and hence the strange form of the if
    if node_locks == locking.ALL_SET or node_locks:
      lock_map["node/[lockset]"] = "joint node lock"

  #TODO add other lock types here when support for these is added
  return lock_map

def _GetBlockingLocks():
  """ Finds out which locks are blocking jobs by invoking "gnt-debug locks".

  @rtype: list of string
  @return: The names of the locks currently blocking any job.

  """
  # Due to mysterious issues when a SSH multiplexer is being used by two
  # threads, we turn it off, and block most of the logging to improve the
  # visibility of the other thread's output
  locks_output = GetOutputFromMaster("gnt-debug locks", use_multiplexer=False,
                                      log_cmd=False)

  # The first non-empty line is the header, which we do not need
  lock_lines = locks_output.splitlines()[1:]

  blocking_locks = []
  for lock_line in lock_lines:
    components = lock_line.split()
    if len(components) != 4:
      raise qa_error.Error("Error while parsing gnt-debug locks output, "
                           "line at fault is: %s" % lock_line)

    lock_name, _, _, pending_jobs = components

    if pending_jobs != '-':
      blocking_locks.append(lock_name)

  return blocking_locks

class QAThread(threading.Thread):
  """ An exception-preserving thread that executes a given function.

  """
  def __init__(self, fn, args, kwargs):
    """ Constructor accepting the function to be invoked later.

    """
    threading.Thread.__init__(self)
    self._fn = fn
    self._args = args
    self._kwargs = kwargs
    self._exc_info = None

  def run(self):
    """ Executes the function, preserving exception info if necessary.

    """
    # pylint: disable=W0702
    # We explicitly want to catch absolutely anything
    try:
      self._fn(*self._args, **self._kwargs)
    except:
      self._exc_info = sys.exc_info()
    # pylint: enable=W0702

  def reraise(self):
    """ Reraises any exceptions that might have occured during thread execution.

    """
    if self._exc_info is not None:
      raise self._exc_info[0], self._exc_info[1], self._exc_info[2]

# TODO: Can this be done as a decorator? Implement as needed.
def RunWithLocks(fn, locks, timeout, block, *args, **kwargs):
  """ Runs the given function, acquiring a set of locks beforehand.

  @type fn: function
  @param fn: The function to invoke.
  @type locks: dict of string to list of string
  @param locks: The locks to acquire, per lock category.
  @type timeout: number
  @param timeout: The number of seconds the locks should be held before
                  expiring.
  @type block: bool
  @param block: Whether the test should block when locks are used or not.

  This function allows a set of locks to be acquired in preparation for a QA
  test, to try and see if the function can run in parallel with other
  operations.

  Locks are acquired by invoking a gnt-debug delay operation which can be
  interrupted as needed. The QA test is then run in a separate thread, with the
  current thread observing jobs waiting for locks. When a job is spotted waiting
  for a lock held by the started delay operation, this is noted, and the delay
  is interrupted, allowing the QA test to continue.

  A default timeout is not provided by design - the test creator must make a
  good conservative estimate.

  """
  if filter(lambda l_type: l_type not in AVAILABLE_LOCKS, locks):
    raise qa_error.Error("Attempted to acquire locks that cannot yet be "
                         "acquired in the course of a QA test.")

  # The watcher may interfere by issuing its own jobs - therefore pause it
  AssertCommand(["gnt-cluster", "watcher", "pause", "12h"])

  # Find out the lock names prior to starting the delay function
  lock_name_map = _FindLockNames(locks)

  blocking_owned_locks = []
  test_blocked = False

  termination_socket = _StartDelayFunction(locks, timeout)
  delay_fn_terminated = False

  try:
    qa_thread = QAThread(fn, args, kwargs)
    qa_thread.start()

    while qa_thread.isAlive():
      blocking_locks = _GetBlockingLocks()
      blocking_owned_locks = \
        set(blocking_locks).intersection(set(lock_name_map))

      if blocking_owned_locks:
        # Set the flag first - if the termination attempt fails, we do not want
        # to redo it in the finally block
        delay_fn_terminated = True
        _TerminateDelayFunction(termination_socket)
        test_blocked = True
        break

      time.sleep(5) # Set arbitrarily

    # The thread should be either finished or unblocked at this point
    qa_thread.join()

    # Raise any errors that might have occured in the thread
    qa_thread.reraise()

  finally:
    if not delay_fn_terminated:
      _TerminateDelayFunction(termination_socket)

  blocking_lock_names = ", ".join(map(lock_name_map.get, blocking_owned_locks))
  if not block and test_blocked:
    raise qa_error.Error("QA test succeded, but was blocked by locks: %s" %
                         blocking_lock_names)
  elif block and not test_blocked:
    raise qa_error.Error("QA test succeded, but was not blocked as it was "
                         "expected to by locks: %s" % blocking_lock_names)
  else:
    pass

  # Revive the watcher
  AssertCommand(["gnt-cluster", "watcher", "continue"])
<commit_msg>Do export _GetOutputFromMaster<commit_after>#
#

# Copyright (C) 2014 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
# 02110-1301, USA.

"""QA utility functions for testing jobs

"""

import re
import sys
import threading
import time

from ganeti import constants
from ganeti import locking
from ganeti import utils
from ganeti.utils import retry

import qa_config
import qa_error

from qa_utils import AssertCommand, GetCommandOutput, GetObjectInfo

AVAILABLE_LOCKS = [locking.LEVEL_NODE, ]

def GetOutputFromMaster(cmd, use_multiplexer=True, log_cmd=True):
  """ Gets the output of a command executed on master.

  """
  if isinstance(cmd, basestring):
    cmdstr = cmd
  else:
    cmdstr = utils.ShellQuoteArgs(cmd)

  # Necessary due to the stderr stream not being captured properly on the
  # buildbot
  cmdstr += " 2>&1"

  return GetCommandOutput(qa_config.GetMasterNode().primary, cmdstr,
                          use_multiplexer=use_multiplexer, log_cmd=log_cmd)

def ExecuteJobProducingCommand(cmd):
  """ Executes a command that contains the --submit flag, and returns a job id.

  @type cmd: list of string
  @param cmd: The command to execute, broken into constituent components.

  """
  job_id_output = GetOutputFromMaster(cmd)

  # Usually, the output contains "JobID: <job_id>", but for instance related
  # commands, the output is of the form "<job_id>: <instance_name>"
  possible_job_ids = re.findall("JobID: ([0-9]+)", job_id_output) or \
                     re.findall("([0-9]+): .+", job_id_output)
  if len(possible_job_ids) != 1:
    raise qa_error.Error("Cannot parse command output to find job id: output "
                         "is %s" % job_id_output)

  return int(possible_job_ids[0])

def _RetrieveTerminationInfo(job_id):
  """ Retrieves the termination info from a job caused by gnt-debug delay.

  @rtype: dict or None
  @return: The termination log entry, or None if no entry was found

  """
  job_info = GetObjectInfo(["gnt-job", "info", str(job_id)])

  opcodes = job_info[0]["Opcodes"]
  if not opcodes:
    raise qa_error.Error("Cannot retrieve a list of opcodes")

  execution_logs = opcodes[0]["Execution log"]
  if not execution_logs:
    return None

  is_termination_info_fn = \
    lambda e: e["Content"][1] == constants.ELOG_DELAY_TEST

  filtered_logs = filter(is_termination_info_fn, execution_logs)

  no_logs = len(filtered_logs)
  if no_logs > 1:
    raise qa_error.Error("Too many interruption information entries found!")
  elif no_logs == 1:
    return filtered_logs[0]
  else:
    return None

def _StartDelayFunction(locks, timeout):
  """ Starts the gnt-debug delay option with the given locks and timeout.

  """
  # The interruptible switch must be used
  cmd = ["gnt-debug", "delay", "-i", "--submit", "--no-master"]

  for node in locks.get(locking.LEVEL_NODE, []):
    cmd.append("-n%s" % node)
  cmd.append(str(timeout))

  job_id = ExecuteJobProducingCommand(cmd)

  # Waits until a non-empty result is returned from the function
  log_entry = retry.SimpleRetry(lambda x: x, _RetrieveTerminationInfo, 2.0,
                                10.0, args=[job_id])

  if not log_entry:
    raise qa_error.Error("Failure when trying to retrieve delay termination "
                         "information")

  _, _, (socket_path, ) = log_entry["Content"]

  return socket_path

def _TerminateDelayFunction(termination_socket):
  """ Terminates the delay function by communicating with the domain socket.

  """
  AssertCommand("echo a | socat -u stdin UNIX-CLIENT:%s" % termination_socket)

def _GetNodeUUIDMap(nodes):
  """ Given a list of nodes, retrieves a mapping of their names to UUIDs.

  @type nodes: list of string
  @param nodes: The nodes to retrieve a map for. If empty, returns information
                for all the nodes.

  """
  cmd = ["gnt-node", "list", "--no-header", "-o", "name,uuid"]
  cmd.extend(nodes)
  output = GetOutputFromMaster(cmd)
  return dict(map(lambda x: x.split(), output.splitlines()))

def _FindLockNames(locks):
  """ Finds the ids and descriptions of locks that given locks can block.

  @type locks: dict of locking level to list
  @param locks: The locks that gnt-debug delay is holding.

  @rtype: dict of string to string
  @return: The lock name to entity name map.

  For a given set of locks, some internal locks (e.g. ALL_SET locks) can be
  blocked even though they were not listed explicitly. This function has to take
  care and list all locks that can be blocked by the locks given as parameters.

  """
  lock_map = {}

  if locking.LEVEL_NODE in locks:
    node_locks = locks[locking.LEVEL_NODE]
    if node_locks == locking.ALL_SET:
      # Empty list retrieves all info
      name_uuid_map = _GetNodeUUIDMap([])
    else:
      name_uuid_map = _GetNodeUUIDMap(node_locks)

    for name in name_uuid_map:
      lock_map["node/%s" % name_uuid_map[name]] = name

    # If ALL_SET was requested explicitly, or there is at least one lock
    # Note that locking.ALL_SET is None and hence the strange form of the if
    if node_locks == locking.ALL_SET or node_locks:
      lock_map["node/[lockset]"] = "joint node lock"

  #TODO add other lock types here when support for these is added
  return lock_map

def _GetBlockingLocks():
  """ Finds out which locks are blocking jobs by invoking "gnt-debug locks".

  @rtype: list of string
  @return: The names of the locks currently blocking any job.

  """
  # Due to mysterious issues when a SSH multiplexer is being used by two
  # threads, we turn it off, and block most of the logging to improve the
  # visibility of the other thread's output
  locks_output = GetOutputFromMaster("gnt-debug locks", use_multiplexer=False,
                                     log_cmd=False)

  # The first non-empty line is the header, which we do not need
  lock_lines = locks_output.splitlines()[1:]

  blocking_locks = []
  for lock_line in lock_lines:
    components = lock_line.split()
    if len(components) != 4:
      raise qa_error.Error("Error while parsing gnt-debug locks output, "
                           "line at fault is: %s" % lock_line)

    lock_name, _, _, pending_jobs = components

    if pending_jobs != '-':
      blocking_locks.append(lock_name)

  return blocking_locks

class QAThread(threading.Thread):
  """ An exception-preserving thread that executes a given function.

  """
  def __init__(self, fn, args, kwargs):
    """ Constructor accepting the function to be invoked later.

    """
    threading.Thread.__init__(self)
    self._fn = fn
    self._args = args
    self._kwargs = kwargs
    self._exc_info = None

  def run(self):
    """ Executes the function, preserving exception info if necessary.

    """
    # pylint: disable=W0702
    # We explicitly want to catch absolutely anything
    try:
      self._fn(*self._args, **self._kwargs)
    except:
      self._exc_info = sys.exc_info()
    # pylint: enable=W0702

  def reraise(self):
    """ Reraises any exceptions that might have occured during thread execution.

    """
    if self._exc_info is not None:
      raise self._exc_info[0], self._exc_info[1], self._exc_info[2]

# TODO: Can this be done as a decorator? Implement as needed.
def RunWithLocks(fn, locks, timeout, block, *args, **kwargs):
  """ Runs the given function, acquiring a set of locks beforehand.

  @type fn: function
  @param fn: The function to invoke.
  @type locks: dict of string to list of string
  @param locks: The locks to acquire, per lock category.
  @type timeout: number
  @param timeout: The number of seconds the locks should be held before
                  expiring.
  @type block: bool
  @param block: Whether the test should block when locks are used or not.

  This function allows a set of locks to be acquired in preparation for a QA
  test, to try and see if the function can run in parallel with other
  operations.

  Locks are acquired by invoking a gnt-debug delay operation which can be
  interrupted as needed. The QA test is then run in a separate thread, with the
  current thread observing jobs waiting for locks. When a job is spotted waiting
  for a lock held by the started delay operation, this is noted, and the delay
  is interrupted, allowing the QA test to continue.

  A default timeout is not provided by design - the test creator must make a
  good conservative estimate.

  """
  if filter(lambda l_type: l_type not in AVAILABLE_LOCKS, locks):
    raise qa_error.Error("Attempted to acquire locks that cannot yet be "
                         "acquired in the course of a QA test.")

  # The watcher may interfere by issuing its own jobs - therefore pause it
  AssertCommand(["gnt-cluster", "watcher", "pause", "12h"])

  # Find out the lock names prior to starting the delay function
  lock_name_map = _FindLockNames(locks)

  blocking_owned_locks = []
  test_blocked = False

  termination_socket = _StartDelayFunction(locks, timeout)
  delay_fn_terminated = False

  try:
    qa_thread = QAThread(fn, args, kwargs)
    qa_thread.start()

    while qa_thread.isAlive():
      blocking_locks = _GetBlockingLocks()
      blocking_owned_locks = \
        set(blocking_locks).intersection(set(lock_name_map))

      if blocking_owned_locks:
        # Set the flag first - if the termination attempt fails, we do not want
        # to redo it in the finally block
        delay_fn_terminated = True
        _TerminateDelayFunction(termination_socket)
        test_blocked = True
        break

      time.sleep(5) # Set arbitrarily

    # The thread should be either finished or unblocked at this point
    qa_thread.join()

    # Raise any errors that might have occured in the thread
    qa_thread.reraise()

  finally:
    if not delay_fn_terminated:
      _TerminateDelayFunction(termination_socket)

  blocking_lock_names = ", ".join(map(lock_name_map.get, blocking_owned_locks))
  if not block and test_blocked:
    raise qa_error.Error("QA test succeded, but was blocked by locks: %s" %
                         blocking_lock_names)
  elif block and not test_blocked:
    raise qa_error.Error("QA test succeded, but was not blocked as it was "
                         "expected to by locks: %s" % blocking_lock_names)
  else:
    pass

  # Revive the watcher
  AssertCommand(["gnt-cluster", "watcher", "continue"])
<|endoftext|>


<commit_before># test mod_md status resources

import json
import os
import pytest
import re
import socket
import ssl
import sys
import time

from datetime import datetime
from TestEnv import TestEnv
from TestHttpdConf import HttpdConf
from TestCertUtil import CertUtil
from shutil import copyfile

def setup_module(module):
    print("setup_module    module:%s" % module.__name__)
    TestEnv.init()
    TestEnv.APACHE_CONF_SRC = "data/test_auto"
    TestEnv.check_acme()
    TestEnv.clear_store()
    HttpdConf().install();

def teardown_module(module):
    print("teardown_module module:%s" % module.__name__)
    assert TestEnv.apache_stop() == 0

class TestStatus:

    def setup_method(self, method):
        print("setup_method: %s" % method.__name__)
        TestEnv.clear_store()
        self.test_domain = TestEnv.get_method_domain(method)

    def teardown_method(self, method):
        print("teardown_method: %s" % method.__name__)

    # simple MD, drive it, check status before activation
    def test_920_001(self):
        domain = self.test_domain
        domains = [ domain ]
        conf = HttpdConf()
        conf.add_admin( "admin@not-forbidden.org" )
        conf.add_md( domains )
        conf.add_vhost(domain)
        conf.install()
        assert TestEnv.apache_restart() == 0
        assert TestEnv.await_completion( [ domain ], restart=False )
        # we started without a valid certificate, so we expect /.httpd/certificate-status
        # to not give information about one and - since we waited for the ACME signup
        # to complete - to give information in 'renewal' about the new cert.
        status = TestEnv.get_certificate_status( domain )
        assert not 'sha256-fingerprint' in status
        assert not 'valid' in status
        assert 'renewal' in status
        assert 'valid' in status['renewal']
        assert 'sha256-fingerprint' in status['renewal']
        # restart and activate
        # once activated, the staging must be gone and attributes exist for the active cert
        assert TestEnv.apache_restart() == 0
        status = TestEnv.get_certificate_status( domain )
        assert not 'renewal' in status
        assert 'sha256-fingerprint' in status
        assert 'valid' in status
        assert 'from' in status['valid']

    # simple MD, drive it, manipulate staged credentials and check status
    def test_920_002(self):
        domain = self.test_domain
        domains = [ domain ]
        conf = HttpdConf()
        conf.add_admin( "admin@not-forbidden.org" )
        conf.add_md( domains )
        conf.add_vhost(domain)
        conf.install()
        assert TestEnv.apache_restart() == 0
        assert TestEnv.await_completion( [ domain ], restart=False )
        # copy a real certificate from LE over to staging
        staged_cert = os.path.join(TestEnv.STORE_DIR, 'staging', domain, 'pubcert.pem')
        real_cert = os.path.join('data', 'test_920', '002.pubcert')
        assert copyfile(real_cert, staged_cert)
        status = TestEnv.get_certificate_status( domain )
        # status shows the copied cert's properties as staged
        assert 'renewal' in status
        assert 'Thu, 29 Aug 2019 16:06:35 GMT' == status['renewal']['valid']['until']
        assert 'Fri, 31 May 2019 16:06:35 GMT' == status['renewal']['valid']['from']
        assert '03039C464D454EDE79FCD2CAE859F668F269' ==  status['renewal']['serial']
        assert 'sha256-fingerprint' in status['renewal']
        if 0 == 1:
            assert len(status['renewal']['scts']) == 2
            assert status['renewal']['scts'][0]['logid'] == '747eda8331ad331091219cce254f4270c2bffd5e422008c6373579e6107bcc56'
            assert status['renewal']['scts'][0]['signed'] == 'Fri, 31 May 2019 17:06:35 GMT'
            assert status['renewal']['scts'][1]['logid'] == '293c519654c83965baaa50fc5807d4b76fbf587a2972dca4c30cf4e54547f478'
            assert status['renewal']['scts'][1]['signed'] == 'Fri, 31 May 2019 17:06:35 GMT'

    # test if switching status off has effect
    def test_920_003(self):
        domain = self.test_domain
        domains = [ domain ]
        conf = HttpdConf()
        conf.add_admin( "admin@not-forbidden.org" )
        conf.add_md( domains )
        conf.add_line("MDCertificateStatus off")
        conf.add_vhost(domain)
        conf.install()
        assert TestEnv.apache_restart() == 0
        assert TestEnv.await_completion( [ domain ], restart=False )
        status = TestEnv.get_certificate_status( domain )
        assert not status

    # get the complete md-status JSON, check that it
    def test_920_004(self):
        domain = self.test_domain
        domains = [ domain ]
        conf = HttpdConf()
        conf.add_admin( "admin@not-forbidden.org" )
        conf.add_md( domains )
        conf.add_line("MDCertificateStatus off")
        conf.add_vhost(domain)
        conf.install()
        assert TestEnv.apache_restart() == 0
        assert TestEnv.await_completion( [ domain ] )
        status = TestEnv.get_md_status( "" )
        assert "version" in status
        assert "managed-domains" in status
        assert 1 == len(status["managed-domains"])

    # get the status of a domain on base server
    def test_920_010(self):
        domain = self.test_domain
        domains = [ domain ]
        conf = HttpdConf(std_vhosts=False, text="""
LogLevel md:trace2
LogLevel ssl:debug

MDBaseServer on
MDPortMap http:- https:%s

Listen %s
ServerAdmin admin@not-forbidden.org
ServerName %s
SSLEngine on
Protocols h2 http/1.1 acme-tls/1

<Location "/server-status">
    SetHandler server-status
</Location>
<Location "/md-status">
    SetHandler md-status
</Location>
            """ % (TestEnv.HTTPS_PORT, TestEnv.HTTPS_PORT, domain))
        conf.add_md( domains )
        conf.install()
        TestEnv.HTTPD_CHECK_URL = TestEnv.HTTPD_URL_SSL
        assert TestEnv.apache_restart() == 0
        assert TestEnv.await_completion( [ domain ], restart=False )
        status = TestEnv.get_md_status( "" )
        assert "version" in status
        assert "managed-domains" in status
        assert 1 == len(status["managed-domains"])
        # get the html page
        status = TestEnv.get_server_status()
        assert re.search(r'
Managed Certificates
', status, re.MULTILINE)
        # get the ascii summary
        status = TestEnv.get_server_status(query="?auto")
        m = re.search(r'Managed Certificates: total=(\d+), ok=(\d+) renew=(\d+) errored=(\d+) ready=(\d+)', status, re.MULTILINE)
        assert 1 == int(m.group(1))
        assert 0 == int(m.group(2))
        assert 1 == int(m.group(3))
        assert 0 == int(m.group(4))
        assert 1 == int(m.group(5))
<commit_msg>Adding test case for issue #161. Seems to work just fine.<commit_after># test mod_md status resources

import json
import os
import pytest
import re
import socket
import ssl
import sys
import time

from datetime import datetime
from TestEnv import TestEnv
from TestHttpdConf import HttpdConf
from TestCertUtil import CertUtil
from shutil import copyfile

def setup_module(module):
    print("setup_module    module:%s" % module.__name__)
    TestEnv.init()
    TestEnv.APACHE_CONF_SRC = "data/test_auto"
    TestEnv.check_acme()
    TestEnv.clear_store()
    HttpdConf().install();

def teardown_module(module):
    print("teardown_module module:%s" % module.__name__)
    assert TestEnv.apache_stop() == 0

class TestStatus:

    def setup_method(self, method):
        print("setup_method: %s" % method.__name__)
        TestEnv.clear_store()
        self.test_domain = TestEnv.get_method_domain(method)

    def teardown_method(self, method):
        print("teardown_method: %s" % method.__name__)

    # simple MD, drive it, check status before activation
    def test_920_001(self):
        domain = self.test_domain
        domains = [ domain ]
        conf = HttpdConf()
        conf.add_admin( "admin@not-forbidden.org" )
        conf.add_md( domains )
        conf.add_vhost(domain)
        conf.install()
        assert TestEnv.apache_restart() == 0
        assert TestEnv.await_completion( [ domain ], restart=False )
        # we started without a valid certificate, so we expect /.httpd/certificate-status
        # to not give information about one and - since we waited for the ACME signup
        # to complete - to give information in 'renewal' about the new cert.
        status = TestEnv.get_certificate_status( domain )
        assert not 'sha256-fingerprint' in status
        assert not 'valid' in status
        assert 'renewal' in status
        assert 'valid' in status['renewal']
        assert 'sha256-fingerprint' in status['renewal']
        # restart and activate
        # once activated, the staging must be gone and attributes exist for the active cert
        assert TestEnv.apache_restart() == 0
        status = TestEnv.get_certificate_status( domain )
        assert not 'renewal' in status
        assert 'sha256-fingerprint' in status
        assert 'valid' in status
        assert 'from' in status['valid']

    # simple MD, drive it, manipulate staged credentials and check status
    def test_920_002(self):
        domain = self.test_domain
        domains = [ domain ]
        conf = HttpdConf()
        conf.add_admin( "admin@not-forbidden.org" )
        conf.add_md( domains )
        conf.add_vhost(domain)
        conf.install()
        assert TestEnv.apache_restart() == 0
        assert TestEnv.await_completion( [ domain ], restart=False )
        # copy a real certificate from LE over to staging
        staged_cert = os.path.join(TestEnv.STORE_DIR, 'staging', domain, 'pubcert.pem')
        real_cert = os.path.join('data', 'test_920', '002.pubcert')
        assert copyfile(real_cert, staged_cert)
        status = TestEnv.get_certificate_status( domain )
        # status shows the copied cert's properties as staged
        assert 'renewal' in status
        assert 'Thu, 29 Aug 2019 16:06:35 GMT' == status['renewal']['valid']['until']
        assert 'Fri, 31 May 2019 16:06:35 GMT' == status['renewal']['valid']['from']
        assert '03039C464D454EDE79FCD2CAE859F668F269' ==  status['renewal']['serial']
        assert 'sha256-fingerprint' in status['renewal']
        if 0 == 1:
            assert len(status['renewal']['scts']) == 2
            assert status['renewal']['scts'][0]['logid'] == '747eda8331ad331091219cce254f4270c2bffd5e422008c6373579e6107bcc56'
            assert status['renewal']['scts'][0]['signed'] == 'Fri, 31 May 2019 17:06:35 GMT'
            assert status['renewal']['scts'][1]['logid'] == '293c519654c83965baaa50fc5807d4b76fbf587a2972dca4c30cf4e54547f478'
            assert status['renewal']['scts'][1]['signed'] == 'Fri, 31 May 2019 17:06:35 GMT'

    # test if switching status off has effect
    def test_920_003(self):
        domain = self.test_domain
        domains = [ domain ]
        conf = HttpdConf()
        conf.add_admin( "admin@not-forbidden.org" )
        conf.add_md( domains )
        conf.add_line("MDCertificateStatus off")
        conf.add_vhost(domain)
        conf.install()
        assert TestEnv.apache_restart() == 0
        assert TestEnv.await_completion( [ domain ], restart=False )
        status = TestEnv.get_certificate_status( domain )
        assert not status

    # get the complete md-status JSON, check that it
    def test_920_004(self):
        domain = self.test_domain
        domains = [ domain ]
        conf = HttpdConf()
        conf.add_admin( "admin@not-forbidden.org" )
        conf.add_md( domains )
        conf.add_line("MDCertificateStatus off")
        conf.add_vhost(domain)
        conf.install()
        assert TestEnv.apache_restart() == 0
        assert TestEnv.await_completion( [ domain ] )
        status = TestEnv.get_md_status( "" )
        assert "version" in status
        assert "managed-domains" in status
        assert 1 == len(status["managed-domains"])

    # get the status of a domain on base server
    def test_920_010(self):
        domain = self.test_domain
        domains = [ domain ]
        conf = HttpdConf(std_vhosts=False, text="""
LogLevel md:trace2
LogLevel ssl:debug

MDBaseServer on
MDPortMap http:- https:%s

Listen %s
ServerAdmin admin@not-forbidden.org
ServerName %s
SSLEngine on
Protocols h2 http/1.1 acme-tls/1

<Location "/server-status">
    SetHandler server-status
</Location>
<Location "/md-status">
    SetHandler md-status
</Location>
            """ % (TestEnv.HTTPS_PORT, TestEnv.HTTPS_PORT, domain))
        conf.add_md( domains )
        conf.install()
        TestEnv.HTTPD_CHECK_URL = TestEnv.HTTPD_URL_SSL
        assert TestEnv.apache_restart() == 0
        assert TestEnv.await_completion( [ domain ], restart=False )
        status = TestEnv.get_md_status( "" )
        assert "version" in status
        assert "managed-domains" in status
        assert 1 == len(status["managed-domains"])
        # get the html page
        status = TestEnv.get_server_status()
        assert re.search(r'
Managed Certificates
', status, re.MULTILINE)
        # get the ascii summary
        status = TestEnv.get_server_status(query="?auto")
        m = re.search(r'Managed Certificates: total=(\d+), ok=(\d+) renew=(\d+) errored=(\d+) ready=(\d+)', status, re.MULTILINE)
        assert 1 == int(m.group(1))
        assert 0 == int(m.group(2))
        assert 1 == int(m.group(3))
        assert 0 == int(m.group(4))
        assert 1 == int(m.group(5))

    def test_920_011(self):
        # MD with static cert files in base server, see issue #161
        domain = self.test_domain
        domains = [ domain, 'www.%s' % domain ]
        testpath = os.path.join(TestEnv.GEN_DIR, 'test_920_011')
        # cert that is only 10 more days valid
        TestEnv.create_self_signed_cert(domains, { "notBefore": -70, "notAfter": 20  },
            serial=920011, path=testpath)
        cert_file = os.path.join(testpath, 'pubcert.pem')
        pkey_file = os.path.join(testpath, 'privkey.pem')
        assert os.path.exists(cert_file)
        assert os.path.exists(pkey_file)
        conf = HttpdConf()
        conf = HttpdConf(std_vhosts=False, text= f"""
LogLevel md:trace2
LogLevel ssl:debug

MDPortMap http:- https:{TestEnv.HTTPS_PORT}

Listen {TestEnv.HTTPS_PORT}
ServerAdmin admin@not-forbidden.org
ServerName {domain}
SSLEngine on
Protocols h2 http/1.1 acme-tls/1

MDBaseServer on

<Location "/server-status">
    SetHandler server-status
</Location>
<Location "/md-status">
    SetHandler md-status
</Location>
            """)
        conf.start_md(domains)
        conf.add_line(f"MDCertificateFile {cert_file}")
        conf.add_line(f"MDCertificateKeyFile {pkey_file}")
        conf.end_md()
        conf.install()
        TestEnv.HTTPD_CHECK_URL = TestEnv.HTTPD_URL_SSL
        assert TestEnv.apache_restart() == 0
        status = TestEnv.get_md_status( domain )
        assert status
        assert not 'renewal' in status
        print(status)
        assert status['state'] == TestEnv.MD_S_COMPLETE
        assert status['renew-mode'] == 1 # manual
<|endoftext|>


<commit_before>#!/usr/bin/env python

from devilryclient.utils import findconffolder, get_config, get_metadata, save_metadata
from os.path import dirname, sep, join, exists
from os import getcwd

class DevilryClientAdd(object):
    """Mark a feedback as done and ready for upload"""

    def __init__(self):
        self.conf = get_config()
        self.root_dir = dirname(findconffolder()) + sep
        self.metadata = get_metadata()

    def set_and_save(self, path):
        self.metadata[dirname(path).replace(self.root_dir, '')]['done'] = True
        save_metadata(self.metadata)

    def check_context(self):

        split_path = getcwd().replace(self.root_dir, '').split(sep)

        # 7 is the depth for deliveries:
        # <root_dir>/subject/period/assignment/assignmentgroup/deadline/delivery/
        # There should be a 'feedback.rst' at this depth
        if len(split_path) < 6:
            print "This directory is not a delivery directory.", "'" + sep.join(split_path) + "'"
            return

        feedback = join(self.root_dir, sep.join(split_path[0:6]), 'feedback')
        print feedback
        if exists(feedback):
            self.set_and_save(feedback)
            print "Added"
        else:
            print "feedback not found in:", dirname(feedback)

    def run(self):
        self.check_context()

if __name__ == '__main__':
    DevilryClientAdd().run()
<commit_msg>devilryclient.plugins.add: Changed key-name from done to corrected<commit_after>#!/usr/bin/env python

from devilryclient.utils import findconffolder, get_config, get_metadata, save_metadata
from os.path import dirname, sep, join, exists
from os import getcwd

class DevilryClientAdd(object):
    """Mark a feedback as done and ready for upload"""

    def __init__(self):
        self.conf = get_config()
        self.root_dir = dirname(findconffolder()) + sep
        self.metadata = get_metadata()

    def set_and_save(self, path):
        self.metadata[dirname(path).replace(self.root_dir, '')]['corrected'] = True
        save_metadata(self.metadata)

    def check_context(self):

        split_path = getcwd().replace(self.root_dir, '').split(sep)

        # 7 is the depth for deliveries:
        # <root_dir>/subject/period/assignment/assignmentgroup/deadline/delivery/
        # There should be a 'feedback.rst' at this depth
        if len(split_path) < 6:
            print "This directory is not a delivery directory.", "'" + sep.join(split_path) + "'"
            return

        feedback = join(self.root_dir, sep.join(split_path[0:6]), 'feedback')
        print feedback
        if exists(feedback):
            self.set_and_save(feedback)
            print "Added"
        else:
            print "feedback not found in:", dirname(feedback)

    def run(self):
        self.check_context()

if __name__ == '__main__':
    DevilryClientAdd().run()
<|endoftext|>


<commit_before>import urllib.request
import os
import sys
import configparser
import psutil
import subprocess
from time import sleep
from datetime import datetime
from bs4 import BeautifulSoup

def current_time():
    return datetime.now().strftime('%H:%M:%S')

def write_to_map_list(table_content):
    try:
        new_file = ''
        with open('MapList.txt', 'r') as f:
            for line in f:
                for row in table_content:
                    map_name = row[0][:-5] + ' Map'
                    if line.startswith('mapList["' + map_name):
                        line = line[:-2] + '`n`nTempest on map:`n- ' + row[1] + '`n- ' + row[2] + '"\n'
                new_file += line

        try:
            approot = os.path.dirname(os.path.abspath(__file__))
        except NameError:
            approot = os.path.dirname(os.path.abspath(sys.argv[0]))

        with open(os.path.abspath(os.path.join(approot, '..', 'data', 'MapList.txt')), 'w') as f:
            f.write(new_file)

    except IOError:
        print('[%s] ERROR: Could not find file "MapList.txt"' % current_time())
        sys.exit(1)

def get_sourcepage(url):
    successful = False
    while not successful:
        try:
            page = urllib.request.urlopen(url).read()
            page = str(page, 'utf-8')
            soup = BeautifulSoup(''.join(page), 'html.parser')
            successful = True
        except urllib.error.URLError as e:
            print('[%s] %s' % e.reason)
            sleep(2)
    return soup

def find_table_content(soup):
    table = soup.find('tbody')
    row = table.find_all('tr')
    table_content = []
    for elemtents in row:
        table_content.append(elemtents.getText().strip().split('\n'))
    return table_content

def read_config():
    config = configparser.ConfigParser()
    config.read('config.ini')

    try:
        auto_hot_key = config['Path']['AutoHotKey']
        file_to_start = config['Path']['ItemInfo']
        reaload = int(config['RefreshInterval']['reaload'])
    except KeyError:
        print("You need to adjust the config.ini")
        sleep(5)
        sys.exit()

    return auto_hot_key, reaload, file_to_start

def end_autohotkey(prog_path):
    procname = prog_path.split('\\')[-1]

    for proc in psutil.process_iter():
        if proc.name() == procname:
            proc.kill()

if __name__ == '__main__':
    prog_path, reload_time, file_to_start = read_config()
    try:
        while True:
            end_autohotkey(prog_path)
            print('[%s] Grabbing new data...' % current_time())
            url = 'http://poetempest.com/'
            soup = get_sourcepage(url)
            table_content = find_table_content(soup)
            write_to_map_list(table_content)
            print('[%s] Done. Next try in %d min. Restarting "AutoHotKey"' % (current_time(), reload_time))
            subprocess.Popen([prog_path, file_to_start])
            sleep(reload_time * 60)
    except KeyboardInterrupt:
        end_autohotkey(prog_path)
        sys.exit(1)
<commit_msg>update for new webpage<commit_after>import urllib.request
import os
import sys
import configparser
import psutil
import subprocess
import winsound
from time import sleep
from datetime import datetime
from bs4 import BeautifulSoup

clear = lambda: os.system("cls")

def exit_with_input():
    input("Exit with any key")
    sys.exit()

def current_time():
    return datetime.now().strftime('%H:%M:%S')

def dummy_html():
    with open('quellcode.html', 'r') as f:
        return BeautifulSoup(''.join(f), 'html.parser')

def write_to_map_list(table_content):
    try:
        new_file = ''
        with open('MapList.txt', 'r') as f:
            for line in f:
                for row in table_content:
                    map_name = row[0][:-5] + ' Map'
                    if line.startswith('mapList["' + map_name):
                        line = line[:-2] + '`n`nTempest on map:`n- ' + row[1] + '`n- ' + row[2] + '"\n'
                new_file += line
    except IOError:
        print('[%s] ERROR: Could not find file "MapList.txt" in script folder.' % current_time())
        exit_with_input()

    try:
        approot = os.path.dirname(os.path.abspath(__file__))
    except NameError:
        approot = os.path.dirname(os.path.abspath(sys.argv[0]))

    try:
        with open(os.path.abspath(os.path.join(approot, '..', 'data', 'MapList.txt')), 'w') as f:
            f.write(new_file)
    except IOError:
        print('[%s] ERROR: Could not find file "MapList.txt" in data folder. Maybe you placed this script into the wrong location.' % current_time())
        exit_with_input()

def get_sourcepage(url):
    successful = False
    while not successful:
        try:
            page = urllib.request.urlopen(url).read()
            page = str(page, 'utf-8')
            soup = BeautifulSoup(''.join(page), 'html.parser')
            successful = True
        except urllib.error.URLError as e:
            print('[%s] %s' % (current_time(), e.reason))
            sleep(2)
    return soup

def play_sound():
    fname = 'beep.wav'
    winsound.PlaySound(fname, winsound.SND_FILENAME)

def show_tempest_info(great):
    print('')
    print('[%s] Good tempests:' % current_time())
    print('------------------------------------')
    for items in great:
        tlist = items.getText().strip().split('\n')
        tlist = [i.strip() for i in tlist if i.strip() and i.strip() not in ['thumb_up', 'thumb_down']]
        print('[%s] Map:          %s' % (current_time(), tlist[0]))
        print('[%s] Votes:        %s' % (current_time(), tlist[1]))
        print('[%s] Tempest:      %s' % (current_time(), tlist[2]))
        print('[%s] Description:  %s' % (current_time(), tlist[3]))
        print('------------------------------------')

def find_table_content(soup, play_mp3):
    table = soup.find('tbody')
    row = table.find_all('tr', {'class': ['', 'great', 'dangerous']})
    great = table.find_all('tr', {'class': 'great'})

    if great:
        show_tempest_info(great)
        if play_mp3.lower() == 'true':
            play_sound()

    table_content = []
    for elemtents in row:
        tlist = elemtents.getText().strip().split('\n')
        tlist = [i.strip() for i in tlist if i.strip() and i.strip() not in ['thumb_up', 'thumb_down']]

        if len(tlist) == 4:
            del tlist[1]
        table_content.append(tlist)

    return table_content

def read_config():
    config = configparser.ConfigParser()
    config.read('config.ini')

    try:
        auto_hot_key = config['Path']['AutoHotKey']
        file_to_start = config['Path']['ItemInfo']
        reaload = int(config['RefreshInterval']['reaload'])
        play_mp3 = config['PlaySounds']['play']
    except KeyError:
        config['Path'] = {'AutoHotKey': 'C:\\path\\to\\AutoHotKey.exe',
                          'ItemInfo': 'C:\\path\\to\\ItemInfo.ahk'}
        config['RefreshInterval'] = {'reaload': '5'}
        config['PlaySounds'] = {'play': 'True'}
        with open('config.ini', 'w') as configfile:
            config.write(configfile)
        print("Wrote new config.ini.\nYou need to adjust this config file first.")
        exit_with_input()

    return auto_hot_key, reaload, file_to_start, play_mp3

def end_autohotkey(prog_path):
    procname = prog_path.split('\\')[-1]

    for proc in psutil.process_iter():
        if proc.name() == procname:
            proc.kill()

if __name__ == '__main__':
    # TODO: READ LOGFILE FROM POE
    prog_path, reload_time, file_to_start, play_mp3 = read_config()
    try:
        while True:
            clear()
            end_autohotkey(prog_path)
            print('[%s] Grabbing new data...' % current_time())
            url = 'http://poetempest.com/'
            soup = get_sourcepage(url)
            #soup = dummy_html()
            table_content = find_table_content(soup, play_mp3)
            write_to_map_list(table_content)
            print('[%s] Done. Next try in %d min. Restarting "AutoHotKey"' % (current_time(), reload_time))
            subprocess.Popen([prog_path, file_to_start])
            sleep(reload_time * 60)
    except KeyboardInterrupt:
        end_autohotkey(prog_path)
<|endoftext|>


<commit_before>#!/usr/bin/env python

###
# Copyright (c) 2002, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#   * Redistributions of source code must retain the above copyright notice,
#     this list of conditions, and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions, and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#   * Neither the name of the author of this software nor the name of
#     contributors to this software may be used to endorse or promote products
#     derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###

from test import *

import re

import utils

class FunCommandsTest(PluginTestCase):
    plugins = ('FunCommands',)
    def testNoErrors(self):
        self.assertNotError('netstats')
        self.assertNotError('cpustats')
        self.assertNotError('uptime')
        self.assertNotError('leet foobar')
        self.assertNotError('lithp meghan sweeney')
        self.assertNotError('objects')
        self.assertNotError('levenshtein Python Perl')
        self.assertNotError('soundex jemfinch')

    def testBinary(self):
        self.assertResponse('binary A', '01000001')

    def testRot13(self):
        for s in nicks[:10]: # 10 is probably enough.
            self.assertResponse('rot13 [rot13 %s]' % s, s)

    def testCalc(self):
        self.assertResponse('calc 5*0.06', str(5*0.06))

    def testChr(self):
        for i in range(256):
            c = chr(i)
            regexp = r'%s|%s' % (re.escape(c), re.escape(repr(c)))
            self.assertRegexp('chr %s' % i, regexp)

    def testHexlifyUnhexlify(self):
        for s in nicks[:10]: # 10, again, is probably enough.
            self.assertResponse('unhexlify [hexlify %s]' % s, s)

    def testXor(self):
        for s0, s1, s2, s3, s4, s5, s6, s7, s8, s9 in group(nicks, 10):
            data = '%s%s%s%s%s%s%s%s%s' % (s0, s1, s2, s3, s4, s5, s6, s7, s8)
            self.assertResponse('xor %s [xor %s %s]' % (s9, s9, data), data)

    def testUrlquoteUrlunquote(self):
        self.assertResponse('urlunquote [urlquote ~jfincher]', '~jfincher')

    def testPydoc(self):
        self.assertNotError('pydoc str')
        self.assertError('pydoc foobar')
        self.assertError('pydoc assert')

    def testOrd(self):
        for c in map(chr, range(256)):
            i = ord(c)
            self.assertResponse('ord %s' % utils.dqrepr(c), str(i))

    def testTell(self):
        m = self.getMsg('tell foo [rot13 foobar]')
        self.failUnless(m.args[0] == 'foo')
        self.failUnless('sbbone' in m.args[1])
        m = self.getMsg('tell #foo [rot13 foobar]')
        self.failUnless(m.args[0] == '#foo')
        self.failUnless('sbbone' in m.args[1])

    def testZen(self):
        self.assertNotError('zen')

    def testDns(self):
        self.assertNotError('dns slashdot.org')

    def testWhois(self):
        self.assertNotError('whois ohio-state.edu')
        self.assertError('whois slashdot.org')

# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:
<commit_msg>Added tests for the longest, stupidest bug in existence.<commit_after>#!/usr/bin/env python

###
# Copyright (c) 2002, Jeremiah Fincher
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#   * Redistributions of source code must retain the above copyright notice,
#     this list of conditions, and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions, and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#   * Neither the name of the author of this software nor the name of
#     contributors to this software may be used to endorse or promote products
#     derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###

from test import *

import re

import utils

class FunCommandsTest(PluginTestCase):
    plugins = ('FunCommands',)
    def testNoErrors(self):
        self.assertNotError('netstats')
        self.assertNotError('cpustats')
        self.assertNotError('uptime')
        self.assertNotError('leet foobar')
        self.assertNotError('lithp meghan sweeney')
        self.assertNotError('objects')
        self.assertNotError('levenshtein Python Perl')
        self.assertNotError('soundex jemfinch')

    def testBinary(self):
        self.assertResponse('binary A', '01000001')

    def testRot13(self):
        for s in nicks[:10]: # 10 is probably enough.
            self.assertResponse('rot13 [rot13 %s]' % s, s)

    def testCalc(self):
        self.assertResponse('calc 5*0.06', str(5*0.06))
        self.assertResponse('calc 2.0-7.0', str(2-7))

    def testChr(self):
        for i in range(256):
            c = chr(i)
            regexp = r'%s|%s' % (re.escape(c), re.escape(repr(c)))
            self.assertRegexp('chr %s' % i, regexp)

    def testHexlifyUnhexlify(self):
        for s in nicks[:10]: # 10, again, is probably enough.
            self.assertResponse('unhexlify [hexlify %s]' % s, s)

    def testXor(self):
        for s0, s1, s2, s3, s4, s5, s6, s7, s8, s9 in group(nicks, 10):
            data = '%s%s%s%s%s%s%s%s%s' % (s0, s1, s2, s3, s4, s5, s6, s7, s8)
            self.assertResponse('xor %s [xor %s %s]' % (s9, s9, data), data)

    def testUrlquoteUrlunquote(self):
        self.assertResponse('urlunquote [urlquote ~jfincher]', '~jfincher')

    def testPydoc(self):
        self.assertNotError('pydoc str')
        self.assertError('pydoc foobar')
        self.assertError('pydoc assert')

    def testOrd(self):
        for c in map(chr, range(256)):
            i = ord(c)
            self.assertResponse('ord %s' % utils.dqrepr(c), str(i))

    def testTell(self):
        m = self.getMsg('tell foo [rot13 foobar]')
        self.failUnless(m.args[0] == 'foo')
        self.failUnless('sbbone' in m.args[1])
        m = self.getMsg('tell #foo [rot13 foobar]')
        self.failUnless(m.args[0] == '#foo')
        self.failUnless('sbbone' in m.args[1])

    def testZen(self):
        self.assertNotError('zen')

    def testDns(self):
        self.assertNotError('dns slashdot.org')

    def testWhois(self):
        self.assertNotError('whois ohio-state.edu')
        self.assertError('whois slashdot.org')

    def testRpn(self):
        self.assertResponse('rpn 5 2 +', '7')
        self.assertResponse('rpn 1 2 3 +', 'Stack: [1, 5]')
        self.assertResponse('rpn 1 dup', 'Stack: [1, 1]')
        self.assertResponse('rpn 2 3 4 + -', str(2-7))

# vim:set shiftwidth=4 tabstop=8 expandtab textwidth=78:
<|endoftext|>


<commit_before>"""
Extended SymPy hierarchy.
"""

import sympy
from sympy import Expr, Float
from sympy.core.basic import _aresame
from sympy.functions.elementary.trigonometric import TrigonometricFunction

__all__ = ['FrozenExpr', 'Eq', 'Mul', 'Add', 'FunctionFromPointer', 'ListInitializer',
           'taylor_sin', 'taylor_cos', 'bhaskara_sin', 'bhaskara_cos']

class FrozenExpr(Expr):

    """
    Use :class:`FrozenExpr` in place of :class:`sympy.Expr` to make sure than
    an expression is no longer transformable; that is, standard manipulations
    such as xreplace, collect, expand, ... have no effect, thus building a
    new expression identical to self.

    :Notes:

    At the moment, only xreplace is overridded (to prevent unpicking factorizations)
    """

    def xreplace(self, rule):
        if self in rule:
            return rule[self]
        elif rule:
            args = []
            for a in self.args:
                try:
                    args.append(a.xreplace(rule))
                except AttributeError:
                    args.append(a)
            args = tuple(args)
            if not _aresame(args, self.args):
                return self.func(*args, evaluate=False)
        return self

class Eq(sympy.Eq, FrozenExpr):

    """
    A customized version of :class:`sympy.Eq` which suppresses
    evaluation.
    """

    def __new__(cls, *args, **kwargs):
        kwargs['evaluate'] = False
        obj = sympy.Eq.__new__(cls, *args, **kwargs)
        return obj

class Mul(sympy.Mul, FrozenExpr):
    pass

class Add(sympy.Add, FrozenExpr):
    pass

class FunctionFromPointer(sympy.Symbol):

    """
    Symbolic representation of the C notation ``pointer->function(params)``.
    """

    def __new__(cls, function, pointer, params=None):
        obj = sympy.Symbol.__new__(cls, '%s->%s' % (pointer, function))
        obj.function = function
        obj.pointer = pointer
        obj.params = params or ()
        return obj

    def __str__(self):
        return "%s->%s(%s)" % (self.pointer, self.function,
                               ', '.join([str(i) for i in self.params]))

    __repr__ = __str__

class ListInitializer(sympy.Symbol):

    """
    Symbolic representation of the C++ list initializer notation ``{a, b, ...}``.
    """

    def __new__(cls, params):
        obj = sympy.Symbol.__new__(cls, ','.join('%s' % i for i in params))
        obj.params = params or ()
        return obj

    def __str__(self):
        return "{%s}" % ", ".join(str(i) for i in self.params)

    __repr__ = __str__

class taylor_sin(TrigonometricFunction):

    """
    Approximation of the sine function using a Taylor polynomial.
    """

    @classmethod
    def eval(cls, arg):
        return eval_taylor_sin(arg)

class taylor_cos(TrigonometricFunction):

    """
    Approximation of the cosine function using a Taylor polynomial.
    """

    @classmethod
    def eval(cls, arg):
        return 1.0 if arg == 0.0 else eval_taylor_cos(arg)

class bhaskara_sin(TrigonometricFunction):

    """
    Approximation of the sine function using a Bhaskara polynomial.
    """

    @classmethod
    def eval(cls, arg):
        return eval_bhaskara_sin(arg)

class bhaskara_cos(TrigonometricFunction):

    """
    Approximation of the cosine function using a Bhaskara polynomial.
    """

    @classmethod
    def eval(cls, arg):
        return 1.0 if arg == 0.0 else eval_bhaskara_sin(arg + 1.5708)

# Utils

def eval_bhaskara_sin(expr):
    return 16.0*expr*(3.1416-abs(expr))/(49.3483-4.0*abs(expr)*(3.1416-abs(expr)))

def eval_taylor_sin(expr):
    v = expr + Mul(-1/6.0,
                   Mul(expr, expr, expr, evaluate=False),
                   1.0 + Mul(Mul(expr, expr, evaluate=False), -0.05, evaluate=False),
                   evaluate=False)
    try:
        Float(expr)
        return v.doit()
    except (TypeError, ValueError):
        return v

def eval_taylor_cos(expr):
    v = 1.0 + Mul(-0.5,
                  Mul(expr, expr, evaluate=False),
                  1.0 + Mul(expr, expr, -1/12.0, evaluate=False),
                  evaluate=False)
    try:
        Float(expr)
        return v.doit()
    except (TypeError, ValueError):
        return v
<commit_msg>dse/yask: Fix FunctionFromPointer hitting SymPy cache<commit_after>"""
Extended SymPy hierarchy.
"""

import sympy
from sympy import Expr, Float
from sympy.core.basic import _aresame
from sympy.functions.elementary.trigonometric import TrigonometricFunction

from devito.tools import as_tuple

__all__ = ['FrozenExpr', 'Eq', 'Mul', 'Add', 'FunctionFromPointer', 'ListInitializer',
           'taylor_sin', 'taylor_cos', 'bhaskara_sin', 'bhaskara_cos']

class FrozenExpr(Expr):

    """
    Use :class:`FrozenExpr` in place of :class:`sympy.Expr` to make sure than
    an expression is no longer transformable; that is, standard manipulations
    such as xreplace, collect, expand, ... have no effect, thus building a
    new expression identical to self.

    :Notes:

    At the moment, only xreplace is overridded (to prevent unpicking factorizations)
    """

    def xreplace(self, rule):
        if self in rule:
            return rule[self]
        elif rule:
            args = []
            for a in self.args:
                try:
                    args.append(a.xreplace(rule))
                except AttributeError:
                    args.append(a)
            args = tuple(args)
            if not _aresame(args, self.args):
                return self.func(*args, evaluate=False)
        return self

class Eq(sympy.Eq, FrozenExpr):

    """
    A customized version of :class:`sympy.Eq` which suppresses
    evaluation.
    """

    def __new__(cls, *args, **kwargs):
        kwargs['evaluate'] = False
        obj = sympy.Eq.__new__(cls, *args, **kwargs)
        return obj

class Mul(sympy.Mul, FrozenExpr):
    pass

class Add(sympy.Add, FrozenExpr):
    pass

class FunctionFromPointer(sympy.Symbol):

    """
    Symbolic representation of the C notation ``pointer->function(params)``.
    """

    def __new__(cls, function, pointer, params=None):
        flatten_name = '%s->%s(%s)' % (pointer, function,
                                       ", ".join(str(i) for i in as_tuple(params)))
        obj = sympy.Symbol.__new__(cls, flatten_name)
        obj.function = function
        obj.pointer = pointer
        obj.params = as_tuple(params)
        return obj

class ListInitializer(sympy.Symbol):

    """
    Symbolic representation of the C++ list initializer notation ``{a, b, ...}``.
    """

    def __new__(cls, params):
        obj = sympy.Symbol.__new__(cls, ','.join('%s' % i for i in params))
        obj.params = params or ()
        return obj

    def __str__(self):
        return "{%s}" % ", ".join(str(i) for i in self.params)

    __repr__ = __str__

class taylor_sin(TrigonometricFunction):

    """
    Approximation of the sine function using a Taylor polynomial.
    """

    @classmethod
    def eval(cls, arg):
        return eval_taylor_sin(arg)

class taylor_cos(TrigonometricFunction):

    """
    Approximation of the cosine function using a Taylor polynomial.
    """

    @classmethod
    def eval(cls, arg):
        return 1.0 if arg == 0.0 else eval_taylor_cos(arg)

class bhaskara_sin(TrigonometricFunction):

    """
    Approximation of the sine function using a Bhaskara polynomial.
    """

    @classmethod
    def eval(cls, arg):
        return eval_bhaskara_sin(arg)

class bhaskara_cos(TrigonometricFunction):

    """
    Approximation of the cosine function using a Bhaskara polynomial.
    """

    @classmethod
    def eval(cls, arg):
        return 1.0 if arg == 0.0 else eval_bhaskara_sin(arg + 1.5708)

# Utils

def eval_bhaskara_sin(expr):
    return 16.0*expr*(3.1416-abs(expr))/(49.3483-4.0*abs(expr)*(3.1416-abs(expr)))

def eval_taylor_sin(expr):
    v = expr + Mul(-1/6.0,
                   Mul(expr, expr, expr, evaluate=False),
                   1.0 + Mul(Mul(expr, expr, evaluate=False), -0.05, evaluate=False),
                   evaluate=False)
    try:
        Float(expr)
        return v.doit()
    except (TypeError, ValueError):
        return v

def eval_taylor_cos(expr):
    v = 1.0 + Mul(-0.5,
                  Mul(expr, expr, evaluate=False),
                  1.0 + Mul(expr, expr, -1/12.0, evaluate=False),
                  evaluate=False)
    try:
        Float(expr)
        return v.doit()
    except (TypeError, ValueError):
        return v
<|endoftext|>


<commit_before>import logging; log = logging.getLogger('qmsk.e2.presets')
import yaml

class E2Presets:
    @classmethod
    def load_yaml (cls, file):
        data = yaml.safe_load(file)

        log.debug("%s: %s")

        return cls(**data)

    def __init__ (self, presets):
        self.presets = presets

<commit_msg>qmsk.e2.presets: Preset with E2Presets iter/getitem<commit_after>import logging; log = logging.getLogger('qmsk.e2.presets')
import yaml

class Preset:
    def __init__ (self, preset, *, title):
        self.preset = preset
        self.title = title

class E2Presets:
    @classmethod
    def load_yaml (cls, file):
        data = yaml.safe_load(file)

        log.debug("%s: %s")

        return cls(**data)

    def __init__ (self, presets):
        self.presets = {id: Preset(id, **values) for id, values in presets.items()}

    def __iter__ (self):
        for preset in self.presets.values():
            yield preset

    def __getitem__ (self, key):
        return self.presets[key]
<|endoftext|>


<commit_before>#
# Copyright (c) 2004-2005 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

from conary.deps import deps
import os

def x86flags(archTag, baseArch, extraFlags, ofInterest):
    try:
        lines = open("/proc/cpuinfo").read().split("\n")
    except IOError:
        lines=[]

    rc = [ (x, deps.FLAG_SENSE_PREFERRED) for x in extraFlags ]

    for line in lines:
	if not line.startswith("flags"): continue
	fields = line.split()
	if fields[0] != "flags": continue

	for flag in fields[2:]:
	    if ofInterest.has_key(flag):
                rc.append((flag, deps.FLAG_SENSE_PREFERRED))

	return deps.Dependency(archTag, rc)

    return deps.Dependency(archTag)

def flags_ix86(baseArch):
    baseFlagMap = [ 'i686', 'i586', 'i486' ]
    i = baseFlagMap.index(baseArch)

    ofInterest = {}.fromkeys([ '3dnow', '3dnowext', 'mmx', 'mmxext', 'sse',
                               'sse2', 'sse3', 'cmov', 'nx'])
    return [ [ x86flags('x86', baseArch, baseFlagMap[i:], ofInterest) ] ]

def flags_i686():
    return flags_ix86(baseArch = 'i686')

def flags_i586():
    return flags_ix86(baseArch = 'i586')

def flags_mips64():
    return [[ deps.Dependency('mipseb', [ ('mips64', deps.FLAG_SENSE_REQUIRED) ]) ]]

def flags_x86_64():
    baseFlagMap = [ ]
    ofInterest = {}.fromkeys([ '3dnow', '3dnowext', 'nx', 'sse3' ])

    x86_64 = x86flags('x86_64', baseArch, baseFlagMap, ofInterest)
    multiarch = flags_i686()
    multiarch[0].append(x86_64)
    return multiarch

def current():
    return currentArch

def any():
    return "any"

def canInstall(other):
    return other == "any" or other == currentArch

def initializeArch():
    global currentArch
    localNamespace = globals()
    if localNamespace.has_key("flags_" + baseArch):
        currentArch = localNamespace["flags_" + baseArch]()

    del localNamespace

baseArch = os.uname()[4]
currentArch = [[ deps.Dependency(baseArch) ]]
initializeArch()

class FlavorPreferences:
    @staticmethod
    def _getCurrentArchIS(arch):
        # Returns just the name of the current arch
        return ' '.join(sorted(dep.name for dep in arch[0]))

    # The flavor preferences table is keyed on the current arch
    flavorPreferences = {
        'x86'           : ['is: x86'],
        'x86_64'        : ['is: x86 x86_64', 'is: x86_64', 'is: x86', ],
        'x86 x86_64'    : ['is: x86 x86_64', 'is: x86_64', 'is: x86', ],
    }

    @staticmethod
    def getStringFlavorPreferences(arch):
        key = FlavorPreferences._getCurrentArchIS(arch)
        return FlavorPreferences.flavorPreferences.get(key, [])

    @staticmethod
    def getFlavorPreferences(arch):
        return [ deps.parseFlavor(x)
            for x in FlavorPreferences.getStringFlavorPreferences(arch)
        ]

def getFlavorPreferences(arch = currentArch):
    return FlavorPreferences.getFlavorPreferences(arch)
<commit_msg>Added more flavor preferences<commit_after>#
# Copyright (c) 2004-2005 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#

from conary.deps import deps
import os

def x86flags(archTag, baseArch, extraFlags, ofInterest):
    try:
        lines = open("/proc/cpuinfo").read().split("\n")
    except IOError:
        lines=[]

    rc = [ (x, deps.FLAG_SENSE_PREFERRED) for x in extraFlags ]

    for line in lines:
	if not line.startswith("flags"): continue
	fields = line.split()
	if fields[0] != "flags": continue

	for flag in fields[2:]:
	    if ofInterest.has_key(flag):
                rc.append((flag, deps.FLAG_SENSE_PREFERRED))

	return deps.Dependency(archTag, rc)

    return deps.Dependency(archTag)

def flags_ix86(baseArch):
    baseFlagMap = [ 'i686', 'i586', 'i486' ]
    i = baseFlagMap.index(baseArch)

    ofInterest = {}.fromkeys([ '3dnow', '3dnowext', 'mmx', 'mmxext', 'sse',
                               'sse2', 'sse3', 'cmov', 'nx'])
    return [ [ x86flags('x86', baseArch, baseFlagMap[i:], ofInterest) ] ]

def flags_i686():
    return flags_ix86(baseArch = 'i686')

def flags_i586():
    return flags_ix86(baseArch = 'i586')

def flags_mips64():
    return [[ deps.Dependency('mipseb', [ ('mips64', deps.FLAG_SENSE_REQUIRED) ]) ]]

def flags_x86_64():
    baseFlagMap = [ ]
    ofInterest = {}.fromkeys([ '3dnow', '3dnowext', 'nx', 'sse3' ])

    x86_64 = x86flags('x86_64', baseArch, baseFlagMap, ofInterest)
    multiarch = flags_i686()
    multiarch[0].append(x86_64)
    return multiarch

def current():
    return currentArch

def any():
    return "any"

def canInstall(other):
    return other == "any" or other == currentArch

def initializeArch():
    global currentArch
    localNamespace = globals()
    if localNamespace.has_key("flags_" + baseArch):
        currentArch = localNamespace["flags_" + baseArch]()

    del localNamespace

baseArch = os.uname()[4]
currentArch = [[ deps.Dependency(baseArch) ]]
initializeArch()

class FlavorPreferences:
    @staticmethod
    def _getCurrentArchIS(arch):
        # Returns just the name of the current arch
        return ' '.join(sorted(dep.name for dep in arch[0]))

    # The flavor preferences table is keyed on the current arch
    flavorPreferences = {
        'alpha'         : ['is: alpha'],
        'ppc'           : ['is: ppc'],
        'ppc64'         : ['is: ppc ppc64', 'is: ppc64', 'is: ppc'],
        'ppc ppc64'     : ['is: ppc ppc64', 'is: ppc64', 'is: ppc'],
        's390'          : ['is: s390'],
        'sparc'         : ['is: sparc'],
        'sparc64'       : ['is: sparc sparc64', 'is: sparc64', 'is: sparc'],
        'sparc sparc64' : ['is: sparc sparc64', 'is: sparc64', 'is: sparc'],
        'x86'           : ['is: x86'],
        'x86_64'        : ['is: x86 x86_64', 'is: x86_64', 'is: x86', ],
        'x86 x86_64'    : ['is: x86 x86_64', 'is: x86_64', 'is: x86', ],
    }

    @staticmethod
    def getStringFlavorPreferences(arch):
        key = FlavorPreferences._getCurrentArchIS(arch)
        return FlavorPreferences.flavorPreferences.get(key, [])

    @staticmethod
    def getFlavorPreferences(arch):
        return [ deps.parseFlavor(x)
            for x in FlavorPreferences.getStringFlavorPreferences(arch)
        ]

def getFlavorPreferences(arch = currentArch):
    return FlavorPreferences.getFlavorPreferences(arch)
<|endoftext|>


<commit_before>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 02-08-2017 16:40
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This module contains models for dealing with PoGo stuff
"""

import os
import abc
import threading
# Application imports
import config_manager as main_app_config_manager
from parallel.models import ParallelRunner, CommandLineRunnerFactory
from parallel.exceptions import CommandLineRunnerException
from . import config_manager as module_config_manager
from .exceptions import PogoRunnerException

class PogoRunResult:
    # TODO - Needs to be extended for abstracting from results files from '-mm' parameter use
    def __init__(self,
                 ncbi_taxonomy_id=None,
                 pogo_source_file_path=None,
                 protein_sequence_file_path=None,
                 gtf_file_path=None):
        """
        Just the constructor, I had this implemented as syntactic sugar, but I was wrong
        :param ncbi_taxonomy_id: ncbi taxonomy id for this PoGo run results
        :param pogo_source_file_path: path of the source file used to run PoGo
        :param protein_sequence_file_path: FASTA file
        :param gtf_file_path: GTF file
        """
        # Logging
        self.__logger = main_app_config_manager.get_app_config_manager().get_logger_for(
            "{}.{}".format(__name__, type(self).__name__))
        # Map<pogo_result_file_extension, pogo_result_file_path>
        self.__pogo_result_file_paths = {}
        self.__pogo_source_file_path = None
        # I knew this was going to give me trouble
        self.set_ncbi_taxonomy_id(ncbi_taxonomy_id)
        self.set_protein_sequence_file_path(protein_sequence_file_path)
        self.set_gtf_file_path(gtf_file_path)
        self.set_pogo_source_file_path(pogo_source_file_path)

    def __generate_pogo_result_file_paths(self, pogo_source_file_path):
        self.__pogo_result_file_paths = {}
        pogo_results_folder_path = os.path.dirname(pogo_source_file_path)
        pogo_source_file_name = os.path.basename(pogo_source_file_path)
        for root, dirs, files in os.walk(pogo_results_folder_path):
            for pogo_result_file_path in files:
                pogo_result_file_name = os.path.basename(pogo_result_file_path)
                if pogo_result_file_name.startswith(pogo_source_file_name) and (
                            pogo_result_file_name != pogo_source_file_name):
                    # It is file generated by PoGo
                    pogo_result_file_extension = pogo_result_file_name.split(pogo_source_file_name)[1]
                    # self.__pogo_result_file_paths[pogo_result_file_extension] = pogo_result_file_path
                    # I need the full path for the pogo result file path
                    self.__pogo_result_file_paths[pogo_result_file_extension] = os.path.join(pogo_results_folder_path,
                                                                                pogo_result_file_path)
                    self._get_logger().info(
                        "PoGo run result '{}' ---> '{}', at full path '{}'".format(pogo_result_file_extension,
                                                                                pogo_result_file_path,
                                                                                self.__pogo_result_file_paths[
                                                                                pogo_result_file_extension]))

    def __get_pogo_result_file_path(self, file_extension):
        if file_extension in self.__pogo_result_file_paths:
            return self.__pogo_result_file_paths[file_extension]
        return None

    def _get_logger(self):
        return self.__logger

    def set_ncbi_taxonomy_id(self, taxonomy_id):
        self.__ncbi_taxonomy_id = taxonomy_id

    def set_pogo_source_file_path(self, pogo_source_file_path):
        if not self.__pogo_source_file_path or (self.__pogo_source_file_path != pogo_source_file_path):
            self.__pogo_source_file_path = pogo_source_file_path
            # Regenerate pogo result file paths
            self.__generate_pogo_result_file_paths(pogo_source_file_path)

    def set_protein_sequence_file_path(self, protein_sequence_file_path):
        self.__protein_sequence_file_path = protein_sequence_file_path

    def set_gtf_file_path(self, gtf_file_path):
        self.__gtf_file_path = gtf_file_path

    def get_ncbi_taxonomy_id(self):
        return self.__ncbi_taxonomy_id

    def get_pogo_result_main_bed_file_path(self):
        return self.__get_pogo_result_file_path(
            module_config_manager.get_configuration_service().get_pogo_result_file_extension_for_main_bed_file())

    def get_pogo_result_main_ptm_bed_file_path(self):
        return self.__get_pogo_result_file_path(
            module_config_manager.get_configuration_service().get_pogo_result_file_extension_for_main_ptm_bed_file()
        )

# PoGo Runners
class PogoRunnerFactory:
    @staticmethod
    def get_pogo_runner(ncbi_taxonomy_id, pogo_input_file, protein_sequence_file_path, gtf_file_path):
        # TODO - Needs to be extended for abstracting from results files from '-mm' parameter use
        # TODO - In the future, more PoGo runners will be implemented
        return PogoRunnerLocalThread(ncbi_taxonomy_id, pogo_input_file, protein_sequence_file_path, gtf_file_path)

class PogoRunner(ParallelRunner):
    # TODO - Needs to be extended for abstracting from results files from '-mm' parameter use
    def __init__(self,
                 ncbi_taxonomy_id=None,
                 pogo_input_file=None,
                 protein_sequence_file_path=None,
                 gtf_file_path=None,
                 mm_gap=None):
        super().__init__()
        self.ncbi_taxonomy_id = ncbi_taxonomy_id
        self.pogo_input_file = pogo_input_file
        self.protein_sequence_file_path = protein_sequence_file_path
        self.gtf_file_path = gtf_file_path
        self.mm_gap = mm_gap
        self._stdout = b' '
        self._stderr = b' '
        self.__success = True
        self.__pogo_run_result = None

    def _set_failed_to_run_pogo(self):
        self.__success = False

    def _set_success_to_run_pogo(self):
        self.__success = self.__success and True

    def _validate_environment_for_running_pogo(self):
        validation_ok = True
        if not self.ncbi_taxonomy_id:
            self._logger.warning("NO NCBI TAXONOMY ID has been specified for running PoGo with input file '{}'"
                                 .format(self.pogo_input_file))
        if not os.path.isfile(self.pogo_input_file):
            self._logger.error("PoGo input file '{}' IS NOT VALID".format(self.pogo_input_file))
            validation_ok = False
        if not os.path.isfile(self.protein_sequence_file_path):
            self._logger.error("Protein sequence file '{}' IS NOT VALID".format(self.protein_sequence_file_path))
            validation_ok = False
        if not os.path.isfile(self.gtf_file_path):
            self._logger.error("GTF file '{}' IS NOT VALID".format(self.gtf_file_path))
            validation_ok = False
        if not os.path.isfile(module_config_manager.get_configuration_service().get_pogo_binary_file_path()):
            self._logger.error("PoGo binary file '{}' IS NOT VALID"
                               .format(module_config_manager.get_configuration_service().get_pogo_binary_file_path()))
            validation_ok = False
        return validation_ok

    def get_stdout(self):
        # Never give it back until the runner is done with whatever it is doing
        if not self._done:
            raise PogoRunnerException("PoGo Runner is NOT DONE doing its job, thus 'stdout' is NOT AVAILABLE")
        return self._stdout

    def get_stderr(self):
        # Never give it back until the runner is done with whatever it is doing
        if not self._done:
            raise PogoRunnerException("PoGo Runner is NOT DONE doing its job, thus 'stderr' is NOT AVAILABLE")
        return self._stderr

    def get_pogo_run_command(self):
        pogo_parameter_species = ''
        if self.ncbi_taxonomy_id:
            pogo_parameter_species = " -species {} ".format(self.ncbi_taxonomy_id)
        pogo_command = "time {}{} -fasta {} -gtf {} -in {}" \
            .format(module_config_manager.get_configuration_service().get_pogo_binary_file_path(),
                    pogo_parameter_species,
                    self.protein_sequence_file_path,
                    self.gtf_file_path,
                    self.pogo_input_file)
        self._logger.debug("Command for running PoGo - '{}'".format(pogo_command))
        return pogo_command

    @abc.abstractmethod
    def _get_command_line_runner(self):
        """
        Although a factory is used to get a command line runner, its initialization differs depending on the final
        PogoRunner, i.e. we delegate to subclasses the provisioning of an initialized CommandLineRunner, how we use
        it, is the same for all subclasses.
        It will prepare the command line runner completely, including the command it has to run.
        :return: a CommandLineRunner for running PoGo
        """
        ...

    def _run(self):
        if not self._validate_environment_for_running_pogo():
            self._logger.error("Can't run PoGo for file '{}'".format(self.pogo_input_file))
            self._set_failed_to_run_pogo()
        else:
            command_line_runner = self._get_command_line_runner()
            try:
                command_line_runner.start()
                self._logger.debug("PoGo --- STARTED --- command '{}'".format(command_line_runner.command))
                command_line_runner.wait()
            except CommandLineRunnerException as e:
                self._logger.error("ERROR RUNNING PoGo - {}, STDOUT ---> '{}', STDERR ---> '{}'"
                                   .format(e.value,
                                           command_line_runner.get_stdout().decode('utf8'),
                                           command_line_runner.get_stderr().decode('utf8')))
                self._set_failed_to_run_pogo()
            finally:
                self._stdout = command_line_runner.get_stdout()
                self._stderr = command_line_runner.get_stderr()
            if not command_line_runner.command_success:
                self._set_failed_to_run_pogo()
                self._logger.error("PoGo FAILED on input file '{}', command '{}'"
                                   .format(self.pogo_input_file, command_line_runner.command))
            else:
                # Successful run of PoGo
                self._set_success_to_run_pogo()
                self._logger.info("PoGo SUCCESS on input file '{}', command '{}'"
                                  .format(self.pogo_input_file, command_line_runner.command))

    def get_pogo_run_result(self):
        if not self.is_done():
            raise PogoRunnerException("PoGo runner IS NOT FINISHED YET, thus, results can't be retrieved")
        if not self.__pogo_run_result:
            self.__pogo_run_result = PogoRunResult(self.ncbi_taxonomy_id,
                                                   self.pogo_input_file,
                                                   self.protein_sequence_file_path,
                                                   self.gtf_file_path)
        return self.__pogo_run_result

    def is_success(self):
        return self.__success

class PogoRunnerLocalThread(PogoRunner):
    def __init__(self,
                 ncbi_taxonomy_id=None,
                 pogo_input_file=None,
                 protein_sequence_file_path=None,
                 gtf_file_path=None):
        super().__init__(ncbi_taxonomy_id, pogo_input_file, protein_sequence_file_path, gtf_file_path)
        self._logger = main_app_config_manager \
            .get_app_config_manager() \
            .get_logger_for("{}.{}-{}".format(__name__, type(self).__name__, threading.current_thread().getName()))

    def _get_command_line_runner(self):
        command_line_runner = CommandLineRunnerFactory.get_multithread_command_line_runner()
        command_line_runner.command = self.get_pogo_run_command()
        command_line_runner.timeout = module_config_manager.get_configuration_service().get_pogo_run_timeout()
        return command_line_runner

class PogoRunnerHpc(PogoRunner):
    pass

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<commit_msg>workout the command depending on the presence of the '-mm' parameter<commit_after>#
# Author    : Manuel Bernal Llinares
# Project   : trackhub-creator
# Timestamp : 02-08-2017 16:40
# ---
# © 2017 Manuel Bernal Llinares <mbdebian@gmail.com>
# All rights reserved.
#

"""
This module contains models for dealing with PoGo stuff
"""

import os
import abc
import threading
# Application imports
import config_manager as main_app_config_manager
from parallel.models import ParallelRunner, CommandLineRunnerFactory
from parallel.exceptions import CommandLineRunnerException
from . import config_manager as module_config_manager
from .exceptions import PogoRunnerException

class PogoRunResult:
    # TODO - Needs to be extended for abstracting from results files from '-mm' parameter use
    def __init__(self,
                 ncbi_taxonomy_id=None,
                 pogo_source_file_path=None,
                 protein_sequence_file_path=None,
                 gtf_file_path=None):
        """
        Just the constructor, I had this implemented as syntactic sugar, but I was wrong
        :param ncbi_taxonomy_id: ncbi taxonomy id for this PoGo run results
        :param pogo_source_file_path: path of the source file used to run PoGo
        :param protein_sequence_file_path: FASTA file
        :param gtf_file_path: GTF file
        """
        # Logging
        self.__logger = main_app_config_manager.get_app_config_manager().get_logger_for(
            "{}.{}".format(__name__, type(self).__name__))
        # Map<pogo_result_file_extension, pogo_result_file_path>
        self.__pogo_result_file_paths = {}
        self.__pogo_source_file_path = None
        # I knew this was going to give me trouble
        self.set_ncbi_taxonomy_id(ncbi_taxonomy_id)
        self.set_protein_sequence_file_path(protein_sequence_file_path)
        self.set_gtf_file_path(gtf_file_path)
        self.set_pogo_source_file_path(pogo_source_file_path)

    def __generate_pogo_result_file_paths(self, pogo_source_file_path):
        self.__pogo_result_file_paths = {}
        pogo_results_folder_path = os.path.dirname(pogo_source_file_path)
        pogo_source_file_name = os.path.basename(pogo_source_file_path)
        for root, dirs, files in os.walk(pogo_results_folder_path):
            for pogo_result_file_path in files:
                pogo_result_file_name = os.path.basename(pogo_result_file_path)
                if pogo_result_file_name.startswith(pogo_source_file_name) and (
                            pogo_result_file_name != pogo_source_file_name):
                    # It is file generated by PoGo
                    pogo_result_file_extension = pogo_result_file_name.split(pogo_source_file_name)[1]
                    # self.__pogo_result_file_paths[pogo_result_file_extension] = pogo_result_file_path
                    # I need the full path for the pogo result file path
                    self.__pogo_result_file_paths[pogo_result_file_extension] = os.path.join(pogo_results_folder_path,
                                                                                pogo_result_file_path)
                    self._get_logger().info(
                        "PoGo run result '{}' ---> '{}', at full path '{}'".format(pogo_result_file_extension,
                                                                                pogo_result_file_path,
                                                                                self.__pogo_result_file_paths[
                                                                                pogo_result_file_extension]))

    def __get_pogo_result_file_path(self, file_extension):
        if file_extension in self.__pogo_result_file_paths:
            return self.__pogo_result_file_paths[file_extension]
        return None

    def _get_logger(self):
        return self.__logger

    def set_ncbi_taxonomy_id(self, taxonomy_id):
        self.__ncbi_taxonomy_id = taxonomy_id

    def set_pogo_source_file_path(self, pogo_source_file_path):
        if not self.__pogo_source_file_path or (self.__pogo_source_file_path != pogo_source_file_path):
            self.__pogo_source_file_path = pogo_source_file_path
            # Regenerate pogo result file paths
            self.__generate_pogo_result_file_paths(pogo_source_file_path)

    def set_protein_sequence_file_path(self, protein_sequence_file_path):
        self.__protein_sequence_file_path = protein_sequence_file_path

    def set_gtf_file_path(self, gtf_file_path):
        self.__gtf_file_path = gtf_file_path

    def get_ncbi_taxonomy_id(self):
        return self.__ncbi_taxonomy_id

    def get_pogo_result_main_bed_file_path(self):
        return self.__get_pogo_result_file_path(
            module_config_manager.get_configuration_service().get_pogo_result_file_extension_for_main_bed_file())

    def get_pogo_result_main_ptm_bed_file_path(self):
        return self.__get_pogo_result_file_path(
            module_config_manager.get_configuration_service().get_pogo_result_file_extension_for_main_ptm_bed_file()
        )

# PoGo Runners
class PogoRunnerFactory:
    @staticmethod
    def get_pogo_runner(ncbi_taxonomy_id, pogo_input_file, protein_sequence_file_path, gtf_file_path):
        # TODO - Needs to be extended for abstracting from results files from '-mm' parameter use
        # TODO - In the future, more PoGo runners will be implemented
        return PogoRunnerLocalThread(ncbi_taxonomy_id, pogo_input_file, protein_sequence_file_path, gtf_file_path)

class PogoRunner(ParallelRunner):
    # TODO - Needs to be extended for abstracting from results files from '-mm' parameter use
    def __init__(self,
                 ncbi_taxonomy_id=None,
                 pogo_input_file=None,
                 protein_sequence_file_path=None,
                 gtf_file_path=None,
                 mm_gap=None):
        super().__init__()
        self.ncbi_taxonomy_id = ncbi_taxonomy_id
        self.pogo_input_file = pogo_input_file
        self.protein_sequence_file_path = protein_sequence_file_path
        self.gtf_file_path = gtf_file_path
        self.mm_gap = mm_gap
        self._stdout = b' '
        self._stderr = b' '
        self.__success = True
        self.__pogo_run_result = None

    def _set_failed_to_run_pogo(self):
        self.__success = False

    def _set_success_to_run_pogo(self):
        self.__success = self.__success and True

    def _validate_environment_for_running_pogo(self):
        validation_ok = True
        if not self.ncbi_taxonomy_id:
            self._logger.warning("NO NCBI TAXONOMY ID has been specified for running PoGo with input file '{}'"
                                 .format(self.pogo_input_file))
        if not os.path.isfile(self.pogo_input_file):
            self._logger.error("PoGo input file '{}' IS NOT VALID".format(self.pogo_input_file))
            validation_ok = False
        if not os.path.isfile(self.protein_sequence_file_path):
            self._logger.error("Protein sequence file '{}' IS NOT VALID".format(self.protein_sequence_file_path))
            validation_ok = False
        if not os.path.isfile(self.gtf_file_path):
            self._logger.error("GTF file '{}' IS NOT VALID".format(self.gtf_file_path))
            validation_ok = False
        if not os.path.isfile(module_config_manager.get_configuration_service().get_pogo_binary_file_path()):
            self._logger.error("PoGo binary file '{}' IS NOT VALID"
                               .format(module_config_manager.get_configuration_service().get_pogo_binary_file_path()))
            validation_ok = False
        return validation_ok

    def get_stdout(self):
        # Never give it back until the runner is done with whatever it is doing
        if not self._done:
            raise PogoRunnerException("PoGo Runner is NOT DONE doing its job, thus 'stdout' is NOT AVAILABLE")
        return self._stdout

    def get_stderr(self):
        # Never give it back until the runner is done with whatever it is doing
        if not self._done:
            raise PogoRunnerException("PoGo Runner is NOT DONE doing its job, thus 'stderr' is NOT AVAILABLE")
        return self._stderr

    def get_pogo_run_command(self):
        pogo_parameter_species = ''
        if self.ncbi_taxonomy_id:
            pogo_parameter_species = " -species {} ".format(self.ncbi_taxonomy_id)
        pogo_parameter_mm_gap = ''
        if self.mm_gap:
            pogo_parameter_mm_gap = " -mm {}".format(self.mm_gap)
        pogo_command = "time {}{}{} -fasta {} -gtf {} -in {}" \
            .format(module_config_manager.get_configuration_service().get_pogo_binary_file_path(),
                    pogo_parameter_species,
                    pogo_parameter_mm_gap,
                    self.protein_sequence_file_path,
                    self.gtf_file_path,
                    self.pogo_input_file)
        self._logger.debug("Command for running PoGo - '{}'".format(pogo_command))
        return pogo_command

    @abc.abstractmethod
    def _get_command_line_runner(self):
        """
        Although a factory is used to get a command line runner, its initialization differs depending on the final
        PogoRunner, i.e. we delegate to subclasses the provisioning of an initialized CommandLineRunner, how we use
        it, is the same for all subclasses.
        It will prepare the command line runner completely, including the command it has to run.
        :return: a CommandLineRunner for running PoGo
        """
        ...

    def _run(self):
        if not self._validate_environment_for_running_pogo():
            self._logger.error("Can't run PoGo for file '{}'".format(self.pogo_input_file))
            self._set_failed_to_run_pogo()
        else:
            command_line_runner = self._get_command_line_runner()
            try:
                command_line_runner.start()
                self._logger.debug("PoGo --- STARTED --- command '{}'".format(command_line_runner.command))
                command_line_runner.wait()
            except CommandLineRunnerException as e:
                self._logger.error("ERROR RUNNING PoGo - {}, STDOUT ---> '{}', STDERR ---> '{}'"
                                   .format(e.value,
                                           command_line_runner.get_stdout().decode('utf8'),
                                           command_line_runner.get_stderr().decode('utf8')))
                self._set_failed_to_run_pogo()
            finally:
                self._stdout = command_line_runner.get_stdout()
                self._stderr = command_line_runner.get_stderr()
            if not command_line_runner.command_success:
                self._set_failed_to_run_pogo()
                self._logger.error("PoGo FAILED on input file '{}', command '{}'"
                                   .format(self.pogo_input_file, command_line_runner.command))
            else:
                # Successful run of PoGo
                self._set_success_to_run_pogo()
                self._logger.info("PoGo SUCCESS on input file '{}', command '{}'"
                                  .format(self.pogo_input_file, command_line_runner.command))

    def get_pogo_run_result(self):
        if not self.is_done():
            raise PogoRunnerException("PoGo runner IS NOT FINISHED YET, thus, results can't be retrieved")
        if not self.__pogo_run_result:
            self.__pogo_run_result = PogoRunResult(self.ncbi_taxonomy_id,
                                                   self.pogo_input_file,
                                                   self.protein_sequence_file_path,
                                                   self.gtf_file_path)
        return self.__pogo_run_result

    def is_success(self):
        return self.__success

class PogoRunnerLocalThread(PogoRunner):
    def __init__(self,
                 ncbi_taxonomy_id=None,
                 pogo_input_file=None,
                 protein_sequence_file_path=None,
                 gtf_file_path=None):
        super().__init__(ncbi_taxonomy_id, pogo_input_file, protein_sequence_file_path, gtf_file_path)
        self._logger = main_app_config_manager \
            .get_app_config_manager() \
            .get_logger_for("{}.{}-{}".format(__name__, type(self).__name__, threading.current_thread().getName()))

    def _get_command_line_runner(self):
        command_line_runner = CommandLineRunnerFactory.get_multithread_command_line_runner()
        command_line_runner.command = self.get_pogo_run_command()
        command_line_runner.timeout = module_config_manager.get_configuration_service().get_pogo_run_timeout()
        return command_line_runner

class PogoRunnerHpc(PogoRunner):
    pass

if __name__ == '__main__':
    print("ERROR: This script is part of a pipeline collection and it is not meant to be run in stand alone mode")
<|endoftext|>