<commit_before># Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Handles a single Python breakpoint."""

from datetime import datetime
from datetime import timedelta
import os
from threading import Lock

import capture_collector
import cdbg_native as native
import deferred_modules
import module_explorer
import module_lookup

# TODO(vlif): move to messages.py module.
BREAKPOINT_ONLY_SUPPORTS_PY_FILES = (
    'Only files with .py or .pyc extension are supported')
MODULE_NOT_FOUND = (
    'Python module not found')
NO_CODE_FOUND_AT_LINE = (
    'No code found at line $0')
BREAKPOINTS_EMULATOR_QUOTA_EXCEEDED = (
    'Active snapshots might affect the application performance')
GLOBAL_CONDITION_QUOTA_EXCEEDED = (
    'Snapshot cancelled. The condition evaluation cost for all active '
    'snapshots might affect the application performance.')
BREAKPOINT_CONDITION_QUOTA_EXCEEDED = (
    'Snapshot cancelled. The condition evaluation at this location might '
    'affect application performance. Please simplify the condition or move '
    'the snapshot to a less frequently called statement.')
MUTABLE_CONDITION = (
    'Only immutable expressions can be used in snapshot conditions')
BREAKPOINT_EXPIRED = (
    'The snapshot has expired')

# Status messages for different breakpoint events (except of "hit").
_BREAKPOINT_EVENT_STATUS = dict(
    [(native.BREAKPOINT_EVENT_GLOBAL_CONDITION_QUOTA_EXCEEDED,
      {'isError': True,
       'refersTo': 'BREAKPOINT_CONDITION',
       'description': {'format': GLOBAL_CONDITION_QUOTA_EXCEEDED}}),
     (native.BREAKPOINT_EVENT_BREAKPOINT_CONDITION_QUOTA_EXCEEDED,
      {'isError': True,
       'refersTo': 'BREAKPOINT_CONDITION',
       'description': {'format': BREAKPOINT_CONDITION_QUOTA_EXCEEDED}}),
     (native.BREAKPOINT_EVENT_CONDITION_EXPRESSION_MUTABLE,
      {'isError': True,
       'refersTo': 'BREAKPOINT_CONDITION',
       'description': {'format': MUTABLE_CONDITION}})])

class PythonBreakpoint(object):
  """Handles a single Python breakpoint.

  Taking care of a breakpoint starts with setting one and evaluating
  condition. When a breakpoint we need to evaluate all the watched expressions
  and take an action. The action can be either to collect all the data or
  to log a statement.
  """

  def __init__(self, definition, hub_client, breakpoints_manager):
    """Class constructor.

    Tries to set the breakpoint. If the source location is invalid, the
    breakpoint is completed with an error message. If the source location is
    valid, but the module hasn't been loaded yet, the breakpoint is initialized
    as deferred.

    Args:
      definition: breakpoint definition as it came from the backend.
      hub_client: asynchronously sends breakpoint updates to the backend.
      breakpoints_manager: parent object managing active breakpoints.
    """
    self.definition = definition

    # Breakpoint expiration time.
    self.expiration_period = timedelta(hours=24)

    self._hub_client = hub_client
    self._breakpoints_manager = breakpoints_manager
    self._cookie = None
    self._import_hook_cleanup = None

    self._lock = Lock()
    self._completed = False

    if not self._TryActivateBreakpoint() and not self._completed:
      self._DeferBreakpoint()

  def Clear(self):
    """Clears the breakpoint and releases all breakpoint resources.

    This function is assumed to be called by BreakpointsManager. Therefore we
    don't call CompleteBreakpoint from here.
    """
    self._RemoveImportHook()
    if self._cookie is not None:
      native.LogInfo('Clearing breakpoint %s' % self.GetBreakpointId())
      native.ClearConditionalBreakpoint(self._cookie)
      self._cookie = None

    self._completed = True  # Never again send updates for this breakpoint.

  def BreakpointsEmulatorQuotaExceeded(self):
    self._CompleteBreakpoint({
        'status': {
            'isError': True,
            'description': {'format': BREAKPOINTS_EMULATOR_QUOTA_EXCEEDED}}})

  def GetBreakpointId(self):
    return self.definition['id']

  def GetExpirationTime(self):
    """Computes the timestamp at which this breakpoint will expire."""
    create_datetime = datetime.strptime(
        self.definition['createTime'].replace('Z', 'UTC'),
        '%Y-%m-%dT%H:%M:%S.%f%Z')
    return create_datetime + self.expiration_period

  def ExpireBreakpoint(self):
    """Expires this breakpoint."""
    # Let only one thread capture the data and complete the breakpoint.
    if not self._SetCompleted():
      return

    self._CompleteBreakpoint({
        'status': {
            'isError': True,
            'refersTo': 'UNSPECIFIED',
            'description': {'format': BREAKPOINT_EXPIRED}}})

  def _TryActivateBreakpoint(self):
    """Sets the breakpoint if the module has already been loaded.

    This function will complete the breakpoint with error if breakpoint
    definition is incorrect. Examples: invalid line or bad condition.

    If the code object corresponding to the source path can't be found,
    this function returns False. In this case, the breakpoint is not
    completed, since the breakpoint may be deferred.

    Returns:
      True if breakpoint was set or false otherwise. False can be returned
      for potentially deferred breakpoints or in case of a bad breakpoint
      definition. The self._completed flag distinguishes between the two cases.
    """

    # Find the code object in which the breakpoint is being set.
    code_object = self._FindCodeObject()
    if not code_object:
      return False

    # Compile the breakpoint condition.
    condition = None
    if self.definition.get('condition'):
      try:
        condition = compile(self.definition.get('condition'),
                            '<condition_expression>',
                            'eval')
      except TypeError as e:  # condition string contains null bytes.
        self._CompleteBreakpoint({
            'status': {
                'isError': True,
                'refersTo': 'BREAKPOINT_CONDITION',
                'description': {
                    'format': 'Invalid expression',
                    'parameters': [str(e)]}}})
        return False
      except SyntaxError as e:
        self._CompleteBreakpoint({
            'status': {
                'isError': True,
                'refersTo': 'BREAKPOINT_CONDITION',
                'description': {
                    'format': 'Expression could not be compiled: $0',
                    'parameters': [e.msg]}}})
        return False

    line = self.definition['location']['line']

    native.LogInfo('Creating new Python breakpoint %s in %s, line %d' % (
        self.GetBreakpointId(), code_object, line))

    self._cookie = native.SetConditionalBreakpoint(
        code_object,
        line,
        condition,
        self._BreakpointEvent)

    return True

  def _FindCodeObject(self):
    """Finds the target code object for the breakpoint.

    This function completes breakpoint with error if the module was found,
    but the line number is invalid. When code object is not found for the
    breakpoint source location, this function just returns None. It does not
    assume error, because it might be a deferred breakpoint.

    Returns:
      Python code object object in which the breakpoint will be set or None if
      module not found or if there is no code at the specified line.
    """
    path = self.definition['location']['path']
    line = self.definition['location']['line']

    module = module_lookup.FindModule(path)
    if not module:
      return None

    code_object = module_explorer.GetCodeObjectAtLine(module, line)
    if code_object is None:
      self._CompleteBreakpoint({
          'status': {
              'isError': True,
              'refersTo': 'BREAKPOINT_SOURCE_LOCATION',
              'description': {
                  'format': NO_CODE_FOUND_AT_LINE,
                  'parameters': [str(line)]}}})
      return None

    return code_object

  # Enables deferred breakpoints.
  def _DeferBreakpoint(self):
    """Defers breakpoint activation until the module has been loaded.

    This function first verifies that a module corresponding to breakpoint
    location exists. This way if the user sets breakpoint in a file that
    doesn't even exist, the debugger will not be waiting forever. If there
    is definitely no module that matches this breakpoint, this function
    completes the breakpoint with error status.

    Otherwise the debugger assumes that the module corresponding to breakpoint
    location hasn't been loaded yet. The debugger will then start waiting for
    the module to get loaded. Once the module is loaded, the debugger
    will automatically try to activate the breakpoint.
    """
    path = self.definition['location']['path']

    if os.path.splitext(path)[1] != '.py':
      self._CompleteBreakpoint({
          'status': {
              'isError': True,
              'refersTo': 'BREAKPOINT_SOURCE_LOCATION',
              'description': {'format': BREAKPOINT_ONLY_SUPPORTS_PY_FILES}}})
      return

    if not deferred_modules.IsValidSourcePath(path):
      self._CompleteBreakpoint({
          'status': {
              'isError': True,
              'refersTo': 'BREAKPOINT_SOURCE_LOCATION',
              'description': {'format': MODULE_NOT_FOUND}}})

    assert not self._import_hook_cleanup
    self._import_hook_cleanup = deferred_modules.AddImportCallback(
        self.definition['location']['path'],
        lambda unused_module_name: self._TryActivateBreakpoint())

  def _RemoveImportHook(self):
    """Removes the import hook if one was installed."""
    if self._import_hook_cleanup:
      self._import_hook_cleanup()
      self._import_hook_cleanup = None

  def _CompleteBreakpoint(self, data, is_incremental=True):
    """Sends breakpoint update and deactivates the breakpoint."""
    if is_incremental:
      data = dict(self.definition, **data)
    data['isFinalState'] = True

    self._hub_client.EnqueueBreakpointUpdate(data)
    self._breakpoints_manager.CompleteBreakpoint(self.GetBreakpointId())
    self.Clear()

  def _SetCompleted(self):
    """Atomically marks the breakpoint as completed.

    Returns:
      True if the breakpoint wasn't marked already completed or False if the
      breakpoint was already completed.
    """
    with self._lock:
      if self._completed:
        return False
      self._completed = True
      return True

  def _BreakpointEvent(self, event, frame):
    """Callback invoked by cdbg_native when breakpoint hits.

    Args:
      event: breakpoint event (see kIntegerConstants in native_module.cc).
      frame: Python stack frame of breakpoint hit or None for other events.
    """
    # TODO(vlif): support dynamic log breakpoints.

    # Let only one thread capture the data and complete the breakpoint.
    if not self._SetCompleted():
      return

    self.Clear()

    if event == native.BREAKPOINT_EVENT_HIT:
      collector = capture_collector.CaptureCollector(self.definition)
      collector.Collect(frame)

      self._CompleteBreakpoint(collector.breakpoint, is_incremental=False)
    elif event == native.BREAKPOINT_EVENT_EMULATOR_QUOTA_EXCEEDED:
      self._breakpoints_manager.BreakpointsEmulatorQuotaExceeded()
    else:
      self._CompleteBreakpoint({'status': _BREAKPOINT_EVENT_STATUS[event]})
<commit_msg>Add proper support for internal error in Python Cloud Debugger ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=105371517<commit_after># Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS-IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Handles a single Python breakpoint."""

from datetime import datetime
from datetime import timedelta
import os
from threading import Lock

import capture_collector
import cdbg_native as native
import deferred_modules
import module_explorer
import module_lookup

# TODO(vlif): move to messages.py module.
BREAKPOINT_ONLY_SUPPORTS_PY_FILES = (
    'Only files with .py or .pyc extension are supported')
MODULE_NOT_FOUND = (
    'Python module not found')
NO_CODE_FOUND_AT_LINE = (
    'No code found at line $0')
BREAKPOINTS_EMULATOR_QUOTA_EXCEEDED = (
    'Active snapshots might affect the application performance')
GLOBAL_CONDITION_QUOTA_EXCEEDED = (
    'Snapshot cancelled. The condition evaluation cost for all active '
    'snapshots might affect the application performance.')
BREAKPOINT_CONDITION_QUOTA_EXCEEDED = (
    'Snapshot cancelled. The condition evaluation at this location might '
    'affect application performance. Please simplify the condition or move '
    'the snapshot to a less frequently called statement.')
MUTABLE_CONDITION = (
    'Only immutable expressions can be used in snapshot conditions')
BREAKPOINT_EXPIRED = (
    'The snapshot has expired')
INTERNAL_ERROR = (
    'Internal error occurred')

# Status messages for different breakpoint events (except of "hit").
_BREAKPOINT_EVENT_STATUS = dict(
    [(native.BREAKPOINT_EVENT_ERROR,
      {'isError': True,
       'description': {'format': INTERNAL_ERROR}}),
     (native.BREAKPOINT_EVENT_GLOBAL_CONDITION_QUOTA_EXCEEDED,
      {'isError': True,
       'refersTo': 'BREAKPOINT_CONDITION',
       'description': {'format': GLOBAL_CONDITION_QUOTA_EXCEEDED}}),
     (native.BREAKPOINT_EVENT_BREAKPOINT_CONDITION_QUOTA_EXCEEDED,
      {'isError': True,
       'refersTo': 'BREAKPOINT_CONDITION',
       'description': {'format': BREAKPOINT_CONDITION_QUOTA_EXCEEDED}}),
     (native.BREAKPOINT_EVENT_CONDITION_EXPRESSION_MUTABLE,
      {'isError': True,
       'refersTo': 'BREAKPOINT_CONDITION',
       'description': {'format': MUTABLE_CONDITION}})])

class PythonBreakpoint(object):
  """Handles a single Python breakpoint.

  Taking care of a breakpoint starts with setting one and evaluating
  condition. When a breakpoint we need to evaluate all the watched expressions
  and take an action. The action can be either to collect all the data or
  to log a statement.
  """

  def __init__(self, definition, hub_client, breakpoints_manager):
    """Class constructor.

    Tries to set the breakpoint. If the source location is invalid, the
    breakpoint is completed with an error message. If the source location is
    valid, but the module hasn't been loaded yet, the breakpoint is initialized
    as deferred.

    Args:
      definition: breakpoint definition as it came from the backend.
      hub_client: asynchronously sends breakpoint updates to the backend.
      breakpoints_manager: parent object managing active breakpoints.
    """
    self.definition = definition

    # Breakpoint expiration time.
    self.expiration_period = timedelta(hours=24)

    self._hub_client = hub_client
    self._breakpoints_manager = breakpoints_manager
    self._cookie = None
    self._import_hook_cleanup = None

    self._lock = Lock()
    self._completed = False

    if not self._TryActivateBreakpoint() and not self._completed:
      self._DeferBreakpoint()

  def Clear(self):
    """Clears the breakpoint and releases all breakpoint resources.

    This function is assumed to be called by BreakpointsManager. Therefore we
    don't call CompleteBreakpoint from here.
    """
    self._RemoveImportHook()
    if self._cookie is not None:
      native.LogInfo('Clearing breakpoint %s' % self.GetBreakpointId())
      native.ClearConditionalBreakpoint(self._cookie)
      self._cookie = None

    self._completed = True  # Never again send updates for this breakpoint.

  def BreakpointsEmulatorQuotaExceeded(self):
    self._CompleteBreakpoint({
        'status': {
            'isError': True,
            'description': {'format': BREAKPOINTS_EMULATOR_QUOTA_EXCEEDED}}})

  def GetBreakpointId(self):
    return self.definition['id']

  def GetExpirationTime(self):
    """Computes the timestamp at which this breakpoint will expire."""
    create_datetime = datetime.strptime(
        self.definition['createTime'].replace('Z', 'UTC'),
        '%Y-%m-%dT%H:%M:%S.%f%Z')
    return create_datetime + self.expiration_period

  def ExpireBreakpoint(self):
    """Expires this breakpoint."""
    # Let only one thread capture the data and complete the breakpoint.
    if not self._SetCompleted():
      return

    self._CompleteBreakpoint({
        'status': {
            'isError': True,
            'refersTo': 'UNSPECIFIED',
            'description': {'format': BREAKPOINT_EXPIRED}}})

  def _TryActivateBreakpoint(self):
    """Sets the breakpoint if the module has already been loaded.

    This function will complete the breakpoint with error if breakpoint
    definition is incorrect. Examples: invalid line or bad condition.

    If the code object corresponding to the source path can't be found,
    this function returns False. In this case, the breakpoint is not
    completed, since the breakpoint may be deferred.

    Returns:
      True if breakpoint was set or false otherwise. False can be returned
      for potentially deferred breakpoints or in case of a bad breakpoint
      definition. The self._completed flag distinguishes between the two cases.
    """

    # Find the code object in which the breakpoint is being set.
    code_object = self._FindCodeObject()
    if not code_object:
      return False

    # Compile the breakpoint condition.
    condition = None
    if self.definition.get('condition'):
      try:
        condition = compile(self.definition.get('condition'),
                            '<condition_expression>',
                            'eval')
      except TypeError as e:  # condition string contains null bytes.
        self._CompleteBreakpoint({
            'status': {
                'isError': True,
                'refersTo': 'BREAKPOINT_CONDITION',
                'description': {
                    'format': 'Invalid expression',
                    'parameters': [str(e)]}}})
        return False
      except SyntaxError as e:
        self._CompleteBreakpoint({
            'status': {
                'isError': True,
                'refersTo': 'BREAKPOINT_CONDITION',
                'description': {
                    'format': 'Expression could not be compiled: $0',
                    'parameters': [e.msg]}}})
        return False

    line = self.definition['location']['line']

    native.LogInfo('Creating new Python breakpoint %s in %s, line %d' % (
        self.GetBreakpointId(), code_object, line))

    self._cookie = native.SetConditionalBreakpoint(
        code_object,
        line,
        condition,
        self._BreakpointEvent)

    return True

  def _FindCodeObject(self):
    """Finds the target code object for the breakpoint.

    This function completes breakpoint with error if the module was found,
    but the line number is invalid. When code object is not found for the
    breakpoint source location, this function just returns None. It does not
    assume error, because it might be a deferred breakpoint.

    Returns:
      Python code object object in which the breakpoint will be set or None if
      module not found or if there is no code at the specified line.
    """
    path = self.definition['location']['path']
    line = self.definition['location']['line']

    module = module_lookup.FindModule(path)
    if not module:
      return None

    code_object = module_explorer.GetCodeObjectAtLine(module, line)
    if code_object is None:
      self._CompleteBreakpoint({
          'status': {
              'isError': True,
              'refersTo': 'BREAKPOINT_SOURCE_LOCATION',
              'description': {
                  'format': NO_CODE_FOUND_AT_LINE,
                  'parameters': [str(line)]}}})
      return None

    return code_object

  # Enables deferred breakpoints.
  def _DeferBreakpoint(self):
    """Defers breakpoint activation until the module has been loaded.

    This function first verifies that a module corresponding to breakpoint
    location exists. This way if the user sets breakpoint in a file that
    doesn't even exist, the debugger will not be waiting forever. If there
    is definitely no module that matches this breakpoint, this function
    completes the breakpoint with error status.

    Otherwise the debugger assumes that the module corresponding to breakpoint
    location hasn't been loaded yet. The debugger will then start waiting for
    the module to get loaded. Once the module is loaded, the debugger
    will automatically try to activate the breakpoint.
    """
    path = self.definition['location']['path']

    if os.path.splitext(path)[1] != '.py':
      self._CompleteBreakpoint({
          'status': {
              'isError': True,
              'refersTo': 'BREAKPOINT_SOURCE_LOCATION',
              'description': {'format': BREAKPOINT_ONLY_SUPPORTS_PY_FILES}}})
      return

    if not deferred_modules.IsValidSourcePath(path):
      self._CompleteBreakpoint({
          'status': {
              'isError': True,
              'refersTo': 'BREAKPOINT_SOURCE_LOCATION',
              'description': {'format': MODULE_NOT_FOUND}}})

    assert not self._import_hook_cleanup
    self._import_hook_cleanup = deferred_modules.AddImportCallback(
        self.definition['location']['path'],
        lambda unused_module_name: self._TryActivateBreakpoint())

  def _RemoveImportHook(self):
    """Removes the import hook if one was installed."""
    if self._import_hook_cleanup:
      self._import_hook_cleanup()
      self._import_hook_cleanup = None

  def _CompleteBreakpoint(self, data, is_incremental=True):
    """Sends breakpoint update and deactivates the breakpoint."""
    if is_incremental:
      data = dict(self.definition, **data)
    data['isFinalState'] = True

    self._hub_client.EnqueueBreakpointUpdate(data)
    self._breakpoints_manager.CompleteBreakpoint(self.GetBreakpointId())
    self.Clear()

  def _SetCompleted(self):
    """Atomically marks the breakpoint as completed.

    Returns:
      True if the breakpoint wasn't marked already completed or False if the
      breakpoint was already completed.
    """
    with self._lock:
      if self._completed:
        return False
      self._completed = True
      return True

  def _BreakpointEvent(self, event, frame):
    """Callback invoked by cdbg_native when breakpoint hits.

    Args:
      event: breakpoint event (see kIntegerConstants in native_module.cc).
      frame: Python stack frame of breakpoint hit or None for other events.
    """
    # TODO(vlif): support dynamic log breakpoints.

    # Let only one thread capture the data and complete the breakpoint.
    if not self._SetCompleted():
      return

    self.Clear()

    if event == native.BREAKPOINT_EVENT_HIT:
      collector = capture_collector.CaptureCollector(self.definition)
      collector.Collect(frame)

      self._CompleteBreakpoint(collector.breakpoint, is_incremental=False)
    elif event == native.BREAKPOINT_EVENT_EMULATOR_QUOTA_EXCEEDED:
      self._breakpoints_manager.BreakpointsEmulatorQuotaExceeded()
    else:
      self._CompleteBreakpoint({'status': _BREAKPOINT_EVENT_STATUS[event]})
<|endoftext|>


<commit_before>from datetime import datetime, date
from time import sleep

from pyepm import api, config
from bitcoin import *

try:
    from urllib.request import build_opener
except:
    from urllib2 import build_opener

# Makes a request to a given URL (first arg) and optional params (second arg)
def make_request(*args):
    opener = build_opener()
    opener.addheaders = [('User-agent',
                          'Mozilla/5.0'+str(random.randrange(1000000)))]
    try:
        return opener.open(*args).read().strip()
    except Exception as e:
        try:
            p = e.read().strip()
        except:
            p = e
        raise Exception(p)

api_config = config.read_config()
instance = api.Api(api_config)

# CPP
instance.address = "0xcd2a3d9f938e13cd947ec05abc7fe734df8dd826"
to = "0xba164d1e85526bd5e27fd15ad14b0eae91c45a93"

# GO
# instance.address = "0x9dc2299a76b68b7ffa9e3ba0fd8cd7646d21d409"
# to = "0x0fd51f042310093b9d8df57d37a42c3523537a99"

def main():
    run()

def run(doFetch=False):
    network = 'testnet'
    chainHead = blockHashHex(getBlockchainHead())

    blockInfoUrl = "https://tbtc.blockr.io/api/v1/block/info/"

    data = make_request(blockInfoUrl + chainHead)
    jsonobj = json.loads(data)
    contractHeight = jsonobj["data"]["nb"]

    actualHeight = last_block_height(network)

    print('@@@ heights: {0} {1}').format(contractHeight, actualHeight)

    print('@@@ chainHead: %s' % chainHead)

    chunkSize = 5
    fetchNum =  actualHeight - contractHeight
    numChunk = fetchNum / chunkSize
    leftoverToFetch = fetchNum % chunkSize

    print('@@@ numChunk: {0} leftoverToFetch: {1}').format(numChunk, leftoverToFetch)

    if doFetch:
        fetchHeaders(contractHeight+1, chunkSize, numChunk)
        fetchHeaders(actualHeight-leftoverToFetch+1, 1, leftoverToFetch)

        sleep(3)
        print('@@@ chainHead: %s' % chainHead)

def fetchHeaders(chunkStartNum, chunkSize, numChunk):
    for j in range(numChunk):
        strings = ""
        for i in range(chunkSize):
            blockNum = chunkStartNum  + i
            bhJson = blockr_get_block_header_data(blockNum, network='testnet')
            bhStr = serialize_header(bhJson)
            print("@@@ {0}: {1}").format(blockNum, bhStr)
            strings += bhStr

        storeHeaders(strings.decode('hex'), chunkSize)

        chainHead = getBlockchainHead()
        print('@@@ DONE hexHead: %s' % blockHashHex(chainHead))

        chunkStartNum += chunkSize

def storeHeaders(bhBinary, chunkSize):

    txCount = instance.transaction_count(defaultBlock='pending')
    print('----------------------------------')
    print('txCount: %s' % txCount)

    hashOne = blockHashHex(int(bin_dbl_sha256(bhBinary[:80])[::-1].encode('hex'), 16))
    hashLast = blockHashHex(int(bin_dbl_sha256(bhBinary[-80:])[::-1].encode('hex'), 16))
    print('hashOne: %s' % hashOne)
    print('hashLast: %s' % hashLast)

    firstH = bhBinary[:80].encode('hex')
    lastH = bhBinary[-80:].encode('hex')
    print('firstH: %s' % firstH)
    print('lastH: %s' % lastH)

    fun_name = "bulkStoreHeader"
    sig = "si"

    # bhBinary = '\x02\x00\x00\x00~\xf0U\xe1gM.eQ\xdb\xa4\x1c\xd2\x14\xde\xbb\xee4\xae\xb5D\xc7\xecg\x00\x00\x00\x00\x00\x00\x00\x00\xd3\x99\x89c\xf8\x0c[\xabC\xfe\x8c&"\x8e\x98\xd00\xed\xf4\xdc\xbeH\xa6f\xf5\xc3\x9e-z\x88\\\x91\x02\xc8mSl\x89\x00\x19Y:G\r\x02\x00\x00\x00Tr\xac\x8b\x11\x87\xbf\xcf\x91\xd6\xd2\x18\xbb\xda\x1e\xb2@]|U\xf1\xf8\xcc\x82\x00\x00\x00\x00\x00\x00\x00\x00\xab\n\xaa7|\xa3\xf4\x9b\x15E\xe2\xaek\x06g\xa0\x8fB\xe7-\x8c$\xae#q@\xe2\x8f\x14\xf3\xbb|k\xccmSl\x89\x00\x19\xed\xd8<\xcf\x02\x00\x00\x00\xa9\xab\x12\xe3,\xed\xdc+\xa5\xe6\xade\x1f\xacw,\x986\xdf\x83M\x91\xa0I\x00\x00\x00\x00\x00\x00\x00\x00\xdfuu\xc7\x8f\x83\x1f \xaf\x14~\xa7T\xe5\x84\xaa\xd9Yeiic-\xa9x\xd2\xddq\x86#\xfd0\xc5\xccmSl\x89\x00\x19\xe6Q\x07\xe9\x02\x00\x00\x00,P\x1f\xc0\xb0\xfd\xe9\xb3\xc1\x0e#S\xc1TI*5k\x1a\x02)^+\x86\x00\x00\x00\x00\x00\x00\x00\x00\xa7\xaaa\xc8\xd3|\x88v\xba\xa0\x17\x9ej2\x94D4\xbf\xd3\xe1\xccug\x89*1K\x0c{\x9e]\x92\'\xcemSl\x89\x00\x19\xa4\xa0<{\x02\x00\x00\x00\xe7\xfc\x91>+y\n0v\x0c\xaa\xfb\x9b_\xaa\xe1\xb5\x1dlT\xff\xe4\xae\x82\x00\x00\x00\x00\x00\x00\x00\x00P\xad\x11k\xfb\x11c\x03\x03a\xd9}H\xb4\xca\x90\'\xa4\x9b\xca\xf8\xb8\xd4!\x1b\xaa\x92\xccr\xe7\xe1#f\xcfmSl\x89\x00\x19\xe6\x13\x9c\x82'
    data = [bhBinary, chunkSize]

    gas = 3000000
    gas_price = 10000000000000  # if CPP, may need to make this 10szabo (10e12)
    value = 0

    #
    # store the headers
    #
    wait = True
    from_count = instance.transaction_count(defaultBlock='pending')
    if wait:
        from_block = instance.last_block()

    instance.transact(to, fun_name=fun_name, sig=sig, data=data, gas=gas, gas_price=gas_price, value=value)

    # instance.wait_for_transaction(
    #     from_count=from_count,
    #     #verbose=(True if api_config.get('misc', 'verbosity') > 1 else False))
    #     verbose=True)

    if wait:
        for i in range(1):
            instance.wait_for_next_block(from_block=from_block,
                #verbose=(True if api_config.get('misc', 'verbosity') > 1 else False))
                verbose=True)
            from_block = instance.last_block()

    # 2nd try
    # instance.transact(to, fun_name=fun_name, sig=sig, data=data, gas=gas, gas_price=gas_price, value=value)
    # instance.wait_for_transaction(
    #     from_count=from_count,
    #     #verbose=(True if api_config.get('misc', 'verbosity') > 1 else False))
    #     verbose=True)
    #
    # if wait:
    #     instance.wait_for_next_block(from_block=from_block,
    #         #verbose=(True if api_config.get('misc', 'verbosity') > 1 else False))
    #         verbose=True)

    chainHead = getBlockchainHead()
    expHead = int(bin_dbl_sha256(bhBinary[-80:])[::-1].encode('hex'), 16)

    if chainHead != expHead:
        print('@@@@@ MISMATCH chainHead: {0} expHead: {1}').format(
            blockHashHex(chainHead), blockHashHex(expHead))
        # sys.exit(1)

def getBlockchainHead():
    gas = 3000000
    gas_price = 1

    fun_name = 'getBlockchainHead'
    sig = ''
    data = []

    callResult = instance.call(to, fun_name=fun_name, sig=sig, data=data, gas=gas, gas_price=gas_price)
    chainHead = callResult[0] if len(callResult) else callResult
    return chainHead

def blockHashHex(number):
    hexHead = hex(number)[2:-1] # snip off the 0x and trailing L
    hexHead = '0'*(64-len(hexHead)) + hexHead
    return hexHead

if __name__ == '__main__':
    main()
<commit_msg>params for From and RelayContract addresses<commit_after>from datetime import datetime, date
from time import sleep
from argparse import ArgumentParser

from pyepm import api, config
from bitcoin import *

try:
    from urllib.request import build_opener
except:
    from urllib2 import build_opener

# Makes a request to a given URL (first arg) and optional params (second arg)
def make_request(*args):
    opener = build_opener()
    opener.addheaders = [('User-agent',
                          'Mozilla/5.0'+str(random.randrange(1000000)))]
    try:
        return opener.open(*args).read().strip()
    except Exception as e:
        try:
            p = e.read().strip()
        except:
            p = e
        raise Exception(p)

api_config = config.read_config()
instance = api.Api(api_config)

# instance.address = "0xcd2a3d9f938e13cd947ec05abc7fe734df8dd826"
# instance.relayContract = "0xba164d1e85526bd5e27fd15ad14b0eae91c45a93"

def main():
    parser = ArgumentParser()
    parser.add_argument('--fetch', action='store_true', help='fetch blockheaders')
    parser.add_argument('-s', '--sender', required=True, help='sender of transaction')
    parser.add_argument('-r', '--relay', required=True, help='relay contract address')
    args = parser.parse_args()

    instance.address = args.sender
    instance.relayContract = args.relay

    run(doFetch=args.fetch)

def run(doFetch=False):
    network = 'testnet'
    chainHead = blockHashHex(getBlockchainHead())

    blockInfoUrl = "https://tbtc.blockr.io/api/v1/block/info/"

    data = make_request(blockInfoUrl + chainHead)
    jsonobj = json.loads(data)
    contractHeight = jsonobj["data"]["nb"]

    actualHeight = last_block_height(network)

    print('@@@ heights: {0} {1}').format(contractHeight, actualHeight)

    print('@@@ chainHead: %s' % chainHead)

    chunkSize = 5
    fetchNum =  actualHeight - contractHeight
    numChunk = fetchNum / chunkSize
    leftoverToFetch = fetchNum % chunkSize

    print('@@@ numChunk: {0} leftoverToFetch: {1}').format(numChunk, leftoverToFetch)

    if doFetch:
        fetchHeaders(contractHeight+1, chunkSize, numChunk)
        fetchHeaders(actualHeight-leftoverToFetch+1, 1, leftoverToFetch)

        sleep(3)
        print('@@@ chainHead: %s' % chainHead)

def fetchHeaders(chunkStartNum, chunkSize, numChunk):
    for j in range(numChunk):
        strings = ""
        for i in range(chunkSize):
            blockNum = chunkStartNum  + i
            bhJson = blockr_get_block_header_data(blockNum, network='testnet')
            bhStr = serialize_header(bhJson)
            print("@@@ {0}: {1}").format(blockNum, bhStr)
            strings += bhStr

        storeHeaders(strings.decode('hex'), chunkSize)

        chainHead = getBlockchainHead()
        print('@@@ DONE hexHead: %s' % blockHashHex(chainHead))

        chunkStartNum += chunkSize

def storeHeaders(bhBinary, chunkSize):

    txCount = instance.transaction_count(defaultBlock='pending')
    print('----------------------------------')
    print('txCount: %s' % txCount)

    hashOne = blockHashHex(int(bin_dbl_sha256(bhBinary[:80])[::-1].encode('hex'), 16))
    hashLast = blockHashHex(int(bin_dbl_sha256(bhBinary[-80:])[::-1].encode('hex'), 16))
    print('hashOne: %s' % hashOne)
    print('hashLast: %s' % hashLast)

    firstH = bhBinary[:80].encode('hex')
    lastH = bhBinary[-80:].encode('hex')
    print('firstH: %s' % firstH)
    print('lastH: %s' % lastH)

    fun_name = "bulkStoreHeader"
    sig = "si"

    # bhBinary = '\x02\x00\x00\x00~\xf0U\xe1gM.eQ\xdb\xa4\x1c\xd2\x14\xde\xbb\xee4\xae\xb5D\xc7\xecg\x00\x00\x00\x00\x00\x00\x00\x00\xd3\x99\x89c\xf8\x0c[\xabC\xfe\x8c&"\x8e\x98\xd00\xed\xf4\xdc\xbeH\xa6f\xf5\xc3\x9e-z\x88\\\x91\x02\xc8mSl\x89\x00\x19Y:G\r\x02\x00\x00\x00Tr\xac\x8b\x11\x87\xbf\xcf\x91\xd6\xd2\x18\xbb\xda\x1e\xb2@]|U\xf1\xf8\xcc\x82\x00\x00\x00\x00\x00\x00\x00\x00\xab\n\xaa7|\xa3\xf4\x9b\x15E\xe2\xaek\x06g\xa0\x8fB\xe7-\x8c$\xae#q@\xe2\x8f\x14\xf3\xbb|k\xccmSl\x89\x00\x19\xed\xd8<\xcf\x02\x00\x00\x00\xa9\xab\x12\xe3,\xed\xdc+\xa5\xe6\xade\x1f\xacw,\x986\xdf\x83M\x91\xa0I\x00\x00\x00\x00\x00\x00\x00\x00\xdfuu\xc7\x8f\x83\x1f \xaf\x14~\xa7T\xe5\x84\xaa\xd9Yeiic-\xa9x\xd2\xddq\x86#\xfd0\xc5\xccmSl\x89\x00\x19\xe6Q\x07\xe9\x02\x00\x00\x00,P\x1f\xc0\xb0\xfd\xe9\xb3\xc1\x0e#S\xc1TI*5k\x1a\x02)^+\x86\x00\x00\x00\x00\x00\x00\x00\x00\xa7\xaaa\xc8\xd3|\x88v\xba\xa0\x17\x9ej2\x94D4\xbf\xd3\xe1\xccug\x89*1K\x0c{\x9e]\x92\'\xcemSl\x89\x00\x19\xa4\xa0<{\x02\x00\x00\x00\xe7\xfc\x91>+y\n0v\x0c\xaa\xfb\x9b_\xaa\xe1\xb5\x1dlT\xff\xe4\xae\x82\x00\x00\x00\x00\x00\x00\x00\x00P\xad\x11k\xfb\x11c\x03\x03a\xd9}H\xb4\xca\x90\'\xa4\x9b\xca\xf8\xb8\xd4!\x1b\xaa\x92\xccr\xe7\xe1#f\xcfmSl\x89\x00\x19\xe6\x13\x9c\x82'
    data = [bhBinary, chunkSize]

    gas = 3000000
    gas_price = 10000000000000  # if CPP, may need to make this 10szabo (10e12)
    value = 0

    #
    # store the headers
    #
    wait = True
    from_count = instance.transaction_count(defaultBlock='pending')
    if wait:
        from_block = instance.last_block()

    instance.transact(instance.relayContract, fun_name=fun_name, sig=sig, data=data, gas=gas, gas_price=gas_price, value=value)

    # instance.wait_for_transaction(
    #     from_count=from_count,
    #     #verbose=(True if api_config.get('misc', 'verbosity') > 1 else False))
    #     verbose=True)

    if wait:
        for i in range(1):
            instance.wait_for_next_block(from_block=from_block,
                #verbose=(True if api_config.get('misc', 'verbosity') > 1 else False))
                verbose=True)
            from_block = instance.last_block()

    chainHead = getBlockchainHead()
    expHead = int(bin_dbl_sha256(bhBinary[-80:])[::-1].encode('hex'), 16)

    if chainHead != expHead:
        print('@@@@@ MISMATCH chainHead: {0} expHead: {1}').format(
            blockHashHex(chainHead), blockHashHex(expHead))
        # sys.exit(1)

def getBlockchainHead():
    gas = 3000000
    gas_price = 1

    fun_name = 'getBlockchainHead'
    sig = ''
    data = []

    callResult = instance.call(instance.relayContract, fun_name=fun_name, sig=sig, data=data, gas=gas, gas_price=gas_price)
    chainHead = callResult[0] if len(callResult) else callResult
    return chainHead

def blockHashHex(number):
    hexHead = hex(number)[2:-1] # snip off the 0x and trailing L
    hexHead = '0'*(64-len(hexHead)) + hexHead
    return hexHead

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import re

from .utils import _perform_request, format_datetime, format_utc, markdown_table_row

conf = None

class IssueThread(object):
    '''
    Everything related to an issue in GitLab, e.g. the issue itself and subsequent comments.
    '''
    def __init__(self, config, fields):
        global conf
        conf = config
        self.load_objects(fields)

    def load_objects(self, fields):
        '''
        Load the issue object and the comment objects.
        If conf.dry_run=False, then Attachments are created in GitLab in this step.
        '''
        self.issue = Issue(fields)
        self.comments = []
        '''
        fields["long_desc"] gets peared down in Issue creation (above). This is because bugzilla
        lacks the concept of an issue description, so the first comment is harvested for
        the issue description, as well as any subsequent comments that are simply attachments
        from the original reporter. What remains below should be a list of genuine comments.
        '''

        for comment_fields in fields["long_desc"]:
            if comment_fields.get("thetext"):
                self.comments.append(Comment(comment_fields))

    def save(self):
        '''
        Save the issue and all of the comments to GitLab.
        If conf.dry_run=True, then only the HTTP request that would be made is printed.
        '''
        self.issue.save()

        for comment in self.comments:
            comment.issue_id = self.issue.id
            comment.save()

        # close the issue in GitLab, if it is resolved in Bugzilla
        if self.issue.status in conf.bugzilla_closed_states:
            self.issue.close()

class Issue(object):
    '''
    The issue model
    '''
    required_fields = ["sudo", "title", "description"]
    data_fields = ["sudo", "created_at", "title", "description", "assignee_ids", "milestone_id",
                   "labels"]

    def __init__(self, bugzilla_fields):
        self.headers = conf.default_headers
        validate_user(bugzilla_fields["reporter"])
        validate_user(bugzilla_fields["assigned_to"])
        self.load_fields(bugzilla_fields)

    def load_fields(self, fields):
        self.title = fields["short_desc"]
        self.sudo = conf.gitlab_users[conf.bugzilla_users[fields["reporter"]]]
        self.assignee_ids = [conf.gitlab_users[conf.bugzilla_users[fields["assigned_to"]]]]
        self.created_at = format_utc(fields["creation_ts"])
        self.status = fields["bug_status"]
        self.create_milestone(fields["target_milestone"])
        self.create_labels(fields["component"], fields.get("op_sys"))
        self.create_description(fields)

    def create_labels(self, component, operating_system):
        '''
        Creates 3 types of labels: default labels listed in the configuration, component labels,
        and operating system labels.
        '''
        labels = []
        if conf.default_gitlab_labels:
            labels.extend(conf.default_gitlab_labels)

        component_label = conf.component_mappings.get(component)
        if component_label:
            labels.append(component_label)

        # Do not create a label if the OS is other. That is a meaningless label.
        if operating_system and operating_system != "Other":
            labels.append(operating_system)

        self.labels = ",".join(labels)

    def create_milestone(self, title):
        '''
        Looks up milestone id given its title or creates a new one. Does nothing if milestones
        are not mapped.
        '''
        if not conf.map_milestones or title in conf.milestones_to_skip:
            return

        if title not in conf.gitlab_milestones:
            url = "{}/projects/{}/milestones".format(conf.gitlab_base_url, conf.gitlab_project_id)
            response = _perform_request(url, "post", headers=self.headers, data={"title": title})
            conf.gitlab_milestones[title] = response["id"]

        self.milestone_id = conf.gitlab_milestones[title]

    def create_description(self, fields):
        '''
        An opinionated description body creator.
        '''
        ext_description = ""

        # markdown table header
        self.description = markdown_table_row("", "")
        self.description += markdown_table_row("---", "---")

        if conf.include_bugzilla_link:
            bug_id = fields["bug_id"]
            link = "{}/show_bug.cgi?id={}".format(conf.bugzilla_base_url, bug_id)
            self.description += markdown_table_row("Bugzilla Link",
                                                   "[{}]({})".format(bug_id, link))

        formatted_dt = format_datetime(fields["creation_ts"], conf.datetime_format_string)
        self.description += markdown_table_row("Created on", formatted_dt)

        if fields.get("resolution"):
            self.description += markdown_table_row("Resolution", fields["resolution"])
            self.description += markdown_table_row("Resolved on",
                                                   format_datetime(fields["delta_ts"],
                                                                   conf.datetime_format_string))

        self.description += markdown_table_row("Version", fields.get("version"))
        self.description += markdown_table_row("OS", fields.get("op_sys"))
        self.description += markdown_table_row("Architecture", fields.get("rep_platform"))

        # add first comment to the issue description
        attachments = []
        to_delete = []
        comment0 = fields["long_desc"][0]
        if (fields["reporter"] == comment0["who"] and comment0["thetext"]):
            ext_description += "\n## Extended Description \n"
            ext_description += "\n\n".join(re.split("\n*", comment0["thetext"]))
            self.update_attachments(fields["reporter"], comment0, attachments)
            del fields["long_desc"][0]

        for i in range(0, len(fields["long_desc"])):
            comment = fields["long_desc"][i]
            if self.update_attachments(fields["reporter"], comment, attachments):
                to_delete.append(i)

        # delete comments that have already added to the issue description
        for i in reversed(to_delete):
            del fields["long_desc"][i]

        if attachments:
            self.description += markdown_table_row("Attachments", ", ".join(attachments))

        if ext_description:
            # for situations where the reporter is a generic or old user, specify the original
            # reporter in the description body
            if fields["reporter"] == conf.bugzilla_auto_reporter:
                # try to get reporter email from the body
                description, part, user_data = ext_description.rpartition("Submitter was ")
                # partition found matching string
                if part:
                    regex = r"^(\S*)\s?.*$"
                    email = re.match(regex, user_data, flags=re.M).group(1)
                    self.description += markdown_table_row("Reporter", email)
            # Add original reporter to the markdown table
            elif conf.bugzilla_users[fields["reporter"]] == conf.gitlab_misc_user:
                self.description += markdown_table_row("Reporter", fields["reporter"])

            self.description += ext_description

    def update_attachments(self, reporter, comment, attachments):
        '''
        Fetches attachments from comment if authored by reporter.
        '''
        if comment.get("attachid") and comment.get("who") == reporter:
            filename = Attachment.parse_file_description(comment.get("thetext"))
            attachment_markdown = Attachment(comment.get("attachid"), filename).save()
            attachments.append(attachment_markdown)
            return True
        return False

    def validate(self):
        for field in self.required_fields:
            value = getattr(self, field)
            if not value:
                raise Exception("Missing value for required field: {}".format(field))
        return True

    def save(self):
        self.validate()
        url = "{}/projects/{}/issues".format(conf.gitlab_base_url, conf.gitlab_project_id)
        data = {k: v for k, v in self.__dict__.items() if k in self.data_fields}
        self.headers["sudo"] = self.sudo

        response = _perform_request(url, "post", headers=self.headers, data=data, json=True,
                                    dry_run=conf.dry_run)

        if conf.dry_run:
            # assign a random number so that program can continue
            self.id = 5
            return

        self.id = response["iid"]

    def close(self):
        url = "{}/projects/{}/issues/{}".format(conf.gitlab_base_url, conf.gitlab_project_id,
                                                self.id)
        data = {
            "state_event": "close",
        }
        self.headers["sudo"] = self.sudo

        _perform_request(url, "put", headers=self.headers, data=data, dry_run=conf.dry_run)

class Comment(object):
    '''
    The comment model
    '''

    required_fields = ["sudo", "body", "issue_id"]
    data_fields = ["created_at", "body"]

    def __init__(self, bugzilla_fields):
        self.headers = conf.default_headers
        validate_user(bugzilla_fields["who"])
        self.load_fields(bugzilla_fields)

    def load_fields(self, fields):
        self.sudo = conf.gitlab_users[conf.bugzilla_users[fields["who"]]]
        # if unable to comment as the original user, put username in comment body
        self.created_at = format_utc(fields["bug_when"])
        if conf.bugzilla_users[fields["who"]] == conf.gitlab_misc_user:
            self.body = "By {} on {}\n\n".format(fields["who"],
                                                 format_datetime(fields["bug_when"],
                                                 conf.datetime_format_string))
        else:
            self.body = format_datetime(fields["bug_when"], conf.datetime_format_string)
            self.body += "\n\n"

        # if this comment is actually an attachment, upload the attachment and add the
        # markdown to the comment body
        if fields.get("attachid"):
            filename = Attachment.parse_file_description(fields["thetext"])
            attachment_markdown = Attachment(fields["attachid"], filename).save()
            self.body += attachment_markdown
        else:
            self.body += fields["thetext"]

    def validate(self):
        for field in self.required_fields:
            value = getattr(self, field)
            if not value:
                raise Exception("Missing value for required field: {}".format(field))

    def save(self):
        self.validate()
        self.headers["sudo"] = self.sudo
        url = "{}/projects/{}/issues/{}/notes".format(conf.gitlab_base_url, conf.gitlab_project_id,
                                                      self.issue_id)
        data = {k: v for k, v in self.__dict__.items() if k in self.data_fields}

        _perform_request(url, "post", headers=self.headers, data=data, json=True,
                         dry_run=conf.dry_run)

class Attachment(object):
    '''
    The attachment model
    '''
    def __init__(self, bugzilla_attachment_id, file_description):
        self.id = bugzilla_attachment_id
        self.file_description = file_description
        self.headers = conf.default_headers

    @classmethod
    def parse_file_description(cls, comment):
        regex = "^Created attachment (\d*)\s?(.*)$"
        matches = re.match(regex, comment, flags=re.M)
        if not matches:
            raise Exception("Failed to match comment string: {}".format(comment))
        return matches.group(2)

    def parse_file_name(self, headers):
        # Use real filename to store attachment but descriptive name for issue text
        if 'Content-disposition' not in headers:
            raise Exception(u"No file name returned for attachment {}"
                            .format(self.file_description))
        # Content-disposition: application/zip; filename="mail_route.zip"
        regex = "^.*; filename=\"(.*)\"$"
        matches = re.match(regex, headers['Content-disposition'], flags=re.M)
        if not matches:
            raise Exception("Failed to match file name for string: {}"
                            .format(headers['Content-disposition']))
        return matches.group(1)

    def parse_upload_link(self, attachment):
        if not (attachment and attachment["markdown"]):
            raise Exception(u"No markdown returned for upload of attachment {}"
                            .format(self.file_description))
        # ![mail_route.zip](/uploads/e943e69eb2478529f2f1c7c7ea00fb46/mail_route.zip)
        regex = "^!?\[.*\]\((.*)\)$"
        matches = re.match(regex, attachment["markdown"], flags=re.M)
        if not matches:
            raise Exception("Failed to match upload link for string: {}"
                            .format(attachment["markdown"]))
        return matches.group(1)

    def save(self):
        url = "{}/attachment.cgi?id={}".format(conf.bugzilla_base_url, self.id)
        result = _perform_request(url, "get", json=False)
        filename = self.parse_file_name(result.headers)

        url = "{}/projects/{}/uploads".format(conf.gitlab_base_url, conf.gitlab_project_id)
        f = {"file": (filename, result.content)}
        attachment = _perform_request(url, "post", headers=self.headers, files=f, json=True,
                                      dry_run=conf.dry_run)
        # For dry run, nothing is uploaded, so upload link is faked just to let the process continue
        upload_link = self.file_description if conf.dry_run else self.parse_upload_link(attachment)

        return u"[{}]({})".format(self.file_description, upload_link)

def validate_user(bugzilla_user):
    if bugzilla_user not in conf.bugzilla_users:
        raise Exception("Bugzilla user `{}` not found in user_mappings.yml. "
                        "Please add them before continuing.".format(bugzilla_user))
<commit_msg>check for the need to map a milestone outside of a mapper method<commit_after>import re

from .utils import _perform_request, format_datetime, format_utc, markdown_table_row

conf = None

class IssueThread(object):
    '''
    Everything related to an issue in GitLab, e.g. the issue itself and subsequent comments.
    '''
    def __init__(self, config, fields):
        global conf
        conf = config
        self.load_objects(fields)

    def load_objects(self, fields):
        '''
        Load the issue object and the comment objects.
        If conf.dry_run=False, then Attachments are created in GitLab in this step.
        '''
        self.issue = Issue(fields)
        self.comments = []
        '''
        fields["long_desc"] gets peared down in Issue creation (above). This is because bugzilla
        lacks the concept of an issue description, so the first comment is harvested for
        the issue description, as well as any subsequent comments that are simply attachments
        from the original reporter. What remains below should be a list of genuine comments.
        '''

        for comment_fields in fields["long_desc"]:
            if comment_fields.get("thetext"):
                self.comments.append(Comment(comment_fields))

    def save(self):
        '''
        Save the issue and all of the comments to GitLab.
        If conf.dry_run=True, then only the HTTP request that would be made is printed.
        '''
        self.issue.save()

        for comment in self.comments:
            comment.issue_id = self.issue.id
            comment.save()

        # close the issue in GitLab, if it is resolved in Bugzilla
        if self.issue.status in conf.bugzilla_closed_states:
            self.issue.close()

class Issue(object):
    '''
    The issue model
    '''
    required_fields = ["sudo", "title", "description"]
    data_fields = ["sudo", "created_at", "title", "description", "assignee_ids", "milestone_id",
                   "labels"]

    def __init__(self, bugzilla_fields):
        self.headers = conf.default_headers
        validate_user(bugzilla_fields["reporter"])
        validate_user(bugzilla_fields["assigned_to"])
        self.load_fields(bugzilla_fields)

    def load_fields(self, fields):
        self.title = fields["short_desc"]
        self.sudo = conf.gitlab_users[conf.bugzilla_users[fields["reporter"]]]
        self.assignee_ids = [conf.gitlab_users[conf.bugzilla_users[fields["assigned_to"]]]]
        self.created_at = format_utc(fields["creation_ts"])
        self.status = fields["bug_status"]
        milestone = fields["target_milestone"]
        if conf.map_milestones and milestone not in conf.milestones_to_skip:
            self.create_milestone(milestone)
        self.create_labels(fields["component"], fields.get("op_sys"))
        self.create_description(fields)

    def create_labels(self, component, operating_system):
        '''
        Creates 3 types of labels: default labels listed in the configuration, component labels,
        and operating system labels.
        '''
        labels = []
        if conf.default_gitlab_labels:
            labels.extend(conf.default_gitlab_labels)

        component_label = conf.component_mappings.get(component)
        if component_label:
            labels.append(component_label)

        # Do not create a label if the OS is other. That is a meaningless label.
        if operating_system and operating_system != "Other":
            labels.append(operating_system)

        self.labels = ",".join(labels)

    def create_milestone(self, milestone):
        '''
        Looks up milestone id given its title or creates a new one.
        '''
        if milestone not in conf.gitlab_milestones:
            url = "{}/projects/{}/milestones".format(conf.gitlab_base_url, conf.gitlab_project_id)
            response = _perform_request(
                url, "post", headers=self.headers, data={"title": milestone})
            conf.gitlab_milestones[milestone] = response["id"]

        self.milestone_id = conf.gitlab_milestones[milestone]

    def create_description(self, fields):
        '''
        An opinionated description body creator.
        '''
        ext_description = ""

        # markdown table header
        self.description = markdown_table_row("", "")
        self.description += markdown_table_row("---", "---")

        if conf.include_bugzilla_link:
            bug_id = fields["bug_id"]
            link = "{}/show_bug.cgi?id={}".format(conf.bugzilla_base_url, bug_id)
            self.description += markdown_table_row("Bugzilla Link",
                                                   "[{}]({})".format(bug_id, link))

        formatted_dt = format_datetime(fields["creation_ts"], conf.datetime_format_string)
        self.description += markdown_table_row("Created on", formatted_dt)

        if fields.get("resolution"):
            self.description += markdown_table_row("Resolution", fields["resolution"])
            self.description += markdown_table_row("Resolved on",
                                                   format_datetime(fields["delta_ts"],
                                                                   conf.datetime_format_string))

        self.description += markdown_table_row("Version", fields.get("version"))
        self.description += markdown_table_row("OS", fields.get("op_sys"))
        self.description += markdown_table_row("Architecture", fields.get("rep_platform"))

        # add first comment to the issue description
        attachments = []
        to_delete = []
        comment0 = fields["long_desc"][0]
        if (fields["reporter"] == comment0["who"] and comment0["thetext"]):
            ext_description += "\n## Extended Description \n"
            ext_description += "\n\n".join(re.split("\n*", comment0["thetext"]))
            self.update_attachments(fields["reporter"], comment0, attachments)
            del fields["long_desc"][0]

        for i in range(0, len(fields["long_desc"])):
            comment = fields["long_desc"][i]
            if self.update_attachments(fields["reporter"], comment, attachments):
                to_delete.append(i)

        # delete comments that have already added to the issue description
        for i in reversed(to_delete):
            del fields["long_desc"][i]

        if attachments:
            self.description += markdown_table_row("Attachments", ", ".join(attachments))

        if ext_description:
            # for situations where the reporter is a generic or old user, specify the original
            # reporter in the description body
            if fields["reporter"] == conf.bugzilla_auto_reporter:
                # try to get reporter email from the body
                description, part, user_data = ext_description.rpartition("Submitter was ")
                # partition found matching string
                if part:
                    regex = r"^(\S*)\s?.*$"
                    email = re.match(regex, user_data, flags=re.M).group(1)
                    self.description += markdown_table_row("Reporter", email)
            # Add original reporter to the markdown table
            elif conf.bugzilla_users[fields["reporter"]] == conf.gitlab_misc_user:
                self.description += markdown_table_row("Reporter", fields["reporter"])

            self.description += ext_description

    def update_attachments(self, reporter, comment, attachments):
        '''
        Fetches attachments from comment if authored by reporter.
        '''
        if comment.get("attachid") and comment.get("who") == reporter:
            filename = Attachment.parse_file_description(comment.get("thetext"))
            attachment_markdown = Attachment(comment.get("attachid"), filename).save()
            attachments.append(attachment_markdown)
            return True
        return False

    def validate(self):
        for field in self.required_fields:
            value = getattr(self, field)
            if not value:
                raise Exception("Missing value for required field: {}".format(field))
        return True

    def save(self):
        self.validate()
        url = "{}/projects/{}/issues".format(conf.gitlab_base_url, conf.gitlab_project_id)
        data = {k: v for k, v in self.__dict__.items() if k in self.data_fields}
        self.headers["sudo"] = self.sudo

        response = _perform_request(url, "post", headers=self.headers, data=data, json=True,
                                    dry_run=conf.dry_run)

        if conf.dry_run:
            # assign a random number so that program can continue
            self.id = 5
            return

        self.id = response["iid"]

    def close(self):
        url = "{}/projects/{}/issues/{}".format(conf.gitlab_base_url, conf.gitlab_project_id,
                                                self.id)
        data = {
            "state_event": "close",
        }
        self.headers["sudo"] = self.sudo

        _perform_request(url, "put", headers=self.headers, data=data, dry_run=conf.dry_run)

class Comment(object):
    '''
    The comment model
    '''

    required_fields = ["sudo", "body", "issue_id"]
    data_fields = ["created_at", "body"]

    def __init__(self, bugzilla_fields):
        self.headers = conf.default_headers
        validate_user(bugzilla_fields["who"])
        self.load_fields(bugzilla_fields)

    def load_fields(self, fields):
        self.sudo = conf.gitlab_users[conf.bugzilla_users[fields["who"]]]
        # if unable to comment as the original user, put username in comment body
        self.created_at = format_utc(fields["bug_when"])
        if conf.bugzilla_users[fields["who"]] == conf.gitlab_misc_user:
            self.body = "By {} on {}\n\n".format(fields["who"],
                                                 format_datetime(fields["bug_when"],
                                                 conf.datetime_format_string))
        else:
            self.body = format_datetime(fields["bug_when"], conf.datetime_format_string)
            self.body += "\n\n"

        # if this comment is actually an attachment, upload the attachment and add the
        # markdown to the comment body
        if fields.get("attachid"):
            filename = Attachment.parse_file_description(fields["thetext"])
            attachment_markdown = Attachment(fields["attachid"], filename).save()
            self.body += attachment_markdown
        else:
            self.body += fields["thetext"]

    def validate(self):
        for field in self.required_fields:
            value = getattr(self, field)
            if not value:
                raise Exception("Missing value for required field: {}".format(field))

    def save(self):
        self.validate()
        self.headers["sudo"] = self.sudo
        url = "{}/projects/{}/issues/{}/notes".format(conf.gitlab_base_url, conf.gitlab_project_id,
                                                      self.issue_id)
        data = {k: v for k, v in self.__dict__.items() if k in self.data_fields}

        _perform_request(url, "post", headers=self.headers, data=data, json=True,
                         dry_run=conf.dry_run)

class Attachment(object):
    '''
    The attachment model
    '''
    def __init__(self, bugzilla_attachment_id, file_description):
        self.id = bugzilla_attachment_id
        self.file_description = file_description
        self.headers = conf.default_headers

    @classmethod
    def parse_file_description(cls, comment):
        regex = "^Created attachment (\d*)\s?(.*)$"
        matches = re.match(regex, comment, flags=re.M)
        if not matches:
            raise Exception("Failed to match comment string: {}".format(comment))
        return matches.group(2)

    def parse_file_name(self, headers):
        # Use real filename to store attachment but descriptive name for issue text
        if 'Content-disposition' not in headers:
            raise Exception(u"No file name returned for attachment {}"
                            .format(self.file_description))
        # Content-disposition: application/zip; filename="mail_route.zip"
        regex = "^.*; filename=\"(.*)\"$"
        matches = re.match(regex, headers['Content-disposition'], flags=re.M)
        if not matches:
            raise Exception("Failed to match file name for string: {}"
                            .format(headers['Content-disposition']))
        return matches.group(1)

    def parse_upload_link(self, attachment):
        if not (attachment and attachment["markdown"]):
            raise Exception(u"No markdown returned for upload of attachment {}"
                            .format(self.file_description))
        # ![mail_route.zip](/uploads/e943e69eb2478529f2f1c7c7ea00fb46/mail_route.zip)
        regex = "^!?\[.*\]\((.*)\)$"
        matches = re.match(regex, attachment["markdown"], flags=re.M)
        if not matches:
            raise Exception("Failed to match upload link for string: {}"
                            .format(attachment["markdown"]))
        return matches.group(1)

    def save(self):
        url = "{}/attachment.cgi?id={}".format(conf.bugzilla_base_url, self.id)
        result = _perform_request(url, "get", json=False)
        filename = self.parse_file_name(result.headers)

        url = "{}/projects/{}/uploads".format(conf.gitlab_base_url, conf.gitlab_project_id)
        f = {"file": (filename, result.content)}
        attachment = _perform_request(url, "post", headers=self.headers, files=f, json=True,
                                      dry_run=conf.dry_run)
        # For dry run, nothing is uploaded, so upload link is faked just to let the process continue
        upload_link = self.file_description if conf.dry_run else self.parse_upload_link(attachment)

        return u"[{}]({})".format(self.file_description, upload_link)

def validate_user(bugzilla_user):
    if bugzilla_user not in conf.bugzilla_users:
        raise Exception("Bugzilla user `{}` not found in user_mappings.yml. "
                        "Please add them before continuing.".format(bugzilla_user))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#  licence AGPL version 3 or later
#  see licence in __openerp__.py or http://www.gnu.org/licenses/agpl-3.0.txt
#  Copyright (C) 2016 Akretion (https://www.akretion.com).
#  @author Raphael Reverdy <raphael.reverdy@akretion.com>
#
##############################################################################
from openerp import models

DUMMY_CARRIER_TYPE = 'dummy'

class StockPicking(models.Model):
    _inherit = 'stock.picking'

    def _dummy_is_our(self):
        return self.carrier_id.type == DUMMY_CARRIER_TYPE

    def _dummy_before_call(self, package_id, request):
        request['parcel']['reference'] = (
            "%s/%s" % (package_id, len(self._get_packages_from_picking()))
        )
        return request

    def _dummy_after_call(self, package_id, response):
        return {
            "data": response['zpl'],
            "tracking_id": "",
            "name": package_id.name,
        }
<commit_msg>Remove useless method<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#  licence AGPL version 3 or later
#  see licence in __openerp__.py or http://www.gnu.org/licenses/agpl-3.0.txt
#  Copyright (C) 2016 Akretion (https://www.akretion.com).
#  @author Raphael Reverdy <raphael.reverdy@akretion.com>
#
##############################################################################
from openerp import models

class StockPicking(models.Model):
    _inherit = 'stock.picking'

    def _dummy_before_call(self, package_id, request):
        request['parcel']['reference'] = (
            "%s/%s" % (package_id, len(self._get_packages_from_picking()))
        )
        return request

    def _dummy_after_call(self, package_id, response):
        return {
            "data": response['zpl'],
            "tracking_id": "",
            "name": package_id.name,
        }
<|endoftext|>


<commit_before>#BEGIN_HEADER
import os
import sys
import requests
import uuid
import numpy as np
from pprint import pformat

from Bio import SeqIO

from ReadsUtils.ReadsUtilsClient import ReadsUtils  # @IgnorePep8
from ReadsUtils.baseclient import ServerError
from ReadsAPI.ReadsAPIClient import ReadsAPI  # @IgnorePep8
from AssemblyUtil.AssemblyUtilClient import AssemblyUtil
from KBaseReport.KBaseReportClient import KBaseReport
from KBaseReport.baseclient import ServerError as _RepError
from kb_quast.kb_quastClient import kb_quast
from kb_quast.baseclient import ServerError as QUASTError
#from kb_ea_utils.kb_ea_utilsClient import kb_ea_utils

import requests.packages.urllib3
requests.packages.urllib3.disable_warnings()
#END_HEADER

class hipmer:
    '''
    Module Name:
    hipmer

    Module Description:
    A KBase module: hipmer
    '''

    # WARNING FOR GEVENT USERS #######
    # Since asynchronous IO can lead to methods - even the same method -
    # interrupting each other, you must be *very* careful when using global
    # state. A method could easily clobber the state set by another while
    # the latter method is running.
    #########################################
    VERSION = "0.0.1"
    GIT_URL = "git@github.com:kbaseapps/kbase_hipmer.git"
    GIT_COMMIT_HASH = "cf9790a0f15a0f8a08a6e1a654af969bd072cb5a"

    #BEGIN_CLASS_HEADER
    workspaceURL = None

    # target is a list for collecting log messages
    def log(self, target, message):
        # we should do something better here...
        if target is not None:
            target.append(message)
        print(message)
        sys.stdout.flush()

    def check_reads(self, ctx, refs, console):
        # Hipmer requires some parameters to be set for the reads.
        # Let's check those first before wasting time with downloads.
        rapi = ReadsAPI(self.callbackURL, token=ctx['token'],
                        service_ver='dev')

        for ref in refs:
            p = {'workspace_obj_ref': ref}
            info = rapi.get_reads_info_all_formatted(p)
            if info['Insert_Size_Mean'] == 'Not Specified' or \
               info['Insert_Size_Std_Dev'] == 'Not Specified':
                return False

        return True

    def get_reads_RU(self, ctx, refs, console):
        readcli = ReadsUtils(self.callbackURL, token=ctx['token'],
                             service_ver='dev')

        typeerr = ('Supported types: KBaseFile.SingleEndLibrary ' +
                   'KBaseFile.PairedEndLibrary ' +
                   'KBaseAssembly.SingleEndLibrary ' +
                   'KBaseAssembly.PairedEndLibrary')
        try:
            reads = readcli.download_reads({'read_libraries': refs,
                                            'interleaved': 'true',
                                            'gzipped': None
                                            })['files']
        except ServerError as se:
            self.log(console, 'logging stacktrace from dynamic client error')
            self.log(console, se.data)
            if typeerr in se.message:
                prefix = se.message.split('.')[0]
                raise ValueError(
                    prefix + '. Only the types ' +
                    'KBaseAssembly.PairedEndLibrary ' +
                    'and KBaseFile.PairedEndLibrary are supported')
            else:
                raise

        self.log(console, 'Got reads data from converter:\n' + pformat(reads))
        return reads

    def generate_config(self, params):
        """
        Generate the HipMer config
        """
        self.config_file = '%s/%s' % (self.scratch, 'hipmer.config')
        with open(self.config_file, 'w') as f:
            # Describe the libraries ( one line per library )
            # lib_seq [ wildcard ][ prefix ][ insAvg ][ insSdev ][ avgReadLen ]
            #         [ hasInnieArtifact ][ isRevComped ][ useForContigging ]
            #         [ onoSetId ][ useForGapClosing ][ 5pWiggleRoom ]
            #         [3pWiggleRoom] [FilesPerPair] [ useForSplinting ]
            #
            # TODO: make these params
            #
            # FilesPerPair = 2
            fmt = 'lib_seq %s %s %d %d   %d %d %d   %d %d %d  %d %d %d %d\n'
            wd = '/kb/module/work/tmp'
            total_bases = 0
            for r in params['reads']:
                # TODO: check read type and set count
                files_obj = params['readsfiles'][r['ref']]['files']
                filelist = [files_obj['fwd'].replace(wd, '.')]
                if 'rev' in files_obj and files_obj['rev'] is not None:
                    rfile = files_obj['rev'].replace(wd, '.')
                    filelist.append(rfile)
                reads_obj = params['readsfiles'][r['ref']]
                r['ins_avg'] = int(reads_obj['insert_size_mean'])
                r['ins_dev'] = int(reads_obj['insert_size_std_dev'])
                r['avg_read_len'] = int(reads_obj['read_length_mean'])
                r['is_rev_comped'] = 0
                if 'read_orientation_outward' in reads_obj and \
                        reads_obj['read_orientation_outward'][0].lower() == 't':
                    r['is_rev_comped'] = 1

                count = len(filelist)
                files = ','.join(filelist)
                # lib_seq small.forward.fq,small.reverse.fq   small  215  10   \
                #    101 0 0      1 1 1  0 0 2 1
                f.write(fmt % (
                    files, r['prefix'], r['ins_avg'], r['ins_dev'],
                    r['avg_read_len'], r['has_innie_artifact'],
                    r['is_rev_comped'], r['use_for_contigging'],
                    r['ono_set_id'], r['use_for_gap_closing'],
                    r['fp_wiggle_room'], r['tp_wiggle_room'],
                    count, r['use_for_splinting']))
                total_bases += reads_obj['total_bases']
            f.write('\n')
            paramf = {
                'is_diploid': 'is_diploid %d\n',
                'dynamic_min_depth': 'dynamic_min_depth %f\n',
                'mer_size': 'mer_size %d\n',
                'min_depth_cutoff': 'min_depth_cutoff %d\n',
                'gap_close_rpt_depth_ratio': 'gap_close_rpt_depth_ratio %f\n',
                'assm_scaff_len_cutoff': 'assm_scaff_len_cutoff %d\n'
            }
            if params['is_diploid'] == 1:
                paramf['bubble_min_depth_cutoff'] = 'bubble_min_depth_cutoff %d'
            for param in paramf:
                f.write(paramf[param] % (params[param]))
            f.close()

        return total_bases

    def generate_submit(self, tsize):
        """
        Generate SLURM submit script
        """
        bpn = 1000000000
        nodes = int((tsize + bpn - 1) / bpn)
        # It seems like hipmer fails with odd numbers of nodes
        # So let's add one if it is odd.
        if nodes % 2:
            nodes += 1
        self.submit = '%s/%s' % (self.scratch, 'slurm.submit')
        with open(self.submit, 'w') as f:
            f.write('#!/bin/bash\n')
            f.write('#SBATCH --partition=debug\n')
            f.write('#SBATCH --nodes=%d\n' % nodes)
            f.write('#SBATCH --ntasks-per-node=32\n')
            f.write('#SBATCH --time=00:30:00\n')
            f.write('#SBATCH --job-name=HipMer\n')
            f.write('export CORES_PER_NODE=${CORES_PER_NODE:=${SLURM_TASKS_PER_NODE%%\(*}}\n')
            f.write('N=${N:=${SLURM_NTASKS}}\n')
            f.write('HIPMER_INSTALL=${HIPMER_INSTALL:=${SCRATCH}/hipmer-install-cori}\n')
            f.write('INST=${HIPMER_INSTALL:=$1}\n')
            f.write('. $INST/env.sh\n')
            f.write('\n')
            f.write('export RUNDIR=${RUNDIR:=$(pwd)}\n')
            f.write('${INST}/bin/run_hipmer.sh ${RUNDIR}/hipmer.config\n')
            f.close()

    def save_assembly(self, wsname, output_contigs, token, name, console):
        self.log(console, 'Uploading FASTA file to Assembly')
        assemblyUtil = AssemblyUtil(self.callbackURL, token=token,
                                    service_ver='dev')
        assemblyUtil.save_assembly_from_fasta({'file': {'path': output_contigs},
                                               'workspace_name': wsname,
                                               'assembly_name': name
                                               })
    #END_CLASS_HEADER

    # config contains contents of config file in a hash or None if it couldn't
    # be found
    def __init__(self, config):
        #BEGIN_CONSTRUCTOR
        self.workspaceURL = config['workspace-url']
        self.scratch = os.path.abspath(config['scratch'])
        self.callbackURL = os.environ.get('SDK_CALLBACK_URL')
        print "Callback=%s" % (self.callbackURL)
        #END_CONSTRUCTOR
        pass

    def run_hipmer_hpc(self, ctx, params):
        """
        :param params: instance of type "AssemblyParams" (Run assembler
           workspace_name - the name of the workspace for input/output
           read_library_name - the name of the PE read library (SE library
           support in the future) output_contig_set_name - the name of the
           output contigset extra_params - assembler specific parameters
           min_contig_length - minimum length of contigs to output, default
           200 @optional min_contig_len @optional extra_params) -> structure:
           parameter "workspace_name" of String, parameter
           "read_library_name" of String, parameter "output_contigset_name"
           of String, parameter "min_contig_len" of Long, parameter
           "extra_params" of list of String
        :returns: instance of type "AssemblyOutput" -> structure: parameter
           "report_name" of String, parameter "report_ref" of String
        """
        # ctx is the context object
        # return variables are: output
        #BEGIN run_hipmer_hpc
        console = []
        self.log(console, 'Running run_hipmer_hpc with params=')
        self.log(console, pformat(params))

        # do some basic checks
        if 'workspace_name' not in params:
            raise ValueError('workspace_name parameter is required')
        if 'reads' not in params:
            raise ValueError('reads parameter is required')
        if 'output_contigset_name' not in params:
            raise ValueError('output_contigset_name parameter is required')
        ws_name = params['workspace_name']
        #ws = workspaceService(self.workspaceURL, token=ctx['token'])

        if 'POST' not in os.environ:
            # Get the read library
            print "Running pre stage"
            refs = []
            for read in params['reads']:
                read_name = read['read_library_name']
                if '/' in read_name:
                    ref = read_name
                else:
                    ref = ws_name + '/' + read_name
                refs.append(ref)
                read['ref'] = ref
            if not self.check_reads(ctx, refs, console):
                sys.exit(1)

            params['readsfiles'] = self.get_reads_RU(ctx, refs, console)

            # set the output location
            output_dir = self.scratch
            # Generate submit script
            ts = self.generate_config(params)
            self.generate_submit(ts)
            return

        print "Running POST stage"

        # run hipmer, capture output as it happens
        self.log(console, 'running hipmer:')

        output_dir = self.scratch
        output_contigs = os.path.join(output_dir, 'final_assembly.fa')
        output_name = params['output_contigset_name']
        wsname = params['workspace_name']
        #output_data_ref = self.save_assembly(wsname,
        #                                     output_contigs,
        #                                     ctx['token'],
        #                                     output_name,
        #                                     console)
        self.log(console, 'Uploading FASTA file to Assembly')
        assemblyUtil = AssemblyUtil(self.callbackURL, token=ctx['token'],
                                    service_ver='dev')
        save_input = {'file': {'path': output_contigs},
                      'workspace_name': wsname,
                      'assembly_name': output_name
                      }
        output_data_ref = assemblyUtil.save_assembly_from_fasta(save_input)
        print 'ref: ' + output_data_ref
        # create a Report
        # compute a simple contig length distribution for the report
        lengths = []
        for seq_record in SeqIO.parse(output_contigs, 'fasta'):
            lengths.append(len(seq_record.seq))

        report = ''
        report += 'ContigSet saved to: ' + params['workspace_name'] + '/'
        report += params['output_contigset_name'] + '\n'
        report += 'Assembled into ' + str(len(lengths)) + ' contigs.\n'
        report += 'Avg Length: ' + str(sum(lengths) / float(len(lengths))) + ' bp.\n'

        bins = 10
        counts, edges = np.histogram(lengths, bins)
        report += 'Contig Length Distribution (# of contigs -- min to max basepairs):\n'
        for c in range(bins):
            report += '   \%d\t--\t%d' % (counts[c], edges[c])
            report += ' to %d bp\n' % (edges[c + 1])

        print('Running QUAST')
        kbq = kb_quast(self.callbackURL)
        try:
            quastret = kbq.run_QUAST({'files': [{'path': output_contigs,
                                                 'label': params['output_contigset_name']}]})
        except QUASTError as qe:
            # not really any way to test this, all inputs have been checked
            # earlier and should be ok
            print('Logging exception from running QUAST')
            print(str(qe))
            # TODO delete shock node
            raise

        print('Saving report')
        kbr = KBaseReport(self.callbackURL)
        try:
            report_info = kbr.create_extended_report(
                {'message': report,
                 'objects_created': [{'ref': output_data_ref,
                                      'description': 'Assembled contigs'}],
                 'direct_html_link_index': 0,
                 'html_links': [{'shock_id': quastret['shock_id'],
                                 'name': 'report.html',
                                 'label': 'QUAST report'}
                                ],
                 'report_object_name': 'kb_megahit_report_' + str(uuid.uuid4()),
                 'workspace_name': params['workspace_name']
                 })
        except _RepError as re:
            # not really any way to test this, all inputs have been checked earlier and should be
            # ok
            print('Logging exception from creating report object')
            print(str(re))
            # TODO delete shock node
            raise

        # STEP 6: contruct the output to send back
        output = {'report_name': report_info['name'],
                  'report_ref': report_info['ref']
                  }
        return [output]
        #END run_hipmer_hpc

        # At some point might do deeper type checking...
        if not isinstance(output, dict):
            raise ValueError('Method run_hipmer_hpc return value ' +
                             'output is not type dict as required.')
        # return the results
        return [output]

    def status(self, ctx):
        #BEGIN_STATUS
        returnVal = {'state': "OK", 'message': "", 'version': self.VERSION,
                     'git_url': self.GIT_URL, 'git_commit_hash': self.GIT_COMMIT_HASH}
        #END_STATUS
        return [returnVal]
<commit_msg>Handle UI changes<commit_after>#BEGIN_HEADER
import os
import sys
import requests
import uuid
import numpy as np
from pprint import pformat

from Bio import SeqIO

from ReadsUtils.ReadsUtilsClient import ReadsUtils  # @IgnorePep8
from ReadsUtils.baseclient import ServerError
from ReadsAPI.ReadsAPIClient import ReadsAPI  # @IgnorePep8
from AssemblyUtil.AssemblyUtilClient import AssemblyUtil
from KBaseReport.KBaseReportClient import KBaseReport
from KBaseReport.baseclient import ServerError as _RepError
from kb_quast.kb_quastClient import kb_quast
from kb_quast.baseclient import ServerError as QUASTError
#from kb_ea_utils.kb_ea_utilsClient import kb_ea_utils

import requests.packages.urllib3
requests.packages.urllib3.disable_warnings()
#END_HEADER

class hipmer:
    '''
    Module Name:
    hipmer

    Module Description:
    A KBase module: hipmer
    '''

    # WARNING FOR GEVENT USERS #######
    # Since asynchronous IO can lead to methods - even the same method -
    # interrupting each other, you must be *very* careful when using global
    # state. A method could easily clobber the state set by another while
    # the latter method is running.
    #########################################
    VERSION = "0.0.1"
    GIT_URL = "git@github.com:kbaseapps/kbase_hipmer.git"
    GIT_COMMIT_HASH = "cf9790a0f15a0f8a08a6e1a654af969bd072cb5a"

    #BEGIN_CLASS_HEADER
    workspaceURL = None

    # target is a list for collecting log messages
    def log(self, target, message):
        # we should do something better here...
        if target is not None:
            target.append(message)
        print(message)
        sys.stdout.flush()

    def check_reads(self, ctx, refs, console):
        # Hipmer requires some parameters to be set for the reads.
        # Let's check those first before wasting time with downloads.
        rapi = ReadsAPI(self.callbackURL, token=ctx['token'],
                        service_ver='dev')

        for ref in refs:
            p = {'workspace_obj_ref': ref}
            info = rapi.get_reads_info_all_formatted(p)
            if info['Insert_Size_Mean'] == 'Not Specified' or \
               info['Insert_Size_Std_Dev'] == 'Not Specified':
                return False

        return True

    def get_reads_RU(self, ctx, refs, console):
        readcli = ReadsUtils(self.callbackURL, token=ctx['token'],
                             service_ver='dev')

        typeerr = ('Supported types: KBaseFile.SingleEndLibrary ' +
                   'KBaseFile.PairedEndLibrary ' +
                   'KBaseAssembly.SingleEndLibrary ' +
                   'KBaseAssembly.PairedEndLibrary')
        try:
            reads = readcli.download_reads({'read_libraries': refs,
                                            'interleaved': 'true',
                                            'gzipped': None
                                            })['files']
        except ServerError as se:
            self.log(console, 'logging stacktrace from dynamic client error')
            self.log(console, se.data)
            if typeerr in se.message:
                prefix = se.message.split('.')[0]
                raise ValueError(
                    prefix + '. Only the types ' +
                    'KBaseAssembly.PairedEndLibrary ' +
                    'and KBaseFile.PairedEndLibrary are supported')
            else:
                raise

        self.log(console, 'Got reads data from converter:\n' + pformat(reads))
        return reads

    def generate_config(self, params):
        """
        Generate the HipMer config
        """
        self.config_file = '%s/%s' % (self.scratch, 'hipmer.config')
        with open(self.config_file, 'w') as f:
            # Describe the libraries ( one line per library )
            # lib_seq [ wildcard ][ prefix ][ insAvg ][ insSdev ][ avgReadLen ]
            #         [ hasInnieArtifact ][ isRevComped ][ useForContigging ]
            #         [ onoSetId ][ useForGapClosing ][ 5pWiggleRoom ]
            #         [3pWiggleRoom] [FilesPerPair] [ useForSplinting ]
            #
            # TODO: make these params
            #
            # FilesPerPair = 2
            fmt = 'lib_seq %s %s %d %d   %d %d %d   %d %d %d  %d %d %d %d\n'
            wd = '/kb/module/work/tmp'
            total_bases = 0
            for r in params['reads']:
                # TODO: check read type and set count
                files_obj = params['readsfiles'][r['ref']]['files']
                filelist = [files_obj['fwd'].replace(wd, '.')]
                if 'rev' in files_obj and files_obj['rev'] is not None:
                    rfile = files_obj['rev'].replace(wd, '.')
                    filelist.append(rfile)
                reads_obj = params['readsfiles'][r['ref']]
                r['ins_avg'] = int(reads_obj['insert_size_mean'])
                r['ins_dev'] = int(reads_obj['insert_size_std_dev'])
                r['avg_read_len'] = int(reads_obj['read_length_mean'])
                r['is_rev_comped'] = 0
                if 'read_orientation_outward' in reads_obj and \
                        reads_obj['read_orientation_outward'][0].lower() == 't':
                    r['is_rev_comped'] = 1

                count = len(filelist)
                files = ','.join(filelist)
                # lib_seq small.forward.fq,small.reverse.fq   small  215  10   \
                #    101 0 0      1 1 1  0 0 2 1
                f.write(fmt % (
                    files, r['prefix'], r['ins_avg'], r['ins_dev'],
                    r['avg_read_len'], r['has_innie_artifact'],
                    r['is_rev_comped'], r['use_for_contigging'],
                    r['ono_set_id'], r['use_for_gap_closing'],
                    r['fp_wiggle_room'], r['tp_wiggle_room'],
                    count, r['use_for_splinting']))
                total_bases += reads_obj['total_bases']
            f.write('\n')
            paramf = {
                'diploid': 'is_diploid %d\n',
                'dynamic_min_depth': 'dynamic_min_depth %f\n',
                'mer_size': 'mer_size %d\n',
                'min_depth_cutoff': 'min_depth_cutoff %d\n',
                'gap_close_rpt_depth_ratio': 'gap_close_rpt_depth_ratio %f\n',
                'assm_scaff_len_cutoff': 'assm_scaff_len_cutoff %d\n'
            }
            if params['is_diploid'] is not None:
                paramf['bubble'] = 'bubble_min_depth_cutoff %d'
                params['bubble'] = params['is_diploid']['bubble_min_depth_cutoff']
                params['diploid'] = 1
            else:
                params['diploid'] = 0
            for param in paramf:
                f.write(paramf[param] % (params[param]))
            f.close()

        return total_bases

    def generate_submit(self, tsize):
        """
        Generate SLURM submit script
        """
        bpn = 1000000000
        nodes = int((tsize + bpn - 1) / bpn)
        # It seems like hipmer fails with odd numbers of nodes
        # So let's add one if it is odd.
        if nodes % 2:
            nodes += 1
        self.submit = '%s/%s' % (self.scratch, 'slurm.submit')
        with open(self.submit, 'w') as f:
            f.write('#!/bin/bash\n')
            f.write('#SBATCH --partition=debug\n')
            f.write('#SBATCH --nodes=%d\n' % nodes)
            f.write('#SBATCH --ntasks-per-node=32\n')
            f.write('#SBATCH --time=00:30:00\n')
            f.write('#SBATCH --job-name=HipMer\n')
            f.write('export CORES_PER_NODE=${CORES_PER_NODE:=${SLURM_TASKS_PER_NODE%%\(*}}\n')
            f.write('N=${N:=${SLURM_NTASKS}}\n')
            f.write('HIPMER_INSTALL=${HIPMER_INSTALL:=${SCRATCH}/hipmer-install-cori}\n')
            f.write('INST=${HIPMER_INSTALL:=$1}\n')
            f.write('. $INST/env.sh\n')
            f.write('\n')
            f.write('export RUNDIR=${RUNDIR:=$(pwd)}\n')
            f.write('${INST}/bin/run_hipmer.sh ${RUNDIR}/hipmer.config\n')
            f.close()

    def save_assembly(self, wsname, output_contigs, token, name, console):
        self.log(console, 'Uploading FASTA file to Assembly')
        assemblyUtil = AssemblyUtil(self.callbackURL, token=token,
                                    service_ver='dev')
        assemblyUtil.save_assembly_from_fasta({'file': {'path': output_contigs},
                                               'workspace_name': wsname,
                                               'assembly_name': name
                                               })
    #END_CLASS_HEADER

    # config contains contents of config file in a hash or None if it couldn't
    # be found
    def __init__(self, config):
        #BEGIN_CONSTRUCTOR
        self.workspaceURL = config['workspace-url']
        self.scratch = os.path.abspath(config['scratch'])
        self.callbackURL = os.environ.get('SDK_CALLBACK_URL')
        print "Callback=%s" % (self.callbackURL)
        #END_CONSTRUCTOR
        pass

    def run_hipmer_hpc(self, ctx, params):
        """
        :param params: instance of type "AssemblyParams" (Run assembler
           workspace_name - the name of the workspace for input/output
           read_library_name - the name of the PE read library (SE library
           support in the future) output_contig_set_name - the name of the
           output contigset extra_params - assembler specific parameters
           min_contig_length - minimum length of contigs to output, default
           200 @optional min_contig_len @optional extra_params) -> structure:
           parameter "workspace_name" of String, parameter
           "read_library_name" of String, parameter "output_contigset_name"
           of String, parameter "min_contig_len" of Long, parameter
           "extra_params" of list of String
        :returns: instance of type "AssemblyOutput" -> structure: parameter
           "report_name" of String, parameter "report_ref" of String
        """
        # ctx is the context object
        # return variables are: output
        #BEGIN run_hipmer_hpc
        console = []
        self.log(console, 'Running run_hipmer_hpc with params=')
        self.log(console, pformat(params))

        # do some basic checks
        if 'workspace_name' not in params:
            raise ValueError('workspace_name parameter is required')
        if 'reads' not in params:
            raise ValueError('reads parameter is required')
        if 'output_contigset_name' not in params:
            raise ValueError('output_contigset_name parameter is required')
        ws_name = params['workspace_name']
        #ws = workspaceService(self.workspaceURL, token=ctx['token'])

        if 'POST' not in os.environ:
            # Get the read library
            print "Running pre stage"
            refs = []
            for read in params['reads']:
                read_name = read['read_library_name']
                if '/' in read_name:
                    ref = read_name
                else:
                    ref = ws_name + '/' + read_name
                refs.append(ref)
                read['ref'] = ref
            if not self.check_reads(ctx, refs, console):
                sys.exit(1)

            params['readsfiles'] = self.get_reads_RU(ctx, refs, console)

            # set the output location
            output_dir = self.scratch
            # Generate submit script
            ts = self.generate_config(params)
            self.generate_submit(ts)
            return

        print "Running POST stage"

        # run hipmer, capture output as it happens
        self.log(console, 'running hipmer:')

        output_dir = self.scratch
        output_contigs = os.path.join(output_dir, 'final_assembly.fa')
        output_name = params['output_contigset_name']
        wsname = params['workspace_name']
        #output_data_ref = self.save_assembly(wsname,
        #                                     output_contigs,
        #                                     ctx['token'],
        #                                     output_name,
        #                                     console)
        self.log(console, 'Uploading FASTA file to Assembly')
        assemblyUtil = AssemblyUtil(self.callbackURL, token=ctx['token'],
                                    service_ver='dev')
        save_input = {'file': {'path': output_contigs},
                      'workspace_name': wsname,
                      'assembly_name': output_name
                      }
        output_data_ref = assemblyUtil.save_assembly_from_fasta(save_input)
        print 'ref: ' + output_data_ref
        # create a Report
        # compute a simple contig length distribution for the report
        lengths = []
        for seq_record in SeqIO.parse(output_contigs, 'fasta'):
            lengths.append(len(seq_record.seq))

        report = ''
        report += 'ContigSet saved to: ' + params['workspace_name'] + '/'
        report += params['output_contigset_name'] + '\n'
        report += 'Assembled into ' + str(len(lengths)) + ' contigs.\n'
        report += 'Avg Length: ' + str(sum(lengths) / float(len(lengths))) + ' bp.\n'

        bins = 10
        counts, edges = np.histogram(lengths, bins)
        report += 'Contig Length Distribution (# of contigs -- min to max basepairs):\n'
        for c in range(bins):
            report += '   \%d\t--\t%d' % (counts[c], edges[c])
            report += ' to %d bp\n' % (edges[c + 1])

        print('Running QUAST')
        kbq = kb_quast(self.callbackURL)
        try:
            quastret = kbq.run_QUAST({'files': [{'path': output_contigs,
                                                 'label': params['output_contigset_name']}]})
        except QUASTError as qe:
            # not really any way to test this, all inputs have been checked
            # earlier and should be ok
            print('Logging exception from running QUAST')
            print(str(qe))
            # TODO delete shock node
            raise

        print('Saving report')
        kbr = KBaseReport(self.callbackURL)
        try:
            report_info = kbr.create_extended_report(
                {'message': report,
                 'objects_created': [{'ref': output_data_ref,
                                      'description': 'Assembled contigs'}],
                 'direct_html_link_index': 0,
                 'html_links': [{'shock_id': quastret['shock_id'],
                                 'name': 'report.html',
                                 'label': 'QUAST report'}
                                ],
                 'report_object_name': 'kb_megahit_report_' + str(uuid.uuid4()),
                 'workspace_name': params['workspace_name']
                 })
        except _RepError as re:
            # not really any way to test this, all inputs have been checked earlier and should be
            # ok
            print('Logging exception from creating report object')
            print(str(re))
            # TODO delete shock node
            raise

        # STEP 6: contruct the output to send back
        output = {'report_name': report_info['name'],
                  'report_ref': report_info['ref']
                  }
        return [output]
        #END run_hipmer_hpc

        # At some point might do deeper type checking...
        if not isinstance(output, dict):
            raise ValueError('Method run_hipmer_hpc return value ' +
                             'output is not type dict as required.')
        # return the results
        return [output]

    def status(self, ctx):
        #BEGIN_STATUS
        returnVal = {'state': "OK", 'message': "", 'version': self.VERSION,
                     'git_url': self.GIT_URL, 'git_commit_hash': self.GIT_COMMIT_HASH}
        #END_STATUS
        return [returnVal]
<|endoftext|>