<commit_before># Copyright (C) 2016 Google Inc.
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>

"""Module containing custom attributable mixin."""

import collections

from sqlalchemy import and_
from sqlalchemy import orm
from sqlalchemy import or_
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.orm import foreign
from sqlalchemy.orm import relationship

from ggrc import db
from ggrc import utils
from ggrc.models.reflection import AttributeInfo

class CustomAttributable(object):

  @declared_attr
  def custom_attribute_values(self):
    """Load custom attribute values"""
    from ggrc.models.custom_attribute_value import CustomAttributeValue

    def join_function():
      return and_(
          foreign(CustomAttributeValue.attributable_id) == self.id,
          foreign(CustomAttributeValue.attributable_type) == self.__name__)
    return relationship(
        "CustomAttributeValue",
        primaryjoin=join_function,
        backref='{0}_custom_attributable'.format(self.__name__),
        cascade='all, delete-orphan',
    )

  def insert_definition(self, definition):
    """Insert a new custom attribute definition into database

    Args:
      definition: dictionary with field_name: value
    Returns:
      Nothing.
    """
    from ggrc.models.custom_attribute_definition \
        import CustomAttributeDefinition
    field_names = AttributeInfo.gather_create_attrs(
        CustomAttributeDefinition)

    data = {fname: definition.get(fname) for fname in field_names}
    data["definition_type"] = self._inflector.table_singular
    cad = CustomAttributeDefinition(**data)
    db.session.add(cad)

  def process_definitions(self, definitions):
    """
    Process custom attribute definitions

    If present, delete all related custom attribute definition and insert new
    custom attribute definitions in the order provided.

    Args:
      definitions: Ordered list of custom attribute definitions
    Returns:
      Nothing
    """
    from ggrc.models.custom_attribute_definition \
        import CustomAttributeDefinition as CADef

    if not hasattr(self, "PER_OBJECT_CUSTOM_ATTRIBUTABLE"):
      return

    if self.id is not None:
      db.session.query(CADef).filter(
          CADef.definition_id == self.id,
          CADef.definition_type == self._inflector.table_singular
      ).delete()
      db.session.commit()

    for definition in definitions:
      if "_pending_delete" in definition and definition["_pending_delete"]:
        continue
      self.insert_definition(definition)

  @declared_attr
  def custom_attribute_definitions(self):
    """Load custom attribute definitions"""
    from ggrc.models.custom_attribute_definition\
        import CustomAttributeDefinition

    def join_function():
      definition_id = foreign(CustomAttributeDefinition.definition_id)
      definition_type = foreign(CustomAttributeDefinition.definition_type)
      return and_(or_(definition_id == self.id, definition_id.is_(None)),
                  definition_type == self._inflector.table_singular)

    return relationship(
        "CustomAttributeDefinition",
        primaryjoin=join_function,
        backref='{0}_custom_attributable_definition'.format(self.__name__),
        viewonly=True,
    )

  @declared_attr
  def _custom_attributes_deletion(self):
    """This declared attribute is used only for handling cascade deletions
       for CustomAttributes. This is done in order not to try to delete
       "global" custom attributes that don't have any definition_id related.
       Attempt to delete custom attributes with definition_id=None causes the
       IntegrityError as we shouldn't be able to delete global attributes along
       side with any other object (e.g. Assessments).
    """
    from ggrc.models.custom_attribute_definition import (
        CustomAttributeDefinition
    )

    def join_function():
      """Join condition used for deletion"""
      definition_id = foreign(CustomAttributeDefinition.definition_id)
      definition_type = foreign(CustomAttributeDefinition.definition_type)
      return and_(definition_id == self.id,
                  definition_type == self._inflector.table_singular)

    return relationship(
        "CustomAttributeDefinition",
        primaryjoin=join_function,
        cascade='all, delete-orphan',
        order_by="CustomAttributeDefinition.id"
    )

  def custom_attributes(self, attributes):
    from ggrc.fulltext.mysql import MysqlRecordProperty
    from ggrc.models.custom_attribute_value import CustomAttributeValue
    from ggrc.services import signals

    definitions = attributes.get("custom_attribute_definitions")
    if definitions:
      self.process_definitions(definitions)

    if 'custom_attributes' not in attributes:
      return

    attributes = attributes['custom_attributes']

    old_values = collections.defaultdict(list)
    last_values = dict()

    # attributes looks like this:
    #    [ {<id of attribute definition> : attribute value, ... }, ... ]

    # 1) Get all custom attribute values for the CustomAttributable instance
    attr_values = db.session.query(CustomAttributeValue).filter(and_(
        CustomAttributeValue.attributable_type == self.__class__.__name__,
        CustomAttributeValue.attributable_id == self.id)).all()

    attr_value_ids = [v.id for v in attr_values]
    ftrp_properties = [
        "attribute_value_{id}".format(id=_id) for _id in attr_value_ids]

    # Save previous value of custom attribute. This is a bit complicated by
    # the fact that imports can save multiple values at the time of writing.
    # old_values holds all previous values of attribute, last_values holds
    # chronologically last value.
    for v in attr_values:
      old_values[v.custom_attribute_id] += [(v.created_at, v.attribute_value)]

    last_values = {str(key): max(old_vals,
                                 key=lambda (created_at, _): created_at)
                   for key, old_vals in old_values.iteritems()}

    # 2) Delete all fulltext_record_properties for the list of values
    if len(attr_value_ids) > 0:
      db.session.query(MysqlRecordProperty)\
          .filter(
              and_(
                  MysqlRecordProperty.type == self.__class__.__name__,
                  MysqlRecordProperty.property.in_(ftrp_properties)))\
          .delete(synchronize_session='fetch')

      # 3) Delete the list of custom attribute values
      db.session.query(CustomAttributeValue)\
          .filter(CustomAttributeValue.id.in_(attr_value_ids))\
          .delete(synchronize_session='fetch')

      db.session.commit()

    # 4) Instantiate custom attribute values for each of the definitions
    #    passed in (keys)
    # pylint: disable=not-an-iterable
    definitions = {d.id: d for d in self.get_custom_attribute_definitions()}
    for ad_id in attributes.keys():
      obj_type = self.__class__.__name__
      obj_id = self.id
      new_value = CustomAttributeValue(
          custom_attribute_id=ad_id,
          attributable_id=obj_id,
          attributable_type=obj_type,
          attribute_value=attributes[ad_id],
      )
      if definitions[int(ad_id)].attribute_type.startswith("Map:"):
        obj_type, obj_id = new_value.attribute_value.split(":")
        new_value.attribute_value = obj_type
        new_value.attribute_object_id = obj_id
      # 5) Set the context_id for each custom attribute value to the context id
      #    of the custom attributable.
      # TODO: We are ignoring contexts for now
      # new_value.context_id = cls.context_id
      self.custom_attribute_values.append(new_value)
      if ad_id in last_values:
        ca, pv = last_values[ad_id]  # created_at, previous_value
        if pv != attributes[ad_id]:
          signals.Signals.custom_attribute_changed.send(
              self.__class__,
              obj=self,
              src={
                  "type": obj_type,
                  "id": obj_id,
                  "operation": "UPDATE",
                  "value": new_value,
                  "old": pv
              }, service=self.__class__.__name__)
      else:
        signals.Signals.custom_attribute_changed.send(
            self.__class__,
            obj=self,
            src={
                "type": obj_type,
                "id": obj_id,
                "operation": "INSERT",
                "value": new_value,
            }, service=self.__class__.__name__)

  _publish_attrs = ['custom_attribute_values', 'custom_attribute_definitions']
  _update_attrs = ['custom_attributes']
  _include_links = ['custom_attribute_definitions']

  @classmethod
  def get_custom_attribute_definitions(cls):
    from ggrc.models.custom_attribute_definition import \
        CustomAttributeDefinition as cad
    if cls.__name__ == "Assessment":
      return cad.query.filter(or_(
          cad.definition_type == utils.underscore_from_camelcase(cls.__name__),
          cad.definition_type == "assessment_template",
      )).all()
    else:
      return cad.query.filter(
          cad.definition_type == utils.underscore_from_camelcase(cls.__name__)
      ).all()

  @classmethod
  def eager_query(cls):
    query = super(CustomAttributable, cls).eager_query()
    return query.options(
        orm.subqueryload('custom_attribute_values'),
        orm.subqueryload('custom_attribute_definitions')
           .undefer_group('CustomAttributeDefinition_complete')
    )

  def log_json(self):
    """Log custom attribute values."""
    # pylint: disable=not-an-iterable
    from ggrc.models.custom_attribute_definition import \
        CustomAttributeDefinition
    # to integrate with Base mixin without order dependencies
    res = getattr(super(CustomAttributable, self), "log_json", lambda: {})()

    if self.custom_attribute_values:
      res["custom_attributes"] = [value.log_json()
                                  for value in self.custom_attribute_values]
      # fetch definitions form database because `self.custom_attribute`
      # may not be populated
      defs = CustomAttributeDefinition.query.filter(
          CustomAttributeDefinition.definition_type == self.type,
          CustomAttributeDefinition.id.in_([
              value.custom_attribute_id
              for value in self.custom_attribute_values
          ])
      )
      # also log definitions to freeze field names in time
      res["custom_attribute_definitions"] = [definition.log_json()
                                             for definition in defs]
    else:
      res["custom_attribute_definitions"] = []
      res["custom_attributes"] = []

    return res
<commit_msg>Move declared attrs to top of CA mixin<commit_after># Copyright (C) 2016 Google Inc.
# Licensed under http://www.apache.org/licenses/LICENSE-2.0 <see LICENSE file>

"""Module containing custom attributable mixin."""

import collections

from sqlalchemy import and_
from sqlalchemy import orm
from sqlalchemy import or_
from sqlalchemy.ext.declarative import declared_attr
from sqlalchemy.orm import foreign
from sqlalchemy.orm import relationship

from ggrc import db
from ggrc import utils
from ggrc.models.reflection import AttributeInfo

class CustomAttributable(object):
  """Custom Attributable mixin."""

  _publish_attrs = ['custom_attribute_values', 'custom_attribute_definitions']
  _update_attrs = ['custom_attributes']
  _include_links = ['custom_attribute_definitions']

  @declared_attr
  def custom_attribute_definitions(self):
    """Load custom attribute definitions"""
    from ggrc.models.custom_attribute_definition\
        import CustomAttributeDefinition

    def join_function():
      definition_id = foreign(CustomAttributeDefinition.definition_id)
      definition_type = foreign(CustomAttributeDefinition.definition_type)
      return and_(or_(definition_id == self.id, definition_id.is_(None)),
                  definition_type == self._inflector.table_singular)

    return relationship(
        "CustomAttributeDefinition",
        primaryjoin=join_function,
        backref='{0}_custom_attributable_definition'.format(self.__name__),
        viewonly=True,
    )

  @declared_attr
  def _custom_attributes_deletion(self):
    """This declared attribute is used only for handling cascade deletions
       for CustomAttributes. This is done in order not to try to delete
       "global" custom attributes that don't have any definition_id related.
       Attempt to delete custom attributes with definition_id=None causes the
       IntegrityError as we shouldn't be able to delete global attributes along
       side with any other object (e.g. Assessments).
    """
    from ggrc.models.custom_attribute_definition import (
        CustomAttributeDefinition
    )

    def join_function():
      """Join condition used for deletion"""
      definition_id = foreign(CustomAttributeDefinition.definition_id)
      definition_type = foreign(CustomAttributeDefinition.definition_type)
      return and_(definition_id == self.id,
                  definition_type == self._inflector.table_singular)

    return relationship(
        "CustomAttributeDefinition",
        primaryjoin=join_function,
        cascade='all, delete-orphan',
        order_by="CustomAttributeDefinition.id"
    )

  @declared_attr
  def custom_attribute_values(self):
    """Load custom attribute values"""
    from ggrc.models.custom_attribute_value import CustomAttributeValue

    def join_function():
      return and_(
          foreign(CustomAttributeValue.attributable_id) == self.id,
          foreign(CustomAttributeValue.attributable_type) == self.__name__)
    return relationship(
        "CustomAttributeValue",
        primaryjoin=join_function,
        backref='{0}_custom_attributable'.format(self.__name__),
        cascade='all, delete-orphan',
    )

  def insert_definition(self, definition):
    """Insert a new custom attribute definition into database

    Args:
      definition: dictionary with field_name: value
    Returns:
      Nothing.
    """
    from ggrc.models.custom_attribute_definition \
        import CustomAttributeDefinition
    field_names = AttributeInfo.gather_create_attrs(
        CustomAttributeDefinition)

    data = {fname: definition.get(fname) for fname in field_names}
    data["definition_type"] = self._inflector.table_singular
    cad = CustomAttributeDefinition(**data)
    db.session.add(cad)

  def process_definitions(self, definitions):
    """
    Process custom attribute definitions

    If present, delete all related custom attribute definition and insert new
    custom attribute definitions in the order provided.

    Args:
      definitions: Ordered list of custom attribute definitions
    Returns:
      Nothing
    """
    from ggrc.models.custom_attribute_definition \
        import CustomAttributeDefinition as CADef

    if not hasattr(self, "PER_OBJECT_CUSTOM_ATTRIBUTABLE"):
      return

    if self.id is not None:
      db.session.query(CADef).filter(
          CADef.definition_id == self.id,
          CADef.definition_type == self._inflector.table_singular
      ).delete()
      db.session.commit()

    for definition in definitions:
      if "_pending_delete" in definition and definition["_pending_delete"]:
        continue
      self.insert_definition(definition)

  def custom_attributes(self, attributes):
    from ggrc.fulltext.mysql import MysqlRecordProperty
    from ggrc.models.custom_attribute_value import CustomAttributeValue
    from ggrc.services import signals

    definitions = attributes.get("custom_attribute_definitions")
    if definitions:
      self.process_definitions(definitions)

    if 'custom_attributes' not in attributes:
      return

    attributes = attributes['custom_attributes']

    old_values = collections.defaultdict(list)
    last_values = dict()

    # attributes looks like this:
    #    [ {<id of attribute definition> : attribute value, ... }, ... ]

    # 1) Get all custom attribute values for the CustomAttributable instance
    attr_values = db.session.query(CustomAttributeValue).filter(and_(
        CustomAttributeValue.attributable_type == self.__class__.__name__,
        CustomAttributeValue.attributable_id == self.id)).all()

    attr_value_ids = [v.id for v in attr_values]
    ftrp_properties = [
        "attribute_value_{id}".format(id=_id) for _id in attr_value_ids]

    # Save previous value of custom attribute. This is a bit complicated by
    # the fact that imports can save multiple values at the time of writing.
    # old_values holds all previous values of attribute, last_values holds
    # chronologically last value.
    for v in attr_values:
      old_values[v.custom_attribute_id] += [(v.created_at, v.attribute_value)]

    last_values = {str(key): max(old_vals,
                                 key=lambda (created_at, _): created_at)
                   for key, old_vals in old_values.iteritems()}

    # 2) Delete all fulltext_record_properties for the list of values
    if len(attr_value_ids) > 0:
      db.session.query(MysqlRecordProperty)\
          .filter(
              and_(
                  MysqlRecordProperty.type == self.__class__.__name__,
                  MysqlRecordProperty.property.in_(ftrp_properties)))\
          .delete(synchronize_session='fetch')

      # 3) Delete the list of custom attribute values
      db.session.query(CustomAttributeValue)\
          .filter(CustomAttributeValue.id.in_(attr_value_ids))\
          .delete(synchronize_session='fetch')

      db.session.commit()

    # 4) Instantiate custom attribute values for each of the definitions
    #    passed in (keys)
    # pylint: disable=not-an-iterable
    definitions = {d.id: d for d in self.get_custom_attribute_definitions()}
    for ad_id in attributes.keys():
      obj_type = self.__class__.__name__
      obj_id = self.id
      new_value = CustomAttributeValue(
          custom_attribute_id=ad_id,
          attributable_id=obj_id,
          attributable_type=obj_type,
          attribute_value=attributes[ad_id],
      )
      if definitions[int(ad_id)].attribute_type.startswith("Map:"):
        obj_type, obj_id = new_value.attribute_value.split(":")
        new_value.attribute_value = obj_type
        new_value.attribute_object_id = obj_id
      # 5) Set the context_id for each custom attribute value to the context id
      #    of the custom attributable.
      # TODO: We are ignoring contexts for now
      # new_value.context_id = cls.context_id
      self.custom_attribute_values.append(new_value)
      if ad_id in last_values:
        ca, pv = last_values[ad_id]  # created_at, previous_value
        if pv != attributes[ad_id]:
          signals.Signals.custom_attribute_changed.send(
              self.__class__,
              obj=self,
              src={
                  "type": obj_type,
                  "id": obj_id,
                  "operation": "UPDATE",
                  "value": new_value,
                  "old": pv
              }, service=self.__class__.__name__)
      else:
        signals.Signals.custom_attribute_changed.send(
            self.__class__,
            obj=self,
            src={
                "type": obj_type,
                "id": obj_id,
                "operation": "INSERT",
                "value": new_value,
            }, service=self.__class__.__name__)

  @classmethod
  def get_custom_attribute_definitions(cls):
    from ggrc.models.custom_attribute_definition import \
        CustomAttributeDefinition as cad
    if cls.__name__ == "Assessment":
      return cad.query.filter(or_(
          cad.definition_type == utils.underscore_from_camelcase(cls.__name__),
          cad.definition_type == "assessment_template",
      )).all()
    else:
      return cad.query.filter(
          cad.definition_type == utils.underscore_from_camelcase(cls.__name__)
      ).all()

  @classmethod
  def eager_query(cls):
    query = super(CustomAttributable, cls).eager_query()
    return query.options(
        orm.subqueryload('custom_attribute_values'),
        orm.subqueryload('custom_attribute_definitions')
           .undefer_group('CustomAttributeDefinition_complete')
    )

  def log_json(self):
    """Log custom attribute values."""
    # pylint: disable=not-an-iterable
    from ggrc.models.custom_attribute_definition import \
        CustomAttributeDefinition
    # to integrate with Base mixin without order dependencies
    res = getattr(super(CustomAttributable, self), "log_json", lambda: {})()

    if self.custom_attribute_values:
      res["custom_attributes"] = [value.log_json()
                                  for value in self.custom_attribute_values]
      # fetch definitions form database because `self.custom_attribute`
      # may not be populated
      defs = CustomAttributeDefinition.query.filter(
          CustomAttributeDefinition.definition_type == self.type,
          CustomAttributeDefinition.id.in_([
              value.custom_attribute_id
              for value in self.custom_attribute_values
          ])
      )
      # also log definitions to freeze field names in time
      res["custom_attribute_definitions"] = [definition.log_json()
                                             for definition in defs]
    else:
      res["custom_attribute_definitions"] = []
      res["custom_attributes"] = []

    return res
<|endoftext|>


<commit_before>"""
Provides a simple launcher adapter that will launch processes in a shell
"""

"""
This file is part of RAPD

Copyright (C) 2016-2017 Cornell University
All rights reserved.

RAPD is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, version 3.

RAPD is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

__created__ = "2016-03-02"
__maintainer__ = "Frank Murphy"
__email__ = "fmurphy@anl.gov"
__status__ = "Development"

import logging
import json
from subprocess import Popen

# RAPD imports
import utils.launch_tools as launch_tools

class LauncherAdapter(object):
    """
    An adapter for launcher process.

    Will launch requested job in shell on the current machine
    """

    def __init__(self, site, message, settings):
        """
        Initialize the adapter
        """

        # Get the logger Instance
        self.logger = logging.getLogger("RAPDLogger")
        self.logger.debug("__init__")

        self.site = site
        self.message = message
        self.settings = settings

        self.run()

    def run(self):
        """
        Orchestrate the adapter's actions
        """

        # Decode message
        command = json.loads(self.message)["command"]

        # Put the command into a file
        command_file = launch_tools.write_command_file(self.settings["launch_dir"], command, self.message)

        # Call the launch process on the command file
        self.logger.debug("rapd.launch", "-s", self.site.ID.lower(), command_file)
        Popen(["rapd.launch", "-s", self.site.ID.lower(), command_file])

if __name__ == "__main__":

    LauncherAdapter('["test1", "test2", "test3"]', {"launch_dir":"/tmp/log"})
<commit_msg>Trying to get simple shell up and running<commit_after>"""
Provides a simple launcher adapter that will launch processes in a shell
"""

"""
This file is part of RAPD

Copyright (C) 2016-2017 Cornell University
All rights reserved.

RAPD is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, version 3.

RAPD is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU Affero General Public License for more details.

You should have received a copy of the GNU Affero General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

__created__ = "2016-03-02"
__maintainer__ = "Frank Murphy"
__email__ = "fmurphy@anl.gov"
__status__ = "Development"

import logging
import json
from subprocess import Popen

# RAPD imports
import utils.launch_tools as launch_tools

class LauncherAdapter(object):
    """
    An adapter for launcher process.

    Will launch requested job in shell on the current machine
    """

    def __init__(self, site, message, settings):
        """
        Initialize the adapter
        """

        # Get the logger Instance
        self.logger = logging.getLogger("RAPDLogger")
        self.logger.debug("__init__")

        self.site = site
        self.message = message
        self.settings = settings

        self.run()

    def run(self):
        """
        Orchestrate the adapter's actions
        """

        # Decode message
        command = self.message["command"]

        # Put the command into a file
        command_file = launch_tools.write_command_file(self.settings["launch_dir"], command, self.message)

        # Call the launch process on the command file
        self.logger.debug("rapd.launch", "-s", self.site.ID.lower(), command_file)
        Popen(["rapd.launch", "-s", self.site.ID.lower(), command_file])

if __name__ == "__main__":

    LauncherAdapter('["test1", "test2", "test3"]', {"launch_dir":"/tmp/log"})
<|endoftext|>


<commit_before># Copyright 2015 Canonical, Ltd.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

""" UI utilities """

from urwid import Padding as _Padding
from urwid import AttrMap
from functools import partialmethod
from subiquity.palette import STYLES

def apply_padders(cls):
    """ Decorator for generating useful padding methods

    Loops through and generates methods like:

      Padding.push_1(Widget)

      Sets the left padding attribute by 1

      Padding.pull_24(Widget)

      Sets right padding attribute by 24.

      Padding.center_50(Widget)

      Provides center padding with a relative width of 50
    """
    padding_count = 100

    for i in range(1, padding_count):
        setattr(cls, 'push_{}'.format(i), partialmethod(_Padding, left=i))
        setattr(cls, 'pull_{}'.format(i), partialmethod(_Padding, right=i))
        setattr(cls, 'center_{}'.format(i),
                partialmethod(_Padding, align='center',
                              width=('relative', i)))
        setattr(cls, 'left_{}'.format(i),
                partialmethod(_Padding, align='left',
                              width=('relative', i)))
        setattr(cls, 'right_{}'.format(i),
                partialmethod(_Padding, align='right',
                              width=('relative', i)))
    return cls

@apply_padders
class Padding:
    """ Padding methods

    .. py:meth:: push_X(:class:`urwid.Widget`)

       This method supports padding the left side of the widget
       from 1-99, for example:

       .. code::

          Padding.push_20(Text("This will be indented 20 columns")

    .. py:meth:: pull_X(:class:`urwid.Widget`)

       This method supports padding the right side of the widget
       from 1-99, for example:

       .. code::

          Padding.pull_20(Text("This will be right indented 20 columns")

    .. py:meth:: center_X(:class:`urwid.Widget`)

       This method centers a widget with X being the relative width of
       the widget.

       .. code::

          Padding.center_10(Text("This will be centered with a "
                                 "width of 10 columns"))

    .. py:meth:: left_X(:class:`urwid.Widget`)

       This method aligns a widget left with X being the relative width of
       the widget.

       .. code::

          Padding.left_10(Text("This will be left aligned with a "
                               "width of 10 columns"))

    .. py:meth:: right_X(:class:`urwid.Widget`)

       This method right aligns a widget with X being the relative width of
       the widget.

       .. code::

          Padding.right_10(Text("This will be right aligned with a "
                                "width of 10 columns"))

    """
    pass

def apply_style_map(cls):
    """ Applies AttrMap attributes to Color class

    Eg:

      Color.frame_header(Text("I'm text in the Orange frame header"))
      Color.body(Text("Im text in wrapped with the body color"))
    """
    for k in STYLES:
        setattr(cls, k[0], partialmethod(AttrMap, attr_map=k[0]))
    return cls

@apply_style_map
class Color:
    """ Partial methods for :class:`subiquity.palette.STYLES`

    .. py:meth:: frame_header(:class:`urwid.Widget`)

       This method colors widget based on the style map used.

       .. code::

          Color.frame_header(Text("This will use foreground and background "
                                  "defined from the STYLES attribute"))

    """
    pass
<commit_msg>Update docstring in utils<commit_after># Copyright 2015 Canonical, Ltd.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

""" UI utilities """

from urwid import Padding as _Padding
from urwid import AttrMap
from functools import partialmethod
from subiquity.palette import STYLES

def apply_padders(cls):
    """ Decorator for generating useful padding methods

    Loops through and generates methods like:

      Padding.push_1(Widget)

      Sets the left padding attribute by 1

      Padding.pull_24(Widget)

      Sets right padding attribute by 24.

      Padding.center_50(Widget)

      Provides center padding with a relative width of 50
    """
    padding_count = 100

    for i in range(1, padding_count):
        setattr(cls, 'push_{}'.format(i), partialmethod(_Padding, left=i))
        setattr(cls, 'pull_{}'.format(i), partialmethod(_Padding, right=i))
        setattr(cls, 'center_{}'.format(i),
                partialmethod(_Padding, align='center',
                              width=('relative', i)))
        setattr(cls, 'left_{}'.format(i),
                partialmethod(_Padding, align='left',
                              width=('relative', i)))
        setattr(cls, 'right_{}'.format(i),
                partialmethod(_Padding, align='right',
                              width=('relative', i)))
    return cls

@apply_padders
class Padding:
    """ Padding methods

    .. py:meth:: push_X(:class:`urwid.Widget`)

       This method supports padding the left side of the widget
       from 1-99, for example:

       .. code::

          Padding.push_20(Text("This will be indented 20 columns")

    .. py:meth:: pull_X(:class:`urwid.Widget`)

       This method supports padding the right side of the widget
       from 1-99, for example:

       .. code::

          Padding.pull_20(Text("This will be right indented 20 columns")

    .. py:meth:: center_X(:class:`urwid.Widget`)

       This method centers a widget with X being the relative width of
       the widget.

       .. code::

          Padding.center_10(Text("This will be centered with a "
                                 "width of 10 columns"))

    .. py:meth:: left_X(:class:`urwid.Widget`)

       This method aligns a widget left with X being the relative width of
       the widget.

       .. code::

          Padding.left_10(Text("This will be left aligned with a "
                               "width of 10 columns"))

    .. py:meth:: right_X(:class:`urwid.Widget`)

       This method right aligns a widget with X being the relative width of
       the widget.

       .. code::

          Padding.right_10(Text("This will be right aligned with a "
                                "width of 10 columns"))

    """
    pass

def apply_style_map(cls):
    """ Applies AttrMap attributes to Color class

    Eg:

      Color.frame_header(Text("I'm text in the Orange frame header"))
      Color.body(Text("Im text in wrapped with the body color"))
    """
    for k in STYLES:
        setattr(cls, k[0], partialmethod(AttrMap, attr_map=k[0]))
    return cls

@apply_style_map
class Color:
    """ Partial methods for :class:`~subiquity.palette.STYLES`

    .. py:meth:: frame_header(:class:`urwid.Widget`)

       This method colors widget based on the style map used.

       .. code::

          Color.frame_header(Text("This will use foreground and background "
                                  "defined from the STYLES attribute"))

    """
    pass
<|endoftext|>


<commit_before>def load_style(styles_f):
    """Load a json file containing the keyword arguments to use for plot styling

    """
    import json

    with open(styles_f, 'r') as styles_src:
        styles = json.load(styles_src)
    return styles

def check_crs(source_crs, base_crs=None):
    """Check the crs for correctness

    """
    from gdal import osr

    crs=osr.SpatialReference(wkt=source_crs)
    if crs.IsProjected and base_crs == None:
        status = 'success'
    elif crs.IsProjected and crs == base_crs:
        status = 'success'
    elif crs.IsProjected and crs != base_crs:
        status = 'unequal'
    else:
        status = 'unprojected'

    return crs, status

def read_geometries(feature_f):
    """    Read feature source geometries

    """
    import click
    import fiona
    import rasterio
    from shapely.geometry import shape

    with fiona.open(feature_f) as feature_src:
        supported = ['Point', 'LineString', '3D Point', '3D LineString']
        schema_geometry = feature_src.schema['geometry']
        try:
            if schema_geometry not in supported:
                raise click.BadParameter('Geometry must be one of: {}'.format(supported))
        except:
            raise click.BadParameter('Unable to obtain schema from {}'.format(feature_f))
        geometries = [shape(feature['geometry']) for feature in feature_src]
        feature_crs = feature_src.crs_wkt
    return schema_geometry, feature_crs, geometries

"""
def read_geometries(feature_f, elevation_f=None, keep_z=False):

    import click
    import fiona
    import rasterio
    from shapely.geometry import shape

    import drapery

    with fiona.open(feature_f) as feature_src:
        supported = ['Point', 'LineString', '3D Point', '3D LineString']
        try:
            if feature_src.schema['geometry'] not in supported:
                raise click.BadParameter('Geometry must be one of: {}'.format(supported))
        except:
            raise click.BadParameter('Unable to obtain schema from {}'.format(feature_f))
        if elevation_f and not keep_z:
            with rasterio.open(elevation_f) as raster:
                if feature_src.crs != raster.crs:
                    raise click.BadParameter('{} and {} use different CRS'.format(feature_f, elevation_f))
                geometries = [drapery.drape(raster, feature) for feature in feature_src]
        else:
            if feature_src.schema['geometry'] in ['LineString', 'Point'] and not keep_z:
                raise click.BadParameter('{} is 2D. Provide an elevation source, or convert to 3D geometry'.format(feature_f))
            geometries = [shape(feature['geometry']) for feature in feature_src]

        feature_crs = feature_src.crs_wkt

    return feature_crs, geometries
"""
<commit_msg>add docstrings to cli utility functions<commit_after>def load_style(styles_f):
    """Load a json file containing the keyword arguments to use for plot styling

    Parameters:
        styles_f: path to json file containing matplotlib style keyword arguments

    Returns:
        styles (dict): dictionary of matplotlib keyword arguments

    """
    import json

    with open(styles_f, 'r') as styles_src:
        styles = json.load(styles_src)
    return styles

def check_crs(source_crs, base_crs=None):
    """Check the crs for correctness

    Parameters:
        source_crs (str): coordinate reference system in well-known text (WKT) format

    Other Parameters:
        base_crs (str): coordinate reference system for comparison

    Returns:
        crs (SpatialReference): coordinate reference system of the source data
        status (str): status message

    """
    from gdal import osr

    crs=osr.SpatialReference(wkt=source_crs)
    if crs.IsProjected and base_crs == None:
        status = 'success'
    elif crs.IsProjected and crs == base_crs:
        status = 'success'
    elif crs.IsProjected and crs != base_crs:
        status = 'unequal'
    else:
        status = 'unprojected'

    return crs, status

def read_geometries(feature_f):
    """Read feature source geometries

    Parameters:
        feature_f: path to the feature data to read

    Returns:
        schema_geometry (str): feature type
        feature_crs (str): feature source crs in well-known text (WKT) format
        geometries: list of shapely geometries

    """
    import click
    import fiona
    import rasterio
    from shapely.geometry import shape

    with fiona.open(feature_f) as feature_src:
        supported = ['Point', 'LineString', '3D Point', '3D LineString']
        schema_geometry = feature_src.schema['geometry']
        try:
            if schema_geometry not in supported:
                raise click.BadParameter('Geometry must be one of: {}'.format(supported))
        except:
            raise click.BadParameter('Unable to obtain schema from {}'.format(feature_f))
        geometries = [shape(feature['geometry']) for feature in feature_src]
        feature_crs = feature_src.crs_wkt
    return schema_geometry, feature_crs, geometries

"""
def read_geometries(feature_f, elevation_f=None, keep_z=False):

    import click
    import fiona
    import rasterio
    from shapely.geometry import shape

    import drapery

    with fiona.open(feature_f) as feature_src:
        supported = ['Point', 'LineString', '3D Point', '3D LineString']
        try:
            if feature_src.schema['geometry'] not in supported:
                raise click.BadParameter('Geometry must be one of: {}'.format(supported))
        except:
            raise click.BadParameter('Unable to obtain schema from {}'.format(feature_f))
        if elevation_f and not keep_z:
            with rasterio.open(elevation_f) as raster:
                if feature_src.crs != raster.crs:
                    raise click.BadParameter('{} and {} use different CRS'.format(feature_f, elevation_f))
                geometries = [drapery.drape(raster, feature) for feature in feature_src]
        else:
            if feature_src.schema['geometry'] in ['LineString', 'Point'] and not keep_z:
                raise click.BadParameter('{} is 2D. Provide an elevation source, or convert to 3D geometry'.format(feature_f))
            geometries = [shape(feature['geometry']) for feature in feature_src]

        feature_crs = feature_src.crs_wkt

    return feature_crs, geometries
"""
<|endoftext|>


<commit_before>from suspect import MRSSpectrum

import subprocess
import parse
import re
import numpy as np

def save_dpt(filename, data):
    with open(filename, 'wb') as fout:
        fout.write("Dangerplot_version\t1.0\n".encode())
        fout.write("Number_of_points\t{}\n".format(data.np).encode())
        fout.write("Sampling_frequency\t{0:8.8e}\n".format(1.0 / data.dt).encode())
        fout.write("Transmitter_frequency\t{0:8.8e}\n".format(data.f0 * 1e6).encode())
        fout.write("Phi0\t{0:8.8e}\n".format(0).encode())
        fout.write("Phi1\t{0:8.8e}\n".format(0).encode())
        fout.write("PPM_reference\t{0:8.8e}\n".format(data.ppm0).encode())
        fout.write("Echo_time\t{0:8.8e}\n".format(data.te * 1e-3).encode())
        fout.write("Real_FID\tImag_FID\t\n".encode())
        for x in data:
            fout.write("{0.real:8.8e} {0.imag:8.8e}\n".format(x).encode())

def read_output(filename):
    """Reads in a Tarquin txt results file and returns a dict of the information

    Parameters
    ----------
    filename : txt file
        The filename to read from

    Returns
    -------
    result : dict

    """
    with open(filename) as fin:
        data = fin.read()

        metabolite_fits = {}
        fit_quality = {}

        sections = data.split("\n\n")

        # first section is the metabolite concentrations
        metabolite_lines = sections[0].splitlines()[2:]
        for line in metabolite_lines:
            name, concentration, pc_sd, sd = line.split()
            metabolite_fits[name] = {
                "concentration": concentration,
                "sd": pc_sd,
            }

        # second section is the fit quality
        lines = sections[1].splitlines()[2:]
        for line in lines:
            key, value = line.split(":")
            fit_quality[key.strip()] = float(value.strip())

        return {
            "metabolite_fits": metabolite_fits,
            "quality": fit_quality
        }

def read_fit_file(filename):
    """Reads in a Tarquin txt fit file and returns all the fitted plots

    Parameters
    ----------
    filename : txt file
        The filename to read from

    Returns
    -------
    result : dict
    """
    with open(filename) as fin:
        contents = fin.read()

    # we don't know the exact size and shape of the grid of voxels
    # we are reading in yet, so start by making a dictionary of
    # positions and arrange them later
    voxel_dict = {}
    metabolite_names = None

    # the file is divided into data from each voxel sequentially
    # we want to split the file into separate voxel components
    # each voxel data starts with Row : x, Col : y ...
    # use re.split rather than str.split to avoid getting an
    # empty string before the first Row, by matching the new line
    # followed by Row instead.
    for voxel_string in re.split("\n(?=Row)", contents):

        # first two lines contain metadata, the rest is spectral points
        position_string, metabolite_string, data_string = voxel_string.split("\n", 2)
        position = parse.parse("Row : {:d}, Col : {:d}, Slice : {:d}", position_string)

        # metabolite names are only necessary once
        # also, the first four are fixed (ppm, data, fit, baseline)
        # only collect the rest
        if metabolite_names is None:
            metabolite_names = [name.strip() for name in metabolite_string.split(",")[4:]]

        voxel_dict[position] = np.array([[float(j) for j in i.split(',')] for i in data_string.splitlines()])

    # get the maximum row, col and slice
    max_row = max([pos[0] for pos in voxel_dict.keys()])
    max_col = max([pos[1] for pos in voxel_dict.keys()])
    max_slice = max([pos[2] for pos in voxel_dict.keys()])

    voxel_shape = voxel_dict.values().__iter__().__next__().shape
    combined_data = np.zeros((max_row, max_col, max_slice, *voxel_shape), 'complex')
    for (row, col, slc), voxel_data in voxel_dict.items():
        combined_data[row - 1, col - 1, slc - 1] = voxel_data

    return metabolite_names, combined_data

def _extract_fit_data(ref_data, metabolite_names, combined_data):
    # helper function to save space
    def make_spectrum(spectrum_data):
        return MRSSpectrum(spectrum_data,
                           ref_data.dt,
                           ref_data.f0,
                           ref_data.te,
                           ref_data.ppm0)

    # split out the individual lines (data, fit, baseline etc.)
    data = make_spectrum(combined_data[:, :, :, :, 1].squeeze())
    fit = make_spectrum(combined_data[:, :, :, :, 2].squeeze())
    baseline = make_spectrum(combined_data[:, :, :, :, 3].squeeze())
    metabolite_dict = {
        metabolite_name: make_spectrum(combined_data[:, :, :, :, i + 4].squeeze())
        for i, metabolite_name in enumerate(metabolite_names)
    }
    return {
        "data": data,
        "fit": fit,
        "baseline": baseline,
        "metabolites": metabolite_dict
    }

def process(data, options={}):
    save_dpt("/tmp/temp.dpt", data)
    option_string = ""
    for key, value in options.items():
        option_string += " --{} {}".format(key, value)
    subprocess.run("tarquin --input {} --format dpt --output_txt {} --output_fit {}{}".format(
        "/tmp/temp.dpt", "/tmp/output.txt", "/tmp/fit.txt", option_string
    ), shell=True)
    # with open("/tmp/output.txt") as fin:
    #    result = fin.read()
    result = read_output("/tmp/output.txt")
    metabolite_names, fit_data = read_fit_file("/tmp/fit.txt")
    fit_results = _extract_fit_data(data, metabolite_names, fit_data)
    result["plots"] = fit_results
    return result
<commit_msg>BUG: Pipe Tarquin output to /dev/null<commit_after>from suspect import MRSSpectrum

import subprocess
import parse
import re
import numpy as np

def save_dpt(filename, data):
    with open(filename, 'wb') as fout:
        fout.write("Dangerplot_version\t1.0\n".encode())
        fout.write("Number_of_points\t{}\n".format(data.np).encode())
        fout.write("Sampling_frequency\t{0:8.8e}\n".format(1.0 / data.dt).encode())
        fout.write("Transmitter_frequency\t{0:8.8e}\n".format(data.f0 * 1e6).encode())
        fout.write("Phi0\t{0:8.8e}\n".format(0).encode())
        fout.write("Phi1\t{0:8.8e}\n".format(0).encode())
        fout.write("PPM_reference\t{0:8.8e}\n".format(data.ppm0).encode())
        fout.write("Echo_time\t{0:8.8e}\n".format(data.te * 1e-3).encode())
        fout.write("Real_FID\tImag_FID\t\n".encode())
        for x in data:
            fout.write("{0.real:8.8e} {0.imag:8.8e}\n".format(x).encode())

def read_output(filename):
    """Reads in a Tarquin txt results file and returns a dict of the information

    Parameters
    ----------
    filename : txt file
        The filename to read from

    Returns
    -------
    result : dict

    """
    with open(filename) as fin:
        data = fin.read()

        metabolite_fits = {}
        fit_quality = {}

        sections = data.split("\n\n")

        # first section is the metabolite concentrations
        metabolite_lines = sections[0].splitlines()[2:]
        for line in metabolite_lines:
            name, concentration, pc_sd, sd = line.split()
            metabolite_fits[name] = {
                "concentration": concentration,
                "sd": pc_sd,
            }

        # second section is the fit quality
        lines = sections[1].splitlines()[2:]
        for line in lines:
            key, value = line.split(":")
            fit_quality[key.strip()] = float(value.strip())

        return {
            "metabolite_fits": metabolite_fits,
            "quality": fit_quality
        }

def read_fit_file(filename):
    """Reads in a Tarquin txt fit file and returns all the fitted plots

    Parameters
    ----------
    filename : txt file
        The filename to read from

    Returns
    -------
    result : dict
    """
    with open(filename) as fin:
        contents = fin.read()

    # we don't know the exact size and shape of the grid of voxels
    # we are reading in yet, so start by making a dictionary of
    # positions and arrange them later
    voxel_dict = {}
    metabolite_names = None

    # the file is divided into data from each voxel sequentially
    # we want to split the file into separate voxel components
    # each voxel data starts with Row : x, Col : y ...
    # use re.split rather than str.split to avoid getting an
    # empty string before the first Row, by matching the new line
    # followed by Row instead.
    for voxel_string in re.split("\n(?=Row)", contents):

        # first two lines contain metadata, the rest is spectral points
        position_string, metabolite_string, data_string = voxel_string.split("\n", 2)
        position = parse.parse("Row : {:d}, Col : {:d}, Slice : {:d}", position_string)

        # metabolite names are only necessary once
        # also, the first four are fixed (ppm, data, fit, baseline)
        # only collect the rest
        if metabolite_names is None:
            metabolite_names = [name.strip() for name in metabolite_string.split(",")[4:]]

        voxel_dict[position] = np.array([[float(j) for j in i.split(',')] for i in data_string.splitlines()])

    # get the maximum row, col and slice
    max_row = max([pos[0] for pos in voxel_dict.keys()])
    max_col = max([pos[1] for pos in voxel_dict.keys()])
    max_slice = max([pos[2] for pos in voxel_dict.keys()])

    voxel_shape = voxel_dict.values().__iter__().__next__().shape
    combined_data = np.zeros((max_row, max_col, max_slice, *voxel_shape), 'complex')
    for (row, col, slc), voxel_data in voxel_dict.items():
        combined_data[row - 1, col - 1, slc - 1] = voxel_data

    return metabolite_names, combined_data

def _extract_fit_data(ref_data, metabolite_names, combined_data):
    # helper function to save space
    def make_spectrum(spectrum_data):
        return MRSSpectrum(spectrum_data,
                           ref_data.dt,
                           ref_data.f0,
                           ref_data.te,
                           ref_data.ppm0)

    # split out the individual lines (data, fit, baseline etc.)
    data = make_spectrum(combined_data[:, :, :, :, 1].squeeze())
    fit = make_spectrum(combined_data[:, :, :, :, 2].squeeze())
    baseline = make_spectrum(combined_data[:, :, :, :, 3].squeeze())
    metabolite_dict = {
        metabolite_name: make_spectrum(combined_data[:, :, :, :, i + 4].squeeze())
        for i, metabolite_name in enumerate(metabolite_names)
    }
    return {
        "data": data,
        "fit": fit,
        "baseline": baseline,
        "metabolites": metabolite_dict
    }

def process(data, options={}):
    save_dpt("/tmp/temp.dpt", data)
    option_string = ""
    for key, value in options.items():
        option_string += " --{} {}".format(key, value)
    subprocess.run("tarquin --input {} --format dpt --output_txt {} --output_fit {}{}".format(
        "/tmp/temp.dpt", "/tmp/output.txt", "/tmp/fit.txt", option_string
    ), shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
    # with open("/tmp/output.txt") as fin:
    #    result = fin.read()
    result = read_output("/tmp/output.txt")
    metabolite_names, fit_data = read_fit_file("/tmp/fit.txt")
    fit_results = _extract_fit_data(data, metabolite_names, fit_data)
    result["plots"] = fit_results
    return result
<|endoftext|>


<commit_before>import l_bp

class Breakpoint:
    '''
    Class for storing information about Breakpoints for merging
    '''
    def __init__(self, line, percent_slop=0, fixed_slop=0):
        '''
        Initialize with slop for probabilities
        '''
        self.l = line

        (self.sv_type,
        self.chr_l,
        self.chr_r,
        self.strands,
        self.start_l,
        self.end_l,
        self.start_r,
        self.end_r,
        m) = l_bp.split_v(line)

        self.p_l = self.floats_from_tag(m, 'PRPOS')
        self.p_r = self.floats_from_tag(m, 'PREND')

        slop_prob = 1e-100 # FIXME This is a constant. Pull out to make more obvious
        if ((percent_slop > 0) or (fixed_slop > 0)):

            l_slop = int(max(percent_slop * (self.end_l - self.start_l + 1), fixed_slop))
            r_slop = int(max(percent_slop * (self.end_r - self.start_r + 1), fixed_slop))

            # pad each interval with slop_prob on each side. TODO This should be a method
            self.start_l = self.start_l - l_slop
            self.end_l = self.end_l + l_slop
            new_p_l = [slop_prob] * l_slop + self.p_l + [slop_prob] * l_slop

            self.start_r = self.start_r - r_slop
            self.end_r = self.end_r + r_slop
            new_p_r = [slop_prob] * r_slop + self.p_r + [slop_prob] * r_slop

            # chew off overhang if self.start_l or self.start_r less than 0 TODO This should also be a method
            if self.start_l < 0:
                new_p_l = new_p_l[-self.start_l:]
                self.start_l = 0
            if self.start_r < 0:
                new_p_r = new_p_r[-self.start_r:]
                self.start_r = 0

            # normalize so each probability curve sums to 1. TODO Should be a method
            sum_p_l = sum(new_p_l)
            self.p_l = [float(x)/sum_p_l for x in new_p_l]
            sum_p_r = sum(new_p_r)
            self.p_r = [float(x)/sum_p_r for x in new_p_r]

    def __str__(self):
        '''
        Convert back to a string
        '''
        return '\t'.join([str(x) for x in [self.chr_l,
                                           self.start_l,
                                           self.end_l,
                                           self.chr_r,
                                           self.start_r,
                                           self.end_r,
                                           self.sv_type,
                                           self.strands,
                                           self.p_l,
                                           self.p_r]])
    def ovl(self, b):
        '''
        Calculate overlapping cumulative probability value as weight?
        0 if not overlapping.
        '''
        if ((self.chr_l != b.chr_l) or
            (self.chr_r != b.chr_r) or
            (self.sv_type != b.sv_type)):
                return 0
        #get left common interval
        c_start_l, c_end_l = max(self.start_l, b.start_l), min(self.end_l, b.end_l)
        #get right common interval
        c_start_r, c_end_r = max(self.start_r, b.start_r), min(self.end_r, b.end_r)

        c_l_len = c_end_l - c_start_l + 1
        c_r_len = c_end_r - c_start_r + 1

        if (c_l_len < 1) or (c_r_len < 1):
            return 0

        # TODO This should probably be a method as well
        self_start_off_l = c_start_l - self.start_l
        b_start_off_l = c_start_l - b.start_l

        self_start_off_r = c_start_r - self.start_r
        b_start_off_r = c_start_r - b.start_r

        ovl_l = 0
        for i in range(c_l_len):
            ovl_l += min(self.p_l[i + self_start_off_l], b.p_l[i + b_start_off_l])

        ovl_r = 0
        for i in range(c_r_len):
            ovl_r += min(self.p_r[i + self_start_off_r], b.p_r[i + b_start_off_r])

        return ovl_l * ovl_r

    @staticmethod
    def floats_from_tag(info_dict, tag):
        if tag in info_dict:
            return [float(x) for x in info_dict[tag].split(',')]
        else:
            raise RuntimeError('Required tag {0} not found.'.format(tag))
<commit_msg>Pull out additional static methods<commit_after>import l_bp

class Breakpoint:
    '''
    Class for storing information about Breakpoints for merging
    '''
    # Constant value for slop padding
    SLOP_PROB = 1e-100

    def __init__(self, line, percent_slop=0, fixed_slop=0):
        '''
        Initialize with slop for probabilities
        '''
        self.l = line

        (self.sv_type,
        self.chr_l,
        self.chr_r,
        self.strands,
        self.start_l,
        self.end_l,
        self.start_r,
        self.end_r,
        m) = l_bp.split_v(line)

        self.p_l = self.floats_from_tag(m, 'PRPOS')
        self.p_r = self.floats_from_tag(m, 'PREND')

        if ((percent_slop > 0) or (fixed_slop > 0)):

            self.start_l, self.end_l, new_p_l = self.pad_slop(self.start_l, self.end_l, self.p_l, percent_slop, fixed_slop)
            self.start_r, self.end_r, new_p_r = self.pad_slop(self.start_r, self.end_r, self.p_r, percent_slop, fixed_slop)

            # chew off overhang if self.start_l or self.start_r less than 0 TODO This should also be a method
            self.start_l, new_p_l = self.trim_slop(self.start_l, new_p_l)
            self.start_r, new_p_r = self.trim_slop(self.start_r, new_p_r)

            # normalize so each probability curve sums to 1. TODO Should be a method
            self.p_l = self.normalize(new_p_l)
            self.p_r = self.normalize(new_p_r)

    def __str__(self):
        '''
        Convert back to a string
        '''
        return '\t'.join([str(x) for x in [self.chr_l,
                                           self.start_l,
                                           self.end_l,
                                           self.chr_r,
                                           self.start_r,
                                           self.end_r,
                                           self.sv_type,
                                           self.strands,
                                           self.p_l,
                                           self.p_r]])
    def ovl(self, b):
        '''
        Calculate overlapping cumulative probability value as weight?
        0 if not overlapping.
        '''
        if ((self.chr_l != b.chr_l) or
            (self.chr_r != b.chr_r) or
            (self.sv_type != b.sv_type)):
                return 0
        #get left common interval
        c_start_l, c_end_l = max(self.start_l, b.start_l), min(self.end_l, b.end_l)
        #get right common interval
        c_start_r, c_end_r = max(self.start_r, b.start_r), min(self.end_r, b.end_r)

        c_l_len = c_end_l - c_start_l + 1
        c_r_len = c_end_r - c_start_r + 1

        if (c_l_len < 1) or (c_r_len < 1):
            return 0

        # TODO This should probably be a method as well
        self_start_off_l = c_start_l - self.start_l
        b_start_off_l = c_start_l - b.start_l

        self_start_off_r = c_start_r - self.start_r
        b_start_off_r = c_start_r - b.start_r

        ovl_l = 0
        for i in range(c_l_len):
            ovl_l += min(self.p_l[i + self_start_off_l], b.p_l[i + b_start_off_l])

        ovl_r = 0
        for i in range(c_r_len):
            ovl_r += min(self.p_r[i + self_start_off_r], b.p_r[i + b_start_off_r])

        return ovl_l * ovl_r

    @staticmethod
    def floats_from_tag(info_dict, tag):
        if tag in info_dict:
            return [float(x) for x in info_dict[tag].split(',')]
        else:
            raise RuntimeError('Required tag {0} not found.'.format(tag))

    @staticmethod
    def pad_slop(start, end, p, percent_slop, fixed_slop):
        slop = int(max(percent_slop * (end - start + 1), fixed_slop))
        new_start = start - slop
        new_end = end + slop
        new_p = [Breakpoint.SLOP_PROB] * slop + p + [Breakpoint.SLOP_PROB] * slop
        return (new_start, new_end, new_p)

    @staticmethod
    def trim_slop(start, p):
        if start < 0:
            new_p = p[-start:]
            return (0, new_p)
        else:
            return (start, p)

    @staticmethod
    def normalize(p):
        sum_p = sum(p)
        return [float(x)/sum_p for x in p]
<|endoftext|>


<commit_before># Monocyte - Monocyte - Search and Destroy unwanted AWS Resources relentlessly.
# Copyright 2015 Immobilien Scout GmbH
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
import warnings
import logging

class Resource(object):
    def __init__(self, resource, resource_type, resource_id, creation_date, region=None):
        self.wrapped = resource
        self.region = region
        self.resource_type = resource_type
        self.resource_id = resource_id
        self.creation_date = creation_date

HANDLER_PREFIX = "monocyte.handler."

class Handler(object):

    def __init__(self, region_filter, dry_run=True, logger=None, ignored_resources=None):
        warnings.filterwarnings('error')
        self.region_filter = region_filter
        self.regions = [region for region in self.fetch_regions() if self.region_filter(region.name)]
        self.dry_run = dry_run
        self.ignored_resources = ignored_resources or []
        self.logger = logger or logging.getLogger(__name__)

    @property
    def resource_type(self):
        return "%s %s" % (self.__class__.__module__, self.__class__.__name__)

    @property
    def name(self):
        full_name = "%s.%s" % (self.__class__.__module__, self.__class__.__name__)
        return full_name.replace(HANDLER_PREFIX, "")

    def fetch_regions(self):
        raise NotImplementedError("Should have implemented this")

    def fetch_unwanted_resources(self):
        raise NotImplementedError("Should have implemented this")

    def to_string(self, resource):
        raise NotImplementedError("Should have implemented this")

    def delete(self, resource):
        raise NotImplementedError("Should have implemented this")
<commit_msg>PIO-125 monocyte code formation<commit_after># Monocyte - Monocyte - Search and Destroy unwanted AWS Resources relentlessly.
# Copyright 2015 Immobilien Scout GmbH
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
import warnings
import logging

class Resource(object):
    def __init__(self, resource, resource_type, resource_id, creation_date,
                 region=None):
        self.wrapped = resource
        self.region = region
        self.resource_type = resource_type
        self.resource_id = resource_id
        self.creation_date = creation_date

HANDLER_PREFIX = "monocyte.handler."

class Handler(object):

    def __init__(self, region_filter, dry_run=True, logger=None, ignored_resources=None):
        warnings.filterwarnings('error')
        self.region_filter = region_filter
        self.regions = [region for region in self.fetch_regions() if self.region_filter(region.name)]
        self.dry_run = dry_run
        self.ignored_resources = ignored_resources or []
        self.logger = logger or logging.getLogger(__name__)

    @property
    def resource_type(self):
        return "%s %s" % (self.__class__.__module__, self.__class__.__name__)

    @property
    def name(self):
        full_name = "%s.%s" % (self.__class__.__module__, self.__class__.__name__)
        return full_name.replace(HANDLER_PREFIX, "")

    def fetch_regions(self):
        raise NotImplementedError("Should have implemented this")

    def fetch_unwanted_resources(self):
        raise NotImplementedError("Should have implemented this")

    def to_string(self, resource):
        raise NotImplementedError("Should have implemented this")

    def delete(self, resource):
        raise NotImplementedError("Should have implemented this")
<|endoftext|>


<commit_before># Monocyte - Search and Destroy unwanted AWS Resources relentlessly.
# Copyright 2015 Immobilien Scout GmbH
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import datetime

import boto
import boto.dynamodb2
import boto.dynamodb2.exceptions

from monocyte.handler import Resource, Handler

class Table(Handler):
    def fetch_regions(self):
        return boto.dynamodb2.regions()

    def fetch_unwanted_resources(self):
        for region in self.regions:
            connection = boto.dynamodb2.connect_to_region(region.name)
            names = connection.list_tables(limit=100) or {}  # TODO ahilmann: what happens with more than 100 tables per region?
            for name in names.get("TableNames"):
                resource = connection.describe_table(name)
                yield Resource(resource["Table"], region.name)

    def to_string(self, resource):
        table = resource.wrapped
        return "DynamoDB Table found in {0}, ".format(resource.region) + \
               "with name {0}, created {1}, with state {2}".format(
                   table["TableName"],
                   datetime.datetime.fromtimestamp(table["CreationDateTime"]).strftime('%Y-%m-%d %H:%M:%S.%f'),
                   table["TableStatus"])

    def delete(self, resource):
        if self.dry_run:
            return
        connection = boto.dynamodb2.connect_to_region(resource.region)
        self.logger.info("Initiating deletion sequence for {0}.".format(resource.wrapped["TableName"]))
        try:
            connection.delete_table(resource.wrapped["TableName"])
        except boto.dynamodb2.exceptions.ResourceInUseException as exc:
            self.logger.exception(exc)
<commit_msg>process todo: just process the first 100 resources, leave the rest for the next monocyte run<commit_after># Monocyte - Search and Destroy unwanted AWS Resources relentlessly.
# Copyright 2015 Immobilien Scout GmbH
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import datetime

import boto
import boto.dynamodb2
import boto.dynamodb2.exceptions

from monocyte.handler import Resource, Handler

class Table(Handler):
    def fetch_regions(self):
        return boto.dynamodb2.regions()

    def fetch_unwanted_resources(self):
        for region in self.regions:
            connection = boto.dynamodb2.connect_to_region(region.name)
            names = connection.list_tables(limit=100) or {}
            for name in names.get("TableNames"):
                resource = connection.describe_table(name)
                yield Resource(resource["Table"], region.name)

    def to_string(self, resource):
        table = resource.wrapped
        return "DynamoDB Table found in {0}, ".format(resource.region) + \
               "with name {0}, created {1}, with state {2}".format(
                   table["TableName"],
                   datetime.datetime.fromtimestamp(table["CreationDateTime"]).strftime('%Y-%m-%d %H:%M:%S.%f'),
                   table["TableStatus"])

    def delete(self, resource):
        if self.dry_run:
            return
        connection = boto.dynamodb2.connect_to_region(resource.region)
        self.logger.info("Initiating deletion sequence for {0}.".format(resource.wrapped["TableName"]))
        try:
            connection.delete_table(resource.wrapped["TableName"])
        except boto.dynamodb2.exceptions.ResourceInUseException as exc:
            self.logger.exception(exc)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: sw=4:ts=4:expandtab

"""
bump
~~~~

An automated way to follow the Semantic Versioning Specification

Examples:
    basic usage::

        >>> from os import path as p
        >>> project = Project(p.abspath(p.dirname(p.dirname(__file__))))
        >>> project.current_version  == __version__
        True

Attributes:
    DEFAULT_TAG_FMT (str): The default tag format
    DEFAULT_TAG_MSG_FMT (str): The default tag message format
    DEFAULT_COMMIT_MSG_FMT (str): The default commit message format
"""

from __future__ import (
    absolute_import, division, print_function, with_statement,
    unicode_literals)

import os
import semver
import pygogo as gogo

from fnmatch import fnmatch
from subprocess import CalledProcessError

from .git_utils import Git
from .shell_utils import sh

__version__ = '1.4.0'

__title__ = 'bump'
__author__ = 'Reuben Cummings'
__description__ = 'An automated way to follow the Semantic Versioning'
__description__ += 'Specification'
__email__ = 'reubano@gmail.com'
__license__ = 'MIT'
__copyright__ = 'Copyright 2015 Reuben Cummings'

DEFAULT_TAG_FMT = 'v{version}'
DEFAULT_TAG_MSG_FMT = 'Version {version} Release'
DEFAULT_COMMIT_MSG_FMT = 'bump to version {version}'

logger = gogo.Gogo(__name__).logger

class Project(Git):
    """
    class representing a project object.
    """

    def __init__(self, dir_, file_=None, version=None, verbose=False):
        """
        Parameters
        ----------
        dir : str
            the project directory

        file_ : str
            the file to search for a version

        Examples
        --------
        >>> Project(os.curdir)  # doctest: +ELLIPSIS
        <bump.Project object at 0x...>
        """
        if not os.path.isdir(dir_):
            raise Exception('%s is not a directory' % dir_)

        super(Project, self).__init__(dir_, verbose)

        self.bumped = False
        self.file = file_

        if version:
            self.version = version
        else:
            self.version = self.current_version

    @property
    def current_version(self):
        """
        :returns: string of the current version parsed from most recent git tag
        """
        # what to do on first time run? no tags yet..
        if self.current_tag:
            version = self.current_tag.lstrip('v')
        else:
            version = None

        if version and not version_is_valid(version):
            version = None

        return version

    @property
    def versions(self):
        """
        :returns: iterator of all valid versions parsed from the git tags
        """
        versions = (t.lstrip('v') for t in self.tags)
        return (v for v in versions if version_is_valid(v))

    def gen_versioned_files(self, wave):
        if self.file:
            yield self.file
        else:
            wave_one = [
                '*.spec', 'setup.cfg', 'setup.py', '*/__init__.py',
                '*.xml', '*.json']

            switch = {1: wave_one, 2: ['*.php', '*.py']}

            for git_file in self.files:
                if any(fnmatch(git_file, file) for file in switch[wave]):
                    yield git_file

    def set_versions(self, new_version, wave=1):
        for file_ in self.gen_versioned_files(wave):
            if not self.version:
                # get all lines in file
                cmd = 'grep -ine "" %s' % file_

                try:
                    lines = sh(cmd, True, path=self.dir)
                except CalledProcessError:
                    lines = None

                if lines:
                    # escape double quotes
                    escaped = lines.replace('"', '\\"')
                    # find first line containing a version number and the word
                    # 'version'
                    cmd = 'echo "%s" | grep version' % escaped
                    cmd += ' | grep -m1 "[0-9]*\.[0-9]*\.[0-9]*"'

                    try:
                        rep_line = sh(cmd, True, path=self.dir)
                    except Exception:
                        cmd = None
                    else:
                        rep_line_num = rep_line.split(':')[0]
                        # replace with new version number
                        cmd = ("sed -i '' '%ss/[0-9]*\.[0-9]*\.[0-9]*/%s/g' %s"
                            % (rep_line_num, new_version, file_))
                else:
                    cmd = None
            else:
                # replace current version with new version only if the line
                # contains the word 'version'
                cmd = ("sed -i '' '/version/s/%s/%s/g' %s"
                    % (self.version, new_version, file_))

            sh(cmd, path=self.dir) if cmd else None

        self.bumped = self.is_dirty

    def bump(self, bump_type):
        """
        Parameters
        ----------
        version_num: string version name.
        bump_type: version bump type. one of:
            m = major - [x].0.0
            n = minor - x.[y].0
            p = patch - x.y.[z]

        Returns
        -------
        concatenated string of the incremented version name.
        """
        switch = {
            'm': semver.bump_major,
            'n': semver.bump_minor,
            'p': semver.bump_patch}

        new_version = switch.get(bump_type)(self.version)

        if new_version in set(self.versions):
            self.logger.error('version `%s` already present', new_version)
            new_version = None

        return new_version

def version_is_valid(version):
    try:
        return semver.parse(version)
    except ValueError as err:
        logger.error(err.message)
        return False
<commit_msg>BUGFIX: return if no version given<commit_after># -*- coding: utf-8 -*-
# vim: sw=4:ts=4:expandtab

"""
bump
~~~~

An automated way to follow the Semantic Versioning Specification

Examples:
    basic usage::

        >>> from os import path as p
        >>> project = Project(p.abspath(p.dirname(p.dirname(__file__))))
        >>> project.current_version  == __version__
        True

Attributes:
    DEFAULT_TAG_FMT (str): The default tag format
    DEFAULT_TAG_MSG_FMT (str): The default tag message format
    DEFAULT_COMMIT_MSG_FMT (str): The default commit message format
"""

from __future__ import (
    absolute_import, division, print_function, with_statement,
    unicode_literals)

import os
import semver
import pygogo as gogo

from fnmatch import fnmatch
from subprocess import CalledProcessError

from .git_utils import Git
from .shell_utils import sh

__version__ = '1.4.0'

__title__ = 'bump'
__author__ = 'Reuben Cummings'
__description__ = 'An automated way to follow the Semantic Versioning'
__description__ += 'Specification'
__email__ = 'reubano@gmail.com'
__license__ = 'MIT'
__copyright__ = 'Copyright 2015 Reuben Cummings'

DEFAULT_TAG_FMT = 'v{version}'
DEFAULT_TAG_MSG_FMT = 'Version {version} Release'
DEFAULT_COMMIT_MSG_FMT = 'bump to version {version}'

logger = gogo.Gogo(__name__).logger

class Project(Git):
    """
    class representing a project object.
    """

    def __init__(self, dir_, file_=None, version=None, verbose=False):
        """
        Parameters
        ----------
        dir : str
            the project directory

        file_ : str
            the file to search for a version

        Examples
        --------
        >>> Project(os.curdir)  # doctest: +ELLIPSIS
        <bump.Project object at 0x...>
        """
        if not os.path.isdir(dir_):
            raise Exception('%s is not a directory' % dir_)

        super(Project, self).__init__(dir_, verbose)

        self.bumped = False
        self.file = file_

        if version:
            self.version = version
        else:
            self.version = self.current_version

    @property
    def current_version(self):
        """
        :returns: string of the current version parsed from most recent git tag
        """
        # what to do on first time run? no tags yet..
        if self.current_tag:
            version = self.current_tag.lstrip('v')
        else:
            version = None

        if version and not version_is_valid(version):
            version = None

        return version

    @property
    def versions(self):
        """
        :returns: iterator of all valid versions parsed from the git tags
        """
        versions = (t.lstrip('v') for t in self.tags)
        return (v for v in versions if version_is_valid(v))

    def gen_versioned_files(self, wave):
        if self.file:
            yield self.file
        else:
            wave_one = [
                '*.spec', 'setup.cfg', 'setup.py', '*/__init__.py',
                '*.xml', '*.json']

            switch = {1: wave_one, 2: ['*.php', '*.py']}

            for git_file in self.files:
                if any(fnmatch(git_file, file) for file in switch[wave]):
                    yield git_file

    def set_versions(self, new_version, wave=1):
        if not new_version:
            return

        for file_ in self.gen_versioned_files(wave):
            if not self.version:
                # get all lines in file
                cmd = 'grep -ine "" %s' % file_

                try:
                    lines = sh(cmd, True, path=self.dir)
                except CalledProcessError:
                    lines = None

                if lines:
                    # escape double quotes
                    escaped = lines.replace('"', '\\"')
                    # find first line containing a version number and the word
                    # 'version'
                    cmd = 'echo "%s" | grep version' % escaped
                    cmd += ' | grep -m1 "[0-9]*\.[0-9]*\.[0-9]*"'

                    try:
                        rep_line = sh(cmd, True, path=self.dir)
                    except Exception:
                        cmd = None
                    else:
                        rep_line_num = rep_line.split(':')[0]
                        # replace with new version number
                        cmd = ("sed -i '' '%ss/[0-9]*\.[0-9]*\.[0-9]*/%s/g' %s"
                            % (rep_line_num, new_version, file_))
                else:
                    cmd = None
            else:
                # replace current version with new version only if the line
                # contains the word 'version'
                cmd = ("sed -i '' '/version/s/%s/%s/g' %s"
                    % (self.version, new_version, file_))

            sh(cmd, path=self.dir) if cmd else None

        self.bumped = self.is_dirty

    def bump(self, bump_type):
        """
        Parameters
        ----------
        version_num: string version name.
        bump_type: version bump type. one of:
            m = major - [x].0.0
            n = minor - x.[y].0
            p = patch - x.y.[z]

        Returns
        -------
        concatenated string of the incremented version name.
        """
        switch = {
            'm': semver.bump_major,
            'n': semver.bump_minor,
            'p': semver.bump_patch}

        new_version = switch.get(bump_type)(self.version)

        if new_version in set(self.versions):
            self.logger.error('version `%s` already present', new_version)
            new_version = None

        return new_version

def version_is_valid(version):
    try:
        return semver.parse(version)
    except ValueError as err:
        logger.error(err.message)
        return False
<|endoftext|>


<commit_before># the code below is mostly copied from django's db backend code
import os
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured

__all__ = ('backend')

if not settings.RCS_BACKEND:
    settings.RCS_BACKEND = 'dummy'

try:
    # Try to import one of the bundled backends
    _import_path = 'rcsfield.backends.'
    backend = __import__('%s%s' % (_import_path, settings.RCS_BACKEND), {}, {}, [''])
except ImportError, e:
    # If the above import failed try to load an external backend
    try:
        _import_path = ''
        backend = __import__('%s%s' % (_import_path, settings.RCS_BACKEND), {}, {}, [''])
    except ImportError, e_user:
        # No backend found, display an error message and a list of all
        # bundled backends.
        backend_dir = __path__[0]
        available_backends = [f.split('.py')[0] for f in os.listdir(backend_dir) if not f.startswith('_') and not f.startswith('.') and not f.endswith('.pyc')]
        available_backends.sort()
        if settings.RCS_BACKEND not in available_backends:
            raise ImproperlyConfigured("%s isn't an available revision control (rcsfield) backend. Available options are: %s" % \
                                        (settings.RCS_BACKEND, ', '.join(map(repr, available_backends))))
        else:
            raise
    <commit_msg>made backend loading mechanism a bit more robust<commit_after># the code below is mostly copied from django's db backend code
import os
from django.conf import settings
from django.core.exceptions import ImproperlyConfigured

__all__ = ('backend')

RCS_BACKEND = getattr(settings, 'RCS_BACKEND', 'dummy')

try:
    # Try to import one of the bundled backends
    _import_path = 'rcsfield.backends.'
    backend = __import__('%s%s' % (_import_path, RCS_BACKEND), {}, {}, [''])
except ImportError, e:
    # If the above import failed try to load an external backend
    try:
        _import_path = ''
        backend = __import__('%s%s' % (_import_path, RCS_BACKEND), {}, {}, [''])
    except ImportError, e_user:
        # No backend found, display an error message and a list of all
        # bundled backends.
        backend_dir = __path__[0]
        available_backends = [f.split('.py')[0] for f in os.listdir(backend_dir) if not f.startswith('_') and not f.startswith('.') and not f.endswith('.pyc')]
        available_backends.sort()
        if RCS_BACKEND not in available_backends:
            raise ImproperlyConfigured("%s isn't an available revision control (rcsfield) backend. Available options are: %s" % \
                                        (RCS_BACKEND, ', '.join(map(repr, available_backends))))
        else:
            raise
<|endoftext|>