<commit_before>#! /usr/bin/python

from xbee import ZigBee
import serial
import struct
from datetime import datetime
import logging

logging.basicConfig(filename="./datalogger.log", format='%(asctime)s %(levelname)s:%(message)s', level=logging.INFO)

class zbDataLogger:
    def __init__(self, port='/dev/ttyUSB0', baud=9600, escaped=True):
        self.port = port
        self.baud = baud
        self.escaped = escaped
        try:
            self.serial_port = serial.Serial(self.port, self.baud)
            self.xbee = ZigBee(self.serial_port, escaped=self.escaped)
            logging.info("Successfully initialised ZigBee on " + self.port + " at " + str(self.baud) + " baud")
        except:
            logging.error("Unable to initialise Zigbee on " + self.port + " at " + str(self.baud) + " baud")
            raise
        self.frame = ""
        self.msg = {}
        self.data = ""
        self.appHandlers = {}

    def getMsg(self):
        self.frame = self.xbee.wait_read_frame() # blocking
        self.rfdata = self.frame.get("rf_data")
        logging.info("Got message")
        self.msg["logtime"] = datetime.isoformat(datetime.now())

        decodeHdr = struct.unpack("HHHH", self.rfdata[0:8])
        self.msg["appID"] = decodeHdr[0]
        self.msg["msgType"] = decodeHdr[1]
        self.msg["reserved"] = decodeHdr[2]
        self.msg["length"] = decodeHdr[3]
        self.msg["data"] = self.rfdata[8:]
        if self.msg["length"] != len(self.msg["data"]):
            logging.error("Incorrect data length in received packet. Rx: " + str(len(self.msg["data"])) + " Expected: " + str(self.header["length"]))
        else:
            if self.msg["appID"] in self.appHandlers:
                logging.info("Handling application ID: " + str(self.msg["appID"]))
                return self.appHandlers[self.msg["appID"]].decode(self.msg)
            else:
                logging.warn("No handler registered for appID " + str(self.msg["appID"]) + " dropping message...")
        return []

    def register(self, appID, handler):
        self.appHandlers[appID] = handler
        logging.info("Registered handler for appID: " + str(appID))

# a handler class to allow indirection of message handling
class appHandler:
    def __init__(self, parent, appID):
        self.parent = parent
        self.appID = appID
        self.msgHandlers = {}
        self.parent.register(self.appID, self)

    def register(self, msgType, handler):
        self.msgHandlers[msgType] = handler
        logging.info("Registered handler for msgType: " + str(msgType))

    def decode(self, msg):
        if self.appID != msg["appID"]:
            logging.error("Passed a message that I didn't register for.  My appID: " + str(self.appID) + ", message appID " + str(msg["appID"]))
        else:
            if msg["msgType"] in self.msgHandlers:
                logging.info("Handling message type: " + str(msg["msgType"]))
                return self.msgHandlers[msg["msgType"]].decode(msg)
            else:
                logging.warn("No handler registered for msgType " + str(msg["msgType"]) + " dropping message...")
        return []

class msgHandler:
    def __init__(self, parent, msgTypes=[]):
        self.msgTypes = msgTypes
        for msgType in msgTypes:
            parent.register(msgType, self)

    def decode(self, msg):
        # this is where the final message decoding happens - return an object containing the message items
        # do the work here and add values into the msg Dictionary
        # you MUST also provide a "csv" member containing a CSV of the data to be logged
        return msg

    def createCSV(self, msg, listOfRefs):
        csv = []
        for ref in listOfRefs:
            if ref in msg:
                csv.append(str(msg[ref]))
            else:
                logging.error("Error creating CSV, requested item not available: " + str(ref))
        return ','.join(csv)

class weatherHandler(msgHandler):
    def __init__(self, parent):
        msgHandler.__init__(self, parent, [0x0001])

    def decode(self, msg):
        values = struct.unpack("Iihhhhhh", msg["data"])
        msg["millis"] = values[0]
        msg["pressure"] = values[1]/100.0
        msg["inT1"] = values[2]/100.0
        msg["inT2"] = values[3]/100.0
        msg["inT3"] = values[4]/100.0
        msg["outT"] = values[5]/100.0
        msg["humidity"] = values[6]/100.0
        msg["light"] = values[7]

        msg["csv"]= self.createCSV(msg, ["logtime", "millis", "inT1", "inT2", "inT3", "outT", "pressure", "humidity", "light"])

        return msg;

if __name__ == '__main__':
    zbdl = zbDataLogger()
    weatherApp = appHandler(zbdl, 0x10a1)
    weatherMsg = weatherHandler(weatherApp)
    with open('data.log', 'a', 1) as dataLogfile:
        while True:
            data = zbdl.getMsg()
            dataLogfile.write(data["csv"] + '\n')

<commit_msg>Updated log messages so that hex values are properly represented to easy tying up with Arduino code<commit_after>#! /usr/bin/python

from xbee import ZigBee
import serial
import struct
from datetime import datetime
import logging

logging.basicConfig(filename="./datalogger.log", format='%(asctime)s %(levelname)s:%(message)s', level=logging.INFO)

class zbDataLogger:
    def __init__(self, port='/dev/ttyUSB0', baud=9600, escaped=True):
        self.port = port
        self.baud = baud
        self.escaped = escaped
        try:
            self.serial_port = serial.Serial(self.port, self.baud)
            self.xbee = ZigBee(self.serial_port, escaped=self.escaped)
            logging.info("Successfully initialised ZigBee on " + self.port + " at " + str(self.baud) + " baud")
        except:
            logging.error("Unable to initialise Zigbee on " + self.port + " at " + str(self.baud) + " baud")
            raise
        self.frame = ""
        self.msg = {}
        self.data = ""
        self.appHandlers = {}

    def getMsg(self):
        self.frame = self.xbee.wait_read_frame() # blocking
        self.rfdata = self.frame.get("rf_data")
        logging.info("Got message")
        self.msg["logtime"] = datetime.isoformat(datetime.now())

        decodeHdr = struct.unpack("HHHH", self.rfdata[0:8])
        self.msg["appID"] = decodeHdr[0]
        self.msg["msgType"] = decodeHdr[1]
        self.msg["reserved"] = decodeHdr[2]
        self.msg["length"] = decodeHdr[3]
        self.msg["data"] = self.rfdata[8:]
        if self.msg["length"] != len(self.msg["data"]):
            logging.error("Incorrect data length in received packet. Rx: %s, Expected: %s" % (len(self.msg["data"]), self.header["length"]))
        else:
            if self.msg["appID"] in self.appHandlers:
                logging.info("Handling application ID: 0x%0.4X" % self.msg["appID"])
                return self.appHandlers[self.msg["appID"]].decode(self.msg)
            else:
                logging.warn("No handler registered for appID 0x%0.4X, dropping message..." % self.msg["appID"])
        return []

    def register(self, appID, handler):
        self.appHandlers[appID] = handler
        logging.info("Registered handler for appID: 0x%0.4X" % appID)

# a handler class to allow indirection of message handling
class appHandler:
    def __init__(self, parent, appID):
        self.parent = parent
        self.appID = appID
        self.msgHandlers = {}
        self.parent.register(self.appID, self)

    def register(self, msgType, handler):
        self.msgHandlers[msgType] = handler
        logging.info("Registered handler for msgType: 0x%0.4X" % msgType)

    def decode(self, msg):
        if self.appID != msg["appID"]:
            logging.error("Passed a message that I didn't register for.  My appID: 0x%0.4X, message appID: 0x%0.4X" % (self.appID, msg["appID"]))
        else:
            if msg["msgType"] in self.msgHandlers:
                logging.info("Handling message type: 0x%0.4X" % msg["msgType"])
                return self.msgHandlers[msg["msgType"]].decode(msg)
            else:
                logging.warn("No handler registered for msgType 0x%0.4X, dropping message..." % msg["msgType"])
        return []

class msgHandler:
    def __init__(self, parent, msgTypes=[]):
        self.msgTypes = msgTypes
        for msgType in msgTypes:
            parent.register(msgType, self)

    def decode(self, msg):
        # this is where the final message decoding happens - return an object containing the message items
        # do the work here and add values into the msg Dictionary
        # you MUST also provide a "csv" member containing a CSV of the data to be logged
        return msg

    def createCSV(self, msg, listOfRefs):
        csv = []
        for ref in listOfRefs:
            if ref in msg:
                csv.append(str(msg[ref]))
            else:
                logging.error("Error creating CSV, requested item not available: %s" % ref)
        return ','.join(csv)

class weatherHandler(msgHandler):
    def __init__(self, parent):
        msgHandler.__init__(self, parent, [0x0001])

    def decode(self, msg):
        values = struct.unpack("Iihhhhhh", msg["data"])
        msg["millis"] = values[0]
        msg["pressure"] = values[1]/100.0
        msg["inT1"] = values[2]/100.0
        msg["inT2"] = values[3]/100.0
        msg["inT3"] = values[4]/100.0
        msg["outT"] = values[5]/100.0
        msg["humidity"] = values[6]/100.0
        msg["light"] = values[7]

        msg["csv"]= self.createCSV(msg, ["logtime", "millis", "inT1", "inT2", "inT3", "outT", "pressure", "humidity", "light"])

        return msg;

if __name__ == '__main__':
    zbdl = zbDataLogger()
    weatherApp = appHandler(zbdl, 0x10a1)
    weatherMsg = weatherHandler(weatherApp)
    with open('data.log', 'a', 1) as dataLogfile:
        while True:
            data = zbdl.getMsg()
            dataLogfile.write(data["csv"] + '\n')

<|endoftext|>


<commit_before># Licensed under a 3-clause BSD style license - see LICENSE.rst
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

from ...extern import six

import io

from ..xml import check, unescaper, writer

def test_writer():
    fh = io.StringIO()
    w = writer.XMLWriter(fh)
    with w.tag("html"):
        with w.tag("body"):
            w.data("This is the content")
            w.comment("comment")

    value = ''.join(fh.getvalue().split())
    assert value == 'Thisisthecontent'

def test_check_id():
    assert check.check_id("Fof32")
    assert check.check_id("_Fof32")
    assert not check.check_id("32Fof")

def test_fix_id():
    assert check.fix_id("Fof32") == "Fof32"
    assert check.fix_id("@#f") == "___f"

def test_check_token():
    assert check.check_token("token")
    assert not check.check_token("token\rtoken")

def test_check_mime_content_type():
    assert check.check_mime_content_type("image/jpeg")
    assert not check.check_mime_content_type("image")

def test_check_anyuri():
    assert check.check_anyuri("https://github.com/astropy/astropy")

def test_unescape_all():
    # str
    url_in = 'http://casu.ast.cam.ac.uk/ag/iphas-dsa/SubmitCone?' \
             'DSACAT=IDR&amp;DSATAB=Emitters&amp;'
    url_out = 'http://casu.ast.cam.ac.uk/ag/iphas-dsa/SubmitCone?' \
              'DSACAT=IDR&DSATAB=Emitters&'
    assert unescaper.unescape_all(url_in) == url_out

    # bytes
    url_in = b'http://casu.ast.cam.ac.uk/ag/iphas-dsa/SubmitCone?' \
             b'DSACAT=IDR&amp;DSATAB=Emitters&amp;'
    url_out = b'http://casu.ast.cam.ac.uk/ag/iphas-dsa/SubmitCone?' \
              b'DSACAT=IDR&DSATAB=Emitters&'
    assert unescaper.unescape_all(url_in) == url_out

def test_escape_xml():
    s = writer.xml_escape('This & That')
    assert type(s) == six.text_type
    assert s == u'This & That'

    s = writer.xml_escape('This & That')
    assert type(s) == six.text_type
    assert s == 'This & That'

    s = writer.xml_escape(1)
    assert type(s) == six.text_type
    assert s == '1'
<commit_msg>Fix test on Python 3.2<commit_after># Licensed under a 3-clause BSD style license - see LICENSE.rst
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

from ...extern import six

import io

from ..xml import check, unescaper, writer

def test_writer():
    fh = io.StringIO()
    w = writer.XMLWriter(fh)
    with w.tag("html"):
        with w.tag("body"):
            w.data("This is the content")
            w.comment("comment")

    value = ''.join(fh.getvalue().split())
    assert value == 'Thisisthecontent'

def test_check_id():
    assert check.check_id("Fof32")
    assert check.check_id("_Fof32")
    assert not check.check_id("32Fof")

def test_fix_id():
    assert check.fix_id("Fof32") == "Fof32"
    assert check.fix_id("@#f") == "___f"

def test_check_token():
    assert check.check_token("token")
    assert not check.check_token("token\rtoken")

def test_check_mime_content_type():
    assert check.check_mime_content_type("image/jpeg")
    assert not check.check_mime_content_type("image")

def test_check_anyuri():
    assert check.check_anyuri("https://github.com/astropy/astropy")

def test_unescape_all():
    # str
    url_in = 'http://casu.ast.cam.ac.uk/ag/iphas-dsa/SubmitCone?' \
             'DSACAT=IDR&amp;DSATAB=Emitters&amp;'
    url_out = 'http://casu.ast.cam.ac.uk/ag/iphas-dsa/SubmitCone?' \
              'DSACAT=IDR&DSATAB=Emitters&'
    assert unescaper.unescape_all(url_in) == url_out

    # bytes
    url_in = b'http://casu.ast.cam.ac.uk/ag/iphas-dsa/SubmitCone?' \
             b'DSACAT=IDR&amp;DSATAB=Emitters&amp;'
    url_out = b'http://casu.ast.cam.ac.uk/ag/iphas-dsa/SubmitCone?' \
              b'DSACAT=IDR&DSATAB=Emitters&'
    assert unescaper.unescape_all(url_in) == url_out

def test_escape_xml():
    s = writer.xml_escape('This & That')
    assert type(s) == six.text_type
    assert s == 'This & That'

    s = writer.xml_escape('This & That')
    assert type(s) == six.text_type
    assert s == 'This & That'

    s = writer.xml_escape(1)
    assert type(s) == six.text_type
    assert s == '1'
<|endoftext|>


<commit_before>"""
A collection of small helper functions for generating small pieces of datatable output in custom
methods on a view.

Because of uncontrolled external use of ``preload_record_data()`` by the view, each of these utility
functions allows for generic ``*args`` to pass through to the function, even though they aren't used
in any way.

"""

def keyed_helper(helper):
    """
    Decorator for helper functions that operate on direct values instead of model instances.

    A keyed helper is one that can be used normally in the view's own custom callbacks, but also
    supports direct access in the column declaration, such as in the example:

        datatable_options = {
            'columns': [
                ('Field Name', 'fieldname', make_boolean_checkmark(key=attrgetter('fieldname'))),
            ],
        }

    With the help of a ``sort``-style ``key`` argument, the helper can receive all the information
    it requires in advance, so that the view doesn't have to go through the trouble of declaring
    a custom callback method that simply returns the value of the ``make_boolean_checkmark()``
    helper.

    If the attribute being fetched is identical to the one pointed to in the column declaration,
    even the ``key`` argument can be omitted:

        ('Field Name', 'fieldname', make_boolean_checkmark)),

    """

    def wrapper(instance=None, key=None, *args, **kwargs):
        if instance and not key:
            # if key:
            #     # Helper was provided in the column declaration without being instantiated and
            #     # given a "key" argument.  It received the "key" function implicitly.
            #     value = key(instance)
            # else:
            #     # Helper is only being used within in a custom callback
            value = instance
            return helper(value, *args, **kwargs)
        elif key and not instance:
            # Helper is used directly in the columns declaration.  A new callable is
            # returned to take the place of a callback.
            def helper_wrapper(instance, *args, **kwargs):
                return helper(key(instance), *args, **kwargs)
            return helper_wrapper
        elif not key and not instance:
            # Helper was provided in the column declaration and was called in place with no
            # arguments.  We return the helper back, negating the premature call.
            return wrapper
    wrapper._is_wrapped = True
    return wrapper

def link_to_model(instance, text=None, *args, **kwargs):
    if not text:
        text = kwargs['default_value'] or unicode(instance)
    return """{}""".format(instance.get_absolute_url(), text)

@keyed_helper
def make_boolean_checkmark(value, false_value="", *args, **kwargs):
    if value:
        return "✔"
    return false_value

def attrgetter(attr):
    def helper(instance, *args, **kwargs):
        value = getattr(instance, attr)

        if callable(value):
            return value()
        return value

    return helper

def format_date(format, key=None):
    def helper(value, *args, **kwargs):
        if key:
            value = key(value)
        return value.strftime(format)

    if key:
        return helper
    return keyed_helper(helper)
<commit_msg>Changed name of format_date argument<commit_after>"""
A collection of small helper functions for generating small pieces of datatable output in custom
methods on a view.

Because of uncontrolled external use of ``preload_record_data()`` by the view, each of these utility
functions allows for generic ``*args`` to pass through to the function, even though they aren't used
in any way.

"""

def keyed_helper(helper):
    """
    Decorator for helper functions that operate on direct values instead of model instances.

    A keyed helper is one that can be used normally in the view's own custom callbacks, but also
    supports direct access in the column declaration, such as in the example:

        datatable_options = {
            'columns': [
                ('Field Name', 'fieldname', make_boolean_checkmark(key=attrgetter('fieldname'))),
            ],
        }

    With the help of a ``sort``-style ``key`` argument, the helper can receive all the information
    it requires in advance, so that the view doesn't have to go through the trouble of declaring
    a custom callback method that simply returns the value of the ``make_boolean_checkmark()``
    helper.

    If the attribute being fetched is identical to the one pointed to in the column declaration,
    even the ``key`` argument can be omitted:

        ('Field Name', 'fieldname', make_boolean_checkmark)),

    """

    def wrapper(instance=None, key=None, *args, **kwargs):
        if instance and not key:
            # if key:
            #     # Helper was provided in the column declaration without being instantiated and
            #     # given a "key" argument.  It received the "key" function implicitly.
            #     value = key(instance)
            # else:
            #     # Helper is only being used within in a custom callback
            value = instance
            return helper(value, *args, **kwargs)
        elif key and not instance:
            # Helper is used directly in the columns declaration.  A new callable is
            # returned to take the place of a callback.
            def helper_wrapper(instance, *args, **kwargs):
                return helper(key(instance), *args, **kwargs)
            return helper_wrapper
        elif not key and not instance:
            # Helper was provided in the column declaration and was called in place with no
            # arguments.  We return the helper back, negating the premature call.
            return wrapper
    wrapper._is_wrapped = True
    return wrapper

def link_to_model(instance, text=None, *args, **kwargs):
    if not text:
        text = kwargs['default_value'] or unicode(instance)
    return """{}""".format(instance.get_absolute_url(), text)

@keyed_helper
def make_boolean_checkmark(value, false_value="", *args, **kwargs):
    if value:
        return "✔"
    return false_value

def attrgetter(attr):
    def helper(instance, *args, **kwargs):
        value = getattr(instance, attr)

        if callable(value):
            return value()
        return value

    return helper

def format_date(format_string, key=None):
    def helper(value, *args, **kwargs):
        if key:
            value = key(value)
        return value.strftime(format)

    if key:
        return helper
    return keyed_helper(helper)<|endoftext|>


<commit_before>from Bio import SeqIO
from Bio.Seq import Seq
from copy import copy
import itertools
from collections import Counter
import logging
import datetime
import math

def unique(seq):
    seen = set()
    seen_add = seen.add
    return [ x for x in seq if x not in seen and not seen_add(x)]

class Variant(object):

    def __init__(self, ref, pos, alt):
        self.ref = ref
        self.pos = pos
        self.alt = alt

    def __str__(self):
        return "".join([self.ref, str(self.pos), self.alt])

    def __repr__(self):
        return "".join([self.ref, str(self.pos), self.alt])

    def __gt__(self, other):
        return self.pos > other.pos

    def __lt__(self, other):
        return self.pos < other.pos

    def overlapping(self, other):
        return (other.pos in self.ref_range) or (self.pos in other.ref_range)

    @property
    def ref_range(self):
        return range(self.pos, self.pos + len(self.ref))

    @property
    def length(self):
        return abs(len(self.ref) - len(self.alt))

    @property
    def is_indel(self):
        """ Return whether or not the variant is an INDEL """
        if len(self.ref) > 1:
            return True
        if self.alt is None:
            return True
        elif len(self.alt) != len(self.ref):
            return True
        return False

    @property
    def is_deletion(self):
        """ Return whether or not the INDEL is a deletion """
        if self.is_indel:
            # just one alt allele
            if self.alt is None:
                return True
            if len(self.ref) > len(self.alt):
                return True
            else:
                return False
        else:
            return False

    @property
    def is_insertion(self):
        if self.is_indel:
            if self.alt is None:
                return False
            if len(self.alt) > len(self.ref):
                return True
            else:
                return False
        else:
            return False

class AlleleGenerator(object):
    """docstring for PanelGenerator"""
    def __init__(self, reference_filepath, kmer = 31):
        self.reference_filepath = reference_filepath
        self.kmer = kmer
        self.ref = []
        self.ref_length = 0
        self._read_reference()

    def _read_reference(self):
        for record in SeqIO.parse(self.reference_filepath, 'fasta'):
            self.ref += list(record.seq)
        self.ref_length = len(self.ref)

    def create(self, v, context = []):
        ## Position should be 1 based
        self._check_valid_variant(v)
        context = self._remove_contexts_spanning_del(v, context)
        context = self._remove_contexts_not_within_k(v, context)
        wild_type_reference = self._get_wildtype_reference(v)
        alternates = self._generate_alternates_on_all_backgrounds(v,  context)
        return Panel(wild_type_reference, v.pos, alternates)

    def _check_valid_variant(self, v):
        index = v.pos - 1
        if "".join(self.ref[index:(index + len(v.ref))]) != v.ref:
            raise ValueError("""Cannot create alleles as ref at pos %i is not %s
                                (it's %s) are you sure you're using one-based
                                co-ordinates?
                             """ % (v.pos, v.ref, "".join(self.ref[index: (index + len(v.ref))])))
        if v.pos <= 0 :
            raise ValueError("Position should be 1 based")

    def _remove_contexts_spanning_del(self, v, context):
        """If there's a var within the range of an DEL remove from context"""
        if v.is_deletion:
            context = [c for c in context if not c.pos in v.ref_range]
        return context

    def _remove_contexts_not_within_k(self, v, context):
        new_context = []
        for c in context:
            if c.is_insertion:
                effective_pos = c.pos - c.length
            elif c.is_deletion:
                effective_pos = c.pos + c.length
            else:
                effective_pos = c.pos
            if abs(v.pos - effective_pos) < self.kmer:
                new_context.append(c)
        return new_context

    def _get_wildtype_reference(self, v):
        i, start_index, end_index = self._get_start_end(v)
        return self._get_reference_segment(start_index, end_index)

    def _get_reference_segment(self, start_index, end_index):
        return copy(self.ref[start_index:end_index])

    def _get_alternate_reference_segment(self, v, context):
        ref_segment_length_delta = self._calculate_length_delta_from_indels(v, context)
        i, start_index, end_index = self._get_start_end(v, delta = ref_segment_length_delta)
        return self._get_reference_segment(start_index, end_index)

    def _generate_alternates_on_all_backgrounds(self, v, context):
        ## First, create all the context combinations
        context_combinations = self._get_all_context_combinations(context)
        ## For each context, create the background and alternate
        alternates = []
        for context_combo in context_combinations:
            ref_segment_length_delta = self._calculate_length_delta_from_indels(v, context_combo)
            i, start_index, end_index = self._get_start_end(v, delta = ref_segment_length_delta)
            alternate_reference_segment = self._get_reference_segment(start_index, end_index)
            background = self._generate_background_using_context(i, v, alternate_reference_segment, context_combo)
            alternate = copy(background)
            i -=  self._calculate_length_delta_from_variant_list([c for c in context_combo if c.pos <= v.pos and c.is_indel])
            print (i, len(v.ref))
            print ("".join(alternate[i:(i + len(v.ref))]) , v.ref)
            assert "".join(alternate[i:(i + len(v.ref))]) == v.ref
            alternate[i : i + len(v.ref)] = v.alt
            alternates.append(alternate)
        return alternates

    def _get_all_context_combinations(self, context):
        context_list = [[]]
        if context:
            contexts = self._create_multiple_contexts(context) ## Create contexts if multiple variants at given posision
            for context in contexts:
                context_combinations = self._get_combinations_of_backgrounds(context)
                context_list.extend(context_combinations)
        return context_list

    def _generate_background_using_context(self, i, v, alternate_reference_segment, context ):
        backgrounds = [alternate_reference_segment]
        new_background = copy(alternate_reference_segment)

        variants_added = []
        for variant in context:
            j = i + variant.pos - v.pos
            j -= self._calculate_length_delta_from_variant_list([c for c in variants_added if c.pos <= variant.pos and c.is_indel])
            assert "".join(new_background[j : j+ len(variant.ref)]) == variant.ref
            new_background[j : j + len(variant.ref)] = variant.alt
            variants_added.append(variant)
        return new_background

    def _create_multiple_contexts(self, context):
        new_contexts = self._recursive_context_creator([context])
        return new_contexts

    def _recursive_context_creator(self, contexts):
        ## This is only run when there are multiple variants at the same position
        compatiblity_of_contexts = [self._all_variants_are_combatible(context) for context in contexts]
        if self._are_contexts_all_valid(compatiblity_of_contexts):
            return contexts
        else:
            ## If there are variants that are incompatible (e.g. SNPs at the same position or overlapping INDELs)
            ## we need to distangle them into multiple contexts which are all compatible with each other
            ## Get the first incompatible context
            i = compatiblity_of_contexts.index(False)
            incompatible_context = contexts.pop(i)
            ## Split it into two smaller contexts by removing incompatible variants
            split_context = self._split_context(incompatible_context)
            contexts.extend(split_context)
            return self._recursive_context_creator(contexts)

    def _split_context(self, context):
        assert not self._all_variants_are_combatible(context)
        v1, v2 = self._get_first_two_incompatible_variants(context)
        nov1 = []
        nov2 = []
        for x in context:
            if not x is v1:
                nov1.append(x)
            if not x is v2:
                nov2.append(x)
        return [nov1, nov2]

    def _get_first_two_incompatible_variants(self, context):
        for pair in itertools.combinations(context, 2):
            if pair[0].overlapping(pair[1]):
                return pair

    def _are_contexts_all_valid(self, compatiblity_of_contexts):
        return all(compatiblity_of_contexts)

    def _all_variants_are_combatible(self, variants):
        pairs = list(itertools.combinations(variants, 2))
        return all([not v1.overlapping(v2) for v1,v2 in pairs])

    def _get_combinations_of_backgrounds(self, context):
        combination_context = []
        for L in range(1, len(context)+1):
          for subset in itertools.combinations(context, L):
            combination_context.append(list(subset))
        return combination_context

    def _get_start_end(self, v, delta = 0):
        ## Is large var
        shift = 0
        kmer = self.kmer
        if len(v.ref) > 2 * kmer:
            kmer = int(math.ceil(float(len(v.ref)) / 2)) + 5
        elif  (v.length > 2 * kmer):
            kmer = int(math.ceil(float(v.length) / 2)) + 5

        if len(v.ref) > kmer:
            shift = int( (kmer - 1) - math.floor(float((2 * kmer + 1) - len(v.ref)) / 2))
        pos = v.pos
        start_delta = int(math.floor(float(delta) / 2))
        end_delta = int(math.ceil(float(delta) / 2))
        start_index = pos - kmer - start_delta
        end_index =  pos + kmer + end_delta + 1
        i = kmer - 1 + start_delta
        if start_index < 0:
            diff = abs(start_index)
            start_index = 0
            end_index += diff
            i -= diff
        elif end_index > self.ref_length:
            diff = abs(end_index - self.ref_length)
            end_index = self.ref_length
            start_index -= diff
            i += diff
        start_index += shift
        end_index += shift
        i -= shift
        return (i, start_index, end_index)

    def _calculate_length_delta_from_indels(self, v, context):
        """Calculates the change in required bases for given variant.
        For deletions we need extra bases to get same length flanks"""
        variants = context + [v]
        return self._calculate_length_delta_from_variant_list(variants)

    def _calculate_length_delta_from_variant_list(self, variants):
        deletions_length = [v.length for v in variants if v.is_deletion]
        insertions_length = [v.length for v in variants if v.is_insertion]
        return sum(deletions_length) - sum(insertions_length)

class Panel(object):

    def __init__(self, ref, pos, alts):
        self.ref = "".join(ref)
        self.pos = pos
        self.alts = unique(["".join(alt) for alt in alts])

        <commit_msg>Remove print statement<commit_after>from Bio import SeqIO
from Bio.Seq import Seq
from copy import copy
import itertools
from collections import Counter
import logging
import datetime
import math

def unique(seq):
    seen = set()
    seen_add = seen.add
    return [ x for x in seq if x not in seen and not seen_add(x)]

class Variant(object):

    def __init__(self, ref, pos, alt):
        self.ref = ref
        self.pos = pos
        self.alt = alt

    def __str__(self):
        return "".join([self.ref, str(self.pos), self.alt])

    def __repr__(self):
        return "".join([self.ref, str(self.pos), self.alt])

    def __gt__(self, other):
        return self.pos > other.pos

    def __lt__(self, other):
        return self.pos < other.pos

    def overlapping(self, other):
        return (other.pos in self.ref_range) or (self.pos in other.ref_range)

    @property
    def ref_range(self):
        return range(self.pos, self.pos + len(self.ref))

    @property
    def length(self):
        return abs(len(self.ref) - len(self.alt))

    @property
    def is_indel(self):
        """ Return whether or not the variant is an INDEL """
        if len(self.ref) > 1:
            return True
        if self.alt is None:
            return True
        elif len(self.alt) != len(self.ref):
            return True
        return False

    @property
    def is_deletion(self):
        """ Return whether or not the INDEL is a deletion """
        if self.is_indel:
            # just one alt allele
            if self.alt is None:
                return True
            if len(self.ref) > len(self.alt):
                return True
            else:
                return False
        else:
            return False

    @property
    def is_insertion(self):
        if self.is_indel:
            if self.alt is None:
                return False
            if len(self.alt) > len(self.ref):
                return True
            else:
                return False
        else:
            return False

class AlleleGenerator(object):
    """docstring for PanelGenerator"""
    def __init__(self, reference_filepath, kmer = 31):
        self.reference_filepath = reference_filepath
        self.kmer = kmer
        self.ref = []
        self.ref_length = 0
        self._read_reference()

    def _read_reference(self):
        for record in SeqIO.parse(self.reference_filepath, 'fasta'):
            self.ref += list(record.seq)
        self.ref_length = len(self.ref)

    def create(self, v, context = []):
        ## Position should be 1 based
        self._check_valid_variant(v)
        context = self._remove_contexts_spanning_del(v, context)
        context = self._remove_contexts_not_within_k(v, context)
        wild_type_reference = self._get_wildtype_reference(v)
        alternates = self._generate_alternates_on_all_backgrounds(v,  context)
        return Panel(wild_type_reference, v.pos, alternates)

    def _check_valid_variant(self, v):
        index = v.pos - 1
        if "".join(self.ref[index:(index + len(v.ref))]) != v.ref:
            raise ValueError("""Cannot create alleles as ref at pos %i is not %s
                                (it's %s) are you sure you're using one-based
                                co-ordinates?
                             """ % (v.pos, v.ref, "".join(self.ref[index: (index + len(v.ref))])))
        if v.pos <= 0 :
            raise ValueError("Position should be 1 based")

    def _remove_contexts_spanning_del(self, v, context):
        """If there's a var within the range of an DEL remove from context"""
        if v.is_deletion:
            context = [c for c in context if not c.pos in v.ref_range]
        return context

    def _remove_contexts_not_within_k(self, v, context):
        new_context = []
        for c in context:
            if c.is_insertion:
                effective_pos = c.pos - c.length
            elif c.is_deletion:
                effective_pos = c.pos + c.length
            else:
                effective_pos = c.pos
            if abs(v.pos - effective_pos) < self.kmer:
                new_context.append(c)
        return new_context

    def _get_wildtype_reference(self, v):
        i, start_index, end_index = self._get_start_end(v)
        return self._get_reference_segment(start_index, end_index)

    def _get_reference_segment(self, start_index, end_index):
        return copy(self.ref[start_index:end_index])

    def _get_alternate_reference_segment(self, v, context):
        ref_segment_length_delta = self._calculate_length_delta_from_indels(v, context)
        i, start_index, end_index = self._get_start_end(v, delta = ref_segment_length_delta)
        return self._get_reference_segment(start_index, end_index)

    def _generate_alternates_on_all_backgrounds(self, v, context):
        ## First, create all the context combinations
        context_combinations = self._get_all_context_combinations(context)
        ## For each context, create the background and alternate
        alternates = []
        for context_combo in context_combinations:
            ref_segment_length_delta = self._calculate_length_delta_from_indels(v, context_combo)
            i, start_index, end_index = self._get_start_end(v, delta = ref_segment_length_delta)
            alternate_reference_segment = self._get_reference_segment(start_index, end_index)
            background = self._generate_background_using_context(i, v, alternate_reference_segment, context_combo)
            alternate = copy(background)
            i -=  self._calculate_length_delta_from_variant_list([c for c in context_combo if c.pos <= v.pos and c.is_indel])
            # print ("".join(alternate[i:(i + len(v.ref))]) , v.ref)
            assert "".join(alternate[i:(i + len(v.ref))]) == v.ref
            alternate[i : i + len(v.ref)] = v.alt
            alternates.append(alternate)
        return alternates

    def _get_all_context_combinations(self, context):
        context_list = [[]]
        if context:
            contexts = self._create_multiple_contexts(context) ## Create contexts if multiple variants at given posision
            for context in contexts:
                context_combinations = self._get_combinations_of_backgrounds(context)
                context_list.extend(context_combinations)
        return context_list

    def _generate_background_using_context(self, i, v, alternate_reference_segment, context ):
        backgrounds = [alternate_reference_segment]
        new_background = copy(alternate_reference_segment)

        variants_added = []
        for variant in context:
            j = i + variant.pos - v.pos
            j -= self._calculate_length_delta_from_variant_list([c for c in variants_added if c.pos <= variant.pos and c.is_indel])
            assert "".join(new_background[j : j+ len(variant.ref)]) == variant.ref
            new_background[j : j + len(variant.ref)] = variant.alt
            variants_added.append(variant)
        return new_background

    def _create_multiple_contexts(self, context):
        new_contexts = self._recursive_context_creator([context])
        return new_contexts

    def _recursive_context_creator(self, contexts):
        ## This is only run when there are multiple variants at the same position
        compatiblity_of_contexts = [self._all_variants_are_combatible(context) for context in contexts]
        if self._are_contexts_all_valid(compatiblity_of_contexts):
            return contexts
        else:
            ## If there are variants that are incompatible (e.g. SNPs at the same position or overlapping INDELs)
            ## we need to distangle them into multiple contexts which are all compatible with each other
            ## Get the first incompatible context
            i = compatiblity_of_contexts.index(False)
            incompatible_context = contexts.pop(i)
            ## Split it into two smaller contexts by removing incompatible variants
            split_context = self._split_context(incompatible_context)
            contexts.extend(split_context)
            return self._recursive_context_creator(contexts)

    def _split_context(self, context):
        assert not self._all_variants_are_combatible(context)
        v1, v2 = self._get_first_two_incompatible_variants(context)
        nov1 = []
        nov2 = []
        for x in context:
            if not x is v1:
                nov1.append(x)
            if not x is v2:
                nov2.append(x)
        return [nov1, nov2]

    def _get_first_two_incompatible_variants(self, context):
        for pair in itertools.combinations(context, 2):
            if pair[0].overlapping(pair[1]):
                return pair

    def _are_contexts_all_valid(self, compatiblity_of_contexts):
        return all(compatiblity_of_contexts)

    def _all_variants_are_combatible(self, variants):
        pairs = list(itertools.combinations(variants, 2))
        return all([not v1.overlapping(v2) for v1,v2 in pairs])

    def _get_combinations_of_backgrounds(self, context):
        combination_context = []
        for L in range(1, len(context)+1):
          for subset in itertools.combinations(context, L):
            combination_context.append(list(subset))
        return combination_context

    def _get_start_end(self, v, delta = 0):
        ## Is large var
        shift = 0
        kmer = self.kmer
        if len(v.ref) > 2 * kmer:
            kmer = int(math.ceil(float(len(v.ref)) / 2)) + 5
        elif  (v.length > 2 * kmer):
            kmer = int(math.ceil(float(v.length) / 2)) + 5

        if len(v.ref) > kmer:
            shift = int( (kmer - 1) - math.floor(float((2 * kmer + 1) - len(v.ref)) / 2))
        pos = v.pos
        start_delta = int(math.floor(float(delta) / 2))
        end_delta = int(math.ceil(float(delta) / 2))
        start_index = pos - kmer - start_delta
        end_index =  pos + kmer + end_delta + 1
        i = kmer - 1 + start_delta
        if start_index < 0:
            diff = abs(start_index)
            start_index = 0
            end_index += diff
            i -= diff
        elif end_index > self.ref_length:
            diff = abs(end_index - self.ref_length)
            end_index = self.ref_length
            start_index -= diff
            i += diff
        start_index += shift
        end_index += shift
        i -= shift
        return (i, start_index, end_index)

    def _calculate_length_delta_from_indels(self, v, context):
        """Calculates the change in required bases for given variant.
        For deletions we need extra bases to get same length flanks"""
        variants = context + [v]
        return self._calculate_length_delta_from_variant_list(variants)

    def _calculate_length_delta_from_variant_list(self, variants):
        deletions_length = [v.length for v in variants if v.is_deletion]
        insertions_length = [v.length for v in variants if v.is_insertion]
        return sum(deletions_length) - sum(insertions_length)

class Panel(object):

    def __init__(self, ref, pos, alts):
        self.ref = "".join(ref)
        self.pos = pos
        self.alts = unique(["".join(alt) for alt in alts])

        <|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# © 2015 ABF OSIELL <http://osiell.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from odoo import models, fields, api
from odoo.http import request

class AuditlogHTTPRequest(models.Model):
    _name = 'auditlog.http.request'
    _description = u"Auditlog - HTTP request log"
    _order = "create_date DESC"

    display_name = fields.Char(
        u"Name", compute="_compute_display_name", store=True)
    name = fields.Char(u"Path")
    root_url = fields.Char(u"Root URL")
    user_id = fields.Many2one(
        'res.users', string=u"User")
    http_session_id = fields.Many2one(
        'auditlog.http.session', string=u"Session")
    user_context = fields.Char(u"Context")
    log_ids = fields.One2many(
        'auditlog.log', 'http_request_id', string=u"Logs")

    @api.depends('create_date', 'name')
    def _compute_display_name(self):
        for httprequest in self:
            create_date = fields.Datetime.from_string(httprequest.create_date)
            tz_create_date = fields.Datetime.context_timestamp(
                httprequest, create_date)
            httprequest.display_name = u"%s (%s)" % (
                httprequest.name or '?',
                fields.Datetime.to_string(tz_create_date))

    @api.multi
    def name_get(self):
        return [(request.id, request.display_name) for request in self]

    @api.model
    def current_http_request(self):
        """Create a log corresponding to the current HTTP request, and returns
        its ID. This method can be called several times during the
        HTTP query/response cycle, it will only log the request on the
        first call.
        If no HTTP request is available, returns `False`.
        """
        if not request:
            return False
        http_session_model = self.env['auditlog.http.session']
        httprequest = request.httprequest
        if httprequest:
            if hasattr(httprequest, 'auditlog_http_request_id'):
                return httprequest.auditlog_http_request_id
            vals = {
                'name': httprequest.path,
                'root_url': httprequest.url_root,
                'user_id': request.uid,
                'http_session_id': http_session_model.current_http_session(),
                'user_context': request.context,
            }
            httprequest.auditlog_http_request_id = self.create(vals).id
            return httprequest.auditlog_http_request_id
        return False
<commit_msg>[FIX] Request id no longer exists after concurrency rollback<commit_after># -*- coding: utf-8 -*-
# © 2015 ABF OSIELL <http://osiell.com>
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl).

from psycopg2.extensions import AsIs

from odoo import models, fields, api
from odoo.http import request

class AuditlogHTTPRequest(models.Model):
    _name = 'auditlog.http.request'
    _description = u"Auditlog - HTTP request log"
    _order = "create_date DESC"

    display_name = fields.Char(
        u"Name", compute="_compute_display_name", store=True)
    name = fields.Char(u"Path")
    root_url = fields.Char(u"Root URL")
    user_id = fields.Many2one(
        'res.users', string=u"User")
    http_session_id = fields.Many2one(
        'auditlog.http.session', string=u"Session")
    user_context = fields.Char(u"Context")
    log_ids = fields.One2many(
        'auditlog.log', 'http_request_id', string=u"Logs")

    @api.depends('create_date', 'name')
    def _compute_display_name(self):
        for httprequest in self:
            create_date = fields.Datetime.from_string(httprequest.create_date)
            tz_create_date = fields.Datetime.context_timestamp(
                httprequest, create_date)
            httprequest.display_name = u"%s (%s)" % (
                httprequest.name or '?',
                fields.Datetime.to_string(tz_create_date))

    @api.multi
    def name_get(self):
        return [(request.id, request.display_name) for request in self]

    @api.model
    def current_http_request(self):
        """Create a log corresponding to the current HTTP request, and returns
        its ID. This method can be called several times during the
        HTTP query/response cycle, it will only log the request on the
        first call.
        If no HTTP request is available, returns `False`.
        """
        if not request:
            return False
        http_session_model = self.env['auditlog.http.session']
        httprequest = request.httprequest
        if httprequest:
            if hasattr(httprequest, 'auditlog_http_request_id'):
                # Verify existence. Could have been rolled back after a
                # concurrency error
                self.env.cr.execute(
                    "SELECT id FROM %s WHERE id = %s", (
                        AsIs(self._table),
                        httprequest.auditlog_http_request_id))
                if self.env.cr.fetchone():
                    return httprequest.auditlog_http_request_id
            vals = {
                'name': httprequest.path,
                'root_url': httprequest.url_root,
                'user_id': request.uid,
                'http_session_id': http_session_model.current_http_session(),
                'user_context': request.context,
            }
            httprequest.auditlog_http_request_id = self.create(vals).id
            return httprequest.auditlog_http_request_id
        return False
<|endoftext|>


<commit_before>import autonetkit.log as log
import autonetkit.ank as ank_utils

from autonetkit.ank_utils import call_log

def build_layer2(anm):
        g_l2 = anm.add_overlay('layer2')
        g_phy = anm['phy']
        g_l2.add_nodes_from(g_phy)
        g_l2.add_edges_from(g_phy.edges())
        # Don't aggregate managed switches
        unmanaged_switches = [n for n in g_l2.switches()
        if n.device_subtype != "managed"]
        ank_utils.aggregate_nodes(g_l2, unmanaged_switches)

def check_layer2(anm):
    """Sanity checks on topology"""
    from collections import defaultdict
    g_l2 = anm['layer2']

    # check for igp and ebgp on same switch
    for switch in sorted(g_l2.switches()):
        neigh_asns = defaultdict(int)
        for neigh in switch.neighbors():
            if neigh.asn is None:
                continue # don't add if not set
            neigh_asns[neigh.asn] += 1

        # IGP if two or more neighbors share the same ASN
        is_igp = any(asns > 1 for asns in neigh_asns.values())
        # eBGP if more than one unique neigh ASN
        is_ebgp = len(neigh_asns.keys()) > 1
        if is_igp and is_ebgp:
            log.warning("Switch %s contains both IGP and eBGP neighbors"
                % switch)

    # check for multiple links from nodes to switch
    for switch in sorted(g_l2.switches()):
        for neighbor in sorted(switch.neighbors()):
            edges = g_l2.edges(switch, neighbor)
            if len(edges) > 1:
                # more than one edge between the (src, dst) pair -> parallel
                log.warning("Multiple edges (%s) between %s and device %s"
                    % (len(edges), switch, neighbor))

def build_layer2_broadcast(anm):
        g_l2 = anm['layer2']
        g_phy = anm['phy']
        g_graphics = anm['graphics']
        g_l2_bc = anm.add_overlay('layer2_bc')
        g_l2_bc.add_nodes_from(g_l2.l3devices())
        g_l2_bc.add_nodes_from(g_l2.switches())
        g_l2_bc.add_edges_from(g_l2.edges())

        # remove external connectors

        edges_to_split = [edge for edge in g_l2_bc.edges()
            if edge.src.is_l3device() and edge.dst.is_l3device()]
        #TODO: debug the edges to split
        #print "edges to split", edges_to_split
        for edge in edges_to_split:
            edge.split = True  # mark as split for use in building nidb

        split_created_nodes = list(ank_utils.split(g_l2_bc, edges_to_split,
                                   retain=['split'],
                                   id_prepend='cd_'))

        #TODO: if parallel nodes, offset
        #TODO: remove graphics, assign directly
        for node in split_created_nodes:
            node['graphics'].x = ank_utils.neigh_average(g_l2_bc, node, 'x',
                    g_graphics) + 0.1

             # temporary fix for gh-90

            node['graphics'].y = ank_utils.neigh_average(g_l2_bc, node, 'y',
                    g_graphics) + 0.1

                # temporary fix for gh-90

            asn = ank_utils.neigh_most_frequent(g_l2_bc, node, 'asn', g_phy)  # arbitrary choice
            node['graphics'].asn = asn
            node.asn = asn  # need to use asn in IP overlay for aggregating subnets

        # also allocate an ASN for virtual switches
        vswitches = [n for n in g_l2_bc.nodes()
        if n['layer2'].device_type == "switch"
        and n['layer2'].device_subtype == "virtual"]
        for node in vswitches:
            #TODO: refactor neigh_most_frequent to allow fallthrough attributes
            #asn = ank_utils.neigh_most_frequent(g_l2_bc, node, 'asn', g_l2)  # arbitrary choice
            asns = [n['layer2'].asn for n in node.neighbors()]
            asns = [x for x in asns if x is not None]
            asn = ank_utils.most_frequent(asns)
            node.asn = asn  # need to use asn in IP overlay for aggregating subnets
            # also mark as broadcast domain

        from collections import defaultdict
        coincident_nodes = defaultdict(list)
        for node in split_created_nodes:
            coincident_nodes[(node['graphics'].x, node['graphics'].y)].append(node)

        coincident_nodes = {k: v for k, v in coincident_nodes.items()
        if len(v) > 1} # trim out single node co-ordinates
        import math
        for key, val in coincident_nodes.items():
            for index, item in enumerate(val):
                index = index + 1
                x_offset = 25*math.floor(index/2) * math.pow(-1, index)
                y_offset = -1 * 25*math.floor(index/2) * math.pow(-1, index)
                item['graphics'].x  = item['graphics'].x + x_offset
                item['graphics'].y  = item['graphics'].y + y_offset

        switch_nodes = g_l2_bc.switches()  # regenerate due to aggregated
        g_l2_bc.update(switch_nodes, broadcast_domain=True)

                     # switches are part of collision domain
        g_l2_bc.update(split_created_nodes, broadcast_domain=True)

    # Assign collision domain to a host if all neighbours from same host

        for node in split_created_nodes:
            if ank_utils.neigh_equal(g_l2_bc, node, 'host', g_phy):
                node.host = ank_utils.neigh_attr(g_l2_bc, node, 'host',
                        g_phy).next()  # first attribute

        # set collision domain IPs
        #TODO; work out why this throws a json exception
        #autonetkit.ank.set_node_default(g_l2_bc,  broadcast_domain=False)

        for node in g_l2_bc.nodes('broadcast_domain'):
            graphics_node = g_graphics.node(node)
            #graphics_node.device_type = 'broadcast_domain'
            if node.is_switch():
                #TODO: check not virtual
                node['phy'].broadcast_domain = True
            if not node.is_switch():
                # use node sorting, as accomodates for numeric/string names
                graphics_node.device_type = 'broadcast_domain'
                neighbors = sorted(neigh for neigh in node.neighbors())
                label = '_'.join(neigh.label for neigh in neighbors)
                cd_label = 'cd_%s' % label  # switches keep their names
                node.label = cd_label
                graphics_node.label = cd_label
                node.device_type = "broadcast_domain"
                node.label = node.id
                graphics_node.label = node.id

        for node in vswitches:
            node.broadcast_domain = True

def build_layer3(anm):
    """ l3_connectivity graph: switch nodes aggregated and exploded"""
    g_in = anm['input']
    g_l2 = anm['layer2']
    g_l3 = anm.add_overlay("layer3")
    g_l3.add_nodes_from(g_l2 , retain=['label'])
    g_l3.add_nodes_from(g_in.switches(), retain=['asn'])
    g_l3.add_edges_from(g_l2.edges())

    ank_utils.aggregate_nodes(g_l3, g_l3.switches())
    exploded_edges = ank_utils.explode_nodes(g_l3,
                                             g_l3.switches())

    # also explode virtual switches
    vswitches = [n for n in g_l3.nodes()
    if n['layer2'].device_type == "switch"
    and n['layer2'].device_subtype == "virtual"]

    # explode each seperately?
    vswitch_edges = []
    for vswitch in vswitches:
        vswitch_edges += ank_utils.explode_nodes(g_l3, [vswitch] )

    for edge in exploded_edges:
        edge.multipoint = True
        edge.src_int.multipoint = True
        edge.dst_int.multipoint = True
<commit_msg>chg: dev: refactor handling with custom VIRL design rules<commit_after>import autonetkit.log as log
import autonetkit.ank as ank_utils

from autonetkit.ank_utils import call_log

def build_layer2(anm):
        g_l2 = anm.add_overlay('layer2')
        g_phy = anm['phy']
        g_l2.add_nodes_from(g_phy)
        g_l2.add_edges_from(g_phy.edges())
        # Don't aggregate managed switches
        unmanaged_switches = [n for n in g_l2.switches()
        if n.device_subtype != "managed"]
        ank_utils.aggregate_nodes(g_l2, unmanaged_switches)

        try:
            from autonetkit_cisco import build_network as cisco_build_network
        except ImportError, e:
            pass
        else:
            cisco_build_network.post_layer2(anm)

def check_layer2(anm):
    """Sanity checks on topology"""
    from collections import defaultdict
    g_l2 = anm['layer2']

    # check for igp and ebgp on same switch
    for switch in sorted(g_l2.switches()):
        neigh_asns = defaultdict(int)
        for neigh in switch.neighbors():
            if neigh.asn is None:
                continue # don't add if not set
            neigh_asns[neigh.asn] += 1

        # IGP if two or more neighbors share the same ASN
        is_igp = any(asns > 1 for asns in neigh_asns.values())
        # eBGP if more than one unique neigh ASN
        is_ebgp = len(neigh_asns.keys()) > 1
        if is_igp and is_ebgp:
            log.warning("Switch %s contains both IGP and eBGP neighbors"
                % switch)

    # check for multiple links from nodes to switch
    for switch in sorted(g_l2.switches()):
        for neighbor in sorted(switch.neighbors()):
            edges = g_l2.edges(switch, neighbor)
            if len(edges) > 1:
                # more than one edge between the (src, dst) pair -> parallel
                log.warning("Multiple edges (%s) between %s and device %s"
                    % (len(edges), switch, neighbor))

def build_layer2_broadcast(anm):
        g_l2 = anm['layer2']
        g_phy = anm['phy']
        g_graphics = anm['graphics']
        g_l2_bc = anm.add_overlay('layer2_bc')
        g_l2_bc.add_nodes_from(g_l2.l3devices())
        g_l2_bc.add_nodes_from(g_l2.switches())
        g_l2_bc.add_edges_from(g_l2.edges())

        # remove external connectors

        edges_to_split = [edge for edge in g_l2_bc.edges()
            if edge.src.is_l3device() and edge.dst.is_l3device()]
        #TODO: debug the edges to split
        #print "edges to split", edges_to_split
        for edge in edges_to_split:
            edge.split = True  # mark as split for use in building nidb

        split_created_nodes = list(ank_utils.split(g_l2_bc, edges_to_split,
                                   retain=['split'],
                                   id_prepend='cd_'))

        #TODO: if parallel nodes, offset
        #TODO: remove graphics, assign directly
        for node in split_created_nodes:
            node['graphics'].x = ank_utils.neigh_average(g_l2_bc, node, 'x',
                    g_graphics) + 0.1

             # temporary fix for gh-90

            node['graphics'].y = ank_utils.neigh_average(g_l2_bc, node, 'y',
                    g_graphics) + 0.1

                # temporary fix for gh-90

            asn = ank_utils.neigh_most_frequent(g_l2_bc, node, 'asn', g_phy)  # arbitrary choice
            node['graphics'].asn = asn
            node.asn = asn  # need to use asn in IP overlay for aggregating subnets

        # also allocate an ASN for virtual switches
        vswitches = [n for n in g_l2_bc.nodes()
        if n['layer2'].device_type == "switch"
        and n['layer2'].device_subtype == "virtual"]
        for node in vswitches:
            #TODO: refactor neigh_most_frequent to allow fallthrough attributes
            #asn = ank_utils.neigh_most_frequent(g_l2_bc, node, 'asn', g_l2)  # arbitrary choice
            asns = [n['layer2'].asn for n in node.neighbors()]
            asns = [x for x in asns if x is not None]
            asn = ank_utils.most_frequent(asns)
            node.asn = asn  # need to use asn in IP overlay for aggregating subnets
            # also mark as broadcast domain

        from collections import defaultdict
        coincident_nodes = defaultdict(list)
        for node in split_created_nodes:
            coincident_nodes[(node['graphics'].x, node['graphics'].y)].append(node)

        coincident_nodes = {k: v for k, v in coincident_nodes.items()
        if len(v) > 1} # trim out single node co-ordinates
        import math
        for key, val in coincident_nodes.items():
            for index, item in enumerate(val):
                index = index + 1
                x_offset = 25*math.floor(index/2) * math.pow(-1, index)
                y_offset = -1 * 25*math.floor(index/2) * math.pow(-1, index)
                item['graphics'].x  = item['graphics'].x + x_offset
                item['graphics'].y  = item['graphics'].y + y_offset

        switch_nodes = g_l2_bc.switches()  # regenerate due to aggregated
        g_l2_bc.update(switch_nodes, broadcast_domain=True)

                     # switches are part of collision domain
        g_l2_bc.update(split_created_nodes, broadcast_domain=True)

    # Assign collision domain to a host if all neighbours from same host

        for node in split_created_nodes:
            if ank_utils.neigh_equal(g_l2_bc, node, 'host', g_phy):
                node.host = ank_utils.neigh_attr(g_l2_bc, node, 'host',
                        g_phy).next()  # first attribute

        # set collision domain IPs
        #TODO; work out why this throws a json exception
        #autonetkit.ank.set_node_default(g_l2_bc,  broadcast_domain=False)

        for node in g_l2_bc.nodes('broadcast_domain'):
            graphics_node = g_graphics.node(node)
            #graphics_node.device_type = 'broadcast_domain'
            if node.is_switch():
                #TODO: check not virtual
                node['phy'].broadcast_domain = True
            if not node.is_switch():
                # use node sorting, as accomodates for numeric/string names
                graphics_node.device_type = 'broadcast_domain'
                neighbors = sorted(neigh for neigh in node.neighbors())
                label = '_'.join(neigh.label for neigh in neighbors)
                cd_label = 'cd_%s' % label  # switches keep their names
                node.label = cd_label
                graphics_node.label = cd_label
                node.device_type = "broadcast_domain"
                node.label = node.id
                graphics_node.label = node.id

        for node in vswitches:
            node.broadcast_domain = True

def build_layer3(anm):
    """ l3_connectivity graph: switch nodes aggregated and exploded"""
    g_in = anm['input']
    g_l2 = anm['layer2']
    g_l3 = anm.add_overlay("layer3")
    g_l3.add_nodes_from(g_l2 , retain=['label'])
    g_l3.add_nodes_from(g_in.switches(), retain=['asn'])
    g_l3.add_edges_from(g_l2.edges())

    switches = g_l3.switches()

    ank_utils.aggregate_nodes(g_l3, switches)
    exploded_edges = ank_utils.explode_nodes(g_l3,
                                             switches)

    # also explode virtual switches
    vswitches = [n for n in g_l3.nodes()
    if n['layer2'].device_type == "switch"
    and n['layer2'].device_subtype == "virtual"]

    # explode each seperately?
    vswitch_edges = []
    for vswitch in vswitches:
        vswitch_edges += ank_utils.explode_nodes(g_l3, [vswitch] )

    for edge in exploded_edges:
        edge.multipoint = True
        edge.src_int.multipoint = True
        edge.dst_int.multipoint = True
<|endoftext|>


<commit_before>import os
import ast
import select
import platform
import threading
from ...state import State
from ..store import PluginStore
from ..process import flush_stdout
from ..process import multi_lines_output_handler
from ...components import _BaseComponent
from ...queues import QueuePluginCommand, QueueTtS
from avasdk.plugins.log import ERROR, IMPORT, REQUEST, RESPONSE

def daemon(fn):
    """@decorator
    """
    def wrapper(*args, **kwargs):
        thread = threading.Thread(target=fn, args=args, kwargs=kwargs)
        thread.daemon = True
        thread.start()
        return thread
    return wrapper

class PluginInvoker(_BaseComponent):
    """The entity responsible of executing the according plugin depending on the
        user's input.
    """

    def __init__(self):
        """Initializer.
        """
        super().__init__()
        self.observer = None
        self.observing = False
        self.stop_observing = threading.Event()
        self.state = State()
        self.store = PluginStore()
        self.queue_plugin_command = QueuePluginCommand()
        self.queue_tts = QueueTtS()

    def _process_result(self, plugin_name, process):
        """This functions flushes the stdout of the given process and process the
            data read.

            @param:
                - plugin_name: The name of the plugin (string).
                - process: The process object (subprocess.Popen)
        """
        output, import_flushed = flush_stdout(process)
        if ERROR in output:
            self.queue_tts.put('Plugin {} just crashed... Restarting'.format(plugin_name))
            self.store.get_plugin(plugin_name).kill()
            self.store.get_plugin(plugin_name).restart()
            return
        if IMPORT in output:
            return
        if REQUEST in output:
            output.remove(REQUEST)
            self.state.plugin_requires_user_interaction(plugin_name)
            self.queue_tts.put(ast.literal_eval(''.join(output)).get('tts'))
            return
        output.remove(RESPONSE)
        # TODO improve this part
        result, multi_lines = multi_lines_output_handler(output)
        if multi_lines:
            print(result)
            self.queue_tts.put('Result of [{}] has been print.'.format(plugin_name))
        else:
            self.queue_tts.put(result)

    @daemon
    def _observe(self):
        """Thread routine
            Poll each plugin process stdout to detect when there is data to read.
        """
        while not self.stop_observing.is_set():
            if platform.system() == 'Windows':
                pass
            else:
                OBSERVED = {}
                POLL = select.poll()
                READ_ONLY = select.POLLIN | select.POLLPRI | select.POLLHUP | select.POLLERR
                for _, plugin in self.store.plugins.items():
                    fd = int(plugin.get_process().stdout.name)
                    OBSERVED[fd] = plugin.get_process()
                    POLL.register(fd, READ_ONLY)
                try:
                    result = POLL.poll(0)
                except:
                    raise
                for fd, _ in result:
                    if OBSERVED.get(fd) is None:
                        continue
                    p = OBSERVED.get(fd)
                    self._process_result(''.join('{}'.format(k) for k, v in self.store.plugins.items() if p == v.get_process()), p)

    def _exec_event(self, event, expected=False, plugin_name=None):
        """Execute an event related to a plugin feature.

            @param:
                - event: A dictionary containing the event to execute (dictionary).
                optional: - exepected: A boolean to determine if this specific event is expected. (boolean).
                optional: - plugin_name: The name of the plugin waiting for an user input (string).
        """
        if expected:
            self.state.plugin_stops_waiting_for_user_interaction()
            command = ' '.join('{}'.format(value) for key, value in event.items() if value)
        else:
            plugin_name = event['action']
            command = ' '.join('{}'.format(value) for key, value in event.items() if key != 'action' and value)
        assert plugin_name is not None
        process = self.store.get_plugin(plugin_name).get_process()
        assert process is not None
        process.stdin.write(command + '\n')
        process.stdin.flush()

    def _process_event(self, event):
        """Handler to dertimine what kind of event, the invoker is currently dealing with.

            @param:
                - event: A dictionary containing the event to proceed (dictionary).
        """
        waiting, plugin = self.state.is_plugin_waiting_for_user_interaction()
        if waiting:
            self._exec_event(event, expected=True, plugin_name=plugin)
            return
        if not event['target']:
            self.queue_tts.put('In order to use a plugin, you must specify one command.')
            return
        if self.store.is_plugin_disabled(event['action']):
            self.queue_tts.put('The plugin {} is currently disabled.'.format(event['action']))
            return
        if not self.store.get_plugin(event['action']):
            self.queue_tts.put('No plugin named {} found.'.format(event['action']))
            return
        self._exec_event(event)

    def run(self):
        """The main function of the invoker.
            Waiting on self.queue_plugin_command, when an event is enqueued, this
            function processes end executes it.
        """
        try:
            if not self.observing:
                self.observing = True
                self._observer = self._observe()
            event = self.queue_plugin_command.get()
            print('PluginInvoker current event: ', event)
            self._process_event(event)
            self.queue_plugin_command.task_done()
        except:
            import traceback
            traceback.print_exc()
            self.queue_tts.put('RuntimeError')
            raise

    def shutdown(self):
        """Shutdown gracefully the invoker.
        """
        print('Shutting down the PluginInvoker ...')
        assert self.observer is not None and not self.stop_observing.is_set()
        self.stop_observing.set()
        self.observer.join()
<commit_msg>need to implement select on windows to detect when there is data to read from plugin process stdout<commit_after>import os
import ast
import select
import platform
import threading
from ...state import State
from ..store import PluginStore
from ..process import flush_stdout
from ..process import multi_lines_output_handler
from ...components import _BaseComponent
from ...queues import QueuePluginCommand, QueueTtS
from avasdk.plugins.log import ERROR, IMPORT, REQUEST, RESPONSE

def daemon(fn):
    """@decorator
    """
    def wrapper(*args, **kwargs):
        thread = threading.Thread(target=fn, args=args, kwargs=kwargs)
        thread.daemon = True
        thread.start()
        return thread
    return wrapper

class PluginInvoker(_BaseComponent):
    """The entity responsible of executing the according plugin depending on the
        user's input.
    """

    def __init__(self):
        """Initializer.
        """
        super().__init__()
        self.observer = None
        self.observing = False
        self.stop_observing = threading.Event()
        self.state = State()
        self.store = PluginStore()
        self.queue_plugin_command = QueuePluginCommand()
        self.queue_tts = QueueTtS()

    def _process_result(self, plugin_name, process):
        """This functions flushes the stdout of the given process and process the
            data read.

            @param:
                - plugin_name: The name of the plugin (string).
                - process: The process object (subprocess.Popen)
        """
        output, import_flushed = flush_stdout(process)
        if ERROR in output:
            self.queue_tts.put('Plugin {} just crashed... Restarting'.format(plugin_name))
            self.store.get_plugin(plugin_name).kill()
            self.store.get_plugin(plugin_name).restart()
            return
        if IMPORT in output:
            return
        if REQUEST in output:
            output.remove(REQUEST)
            self.state.plugin_requires_user_interaction(plugin_name)
            self.queue_tts.put(ast.literal_eval(''.join(output)).get('tts'))
            return
        output.remove(RESPONSE)
        # TODO improve this part
        result, multi_lines = multi_lines_output_handler(output)
        if multi_lines:
            print(result)
            self.queue_tts.put('Result of [{}] has been print.'.format(plugin_name))
        else:
            self.queue_tts.put(result)

    @daemon
    def _observe(self):
        """Thread routine
            Poll each plugin process stdout to detect when there is data to read.
        """
        while not self.stop_observing.is_set():
            if platform.system() == 'Windows':
                pass
            else:
                OBSERVED = {}
                POLL = select.poll()
                READ_ONLY = select.POLLIN | select.POLLPRI | select.POLLHUP | select.POLLERR
                for _, plugin in self.store.plugins.items():
                    fd = int(plugin.get_process().stdout.name)
                    OBSERVED[fd] = plugin.get_process()
                    POLL.register(fd, READ_ONLY)
                try:
                    result = POLL.poll(0)
                except:
                    raise
                for fd, _ in result:
                    if OBSERVED.get(fd) is None:
                        continue
                    p = OBSERVED.get(fd)
                    self._process_result(''.join('{}'.format(k) for k, v in self.store.plugins.items() if p == v.get_process()), p)

    def _exec_event(self, event, expected=False, plugin_name=None):
        """Execute an event related to a plugin feature.

            @param:
                - event: A dictionary containing the event to execute (dictionary).
                optional: - exepected: A boolean to determine if this specific event is expected. (boolean).
                optional: - plugin_name: The name of the plugin waiting for an user input (string).
        """
        if expected:
            self.state.plugin_stops_waiting_for_user_interaction()
            command = ' '.join('{}'.format(value) for key, value in event.items() if value)
        else:
            plugin_name = event['action']
            command = ' '.join('{}'.format(value) for key, value in event.items() if key != 'action' and value)
        assert plugin_name is not None
        process = self.store.get_plugin(plugin_name).get_process()
        assert process is not None
        process.stdin.write(command + '\n');
        process.stdin.flush();

    def _process_event(self, event):
        """Handler to dertimine what kind of event, the invoker is currently dealing with.

            @param:
                - event: A dictionary containing the event to proceed (dictionary).
        """
        waiting, plugin = self.state.is_plugin_waiting_for_user_interaction()
        if waiting:
            self._exec_event(event, expected=True, plugin_name=plugin)
            return
        if not event['target']:
            self.queue_tts.put('In order to use a plugin, you must specify one command.')
            return
        if self.store.is_plugin_disabled(event['action']):
            self.queue_tts.put('The plugin {} is currently disabled.'.format(event['action']))
            return
        if not self.store.get_plugin(event['action']):
            self.queue_tts.put('No plugin named {} found.'.format(event['action']))
            return
        self._exec_event(event)

    def run(self):
        """The main function of the invoker.
            Waiting on self.queue_plugin_command, when an event is enqueued, this
            function processes end executes it.
        """
        try:
            if not self.observing:
                self.observing = True
                self._observer = self._observe()
            event = self.queue_plugin_command.get()
            print('PluginInvoker current event: ', event)
            self._process_event(event)
            self.queue_plugin_command.task_done()
        except:
            import traceback
            traceback.print_exc()
            self.queue_tts.put('RuntimeError')
            raise

    def shutdown(self):
        """Shutdown gracefully the invoker.
        """
        print('Shutting down the PluginInvoker ...')
        assert self.observer is not None and not self.stop_observing.is_set()
        self.stop_observing.set()
        self.observer.join()
<|endoftext|>