<commit_before>import xml.etree.ElementTree as XML
import jenkins_jobs.modules.base
from jenkins_jobs.errors import JenkinsJobsException
import logging

def ansibleplaybook(parser, xml_parent, data):
    """yaml: ansibleplaybook
    """
    logger = logging.getLogger("%s:ansibleplaybook" % __name__)
    apb = XML.SubElement(xml_parent, 'org.jenkinsci.plugins.ansible.AnsiblePlaybookBuilder')
    inventoryPath = XML.SubElement(apb, 'org.jenkinsci.plugins.ansible.InventoryPathorg.jenkinsci.plugins.ansible.InventoryPath')

    XML.SubElement(apb, 'playbook').text = data.get('playbook', 'deploy.yml')
    XML.SubElement(inventoryPath, 'path').text = data.get('inventory_path', '/etc/ansible/ec2.py')
    XML.SubElement(apb, 'limit').text = data.get('limit', '')

    XML.SubElement(apb, 'tags').text = data.get('tags', '')
    XML.SubElement(apb, 'skippedTags').text = data.get('skippedTags', '')
    XML.SubElement(apb, 'startAtTask').text = data.get('startAtTask', '')
    XML.SubElement(apb, 'credentialsId').text = data.get('credentialsId')
    XML.SubElement(apb, 'forks').text = str(data.get('forks', 5))
    XML.SubElement(apb, 'sudo').text = str(data.get('sudo', False)).lower()
    XML.SubElement(apb, 'sudoUser').text = data.get('sudoUser', 'root')
    XML.SubElement(apb, 'unbufferedOutput').text = str(data.get('unbufferedOutput', True)).lower()
    XML.SubElement(apb, 'colorizedOutput').text = str(data.get('colorizedOutput', False)).lower()
    XML.SubElement(apb, 'hostKeyChecking').text = str(data.get('hostKeyChecking', False)).lower()
    XML.SubElement(apb, 'additionalParameters').text = data.get('additionalParameters', '')
    XML.SubElement(apb, 'copyCredentialsInWorkspace').text = str(data.get('copyCredentialsInWorkspace', False)).lower()
<commit_msg>inventory path fix<commit_after>import xml.etree.ElementTree as XML
import jenkins_jobs.modules.base
from jenkins_jobs.errors import JenkinsJobsException
import logging

def ansibleplaybook(parser, xml_parent, data):
    """yaml: ansibleplaybook
    """
    logger = logging.getLogger("%s:ansibleplaybook" % __name__)
    apb = XML.SubElement(xml_parent, 'org.jenkinsci.plugins.ansible.AnsiblePlaybookBuilder')
    inventoryPath = XML.SubElement(apb, 'inventory', {'class':'org.jenkinsci.plugins.ansible.InventoryPath'})

    XML.SubElement(apb, 'playbook').text = data.get('playbook', 'deploy.yml')
    XML.SubElement(inventoryPath, 'path').text = data.get('inventory_path', '/etc/ansible/ec2.py')
    XML.SubElement(apb, 'limit').text = data.get('limit', '')

    XML.SubElement(apb, 'tags').text = data.get('tags', '')
    XML.SubElement(apb, 'skippedTags').text = data.get('skippedTags', '')
    XML.SubElement(apb, 'startAtTask').text = data.get('startAtTask', '')
    XML.SubElement(apb, 'credentialsId').text = data.get('credentialsId')
    XML.SubElement(apb, 'forks').text = str(data.get('forks', 5))
    XML.SubElement(apb, 'sudo').text = str(data.get('sudo', False)).lower()
    XML.SubElement(apb, 'sudoUser').text = data.get('sudoUser', 'root')
    XML.SubElement(apb, 'unbufferedOutput').text = str(data.get('unbufferedOutput', True)).lower()
    XML.SubElement(apb, 'colorizedOutput').text = str(data.get('colorizedOutput', False)).lower()
    XML.SubElement(apb, 'hostKeyChecking').text = str(data.get('hostKeyChecking', False)).lower()
    XML.SubElement(apb, 'additionalParameters').text = data.get('additionalParameters', '')
    XML.SubElement(apb, 'copyCredentialsInWorkspace').text = str(data.get('copyCredentialsInWorkspace', False)).lower()
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# Dan Levin <dlevin@net.t-labs.tu-berlin.de>

import argparse
import logging
import logging.config
from math import log
#import plot #automatically plot selected ouputs directly after running
from sim.simulation import LinkBalancerSim
from sim.workload import dual_offset_workload, sawtooth, wave, expo_workload
import sys
from test.test_helper import two_ctrls, two_separate_state_ctrls, two_random_ctrls, two_greedy_ctrls, two_switch_topo, strictly_local_ctrls

parser = argparse.ArgumentParser()
parser.add_argument('--demand', '-d',
                    help="max demand values",
                    action="store",
                    nargs='+',
                    dest="demands")
parser.add_argument('--staleness', '-s',
                    help="staleness values",
                    action="store",
                    nargs='+',
                    dest="stalenesses")
args = parser.parse_args()

logging.config.fileConfig('setup.cfg')
logger= logging.getLogger(__name__)

def main():
#    demo_strictly_local_ctrls()
    for demand in [32,64,128]:
        #for staleness in [0,1]:
        for staleness in [0]:
            sync_expo_improves_metric(max_demand=demand, staleness=staleness)
            sync_expo_separate_improves_metric(max_demand=demand, staleness=staleness)
            sync_separate_state_improves_metric(max_demand=demand, staleness=staleness)
            sync_improves_metric(max_demand=demand, staleness=staleness)
        for greedylimit in [0,0.25,0.5,0.75,1]:
            compare_greedy_dist_to_centralized(max_demand=demand, greedylimit=greedylimit)
#    synced_dist_equals_central()
#    compare_random_dist_to_centralized()
#    plot.plot_timeseries()
#    plot.plot_boxplot()

def demo_strictly_local_ctrls(max_demand=8, show_graph=False):
    """Demonstrate synchronization across domains makes no difference when
    LinkBalanerCtrl only handles requests within its own domain"""

    #TODO: demonstrate with more than 1 srv per controller domain
    period = 32
    timesteps = period * 4
    for sync_period in [0] + [2**x for x in range(0, int(log(period,2)))]:
        myname = '%(fname)s_%(num)02d' % {"fname": sys._getframe().f_code.co_name, "num": sync_period}
        logger.info("starting %s", myname)
        workload = dual_offset_workload(switches=['sw1', 'sw2'],
                                        period=period, offset=period/2.0,
                                        max_demand=max_demand, size=1,
                                        duration=1, timesteps=timesteps,
                                        workload_fcn=wave)

        ctrls = strictly_local_ctrls()
        sim = LinkBalancerSim(two_switch_topo(), ctrls)
        sim.run_and_trace(myname, workload, old=True, sync_period=sync_period,
                          show_graph=show_graph)
        logger.info("ending %s", myname)

def sync_improves_metric(period=64, max_demand=200, show_graph=False,
                         staleness=0):
    """Evalute the value of synchronization for a LinkBalanerCtrl by showing
    its effect on performance metric. We expect that for a workload which
    imparts server link imbalance across multiple domains, syncing will help
    improve the rmse_server metric."""

    timesteps = period * 4
    for sync_period in [0] + [2**x for x in range(0, int(log(period,2)))]:
        myname = '%(fname)s_%(demand)d_%(num)02d_%(staleness)d' % {"fname": sys._getframe().f_code.co_name,
                                                     "demand": max_demand,
                                                     "num": sync_period,
                                                     "staleness": staleness}
        logger.info("starting %s", myname)
        workload = dual_offset_workload(switches=['sw1', 'sw2'],
                                        period=period, offset=period/2.0,
                                        max_demand=max_demand, size=1,
                                    duration=2, timesteps=timesteps,
                                    workload_fcn=wave)

        ctrls = two_ctrls()
        sim = LinkBalancerSim(two_switch_topo(), ctrls)
        sim.run_and_trace(myname, workload, old=True, sync_period=sync_period,
                          show_graph=show_graph, staleness=staleness)
        logger.info("ending %s", myname)

def sync_expo_improves_metric(max_demand, timesteps=64, show_graph=False, staleness=0):
    """Evalute the value of synchronization for a LinkBalanerCtrl by showing
    its effect on performance metric. We expect that for a workload which
    imparts server link imbalance across multiple domains, syncing will help
    improve the rmse_server metric."""

    for sync_period in [0] + [2**x for x in range(0, int(log(timesteps,2)))]:
        myname = '%(fname)s_%(demand)d_%(num)02d_%(staleness)d' % {"fname": sys._getframe().f_code.co_name,
                                                     "demand": max_demand,
                                                     "num": sync_period,
                                                     "staleness": staleness}
        logger.info("starting %s", myname)
        workload = expo_workload(switches=['sw1', 'sw2'],
                workload_duration=timesteps)

        ctrls = two_ctrls()
        #ctrls = two_separate_state_ctrls()
        sim = LinkBalancerSim(two_switch_topo(), ctrls)
        sim.run_and_trace(myname, workload, old=False, sync_period=sync_period,
                          show_graph=show_graph, staleness=staleness)
        logger.info("ending %s", myname)

def sync_expo_separate_improves_metric(max_demand, timesteps=64, show_graph=False, staleness=0):
    """Evalute the value of synchronization for a LinkBalanerCtrl by showing
    its effect on performance metric. We expect that for a workload which
    imparts server link imbalance across multiple domains, syncing will help
    improve the rmse_server metric."""

    for sync_period in [0] + [2**x for x in range(0, int(log(timesteps,2)))]:
        myname = '%(fname)s_%(demand)d_%(num)02d_%(staleness)d' % {"fname": sys._getframe().f_code.co_name,
                                                     "demand": max_demand,
                                                     "num": sync_period,
                                                     "staleness": staleness}
        logger.info("starting %s", myname)
        workload = expo_workload(switches=['sw1', 'sw2'],
                workload_duration=timesteps)

        #ctrls = two_ctrls()
        ctrls = two_separate_state_ctrls()
        sim = LinkBalancerSim(two_switch_topo(), ctrls)
        sim.run_and_trace(myname, workload, old=False, sync_period=sync_period,
                          show_graph=show_graph, staleness=staleness)
        logger.info("ending %s", myname)

def sync_separate_state_improves_metric(period=64, max_demand=200, show_graph=False,
                         staleness=0):
    """
    Same as above, except using separate state tracking controllers which keep
    synchronization-shared state from extra-domain links sepatate from
    locally-originating inferred "contributed" extra-domain link utilization
    """

    timesteps = period * 4
    for sync_period in [0] + [2**x for x in range(0, int(log(period,2)))]:
        myname = '%(fname)s_%(demand)d_%(num)02d_%(staleness)d' % {"fname": sys._getframe().f_code.co_name,
                                                     "demand": max_demand,
                                                     "num": sync_period,
                                                     "staleness": staleness}
        logger.info("starting %s", myname)
        workload = dual_offset_workload(switches=['sw1', 'sw2'],
                                        period=period, offset=period/2.0,
                                        max_demand=max_demand, size=1,
                                    duration=2, timesteps=timesteps,
                                    workload_fcn=wave)

        ctrls = two_separate_state_ctrls()
        sim = LinkBalancerSim(two_switch_topo(), ctrls)
        sim.run_and_trace(myname, workload, old=True, sync_period=sync_period,
                          show_graph=show_graph, staleness=staleness)
        logger.info("ending %s", myname)

def synced_dist_equals_central(period=64, max_demand=4, show_graph=False):
    """Ensure that a distributed controller simulation run with sync_period=0
    yields exactly the same result as the same toplology and workload with a
    single controller."""

    timesteps = period * 4
    for sync_period in [0] + [2**x for x in range(0, int(log(period,2)))]:
        myname = '%(fname)s_%(num)02d' % {"fname": sys._getframe().f_code.co_name, "num": sync_period}
        logger.info("starting %s", myname)
        workload = dual_offset_workload(switches=['sw1', 'sw2'],
                                        period=period, offset=0,
                                        max_demand=max_demand, size=1,
                                        duration=1, timesteps=timesteps,
                                        workload_fcn=wave)

        ctrls = two_ctrls()
        sim = LinkBalancerSim(two_switch_topo(), ctrls)
        sim.run_and_trace(myname+str(sync_period), workload, old=True, sync_period=sync_period,
                          show_graph=show_graph)
        logger.info("ending %s", myname)

def compare_random_dist_to_centralized(period=64, max_demand=8, show_graph=False):
    """ """
    timesteps = period * 4
    for sync_period in [0] + [2**x for x in range(0, int(log(period,2)))]:
        myname = '%(fname)s_%(num)02d' % {"fname": sys._getframe().f_code.co_name, "num": sync_period}
        logger.info("starting %s", myname)
        workload = dual_offset_workload(switches=['sw1', 'sw2'],
                                        period=period, offset=period/2.0,
                                        max_demand=max_demand, size=1,
                                        duration=2, timesteps=timesteps,
                                        workload_fcn=wave)

        ctrls = two_random_ctrls()
        sim = LinkBalancerSim(two_switch_topo(), ctrls)
        sim.run_and_trace(myname, workload, old=True, sync_period=sync_period,
                          show_graph=show_graph)
        logger.info("ending %s", myname)

def compare_greedy_dist_to_centralized(period=64, max_demand=30,
                                       greedylimit=0.5, show_graph=False):
    """Ensure that a distributed controller simulation run with sync_period=0
    yields exactly the same result as the same toplology and workload with a
    single controller."""
    timesteps = period * 4
    for sync_period in [0] + [2**x for x in range(0, int(log(period,2)))]:
        myname = '%(fname)s_%(demand)d_%(gl)s_%(num)02d' % {"fname": sys._getframe().f_code.co_name,
                                                     "demand": max_demand,
                                                     "gl": str(greedylimit),
                                                     "num": sync_period}

        logger.info("starting %s", myname)
        workload = dual_offset_workload(switches=['sw1', 'sw2'],
                                        period=period, offset=period/2.0,
                                        max_demand=max_demand, size=1,
                                        duration=2, timesteps=timesteps,
                                        workload_fcn=wave)

        ctrls = two_greedy_ctrls(greedylimit=greedylimit)
        sim = LinkBalancerSim(two_switch_topo(), ctrls)
        sim.run_and_trace(myname, workload, old=True, sync_period=sync_period,
                          show_graph=show_graph)
        logger.info("ending %s", myname)

def compare_greedy_dist_to_sync_dist(max_demand=8, show_graph=False):
    """Understand what improvement synchronization gives us over a greedy
    algorithm in a dynamic, discrete loadbalancing environment"""
    pass

#Not enough brainpower left tonight to implement these, but I have been
#looking through the code and tests.  Here are some thoughts for regression
#test-ish graphs to generate.
#
#(1) Line topology w/3 controllers, each responsible for one server.   Let's
#say all requests go to the center controller first.  If the main parameter
#to vary is the relative size of requests, this should expose the effect of
#bins on our load-packing metric, plus ensure that LinkBalancerCtrl properly
#handles shortest-path ties in a reasonable way.
#
#(2) Line topology w/N controllers, N servers.  I have to think more about
#this one, but it seems like the nearest servers are always a better choice
#on a restricted topology where the demand is at one location, because
#they'd yield strictly less link usage.  But when demands are more spread
#out I could see a model solver in a central node doing a better job.
#
#(3) Ring topology w/N controllers, N servers.  Like 2.  Faster sync should
#yield closer-to-optimal balancing.
#
#(4) Ring topology w/N servers and varying number of controllers C, each
#commanding N / C servers.  With more controllers do we see crappier
#balancing for random workloads?
#
#-b

if __name__ == "__main__":
    main()
<commit_msg>Runsim was getting very ugly as Anja and I have been working with it, cleaned it up and added our latest exponentitally distributed workload simulation runs<commit_after>#!/usr/bin/env python
#
# Dan Levin <dlevin@net.t-labs.tu-berlin.de>

import argparse
import logging
import logging.config
from math import log
#import plot #automatically plot selected ouputs directly after running
from sim.simulation import LinkBalancerSim
from sim.workload import dual_offset_workload, sawtooth, wave, expo_workload
import sys
from test.test_helper import two_ctrls, two_separate_state_ctrls, two_random_ctrls, two_greedy_ctrls, two_switch_topo, strictly_local_ctrls

parser = argparse.ArgumentParser()
parser.add_argument('--demand', '-d',
                    help="max demand values",
                    action="store",
                    nargs='+',
                    default=[32,64,128],
                    dest="demands")
parser.add_argument('--staleness', '-s',
                    help="staleness values",
                    action="store",
                    nargs='+',
                    default=[0],
                    dest="stalenesses")
parser.add_argument('--timesteps', '-t',
                    help="number of simulation timesteps",
                    action="store",
                    nargs='1',
                    default=256,
                    dest="timesteps")
#parser.add_argument('--workloads', '-w',
#                    help="workload function",
#                    action="store",
#                    nargs='+',
#                    default=[expo_workload, dual_offset_workload]
#                    dest="timesteps")
args = parser.parse_args()

logging.config.fileConfig('setup.cfg')
logger= logging.getLogger(__name__)

def main():
    timesteps = args.timesteps
    for demand in args.demands:
        for staleness in args.stalenesses:
            sync_improves_metric(max_demand=demand, staleness=staleness)
            sync_separate_improves_metric(max_demand=demand, staleness=staleness)
            sync_expo_improves_metric(max_demand=demand, timesteps=timesteps, staleness=staleness)
            sync_expo_separate_improves_metric(max_demand=demand, timesteps=timesteps, staleness=staleness)
#        for greedylimit in [0,0.25,0.5,0.75,1]:
#            compare_greedy_dist_to_centralized(max_demand=demand, greedylimit=greedylimit)
#    synced_dist_equals_central()
#    compare_random_dist_to_centralized()
#    demo_strictly_local_ctrls()

def sync_improves_metric(max_demand, period=64, name=None, show_graph=False, staleness=0):
    """Evalute the value of synchronization for a LinkBalanerCtrl by showing
    its effect on performance metric. We expect that for a workload which
    imparts server link imbalance across multiple domains, syncing will help
    improve the rmse_server metric."""

    if name == None:
        name = sys._getframe().f_code.co_name

    timesteps = period * 4
    for sync_period in [0] + [2**x for x in range(0, int(log(period,2)))]:
        myname = '%(a)s_%(b)d_%(c)02d_%(d)d' % {"a": name,
                                                "b": max_demand,
                                                "c": sync_period,
                                                "d": staleness}

        logger.info("starting %s", myname)
        workload = dual_offset_workload(switches=['sw1', 'sw2'],
                                        period=period, offset=period/2.0,
                                        max_demand=max_demand, size=1,
                                        duration=2, timesteps=timesteps,
                                        workload_fcn=wave, y_shift=(1.0/3))

        ctrls = two_ctrls()
        sim = LinkBalancerSim(two_switch_topo(), ctrls)
        sim.run_and_trace(myname, workload, old=True, sync_period=sync_period,
                          show_graph=show_graph, staleness=staleness)
        logger.info("ending %s", myname)

def sync_expo_improves_metric(max_demand, timesteps, ia=10, shape=5, show_graph=False, staleness=0):
    """ Same as sync_improves_metric, but using an exponential inter-arrival
    distribution workload """

    for sync_period in [0] + [2**x for x in range(0, int(log(timesteps,2)))]:
        myname = '%(fname)s_%(demand)d_%(num)02d_%(staleness)d' % {"fname": sys._getframe().f_code.co_name,
                                                     "demand": max_demand,
                                                     "num": sync_period,
                                                     "staleness": staleness}
        logger.info("starting %s", myname)
        workload = expo_workload(switches=['sw1', 'sw2'], interarrival_alpha=ia,
                duration_shape=shape, timesteps=timesteps)

        ctrls = two_ctrls()
        #ctrls = two_separate_state_ctrls()
        sim = LinkBalancerSim(two_switch_topo(), ctrls)
        sim.run_and_trace(myname, workload, old=False, sync_period=sync_period,
                          show_graph=show_graph, staleness=staleness)
        logger.info("ending %s", myname)

def sync_separate_improves_metric(max_demand, period=64, name=None, show_graph=False, staleness=0):
    """ Same as above, except using separate state tracking controllers which keep
    synchronization-shared state from extra-domain links sepatate from
    locally-originating inferred "contributed" extra-domain link utilization """

    if name == None:
        name = sys._getframe().f_code.co_name

    timesteps = period * 4
    for sync_period in [0] + [2**x for x in range(0, int(log(period,2)))]:
        myname = '%(a)s_%(b)d_%(c)02d_%(d)d' % {"a": name,
                                                "b": max_demand,
                                                "c": sync_period,
                                                "d": staleness}

        logger.info("starting %s", myname)
        workload = dual_offset_workload(switches=['sw1', 'sw2'],
                                        period=period, offset=period/2.0,
                                        max_demand=max_demand, size=1,
                                        duration=2, timesteps=timesteps,
                                        workload_fcn=wave, y_shift=(1.0/3))

        ctrls = two_separate_state_ctrls()
        sim = LinkBalancerSim(two_switch_topo(), ctrls)
        sim.run_and_trace(myname, workload, old=True, sync_period=sync_period,
                          show_graph=show_graph, staleness=staleness)
        logger.info("ending %s", myname)

def sync_expo_separate_improves_metric(max_demand, timesteps, ia=10, shape=5, show_graph=False, staleness=0):
    """ """

    for sync_period in [0] + [2**x for x in range(0, int(log(timesteps,2)))]:
        myname = '%(fname)s_%(demand)d_%(num)02d_%(staleness)d' % {"fname": sys._getframe().f_code.co_name,
                                                     "demand": max_demand,
                                                     "num": sync_period,
                                                     "staleness": staleness}
        logger.info("starting %s", myname)
        workload = expo_workload(switches=['sw1', 'sw2'], interarrival_alpha=ia,
                duration_shape=shape, timesteps=timesteps)

        ctrls = two_separate_state_ctrls()
        sim = LinkBalancerSim(two_switch_topo(), ctrls)
        sim.run_and_trace(myname, workload, old=False, sync_period=sync_period,
                          show_graph=show_graph, staleness=staleness)
        logger.info("ending %s", myname)

#######################################

def synced_dist_equals_central(period=64, max_demand=4, show_graph=False):
    """Ensure that a distributed controller simulation run with sync_period=0
    yields exactly the same result as the same toplology and workload with a
    single controller."""

    timesteps = period * 4
    for sync_period in [0] + [2**x for x in range(0, int(log(period,2)))]:
        myname = '%(fname)s_%(num)02d' % {"fname": sys._getframe().f_code.co_name, "num": sync_period}
        logger.info("starting %s", myname)
        workload = dual_offset_workload(switches=['sw1', 'sw2'],
                                        period=period, offset=0,
                                        max_demand=max_demand, size=1,
                                        duration=1, timesteps=timesteps,
                                        workload_fcn=wave)

        ctrls = two_ctrls()
        sim = LinkBalancerSim(two_switch_topo(), ctrls)
        sim.run_and_trace(myname+str(sync_period), workload, old=True, sync_period=sync_period,
                          show_graph=show_graph)
        logger.info("ending %s", myname)

def compare_random_dist_to_centralized(period=64, max_demand=8, show_graph=False):
    """ """
    timesteps = period * 4
    for sync_period in [0] + [2**x for x in range(0, int(log(period,2)))]:
        myname = '%(fname)s_%(num)02d' % {"fname": sys._getframe().f_code.co_name, "num": sync_period}
        logger.info("starting %s", myname)
        workload = dual_offset_workload(switches=['sw1', 'sw2'],
                                        period=period, offset=period/2.0,
                                        max_demand=max_demand, size=1,
                                        duration=2, timesteps=timesteps,
                                        workload_fcn=wave)

        ctrls = two_random_ctrls()
        sim = LinkBalancerSim(two_switch_topo(), ctrls)
        sim.run_and_trace(myname, workload, old=True, sync_period=sync_period,
                          show_graph=show_graph)
        logger.info("ending %s", myname)

def compare_greedy_dist_to_centralized(period=64, max_demand=30,
                                       greedylimit=0.5, show_graph=False):
    """Ensure that a distributed controller simulation run with sync_period=0
    yields exactly the same result as the same toplology and workload with a
    single controller."""
    timesteps = period * 4
    for sync_period in [0] + [2**x for x in range(0, int(log(period,2)))]:
        myname = '%(fname)s_%(demand)d_%(gl)s_%(num)02d' % {"fname": sys._getframe().f_code.co_name,
                                                     "demand": max_demand,
                                                     "gl": str(greedylimit),
                                                     "num": sync_period}

        logger.info("starting %s", myname)
        workload = dual_offset_workload(switches=['sw1', 'sw2'],
                                        period=period, offset=period/2.0,
                                        max_demand=max_demand, size=1,
                                        duration=2, timesteps=timesteps,
                                        workload_fcn=wave)

        ctrls = two_greedy_ctrls(greedylimit=greedylimit)
        sim = LinkBalancerSim(two_switch_topo(), ctrls)
        sim.run_and_trace(myname, workload, old=True, sync_period=sync_period,
                          show_graph=show_graph)
        logger.info("ending %s", myname)

def compare_greedy_dist_to_sync_dist(max_demand=8, show_graph=False):
    """Understand what improvement synchronization gives us over a greedy
    algorithm in a dynamic, discrete loadbalancing environment"""
    pass

def demo_strictly_local_ctrls(max_demand=8, show_graph=False):
    """Demonstrate synchronization across domains makes no difference when
    LinkBalanerCtrl only handles requests within its own domain"""

    #TODO: demonstrate with more than 1 srv per controller domain
    period = 32
    timesteps = period * 4
    for sync_period in [0] + [2**x for x in range(0, int(log(period,2)))]:
        myname = '%(fname)s_%(num)02d' % {"fname": sys._getframe().f_code.co_name, "num": sync_period}
        logger.info("starting %s", myname)
        workload = dual_offset_workload(switches=['sw1', 'sw2'],
                                        period=period, offset=period/2.0,
                                        max_demand=max_demand, size=1,
                                        duration=1, timesteps=timesteps,
                                        workload_fcn=wave)

        ctrls = strictly_local_ctrls()
        sim = LinkBalancerSim(two_switch_topo(), ctrls)
        sim.run_and_trace(myname, workload, old=True, sync_period=sync_period,
                          show_graph=show_graph)
        logger.info("ending %s", myname)

#Not enough brainpower left tonight to implement these, but I have been
#looking through the code and tests.  Here are some thoughts for regression
#test-ish graphs to generate.
#
#(1) Line topology w/3 controllers, each responsible for one server.   Let's
#say all requests go to the center controller first.  If the main parameter
#to vary is the relative size of requests, this should expose the effect of
#bins on our load-packing metric, plus ensure that LinkBalancerCtrl properly
#handles shortest-path ties in a reasonable way.
#
#(2) Line topology w/N controllers, N servers.  I have to think more about
#this one, but it seems like the nearest servers are always a better choice
#on a restricted topology where the demand is at one location, because
#they'd yield strictly less link usage.  But when demands are more spread
#out I could see a model solver in a central node doing a better job.
#
#(3) Ring topology w/N controllers, N servers.  Like 2.  Faster sync should
#yield closer-to-optimal balancing.
#
#(4) Ring topology w/N servers and varying number of controllers C, each
#commanding N / C servers.  With more controllers do we see crappier
#balancing for random workloads?
#
#-b

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>
import functools

import sqlalchemy
from sqlalchemy import inspect, orm, and_, or_
from sqlalchemy.ext.declarative import declarative_base, DeclarativeMeta
from sqlalchemy.orm.exc import UnmappedClassError

import query
from utils import classproperty, is_sequence, has_primary_key, camelcase_to_underscore

class ModelMeta(DeclarativeMeta):
    def __new__(cls, name, bases, dict_):
        # set __tablename__ (if not defined) to underscore version of class name
        if not dict_.get('__tablename__') and not dict_.get('__table__') is not None and has_primary_key(dict_):
            dict_['__tablename__'] = camelcase_to_underscore(name)

        return DeclarativeMeta.__new__(cls, name, bases, dict_)

    def __init__(cls, name, bases, dict_):
        decorated_events = {}
        for attr, value in dict_.iteritems():
            if hasattr(value, '__event__'):
                event_name,_ = value.__event__

                # initialize event namespace
                decorated_events.setdefault(event_name, [])

                # convert namespace to list if not already
                if not isinstance(decorated_events[event_name], list):
                    decorated_events[event_name] = [decorated_events[event_name]]

                # register our event
                decorated_events[event_name].append(value.__event__)

        ##
        # @todo: merge decorated events with dict_['__events__']
        ##

        events = dict_.get('__events__', {})

        for event_name, listeners in events.iteritems():
            if not isinstance(listeners, list):
                listeners = [listeners]

            for listener in listeners:
                if isinstance(listener, tuple):
                    # listener definition includes listen keyword args
                    listener, listen_kargs = listener
                else:
                    listen_kargs = {}

                if not hasattr(listener, '__call__'):
                    # assume listener is string reference to class method
                    listener = dict_[listener]

                sqlalchemy.event.listen(cls, event_name, listener, **listen_kargs)

        super(ModelMeta, cls).__init__(name, bases, dict_)

class ModelBase(object):
    '''Augmentable Base class for adding shared model properties/functions'''

    # default table args
    __table_args__ = {}

    # define a default order by when not specified by query operation
    # eg: { 'order_by': [column1, column2] }
    __mapper_args__ = {}

    # register orm event listeners
    __events__ = {}

    # query class to use for self.query
    query_class = query.Query

    # an instance of `query_class`
    # can be used to query the database for instances of this model
    # @note: requires setting Class.query = QueryProperty(session) when session available
    # see `make_declarative_base()` for automatic implementation
    query = None

    # specify sqla.session.query.filter() options for advanced and simple searches
    # eg: { key: lambda value: Model.column_name == val }
    __advanced_search__ = {}
    __simple_search__ = {}

    def __init__(self, *args, **kargs):
        self.update(*args, **kargs)

    def __repr__(self):
        '''
        Default representation of model
        '''
        values = ', '.join(['{0}={1}'.format(c, repr(getattr(self, c))) for c in self.columns])
        return '<{0}({1})>'.format(self.__class__.__name__, values)

    def update(self, data_dict=None, strict=False, **kargs):
        '''
        Update model with arbitrary set of data
        '''

        data = data_dict if isinstance(data_dict, dict) else kargs

        updatable_fields = self.strict_update_fields if strict else data.keys()
        relationships = self.relationships

        for k,v in data.iteritems():
            if hasattr(self, k) and k in updatable_fields:
                # consider v a dict if any of its elements are a dict
                v_is_dict = any([isinstance(_v, dict) for _v in v]) if is_sequence(v) else isinstance(v, dict)

                attr = getattr(self, k)

                if hasattr(attr, 'update') and v_is_dict:
                    # nest calls to attr.update if available and input is a data dict
                    attr.update(v)
                else:
                    if k in relationships and v_is_dict and not v:
                        # typically, if v is {}, then we're usually updating a relationship attribute
                        # where the relationship has an empty/null value in the database
                        # (e.g. a frontend sends missing relationship attribute as {})
                        # but if we set a relationship attribute = {}, things blow up
                        # so instead, convert {} to None which is valid for standard relationship attribute
                        v = None
                    setattr(self, k, v)

    def to_dict(self):
        '''
        Return dict representation of model.

        Drill down to any relationships and serialize those too.

        Assume that the current object has been loaded (lazy/joined/etc)
        and don't try to expand anything, i.e., just serialize the currently loaded data
        '''
        d = {}

        for field, value in self.__dict__.iteritems():
            if field.startswith('_sa'):
                # skip sqlalchemy properties
                continue

            d[field] = [v.to_dict() for v in value] if is_sequence(value) else value

        return d

    @property
    def strict_update_fields(self):
        '''
        Model fields which are allowed to be updated during strict mode

        Default is to limit to table columns
        Override as needed in child classes
        '''
        return self.columns

    ##
    # search/filtering methods
    ##

    @classmethod
    def get_search(cls, search_dict, filter_fns):
        filters = []

        for key, filter_fn in [(k,v) for k,v in filter_fns.iteritems() if k in search_dict]:
            filters.append(filter_fn(search_dict[key]))

        return filters

    @classmethod
    def advanced_search(cls, search_dict):
        filters = None
        if cls.__advanced_search__:
            _filters = cls.get_search(search_dict, cls.__advanced_search__)

            if _filters:
                filters = and_(*_filters)

        return filters

    @classmethod
    def simple_search(cls, search_string):
        filters = None

        if cls.__simple_search__:
            terms = [s for s in search_string.split()]
            fields = cls.__simple_search__.keys()
            field_count = len(fields)

            search_filters = []

            for term in terms:
                # create a dict with each `config_search` key and `term` so filters can be applied to each combination
                # i.e. { config_search_key1: term, config_search_key2: term, ..., config_search_keyN, term }
                search_dict = dict(zip(fields, [term]*field_count))
                term_filters = cls.get_search(search_dict, cls.__simple_search__)

                if term_filters:
                    # `or` filters together since only 1 filter needs to match for `term`
                    search_filters.append(or_(*term_filters))

            if search_filters:
                # `and` all search conditions together
                # each `term` should have an OR'd set of filters that evaluates to True
                filters = and_(*search_filters)

        return filters

    ##
    # session based methods/properties
    ##

    @property
    def session(self):
        '''Return session belonging to self'''
        return orm.object_session(self)

    def flush(self, *args, **kargs):
        return self.session.flush([self], *args, **kargs)

    def delete(self, *args, **kargs):
        return self.session.delete(self, *args, **kargs)

    def expire(self, *args, **kargs):
        return self.session.expire(self, *args, **kargs)

    def refresh(self, *args, **kargs):
        return self.session.refresh(self, *args, **kargs)

    def expunge(self, *args, **kargs):
        return self.session.expunge(self, *args, **kargs)

    @classmethod
    def get(cls, *args, **kargs):
        return cls.query.get(*args, **kargs)

    ##
    # inspect based methods/properties
    ##

    @classproperty
    def attrs(cls):
        '''Return ORM attributes'''
        return inspect(cls).attrs.keys()

    @classproperty
    def descriptors(cls):
        '''Return all ORM descriptors'''
        return [d for d in inspect(cls).mapper.all_orm_descriptors.keys() if not d.startswith('__')]

    @classproperty
    def relationships(cls):
        '''Return ORM relationships'''
        return inspect(cls).mapper.relationships.keys()

    @classproperty
    def column_attrs(cls):
        '''Return table columns as list of class attributes at the class level'''
        return inspect(cls).mapper.column_attrs

    @classproperty
    def columns(cls):
        '''Return table columns'''
        return inspect(cls).mapper.columns.keys()

class QueryProperty(object):
    def __init__(self, session):
        self.session = session

    def __get__(self, model, Model):
        try:
            mapper = orm.class_mapper(Model)
            if mapper:
                if not getattr(Model, 'query_class', None):
                    Model.query_class = query.Query

                q = Model.query_class(mapper, session=self.session())
                q.__model__ = Model

                return q
        except UnmappedClassError:
            return None

def make_declarative_base(session=None, query_property=None, Model=None):
    Model = Model or declarative_base(cls=ModelBase, constructor=ModelBase.__init__, metaclass=ModelMeta)
    extend_declarative_base(Model, session, query_property)
    return Model

def extend_declarative_base(Model, session=None, query_property=None):
    # attach query attribute to Model if `session` object passed in
    if session:
        Model.query = query_property(session) if query_property else QueryProperty(session)

def event(event_name, **kargs):
    def _event(f, *args, **kargs):
        f.__event__ = (event_name, kargs)

        @functools.wraps(f)
        def wrapper(cls, *args, **kargs):
            return f(*args, **kargs)

        return wrapper

    return _event

<commit_msg>Integrate decorated event methods with __events__ class attribute.<commit_after>
import functools
from collections import namedtuple

import sqlalchemy
from sqlalchemy import inspect, orm, and_, or_
from sqlalchemy.ext.declarative import declarative_base, DeclarativeMeta
from sqlalchemy.orm.exc import UnmappedClassError

import query
from utils import classproperty, is_sequence, has_primary_key, camelcase_to_underscore

Event = namedtuple('Event', ['name', 'listener', 'kargs'])

class ModelMeta(DeclarativeMeta):
    def __new__(cls, name, bases, dict_):
        # set __tablename__ (if not defined) to underscore version of class name
        if not dict_.get('__tablename__') and not dict_.get('__table__') is not None and has_primary_key(dict_):
            dict_['__tablename__'] = camelcase_to_underscore(name)

        # set __events__ to expected default so that it's updatable when initializing
        # e.g. if class definition sets __events__=None but defines decorated events,
        # then we want the final __events__ attribute to reflect the registered events.
        # if set to anything that's non-empty/non-dict will lead to an error if decorated events defined
        if not dict_.get('__events__'):
            dict_['__events__'] = {}

        return DeclarativeMeta.__new__(cls, name, bases, dict_)

    def __init__(cls, name, bases, dict_):
        events = []

        # append class attribute defined events
        if dict_.get('__events__'):
            # events defined on __events__ can have many forms (e.g. string based, list of tuples, etc)
            # so we need to iterate over them and parse into standardized Event object
            for event_name, listeners in dict_['__events__'].iteritems():
                if not isinstance(listeners, list):
                    listeners = [listeners]

                for listener in listeners:
                    if isinstance(listener, tuple):
                        # listener definition includes event.listen keyword args
                        listener, kargs = listener
                    else:
                        kargs = {}

                    if not hasattr(listener, '__call__'):
                        # no __call__ attribute? then assume listener is a string reference to class method
                        listener = dict_[listener]

                    events.append(Event(event_name, listener, kargs))

        # append events which were added via @event decorator
        events += [value.__event__ for value in dict_.values() if hasattr(value, '__event__')]

        if events:
            # reassemble events dict into consistent form using Event objects as values
            events_dict = {}
            for event in events:
                sqlalchemy.event.listen(cls, event.name, event.listener, **event.kargs)
                events_dict.setdefault(event.name, []).append(event)

            dict_['__events__'].update(events_dict)

        super(ModelMeta, cls).__init__(name, bases, dict_)

class ModelBase(object):
    '''Augmentable Base class for adding shared model properties/functions'''

    # default table args
    __table_args__ = {}

    # define a default order by when not specified by query operation
    # eg: { 'order_by': [column1, column2] }
    __mapper_args__ = {}

    # register orm event listeners
    __events__ = {}

    # query class to use for self.query
    query_class = query.Query

    # an instance of `query_class`
    # can be used to query the database for instances of this model
    # @note: requires setting Class.query = QueryProperty(session) when session available
    # see `make_declarative_base()` for automatic implementation
    query = None

    # specify sqla.session.query.filter() options for advanced and simple searches
    # eg: { key: lambda value: Model.column_name == val }
    __advanced_search__ = {}
    __simple_search__ = {}

    def __init__(self, *args, **kargs):
        self.update(*args, **kargs)

    def __repr__(self):
        '''
        Default representation of model
        '''
        values = ', '.join(['{0}={1}'.format(c, repr(getattr(self, c))) for c in self.columns])
        return '<{0}({1})>'.format(self.__class__.__name__, values)

    def update(self, data_dict=None, strict=False, **kargs):
        '''
        Update model with arbitrary set of data
        '''

        data = data_dict if isinstance(data_dict, dict) else kargs

        updatable_fields = self.strict_update_fields if strict else data.keys()
        relationships = self.relationships

        for k,v in data.iteritems():
            if hasattr(self, k) and k in updatable_fields:
                # consider v a dict if any of its elements are a dict
                v_is_dict = any([isinstance(_v, dict) for _v in v]) if is_sequence(v) else isinstance(v, dict)

                attr = getattr(self, k)

                if hasattr(attr, 'update') and v_is_dict:
                    # nest calls to attr.update if available and input is a data dict
                    attr.update(v)
                else:
                    if k in relationships and v_is_dict and not v:
                        # typically, if v is {}, then we're usually updating a relationship attribute
                        # where the relationship has an empty/null value in the database
                        # (e.g. a frontend sends missing relationship attribute as {})
                        # but if we set a relationship attribute = {}, things blow up
                        # so instead, convert {} to None which is valid for standard relationship attribute
                        v = None
                    setattr(self, k, v)

    def to_dict(self):
        '''
        Return dict representation of model.

        Drill down to any relationships and serialize those too.

        Assume that the current object has been loaded (lazy/joined/etc)
        and don't try to expand anything, i.e., just serialize the currently loaded data
        '''
        d = {}

        for field, value in self.__dict__.iteritems():
            if field.startswith('_sa'):
                # skip sqlalchemy properties
                continue

            d[field] = [v.to_dict() for v in value] if is_sequence(value) else value

        return d

    @property
    def strict_update_fields(self):
        '''
        Model fields which are allowed to be updated during strict mode

        Default is to limit to table columns
        Override as needed in child classes
        '''
        return self.columns

    ##
    # search/filtering methods
    ##

    @classmethod
    def get_search(cls, search_dict, filter_fns):
        filters = []

        for key, filter_fn in [(k,v) for k,v in filter_fns.iteritems() if k in search_dict]:
            filters.append(filter_fn(search_dict[key]))

        return filters

    @classmethod
    def advanced_search(cls, search_dict):
        filters = None
        if cls.__advanced_search__:
            _filters = cls.get_search(search_dict, cls.__advanced_search__)

            if _filters:
                filters = and_(*_filters)

        return filters

    @classmethod
    def simple_search(cls, search_string):
        filters = None

        if cls.__simple_search__:
            terms = [s for s in search_string.split()]
            fields = cls.__simple_search__.keys()
            field_count = len(fields)

            search_filters = []

            for term in terms:
                # create a dict with each `config_search` key and `term` so filters can be applied to each combination
                # i.e. { config_search_key1: term, config_search_key2: term, ..., config_search_keyN, term }
                search_dict = dict(zip(fields, [term]*field_count))
                term_filters = cls.get_search(search_dict, cls.__simple_search__)

                if term_filters:
                    # `or` filters together since only 1 filter needs to match for `term`
                    search_filters.append(or_(*term_filters))

            if search_filters:
                # `and` all search conditions together
                # each `term` should have an OR'd set of filters that evaluates to True
                filters = and_(*search_filters)

        return filters

    ##
    # session based methods/properties
    ##

    @property
    def session(self):
        '''Return session belonging to self'''
        return orm.object_session(self)

    def flush(self, *args, **kargs):
        return self.session.flush([self], *args, **kargs)

    def delete(self, *args, **kargs):
        return self.session.delete(self, *args, **kargs)

    def expire(self, *args, **kargs):
        return self.session.expire(self, *args, **kargs)

    def refresh(self, *args, **kargs):
        return self.session.refresh(self, *args, **kargs)

    def expunge(self, *args, **kargs):
        return self.session.expunge(self, *args, **kargs)

    @classmethod
    def get(cls, *args, **kargs):
        return cls.query.get(*args, **kargs)

    ##
    # inspect based methods/properties
    ##

    @classproperty
    def attrs(cls):
        '''Return ORM attributes'''
        return inspect(cls).attrs.keys()

    @classproperty
    def descriptors(cls):
        '''Return all ORM descriptors'''
        return [d for d in inspect(cls).mapper.all_orm_descriptors.keys() if not d.startswith('__')]

    @classproperty
    def relationships(cls):
        '''Return ORM relationships'''
        return inspect(cls).mapper.relationships.keys()

    @classproperty
    def column_attrs(cls):
        '''Return table columns as list of class attributes at the class level'''
        return inspect(cls).mapper.column_attrs

    @classproperty
    def columns(cls):
        '''Return table columns'''
        return inspect(cls).mapper.columns.keys()

class QueryProperty(object):
    def __init__(self, session):
        self.session = session

    def __get__(self, model, Model):
        try:
            mapper = orm.class_mapper(Model)
            if mapper:
                if not getattr(Model, 'query_class', None):
                    Model.query_class = query.Query

                q = Model.query_class(mapper, session=self.session())
                q.__model__ = Model

                return q
        except UnmappedClassError:
            return None

def make_declarative_base(session=None, query_property=None, Model=None):
    Model = Model or declarative_base(cls=ModelBase, constructor=ModelBase.__init__, metaclass=ModelMeta)
    extend_declarative_base(Model, session, query_property)
    return Model

def extend_declarative_base(Model, session=None, query_property=None):
    # attach query attribute to Model if `session` object passed in
    if session:
        Model.query = query_property(session) if query_property else QueryProperty(session)

def event(event_name, **kargs):
    def _event(f, *args, **kargs):
        f.__event__ = Event(event_name, f, kargs)

        @functools.wraps(f)
        def wrapper(cls, *args, **kargs):
            return f(*args, **kargs)

        return wrapper

    return _event

<|endoftext|>


<commit_before>"""
============================
The Vivarium Event Framework
============================

``vivarium`` constructs and manages the flow of :ref:`time <time_concept>`
through the emission of regularly scheduled events. The tools in this module
manage the relationships between event emitters and listeners and provide
an interface for user :ref:`components <components_concept>` to register
themselves as emitters or listeners to particular events.

For more information, see the associated event
:ref:`concept note <event_concept>`.

"""
from collections import defaultdict
from typing import Callable

class Event:
    """An Event object represents the context of an event. It is possible to subclass Event
    to add information for more specialized cases like vivarium.framework.population.PopulationEvent

    Attributes
    ----------
    time  : pandas.Timestamp
            The simulation time at which this event was emitted.
    index : pandas.Index
            An index into the population table containing all simulants effected by this event.
    """

    def __init__(self, index, user_data=None):
        self.index = index
        self.user_data = user_data if user_data is not None else {}
        self.time = None
        self.step_size = None

    def split(self, new_index):
        """Create a new event which is a copy of this one but with a new index.
        """
        new_event = Event(new_index, self.user_data)
        new_event.time = self.time
        new_event.step_size = self.step_size
        return new_event

    def __repr__(self):
        return "Event(user_data={}, time={}, step_size={})".format(self.user_data, self.time, self.step_size)

    def __eq__(self, other):
        return self.__dict__ == other.__dict__

class _EventChannel:
    def __init__(self, manager):
        self.manager = manager

        self.listeners = [[] for _ in range(10)]

    def emit(self, event):
        """Notifies all listeners to this channel that an event has occurred.

        Parameters
        ----------
        event : Event
            The event to be emitted.
        """
        if hasattr(event, 'time'):
            event.step_size = self.manager.step_size()
            event.time = self.manager.clock() + self.manager.step_size()

        for priority_bucket in self.listeners:
            for listener in sorted(priority_bucket, key=lambda x: x.__name__):
                listener(event)
        return event

    def __repr__(self):
        return "_EventChannel(listeners: {})".format([listener for bucket in self.listeners for listener in bucket])

class EventManager:
    """The configuration for the event system.

    Notes
    -----
    Client code should never need to interact with this class
    except through the decorators in this module and the emitter
    function exposed on the builder during the setup phase.
    """

    def __init__(self):
        self._event_types = defaultdict(lambda: _EventChannel(self))

    @property
    def name(self):
        return "event_manager"

    def setup(self, builder):
        """Performs this components simulation setup.

        Parameters
        ----------
        builder : vivarium.framework.engine.Builder
            Object giving access to core framework functionality.
        """
        self.clock = builder.time.clock()
        self.step_size = builder.time.step_size()

    def get_emitter(self, name):
        """Get an emitter function for the named event

        Parameters
        ----------
        name : str
               The name of the event

        Returns
        -------
        callable
            A function that accepts an Event object and distributes
            it to all listeners for this event.
        """
        return self._event_types[name].emit

    def register_listener(self, name, listener, priority=5):
        """Registers a new listener to the named event.

        Parameters
        ----------
        name : str
            The name of the event.
        listener : Callable
            The consumer of the named event.
        priority : int
            Number in range(10) used to assign the ordering in which listeners process the event.
        """
        self._event_types[name].listeners[priority].append(listener)

    def get_listeners(self, name):
        channel = self._event_types[name]
        return {priority: listeners for priority, listeners in enumerate(channel.listeners) if listeners}

    def list_events(self):
        """List all event names known to the event system

        Returns
        -------
        List[Event]
            A list of all known event names

        Notes
        -----
        This value can change after setup if components dynamically create new event labels.
        """
        return list(self._event_types.keys())

    def __contains__(self, item):
        return item in self._event_types

    def __str__(self):
        return "EventManager()"

    def __repr__(self):
        return f"EventManager(event_types: {self._event_types.keys()})"

class EventInterface:

    def __init__(self, event_manager: EventManager):
        self._event_manager = event_manager

    def get_emitter(self, name: str) -> Callable[[Event], Event]:
        """Gets and emitter for a named event.

        Parameters
        ----------
        name :
            The name of the event the requested emitter will emit.
            Users may provide their own named events by requesting an emitter with this function,
            but should do so with caution as it makes time much more difficult to think about.

        Returns
        -------
            An emitter for the named event. The emitter should be called by the requesting component
            at the appropriate point in the simulation lifecycle.
        """
        return self._event_manager.get_emitter(name)

    def register_listener(self, name: str, listener: Callable[[Event], None], priority: int=5) -> None:
        """Registers a callable as a listener to a events with the given name.

        The listening callable will be called with a named ``Event`` as it's only argument any time the
        event emitter is invoked from somewhere in the simulation.

        The framework creates the following events and emits them at different points in the simulation:
            At the end of the setup phase: ``post_setup``
            Every time step: ``time_step__prepare``, ``time_step``, ``time_step__cleanup``, ``collect_metrics``
            At simulation end: ``simulation_end``

        Parameters
        ----------
        name :
            The name of the event to listen for.
        listener :
            The callable to be invoked any time an ``Event`` with the given name is emitted.
        priority : {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
            An indication of the order in which event listeners should be called. Listeners with
            smaller priority values will be called earlier. Listeners with the same priority have
            no guaranteed ordering.  This feature should be avoided if possible. Components should
            strive to obey the Markov property as they transform the state table (the state of the simulation
            at the beginning of the next time step should only depend on the current state of the system).
        """
        self._event_manager.register_listener(name, listener, priority)
<commit_msg>simplify repr of singleton manager<commit_after>"""
============================
The Vivarium Event Framework
============================

``vivarium`` constructs and manages the flow of :ref:`time <time_concept>`
through the emission of regularly scheduled events. The tools in this module
manage the relationships between event emitters and listeners and provide
an interface for user :ref:`components <components_concept>` to register
themselves as emitters or listeners to particular events.

For more information, see the associated event
:ref:`concept note <event_concept>`.

"""
from collections import defaultdict
from typing import Callable

class Event:
    """An Event object represents the context of an event. It is possible to subclass Event
    to add information for more specialized cases like vivarium.framework.population.PopulationEvent

    Attributes
    ----------
    time  : pandas.Timestamp
            The simulation time at which this event was emitted.
    index : pandas.Index
            An index into the population table containing all simulants effected by this event.
    """

    def __init__(self, index, user_data=None):
        self.index = index
        self.user_data = user_data if user_data is not None else {}
        self.time = None
        self.step_size = None

    def split(self, new_index):
        """Create a new event which is a copy of this one but with a new index.
        """
        new_event = Event(new_index, self.user_data)
        new_event.time = self.time
        new_event.step_size = self.step_size
        return new_event

    def __repr__(self):
        return "Event(user_data={}, time={}, step_size={})".format(self.user_data, self.time, self.step_size)

    def __eq__(self, other):
        return self.__dict__ == other.__dict__

class _EventChannel:
    def __init__(self, manager):
        self.manager = manager

        self.listeners = [[] for _ in range(10)]

    def emit(self, event):
        """Notifies all listeners to this channel that an event has occurred.

        Parameters
        ----------
        event : Event
            The event to be emitted.
        """
        if hasattr(event, 'time'):
            event.step_size = self.manager.step_size()
            event.time = self.manager.clock() + self.manager.step_size()

        for priority_bucket in self.listeners:
            for listener in sorted(priority_bucket, key=lambda x: x.__name__):
                listener(event)
        return event

    def __repr__(self):
        return "_EventChannel(listeners: {})".format([listener for bucket in self.listeners for listener in bucket])

class EventManager:
    """The configuration for the event system.

    Notes
    -----
    Client code should never need to interact with this class
    except through the decorators in this module and the emitter
    function exposed on the builder during the setup phase.
    """

    def __init__(self):
        self._event_types = defaultdict(lambda: _EventChannel(self))

    @property
    def name(self):
        return "event_manager"

    def setup(self, builder):
        """Performs this components simulation setup.

        Parameters
        ----------
        builder : vivarium.framework.engine.Builder
            Object giving access to core framework functionality.
        """
        self.clock = builder.time.clock()
        self.step_size = builder.time.step_size()

    def get_emitter(self, name):
        """Get an emitter function for the named event

        Parameters
        ----------
        name : str
               The name of the event

        Returns
        -------
        callable
            A function that accepts an Event object and distributes
            it to all listeners for this event.
        """
        return self._event_types[name].emit

    def register_listener(self, name, listener, priority=5):
        """Registers a new listener to the named event.

        Parameters
        ----------
        name : str
            The name of the event.
        listener : Callable
            The consumer of the named event.
        priority : int
            Number in range(10) used to assign the ordering in which listeners process the event.
        """
        self._event_types[name].listeners[priority].append(listener)

    def get_listeners(self, name):
        channel = self._event_types[name]
        return {priority: listeners for priority, listeners in enumerate(channel.listeners) if listeners}

    def list_events(self):
        """List all event names known to the event system

        Returns
        -------
        List[Event]
            A list of all known event names

        Notes
        -----
        This value can change after setup if components dynamically create new event labels.
        """
        return list(self._event_types.keys())

    def __contains__(self, item):
        return item in self._event_types

    def __repr__(self):
        return "EventManager()"

class EventInterface:

    def __init__(self, event_manager: EventManager):
        self._event_manager = event_manager

    def get_emitter(self, name: str) -> Callable[[Event], Event]:
        """Gets and emitter for a named event.

        Parameters
        ----------
        name :
            The name of the event the requested emitter will emit.
            Users may provide their own named events by requesting an emitter with this function,
            but should do so with caution as it makes time much more difficult to think about.

        Returns
        -------
            An emitter for the named event. The emitter should be called by the requesting component
            at the appropriate point in the simulation lifecycle.
        """
        return self._event_manager.get_emitter(name)

    def register_listener(self, name: str, listener: Callable[[Event], None], priority: int=5) -> None:
        """Registers a callable as a listener to a events with the given name.

        The listening callable will be called with a named ``Event`` as it's only argument any time the
        event emitter is invoked from somewhere in the simulation.

        The framework creates the following events and emits them at different points in the simulation:
            At the end of the setup phase: ``post_setup``
            Every time step: ``time_step__prepare``, ``time_step``, ``time_step__cleanup``, ``collect_metrics``
            At simulation end: ``simulation_end``

        Parameters
        ----------
        name :
            The name of the event to listen for.
        listener :
            The callable to be invoked any time an ``Event`` with the given name is emitted.
        priority : {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
            An indication of the order in which event listeners should be called. Listeners with
            smaller priority values will be called earlier. Listeners with the same priority have
            no guaranteed ordering.  This feature should be avoided if possible. Components should
            strive to obey the Markov property as they transform the state table (the state of the simulation
            at the beginning of the next time step should only depend on the current state of the system).
        """
        self._event_manager.register_listener(name, listener, priority)
<|endoftext|>


<commit_before>#!/usr/bin/python
# <variable> = required
# Usage ./lava-report.py <option> [json]
import argparse
import subprocess
import re
from utils import *

def parse_json(json):
    jobs = load_json(json)
    url = validate_input(jobs['username'], jobs['token'], jobs['server'])
    connection = connect(url)
    duration = jobs['duration']
    # Remove unused data
    jobs.pop('duration')
    jobs.pop('username')
    jobs.pop('token')
    jobs.pop('server')
    return connection, jobs, duration

def boot_report(args):
    connection, jobs, duration =  parse_json(args.boot)
    results_directory = os.getcwd() + '/results'
    results = {}
    dt_tests = False
    mkdir(results_directory)
    for job_id in jobs:
        # Retrieve job details
        job_details = connection.scheduler.job_details(job_id)
        if job_details['requested_device_type_id']:
            device_type = job_details['requested_device_type_id']
        if job_details['description']:
            job_name = job_details['description']
        result = jobs[job_id]['result']
        bundle = jobs[job_id]['bundle']
        # Retrieve the log file
        binary_job_file = connection.scheduler.job_output(job_id)
        # Parse LAVA messages out of log
        raw_job_file = str(binary_job_file)
        job_file = ''
        dt_test = None
        dt_test_result = None
        dt_tests_passed = None
        dt_tests_failed = None
        for line in raw_job_file.splitlines():
            if '<LAVA_DISPATCHER>' not in line:
                if len(line) != 0:
                    job_file += line + '\n'
            if '### dt-test ### end of selftest' in line:
                dt_tests = True
                regex = re.compile("(?P<test>\d+\*?)")
                dt_test_results = regex.findall(line)
                dt_tests_passed = dt_test_results[2]
                dt_tests_failed = dt_test_results[3]
                if int(dt_test_results[3]) > 0:
                    dt_test_result = 'FAIL'
                else:
                    dt_test_result = 'PASS'
        # Retrieve bundle
        if bundle is not None:
            json_bundle = connection.dashboard.get(bundle)
            bundle_data = json.loads(json_bundle['content'])
            bundle_attributes =  bundle_data['test_runs'][0]['attributes']
            boot_meta = {}
            kernel_defconfig = None
            kernel_version = None
            kernel_endian = None
            kernel_boot_time = None
            kernel_tree = None
            kernel_image = None
            kernel_addr = None
            initrd_addr = None
            dtb_addr = None
            dtb_append = None
            fastboot = None
            fastboot_cmd = None
            if in_bundle_attributes(bundle_attributes, 'kernel.defconfig'):
                kernel_defconfig = bundle_attributes['kernel.defconfig']
            if in_bundle_attributes(bundle_attributes, 'kernel.version'):
                kernel_version = bundle_attributes['kernel.version']
            if in_bundle_attributes(bundle_attributes, 'kernel.endian'):
                kernel_endian = bundle_attributes['kernel.endian']
            if in_bundle_attributes(bundle_attributes, 'platform.fastboot'):
                fastboot = bundle_attributes['platform.fastboot']
            if in_bundle_attributes(bundle_attributes, 'kernel-boot-time'):
                kernel_boot_time = bundle_attributes['kernel-boot-time']
            if in_bundle_attributes(bundle_attributes, 'kernel.tree'):
                kernel_tree = bundle_attributes['kernel.tree']
            if in_bundle_attributes(bundle_attributes, 'kernel-image'):
                kernel_image = bundle_attributes['kernel-image']
            if in_bundle_attributes(bundle_attributes, 'kernel-addr'):
                kernel_addr = bundle_attributes['kernel-addr']
            if in_bundle_attributes(bundle_attributes, 'initrd-addr'):
                initrd_addr = bundle_attributes['initrd-addr']
            if in_bundle_attributes(bundle_attributes, 'dtb-addr'):
                dtb_addr = bundle_attributes['dtb-addr']
            if in_bundle_attributes(bundle_attributes, 'dtb-append'):
                dtb_append = bundle_attributes['dtb-append']

        # Record the boot log and result
        # TODO: Will need to map device_types to dashboard device types
        if kernel_defconfig and device_type and result:
            print 'Creating boot log for %s' % device_type
            log = 'boot-%s.log' % device_type
            directory = os.path.join(results_directory, kernel_defconfig)
            ensure_dir(directory)
            write_file(job_file, log, directory)
            if kernel_boot_time is None:
                kernel_boot_time = '0.0'
            if results.has_key(kernel_defconfig):
                results[kernel_defconfig].append({'device_type': device_type, 'dt_test_result': dt_test_result, 'dt_tests_passed': dt_tests_passed, 'dt_tests_failed': dt_tests_failed, 'kernel_boot_time': kernel_boot_time, 'result': result})
            else:
                results[kernel_defconfig] = [{'device_type': device_type, 'dt_test_result': dt_test_result, 'dt_tests_passed': dt_tests_passed, 'dt_tests_failed': dt_tests_failed, 'kernel_boot_time': kernel_boot_time, 'result': result}]
            # Create JSON format boot metadata
            print 'Creating JSON format boot metadata'
            boot_meta['boot_log'] = log
            # TODO: Fix this
            boot_meta['boot_log_html'] = None
            boot_meta['boot_result'] = result
            boot_meta['boot_time'] = kernel_boot_time
            # TODO: Fix this
            boot_meta['boot_warnings'] = None
            # TODO: Fix this
            boot_meta['dtb'] = None
            boot_meta['dtb_addr'] = dtb_addr
            boot_meta['dtb_append'] = dtb_append
            boot_meta['dt_test'] = dt_test
            boot_meta['endian'] = kernel_endian
            boot_meta['fastboot'] = fastboot
            boot_meta['fastboot_cmds'] = fastboot_cmd
            # TODO: Fix this
            boot_meta['initrd'] = None
            boot_meta['initrd_addr'] = initrd_addr
            boot_meta['kernel_image'] = kernel_image
            boot_meta['loadaddr'] = kernel_addr
            json_file = 'boot-%s.json' % device_type
            write_json(json_file, directory, boot_meta)

    if results and kernel_tree and kernel_version:
        print 'Creating boot summary for %s' % kernel_version
        boot = '%s-boot-report.txt' % kernel_version
        passed = 0
        failed = 0
        for defconfig, results_list in results.items():
            for result in results_list:
                if result['result'] == 'PASS':
                    passed += 1
                else:
                    failed += 1
        total = passed + failed
        with open(os.path.join(results_directory, boot), 'a') as f:
            f.write('to : %s\n' % args.email)
            f.write('from : lava@armcloud.us\n')
            f.write('subject : %s boot: %s boots: %s passed, %s failed (%s)\n' % (kernel_tree,
                                                                                str(total),
                                                                                str(passed),
                                                                                str(failed),
                                                                                kernel_version))
            f.write('\n')
            f.write('Full Build Report: http://status.armcloud.us/build/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
            f.write('Full Boot Report: http://status.armcloud.us/boot/all/job/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
            f.write('\n')
            f.write('Total Duration: %.2f minutes\n' % (duration / 60))
            f.write('Tree/Branch: %s\n' % kernel_tree)
            f.write('Git Describe: %s\n' % kernel_version)
            first = True
            for defconfig, results_list in results.items():
                for result in results_list:
                    if result['result'] != 'PASS':
                        if first:
                            f.write('\n')
                            f.write('Failed Boot Tests:\n')
                            first = False
                        f.write('\n')
                        f.write(defconfig)
                        f.write('\n')
                        break
                for result in results_list:
                    if result['result'] != 'PASS':
                        f.write('    %s   %ss   boot-test: %s\n' % (result['device_type'],
                                                                    result['kernel_boot_time'],
                                                                    result['result']))
                        f.write('    http://storage.armcloud.us/kernel-ci/%s/%s/%s/boot-%s.html' % (kernel_tree,
                                                                                kernel_version,
                                                                                defconfig,
                                                                                result['device_type']))
                        f.write('\n')
            f.write('\n')
            f.write('Full Boot Report:\n')
            for defconfig, results_list in results.items():
                f.write('\n')
                f.write(defconfig)
                f.write('\n')
                for result in results_list:
                    f.write('    %s   %ss   boot-test: %s\n' % (result['device_type'], result['kernel_boot_time'], result['result']))

    # dt-self-test
    if results and kernel_tree and kernel_version and dt_tests:
        print 'Creating device tree runtime self test summary for %s' % kernel_version
        dt_self_test = '%s-dt-runtime-self-test-report.txt' % kernel_version
        passed = 0
        failed = 0
        for defconfig, results_list in results.items():
            for result in results_list:
                if result['dt_test_result'] == 'PASS':
                    passed += 1
                elif result['dt_test_result'] == 'FAIL':
                    failed += 1
        total = passed + failed
        with open(os.path.join(results_directory, dt_self_test), 'a') as f:
            f.write('to : %s\n' % args.email)
            f.write('from : lava@armcloud.us\n')
            f.write('subject : %s dt-runtime-unit-tests: %s boards tested: %s passed, %s failed (%s)\n' % (kernel_tree,
                                                                                str(total),
                                                                                str(passed),
                                                                                str(failed),
                                                                                kernel_version))
            f.write('\n')
            f.write('Full Build Report: http://status.armcloud.us/build/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
            f.write('Full Boot Report: http://status.armcloud.us/boot/all/job/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
            f.write('Full Test Report: http://status.armcloud.us/test/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
            f.write('\n')
            f.write('Tree/Branch: %s\n' % kernel_tree)
            f.write('Git Describe: %s\n' % kernel_version)
            first = True
            for defconfig, results_list in results.items():
                for result in results_list:
                    if result['dt_test_result'] == 'FAIL':
                        if first:
                            f.write('\n')
                            f.write('Failed Device Tree Self Tests:\n')
                            first = False
                        f.write('\n')
                        f.write(defconfig)
                        f.write('\n')
                        break
                for result in results_list:
                    if result['dt_test_result'] == "FAIL":
                        f.write('    %s   passed: %s / failed: %s   dt-runtime-unit-tests: %s\n' % (result['device_type'],
                                                                                result['dt_tests_passed'],
                                                                                result['dt_tests_failed'],
                                                                                result['dt_test_result']))
                        f.write('    http://storage.armcloud.us/kernel-ci/%s/%s/%s/boot-%s.html' % (kernel_tree,
                                                                                kernel_version,
                                                                                defconfig,
                                                                                result['device_type']))
                        f.write('\n')
            f.write('\n')
            f.write('Full Test Report:\n')
            for defconfig, results_list in results.items():
                first = True
                for result in results_list:
                    if result['dt_test_result']:
                        if first:
                            f.write('\n')
                            f.write(defconfig)
                            f.write('\n')
                            first = False
                        f.write('    %s   passed: %s / failed: %s   dt-runtime-unit-tests: %s\n' % (result['device_type'],
                                                                                result['dt_tests_passed'],
                                                                                result['dt_tests_failed'],
                                                                                result['dt_test_result']))

    # sendmail
    if args.email:
        print 'Sending e-mail summary to %s' % args.email
        cmd = 'cat %s | sendmail -t' % os.path.join(results_directory, boot)
        subprocess.check_output(cmd, shell=True)
        if dt_tests:
            cmd = 'cat %s | sendmail -t' % os.path.join(results_directory, dt_self_test)
            subprocess.check_output(cmd, shell=True)

def main(args):
    if args.boot:
        boot_report(args)
    exit(0)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--boot", help="creates a kernel-ci boot report from a given json file")
    parser.add_argument("--email", help="email address to send report to")
    args = parser.parse_args()
    main(args)
<commit_msg>kernel-ci: self test -> unit test<commit_after>#!/usr/bin/python
# <variable> = required
# Usage ./lava-report.py <option> [json]
import argparse
import subprocess
import re
from utils import *

def parse_json(json):
    jobs = load_json(json)
    url = validate_input(jobs['username'], jobs['token'], jobs['server'])
    connection = connect(url)
    duration = jobs['duration']
    # Remove unused data
    jobs.pop('duration')
    jobs.pop('username')
    jobs.pop('token')
    jobs.pop('server')
    return connection, jobs, duration

def boot_report(args):
    connection, jobs, duration =  parse_json(args.boot)
    results_directory = os.getcwd() + '/results'
    results = {}
    dt_tests = False
    mkdir(results_directory)
    for job_id in jobs:
        # Retrieve job details
        job_details = connection.scheduler.job_details(job_id)
        if job_details['requested_device_type_id']:
            device_type = job_details['requested_device_type_id']
        if job_details['description']:
            job_name = job_details['description']
        result = jobs[job_id]['result']
        bundle = jobs[job_id]['bundle']
        # Retrieve the log file
        binary_job_file = connection.scheduler.job_output(job_id)
        # Parse LAVA messages out of log
        raw_job_file = str(binary_job_file)
        job_file = ''
        dt_test = None
        dt_test_result = None
        dt_tests_passed = None
        dt_tests_failed = None
        for line in raw_job_file.splitlines():
            if '<LAVA_DISPATCHER>' not in line:
                if len(line) != 0:
                    job_file += line + '\n'
            if '### dt-test ### end of selftest' in line:
                dt_tests = True
                regex = re.compile("(?P<test>\d+\*?)")
                dt_test_results = regex.findall(line)
                dt_tests_passed = dt_test_results[2]
                dt_tests_failed = dt_test_results[3]
                if int(dt_test_results[3]) > 0:
                    dt_test_result = 'FAIL'
                else:
                    dt_test_result = 'PASS'
        # Retrieve bundle
        if bundle is not None:
            json_bundle = connection.dashboard.get(bundle)
            bundle_data = json.loads(json_bundle['content'])
            bundle_attributes =  bundle_data['test_runs'][0]['attributes']
            boot_meta = {}
            kernel_defconfig = None
            kernel_version = None
            kernel_endian = None
            kernel_boot_time = None
            kernel_tree = None
            kernel_image = None
            kernel_addr = None
            initrd_addr = None
            dtb_addr = None
            dtb_append = None
            fastboot = None
            fastboot_cmd = None
            if in_bundle_attributes(bundle_attributes, 'kernel.defconfig'):
                kernel_defconfig = bundle_attributes['kernel.defconfig']
            if in_bundle_attributes(bundle_attributes, 'kernel.version'):
                kernel_version = bundle_attributes['kernel.version']
            if in_bundle_attributes(bundle_attributes, 'kernel.endian'):
                kernel_endian = bundle_attributes['kernel.endian']
            if in_bundle_attributes(bundle_attributes, 'platform.fastboot'):
                fastboot = bundle_attributes['platform.fastboot']
            if in_bundle_attributes(bundle_attributes, 'kernel-boot-time'):
                kernel_boot_time = bundle_attributes['kernel-boot-time']
            if in_bundle_attributes(bundle_attributes, 'kernel.tree'):
                kernel_tree = bundle_attributes['kernel.tree']
            if in_bundle_attributes(bundle_attributes, 'kernel-image'):
                kernel_image = bundle_attributes['kernel-image']
            if in_bundle_attributes(bundle_attributes, 'kernel-addr'):
                kernel_addr = bundle_attributes['kernel-addr']
            if in_bundle_attributes(bundle_attributes, 'initrd-addr'):
                initrd_addr = bundle_attributes['initrd-addr']
            if in_bundle_attributes(bundle_attributes, 'dtb-addr'):
                dtb_addr = bundle_attributes['dtb-addr']
            if in_bundle_attributes(bundle_attributes, 'dtb-append'):
                dtb_append = bundle_attributes['dtb-append']

        # Record the boot log and result
        # TODO: Will need to map device_types to dashboard device types
        if kernel_defconfig and device_type and result:
            print 'Creating boot log for %s' % device_type
            log = 'boot-%s.log' % device_type
            directory = os.path.join(results_directory, kernel_defconfig)
            ensure_dir(directory)
            write_file(job_file, log, directory)
            if kernel_boot_time is None:
                kernel_boot_time = '0.0'
            if results.has_key(kernel_defconfig):
                results[kernel_defconfig].append({'device_type': device_type, 'dt_test_result': dt_test_result, 'dt_tests_passed': dt_tests_passed, 'dt_tests_failed': dt_tests_failed, 'kernel_boot_time': kernel_boot_time, 'result': result})
            else:
                results[kernel_defconfig] = [{'device_type': device_type, 'dt_test_result': dt_test_result, 'dt_tests_passed': dt_tests_passed, 'dt_tests_failed': dt_tests_failed, 'kernel_boot_time': kernel_boot_time, 'result': result}]
            # Create JSON format boot metadata
            print 'Creating JSON format boot metadata'
            boot_meta['boot_log'] = log
            # TODO: Fix this
            boot_meta['boot_log_html'] = None
            boot_meta['boot_result'] = result
            boot_meta['boot_time'] = kernel_boot_time
            # TODO: Fix this
            boot_meta['boot_warnings'] = None
            # TODO: Fix this
            boot_meta['dtb'] = None
            boot_meta['dtb_addr'] = dtb_addr
            boot_meta['dtb_append'] = dtb_append
            boot_meta['dt_test'] = dt_test
            boot_meta['endian'] = kernel_endian
            boot_meta['fastboot'] = fastboot
            boot_meta['fastboot_cmds'] = fastboot_cmd
            # TODO: Fix this
            boot_meta['initrd'] = None
            boot_meta['initrd_addr'] = initrd_addr
            boot_meta['kernel_image'] = kernel_image
            boot_meta['loadaddr'] = kernel_addr
            json_file = 'boot-%s.json' % device_type
            write_json(json_file, directory, boot_meta)

    if results and kernel_tree and kernel_version:
        print 'Creating boot summary for %s' % kernel_version
        boot = '%s-boot-report.txt' % kernel_version
        passed = 0
        failed = 0
        for defconfig, results_list in results.items():
            for result in results_list:
                if result['result'] == 'PASS':
                    passed += 1
                else:
                    failed += 1
        total = passed + failed
        with open(os.path.join(results_directory, boot), 'a') as f:
            f.write('to : %s\n' % args.email)
            f.write('from : lava@armcloud.us\n')
            f.write('subject : %s boot: %s boots: %s passed, %s failed (%s)\n' % (kernel_tree,
                                                                                str(total),
                                                                                str(passed),
                                                                                str(failed),
                                                                                kernel_version))
            f.write('\n')
            f.write('Full Build Report: http://status.armcloud.us/build/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
            f.write('Full Boot Report: http://status.armcloud.us/boot/all/job/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
            f.write('\n')
            f.write('Total Duration: %.2f minutes\n' % (duration / 60))
            f.write('Tree/Branch: %s\n' % kernel_tree)
            f.write('Git Describe: %s\n' % kernel_version)
            first = True
            for defconfig, results_list in results.items():
                for result in results_list:
                    if result['result'] != 'PASS':
                        if first:
                            f.write('\n')
                            f.write('Failed Boot Tests:\n')
                            first = False
                        f.write('\n')
                        f.write(defconfig)
                        f.write('\n')
                        break
                for result in results_list:
                    if result['result'] != 'PASS':
                        f.write('    %s   %ss   boot-test: %s\n' % (result['device_type'],
                                                                    result['kernel_boot_time'],
                                                                    result['result']))
                        f.write('    http://storage.armcloud.us/kernel-ci/%s/%s/%s/boot-%s.html' % (kernel_tree,
                                                                                kernel_version,
                                                                                defconfig,
                                                                                result['device_type']))
                        f.write('\n')
            f.write('\n')
            f.write('Full Boot Report:\n')
            for defconfig, results_list in results.items():
                f.write('\n')
                f.write(defconfig)
                f.write('\n')
                for result in results_list:
                    f.write('    %s   %ss   boot-test: %s\n' % (result['device_type'], result['kernel_boot_time'], result['result']))

    # dt-self-test
    if results and kernel_tree and kernel_version and dt_tests:
        print 'Creating device tree runtime self test summary for %s' % kernel_version
        dt_self_test = '%s-dt-runtime-self-test-report.txt' % kernel_version
        passed = 0
        failed = 0
        for defconfig, results_list in results.items():
            for result in results_list:
                if result['dt_test_result'] == 'PASS':
                    passed += 1
                elif result['dt_test_result'] == 'FAIL':
                    failed += 1
        total = passed + failed
        with open(os.path.join(results_directory, dt_self_test), 'a') as f:
            f.write('to : %s\n' % args.email)
            f.write('from : lava@armcloud.us\n')
            f.write('subject : %s dt-runtime-unit-tests: %s boards tested: %s passed, %s failed (%s)\n' % (kernel_tree,
                                                                                str(total),
                                                                                str(passed),
                                                                                str(failed),
                                                                                kernel_version))
            f.write('\n')
            f.write('Full Build Report: http://status.armcloud.us/build/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
            f.write('Full Boot Report: http://status.armcloud.us/boot/all/job/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
            f.write('Full Test Report: http://status.armcloud.us/test/%s/kernel/%s/\n' % (kernel_tree, kernel_version))
            f.write('\n')
            f.write('Tree/Branch: %s\n' % kernel_tree)
            f.write('Git Describe: %s\n' % kernel_version)
            first = True
            for defconfig, results_list in results.items():
                for result in results_list:
                    if result['dt_test_result'] == 'FAIL':
                        if first:
                            f.write('\n')
                            f.write('Failed Device Tree Unit Tests:\n')
                            first = False
                        f.write('\n')
                        f.write(defconfig)
                        f.write('\n')
                        break
                for result in results_list:
                    if result['dt_test_result'] == "FAIL":
                        f.write('    %s   passed: %s / failed: %s   dt-runtime-unit-tests: %s\n' % (result['device_type'],
                                                                                result['dt_tests_passed'],
                                                                                result['dt_tests_failed'],
                                                                                result['dt_test_result']))
                        f.write('    http://storage.armcloud.us/kernel-ci/%s/%s/%s/boot-%s.html' % (kernel_tree,
                                                                                kernel_version,
                                                                                defconfig,
                                                                                result['device_type']))
                        f.write('\n')
            f.write('\n')
            f.write('Full Unit Test Report:\n')
            for defconfig, results_list in results.items():
                first = True
                for result in results_list:
                    if result['dt_test_result']:
                        if first:
                            f.write('\n')
                            f.write(defconfig)
                            f.write('\n')
                            first = False
                        f.write('    %s   passed: %s / failed: %s   dt-runtime-unit-tests: %s\n' % (result['device_type'],
                                                                                result['dt_tests_passed'],
                                                                                result['dt_tests_failed'],
                                                                                result['dt_test_result']))

    # sendmail
    if args.email:
        print 'Sending e-mail summary to %s' % args.email
        cmd = 'cat %s | sendmail -t' % os.path.join(results_directory, boot)
        subprocess.check_output(cmd, shell=True)
        if dt_tests:
            cmd = 'cat %s | sendmail -t' % os.path.join(results_directory, dt_self_test)
            subprocess.check_output(cmd, shell=True)

def main(args):
    if args.boot:
        boot_report(args)
    exit(0)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--boot", help="creates a kernel-ci boot report from a given json file")
    parser.add_argument("--email", help="email address to send report to")
    args = parser.parse_args()
    main(args)
<|endoftext|>