<commit_before>#!/usr/bin/env python2
#***************************************************************************
#
#   Copyright (c) 2015-2016 PX4 Development Team. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materials provided with the
#    distribution.
# 3. Neither the name PX4 nor the names of its contributors may be
#    used to endorse or promote products derived from this software
#    without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
#***************************************************************************/

#
# @author Andreas Antener <andreas@uaventure.com>
#

# The shebang of this file is currently Python2 because some
# dependencies such as pymavlink don't play well with Python3 yet.
from __future__ import division

PKG = 'px4'

import rospy
import glob
import json
import math
import os
import px4tools
import sys
from mavros import mavlink
from mavros_msgs.msg import Mavlink, Waypoint, WaypointReached
from mavros_test_common import MavrosTestCommon
from pymavlink import mavutil
from threading import Thread

def get_last_log():
    try:
        log_path = os.environ['PX4_LOG_DIR']
    except KeyError:
        log_path = os.path.join(os.environ['HOME'],
                                '.ros/rootfs/fs/microsd/log')
    last_log_dir = sorted(glob.glob(os.path.join(log_path, '*')))[-1]
    last_log = sorted(glob.glob(os.path.join(last_log_dir, '*.ulg')))[-1]
    return last_log

def read_mission(mission_filename):
    wps = []
    with open(mission_filename, 'r') as f:
        for waypoint in read_plan_file(f):
            wps.append(waypoint)
            rospy.logdebug(waypoint)

    # set first item to current
    if wps:
        wps[0].is_current = True

    return wps

def read_plan_file(f):
    d = json.load(f)
    if 'mission' in d:
        d = d['mission']

    if 'items' in d:
        for wp in d['items']:
            yield Waypoint(
                is_current=False,
                frame=int(wp['frame']),
                command=int(wp['command']),
                param1=float('nan'
                             if wp['params'][0] is None else wp['params'][0]),
                param2=float('nan'
                             if wp['params'][1] is None else wp['params'][1]),
                param3=float('nan'
                             if wp['params'][2] is None else wp['params'][2]),
                param4=float('nan'
                             if wp['params'][3] is None else wp['params'][3]),
                x_lat=float(wp['params'][4]),
                y_long=float(wp['params'][5]),
                z_alt=float(wp['params'][6]),
                autocontinue=bool(wp['autoContinue']))
    else:
        raise IOError("no mission items")

class MavrosMissionTest(MavrosTestCommon):
    """
    Run a mission
    """

    def setUp(self):
        super(self.__class__, self).setUp()

        self.mission_item_reached = -1  # first mission item is 0
        self.mission_name = ""

        self.mavlink_pub = rospy.Publisher('mavlink/to', Mavlink, queue_size=1)
        self.mission_item_reached_sub = rospy.Subscriber(
            'mavros/mission/reached', WaypointReached,
            self.mission_item_reached_callback)

        # need to simulate heartbeat to prevent datalink loss detection
        self.hb_mav_msg = mavutil.mavlink.MAVLink_heartbeat_message(
            mavutil.mavlink.MAV_TYPE_GCS, 0, 0, 0, 0, 0)
        self.hb_mav_msg.pack(mavutil.mavlink.MAVLink('', 2, 1))
        self.hb_ros_msg = mavlink.convert_to_rosmsg(self.hb_mav_msg)
        self.hb_thread = Thread(target=self.send_heartbeat, args=())
        self.hb_thread.daemon = True
        self.hb_thread.start()

    def tearDown(self):
        super(MavrosMissionTest, self).tearDown()

    #
    # Helper methods
    #
    def send_heartbeat(self):
        rate = rospy.Rate(2)  # Hz
        while not rospy.is_shutdown():
            self.mavlink_pub.publish(self.hb_ros_msg)
            try:  # prevent garbage in console output when thread is killed
                rate.sleep()
            except rospy.ROSInterruptException:
                pass

    def mission_item_reached_callback(self, data):
        if self.mission_item_reached != data.wp_seq:
            rospy.loginfo("mission item reached: {0}".format(data.wp_seq))
            self.mission_item_reached = data.wp_seq

    def distance_to_wp(self, lat, lon, alt):
        """alt(amsl): meters"""
        R = 6371000  # metres
        rlat1 = math.radians(lat)
        rlat2 = math.radians(self.global_position.latitude)

        rlat_d = math.radians(self.global_position.latitude - lat)
        rlon_d = math.radians(self.global_position.longitude - lon)

        a = (math.sin(rlat_d / 2) * math.sin(rlat_d / 2) + math.cos(rlat1) *
             math.cos(rlat2) * math.sin(rlon_d / 2) * math.sin(rlon_d / 2))
        c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))

        d = R * c
        alt_d = abs(alt - self.altitude.amsl)

        rospy.logdebug("d: {0}, alt_d: {1}".format(d, alt_d))
        return d, alt_d

    def reach_position(self, lat, lon, alt, timeout, index):
        """alt(amsl): meters, timeout(int): seconds"""
        rospy.loginfo(
            "trying to reach waypoint | lat: {0:.9f}, lon: {1:.9f}, alt: {2:.2f}, index: {3}".
            format(lat, lon, alt, index))
        best_pos_xy_d = None
        best_pos_z_d = None
        reached = False
        mission_length = len(self.mission_wp.waypoints)

        # does it reach the position in 'timeout' seconds?
        loop_freq = 2  # Hz
        rate = rospy.Rate(loop_freq)
        for i in xrange(timeout * loop_freq):
            pos_xy_d, pos_z_d = self.distance_to_wp(lat, lon, alt)

            # remember best distances
            if not best_pos_xy_d or best_pos_xy_d > pos_xy_d:
                best_pos_xy_d = pos_xy_d
            if not best_pos_z_d or best_pos_z_d > pos_z_d:
                best_pos_z_d = pos_z_d

            # FCU advanced to the next mission item, or finished mission
            reached = (
                # advanced to next wp
                (index < self.mission_wp.current_seq)
                # end of mission, reset to first wp
                or (mission_length > 1 and index == (mission_length - 1)
                    and self.mission_wp.current_seq == 0)
                # end of mission with only 1 wp
                or (mission_length == 1 and self.mission_item_reached == 0))

            if reached:
                rospy.loginfo(
                    "position reached | pos_xy_d: {0:.2f}, pos_z_d: {1:.2f}, index: {2} | seconds: {3} of {4}".
                    format(pos_xy_d, pos_z_d, index, i / loop_freq, timeout))
                break
            elif i == 0 or ((i / loop_freq) % 10) == 0:
                # log distance first iteration and every 10 sec
                rospy.loginfo(
                    "current distance to waypoint | pos_xy_d: {0:.2f}, pos_z_d: {1:.2f}, index: {2}".
                    format(pos_xy_d, pos_z_d, index))

            try:
                rate.sleep()
            except rospy.ROSException as e:
                self.fail(e)

        self.assertTrue(
            reached,
            "position not reached | lat: {0:.9f}, lon: {1:.9f}, alt: {2:.2f}, current pos_xy_d: {3:.2f}, current pos_z_d: {4:.2f}, best pos_xy_d: {5:.2f}, best pos_z_d: {6:.2f}, index: {7} | timeout(seconds): {8}".
            format(lat, lon, alt, pos_xy_d, pos_z_d, best_pos_xy_d,
                   best_pos_z_d, index, timeout))

    #
    # Test method
    #
    def test_mission(self):
        """Test mission"""
        if len(sys.argv) < 2:
            self.fail("usage: mission_test.py mission_file")
            return

        self.mission_name = sys.argv[1]
        mission_file = os.path.dirname(
            os.path.realpath(__file__)) + "/missions/" + sys.argv[1]

        rospy.loginfo("reading mission {0}".format(mission_file))
        try:
            wps = read_mission(mission_file)
        except IOError as e:
            self.fail(e)

        # make sure the simulation is ready to start the mission
        self.wait_for_topics(60)
        self.wait_for_landed_state(mavutil.mavlink.MAV_LANDED_STATE_ON_GROUND,
                                   10, -1)
        self.wait_for_mav_type(10)

        # push waypoints to FCU and start mission
        self.send_wps(wps, 30)
        self.log_topic_vars()
        self.set_mode("AUTO.MISSION", 5)
        self.set_arm(True, 5)

        rospy.loginfo("run mission {0}".format(self.mission_name))
        for index, waypoint in enumerate(wps):
            # only check position for waypoints where this makes sense
            if (waypoint.frame == Waypoint.FRAME_GLOBAL_REL_ALT
                    or waypoint.frame == Waypoint.FRAME_GLOBAL):
                alt = waypoint.z_alt
                if waypoint.frame == Waypoint.FRAME_GLOBAL_REL_ALT:
                    alt += self.altitude.amsl - self.altitude.relative

                self.reach_position(waypoint.x_lat, waypoint.y_long, alt, 60,
                                    index)

            # check if VTOL transition happens if applicable
            if (waypoint.command == mavutil.mavlink.MAV_CMD_NAV_VTOL_TAKEOFF or
                    waypoint.command == mavutil.mavlink.MAV_CMD_NAV_VTOL_LAND
                    or waypoint.command ==
                    mavutil.mavlink.MAV_CMD_DO_VTOL_TRANSITION):
                transition = waypoint.param1  # used by MAV_CMD_DO_VTOL_TRANSITION
                if waypoint.command == mavutil.mavlink.MAV_CMD_NAV_VTOL_TAKEOFF:  # VTOL takeoff implies transition to FW
                    transition = mavutil.mavlink.MAV_VTOL_STATE_FW
                if waypoint.command == mavutil.mavlink.MAV_CMD_NAV_VTOL_LAND:  # VTOL land implies transition to MC
                    transition = mavutil.mavlink.MAV_VTOL_STATE_MC

                self.wait_for_vtol_state(transition, 60, index)

            # after reaching position, wait for landing detection if applicable
            if (waypoint.command == mavutil.mavlink.MAV_CMD_NAV_VTOL_LAND
                    or waypoint.command == mavutil.mavlink.MAV_CMD_NAV_LAND):
                self.wait_for_landed_state(
                    mavutil.mavlink.MAV_LANDED_STATE_ON_GROUND, 60, index)

        self.set_arm(False, 5)
        self.clear_wps(5)

        rospy.loginfo("mission done, calculating performance metrics")
        last_log = get_last_log()
        rospy.loginfo("log file {0}".format(last_log))
        data = px4tools.read_ulog(last_log).concat(dt=0.1)
        data = px4tools.compute_data(data)
        res = px4tools.estimator_analysis(data, False)

        # enforce performance
        self.assertTrue(abs(res['roll_error_mean']) < 5.0, str(res))
        self.assertTrue(abs(res['pitch_error_mean']) < 5.0, str(res))
        self.assertTrue(abs(res['yaw_error_mean']) < 5.0, str(res))
        self.assertTrue(res['roll_error_std'] < 5.0, str(res))
        self.assertTrue(res['pitch_error_std'] < 5.0, str(res))
        self.assertTrue(res['yaw_error_std'] < 5.0, str(res))

if __name__ == '__main__':
    import rostest
    rospy.init_node('test_node', anonymous=True)

    name = "mavros_mission_test"
    if len(sys.argv) > 1:
        name += "-%s" % sys.argv[1]
    rostest.rosrun(PKG, name, MavrosMissionTest)
<commit_msg>CI: better detect end of mission<commit_after>#!/usr/bin/env python2
#***************************************************************************
#
#   Copyright (c) 2015-2016 PX4 Development Team. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in
#    the documentation and/or other materials provided with the
#    distribution.
# 3. Neither the name PX4 nor the names of its contributors may be
#    used to endorse or promote products derived from this software
#    without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
#***************************************************************************/

#
# @author Andreas Antener <andreas@uaventure.com>
#

# The shebang of this file is currently Python2 because some
# dependencies such as pymavlink don't play well with Python3 yet.
from __future__ import division

PKG = 'px4'

import rospy
import glob
import json
import math
import os
import px4tools
import sys
from mavros import mavlink
from mavros_msgs.msg import Mavlink, Waypoint, WaypointReached
from mavros_test_common import MavrosTestCommon
from pymavlink import mavutil
from threading import Thread

def get_last_log():
    try:
        log_path = os.environ['PX4_LOG_DIR']
    except KeyError:
        log_path = os.path.join(os.environ['HOME'],
                                '.ros/rootfs/fs/microsd/log')
    last_log_dir = sorted(glob.glob(os.path.join(log_path, '*')))[-1]
    last_log = sorted(glob.glob(os.path.join(last_log_dir, '*.ulg')))[-1]
    return last_log

def read_mission(mission_filename):
    wps = []
    with open(mission_filename, 'r') as f:
        for waypoint in read_plan_file(f):
            wps.append(waypoint)
            rospy.logdebug(waypoint)

    # set first item to current
    if wps:
        wps[0].is_current = True

    return wps

def read_plan_file(f):
    d = json.load(f)
    if 'mission' in d:
        d = d['mission']

    if 'items' in d:
        for wp in d['items']:
            yield Waypoint(
                is_current=False,
                frame=int(wp['frame']),
                command=int(wp['command']),
                param1=float('nan'
                             if wp['params'][0] is None else wp['params'][0]),
                param2=float('nan'
                             if wp['params'][1] is None else wp['params'][1]),
                param3=float('nan'
                             if wp['params'][2] is None else wp['params'][2]),
                param4=float('nan'
                             if wp['params'][3] is None else wp['params'][3]),
                x_lat=float(wp['params'][4]),
                y_long=float(wp['params'][5]),
                z_alt=float(wp['params'][6]),
                autocontinue=bool(wp['autoContinue']))
    else:
        raise IOError("no mission items")

class MavrosMissionTest(MavrosTestCommon):
    """
    Run a mission
    """

    def setUp(self):
        super(self.__class__, self).setUp()

        self.mission_item_reached = -1  # first mission item is 0
        self.mission_name = ""

        self.mavlink_pub = rospy.Publisher('mavlink/to', Mavlink, queue_size=1)
        self.mission_item_reached_sub = rospy.Subscriber(
            'mavros/mission/reached', WaypointReached,
            self.mission_item_reached_callback)

        # need to simulate heartbeat to prevent datalink loss detection
        self.hb_mav_msg = mavutil.mavlink.MAVLink_heartbeat_message(
            mavutil.mavlink.MAV_TYPE_GCS, 0, 0, 0, 0, 0)
        self.hb_mav_msg.pack(mavutil.mavlink.MAVLink('', 2, 1))
        self.hb_ros_msg = mavlink.convert_to_rosmsg(self.hb_mav_msg)
        self.hb_thread = Thread(target=self.send_heartbeat, args=())
        self.hb_thread.daemon = True
        self.hb_thread.start()

    def tearDown(self):
        super(MavrosMissionTest, self).tearDown()

    #
    # Helper methods
    #
    def send_heartbeat(self):
        rate = rospy.Rate(2)  # Hz
        while not rospy.is_shutdown():
            self.mavlink_pub.publish(self.hb_ros_msg)
            try:  # prevent garbage in console output when thread is killed
                rate.sleep()
            except rospy.ROSInterruptException:
                pass

    def mission_item_reached_callback(self, data):
        if self.mission_item_reached != data.wp_seq:
            rospy.loginfo("mission item reached: {0}".format(data.wp_seq))
            self.mission_item_reached = data.wp_seq

    def distance_to_wp(self, lat, lon, alt):
        """alt(amsl): meters"""
        R = 6371000  # metres
        rlat1 = math.radians(lat)
        rlat2 = math.radians(self.global_position.latitude)

        rlat_d = math.radians(self.global_position.latitude - lat)
        rlon_d = math.radians(self.global_position.longitude - lon)

        a = (math.sin(rlat_d / 2) * math.sin(rlat_d / 2) + math.cos(rlat1) *
             math.cos(rlat2) * math.sin(rlon_d / 2) * math.sin(rlon_d / 2))
        c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))

        d = R * c
        alt_d = abs(alt - self.altitude.amsl)

        rospy.logdebug("d: {0}, alt_d: {1}".format(d, alt_d))
        return d, alt_d

    def reach_position(self, lat, lon, alt, timeout, index):
        """alt(amsl): meters, timeout(int): seconds"""
        rospy.loginfo(
            "trying to reach waypoint | lat: {0:.9f}, lon: {1:.9f}, alt: {2:.2f}, index: {3}".
            format(lat, lon, alt, index))
        best_pos_xy_d = None
        best_pos_z_d = None
        reached = False
        mission_length = len(self.mission_wp.waypoints)

        # does it reach the position in 'timeout' seconds?
        loop_freq = 2  # Hz
        rate = rospy.Rate(loop_freq)
        for i in xrange(timeout * loop_freq):
            pos_xy_d, pos_z_d = self.distance_to_wp(lat, lon, alt)

            # remember best distances
            if not best_pos_xy_d or best_pos_xy_d > pos_xy_d:
                best_pos_xy_d = pos_xy_d
            if not best_pos_z_d or best_pos_z_d > pos_z_d:
                best_pos_z_d = pos_z_d

            # FCU advanced to the next mission item, or finished mission
            reached = (
                # advanced to next wp
                (index < self.mission_wp.current_seq)
                # end of mission
                or (index == (mission_length - 1)
                    and self.mission_item_reached == index))

            if reached:
                rospy.loginfo(
                    "position reached | pos_xy_d: {0:.2f}, pos_z_d: {1:.2f}, index: {2} | seconds: {3} of {4}".
                    format(pos_xy_d, pos_z_d, index, i / loop_freq, timeout))
                break
            elif i == 0 or ((i / loop_freq) % 10) == 0:
                # log distance first iteration and every 10 sec
                rospy.loginfo(
                    "current distance to waypoint | pos_xy_d: {0:.2f}, pos_z_d: {1:.2f}, index: {2}".
                    format(pos_xy_d, pos_z_d, index))

            try:
                rate.sleep()
            except rospy.ROSException as e:
                self.fail(e)

        self.assertTrue(
            reached,
            "position not reached | lat: {0:.9f}, lon: {1:.9f}, alt: {2:.2f}, current pos_xy_d: {3:.2f}, current pos_z_d: {4:.2f}, best pos_xy_d: {5:.2f}, best pos_z_d: {6:.2f}, index: {7} | timeout(seconds): {8}".
            format(lat, lon, alt, pos_xy_d, pos_z_d, best_pos_xy_d,
                   best_pos_z_d, index, timeout))

    #
    # Test method
    #
    def test_mission(self):
        """Test mission"""
        if len(sys.argv) < 2:
            self.fail("usage: mission_test.py mission_file")
            return

        self.mission_name = sys.argv[1]
        mission_file = os.path.dirname(
            os.path.realpath(__file__)) + "/missions/" + sys.argv[1]

        rospy.loginfo("reading mission {0}".format(mission_file))
        try:
            wps = read_mission(mission_file)
        except IOError as e:
            self.fail(e)

        # make sure the simulation is ready to start the mission
        self.wait_for_topics(60)
        self.wait_for_landed_state(mavutil.mavlink.MAV_LANDED_STATE_ON_GROUND,
                                   10, -1)
        self.wait_for_mav_type(10)

        # push waypoints to FCU and start mission
        self.send_wps(wps, 30)
        self.log_topic_vars()
        self.set_mode("AUTO.MISSION", 5)
        self.set_arm(True, 5)

        rospy.loginfo("run mission {0}".format(self.mission_name))
        for index, waypoint in enumerate(wps):
            # only check position for waypoints where this makes sense
            if (waypoint.frame == Waypoint.FRAME_GLOBAL_REL_ALT
                    or waypoint.frame == Waypoint.FRAME_GLOBAL):
                alt = waypoint.z_alt
                if waypoint.frame == Waypoint.FRAME_GLOBAL_REL_ALT:
                    alt += self.altitude.amsl - self.altitude.relative

                self.reach_position(waypoint.x_lat, waypoint.y_long, alt, 60,
                                    index)

            # check if VTOL transition happens if applicable
            if (waypoint.command == mavutil.mavlink.MAV_CMD_NAV_VTOL_TAKEOFF or
                    waypoint.command == mavutil.mavlink.MAV_CMD_NAV_VTOL_LAND
                    or waypoint.command ==
                    mavutil.mavlink.MAV_CMD_DO_VTOL_TRANSITION):
                transition = waypoint.param1  # used by MAV_CMD_DO_VTOL_TRANSITION
                if waypoint.command == mavutil.mavlink.MAV_CMD_NAV_VTOL_TAKEOFF:  # VTOL takeoff implies transition to FW
                    transition = mavutil.mavlink.MAV_VTOL_STATE_FW
                if waypoint.command == mavutil.mavlink.MAV_CMD_NAV_VTOL_LAND:  # VTOL land implies transition to MC
                    transition = mavutil.mavlink.MAV_VTOL_STATE_MC

                self.wait_for_vtol_state(transition, 60, index)

            # after reaching position, wait for landing detection if applicable
            if (waypoint.command == mavutil.mavlink.MAV_CMD_NAV_VTOL_LAND
                    or waypoint.command == mavutil.mavlink.MAV_CMD_NAV_LAND):
                self.wait_for_landed_state(
                    mavutil.mavlink.MAV_LANDED_STATE_ON_GROUND, 60, index)

        self.set_arm(False, 5)
        self.clear_wps(5)

        rospy.loginfo("mission done, calculating performance metrics")
        last_log = get_last_log()
        rospy.loginfo("log file {0}".format(last_log))
        data = px4tools.read_ulog(last_log).concat(dt=0.1)
        data = px4tools.compute_data(data)
        res = px4tools.estimator_analysis(data, False)

        # enforce performance
        self.assertTrue(abs(res['roll_error_mean']) < 5.0, str(res))
        self.assertTrue(abs(res['pitch_error_mean']) < 5.0, str(res))
        self.assertTrue(abs(res['yaw_error_mean']) < 5.0, str(res))
        self.assertTrue(res['roll_error_std'] < 5.0, str(res))
        self.assertTrue(res['pitch_error_std'] < 5.0, str(res))
        self.assertTrue(res['yaw_error_std'] < 5.0, str(res))

if __name__ == '__main__':
    import rostest
    rospy.init_node('test_node', anonymous=True)

    name = "mavros_mission_test"
    if len(sys.argv) > 1:
        name += "-%s" % sys.argv[1]
    rostest.rosrun(PKG, name, MavrosMissionTest)
<|endoftext|>


<commit_before>from intelmq.lib.bot import Bot, sys
from intelmq.lib.utils import decode
from intelmq.lib.event import Event

class DansParserBot(Bot):

    def process(self):
        report = self.receive_message()

        if report:
            for row in report.split('\n'):
                row = row.strip()

                if len(row) == 0 or row.startswith('#'):
                    continue

                row = row.split()
                event = Event()

                columns = ["url"]
                for key, value in zip(columns, row):
                    event.add(key, value)

                event.add('feed', 'DansGuardian')
                event.add('feed_url', "https://lists.malwarepatrol.net/cgi/getfile?product=8&list=dansguardian")
                event.add('type', 'malware')

                self.send_message(event)

        self.acknowledge_message()

if __name__ == "__main__":
    bot = DansParserBot(sys.argv[1])
    bot.start()<commit_msg>improve malwarepatrol - dansguardian<commit_after>from intelmq.lib.bot import Bot, sys
from intelmq.lib.event import Event
from intelmq.bots import utils

class DansParserBot(Bot):

    def process(self):
        report = self.receive_message()

        if report:
            for row in report.split('\n'):
                row = row.strip()

                if len(row) == 0 or row.startswith('#'):
                    continue

                row = row.split()
                event = Event()

                columns = ["source_url"]
                for key, value in zip(columns, row):
                    event.add(key, value)

                event.add('feed', 'malwarepatrol-dansguardian')
                event.add('type', 'malware')
                event = utils.parse_source_time(event, "source_time")
                event = utils.generate_observation_time(event, "observation_time")
                event = utils.generate_reported_fields(event)

                self.send_message(event)
        self.acknowledge_message()

if __name__ == "__main__":
    bot = DansParserBot(sys.argv[1])
    bot.start()
<|endoftext|>


<commit_before># Copyright (C) 2010 Holger Berndt <hb@gnome.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public
# License along with this program; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

"""Various checks"""

import pygtk
pygtk.require('2.0')
import gtk
import gio

import constants

class Checker(object):
    """Perform various checks on a model"""

    def __init__(self, model):
        """Perform checks already in constructor"""
        # results that can be queried
        self.all_names_stay_the_same = True
        self.highest_problem_level = 0
        self.circular_uris = set()

        # common data
        self._model = model

        # run tests
        # these modify the model and/or set the results
        self._clear_all_warnings_and_errors()
        self._check_if_all_names_stay_the_same()
        # there can't be any problems in this case
        if not self.all_names_stay_the_same:

            # more common data
            self._list_of_source_uris = [(row[constants.FILES_MODEL_COLUMN_URI_DIRNAME] + row[0]) for row in model]
            self._list_of_target_uris = [(row[constants.FILES_MODEL_COLUMN_URI_DIRNAME] + row[1]) for row in model]

            self._dict_source_uri_to_index = {}
            for ii, uri in enumerate(self._list_of_source_uris):
                self._dict_source_uri_to_index[uri] = ii

            self._dict_target_uri_to_indices = {}
            for ii, uri in enumerate(self._list_of_target_uris):
                self._dict_target_uri_to_indices.setdefault(uri, []).append(ii)

            # checks
            self._check_for_double_targets()
            self._check_for_circular_renaming()
            self._check_for_already_existing_names()

    def _clear_all_warnings_and_errors(self):
        for row in self._model:
            row[constants.FILES_MODEL_COLUMN_ICON_STOCK] = None
            row[constants.FILES_MODEL_COLUMN_TOOLTIP] = None

    def _check_if_all_names_stay_the_same(self):
        for row in self._model:
            if row[0] != row[1]:
                self.all_names_stay_the_same = False
        self._all_names_stay_the_same = True

    def _check_for_double_targets(self):
        """Sets pixbuf and tooltip text. Returns True if double targets exist"""
        double_uris = []
        for key,value in self._dict_target_uri_to_indices.iteritems():
            if len(value) > 1:
                double_uris.append(key)

        msg = "ERROR: Double output filepath"
        registered = set()
        found_problem = False
        for uri in double_uris:
            for ii in self._dict_target_uri_to_indices[uri]:
                if ii not in registered:
                    self._model[ii][constants.FILES_MODEL_COLUMN_ICON_STOCK] = gtk.STOCK_DIALOG_ERROR
                    self.highest_problem_level = max(self.highest_problem_level, 2)
                    found_problem = True
                    if self._model[ii][constants.FILES_MODEL_COLUMN_TOOLTIP] == None:
                        self._model[ii][constants.FILES_MODEL_COLUMN_TOOLTIP] = msg
                    else:
                        self._model[ii][constants.FILES_MODEL_COLUMN_TOOLTIP] = self._model[ii][constants.FILES_MODEL_COLUMN_TOOLTIP] + "\n" + msg
                    registered.add(ii)

        return found_problem

    def _check_for_circular_renaming(self):
        self.circular_uris = set()
        # we have a circular renaming if one row's source refers to the same uri as another row's target
        set_of_source_uris = set(self._list_of_source_uris)
        set_of_target_uris = set(self._list_of_target_uris)
        possible_circular_uris = set_of_source_uris.intersection(set_of_target_uris)
        for pu in possible_circular_uris:
            if len(self._dict_target_uri_to_indices[pu]) > 1 or self._dict_source_uri_to_index[pu] != self._dict_target_uri_to_indices[pu][0]:
                self.circular_uris.add(pu)

    def _check_for_already_existing_names(self):
        """Check if a target name already exists on the file system, but is not a circular rename"""
        existing_files = set()
        for row in self._model:
            if row[0] == row[1]:
                continue
            new_name = row[constants.FILES_MODEL_COLUMN_URI_DIRNAME] + row[1]
            if new_name not in self.circular_uris:
                parent = gio.File(uri=row[constants.FILES_MODEL_COLUMN_URI_DIRNAME])
                file = parent.get_child_for_display_name(row[1])
                if file.query_exists():
                    existing_files.add(new_name)
<commit_msg>Add warnings for existing filenames<commit_after># Copyright (C) 2010 Holger Berndt <hb@gnome.org>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
# published by the Free Software Foundation; either version 2 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public
# License along with this program; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.

"""Various checks"""

import pygtk
pygtk.require('2.0')
import gtk
import gio

import constants

class Checker(object):
    """Perform various checks on a model"""

    def __init__(self, model):
        """Perform checks already in constructor"""
        # results that can be queried
        self.all_names_stay_the_same = True
        self.highest_problem_level = 0
        self.circular_uris = set()

        # common data
        self._model = model

        # run tests
        # these modify the model and/or set the results
        self._clear_all_warnings_and_errors()
        self._check_if_all_names_stay_the_same()
        # there can't be any problems in this case
        if not self.all_names_stay_the_same:

            # more common data
            self._list_of_source_uris = [(row[constants.FILES_MODEL_COLUMN_URI_DIRNAME] + row[0]) for row in model]
            self._list_of_target_uris = [(row[constants.FILES_MODEL_COLUMN_URI_DIRNAME] + row[1]) for row in model]

            self._dict_source_uri_to_index = {}
            for ii, uri in enumerate(self._list_of_source_uris):
                self._dict_source_uri_to_index[uri] = ii

            self._dict_target_uri_to_indices = {}
            for ii, uri in enumerate(self._list_of_target_uris):
                self._dict_target_uri_to_indices.setdefault(uri, []).append(ii)

            # checks
            self._check_for_double_targets()
            self._check_for_circular_renaming()
            self._check_for_already_existing_names()

    def _clear_all_warnings_and_errors(self):
        for row in self._model:
            row[constants.FILES_MODEL_COLUMN_ICON_STOCK] = None
            row[constants.FILES_MODEL_COLUMN_TOOLTIP] = None

    def _check_if_all_names_stay_the_same(self):
        for row in self._model:
            if row[0] != row[1]:
                self.all_names_stay_the_same = False
        self._all_names_stay_the_same = True

    def _check_for_double_targets(self):
        """Sets pixbuf and tooltip text. Returns True if double targets exist"""
        double_uris = []
        for key,value in self._dict_target_uri_to_indices.iteritems():
            if len(value) > 1:
                double_uris.append(key)

        msg = "ERROR: Double output filepath"
        registered = set()
        found_problem = False
        for uri in double_uris:
            for ii in self._dict_target_uri_to_indices[uri]:
                if ii not in registered:
                    self._model[ii][constants.FILES_MODEL_COLUMN_ICON_STOCK] = gtk.STOCK_DIALOG_ERROR
                    self.highest_problem_level = max(self.highest_problem_level, 2)
                    found_problem = True
                    self._add_tooltip_msg(ii, msg)
                    registered.add(ii)

        return found_problem

    def _check_for_circular_renaming(self):
        self.circular_uris = set()
        # we have a circular renaming if one row's source refers to the same uri as another row's target
        set_of_source_uris = set(self._list_of_source_uris)
        set_of_target_uris = set(self._list_of_target_uris)
        possible_circular_uris = set_of_source_uris.intersection(set_of_target_uris)
        for pu in possible_circular_uris:
            if len(self._dict_target_uri_to_indices[pu]) > 1 or self._dict_source_uri_to_index[pu] != self._dict_target_uri_to_indices[pu][0]:
                self.circular_uris.add(pu)

    def _check_for_already_existing_names(self):
        """Check if a target name already exists on the file system, but is not a circular rename"""
        existing_files = set()
        for row in self._model:
            if row[0] == row[1]:
                continue
            new_name = row[constants.FILES_MODEL_COLUMN_URI_DIRNAME] + row[1]
            if new_name not in self.circular_uris:
                parent = gio.File(uri=row[constants.FILES_MODEL_COLUMN_URI_DIRNAME])
                file = parent.get_child_for_display_name(row[1])
                if file.query_exists():
                    existing_files.add(new_name)

        # mark files
        msg = "WARNING: Target filename already exists on the filesystem"
        for uri in existing_files:
            for idx in self._dict_target_uri_to_indices[uri]:
                if self._model[idx][constants.FILES_MODEL_COLUMN_ICON_STOCK] != gtk.STOCK_DIALOG_ERROR:
                    self._model[idx][constants.FILES_MODEL_COLUMN_ICON_STOCK] = gtk.STOCK_DIALOG_WARNING
                self._add_tooltip_msg(idx, msg)
                self.highest_problem_level = max(self.highest_problem_level, 1)

    def _add_tooltip_msg(self, row_num, msg):
        if self._model[row_num][constants.FILES_MODEL_COLUMN_TOOLTIP] == None:
            self._model[row_num][constants.FILES_MODEL_COLUMN_TOOLTIP] = msg
        else:
            self._model[row_num][constants.FILES_MODEL_COLUMN_TOOLTIP] = self._model[row_num][constants.FILES_MODEL_COLUMN_TOOLTIP] + "\n" + msg
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys

class BoulderIOFormatter:

    def __init__(self, primer_options=None):
        self.segment_lengths = {}
        if not primer_options:
            primer_options = {}
        self.primer3_core_options = primer_options

    def format_seq(self, seq, excluded_region_entries=None):
        """Writes Sequence object to boulder-io format.

        If optional excluded_region_entry argument is passed,
        appends these entries to the output.
        """
        if not self.segment_lengths:
            sys.stderr.write("BoulderIOFormatter: attempt to format seq without segment lengths dictionary.\n")
            return ""
        if seq.header not in self.segment_lengths.keys():
            sys.stderr.write("BoulderIOFormatter: seq header not found in segment lengths dictionary -- " + seq.header + "\n")
            return ""
        result = "SEQUENCE_ID=" + seq.header + "\n"
        result += "SEQUENCE_TEMPLATE=" + seq.bases + "\n"
        lengths = self.segment_lengths[seq.header]
        total_length = 0
        for length in lengths[:-1]:
            total_length += length
            result += "SEQUENCE_TARGET=" + str(total_length) + ",1\n"
        if excluded_region_entries:
            for entry in excluded_region_entries:
                result += entry
        result += "=\n"
        return result

<commit_msg>segment_lengths optional in BoulderIOFormatter<commit_after>#!/usr/bin/env python

import sys

class BoulderIOFormatter:

    def __init__(self, primer_options=None):
        self.segment_lengths = {}
        if not primer_options:
            primer_options = {}
        self.primer3_core_options = primer_options

    def format_seq(self, seq, excluded_region_entries=None):
        """Writes Sequence object to boulder-io format.

        If optional excluded_region_entry argument is passed,
        appends these entries to the output.
        """
        result = "SEQUENCE_ID=" + seq.header + "\n"
        result += "SEQUENCE_TEMPLATE=" + seq.bases + "\n"
        if self.segment_lengths:
            lengths = self.segment_lengths[seq.header]
            total_length = 0
            for length in lengths[:-1]:
                total_length += length
                result += "SEQUENCE_TARGET=" + str(total_length) + ",1\n"
        if excluded_region_entries:
            for entry in excluded_region_entries:
                result += entry
        result += "=\n"
        return result

<|endoftext|>


<commit_before># -*- test-case-name: go.vumitools.tests.test_api_worker -*-
# -*- coding: utf-8 -*-

"""Vumi application worker for the vumitools API."""

from twisted.internet.defer import inlineCallbacks, returnValue

from vumi.application import ApplicationWorker
from vumi.persist.message_store import MessageStore
from vumi.persist.txriak_manager import TxRiakManager
from vumi.dispatchers.base import BaseDispatchRouter
from vumi.middleware.tagger import TaggingMiddleware
from vumi import log

from go.vumitools.api import VumiApiCommand, get_redis
from go.vumitools.account import AccountStore
from go.vumitools.conversation import ConversationStore
from go.vumitools.middleware import DebitAccountMiddleware

class CommandDispatcher(ApplicationWorker):
    """
    An application worker that forwards commands arriving on the Vumi Api
    queue to the relevant applications. It does this by using the commands
    worker_name parameter to construct the routing key.

    Configuration parameters:

    :type api_routing: dict
    :param api_routing:
        Dictionary describing where to consume API commands.
    :type worker_names: list
    :param worker_names:
        A list of known worker names that we can forward
        VumiApiCommands to.
    """

    def validate_config(self):
        self.api_routing_config = VumiApiCommand.default_routing_config()
        self.api_routing_config.update(self.config.get('api_routing', {}))
        self.api_consumer = None
        self.worker_names = self.config.get('worker_names', [])

    @inlineCallbacks
    def setup_application(self):
        self.worker_publishers = {}
        for worker_name in self.worker_names:
            worker_publisher = yield self.publish_to('%s.control' % (
                worker_name,))
            self.worker_publishers[worker_name] = worker_publisher

        self.api_consumer = yield self.consume(
            self.api_routing_config['routing_key'],
            self.consume_control_command,
            exchange_name=self.api_routing_config['exchange'],
            exchange_type=self.api_routing_config['exchange_type'],
            message_class=VumiApiCommand)

    @inlineCallbacks
    def teardown_application(self):
        if self.api_consumer:
            yield self.api_consumer.stop()
            self.api_consumer = None

    @inlineCallbacks
    def consume_control_command(self, cmd):
        worker_name = cmd.get('worker_name')
        publisher = self.worker_publishers.get(worker_name)
        if publisher:
            yield publisher.publish_message(cmd)
            log.info('Sent %s to %s' % (cmd, worker_name))
        else:
            log.error('No worker publisher available for %s' % (cmd,))

class GoApplicationRouter(BaseDispatchRouter):
    """
    Router for a dispatcher that routes messages
    based on their tags.

    """
    def setup_routing(self):
        # map conversation types to applications that deal with them
        self.conversation_mappings = self.config['conversation_mappings']
        # setup the message_store
        mdb_config = self.config.get('message_store', {})
        self.mdb_prefix = mdb_config.get('store_prefix', 'message_store')
        r_server = get_redis(self.config)
        self.manager = TxRiakManager.from_config({
                'bucket_prefix': self.mdb_prefix})
        self.account_store = AccountStore(self.manager)
        self.message_store = MessageStore(self.manager, r_server,
                                            self.mdb_prefix)

    @inlineCallbacks
    def get_conversation_for_tag(self, tag):
        current_tag = yield self.message_store.get_tag_info(tag)
        if current_tag:
            batch = yield current_tag.current_batch.get()
            if batch:
                account_key = batch.metadata.user_account
                if account_key:
                    conversation_store = ConversationStore(self.manager,
                        account_key)
                    account_submanager = conversation_store.manager
                    all_conversations = yield batch.backlinks.conversations(
                                                            account_submanager)
                    conversations = [c for c in all_conversations if not
                                        c.ended()]
                    if conversations:
                        if len(conversations) > 1:
                            conv_keys = [c.key for c in conversations]
                            log.warning('Multiple conversations found '
                                'going with most recent: %r' % (conv_keys,))
                        conversation = sorted(conversations, reverse=True,
                            key=lambda c: c.start_timestamp)[0]
                        returnValue(conversation)
                    log.error('No conversations found for %r' % (batch,))
                else:
                    log.error('No account_key found for tag: %r, batch: %r' % (
                        current_tag, batch))
            else:
                log.error("No batch found for tag: %r" % (current_tag,))
        else:
            log.error('Cannot find current tag for %s' % (tag,))

    @inlineCallbacks
    def find_application_for_msg(self, msg):
        tag = TaggingMiddleware.map_msg_to_tag(msg)
        if tag:
            conversation = yield self.get_conversation_for_tag(tag)
            if conversation:
                conv_type = conversation.conversation_type
                DebitAccountMiddleware.add_user_to_message(msg,
                        conversation.user_account.key)
                metadata = msg['helper_metadata']
                conv_metadata = metadata.setdefault('conversations', {})
                conv_metadata.update({
                    'conversation_key': conversation.key,
                    'conversation_type': conv_type,
                })
                returnValue(self.conversation_mappings.get(conv_type))

    @inlineCallbacks
    def dispatch_inbound_message(self, msg):
        tag = TaggingMiddleware.map_msg_to_tag(msg)
        self.message_store.add_inbound_message(msg, tag=tag)
        application = yield self.find_application_for_msg(msg)
        if application:
            publisher = self.dispatcher.exposed_publisher[application]
            yield publisher.publish_message(msg)
        else:
            log.error('No application setup for type: %s' % (
                        msg,))

    def dispatch_inbound_event(self, msg):
        tag = TaggingMiddleware.map_msg_to_tag(msg)
        self.message_store.add_event(msg, tag=tag)
        application = self.find_application_for_msg(msg)
        if application:
            publisher = self.dispatcher.exposed_event_publisher[application]
            yield publisher.publish_message(msg)
        else:
            log.error('No application setup for type: %s' % (
                        msg,))

    def dispatch_outbound_message(self, msg):
        tag = TaggingMiddleware.map_msg_to_tag(msg)
        self.message_store.add_outbound_message(msg, tag=tag)
        upstream = self.dispatcher.transport_publisher.keys()[0]
        self.dispatcher.transport_publisher[upstream].publish_message(msg)
<commit_msg>Access user_account via __getitem__ to match new dynamic fields from vumi.persist.<commit_after># -*- test-case-name: go.vumitools.tests.test_api_worker -*-
# -*- coding: utf-8 -*-

"""Vumi application worker for the vumitools API."""

from twisted.internet.defer import inlineCallbacks, returnValue

from vumi.application import ApplicationWorker
from vumi.persist.message_store import MessageStore
from vumi.persist.txriak_manager import TxRiakManager
from vumi.dispatchers.base import BaseDispatchRouter
from vumi.middleware.tagger import TaggingMiddleware
from vumi import log

from go.vumitools.api import VumiApiCommand, get_redis
from go.vumitools.account import AccountStore
from go.vumitools.conversation import ConversationStore
from go.vumitools.middleware import DebitAccountMiddleware

class CommandDispatcher(ApplicationWorker):
    """
    An application worker that forwards commands arriving on the Vumi Api
    queue to the relevant applications. It does this by using the commands
    worker_name parameter to construct the routing key.

    Configuration parameters:

    :type api_routing: dict
    :param api_routing:
        Dictionary describing where to consume API commands.
    :type worker_names: list
    :param worker_names:
        A list of known worker names that we can forward
        VumiApiCommands to.
    """

    def validate_config(self):
        self.api_routing_config = VumiApiCommand.default_routing_config()
        self.api_routing_config.update(self.config.get('api_routing', {}))
        self.api_consumer = None
        self.worker_names = self.config.get('worker_names', [])

    @inlineCallbacks
    def setup_application(self):
        self.worker_publishers = {}
        for worker_name in self.worker_names:
            worker_publisher = yield self.publish_to('%s.control' % (
                worker_name,))
            self.worker_publishers[worker_name] = worker_publisher

        self.api_consumer = yield self.consume(
            self.api_routing_config['routing_key'],
            self.consume_control_command,
            exchange_name=self.api_routing_config['exchange'],
            exchange_type=self.api_routing_config['exchange_type'],
            message_class=VumiApiCommand)

    @inlineCallbacks
    def teardown_application(self):
        if self.api_consumer:
            yield self.api_consumer.stop()
            self.api_consumer = None

    @inlineCallbacks
    def consume_control_command(self, cmd):
        worker_name = cmd.get('worker_name')
        publisher = self.worker_publishers.get(worker_name)
        if publisher:
            yield publisher.publish_message(cmd)
            log.info('Sent %s to %s' % (cmd, worker_name))
        else:
            log.error('No worker publisher available for %s' % (cmd,))

class GoApplicationRouter(BaseDispatchRouter):
    """
    Router for a dispatcher that routes messages
    based on their tags.

    """
    def setup_routing(self):
        # map conversation types to applications that deal with them
        self.conversation_mappings = self.config['conversation_mappings']
        # setup the message_store
        mdb_config = self.config.get('message_store', {})
        self.mdb_prefix = mdb_config.get('store_prefix', 'message_store')
        r_server = get_redis(self.config)
        self.manager = TxRiakManager.from_config({
                'bucket_prefix': self.mdb_prefix})
        self.account_store = AccountStore(self.manager)
        self.message_store = MessageStore(self.manager, r_server,
                                            self.mdb_prefix)

    @inlineCallbacks
    def get_conversation_for_tag(self, tag):
        current_tag = yield self.message_store.get_tag_info(tag)
        if current_tag:
            batch = yield current_tag.current_batch.get()
            if batch:
                account_key = batch.metadata['user_account']
                if account_key:
                    conversation_store = ConversationStore(self.manager,
                        account_key)
                    account_submanager = conversation_store.manager
                    all_conversations = yield batch.backlinks.conversations(
                                                            account_submanager)
                    conversations = [c for c in all_conversations if not
                                        c.ended()]
                    if conversations:
                        if len(conversations) > 1:
                            conv_keys = [c.key for c in conversations]
                            log.warning('Multiple conversations found '
                                'going with most recent: %r' % (conv_keys,))
                        conversation = sorted(conversations, reverse=True,
                            key=lambda c: c.start_timestamp)[0]
                        returnValue(conversation)
                    log.error('No conversations found for %r' % (batch,))
                else:
                    log.error('No account_key found for tag: %r, batch: %r' % (
                        current_tag, batch))
            else:
                log.error("No batch found for tag: %r" % (current_tag,))
        else:
            log.error('Cannot find current tag for %s' % (tag,))

    @inlineCallbacks
    def find_application_for_msg(self, msg):
        tag = TaggingMiddleware.map_msg_to_tag(msg)
        if tag:
            conversation = yield self.get_conversation_for_tag(tag)
            if conversation:
                conv_type = conversation.conversation_type
                DebitAccountMiddleware.add_user_to_message(msg,
                        conversation.user_account.key)
                metadata = msg['helper_metadata']
                conv_metadata = metadata.setdefault('conversations', {})
                conv_metadata.update({
                    'conversation_key': conversation.key,
                    'conversation_type': conv_type,
                })
                returnValue(self.conversation_mappings.get(conv_type))

    @inlineCallbacks
    def dispatch_inbound_message(self, msg):
        tag = TaggingMiddleware.map_msg_to_tag(msg)
        self.message_store.add_inbound_message(msg, tag=tag)
        application = yield self.find_application_for_msg(msg)
        if application:
            publisher = self.dispatcher.exposed_publisher[application]
            yield publisher.publish_message(msg)
        else:
            log.error('No application setup for type: %s' % (
                        msg,))

    def dispatch_inbound_event(self, msg):
        tag = TaggingMiddleware.map_msg_to_tag(msg)
        self.message_store.add_event(msg, tag=tag)
        application = self.find_application_for_msg(msg)
        if application:
            publisher = self.dispatcher.exposed_event_publisher[application]
            yield publisher.publish_message(msg)
        else:
            log.error('No application setup for type: %s' % (
                        msg,))

    def dispatch_outbound_message(self, msg):
        tag = TaggingMiddleware.map_msg_to_tag(msg)
        self.message_store.add_outbound_message(msg, tag=tag)
        upstream = self.dispatcher.transport_publisher.keys()[0]
        self.dispatcher.transport_publisher[upstream].publish_message(msg)
<|endoftext|>


<commit_before># HTTP GET to interact with Random.org

import urllib

# New Code
import urllib2
import os.path

RANDOM_URL = 'http://www.random.org'

def integers(num, minimum, maximum, base=10):
    """Docstring."""
    FUNC_TYPE = 'integers'

    num, minimum, maximum = map(int, [num, minimum, maximum])
    # Attribute error ?

    base = int(base)

    if base not in [2,8,10,16]:
        raise

    opts = {'num':num,
            'min':minimum,
            'max':maximum,
            'col':1,
            'base':base,
            'format':'plain',
            'rnd':'new'}

    url_opts = urllib.urlencode(opts)
    full_url = os.path.join(RANDOM_URL, FUNC_TYPE) + '/?' + url_opts
    data = urllib2.urlopen(full_url)
    if data.code == 200:
        # Status is Good
        arr = np.fromstring(data.read(), sep='\n', dtype='int')
    else:
        print "random.org returned server code: %s" % data.code
    return arr

#################################
####### START OF PROGRAM ########
#################################

quotaCheck = urllib.urlopen('http://www.random.org/quota/?format=plain').read()  # Check to see if you have bits remaining on the server

if quotaCheck > 0:
	getString='http://www.random.org/integers/?num=25&min=0&max=16&col=1&base=16&format=plain&rnd=new'
	randomData = urllib.urlopen(getString).read()
	print(randomData)
	print
	print 'You have', quotaCheck.strip(), 'bits remaining on the server.'

else:
	print 'You have no remaing requests on the Random.org server; wait at least 10 minutes before making another request. It may take 24 hours for you to gain access the server again.'

sitePath = 'http://www.random.org/'

# Integer sequence from random.org
def randInt(num, min, max, base=10, format='plain', columns=1, dataType='new'):
	appPath = 'integers/?'
	params = ['num', str(num),
			  'min', str(min),
			  'max', str(max),
			  'base', str(base),
			  'format', format,
			  'col', str(columns),
			  'rnd', dataType]

	for i in range(1, len(params) + len(params)/2, 3):
		params.insert(i, '=')

	for i in range(3, len(params) + (len(params)/3-1), 4):
		params.insert(i, '&')

	params = "".join(params)
	getString = sitePath + appPath + params
	randomData = urllib.urlopen(getString).read()
	print(getString)
	print(randomData)
	return randomData
	# TODO: learn more about "return" in a function

# TODO: create a function that simulates tossing a coin by calling on randInt(tosses, min=0, max=1), where HEADS is sum/tosses >= 0.5.

# def coinToss(tosses=100):
	# randInt(tosses, 0, 1)
<commit_msg>Added `get_http` and `str_to_arr` functions<commit_after># HTTP GET to interact with Random.org

import urllib
import urllib2
import os.path

# CONFIG
RANDOM_URL = 'http://www.random.org'

def integers(num, minimum, maximum, base=10):
    """Docstring."""
    FUNC_TYPE = 'integers'

    num, minimum, maximum = map(int, [num, minimum, maximum])
    # Attribute error ?

    base = int(base)

    if base not in [2,8,10,16]:
        raise

    opts = {'num':num,
            'min':minimum,
            'max':maximum,
            'col':1,
            'base':base,
            'format':'plain',
            'rnd':'new'}

    url_opts = urllib.urlencode(opts)
    full_url = os.path.join(RANDOM_URL, FUNC_TYPE) + '/?' + url_opts
    data = urllib2.urlopen(full_url)
    if data.code == 200:
        # Status is Good
        arr = np.fromstring(data.read(), sep='\n', dtype='int')
    else:
        print "random.org returned server code: %s" % data.code
    return arr

base_url = 'http://www.random.org'
function = 'integers'
opts = {'num':num,
            'min':minimum,
            'max':maximum,
            'col':1,
            'base':base,
            'format':'plain',
            'rnd':'new'}

def get_http(base_url, function, opts):
    url = (os.path.join(base_url, function) + '/?' +
                 urllib.urlencode(opts))
    data = urllib2.urlopen(url)

    if data.code != 200:
        raise ValueError("Random.rg returned server code: " + str(data.code))

    return data.read()

def str_to_arr(string):
    arr = np.fromstring(string, sep='\n', dtype='int')

    return arr

#################################
####### START OF PROGRAM ########
#################################

quotaCheck = urllib.urlopen('http://www.random.org/quota/?format=plain').read()  # Check to see if you have bits remaining on the server

if quotaCheck > 0:
	getString='http://www.random.org/integers/?num=25&min=0&max=16&col=1&base=16&format=plain&rnd=new'
	randomData = urllib.urlopen(getString).read()
	print(randomData)
	print
	print 'You have', quotaCheck.strip(), 'bits remaining on the server.'

else:
	print 'You have no remaing requests on the Random.org server; wait at least 10 minutes before making another request. It may take 24 hours for you to gain access the server again.'

sitePath = 'http://www.random.org/'

# Integer sequence from random.org
def randInt(num, min, max, base=10, format='plain', columns=1, dataType='new'):
	appPath = 'integers/?'
	params = ['num', str(num),
			  'min', str(min),
			  'max', str(max),
			  'base', str(base),
			  'format', format,
			  'col', str(columns),
			  'rnd', dataType]

	for i in range(1, len(params) + len(params)/2, 3):
		params.insert(i, '=')

	for i in range(3, len(params) + (len(params)/3-1), 4):
		params.insert(i, '&')

	params = "".join(params)
	getString = sitePath + appPath + params
	randomData = urllib.urlopen(getString).read()
	print(getString)
	print(randomData)
	return randomData
	# TODO: learn more about "return" in a function

# TODO: create a function that simulates tossing a coin by calling on randInt(tosses, min=0, max=1), where HEADS is sum/tosses >= 0.5.

# def coinToss(tosses=100):
	# randInt(tosses, 0, 1)
<|endoftext|>


<commit_before>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""Worker utilities for representing MapTasks.

Each MapTask represents a sequence of ParallelInstruction(s): read from a
source, write to a sink, parallel do, etc.
"""

import collections

from apache_beam import coders

# This module is experimental. No backwards-compatibility guarantees.

def build_worker_instruction(*args):
  """Create an object representing a ParallelInstruction protobuf.

  This will be a collections.namedtuple with a custom __str__ method.

  Alas, this wrapper is not known to pylint, which thinks it creates
  constants.  You may have to put a disable=invalid-name pylint
  annotation on any use of this, depending on your names.

  Args:
    *args: first argument is the name of the type to create.  Should
      start with "Worker".  Second arguments is alist of the
      attributes of this object.
  Returns:
    A new class, a subclass of tuple, that represents the protobuf.
  """
  tuple_class = collections.namedtuple(*args)
  tuple_class.__str__ = worker_object_to_string
  tuple_class.__repr__ = worker_object_to_string
  return tuple_class

def worker_printable_fields(workerproto):
  """Returns the interesting fields of a Worker* object."""
  return ['%s=%s' % (name, value)
          # _asdict is the only way and cannot subclass this generated class
          # pylint: disable=protected-access
          for name, value in workerproto._asdict().iteritems()
          # want to output value 0 but not None nor []
          if (value or value == 0)
          and name not in
          ('coder', 'coders', 'output_coders',
           'elements',
           'combine_fn', 'serialized_fn', 'window_fn',
           'append_trailing_newlines', 'strip_trailing_newlines',
           'compression_type', 'context',
           'start_shuffle_position', 'end_shuffle_position',
           'shuffle_reader_config', 'shuffle_writer_config')]

def worker_object_to_string(worker_object):
  """Returns a string compactly representing a Worker* object."""
  return '%s(%s)' % (worker_object.__class__.__name__,
                     ', '.join(worker_printable_fields(worker_object)))

# All the following Worker* definitions will have these lint problems:
# pylint: disable=invalid-name
# pylint: disable=pointless-string-statement

WorkerRead = build_worker_instruction(
    'WorkerRead', ['source', 'output_coders'])
"""Worker details needed to read from a source.

Attributes:
  source: a source object.
  output_coders: 1-tuple of the coder for the output.
"""

WorkerSideInputSource = build_worker_instruction(
    'WorkerSideInputSource', ['source', 'tag'])
"""Worker details needed to read from a side input source.

Attributes:
  source: a source object.
  tag: string tag for this side input.
"""

WorkerGroupingShuffleRead = build_worker_instruction(
    'WorkerGroupingShuffleRead',
    ['start_shuffle_position', 'end_shuffle_position',
     'shuffle_reader_config', 'coder', 'output_coders'])
"""Worker details needed to read from a grouping shuffle source.

Attributes:
  start_shuffle_position: An opaque string to be passed to the shuffle
    source to indicate where to start reading.
  end_shuffle_position: An opaque string to be passed to the shuffle
    source to indicate where to stop reading.
  shuffle_reader_config: An opaque string used to initialize the shuffle
    reader. Contains things like connection endpoints for the shuffle
    server appliance and various options.
  coder: The KV coder used to decode shuffle entries.
  output_coders: 1-tuple of the coder for the output.
"""

WorkerUngroupedShuffleRead = build_worker_instruction(
    'WorkerUngroupedShuffleRead',
    ['start_shuffle_position', 'end_shuffle_position',
     'shuffle_reader_config', 'coder', 'output_coders'])
"""Worker details needed to read from an ungrouped shuffle source.

Attributes:
  start_shuffle_position: An opaque string to be passed to the shuffle
    source to indicate where to start reading.
  end_shuffle_position: An opaque string to be passed to the shuffle
    source to indicate where to stop reading.
  shuffle_reader_config: An opaque string used to initialize the shuffle
    reader. Contains things like connection endpoints for the shuffle
    server appliance and various options.
  coder: The value coder used to decode shuffle entries.
"""

WorkerWrite = build_worker_instruction(
    'WorkerWrite', ['sink', 'input', 'output_coders'])
"""Worker details needed to write to a sink.

Attributes:
  sink: a sink object.
  input: A (producer index, output index) tuple representing the
    ParallelInstruction operation whose output feeds into this operation.
    The output index is 0 except for multi-output operations (like ParDo).
  output_coders: 1-tuple, coder to use to estimate bytes written.
"""

WorkerInMemoryWrite = build_worker_instruction(
    'WorkerInMemoryWrite',
    ['output_buffer', 'write_windowed_values', 'input', 'output_coders'])
"""Worker details needed to write to a in-memory sink.

Used only for unit testing. It makes worker tests less cluttered with code like
"write to a file and then check file contents".

Attributes:
  output_buffer: list to which output elements will be appended
  write_windowed_values: whether to record the entire WindowedValue outputs,
    or just the raw (unwindowed) value
  input: A (producer index, output index) tuple representing the
    ParallelInstruction operation whose output feeds into this operation.
    The output index is 0 except for multi-output operations (like ParDo).
  output_coders: 1-tuple, coder to use to estimate bytes written.
"""

WorkerShuffleWrite = build_worker_instruction(
    'WorkerShuffleWrite',
    ['shuffle_kind', 'shuffle_writer_config', 'input', 'output_coders'])
"""Worker details needed to write to a shuffle sink.

Attributes:
  shuffle_kind: A string describing the shuffle kind. This can control the
    way the worker interacts with the shuffle sink. The possible values are:
    'ungrouped', 'group_keys', and 'group_keys_and_sort_values'.
  shuffle_writer_config: An opaque string used to initialize the shuffle
    write. Contains things like connection endpoints for the shuffle
    server appliance and various options.
  input: A (producer index, output index) tuple representing the
    ParallelInstruction operation whose output feeds into this operation.
    The output index is 0 except for multi-output operations (like ParDo).
  output_coders: 1-tuple of the coder for input elements. If the
    shuffle_kind is grouping, this is expected to be a KV coder.
"""

WorkerDoFn = build_worker_instruction(
    'WorkerDoFn',
    ['serialized_fn', 'output_tags', 'input', 'side_inputs', 'output_coders'])
"""Worker details needed to run a DoFn.
Attributes:
  serialized_fn: A serialized DoFn object to be run for each input element.
  output_tags: The string tags used to identify the outputs of a ParDo
    operation. The tag is present even if the ParDo has just one output
    (e.g., ['out'].
  output_coders: array of coders, one for each output.
  input: A (producer index, output index) tuple representing the
    ParallelInstruction operation whose output feeds into this operation.
    The output index is 0 except for multi-output operations (like ParDo).
  side_inputs: A list of Worker...Read instances describing sources to be
    used for getting values. The types supported right now are
    WorkerInMemoryRead and WorkerTextRead.
"""

WorkerReifyTimestampAndWindows = build_worker_instruction(
    'WorkerReifyTimestampAndWindows',
    ['output_tags', 'input', 'output_coders'])
"""Worker details needed to run a WindowInto.
Attributes:
  output_tags: The string tags used to identify the outputs of a ParDo
    operation. The tag is present even if the ParDo has just one output
    (e.g., ['out'].
  output_coders: array of coders, one for each output.
  input: A (producer index, output index) tuple representing the
    ParallelInstruction operation whose output feeds into this operation.
    The output index is 0 except for multi-output operations (like ParDo).
"""

WorkerMergeWindows = build_worker_instruction(
    'WorkerMergeWindows',
    ['window_fn', 'combine_fn', 'phase', 'output_tags', 'input', 'coders',
     'context', 'output_coders'])
"""Worker details needed to run a MergeWindows (aka. GroupAlsoByWindows).
Attributes:
  window_fn: A serialized Windowing object representing the windowing strategy.
  combine_fn: A serialized CombineFn object to be used after executing the
    GroupAlsoByWindows operation. May be None if not a combining operation.
  phase: Possible values are 'all', 'add', 'merge', and 'extract'.
    A runner optimizer may split the user combiner in 3 separate
    phases (ADD, MERGE, and EXTRACT), on separate VMs, as it sees
    fit. The phase attribute dictates which DoFn is actually running in
    the worker. May be None if not a combining operation.
  output_tags: The string tags used to identify the outputs of a ParDo
    operation. The tag is present even if the ParDo has just one output
    (e.g., ['out'].
  output_coders: array of coders, one for each output.
  input: A (producer index, output index) tuple representing the
    ParallelInstruction operation whose output feeds into this operation.
    The output index is 0 except for multi-output operations (like ParDo).
  coders: A 2-tuple of coders (key, value) to encode shuffle entries.
  context: The ExecutionContext object for the current work item.
"""

WorkerCombineFn = build_worker_instruction(
    'WorkerCombineFn',
    ['serialized_fn', 'phase', 'input', 'output_coders'])
"""Worker details needed to run a CombineFn.
Attributes:
  serialized_fn: A serialized CombineFn object to be used.
  phase: Possible values are 'all', 'add', 'merge', and 'extract'.
    A runner optimizer may split the user combiner in 3 separate
    phases (ADD, MERGE, and EXTRACT), on separate VMs, as it sees
    fit. The phase attribute dictates which DoFn is actually running in
    the worker.
  input: A (producer index, output index) tuple representing the
    ParallelInstruction operation whose output feeds into this operation.
    The output index is 0 except for multi-output operations (like ParDo).
  output_coders: 1-tuple of the coder for the output.
"""

WorkerPartialGroupByKey = build_worker_instruction(
    'WorkerPartialGroupByKey',
    ['combine_fn', 'input', 'output_coders'])
"""Worker details needed to run a partial group-by-key.
Attributes:
  combine_fn: A serialized CombineFn object to be used.
  input: A (producer index, output index) tuple representing the
    ParallelInstruction operation whose output feeds into this operation.
    The output index is 0 except for multi-output operations (like ParDo).
  output_coders: 1-tuple of the coder for the output.
"""

WorkerFlatten = build_worker_instruction(
    'WorkerFlatten',
    ['inputs', 'output_coders'])
"""Worker details needed to run a Flatten.
Attributes:
  inputs: A list of tuples, each (producer index, output index), representing
    the ParallelInstruction operations whose output feeds into this operation.
    The output index is 0 unless the input is from a multi-output
    operation (such as ParDo).
  output_coders: 1-tuple of the coder for the output.
"""

def get_coder_from_spec(coder_spec):
  """Return a coder instance from a coder spec.

  Args:
    coder_spec: A dict where the value of the '@type' key is a pickled instance
      of a Coder instance.

  Returns:
    A coder instance (has encode/decode methods).
  """
  assert coder_spec is not None

  # Ignore the wrappers in these encodings.
  ignored_wrappers = (
      'com.google.cloud.dataflow.sdk.util.TimerOrElement$TimerOrElementCoder')
  if coder_spec['@type'] in ignored_wrappers:
    assert len(coder_spec['component_encodings']) == 1
    coder_spec = coder_spec['component_encodings'][0]
    return get_coder_from_spec(coder_spec)

  # Handle a few well known types of coders.
  if coder_spec['@type'] == 'kind:pair':
    assert len(coder_spec['component_encodings']) == 2
    component_coders = [
        get_coder_from_spec(c) for c in coder_spec['component_encodings']]
    return coders.TupleCoder(component_coders)
  elif coder_spec['@type'] == 'kind:stream':
    assert len(coder_spec['component_encodings']) == 1
    return coders.IterableCoder(
        get_coder_from_spec(coder_spec['component_encodings'][0]))
  elif coder_spec['@type'] == 'kind:windowed_value':
    assert len(coder_spec['component_encodings']) == 2
    value_coder, window_coder = [
        get_coder_from_spec(c) for c in coder_spec['component_encodings']]
    return coders.WindowedValueCoder(value_coder, window_coder=window_coder)
  elif coder_spec['@type'] == 'kind:interval_window':
    assert ('component_encodings' not in coder_spec
            or not coder_spec['component_encodings'])
    return coders.IntervalWindowCoder()
  elif coder_spec['@type'] == 'kind:global_window':
    assert ('component_encodings' not in coder_spec
            or not coder_spec['component_encodings'])
    return coders.coders.GlobalWindowCoder()
  elif coder_spec['@type'] == 'kind:length_prefix':
    assert len(coder_spec['component_encodings']) == 1
    return coders.coders.LengthPrefixCoder(
        get_coder_from_spec(coder_spec['component_encodings'][0]))
  elif coder_spec['@type'] == 'kind:bytes':
    assert ('component_encodings' not in coder_spec
            or len(coder_spec['component_encodings'] == 0))
    return coders.BytesCoder()

  # We pass coders in the form "<coder_name>$<pickled_data>" to make the job
  # description JSON more readable.
  return coders.coders.deserialize_coder(coder_spec['@type'])

class MapTask(object):
  """A map task decoded into operations and ready to be executed.

  Attributes:
    operations: A list of Worker* object created by parsing the instructions
      within the map task.
    stage_name: The name of this map task execution stage.
    system_names: The system names of the step corresponding to each map task
      operation in the execution graph.
    step_names: The names of the step corresponding to each map task operation.
    original_names: The internal name of a step in the original workflow graph.
  """

  def __init__(
      self, operations, stage_name, system_names, step_names, original_names):
    self.operations = operations
    self.stage_name = stage_name
    self.system_names = system_names
    self.step_names = step_names
    self.original_names = original_names

  def __str__(self):
    return '<%s %s steps=%s>' % (self.__class__.__name__, self.stage_name,
                                 '+'.join(self.step_names))
<commit_msg>Closes #3325<commit_after>#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""Worker utilities for representing MapTasks.

Each MapTask represents a sequence of ParallelInstruction(s): read from a
source, write to a sink, parallel do, etc.
"""

import collections

from apache_beam import coders

# This module is experimental. No backwards-compatibility guarantees.

def build_worker_instruction(*args):
  """Create an object representing a ParallelInstruction protobuf.

  This will be a collections.namedtuple with a custom __str__ method.

  Alas, this wrapper is not known to pylint, which thinks it creates
  constants.  You may have to put a disable=invalid-name pylint
  annotation on any use of this, depending on your names.

  Args:
    *args: first argument is the name of the type to create.  Should
      start with "Worker".  Second arguments is alist of the
      attributes of this object.
  Returns:
    A new class, a subclass of tuple, that represents the protobuf.
  """
  tuple_class = collections.namedtuple(*args)
  tuple_class.__str__ = worker_object_to_string
  tuple_class.__repr__ = worker_object_to_string
  return tuple_class

def worker_printable_fields(workerproto):
  """Returns the interesting fields of a Worker* object."""
  return ['%s=%s' % (name, value)
          # _asdict is the only way and cannot subclass this generated class
          # pylint: disable=protected-access
          for name, value in workerproto._asdict().iteritems()
          # want to output value 0 but not None nor []
          if (value or value == 0)
          and name not in
          ('coder', 'coders', 'output_coders',
           'elements',
           'combine_fn', 'serialized_fn', 'window_fn',
           'append_trailing_newlines', 'strip_trailing_newlines',
           'compression_type', 'context',
           'start_shuffle_position', 'end_shuffle_position',
           'shuffle_reader_config', 'shuffle_writer_config')]

def worker_object_to_string(worker_object):
  """Returns a string compactly representing a Worker* object."""
  return '%s(%s)' % (worker_object.__class__.__name__,
                     ', '.join(worker_printable_fields(worker_object)))

# All the following Worker* definitions will have these lint problems:
# pylint: disable=invalid-name
# pylint: disable=pointless-string-statement

WorkerRead = build_worker_instruction(
    'WorkerRead', ['source', 'output_coders'])
"""Worker details needed to read from a source.

Attributes:
  source: a source object.
  output_coders: 1-tuple of the coder for the output.
"""

WorkerSideInputSource = build_worker_instruction(
    'WorkerSideInputSource', ['source', 'tag'])
"""Worker details needed to read from a side input source.

Attributes:
  source: a source object.
  tag: string tag for this side input.
"""

WorkerGroupingShuffleRead = build_worker_instruction(
    'WorkerGroupingShuffleRead',
    ['start_shuffle_position', 'end_shuffle_position',
     'shuffle_reader_config', 'coder', 'output_coders'])
"""Worker details needed to read from a grouping shuffle source.

Attributes:
  start_shuffle_position: An opaque string to be passed to the shuffle
    source to indicate where to start reading.
  end_shuffle_position: An opaque string to be passed to the shuffle
    source to indicate where to stop reading.
  shuffle_reader_config: An opaque string used to initialize the shuffle
    reader. Contains things like connection endpoints for the shuffle
    server appliance and various options.
  coder: The KV coder used to decode shuffle entries.
  output_coders: 1-tuple of the coder for the output.
"""

WorkerUngroupedShuffleRead = build_worker_instruction(
    'WorkerUngroupedShuffleRead',
    ['start_shuffle_position', 'end_shuffle_position',
     'shuffle_reader_config', 'coder', 'output_coders'])
"""Worker details needed to read from an ungrouped shuffle source.

Attributes:
  start_shuffle_position: An opaque string to be passed to the shuffle
    source to indicate where to start reading.
  end_shuffle_position: An opaque string to be passed to the shuffle
    source to indicate where to stop reading.
  shuffle_reader_config: An opaque string used to initialize the shuffle
    reader. Contains things like connection endpoints for the shuffle
    server appliance and various options.
  coder: The value coder used to decode shuffle entries.
"""

WorkerWrite = build_worker_instruction(
    'WorkerWrite', ['sink', 'input', 'output_coders'])
"""Worker details needed to write to a sink.

Attributes:
  sink: a sink object.
  input: A (producer index, output index) tuple representing the
    ParallelInstruction operation whose output feeds into this operation.
    The output index is 0 except for multi-output operations (like ParDo).
  output_coders: 1-tuple, coder to use to estimate bytes written.
"""

WorkerInMemoryWrite = build_worker_instruction(
    'WorkerInMemoryWrite',
    ['output_buffer', 'write_windowed_values', 'input', 'output_coders'])
"""Worker details needed to write to a in-memory sink.

Used only for unit testing. It makes worker tests less cluttered with code like
"write to a file and then check file contents".

Attributes:
  output_buffer: list to which output elements will be appended
  write_windowed_values: whether to record the entire WindowedValue outputs,
    or just the raw (unwindowed) value
  input: A (producer index, output index) tuple representing the
    ParallelInstruction operation whose output feeds into this operation.
    The output index is 0 except for multi-output operations (like ParDo).
  output_coders: 1-tuple, coder to use to estimate bytes written.
"""

WorkerShuffleWrite = build_worker_instruction(
    'WorkerShuffleWrite',
    ['shuffle_kind', 'shuffle_writer_config', 'input', 'output_coders'])
"""Worker details needed to write to a shuffle sink.

Attributes:
  shuffle_kind: A string describing the shuffle kind. This can control the
    way the worker interacts with the shuffle sink. The possible values are:
    'ungrouped', 'group_keys', and 'group_keys_and_sort_values'.
  shuffle_writer_config: An opaque string used to initialize the shuffle
    write. Contains things like connection endpoints for the shuffle
    server appliance and various options.
  input: A (producer index, output index) tuple representing the
    ParallelInstruction operation whose output feeds into this operation.
    The output index is 0 except for multi-output operations (like ParDo).
  output_coders: 1-tuple of the coder for input elements. If the
    shuffle_kind is grouping, this is expected to be a KV coder.
"""

WorkerDoFn = build_worker_instruction(
    'WorkerDoFn',
    ['serialized_fn', 'output_tags', 'input', 'side_inputs', 'output_coders'])
"""Worker details needed to run a DoFn.
Attributes:
  serialized_fn: A serialized DoFn object to be run for each input element.
  output_tags: The string tags used to identify the outputs of a ParDo
    operation. The tag is present even if the ParDo has just one output
    (e.g., ['out'].
  output_coders: array of coders, one for each output.
  input: A (producer index, output index) tuple representing the
    ParallelInstruction operation whose output feeds into this operation.
    The output index is 0 except for multi-output operations (like ParDo).
  side_inputs: A list of Worker...Read instances describing sources to be
    used for getting values. The types supported right now are
    WorkerInMemoryRead and WorkerTextRead.
"""

WorkerReifyTimestampAndWindows = build_worker_instruction(
    'WorkerReifyTimestampAndWindows',
    ['output_tags', 'input', 'output_coders'])
"""Worker details needed to run a WindowInto.
Attributes:
  output_tags: The string tags used to identify the outputs of a ParDo
    operation. The tag is present even if the ParDo has just one output
    (e.g., ['out'].
  output_coders: array of coders, one for each output.
  input: A (producer index, output index) tuple representing the
    ParallelInstruction operation whose output feeds into this operation.
    The output index is 0 except for multi-output operations (like ParDo).
"""

WorkerMergeWindows = build_worker_instruction(
    'WorkerMergeWindows',
    ['window_fn', 'combine_fn', 'phase', 'output_tags', 'input', 'coders',
     'context', 'output_coders'])
"""Worker details needed to run a MergeWindows (aka. GroupAlsoByWindows).
Attributes:
  window_fn: A serialized Windowing object representing the windowing strategy.
  combine_fn: A serialized CombineFn object to be used after executing the
    GroupAlsoByWindows operation. May be None if not a combining operation.
  phase: Possible values are 'all', 'add', 'merge', and 'extract'.
    A runner optimizer may split the user combiner in 3 separate
    phases (ADD, MERGE, and EXTRACT), on separate VMs, as it sees
    fit. The phase attribute dictates which DoFn is actually running in
    the worker. May be None if not a combining operation.
  output_tags: The string tags used to identify the outputs of a ParDo
    operation. The tag is present even if the ParDo has just one output
    (e.g., ['out'].
  output_coders: array of coders, one for each output.
  input: A (producer index, output index) tuple representing the
    ParallelInstruction operation whose output feeds into this operation.
    The output index is 0 except for multi-output operations (like ParDo).
  coders: A 2-tuple of coders (key, value) to encode shuffle entries.
  context: The ExecutionContext object for the current work item.
"""

WorkerCombineFn = build_worker_instruction(
    'WorkerCombineFn',
    ['serialized_fn', 'phase', 'input', 'output_coders'])
"""Worker details needed to run a CombineFn.
Attributes:
  serialized_fn: A serialized CombineFn object to be used.
  phase: Possible values are 'all', 'add', 'merge', and 'extract'.
    A runner optimizer may split the user combiner in 3 separate
    phases (ADD, MERGE, and EXTRACT), on separate VMs, as it sees
    fit. The phase attribute dictates which DoFn is actually running in
    the worker.
  input: A (producer index, output index) tuple representing the
    ParallelInstruction operation whose output feeds into this operation.
    The output index is 0 except for multi-output operations (like ParDo).
  output_coders: 1-tuple of the coder for the output.
"""

WorkerPartialGroupByKey = build_worker_instruction(
    'WorkerPartialGroupByKey',
    ['combine_fn', 'input', 'output_coders'])
"""Worker details needed to run a partial group-by-key.
Attributes:
  combine_fn: A serialized CombineFn object to be used.
  input: A (producer index, output index) tuple representing the
    ParallelInstruction operation whose output feeds into this operation.
    The output index is 0 except for multi-output operations (like ParDo).
  output_coders: 1-tuple of the coder for the output.
"""

WorkerFlatten = build_worker_instruction(
    'WorkerFlatten',
    ['inputs', 'output_coders'])
"""Worker details needed to run a Flatten.
Attributes:
  inputs: A list of tuples, each (producer index, output index), representing
    the ParallelInstruction operations whose output feeds into this operation.
    The output index is 0 unless the input is from a multi-output
    operation (such as ParDo).
  output_coders: 1-tuple of the coder for the output.
"""

def get_coder_from_spec(coder_spec):
  """Return a coder instance from a coder spec.

  Args:
    coder_spec: A dict where the value of the '@type' key is a pickled instance
      of a Coder instance.

  Returns:
    A coder instance (has encode/decode methods).
  """
  assert coder_spec is not None

  # Ignore the wrappers in these encodings.
  ignored_wrappers = (
      'com.google.cloud.dataflow.sdk.util.TimerOrElement$TimerOrElementCoder')
  if coder_spec['@type'] in ignored_wrappers:
    assert len(coder_spec['component_encodings']) == 1
    coder_spec = coder_spec['component_encodings'][0]
    return get_coder_from_spec(coder_spec)

  # Handle a few well known types of coders.
  if coder_spec['@type'] == 'kind:pair':
    assert len(coder_spec['component_encodings']) == 2
    component_coders = [
        get_coder_from_spec(c) for c in coder_spec['component_encodings']]
    return coders.TupleCoder(component_coders)
  elif coder_spec['@type'] == 'kind:stream':
    assert len(coder_spec['component_encodings']) == 1
    return coders.IterableCoder(
        get_coder_from_spec(coder_spec['component_encodings'][0]))
  elif coder_spec['@type'] == 'kind:windowed_value':
    assert len(coder_spec['component_encodings']) == 2
    value_coder, window_coder = [
        get_coder_from_spec(c) for c in coder_spec['component_encodings']]
    return coders.coders.WindowedValueCoder(
        value_coder, window_coder=window_coder)
  elif coder_spec['@type'] == 'kind:interval_window':
    assert ('component_encodings' not in coder_spec
            or not coder_spec['component_encodings'])
    return coders.coders.IntervalWindowCoder()
  elif coder_spec['@type'] == 'kind:global_window':
    assert ('component_encodings' not in coder_spec
            or not coder_spec['component_encodings'])
    return coders.coders.GlobalWindowCoder()
  elif coder_spec['@type'] == 'kind:length_prefix':
    assert len(coder_spec['component_encodings']) == 1
    return coders.coders.LengthPrefixCoder(
        get_coder_from_spec(coder_spec['component_encodings'][0]))
  elif coder_spec['@type'] == 'kind:bytes':
    assert ('component_encodings' not in coder_spec
            or len(coder_spec['component_encodings'] == 0))
    return coders.BytesCoder()

  # We pass coders in the form "<coder_name>$<pickled_data>" to make the job
  # description JSON more readable.
  return coders.coders.deserialize_coder(coder_spec['@type'])

class MapTask(object):
  """A map task decoded into operations and ready to be executed.

  Attributes:
    operations: A list of Worker* object created by parsing the instructions
      within the map task.
    stage_name: The name of this map task execution stage.
    system_names: The system names of the step corresponding to each map task
      operation in the execution graph.
    step_names: The names of the step corresponding to each map task operation.
    original_names: The internal name of a step in the original workflow graph.
  """

  def __init__(
      self, operations, stage_name, system_names, step_names, original_names):
    self.operations = operations
    self.stage_name = stage_name
    self.system_names = system_names
    self.step_names = step_names
    self.original_names = original_names

  def __str__(self):
    return '<%s %s steps=%s>' % (self.__class__.__name__, self.stage_name,
                                 '+'.join(self.step_names))
<|endoftext|>


<commit_before>import os
import base64
import random
from datetime import datetime
from django.db.models import F, Q
from planetstack.config import Config
from observer.openstacksyncstep import OpenStackSyncStep
from core.models.node import Node
from core.models.site import Site, Deployment

class SyncNodes(OpenStackSyncStep):
    provides=[Node]
    requested_interval=0

    def fetch_pending(self, deleted):
        # Nodes come from the back end
        # You can't delete them
        if (deleted):
            return []

        # collect local nodes
        sites = Site.objects.all()
        nodes = Node.objects.all()
        node_hostnames  = [node.name for node in nodes]

        # fetch all nodes from each deployment
        deployments = Deployment.objects.all()
        new_nodes = []
        for deployment in deployments:
            driver = self.driver.admin_driver(deployment=deployment.name)
            compute_nodes = driver.shell.nova.hypervisors.list()
            for compute_node in compute_nodes:
                if compute_node.hypervisor_hostname not in node_hostnames:
                    # XX TODO:figure out how to correctly identify a node's site.
                    # XX pick a random site to add the node to for now
                    site_index = random.randint(0, len(sites))
                    node = Node(name=compute_node.hypervisor_hostname,
                                site=sites[site_index], deployment=deployment)
                    new_nodes.append(node)

        return new_nodes

    def sync_record(self, node):
        node.save()

<commit_msg>skip unreachable/misconfigured deployments in sync_nodes<commit_after>import os
import base64
import random
from datetime import datetime
from django.db.models import F, Q
from planetstack.config import Config
from observer.openstacksyncstep import OpenStackSyncStep
from core.models.node import Node
from core.models.site import Site, Deployment
from util.logger import Logger, logging

logger = Logger(level=logging.INFO)

class SyncNodes(OpenStackSyncStep):
    provides=[Node]
    requested_interval=0

    def fetch_pending(self, deleted):
        # Nodes come from the back end
        # You can't delete them
        if (deleted):
            return []

        # collect local nodes
        sites = Site.objects.all()
        nodes = Node.objects.all()
        node_hostnames  = [node.name for node in nodes]

        # fetch all nodes from each deployment
        deployments = Deployment.objects.all()
        new_nodes = []
        for deployment in deployments:
            try:
                driver = self.driver.admin_driver(deployment=deployment.name)
                compute_nodes = driver.shell.nova.hypervisors.list()
            except:
                logger.log_exc("Failed to get nodes from deployment %s" % str(deployment))
                continue

            for compute_node in compute_nodes:
                if compute_node.hypervisor_hostname not in node_hostnames:
                    # XX TODO:figure out how to correctly identify a node's site.
                    # XX pick a random site to add the node to for now
                    site_index = random.randint(0, len(sites))
                    node = Node(name=compute_node.hypervisor_hostname,
                                site=sites[site_index], deployment=deployment)
                    new_nodes.append(node)

        return new_nodes

    def sync_record(self, node):
        node.save()

<|endoftext|>