<commit_before>#!/usr/bin/env python

import json
import sys
import argparse
import usb.core
import serial
import curses
import curses.wrapper
import time
from datetime import datetime

class DataPoint(object):
    def __init__(self, name, value_type, min_value=0, max_value=0, vocab=None,
            events=False, messages_received=0):
        self.name = name
        self.type = value_type
        self.min_value = min_value
        self.max_value = max_value
        self.range = max_value - min_value
        if self.range <= 0:
            self.range = 1
        self.event = ''
        self.bad_data = False
        self.bad_data_tally = 0
        self.current_data = None
        self.events_active = events
        self.events = []
        self.messages_received = messages_received
        self.rate_messages_received = 0

        # Vocab is a list of acceptable strings for CurrentValue
        self.vocab = vocab or []

        if self.events_active is True:
            for x in range(len(self.vocab)):
                self.events.append("")

    def update(self, message):
        if self.bad_data:
            self.bad_data_tally += 1
            self.bad_data = False

        self.rate_messages_received += 1
        self.messages_received += 1
        self.current_data = message.get('value', None)
        if type(self.current_data) == int:
            self.current_data = float(self.current_data)
        if type(self.current_data) != self.type:
            self.bad_data = True
        else:
            if type(self.current_data) is bool:
                return
            elif type(self.current_data) is unicode:
                if self.current_data in self.vocab:
                    #Save the event in the proper spot.
                    if (len(message) > 2) and (self.events_active is True):
                        self.events[self.vocab.index(self.current_data)
                                ] = message.get('event', None)
                else:
                    self.bad_data = True
            else:
                if self.current_data < self.min_value:
                    self.bad_data = True
                elif self.current_data > self.max_value:
                    self.bad_data = True

    def print_to_window(self, window, row, begin_time):
        window.addstr(row, 0, self.name)
        result = ""
        if self.current_data is not None and not self.bad_data:
            if self.type == float:
                percent = self.current_data - self.min_value
                percent /= self.range
                Count = 0
                graph = "*"
                percent -= .1
                while percent > 0:
                    graph += "-"
                    Count += 1
                    percent -= .1
                graph += "|"
                Count += 1
                while Count < 10:
                    graph += "-"
                    Count +=1
                graph += "* "
                window.addstr(row, 30, graph)

            if self.events_active is False:
                window.addstr(row, 45, str(self.current_data))
            else:
                result = ""
                for item, value in enumerate(self.vocab):
                    if value == "driver":
                        keyword = "dr"
                    elif value == "passenger":
                        keyword = "ps"
                    elif value == "rear_right":
                        keyword = "rr"
                    elif value == "rear_left":
                        keyword = "rl"
                result = (result + keyword + ":" + str(self.events[item]) + " ")
                window.addstr(row, 45, result)

            if self.bad_data_tally > 0:
                bad_data_color = curses.color_pair(1)
            else:
                bad_data_color = curses.color_pair(2)

            window.addstr(row, 80, "Errors: " + str(self.bad_data_tally),
                    bad_data_color)
            window.addstr(row, 95, "Messages: " + str(self.messages_received))
            window.addstr(row, 110, "Frequency: " +
                    str(int(self.rate_messages_received /
                        ((datetime.now() - begin_time).total_seconds()))))

class CanTranslator(object):
    def __init__(self, verbose=False, dump=False, dashboard=False,
            elements=None):
        self.verbose = verbose
        self.dump = dump
        self.dashboard = dashboard
        self.message_buffer = ""
        self.messages_received = 0
        self.good_messages = 0
        self.elements = elements or []
        self.total_bytes_received = 0
        self.rate_bytes_received = 0
        self.begin_time = datetime.now()

    def parse_message(self):
        if "\n" in self.message_buffer:
            message,_,remainder= self.message_buffer.partition("\n")
            try:
                parsed_message = json.loads(message)
                if not isinstance(parsed_message, dict):
                    raise ValueError()
            except ValueError:
                pass
            else:
                self.good_messages += 1
                if self.dump:
                    print "%f: %s" % (time.time(), message)
                if self.verbose:
                    print parsed_message
                if self.dashboard:
                    self.total_bytes_received += sys.getsizeof(parsed_message)
                self.rate_bytes_received += sys.getsizeof(parsed_message)
                for element in self.elements:
                    if element.name == parsed_message.get('name', None):
                        element.update(parsed_message)
                        break
                return parsed_message
            finally:
                self.message_buffer = remainder
                self.messages_received += 1

    # This resets the number of bytes received for both individual messages and
    # the total number of messages and also resets begin_time to set up a time
    # frame of 10 seconds in which to calculate the data rates
    def date_rate_management(self):
        if ((datetime.now() - self.begin_time).total_seconds() > 10):
            self.begin_time = datetime.now()
            self.rate_bytes_received = 0;
            for element in self.elements:
                    element.rate_messages_received = 0

    def run(self, window=None):
        if window is not None:
            curses.use_default_colors()
            curses.init_pair(1, curses.COLOR_RED, -1)
            curses.init_pair(2, curses.COLOR_GREEN, -1)
            curses.init_pair(3, curses.COLOR_YELLOW, -1)

        while True:
            self.message_buffer += self.read()
            self.parse_message()
            self.date_rate_management()

            if self.dashboard and window is not None:
                window.erase()
                for row, element in enumerate(self.elements):
                    element.print_to_window(window, row, self.begin_time)
                percentage_good = 0
                if self.messages_received != 0:
                    percentage_good = (self.good_messages /
                            self.messages_received)
                window.addstr(len(self.elements), 0,
                        "Received %d messages so far (%d%% valid)..." % (
                        self.messages_received, percentage_good * 100),
                        curses.A_REVERSE)
                window.addstr(len(self.elements) + 1, 0,
                        "Total Bytes Received: " +
                        str(self.total_bytes_received), curses.A_REVERSE)
                window.addstr(len(self.elements) + 2, 0, "Overall Data Rate: " +
                    str((self.rate_bytes_received) / (datetime.now()
                    - self.begin_time).total_seconds()) + " Bps",
                     curses.A_REVERSE)
                window.refresh()

class SerialCanTransaltor(CanTranslator):
    def __init__(self, port="/dev/ttyUSB1", baud_rate=115200, verbose=False,
            dump=False, dashboard=False, elements=None):
        super(SerialCanTransaltor, self).__init__(verbose, dump, dashboard,
                elements)
        self.port = port
        self.baud_rate = baud_rate
        self.device = serial.Serial(self.port, self.baud_rate)
        print "Opened serial device at %s" % self.port

    def read(self):
        return self.device.readline()

class UsbCanTranslator(CanTranslator):
    INTERFACE = 0
    VERSION_CONTROL_COMMAND = 0x80
    RESET_CONTROL_COMMAND = 0x81

    def __init__(self, vendor_id=0x04d8, verbose=False, dump=False,
            dashboard=False, elements=None):
        super(UsbCanTranslator, self).__init__(verbose, dump, dashboard,
                elements)
        self.vendor_id = vendor_id

        self.device = usb.core.find(idVendor=vendor_id)
        if not self.device:
            print "Couldn't find a USB device from vendor %s" % self.vendor_id
            sys.exit()
        self.device.set_configuration()
        config = self.device.get_active_configuration()
        interface_number = config[(0,0)].bInterfaceNumber
        interface = usb.util.find_descriptor(config,
                bInterfaceNumber=interface_number)

        self.out_endpoint = usb.util.find_descriptor(interface,
                custom_match = \
                        lambda e: \
                        usb.util.endpoint_direction(e.bEndpointAddress) == \
                        usb.util.ENDPOINT_OUT)
        self.in_endpoint = usb.util.find_descriptor(interface,
                custom_match = \
                        lambda e: \
                        usb.util.endpoint_direction(e.bEndpointAddress) == \
                        usb.util.ENDPOINT_IN)

        if not self.out_endpoint or not self.in_endpoint:
            print "Couldn't find proper endpoints on the USB device"
            sys.exit()

    def read(self):
        return self.in_endpoint.read(64, 1000000).tostring()

    @property
    def version(self):
        raw_version = self.device.ctrl_transfer(0xC0,
                self.VERSION_CONTROL_COMMAND, 0, 0, 100)
        return ''.join([chr(x) for x in raw_version])

    def reset(self):
        self.device.ctrl_transfer(0x40, self.RESET_CONTROL_COMMAND, 0, 0)

    def write(self, name, value):
        if value == "true":
            value = True
        elif value == "false":
            value = False
        else:
            try:
                value = float(value)
            except ValueError:
                pass

        message = json.dumps({'name': name, 'value': value})
        bytes_written = self.out_endpoint.write(message + "\x00")
        assert bytes_written == len(message) + 1

def parse_options():
    parser = argparse.ArgumentParser(description="Receive and print OpenXC "
        "messages over USB")
    parser.add_argument("--vendor",
            action="store",
            dest="vendor",
            default=0x04d8)
    parser.add_argument("--verbose", "-v",
            action="store_true",
            dest="verbose",
            default=False)
    parser.add_argument("--serial", "-s",
            action="store_true",
            dest="serial",
            default=False)
    parser.add_argument("--dump", "-d",
            action="store_true",
            dest="dump",
            default=False)
    parser.add_argument("--version",
            action="store_true",
            dest="version",
            default=False)
    parser.add_argument("--reset",
            action="store_true",
            dest="reset",
            default=False)
    parser.add_argument("--dashboard",
            action="store_true",
            dest="dashboard",
            default=False)
    parser.add_argument("--write", "-w",
            nargs=2,
            action="store",
            dest="write")
    parser.add_argument("--serial-device",
            action="store",
            dest="serial_device",
            default=None)

    arguments = parser.parse_args()
    return arguments

def initialize_elements(dashboard):
    elements = []

    elements.append(DataPoint('steering_wheel_angle', float, -600, 600))
    elements.append(DataPoint('engine_speed', float, 0, 8000))
    elements.append(DataPoint('transmission_gear_position', unicode,
        vocab=['first', 'second', 'third', 'fourth', 'fifth', 'sixth',
            'seventh', 'eighth', 'neutral', 'reverse', 'park']))
    elements.append(DataPoint('ignition_status', unicode,
        vocab=['off', 'accessory', 'run', 'start']))
    elements.append(DataPoint('brake_pedal_status', bool))
    elements.append(DataPoint('parking_brake_status', bool))
    elements.append(DataPoint('headlamp_status', bool))
    elements.append(DataPoint('accelerator_pedal_position', float, 0, 100))
    elements.append(DataPoint('torque_at_transmission', float, -400, 1500))
    elements.append(DataPoint('vehicle_speed', float, 0, 120))
    elements.append(DataPoint('fuel_consumed_since_restart', float, 0, 300))
    elements.append(DataPoint('fine_odometer_since_restart', float, 0, 300))
    elements.append(DataPoint('door_status', unicode,
        vocab=['driver', 'rear_left', 'rear_right', 'passenger'], events=True))
    elements.append(DataPoint('windshield_wiper_status', bool))
    elements.append(DataPoint('odometer', float, 0, 100000))
    elements.append(DataPoint('high_beam_status', bool))
    elements.append(DataPoint('fuel_level', float, 0, 300))
    elements.append(DataPoint('latitude', float, -90, 90))
    elements.append(DataPoint('longitude', float, -180, 180))
    elements.append(DataPoint('heater_status', bool))
    elements.append(DataPoint('air_conditioning_status', bool))
    elements.append(DataPoint('charging_status', bool))
    elements.append(DataPoint('range', float, 0, 500))
    elements.append(DataPoint('gear_lever_position', unicode,
        vocab=['first', 'second', 'third', 'fourth', 'fifth', 'sixth',
            'seventh', 'neutral', 'reverse', 'park', 'drive', 'low', 'sport']))
    elements.append(DataPoint('battery_level', float, 0, 100))
    elements.append(DataPoint('cabin_temperature', float, -50, 150))

    return elements

def main():
    arguments = parse_options()

    if arguments.serial:
        device_class = SerialCanTransaltor
        kwargs = dict()
        if arguments.serial_device:
            kwargs['port'] = arguments.serial_device
    else:
        device_class = UsbCanTranslator
        kwargs = dict(vendor_id=arguments.vendor)

    device = device_class(verbose=arguments.verbose, dump=arguments.dump,
            dashboard=arguments.dashboard,
            elements=initialize_elements(arguments.dashboard), **kwargs)
    if arguments.version:
        print "Device is running version %s" % device.version
    elif arguments.reset:
        print "Resetting device..."
        device.reset()
    elif arguments.dashboard:
        curses.wrapper(device.run)
    elif arguments.write:
        device.write(arguments.write[0], arguments.write[1])
    else:
        device.run()

if __name__ == '__main__':
    main()
<commit_msg>Only import serial if we try to use a serial reader.<commit_after>#!/usr/bin/env python

import json
import sys
import argparse
import usb.core
import curses
import curses.wrapper
import time
from datetime import datetime

class DataPoint(object):
    def __init__(self, name, value_type, min_value=0, max_value=0, vocab=None,
            events=False, messages_received=0):
        self.name = name
        self.type = value_type
        self.min_value = min_value
        self.max_value = max_value
        self.range = max_value - min_value
        if self.range <= 0:
            self.range = 1
        self.event = ''
        self.bad_data = False
        self.bad_data_tally = 0
        self.current_data = None
        self.events_active = events
        self.events = []
        self.messages_received = messages_received
        self.rate_messages_received = 0

        # Vocab is a list of acceptable strings for CurrentValue
        self.vocab = vocab or []

        if self.events_active is True:
            for x in range(len(self.vocab)):
                self.events.append("")

    def update(self, message):
        if self.bad_data:
            self.bad_data_tally += 1
            self.bad_data = False

        self.rate_messages_received += 1
        self.messages_received += 1
        self.current_data = message.get('value', None)
        if type(self.current_data) == int:
            self.current_data = float(self.current_data)
        if type(self.current_data) != self.type:
            self.bad_data = True
        else:
            if type(self.current_data) is bool:
                return
            elif type(self.current_data) is unicode:
                if self.current_data in self.vocab:
                    #Save the event in the proper spot.
                    if (len(message) > 2) and (self.events_active is True):
                        self.events[self.vocab.index(self.current_data)
                                ] = message.get('event', None)
                else:
                    self.bad_data = True
            else:
                if self.current_data < self.min_value:
                    self.bad_data = True
                elif self.current_data > self.max_value:
                    self.bad_data = True

    def print_to_window(self, window, row, begin_time):
        window.addstr(row, 0, self.name)
        result = ""
        if self.current_data is not None and not self.bad_data:
            if self.type == float:
                percent = self.current_data - self.min_value
                percent /= self.range
                Count = 0
                graph = "*"
                percent -= .1
                while percent > 0:
                    graph += "-"
                    Count += 1
                    percent -= .1
                graph += "|"
                Count += 1
                while Count < 10:
                    graph += "-"
                    Count +=1
                graph += "* "
                window.addstr(row, 30, graph)

            if self.events_active is False:
                window.addstr(row, 45, str(self.current_data))
            else:
                result = ""
                for item, value in enumerate(self.vocab):
                    if value == "driver":
                        keyword = "dr"
                    elif value == "passenger":
                        keyword = "ps"
                    elif value == "rear_right":
                        keyword = "rr"
                    elif value == "rear_left":
                        keyword = "rl"
                result = (result + keyword + ":" + str(self.events[item]) + " ")
                window.addstr(row, 45, result)

            if self.bad_data_tally > 0:
                bad_data_color = curses.color_pair(1)
            else:
                bad_data_color = curses.color_pair(2)

            window.addstr(row, 80, "Errors: " + str(self.bad_data_tally),
                    bad_data_color)
            window.addstr(row, 95, "Messages: " + str(self.messages_received))
            window.addstr(row, 110, "Frequency: " +
                    str(int(self.rate_messages_received /
                        ((datetime.now() - begin_time).total_seconds()))))

class CanTranslator(object):
    def __init__(self, verbose=False, dump=False, dashboard=False,
            elements=None):
        self.verbose = verbose
        self.dump = dump
        self.dashboard = dashboard
        self.message_buffer = ""
        self.messages_received = 0
        self.good_messages = 0
        self.elements = elements or []
        self.total_bytes_received = 0
        self.rate_bytes_received = 0
        self.begin_time = datetime.now()

    def parse_message(self):
        if "\n" in self.message_buffer:
            message,_,remainder= self.message_buffer.partition("\n")
            try:
                parsed_message = json.loads(message)
                if not isinstance(parsed_message, dict):
                    raise ValueError()
            except ValueError:
                pass
            else:
                self.good_messages += 1
                if self.dump:
                    print "%f: %s" % (time.time(), message)
                if self.verbose:
                    print parsed_message
                if self.dashboard:
                    self.total_bytes_received += sys.getsizeof(parsed_message)
                self.rate_bytes_received += sys.getsizeof(parsed_message)
                for element in self.elements:
                    if element.name == parsed_message.get('name', None):
                        element.update(parsed_message)
                        break
                return parsed_message
            finally:
                self.message_buffer = remainder
                self.messages_received += 1

    # This resets the number of bytes received for both individual messages and
    # the total number of messages and also resets begin_time to set up a time
    # frame of 10 seconds in which to calculate the data rates
    def date_rate_management(self):
        if ((datetime.now() - self.begin_time).total_seconds() > 10):
            self.begin_time = datetime.now()
            self.rate_bytes_received = 0;
            for element in self.elements:
                    element.rate_messages_received = 0

    def run(self, window=None):
        if window is not None:
            curses.use_default_colors()
            curses.init_pair(1, curses.COLOR_RED, -1)
            curses.init_pair(2, curses.COLOR_GREEN, -1)
            curses.init_pair(3, curses.COLOR_YELLOW, -1)

        while True:
            self.message_buffer += self.read()
            self.parse_message()
            self.date_rate_management()

            if self.dashboard and window is not None:
                window.erase()
                for row, element in enumerate(self.elements):
                    element.print_to_window(window, row, self.begin_time)
                percentage_good = 0
                if self.messages_received != 0:
                    percentage_good = (self.good_messages /
                            self.messages_received)
                window.addstr(len(self.elements), 0,
                        "Received %d messages so far (%d%% valid)..." % (
                        self.messages_received, percentage_good * 100),
                        curses.A_REVERSE)
                window.addstr(len(self.elements) + 1, 0,
                        "Total Bytes Received: " +
                        str(self.total_bytes_received), curses.A_REVERSE)
                window.addstr(len(self.elements) + 2, 0, "Overall Data Rate: " +
                    str((self.rate_bytes_received) / (datetime.now()
                    - self.begin_time).total_seconds()) + " Bps",
                     curses.A_REVERSE)
                window.refresh()

class SerialCanTransaltor(CanTranslator):
    def __init__(self, port="/dev/ttyUSB1", baud_rate=115200, verbose=False,
            dump=False, dashboard=False, elements=None):
        super(SerialCanTransaltor, self).__init__(verbose, dump, dashboard,
                elements)
        self.port = port
        self.baud_rate = baud_rate
        import serial
        self.device = serial.Serial(self.port, self.baud_rate)
        print "Opened serial device at %s" % self.port

    def read(self):
        return self.device.readline()

class UsbCanTranslator(CanTranslator):
    INTERFACE = 0
    VERSION_CONTROL_COMMAND = 0x80
    RESET_CONTROL_COMMAND = 0x81

    def __init__(self, vendor_id=0x04d8, verbose=False, dump=False,
            dashboard=False, elements=None):
        super(UsbCanTranslator, self).__init__(verbose, dump, dashboard,
                elements)
        self.vendor_id = vendor_id

        self.device = usb.core.find(idVendor=vendor_id)
        if not self.device:
            print "Couldn't find a USB device from vendor %s" % self.vendor_id
            sys.exit()
        self.device.set_configuration()
        config = self.device.get_active_configuration()
        interface_number = config[(0,0)].bInterfaceNumber
        interface = usb.util.find_descriptor(config,
                bInterfaceNumber=interface_number)

        self.out_endpoint = usb.util.find_descriptor(interface,
                custom_match = \
                        lambda e: \
                        usb.util.endpoint_direction(e.bEndpointAddress) == \
                        usb.util.ENDPOINT_OUT)
        self.in_endpoint = usb.util.find_descriptor(interface,
                custom_match = \
                        lambda e: \
                        usb.util.endpoint_direction(e.bEndpointAddress) == \
                        usb.util.ENDPOINT_IN)

        if not self.out_endpoint or not self.in_endpoint:
            print "Couldn't find proper endpoints on the USB device"
            sys.exit()

    def read(self):
        return self.in_endpoint.read(64, 1000000).tostring()

    @property
    def version(self):
        raw_version = self.device.ctrl_transfer(0xC0,
                self.VERSION_CONTROL_COMMAND, 0, 0, 100)
        return ''.join([chr(x) for x in raw_version])

    def reset(self):
        self.device.ctrl_transfer(0x40, self.RESET_CONTROL_COMMAND, 0, 0)

    def write(self, name, value):
        if value == "true":
            value = True
        elif value == "false":
            value = False
        else:
            try:
                value = float(value)
            except ValueError:
                pass

        message = json.dumps({'name': name, 'value': value})
        bytes_written = self.out_endpoint.write(message + "\x00")
        assert bytes_written == len(message) + 1

def parse_options():
    parser = argparse.ArgumentParser(description="Receive and print OpenXC "
        "messages over USB")
    parser.add_argument("--vendor",
            action="store",
            dest="vendor",
            default=0x04d8)
    parser.add_argument("--verbose", "-v",
            action="store_true",
            dest="verbose",
            default=False)
    parser.add_argument("--serial", "-s",
            action="store_true",
            dest="serial",
            default=False)
    parser.add_argument("--dump", "-d",
            action="store_true",
            dest="dump",
            default=False)
    parser.add_argument("--version",
            action="store_true",
            dest="version",
            default=False)
    parser.add_argument("--reset",
            action="store_true",
            dest="reset",
            default=False)
    parser.add_argument("--dashboard",
            action="store_true",
            dest="dashboard",
            default=False)
    parser.add_argument("--write", "-w",
            nargs=2,
            action="store",
            dest="write")
    parser.add_argument("--serial-device",
            action="store",
            dest="serial_device",
            default=None)

    arguments = parser.parse_args()
    return arguments

def initialize_elements(dashboard):
    elements = []

    elements.append(DataPoint('steering_wheel_angle', float, -600, 600))
    elements.append(DataPoint('engine_speed', float, 0, 8000))
    elements.append(DataPoint('transmission_gear_position', unicode,
        vocab=['first', 'second', 'third', 'fourth', 'fifth', 'sixth',
            'seventh', 'eighth', 'neutral', 'reverse', 'park']))
    elements.append(DataPoint('ignition_status', unicode,
        vocab=['off', 'accessory', 'run', 'start']))
    elements.append(DataPoint('brake_pedal_status', bool))
    elements.append(DataPoint('parking_brake_status', bool))
    elements.append(DataPoint('headlamp_status', bool))
    elements.append(DataPoint('accelerator_pedal_position', float, 0, 100))
    elements.append(DataPoint('torque_at_transmission', float, -400, 1500))
    elements.append(DataPoint('vehicle_speed', float, 0, 120))
    elements.append(DataPoint('fuel_consumed_since_restart', float, 0, 300))
    elements.append(DataPoint('fine_odometer_since_restart', float, 0, 300))
    elements.append(DataPoint('door_status', unicode,
        vocab=['driver', 'rear_left', 'rear_right', 'passenger'], events=True))
    elements.append(DataPoint('windshield_wiper_status', bool))
    elements.append(DataPoint('odometer', float, 0, 100000))
    elements.append(DataPoint('high_beam_status', bool))
    elements.append(DataPoint('fuel_level', float, 0, 300))
    elements.append(DataPoint('latitude', float, -90, 90))
    elements.append(DataPoint('longitude', float, -180, 180))
    elements.append(DataPoint('heater_status', bool))
    elements.append(DataPoint('air_conditioning_status', bool))
    elements.append(DataPoint('charging_status', bool))
    elements.append(DataPoint('range', float, 0, 500))
    elements.append(DataPoint('gear_lever_position', unicode,
        vocab=['first', 'second', 'third', 'fourth', 'fifth', 'sixth',
            'seventh', 'neutral', 'reverse', 'park', 'drive', 'low', 'sport']))
    elements.append(DataPoint('battery_level', float, 0, 100))
    elements.append(DataPoint('cabin_temperature', float, -50, 150))

    return elements

def main():
    arguments = parse_options()

    if arguments.serial:
        device_class = SerialCanTransaltor
        kwargs = dict()
        if arguments.serial_device:
            kwargs['port'] = arguments.serial_device
    else:
        device_class = UsbCanTranslator
        kwargs = dict(vendor_id=arguments.vendor)

    device = device_class(verbose=arguments.verbose, dump=arguments.dump,
            dashboard=arguments.dashboard,
            elements=initialize_elements(arguments.dashboard), **kwargs)
    if arguments.version:
        print "Device is running version %s" % device.version
    elif arguments.reset:
        print "Resetting device..."
        device.reset()
    elif arguments.dashboard:
        curses.wrapper(device.run)
    elif arguments.write:
        device.write(arguments.write[0], arguments.write[1])
    else:
        device.run()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
__version__ = "1.8.5"
<commit_msg>updated version<commit_after># -*- coding: utf-8 -*-
__version__ = "1.8.6"
<|endoftext|>


<commit_before>import numpy as np
import cv2

cap = cv2.VideoCapture(0)
width, height = 320, 320
cap.set(3, width)
cap.set(4, height)

# create a window for displaying image and move it to a reasonable spot
cv2.namedWindow('frame')
cv2.moveWindow('frame', 100, 100)

while(True):
    # Capture frame-by-frame
    ret, frame = cap.read()

    # rotate frame
    rotation_matrix = cv2.getRotationMatrix2D((width / 2, height / 2), 90, 1)
    rotated = cv2.warpAffine(frame, rotation_matrix, (width, height))

    # Our operations on the frame come here
    # gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    # Display the resulting frame
    cv2.imshow('frame', rotated)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

# When everything done, release the capture
cap.release()
cv2.destroyAllWindows()
<commit_msg>Update<commit_after>import numpy as np
import cv2

cap = cv2.VideoCapture(0)
width, height = 320, 320
cap.set(3, width)
cap.set(4, height)

# create a window for displaying image and move it to a reasonable spot
cv2.namedWindow('frame')
cv2.moveWindow('frame', 100, 100)

while(True):
    # Capture frame-by-frame
    ret, frame = cap.read()

    # rotate frame
    # rotation_matrix = cv2.getRotationMatrix2D((width / 2, height / 2), 90, 1)
    # rotated = cv2.warpAffine(frame, rotation_matrix, (width, height))

    # Our operations on the frame come here
    # gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    # Display the resulting frame
    cv2.imshow('frame', frame)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

# When everything done, release the capture
cap.release()
cv2.destroyAllWindows()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.contrib.auth.models import User
from django.test import TestCase, skipIfDBFeature
from django.test.client import Client
from django.urls import reverse, resolve
from rest_framework import status
from rest_framework.test import force_authenticate

from container.models import ContainerFamily, ContainerApp, Container, ContainerRun
from kive.tests import BaseTestCases

@skipIfDBFeature('is_mocked')
class ContainerAppTests(TestCase):
    def test_default_app_empty(self):
        user = User.objects.first()
        family = ContainerFamily.objects.create(user=user)
        container = Container.objects.create(family=family, user=user)
        client = Client()
        client.force_login(user)
        response = client.post(reverse('container_app_add',
                                       kwargs=dict(container_id=container.id)),
                               {})
        if response.status_code != 302:
            self.assertEqual({}, response.context['form'].errors)

    def test_add(self):
        user = User.objects.first()
        family = ContainerFamily.objects.create(user=user)
        container = Container.objects.create(family=family, user=user)
        client = Client()
        client.force_login(user)
        expected_inputs = 'names_csv greetings_csv'
        expected_outputs = 'messages_csv'

        response = client.post(reverse('container_app_add',
                                       kwargs=dict(container_id=container.id)),
                               dict(inputs=expected_inputs,
                                    outputs=expected_outputs))

        if response.status_code != 302:
            self.assertEqual({}, response.context['form'].errors)
        app, = container.apps.all()
        self.assertEqual(expected_inputs, app.inputs)
        self.assertEqual(expected_outputs, app.outputs)

    def test_write_inputs_replaces(self):
        user = User.objects.first()
        family = ContainerFamily.objects.create(user=user)
        container = Container.objects.create(family=family, user=user)
        app = container.apps.create()

        old_inputs = 'old_in_csv'
        expected_outputs = 'old_out_csv'
        expected_inputs = '--greetings_csv* -- names_csv'
        app.write_inputs(old_inputs)
        app.write_outputs(expected_outputs)

        app.write_inputs(expected_inputs)
        inputs = app.inputs
        outputs = app.outputs

        self.assertEqual(expected_inputs, inputs)
        self.assertEqual(expected_outputs, outputs)

@skipIfDBFeature('is_mocked')
class ContainerAppApiTests(BaseTestCases.ApiTestCase):
    def setUp(self):
        super(ContainerAppApiTests, self).setUp()
        user = User.objects.first()
        family = ContainerFamily.objects.create(user=user)
        container = Container.objects.create(family=family, user=user)
        self.test_app = ContainerApp.objects.create(container=container,
                                                    name='test')

        self.list_path = reverse("containerapp-list")
        self.list_view, _, _ = resolve(self.list_path)

        self.detail_pk = self.test_app.pk
        self.detail_path = reverse("containerapp-detail",
                                   kwargs={'pk': self.detail_pk})
        self.removal_path = reverse("containerapp-removal-plan",
                                    kwargs={'pk': self.detail_pk})

        self.detail_view, _, _ = resolve(self.detail_path)
        self.removal_view, _, _ = resolve(self.removal_path)

        self.container_path = reverse("container-detail",
                                      kwargs={'pk': container.pk})

    def test_add(self):
        request1 = self.factory.get(self.list_path)
        force_authenticate(request1, user=self.kive_user)
        start_count = len(self.list_view(request1).data)
        expected_inputs = 'alpha bravo'
        expected_outputs = 'charlie delta'

        request2 = self.factory.post(self.list_path,
                                     dict(name="zoo app",
                                          container=self.container_path,
                                          description='A really cool app',
                                          inputs=expected_inputs,
                                          outputs=expected_outputs))

        force_authenticate(request2, user=self.kive_user)
        resp = self.list_view(request2).render().data

        self.assertIn('id', resp)
        self.assertEquals(resp['name'], "zoo app")

        request3 = self.factory.get(self.list_path)
        force_authenticate(request3, user=self.kive_user)
        resp = self.list_view(request3).data

        self.assertEquals(len(resp), start_count + 1)
        self.assertEquals(resp[-1]['description'], "A really cool app")
        self.assertEquals(resp[-1]['inputs'], expected_inputs)
        self.assertEquals(resp[-1]['outputs'], expected_outputs)

    def test_removal_plan(self):
        request = self.factory.get(self.removal_path)
        force_authenticate(request, user=self.kive_user)
        response = self.removal_view(request, pk=self.detail_pk)

        self.assertEquals(response.data['ContainerApps'], 1)

    def test_removal(self):
        start_count = ContainerApp.objects.all().count()

        request = self.factory.delete(self.detail_path)
        force_authenticate(request, user=self.kive_user)
        response = self.detail_view(request, pk=self.detail_pk)
        self.assertEquals(response.status_code, status.HTTP_204_NO_CONTENT)

        end_count = ContainerApp.objects.all().count()
        self.assertEquals(end_count, start_count - 1)

@skipIfDBFeature('is_mocked')
class ContainerRunTests(TestCase):
    def test(self):
        user = User.objects.first()
        family = ContainerFamily.objects.create(user=user)
        container = Container.objects.create(family=family, user=user)
        app = ContainerApp.objects.create(container=container, name='test')
        client = Client()
        client.force_login(user)
        start_count = ContainerRun.objects.count()
        expected_count = start_count + 1

        response = client.post(reverse('container_run_add'),
                               data=dict(app=app.id))

        end_count = ContainerRun.objects.count()

        if response.status_code != 302:
            self.assertEqual({}, response.context['form'].errors)
        self.assertEqual(expected_count, end_count)
<commit_msg>Add required fields to test data for #739.<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.contrib.auth.models import User
from django.test import TestCase, skipIfDBFeature
from django.test.client import Client
from django.urls import reverse, resolve
from rest_framework import status
from rest_framework.test import force_authenticate

from container.models import ContainerFamily, ContainerApp, Container, ContainerRun
from kive.tests import BaseTestCases

@skipIfDBFeature('is_mocked')
class ContainerAppTests(TestCase):
    def test_default_app_empty(self):
        user = User.objects.first()
        family = ContainerFamily.objects.create(user=user)
        container = Container.objects.create(family=family, user=user)
        client = Client()
        client.force_login(user)
        response = client.post(reverse('container_app_add',
                                       kwargs=dict(container_id=container.id)),
                               dict(threads=1, memory=100))
        if response.status_code != 302:
            self.assertEqual({}, response.context['form'].errors)

    def test_add(self):
        user = User.objects.first()
        family = ContainerFamily.objects.create(user=user)
        container = Container.objects.create(family=family, user=user)
        client = Client()
        client.force_login(user)
        expected_inputs = 'names_csv greetings_csv'
        expected_outputs = 'messages_csv'

        response = client.post(reverse('container_app_add',
                                       kwargs=dict(container_id=container.id)),
                               dict(threads=1,
                                    memory=100,
                                    inputs=expected_inputs,
                                    outputs=expected_outputs))

        if response.status_code != 302:
            self.assertEqual({}, response.context['form'].errors)
        app, = container.apps.all()
        self.assertEqual(expected_inputs, app.inputs)
        self.assertEqual(expected_outputs, app.outputs)

    def test_write_inputs_replaces(self):
        user = User.objects.first()
        family = ContainerFamily.objects.create(user=user)
        container = Container.objects.create(family=family, user=user)
        app = container.apps.create()

        old_inputs = 'old_in_csv'
        expected_outputs = 'old_out_csv'
        expected_inputs = '--greetings_csv* -- names_csv'
        app.write_inputs(old_inputs)
        app.write_outputs(expected_outputs)

        app.write_inputs(expected_inputs)
        inputs = app.inputs
        outputs = app.outputs

        self.assertEqual(expected_inputs, inputs)
        self.assertEqual(expected_outputs, outputs)

@skipIfDBFeature('is_mocked')
class ContainerAppApiTests(BaseTestCases.ApiTestCase):
    def setUp(self):
        super(ContainerAppApiTests, self).setUp()
        user = User.objects.first()
        family = ContainerFamily.objects.create(user=user)
        container = Container.objects.create(family=family, user=user)
        self.test_app = ContainerApp.objects.create(container=container,
                                                    name='test')

        self.list_path = reverse("containerapp-list")
        self.list_view, _, _ = resolve(self.list_path)

        self.detail_pk = self.test_app.pk
        self.detail_path = reverse("containerapp-detail",
                                   kwargs={'pk': self.detail_pk})
        self.removal_path = reverse("containerapp-removal-plan",
                                    kwargs={'pk': self.detail_pk})

        self.detail_view, _, _ = resolve(self.detail_path)
        self.removal_view, _, _ = resolve(self.removal_path)

        self.container_path = reverse("container-detail",
                                      kwargs={'pk': container.pk})

    def test_add(self):
        request1 = self.factory.get(self.list_path)
        force_authenticate(request1, user=self.kive_user)
        start_count = len(self.list_view(request1).data)
        expected_inputs = 'alpha bravo'
        expected_outputs = 'charlie delta'

        request2 = self.factory.post(self.list_path,
                                     dict(name="zoo app",
                                          container=self.container_path,
                                          description='A really cool app',
                                          inputs=expected_inputs,
                                          outputs=expected_outputs))

        force_authenticate(request2, user=self.kive_user)
        resp = self.list_view(request2).render().data

        self.assertIn('id', resp)
        self.assertEquals(resp['name'], "zoo app")

        request3 = self.factory.get(self.list_path)
        force_authenticate(request3, user=self.kive_user)
        resp = self.list_view(request3).data

        self.assertEquals(len(resp), start_count + 1)
        self.assertEquals(resp[-1]['description'], "A really cool app")
        self.assertEquals(resp[-1]['inputs'], expected_inputs)
        self.assertEquals(resp[-1]['outputs'], expected_outputs)

    def test_removal_plan(self):
        request = self.factory.get(self.removal_path)
        force_authenticate(request, user=self.kive_user)
        response = self.removal_view(request, pk=self.detail_pk)

        self.assertEquals(response.data['ContainerApps'], 1)

    def test_removal(self):
        start_count = ContainerApp.objects.all().count()

        request = self.factory.delete(self.detail_path)
        force_authenticate(request, user=self.kive_user)
        response = self.detail_view(request, pk=self.detail_pk)
        self.assertEquals(response.status_code, status.HTTP_204_NO_CONTENT)

        end_count = ContainerApp.objects.all().count()
        self.assertEquals(end_count, start_count - 1)

@skipIfDBFeature('is_mocked')
class ContainerRunTests(TestCase):
    def test(self):
        user = User.objects.first()
        family = ContainerFamily.objects.create(user=user)
        container = Container.objects.create(family=family, user=user)
        app = ContainerApp.objects.create(container=container, name='test')
        client = Client()
        client.force_login(user)
        start_count = ContainerRun.objects.count()
        expected_count = start_count + 1

        response = client.post(reverse('container_run_add'),
                               data=dict(app=app.id))

        end_count = ContainerRun.objects.count()

        if response.status_code != 302:
            self.assertEqual({}, response.context['form'].errors)
        self.assertEqual(expected_count, end_count)
<|endoftext|>


<commit_before>import sys
import re

def main_find_report(args, argparser):

    if args.infile.isatty() or args.target is None:
        argparser.print_help()
        sys.exit()

    arg_ref = args.target
    infile = args.infile

    variants = {}
    samples = {}
    data = {}

    # BE 1-based
    nts = [-1]
    ref_seq = []
    chro = None
    if arg_ref:
        # CODE for multiple >LOC lines:
        for line in open(arg_ref, "r"):
            line = line.strip()
            if line[0] == '>':
                exon = line.strip(">")
                chro, pos = exon.split(":")
                refstart, refstop = pos.split("-")
                for i in xrange(int(refstart), int(refstop) + 1):
                    nts += [i]
        ref_seq = "X" + ''.join(ref_seq)

    print "\t".join(['Sample', 'Region', 'Location', 'Type', 'Removed',
                     'Added', 'Abnormal', 'Normal', 'Ratio', 'Min coverage',
                     'Variant', 'Target', 'Info', 'Variant sequence',
                     'Reference sequence'])

    for line in infile:
        # filter header
        if line[0] == "#":
            # sys.stderr.write("Filtred: " + line)
            continue

        # filter on info column
        if not re.search(args.info, line):
            # sys.stderr.write("Filtred: " + line)
            continue

        tok = line.strip("\n").split("\t")
        if len(tok) > 1 and tok[0] != "Database":
            pathvals = {
                'project': '',
                'sample': tok[0]
            }
            samp = (pathvals['project'], pathvals['sample'])

            variant = (tok[2], tok[3])
            if variant not in variants:
                variants[variant] = 0
            variants[variant] += 1

            if samp not in samples:
                samples[samp] = set()
            samples[samp].add(variant)

            if samp not in data:
                data[samp] = {}

            data[samp][variant] = tok

            query = tok[1]
            ratio = tok[4]
            alt_ratio = tok[5]
            ref_ratio = tok[9]
            min_cov = tok[6]
            alt_seq = tok[7]
            refSeq = tok[10]

            if int(min_cov) <= args.min_cov:
                continue

            if variant[0] == 'Reference':
                print "\t".join([samp[1], samp[0], '-', variant[0], '0', '0',
                                 '0.0', alt_ratio, tok[4], min_cov, '-', query,
                                 tok[-1], "", ""])
                continue

            start, mod, stop = variant[1].split(":")
            delet, insert = mod.split("/")
            # sys.stderr.write("start: " + str(start) + "\n")
            # sys.stderr.write("stop: " + str(stop) + "\n")
            # sys.stderr.write("delet: " + str(delet) + "\n")
            # sys.stderr.write("insert: " + str(insert) + "\n")

            if int(start) == len(nts) or int(stop) == len(nts):
                sys.stderr.write("WARNING: Mutation point outside reference range")
                sys.stderr.write("WARNING: " + "\t".join([samp[1], samp[0],
                                                          chro + ":" + str(nts[int(stop)-1] + 1),
                                                          variant[0],
                                                          str(len(delet)),
                                                          str(len(insert)),
                                                          alt_ratio,
                                                          ref_ratio, tok[4],
                                                          mod, query]))
                print "\t".join([samp[1], samp[0],
                                 chro + ":" + str(nts[int(stop)-1]+1),
                                 variant[0], str(len(delet)), str(len(insert)),
                                 alt_ratio, ref_ratio, ratio, min_cov, mod,
                                 query, tok[-1], tok[7], refSeq])
            else:
                # Reinterpret mutation for small ITD.
                # INSERTIONS
                if len(delet) == 0 and len(insert) != 0:
                    pos = int(start)-1
                    upstream = alt_seq[pos-len(insert):pos]
                    match = 0
                    # careful, going upstream may put us outside the reference.
                    if pos-len(insert) >= 0:
                        for i in xrange(0, len(insert)):
                            if insert[i] == upstream[i]:
                                match += 1
                        match = float(match)/len(insert)

                    insert_type = "Insertion"
                    region = chro + ":" + str(nts[int(pos-1)]) + "-" + str(nts[int(stop)-1])
                    added = str(len(insert))
                    if pos-len(insert) >= 0 and len(insert) >= 3 and insert == upstream:
                        insert_type = "ITD"
                        region = chro + ":" + str(nts[int(pos-len(insert)+1)]) + "-" + str(nts[int(stop)-1])
                        added += " | " + str(nts[int(stop)-1] - nts[int(pos-len(insert)+1)] + 1)
                    elif pos-len(insert) >= 0 and len(insert) >= 3 and match > 0.5:
                        insert_type = "I&I"
                        region = chro + ":" + str(nts[int(pos-len(insert)+1)]) + "-" + str(nts[int(stop)-1])
                        added += " | " + str(nts[int(stop)-1] - nts[int(pos-len(insert)+1)] + 1)

                    print "\t".join([samp[1],
                                     region,
                                     chro + ":" + str(nts[int(stop)]),
                                     insert_type, str(len(delet)),
                                     added,
                                     alt_ratio, ref_ratio, ratio, min_cov, mod,
                                     query, tok[-1], alt_seq, refSeq])

                elif variant[0] == 'Deletion':
                    print "\t".join([samp[1],
                                     chro + ":" + str(nts[int(start)-1]+1) + "-" + str(nts[int(stop)-1]),
                                     "", variant[0], str(len(delet)),
                                     str(len(insert)), alt_ratio, ref_ratio,
                                     ratio, min_cov, mod, query, tok[-1],
                                     alt_seq, refSeq])
                # SNP
                elif variant[0] == 'Substitution':
                    print "\t".join([samp[1],
                                     chro + ":" + str(nts[int(start)-1]+1) + "-" + str(nts[int(stop)-1]),
                                     chro + ":" + str(nts[int(stop)-1]),
                                     variant[0], str(len(delet)),
                                     str(len(insert)), alt_ratio, ref_ratio,
                                     ratio, min_cov, mod, query, tok[-1],
                                     alt_seq, refSeq])
                else:
                    print "\t".join([samp[1],
                                     chro + ":" + str(nts[int(start)-1]+1) + "-" + str(nts[int(stop)-1]),
                                     chro + ":" + str(nts[int(stop)]),
                                     variant[0], str(len(delet)),
                                     str(len(insert)), alt_ratio, ref_ratio,
                                     ratio, min_cov, mod, query, tok[-1],
                                     alt_seq, refSeq])
<commit_msg>cleaning<commit_after>import sys
import re

def init_ref_seq(arg_ref):
    # BE 1-based
    nts = [-1]
    ref_seq = []
    chro = None
    if arg_ref:
        # CODE for multiple >LOC lines:
        for line in open(arg_ref, "r"):
            line = line.strip()
            if line[0] == '>':
                exon = line.strip(">")
                chro, pos = exon.split(":")
                refstart, refstop = pos.split("-")
                for i in xrange(int(refstart), int(refstop) + 1):
                    nts += [i]
        ref_seq = "X" + ''.join(ref_seq)

    return(nts, ref_seq, chro)

def create_report(arg_ref, infile, args_info, args_min_cov):
    variants = {}
    samples = {}
    data = {}

    (nts, ref_seq, chro) = init_ref_seq(arg_ref)

    print "\t".join(['Sample', 'Region', 'Location', 'Type', 'Removed',
                     'Added', 'Abnormal', 'Normal', 'Ratio', 'Min coverage',
                     'Variant', 'Target', 'Info', 'Variant sequence',
                     'Reference sequence'])

    for line in infile:
        # filter header
        if line[0] == "#":
            # sys.stderr.write("Filtred: " + line)
            continue

        # filter on info column
        if not re.search(args_info, line):
            # sys.stderr.write("Filtred: " + line)
            continue

        tok = line.strip("\n").split("\t")
        if len(tok) > 1 and tok[0] != "Database":
            pathvals = {
                'project': '',
                'sample': tok[0]
            }
            samp = (pathvals['project'], pathvals['sample'])

            variant = (tok[2], tok[3])
            if variant not in variants:
                variants[variant] = 0
            variants[variant] += 1

            if samp not in samples:
                samples[samp] = set()
            samples[samp].add(variant)

            if samp not in data:
                data[samp] = {}

            data[samp][variant] = tok

            query = tok[1]
            ratio = tok[4]
            alt_ratio = tok[5]
            ref_ratio = tok[9]
            min_cov = tok[6]
            alt_seq = tok[7]
            refSeq = tok[10]

            if int(min_cov) <= args_min_cov:
                continue

            if variant[0] == 'Reference':
                print "\t".join([samp[1], samp[0], '-', variant[0], '0', '0',
                                 '0.0', alt_ratio, tok[4], min_cov, '-', query,
                                 tok[-1], "", ""])
                continue

            start, mod, stop = variant[1].split(":")
            delet, insert = mod.split("/")

            if int(start) == len(nts) or int(stop) == len(nts):
                sys.stderr.write("WARNING: Mutation point outside reference range")
                sys.stderr.write("WARNING: " + "\t".join([samp[1], samp[0],
                                                          chro + ":" + str(nts[int(stop)-1] + 1),
                                                          variant[0],
                                                          str(len(delet)),
                                                          str(len(insert)),
                                                          alt_ratio,
                                                          ref_ratio, tok[4],
                                                          mod, query]))
                print "\t".join([samp[1], samp[0],
                                 chro + ":" + str(nts[int(stop)-1]+1),
                                 variant[0], str(len(delet)), str(len(insert)),
                                 alt_ratio, ref_ratio, ratio, min_cov, mod,
                                 query, tok[-1], tok[7], refSeq])
            else:
                # Reinterpret mutation for small ITD.
                # INSERTIONS
                if len(delet) == 0 and len(insert) != 0:
                    pos = int(start)-1
                    upstream = alt_seq[pos-len(insert):pos]
                    match = 0
                    # careful, going upstream may put us outside the reference.
                    if pos-len(insert) >= 0:
                        for i in xrange(0, len(insert)):
                            if insert[i] == upstream[i]:
                                match += 1
                        match = float(match)/len(insert)

                    insert_type = "Insertion"
                    region = chro + ":" + str(nts[int(pos-1)]) + "-" + str(nts[int(stop)-1])
                    added = str(len(insert))
                    if pos-len(insert) >= 0 and len(insert) >= 3 and insert == upstream:
                        insert_type = "ITD"
                        region = chro + ":" + str(nts[int(pos-len(insert)+1)]) + "-" + str(nts[int(stop)-1])
                        added += " | " + str(nts[int(stop)-1] - nts[int(pos-len(insert)+1)] + 1)
                    elif pos-len(insert) >= 0 and len(insert) >= 3 and match > 0.5:
                        insert_type = "I&I"
                        region = chro + ":" + str(nts[int(pos-len(insert)+1)]) + "-" + str(nts[int(stop)-1])
                        added += " | " + str(nts[int(stop)-1] - nts[int(pos-len(insert)+1)] + 1)

                    print "\t".join([samp[1],
                                     region,
                                     chro + ":" + str(nts[int(stop)]),
                                     insert_type, str(len(delet)),
                                     added,
                                     alt_ratio, ref_ratio, ratio, min_cov, mod,
                                     query, tok[-1], alt_seq, refSeq])

                elif variant[0] == 'Deletion':
                    print "\t".join([samp[1],
                                     chro + ":" + str(nts[int(start)-1]+1) + "-" + str(nts[int(stop)-1]),
                                     "", variant[0], str(len(delet)),
                                     str(len(insert)), alt_ratio, ref_ratio,
                                     ratio, min_cov, mod, query, tok[-1],
                                     alt_seq, refSeq])
                # SNP
                elif variant[0] == 'Substitution':
                    print "\t".join([samp[1],
                                     chro + ":" + str(nts[int(start)-1]+1) + "-" + str(nts[int(stop)-1]),
                                     chro + ":" + str(nts[int(stop)-1]),
                                     variant[0], str(len(delet)),
                                     str(len(insert)), alt_ratio, ref_ratio,
                                     ratio, min_cov, mod, query, tok[-1],
                                     alt_seq, refSeq])
                else:
                    print "\t".join([samp[1],
                                     chro + ":" + str(nts[int(start)-1]+1) + "-" + str(nts[int(stop)-1]),
                                     chro + ":" + str(nts[int(stop)]),
                                     variant[0], str(len(delet)),
                                     str(len(insert)), alt_ratio, ref_ratio,
                                     ratio, min_cov, mod, query, tok[-1],
                                     alt_seq, refSeq])

def main_find_report(args, argparser):

    if args.infile.isatty() or args.target is None:
        argparser.print_help()
        sys.exit()

    create_report(args.target, args.infile, args.info, args.min_cov)
<|endoftext|>


<commit_before># coding=utf-8
"""
Show the km3pipe etc. version used to write a H5 file.

Usage:
    h5info FILE [-r]
    h5info (-h | --help)
    h5info --version

Options:
    FILE        Input file.
    -r --raw    Dump raw metadata.
    -h --help   Show this screen.

"""
from __future__ import division, absolute_import, print_function

import tables

__author__ = "Moritz Lotze"
__copyright__ = "Copyright 2016, Moritz Lotze and the KM3NeT collaboration."
__credits__ = []
__license__ = "MIT"
__maintainer__ = "Moritz Lotze"
__email__ = "mlotze@km3net.de"
__status__ = "Development"

def h5info(h5name, raw=False):
    with tables.open_file(h5name) as h5:
        att_list = h5.root._v_attrs._f_list()
        if raw:
            print(h5.root._v_attrs.__repr__())
            return
        if not att_list and not raw:
            print("Sorry, no metadata.")
        for att in h5.root._v_attrs._f_list():
            ver = h5.root._v_attrs[att]
            print("{}: {}".format(att, ver))

def main():
    from docopt import docopt
    arguments = docopt(__doc__)

    h5info(arguments['FILE'], arguments['--raw'])
<commit_msg>Print header<commit_after># coding=utf-8
"""
Show the km3pipe etc. version used to write a H5 file.

Usage:
    h5info FILE [-r]
    h5info (-h | --help)
    h5info --version

Options:
    FILE        Input file.
    -r --raw    Dump raw metadata.
    -h --help   Show this screen.

"""
from __future__ import division, absolute_import, print_function

import tables

__author__ = "Moritz Lotze"
__copyright__ = "Copyright 2016, Moritz Lotze and the KM3NeT collaboration."
__credits__ = []
__license__ = "MIT"
__maintainer__ = "Moritz Lotze"
__email__ = "mlotze@km3net.de"
__status__ = "Development"

def h5info(h5name, raw=False):
    with tables.open_file(h5name) as h5:
        att_list = h5.root._v_attrs._f_list()
        if raw:
            print("HDF5 Meta Data")
            print("--------------")
            print(h5.root._v_attrs.__repr__())
            if '/header' in h5:
                print("\nHeader ('/header')")
                print("----------------")
                print(h5.get_node('/header')._v_attrs.__repr__())
            return
        if not att_list and not raw:
            print("Sorry, no metadata.")
        else:
            print("HDF5 Meta Data")
            print("--------------")
        for att in h5.root._v_attrs._f_list():
            ver = h5.root._v_attrs[att]
            print("{}: {}".format(att, ver))
        if '/header' in h5:
            print("\nHeader (/header)")
            print("----------------")
            node = h5.get_node('/header')
            for att in node._v_attrs._f_list():
                ver = node._v_attrs[att]
                print("{}: {}".format(att, ver))

def main():
    from docopt import docopt
    arguments = docopt(__doc__)

    h5info(arguments['FILE'], arguments['--raw'])
<|endoftext|>


<commit_before># No shebang line, this module is meant to be imported
#
# Copyright 2013 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import re
import netaddr
from textwrap import dedent
from sqlalchemy.schema import UniqueConstraint
from sqlalchemy.orm import validates

from pyfarm.flaskapp import db
from pyfarm.ext.config.core.loader import Loader
from pyfarm.ext.config.enum import AgentState
from pyfarm.ext.system.network import IP_NONNETWORK
from pyfarm.models.mixins import WorkValidationMixin
from pyfarm.models.core.types import IDColumn, IDType
from pyfarm.models.core.cfg import (
    DBCFG, TABLE_AGENT, TABLE_AGENT_TAGS, TABLE_AGENT_SOFTWARE)

REGEX_HOSTNAME = re.compile("^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*"
                            "[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9]"
                            "[A-Za-z0-9\-]*[A-Za-z0-9])$")

class AgentTaggingMixin(object):
    """
    Mixin used which provides some common structures to
    :class:`.AgentTagsModel` and :class:`.AgentSoftwareModel`
    """
    @validates("tag", "software")
    def validate_string_column(self, key, value):
        """
        Ensures `value` is a string or something that can be converted
        to a string.
        """
        if isinstance(value, (int, long)):
            value = str(value)
        elif not isinstance(value, basestring):
            raise ValueError("expected a string for `%s`" % key)

        return value

class AgentTagsModel(db.Model, AgentTaggingMixin):
    """
    Table model used to store tags for an agent.

    .. note::
        This table enforces two forms of uniqueness.  The :attr:`id` column
        must be unique and the combination of these columns must also be
        unique to limit the frequency of duplicate data:

            * :attr:`_agentid`
            * :attr:`tag`

    .. autoattribute:: _agentid
    """
    __tablename__ = TABLE_AGENT_TAGS
    __table_args__ = (UniqueConstraint("_agentid", "tag"), )
    id = IDColumn()
    _agentid = db.Column(IDType, db.ForeignKey("%s.id" % TABLE_AGENT),
                         doc=dedent("""
                         The foreign key which stores :attr:`AgentModel.id`"""))
    tag = db.Column(db.String,
                    doc=dedent("""
                    A string value to tag an :class:`.AgentModel`. Generally
                    this value is used for grouping like resources together
                    on the network but could also be used by jobs as a sort of
                    requirement."""))

class AgentSoftwareModel(db.Model, AgentTaggingMixin):
    """
    Stores information about an the software installed on
    an agent.

    .. note::
        This table enforces two forms of uniqueness.  The :attr:`id` column
        must be unique and the combination of these columns must also be
        unique to limit the frequency of duplicate data:

            * :attr:`_agentid`
            * :attr:`version`
            * :attr:`software`

    .. autoattribute:: _agentid
    """
    __tablename__ = TABLE_AGENT_SOFTWARE
    __table_args__ = (UniqueConstraint("_agentid", "version", "software"), )
    id = IDColumn()
    _agentid = db.Column(IDType, db.ForeignKey("%s.id" % TABLE_AGENT),
                         doc=dedent("""
                         The foreign key which stores :attr:`AgentModel.id`"""))
    software = db.Column(db.String, nullable=False,
                         doc=dedent("""
                         The name of the software installed.  No normalization
                         is performed prior to being stored in the database"""))
    version = db.Column(db.String, default="any", nullable=False,
                        doc=dedent("""
                        The version of the software installed on a host.  This
                        value does not follow any special formatting rules
                        because the format depends on the 3rd party."""))

class AgentModel(db.Model, WorkValidationMixin):
    """
    Stores information about an agent include its network address,
    state, allocation configuration, etc.

    .. note::
        This table enforces two forms of uniqueness.  The :attr:`id` column
        must be unique and the combination of these columns must also be
        unique to limit the frequency of duplicate data:

            * :attr:`hostname`
            * :attr:`ip`
            * :attr:`subnet`
            * :attr:`port`

    """
    __tablename__ = TABLE_AGENT
    __table_args__ = (UniqueConstraint("hostname", "ip", "subnet", "port"), )
    STATE_ENUM = AgentState()
    STATE_DEFAULT = STATE_ENUM.ONLINE
    id = IDColumn()

    # basic host attribute information
    hostname = db.Column(db.String, nullable=False,
                         doc=dedent("""
                         The hostname we should use to talk to this host.
                         Preferably this value will be the fully qualified
                         name instead of the base hostname alone."""))
    ip = db.Column(db.String(15), nullable=False,
                   doc=dedent("""
                   The IPv4 network address this host resides on.  This is
                   'best guess' address using factors described in
                   :meth:`pyfarm.system.network.NetworkInfo.ip`"""))
    subnet = db.Column(db.String(15), nullable=False,
                       doc=dedent("""
                       The subnet associated with the agent at the time we
                       discovered :attr:`ip`"""))
    ram = db.Column(db.Integer, nullable=False,
                    doc=dedent("""
                    The amount of ram (in megabytes) installed on the agent.
                    This value is provided by
                    :attr:`pyfarm.system.memory.MemoryInfo.TOTAL_RAM`"""))
    cpus = db.Column(db.Integer, nullable=False,
                     doc=dedent("""
                     The number of cpus installed on the agent.  This value
                     is provided by
                     :attr:`pyfarm.system.processor.ProcessorInfo.NUM_CPUS`
                     """))
    port = db.Column(db.Integer, nullable=False,
                     doc=dedent("""
                     The port the agent is currently running on"""))

    # host state
    enabled = db.Column(db.Boolean, default=True, nullable=False,
                        doc=dedent("""
                        Tells the queue and various other parts of PyFarm if
                        this host is considered part of the pool.  Setting
                        this to True will prevent:

                        * new work from being sent to an agent
                        * failed job types from rerunning
                        * batch commands being picked up over the pub/sub
                          channel"""))
    state = db.Column(db.Integer, default=STATE_DEFAULT, nullable=False,
                      doc=dedent("""
                      Stores the current state of the host.  This value can be
                      changed either by a master telling the host to do
                      something with a task or from the host via REST api.

                      .. csv-table:: **Values (from enum.yml:AgentState)**
                          :header: Integer, Description
                          :widths: 10, 50

                          16,Offline - host is unreachable
                          17,Online - ready to receive work
                          18,Disabled - same as online but cannot receive work
                          19,Running - currently processing work"""))

    # Max allocation of the two primary resources which `1.0` is 100%
    # allocation.  For `cpu_allocation` 100% allocation typically means
    # one task per cpu.
    ram_allocation = db.Column(db.Float, nullable=False,
                               default=DBCFG.get("agent.ram_allocation", .8),
                               doc=dedent("""
                               The amount of ram the agent is allowed to
                               allocate towards work.  A value of `1.0` would
                               mean to let the agent use all of the memory
                               installed on the system when assigning work.

                               **configured by**: `agent.ram_allocation`"""))
    cpu_allocation = db.Column(db.Float, nullable=False,
                               default=DBCFG.get("agent.cpu_allocation", 1.0),
                               doc=dedent("""
                               The total amount of cpu space an agent is
                               allowed to process work in.  A value of `1.0`
                               would mean an agent can handle as much work
                               as the system could handle given the
                               requirements of a task.  For example if an agent
                               has 8 cpus, cpu_allocation is .5, and a task
                               requires 4 cpus then only that task will run
                               on the system.

                               **configured by**: `agent.cpu_allocation`"""))

    # relationships
    tasks = db.relationship("TaskModel", backref="agent", lazy="dynamic",
                            doc=dedent("""
                            Relationship between an :class:`AgentModel`
                            and any :class:`pyfarm.models.TaskModel`
                            objects"""))
    tags = db.relationship("AgentTagsModel", backref="agent", lazy="dynamic",
                           doc=dedent("""
                           Relationship between an :class:`AgentModel`
                           and any :class:`pyfarm.models.AgentTagsModel`
                           objects"""))
    software = db.relation("AgentSoftwareModel", backref="agent",
                           lazy="dynamic", doc=dedent("""
                           Relationship between an :class:`AgentModel`
                           and any :class:`pyfarm.models.AgentSoftwareModel`
                           objects"""))

    @validates("hostname")
    def validate_hostname(self, key, value):
        """
        Ensures that the hostname provided by `value` matches a regular
        expression that expresses what a valid hostname is.
        """
        # ensure hostname does not contain characters we can't use
        if not REGEX_HOSTNAME.match(value):
            raise ValueError("%s is not valid for %s" % (value, key))

        return value

    @validates("ip", "subnet")
    def validate_address(self, key, value):
        """
        Ensures the :attr:`ip` and :attr:`subnet` are valid.  For ip addresses
        this will make sure `value` is:

            * not a hostmask
            * not link local (:rfc:`3927`)
            * not used for multicast (:rfc:`1112`)
            * not a netmask (:rfc:`4632`)
            * not reserved (:rfc:`6052`)
            * a private address (:rfc:`1918`)

        This method will also value subnet masks to ensure their format is
        valid.
        """
        try:
            ip = netaddr.IPAddress(value)

        except ValueError, e:
            raise ValueError(
                "%s is not a valid address format: %s" % (value, e))

        if key == "ip":
            valid = all([
                not ip.is_hostmask(), not ip.is_link_local(),
                not ip.is_loopback(), not ip.is_multicast(),
                not ip.is_netmask(), ip.is_private(),
                not ip.is_reserved()
            ])
            if not valid:
                raise ValueError("%s it not a private ip address" % value)

        elif key == "subnet":
            valid = all([
                not ip.is_hostmask(), not ip.is_link_local(),
                not ip.is_loopback(), not ip.is_multicast(),
                ip.is_netmask(), not ip.is_private(),
                ip.is_reserved()
            ])
            if not valid:
                raise ValueError("%s is not valid subnet" % value)

        return value

    @validates("ram", "cpus", "port")
    def validate_resource(self, key, value):
        """
        Ensure the `value` provided for `key` is within an expected range as
        specified in `agent.yml`
        """
        if value is None:
            return value

        min_value = DBCFG.get("agent.min_%s" % key)
        max_value = DBCFG.get("agent.max_%s" % key)

        # quick sanity check of the incoming config
        assert isinstance(min_value, int), "db.min_%s must be an integer" % key
        assert isinstance(max_value, int), "db.max_%s must be an integer" % key
        assert min_value >= 1, "db.min_%s must be > 0" % key
        assert max_value >= 1, "db.max_%s must be > 0" % key

        # check the provided input
        if min_value > value or value > max_value:
            msg = "value for `%s` must be between " % key
            msg += "%s and %s" % (min_value, max_value)
            raise ValueError(msg)

        return value

class AgentTag(AgentTagsModel):
    """
    Provides :meth:`__init__` for :class:`.AgentTagsModel` so the model can
    be instanced with initial values.
    """
    def __init__(self, agent, tag):
        agentid = agent
        if isinstance(agent, AgentModel):
            agentid = agent.id

        self._agentid = agentid
        self.tag = tag

class AgentSoftware(AgentSoftwareModel):
    """
    Provides :meth:`__init__` for :class:`AgentSoftwareModel` so the model can
    be instanced with initial values.
    """
    def __init__(self, agent, software, version="any"):
        agentid = agent
        if isinstance(agent, AgentModel):
            agentid = agent.id

        self._agentid = agentid
        self.software = software
        self.version = version

class Agent(AgentModel):
    """
    Provides :meth:`__init__` for :class:`AgentModel` so the model can
    be instanced with initial values.
    """
    def __init__(self, hostname, ip, subnet, port, cpus, ram, state=None,
                 enabled=None, ram_allocation=None, cpu_allocation=None):
        self.hostname = hostname
        self.ip = ip
        self.subnet = subnet
        self.port = port
        self.cpus = cpus
        self.ram = ram

        if state is not None:
            self.state = state

        if enabled is not None:
            self.enabled = enabled

        if ram_allocation is not None:
            self.ram_allocation = ram_allocation

        if cpu_allocation is not None:
            self.cpu_allocation = cpu_allocation<commit_msg>swap to IPv4Address type<commit_after># No shebang line, this module is meant to be imported
#
# Copyright 2013 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import re
import netaddr
from textwrap import dedent
from sqlalchemy.schema import UniqueConstraint
from sqlalchemy.orm import validates

from pyfarm.flaskapp import db
from pyfarm.ext.config.core.loader import Loader
from pyfarm.ext.config.enum import AgentState
from pyfarm.ext.system.network import IP_NONNETWORK
from pyfarm.models.mixins import WorkValidationMixin
from pyfarm.models.core.types import IDColumn, IDType, IPv4Address
from pyfarm.models.core.cfg import (
    DBCFG, TABLE_AGENT, TABLE_AGENT_TAGS, TABLE_AGENT_SOFTWARE)

REGEX_HOSTNAME = re.compile("^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*"
                            "[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9]"
                            "[A-Za-z0-9\-]*[A-Za-z0-9])$")

class AgentTaggingMixin(object):
    """
    Mixin used which provides some common structures to
    :class:`.AgentTagsModel` and :class:`.AgentSoftwareModel`
    """
    @validates("tag", "software")
    def validate_string_column(self, key, value):
        """
        Ensures `value` is a string or something that can be converted
        to a string.
        """
        if isinstance(value, (int, long)):
            value = str(value)
        elif not isinstance(value, basestring):
            raise ValueError("expected a string for `%s`" % key)

        return value

class AgentTagsModel(db.Model, AgentTaggingMixin):
    """
    Table model used to store tags for an agent.

    .. note::
        This table enforces two forms of uniqueness.  The :attr:`id` column
        must be unique and the combination of these columns must also be
        unique to limit the frequency of duplicate data:

            * :attr:`_agentid`
            * :attr:`tag`

    .. autoattribute:: _agentid
    """
    __tablename__ = TABLE_AGENT_TAGS
    __table_args__ = (UniqueConstraint("_agentid", "tag"), )
    id = IDColumn()
    _agentid = db.Column(IDType, db.ForeignKey("%s.id" % TABLE_AGENT),
                         doc=dedent("""
                         The foreign key which stores :attr:`AgentModel.id`"""))
    tag = db.Column(db.String,
                    doc=dedent("""
                    A string value to tag an :class:`.AgentModel`. Generally
                    this value is used for grouping like resources together
                    on the network but could also be used by jobs as a sort of
                    requirement."""))

class AgentSoftwareModel(db.Model, AgentTaggingMixin):
    """
    Stores information about an the software installed on
    an agent.

    .. note::
        This table enforces two forms of uniqueness.  The :attr:`id` column
        must be unique and the combination of these columns must also be
        unique to limit the frequency of duplicate data:

            * :attr:`_agentid`
            * :attr:`version`
            * :attr:`software`

    .. autoattribute:: _agentid
    """
    __tablename__ = TABLE_AGENT_SOFTWARE
    __table_args__ = (UniqueConstraint("_agentid", "version", "software"), )
    id = IDColumn()
    _agentid = db.Column(IDType, db.ForeignKey("%s.id" % TABLE_AGENT),
                         doc=dedent("""
                         The foreign key which stores :attr:`AgentModel.id`"""))
    software = db.Column(db.String, nullable=False,
                         doc=dedent("""
                         The name of the software installed.  No normalization
                         is performed prior to being stored in the database"""))
    version = db.Column(db.String, default="any", nullable=False,
                        doc=dedent("""
                        The version of the software installed on a host.  This
                        value does not follow any special formatting rules
                        because the format depends on the 3rd party."""))

class AgentModel(db.Model, WorkValidationMixin):
    """
    Stores information about an agent include its network address,
    state, allocation configuration, etc.

    .. note::
        This table enforces two forms of uniqueness.  The :attr:`id` column
        must be unique and the combination of these columns must also be
        unique to limit the frequency of duplicate data:

            * :attr:`hostname`
            * :attr:`ip`
            * :attr:`subnet`
            * :attr:`port`

    """
    __tablename__ = TABLE_AGENT
    __table_args__ = (UniqueConstraint("hostname", "ip", "subnet", "port"), )
    STATE_ENUM = AgentState()
    STATE_DEFAULT = STATE_ENUM.ONLINE
    id = IDColumn()

    # basic host attribute information
    hostname = db.Column(db.String, nullable=False,
                         doc=dedent("""
                         The hostname we should use to talk to this host.
                         Preferably this value will be the fully qualified
                         name instead of the base hostname alone."""))
    ip = db.Column(IPv4Address, nullable=False,
                   doc=dedent("""
                   The IPv4 network address this host resides on.  This is
                   'best guess' address using factors described in
                   :meth:`pyfarm.system.network.NetworkInfo.ip`"""))
    subnet = db.Column(IPv4Address, nullable=False,
                       doc=dedent("""
                       The subnet associated with the agent at the time we
                       discovered :attr:`ip`"""))
    ram = db.Column(db.Integer, nullable=False,
                    doc=dedent("""
                    The amount of ram (in megabytes) installed on the agent.
                    This value is provided by
                    :attr:`pyfarm.system.memory.MemoryInfo.TOTAL_RAM`"""))
    cpus = db.Column(db.Integer, nullable=False,
                     doc=dedent("""
                     The number of cpus installed on the agent.  This value
                     is provided by
                     :attr:`pyfarm.system.processor.ProcessorInfo.NUM_CPUS`
                     """))
    port = db.Column(db.Integer, nullable=False,
                     doc=dedent("""
                     The port the agent is currently running on"""))

    # host state
    enabled = db.Column(db.Boolean, default=True, nullable=False,
                        doc=dedent("""
                        Tells the queue and various other parts of PyFarm if
                        this host is considered part of the pool.  Setting
                        this to True will prevent:

                        * new work from being sent to an agent
                        * failed job types from rerunning
                        * batch commands being picked up over the pub/sub
                          channel"""))
    state = db.Column(db.Integer, default=STATE_DEFAULT, nullable=False,
                      doc=dedent("""
                      Stores the current state of the host.  This value can be
                      changed either by a master telling the host to do
                      something with a task or from the host via REST api.

                      .. csv-table:: **Values (from enum.yml:AgentState)**
                          :header: Integer, Description
                          :widths: 10, 50

                          16,Offline - host is unreachable
                          17,Online - ready to receive work
                          18,Disabled - same as online but cannot receive work
                          19,Running - currently processing work"""))

    # Max allocation of the two primary resources which `1.0` is 100%
    # allocation.  For `cpu_allocation` 100% allocation typically means
    # one task per cpu.
    ram_allocation = db.Column(db.Float, nullable=False,
                               default=DBCFG.get("agent.ram_allocation", .8),
                               doc=dedent("""
                               The amount of ram the agent is allowed to
                               allocate towards work.  A value of `1.0` would
                               mean to let the agent use all of the memory
                               installed on the system when assigning work.

                               **configured by**: `agent.ram_allocation`"""))
    cpu_allocation = db.Column(db.Float, nullable=False,
                               default=DBCFG.get("agent.cpu_allocation", 1.0),
                               doc=dedent("""
                               The total amount of cpu space an agent is
                               allowed to process work in.  A value of `1.0`
                               would mean an agent can handle as much work
                               as the system could handle given the
                               requirements of a task.  For example if an agent
                               has 8 cpus, cpu_allocation is .5, and a task
                               requires 4 cpus then only that task will run
                               on the system.

                               **configured by**: `agent.cpu_allocation`"""))

    # relationships
    tasks = db.relationship("TaskModel", backref="agent", lazy="dynamic",
                            doc=dedent("""
                            Relationship between an :class:`AgentModel`
                            and any :class:`pyfarm.models.TaskModel`
                            objects"""))
    tags = db.relationship("AgentTagsModel", backref="agent", lazy="dynamic",
                           doc=dedent("""
                           Relationship between an :class:`AgentModel`
                           and any :class:`pyfarm.models.AgentTagsModel`
                           objects"""))
    software = db.relation("AgentSoftwareModel", backref="agent",
                           lazy="dynamic", doc=dedent("""
                           Relationship between an :class:`AgentModel`
                           and any :class:`pyfarm.models.AgentSoftwareModel`
                           objects"""))

    @validates("hostname")
    def validate_hostname(self, key, value):
        """
        Ensures that the hostname provided by `value` matches a regular
        expression that expresses what a valid hostname is.
        """
        # ensure hostname does not contain characters we can't use
        if not REGEX_HOSTNAME.match(value):
            raise ValueError("%s is not valid for %s" % (value, key))

        return value

    @validates("ip", "subnet")
    def validate_address(self, key, value):
        """
        Ensures the :attr:`ip` and :attr:`subnet` are valid.  For ip addresses
        this will make sure `value` is:

            * not a hostmask
            * not link local (:rfc:`3927`)
            * not used for multicast (:rfc:`1112`)
            * not a netmask (:rfc:`4632`)
            * not reserved (:rfc:`6052`)
            * a private address (:rfc:`1918`)

        This method will also value subnet masks to ensure their format is
        valid.
        """
        try:
            ip = netaddr.IPAddress(value)

        except ValueError, e:
            raise ValueError(
                "%s is not a valid address format: %s" % (value, e))

        if key == "ip":
            valid = all([
                not ip.is_hostmask(), not ip.is_link_local(),
                not ip.is_loopback(), not ip.is_multicast(),
                not ip.is_netmask(), ip.is_private(),
                not ip.is_reserved()
            ])
            if not valid:
                raise ValueError("%s it not a private ip address" % value)

        elif key == "subnet":
            valid = all([
                not ip.is_hostmask(), not ip.is_link_local(),
                not ip.is_loopback(), not ip.is_multicast(),
                ip.is_netmask(), not ip.is_private(),
                ip.is_reserved()
            ])
            if not valid:
                raise ValueError("%s is not valid subnet" % value)

        return value

    @validates("ram", "cpus", "port")
    def validate_resource(self, key, value):
        """
        Ensure the `value` provided for `key` is within an expected range as
        specified in `agent.yml`
        """
        if value is None:
            return value

        min_value = DBCFG.get("agent.min_%s" % key)
        max_value = DBCFG.get("agent.max_%s" % key)

        # quick sanity check of the incoming config
        assert isinstance(min_value, int), "db.min_%s must be an integer" % key
        assert isinstance(max_value, int), "db.max_%s must be an integer" % key
        assert min_value >= 1, "db.min_%s must be > 0" % key
        assert max_value >= 1, "db.max_%s must be > 0" % key

        # check the provided input
        if min_value > value or value > max_value:
            msg = "value for `%s` must be between " % key
            msg += "%s and %s" % (min_value, max_value)
            raise ValueError(msg)

        return value

class AgentTag(AgentTagsModel):
    """
    Provides :meth:`__init__` for :class:`.AgentTagsModel` so the model can
    be instanced with initial values.
    """
    def __init__(self, agent, tag):
        agentid = agent
        if isinstance(agent, AgentModel):
            agentid = agent.id

        self._agentid = agentid
        self.tag = tag

class AgentSoftware(AgentSoftwareModel):
    """
    Provides :meth:`__init__` for :class:`AgentSoftwareModel` so the model can
    be instanced with initial values.
    """
    def __init__(self, agent, software, version="any"):
        agentid = agent
        if isinstance(agent, AgentModel):
            agentid = agent.id

        self._agentid = agentid
        self.software = software
        self.version = version

class Agent(AgentModel):
    """
    Provides :meth:`__init__` for :class:`AgentModel` so the model can
    be instanced with initial values.
    """
    def __init__(self, hostname, ip, subnet, port, cpus, ram, state=None,
                 enabled=None, ram_allocation=None, cpu_allocation=None):
        self.hostname = hostname
        self.ip = ip
        self.subnet = subnet
        self.port = port
        self.cpus = cpus
        self.ram = ram

        if state is not None:
            self.state = state

        if enabled is not None:
            self.enabled = enabled

        if ram_allocation is not None:
            self.ram_allocation = ram_allocation

        if cpu_allocation is not None:
            self.cpu_allocation = cpu_allocation<|endoftext|>


<commit_before># No shebang line, this module is meant to be imported
#
# Copyright 2013 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from sqlalchemy.orm import validates

from pyfarm.flaskapp import db
from pyfarm.config.enum import HostState
from pyfarm.models.mixins import StateMixin, RandIdMixin
from pyfarm.models.constants import (MAX_HOSTNAME_LENGTH,
    MAX_IPV4_LENGTH, REGEX_IPV4, REGEX_HOSTNAME, TABLE_AGENT)

class Agent(db.Model, RandIdMixin, StateMixin):
    __tablename__ = TABLE_AGENT
    STATE_ENUM = HostState()

    # columns
    hostname = db.Column(db.String(MAX_HOSTNAME_LENGTH), nullable=False)
    ip = db.Column(db.String(MAX_IPV4_LENGTH), nullable=False)
    subnet = db.Column(db.String(MAX_IPV4_LENGTH), nullable=False)
    enabled = db.Column(db.Boolean, default=True)

    # relationships
    tasks = db.relationship("Task", backref="agent", lazy="dynamic")

    @validates("hostname")
    def validate_hostname(self, key, value):
        if not REGEX_HOSTNAME.match(value):
            raise ValueError("%s is not valid for %s" % (value, key))

        return value

    @validates("ip", "subnet")
    def validate_address(self, key, value):
        if not REGEX_IPV4.match(value):
            raise ValueError("%s is not valid for %s" % (value, key))

        return value<commit_msg>added cpus/ran/port/os columns<commit_after># No shebang line, this module is meant to be imported
#
# Copyright 2013 Oliver Palmer
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from sqlalchemy.orm import validates

from pyfarm.flaskapp import db
from pyfarm.config.enum import HostState, OperatingSystem
from pyfarm.models.mixins import StateMixin, RandIdMixin
from pyfarm.models.constants import (MAX_HOSTNAME_LENGTH,
    MAX_IPV4_LENGTH, REGEX_IPV4, REGEX_HOSTNAME, TABLE_AGENT,
    MIN_RAM_MB, MAX_RAM_MB, MIN_AGENT_PORT, MAX_AGENT_PORT)

class Agent(db.Model, RandIdMixin, StateMixin):
    __tablename__ = TABLE_AGENT
    STATE_ENUM = HostState()
    OS_ENUM = OperatingSystem()

    # columns
    enabled = db.Column(db.Boolean, default=True)
    hostname = db.Column(db.String(MAX_HOSTNAME_LENGTH), nullable=False)
    ip = db.Column(db.String(MAX_IPV4_LENGTH), nullable=False)
    port = db.Column(db.Integer, nullable=False)
    subnet = db.Column(db.String(MAX_IPV4_LENGTH), nullable=False)
    cpus = db.Column(db.Integer, nullable=False)
    ram = db.Column(db.Integer, nullable=False)
    os = db.Column(db.Integer, nullable=False)

    # relationships
    tasks = db.relationship("Task", backref="agent", lazy="dynamic")

    @validates("hostname")
    def validate_hostname(self, key, value):
        if not REGEX_HOSTNAME.match(value):
            raise ValueError("%s is not valid for %s" % (value, key))

        return value

    @validates("ip", "subnet")
    def validate_address(self, key, value):
        if not REGEX_IPV4.match(value):
            raise ValueError("%s is not valid for %s" % (value, key))

        return value

    @validates("cpus")
    def validate_cpus(self, key, value):
        if value < 1:
            raise ValueError("at least one cpu is required")

        return value

    @validates("ram")
    def validate_ram(self, key, value):
        if value < MIN_RAM_MB or value > MAX_RAM_MB:
            args = (MIN_RAM_MB, MAX_RAM_MB)
            raise ValueError("ram must be between %s and %s megabytes" % args)

        return value

    @validates("os")
    def validates_os(self, key, value):
        if value not in self.OS_ENUM:
            msg = "`%s` is not a valid operating system, " % value
            msg += "valid systems are %s" % self.OS_ENUM.values()
            raise ValueError(msg)

        return value

    @validates("port")
    def validates_port(self, key, value):
        if value < MIN_AGENT_PORT or value > MAX_AGENT_PORT:
            args = (MIN_AGENT_PORT, MAX_AGENT_PORT)
            msg = "port number must be between %s and %s" % args
            raise ValueError(msg)

        return value<|endoftext|>