<commit_before>from __future__ import division
from __future__ import print_function

import cea.plots.demand
import cea.plots.cache
import plotly.graph_objs as go
from plotly.offline import plot
import pandas as pd

from cea.plots.variable_naming import NAMING, LOGO, COLOR

__author__ = "Jimeno A. Fonseca"
__copyright__ = "Copyright 2018, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Jimeno A. Fonseca"]
__license__ = "MIT"
__version__ = "0.1"
__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

class EnergyDemandDistrictPlot(cea.plots.demand.DemandPlotBase):
    """Implement the energy-use plot"""
    name = "Energy Demand"

    def __init__(self, project, parameters, cache):
        super(EnergyDemandDistrictPlot, self).__init__(project, parameters, cache)
        self.analysis_fields = ["E_sys_MWhyr",
                                "Qhs_sys_MWhyr", "Qww_sys_MWhyr",
                                "Qcs_sys_MWhyr", 'Qcdata_sys_MWhyr', 'Qcre_sys_MWhyr']

    @property
    def layout(self):
        return go.Layout(barmode='stack',
                         yaxis=dict(title='Energy Demand [MWh/yr]', domain=[0.35, 1]),
                         xaxis=dict(title='Building Name'), showlegend=True)

    def calc_graph(self):
        graph = []
        analysis_fields = self.remove_unused_fields(self.data, self.analysis_fields)
        dataframe = self.data
        dataframe['total'] = dataframe[analysis_fields].sum(axis=1)
        dataframe.sort_values(by='total', ascending=False, inplace=True)
        dataframe.reset_index(inplace=True, drop=True)
        for field in analysis_fields:
            y = dataframe[field]
            name = NAMING[field]
            total_percent = (y / dataframe['total'] * 100).round(2).values
            total_percent_txt = ["(%.2f %%)" % x for x in total_percent]
            trace = go.Bar(x=dataframe["Name"], y=y, name=name, text=total_percent_txt, orientation='v',
                           marker=dict(color=COLOR[field]))
            graph.append(trace)

        return graph

    def calc_table(self):
        data_frame = self.data
        analysis_fields = self.remove_unused_fields(self.data, self.analysis_fields)
        median = data_frame[analysis_fields].median().round(2).tolist()
        total = data_frame[analysis_fields].sum().round(2).tolist()
        total_perc = [str(x) + " (" + str(round(x / sum(total) * 100, 1)) + " %)" for x in total]
        # calculate graph
        anchors = []
        load_names = []
        for field in analysis_fields:
            anchors.append(', '.join(calc_top_three_anchor_loads(data_frame, field)))
            load_names.append(NAMING[field] + ' (' + field.split('_', 1)[0] + ')')

        column_names = ['Load Name', 'Total [MWh/yr]', 'Median [MWh/yr]', 'Top 3 Consumers']
        table_df = pd.DataFrame({'Load Name': load_names,
                              'Total [MWh/yr]': total_perc,
                              'Median [MWh/yr]': median,
                              'Top 3 Consumers': anchors}, columns=column_names)
        return table_df

def energy_demand_district(data_frame, analysis_fields, title, output_path):
    # CALCULATE GRAPH
    traces_graph = calc_graph(analysis_fields, data_frame)

    # CALCULATE TABLE
    traces_table = calc_table(analysis_fields, data_frame)

    # PLOT GRAPH
    traces_graph.append(traces_table)
    layout = go.Layout(images=LOGO, title=title, barmode='stack',
                       yaxis=dict(title='Energy [MWh/yr]', domain=[0.35, 1]),
                       xaxis=dict(title='Building Name'), showlegend=True)
    fig = go.Figure(data=traces_graph, layout=layout)
    plot(fig, auto_open=False, filename=output_path)

    return {'data': traces_graph, 'layout': layout}

def calc_table(analysis_fields, data_frame):
    median = data_frame[analysis_fields].median().round(2).tolist()
    total = data_frame[analysis_fields].sum().round(2).tolist()
    total_perc = [str(x) + " (" + str(round(x / sum(total) * 100, 1)) + " %)" for x in total]
    # calculate graph
    anchors = []
    load_names = []
    for field in analysis_fields:
        anchors.append(calc_top_three_anchor_loads(data_frame, field))
        load_names.append(NAMING[field] + ' (' + field.split('_', 1)[0] + ')')

    table = go.Table(domain=dict(x=[0, 1.0], y=[0, 0.2]),
                     header=dict(values=['Load Name', 'Total [MWh/yr]', 'Median [MWh/yr]', 'Top 3 Consumers']),
                     cells=dict(values=[load_names, total_perc, median, anchors]))

    return table

def calc_graph(analysis_fields, data):
    graph = []
    data['total'] = data[analysis_fields].sum(axis=1)
    data = data.sort_values(by='total', ascending=False)
    for field in analysis_fields:
        y = data[field]
        name = NAMING[field]
        total_percent = (y / data['total'] * 100).round(2).values
        total_percent_txt = ["(%.2f %%)" % x for x in total_percent]
        trace = go.Bar(x=data["Name"], y=y, name=name, text=total_percent_txt, orientation='v',
                       marker=dict(color=COLOR[field]))
        graph.append(trace)

    return graph

def calc_top_three_anchor_loads(data_frame, field):
    data_frame = data_frame.sort_values(by=field, ascending=False)
    anchor_list = data_frame[:3].Name.values
    return anchor_list

def main():
    import cea.config
    import cea.inputlocator
    config = cea.config.Configuration()
    locator = cea.inputlocator.InputLocator(config.scenario)
    # cache = cea.plots.cache.PlotCache(config.project)
    cache = cea.plots.cache.NullPlotCache()
    EnergyDemandDistrictPlot(config.project, {'buildings': None,
                                              'scenario-name': config.scenario_name},
                             cache).plot(auto_open=True)
    EnergyDemandDistrictPlot(config.project, {'buildings': locator.get_zone_building_names()[0:2],
                                              'scenario-name': config.scenario_name},
                             cache).plot(auto_open=True)
    EnergyDemandDistrictPlot(config.project, {'buildings': [locator.get_zone_building_names()[0]],
                                              'scenario-name': config.scenario_name},
                             cache).plot(auto_open=True)

if __name__ == '__main__':
    main()<commit_msg>fix to table<commit_after>from __future__ import division
from __future__ import print_function

import cea.plots.demand
import cea.plots.cache
import plotly.graph_objs as go
from plotly.offline import plot
import pandas as pd

from cea.plots.variable_naming import NAMING, LOGO, COLOR

__author__ = "Jimeno A. Fonseca"
__copyright__ = "Copyright 2018, Architecture and Building Systems - ETH Zurich"
__credits__ = ["Jimeno A. Fonseca"]
__license__ = "MIT"
__version__ = "0.1"
__maintainer__ = "Daren Thomas"
__email__ = "cea@arch.ethz.ch"
__status__ = "Production"

class EnergyDemandDistrictPlot(cea.plots.demand.DemandPlotBase):
    """Implement the energy-use plot"""
    name = "Energy Demand"

    def __init__(self, project, parameters, cache):
        super(EnergyDemandDistrictPlot, self).__init__(project, parameters, cache)
        self.analysis_fields = ["E_sys_MWhyr",
                                "Qhs_sys_MWhyr", "Qww_sys_MWhyr",
                                "Qcs_sys_MWhyr", 'Qcdata_sys_MWhyr', 'Qcre_sys_MWhyr']

    @property
    def layout(self):
        return go.Layout(barmode='stack',
                         yaxis=dict(title='Energy Demand [MWh/yr]', domain=[0.35, 1]),
                         xaxis=dict(title='Building Name'), showlegend=True)

    def calc_graph(self):
        graph = []
        analysis_fields = self.remove_unused_fields(self.data, self.analysis_fields)
        dataframe = self.data
        dataframe['total'] = dataframe[analysis_fields].sum(axis=1)
        dataframe.sort_values(by='total', ascending=False, inplace=True)
        dataframe.reset_index(inplace=True, drop=True)
        for field in analysis_fields:
            y = dataframe[field]
            name = NAMING[field]
            total_percent = (y / dataframe['total'] * 100).round(2).values
            total_percent_txt = ["(%.2f %%)" % x for x in total_percent]
            trace = go.Bar(x=dataframe["Name"], y=y, name=name, text=total_percent_txt, orientation='v',
                           marker=dict(color=COLOR[field]))
            graph.append(trace)

        return graph

    def calc_table(self):
        data_frame = self.data
        analysis_fields = self.remove_unused_fields(self.data, self.analysis_fields)
        median = data_frame[analysis_fields].median().round(2).tolist()
        total = data_frame[analysis_fields].sum().round(2).tolist()
        total_perc = [str(x) + " (" + str(round(x / sum(total) * 100, 1)) + " %)" for x in total]
        # calculate graph
        anchors = []
        load_names = []
        for field in analysis_fields:
            anchors.append(', '.join(calc_top_three_anchor_loads(data_frame, field)))
            load_names.append(NAMING[field] + ' (' + field.split('_', 1)[0] + ')')

        column_names = ['Load Name', 'Total [MWh/yr]', 'Median [MWh/yr]', 'Top 3 Consumers']
        table_df = pd.DataFrame({'Load Name': load_names + ["Total"],
                                 'Total [MWh/yr]': total_perc + [str(sum(total)) + " (" + str(100) + " %)"],
                                 'Median [MWh/yr]': median + ["-"],
                                 'Top 3 Consumers': anchors + ['-']}, columns=column_names)
        return table_df

def energy_demand_district(data_frame, analysis_fields, title, output_path):
    # CALCULATE GRAPH
    traces_graph = calc_graph(analysis_fields, data_frame)

    # CALCULATE TABLE
    traces_table = calc_table(analysis_fields, data_frame)

    # PLOT GRAPH
    traces_graph.append(traces_table)
    layout = go.Layout(images=LOGO, title=title, barmode='stack',
                       yaxis=dict(title='Energy [MWh/yr]', domain=[0.35, 1]),
                       xaxis=dict(title='Building Name'), showlegend=True)
    fig = go.Figure(data=traces_graph, layout=layout)
    plot(fig, auto_open=False, filename=output_path)

    return {'data': traces_graph, 'layout': layout}

def calc_table(analysis_fields, data_frame):
    median = data_frame[analysis_fields].median().round(2).tolist()
    total = data_frame[analysis_fields].sum().round(2).tolist()
    total_perc = [str(x) + " (" + str(round(x / sum(total) * 100, 1)) + " %)" for x in total]
    # calculate graph
    anchors = []
    load_names = []
    for field in analysis_fields:
        anchors.append(calc_top_three_anchor_loads(data_frame, field))
        load_names.append(NAMING[field] + ' (' + field.split('_', 1)[0] + ')')

    table = go.Table(domain=dict(x=[0, 1.0], y=[0, 0.2]),
                     header=dict(values=['Load Name', 'Total [MWh/yr]', 'Median [MWh/yr]', 'Top 3 Consumers']),
                     cells=dict(values=[load_names, total_perc, median, anchors]))

    return table

def calc_graph(analysis_fields, data):
    graph = []
    data['total'] = data[analysis_fields].sum(axis=1)
    data = data.sort_values(by='total', ascending=False)
    for field in analysis_fields:
        y = data[field]
        name = NAMING[field]
        total_percent = (y / data['total'] * 100).round(2).values
        total_percent_txt = ["(%.2f %%)" % x for x in total_percent]
        trace = go.Bar(x=data["Name"], y=y, name=name, text=total_percent_txt, orientation='v',
                       marker=dict(color=COLOR[field]))
        graph.append(trace)

    return graph

def calc_top_three_anchor_loads(data_frame, field):
    data_frame = data_frame.sort_values(by=field, ascending=False)
    anchor_list = data_frame[:3].Name.values
    return anchor_list

def main():
    import cea.config
    import cea.inputlocator
    config = cea.config.Configuration()
    locator = cea.inputlocator.InputLocator(config.scenario)
    # cache = cea.plots.cache.PlotCache(config.project)
    cache = cea.plots.cache.NullPlotCache()
    EnergyDemandDistrictPlot(config.project, {'buildings': None,
                                              'scenario-name': config.scenario_name},
                             cache).plot(auto_open=True)
    EnergyDemandDistrictPlot(config.project, {'buildings': locator.get_zone_building_names()[0:2],
                                              'scenario-name': config.scenario_name},
                             cache).plot(auto_open=True)
    EnergyDemandDistrictPlot(config.project, {'buildings': [locator.get_zone_building_names()[0]],
                                              'scenario-name': config.scenario_name},
                             cache).plot(auto_open=True)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""
NRM backends which just logs actions performed.

Author: Henrik Thostrup Jensen <htj@nordu.net>
Copyright: NORDUnet (2011)
"""

from twisted.python import log
from twisted.internet import defer

from zope.interface import implements

from opennsa import interface as nsainterface
from opennsa.backends.common import calendar as reservationcalendar, simplebackend

class DUDNSIBackend:

    implements(nsainterface.NSIBackendInterface)

    def __init__(self, network_name):
        self.network_name = network_name
        self.calendar = reservationcalendar.ReservationCalendar()
        self.connections = []
        self.connection_manager = DUDConnectionManager('DUD Connection Manager %s' % network_name)

    def canAllocateLink(self, source_port, dest_port, service_parameters):

        self.calendar.checkReservation(source_port, service_parameters.start_time, service_parameters.end_time)
        self.calendar.checkReservation(dest_port  , service_parameters.start_time, service_parameters.end_time)
        return True

    def createConnection(self, source_port, dest_port, service_parameters):

        self.canAllocateLink(source_port, dest_port, service_parameters)

        self.calendar.addConnection(source_port, service_parameters.start_time, service_parameters.end_time)
        self.calendar.addConnection(dest_port  , service_parameters.start_time, service_parameters.end_time)

        ac = simplebackend.GenericConnection(source_port, dest_port, service_parameters, self.network_name, self.calendar,
                                             'DUD NRM', 'DUD Network %s' % self.network_name, self.connection_manager)
        self.connections.append(ac)
        return ac

class DUDConnectionManager:

    def __init__(self, log_system):
        self.log_system = log_system

    def setupLink(self, source_port, dest_port):
        log.msg('Link %s -> %s up' % (source_port, dest_port), system=self.log_system)
        return defer.succeed(None)
        #return defer.fail(NotImplementedError('Link setup failed'))

    def teardownLink(self, source_port, dest_port):
        log.msg('Link %s -> %s down' % (source_port, dest_port), system=self.log_system)
        return defer.succeed(None)
        #return defer.fail(NotImplementedError('Link teardown failed'))

<commit_msg>update commented out dud failures to use new error infrastructure<commit_after>"""
NRM backends which just logs actions performed.

Author: Henrik Thostrup Jensen <htj@nordu.net>
Copyright: NORDUnet (2011)
"""

from twisted.python import log
from twisted.internet import defer

from zope.interface import implements

from opennsa import interface as nsainterface
from opennsa.backends.common import calendar as reservationcalendar, simplebackend

class DUDNSIBackend:

    implements(nsainterface.NSIBackendInterface)

    def __init__(self, network_name):
        self.network_name = network_name
        self.calendar = reservationcalendar.ReservationCalendar()
        self.connections = []
        self.connection_manager = DUDConnectionManager('DUD Connection Manager %s' % network_name)

    def canAllocateLink(self, source_port, dest_port, service_parameters):

        self.calendar.checkReservation(source_port, service_parameters.start_time, service_parameters.end_time)
        self.calendar.checkReservation(dest_port  , service_parameters.start_time, service_parameters.end_time)
        return True

    def createConnection(self, source_port, dest_port, service_parameters):

        self.canAllocateLink(source_port, dest_port, service_parameters)

        self.calendar.addConnection(source_port, service_parameters.start_time, service_parameters.end_time)
        self.calendar.addConnection(dest_port  , service_parameters.start_time, service_parameters.end_time)

        ac = simplebackend.GenericConnection(source_port, dest_port, service_parameters, self.network_name, self.calendar,
                                             'DUD NRM', 'DUD Network %s' % self.network_name, self.connection_manager)
        self.connections.append(ac)
        return ac

class DUDConnectionManager:

    def __init__(self, log_system):
        self.log_system = log_system

    def setupLink(self, source_port, dest_port):
        log.msg('Link %s -> %s up' % (source_port, dest_port), system=self.log_system)
        return defer.succeed(None)
        #from opennsa import error
        #return defer.fail(error.InternalNRMError('Link setup failed'))

    def teardownLink(self, source_port, dest_port):
        log.msg('Link %s -> %s down' % (source_port, dest_port), system=self.log_system)
        return defer.succeed(None)
        #from opennsa import error
        #return defer.fail(error.InternalNRMError('Link teardown failed'))

<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import datetime
import krpc
import serial
from time import sleep
import sys
import json
import argparse
from ksp_console import *

#port = "COM4"
port = "/dev/ttyS3"
last_scene=""
status_updates = {}

def serial_read_line():
	global ser
	serial_data = ""
	while True:
		data = ser.read(1)
		if len(data) > 0:
			data = data.decode('iso8859-1')
			if data=='\n':
				return serial_data
			serial_data += data

# auf -1 ... 1 normieren
def normiere_joystick(value):
	if value>512:
		value=512
	if value<-512:
		value=-512
	value = float(value)
	value = value/512.0
	return value

def normiere_throttle(value):
	if value<20:
		return 0;
	if value>890:
		return 1;
	return float(value)/1000.0;

def send_handshake():
	send_data = {}
	send_data["start"] = 2016
	send_serial( CMD_INIT, send_data)

def send_serial( command, send_data):
	global args
	global ser
	send_data["cmd"]=command;
	data=json.dumps(send_data,separators=(',',':'))+"\n"
	if args.debugsend:
		print("sending %d bytes " % len(data))
		print("send: "+data)
		sys.stdout.flush()
	data = data.encode('iso8859-1')
	#got to send in 32 byte chunks to avoid loosing stuff
	len_data = str(len(data))+":"
	ser.write(len_data.encode('iso8859-1'))
	while( len(data)>0 ):
		send_pkt = data[:32]
		data = data[32:]
		ser.write(send_pkt)
		ser.flush()
		response = ""
		while( len(response)!=2 ):
			response += ser.read(1).decode('iso8859-1')
#		print( "response: " + response)
#		sys.stdout.flush()
		if response != "OK":
			print( "got the wrong ACK for the serial protocol: " +response )

def decode_json_array(arr):
	res={}
	for index in range( 0, len(arr), 2):
		res[arr[index]]=arr[index+1]
	return res

def encode_json_array(arr):
	res=[]
	for element in arr:
		res.append(element)
		res.append(arr[element])
	return res

def send_flight_data():
	global args
	global status_updates
	global conn
	if not args.noksp:
		if conn.krpc.current_game_scene!=conn.krpc.GameScene.flight:
			return
		try:
			vessel = conn.space_center.active_vessel
			control = vessel.control
			orbit = vessel.orbit
			status_updates[str(INFO_HEIGHT)] = int(vessel.flight().surface_altitude)
			status_updates[str(INFO_SPEED)] = int(vessel.flight(vessel.orbit.body.reference_frame).speed)
			status_updates[str(BUTTON_SAS)] = int(control.sas)
			status_updates[str(BUTTON_RCS)] = int(control.rcs)
			status_updates[str(BUTTON_LIGHTS)] = int(control.lights)
			status_updates[str(BUTTON_GEAR)] = int(control.gear)
			status_updates[str(BUTTON_BREAKS)] = int(control.brakes)
			send_updates()
		except krpc.error.RPCError:
		 	pass

def send_updates():
	global args
	global status_updates
	global conn
	if not args.noksp:
		if conn.krpc.current_game_scene!=conn.krpc.GameScene.flight:
			return
		try:
			send_data = encode_json_array(status_updates)
			send_serial( CMD_UPDATE_CONSOLE, {"data":send_data})
			status_updates={}
		except krpc.error.RPCError:
		 	pass

def time_to_string(secs):
	if secs<0:
		return "n/a"
	tap = ""
	if secs>60:
		mins = int(secs/60)
		tap = str(mins)+ " min"
		if mins<5:
			tap = tap+", "+str(secs-(mins*60))+" sec"
	else:
		tap = str(int(secs)) + " sec"
	return tap

def add_action_group_status():
    global args
    global status_updates
    global conn
    if not args.noksp:
        if conn.krpc.current_game_scene!=conn.krpc.GameScene.flight:
            return
        try:
            vessel = conn.space_center.active_vessel
            control = vessel.control
            status=0
            current_value=1
            for grp in range(0,9):
                if control.get_action_group(grp):
                    status = status + current_value
                    current_value = current_value * 2
            status_updates[str(INFO_ACTION_GROUPS)] = status
        except krpc.error.RPCError:
            pass
        except ValueError:
            pass

def add_landing_info():
    global args
    global status_updates
    global conn
    if not args.noksp:
        if conn.krpc.current_game_scene!=conn.krpc.GameScene.flight:
            return
        try:
            flight = conn.space_center.active_vessel.flight()
            status_updates[str(INFO_SURFACE_HEIGHT)] = int(flight.surface_altitude)
            speed = int( conn.space_center.active_vessel.flight( conn.space_center.active_vessel.orbit.body.reference_frame).vertical_speed)
            if( speed<0.0 ):
                status_updates[str(INFO_SURFACE_TIME)] = time_to_string(int(flight.surface_altitude/abs(speed)))
            else:
                status_updates[str(INFO_SURFACE_TIME)] = "n/a"
        except krpc.error.RPCError:
            pass
        except ValueError:
            pass

def add_orbit_to_status():
	global args
	global status_updates
	global conn
	if not args.noksp:
		if conn.krpc.current_game_scene!=conn.krpc.GameScene.flight:
			return
		try:
			orbit = conn.space_center.active_vessel.orbit
			status_updates[str(INFO_APOAPSIS)] = int(orbit.apoapsis_altitude)
			status_updates[str(INFO_APOAPSIS_TIME)] = time_to_string(int(orbit.time_to_apoapsis))
			if orbit.periapsis_altitude>0:
				status_updates[str(INFO_PERIAPSIS)] = int(orbit.periapsis_altitude)
				status_updates[str(INFO_PERIAPSIS_TIME)] = time_to_string(int(orbit.time_to_periapsis))
			else:
				status_updates[str(INFO_PERIAPSIS)] = "n/a"
				status_updates[str(INFO_PERIAPSIS_TIME)] = "n/a"
		except krpc.error.RPCError:
		 	pass
		except ValueError:
		  	pass

def check_input( data, key, fun, *fargs):
	global args
	if key in data and not args.noksp:
		fun(*fargs)

def expand_solar_arrays( vessel, value):
	global args
	if not args.noksp:
		for solar in vessel.parts.solar_panels:
			solar.deployed = value

def check_input_and_feedback(data, key_str, key, control):
    global args
    global status_updates
    global conn
    if key in data and not args.noksp:
        if bool(data[key]):
            setattr( control, key_str, not getattr( control, key_str))
        status_updates[str(key)] = int(getattr( control, key_str))

def check_analog( data, key, control, ckey):
    if key in data:
        value = data[key]
        if not args.noksp:
            setattr( control, ckey, normiere_joystick(value))

def work_on_json(input_data):
	global args
	global status_updates
	global conn

	if not args.noksp:
		if conn.krpc.current_game_scene!=conn.krpc.GameScene.flight:
			return
	try:
		if args.noksp:
			return
		vessel = conn.space_center.active_vessel
		control = vessel.control
		json_data = json.loads(input_data)
		data = decode_json_array(json_data["data"])
		if args.debugrecv:
			print( data )
			sys.stdout.flush()
		check_analog( data, KSP_INPUT_XTRANS, control, "right")
		check_analog( data, KSP_INPUT_YTRANS, control, "up")
		check_analog( data, KSP_INPUT_ZTRANS, control, "forward")
		check_analog( data, KSP_INPUT_YAW, control, "yaw")
		check_analog( data, KSP_INPUT_PITCH, control, "pitch")
		check_analog( data, KSP_INPUT_ROLL, control, "roll")
		if args.debugrecv:
			print( data )
			sys.stdout.flush()
		if KSP_INPUT_THRUST in data:
			value = data[KSP_INPUT_THRUST]
			control.throttle = normiere_throttle(value)
		if BUTTON_STAGE in data and data[BUTTON_STAGE]==1:
			control.activate_next_stage()
		check_input_and_feedback( data, "sas", BUTTON_SAS, control)
		check_input_and_feedback( data, "rcs", BUTTON_RCS, control)
		check_input_and_feedback( data, "gear", BUTTON_GEAR, control)
		check_input_and_feedback( data, "lights", BUTTON_LIGHTS, control)
		check_input_and_feedback( data, "brakes", BUTTON_BREAKS, control)
		check_input( data, BUTTON_ACTION_1, lambda: control.toggle_action_group(0))
		check_input( data, BUTTON_ACTION_2, lambda: control.toggle_action_group(1))
		check_input( data, BUTTON_ACTION_3, lambda: control.toggle_action_group(2))
		check_input( data, BUTTON_ACTION_4, lambda: control.toggle_action_group(3))
		check_input( data, BUTTON_ACTION_5, lambda: control.toggle_action_group(4))
		check_input( data, BUTTON_ACTION_6, lambda: control.toggle_action_group(5))
		check_input( data, BUTTON_ACTION_7, lambda: control.toggle_action_group(6))
		check_input( data, BUTTON_ACTION_8, lambda: control.toggle_action_group(7))
		check_input( data, BUTTON_ACTION_9, lambda: control.toggle_action_group(8))
		check_input( data, BUTTON_ACTION_10, lambda: control.toggle_action_group(9))
		check_input( data, BUTTON_SOLAR_OFF, lambda: expand_solar_arrays( vessel, False))
		check_input( data, BUTTON_SOLAR_ON, lambda: expand_solar_arrays( vessel, True))
	except ValueError:
		print('Decoding JSON failed')
	except krpc.error.RPCError:
		pass

## main

last_chip_data = 0
parser = argparse.ArgumentParser()
parser.add_argument("--debugsend", help="print data sent to con", action="store_true")
parser.add_argument("--debugrecv", help="print some received from con", action="store_true")
parser.add_argument("--debugchip", help="print chip debug output", action="store_true")
parser.add_argument("--noksp", help="run without connecting to ksp", action="store_true")
args = parser.parse_args()
if args.noksp:
	print("noksp: will not connect to KSP")

# no async receives, so it is ok to set a timeout, should
# make less loops
ser = serial.Serial(port, 115200, timeout=0)
ser.reset_input_buffer()
ser.reset_output_buffer()
if not args.noksp:
	conn = krpc.connect(name='mk console')
sleep(3)
ref_time_short = datetime.datetime.now()
ref_time_long = datetime.datetime.now()
send_handshake()
now = datetime.datetime.now()
diff_short = now - ref_time_short
diff_long  = now - ref_time_long
while True:
	# this works command driven, so we send commands,
	# wait for the reply and done

	# every 2 seconds or so: send update to the arduino
#	if (diff_short.seconds>1 or diff_short.microseconds>200000) and ser.out_waiting == 0:
	if (diff_short.seconds>1 or diff_short.microseconds>200000):
		if not args.noksp:
			if conn.krpc.current_game_scene==conn.krpc.GameScene.flight and diff_long.seconds>1:
				add_action_group_status()
				add_orbit_to_status()
				add_landing_info()
				ref_time_long = datetime.datetime.now()
				send_flight_data()
			ref_time_short = datetime.datetime.now()

	# read the current status and button updates and so on
	send_serial( CMD_GET_UPDATES, {})
	serial_data=serial_read_line()
	if args.debugrecv:
		print( serial_data )
		sys.stdout.flush()
	if args.debugchip:
		try:
			data = json.loads(serial_data)
			if "chip" in data and data["chip"]!=last_chip_data:
				print("Chip: "+str(data["chip"]))
				last_chip_data = data["chip"]
				sys.stdout.flush()
		except ValueError:
			print('Decoding JSON failed for: '+lines[0])
	work_on_json(serial_data)
	# if this generated updates -> send them right away
	if len(status_updates)>0:
		send_updates()
	now = datetime.datetime.now()
	diff_short = now - ref_time_short
	diff_long  = now - ref_time_long
<commit_msg>removed " from json string, ints are better<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import datetime
import krpc
import serial
from time import sleep
import sys
import json
import argparse
from ksp_console import *

#port = "COM4"
port = "/dev/ttyS3"
last_scene=""
status_updates = {}

def serial_read_line():
	global ser
	serial_data = ""
	while True:
		data = ser.read(1)
		if len(data) > 0:
			data = data.decode('iso8859-1')
			if data=='\n':
				return serial_data
			serial_data += data

# auf -1 ... 1 normieren
def normiere_joystick(value):
	if value>512:
		value=512
	if value<-512:
		value=-512
	value = float(value)
	value = value/512.0
	return value

def normiere_throttle(value):
	if value<20:
		return 0;
	if value>890:
		return 1;
	return float(value)/1000.0;

def send_handshake():
	send_data = {}
	send_data["start"] = 2016
	send_serial( CMD_INIT, send_data)

def send_serial( command, send_data):
	global args
	global ser
	send_data["cmd"]=command;
	data=json.dumps(send_data,separators=(',',':'))+"\n"
	if args.debugsend:
		print("sending %d bytes " % len(data))
		print("send: "+data)
		sys.stdout.flush()
	data = data.encode('iso8859-1')
	#got to send in 32 byte chunks to avoid loosing stuff
	len_data = str(len(data))+":"
	ser.write(len_data.encode('iso8859-1'))
	while( len(data)>0 ):
		send_pkt = data[:32]
		data = data[32:]
		ser.write(send_pkt)
		ser.flush()
		response = ""
		while( len(response)!=2 ):
			response += ser.read(1).decode('iso8859-1')
#		print( "response: " + response)
#		sys.stdout.flush()
		if response != "OK":
			print( "got the wrong ACK for the serial protocol: " +response )

def decode_json_array(arr):
	res={}
	for index in range( 0, len(arr), 2):
		res[arr[index]]=arr[index+1]
	return res

def encode_json_array(arr):
	res=[]
	for element in arr:
		res.append(int(element))
		res.append(arr[element])
	return res

def send_flight_data():
	global args
	global status_updates
	global conn
	if not args.noksp:
		if conn.krpc.current_game_scene!=conn.krpc.GameScene.flight:
			return
		try:
			vessel = conn.space_center.active_vessel
			control = vessel.control
			orbit = vessel.orbit
			status_updates[str(INFO_HEIGHT)] = int(vessel.flight().surface_altitude)
			status_updates[str(INFO_SPEED)] = int(vessel.flight(vessel.orbit.body.reference_frame).speed)
			status_updates[str(BUTTON_SAS)] = int(control.sas)
			status_updates[str(BUTTON_RCS)] = int(control.rcs)
			status_updates[str(BUTTON_LIGHTS)] = int(control.lights)
			status_updates[str(BUTTON_GEAR)] = int(control.gear)
			status_updates[str(BUTTON_BREAKS)] = int(control.brakes)
			send_updates()
		except krpc.error.RPCError:
		 	pass

def send_updates():
	global args
	global status_updates
	global conn
	if not args.noksp:
		if conn.krpc.current_game_scene!=conn.krpc.GameScene.flight:
			return
		try:
			send_data = encode_json_array(status_updates)
			send_serial( CMD_UPDATE_CONSOLE, {"data":send_data})
			status_updates={}
		except krpc.error.RPCError:
		 	pass

def time_to_string(secs):
	if secs<0:
		return "n/a"
	tap = ""
	if secs>60:
		mins = int(secs/60)
		tap = str(mins)+ " min"
		if mins<5:
			tap = tap+", "+str(secs-(mins*60))+" sec"
	else:
		tap = str(int(secs)) + " sec"
	return tap

def add_action_group_status():
    global args
    global status_updates
    global conn
    if not args.noksp:
        if conn.krpc.current_game_scene!=conn.krpc.GameScene.flight:
            return
        try:
            vessel = conn.space_center.active_vessel
            control = vessel.control
            status=0
            current_value=1
            for grp in range(0,9):
                if control.get_action_group(grp):
                    status = status + current_value
                    current_value = current_value * 2
            status_updates[str(INFO_ACTION_GROUPS)] = status
        except krpc.error.RPCError:
            pass
        except ValueError:
            pass

def add_landing_info():
    global args
    global status_updates
    global conn
    if not args.noksp:
        if conn.krpc.current_game_scene!=conn.krpc.GameScene.flight:
            return
        try:
            flight = conn.space_center.active_vessel.flight()
            status_updates[str(INFO_SURFACE_HEIGHT)] = int(flight.surface_altitude)
            speed = int( conn.space_center.active_vessel.flight( conn.space_center.active_vessel.orbit.body.reference_frame).vertical_speed)
            if( speed<0.0 ):
                status_updates[str(INFO_SURFACE_TIME)] = time_to_string(int(flight.surface_altitude/abs(speed)))
            else:
                status_updates[str(INFO_SURFACE_TIME)] = "n/a"
        except krpc.error.RPCError:
            pass
        except ValueError:
            pass

def add_orbit_to_status():
	global args
	global status_updates
	global conn
	if not args.noksp:
		if conn.krpc.current_game_scene!=conn.krpc.GameScene.flight:
			return
		try:
			orbit = conn.space_center.active_vessel.orbit
			status_updates[str(INFO_APOAPSIS)] = int(orbit.apoapsis_altitude)
			status_updates[str(INFO_APOAPSIS_TIME)] = time_to_string(int(orbit.time_to_apoapsis))
			if orbit.periapsis_altitude>0:
				status_updates[str(INFO_PERIAPSIS)] = int(orbit.periapsis_altitude)
				status_updates[str(INFO_PERIAPSIS_TIME)] = time_to_string(int(orbit.time_to_periapsis))
			else:
				status_updates[str(INFO_PERIAPSIS)] = "n/a"
				status_updates[str(INFO_PERIAPSIS_TIME)] = "n/a"
		except krpc.error.RPCError:
		 	pass
		except ValueError:
		  	pass

def check_input( data, key, fun, *fargs):
	global args
	if key in data and not args.noksp:
		fun(*fargs)

def expand_solar_arrays( vessel, value):
	global args
	if not args.noksp:
		for solar in vessel.parts.solar_panels:
			solar.deployed = value

def check_input_and_feedback(data, key_str, key, control):
    global args
    global status_updates
    global conn
    if key in data and not args.noksp:
        if bool(data[key]):
            setattr( control, key_str, not getattr( control, key_str))
        status_updates[str(key)] = int(getattr( control, key_str))

def check_analog( data, key, control, ckey):
    if key in data:
        value = data[key]
        if not args.noksp:
            setattr( control, ckey, normiere_joystick(value))

def work_on_json(input_data):
	global args
	global status_updates
	global conn

	if not args.noksp:
		if conn.krpc.current_game_scene!=conn.krpc.GameScene.flight:
			return
	try:
		if args.noksp:
			return
		vessel = conn.space_center.active_vessel
		control = vessel.control
		json_data = json.loads(input_data)
		data = decode_json_array(json_data["data"])
		if args.debugrecv:
			print( data )
			sys.stdout.flush()
		check_analog( data, KSP_INPUT_XTRANS, control, "right")
		check_analog( data, KSP_INPUT_YTRANS, control, "up")
		check_analog( data, KSP_INPUT_ZTRANS, control, "forward")
		check_analog( data, KSP_INPUT_YAW, control, "yaw")
		check_analog( data, KSP_INPUT_PITCH, control, "pitch")
		check_analog( data, KSP_INPUT_ROLL, control, "roll")
		if args.debugrecv:
			print( data )
			sys.stdout.flush()
		if KSP_INPUT_THRUST in data:
			value = data[KSP_INPUT_THRUST]
			control.throttle = normiere_throttle(value)
		if BUTTON_STAGE in data and data[BUTTON_STAGE]==1:
			control.activate_next_stage()
		check_input_and_feedback( data, "sas", BUTTON_SAS, control)
		check_input_and_feedback( data, "rcs", BUTTON_RCS, control)
		check_input_and_feedback( data, "gear", BUTTON_GEAR, control)
		check_input_and_feedback( data, "lights", BUTTON_LIGHTS, control)
		check_input_and_feedback( data, "brakes", BUTTON_BREAKS, control)
		check_input( data, BUTTON_ACTION_1, lambda: control.toggle_action_group(0))
		check_input( data, BUTTON_ACTION_2, lambda: control.toggle_action_group(1))
		check_input( data, BUTTON_ACTION_3, lambda: control.toggle_action_group(2))
		check_input( data, BUTTON_ACTION_4, lambda: control.toggle_action_group(3))
		check_input( data, BUTTON_ACTION_5, lambda: control.toggle_action_group(4))
		check_input( data, BUTTON_ACTION_6, lambda: control.toggle_action_group(5))
		check_input( data, BUTTON_ACTION_7, lambda: control.toggle_action_group(6))
		check_input( data, BUTTON_ACTION_8, lambda: control.toggle_action_group(7))
		check_input( data, BUTTON_ACTION_9, lambda: control.toggle_action_group(8))
		check_input( data, BUTTON_ACTION_10, lambda: control.toggle_action_group(9))
		check_input( data, BUTTON_SOLAR_OFF, lambda: expand_solar_arrays( vessel, False))
		check_input( data, BUTTON_SOLAR_ON, lambda: expand_solar_arrays( vessel, True))
	except ValueError:
		print('Decoding JSON failed')
	except krpc.error.RPCError:
		pass

## main

last_chip_data = 0
parser = argparse.ArgumentParser()
parser.add_argument("--debugsend", help="print data sent to con", action="store_true")
parser.add_argument("--debugrecv", help="print some received from con", action="store_true")
parser.add_argument("--debugchip", help="print chip debug output", action="store_true")
parser.add_argument("--noksp", help="run without connecting to ksp", action="store_true")
args = parser.parse_args()
if args.noksp:
	print("noksp: will not connect to KSP")

# no async receives, so it is ok to set a timeout, should
# make less loops
ser = serial.Serial(port, 115200, timeout=0)
ser.reset_input_buffer()
ser.reset_output_buffer()
if not args.noksp:
	conn = krpc.connect(name='mk console')
sleep(3)
ref_time_short = datetime.datetime.now()
ref_time_long = datetime.datetime.now()
send_handshake()
now = datetime.datetime.now()
diff_short = now - ref_time_short
diff_long  = now - ref_time_long
while True:
	# this works command driven, so we send commands,
	# wait for the reply and done

	# every 2 seconds or so: send update to the arduino
#	if (diff_short.seconds>1 or diff_short.microseconds>200000) and ser.out_waiting == 0:
	if (diff_short.seconds>1 or diff_short.microseconds>200000):
		if not args.noksp:
			if conn.krpc.current_game_scene==conn.krpc.GameScene.flight and diff_long.seconds>1:
				add_action_group_status()
				add_orbit_to_status()
				add_landing_info()
				ref_time_long = datetime.datetime.now()
				send_flight_data()
			ref_time_short = datetime.datetime.now()

	# read the current status and button updates and so on
	send_serial( CMD_GET_UPDATES, {})
	serial_data=serial_read_line()
	if args.debugrecv:
		print( serial_data )
		sys.stdout.flush()
	if args.debugchip:
		try:
			data = json.loads(serial_data)
			if "chip" in data and data["chip"]!=last_chip_data:
				print("Chip: "+str(data["chip"]))
				last_chip_data = data["chip"]
				sys.stdout.flush()
		except ValueError:
			print('Decoding JSON failed for: '+lines[0])
	work_on_json(serial_data)
	# if this generated updates -> send them right away
	if len(status_updates)>0:
		send_updates()
	now = datetime.datetime.now()
	diff_short = now - ref_time_short
	diff_long  = now - ref_time_long
<|endoftext|>


<commit_before>import os
import pytest
import asyncio
from concurrent.futures import ProcessPoolExecutor

import virtool.job
import virtool.virus_index

@pytest.fixture
def test_rebuild_job(tmpdir, loop, test_motor, test_dispatch):
    tmpdir.mkdir("reference").mkdir("viruses")
    tmpdir.mkdir("logs").mkdir("jobs")

    exec = ProcessPoolExecutor()

    settings = {
        "data_path": str(tmpdir)
    }

    job = virtool.virus_index.RebuildIndex(
        loop,
        exec,
        test_motor,
        settings,
        test_dispatch,
        "foobar",
        "rebuild_index",
        dict(index_id="foobar"),
        1,
        4
    )

    return job

async def test_mk_index_dir(tmpdir, test_rebuild_job):
    await test_rebuild_job.mk_index_dir()

    assert os.listdir(os.path.join(str(tmpdir), "reference", "viruses")) == [
        "foobar"
    ]

async def test_write_fasta(mocker, test_rebuild_job):
    m = mocker.stub(name="patch_virus_to_version")

    async def patch_virus_to_version(*args):
        m(*args)
        return None, {
            "isolates": [
                {
                    "default": True,
                    "sequences": [
                        {
                            "_id": "foo",
                            "sequence": "ATAGAGATATAGAGACACACTTACTTATCA"
                        },
                        {
                            "_id": "bar",
                            "sequence": "GGCTTTCTCTATCAGGGAGGACTAGGCTAC"
                        }
                    ]
                },
                {
                    "default": True,
                    "sequences": [
                        {
                            "_id": "baz",
                            "sequence": "ATAGAGATATAGAGACACACTTACTTATCA"
                        },
                        {
                            "_id": "test",
                            "sequence": "GGCTTTCTCTATCAGGGAGGACTAGGCTAC"
                        }
                    ]
                }
            ]
        }, None

    mocker.patch("virtool.virus_history.patch_virus_to_version", patch_virus_to_version)

    test_rebuild_job.task_args["virus_manifest"] = {
        "foobar": 2
    }

    os.mkdir(test_rebuild_job.reference_path)

    await test_rebuild_job.write_fasta()

    with open(os.path.join(test_rebuild_job.reference_path, "ref.fa"), "r") as handle:
        assert handle.read() in [
            ">foo\nATAGAGATATAGAGACACACTTACTTATCA\n>bar\nGGCTTTCTCTATCAGGGAGGACTAGGCTAC\n",
            ">bar\nGGCTTTCTCTATCAGGGAGGACTAGGCTAC\n>foo\nATAGAGATATAGAGACACACTTACTTATCA\n"
        ]

@pytest.mark.parametrize("error", [False, True])
async def test_bowtie_build(error, capsys, tmpdir, test_rebuild_job):
    root_path = os.path.join(str(tmpdir), "reference", "viruses")
    os.mkdir(os.path.join(root_path, "foobar"))

    if not error:
        fasta_path = os.path.join(str(tmpdir), "reference", "viruses", "foobar", "ref.fa")

        with open(fasta_path, "w") as handle:
            handle.write(">test_1\nTACGTATGACTGAGCTACGGGGCTACGACTTACCCTTCACGATCAC")
            handle.write(">test_2\nGGCTTCGGCTGATCACGACTGGACTAGCATCTGACTACGATGCTGA")

    with capsys.disabled():
        if error:
            with pytest.raises(virtool.job.SubprocessError) as err:
                await test_rebuild_job.bowtie_build()

            assert "virtool.job.SubprocessError" in str(err)
            assert "Command failed: bowtie2-build -f" in str(err)
        else:
            await test_rebuild_job.bowtie_build()

    await test_rebuild_job.flush_log()

    log_path = os.path.join(str(tmpdir), "logs", "jobs", test_rebuild_job.id)

    with open(log_path, "r") as handle:
        content = handle.read()

        if error:
            assert "Error: could not open" in content
            assert "Error: Encountered internal Bowtie 2 exception (#1)" in content
        else:
            assert "Building a SMALL index" in content

@pytest.mark.parametrize("in_use", [True, False])
async def test_replace_old(in_use, mocker, tmpdir, test_motor, test_rebuild_job):
    m = mocker.Mock()

    async def run_in_executor(*args):
        return m(*args)

    mocker.patch.object(test_rebuild_job, "run_in_executor", run_in_executor)

    await test_motor.indexes.insert_many([
        {
            "_id": "foobar",
            "version": 2,
            "ready": False,
            "has_files": True
        },
        {
            "_id": "foo",
            "version": 1,
            "ready": True,
            "has_files": True
        },
        {
            "_id": "baz",
            "version": 0,
            "ready": True,
            "has_files": True
        }
    ])

    if in_use:
        await test_motor.analyses.insert_one({
            "_id": "test",
            "ready": False,
            "index": {
                "id": "foo"
            }
        })

    await test_rebuild_job.replace_old()

    assert await test_motor.indexes.find_one("foobar") == {
        "_id": "foobar",
        "version": 2,
        "ready": True,
        "has_files": True
    }

    expected = {"foo", "foobar"} if in_use else {"foobar"}

    assert m.called

    assert m.call_args[0][0:2] == (
        virtool.virus_index.remove_unused_index_files,
        os.path.join(str(tmpdir), "reference", "viruses")
    )

    assert set(m.call_args[0][2]) == expected

    # Make sure that ``has_files`` was set to false for non-active indexes.
    assert set(await test_motor.indexes.find({"has_files": True}).distinct("_id")) == expected

async def test_cleanup(test_motor, test_rebuild_job):
    await test_motor.indexes.insert_one({"_id": "foobar"})

    await test_motor.history.insert_many([
        {
            "_id": "foo",
            "index": {
                "id": "foobar",
                "version": 2
            }
        },
        {
            "_id": "bar",
            "index": {
                "id": "foobar",
                "version": 2
            }
        },
        {
            "_id": "baz",
            "index": {
                "id": "aaa111",
                "version": 1
            }
        }
    ])

    os.mkdir(test_rebuild_job.reference_path)

    with open(os.path.join(test_rebuild_job.reference_path, "test.txt"), "w") as handle:
        handle.write("hello world")

    await test_rebuild_job.cleanup()

    assert not os.path.isdir(test_rebuild_job.reference_path)

    assert not await test_motor.indexes.count()

    assert await test_motor.history.count({"index.id": "unbuilt", "index.version": "unbuilt"}) == 2
<commit_msg>Change exec to executor in test to avoid builtin conflict<commit_after>import os
import pytest
import asyncio
from concurrent.futures import ProcessPoolExecutor

import virtool.job
import virtool.virus_index

@pytest.fixture
def test_rebuild_job(tmpdir, loop, test_motor, test_dispatch):
    tmpdir.mkdir("reference").mkdir("viruses")
    tmpdir.mkdir("logs").mkdir("jobs")

    executor = ProcessPoolExecutor()

    settings = {
        "data_path": str(tmpdir)
    }

    job = virtool.virus_index.RebuildIndex(
        loop,
        executor,
        test_motor,
        settings,
        test_dispatch,
        "foobar",
        "rebuild_index",
        dict(index_id="foobar"),
        1,
        4
    )

    return job

async def test_mk_index_dir(tmpdir, test_rebuild_job):
    await test_rebuild_job.mk_index_dir()

    assert os.listdir(os.path.join(str(tmpdir), "reference", "viruses")) == [
        "foobar"
    ]

async def test_write_fasta(mocker, test_rebuild_job):
    m = mocker.stub(name="patch_virus_to_version")

    async def patch_virus_to_version(*args):
        m(*args)
        return None, {
            "isolates": [
                {
                    "default": True,
                    "sequences": [
                        {
                            "_id": "foo",
                            "sequence": "ATAGAGATATAGAGACACACTTACTTATCA"
                        },
                        {
                            "_id": "bar",
                            "sequence": "GGCTTTCTCTATCAGGGAGGACTAGGCTAC"
                        }
                    ]
                },
                {
                    "default": True,
                    "sequences": [
                        {
                            "_id": "baz",
                            "sequence": "ATAGAGATATAGAGACACACTTACTTATCA"
                        },
                        {
                            "_id": "test",
                            "sequence": "GGCTTTCTCTATCAGGGAGGACTAGGCTAC"
                        }
                    ]
                }
            ]
        }, None

    mocker.patch("virtool.virus_history.patch_virus_to_version", patch_virus_to_version)

    test_rebuild_job.task_args["virus_manifest"] = {
        "foobar": 2
    }

    os.mkdir(test_rebuild_job.reference_path)

    await test_rebuild_job.write_fasta()

    with open(os.path.join(test_rebuild_job.reference_path, "ref.fa"), "r") as handle:
        assert handle.read() in [
            ">foo\nATAGAGATATAGAGACACACTTACTTATCA\n>bar\nGGCTTTCTCTATCAGGGAGGACTAGGCTAC\n",
            ">bar\nGGCTTTCTCTATCAGGGAGGACTAGGCTAC\n>foo\nATAGAGATATAGAGACACACTTACTTATCA\n"
        ]

@pytest.mark.parametrize("error", [False, True])
async def test_bowtie_build(error, capsys, tmpdir, test_rebuild_job):
    root_path = os.path.join(str(tmpdir), "reference", "viruses")
    os.mkdir(os.path.join(root_path, "foobar"))

    if not error:
        fasta_path = os.path.join(str(tmpdir), "reference", "viruses", "foobar", "ref.fa")

        with open(fasta_path, "w") as handle:
            handle.write(">test_1\nTACGTATGACTGAGCTACGGGGCTACGACTTACCCTTCACGATCAC")
            handle.write(">test_2\nGGCTTCGGCTGATCACGACTGGACTAGCATCTGACTACGATGCTGA")

    with capsys.disabled():
        if error:
            with pytest.raises(virtool.job.SubprocessError) as err:
                await test_rebuild_job.bowtie_build()

            assert "virtool.job.SubprocessError" in str(err)
            assert "Command failed: bowtie2-build -f" in str(err)
        else:
            await test_rebuild_job.bowtie_build()

    await test_rebuild_job.flush_log()

    log_path = os.path.join(str(tmpdir), "logs", "jobs", test_rebuild_job.id)

    with open(log_path, "r") as handle:
        content = handle.read()

        if error:
            assert "Error: could not open" in content
            assert "Error: Encountered internal Bowtie 2 exception (#1)" in content
        else:
            assert "Building a SMALL index" in content

@pytest.mark.parametrize("in_use", [True, False])
async def test_replace_old(in_use, mocker, tmpdir, test_motor, test_rebuild_job):
    m = mocker.Mock()

    async def run_in_executor(*args):
        return m(*args)

    mocker.patch.object(test_rebuild_job, "run_in_executor", run_in_executor)

    await test_motor.indexes.insert_many([
        {
            "_id": "foobar",
            "version": 2,
            "ready": False,
            "has_files": True
        },
        {
            "_id": "foo",
            "version": 1,
            "ready": True,
            "has_files": True
        },
        {
            "_id": "baz",
            "version": 0,
            "ready": True,
            "has_files": True
        }
    ])

    if in_use:
        await test_motor.analyses.insert_one({
            "_id": "test",
            "ready": False,
            "index": {
                "id": "foo"
            }
        })

    await test_rebuild_job.replace_old()

    assert await test_motor.indexes.find_one("foobar") == {
        "_id": "foobar",
        "version": 2,
        "ready": True,
        "has_files": True
    }

    expected = {"foo", "foobar"} if in_use else {"foobar"}

    assert m.called

    assert m.call_args[0][0:2] == (
        virtool.virus_index.remove_unused_index_files,
        os.path.join(str(tmpdir), "reference", "viruses")
    )

    assert set(m.call_args[0][2]) == expected

    # Make sure that ``has_files`` was set to false for non-active indexes.
    assert set(await test_motor.indexes.find({"has_files": True}).distinct("_id")) == expected

async def test_cleanup(test_motor, test_rebuild_job):
    await test_motor.indexes.insert_one({"_id": "foobar"})

    await test_motor.history.insert_many([
        {
            "_id": "foo",
            "index": {
                "id": "foobar",
                "version": 2
            }
        },
        {
            "_id": "bar",
            "index": {
                "id": "foobar",
                "version": 2
            }
        },
        {
            "_id": "baz",
            "index": {
                "id": "aaa111",
                "version": 1
            }
        }
    ])

    os.mkdir(test_rebuild_job.reference_path)

    with open(os.path.join(test_rebuild_job.reference_path, "test.txt"), "w") as handle:
        handle.write("hello world")

    await test_rebuild_job.cleanup()

    assert not os.path.isdir(test_rebuild_job.reference_path)

    assert not await test_motor.indexes.count()

    assert await test_motor.history.count({"index.id": "unbuilt", "index.version": "unbuilt"}) == 2
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# This file is part of Shuup.
#
# Copyright (c) 2012-2016, Shoop Commerce Ltd. All rights reserved.
#
# This source code is licensed under the AGPLv3 license found in the
# LICENSE file in the root directory of this source tree.
from django.db.models import ManyToManyField

from shuup.apps.provides import get_provide_objects
from shuup.core.models import Category, Product, ProductType, ShopProduct

def get_matching_for_product(shop_product, provide_category):
    """
    Get matching ids for shop product based on provide category

    For example:
    matching_ids = get_matching_for_product(shop_product, "campaign_catalog_filter")

    :param shop_product: A Shop Product
    :type shop_product: shuup.core.models.ShopProduct
    :param provide_category: Provide category name
    :type provide_category: str
    :return: list of collected item ids
    :rtype: list[int]
    """
    collected = set()
    matcher = ProductCampaignMatcher(shop_product)
    for item in get_provide_objects(provide_category):
        for obj in item._meta.model.objects.all():
            if matcher.matches(obj):
                collected.add(obj.pk)
    return collected

class ProductCampaignMatcher(object):
    """
    A class to match shop product to existing campaigns
    """

    def __init__(self, shop_product, **kwargs):
        """
        :param shop_product: a Shop Product to find matches for
        :type shop_product: shuup.core.models.ShopProduct
        """
        assert isinstance(shop_product, ShopProduct)

        self.shop = shop_product.shop
        self.product = shop_product.product
        self.shop_product = shop_product

    def matches(self, obj):
        """
        Tries to match filters, conditions, effects etc to shop product

        :return: True or False based on the match
        :rtype: bool
        """
        _matches = []
        for field in obj._meta.get_fields():
            related_model = field.related_model
            if self._is_product_kind(related_model):
                _matches.append(self.product_matcher(field, obj))
            elif self._is_category(related_model):
                _matches.append(self.category_matcher(field, obj))
            elif self._is_product_type(related_model):
                _matches.append(self.product_type_matcher(field, obj))
            else:
                _matches.append(False)
        return any(_matches)

    def product_type_matcher(self, field, obj):
        if isinstance(field, ManyToManyField):
            return self._product_types_match(field.name, obj)
        else:
            return self._product_type_matches(field.name, obj)

    def category_matcher(self, field, obj):
        if isinstance(field, ManyToManyField):
            return self._categories_match(field.name, obj)
        else:
            return self._category_matches(field.name, obj)

    def product_matcher(self, field, obj):
        if isinstance(field, ManyToManyField):
            return self._products_match(field, obj)
        else:
            return self._product_matches(field, obj)

    def _product_types_match(self, field_name, obj):
        return getattr(obj, field_name).filter(pk=self.product.type.pk).exists()

    def _product_type_matches(self, field_name, obj):
        attr = getattr(obj, field_name)
        if not attr:
            return False
        return (self.product.type.pk == attr.pk)

    def _products_match(self, field, obj):
        attr = getattr(obj, field.name)
        if not attr:
            return False
        if self._is_product(field.related_model):
            return attr.filter(pk=self.product.pk).exists()
        elif self._is_shop_product(field.related_model):
            return attr.filter(product__id=self.product.pk).exists()
        return False

    def _product_matches(self, field, obj):
        attr = getattr(obj, field.name)
        if not attr:
            return False
        if self._is_product(field.related_model):
            return (self.product.pk == attr.pk)
        elif self._is_shop_product(field.related_model):
            return (self.product.pk == attr.product.pk)
        return False

    def _categories_match(self, field_name, obj):
        existing = set(getattr(obj, field_name).values_list("pk", flat=True))
        if self.shop_product.primary_category and self.shop_product.primary_category.pk in existing:
            return True
        cats = set(self.shop_product.categories.values_list("pk", flat=True))
        if cats.intersection(existing):
            return True
        return False

    def _category_matches(self, field_name, obj):
        attr = getattr(obj, field_name)
        if not attr:
            return False
        if self.shop_product.primary_category and self.shop_product.primary_category.pk == attr.pk:
            return True
        return self.shop_product.categories.filter(pk=attr.pk).exists()

    def _is_product_kind(self, model):
        return (self._is_product(model) or self._is_shop_product(model))

    def _is_product(self, model):
        if model is None:
            return False
        return isinstance(model(), Product)

    def _is_shop_product(self, model):
        if model is None:
            return False
        return isinstance(model(), ShopProduct)

    def _is_category(self, model):
        if model is None:
            return False
        return isinstance(model(), Category)

    def _is_product_type(self, model):
        if model is None:
            return False
        return isinstance(model(), ProductType)
<commit_msg>Add skippable_classes param to matcher<commit_after># -*- coding: utf-8 -*-
# This file is part of Shuup.
#
# Copyright (c) 2012-2016, Shoop Commerce Ltd. All rights reserved.
#
# This source code is licensed under the AGPLv3 license found in the
# LICENSE file in the root directory of this source tree.
from django.db.models import ManyToManyField

from shuup.apps.provides import get_provide_objects
from shuup.core.models import Category, Product, ProductType, ShopProduct

def get_matching_for_product(shop_product, provide_category, skippable_classes=None):
    """
    Get matching ids for shop product based on provide category

    For example:
    matching_ids = get_matching_for_product(shop_product, "campaign_catalog_filter")

    :param shop_product: A Shop Product
    :type shop_product: shuup.core.models.ShopProduct
    :param provide_category: Provide category name
    :type provide_category: str
    :param skip: Classes to skip
    :type skip: None or list
    :return: list of collected item ids
    :rtype: list[int]
    """
    collected = set()
    matcher = ProductCampaignMatcher(shop_product)
    for item in get_provide_objects(provide_category):
        if skippable_classes:
            objects = item._meta.model.objects.not_instance_of(*skippable_classes).all()
        else:
            objects = item._meta.model.objects.all()
        for obj in objects:
            if matcher.matches(obj):
                collected.add(obj.pk)
    return collected

class ProductCampaignMatcher(object):
    """
    A class to match shop product to existing campaigns
    """

    def __init__(self, shop_product, **kwargs):
        """
        :param shop_product: a Shop Product to find matches for
        :type shop_product: shuup.core.models.ShopProduct
        """
        assert isinstance(shop_product, ShopProduct)

        self.shop = shop_product.shop
        self.product = shop_product.product
        self.shop_product = shop_product

    def matches(self, obj):
        """
        Tries to match filters, conditions, effects etc to shop product

        :return: True or False based on the match
        :rtype: bool
        """
        _matches = []
        for field in obj._meta.get_fields():
            related_model = field.related_model
            if self._is_product_kind(related_model):
                _matches.append(self.product_matcher(field, obj))
            elif self._is_category(related_model):
                _matches.append(self.category_matcher(field, obj))
            elif self._is_product_type(related_model):
                _matches.append(self.product_type_matcher(field, obj))
            else:
                _matches.append(False)
        return any(_matches)

    def product_type_matcher(self, field, obj):
        if isinstance(field, ManyToManyField):
            return self._product_types_match(field.name, obj)
        else:
            return self._product_type_matches(field.name, obj)

    def category_matcher(self, field, obj):
        if isinstance(field, ManyToManyField):
            return self._categories_match(field.name, obj)
        else:
            return self._category_matches(field.name, obj)

    def product_matcher(self, field, obj):
        if isinstance(field, ManyToManyField):
            return self._products_match(field, obj)
        else:
            return self._product_matches(field, obj)

    def _product_types_match(self, field_name, obj):
        return getattr(obj, field_name).filter(pk=self.product.type.pk).exists()

    def _product_type_matches(self, field_name, obj):
        attr = getattr(obj, field_name)
        if not attr:
            return False
        return (self.product.type.pk == attr.pk)

    def _products_match(self, field, obj):
        attr = getattr(obj, field.name)
        if not attr:
            return False
        if self._is_product(field.related_model):
            return attr.filter(pk=self.product.pk).exists()
        elif self._is_shop_product(field.related_model):
            return attr.filter(product__id=self.product.pk).exists()
        return False

    def _product_matches(self, field, obj):
        attr = getattr(obj, field.name)
        if not attr:
            return False
        if self._is_product(field.related_model):
            return (self.product.pk == attr.pk)
        elif self._is_shop_product(field.related_model):
            return (self.product.pk == attr.product.pk)
        return False

    def _categories_match(self, field_name, obj):
        existing = set(getattr(obj, field_name).values_list("pk", flat=True))
        if self.shop_product.primary_category and self.shop_product.primary_category.pk in existing:
            return True
        cats = set(self.shop_product.categories.values_list("pk", flat=True))
        if cats.intersection(existing):
            return True
        return False

    def _category_matches(self, field_name, obj):
        attr = getattr(obj, field_name)
        if not attr:
            return False
        if self.shop_product.primary_category and self.shop_product.primary_category.pk == attr.pk:
            return True
        return self.shop_product.categories.filter(pk=attr.pk).exists()

    def _is_product_kind(self, model):
        return (self._is_product(model) or self._is_shop_product(model))

    def _is_product(self, model):
        if model is None:
            return False
        return isinstance(model(), Product)

    def _is_shop_product(self, model):
        if model is None:
            return False
        return isinstance(model(), ShopProduct)

    def _is_category(self, model):
        if model is None:
            return False
        return isinstance(model(), Category)

    def _is_product_type(self, model):
        if model is None:
            return False
        return isinstance(model(), ProductType)
<|endoftext|>


<commit_before>import numpy as np
from scipy.signal import hilbert
import math

from synthesis import sample_time, sine
from playback import generate_and_play

def amplitude_envelope(x):
    return abs(hilbert(x))

def split_to_blocks(x, block_size=1024, hop_size=None, fs=44100):
    '''
    Splits the input audio signal to block of given size (in samples).
    Start position of each block is determined by given hop size.
    The last block is right zero-padded if needed.
    input:
    x - array-like representing the audio signal
    '''
    if hop_size is None:
        hop_size = block_size
    block_count = math.ceil(len(x) / hop_size)
    def pad(x, size, value=0):
        padding_size = size - len(x)
        if padding_size:
            x = np.pad(x, (0, padding_size), 'constant', constant_values=(0, 0))
        return x
    blocks = np.vstack(
        pad(x[start:start + block_size], block_size) \
        for start in range(0, hop_size * block_count, hop_size))
    times = np.arange(0, len(x)/fs, hop_size/fs)
    return blocks, times

def test_split_to_blocks():
    assert np.array([
        [ 0,  1,  2,  3,  4,  5,  6,  7],
        [ 6,  7,  8,  9, 10, 11, 12, 13],
        [12, 13, 14, 15, 16, 17, 18, 19],
        [18, 19, 20, 21, 22,  0,  0,  0],
    ]) == split_to_blocks(np.arange(23), 8, 6)

if __name__ == '__main__':

    # Dissonance of two sines f_1 and f_2 is like
    # amplitude modulation of abs(f_1 - f_2) onto mean(f_1, f_2)
    # The amplitude envelope can be obtained as the
    # absolute value of the analytical signal (x + i * h(x))
    # where h is the Hilbert transform.
    # abs(scipy.signal.hilbert(x))
    # It corresponds to the AM demodulation.
    # The beating frequency is 2 * (f_1 - f_2), since its absolute
    # value has twice higher the period.
    t = sample_time(0, 1, 200)
    x = sine(t, 10)  + sine(t, 12)
    e = amplitude_envelope(x)
    plot(t, x)
    plot(t, e)
    # generate_and_play(lambda t: np.sum(sine(t, f) for f in (440, 445)), duration=3)

    # derivative of the amplitude envelope
    # plot(t[:-1], abs(np.diff(abs(hilbert(x)))))
<commit_msg>Extract a function to compute STFT block start times.<commit_after>import numpy as np
from scipy.signal import hilbert
import math

from synthesis import sample_time, sine
from playback import generate_and_play

def amplitude_envelope(x):
    return abs(hilbert(x))

def split_to_blocks(x, block_size=1024, hop_size=None, fs=44100):
    '''
    Splits the input audio signal to block of given size (in samples).
    Start position of each block is determined by given hop size.
    The last block is right zero-padded if needed.
    input:
    x - array-like representing the audio signal
    '''
    if hop_size is None:
        hop_size = block_size
    block_count = math.ceil(len(x) / hop_size)
    def pad(x, size, value=0):
        padding_size = size - len(x)
        if padding_size:
            x = np.pad(x, (0, padding_size), 'constant', constant_values=(0, 0))
        return x
    blocks = np.vstack(
        pad(x[start:start + block_size], block_size) \
        for start in range(0, hop_size * block_count, hop_size))
    times = split_block_times(len(x), fs, hop_size)
    return blocks, times

def split_block_times(N, fs, hop_size):
    return np.arange(0, N/fs, hop_size/fs)

def test_split_to_blocks():
    assert np.array([
        [ 0,  1,  2,  3,  4,  5,  6,  7],
        [ 6,  7,  8,  9, 10, 11, 12, 13],
        [12, 13, 14, 15, 16, 17, 18, 19],
        [18, 19, 20, 21, 22,  0,  0,  0],
    ]) == split_to_blocks(np.arange(23), 8, 6)

if __name__ == '__main__':

    # Dissonance of two sines f_1 and f_2 is like
    # amplitude modulation of abs(f_1 - f_2) onto mean(f_1, f_2)
    # The amplitude envelope can be obtained as the
    # absolute value of the analytical signal (x + i * h(x))
    # where h is the Hilbert transform.
    # abs(scipy.signal.hilbert(x))
    # It corresponds to the AM demodulation.
    # The beating frequency is 2 * (f_1 - f_2), since its absolute
    # value has twice higher the period.
    t = sample_time(0, 1, 200)
    x = sine(t, 10)  + sine(t, 12)
    e = amplitude_envelope(x)
    plot(t, x)
    plot(t, e)
    # generate_and_play(lambda t: np.sum(sine(t, f) for f in (440, 445)), duration=3)

    # derivative of the amplitude envelope
    # plot(t[:-1], abs(np.diff(abs(hilbert(x)))))
<|endoftext|>


<commit_before>#from PIL import Image
import sys
import time
import os
import struct
import threading
import uuid

import usb.core
import usb.util
import binascii
import zmq
import json

USB_TIMEOUT = 5000 # Timeout in MS

class Serial_Bus(object):
    def __init__(self):
        import serial
        self.ser = serial.Serial('/dev/ttyACM2')  # open serial port
        self.long_values = {}
        self.long_names  = {}
        self.hosts       = {}
    def read(self):

        values_list = []

        recv_data = bytearray(self._read_device())

        packets = self._decode_packet(recv_data)

        for packet in packets:
            print("packet:",packet,"::",packet[2] & 1)
            if len(packet) >= 2:
                if packet[0] not in self.hosts:
                    self.req_addr(packet[0])
                elif packet[2] & 1 == 1:
                    self.long_names[self.hosts[packet[0]]+"-"+str(packet[2] & ~1)] = packet[3]
                    self.long_values[packet[3]] = (self.hosts[packet[0]],packet[2] & ~1)
                    #print(self.long_names)
                else:
                    long_name = str(self.hosts[packet[0]])+"-"+str(packet[2] & ~1)
                    #print("long_name",self.long_names,long_name)
                    if (long_name in self.long_names) and (packet[0] in self.hosts):
                        values_dict = { "station_id": self.hosts[packet[0]], "value" : packet[4], "sequence_no": packet[3] }
                        values_long = [self.long_names[long_name],"*",json.dumps(values_dict),str(packet[4])]
                        values_list.append(values_long)
        return values_list

    def _decode_packet(self,recv_data):

        if len(recv_data) < 8:
             return []

        args = list(struct.unpack("<BBBBi", recv_data[:8]))
        print("line:",args)
        args[3] = args[3] * 2
        if args[2] == 0x88:
            #print("data:",bytes(recv_data[4:22]))
            uuid_str = str(uuid.UUID(bytes=bytes(recv_data[3:21])))

            self.hosts[args[0]] = uuid_str

            data = ["UUID/"+uuid_str]
            return [tuple(list(args[0:3]) + data)]
        elif(args[2] & 1 == 1):
            #print("descriptor type",recv_data[5] >> 4)

            if args[4] & 0xf == 1: # Onewire bus address of sensor.
                addr=[]
                #print("raw addr:",recv_data)
                for byte_data in recv_data[5:13]:
                    addr = [byte_data] + addr
                data = ["Temp/"+binascii.hexlify(bytearray([recv_data[6]]))+'-'+binascii.hexlify(bytearray(addr))]
            else:
                name = []
                for ch in recv_data[4:]:
                    if ch != 0:
                        name.append(ch)
                    else:
                        break
                data = [bytearray(name).decode('UTF-8')+"/"+self.hosts[args[0]]+"/"+str(args[2] & ~0x1),recv_data[3] >> 4]
            return [tuple(list(args[0:3]) + data)]
        else:
            lst = [tuple(args)]
            pos = 8
            for item in range((len(recv_data) - 8) // 4):
               lst.append((args[0],args[1],args[2]+2+item*2,(args[3]+item+1),struct.unpack("<i",recv_data[pos:pos+4])[0]))
               pos = pos + 4
            return lst

    def _read_device(self):
        line=self.ser.readline();
        print(line[0],' ',line)
        if line[0] == 68:
            args = line.strip()[1:-1].split(b',')
            args = map(lambda x: int(x,16),args)
            return args
        if line[0] == 80:
            print('Device DBG:',line[1:])
        return ()

class TFT_FastBus(Serial_Bus):
    def __init__(self):
        self.long_values = {}
        self.long_names  = {}
        self.hosts       = {}
        #super(TFT_FastBus, self).__init__()
        self.dev = usb.core.find(idVendor=0x0483,idProduct=0x5740)
        self.write_lock = threading.RLock()

        for interface in self.dev[0]:
            if interface.bInterfaceClass == 0xff:
                print(interface)
                self.fast_endpoint = interface[0]
                self.rx_endpoint = interface[1]
                #print self.rx_endpoint
        #self.rx_endpoint = self.dev[0][(1,0)][1]
        #self.endpoint_out  = self.dev[0][(1,0)][0] #.bEndpointAddress
        #self.endpoint_in   = self.dev[0][(1,0)][1]
        self.tm_sec = -1

    def write(self,str):
        self.write_lock.acquire()
        self.fast_endpoint.write(struct.pack("<H",0x1000) + str)
        self.write_lock.release()

    def write_addr(self,str):
        self.write_lock.acquire()
        self.fast_endpoint.write(struct.pack("<H",0x1000) + str)
        self.write_lock.release()

    def req_addr(self,count):
        self.write_lock.acquire()
        self.fast_endpoint.write(struct.pack("<HB",0x2000,count))
        self.write_lock.release()

    def _read_device(self):
        try:
            print(self.rx_endpoint.wMaxPacketSize)
            recv_data = self.rx_endpoint.read(self.rx_endpoint.wMaxPacketSize, timeout=5000)
            print("recv_data:",recv_data)
            if (len(recv_data) < 8):
                return ()

            return recv_data
            #str_out = ""
            #for ch in str[2:]:
            #    str_out = str_out + chr(ch)
            #if len(str) == 4:
            #return struct.unpack("<HH",str)
            #return str_out
        except usb.core.USBError as e:
            if e.backend_error_code != -116:
                print(e.backend_error_code)
                raise e
            #print(e)
            return ()

if __name__ == "__main__":
    data_bus = TFT_FastBus()
    thread_context = zmq.Context()
    sock_send = thread_context.socket(zmq.PUB)
    sock_send.connect("tcp://192.168.1.116:10901")
    try:
      #data_bus.write("wibble".encode('utf-8'))
      data_bus.req_addr(10);
      while True:
        values_lst = data_bus.read()
        for values in values_lst:
            print(values)
            if values:
                sock_send.send_multipart([str(i).encode('UTF-8') for i in values])

    except KeyError:
        tempreture_log_file.close()
        #tft.fast_endpoint.write(struct.pack("<H",0x4000) + str(x))
        #print x
        #print time.gmtime().tm_min,

        #.split('$')
        #if (int(args[2]) < 3500):
<commit_msg>Auto numbering support in the python interface library.<commit_after>#from PIL import Image
import sys
import time
import os
import struct
import threading
import uuid
import traceback

import usb.core
import usb.util
import binascii
import zmq
import json

USB_TIMEOUT = 5000 # Timeout in MS

class Serial_Bus(object):
    def __init__(self):
        import serial
        self.ser = serial.Serial('/dev/ttyACM2')  # open serial port
        self.long_values = {}
        self.long_names  = {}
        self.hosts       = {}
    def read(self):

        values_list = []

        recv_data = bytearray(self._read_device())

        packets = self._decode_packet(recv_data)

        for packet in packets:
            print("packet:",packet,"::",packet[2] & 1)
            if len(packet) >= 2:
                if packet[0] not in self.hosts:
                    self.req_addr(packet[0])
                elif packet[2] & 1 == 1:
                    self.long_names[self.hosts[packet[0]]+"-"+str(packet[2] & ~1)] = packet[3]
                    self.long_values[packet[3]] = (self.hosts[packet[0]],packet[2] & ~1)
                    #print(self.long_names)
                else:
                    long_name = str(self.hosts[packet[0]])+"-"+str(packet[2] & ~1)
                    #print("long_name",self.long_names,long_name)
                    if (long_name in self.long_names) and (packet[0] in self.hosts):
                        values_dict = { "station_id": packet[0], "value" : packet[4], "sequence_no": packet[3] }
                        values_long = [self.long_names[long_name],"*",json.dumps(values_dict),str(packet[4])]
                        values_list.append(values_long)
                    elif (packet[2] < 128):
                        print("Request descriptor",packet[0])
                        self.req_descriptors(packet[0])
        return values_list

    def _decode_packet(self,recv_data):

        if len(recv_data) < 8:
             return []

        args = list(struct.unpack("<BBBBi", recv_data[:8]))
        print("line:",args)
        args[3] = args[3] * 2
        if args[2] == 0x88:
            print("uuid:",bytes(recv_data[4:22]))
            uuid_str = str(uuid.UUID(bytes=bytes(recv_data[3:21])))

            self.hosts[args[0]] = uuid_str

            data = ["UUID/"+uuid_str]
            return [tuple(list(args[0:3]) + data)]
        elif args[2] == 0x86:
            print("uuid:",bytes(recv_data[4:22]))
            uuid_str = str(uuid.UUID(bytes=bytes(recv_data[4:22])))

            self.hosts[args[0]] = uuid_str

            data = ["UUID/"+uuid_str]
            return [tuple(list(args[0:3]) + data)]
        elif(args[2] & 1 == 1):
            #print("descriptor type",recv_data[5] >> 4)

            if args[4] & 0xf == 1: # Onewire bus address of sensor.
                addr=[]
                #print("raw addr:",recv_data)
                for byte_data in recv_data[5:13]:
                    addr = [byte_data] + addr
                data = ["Temp/"+binascii.hexlify(bytearray([recv_data[6]]))+'-'+binascii.hexlify(bytearray(addr))]
            else:
                name = []
                for ch in recv_data[4:]:
                    if ch != 0:
                        name.append(ch)
                    else:
                        break
                data = [bytearray(name).decode('UTF-8')+"/"+self.hosts[args[0]]+"/"+str(args[2] & ~0x1),recv_data[3] >> 4]
            return [tuple(list(args[0:3]) + data)]
        else:
            lst = [tuple(args)]
            pos = 8
            for item in range((len(recv_data) - 8) // 4):
               lst.append((args[0],args[1],args[2]+2+item*2,(args[3]+item+1),struct.unpack("<i",recv_data[pos:pos+4])[0]))

               print(pos,recv_data[pos:pos+4])

               pos = pos + 4
            print(lst)
            return lst

    def _read_device(self):
        line=self.ser.readline();
        print(line[0],' ',line)
        if line[0] == 68:
            args = line.strip()[1:-1].split(b',')
            args = map(lambda x: int(x,16),args)
            return args
        if line[0] == 80:
            print('Device DBG:',line[1:])
        return ()

class TFT_FastBus(Serial_Bus):
    def __init__(self):
        self.long_values = {}
        self.long_names  = {}
        self.hosts       = {}
        #super(TFT_FastBus, self).__init__()
        self.dev = usb.core.find(idVendor=0x0483,idProduct=0x5740)
        self.write_lock = threading.RLock()

        for interface in self.dev[0]:
            if interface.bInterfaceClass == 0xff:
                print(interface)
                self.fast_endpoint = interface[0]
                self.rx_endpoint = interface[1]
                #print self.rx_endpoint
        #self.rx_endpoint = self.dev[0][(1,0)][1]
        #self.endpoint_out  = self.dev[0][(1,0)][0] #.bEndpointAddress
        #self.endpoint_in   = self.dev[0][(1,0)][1]
        self.tm_sec = -1

    def write(self,str):
        self.write_lock.acquire()
        self.fast_endpoint.write(struct.pack("<H",0x1000) + str)
        self.write_lock.release()

    def write_addr(self,str):
        self.write_lock.acquire()
        self.fast_endpoint.write(struct.pack("<H",0x1000) + str)
        self.write_lock.release()

    def req_descriptors(self,count):
        self.write_lock.acquire()
        self.fast_endpoint.write(struct.pack("<HBB",0x2001,count,0x2)) # 0x2 Request descriptors flag.
        self.write_lock.release()

    def req_addr(self,count):
        self.write_lock.acquire()
        self.fast_endpoint.write(struct.pack("<HB",0x2000,count))
        self.write_lock.release()

    def _read_device(self):
        try:
            print(self.rx_endpoint.wMaxPacketSize)
            recv_data = self.rx_endpoint.read(self.rx_endpoint.wMaxPacketSize, timeout=5000)
            print("recv_data:",recv_data)
            if (len(recv_data) < 8):
                return ()

            return recv_data
            #str_out = ""
            #for ch in str[2:]:
            #    str_out = str_out + chr(ch)
            #if len(str) == 4:
            #return struct.unpack("<HH",str)
            #return str_out
        except usb.core.USBError as e:
            if e.backend_error_code != -116 and e.backend_error_code != -7:
                print("e.backend_error_code",e.backend_error_code)
                raise e
            #print(e)
            return ()

if __name__ == "__main__":
    data_bus = TFT_FastBus()
    thread_context = zmq.Context()
    sock_send = thread_context.socket(zmq.PUB)
    sock_send.connect("tcp://192.168.1.116:10901")
    try:
      #data_bus.write("wibble".encode('utf-8'))
      while True:
        try:
          values_lst = data_bus.read()
          for values in values_lst:
            print("values:",values)
            if values:
                sock_send.send_multipart([str(i).encode('UTF-8') for i in values])
        except:
          traceback.print_exc(file=sys.stderr)
    except KeyError:
        pass
        #tempreture_log_file.close()
        #tft.fast_endpoint.write(struct.pack("<H",0x4000) + str(x))
        #print x
        #print time.gmtime().tm_min,

        #.split('$')
        #if (int(args[2]) < 3500):
<|endoftext|>


<commit_before>import numpy as np
from scipy.ndimage.measurements import maximum_position, label, find_objects
from scipy.ndimage.morphology import generate_binary_structure
from skimage.measure import regionprops

# TODO: add algorithm in 1D - just searching peaks in freq. averaged
# de-dispersed auto-spectra.

def find_peaks(array, nstd=4, med_width=30, gauss_width=2):
    import scipy
    array = scipy.signal.medfilt(array, med_width)
    garray = scipy.ndimage.filters.gaussian_filter1d(array, gauss_width)
    ind = garray[garray > nstd * np.std(garray)]
    return ind

def max_pos(object, image):
    """
    Returns maximum position and widths in both direction.

    :param object:
        ``skimage.measure._regionprops._RegionProperties`` instance.
    :param image:
        Original image
    :return:
        Tuple of max position & 2 widths of region.
    """
    subimage = image[object.bbox[0]: object.bbox[2],
               object.bbox[1]: object.bbox[3]]
    indx = np.unravel_index(subimage.argmax(), subimage.shape)
    return (object.bbox[0] + indx[0], object.bbox[1] + indx[1]),\
            object.bbox[2] - object.bbox[0], object.bbox[3] - object.bbox[1]

def search_candidates(image, threshold, n_d_x, n_d_y):

    threshold = np.percentile(image.ravel(), threshold)
    a = image.copy()
    # Keep only tail of image values distribution with signal
    a[a < threshold] = 0
    s = generate_binary_structure(2, 2)
    # Label image
    labeled_array, num_features = label(a, structure=s)
    # Find objects
    objects = find_objects(labeled_array)
    props = regionprops(labeled_array, intensity_image=image)
    # Container of object's properties
    _objects = np.empty(num_features, dtype=[('label', 'int'),
                                             ('dx', '<f8'),
                                             ('dy', '<f8'),
                                             ('max_pos', 'int',
                                              (2,))])

    labels = np.arange(num_features) + 1
    dx = [int(obj[1].stop - obj[1].start) for obj in objects]
    dy = [int(obj[0].stop - obj[0].start) for obj in objects]

    # Filling objects structured array
    _objects['label'] = labels
    _objects['dx'] = dx
    _objects['dy'] = dy
    # Classify objects
    _objects = _objects[np.logical_and(_objects['dy'] > n_d_y,
                                       _objects['dx'] > n_d_x)]
    # Fetch positions of only successfuly classified objects
    _objects['max_pos'] = maximum_position(image, labels=labeled_array,
                                           index=_objects['label'])
    _objects = _objects[np.lexsort((_objects['dx'], _objects['dy']))[::-1]]

    return _objects

def get_props(image, threshold):
    """
    Rerurn measured properties list of imaged labeled at specified threshold.

    :param image:
        Numpy 2D array with image.
    :param threshold:
        Threshold to label image. [0.-100.]

    :return:
        List of RegionProperties -
        (``skimage.measure._regionprops._RegionProperties`` instances)
    """
    threshold = np.percentile(image.ravel(), threshold)
    a = image.copy()
    # Keep only tail of image values distribution with signal
    a[a < threshold] = 0
    s = generate_binary_structure(2, 2)
    # Label image
    labeled_array, num_features = label(a, structure=s)
    return regionprops(labeled_array, intensity_image=image)

<commit_msg>+docs for 1d peak finding<commit_after>import numpy as np
from scipy.ndimage.measurements import maximum_position, label, find_objects
from scipy.ndimage.morphology import generate_binary_structure
from skimage.measure import regionprops

# TODO: add algorithm in 1D - just searching peaks in freq. averaged
# de-dispersed auto-spectra.

def find_peaks(array_like, n_std=4, med_width=30, gauss_width=2):
    """
    Find peaks in 1D array.

    Data first median filtered with window size ``med_width``. Then it convolved
    with gaussian filter of width ``gauss_width``. Indexes of data with values
    higher then ``n_std`` are returned.

    :param array_like:
        Iterable of data values.
    :param med_width: (optional)
        Width of median filter to preprocess data. (default: ``30``)
    :param gauss_width: (optional)
        Width of gaussian filter to convolve median filtered data. (default:
        ``2``)
    :param n_std: (optional)
        Number of standard deviations to consider when searching peaks.
        (default: 4)

    :return:
        Numpy array of indexes of peak values.
    """
    import scipy
    array = np.asarray(array_like)
    array = scipy.signal.medfilt(array, med_width)
    garray = scipy.ndimage.filters.gaussian_filter1d(array, gauss_width)
    ind = garray[garray > n_std * np.std(garray)]
    return ind

def max_pos(object, image):
    """
    Returns maximum position and widths in both direction.

    :param object:
        ``skimage.measure._regionprops._RegionProperties`` instance.
    :param image:
        Original image
    :return:
        Tuple of max position & 2 widths of region.
    """
    subimage = image[object.bbox[0]: object.bbox[2],
               object.bbox[1]: object.bbox[3]]
    indx = np.unravel_index(subimage.argmax(), subimage.shape)
    return (object.bbox[0] + indx[0], object.bbox[1] + indx[1]),\
            object.bbox[2] - object.bbox[0], object.bbox[3] - object.bbox[1]

def search_candidates(image, threshold, n_d_x, n_d_y):

    threshold = np.percentile(image.ravel(), threshold)
    a = image.copy()
    # Keep only tail of image values distribution with signal
    a[a < threshold] = 0
    s = generate_binary_structure(2, 2)
    # Label image
    labeled_array, num_features = label(a, structure=s)
    # Find objects
    objects = find_objects(labeled_array)
    props = regionprops(labeled_array, intensity_image=image)
    # Container of object's properties
    _objects = np.empty(num_features, dtype=[('label', 'int'),
                                             ('dx', '<f8'),
                                             ('dy', '<f8'),
                                             ('max_pos', 'int',
                                              (2,))])

    labels = np.arange(num_features) + 1
    dx = [int(obj[1].stop - obj[1].start) for obj in objects]
    dy = [int(obj[0].stop - obj[0].start) for obj in objects]

    # Filling objects structured array
    _objects['label'] = labels
    _objects['dx'] = dx
    _objects['dy'] = dy
    # Classify objects
    _objects = _objects[np.logical_and(_objects['dy'] > n_d_y,
                                       _objects['dx'] > n_d_x)]
    # Fetch positions of only successfuly classified objects
    _objects['max_pos'] = maximum_position(image, labels=labeled_array,
                                           index=_objects['label'])
    _objects = _objects[np.lexsort((_objects['dx'], _objects['dy']))[::-1]]

    return _objects

def get_props(image, threshold):
    """
    Rerurn measured properties list of imaged labeled at specified threshold.

    :param image:
        Numpy 2D array with image.
    :param threshold:
        Threshold to label image. [0.-100.]

    :return:
        List of RegionProperties -
        (``skimage.measure._regionprops._RegionProperties`` instances)
    """
    threshold = np.percentile(image.ravel(), threshold)
    a = image.copy()
    # Keep only tail of image values distribution with signal
    a[a < threshold] = 0
    s = generate_binary_structure(2, 2)
    # Label image
    labeled_array, num_features = label(a, structure=s)
    return regionprops(labeled_array, intensity_image=image)

<|endoftext|>