<commit_before>#!/usr/bin/env python
"""
HiveMind Node
Developed by Trevor Stanhope
Hive sensor node based on RaspberryPi and Arduino.

TODO:
- Authenticate to aggregator?
- Authenticate to server?
- Validate data received from Arduino
- Add computer vision components
"""

# Libraries
import zmq
import ast
import json
import os
import sys
try:
    import pyaudio
except Exception:
    pass
import cherrypy
import numpy as np
import random
import urllib2
from datetime import datetime
from serial import Serial, SerialException
from ctypes import *
from cherrypy.process.plugins import Monitor
from cherrypy import tools
import logging
import socket
import cv2

try:
    import Adafruit_DHT
except Exception:
    pass

try:
    import Adafruit_BMP.BMP085 as BMP085
except Exception:
    pass

# Constants
try:
    CONFIG_FILE = sys.argv[1]
except Exception as err:
    CONFIG_FILE = None

# Error Handling
ERROR_HANDLER_FUNC = CFUNCTYPE(None, c_char_p, c_int, c_char_p, c_int, c_char_p)
def py_error_handler(filename, line, function, err, fmt):
  pass
C_ERROR_HANDLER = ERROR_HANDLER_FUNC(py_error_handler)

# Node
class HiveNode:

    ## Initialize
    def __init__(self, config):

        # Configuration
        if not config:
            self.REBOOT_ENABLED = False
            self.ZMQ_SERVER = "tcp://192.168.0.100:1980"
            self.ZMQ_TIMEOUT = 5000
            self.WAN_URL = "http://127.0.0.1:5000/new"
            self.ARDUINO_DEV = "/dev/ttyS0"
            self.ARDUINO_BAUD = 9600
            self.ARDUINO_TIMEOUT = 3
            self.MICROPHONE_CHANNELS = 1
            self.MICROPHONE_RATE = 44100
            self.MICROPHONE_CHUNK = 4096
            self.MICROPHONE_FORMAT = 8
            self.CAMERA_INDEX = 0
            self.CHERRYPY_PORT = 8081
            self.CHERRYPY_ADDR ="0.0.0.0"
            self.PING_INTERVAL = 1
            self.LOG_FILE = "log.txt"
            self.PARAMS = ["int_t","ext_t","int_h","ext_h","volts","amps","hz","db","pa"]
            self.HIVE_ID = socket.gethostname()
        else:
            self.load_config(config)

        # Initializers
        self.init_tasks()
        self.init_csv()
        self.init_zmq()
        self.init_logging()
        self.init_arduino()
        self.init_DHT()
        self.init_BMP()
        self.init_mic()
        self.init_cam()

    ## Load Config File
    def load_config(self, config):
        self.log_msg('CONFIG', 'Loading Config File')
        with open(config) as config_file:
            settings = json.loads(config_file.read())
            for key in settings:
                try:
                    getattr(self, key)
                except AttributeError as error:
                    self.log_msg('CONFIG', '%s : %s' % (key, str(settings[key])))
                    setattr(self, key, settings[key])

    ## Initialize tasks
    def init_tasks(self):
        self.log_msg('ENGINE', 'Initializing cherrypy monitor tasks ...')
        try:
            Monitor(cherrypy.engine, self.update, frequency=self.PING_INTERVAL).subscribe()
        except Exception as error:
            self.log_msg('ENGINE', 'Error: %s' % str(error))

    ## Initialize CSV backups
    def init_csv(self):
        self.NODE_DIR = os.path.dirname(os.path.abspath(__file__))
        for param in self.PARAMS:
            try:
                csv_path = os.path.join(self.NODE_DIR, 'data', param + '.csv')
                csv_file = open(csv_path, 'a')
                self.log_msg('CSV', 'Using EXISTING file for %s' % param)
            except Exception:
                self.log_msg('CSV', 'Using NEW file for %s' % param)
                with open(csv_path, 'w') as csv_file:
                    csv_file.write('date,val,\n') # no spaces!

    ## Initialize ZMQ messenger
    def init_zmq(self):
        self.log_msg('ZMQ', 'Initializing ZMQ client ...')
        try:
            self.context = zmq.Context()
            self.socket = self.context.socket(zmq.REQ)
            self.socket.connect(self.ZMQ_SERVER)
            self.poller = zmq.Poller()
            self.poller.register(self.socket, zmq.POLLIN)
            msg = 'OK'
        except Exception as error:
            msg = 'Error: %s' % str(error)
        self.log_msg('ZMQ', msg)

    ## Initialize Logging
    def init_logging(self):
        self.log_msg('LOG', 'Initializing logging ...')
        try:
            logging.basicConfig(filename=self.LOG_FILE,level=logging.DEBUG)
            msg = 'OK'
        except Exception as error:
            msg = 'Error: %s' % str(error)
        self.log_msg('LOG', msg)

    ## Initialize Arduino
    def init_arduino(self):
        self.log_msg('CTRL', 'Initializing controller ...')
        try:
            self.arduino = Serial(self.ARDUINO_DEV, self.ARDUINO_BAUD, timeout=self.ARDUINO_TIMEOUT)
            msg = 'OK'
        except Exception as error:
            msg = 'Error: %s' % str(error)
        self.log_msg('CTRL', msg)

    ## Initialize DHT Sensor
    def init_DHT(self, sensor=Adafruit_DHT.AM2302, pin=11):
        self.log_msg('DHT', 'Initializing DHT sensor ...')
        try:
            humidity, temperature = Adafruit_DHT.read_retry(sensor, pin)
        except Exception as error:
            self.log_msg('DHT', 'Error: %s' % str(error))

    ## Initialize BMP Sensor
    def init_BMP(self):
        self.log_msg('BMP', 'Initializing BMP sensor ...')
        try:
            self.BMP085 = BMP085.BMP085()
        except Exception as error:
            self.log_msg('BMP', 'Error: %s' % str(error))

    ## Initialize Microphone
    def init_mic(self):
        self.log_msg('MIC', 'Initializing microphone ...')
        try:
            asound = cdll.LoadLibrary('libasound.so')
            asound.snd_lib_error_set_handler(C_ERROR_HANDLER) # Set error handler
            mic = pyaudio.PyAudio()
            self.microphone = mic.open(
                format=self.MICROPHONE_FORMAT,
                channels=self.MICROPHONE_CHANNELS,
                rate=self.MICROPHONE_RATE,
                input=True,
                frames_per_buffer=self.MICROPHONE_CHUNK
            )
            self.microphone.stop_stream()
            msg = 'OK'
        except Exception as error:
            msg = 'Error: %s' % str(error)
        self.log_msg('MIC', msg)

    ## Initialize camera
    def init_cam(self):
        self.log_msg('CAM', 'Initializing camera ...')
        try:
            self.camera = cv2.VideoCapture(self.CAMERA_INDEX)
            self.log_msg('CAM', 'OK')
        except Exception as error:
            self.log_msg('CAM', 'Error: %s' % str(error))

    ## Capture Audio
    def capture_audio(self):
        self.log_msg('MIC', 'Capturing audio segment ...')
        rms_decibels = None
        dominant_hertz = None
        try:
            self.microphone.start_stream()
            data = self.microphone.read(self.MICROPHONE_CHUNK)
            self.microphone.stop_stream()
            wave_array = np.fromstring(data, dtype='int16')
            wave_fft = np.fft.fft(wave_array)
            wave_freqs = np.fft.fftfreq(len(wave_fft))
            dominant_peak = np.argmax(np.abs(wave_fft))
            dominant_hertz = self.MICROPHONE_RATE*abs(wave_freqs[dominant_peak])
            dominant_amplitude = np.sqrt(np.abs(wave_fft[dominant_peak])**2)
            dominant_decibels = 10*np.log10(dominant_amplitude)
            rms_amplitude = np.sqrt(np.mean(np.abs(wave_fft)**2))
            rms_decibels =  10*np.log10(rms_amplitude)
            sorted_peaks = np.argsort(np.abs(wave_fft))
            sorted_hertz = self.MICROPHONE_RATE*abs(wave_freqs[sorted_peaks])
        except Exception as error:
            self.log_msg('MIC', 'Error: %s' % str(error))
        result = {
            'db' : rms_decibels,
            'hz' : dominant_hertz
        }
        self.log_msg('MIC', str(result))
        return result

    ## Capture Video
    def capture_video(self):
        self.log_msg('CAM', 'Capturing video ...')
        num_bees = None
        try:
            (s, bgr) = self.camera.read()
            if s:
                #! TODO Add computer vision analysis to function
                self.log_msg('CAM', 'OK')
            else:
                self.log_msg('CAM', 'Error: failed to get image')
        except Exception as error:
            self.log_msg('CAM', 'Error: %s' % str(error))
        result = {}
        return result

    ## Read Arduino (if available))
    def read_arduino(self):
        self.log_msg('CTRL', 'Reading from controller ...')
        try:
            string = self.arduino.readline()
            result = ast.literal_eval(string)
            self.log_msg('CTRL', 'OK: %s' % str(result))
        except Exception as error:
            result = {}
            self.log_msg('CTRL', 'Error: %s' % str(error))
        return result

    ## Read DHT (if available)
    def read_DHT(self, sensor=Adafruit_DHT.AM2302, pin=11):
        try:
            humidity, temperature = Adafruit_DHT.read_retry(sensor, pin)
            result = {
                "dht_t" : temperature,
                "dht_h" : humidity
            }
        except Exception as error:
            result = {}
            self.log_msg('DHT', 'Error: %s' % str(error))
        return result

    ## Read BMP (if available)
    def read_BMP(self):
        try:
            temperature = self.BMP085.read_temperature()
            pressure = self.BMP085.read_pressure()
            altitude = self.BMP085.read_altitude()
            sealevel_pressure = self.BMP085.read_read_sealevel_pressure()
            result = {
                "bmp_t" : temperature,
                "bmp_a" : altitude,
                "bmp_p" : pressure,
                "bmp_s" : sealevel_pressure
            }
        except Exception as error:
            result = {}
            self.log_msg('BMP', 'Error: %s' % str(error))
        return result

    ## Post sample to server
    def post_sample(self, sample):
        self.log_msg('REST', 'Posting to webserver ...')
        try:
            dump = json.dumps(sample)
            req = urllib2.Request(self.WAN_URL)
            req.add_header('Content-Type','application/json')
            response = urllib2.urlopen(req, dump)
            self.log_msg('REST', 'OK: %s' % str(response.getcode()))
            return response
        except Exception as error:
            self.log_msg('REST', 'Error: %s' % str(error))

    ## Send sample to aggregator
    def zmq_sample(self, sample):
        self.log_msg('ZMQ', 'Pushing to aggregator ...')
        try:
            dump = json.dumps(sample)
            self.socket.send(dump)
            socks = dict(self.poller.poll(self.ZMQ_TIMEOUT))
            if socks:
                if socks.get(self.socket) == zmq.POLLIN:
                    dump = self.socket.recv(zmq.NOBLOCK)
                    response = json.loads(dump)
                    self.log_msg('ZMQ', str(response))
                    result = response
                else:
                    self.log_msg('ZMQ', 'Error: Poll Timeout')
                    result = None
            else:
                self.log_msg('ZMQ', 'Error: Socket Timeout')
        except Exception as error:
            self.log_msg('ZMQ', 'Error: %s' % str(error))
        return result

    ## Save Data
    def save_sample(self, sample):
        self.log_msg('CSV', 'Saving sample to file ...')
        if sample:
            time = datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
            for param in self.PARAMS:
                try:
                    csv_path = os.path.join(self.NODE_DIR, 'data', param + '.csv')
                    with open(csv_path, 'a') as csv_file:
                        csv_file.write(','.join([time, str(sample[param]), '\n']))
                except Exception as error:
                    self.log_msg('CSV', 'Error: Could not write key (%s)' % str(error))

    ## Generate blank sample
    def blank_sample(self):
        sample = {
            'type' : 'sample',
            'time' : datetime.strftime(datetime.now(), '%Y-%m-%d %H:%M:%S'),
            'hive_id' : self.HIVE_ID
            }
        return sample

    ## Update Clock
    def update_clock(self, secs):
        self.TIME_ERROR = time.time() - secs
        self.log_msg('CLOCK', 'dt=%d' % secs)

    ## Log Message
    def log_msg(self, task, msg):
        date = datetime.strftime(datetime.now(), '%d/%b/%Y:%H:%M:%S')
        print('[%s] %s %s' % (date, task, msg))

    ## Shutdown
    def shutdown(self):
        self.log_msg('ENGINE', 'Shutting Down')
        try:
            self.arduino.close()
        except Exception as e:
            self.log_msg('CTRL', str(e))
        try:
            self.microphone.close()
        except Exception as e:
            self.log_msg('MIC', str(e))
        try:
            self.camera.release()
        except Exception as e:
            self.log_msg('CAM', str(e))
        os.system("sudo reboot")

    ## Update to Aggregator
    def update(self):
        sample = self.blank_sample()

        # Arduino
        arduino_result = self.read_arduino()
        sample.update(arduino_result)

        # Mic
        microphone_result = self.capture_audio()
        sample.update(microphone_result)

        # Camera
        camera_result = self.capture_video()
        sample.update(camera_result)

        # BMP
        BMP_result = self.read_BMP()
        sample.update(BMP_result)

        # DHT
        DHT_result = self.read_DHT()
        sample.update(DHT_result)

        try:
            response = self.zmq_sample(sample)
            if response['type'] == 'clock':
                self.log_msg('ZMQ', 'Caught time update request', '')
                self.update_clock['secs']
            if response['type'] == 'config':
                self.log_msg('ZMQ', 'Caught Reload Config Request', '')
        except:
            if self.REBOOT_ENABLED:
                self.shutdown()
        self.post_sample(sample)
        self.save_sample(sample)

    ## Render Index
    @cherrypy.expose
    def index(self):
        with open('static/index.html') as html:
            return html.read()

# Main
if __name__ == '__main__':
    node = HiveNode(config=CONFIG_FILE)
    currdir = os.path.dirname(os.path.abspath(__file__))
    cherrypy.server.socket_host = node.CHERRYPY_ADDR
    cherrypy.server.socket_port = node.CHERRYPY_PORT
    conf = {
        '/': {'tools.staticdir.on':True, 'tools.staticdir.dir':os.path.join(currdir,'static')},
        '/data': {'tools.staticdir.on':True, 'tools.staticdir.dir':os.path.join(currdir,'data')},
        '/js': {'tools.staticdir.on':True, 'tools.staticdir.dir':os.path.join(currdir,'static','js')},
    }
    cherrypy.quickstart(node, '/', config=conf)
<commit_msg>fixed<commit_after>#!/usr/bin/env python
"""
HiveMind Node
Developed by Trevor Stanhope
Hive sensor node based on RaspberryPi and Arduino.

TODO:
- Authenticate to aggregator?
- Authenticate to server?
- Validate data received from Arduino
- Add computer vision components
"""

# Libraries
import zmq
import ast
import json
import os
import sys
try:
    import pyaudio
except Exception:
    pass
import cherrypy
import numpy as np
import random
import urllib2
from datetime import datetime
from serial import Serial, SerialException
from ctypes import *
from cherrypy.process.plugins import Monitor
from cherrypy import tools
import logging
import socket
import cv2

try:
    import Adafruit_DHT
except Exception:
    pass

try:
    import Adafruit_BMP.BMP085 as BMP085
except Exception:
    pass

# Constants
try:
    CONFIG_FILE = sys.argv[1]
except Exception as err:
    CONFIG_FILE = None

# Error Handling
ERROR_HANDLER_FUNC = CFUNCTYPE(None, c_char_p, c_int, c_char_p, c_int, c_char_p)
def py_error_handler(filename, line, function, err, fmt):
  pass
C_ERROR_HANDLER = ERROR_HANDLER_FUNC(py_error_handler)

# Node
class HiveNode:

    ## Initialize
    def __init__(self, config):

        # Configuration
        if not config:
            self.REBOOT_ENABLED = False
            self.ZMQ_SERVER = "tcp://192.168.0.100:1980"
            self.ZMQ_TIMEOUT = 5000
            self.WAN_URL = "http://127.0.0.1:5000/new"
            self.ARDUINO_DEV = "/dev/ttyS0"
            self.ARDUINO_BAUD = 9600
            self.ARDUINO_TIMEOUT = 3
            self.MICROPHONE_CHANNELS = 1
            self.MICROPHONE_RATE = 44100
            self.MICROPHONE_CHUNK = 4096
            self.MICROPHONE_FORMAT = 8
            self.CAMERA_INDEX = 0
            self.CHERRYPY_PORT = 8081
            self.CHERRYPY_ADDR ="0.0.0.0"
            self.PING_INTERVAL = 1
            self.LOG_FILE = "log.txt"
            self.PARAMS = ["int_t","ext_t","int_h","ext_h","volts","amps","hz","db","pa"]
            self.HIVE_ID = socket.gethostname()
        else:
            self.load_config(config)

        # Initializers
        self.init_tasks()
        self.init_csv()
        self.init_zmq()
        self.init_logging()
        self.init_arduino()
        self.init_DHT()
        self.init_BMP()
        self.init_mic()
        self.init_cam()

    ## Load Config File
    def load_config(self, config):
        self.log_msg('CONFIG', 'Loading Config File')
        with open(config) as config_file:
            settings = json.loads(config_file.read())
            for key in settings:
                try:
                    getattr(self, key)
                except AttributeError as error:
                    self.log_msg('CONFIG', '%s : %s' % (key, str(settings[key])))
                    setattr(self, key, settings[key])

    ## Initialize tasks
    def init_tasks(self):
        self.log_msg('ENGINE', 'Initializing cherrypy monitor tasks ...')
        try:
            Monitor(cherrypy.engine, self.update, frequency=self.PING_INTERVAL).subscribe()
        except Exception as error:
            self.log_msg('ENGINE', 'Error: %s' % str(error))

    ## Initialize CSV backups
    def init_csv(self):
        self.NODE_DIR = os.path.dirname(os.path.abspath(__file__))
        for param in self.PARAMS:
            try:
                csv_path = os.path.join(self.NODE_DIR, 'data', param + '.csv')
                csv_file = open(csv_path, 'a')
                self.log_msg('CSV', 'Using EXISTING file for %s' % param)
            except Exception:
                self.log_msg('CSV', 'Using NEW file for %s' % param)
                with open(csv_path, 'w') as csv_file:
                    csv_file.write('date,val,\n') # no spaces!

    ## Initialize ZMQ messenger
    def init_zmq(self):
        self.log_msg('ZMQ', 'Initializing ZMQ client ...')
        try:
            self.context = zmq.Context()
            self.socket = self.context.socket(zmq.REQ)
            self.socket.connect(self.ZMQ_SERVER)
            self.poller = zmq.Poller()
            self.poller.register(self.socket, zmq.POLLIN)
            msg = 'OK'
        except Exception as error:
            msg = 'Error: %s' % str(error)
        self.log_msg('ZMQ', msg)

    ## Initialize Logging
    def init_logging(self):
        self.log_msg('LOG', 'Initializing logging ...')
        try:
            logging.basicConfig(filename=self.LOG_FILE,level=logging.DEBUG)
            msg = 'OK'
        except Exception as error:
            msg = 'Error: %s' % str(error)
        self.log_msg('LOG', msg)

    ## Initialize Arduino
    def init_arduino(self):
        self.log_msg('CTRL', 'Initializing controller ...')
        try:
            self.arduino = Serial(self.ARDUINO_DEV, self.ARDUINO_BAUD, timeout=self.ARDUINO_TIMEOUT)
            msg = 'OK'
        except Exception as error:
            msg = 'Error: %s' % str(error)
        self.log_msg('CTRL', msg)

    ## Initialize DHT Sensor
    def init_DHT(self, sensor=Adafruit_DHT.AM2302, pin=11):
        self.log_msg('DHT', 'Initializing DHT sensor ...')
        try:
            humidity, temperature = Adafruit_DHT.read_retry(sensor, pin)
        except Exception as error:
            self.log_msg('DHT', 'Error: %s' % str(error))

    ## Initialize BMP Sensor
    def init_BMP(self):
        self.log_msg('BMP', 'Initializing BMP sensor ...')
        try:
            self.BMP085 = BMP085.BMP085()
        except Exception as error:
            self.log_msg('BMP', 'Error: %s' % str(error))

    ## Initialize Microphone
    def init_mic(self):
        self.log_msg('MIC', 'Initializing microphone ...')
        try:
            asound = cdll.LoadLibrary('libasound.so')
            asound.snd_lib_error_set_handler(C_ERROR_HANDLER) # Set error handler
            mic = pyaudio.PyAudio()
            self.microphone = mic.open(
                format=self.MICROPHONE_FORMAT,
                channels=self.MICROPHONE_CHANNELS,
                rate=self.MICROPHONE_RATE,
                input=True,
                frames_per_buffer=self.MICROPHONE_CHUNK
            )
            self.microphone.stop_stream()
            msg = 'OK'
        except Exception as error:
            msg = 'Error: %s' % str(error)
        self.log_msg('MIC', msg)

    ## Initialize camera
    def init_cam(self):
        self.log_msg('CAM', 'Initializing camera ...')
        try:
            self.camera = cv2.VideoCapture(self.CAMERA_INDEX)
            self.log_msg('CAM', 'OK')
        except Exception as error:
            self.log_msg('CAM', 'Error: %s' % str(error))

    ## Capture Audio
    def capture_audio(self):
        self.log_msg('MIC', 'Capturing audio segment ...')
        rms_decibels = None
        dominant_hertz = None
        try:
            self.microphone.start_stream()
            data = self.microphone.read(self.MICROPHONE_CHUNK)
            self.microphone.stop_stream()
            wave_array = np.fromstring(data, dtype='int16')
            wave_fft = np.fft.fft(wave_array)
            wave_freqs = np.fft.fftfreq(len(wave_fft))
            dominant_peak = np.argmax(np.abs(wave_fft))
            dominant_hertz = self.MICROPHONE_RATE*abs(wave_freqs[dominant_peak])
            dominant_amplitude = np.sqrt(np.abs(wave_fft[dominant_peak])**2)
            dominant_decibels = 10*np.log10(dominant_amplitude)
            rms_amplitude = np.sqrt(np.mean(np.abs(wave_fft)**2))
            rms_decibels =  10*np.log10(rms_amplitude)
            sorted_peaks = np.argsort(np.abs(wave_fft))
            sorted_hertz = self.MICROPHONE_RATE*abs(wave_freqs[sorted_peaks])
        except Exception as error:
            self.log_msg('MIC', 'Error: %s' % str(error))
        result = {
            'db' : rms_decibels,
            'hz' : dominant_hertz
        }
        self.log_msg('MIC', str(result))
        return result

    ## Capture Video
    def capture_video(self):
        self.log_msg('CAM', 'Capturing video ...')
        num_bees = None
        try:
            (s, bgr) = self.camera.read()
            if s:
                #! TODO Add computer vision analysis to function
                self.log_msg('CAM', 'OK')
            else:
                self.log_msg('CAM', 'Error: failed to get image')
        except Exception as error:
            self.log_msg('CAM', 'Error: %s' % str(error))
        result = {}
        return result

    ## Read Arduino (if available))
    def read_arduino(self):
        self.log_msg('CTRL', 'Reading from controller ...')
        try:
            string = self.arduino.readline()
            result = ast.literal_eval(string)
            self.log_msg('CTRL', 'OK: %s' % str(result))
        except Exception as error:
            result = {}
            self.log_msg('CTRL', 'Error: %s' % str(error))
        return result

    ## Read DHT (if available)
    def read_DHT(self, sensor=Adafruit_DHT.AM2302, pin=11):
        self.log_msg('DHT', 'Reading from DHT ...')
        try:
            humidity, temperature = Adafruit_DHT.read_retry(sensor, pin)
            result = {
                "dht_t" : temperature,
                "dht_h" : humidity
            }
        except Exception as error:
            result = {}
            self.log_msg('DHT', 'Error: %s' % str(error))
        return result

    ## Read BMP (if available)
    def read_BMP(self):
        try:
            temperature = self.BMP085.read_temperature()
            pressure = self.BMP085.read_pressure()
            altitude = self.BMP085.read_altitude()
            sealevel_pressure = self.BMP085.read_read_sealevel_pressure()
            result = {
                "bmp_t" : temperature,
                "bmp_a" : altitude,
                "bmp_p" : pressure,
                "bmp_s" : sealevel_pressure
            }
        except Exception as error:
            result = {}
            self.log_msg('BMP', 'Error: %s' % str(error))
        return result

    ## Post sample to server
    def post_sample(self, sample):
        self.log_msg('REST', 'Posting to webserver ...')
        try:
            dump = json.dumps(sample)
            req = urllib2.Request(self.WAN_URL)
            req.add_header('Content-Type','application/json')
            response = urllib2.urlopen(req, dump)
            self.log_msg('REST', 'OK: %s' % str(response.getcode()))
            return response
        except Exception as error:
            self.log_msg('REST', 'Error: %s' % str(error))

    ## Send sample to aggregator
    def zmq_sample(self, sample):
        self.log_msg('ZMQ', 'Pushing to aggregator ...')
        try:
            dump = json.dumps(sample)
            self.socket.send(dump)
            socks = dict(self.poller.poll(self.ZMQ_TIMEOUT))
            if socks:
                if socks.get(self.socket) == zmq.POLLIN:
                    dump = self.socket.recv(zmq.NOBLOCK)
                    response = json.loads(dump)
                    self.log_msg('ZMQ', str(response))
                    result = response
                else:
                    self.log_msg('ZMQ', 'Error: Poll Timeout')
                    result = None
            else:
                self.log_msg('ZMQ', 'Error: Socket Timeout')
        except Exception as error:
            self.log_msg('ZMQ', 'Error: %s' % str(error))
        return result

    ## Save Data
    def save_sample(self, sample):
        self.log_msg('CSV', 'Saving sample to file ...')
        if sample:
            time = datetime.now().strftime('%Y-%m-%d-%H-%M-%S')
            for param in self.PARAMS:
                try:
                    csv_path = os.path.join(self.NODE_DIR, 'data', param + '.csv')
                    with open(csv_path, 'a') as csv_file:
                        csv_file.write(','.join([time, str(sample[param]), '\n']))
                except Exception as error:
                    self.log_msg('CSV', 'Error: Could not write key (%s)' % str(error))

    ## Generate blank sample
    def blank_sample(self):
        sample = {
            'type' : 'sample',
            'time' : datetime.strftime(datetime.now(), '%Y-%m-%d %H:%M:%S'),
            'hive_id' : self.HIVE_ID
            }
        return sample

    ## Update Clock
    def update_clock(self, secs):
        self.TIME_ERROR = time.time() - secs
        self.log_msg('CLOCK', 'dt=%d' % secs)

    ## Log Message
    def log_msg(self, task, msg):
        date = datetime.strftime(datetime.now(), '%d/%b/%Y:%H:%M:%S')
        print('[%s] %s %s' % (date, task, msg))

    ## Shutdown
    def shutdown(self):
        self.log_msg('ENGINE', 'Shutting Down')
        try:
            self.arduino.close()
        except Exception as e:
            self.log_msg('CTRL', str(e))
        try:
            self.microphone.close()
        except Exception as e:
            self.log_msg('MIC', str(e))
        try:
            self.camera.release()
        except Exception as e:
            self.log_msg('CAM', str(e))
        os.system("sudo reboot")

    ## Update to Aggregator
    def update(self):
        sample = self.blank_sample()

        # Arduino
        arduino_result = self.read_arduino()
        sample.update(arduino_result)

        # Mic
        microphone_result = self.capture_audio()
        sample.update(microphone_result)

        # Camera
        camera_result = self.capture_video()
        sample.update(camera_result)

        # BMP
        BMP_result = self.read_BMP()
        sample.update(BMP_result)

        # DHT
        DHT_result = self.read_DHT()
        sample.update(DHT_result)

        print sample
        try:
            response = self.zmq_sample(sample)
            if response['type'] == 'clock':
                self.log_msg('ZMQ', 'Caught time update request', '')
                self.update_clock['secs']
            if response['type'] == 'config':
                self.log_msg('ZMQ', 'Caught Reload Config Request', '')
        except:
            if self.REBOOT_ENABLED:
                self.shutdown()
        self.post_sample(sample)
        self.save_sample(sample)

    ## Render Index
    @cherrypy.expose
    def index(self):
        with open('static/index.html') as html:
            return html.read()

# Main
if __name__ == '__main__':
    node = HiveNode(config=CONFIG_FILE)
    currdir = os.path.dirname(os.path.abspath(__file__))
    cherrypy.server.socket_host = node.CHERRYPY_ADDR
    cherrypy.server.socket_port = node.CHERRYPY_PORT
    conf = {
        '/': {'tools.staticdir.on':True, 'tools.staticdir.dir':os.path.join(currdir,'static')},
        '/data': {'tools.staticdir.on':True, 'tools.staticdir.dir':os.path.join(currdir,'data')},
        '/js': {'tools.staticdir.on':True, 'tools.staticdir.dir':os.path.join(currdir,'static','js')},
    }
    cherrypy.quickstart(node, '/', config=conf)
<|endoftext|>


<commit_before>from __future__ import print_function
import unittest
from SimPEG.Utils import Zero, Identity, sdiag, mkvc
import numpy as np

class Tests(unittest.TestCase):

    def test_zero(self):
        z = Zero()
        assert z == 0
        assert not (z < 0)
        assert z <= 0
        assert not (z > 0)
        assert z >= 0
        assert +z == z
        assert -z == z
        assert z + 1 == 1
        assert z + 3 + z == 3
        assert z - 3 == -3
        assert z - 3 - z == -3
        assert 3*z == 0
        assert z*3 == 0
        assert z/3 == 0

        a = 1
        a += z
        assert a == 1
        a = 1
        a += z
        assert a == 1
        self.assertRaises(ZeroDivisionError, lambda: 3/z)

        assert mkvc(z) == 0
        assert sdiag(z)*a == 0
        assert z.T == 0
        assert z.transpose == 0

    def test_mat_zero(self):
        z = Zero()
        S = sdiag(np.r_[2, 3])
        assert S*z == 0

    def test_numpy_multiply(self):
        z = Zero()
        x = np.r_[1, 2, 3]
        a = x * z
        assert isinstance(a, Zero)

        z = Zero()
        x = np.r_[1, 2, 3]
        a = z * x
        assert isinstance(a, Zero)

    def test_one(self):
        o = Identity()
        assert o == 1
        assert not (o < 1)
        assert o <= 1
        assert not (o > 1)
        assert o >= 1
        o = -o
        assert o == -1
        assert not (o < -1)
        assert o <= -1
        assert not (o > -1)
        assert o >= -1
        assert -1.*(-o)*o == -o
        o = Identity()
        assert +o == o
        assert -o == -o
        assert o*3 == 3
        assert -o*3 == -3
        assert -o*o == -1
        assert -o*o*-o == 1
        assert -o + 3 == 2
        assert 3 + -o == 2

        assert -o - 3 == -4
        assert o - 3 == -2
        assert 3 - -o == 4
        assert 3 - o == 2

        assert o//2 == 0
        assert o/2. == 0.5
        assert -o//2 == -1
        assert -o/2. == -0.5
        assert 2/o == 2
        assert 2/-o == -2

    def test_mat_one(self):

        o = Identity()
        S = sdiag(np.r_[2, 3])

        def check(exp, ans):
            assert np.all((exp).todense() == ans)

        check(S * o, [[2, 0], [0, 3]])
        check(o * S, [[2, 0], [0, 3]])
        check(S * -o, [[-2, 0], [0, -3]])
        check(-o * S, [[-2, 0], [0, -3]])
        check(S/o, [[2, 0], [0, 3]])
        check(S/-o, [[-2, 0], [0, -3]])
        self.assertRaises(NotImplementedError, lambda: o/S)

        check(S + o, [[3, 0], [0, 4]])
        check(o + S, [[3, 0], [0, 4]])
        check(S - o, [[1, 0], [0, 2]])

        check(S + - o, [[1, 0], [0, 2]])
        check(- o + S, [[1, 0], [0, 2]])

    def test_mat_shape(self):
        o = Identity()
        S = sdiag(np.r_[2, 3])[:1, :]
        self.assertRaises(ValueError, lambda: S + o)

        def check(exp, ans):
            assert np.all((exp).todense() == ans)

        check(S * o, [[2, 0]])
        check(S * -o, [[-2, 0]])

    def test_numpy_one(self):
        o = Identity()
        n = np.r_[2., 3]

        assert np.all(n+1 == n+o)
        assert np.all(1+n == o+n)
        assert np.all(n-1 == n-o)
        assert np.all(1-n == o-n)
        assert np.all(n/1 == n/o)
        assert np.all(n/-1 == n/-o)
        assert np.all(1/n == o/n)
        assert np.all(-1/n == -o/n)
        assert np.all(n*1 == n*o)
        assert np.all(n*-1 == n*-o)
        assert np.all(1*n == o*n)
        assert np.all(-1*n == -o*n)

    def test_both(self):
        z = Zero()
        o = Identity()
        assert o*z == 0
        assert o*z + o == 1
        assert o-z == 1

if __name__ == '__main__':
    unittest.main()
<commit_msg>delete zero test<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python3
import logging, logging.config, logging.handlers
import sys
import asyncio, asyncssh
from queue import Queue

# raised when asyncio.wait_for timeout reached
from concurrent.futures._base import TimeoutError

from simpleton import *

## Global configuration

# 100ms poll cycle (used in SSHManager)
YIELD_TIMEOUT=0.1
# Max. 50 concurrent sessions
MAX_CONCURRENT=50
# Wait at most 5s to initiate connection
CONNECT_TIMEOUT=5

## Definitions
_host_dict = {}
_delimiter = '-'*80
connectfailures = []
sessionfailures = []

def parse_hosts(path):
    with open(path) as f:
        hosts = f.readlines()
    # Parse out comments and blank lines
    hosts = [line.strip().split() for line in hosts if not line.startswith('#') and line.strip()]
    log.debug('Parsed %d entries from %s' % (len(hosts), path))
    # Return dictionary of ip: [hostnames,..]
    return {line[0]: line[1:] for line in hosts}

def ip2host(ip):
    global _host_dict

    try:
        return _host_dict[ip][0]
    except:
        return ip

class SSHClientSession(asyncssh.SSHClientSession):

    global output

    def __init__(self):
        self.user = "null"
        self.cmd  = "null"
        self.host = None
        self.fail = False
        self.first = True

    def connection_made(self, chan):
        self.user = chan.get_extra_info('connection')._usr
        self.cmd  = chan.get_extra_info('connection')._cmd
        self.host = ip2host(chan.get_extra_info('peername')[0])

    def data_received(self, data, datatype):
        global _delimiter
        if self.first:
            print('cat <<_EOF >>%s\n\n# %s\n%s\n%s\n_EOF\n' %
                 (self.host+'_'+self.user+'.out', self.cmd, _delimiter, data.strip()), file=output)
            log.info('[%s:%s] %s\n%s' %
                    (self.host, self.user, self.cmd, data.strip()))
            self.first = False
        else:
            print('cat <<_EOF >>%s\n%s\n_EOF' %
                 (self.host+'_'+self.user+'.out', data.strip()), file=output)
            log.info('[%s:%s] %s\n%s' %
                    (self.host, self.user, self.cmd, data.strip()))

    def exit_status_received(self, status):
        if status:
            self.fail = True
            log.error('[%s:%s] %s -> error code %d' % (self.host, self.user, self.cmd, status))

    def connection_lost(self, exc):
        if exc:
            self.fail = True
            log.error('[%s:%s] SSH session error: %s' % (self.host, self.user, str(exc) if str(exc) else "Timeout"))

@asyncio.coroutine
def SSHClient(host, cmdlist):

    global connectfailures
    global sessionfailures

    try:
        with (yield from asyncio.wait_for(asyncssh.connect(host, known_hosts=None), CONNECT_TIMEOUT)) as conn:
            conn._usr = conn.get_extra_info("username")
            log.debug('[%s:%s] Connection initiated' % (host, conn._usr))
            for cmd in cmdlist:
                conn._cmd = cmd
                chan, session = yield from conn.create_session(SSHClientSession, cmd)
                yield from chan.wait_closed()
                if session.fail:
                    log.critical('[%s:%s] Failure detected, breaking...' % (host, conn._usr))
                    sessionfailures.append(host)
                    break
    except (OSError, asyncssh.Error, TimeoutError) as exc:
        log.error('[%s] SSH connection failed: %s' % (host, str(exc) if str(exc) else "Timeout"))
        connectfailures.append(host)

@asyncio.coroutine
def SSHManager(loop, queue, cmdlist):

    tasks = []
    while True:
        while (len(tasks) > MAX_CONCURRENT) or (tasks and queue.empty()):
            for task in tasks:
                if task.done():
                    tasks.remove(task)
            yield from asyncio.sleep(YIELD_TIMEOUT)

        if not queue.empty():
            host = yield from queue.get()
            tasks.append(asyncio.async(SSHClient(host, cmdlist)))
        else:
            if not tasks:
                break
    loop.stop()

if __name__ == '__main__':

    args = parser.parse_args()

    if len(sys.argv) == 1:
        parser.print_help()
        sys.exit(1)

    output = open(args.output, 'w')

    # Configure logging format
    logging.config.dictConfig(SIMPLETON_LOGGING)
    log = logging.getLogger("default")

    _log_queue = Queue()
    log_async  = logging.handlers.QueueHandler(_log_queue)
    log_queue  = logging.handlers.QueueListener(_log_queue, *log.handlers)

    log_queue.start()
    log.handlers = [log_async,]

    loop  = asyncio.get_event_loop()
    queue = asyncio.Queue()

    if args.cmdfile:
        with open(args.cmdfile) as f:
            args.cmdlist = f.readlines()

    _host_dict = parse_hosts(args.hostsfile)
    _hosts = []
    if args.hostmatch:
        for ip in _host_dict:
            for match in args.hostmatch:
                #i = 'i'*10
                #for c in i:
                [_hosts.append(hostname) for hostname in _host_dict[ip] if match in hostname]

    _uniq = set(_hosts)
    if args.hostexclude:
        for exclude in args.hostexclude:
            [_uniq.remove(hostname) for hostname in _hosts if exclude in hostname]

    if not _uniq:
        log.critical('No hosts matched')
        log_queue.stop()
        sys.exit(1)

    try:
        [queue.put_nowait(hostname) for hostname in _uniq]
        _host_count = queue.qsize()

        _start = loop.time()
        loop.run_until_complete(SSHManager(loop, queue, args.cmdlist))

    finally:
        _end   = loop.time()
        log.critical('Connected to %d hosts successfully, %d hosts failed: %s' % (_host_count-len(connectfailures), len(connectfailures), ' '.join(sorted(connectfailures))))
        log.critical('Failed commands on %d hosts: %s' % (len(sessionfailures), ' '.join(sorted(sessionfailures))))
        log.critical('Finished run in %.03fms' % ((_end-_start)*1000))
        log_queue.stop()
<commit_msg>reworked fail state variables<commit_after>#!/usr/bin/env python3
"""
https://github.com/xstaticxgpx/simpleton
"""

import logging, logging.config, logging.handlers
import sys
import asyncio, asyncssh
from queue import Queue

# raised when asyncio.wait_for timeout reached
from concurrent.futures._base import TimeoutError

from simpleton import *

## Global configuration

# 100ms poll cycle (used in SSHManager)
YIELD_TIMEOUT=0.1
# Max. 50 concurrent sessions
MAX_CONCURRENT=50
# Wait max. 10s for connection
CONNECT_TIMEOUT=10

## Definitions
_host_dict = {}
_delimiter = '-'*80
connectfailures = {}
sessionfailures = {}

## Static variables
# pointers for sessionfailures list values
_CMD = 0
_EXITCODE = 1

def parse_hosts(path):
    with open(path) as f:
        hosts = f.readlines()
    # Parse out comments and blank lines
    hosts = [line.strip().split() for line in hosts if not line.startswith('#') and line.strip()]
    log.debug('Parsed %d entries from %s' % (len(hosts), path))
    # Return dictionary of ip: [hostnames,..]
    return {line[0]: line[1:] for line in hosts}

def ip2host(ip):
    global _host_dict

    try:
        return _host_dict[ip][0]
    except:
        return ip

class SSHClientSession(asyncssh.SSHClientSession):

    global output

    def __init__(self):
        self.user = "null"
        self.cmd  = "null"
        self.host = None
        self.fail = False
        self.first = True

    def connection_made(self, chan):
        self.user = chan.get_extra_info('connection')._usr
        self.cmd  = chan.get_extra_info('connection')._cmd
        self.host = ip2host(chan.get_extra_info('peername')[0])

    def data_received(self, data, datatype):
        global _delimiter
        if self.first:
            print('cat <<_EOF >>%s\n\n# %s\n%s\n%s\n_EOF\n' %
                 (self.host+'_'+self.user+'.out', self.cmd, _delimiter, data.strip()), file=output)
            log.info('[%s:%s] %s\n%s' %
                    (self.host, self.user, self.cmd, data.strip()))
            self.first = False
        else:
            print('cat <<_EOF >>%s\n%s\n_EOF' %
                 (self.host+'_'+self.user+'.out', data.strip()), file=output)
            log.info('[%s:%s] %s\n%s' %
                    (self.host, self.user, self.cmd, data.strip()))

    def exit_status_received(self, status):
        if status:
            self.fail = status
            log.error('[%s:%s] %s -> error code %d' % (self.host, self.user, self.cmd, status))

    def connection_lost(self, exc):
        if exc:
            self.fail = True
            log.error('[%s:%s] SSH session error: %s' % (self.host, self.user, str(exc) if str(exc) else "Timeout"))

@asyncio.coroutine
def SSHClient(host, cmdlist):

    global connectfailures
    global sessionfailures

    try:
        with (yield from asyncio.wait_for(asyncssh.connect(host, known_hosts=None), CONNECT_TIMEOUT)) as conn:
            conn._usr = conn.get_extra_info("username")
            log.debug('[%s:%s] Connection initiated' % (host, conn._usr))
            for cmd in cmdlist:
                conn._cmd = cmd
                chan, session = yield from conn.create_session(SSHClientSession, cmd)
                yield from chan.wait_closed()
                if session.fail:
                    log.critical('[%s:%s] Failure detected, breaking...' % (host, conn._usr))
                    sessionfailures[host] = [cmd, session.fail]
                    break
    except (OSError, asyncssh.Error, TimeoutError) as exc:
        exc = str(exc) if str(exc) else "Timeout"
        log.error('[%s] SSH connection failed: %s' % (host, exc))
        connectfailures[host] = exc

@asyncio.coroutine
def SSHManager(loop, queue, cmdlist):

    tasks = []
    while True:
        while (len(tasks) > MAX_CONCURRENT) or (tasks and queue.empty()):
            for task in tasks:
                if task.done():
                    tasks.remove(task)
            yield from asyncio.sleep(YIELD_TIMEOUT)

        if not queue.empty():
            host = yield from queue.get()
            tasks.append(asyncio.async(SSHClient(host, cmdlist)))
        else:
            if not tasks:
                break
    loop.stop()

if __name__ == '__main__':

    args = parser.parse_args()

    if len(sys.argv) == 1:
        parser.print_help()
        sys.exit(1)

    output = open(args.output, 'w')

    # Configure logging format
    logging.config.dictConfig(SIMPLETON_LOGGING)
    log = logging.getLogger("default")

    _log_queue = Queue()
    log_async  = logging.handlers.QueueHandler(_log_queue)
    log_queue  = logging.handlers.QueueListener(_log_queue, *log.handlers)

    log_queue.start()
    log.handlers = [log_async,]

    loop  = asyncio.get_event_loop()
    queue = asyncio.Queue()

    if args.cmdfile:
        with open(args.cmdfile) as f:
            args.cmdlist = f.readlines()

    _host_dict = parse_hosts(args.hostsfile)
    _hosts = []
    if args.hostmatch:
        for ip in _host_dict:
            for match in args.hostmatch:
                #i = 'i'*10
                #for c in i:
                [_hosts.append(hostname) for hostname in _host_dict[ip] if match in hostname]

    _uniq = set(_hosts)
    if args.hostexclude:
        for exclude in args.hostexclude:
            [_uniq.remove(hostname) for hostname in _hosts if exclude in hostname]

    if not _uniq:
        log.critical('No hosts matched')
        log_queue.stop()
        sys.exit(1)

    try:
        [queue.put_nowait(hostname) for hostname in _uniq]
        _host_count = queue.qsize()

        _start = loop.time()
        loop.run_until_complete(SSHManager(loop, queue, args.cmdlist))

    finally:
        _end   = loop.time()
        #log.critical('Completed %d hosts, failed %d hosts: %s %s' % (_host_count-len(connectfailures), len(connectfailures), ' '.join(sorted(connectfailures)), ' '.join(sorted(sessionfailures))))
        log.debug(_delimiter)
        log.critical('Finished run in %.03fms' % ((_end-_start)*1000))
        for host in sorted(_uniq):
            if host in sessionfailures:
                log.warning('%s command failed: %s (exit code: %d)' % (host, sessionfailures[host][_CMD], sessionfailures[host][_EXITCODE]))
            elif host in connectfailures:
                log.critical('%s connection failed: %s' % (host, connectfailures[host]))
        log.debug(_delimiter)
        log.info('Saved output script to %s' % args.output)
        log_queue.stop()
<|endoftext|>


<commit_before>import sys
import time

import numpy as np
import tensorflow as tf
from tensorflow.python.client import timeline

from language_model import LM
from common import CheckpointLoader

def run_train(dataset, hps, logdir, ps_device, task=0, master=""):
    with tf.variable_scope("model"):
        model = LM(hps, "train", ps_device)

    print("ALL VARIABLES")
    for v in tf.all_variables():
        print("%s %s %s" % (v.name, v.get_shape(), v.device))
    print("TRAINABLE VARIABLES")
    for v in tf.trainable_variables():
        print("%s %s %s" % (v.name, v.get_shape(), v.device))
    print("LOCAL VARIABLES")
    for v in tf.local_variables():
        print("%s %s %s" % (v.name, v.get_shape(), v.device))

    sv = tf.train.Supervisor(is_chief=(task == 0),
                             logdir=logdir,
                             summary_op=None,  # Automatic summaries don't work with placeholders.
                             global_step=model.global_step,
                             save_summaries_secs=30,
                             save_model_secs=120 * 5)

    config = tf.ConfigProto(allow_soft_placement=True,
                            intra_op_parallelism_threads=2,
                            inter_op_parallelism_threads=20)
    with sv.managed_session(master, config=config) as sess:
        # Slowly increase the number of workers during beginning of the training.
        while not sv.should_stop():
            step = int(sess.run(model.global_step))
            waiting_until_step = task * hps.num_delayed_steps
            if step >= waiting_until_step:
                break
            else:
                print("Current step is %d. Waiting until: %d" % (step, waiting_until_step))
            time.sleep(10.0)

        local_step = 0
        prev_global_step = sess.run(model.global_step)
        prev_time = time.time()
        data_iterator = dataset.iterate_forever(hps.batch_size * hps.num_gpus, hps.num_steps)
        while not sv.should_stop():
            fetches = [model.global_step, model.loss, model.train_op]
            # Chief worker computes summaries every 20 steps.
            should_compute_summary = (task == 0 and local_step > 0 and local_step % 20 == 0)
            if should_compute_summary:
                fetches += [model.summary_op]

            #x, y, w = next(data_iterator)
            x, y = next(data_iterator)
            should_run_profiler = (hps.run_profiler and task == 0 and local_step % 1000 == 13)
            if should_run_profiler:
                run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
                run_metadata = tf.RunMetadata()
                #fetched = sess.run(fetches, {model.x: x, model.y: y, model.w: w},
                fetched = sess.run(fetches, {model.x: x, model.y: y},
                                   options=run_options, run_metadata=run_metadata)
                # Create the Timeline object, and write it to a json
                tl = timeline.Timeline(run_metadata.step_stats)
                ctf = tl.generate_chrome_trace_format()
                print("Running profiler")
                with open(logdir + "/timeline.json", 'w') as f:
                    f.write(ctf)
                print("Finished profiling!")
            else:
                #fetched = sess.run(fetches, {model.x: x, model.y: y, model.w: w})
                fetched = sess.run(fetches, {model.x: x, model.y: y})

            local_step += 1
            if should_compute_summary:
                sv.summary_computed(sess, fetched[-1])

            if local_step < 10 or local_step % 20 == 0:
                cur_time = time.time()
                num_words = hps.batch_size * hps.num_gpus * hps.num_steps
                wps = (fetched[0] - prev_global_step) * num_words / (cur_time - prev_time)
                prev_global_step = fetched[0]
                print("Iteration %d, time = %.2fs, wps = %.0f, train loss = %.4f" % (
                    fetched[0], cur_time - prev_time, wps, fetched[1]))
                prev_time = cur_time
    sv.stop()

def run_eval(dataset, hps, logdir, mode, num_eval_steps):
    with tf.variable_scope("model"):
        hps.num_sampled = 0  # Always using full softmax at evaluation.
        hps.keep_prob = 1.0
        model = LM(hps, "eval", "/cpu:0")

    if hps.average_params:
        print("Averaging parameters for evaluation.")
        saver = tf.train.Saver(model.avg_dict)
    else:
        saver = tf.train.Saver()

    # Use only 4 threads for the evaluation.
    config = tf.ConfigProto(allow_soft_placement=True,
                            intra_op_parallelism_threads=20,
                            inter_op_parallelism_threads=1)
    sess = tf.Session(config=config)
    sw = tf.train.SummaryWriter(logdir + "/" + mode, sess.graph)
    ckpt_loader = CheckpointLoader(saver, model.global_step, logdir + "/train")

    with sess.as_default():
        while ckpt_loader.load_checkpoint():
            global_step = ckpt_loader.last_global_step
            data_iterator = dataset.iterate_once(hps.batch_size * hps.num_gpus, hps.num_steps)
            tf.initialize_local_variables().run()
            loss_nom = 0.0
            loss_den = 0.0
            #for i, (x, y, w) in enumerate(data_iterator):
            for i, (x, y) in enumerate(data_iterator):
                if i >= num_eval_steps and mode!="eval_full":
                    break

                #loss = sess.run(model.loss, {model.x: x, model.y: y, model.w: w})
                loss = sess.run(model.loss, {model.x: x, model.y: y})
                loss_nom += loss
                loss_den += w.mean()
                loss = loss_nom / loss_den
                sys.stdout.write("%d: %.3f (%.3f) ... " % (i, loss, np.exp(loss)))
                sys.stdout.flush()
            sys.stdout.write("\n")

            log_perplexity = loss_nom / loss_den
            print("Results at %d: log_perplexity = %.3f perplexity = %.3f" % (
                global_step, log_perplexity, np.exp(log_perplexity)))

            summary = tf.Summary()
            summary.value.add(tag='eval/log_perplexity', simple_value=log_perplexity)
            summary.value.add(tag='eval/perplexity', simple_value=np.exp(log_perplexity))
            sw.add_summary(summary, global_step)
            sw.flush()
<commit_msg>bugfix in runutils<commit_after>import sys
import time

import numpy as np
import tensorflow as tf
from tensorflow.python.client import timeline

from language_model import LM
from common import CheckpointLoader

def run_train(dataset, hps, logdir, ps_device, task=0, master=""):
    with tf.variable_scope("model"):
        model = LM(hps, "train", ps_device)

    print("ALL VARIABLES")
    for v in tf.all_variables():
        print("%s %s %s" % (v.name, v.get_shape(), v.device))
    print("TRAINABLE VARIABLES")
    for v in tf.trainable_variables():
        print("%s %s %s" % (v.name, v.get_shape(), v.device))
    print("LOCAL VARIABLES")
    for v in tf.local_variables():
        print("%s %s %s" % (v.name, v.get_shape(), v.device))

    sv = tf.train.Supervisor(is_chief=(task == 0),
                             logdir=logdir,
                             summary_op=None,  # Automatic summaries don't work with placeholders.
                             global_step=model.global_step,
                             save_summaries_secs=30,
                             save_model_secs=120 * 5)

    config = tf.ConfigProto(allow_soft_placement=True,
                            intra_op_parallelism_threads=2,
                            inter_op_parallelism_threads=20)
    with sv.managed_session(master, config=config) as sess:
        # Slowly increase the number of workers during beginning of the training.
        while not sv.should_stop():
            step = int(sess.run(model.global_step))
            waiting_until_step = task * hps.num_delayed_steps
            if step >= waiting_until_step:
                break
            else:
                print("Current step is %d. Waiting until: %d" % (step, waiting_until_step))
            time.sleep(10.0)

        local_step = 0
        prev_global_step = sess.run(model.global_step)
        prev_time = time.time()
        data_iterator = dataset.iterate_forever(hps.batch_size * hps.num_gpus, hps.num_steps)
        while not sv.should_stop():
            fetches = [model.global_step, model.loss, model.train_op]
            # Chief worker computes summaries every 20 steps.
            should_compute_summary = (task == 0 and local_step > 0 and local_step % 20 == 0)
            if should_compute_summary:
                fetches += [model.summary_op]

            #x, y, w = next(data_iterator)
            x, y = next(data_iterator)
            should_run_profiler = (hps.run_profiler and task == 0 and local_step % 1000 == 13)
            if should_run_profiler:
                run_options = tf.RunOptions(trace_level=tf.RunOptions.FULL_TRACE)
                run_metadata = tf.RunMetadata()
                #fetched = sess.run(fetches, {model.x: x, model.y: y, model.w: w},
                fetched = sess.run(fetches, {model.x: x, model.y: y},
                                   options=run_options, run_metadata=run_metadata)
                # Create the Timeline object, and write it to a json
                tl = timeline.Timeline(run_metadata.step_stats)
                ctf = tl.generate_chrome_trace_format()
                print("Running profiler")
                with open(logdir + "/timeline.json", 'w') as f:
                    f.write(ctf)
                print("Finished profiling!")
            else:
                #fetched = sess.run(fetches, {model.x: x, model.y: y, model.w: w})
                fetched = sess.run(fetches, {model.x: x, model.y: y})

            local_step += 1
            if should_compute_summary:
                sv.summary_computed(sess, fetched[-1])

            if local_step < 10 or local_step % 20 == 0:
                cur_time = time.time()
                num_words = hps.batch_size * hps.num_gpus * hps.num_steps
                wps = (fetched[0] - prev_global_step) * num_words / (cur_time - prev_time)
                prev_global_step = fetched[0]
                print("Iteration %d, time = %.2fs, wps = %.0f, train loss = %.4f" % (
                    fetched[0], cur_time - prev_time, wps, fetched[1]))
                prev_time = cur_time
    sv.stop()

def run_eval(dataset, hps, logdir, mode, num_eval_steps):
    with tf.variable_scope("model"):
        hps.num_sampled = 0  # Always using full softmax at evaluation.
        hps.keep_prob = 1.0
        model = LM(hps, "eval", "/cpu:0")

    if hps.average_params:
        print("Averaging parameters for evaluation.")
        saver = tf.train.Saver(model.avg_dict)
    else:
        saver = tf.train.Saver()

    # Use only 4 threads for the evaluation.
    config = tf.ConfigProto(allow_soft_placement=True,
                            intra_op_parallelism_threads=20,
                            inter_op_parallelism_threads=1)
    sess = tf.Session(config=config)
    sw = tf.train.SummaryWriter(logdir + "/" + mode, sess.graph)
    ckpt_loader = CheckpointLoader(saver, model.global_step, logdir + "/train")

    with sess.as_default():
        while ckpt_loader.load_checkpoint():
            global_step = ckpt_loader.last_global_step
            data_iterator = dataset.iterate_once(hps.batch_size * hps.num_gpus, hps.num_steps)
            tf.initialize_local_variables().run()
            loss_nom = 0.0
            loss_den = 0.0
            #for i, (x, y, w) in enumerate(data_iterator):
            for i, (x, y) in enumerate(data_iterator):
                if i >= num_eval_steps and mode!="eval_full":
                    break

                #loss = sess.run(model.loss, {model.x: x, model.y: y, model.w: w})
                loss = sess.run(model.loss, {model.x: x, model.y: y})
                loss_nom += loss
                #loss_den += w.mean()
                loss = loss_nom / loss_den
                sys.stdout.write("%d: %.3f (%.3f) ... " % (i, loss, np.exp(loss)))
                sys.stdout.flush()
            sys.stdout.write("\n")

            log_perplexity = loss_nom / loss_den
            print("Results at %d: log_perplexity = %.3f perplexity = %.3f" % (
                global_step, log_perplexity, np.exp(log_perplexity)))

            summary = tf.Summary()
            summary.value.add(tag='eval/log_perplexity', simple_value=log_perplexity)
            summary.value.add(tag='eval/perplexity', simple_value=np.exp(log_perplexity))
            sw.add_summary(summary, global_step)
            sw.flush()
<|endoftext|>


<commit_before>#!/usr/bin/env python
'graph dependencies in projects'
import json
from subprocess import Popen, PIPE
import sys
import textwrap

# Typical command line usage:
#
# python graphdeps.py TASKFILTER
#
# TASKFILTER is a taskwarrior filter, documentation can be found here: http://taskwarrior.org/projects/taskwarrior/wiki/Feature_filters
#
# Probably the most helpful commands are:
#
# python graphdeps.py project:fooproject status:pending
#  --> graph pending tasks in project 'fooproject'
#
# python graphdeps.py project:fooproject
#  --> graphs all tasks in 'fooproject', pending, completed, deleted
#
# python graphdeps.py status:pending
#  --> graphs all pending tasks in all projects
#
# python graphdeps.py
#  --> graphs everything - could be massive
#

#Wrap label text at this number of characters
charsPerLine = 20;

#full list of colors here: http://www.graphviz.org/doc/info/colors.html
blockedColor = 'gold4'
maxUrgencyColor = 'red2' #color of the tasks that have absolutely the highest urgency
unblockedColor = 'green'
doneColor = 'grey'
waitColor = 'white'
deletedColor = 'pink';

#The width of the border around the tasks:
penWidth = 1

#Have one HEADER (and only one) uncommented at a time, or the last uncommented value will be the only one considered

#Left to right layout, my favorite, ganntt-ish
#HEADER = "digraph  dependencies { splines=true; overlap=ortho; rankdir=LR; weight=2;"

#Spread tasks on page
HEADER = "digraph  dependencies { layout=neato;   splines=true; overlap=scalexy;  rankdir=LR; weight=2;"

#More information on setting up graphviz: http://www.graphviz.org/doc/info/attrs.html

#-----------------------------------------#
#  Editing under this might break things  #
#-----------------------------------------#

FOOTER = "}"

JSON_START = '['
JSON_END = ']'

validUuids = list()

def call_taskwarrior(cmd):
    'call taskwarrior, returning output and error'
    tw = Popen(['task'] + cmd.split(), stdout=PIPE, stderr=PIPE)
    return tw.communicate()

def get_json(query):
    'call taskwarrior, returning objects from json'
    result, err = call_taskwarrior('export %s rc.json.array=on rc.verbose=nothing' % query)
    return json.loads(str(result))

def call_dot(instr):
    'call dot, returning stdout and stdout'
    dot = Popen('dot -T png'.split(), stdout=PIPE, stderr=PIPE, stdin=PIPE)
    return dot.communicate(instr)

if __name__ == '__main__':
    query = sys.argv[1:]
    print ('Calling TaskWarrior')
    data = get_json(' '.join(query))
    #print data

    maxUrgency = -9999;
    for datum in data:
        if float(datum['urgency']) > maxUrgency:
            maxUrgency = float(datum['urgency'])

    # first pass: labels
    lines = [HEADER]
    print ('Printing Labels')
    for datum in data:
        validUuids.append(datum['uuid'])
        if datum['description']:

            style = ''
            color = ''
            style = 'filled'

            if datum['status']=='pending':
                prefix = datum['id']
                if not datum.get('depends','') : color = unblockedColor
                else :
                    hasPendingDeps = 0
                    for depend in datum['depends'].split(','):
                        for datum2 in data:
                            if datum2['uuid'] == depend and datum2['status'] == 'pending':
                               hasPendingDeps = 1
                    if hasPendingDeps == 1 : color = blockedColor
                    else : color = unblockedColor

            elif datum['status'] == 'waiting':
                prefix = 'WAIT'
                color = waitColor
            elif datum['status'] == 'completed':
                prefix = 'DONE'
                color = doneColor
            elif datum['status'] == 'deleted':
                prefix = 'DELETED'
                color = deletedColor
            else:
                prefix = ''
                color = 'white'

            if float(datum['urgency']) == maxUrgency:
                color = maxUrgencyColor

            label = '';
            descriptionLines = textwrap.wrap(datum['description'],charsPerLine);
            for descLine in descriptionLines:
                label += descLine+"\\n";

            lines.append('"%s"[shape=box][penwidth=%d][label="%s\:%s"][fillcolor=%s][style=%s]' % (datum['uuid'], penWidth, prefix, label, color, style))
            #documentation http://www.graphviz.org/doc/info/attrs.html

    # second pass: dependencies
    print ('Resolving Dependencies')
    for datum in data:
        if datum['description']:
            for dep in datum.get('depends', '').split(','):
                #print ("\naaa %s" %dep)
                if dep!='' and dep in validUuids:
                    lines.append('"%s" -> "%s";' % (dep, datum['uuid']))
                    continue

    lines.append(FOOTER)

    print ('Calling dot')
    png, err = call_dot('\n'.join(lines))
    if err != '':
        print ('Error calling dot:')
        print (err.strip())

    print ('Writing to deps.png')
    with open('deps.png', 'w') as f:
        f.write(png)
<commit_msg>Switched default GraphViz header<commit_after>#!/usr/bin/env python
'graph dependencies in projects'
import json
from subprocess import Popen, PIPE
import sys
import textwrap

# Typical command line usage:
#
# python graphdeps.py TASKFILTER
#
# TASKFILTER is a taskwarrior filter, documentation can be found here: http://taskwarrior.org/projects/taskwarrior/wiki/Feature_filters
#
# Probably the most helpful commands are:
#
# python graphdeps.py project:fooproject status:pending
#  --> graph pending tasks in project 'fooproject'
#
# python graphdeps.py project:fooproject
#  --> graphs all tasks in 'fooproject', pending, completed, deleted
#
# python graphdeps.py status:pending
#  --> graphs all pending tasks in all projects
#
# python graphdeps.py
#  --> graphs everything - could be massive
#

#Wrap label text at this number of characters
charsPerLine = 20;

#full list of colors here: http://www.graphviz.org/doc/info/colors.html
blockedColor = 'gold4'
maxUrgencyColor = 'red2' #color of the tasks that have absolutely the highest urgency
unblockedColor = 'green'
doneColor = 'grey'
waitColor = 'white'
deletedColor = 'pink';

#The width of the border around the tasks:
penWidth = 1

#Have one HEADER (and only one) uncommented at a time, or the last uncommented value will be the only one considered

#Left to right layout, my favorite, ganntt-ish
HEADER = "digraph  dependencies { splines=true; overlap=ortho; rankdir=LR; weight=2;"

#Spread tasks on page
#HEADER = "digraph  dependencies { layout=neato;   splines=true; overlap=scalexy;  rankdir=LR; weight=2;"

#More information on setting up graphviz: http://www.graphviz.org/doc/info/attrs.html

#-----------------------------------------#
#  Editing under this might break things  #
#-----------------------------------------#

FOOTER = "}"

JSON_START = '['
JSON_END = ']'

validUuids = list()

def call_taskwarrior(cmd):
    'call taskwarrior, returning output and error'
    tw = Popen(['task'] + cmd.split(), stdout=PIPE, stderr=PIPE)
    return tw.communicate()

def get_json(query):
    'call taskwarrior, returning objects from json'
    result, err = call_taskwarrior('export %s rc.json.array=on rc.verbose=nothing' % query)
    return json.loads(str(result))

def call_dot(instr):
    'call dot, returning stdout and stdout'
    dot = Popen('dot -T png'.split(), stdout=PIPE, stderr=PIPE, stdin=PIPE)
    return dot.communicate(instr)

if __name__ == '__main__':
    query = sys.argv[1:]
    print ('Calling TaskWarrior')
    data = get_json(' '.join(query))
    #print data

    maxUrgency = -9999;
    for datum in data:
        if float(datum['urgency']) > maxUrgency:
            maxUrgency = float(datum['urgency'])

    # first pass: labels
    lines = [HEADER]
    print ('Printing Labels')
    for datum in data:
        validUuids.append(datum['uuid'])
        if datum['description']:

            style = ''
            color = ''
            style = 'filled'

            if datum['status']=='pending':
                prefix = datum['id']
                if not datum.get('depends','') : color = unblockedColor
                else :
                    hasPendingDeps = 0
                    for depend in datum['depends'].split(','):
                        for datum2 in data:
                            if datum2['uuid'] == depend and datum2['status'] == 'pending':
                               hasPendingDeps = 1
                    if hasPendingDeps == 1 : color = blockedColor
                    else : color = unblockedColor

            elif datum['status'] == 'waiting':
                prefix = 'WAIT'
                color = waitColor
            elif datum['status'] == 'completed':
                prefix = 'DONE'
                color = doneColor
            elif datum['status'] == 'deleted':
                prefix = 'DELETED'
                color = deletedColor
            else:
                prefix = ''
                color = 'white'

            if float(datum['urgency']) == maxUrgency:
                color = maxUrgencyColor

            label = '';
            descriptionLines = textwrap.wrap(datum['description'],charsPerLine);
            for descLine in descriptionLines:
                label += descLine+"\\n";

            lines.append('"%s"[shape=box][penwidth=%d][label="%s\:%s"][fillcolor=%s][style=%s]' % (datum['uuid'], penWidth, prefix, label, color, style))
            #documentation http://www.graphviz.org/doc/info/attrs.html

    # second pass: dependencies
    print ('Resolving Dependencies')
    for datum in data:
        if datum['description']:
            for dep in datum.get('depends', '').split(','):
                #print ("\naaa %s" %dep)
                if dep!='' and dep in validUuids:
                    lines.append('"%s" -> "%s";' % (dep, datum['uuid']))
                    continue

    lines.append(FOOTER)

    print ('Calling dot')
    png, err = call_dot('\n'.join(lines))
    if err != '':
        print ('Error calling dot:')
        print (err.strip())

    print ('Writing to deps.png')
    with open('deps.png', 'w') as f:
        f.write(png)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse
import os
import subprocess
from functools import partial
from breakseq_pre import print_candidate_reads
import logging
import multiprocessing
import traceback
import shutil

def add_options(main_parser):
    local_parser = main_parser.add_argument_group("Read alignment options")
    local_parser.add_argument("--bplib", help="Breakpoint library FASTA", required=False)
    local_parser.add_argument("--bwa", help="Path to BWA executable", required=True)
    local_parser.add_argument("--samtools", help="Path to SAMtools executable", required=True)

def preprocess_and_align(bplib=None, bwa=None, samtools=None, bam=None, prefix=None, chromosome=None, keep_temp=False):
    func_logger = logging.getLogger("%s-%s" % (preprocess_and_align.__name__, multiprocessing.current_process()))
    outfq = prefix + ".fq"
    # First generate the FASTQ for alignment
    outbam = None
    try:
        func_logger.info("Extracting candidate reads from {bam} for chromosome {chromosome} and aligning against {bplib}".format(bam=bam, chromosome=chromosome, bplib=bplib))
        readcount = print_candidate_reads([bam], chromosome, outfile=outfq)
        if readcount > 0:
            outbam = prefix + ".bam"
            outlog = prefix + ".bwa.log"
            bash_cmd = "bash -c \"{bwa} samse {bplib} <({bwa} aln {bplib} {outfq}) {outfq} | {samtools} view -S - -1 -F 4 -bo {outbam}\"".format(bwa=bwa, bplib=bplib, samtools=samtools, outfq=outfq, outbam=outbam)
            func_logger.info("Running %s" % bash_cmd)
            with open(outlog, "w") as logfd:
                subprocess.check_call(bash_cmd, shell=True, stderr=logfd)
        else:
            func_logger.info("No reads extracted from {bam} so no alignment will be done".format(bam=bam))
    except Exception as e:
        func_logger.error('Caught exception in worker thread')
        traceback.print_exc()

        print()
        raise e

    if not keep_temp:
        os.remove(outfq)
    return outbam

def preprocess_and_align_callback(result, result_list):
    if result is not None:
        result_list.append(result)

# Parallelize over BAMs and chromosomes
def parallel_preprocess_and_align(bplib, bwa, samtools, bams, work, chromosomes=[], nthreads=1, merge=False, keep_temp=False):
    func_logger = logging.getLogger(parallel_preprocess_and_align.__name__)

    if not os.path.isdir(work):
        os.makedirs(work)

    if not chromosomes: chromosomes = [None]
    else: chromosomes.append("") # Special contig for unmapped reads

    pool = multiprocessing.Pool(nthreads)
    count = 0
    aligned_bams = []
    for bam  in bams:
        for chromosome in chromosomes:
            kwargs_dict = {"bplib": bplib, "bwa": bwa, "samtools": samtools, "bam": bam, "prefix": "%s/%d" % (work, count), "chromosome": chromosome, "keep_temp": keep_temp}
            pool.apply_async(preprocess_and_align, args=[], kwds=kwargs_dict, callback=partial(preprocess_and_align_callback, result_list=aligned_bams))
            count += 1

    pool.close()
    pool.join()

    # Now merge the BAMs together
    if not aligned_bams:
        func_logger.info("No alignment generated after read extraction")
        return []

    finalbam = os.path.join(work, "final.bam")
    if len(aligned_bams) > 1:
        merged_str = " ".join(aligned_bams)
        bash_cmd = "bash -c \"{samtools} merge -f {finalbam} {merged_str}\"".format(samtools=samtools, finalbam=finalbam, merged_str=merged_str)
        subprocess.check_call(bash_cmd, shell=True)
    else:
        shutil.copy(aligned_bams[0], finalbam)

    if not keep_temp:
        for bam in aligned_bams:
            os.remove(bam)

    return [finalbam]

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Process BAMs to generate candidate reads and align them against the breakpoint library")
    add_options(parser)
    parser.add_argument("--bams", nargs="+", help="BAMs to extract reads from", required=True)
    parser.add_argument("--chromosomes", nargs="+", help="List of chromosomes to process", default=[])
    parser.add_argument("--work", help="Working directory")
    parser.add_argument("--nthreads", help="Number of processes to use", type=int, default=1)
    parser.add_argument("--keep_temp", help="Keep temporary files", action="store_true")

    args = parser.parse_args()
    preprocess_and_align(args.bplib, args.bwa, args.samtools, args.bams, args.chromosomes, args.nthreads, args.keep_temp)<commit_msg>Remove merge param from preprocess and align<commit_after>#!/usr/bin/env python

import argparse
import os
import subprocess
from functools import partial
from breakseq_pre import print_candidate_reads
import logging
import multiprocessing
import traceback
import shutil

def add_options(main_parser):
    local_parser = main_parser.add_argument_group("Read alignment options")
    local_parser.add_argument("--bplib", help="Breakpoint library FASTA", required=False)
    local_parser.add_argument("--bwa", help="Path to BWA executable", required=True)
    local_parser.add_argument("--samtools", help="Path to SAMtools executable", required=True)

def preprocess_and_align(bplib=None, bwa=None, samtools=None, bam=None, prefix=None, chromosome=None, keep_temp=False):
    func_logger = logging.getLogger("%s-%s" % (preprocess_and_align.__name__, multiprocessing.current_process()))
    outfq = prefix + ".fq"
    # First generate the FASTQ for alignment
    outbam = None
    try:
        func_logger.info("Extracting candidate reads from {bam} for chromosome {chromosome} and aligning against {bplib}".format(bam=bam, chromosome=chromosome, bplib=bplib))
        readcount = print_candidate_reads([bam], chromosome, outfile=outfq)
        if readcount > 0:
            outbam = prefix + ".bam"
            outlog = prefix + ".bwa.log"
            bash_cmd = "bash -c \"{bwa} samse {bplib} <({bwa} aln {bplib} {outfq}) {outfq} | {samtools} view -S - -1 -F 4 -bo {outbam}\"".format(bwa=bwa, bplib=bplib, samtools=samtools, outfq=outfq, outbam=outbam)
            func_logger.info("Running %s" % bash_cmd)
            with open(outlog, "w") as logfd:
                subprocess.check_call(bash_cmd, shell=True, stderr=logfd)
        else:
            func_logger.info("No reads extracted from {bam} so no alignment will be done".format(bam=bam))
    except Exception as e:
        func_logger.error('Caught exception in worker thread')
        traceback.print_exc()

        print()
        raise e

    if not keep_temp:
        os.remove(outfq)
    return outbam

def preprocess_and_align_callback(result, result_list):
    if result is not None:
        result_list.append(result)

# Parallelize over BAMs and chromosomes
def parallel_preprocess_and_align(bplib, bwa, samtools, bams, work, chromosomes=[], nthreads=1, keep_temp=False):
    func_logger = logging.getLogger(parallel_preprocess_and_align.__name__)

    if not os.path.isdir(work):
        os.makedirs(work)

    if not chromosomes: chromosomes = [None]
    else: chromosomes.append("") # Special contig for unmapped reads

    pool = multiprocessing.Pool(nthreads)
    count = 0
    aligned_bams = []
    for bam  in bams:
        for chromosome in chromosomes:
            kwargs_dict = {"bplib": bplib, "bwa": bwa, "samtools": samtools, "bam": bam, "prefix": "%s/%d" % (work, count), "chromosome": chromosome, "keep_temp": keep_temp}
            pool.apply_async(preprocess_and_align, args=[], kwds=kwargs_dict, callback=partial(preprocess_and_align_callback, result_list=aligned_bams))
            count += 1

    pool.close()
    pool.join()

    # Now merge the BAMs together
    if not aligned_bams:
        func_logger.info("No alignment generated after read extraction")
        return []

    finalbam = os.path.join(work, "final.bam")
    if len(aligned_bams) > 1:
        merged_str = " ".join(aligned_bams)
        bash_cmd = "bash -c \"{samtools} merge -f {finalbam} {merged_str}\"".format(samtools=samtools, finalbam=finalbam, merged_str=merged_str)
        subprocess.check_call(bash_cmd, shell=True)
    else:
        shutil.copy(aligned_bams[0], finalbam)

    if not keep_temp:
        for bam in aligned_bams:
            os.remove(bam)

    return [finalbam]

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="Process BAMs to generate candidate reads and align them against the breakpoint library")
    add_options(parser)
    parser.add_argument("--bams", nargs="+", help="BAMs to extract reads from", required=True)
    parser.add_argument("--chromosomes", nargs="+", help="List of chromosomes to process", default=[])
    parser.add_argument("--work", help="Working directory")
    parser.add_argument("--nthreads", help="Number of processes to use", type=int, default=1)
    parser.add_argument("--keep_temp", help="Keep temporary files", action="store_true")

    args = parser.parse_args()
    preprocess_and_align(args.bplib, args.bwa, args.samtools, args.bams, args.chromosomes, args.nthreads, args.keep_temp)
<|endoftext|>


<commit_before>import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

def simple_prettifier(scraped_data):
    """Return more presentable data (in a list) provided by scrape_target_elements()

    :param bs4.element.ResultSet scraped_data: all of the data scraped by scrape_target_elements()
    :return: list of presentable data
    :rtype: list
    """
    data_list = []
    for data in scraped_data:
        data_list.append(data.text)
    return data_list
<commit_msg>Add regex prettifier<commit_after>import os
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

import re

def simple_prettifier(scraped_data):
    """Return more presentable data (in a list) provided by scrape_target_elements()

    :param bs4.element.ResultSet scraped_data: all of the data scraped by scrape_target_elements()
    :return: list of presentable data
    :rtype: list
    """
    data_list = []
    for data in scraped_data:
        data_list.append(data.text)
    return data_list

def regex_prettifier(scraped_data, regex):
    """Return regex modified data (in a list).

    :param list scraped_data: all the scraped data
    :param str regex: the regular expression you want to use to prettify the
    data
    :return: list of regex modified data
    :rtype: list
    """
    data_list = []
    for data in scraped_data:
        data_list.append(re.sub(regex, '', data))
    return data_list

<|endoftext|>


<commit_before>
import Utils

class CppBuild:

    def __init__(self, env, dirs, target, ext_libs = [], libs = [], build = True):

        # Gather source/header files
        self.cpp_files = []
        self.hpp_files = []
        for dir in dirs:
            self.cpp_files += Utils.Glob(dir, "*.cpp")
            self.cpp_files += Utils.Glob(dir, "*.c")
            self.hpp_files += Utils.Glob(dir, "*.h")

        # Compile C++ files and create file nodes out of external library references
        self.obj_files = [ env.CPPFile(file) for file in self.cpp_files ]
        self.lib_files = [ env.NewFile(file) for file in ext_libs ]

        # Link or use librarian dependent on output path
        target = target.lower()
        self.output = None
        if target.endswith(".exe"):
            self.output = self.exe = env.Link(target, self.obj_files, self.lib_files + libs)
        elif target.endswith(".dll"):
            self.output = self.dll = env.Link(target, self.obj_files, self.lib_files + libs)
        elif target.endswith(".lib"):
            self.output = self.lib = env.Lib(target, self.obj_files, self.lib_files + libs)

        # Build all the config command lines
        for config in env.Configs.values():
            config.UpdateCommandLines()

        if build:
            env.Build(self.output, target[:-4])

    def OverrideCPPOptions(self, cpp_file_match, override_cpp_opts):

        # Find all C++ files that match the input string and apply the override options
        cpp_file_match = cpp_file_match.lower()
        for obj_file in self.obj_files:
            if cpp_file_match in obj_file.Path.lower():
                obj_file.SetCPPOptions(override_cpp_opts)
<commit_msg>Remove lowercase transform from CppBuild targets.<commit_after>
import Utils

class CppBuild:

    def __init__(self, env, dirs, target, ext_libs = [], libs = [], build = True):

        # Gather source/header files
        self.cpp_files = []
        self.hpp_files = []
        for dir in dirs:
            self.cpp_files += Utils.Glob(dir, "*.cpp")
            self.cpp_files += Utils.Glob(dir, "*.c")
            self.hpp_files += Utils.Glob(dir, "*.h")

        # Compile C++ files and create file nodes out of external library references
        self.obj_files = [ env.CPPFile(file) for file in self.cpp_files ]
        self.lib_files = [ env.NewFile(file) for file in ext_libs ]

        # Link or use librarian dependent on output path
        self.output = None
        if target.endswith(".exe"):
            self.output = self.exe = env.Link(target, self.obj_files, self.lib_files + libs)
        elif target.endswith(".dll"):
            self.output = self.dll = env.Link(target, self.obj_files, self.lib_files + libs)
        elif target.endswith(".lib"):
            self.output = self.lib = env.Lib(target, self.obj_files, self.lib_files + libs)

        # Build all the config command lines
        for config in env.Configs.values():
            config.UpdateCommandLines()

        if build:
            env.Build(self.output, target[:-4])

    def OverrideCPPOptions(self, cpp_file_match, override_cpp_opts):

        # Find all C++ files that match the input string and apply the override options
        cpp_file_match = cpp_file_match.lower()
        for obj_file in self.obj_files:
            if cpp_file_match in obj_file.Path.lower():
                obj_file.SetCPPOptions(override_cpp_opts)
<|endoftext|>


<commit_before>'''
Modify this file, by platform, to handle nonstandard options for third-party
dependencies. If you do modify this file, you should make it read-only
(or otherwise protect it) so that CVS will not overwrite it.

Order is important: place local paths ahead of system paths.
'''
import sys

customCPPPATH = []
customCCFLAGS = []
customCXXFLAGS = []
customLIBS = []
customLIBPATH = []
customSHLINKFLAGS = []
customSWIGFLAGS = []

if sys.platform[:5] == 'linux':
    platform = 'linux'
elif sys.platform == 'cygwin':
    platform = 'cygwin'
    customCPPPATH.append('c:/apcc/tools/Python23/include')
    customLIBPATH.append('cygwin_import_libs')
elif sys.platform[:3] == 'win':
    # For the basic build you need MinGW, MSys, and libsndfile.
    # Add them here:
    customLIBPATH.append('c:/apcc/tools/mingw/lib')
    customCPPPATH.append('c:/apcc/tools/msys/1.0/local/include')
    customLIBPATH.append('c:/apcc/tools/msys/1.0/local/lib')
    # If libsndfile is not in a standard location add it here:
    customCPPPATH.append('C:/apcc/tools/libsndfile-1.0.10/src')
    customLIBPATH.append('C:/apcc/tools/libsndfile-1.0.10')
    ################################################################
    # If you want real-time audio you need PortAudio.
    # If it is not in a standard location add it here
    # (of course you must build it first):
    # customCPPPATH.append('C:/projects/portaudio/pa_common')
    # customLIBPATH.append('C:/projects/portaudio/lib')
    ################################################################
    # If you want FLTK widgets or if you want to build CsoundVST,
    # you need FLTK. If it is not in a standard location,
    # add it here (of course you must build it first):
    # customCPPPATH.append('C:/tools/fltk-1.1.5rc1')
    # customLIBPATH.append('C:/tools/fltk-1.1.5rc1/lib')
    ################################################################
    # If you want to build CsoundVST you need Python and
    # a MinGW import library for Python. Add them here:
    customCPPPATH.append('c:/apcc/tools/Python23/include')
    customCPPPATH.append('c:/apcc/tools/mingw/include')
    customLIBPATH.append('c:/projects/csound5/cygwin_import_libs')
    customSWIGFLAGS.append('-Ic:/apcc/tools/mingw/lib/gcc-lib/mingw32/3.3.1/include')
    ################################################################
    # If you want to build CsoundVST you need boost.
    # If it is not in a standard lcoation add it here
    # (you do NOT need to build it first):
    customCPPPATH.append('c:/apcc/tools/boost')
    ################################################################
    # If you want to build the FluidSynth opcodes
    # you need FluidSynth. If it is not a standard location,
    # add it here (you do NOT need to build it first):
    customLIBPATH.append('c:/apcc/tools/fluidsynth-1.0.3-win32')
    customCPPPATH.append('c:/apcc/tools/fluidsynth-1.0.3-win32/include')
    ################################################################
    platform = 'mingw'
else:
    platform = 'unsupported platform'

<commit_msg>Updated for more current versions of third-party libraries.<commit_after>'''
Modify this file, by platform, to handle nonstandard options for third-party
dependencies. If you do modify this file, you should make it read-only
(or otherwise protect it) so that CVS will not overwrite it.

Order is important: place local paths ahead of system paths.
'''
import sys

customCPPPATH = []
customCCFLAGS = []
customCXXFLAGS = []
customLIBS = []
customLIBPATH = []
customSHLINKFLAGS = []
customSWIGFLAGS = []

if sys.platform[:5] == 'linux':
    platform = 'linux'
elif sys.platform == 'cygwin':
    platform = 'cygwin'
    customCPPPATH.append('c:/apcc/tools/Python23/include')
    customLIBPATH.append('cygwin_import_libs')
elif sys.platform[:3] == 'win':
    # For the basic build you need MinGW, MSys, and libsndfile.
    # Add them here:
    customLIBPATH.append('c:/apcc/tools/mingw/lib')
    customCPPPATH.append('c:/apcc/tools/msys/1.0/local/include')
    customLIBPATH.append('c:/apcc/tools/msys/1.0/local/lib')
    # If libsndfile is not in a standard location add it here:
    #customCPPPATH.append('C:/apcc/tools/libsndfile-1.0.11/src')
    #customLIBPATH.append('C:/apcc/tools/libsndfile-1.0.11')
    ################################################################
    # If you want real-time audio you need PortAudio.
    # If it is not in a standard location add it here
    # (of course you must build it first):
    # customCPPPATH.append('C:/projects/portaudio/pa_common')
    # customLIBPATH.append('C:/projects/portaudio/lib')
    ################################################################
    # If you want FLTK widgets or if you want to build CsoundVST,
    # you need FLTK. If it is not in a standard location,
    # add it here (of course you must build it first):
    # customCPPPATH.append('C:/tools/fltk-1.1.6')
    # customLIBPATH.append('C:/tools/fltk-1.1.6/lib')
    ################################################################
    # If you want to build CsoundVST you need Python and
    # a MinGW import library for Python. Add them here:
    customCPPPATH.append('c:/apcc/tools/Python23/include')
    customCPPPATH.append('c:/apcc/tools/mingw/include')
    customLIBPATH.append('c:/projects/csound5/cygwin_import_libs')
    customSWIGFLAGS.append('-Ic:/apcc/tools/mingw/lib/gcc-lib/mingw32/3.3.1/include')
    ################################################################
    # If you want to build CsoundVST you need boost.
    # If it is not in a standard lcoation add it here
    # (you do NOT need to build it first):
    customCPPPATH.append('c:/apcc/tools/boost')
    ################################################################
    # If you want to build the FluidSynth opcodes
    # you need FluidSynth. If it is not a standard location,
    # add it here (you do NOT need to build it first):
    customLIBPATH.append('c:/apcc/tools/fluidsynth-1.0.3-win32')
    customCPPPATH.append('c:/apcc/tools/fluidsynth-1.0.3-win32/include')
    ################################################################
    platform = 'mingw'
else:
    platform = 'unsupported platform'

<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

# (c) 2012, Franck Cuny <franck@lumberjaph.net>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#

DOCUMENTATION = '''
---
module: cpanm
short_description: Manages Perl library dependencies.
description:
  - Manage Perl library dependencies.
version_added: "1.6"
options:
  name:
    description:
      - The name of the Perl library to install. You may use the "full distribution path", e.g.  MIYAGAWA/Plack-0.99_05.tar.gz
    required: false
    default: null
    aliases: ["pkg"]
  from_path:
    description:
      - The local directory from where to install
    required: false
    default: null
  notest:
    description:
      - Do not run unit tests
    required: false
    default: false
  locallib:
    description:
      - Specify the install base to install modules
    required: false
    default: false
  mirror:
    description:
      - Specifies the base URL for the CPAN mirror to use
    required: false
    default: false
  mirror_only:
    description:
      - Use the mirror's index file instead of the CPAN Meta DB
    required: false
    default: false
examples:
   - code: "cpanm: name=Dancer"
     description: Install I(Dancer) perl package.
   - code: "cpanm: name=MIYAGAWA/Plack-0.99_05.tar.gz"
     description: Install version 0.99_05 of the I(Plack) perl package.
   - code: "cpanm: name=Dancer locallib=/srv/webapps/my_app/extlib"
     description: "Install I(Dancer) (U(http://perldancer.org/)) into the specified I(locallib)"
   - code: "cpanm: from_path=/srv/webapps/my_app/src/"
     description: Install perl dependencies from local directory.
   - code: "cpanm: name=Dancer notest=True locallib=/srv/webapps/my_app/extlib"
     description: Install I(Dancer) perl package without running the unit tests in indicated I(locallib).
   - code: "cpanm: name=Dancer mirror=http://cpan.cpantesters.org/"
     description: Install I(Dancer) perl package from a specific mirror
notes:
   - Please note that U(http://search.cpan.org/dist/App-cpanminus/bin/cpanm, cpanm) must be installed on the remote host.
author: "Franck Cuny (@franckcuny)"
'''

def _is_package_installed(module, name, locallib, cpanm):
    cmd = ""
    if locallib:
        os.environ["PERL5LIB"] = "%s/lib/perl5" % locallib
    cmd = "%s perl -M%s -e '1'" % (cmd, name)
    res, stdout, stderr = module.run_command(cmd, check_rc=False)
    if res == 0:
       return True
    else:
       return False

def _build_cmd_line(name, from_path, notest, locallib, mirror, mirror_only, cpanm):
    # this code should use "%s" like everything else and just return early but not fixing all of it now.
    # don't copy stuff like this
    if from_path:
        cmd = "{cpanm} {path}".format(cpanm=cpanm, path=from_path)
    else:
        cmd = "{cpanm} {name}".format(cpanm=cpanm, name=name)

    if notest is True:
        cmd = "{cmd} -n".format(cmd=cmd)

    if locallib is not None:
        cmd = "{cmd} -l {locallib}".format(cmd=cmd, locallib=locallib)

    if mirror is not None:
        cmd = "{cmd} --mirror {mirror}".format(cmd=cmd, mirror=mirror)

    if mirror_only is True:
        cmd = "{cmd} --mirror-only".format(cmd=cmd)

    return cmd

def main():
    arg_spec = dict(
        name=dict(default=None, required=False, aliases=['pkg']),
        from_path=dict(default=None, required=False),
        notest=dict(default=False, type='bool'),
        locallib=dict(default=None, required=False),
        mirror=dict(default=None, required=False),
        mirror_only=dict(default=False, type='bool'),
    )

    module = AnsibleModule(
        argument_spec=arg_spec,
        required_one_of=[['name', 'from_path']],
    )

    cpanm       = module.get_bin_path('cpanm', True)
    name        = module.params['name']
    from_path   = module.params['from_path']
    notest      = module.boolean(module.params.get('notest', False))
    locallib    = module.params['locallib']
    mirror      = module.params['mirror']
    mirror_only = module.params['mirror_only']

    changed   = False

    installed = _is_package_installed(module, name, locallib, cpanm)

    if not installed:
        out_cpanm = err_cpanm = ''
        cmd       = _build_cmd_line(name, from_path, notest, locallib, mirror, mirror_only, cpanm)

        rc_cpanm, out_cpanm, err_cpanm = module.run_command(cmd, check_rc=False)

        if rc_cpanm != 0:
            module.fail_json(msg=err_cpanm, cmd=cmd)

        if err_cpanm and 'is up to date' not in err_cpanm:
            changed = True

    module.exit_json(changed=changed, binary=cpanm, name=name)

# import module snippets
from ansible.module_utils.basic import *

main()
<commit_msg>Move examples to EXAMPLES variable<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

# (c) 2012, Franck Cuny <franck@lumberjaph.net>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible.  If not, see <http://www.gnu.org/licenses/>.
#

DOCUMENTATION = '''
---
module: cpanm
short_description: Manages Perl library dependencies.
description:
  - Manage Perl library dependencies.
version_added: "1.6"
options:
  name:
    description:
      - The name of the Perl library to install. You may use the "full distribution path", e.g.  MIYAGAWA/Plack-0.99_05.tar.gz
    required: false
    default: null
    aliases: ["pkg"]
  from_path:
    description:
      - The local directory from where to install
    required: false
    default: null
  notest:
    description:
      - Do not run unit tests
    required: false
    default: false
  locallib:
    description:
      - Specify the install base to install modules
    required: false
    default: false
  mirror:
    description:
      - Specifies the base URL for the CPAN mirror to use
    required: false
    default: false
  mirror_only:
    description:
      - Use the mirror's index file instead of the CPAN Meta DB
    required: false
    default: false
notes:
   - Please note that U(http://search.cpan.org/dist/App-cpanminus/bin/cpanm, cpanm) must be installed on the remote host.
author: "Franck Cuny (@franckcuny)"
'''

EXAMPLES = '''
# install Dancer perl package
- cpanm: name=Dancer

# install version 0.99_05 of the Plack perl package
- cpanm: name=MIYAGAWA/Plack-0.99_05.tar.gz

# install Dancer into the specified locallib
- cpanm: name=Dancer locallib=/srv/webapps/my_app/extlib

# install perl dependencies from local directory
- cpanm: from_path=/srv/webapps/my_app/src/

# install Dancer perl package without running the unit tests in indicated locallib
- cpanm: name=Dancer notest=True locallib=/srv/webapps/my_app/extlib

# install Dancer perl package from a specific mirror
- cpanm: name=Dancer mirror=http://cpan.cpantesters.org/
'''

def _is_package_installed(module, name, locallib, cpanm):
    cmd = ""
    if locallib:
        os.environ["PERL5LIB"] = "%s/lib/perl5" % locallib
    cmd = "%s perl -M%s -e '1'" % (cmd, name)
    res, stdout, stderr = module.run_command(cmd, check_rc=False)
    if res == 0:
       return True
    else:
       return False

def _build_cmd_line(name, from_path, notest, locallib, mirror, mirror_only, cpanm):
    # this code should use "%s" like everything else and just return early but not fixing all of it now.
    # don't copy stuff like this
    if from_path:
        cmd = "{cpanm} {path}".format(cpanm=cpanm, path=from_path)
    else:
        cmd = "{cpanm} {name}".format(cpanm=cpanm, name=name)

    if notest is True:
        cmd = "{cmd} -n".format(cmd=cmd)

    if locallib is not None:
        cmd = "{cmd} -l {locallib}".format(cmd=cmd, locallib=locallib)

    if mirror is not None:
        cmd = "{cmd} --mirror {mirror}".format(cmd=cmd, mirror=mirror)

    if mirror_only is True:
        cmd = "{cmd} --mirror-only".format(cmd=cmd)

    return cmd

def main():
    arg_spec = dict(
        name=dict(default=None, required=False, aliases=['pkg']),
        from_path=dict(default=None, required=False),
        notest=dict(default=False, type='bool'),
        locallib=dict(default=None, required=False),
        mirror=dict(default=None, required=False),
        mirror_only=dict(default=False, type='bool'),
    )

    module = AnsibleModule(
        argument_spec=arg_spec,
        required_one_of=[['name', 'from_path']],
    )

    cpanm       = module.get_bin_path('cpanm', True)
    name        = module.params['name']
    from_path   = module.params['from_path']
    notest      = module.boolean(module.params.get('notest', False))
    locallib    = module.params['locallib']
    mirror      = module.params['mirror']
    mirror_only = module.params['mirror_only']

    changed   = False

    installed = _is_package_installed(module, name, locallib, cpanm)

    if not installed:
        out_cpanm = err_cpanm = ''
        cmd       = _build_cmd_line(name, from_path, notest, locallib, mirror, mirror_only, cpanm)

        rc_cpanm, out_cpanm, err_cpanm = module.run_command(cmd, check_rc=False)

        if rc_cpanm != 0:
            module.fail_json(msg=err_cpanm, cmd=cmd)

        if err_cpanm and 'is up to date' not in err_cpanm:
            changed = True

    module.exit_json(changed=changed, binary=cpanm, name=name)

# import module snippets
from ansible.module_utils.basic import *

main()
<|endoftext|>


<commit_before>from waitress import serve
from samil import app

app.run(debug=True, host="0.0.0.0", port=80)
# serve(app, port=8080)
# serve(app)

# test 2
<commit_msg>test 3<commit_after>from waitress import serve
from samil import app

app.run(debug=True, host="0.0.0.0", port=80)
# serve(app, port=8080)
# serve(app)

# test 3
<|endoftext|>