<commit_before>import subprocess
import shlex
import re
import os
import time
import urllib2
import platform
import json
import sys
import base64
import random

server = "runmyrobot.com"
#server = "52.52.213.92"

from socketIO_client import SocketIO, LoggingNamespace

print "test1"

if len(sys.argv) >= 4:
    print "using dev port 8122"
    port = 8122
else:
    print "using prod port 8022"
    port = 8022

print "initializing socket io"
print "server:", server
print "port:", port
socketIO = SocketIO(server, port, LoggingNamespace)
print "finished initializing socket io"

#ffmpeg -f qtkit -i 0 -f mpeg1video -b 400k -r 30 -s 320x240 http://52.8.81.124:8082/hello/320/240/

def onHandleCameraCommand(*args):
    #thread.start_new_thread(handle_command, args)
    print args

socketIO.on('command_to_camera', onHandleCameraCommand)

def randomSleep():
    """A short wait is good for quick recovery, but sometimes a longer delay is needed or it will just keep trying and failing short intervals, like because the system thinks the port is still in use and every retry makes the system think it's still in use. So, this has a high likelihood of picking a short interval, but will pick a long one sometimes."""

    timeToWait = random.choice((0.25, 0.25, 0.25, 0.25, 0.25, 0.25, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 5))
    print "sleeping", timeToWait
    time.sleep(timeToWait)

def getVideoPort():

    url = 'http://%s/get_video_port/%s' % (server, cameraIDAnswer)

    for retryNumber in range(2000):
        try:
            print "GET", url
            response = urllib2.urlopen(url).read()
            break
        except:
            print "could not open url ", url
            time.sleep(2)

    return json.loads(response)['mpeg_stream_port']

def runFfmpeg(commandLine):

    print commandLine
    ffmpegProcess = subprocess.Popen(shlex.split(commandLine))
    print "command started"

    return ffmpegProcess

def handleDarwin(deviceNumber, videoPort):

    p = subprocess.Popen(["ffmpeg", "-list_devices", "true", "-f", "qtkit", "-i", "dummy"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    out, err = p.communicate()

    print err

    deviceAnswer = raw_input("Enter the number of the camera device for your robot from the list above: ")
    commandLine = 'ffmpeg -f qtkit -i %s -f mpeg1video -b 400k -r 30 -s 320x240 http://%s:%s/hello/320/240/' % (deviceAnswer, server, videoPort)

    process = runFfmpeg(commandLine)

    return {'process': process, 'device_answer': deviceAnswer}

def handleLinux(deviceNumber, videoPort):

    print "sleeping to give the camera time to start working"
    randomSleep()
    print "finished sleeping"

    #p = subprocess.Popen(["ffmpeg", "-list_devices", "true", "-f", "qtkit", "-i", "dummy"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    #out, err = p.communicate()
    #print err

    #os.system("v4l2-ctl -c gain=60 -c brightness=10 -c contrast=25 -c saturation=30") # Timmy
    #os.system("v4l2-ctl -c brightness=10 -c contrast=25 -c saturation=40") # Timmy
    #os.system("v4l2-ctl -c brightness=10 -c contrast=25 -c saturation=40")
    #os.system("v4l2-ctl -c brightness=240 -c contrast=75 -c saturation=60") # Skippy
    #os.system("v4l2-ctl -c brightness=150 -c contrast=50 -c saturation=80") # Marvin
    #os.system("v4l2-ctl -c brightness=150 -c contrast=50 -c saturation=80") # Marvin
    #os.system("v4l2-ctl -c brightness=10 -c contrast=70 -c saturation=80") # RedBird
    #os.system("v4l2-ctl -c brightness=40 -c contrast=70 -c saturation=80") # ClawDaddy
    #os.system("v4l2-ctl -c brightness=50 -c contrast=50 -c saturation=80")
    #os.system("v4l2-ctl -c brightness=200 -c contrast=100 -c saturation=100")
    #os.system("v4l2-ctl -c brightness=20 -c contrast=50 -c saturation=60") # Jenny
    os.system("v4l2-ctl -c brightness=10 -c contrast=50 -c saturation=60") # BlueberrySurprise

    if deviceNumber is None:
        deviceAnswer = raw_input("Enter the number of the camera device for your robot: ")
    else:
        deviceAnswer = str(deviceNumber)

    #commandLine = '/usr/local/bin/ffmpeg -s 320x240 -f video4linux2 -i /dev/video%s -f mpeg1video -b 1k -r 20 http://runmyrobot.com:%s/hello/320/240/' % (deviceAnswer, videoPort)
    #commandLine = '/usr/local/bin/ffmpeg -s 640x480 -f video4linux2 -i /dev/video%s -f mpeg1video -b 150k -r 20 http://%s:%s/hello/640/480/' % (deviceAnswer, server, videoPort)
    commandLine = '/usr/local/bin/ffmpeg -s 640x480 -f video4linux2 -i /dev/video%s -f mpeg1video -b 200k -r 20 http://%s:%s/hello/640/480/' % (deviceAnswer, server, videoPort) # ClawDaddy
    #commandLine = '/usr/local/bin/ffmpeg -s 1280x720 -f video4linux2 -i /dev/video%s -f mpeg1video -b 1k -r 20 http://runmyrobot.com:%s/hello/1280/720/' % (deviceAnswer, videoPort)

    process = runFfmpeg(commandLine)

    return {'process': process, 'device_answer': deviceAnswer}

def handleWindows(deviceNumber, videoPort):

    p = subprocess.Popen(["ffmpeg", "-list_devices", "true", "-f", "dshow", "-i", "dummy"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    out, err = p.communicate()

    lines = err.split('\n')

    count = 0

    devices = []

    for line in lines:

        #if "]  \"" in line:
        #    print "line:", line

        m = re.search('.*\\"(.*)\\"', line)
        if m != None:
            #print line
            if m.group(1)[0:1] != '@':
                print count, m.group(1)
                devices.append(m.group(1))
                count += 1

    if deviceNumber is None:
        deviceAnswer = raw_input("Enter the number of the camera device for your robot from the list above: ")
    else:
        deviceAnswer = str(deviceNumber)

    device = devices[int(deviceAnswer)]
    commandLine = 'ffmpeg -s 320x240 -f dshow -i video="%s" -f mpeg1video -b 200k -r 20 http://%s:%s/hello/320/240/' % (device, server, videoPort)

    process = runFfmpeg(commandLine)

    return {'process': process, 'device_answer': device}

def handleWindowsScreenCapture(deviceNumber, videoPort):

    p = subprocess.Popen(["ffmpeg", "-list_devices", "true", "-f", "dshow", "-i", "dummy"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    out, err = p.communicate()

    lines = err.split('\n')

    count = 0

    devices = []

    for line in lines:

        #if "]  \"" in line:
        #    print "line:", line

        m = re.search('.*\\"(.*)\\"', line)
        if m != None:
            #print line
            if m.group(1)[0:1] != '@':
                print count, m.group(1)
                devices.append(m.group(1))
                count += 1

    if deviceNumber is None:
        deviceAnswer = raw_input("Enter the number of the camera device for your robot from the list above: ")
    else:
        deviceAnswer = str(deviceNumber)

    device = devices[int(deviceAnswer)]
    commandLine = 'ffmpeg -f dshow -i video="screen-capture-recorder" -vf "scale=640:480" -f mpeg1video -b 50k -r 20 http://%s:%s/hello/640/480/' % (server, videoPort)

    print "command line:", commandLine

    process = runFfmpeg(commandLine)

    return {'process': process, 'device_answer': device}

def snapShot(operatingSystem, inputDeviceID, filename="snapshot.jpg"):

    try:
        os.remove('snapshot.jpg')
    except:
        print "did not remove file"

    commandLineDict = {
        'Darwin': 'ffmpeg -y -f qtkit -i %s -vframes 1 %s' % (inputDeviceID, filename),
        'Linux': '/usr/local/bin/ffmpeg -y -f video4linux2 -i /dev/video%s -vframes 1 -q:v 1000 -vf scale=320:240 %s' % (inputDeviceID, filename),
        'Windows': 'ffmpeg -y -s 320x240 -f dshow -i video="%s" -vframes 1 %s' % (inputDeviceID, filename)}

    print commandLineDict[operatingSystem]
    os.system(commandLineDict[operatingSystem])

def startVideoCapture():

    videoPort = getVideoPort()
    print "video port:", videoPort

    if len(sys.argv) >= 3:
        deviceNumber = sys.argv[2]
    else:
        deviceNumber = None

    result = None
    if platform.system() == 'Darwin':
        result = handleDarwin(deviceNumber, videoPort)
    elif platform.system() == 'Linux':
        result = handleLinux(deviceNumber, videoPort)
    elif platform.system() == 'Windows':
        #result = handleWindowsScreenCapture(deviceNumber, videoPort)
        result = handleWindows(deviceNumber, videoPort)
    else:
        print "unknown platform", platform.system()

    return result

def timeInMilliseconds():
    return int(round(time.time() * 1000))

def main():

    print "main"

    streamProcessDict = None

    twitterSnapCount = 0

    while True:

        socketIO.emit('send_video_status', {'send_video_process_exists': True,
                                            'camera_id':cameraIDAnswer})

        if streamProcessDict is not None:
            print "stopping previously running ffmpeg (needs to happen if this is not the first iteration)"
            streamProcessDict['process'].kill()

        print "starting process just to get device result" # this should be a separate function so you don't have to do this
        streamProcessDict = startVideoCapture()
        inputDeviceID = streamProcessDict['device_answer']
        print "stopping video capture"
        streamProcessDict['process'].kill()

        #print "sleeping"
        #time.sleep(3)
        #frameCount = int(round(time.time() * 1000))

        videoWithSnapshots = False
        while videoWithSnapshots:

            frameCount = timeInMilliseconds()

            print "taking single frame image"
            snapShot(platform.system(), inputDeviceID, filename="single_frame_image.jpg")

            with open ("single_frame_image.jpg", 'rb') as f:

                # every so many frames, post a snapshot to twitter
                #if frameCount % 450 == 0:
                if frameCount % 6000 == 0:
                        data = f.read()
                        print "emit"
                        socketIO.emit('snapshot', {'frame_count':frameCount, 'image':base64.b64encode(data)})
                data = f.read()

            print "emit"
            socketIO.emit('single_frame_image', {'frame_count':frameCount, 'image':base64.b64encode(data)})
            time.sleep(0)

            #frameCount += 1

        if False:
         if platform.system() != 'Windows':
            print "taking snapshot"
            snapShot(platform.system(), inputDeviceID)
            with open ("snapshot.jpg", 'rb') as f:
                data = f.read()
            print "emit"

            # skip sending the first image because it's mostly black, maybe completely black
            #todo: should find out why this black image happens
            if twitterSnapCount > 0:
                socketIO.emit('snapshot', {'image':base64.b64encode(data)})

        print "starting video capture"
        streamProcessDict = startVideoCapture()

        # This loop counts out a delay that occurs between twitter snapshots.
        # Every 50 seconds, it kills and restarts ffmpeg.
        # Every 40 seconds, it sends a signal to the server indicating status of processes.
        period = 2*60*60 # period in seconds between snaps
        for count in range(period):
            time.sleep(1)

            if count % 20 == 0:
                socketIO.emit('send_video_status', {'send_video_process_exists': True,
                                                    'camera_id':cameraIDAnswer})

            if count % 161 == 30:
                print "stopping video capture just in case it has reached a state where it's looping forever, not sending video, and not dying as a process, which can happen"
                streamProcessDict['process'].kill()

            if count % 80 == 75:
                print "send status about this process and its child process ffmpeg"
                ffmpegProcessExists = streamProcessDict['process'].poll() is None
                socketIO.emit('send_video_status', {'send_video_process_exists': True,
                                                    'ffmpeg_process_exists': ffmpegProcessExists,
                                                    'camera_id':cameraIDAnswer})

            #if count % 190 == 180:
            #    print "reboot system in case the webcam is not working"
            #    os.system("sudo reboot")

            # if the video stream process dies, restart it
            if streamProcessDict['process'].poll() is not None:
                # wait before trying to start ffmpeg
                print "ffmpeg process is dead, waiting before trying to restart"
                randomSleep()
                streamProcessDict = startVideoCapture()

        #print "taking snapshot"
        #snapShot(platform.system(), inputDeviceID)
        #with open ("snapshot.jpg", 'rb') as f:
        #    data = f.read()
        #print "emit"
        #socketIO.emit('snapshot', {'image':base64.b64encode(data)})

        twitterSnapCount += 1

print "test2"

if __name__ == "__main__":

    if len(sys.argv) > 1:
        cameraIDAnswer = sys.argv[1]
    else:
        cameraIDAnswer = raw_input("Enter the Camera ID for your robot, you can get it by pointing a browser to the runmyrobot server %s: " % server)

    main()

<commit_msg>added take snapshot command<commit_after>import subprocess
import shlex
import re
import os
import time
import urllib2
import platform
import json
import sys
import base64
import random

server = "runmyrobot.com"
#server = "52.52.213.92"

from socketIO_client import SocketIO, LoggingNamespace

print "test1"

if len(sys.argv) >= 4:
    print "using dev port 8122"
    port = 8122
else:
    print "using prod port 8022"
    port = 8022

print "initializing socket io"
print "server:", server
print "port:", port
socketIO = SocketIO(server, port, LoggingNamespace)
print "finished initializing socket io"

#ffmpeg -f qtkit -i 0 -f mpeg1video -b 400k -r 30 -s 320x240 http://52.8.81.124:8082/hello/320/240/

def onHandleCameraCommand(*args):
    #thread.start_new_thread(handle_command, args)
    print args

socketIO.on('command_to_camera', onHandleCameraCommand)

def onHandleTakeSnapshotCommand(*args):
    print "taking snapshot"
    inputDeviceID = streamProcessDict['device_answer']
    snapShot(platform.system(), inputDeviceID)
    with open ("snapshot.jpg", 'rb') as f:
        data = f.read()
    print "emit"

    socketIO.emit('snapshot', {'image':base64.b64encode(data)})

socketIO.on('take_snapshot_command', onHandleTakeSnapshotCommand)

def randomSleep():
    """A short wait is good for quick recovery, but sometimes a longer delay is needed or it will just keep trying and failing short intervals, like because the system thinks the port is still in use and every retry makes the system think it's still in use. So, this has a high likelihood of picking a short interval, but will pick a long one sometimes."""

    timeToWait = random.choice((0.25, 0.25, 0.25, 0.25, 0.25, 0.25, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 5))
    print "sleeping", timeToWait
    time.sleep(timeToWait)

def getVideoPort():

    url = 'http://%s/get_video_port/%s' % (server, cameraIDAnswer)

    for retryNumber in range(2000):
        try:
            print "GET", url
            response = urllib2.urlopen(url).read()
            break
        except:
            print "could not open url ", url
            time.sleep(2)

    return json.loads(response)['mpeg_stream_port']

def runFfmpeg(commandLine):

    print commandLine
    ffmpegProcess = subprocess.Popen(shlex.split(commandLine))
    print "command started"

    return ffmpegProcess

def handleDarwin(deviceNumber, videoPort):

    p = subprocess.Popen(["ffmpeg", "-list_devices", "true", "-f", "qtkit", "-i", "dummy"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    out, err = p.communicate()

    print err

    deviceAnswer = raw_input("Enter the number of the camera device for your robot from the list above: ")
    commandLine = 'ffmpeg -f qtkit -i %s -f mpeg1video -b 400k -r 30 -s 320x240 http://%s:%s/hello/320/240/' % (deviceAnswer, server, videoPort)

    process = runFfmpeg(commandLine)

    return {'process': process, 'device_answer': deviceAnswer}

def handleLinux(deviceNumber, videoPort):

    print "sleeping to give the camera time to start working"
    randomSleep()
    print "finished sleeping"

    #p = subprocess.Popen(["ffmpeg", "-list_devices", "true", "-f", "qtkit", "-i", "dummy"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    #out, err = p.communicate()
    #print err

    #os.system("v4l2-ctl -c gain=60 -c brightness=10 -c contrast=25 -c saturation=30") # Timmy
    #os.system("v4l2-ctl -c brightness=10 -c contrast=25 -c saturation=40") # Timmy
    #os.system("v4l2-ctl -c brightness=10 -c contrast=25 -c saturation=40")
    #os.system("v4l2-ctl -c brightness=240 -c contrast=75 -c saturation=60") # Skippy
    #os.system("v4l2-ctl -c brightness=150 -c contrast=50 -c saturation=80") # Marvin
    #os.system("v4l2-ctl -c brightness=150 -c contrast=50 -c saturation=80") # Marvin
    #os.system("v4l2-ctl -c brightness=10 -c contrast=70 -c saturation=80") # RedBird
    #os.system("v4l2-ctl -c brightness=40 -c contrast=70 -c saturation=80") # ClawDaddy
    #os.system("v4l2-ctl -c brightness=50 -c contrast=50 -c saturation=80")
    #os.system("v4l2-ctl -c brightness=200 -c contrast=100 -c saturation=100")
    #os.system("v4l2-ctl -c brightness=20 -c contrast=50 -c saturation=60") # Jenny
    os.system("v4l2-ctl -c brightness=10 -c contrast=50 -c saturation=60") # BlueberrySurprise

    if deviceNumber is None:
        deviceAnswer = raw_input("Enter the number of the camera device for your robot: ")
    else:
        deviceAnswer = str(deviceNumber)

    #commandLine = '/usr/local/bin/ffmpeg -s 320x240 -f video4linux2 -i /dev/video%s -f mpeg1video -b 1k -r 20 http://runmyrobot.com:%s/hello/320/240/' % (deviceAnswer, videoPort)
    #commandLine = '/usr/local/bin/ffmpeg -s 640x480 -f video4linux2 -i /dev/video%s -f mpeg1video -b 150k -r 20 http://%s:%s/hello/640/480/' % (deviceAnswer, server, videoPort)
    commandLine = '/usr/local/bin/ffmpeg -s 640x480 -f video4linux2 -i /dev/video%s -f mpeg1video -b 200k -r 20 http://%s:%s/hello/640/480/' % (deviceAnswer, server, videoPort) # ClawDaddy
    #commandLine = '/usr/local/bin/ffmpeg -s 1280x720 -f video4linux2 -i /dev/video%s -f mpeg1video -b 1k -r 20 http://runmyrobot.com:%s/hello/1280/720/' % (deviceAnswer, videoPort)

    process = runFfmpeg(commandLine)

    return {'process': process, 'device_answer': deviceAnswer}

def handleWindows(deviceNumber, videoPort):

    p = subprocess.Popen(["ffmpeg", "-list_devices", "true", "-f", "dshow", "-i", "dummy"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    out, err = p.communicate()

    lines = err.split('\n')

    count = 0

    devices = []

    for line in lines:

        #if "]  \"" in line:
        #    print "line:", line

        m = re.search('.*\\"(.*)\\"', line)
        if m != None:
            #print line
            if m.group(1)[0:1] != '@':
                print count, m.group(1)
                devices.append(m.group(1))
                count += 1

    if deviceNumber is None:
        deviceAnswer = raw_input("Enter the number of the camera device for your robot from the list above: ")
    else:
        deviceAnswer = str(deviceNumber)

    device = devices[int(deviceAnswer)]
    commandLine = 'ffmpeg -s 320x240 -f dshow -i video="%s" -f mpeg1video -b 200k -r 20 http://%s:%s/hello/320/240/' % (device, server, videoPort)

    process = runFfmpeg(commandLine)

    return {'process': process, 'device_answer': device}

def handleWindowsScreenCapture(deviceNumber, videoPort):

    p = subprocess.Popen(["ffmpeg", "-list_devices", "true", "-f", "dshow", "-i", "dummy"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    out, err = p.communicate()

    lines = err.split('\n')

    count = 0

    devices = []

    for line in lines:

        #if "]  \"" in line:
        #    print "line:", line

        m = re.search('.*\\"(.*)\\"', line)
        if m != None:
            #print line
            if m.group(1)[0:1] != '@':
                print count, m.group(1)
                devices.append(m.group(1))
                count += 1

    if deviceNumber is None:
        deviceAnswer = raw_input("Enter the number of the camera device for your robot from the list above: ")
    else:
        deviceAnswer = str(deviceNumber)

    device = devices[int(deviceAnswer)]
    commandLine = 'ffmpeg -f dshow -i video="screen-capture-recorder" -vf "scale=640:480" -f mpeg1video -b 50k -r 20 http://%s:%s/hello/640/480/' % (server, videoPort)

    print "command line:", commandLine

    process = runFfmpeg(commandLine)

    return {'process': process, 'device_answer': device}

def snapShot(operatingSystem, inputDeviceID, filename="snapshot.jpg"):

    try:
        os.remove('snapshot.jpg')
    except:
        print "did not remove file"

    commandLineDict = {
        'Darwin': 'ffmpeg -y -f qtkit -i %s -vframes 1 %s' % (inputDeviceID, filename),
        'Linux': '/usr/local/bin/ffmpeg -y -f video4linux2 -i /dev/video%s -vframes 1 -q:v 1000 -vf scale=320:240 %s' % (inputDeviceID, filename),
        'Windows': 'ffmpeg -y -s 320x240 -f dshow -i video="%s" -vframes 1 %s' % (inputDeviceID, filename)}

    print commandLineDict[operatingSystem]
    os.system(commandLineDict[operatingSystem])

def startVideoCapture():

    videoPort = getVideoPort()
    print "video port:", videoPort

    if len(sys.argv) >= 3:
        deviceNumber = sys.argv[2]
    else:
        deviceNumber = None

    result = None
    if platform.system() == 'Darwin':
        result = handleDarwin(deviceNumber, videoPort)
    elif platform.system() == 'Linux':
        result = handleLinux(deviceNumber, videoPort)
    elif platform.system() == 'Windows':
        #result = handleWindowsScreenCapture(deviceNumber, videoPort)
        result = handleWindows(deviceNumber, videoPort)
    else:
        print "unknown platform", platform.system()

    return result

def timeInMilliseconds():
    return int(round(time.time() * 1000))

def main():

    print "main"

    streamProcessDict = None

    twitterSnapCount = 0

    while True:

        socketIO.emit('send_video_status', {'send_video_process_exists': True,
                                            'camera_id':cameraIDAnswer})

        if streamProcessDict is not None:
            print "stopping previously running ffmpeg (needs to happen if this is not the first iteration)"
            streamProcessDict['process'].kill()

        print "starting process just to get device result" # this should be a separate function so you don't have to do this
        streamProcessDict = startVideoCapture()
        inputDeviceID = streamProcessDict['device_answer']
        print "stopping video capture"
        streamProcessDict['process'].kill()

        #print "sleeping"
        #time.sleep(3)
        #frameCount = int(round(time.time() * 1000))

        videoWithSnapshots = False
        while videoWithSnapshots:

            frameCount = timeInMilliseconds()

            print "taking single frame image"
            snapShot(platform.system(), inputDeviceID, filename="single_frame_image.jpg")

            with open ("single_frame_image.jpg", 'rb') as f:

                # every so many frames, post a snapshot to twitter
                #if frameCount % 450 == 0:
                if frameCount % 6000 == 0:
                        data = f.read()
                        print "emit"
                        socketIO.emit('snapshot', {'frame_count':frameCount, 'image':base64.b64encode(data)})
                data = f.read()

            print "emit"
            socketIO.emit('single_frame_image', {'frame_count':frameCount, 'image':base64.b64encode(data)})
            time.sleep(0)

            #frameCount += 1

        if False:
         if platform.system() != 'Windows':
            print "taking snapshot"
            snapShot(platform.system(), inputDeviceID)
            with open ("snapshot.jpg", 'rb') as f:
                data = f.read()
            print "emit"

            # skip sending the first image because it's mostly black, maybe completely black
            #todo: should find out why this black image happens
            if twitterSnapCount > 0:
                socketIO.emit('snapshot', {'image':base64.b64encode(data)})

        print "starting video capture"
        streamProcessDict = startVideoCapture()

        # This loop counts out a delay that occurs between twitter snapshots.
        # Every 50 seconds, it kills and restarts ffmpeg.
        # Every 40 seconds, it sends a signal to the server indicating status of processes.
        period = 2*60*60 # period in seconds between snaps
        for count in range(period):
            time.sleep(1)

            if count % 20 == 0:
                socketIO.emit('send_video_status', {'send_video_process_exists': True,
                                                    'camera_id':cameraIDAnswer})

            if count % 161 == 30:
                print "stopping video capture just in case it has reached a state where it's looping forever, not sending video, and not dying as a process, which can happen"
                streamProcessDict['process'].kill()

            if count % 80 == 75:
                print "send status about this process and its child process ffmpeg"
                ffmpegProcessExists = streamProcessDict['process'].poll() is None
                socketIO.emit('send_video_status', {'send_video_process_exists': True,
                                                    'ffmpeg_process_exists': ffmpegProcessExists,
                                                    'camera_id':cameraIDAnswer})

            #if count % 190 == 180:
            #    print "reboot system in case the webcam is not working"
            #    os.system("sudo reboot")

            # if the video stream process dies, restart it
            if streamProcessDict['process'].poll() is not None:
                # wait before trying to start ffmpeg
                print "ffmpeg process is dead, waiting before trying to restart"
                randomSleep()
                streamProcessDict = startVideoCapture()

        #print "taking snapshot"
        #snapShot(platform.system(), inputDeviceID)
        #with open ("snapshot.jpg", 'rb') as f:
        #    data = f.read()
        #print "emit"
        #socketIO.emit('snapshot', {'image':base64.b64encode(data)})

        twitterSnapCount += 1

print "test2"

if __name__ == "__main__":

    if len(sys.argv) > 1:
        cameraIDAnswer = sys.argv[1]
    else:
        cameraIDAnswer = raw_input("Enter the Camera ID for your robot, you can get it by pointing a browser to the runmyrobot server %s: " % server)

    main()

<|endoftext|>


<commit_before>#!/usr/bin/python
import os
import subprocess
import os.path
import sys
from multiprocessing import Process, JoinableQueue as Queue, Pool
from optparse import OptionParser

# import hanging_threads

MAX_PROCESSES = 16
unsearched = Queue()

def fscat(options, queue, results_q, name, is_multithread=True):
    problematic_ranges = []
    problem = None
    while not queue.empty():
        try:
            file = queue.get()
            print name + ": getting %s from queue" % file
            if not os.path.isfile(file):
                continue
            rangeslength = [0, 0, 0]  # Null, oca, disk
            output = "file " + file
            problem = False
            over64KBwithoutOCAorNULL = False
            command = "fscat"
            p = subprocess.Popen([command, "-M", file], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            c = p.communicate()
            if p.returncode != 0:
                if options.verbose:
                    print c[1]
                    print c[0]
                print name + ": fscat returned " + str(p.returncode) + " for file " + file + ", skipping."
                continue
            if options.fscatoutput:
                print c[1]
                print c[0]
            if len(c[0]) == 0:
                output += " is inline"
            startofdiskrange = 0
            for line in c[0].splitlines():
                line = line.split(" -> ")
                line = [line[0].split("{Mapping: [")[1].split("]")[0].split("-"), line[1]]
                if not line[0][1] == "8000000000000000":  # last range points outside the file and is always Null
                    #  - just ignore it
                    line[0] = [int(x, 16) for x in line[0]]
                    rangelen = line[0][1] - line[0][0]
                    if "-sec" in line[1]:
                        print name + ": " + output + " contains redundant data!"
                        continue
                    if "oca" in line[1]:
                        rangeslength[1] = rangeslength[1] + rangelen
                        startofdiskrange = line[0][1]  # reset consecutive disk range count
                    elif "[0-0]" in line[1]:
                        rangeslength[0] = rangeslength[0] + rangelen
                        startofdiskrange = line[0][1]
                    elif "disk" in line[1]:
                        rangeslength[2] = rangeslength[2] + rangelen
                        if (line[0][
                                1] - startofdiskrange) >= 65536:  # consecutive disk range larger than 64KB - shouldn't happen
                            # after dedup for files of at least 64KB
                            problem = True
                            problematic_ranges.append("[ %x - %x ]" % (int(line[0][0]), int(line[0][1])))
                    elif "shared null" in line[1] or "owned null" in line[1]:  # Skipping "shared null" lines
                        startofdiskrange = line[0][1]
                        if options.verbose:
                            print name + ": " + output + " null mapping"
                    else:
                        print name + ": " + output + " parsing error!"
                        print "Parsing error DEBUG: %s \n" % line[1]
                        break
            output = "Null ranges: " + str(rangeslength[0]) + " \tOca ranges: " + str(
                rangeslength[1]) + " \tDisk ranges: " + str(rangeslength[2]) + " " + output
            if problem or options.filesize or (
                                options.outputnondeduped and rangeslength[0] == 0 and rangeslength[1] == 0 and
                            rangeslength[
                                2] >= 65536):
                filesize = os.path.getsize(file)
                output = output + " Size: " + str(filesize)
                if filesize < 65536:  # file too small to be deduped (e.g. a 63KB file is mapped to a 64KB disk range)
                    problem = False
                else:
                    if rangeslength[0] == 0 and rangeslength[1] == 0 and rangeslength[2] >= 65536:
                        over64KBwithoutOCAorNULL = True
                        output += " is over 64KB without OCA or NULL ranges!!!"
                if problem:
                    if options.verbose:
                        for mrange in problematic_ranges:  # Will print all ranges that are expected to be deduped
                            output += " contains a disk range %s that should have been deduped!\n" % mrange
                    else:
                        output += " contains a disk range/s that should have been deduped!"
            if options.verbose or problem or over64KBwithoutOCAorNULL:
                print name + ": " + output

            if is_multithread is False:  # in case function is called as single execution, we won't loop
                break

        except Exception as e:
            raise e

    results_q.put(problem)
    print name + ": finished"

def explore_path(path):
    directories = []
    print "Exploring path " + path
    for filename in os.listdir(path):
        fullname = os.path.join(path, filename)
        if os.path.isdir(fullname):
            directories.append(fullname)
    return directories

def dir_scan_worker():
    while True:
        path = unsearched.get()
        dirs = explore_path(path)
        for newdir in dirs:
            unsearched.put(newdir)
        unsearched.task_done()

def fscat_stub(options, queue, results_q, name, is_multithread=True):
    print name + ": running fscat_stub"

def run_recursive_scan(options, queue, results_q):
    process_pool = []
    first_level_dirs = next(os.walk(options.path))[1]

    folders_scan_pool = Pool(MAX_PROCESSES)
    for path in first_level_dirs:
        unsearched.put(path)

    for i in range(MAX_PROCESSES):
        folders_scan_pool.apply_async(dir_scan_worker)

    # for dirpath, dirnames, filenames in os.walk(options.path):
    #     for name in filenames:
    #         print "Putting in queue: " + dirpath + "/" + name
    #         queue.put(os.path.join(dirpath, name))

    for i in range(0, MAX_PROCESSES):
        p = Process(target=fscat_stub, name=("process-%d" % i),
                    args=(options, queue, results_q, ("process-%d" % i)))
        process_pool.append(p)

    for p in process_pool:
        print "process %s started" % p.name
        p.start()

    for p in process_pool:
        p.join()

    while not results_q.empty():
        q = results_q.get()
        if q is True:  # if 'True', there is a problem
            return q

def run_single_folder_scan(options, queue, results_q):
    process_pool = []

    for name in os.listdir(options.path):
        queue.put(os.path.join(options.path, name))

    for i in range(0, MAX_PROCESSES):
        p = Process(target=fscat, name=("process-%d" % i), args=(options, queue, results_q, ("process-%d" % i)))
        process_pool.append(p)

    for p in process_pool:
        print "thread %s started" % p.name
        p.start()

    for p in process_pool:
        p.join()

    while not results_q.empty():
        q = results_q.get()
        if q is True:
            return q

def run_single_file_scan(options, queue, results_q):
    queue.put(options.path)

    p = Process(target=fscat, name="process-1", args=(options, queue, results_q, "process-1", False))

    print "thread %s started" % p.name
    p.start()
    p.join()

    q = results_q.get()
    if q is True:
        return q

def main():
    """ Main function
    Returns:
    0 - all relevant files are deduped
    1 - dedup error
    2 - bad argument passed
    """

    queue = Queue()
    results_q = Queue()

    parser = OptionParser()
    parser.add_option("-v", "--verbose", action="store_true", dest="verbose")
    parser.add_option("-r", "--recursive", action="store_true", dest="recursive")
    parser.add_option("-f", "--fscatoutput", action="store_true", dest="fscatoutput", help="Print fscat -M's output")
    parser.add_option("-s", "--filesize", action="store_true", dest="filesize", help="Always print file size")
    parser.add_option("-p", "--path", help="file/dir path", action="store", dest="path", default=".")
    parser.add_option("-o", "--outputnondeduped", action="store_true", dest="outputnondeduped",
                      help="Indicates files that are over 64KB, without OCA or NULL ranges")
    (options, args) = parser.parse_args()

    if options.path[0] != "/":
        options.path = os.path.abspath(options.path)
        print "Using path: " + options.path
    # if options.path[:9] != "/mnt/mgmt":
    #     print "Path should start with /mnt/mgmt"
    #     sys.exit(2)

    if os.path.isfile(options.path):
        print "scanning single file..."
        if run_single_file_scan(options, queue, results_q) is True:
            sys.exit(1)
    elif os.path.isdir(options.path):
        if options.recursive:
            print "Scanning directory tree..."
            if run_recursive_scan(options, queue, results_q) is True:
                sys.exit(1)
        else:
            print "Scanning folder for files..."
            if run_single_folder_scan(options, queue, results_q) is True:
                sys.exit(1)

if __name__ == '__main__':
    try:
        main()
    except Exception as e:
        print e
<commit_msg>multiprocessing folder scan<commit_after>#!/usr/bin/python
import os
import subprocess
import os.path
import sys
from multiprocessing import Process, JoinableQueue as Queue, Pool
from optparse import OptionParser

# import hanging_threads

MAX_PROCESSES = 16
unsearched = Queue()

def fscat(options, queue, results_q, name, is_multithread=True):
    problematic_ranges = []
    problem = None
    while not queue.empty():
        try:
            file = queue.get()
            print name + ": getting %s from queue" % file
            if not os.path.isfile(file):
                continue
            rangeslength = [0, 0, 0]  # Null, oca, disk
            output = "file " + file
            problem = False
            over64KBwithoutOCAorNULL = False
            command = "fscat"
            p = subprocess.Popen([command, "-M", file], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            c = p.communicate()
            if p.returncode != 0:
                if options.verbose:
                    print c[1]
                    print c[0]
                print name + ": fscat returned " + str(p.returncode) + " for file " + file + ", skipping."
                continue
            if options.fscatoutput:
                print c[1]
                print c[0]
            if len(c[0]) == 0:
                output += " is inline"
            startofdiskrange = 0
            for line in c[0].splitlines():
                line = line.split(" -> ")
                line = [line[0].split("{Mapping: [")[1].split("]")[0].split("-"), line[1]]
                if not line[0][1] == "8000000000000000":  # last range points outside the file and is always Null
                    #  - just ignore it
                    line[0] = [int(x, 16) for x in line[0]]
                    rangelen = line[0][1] - line[0][0]
                    if "-sec" in line[1]:
                        print name + ": " + output + " contains redundant data!"
                        continue
                    if "oca" in line[1]:
                        rangeslength[1] = rangeslength[1] + rangelen
                        startofdiskrange = line[0][1]  # reset consecutive disk range count
                    elif "[0-0]" in line[1]:
                        rangeslength[0] = rangeslength[0] + rangelen
                        startofdiskrange = line[0][1]
                    elif "disk" in line[1]:
                        rangeslength[2] = rangeslength[2] + rangelen
                        if (line[0][
                                1] - startofdiskrange) >= 65536:  # consecutive disk range larger than 64KB - shouldn't happen
                            # after dedup for files of at least 64KB
                            problem = True
                            problematic_ranges.append("[ %x - %x ]" % (int(line[0][0]), int(line[0][1])))
                    elif "shared null" in line[1] or "owned null" in line[1]:  # Skipping "shared null" lines
                        startofdiskrange = line[0][1]
                        if options.verbose:
                            print name + ": " + output + " null mapping"
                    else:
                        print name + ": " + output + " parsing error!"
                        print "Parsing error DEBUG: %s \n" % line[1]
                        break
            output = "Null ranges: " + str(rangeslength[0]) + " \tOca ranges: " + str(
                rangeslength[1]) + " \tDisk ranges: " + str(rangeslength[2]) + " " + output
            if problem or options.filesize or (
                                options.outputnondeduped and rangeslength[0] == 0 and rangeslength[1] == 0 and
                            rangeslength[
                                2] >= 65536):
                filesize = os.path.getsize(file)
                output = output + " Size: " + str(filesize)
                if filesize < 65536:  # file too small to be deduped (e.g. a 63KB file is mapped to a 64KB disk range)
                    problem = False
                else:
                    if rangeslength[0] == 0 and rangeslength[1] == 0 and rangeslength[2] >= 65536:
                        over64KBwithoutOCAorNULL = True
                        output += " is over 64KB without OCA or NULL ranges!!!"
                if problem:
                    if options.verbose:
                        for mrange in problematic_ranges:  # Will print all ranges that are expected to be deduped
                            output += " contains a disk range %s that should have been deduped!\n" % mrange
                    else:
                        output += " contains a disk range/s that should have been deduped!"
            if options.verbose or problem or over64KBwithoutOCAorNULL:
                print name + ": " + output

            if is_multithread is False:  # in case function is called as single execution, we won't loop
                break

        except Exception as e:
            raise e

    results_q.put(problem)
    print name + ": finished"

def explore_path(id, path):
    directories = []
    print "process-" + id + " -- Exploring path " + path
    for filename in os.listdir(path):
        fullname = os.path.join(path, filename)
        if os.path.isdir(fullname):
            directories.append(fullname)
    return directories

def dir_scan_worker(id):
    while True:
        path = unsearched.get()
        dirs = explore_path(id, path)
        for newdir in dirs:
            unsearched.put(newdir)
        unsearched.task_done()

def fscat_stub(options, queue, results_q, name, is_multithread=True):
    print name + ": running fscat_stub on path: " + queue.get()

def run_recursive_scan(options, queue, results_q):
    process_pool = []
    first_level_dirs = next(os.walk(options.path))[1]

    folders_scan_pool = Pool(MAX_PROCESSES)
    for path in first_level_dirs:
        unsearched.put(path)

    for i in range(MAX_PROCESSES):
        folders_scan_pool.apply_async(dir_scan_worker, args=i)

    # for dirpath, dirnames, filenames in os.walk(options.path):
    #     for name in filenames:
    #         print "Putting in queue: " + dirpath + "/" + name
    #         queue.put(os.path.join(dirpath, name))

    for i in range(0, MAX_PROCESSES):
        p = Process(target=fscat_stub, name=("process-%d" % i),
                    args=(options, queue, results_q, ("process-%d" % i)))
        process_pool.append(p)

    for p in process_pool:
        print "process %s started" % p.name
        p.start()

    for p in process_pool:
        p.join()

    while not results_q.empty():
        q = results_q.get()
        if q is True:  # if 'True', there is a problem
            return q

def run_single_folder_scan(options, queue, results_q):
    process_pool = []

    for name in os.listdir(options.path):
        queue.put(os.path.join(options.path, name))

    for i in range(0, MAX_PROCESSES):
        p = Process(target=fscat, name=("process-%d" % i), args=(options, queue, results_q, ("process-%d" % i)))
        process_pool.append(p)

    for p in process_pool:
        print "thread %s started" % p.name
        p.start()

    for p in process_pool:
        p.join()

    while not results_q.empty():
        q = results_q.get()
        if q is True:
            return q

def run_single_file_scan(options, queue, results_q):
    queue.put(options.path)

    p = Process(target=fscat, name="process-1", args=(options, queue, results_q, "process-1", False))

    print "thread %s started" % p.name
    p.start()
    p.join()

    q = results_q.get()
    if q is True:
        return q

def main():
    """ Main function
    Returns:
    0 - all relevant files are deduped
    1 - dedup error
    2 - bad argument passed
    """

    queue = Queue()
    results_q = Queue()

    parser = OptionParser()
    parser.add_option("-v", "--verbose", action="store_true", dest="verbose")
    parser.add_option("-r", "--recursive", action="store_true", dest="recursive")
    parser.add_option("-f", "--fscatoutput", action="store_true", dest="fscatoutput", help="Print fscat -M's output")
    parser.add_option("-s", "--filesize", action="store_true", dest="filesize", help="Always print file size")
    parser.add_option("-p", "--path", help="file/dir path", action="store", dest="path", default=".")
    parser.add_option("-o", "--outputnondeduped", action="store_true", dest="outputnondeduped",
                      help="Indicates files that are over 64KB, without OCA or NULL ranges")
    (options, args) = parser.parse_args()

    if options.path[0] != "/":
        options.path = os.path.abspath(options.path)
        print "Using path: " + options.path
    # if options.path[:9] != "/mnt/mgmt":
    #     print "Path should start with /mnt/mgmt"
    #     sys.exit(2)

    if os.path.isfile(options.path):
        print "scanning single file..."
        if run_single_file_scan(options, queue, results_q) is True:
            sys.exit(1)
    elif os.path.isdir(options.path):
        if options.recursive:
            print "Scanning directory tree..."
            if run_recursive_scan(options, queue, results_q) is True:
                sys.exit(1)
        else:
            print "Scanning folder for files..."
            if run_single_folder_scan(options, queue, results_q) is True:
                sys.exit(1)

if __name__ == '__main__':
    try:
        main()
    except Exception as e:
        print e
<|endoftext|>


<commit_before>"""
Adapted (stolen) from http://www.astrobetter.com/visualization-fun-with-python-2d-histogram-with-1d-histograms-on-axes/
Thanks Jess K!
"""

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.ticker import NullFormatter, MaxNLocator
from numpy import linspace
plt.ion()

# Define a function to make the ellipses
def ellipse(ra,rb,ang,x0,y0,Nb=100):
    xpos, ypos = x0, y0
    radm, radn = ra, rb
    an = ang
    co, si = np.cos(an), np.sin(an)
    the = linspace(0,2*np.pi,Nb)
    X = radm * np.cos(the) * co - si * radn * np.sin(the) + xpos
    Y = radm * np.cos(the) * si + co * radn * np.sin(the) + ypos
    return X, Y

def plotter(x, y, bin_size, grid_size, fiber_size):
    # Set up default x and y limits
    xlims = [np.min(x),np.max(x)]
    ylims = [np.min(y),np.max(y)]

    # Set up your x and y labels
    xlabel = '$\mathrm{pixels}$'
    ylabel = '$\mathrm{pixels}$'

    # Define the locations for the axes
    left, width = 0.12, 0.55
    bottom, height = 0.12, 0.55
    bottom_h = left_h = left+width+0.02

    # Set up the geometry of the three plots
    rect_temperature = [left, bottom, width, height] # dimensions of temp plot
    rect_histx = [left, bottom_h, width, 0.25] # dimensions of x-histogram
    rect_histy = [left_h, bottom, 0.25, height] # dimensions of y-histogram

    # Set up the size of the figure
    fig = plt.figure(1, figsize=(9.5,9))

    # Make the three plots
    axTemperature = plt.axes(rect_temperature) # temperature plot
    axHistx = plt.axes(rect_histx) # x histogram
    axHisty = plt.axes(rect_histy) # y histogram

    # Remove the inner axes numbers of the histograms
    nullfmt = NullFormatter()
    axHistx.xaxis.set_major_formatter(nullfmt)
    axHisty.yaxis.set_major_formatter(nullfmt)

    # Find the min/max of the data
    xmin = np.min(xlims)
    xmax = np.max(xlims)
    ymin = np.min(ylims)
    ymax = np.max(y)

    # Make the 'main' temperature plot
    # Define the number of bins
    nxbins = 50
    nybins = 50
    nbins = 100

    xbins = linspace(start = xmin, stop = xmax, num = nxbins)
    ybins = linspace(start = ymin, stop = ymax, num = nybins)
    xcenter = (xbins[0:-1]+xbins[1:])/2.0
    ycenter = (ybins[0:-1]+ybins[1:])/2.0
    aspectratio = 1.0*(xmax - 0)/(1.0*ymax - 0)

    H, xedges,yedges = np.histogram2d(x,y,bins=(xbins,ybins))
    X = xcenter
    Y = ycenter
    Z = H

    # Plot the temperature data
    cax = (axTemperature.imshow(H, extent=[xmin,xmax,ymin,ymax],
           interpolation='nearest', origin='lower',aspect=aspectratio))

    # Plot the temperature plot contours
    contourcolor = 'white'
    xcenter = np.mean(x)
    ycenter = np.mean(y)
    ra = np.std(x)
    rb = np.std(y)
    ang = 0

    X,Y=ellipse(ra,rb,ang,xcenter,ycenter)
    axTemperature.plot(X,Y,"k:",ms=1,linewidth=2.0)
    axTemperature.annotate('$1\\sigma$', xy=(X[15], Y[15]), xycoords='data',xytext=(10, 10),
                           textcoords='offset points', horizontalalignment='right',
                           verticalalignment='bottom',fontsize=25)

    X,Y=ellipse(2*ra,2*rb,ang,xcenter,ycenter)
    axTemperature.plot(X,Y,"k:",color = contourcolor,ms=1,linewidth=2.0)
    axTemperature.annotate('$2\\sigma$', xy=(X[15], Y[15]), xycoords='data',xytext=(10, 10),
                           textcoords='offset points',horizontalalignment='right',
                           verticalalignment='bottom',fontsize=25, color = contourcolor)

    X,Y=ellipse(3*ra,3*rb,ang,xcenter,ycenter)
    axTemperature.plot(X,Y,"k:",color = contourcolor, ms=1,linewidth=2.0)
    axTemperature.annotate('$3\\sigma$', xy=(X[15], Y[15]), xycoords='data',xytext=(10, 10),
                           textcoords='offset points',horizontalalignment='right',
                           verticalalignment='bottom',fontsize=25, color = contourcolor)

    #Plot the axes labels
    axTemperature.set_xlabel(xlabel,fontsize=25)
    axTemperature.set_ylabel(ylabel,fontsize=25)

    #Make the tickmarks pretty
    ticklabels = axTemperature.get_xticklabels()
    for label in ticklabels:
        label.set_fontsize(18)
        label.set_family('serif')

    ticklabels = axTemperature.get_yticklabels()
    for label in ticklabels:
        label.set_fontsize(18)
        label.set_family('serif')

    #Set up the plot limits
    axTemperature.set_xlim(xlims)
    axTemperature.set_ylim(ylims)

    #Set up the histogram bins
    xbins = np.arange(xmin, xmax, (xmax-xmin)/nbins)
    ybins = np.arange(ymin, ymax, (ymax-ymin)/nbins)

    #Plot the histograms
    axHistx.hist(x, bins=xbins, color = 'blue')
    axHisty.hist(y, bins=ybins, orientation='horizontal', color = 'red')

    #Set up the histogram limits
    axHistx.set_xlim( np.min(x), np.max(x) )
    axHisty.set_ylim( np.min(y), np.max(y) )

    #Make the tickmarks pretty
    ticklabels = axHistx.get_yticklabels()
    for label in ticklabels:
        label.set_fontsize(12)
        label.set_family('serif')

    #Make the tickmarks pretty
    ticklabels = axHisty.get_xticklabels()
    for label in ticklabels:
        label.set_fontsize(12)
        label.set_family('serif')

    #Cool trick that changes the number of tickmarks for the histogram axes
    axHisty.xaxis.set_major_locator(MaxNLocator(4))
    axHistx.yaxis.set_major_locator(MaxNLocator(4))

    #Show the plot
    plt.draw()<commit_msg>Changing limits<commit_after>"""
Adapted (stolen) from http://www.astrobetter.com/visualization-fun-with-python-2d-histogram-with-1d-histograms-on-axes/
Thanks Jess K!
"""

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.ticker import NullFormatter, MaxNLocator
from numpy import linspace
plt.ion()

# Define a function to make the ellipses
def ellipse(ra,rb,ang,x0,y0,Nb=100):
    xpos, ypos = x0, y0
    radm, radn = ra, rb
    an = ang
    co, si = np.cos(an), np.sin(an)
    the = linspace(0,2*np.pi,Nb)
    X = radm * np.cos(the) * co - si * radn * np.sin(the) + xpos
    Y = radm * np.cos(the) * si + co * radn * np.sin(the) + ypos
    return X, Y

def plotter(x, y, n_pix, pixel_size, grid_size, fiber_size):
    # Set up default x and y limits
    xlims = [-(np_pix/2.0), (n_pix/2.0)] * pixel_size
    ylims = [-(np_pix/2.0), (n_pix/2.0)] * pixel_size

    # Set up your x and y labels
    xlabel = '$\mathrm{pixels}$'
    ylabel = '$\mathrm{pixels}$'

    # Define the locations for the axes
    left, width = 0.12, 0.55
    bottom, height = 0.12, 0.55
    bottom_h = left_h = left+width+0.02

    # Set up the geometry of the three plots
    rect_temperature = [left, bottom, width, height] # dimensions of temp plot
    rect_histx = [left, bottom_h, width, 0.25] # dimensions of x-histogram
    rect_histy = [left_h, bottom, 0.25, height] # dimensions of y-histogram

    # Set up the size of the figure
    fig = plt.figure(1, figsize=(9.5,9))

    # Make the three plots
    axTemperature = plt.axes(rect_temperature) # temperature plot
    axHistx = plt.axes(rect_histx) # x histogram
    axHisty = plt.axes(rect_histy) # y histogram

    # Remove the inner axes numbers of the histograms
    nullfmt = NullFormatter()
    axHistx.xaxis.set_major_formatter(nullfmt)
    axHisty.yaxis.set_major_formatter(nullfmt)

    # Find the min/max of the data
    xmin = np.min(xlims)
    xmax = np.max(xlims)
    ymin = np.min(ylims)
    ymax = np.max(y)

    # Make the 'main' temperature plot
    # Define the number of bins
    nxbins = 50
    nybins = 50
    nbins = 100

    xbins = linspace(start = xmin, stop = xmax, num = nxbins)
    ybins = linspace(start = ymin, stop = ymax, num = nybins)
    xcenter = (xbins[0:-1]+xbins[1:])/2.0
    ycenter = (ybins[0:-1]+ybins[1:])/2.0
    aspectratio = 1.0*(xmax - 0)/(1.0*ymax - 0)

    H, xedges,yedges = np.histogram2d(x, y, pixel_size))
    X = xcenter
    Y = ycenter
    Z = H

    # Plot the temperature data
    cax = (axTemperature.imshow(H, extent=[xmin,xmax,ymin,ymax],
           interpolation='nearest', origin='lower',aspect=aspectratio))

    # Plot the temperature plot contours
    contourcolor = 'white'
    xcenter = np.mean(x)
    ycenter = np.mean(y)
    ra = np.std(x)
    rb = np.std(y)
    ang = 0

    X,Y=ellipse(ra,rb,ang,xcenter,ycenter)
    axTemperature.plot(X,Y,"k:",ms=1,linewidth=2.0)
    axTemperature.annotate('$1\\sigma$', xy=(X[15], Y[15]), xycoords='data',xytext=(10, 10),
                           textcoords='offset points', horizontalalignment='right',
                           verticalalignment='bottom',fontsize=25)

    X,Y=ellipse(2*ra,2*rb,ang,xcenter,ycenter)
    axTemperature.plot(X,Y,"k:",color = contourcolor,ms=1,linewidth=2.0)
    axTemperature.annotate('$2\\sigma$', xy=(X[15], Y[15]), xycoords='data',xytext=(10, 10),
                           textcoords='offset points',horizontalalignment='right',
                           verticalalignment='bottom',fontsize=25, color = contourcolor)

    X,Y=ellipse(3*ra,3*rb,ang,xcenter,ycenter)
    axTemperature.plot(X,Y,"k:",color = contourcolor, ms=1,linewidth=2.0)
    axTemperature.annotate('$3\\sigma$', xy=(X[15], Y[15]), xycoords='data',xytext=(10, 10),
                           textcoords='offset points',horizontalalignment='right',
                           verticalalignment='bottom',fontsize=25, color = contourcolor)

    #Plot the axes labels
    axTemperature.set_xlabel(xlabel,fontsize=25)
    axTemperature.set_ylabel(ylabel,fontsize=25)

    #Make the tickmarks pretty
    ticklabels = axTemperature.get_xticklabels()
    for label in ticklabels:
        label.set_fontsize(18)
        label.set_family('serif')

    ticklabels = axTemperature.get_yticklabels()
    for label in ticklabels:
        label.set_fontsize(18)
        label.set_family('serif')

    #Set up the plot limits
    axTemperature.set_xlim(xlims)
    axTemperature.set_ylim(ylims)

    #Set up the histogram bins
    xbins = np.arange(xmin, xmax, (xmax-xmin)/nbins)
    ybins = np.arange(ymin, ymax, (ymax-ymin)/nbins)

    #Plot the histograms
    axHistx.hist(x, bins=xbins, color = 'blue')
    axHisty.hist(y, bins=ybins, orientation='horizontal', color = 'red')

    #Set up the histogram limits
    axHistx.set_xlim( np.min(x), np.max(x) )
    axHisty.set_ylim( np.min(y), np.max(y) )

    #Make the tickmarks pretty
    ticklabels = axHistx.get_yticklabels()
    for label in ticklabels:
        label.set_fontsize(12)
        label.set_family('serif')

    #Make the tickmarks pretty
    ticklabels = axHisty.get_xticklabels()
    for label in ticklabels:
        label.set_fontsize(12)
        label.set_family('serif')

    #Cool trick that changes the number of tickmarks for the histogram axes
    axHisty.xaxis.set_major_locator(MaxNLocator(4))
    axHistx.yaxis.set_major_locator(MaxNLocator(4))

    #Show the plot
    plt.draw()<|endoftext|>


<commit_before>from flask import flash, request, Markup, url_for, render_template, redirect, \
    session
from flask.ext.login import current_user

from viaduct import application, login_manager
from markdown import markdown
from resource import Resource
from viaduct.forms import SignInForm
from viaduct.api.user import UserAPI
from viaduct.api.group import GroupPermissionAPI

from viaduct.models.activity import Activity

import datetime

markdown_extensions = [
    'toc'
]

@login_manager.unauthorized_handler
def unauthorized():
    # Save the path the user was rejected from.
    session['denied_from'] = request.path

    flash('You must be logged in to view this page.', 'error')
    return redirect(url_for('user.sign_in'))

@application.errorhandler(403)
def permission_denied(e):
    """ When permission denied and not logged in you will be redirected. """
    content = "403, The police has been notified!"
    image = '/static/img/403.jpg'

    # Save the path you were rejected from.
    session['denied_from'] = request.path

    if not current_user or current_user.is_anonymous():
        flash('Je hebt geen rechten om deze pagina te bekijken.')
        return redirect(url_for('user.sign_in'))

    return render_template('page/403.htm', content=content, image=image)

@application.errorhandler(500)
def internal_server_error(e):
    return render_template('page/500.htm')

@application.errorhandler(404)
def page_not_found(e):
    return render_template('page/404.htm')

def flash_form_errors(form):
    for field, errors in form.errors.items():
        for error in errors:
            flash(error, 'error')

def get_login_form():
    form = SignInForm()
    return form

@application.template_filter('markdown')
def markdown_filter(data, filter_html=True):
    if filter_html:
        safe_mode = False
    else:
        safe_mode = 'escape'

    return Markup(markdown(data, safe_mode=safe_mode, enable_attributes=False,
                        extensions=markdown_extensions))

@application.template_filter('markup')
def markup_filter(data):
    return Markup(data)

@application.template_filter('safe_markdown')
def safe_markdown_filter(data):
    return Markup(markdown(data, extensions=markdown_extensions))

@application.template_filter('pages')
def pages_filter(data):

    content = '
'

    for i in range(len(data)):
        if i % 2 == 0:
            content += '
'

        if i == len(data) - 1 and i % 2 == 0:
            content += '
'
        else:
            content += '
'

        content += '
' if data[i].is_main_page else '">'

        page = data[i].page if data[i].page.id else None
        if (page and current_user and (UserAPI.can_write(page) or
                                    GroupPermissionAPI.can_write('page'))) \
                or (not page and GroupPermissionAPI.can_write('page')):
            content += '
'
            content += ' View History'
            content += ' Edit Page'
            content += '
'

        # if we render stuff for the main page we want to make sure
        # the individual pages are rendered correctly, this is super
        # hard coded but, well, what can you do?
        if data[i].is_main_page:
            if data[i].path == 'twitter':
                if data[i].filter_html:
                    safe_mode = False
                else:
                    safe_mode = 'escape'

                content += '
{0}
'.format(data[i].title)
                content += markdown(data[i].content, enable_attributes=False,
                                    safe_mode=safe_mode,
                                    extensions=markdown_extensions)
            elif data[i].path == 'activities':
                activities = Activity.query \
                    .filter(Activity.end_time > datetime.datetime.now()) \
                    .order_by(Activity.start_time.asc())
                content += render_template('activity/view_simple.htm',
                                        activities=activities
                                        .paginate(1, 12, False))
            elif data[i].path == 'contact' \
                    or data[i].path == 'laatste_bestuursblog':
                if data[i].filter_html:
                    safe_mode = False
                else:
                    safe_mode = 'escape'

                content += '
{0}
'.format(data[i].title)
                content += markdown(data[i].content, safe_mode=safe_mode,
                                    extensions=markdown_extensions)
        else:
            if data[i].filter_html:
                safe_mode = False
            else:
                safe_mode = 'escape'

            #print data[i].path
            content += '
{0}
'.format(data[i].title)
            content += '
Hello
'
            content += markdown(data[i].content, safe_mode=safe_mode,
                                extensions=markdown_extensions)

        content += '

'

        if i == len(data) - 1 or i % 2 != 0:
            content += '
'

    content += '
'

    return Markup(content)

@application.template_filter('pimpy_minute_line_numbers')
def pimpy_minute_line_numbers(data):
    #assert False
    s = ''
    for i, line in enumerate(data.split('\n')):
        #s += '%d%s\n' % (i, line[:-1])
        s += '%s\n' % (i, line[:-1])
    return s

<commit_msg>Error -> danger<commit_after>from flask import flash, request, Markup, url_for, render_template, redirect, \
    session
from flask.ext.login import current_user

from viaduct import application, login_manager
from markdown import markdown
from resource import Resource
from viaduct.forms import SignInForm
from viaduct.api.user import UserAPI
from viaduct.api.group import GroupPermissionAPI

from viaduct.models.activity import Activity

import datetime

markdown_extensions = [
    'toc'
]

@login_manager.unauthorized_handler
def unauthorized():
    # Save the path the user was rejected from.
    session['denied_from'] = request.path

    flash('You must be logged in to view this page.', 'error')
    return redirect(url_for('user.sign_in'))

@application.errorhandler(403)
def permission_denied(e):
    """ When permission denied and not logged in you will be redirected. """
    content = "403, The police has been notified!"
    image = '/static/img/403.jpg'

    # Save the path you were rejected from.
    session['denied_from'] = request.path

    if not current_user or current_user.is_anonymous():
        flash('Je hebt geen rechten om deze pagina te bekijken.')
        return redirect(url_for('user.sign_in'))

    return render_template('page/403.htm', content=content, image=image)

@application.errorhandler(500)
def internal_server_error(e):
    return render_template('page/500.htm')

@application.errorhandler(404)
def page_not_found(e):
    return render_template('page/404.htm')

def flash_form_errors(form):
    for field, errors in form.errors.items():
        for error in errors:
            flash(error, 'danger')

def get_login_form():
    form = SignInForm()
    return form

@application.template_filter('markdown')
def markdown_filter(data, filter_html=True):
    if filter_html:
        safe_mode = False
    else:
        safe_mode = 'escape'

    return Markup(markdown(data, safe_mode=safe_mode, enable_attributes=False,
                        extensions=markdown_extensions))

@application.template_filter('markup')
def markup_filter(data):
    return Markup(data)

@application.template_filter('safe_markdown')
def safe_markdown_filter(data):
    return Markup(markdown(data, extensions=markdown_extensions))

@application.template_filter('pages')
def pages_filter(data):

    content = '
'

    for i in range(len(data)):
        if i % 2 == 0:
            content += '
'

        if i == len(data) - 1 and i % 2 == 0:
            content += '
'
        else:
            content += '
'

        content += '
' if data[i].is_main_page else '">'

        page = data[i].page if data[i].page.id else None
        if (page and current_user and (UserAPI.can_write(page) or
                                    GroupPermissionAPI.can_write('page'))) \
                or (not page and GroupPermissionAPI.can_write('page')):
            content += '
'
            content += ' View History'
            content += ' Edit Page'
            content += '
'

        # if we render stuff for the main page we want to make sure
        # the individual pages are rendered correctly, this is super
        # hard coded but, well, what can you do?
        if data[i].is_main_page:
            if data[i].path == 'twitter':
                if data[i].filter_html:
                    safe_mode = False
                else:
                    safe_mode = 'escape'

                content += '
{0}
'.format(data[i].title)
                content += markdown(data[i].content, enable_attributes=False,
                                    safe_mode=safe_mode,
                                    extensions=markdown_extensions)
            elif data[i].path == 'activities':
                activities = Activity.query \
                    .filter(Activity.end_time > datetime.datetime.now()) \
                    .order_by(Activity.start_time.asc())
                content += render_template('activity/view_simple.htm',
                                        activities=activities
                                        .paginate(1, 12, False))
            elif data[i].path == 'contact' \
                    or data[i].path == 'laatste_bestuursblog':
                if data[i].filter_html:
                    safe_mode = False
                else:
                    safe_mode = 'escape'

                content += '
{0}
'.format(data[i].title)
                content += markdown(data[i].content, safe_mode=safe_mode,
                                    extensions=markdown_extensions)
        else:
            if data[i].filter_html:
                safe_mode = False
            else:
                safe_mode = 'escape'

            #print data[i].path
            content += '
{0}
'.format(data[i].title)
            content += '
Hello
'
            content += markdown(data[i].content, safe_mode=safe_mode,
                                extensions=markdown_extensions)

        content += '

'

        if i == len(data) - 1 or i % 2 != 0:
            content += '
'

    content += '
'

    return Markup(content)

@application.template_filter('pimpy_minute_line_numbers')
def pimpy_minute_line_numbers(data):
    #assert False
    s = ''
    for i, line in enumerate(data.split('\n')):
        #s += '%d%s\n' % (i, line[:-1])
        s += '%s\n' % (i, line[:-1])
    return s

<|endoftext|>


<commit_before>#!/usr/bin/env python

import re
from optparse import OptionParser

confPath = 'resources/cassandra/conf/'
hconfPath = 'resources/hadoop/conf/'
opsConfPath = '/etc/opscenter/'

clusterName = ''
seedList = ''
clusterSize = False
autoBootstrap = False
internalIP = ''
tokenPosition = -1

DEBUG = True

def promptUserInfo():
    global clusterName, seedList, clusterSize, autoBootstrap, internalIP, tokenPosition
    clusterName = raw_input("Cluster name:\n")

    seedList = raw_input("Seed list (comma-delimited):\n")
    seedList = seedList.replace(' ', '')
    seedList = seedList.split(',')
    seedList.sort()

    while (not type(clusterSize) is int):
        clusterSize = raw_input("Current cluster size:\n")
        try:
            clusterSize = int(clusterSize)
        except:
            print "Please enter a valid number."

    while not (autoBootstrap == 'y' or autoBootstrap == 'n'):
        autoBootstrap = raw_input("Is this node being autoBootStrapped? [y/n]\n").strip()
    if autoBootstrap == 'y':
        autoBootstrap = 'true'
    else:
        autoBootstrap = 'false'

    internalIP = raw_input("This node's internal IP address:\n")

    while (tokenPosition < 0) or (tokenPosition >= clusterSize):
        tokenPosition = raw_input("This node's token position (position < cluster size):\n")
        try:
            tokenPosition = int(tokenPosition)
        except:
            print "Please enter a valid number."

def commandLineSwitches():
    global clusterName, seedList, clusterSize, autoBootstrap, internalIP, tokenPosition, confPath, hconfPath, opsConfPath
    parser = OptionParser()
    parser.add_option("-n", "--clusterName", action="store", type="string", dest="clusterName", help="Set the cluster name.")
    parser.add_option("-l", "--seedList", action="store", type="string", dest="seedList", help="Provide a comma-delimited list of seeds.")
    parser.add_option("-s", "--clusterSize", action="store", type="int", dest="clusterSize", help="Provide the size of the cluster for equal partitioning.")
    parser.add_option("-a", "--autoBootstrap", action="store_true", dest="autoBootstrap", help="Set autobootstrap to true. (Defaults to false.)")
    parser.add_option("-i", "--internalIP", action="store", type="string", dest="internalIP", help="Set this nodes internal ip address.")
    parser.add_option("-t", "--tokenPosition", action="store", type="int", dest="tokenPosition", help="Set this node's token position.")

    parser.add_option("-c", "--confPath", action="store", type="string", dest="confPath", help="Set cassandra/conf/ path.")
    parser.add_option("-p", "--hconfPath", action="store", type="string", dest="hconfPath", help="Set hadoop/conf/ path.")

    (options, args) = parser.parse_args()
    if options:
        if options.clusterName:
            clusterName = options.clusterName
        if options.seedList:
            seedList = options.seedList
            seedList = seedList.replace(' ', '')
            seedList = seedList.split(',')
            seedList.sort()
        if options.clusterSize:
            clusterSize = options.clusterSize
        if options.autoBootstrap:
            autoBootstrap = options.autoBootstrap
        if options.internalIP:
            internalIP = options.internalIP
        if options.tokenPosition:
            tokenPosition = options.tokenPosition
        if options.confPath:
            confPath = options.confPath
        if options.hconfPath:
            hconfPath = options.hconfPath
        return True
    else:
        return False

def configureCassandraYaml():
    with open(confPath + 'cassandra.yaml', 'r') as f:
        yaml = f.read()

    # Set auto_bootstrap to true if expanding
    if autoBootstrap:
        yaml = yaml.replace('auto_bootstrap: false', 'auto_bootstrap: true')
    else:
        yaml = yaml.replace('auto_bootstrap: true', 'auto_bootstrap: false')

    # Create the seed list
    seedsYaml = ''
    for ip in seedList:
        seedsYaml += '     - ' + ip + '\n'
    if DEBUG:
        print "[DEBUG] seedsYaml: " + seedsYaml

    # Set seeds
    p = re.compile('seeds:(\s*-.*)*\s*#')
    yaml = p.sub('seeds:\n' + seedsYaml + '\n\n#', yaml)

    # Set listen_address
    p = re.compile('listen_address:.*\s*#')
    yaml = p.sub('listen_address: ' + internalIP + '\n\n#', yaml)
    if DEBUG:
        print "[DEBUG] listen_address: " + internalIP

    # Set rpc_address
    yaml = yaml.replace('rpc_address: localhost', 'rpc_address: 0.0.0.0')

    # Set cluster_name to reservationid
    p = re.compile('cluster_name: .*\s*#')
    yaml = p.sub("cluster_name: '" + clusterName + "'\n\n#", yaml)
    if DEBUG:
        print "[DEBUG] clusterName: " + clusterName

    # Construct token for an equally split ring
    if clusterSize and tokenPosition:
        token = tokenPosition * (2**127 / clusterSize)
        p = re.compile( 'initial_token:(\s)*#')
        yaml = p.sub( 'initial_token: ' + str(token) + "\n\n#", yaml)
        if DEBUG:
            print "[DEBUG] clusterSize: " + clusterSize
            print "[DEBUG] tokenPosition: " + tokenPosition
            print "[DEBUG] token: " + token

    # Brisk: Set to use different datacenters
    # yaml = yaml.replace('endpoint_snitch: org.apache.cassandra.locator.SimpleSnitch', 'endpoint_snitch: org.apache.cassandra.locator.PropertyFileSnitch')

    with open(confPath + 'cassandra.yaml', 'w') as f:
        f.write(yaml)

    print '[INFO] cassandra.yaml configured.'

def configureMapredSiteXML():
    with open(hconfPath + 'mapred-site.xml', 'r') as f:
        mapredSite = f.read()

    if len(seedList) > 0:
        mapredSite = mapredSite.replace('<value>localhost:8012</value>', '<value>' + seedList[0] + ':8012</value>')

    with open(hconfPath + 'mapred-site.xml', 'w') as f:
        f.write(mapredSite)

    print '[INFO] mapred-site.xml configured.'

def configureOpsCenterConf():
    try:
        with open(opsConfPath + 'opscenterd.conf', 'r') as f:
            opsConf = f.read()

        if len(seedList) > 0:
            opsConf = opsConf.replace('interface = 127.0.0.1', 'interface = 0.0.0.0')
            opsConf = opsConf.replace('seed_hosts = localhost', 'seed_hosts = ' + seedList[0])

        with open(opsConfPath + 'opscenterd.conf', 'w') as f:
            f.write(opsConf)

        print '[INFO] opscenterd.conf configured.'
    except:
        print '[INFO] opscenterd.conf not configured.'

if not commandLineSwitches():
    promptUserInfo()

configureCassandraYaml()
configureMapredSiteXML()
configureOpsCenterConf()
<commit_msg>configtool.py<commit_after>#!/usr/bin/env python

import re
from optparse import OptionParser

confPath = 'resources/cassandra/conf/'
hconfPath = 'resources/hadoop/conf/'
opsConfPath = '/etc/opscenter/'

clusterName = ''
seedList = ''
clusterSize = False
autoBootstrap = False
internalIP = ''
tokenPosition = -1

DEBUG = True

def promptUserInfo():
    global clusterName, seedList, clusterSize, autoBootstrap, internalIP, tokenPosition
    clusterName = raw_input("Cluster name:\n")

    seedList = raw_input("Seed list (comma-delimited):\n")
    seedList = seedList.replace(' ', '')
    seedList = seedList.split(',')
    seedList.sort()

    while (not type(clusterSize) is int):
        clusterSize = raw_input("Current cluster size:\n")
        try:
            clusterSize = int(clusterSize)
        except:
            print "Please enter a valid number."

    while not (autoBootstrap == 'y' or autoBootstrap == 'n'):
        autoBootstrap = raw_input("Is this node being autoBootStrapped? [y/n]\n").strip()
    if autoBootstrap == 'y':
        autoBootstrap = 'true'
    else:
        autoBootstrap = 'false'

    internalIP = raw_input("This node's internal IP address:\n")

    while (tokenPosition < 0) or (tokenPosition >= clusterSize):
        tokenPosition = raw_input("This node's token position (position < cluster size):\n")
        try:
            tokenPosition = int(tokenPosition)
        except:
            print "Please enter a valid number."

def commandLineSwitches():
    global clusterName, seedList, clusterSize, autoBootstrap, internalIP, tokenPosition, confPath, hconfPath, opsConfPath
    parser = OptionParser()
    parser.add_option("-n", "--clusterName", action="store", type="string", dest="clusterName", help="Set the cluster name.")
    parser.add_option("-l", "--seedList", action="store", type="string", dest="seedList", help="Provide a comma-delimited list of seeds.")
    parser.add_option("-s", "--clusterSize", action="store", type="int", dest="clusterSize", help="Provide the size of the cluster for equal partitioning.")
    parser.add_option("-a", "--autoBootstrap", action="store_true", dest="autoBootstrap", help="Set autobootstrap to true. (Defaults to false.)")
    parser.add_option("-i", "--internalIP", action="store", type="string", dest="internalIP", help="Set this nodes internal ip address.")
    parser.add_option("-t", "--tokenPosition", action="store", type="int", dest="tokenPosition", help="Set this node's token position.")

    parser.add_option("-c", "--confPath", action="store", type="string", dest="confPath", help="Set cassandra/conf/ path.")
    parser.add_option("-p", "--hconfPath", action="store", type="string", dest="hconfPath", help="Set hadoop/conf/ path.")

    (options, args) = parser.parse_args()
    if options:
        if options.clusterName:
            clusterName = options.clusterName
        if options.seedList:
            seedList = options.seedList
            seedList = seedList.replace(' ', '')
            seedList = seedList.split(',')
            seedList.sort()
        if options.clusterSize:
            clusterSize = options.clusterSize
        if options.autoBootstrap:
            autoBootstrap = options.autoBootstrap
        if options.internalIP:
            internalIP = options.internalIP
        if options.tokenPosition:
            tokenPosition = options.tokenPosition
        if options.confPath:
            confPath = options.confPath
        if options.hconfPath:
            hconfPath = options.hconfPath
        return True
    else:
        return False

def configureCassandraYaml():
    with open(confPath + 'cassandra.yaml', 'r') as f:
        yaml = f.read()

    # Set auto_bootstrap to true if expanding
    if autoBootstrap:
        yaml = yaml.replace('auto_bootstrap: false', 'auto_bootstrap: true')
    else:
        yaml = yaml.replace('auto_bootstrap: true', 'auto_bootstrap: false')

    # Create the seed list
    seedsYaml = ''
    for ip in seedList:
        seedsYaml += '     - ' + ip + '\n'
    if DEBUG:
        print "[DEBUG] seedsYaml: " + seedsYaml

    # Set seeds
    p = re.compile('seeds:(\s*-.*)*\s*#')
    yaml = p.sub('seeds:\n' + seedsYaml + '\n\n#', yaml)

    # Set listen_address
    p = re.compile('listen_address:.*\s*#')
    yaml = p.sub('listen_address: ' + internalIP + '\n\n#', yaml)
    if DEBUG:
        print "[DEBUG] listen_address: " + internalIP

    # Set rpc_address
    yaml = yaml.replace('rpc_address: localhost', 'rpc_address: 0.0.0.0')

    # Set cluster_name to reservationid
    p = re.compile('cluster_name: .*\s*#')
    yaml = p.sub("cluster_name: '" + clusterName + "'\n\n#", yaml)
    if DEBUG:
        print "[DEBUG] clusterName: " + clusterName

    # Construct token for an equally split ring
    if clusterSize and tokenPosition:
        token = tokenPosition * (2**127 / clusterSize)
        p = re.compile( 'initial_token:(\s)*#')
        yaml = p.sub( 'initial_token: ' + str(token) + "\n\n#", yaml)
        if DEBUG:
            print "[DEBUG] clusterSize: " + str(clusterSize)
            print "[DEBUG] tokenPosition: " + str(tokenPosition)
            print "[DEBUG] token: " + str(token)

    # Brisk: Set to use different datacenters
    # yaml = yaml.replace('endpoint_snitch: org.apache.cassandra.locator.SimpleSnitch', 'endpoint_snitch: org.apache.cassandra.locator.PropertyFileSnitch')

    with open(confPath + 'cassandra.yaml', 'w') as f:
        f.write(yaml)

    print '[INFO] cassandra.yaml configured.'

def configureMapredSiteXML():
    with open(hconfPath + 'mapred-site.xml', 'r') as f:
        mapredSite = f.read()

    if len(seedList) > 0:
        mapredSite = mapredSite.replace('<value>localhost:8012</value>', '<value>' + seedList[0] + ':8012</value>')

    with open(hconfPath + 'mapred-site.xml', 'w') as f:
        f.write(mapredSite)

    print '[INFO] mapred-site.xml configured.'

def configureOpsCenterConf():
    try:
        with open(opsConfPath + 'opscenterd.conf', 'r') as f:
            opsConf = f.read()

        if len(seedList) > 0:
            opsConf = opsConf.replace('interface = 127.0.0.1', 'interface = 0.0.0.0')
            opsConf = opsConf.replace('seed_hosts = localhost', 'seed_hosts = ' + seedList[0])

        with open(opsConfPath + 'opscenterd.conf', 'w') as f:
            f.write(opsConf)

        print '[INFO] opscenterd.conf configured.'
    except:
        print '[INFO] opscenterd.conf not configured.'

if not commandLineSwitches():
    promptUserInfo()

configureCassandraYaml()
configureMapredSiteXML()
configureOpsCenterConf()
<|endoftext|>


<commit_before>import subprocess
import shlex
import re
import os
import time
import urllib2
import platform
import json
import sys
import base64
import random
import datetime
import traceback

import argparse

class DummyProcess:
    def poll(self):
        return None
    def __init__(self):
        self.pid = 123456789

parser = argparse.ArgumentParser(description='robot control')
parser.add_argument('camera_id')
parser.add_argument('--xres', type=int, default=640)
parser.add_argument('--yres', type=int, default=480)
parser.add_argument('video_device_number', default=0, type=int)
parser.add_argument('--kbps', default=350, type=int)
parser.add_argument('--brightness', type=int, help='camera brightness')
parser.add_argument('--contrast', type=int, help='camera contrast')
parser.add_argument('--saturation', type=int, help='camera saturation')
parser.add_argument('--rotate180', default=False, type=bool, help='rotate image 180 degrees')
parser.add_argument('--env', default="prod")
parser.add_argument('--screen-capture', dest='screen_capture', action='store_true') # tells windows to pull from different camera, this should just be replaced with a video input device option
parser.set_defaults(screen_capture=False)
parser.add_argument('--no-mic', dest='mic_enabled', action='store_false')
parser.set_defaults(mic_enabled=True)
parser.add_argument('--no-camera', dest='camera_enabled', action='store_false')
parser.set_defaults(camera_enabled=True)
parser.add_argument('--dry-run', dest='dry_run', action='store_true')
parser.add_argument('--mic-channels', type=int, help='microphone channels, typically 1 or 2', default=1)
parser.add_argument('--audio-input-device', default='Microphone (HD Webcam C270)') # currently, this option is only used for windows screen capture

#global numVideoRestarts
#numVideoRestarts = 0
#global numAudioRestarts
#numAudioRestarts = 0

args = parser.parse_args()

print "args", args

server = "runmyrobot.com"

from socketIO_client import SocketIO, LoggingNamespace

# enable raspicam driver in case a raspicam is being used
os.system("sudo modprobe bcm2835-v4l2")

if args.env == "dev":
    print "using dev port 8122"
    port = 8122
elif args.env == "prod":
    print "using prod port 8022"
    port = 8022
else:
    print "invalid environment"
    sys.exit(0)

print "initializing socket io"
print "server:", server
print "port:", port
socketIO = SocketIO(server, port, LoggingNamespace)
print "finished initializing socket io"

#ffmpeg -f qtkit -i 0 -f mpeg1video -b 400k -r 30 -s 320x240 http://52.8.81.124:8082/hello/320/240/

def getVideoPort():

    url = 'http://%s/get_video_port/%s' % (server, args.camera_id)

    for retryNumber in range(2000):
        try:
            print "GET", url
            response = urllib2.urlopen(url).read()
            break
        except:
            print "could not open url ", url
            time.sleep(2)

    return json.loads(response)['mpeg_stream_port']

def getAudioPort():

    url = 'http://%s/get_audio_port/%s' % (server, args.camera_id)

    for retryNumber in range(2000):
        try:
            print "GET", url
            response = urllib2.urlopen(url).read()
            break
        except:
            print "could not open url ", url
            time.sleep(2)

    return json.loads(response)['audio_stream_port']

def randomSleep():
    """A short wait is good for quick recovery, but sometimes a longer delay is needed or it will just keep trying and failing short intervals, like because the system thinks the port is still in use and every retry makes the system think it's still in use. So, this has a high likelihood of picking a short interval, but will pick a long one sometimes."""

    timeToWait = random.choice((0.25, 0.25, 0.25, 0.25, 0.25, 0.25, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 5))
    print "sleeping", timeToWait
    time.sleep(timeToWait)

def startVideoCaptureLinux():

    videoPort = getVideoPort()

    # set brightness
    if (args.brightness is not None):
        print "brightness"
        os.system("v4l2-ctl -c brightness={brightness}".format(brightness=args.brightness))

    # set contrast
    if (args.contrast is not None):
        print "contrast"
        os.system("v4l2-ctl -c contrast={contrast}".format(contrast=args.contrast))

    # set saturation
    if (args.saturation is not None):
        print "saturation"
        os.system("v4l2-ctl -c saturation={saturation}".format(saturation=args.saturation))

    videoCommandLine = '/usr/local/bin/ffmpeg -f v4l2 -framerate 25 -video_size 640x480 -i /dev/video{video_device_number} {rotation_option} -f mpegts -codec:v mpeg1video -s {xres}x{yres} -b:v {kbps}k -bf 0 -muxdelay 0.001 http://{server}:{video_port}/hello/{xres}/{yres}/'.format(video_device_number=args.video_device_number, rotation_option=rotationOption(), kbps=args.kbps, server=server, video_port=videoPort, xres=args.xres, yres=args.yres)
    print videoCommandLine
    return subprocess.Popen(shlex.split(videoCommandLine))

def startAudioCaptureLinux():

    audioPort = getAudioPort()

    audioCommandLine = '/usr/local/bin/ffmpeg -f alsa -ar 44100 -ac %d -i hw:1 -f mpegts -codec:a mp2 -b:a 32k -muxdelay 0.001 http://%s:%s/hello/640/480/' % (args.mic_channels, server, audioPort)
    print audioCommandLine
    return subprocess.Popen(shlex.split(audioCommandLine))

def rotationOption():

    if args.rotate180:
        return "-vf transpose=2,transpose=2"
    else:
        return ""

def main():

    if args.camera_enabled:
        if not args.dry_run:
            videoProcess = startVideoCaptureLinux()
        else:
            videoProcess = DummyProcess()

    if args.mic_enabled:
        if not args.dry_run:
            audioProcess = startAudioCaptureLinux()
            time.sleep(30)
            os.system("killall ffmpeg")
            #socketIO.emit('send_video_process_start_event', {'camera_id': args.camera_id})
        else:
            audioProcess = DummyProcess()

    numVideoRestarts = 0
    numAudioRestarts = 0

    count = 0

    # loop forever and monitor status of ffmpeg processes
    while True:

        print "------------------------------------------" + str(count) + "-----------------------------------------------------"

        time.sleep(1)

        # todo: note about the following ffmpeg_process_exists is not technically true, but need to update
        # server code to check for send_video_process_exists if you want to set it technically accurate
        # because the process doesn't always exist, like when the relay is not started yet.

        # send status to server
        socketIO.emit('send_video_status', {'send_video_process_exists': True,
                                            'ffmpeg_process_exists': True,
                                            'camera_id':args.camera_id})

        if count % 10 == 0:
            try:
                with os.fdopen(os.open('/tmp/send_video_summary.txt', os.O_WRONLY | os.O_CREAT, 0o777), 'w') as statusFile:
                    statusFile.write("time" + str(datetime.datetime.now()) + "\n")
                    statusFile.write("video process poll " + str(videoProcess.poll()) + " pid " + str(videoProcess.pid) + " restarts " + str(numVideoRestarts) + " \n")
                    statusFile.write("audio process poll " + str(audioProcess.poll()) + " pid " + str(audioProcess.pid) + " restarts " + str(numAudioRestarts) + " \n")
                print "status file written"
                sys.stdout.flush()
            except:
                print "status file could not be written"
                traceback.print_exc()
                sys.stdout.flush()

        if args.camera_enabled:

            print "video process poll", videoProcess.poll(), "pid", videoProcess.pid, "restarts", numVideoRestarts

            # restart video if needed
            if videoProcess.poll() != None:
                randomSleep()
                videoProcess = startVideoCaptureLinux()
                numVideoRestarts += 1

        if args.mic_enabled:

            print "audio process poll", audioProcess.poll(), "pid", audioProcess.pid, "restarts", numAudioRestarts

            # restart audio if needed
            if audioProcess.poll() != None:
                randomSleep()
                audioProcess = startAudioCaptureLinux()
                #time.sleep(30)
                #socketIO.emit('send_video_process_start_event', {'camera_id': args.camera_id})
                numAudioRestarts += 1

        count += 1

main()

<commit_msg>added ability to turn video feed off and on<commit_after>import subprocess
import shlex
import re
import os
import time
import urllib2
import platform
import json
import sys
import base64
import random
import datetime
import traceback

import argparse

class DummyProcess:
    def poll(self):
        return None
    def __init__(self):
        self.pid = 123456789

parser = argparse.ArgumentParser(description='robot control')
parser.add_argument('camera_id')
parser.add_argument('--xres', type=int, default=640)
parser.add_argument('--yres', type=int, default=480)
parser.add_argument('video_device_number', default=0, type=int)
parser.add_argument('--kbps', default=350, type=int)
parser.add_argument('--brightness', type=int, help='camera brightness')
parser.add_argument('--contrast', type=int, help='camera contrast')
parser.add_argument('--saturation', type=int, help='camera saturation')
parser.add_argument('--rotate180', default=False, type=bool, help='rotate image 180 degrees')
parser.add_argument('--env', default="prod")
parser.add_argument('--screen-capture', dest='screen_capture', action='store_true') # tells windows to pull from different camera, this should just be replaced with a video input device option
parser.set_defaults(screen_capture=False)
parser.add_argument('--no-mic', dest='mic_enabled', action='store_false')
parser.set_defaults(mic_enabled=True)
parser.add_argument('--no-camera', dest='camera_enabled', action='store_false')
parser.set_defaults(camera_enabled=True)
parser.add_argument('--dry-run', dest='dry_run', action='store_true')
parser.add_argument('--mic-channels', type=int, help='microphone channels, typically 1 or 2', default=1)
parser.add_argument('--audio-input-device', default='Microphone (HD Webcam C270)') # currently, this option is only used for windows screen capture

#global numVideoRestarts
#numVideoRestarts = 0
#global numAudioRestarts
#numAudioRestarts = 0

commandArgs = parser.parse_args()

print "commandArgs", commandArgs

server = "runmyrobot.com"

from socketIO_client import SocketIO, LoggingNamespace

# enable raspicam driver in case a raspicam is being used
os.system("sudo modprobe bcm2835-v4l2")

if commandArgs.env == "dev":
    print "using dev port 8122"
    port = 8122
elif commandArgs.env == "prod":
    print "using prod port 8022"
    port = 8022
else:
    print "invalid environment"
    sys.exit(0)

print "initializing socket io"
print "server:", server
print "port:", port
socketIO = SocketIO(server, port, LoggingNamespace)
print "finished initializing socket io"

#ffmpeg -f qtkit -i 0 -f mpeg1video -b 400k -r 30 -s 320x240 http://52.8.81.124:8082/hello/320/240/

def getWithRetry(url):

    for retryNumber in range(2000):
        try:
            print "GET", url
            response = urllib2.urlopen(url).read()
            break
        except:
            print "could not open url ", url
            time.sleep(2)

    return response

def getVideoPort():

    url = 'http://%s/get_video_port/%s' % (server, commandArgs.camera_id)
    response = getWithRetry(url)
    return json.loads(response)['mpeg_stream_port']

def getAudioPort():

    url = 'http://%s/get_audio_port/%s' % (server, commandArgs.camera_id)
    response = getWithRetry(url)
    return json.loads(response)['audio_stream_port']

def getRobotID():

    url = 'http://%s/get_robot_id/%s' % (server, commandArgs.camera_id)
    response = getWithRetry(url)
    return json.loads(response)['robot_id']

def randomSleep():
    """A short wait is good for quick recovery, but sometimes a longer delay is needed or it will just keep trying and failing short intervals, like because the system thinks the port is still in use and every retry makes the system think it's still in use. So, this has a high likelihood of picking a short interval, but will pick a long one sometimes."""

    timeToWait = random.choice((0.25, 0.25, 0.25, 0.25, 0.25, 0.25, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 5))
    print "sleeping", timeToWait
    time.sleep(timeToWait)

def startVideoCaptureLinux():

    videoPort = getVideoPort()

    # set brightness
    if (commandArgs.brightness is not None):
        print "brightness"
        os.system("v4l2-ctl -c brightness={brightness}".format(brightness=commandArgs.brightness))

    # set contrast
    if (commandArgs.contrast is not None):
        print "contrast"
        os.system("v4l2-ctl -c contrast={contrast}".format(contrast=commandArgs.contrast))

    # set saturation
    if (commandArgs.saturation is not None):
        print "saturation"
        os.system("v4l2-ctl -c saturation={saturation}".format(saturation=commandArgs.saturation))

    videoCommandLine = '/usr/local/bin/ffmpeg -f v4l2 -framerate 25 -video_size 640x480 -i /dev/video{video_device_number} {rotation_option} -f mpegts -codec:v mpeg1video -s {xres}x{yres} -b:v {kbps}k -bf 0 -muxdelay 0.001 http://{server}:{video_port}/hello/{xres}/{yres}/'.format(video_device_number=commandArgs.video_device_number, rotation_option=rotationOption(), kbps=commandArgs.kbps, server=server, video_port=videoPort, xres=commandArgs.xres, yres=commandArgs.yres)
    print videoCommandLine
    return subprocess.Popen(shlex.split(videoCommandLine))

def startAudioCaptureLinux():

    audioPort = getAudioPort()

    audioCommandLine = '/usr/local/bin/ffmpeg -f alsa -ar 44100 -ac %d -i hw:1 -f mpegts -codec:a mp2 -b:a 32k -muxdelay 0.001 http://%s:%s/hello/640/480/' % (commandArgs.mic_channels, server, audioPort)
    print audioCommandLine
    return subprocess.Popen(shlex.split(audioCommandLine))

def rotationOption():

    if commandArgs.rotate180:
        return "-vf transpose=2,transpose=2"
    else:
        return ""

def onCommandToRobot(*args):
    global robotID
    if len(args) > 0 and 'robot_id' in args[0] and args[0]['robot_id'] == robotID:
        commandMessage = args[0]
        print('command for this robot received:', commandMessage)
        command = commandMessage['command']

        if command == 'VIDOFF':
            print ('disabling camera capture process')
            print "args", args
            commandArgs.camera_enabled = False
            os.system("killall ffmpeg")

        if command == 'VIDON':
            print ('enabling camera capture process')
            print "args", args
            commandArgs.camera_enabled = True

        sys.stdout.flush()

def onConnection(*args):
    print('connection:', args)
    sys.stdout.flush()

def main():

    global robotID
    robotID = getRobotID()

    socketIO.on('command_to_robot', onCommandToRobot)
    socketIO.on('connection', onConnection)

    print "robot id:", robotID
    sys.stdout.flush()

    if commandArgs.camera_enabled:
        if not commandArgs.dry_run:
            videoProcess = startVideoCaptureLinux()
        else:
            videoProcess = DummyProcess()

    if commandArgs.mic_enabled:
        if not commandArgs.dry_run:
            audioProcess = startAudioCaptureLinux()
            time.sleep(30)
            os.system("killall ffmpeg")
            #socketIO.emit('send_video_process_start_event', {'camera_id': commandArgs.camera_id})
        else:
            audioProcess = DummyProcess()

    numVideoRestarts = 0
    numAudioRestarts = 0

    count = 0

    # loop forever and monitor status of ffmpeg processes
    while True:

        print "------------------------------------------" + str(count) + "-----------------------------------------------------"

        socketIO.wait(seconds=1)

        # todo: note about the following ffmpeg_process_exists is not technically true, but need to update
        # server code to check for send_video_process_exists if you want to set it technically accurate
        # because the process doesn't always exist, like when the relay is not started yet.

        # send status to server
        socketIO.emit('send_video_status', {'send_video_process_exists': True,
                                            'ffmpeg_process_exists': True,
                                            'camera_id':commandArgs.camera_id})

        if count % 20 == 0:
            try:
                with os.fdopen(os.open('/tmp/send_video_summary.txt', os.O_WRONLY | os.O_CREAT, 0o777), 'w') as statusFile:
                    statusFile.write("time" + str(datetime.datetime.now()) + "\n")
                    statusFile.write("video process poll " + str(videoProcess.poll()) + " pid " + str(videoProcess.pid) + " restarts " + str(numVideoRestarts) + " \n")
                    statusFile.write("audio process poll " + str(audioProcess.poll()) + " pid " + str(audioProcess.pid) + " restarts " + str(numAudioRestarts) + " \n")
                print "status file written"
                sys.stdout.flush()
            except:
                print "status file could not be written"
                traceback.print_exc()
                sys.stdout.flush()

        if commandArgs.camera_enabled:

            print "video process poll", videoProcess.poll(), "pid", videoProcess.pid, "restarts", numVideoRestarts

            # restart video if needed
            if videoProcess.poll() != None:
                randomSleep()
                videoProcess = startVideoCaptureLinux()
                numVideoRestarts += 1

        if commandArgs.mic_enabled:

            print "audio process poll", audioProcess.poll(), "pid", audioProcess.pid, "restarts", numAudioRestarts

            # restart audio if needed
            if audioProcess.poll() != None:
                randomSleep()
                audioProcess = startAudioCaptureLinux()
                #time.sleep(30)
                #socketIO.emit('send_video_process_start_event', {'camera_id': commandArgs.camera_id})
                numAudioRestarts += 1

        count += 1

main()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('accounts', '0005_auto_20150625_1647'),
        ('transactions', '0002_auto_20150625_0113'),
    ]

    operations = [
        migrations.AddField(
            model_name='transaction',
            name='account',
            field=models.ForeignKey(default=1, to='accounts.Account'),
            preserve_default=False,
        ),
        migrations.AlterField(
            model_name='transaction',
            name='amount',
            field=models.DecimalField(max_digits=19, decimal_places=2),
        ),
        migrations.AlterField(
            model_name='transaction',
            name='balance',
            field=models.DecimalField(max_digits=19, decimal_places=2),
        ),
    ]
<commit_msg>Deletes migration files and sqlite file to apply database changes<commit_after><|endoftext|>