<commit_before>import numpy as np
from astropy.io import fits
import matplotlib.pyplot as plt
import glob
from echelle_funcs import getOrderWavelengths
'''
Plots the CaII H&K line region from a series of 2D (multi-order) ARCES spectra. Hooray!

Note: This program assumes a spectrum of 'KIC 1234567' will have the string '1234567'
somewhere in its filename. If not, you're going to have a bad time.
'''
# Define where important things live, like your spectra and a text file listing your targets
rootdir = '../../../RG_spectra/tequila_201604/'
targets = np.loadtxt('starnames.txt', usecols=(0,), comments='#', unpack=True)
strtargets = []
for target in targets:
    strtargets.append(str(int(target)))

#########
## OPTION 1: create a list of spectrum files via glob
#dirlist = []
#for target in strtargets:
#    dirlist.append(rootdir + 'kplr' + target + '/')
#filelist = []
#for dir in dirlist:
#    for file in glob.glob(dir + '1*ec.fits'):
#        filelist.append(file)
#        #print(file)
#########

#########
## OPTION 2: Once a text file listing the names of the good 2D spectra exists, use that
infiles = '../../../Rawls_tidesactivity/good_caHK_spectra.txt'
filelist = []
with open(infiles) as f1:
    for line in f1:
        filelist.append(rootdir + line.rstrip())
#########

def plotThreeAtATime(filelist):
    '''
    The original plotter is now a function!
    Use this if you're not interested to sort by star yet and just want to plot all
    the spectra in figures, three at a time, for manual inspection.
    It builds up a list of spectra with S/N > 5, defined as having a y-axis values > 25.
    '''
    ## Plot three panes of spectra at a time
    ## (The original idea was to plot a range of orders, but then I realized I only wanted two)
    #orderstart = 90 #89
    #orderstop = 93 #94
    #colors = ['#fc8d59','#ef6548','#d7301f','#b30000','#7f0000']
    ## LOL NOPE let's just manually plot order 90 and 91, which contain the Ca H&K lines
    goodfilelist = []
    for i in np.arange(0, len(filelist)-3, 3):
        fig = plt.figure(figsize=(10,18))
        # LET'S PLOT ONE SPECTRUM in the top of three panels
        ax = fig.add_subplot(311)
        ax.set_xlim([3910, 3990])
        fitsspec = fits.open(filelist[i])
        fitsfluxes = fitsspec[0].data
        fitswaves = getOrderWavelengths(fitsspec)
        # sort the damn orders so they're not backwards
        for idx, (waves, fluxes) in enumerate(zip(fitswaves, fitsfluxes)):
            fitsfluxes[idx] = fluxes[np.argsort(waves)]
            fitswaves[idx] = waves[np.argsort(waves)]
        # truncate the spectra somewhat arbitrarily (one order for H line region, one order for K line)
        idxtrunc90 = np.where(fitswaves[90] > 3950)[0][0]
        idxtrunc91 = np.where(fitswaves[91] > 3950)[0][0]
        waves90 = fitswaves[90][idxtrunc90::]; fluxes90 = fitsfluxes[90][idxtrunc90::]
        waves91 = fitswaves[91][0:idxtrunc91]; fluxes91 = fitsfluxes[91][0:idxtrunc91]
        if (np.max(fluxes90) < 25) or (np.max(fluxes91) < 25) or (len(fluxes90) < 50):
            colors = ['0.75', '0.75'] # this is a bad spectrum so we plot it in gray and feel bad for it
        else:
            colors = ['#3182bd','#08519c'] # this is a good spectrum so it gets pretty blue colors
            goodfilelist.append(filelist[i])
        plt.plot(waves90, fluxes90, color=colors[0])
        plt.plot(waves91, fluxes91, color=colors[1])
        ## (You could use this with the original idea of plotting a range of orders!)
    #    for idx, (waves, fluxes) in enumerate(zip(fitswaves[orderstart:orderstop], fitsfluxes[orderstart:orderstop])):
    #        plt.plot(waves, fluxes, color=colors[idx])
        plt.axvline(x=3968.468, ls=':', color='k')
        plt.axvline(x=3933.663, ls=':', color='k')
        plt.title(filelist[i][-28:], size=20)
        fitsspec.close()
        # DO IT ALL AGAIN for the middle panel
        ax = fig.add_subplot(312)
        ax.set_xlim([3910, 3990])
        fitsspec = fits.open(filelist[i+1])
        fitsfluxes = fitsspec[0].data
        fitswaves = getOrderWavelengths(fitsspec)
        for idx, (waves, fluxes) in enumerate(zip(fitswaves, fitsfluxes)):
            fitsfluxes[idx] = fluxes[np.argsort(waves)]
            fitswaves[idx] = waves[np.argsort(waves)]
        idxtrunc90 = np.where(fitswaves[90] > 3950)[0][0]
        idxtrunc91 = np.where(fitswaves[91] > 3950)[0][0]
        waves90 = fitswaves[90][idxtrunc90::]; fluxes90 = fitsfluxes[90][idxtrunc90::]
        waves91 = fitswaves[91][0:idxtrunc91]; fluxes91 = fitsfluxes[91][0:idxtrunc91]
        if (np.max(fluxes90) < 25) or (np.max(fluxes91) < 25) or (len(fluxes90) < 50):
            colors = ['0.75', '0.75']
        else:
            colors = ['#3182bd','#08519c']
            goodfilelist.append(filelist[i+1])
        plt.plot(waves90, fluxes90, color=colors[0])
        plt.plot(waves91, fluxes91, color=colors[1])
        plt.axvline(x=3968.468, ls=':', color='k')
        plt.axvline(x=3933.663, ls=':', color='k')
        plt.title(filelist[i+1][-28:], size=20)
        fitsspec.close()
        # DO IT ALL ONE LAST TIME for the bottom panel
        ax = fig.add_subplot(313)
        ax.set_xlim([3910, 3990])
        fitsspec = fits.open(filelist[i+2])
        fitsfluxes = fitsspec[0].data
        fitswaves = getOrderWavelengths(fitsspec)
        for idx, (waves, fluxes) in enumerate(zip(fitswaves, fitsfluxes)):
            fitsfluxes[idx] = fluxes[np.argsort(waves)]
            fitswaves[idx] = waves[np.argsort(waves)]
        idxtrunc90 = np.where(fitswaves[90] > 3950)[0][0]
        idxtrunc91 = np.where(fitswaves[91] > 3950)[0][0]
        waves90 = fitswaves[90][idxtrunc90::]; fluxes90 = fitsfluxes[90][idxtrunc90::]
        waves91 = fitswaves[91][0:idxtrunc91]; fluxes91 = fitsfluxes[91][0:idxtrunc91]
        if (np.max(fluxes90) < 25) or (np.max(fluxes91) < 25) or (len(fluxes90) < 50):
            colors = ['0.75', '0.75']
        else:
            colors = ['#3182bd','#08519c']
            goodfilelist.append(filelist[i+2])
        plt.plot(waves90, fluxes90, color=colors[0])
        plt.plot(waves91, fluxes91, color=colors[1])
        plt.axvline(x=3968.468, ls=':', color='k')
        plt.axvline(x=3933.663, ls=':', color='k')
        plt.title(filelist[i+2][-28:], size=20)
        plt.xlabel('Wavelength (\AA)')
        fitsspec.close()
        plt.show()
    for file in goodfilelist:
        print(file)

## Use this if you want to plot spectra three at a time via the function above
#plotThreeAtATime(filelist)

## Use this if you want to make one figure per star with all its spectra stacked and normalized
for star in strtargets:
    yoffset = 0
    fig = plt.figure() # note you'll get a blank figure if the star isn't in any of the filenames
    plt.title(star)
    for file in filelist:
        if star in file:
            #print(star, file) # I can write nested loops yes I can
            fitsspec = fits.open(file)
            fitsfluxes = fitsspec[0].data
            fitswaves = getOrderWavelengths(fitsspec)
            for idx, (waves, fluxes) in enumerate(zip(fitswaves, fitsfluxes)):
                fitsfluxes[idx] = fluxes[np.argsort(waves)]
                fitswaves[idx] = waves[np.argsort(waves)]
            idxtrunc90 = np.where(fitswaves[90] > 3950)[0][0]
            idxtrunc91 = np.where(fitswaves[91] > 3950)[0][0]
            waves90 = fitswaves[90][idxtrunc90::]; fluxes90 = fitsfluxes[90][idxtrunc90::]
            waves91 = fitswaves[91][0:idxtrunc91]; fluxes91 = fitsfluxes[91][0:idxtrunc91]
            waves = np.concatenate((waves91, waves90), axis=0)
            fluxes = np.concatenate((fluxes91, fluxes90), axis=0)
            # NORMALIZATION! look out kittens, these fluxes all go from 0 to 1 now
            fluxes = (fluxes-np.min(fluxes))/(np.max(fluxes)-np.min(fluxes))
            plt.plot(waves, fluxes+yoffset, color='#3182bd')
            yoffset += 1
            fitsspec.close()
    plt.axvline(x=3968.468, ls=':', color='k')
    plt.axvline(x=3933.663, ls=':', color='k')
    plt.axis([3910,3990,0,yoffset])
    plt.xlabel('Wavelength (\AA)')
    plt.show()

<commit_msg>begin adding ability to predict where Ca lines should be<commit_after>import numpy as np
from astropy.io import fits
from astropy.time import Time
import matplotlib.pyplot as plt
import glob
from echelle_funcs import getOrderWavelengths
'''
Plots the CaII H&K line region from a series of 2D (multi-order) ARCES spectra. Hooray!

Note: This program assumes a spectrum of 'KIC 1234567' will have the string '1234567'
somewhere in its filename. If not, you're going to have a bad time.
'''
# Define where important things live, like your spectra and a text file listing your targets
rootdir = '../../../RG_spectra/tequila_201604/'
targets = np.loadtxt('starnames.txt', usecols=(0,), comments='#', unpack=True)
strtargets = []
for target in targets:
    strtargets.append(str(int(target)))

#########
## OPTION 1: create a list of spectrum files via glob
#dirlist = []
#for target in strtargets:
#    dirlist.append(rootdir + 'kplr' + target + '/')
#filelist = []
#for dir in dirlist:
#    for file in glob.glob(dir + '1*ec.fits'):
#        filelist.append(file)
#        #print(file)
#########

#########
## OPTION 2: Once a text file listing the names of the good 2D spectra exists, use that
infiles = '../../../Rawls_tidesactivity/good_caHK_spectra.txt'
filelist = []
with open(infiles) as f1:
    for line in f1:
        filelist.append(rootdir + line.rstrip())
useRVdata = True # set True to overplot where each star's Ca line contribution should be
#########

if useRVdata == True:
    # Read in orbital periods for each system
    #periods = np.loadtxt('starnames.txt', comments='#', usecols=(1,), unpack=True)
    # J/K I DON'T THINK WE NEED THAT...
    # (1) Get timestamps for each observation
    rtargets = []; reffiles = []; bjdsaves = []; bcvsaves = []
    for file in filelist:
        ### the below is SUPER GREAT but actually not relevant here ###
        #fitsspec = fits.open(file)
        #fitsspec.verify('silentfix')
        #head = fitsspec[0].header
        #utmiddle = head['UTMIDDLE']
        #if '=' in utmiddle:
        #    time = Time(utmiddle[1::], format='fits', scale='utc')
        #else:
        #    time = Time(head['DATE-OBS'][0:11] + utmiddle, format='fits', scale='utc')
        #target = head['OBJNAME']
        #print(target, time.jd)
        #fitsspec.close()
        ### the above is SUPER GREAT but actually not relevant here ###
        # get date and bcv from jean's handy [target].info.txt files
        fitsspec = fits.open(file)
        target = fitsspec[0].header['OBJNAME']
        fitsspec.close()
        infofile = rootdir + 'kplr' + target + '/' + target + '.info.txt'
        fullspecfiles = np.loadtxt(infofile, usecols=(0,), comments='#', dtype={'names':('fullspecfiles',),'formats':('|S29',)}, unpack=True)
        bjds, bcvs = np.loadtxt(infofile, usecols=(2,4), comments='#', unpack=True)
        for fullspecfile, bjd, bcv in zip(fullspecfiles, bjds, bcvs):
            if file[-19::] in str(fullspecfile):
                rtargets.append(target)
                reffiles.append(file)
                bjdsaves.append(bjd)
                bcvsaves.append(bcv)
    if len(filelist) != len(reffiles):
        raise IOError('Something bad happened when you tried to assign bjds and bcvs with the .info.txt files.')
    # (2) use bjds to look up both stars' RVs in patrick's LaTeX file of doom
    for target, bjd, bcv in zip(rtargets, bjdsaves, bcvsaves):
        print(target, bjd, bcv)
        # KEEP WORKING FROM HERE!!
    # (3) properly combine RVs and BCVs to get predicted location of CaII H and CaII K
    # (4) save this in ca2waves instead of the default values below
    ca2waves = [3968.468, 3933.663] # UPDATE THIS APPROPRIATELY !!!
else:
    ca2waves = [3968.468, 3933.663]

def plotThreeAtATime(filelist):
    '''
    The original plotter is now a function!
    Use this if you're not interested to sort by star yet and just want to plot all
    the spectra in figures, three at a time, for manual inspection.
    It builds up a list of spectra with S/N > 5, defined as having a y-axis values > 25.
    '''
    ## Plot three panes of spectra at a time
    ## (The original idea was to plot a range of orders, but then I realized I only wanted two)
    #orderstart = 90 #89
    #orderstop = 93 #94
    #colors = ['#fc8d59','#ef6548','#d7301f','#b30000','#7f0000']
    ## LOL NOPE let's just manually plot order 90 and 91, which contain the Ca H&K lines
    goodfilelist = []
    for i in np.arange(0, len(filelist)-3, 3):
        fig = plt.figure(figsize=(10,18))
        # LET'S PLOT ONE SPECTRUM in the top of three panels
        ax = fig.add_subplot(311)
        ax.set_xlim([3910, 3990])
        fitsspec = fits.open(filelist[i])
        fitsfluxes = fitsspec[0].data
        fitswaves = getOrderWavelengths(fitsspec)
        # sort the damn orders so they're not backwards
        for idx, (waves, fluxes) in enumerate(zip(fitswaves, fitsfluxes)):
            fitsfluxes[idx] = fluxes[np.argsort(waves)]
            fitswaves[idx] = waves[np.argsort(waves)]
        # truncate the spectra somewhat arbitrarily (one order for H line region, one order for K line)
        idxtrunc90 = np.where(fitswaves[90] > 3950)[0][0]
        idxtrunc91 = np.where(fitswaves[91] > 3950)[0][0]
        waves90 = fitswaves[90][idxtrunc90::]; fluxes90 = fitsfluxes[90][idxtrunc90::]
        waves91 = fitswaves[91][0:idxtrunc91]; fluxes91 = fitsfluxes[91][0:idxtrunc91]
        if (np.max(fluxes90) < 25) or (np.max(fluxes91) < 25) or (len(fluxes90) < 50):
            colors = ['0.75', '0.75'] # this is a bad spectrum so we plot it in gray and feel bad for it
        else:
            colors = ['#3182bd','#08519c'] # this is a good spectrum so it gets pretty blue colors
            goodfilelist.append(filelist[i])
        plt.plot(waves90, fluxes90, color=colors[0])
        plt.plot(waves91, fluxes91, color=colors[1])
        ## (You could use this with the original idea of plotting a range of orders!)
    #    for idx, (waves, fluxes) in enumerate(zip(fitswaves[orderstart:orderstop], fitsfluxes[orderstart:orderstop])):
    #        plt.plot(waves, fluxes, color=colors[idx])
        plt.axvline(x=3968.468, ls=':', color='k')
        plt.axvline(x=3933.663, ls=':', color='k')
        plt.title(filelist[i][-28:], size=20)
        fitsspec.close()
        # DO IT ALL AGAIN for the middle panel
        ax = fig.add_subplot(312)
        ax.set_xlim([3910, 3990])
        fitsspec = fits.open(filelist[i+1])
        fitsfluxes = fitsspec[0].data
        fitswaves = getOrderWavelengths(fitsspec)
        for idx, (waves, fluxes) in enumerate(zip(fitswaves, fitsfluxes)):
            fitsfluxes[idx] = fluxes[np.argsort(waves)]
            fitswaves[idx] = waves[np.argsort(waves)]
        idxtrunc90 = np.where(fitswaves[90] > 3950)[0][0]
        idxtrunc91 = np.where(fitswaves[91] > 3950)[0][0]
        waves90 = fitswaves[90][idxtrunc90::]; fluxes90 = fitsfluxes[90][idxtrunc90::]
        waves91 = fitswaves[91][0:idxtrunc91]; fluxes91 = fitsfluxes[91][0:idxtrunc91]
        if (np.max(fluxes90) < 25) or (np.max(fluxes91) < 25) or (len(fluxes90) < 50):
            colors = ['0.75', '0.75']
        else:
            colors = ['#3182bd','#08519c']
            goodfilelist.append(filelist[i+1])
        plt.plot(waves90, fluxes90, color=colors[0])
        plt.plot(waves91, fluxes91, color=colors[1])
        plt.axvline(x=3968.468, ls=':', color='k')
        plt.axvline(x=3933.663, ls=':', color='k')
        plt.title(filelist[i+1][-28:], size=20)
        fitsspec.close()
        # DO IT ALL ONE LAST TIME for the bottom panel
        ax = fig.add_subplot(313)
        ax.set_xlim([3910, 3990])
        fitsspec = fits.open(filelist[i+2])
        fitsfluxes = fitsspec[0].data
        fitswaves = getOrderWavelengths(fitsspec)
        for idx, (waves, fluxes) in enumerate(zip(fitswaves, fitsfluxes)):
            fitsfluxes[idx] = fluxes[np.argsort(waves)]
            fitswaves[idx] = waves[np.argsort(waves)]
        idxtrunc90 = np.where(fitswaves[90] > 3950)[0][0]
        idxtrunc91 = np.where(fitswaves[91] > 3950)[0][0]
        waves90 = fitswaves[90][idxtrunc90::]; fluxes90 = fitsfluxes[90][idxtrunc90::]
        waves91 = fitswaves[91][0:idxtrunc91]; fluxes91 = fitsfluxes[91][0:idxtrunc91]
        if (np.max(fluxes90) < 25) or (np.max(fluxes91) < 25) or (len(fluxes90) < 50):
            colors = ['0.75', '0.75']
        else:
            colors = ['#3182bd','#08519c']
            goodfilelist.append(filelist[i+2])
        plt.plot(waves90, fluxes90, color=colors[0])
        plt.plot(waves91, fluxes91, color=colors[1])
        plt.axvline(x=3968.468, ls=':', color='k')
        plt.axvline(x=3933.663, ls=':', color='k')
        plt.title(filelist[i+2][-28:], size=20)
        plt.xlabel('Wavelength (\AA)')
        fitsspec.close()
        plt.show()
    for file in goodfilelist:
        print(file)

## Use this if you want to plot spectra three at a time via the function above
#plotThreeAtATime(filelist)

## Use this if you want to make one figure per star with all its spectra stacked and normalized
for star in strtargets:
    yoffset = 0
    fig = plt.figure() # note you'll get a blank figure if the star isn't in any of the filenames
    plt.title(star)
    for file in filelist:
        if star in file:
            #print(star, file) # I can write nested loops yes I can
            fitsspec = fits.open(file)
            fitsfluxes = fitsspec[0].data
            fitswaves = getOrderWavelengths(fitsspec)
            for idx, (waves, fluxes) in enumerate(zip(fitswaves, fitsfluxes)):
                fitsfluxes[idx] = fluxes[np.argsort(waves)]
                fitswaves[idx] = waves[np.argsort(waves)]
            idxtrunc90 = np.where(fitswaves[90] > 3950)[0][0]
            idxtrunc91 = np.where(fitswaves[91] > 3950)[0][0]
            waves90 = fitswaves[90][idxtrunc90::]; fluxes90 = fitsfluxes[90][idxtrunc90::]
            waves91 = fitswaves[91][0:idxtrunc91]; fluxes91 = fitsfluxes[91][0:idxtrunc91]
            waves = np.concatenate((waves91, waves90), axis=0)
            fluxes = np.concatenate((fluxes91, fluxes90), axis=0)
            # NORMALIZATION! look out kittens, these fluxes all go from 0 to 1 now
            fluxes = (fluxes-np.min(fluxes))/(np.max(fluxes)-np.min(fluxes))
            plt.plot(waves, fluxes+yoffset, color='#3182bd')
            yoffset += 1
            fitsspec.close()
    plt.axvline(x=ca2waves[0], ls=':', color='k')
    plt.axvline(x=ca2waves[1], ls=':', color='k')
    plt.axis([3910,3990,0,yoffset])
    plt.xlabel('Wavelength (\AA)')
    plt.show()

<|endoftext|>


<commit_before>import requests

# --------------- Helpers that build all of the responses ----------------------
# These helpers are from amazon

def build_speechlet_response(title, output, reprompt_text, should_end_session):
    return {
        'outputSpeech': {
            'type': 'PlainText',
            'text': output
        },
        'card': {
            'type': 'Simple',
            'title': "SessionSpeechlet - " + title,
            'content': "SessionSpeechlet - " + output
        },
        'reprompt': {
            'outputSpeech': {
                'type': 'PlainText',
                'text': reprompt_text
            }
        },
        'shouldEndSession': should_end_session
    }

def build_login_card_response(title, output, reprompt_text, should_end_session):
    return {
        'outputSpeech': {
            'type': 'PlainText',
            'text': output
        },
        'card': {
            'type': 'LinkAccount',
            'title': "SessionSpeechlet - " + title,
            'content': "SessionSpeechlet - " + output
        },
        'reprompt': {
            'outputSpeech': {
                'type': 'PlainText',
                'text': reprompt_text
            }
        },
        'shouldEndSession': should_end_session
    }

def build_response(session_attributes, speechlet_response):
    return {
        'version': '1.0',
        'sessionAttributes': session_attributes,
        'response': speechlet_response
    }

# ---------------------- Twitch Auth Helpers--------------------------------------
def get_twitch_auth(session):
    if 'accessToken' in session['user']:
        return session['user']['accessToken']
    else:
        return None

# ------------------------Twitch API Helpers -------------------------------------
def twitch_top_streamers(num_of_responses):
    api_info = get_twitch_api_info()
    url = api_info['base_api_url'] + api_info['top_streams']
    headers = {"Accept": "application/json", "Client-ID": api_info['Client-ID']}

    api_response = requests.get(url, headers=headers)
    json = api_response.json()

    streams = ""
    for num in range(0, num_of_responses):
        if streams != "":
            streams = streams + ", "
        streams = streams + json['streams'][num]['channel']['display_name']

    return streams

def twitch_my_top_streamers(num_of_responses, auth_info):
    api_info = get_twitch_api_info()
    url = api_info['base_api_url'] + api_info['my_top_streams']
    headers = {"Accept": "application/json",
               "Client-ID": api_info['Client-ID'],
               "Authorization":"OAuth " + auth_info}

    api_response = requests.get(url, headers=headers)
    json = api_response.json()

    streams = ""
    i = 0

    while i < num_of_responses and i < len(json['streams']):
        if streams != "":
            streams = streams + ", "
        streams = streams + json['streams'][i]['channel']['display_name']

    return streams

def twitch_game_top_streamers(num_of_responses, game):
    api_info = get_twitch_api_info()
    url = api_info['base_api_url'] + api_info['top_game_streams'].format(game)
    headers = {"Accept": "application/json", "Client-ID": api_info['Client-ID']}

    api_response = requests.get(url, headers=headers)
    json = api_response.json()

    streams = ""
    for num in range(0, num_of_responses):
        if streams != "":
            streams = streams + ", "
        streams = streams + json['streams'][num]['channel']['display_name']

    return streams

def get_twitch_api_info():
    api_info = {
        "Client-ID": "fq5m7globoglp4zanbya8c4r5q8e1i",
        "base_api_url": "https://api.twitch.tv/kraken/",
        "top_streams": "streams",
        "top_game_streams": "streams/?game={}",
        "my_top_streams": "streams/followed"
    }
    return api_info

# --------------- Functions that modify behavior --------------------------------

def get_welcome_response(session):
    # TODO tell user if they are authenticated
    session_attributes = {}
    card_title = "Welcome to the My Twitch Streams Alexa Skill"
    speech_output = "Welcome to the My Twitch streams app. To use please to do"
    reprompt_text = "To use this skill, please todo"
    should_end_session = False

    return build_response(session_attributes,
                          build_speechlet_response(card_title, speech_output, reprompt_text, should_end_session))

def get_end_response():
    session_attributes = {}
    card_title = None
    speech_output = None
    reprompt_text = None
    should_end_session = True

    return build_response(session_attributes,
                          build_speechlet_response(card_title, speech_output, reprompt_text, should_end_session))

def get_top_streamers(session):
    num_of_streams = 5
    top_streamers = twitch_top_streamers(num_of_streams)

    session_attributes = {}
    card_title = "Top Twitch Streamers On Now"
    speech_output = "Here are the top {} streamers: {}".format(num_of_streams, top_streamers)
    reprompt_text = None
    should_end_session = True

    return build_response(session_attributes,
                          build_speechlet_response(card_title, speech_output, reprompt_text, should_end_session))

def get_my_top_streamers(session):
    num_of_streams = 5

    auth_info = get_twitch_auth(session)
    if auth_info is None:
        return get_login_card()

    my_top_streamers = twitch_my_top_streamers(num_of_streams, auth_info)
    session_attributes = {}
    card_title = "Top Twitch Streamers On Now"
    speech_output = "Here are your top {} streamers: {}".format(num_of_streams, my_top_streamers)
    reprompt_text = None
    should_end_session = True

    return build_response(session_attributes,
                          build_speechlet_response(card_title, speech_output, reprompt_text, should_end_session))

def get_game_top_streamers(request, session):
    num_of_streams = 5
    game = get_twitch_game_name(request['intent']['slots']['Game']['value'])

    if game is None:
        return get_invalid_game_response()

    streamers = twitch_game_top_streamers(num_of_streams, game)
    session_attributes = {}
    card_title = "Top Twitch Streamers On Now"
    speech_output = "Here are the top {} streamers for {}: {}".format(num_of_streams, game, streamers)
    reprompt_text = None
    should_end_session = True

    return build_response(session_attributes,
                          build_speechlet_response(card_title, speech_output, reprompt_text, should_end_session))

def get_invalid_game_response():
    session_attributes = {}
    card_title = "Unknown game"
    speech_output = "Unknown game, please try again"
    reprompt_text = None
    should_end_session = True

    return build_response(session_attributes,
                          build_speechlet_response(card_title, speech_output, reprompt_text, should_end_session))

def get_login_card():
    session_attributes = {}
    card_title = "Register With Twitch"
    speech_output = "Please register with your twitch account if you wish to get information about your followed streams"
    reprompt_text = None
    should_end_session = True

    return build_response(session_attributes,
                          build_login_card_response(card_title, speech_output, reprompt_text, should_end_session))

# ---------------------------- Events -------------------------------------------

def on_intent(request, session):
    intent_name = request['intent']['name']
    log_action("Intent ({})".format(intent_name), request, session)

    if intent_name == "GetMyStreams":
        return get_my_top_streamers(session)
    elif intent_name == "GetTopStreams":
        return get_top_streamers(session)
    elif intent_name == "GetTopStreamsByGame":
        return get_game_top_streamers(request, session)
    elif intent_name == "AMAZON.CancelIntent" or intent_name == "AMAZON.StopIntent":
        return get_end_response()
    else:
        print ("Invalid intent passed in: {}".format(intent_name))
        raise ValueError("Invalid intent: " + intent_name)

def on_launch(request, session):
    log_action("Launch", request, session)
    return get_welcome_response(session)

def on_end(request, session):
    log_action("End session", request, session)
    return get_end_response()

# ---------------------------- Logging ------------------------------------------
def log_action(action, request, session):
    print("{}: requestId={}, sessionId={}".format(action, request['requestId'], session['sessionId']))

# -------------------------- Main -----------------------------------------------
def lambda_handler(event, context):
    print("event.session.application.applicationId=" +
          event['session']['application']['applicationId'])

    # confirm the API call is from the expected skill
    # if (event['session']['application']['applicationId'] !=
    #         "amzn1.ask.skill.d84e53f8-eec2-48fd-a22e-f31f16177f8c"):
    #     raise ValueError("Invalid Application ID")

    if event['session']['new']:
        log_action("New Session", event['request'], event['session'])

    if event['request']['type'] == "LaunchRequest":
        return on_launch(event['request'], event['session'])
    elif event['request']['type'] == "IntentRequest":
        return on_intent(event['request'], event['session'])
    elif event['request']['type'] == "SessionEndedRequest":
        return on_end(event['request'], event['session'])
    else:
        raise ValueError("Invalid request: " + event['request']['type'])

# ------------------------- Game Name Conversion -----------------------
def get_twitch_game_name(game):
    name_dict = {}
    with open("twitchGameNameConversions.txt") as f:
        for line in f:
            (key, val) = line.split(",")
            name_dict[key.lower().strip()] = val.strip()

    if game.lower() in name_dict:
        return name_dict[game.lower().strip()]
    else:
        return None<commit_msg>cleaned up the communication with user / fixed bug where it would go into infinate loop<commit_after>import requests

# --------------- Helpers that build all of the responses ----------------------
# These helpers are from amazon

def build_speechlet_response(title, output, reprompt_text, should_end_session):
    return {
        'outputSpeech': {
            'type': 'PlainText',
            'text': output
        },
        'card': {
            'type': 'Simple',
            'title': "My Twitch Streams - " + title,
            'content': output
        },
        'reprompt': {
            'outputSpeech': {
                'type': 'PlainText',
                'text': reprompt_text
            }
        },
        'shouldEndSession': should_end_session
    }

def build_login_card_response(title, output, reprompt_text, should_end_session):
    return {
        'outputSpeech': {
            'type': 'PlainText',
            'text': output
        },
        'card': {
            'type': 'LinkAccount',
            'title': "My Twitch Streams - " + title,
            'content': output
        },
        'reprompt': {
            'outputSpeech': {
                'type': 'PlainText',
                'text': reprompt_text
            }
        },
        'shouldEndSession': should_end_session
    }

def build_response(session_attributes, speechlet_response):
    return {
        'version': '1.0',
        'sessionAttributes': session_attributes,
        'response': speechlet_response
    }

# ---------------------- Twitch Auth Helpers--------------------------------------
def get_twitch_auth(session):
    if 'accessToken' in session['user']:
        return session['user']['accessToken']
    else:
        return None

# ------------------------Twitch API Helpers -------------------------------------
def twitch_top_streamers(num_of_responses):
    api_info = get_twitch_api_info()
    url = api_info['base_api_url'] + api_info['top_streams']
    headers = {"Accept": "application/json", "Client-ID": api_info['Client-ID']}

    api_response = requests.get(url, headers=headers)
    json = api_response.json()

    streams = ""
    for num in range(0, num_of_responses):
        if streams != "":
            streams = streams + ", "
        streams = streams + json['streams'][num]['channel']['display_name']

    return streams

def twitch_my_top_streamers(num_of_responses, auth_info):
    api_info = get_twitch_api_info()
    url = api_info['base_api_url'] + api_info['my_top_streams']
    headers = {"Accept": "application/json",
               "Client-ID": api_info['Client-ID'],
               "Authorization":"OAuth " + auth_info}

    api_response = requests.get(url, headers=headers)
    json = api_response.json()

    if 'streams' not in json:
        return None

    streams = ""
    i = 0

    while i < num_of_responses and i < len(json['streams']):
        if streams != "":
            streams = streams + ", "
        streams = streams + json['streams'][i]['channel']['display_name']
        i = i + 1
    return streams

def twitch_game_top_streamers(num_of_responses, game):
    api_info = get_twitch_api_info()
    url = api_info['base_api_url'] + api_info['top_game_streams'].format(game)
    headers = {"Accept": "application/json", "Client-ID": api_info['Client-ID']}

    api_response = requests.get(url, headers=headers)
    json = api_response.json()

    streams = ""
    for num in range(0, num_of_responses):
        if streams != "":
            streams = streams + ", "
        streams = streams + json['streams'][num]['channel']['display_name']

    return streams

def get_twitch_api_info():
    api_info = {
        "Client-ID": "fq5m7globoglp4zanbya8c4r5q8e1i",
        "base_api_url": "https://api.twitch.tv/kraken/",
        "top_streams": "streams",
        "top_game_streams": "streams/?game={}",
        "my_top_streams": "streams/followed"
    }
    return api_info

# --------------- Functions that modify behavior --------------------------------

def get_welcome_response(session):
    if get_twitch_auth(session) is None:
        return get_login_card()

    session_attributes = {}
    card_title = "Welcome to the My Twitch Streams Alexa Skill"
    speech_output = "Welcome to the My Twitch streams skill. " \
                    "To get your followed live streams say ask {0} who is streaming. " \
                    "To get the top streams right now say ask {0} what are the top streams. " \
                    "To get the top streams by game say ask {0} who is streaming League of Legends"\
        .format(get_skill_invocation_name())
    reprompt_text = None
    should_end_session = True

    return build_response(session_attributes,
                          build_speechlet_response(card_title, speech_output, reprompt_text, should_end_session))

def get_end_response():
    session_attributes = {}
    card_title = None
    speech_output = None
    reprompt_text = None
    should_end_session = True

    return build_response(session_attributes,
                          build_speechlet_response(card_title, speech_output, reprompt_text, should_end_session))

def get_top_streamers(session):
    num_of_streams = 5
    top_streamers = twitch_top_streamers(num_of_streams)

    session_attributes = {}
    card_title = "Top Twitch Streamers On Now"
    speech_output = "Here are the top {} streamers: {}".format(num_of_streams, top_streamers)
    reprompt_text = None
    should_end_session = True

    return build_response(session_attributes,
                          build_speechlet_response(card_title, speech_output, reprompt_text, should_end_session))

def get_my_top_streamers(session):
    num_of_streams = 5

    auth_info = get_twitch_auth(session)
    if auth_info is None:
        return get_login_card()

    my_top_streamers = twitch_my_top_streamers(num_of_streams, auth_info)
    session_attributes = {}
    card_title = "Your Twitch Streamers On Now"
    speech_output = "Here are your top {} streamers: {}".format(num_of_streams, my_top_streamers)
    reprompt_text = None
    should_end_session = True

    return build_response(session_attributes,
                          build_speechlet_response(card_title, speech_output, reprompt_text, should_end_session))

def get_game_top_streamers(request, session):
    num_of_streams = 5
    game = get_twitch_game_name(request['intent']['slots']['Game']['value'])

    if game is None:
        return get_invalid_game_response()

    streamers = twitch_game_top_streamers(num_of_streams, game)
    session_attributes = {}
    card_title = "Top Twitch Streamers On Now For {}".format(game)
    speech_output = "Here are the top {} streamers for {}: {}".format(num_of_streams, game, streamers)
    reprompt_text = None
    should_end_session = True

    return build_response(session_attributes,
                          build_speechlet_response(card_title, speech_output, reprompt_text, should_end_session))

def get_invalid_game_response():
    session_attributes = {}
    card_title = "Unknown game"
    speech_output = "Unknown game, please try again"
    reprompt_text = None
    should_end_session = True

    return build_response(session_attributes,
                          build_speechlet_response(card_title, speech_output, reprompt_text, should_end_session))

def get_login_card():
    session_attributes = {}
    card_title = "Register With Twitch"
    speech_output = "Thank you for using my twitch streams. " \
                    "Please register with your twitch account if you wish to get information about your followed streams." \
                    "You can still get the top streams by saying ask {0} what are the top streams," \
                    "or get the top streams by game by saying ask {0} who is streaming League of Legends"\
        .format(get_skill_invocation_name())
    reprompt_text = None
    should_end_session = True

    return build_response(session_attributes,
                          build_login_card_response(card_title, speech_output, reprompt_text, should_end_session))

def get_skill_invocation_name():
    return "twitch"
# ---------------------------- Events -------------------------------------------

def on_intent(request, session):
    intent_name = request['intent']['name']
    log_action("Intent ({})".format(intent_name), request, session)

    if intent_name == "GetMyStreams":
        return get_my_top_streamers(session)
    elif intent_name == "GetTopStreams":
        return get_top_streamers(session)
    elif intent_name == "GetTopStreamsByGame":
        return get_game_top_streamers(request, session)
    elif intent_name == "AMAZON.CancelIntent" or intent_name == "AMAZON.StopIntent":
        return get_end_response()
    else:
        print ("Invalid intent passed in: {}".format(intent_name))
        raise ValueError("Invalid intent: " + intent_name)

def on_launch(request, session):
    log_action("Launch", request, session)
    return get_welcome_response(session)

def on_end(request, session):
    log_action("End session", request, session)
    return get_end_response()

# ---------------------------- Logging ------------------------------------------
def log_action(action, request, session):
    print("{}: requestId={}, sessionId={}".format(action, request['requestId'], session['sessionId']))

# -------------------------- Main -----------------------------------------------
def lambda_handler(event, context):
    print("event.session.application.applicationId=" +
          event['session']['application']['applicationId'])

    # confirm the API call is from the expected skill
    # if (event['session']['application']['applicationId'] !=
    #         "amzn1.ask.skill.d84e53f8-eec2-48fd-a22e-f31f16177f8c"):
    #     raise ValueError("Invalid Application ID")

    if event['session']['new']:
        log_action("New Session", event['request'], event['session'])

    if event['request']['type'] == "LaunchRequest":
        return on_launch(event['request'], event['session'])
    elif event['request']['type'] == "IntentRequest":
        return on_intent(event['request'], event['session'])
    elif event['request']['type'] == "SessionEndedRequest":
        return on_end(event['request'], event['session'])
    else:
        raise ValueError("Invalid request: " + event['request']['type'])

# ------------------------- Game Name Conversion -----------------------
def get_twitch_game_name(game):
    name_dict = {}
    with open("twitchGameNameConversions.txt") as f:
        for line in f:
            (key, val) = line.split(",")
            name_dict[key.lower().strip()] = val.strip()

    if game.lower() in name_dict:
        return name_dict[game.lower().strip()]
    else:
        return None<|endoftext|>


<commit_before>import argparse
import csv
import multiprocessing
import os
import sys
import time

import cloud_setup

# Change this AMI name to whatever AMI you have set up to run the computational tests
DEFAULT_AMI_NAME = "ami-42d42b54"

def create_instances(job, tags, instance_types, verbose=True):
    """
    Simply create an instance for each tag. Uses multiprocessing to create them
    in parallel.
    """

    if verbose:
        print "Launching instances... "
    procs = []
    returninfo = multiprocessing.Queue()
    for tag, instance_type in zip(tags, instance_types):
        if verbose:
            print "  Launching %s ..." % tag

        proc = multiprocessing.Process(
            target=cloud_setup.launch_instance,
            args=(),
            kwargs={
                "tag": tag,
                "key_name": job,
                "group_name": job,
                "inst_type": instance_type,
                "ami_name": DEFAULT_AMI_NAME,
                "user_data": "",
                "wait": True,
                "returninfo": returninfo,
            }
        )
        procs.append(proc)
        proc.start()

    # Wait for all instances to be launched
    for proc in procs:
        proc.join()

    # Count number of processes that have launched
    numstarted = 0
    try:
        while True:
            returninfo.get(False)
            numstarted += 1
    except:
        pass  # Queue is empty

    # Check we all started correctly
    if numstarted != len(tags):
        print "Exiting because", numstarted, "instances started out of",
        print len(tags)
        exit(0)

    if verbose:
        print " All instances launched, but not necessarily ready for"
        print " SSH though - check AWS console to get a better idea."
        print " Hit [RETURN] to proceed to connection attempt."
        raw_input()

def connect_instances(job, tags, verbose=True):
    """
    Connect to the instances. Returns, for every tag, an instance handle and
    a cmdshell handle through which we can execute commands and send and
    receive files.
    Returns:
      insts        Dictionary of tag -> instance
      cmds         Dictionary of tag -> cmdshell
    """

    insts = {}
    cmds = {}
    for tag in tags:
        cmds[tag] = None
    all_done = False
    while not all_done:
        print "Beginning round of connection attempts..."
        all_done = True
        for tag in tags:
            if cmds[tag] is None:
                if verbose:
                    print "  %s" % tag
                # Test if connected
                try:
                    insts[tag], cmds[tag] = cloud_setup.connect_instance(
                        tag=tag,
                        key_name=job,
                        user_name="ubuntu"
                    )
                    cmds[tag].run("ls")
                except:
                    all_done = False
                    cmds[tag] = None

    if verbose:
        print " All connections established, hit [RETURN]"
        raw_input()
    return insts, cmds

def setup_instances(tags, cmds, insts, verbose=True):
    """
    Install dependencies and build so it is ready for the run. This takes
    a while so after copying the files we just fire off a script.
    multiprocessing didn't work nicely with this...
    """

    # Locate the .boto file
    if os.path.exists(".boto"):
        botoloc = ".boto"
    elif os.path.exists(os.path.expanduser("~/.boto")):
        botoloc = os.path.expanduser("~/.boto")
    else:
        print "Could not locate .boto file"
        exit(1)

    if verbose:
        print "Copying keys to instances"

    for tag in tags:
        print "    Copying files to %s ..." % (tag)
        f = cmds[tag].open_sftp()
        f.put(botoloc, ".boto")
        f.close()

        cmds[tag].run("cd ~/PajaritoSupplement; git pull; cd -; touch READY")

    print "    Waiting for all machines"
    while True:
        time.sleep(1)
        done = [cmds[tag].run("ls .")[1].find("READY") >= 0 for tag in tags]
        print "      - Ready on",
        print sum(done), "/", len(done), "boxes"
        if sum(done) == len(done):
            break

    if verbose:
        print " Hit [RETURN] when ready to proceed."
        raw_input()

def dispatch_and_run(job, tags, cmds, commands, verbose=True):
    """
    Spawn the relevant command on each instance
    """
    # Write out and copy to instances
    if verbose:
        print "Writing args file and starting run... "

    for tag, command in zip(tags, commands):
        if verbose:
            print " %s" % tag

        # Make a shell script to run the command and then save the results
        runner_path = "~/runner_%s.sh" % tag
        with open(runner_path, "w") as f:
            f.write("export TAG=%s" % tag)  # Inject tag as environment var
            f.write("\n")
            f.write(command)
            f.write("\n")
            f.write("python ~/PajaritoSupplement/awsscripts/save_results.py %s %s" % (job, tag))

        # Put runner to server
        f = cmds[tag].open_sftp()
        f.put(runner_path, "runner.sh")
        f.close()

        # Cleanup
        try:
            os.remove(runner_path)
        except:
            pass

        cmds[tag].run("chmod +x runner.sh")

        cmds[tag]._ssh_client.exec_command(
            "nohup bash runner.sh &> screen_output.txt &"
        )

    if verbose:
        print "\n  Computation started on all machines"

def extract_job_details(jobfile):
    commands = []
    instance_types = []
    with open(jobfile, "rU") as f:
        reader = csv.reader(f)
        header_line = reader.next()
        if header_line[0] != "instance_type" or header_line[1] != "command":
            print "Error reading specified jobfile: %s" % jobfile
            print ""
            print "Make sure the jobfile is a csv file with instance types "
            print "in the first column and commands in the second."
            print "The headers should be 'instance_type' and 'command'."
            exit(1)

        for line in reader:
            if len(line) > 2:
                print "Error reading specified jobfile: %s" % jobfile
                print ""
                print "Make sure the jobfile is a csv file with instance types "
                print "in the first column and commands in the second."
                print "A row had more than two columns."
                exit(1)
            instance_types.append(line[0])
            commands.append(line[1])
    return commands, instance_types

def run_dispatch(job, commands, instance_types, create,
                 dispatch, verbose, tag_offset):
    """
    Setup machines, run jobs, monitor, then tear them down again.
    """

    if (not len(commands) == len(instance_types)):
        print "Different number of commands and instance types"
        exit(1)

    # Validate that required files exist
    if (not os.path.exists(".boto") and
            not os.path.exists(os.path.expanduser("~/.boto"))):
        print "Please create a .boto file containing your AWS credentials as",
        print "described in README.md. Store this file either in the "
        print "aws-runner folder or in your home directory."
        exit(1)

    tags = ["%s%d" % (job, i + tag_offset) for i in range(len(commands))]

    print "Overview for job %s" % job
    for tag, inst_type, command in zip(tags, instance_types, commands):
        print "   %s%s%s" % (tag.ljust(20), inst_type.ljust(20), command)

    # Setup security group and key pair (these are no-ops if done before)
    cloud_setup.create_security_group(job)
    cloud_setup.create_keypair(job)

    # Do some work upfront (clean out ~/.ssh/known_hosts and wait until all
    # shutting down nodes are shut down).
    cloud_setup.clean_known_hosts()
    cloud_setup.wait_for_shutdown()

    # Create instances if desired
    if create:
        create_instances(job, tags, instance_types, verbose)

    # Connect to all the instances
    if create or dispatch:
        insts, cmds = connect_instances(job, tags, verbose)

    # Set them up (if desired)
    if create:
        setup_instances(tags, cmds, insts, verbose)

    # Send out jobs and start machines working (if desired)
    if dispatch:
        dispatch_and_run(job, tags, cmds, commands, verbose)

    print ""
    print "All dispatcher tasks successfully completed."

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="A helper script to dispatch computation to AWS. "
                    "Please see the README for usage instructions."
    )
    parser.add_argument("jobname", type=str,
                        help="A descriptive name for this job.")
    parser.add_argument("jobfile", type=str,
                        help="Path to the CSV file containing job info.")
    parser.add_argument("-c", "--create", action="store_true",
                        help="Whether to create AWS instances for the jobs.")
    parser.add_argument("-d", "--dispatch", action="store_true",
                        help="Whether to dispatch the jobs to AWS instances.")
    parser.add_argument("-v", "--verbose", action="store_true",
                        help="Output extra progress messages (recommended).")
    parser.add_argument("--tag_offset", type=int, default=0,
                        help="Where to start numbering the machines from. "
                             "Use when you already have machines running for "
                             "this job and want to add more. You should set "
                             "this to a number that is greater than the tag "
                             "number of all currently running machines. "
                             "Defaults to 0.")
    args = parser.parse_args()

    jobname = args.jobname
    jobfile = args.jobfile
    create = args.create
    dispatch = args.dispatch
    verbose = args.verbose
    tag_offset = args.tag_offset

    print "AMI name:"
    print DEFAULT_AMI_NAME
    print ""

    commands, instance_types = extract_job_details(jobfile)

    run_dispatch(jobname, commands, instance_types,
                 create, dispatch, verbose, tag_offset)
<commit_msg>testing aws<commit_after>import argparse
import csv
import multiprocessing
import os
import sys
import time

import cloud_setup

# Change this AMI name to whatever AMI you have set up to run the computational tests
DEFAULT_AMI_NAME = "ami-42d42b54"

def create_instances(job, tags, instance_types, verbose=True):
    """
    Simply create an instance for each tag. Uses multiprocessing to create them
    in parallel.
    """
s
    if verbose:
        print "Launching instances... "
    procs = []
    returninfo = multiprocessing.Queue()
    for tag, instance_type in zip(tags, instance_types):
        if verbose:
            print "  Launching %s ..." % tag

        proc = multiprocessing.Process(
            target=cloud_setup.launch_instance,
            args=(),
            kwargs={
                "tag": tag,
                "key_name": job,
                "group_name": job,
                "inst_type": instance_type,
                "ami_name": DEFAULT_AMI_NAME,
                "user_data": "",
                "wait": True,
                "returninfo": returninfo,
            }
        )
        procs.append(proc)
        proc.start()

    # Wait for all instances to be launched
    for proc in procs:
        proc.join()

    # Count number of processes that have launched
    numstarted = 0
    try:
        while True:
            returninfo.get(False)
            numstarted += 1
    except:
        pass  # Queue is empty

    # Check we all started correctly
    if numstarted != len(tags):
        print "Exiting because", numstarted, "instances started out of",
        print len(tags)
        exit(0)

    if verbose:
        print " All instances launched, but not necessarily ready for"
        print " SSH though - check AWS console to get a better idea."
        print " Hit [RETURN] to proceed to connection attempt."
        raw_input()

def connect_instances(job, tags, verbose=True):
    """
    Connect to the instances. Returns, for every tag, an instance handle and
    a cmdshell handle through which we can execute commands and send and
    receive files.
    Returns:
      insts        Dictionary of tag -> instance
      cmds         Dictionary of tag -> cmdshell
    """

    insts = {}
    cmds = {}
    for tag in tags:
        cmds[tag] = None
    all_done = False
    while not all_done:
        print "Beginning round of connection attempts..."
        all_done = True
        for tag in tags:
            if cmds[tag] is None:
                if verbose:
                    print "  %s" % tag
                # Test if connected
                try:
                    insts[tag], cmds[tag] = cloud_setup.connect_instance(
                        tag=tag,
                        key_name=job,
                        user_name="ubuntu"
                    )
                    cmds[tag].run("ls")
                except:
                    all_done = False
                    cmds[tag] = None

    if verbose:
        print " All connections established, hit [RETURN]"
        raw_input()
    return insts, cmds

def setup_instances(tags, cmds, insts, verbose=True):
    """
    Install dependencies and build so it is ready for the run. This takes
    a while so after copying the files we just fire off a script.
    multiprocessing didn't work nicely with this...
    """

    # Locate the .boto file
    if os.path.exists(".boto"):
        botoloc = ".boto"
    elif os.path.exists(os.path.expanduser("~/.boto")):
        botoloc = os.path.expanduser("~/.boto")
    else:
        print "Could not locate .boto file"
        exit(1)

    if verbose:
        print "Copying keys to instances"

    for tag in tags:
        print "    Copying files to %s ..." % (tag)
        f = cmds[tag].open_sftp()
        f.put(botoloc, ".boto")
        f.close()

        cmds[tag].run("cd ~/PajaritoSupplement; git pull; cd -; touch READY")

    print "    Waiting for all machines"
    while True:
        time.sleep(1)
        done = [cmds[tag].run("ls .")[1].find("READY") >= 0 for tag in tags]
        print "      - Ready on",
        print sum(done), "/", len(done), "boxes"
        if sum(done) == len(done):
            break

    if verbose:
        print " Hit [RETURN] when ready to proceed."
        raw_input()

def dispatch_and_run(job, tags, cmds, commands, verbose=True):
    """
    Spawn the relevant command on each instance
    """
    # Write out and copy to instances
    if verbose:
        print "Writing args file and starting run... "

    for tag, command in zip(tags, commands):
        if verbose:
            print " %s" % tag

        # Make a shell script to run the command and then save the results
        runner_path = "runner_%s.sh" % tag
        with open(runner_path, "w") as f:
            f.write("export TAG=%s" % tag)  # Inject tag as environment var
            f.write("\n")
            f.write(command)
            f.write("\n")
            f.write("python ~/PajaritoSupplement/awsscripts/save_results.py %s %s" % (job, tag))

        # Put runner to server
        f = cmds[tag].open_sftp()
        f.put(runner_path, "runner.sh")
        f.close()

        # Cleanup
        try:
            os.remove(runner_path)
        except:
            pass

        cmds[tag].run("chmod +x runner.sh")

        cmds[tag]._ssh_client.exec_command(
            "nohup bash runner.sh &> screen_output.txt &"
        )

    if verbose:
        print "\n  Computation started on all machines"

def extract_job_details(jobfile):
    commands = []
    instance_types = []
    with open(jobfile, "rU") as f:
        reader = csv.reader(f)
        header_line = reader.next()
        if header_line[0] != "instance_type" or header_line[1] != "command":
            print "Error reading specified jobfile: %s" % jobfile
            print ""
            print "Make sure the jobfile is a csv file with instance types "
            print "in the first column and commands in the second."
            print "The headers should be 'instance_type' and 'command'."
            exit(1)

        for line in reader:
            if len(line) > 2:
                print "Error reading specified jobfile: %s" % jobfile
                print ""
                print "Make sure the jobfile is a csv file with instance types "
                print "in the first column and commands in the second."
                print "A row had more than two columns."
                exit(1)
            instance_types.append(line[0])
            commands.append(line[1])
    return commands, instance_types

def run_dispatch(job, commands, instance_types, create,
                 dispatch, verbose, tag_offset):
    """
    Setup machines, run jobs, monitor, then tear them down again.
    """

    if (not len(commands) == len(instance_types)):
        print "Different number of commands and instance types"
        exit(1)

    # Validate that required files exist
    if (not os.path.exists(".boto") and
            not os.path.exists(os.path.expanduser("~/.boto"))):
        print "Please create a .boto file containing your AWS credentials as",
        print "described in README.md. Store this file either in the "
        print "aws-runner folder or in your home directory."
        exit(1)

    tags = ["%s%d" % (job, i + tag_offset) for i in range(len(commands))]

    print "Overview for job %s" % job
    for tag, inst_type, command in zip(tags, instance_types, commands):
        print "   %s%s%s" % (tag.ljust(20), inst_type.ljust(20), command)

    # Setup security group and key pair (these are no-ops if done before)
    cloud_setup.create_security_group(job)
    cloud_setup.create_keypair(job)

    # Do some work upfront (clean out ~/.ssh/known_hosts and wait until all
    # shutting down nodes are shut down).
    cloud_setup.clean_known_hosts()
    cloud_setup.wait_for_shutdown()

    # Create instances if desired
    if create:
        create_instances(job, tags, instance_types, verbose)

    # Connect to all the instances
    if create or dispatch:
        insts, cmds = connect_instances(job, tags, verbose)

    # Set them up (if desired)
    if create:
        setup_instances(tags, cmds, insts, verbose)

    # Send out jobs and start machines working (if desired)
    if dispatch:
        dispatch_and_run(job, tags, cmds, commands, verbose)

    print ""
    print "All dispatcher tasks successfully completed."

if __name__ == "__main__":
    parser = argparse.ArgumentParser(
        description="A helper script to dispatch computation to AWS. "
                    "Please see the README for usage instructions."
    )
    parser.add_argument("jobname", type=str,
                        help="A descriptive name for this job.")
    parser.add_argument("jobfile", type=str,
                        help="Path to the CSV file containing job info.")
    parser.add_argument("-c", "--create", action="store_true",
                        help="Whether to create AWS instances for the jobs.")
    parser.add_argument("-d", "--dispatch", action="store_true",
                        help="Whether to dispatch the jobs to AWS instances.")
    parser.add_argument("-v", "--verbose", action="store_true",
                        help="Output extra progress messages (recommended).")
    parser.add_argument("--tag_offset", type=int, default=0,
                        help="Where to start numbering the machines from. "
                             "Use when you already have machines running for "
                             "this job and want to add more. You should set "
                             "this to a number that is greater than the tag "
                             "number of all currently running machines. "
                             "Defaults to 0.")
    args = parser.parse_args()

    jobname = args.jobname
    jobfile = args.jobfile
    create = args.create
    dispatch = args.dispatch
    verbose = args.verbose
    tag_offset = args.tag_offset

    print "AMI name:"
    print DEFAULT_AMI_NAME
    print ""

    commands, instance_types = extract_job_details(jobfile)

    run_dispatch(jobname, commands, instance_types,
                 create, dispatch, verbose, tag_offset)
<|endoftext|>


<commit_before>from mcapp import app
from decorators import crossdomain, apikey, jsonp
from flask import request, g
import rethinkdb as r
import error
import dmutil
import access
import args

@app.route('/processes/project/<project_id>', methods=['GET'])
def get_processes_for_project(project_id):
    processes = get_processes(project_id)
    return dmutil.jsoner(processes)

def get_processes(project_id):
    processes = list(r.table("processes")
                     .get_all(project_id, index="project_id")
                     .run(g.conn, time_format="raw"))
    # For fast lookup put processes in a map
    by_ids = {p['id']: p for p in processes}

    # Add additional fields to fill in for the properties
    # and upstream/downstream processes
    for id, process in by_ids.iteritems():
        process['inputs'] = {}
        process['outputs'] = {}
        process['upstream'] = {}
        process['downstream'] = {}

    all_property_sets = get_property_sets(by_ids)
    add_process_properties(all_property_sets, by_ids)
    connect_feeders(all_property_sets, by_ids)
    return processes

def get_property_sets(processes):
    # Get all properties for all of the processes
    all = list(r.table("property_sets")
               .get_all(*processes.keys(), index="item_id")
               .eq_join("id", r.table("properties"), index="item_id")
               .map(lambda row: row.merge({
                   "left": {
                       "ps_id": row["left"]["id"],
                       "ps_name": row["left"]["name"],
                       "ps_process_id": row["left"]["item_id"],
                       "ps_item_type": row["left"]["item_type"]
                   }
               }))
               .without({
                   "left": {
                       "id": True,
                       "name": True,
                       "item_id": True,
                       "item_type": True
                   }
               })
               .zip().run(g.conn, time_format="raw"))
    return all

def add_process_properties(property_sets, processes):
    # Now go through each property set and add the property set
    # and its properties to the relevant process
    for property_set in property_sets:
        process_id = property_set["ps_process_id"]
        process = processes[process_id]
        add_property_set(property_set, process)

def add_property_set(property_set, process):
    ps_name = property_set["ps_name"]
    stype = property_set["stype"]
    if ps_name not in process[stype]:
        process[stype][ps_name] = []
    process[stype][ps_name].append({
        "unit": property_set["unit"],
        "other": property_set["other"],
        "display_name": property_set["display_name"],
        "name": property_set["name"],
        "ptype": property_set["ptype"],
        "value": property_set["value"]
    })

def connect_feeders(property_sets, processes):
    # Get the ids of all input/output properties that
    # could go into or come from another process
    inputs2processes, outputs2processes = get_feeders(property_sets)
    attach_feeders(inputs2processes, outputs2processes, processes)

def get_feeders(property_sets):
    feeder_ids = [prop['value'] for prop in property_sets
                  if prop["ptype"] == "file" or prop["ptype"] == "sample"]
    query = r.table("properties")\
             .get_all(*feeder_ids, index="value")\
             .eq_join("item_id", r.table("property_sets"))\
             .zip()\
             .eq_join("item_id", r.table("processes"))\
             .zip()
    i2p = {}
    o2p = {}
    for p in query.run(g.conn):
        if p["stype"] == "inputs":
            use = i2p
        else:
            use = o2p
        id = p["value"]
        if p["value"] not in use:
            use[id] = {
                "ptype": p["ptype"],
                "name": p["other"]["name"],
                "processes": {}
            }
        pid = p["id"]
        use[id]["processes"][pid] = p["name"]
    return i2p, o2p

def attach_feeders(inputs2processes, outputs2processes, processes):
    attach_feeders_for_type(outputs2processes, "downstream", "inputs",
                            processes)
    attach_feeders_for_type(inputs2processes, "upstream", "outputs",
                            processes)

def attach_feeders_for_type(io2process, feeder_name, io_name, processes):
    for key, process in processes.iteritems():
        for key, io_item in process[io_name].iteritems():
            if key != "files" and key != "sample":
                continue
            for item in io_item:
                item_id = item["value"]
                if item_id in io2process:
                    for process_id in io2process[item_id]["processes"]:
                        if process_id != process["id"]:
                            p_item = io2process[item_id]
                            if process_id not in process[feeder_name]:
                                process[feeder_name][process_id] = {
                                    "name": p_item["processes"][process_id],
                                    "items": []
                                }
                            process[feeder_name][process_id]["items"].append({
                                "item_name": p_item["name"],
                                "item_type": p_item["ptype"],
                            })

@app.route('/processes/<process_id>', methods=['GET'])
@apikey(shared=True)
@jsonp
def get_process(process_id):
    process = dmutil.get_single_from_table('processes', process_id, raw=True)
    if process is None:
        return error.bad_request("Unknown process")
    result = build_sample_file_objects(process, '')
    return args.json_as_format_arg(result)

@app.route('/processes/template/<template_id>', methods=['GET'])
@apikey
@jsonp
def get_all_processes_for_template(template_id):
    rr = r.table('processes').filter({'template': template_id})
    selection = list(rr.run(g.conn, time_format='raw'))
    return args.json_as_format_arg(selection)

@app.route('/processes/sample/<sample_id>', methods=['GET'])
@apikey
@jsonp
def get_processes_by_sample(sample_id):
    rr = r.table('samples_processes_denorm').get_all(sample_id, index='sample_id')
    selection = list(rr.run(g.conn, time_format='raw'))
    return args.json_as_format_arg(selection)

@app.route('/processes/file/<file_id>', methods=['GET'])
@jsonp
def get_processes_by_file(file_id):
    rv = r.table('datafiles_denorm')\
          .filter({'df_id': file_id})\
          .eq_join('process_id', r.table('processes')).zip()
    selection = list(rv.run(g.conn, time_format='raw'))
    result = build_sample_file_objects(selection, 'array')
    return args.json_as_format_arg(result)

def build_sample_file_objects(selection, type):
    sample_list = []
    file_list = []
    file_objs = []
    if type == "array":
        for s in selection:
            inputs = s['inputs']
            outputs = s['outputs']
            for i in inputs:
                if i['attribute'] == 'sample':
                    sample_list.append(i['properties']['id']['value'])
                elif i['attribute'] == 'file':
                    file_list.append(i['properties']['id']['value'])
            for o in outputs:
                if o['attribute'] == 'sample':
                    sample_list.append(o['properties']['id']['value'])
                elif o['attribute'] == 'file':
                    file_list.append(o['properties']['id']['value'])
        if len(sample_list) != 0:
            rr = r.table('samples').get_all(*sample_list)
            sample_objs = list(rr.run(g.conn, time_format='raw'))
        if len(file_list) != 0:
            rr = r.table('datafiles').get_all(*file_list)
            file_objs = list(rr.run(g.conn, time_format='raw'))
        # stick the objects
        for s in selection:
            # get_all_sample_ids
            for i in inputs:
                if i['attribute'] == 'sample':
                    i['properties']['obj'] = get_an_item(i['properties']['id']['value'], sample_objs)
                elif i['attribute'] == 'file':
                    file_list.append(i['properties']['id']['value'])
                    i['properties']['obj'] = get_an_item(i['properties']['id']['value'], file_objs)
            for o in outputs:
                if o['attribute'] == 'sample':
                    o['properties']['obj'] = get_an_item(o['properties']['id']['value'], sample_objs)
                elif o['attribute'] == 'file':
                    file_list.append(o['properties']['id']['value'])
                    o['properties']['obj'] = get_an_item(o['properties']['id']['value'], file_objs)
        return selection
    else:
        s = selection
        inputs = s['inputs']
        outputs = s['outputs']
        for i in inputs:
            if i['attribute'] == 'sample':
                sample_list.append(i['properties']['id']['value'])
            elif i['attribute'] == 'file':
                    file_list.append(i['properties']['id']['value'])
            for o in outputs:
                if o['attribute'] == 'sample':
                    sample_list.append(o['properties']['id']['value'])
                elif o['attribute'] == 'file':
                    file_list.append(o['properties']['id']['value'])
        if len(sample_list) != 0:
            rr = r.table('samples').get_all(*sample_list)
            sample_objs = list(rr.run(g.conn, time_format='raw'))
        if len(file_list) != 0:
            rr = r.table('datafiles').get_all(*file_list)
            file_objs = list(rr.run(g.conn, time_format='raw'))
        # stick the objects
        for s in selection:
            # get_all_sample_ids
            for i in inputs:
                if i['attribute'] == 'sample':
                    i['properties']['obj'] = get_an_item(i['properties']['id']['value'], sample_objs)
                elif i['attribute'] == 'file':
                    file_list.append(i['properties']['id']['value'])
                    i['properties']['obj'] = get_an_item(i['properties']['id']['value'], file_objs)
            for o in outputs:
                if o['attribute'] == 'sample':
                    o['properties']['obj'] = get_an_item(o['properties']['id']['value'], sample_objs)
                elif o['attribute'] == 'file':
                    file_list.append(o['properties']['id']['value'])
                    o['properties']['obj'] = get_an_item(o['properties']['id']['value'], file_objs)
        return selection

def get_an_item(id, obj_list):
    for obj in obj_list:
        if obj['id'] == id:
            return obj

@app.route('/processes/new', methods=['POST'])
@apikey
@crossdomain(origin='*')
def create_process():
    p = dict()
    j = request.get_json()
    p['name'] = dmutil.get_required('name', j)
    user = access.get_user()
    p['owner'] = user
    p['project'] = dmutil.get_required('project', j)
    if not dmutil.item_exists('projects', p['project']):
        return error.not_acceptable("Unknown project id: %s" % (p['project']))
    p['birthtime'] = r.now()
    p['mtime'] = p['birthtime']
    p['machine'] = dmutil.get_optional('machine', j)
    p['process_type'] = dmutil.get_required('process_type', j)
    p['description'] = dmutil.get_required('description', j)
    p['version'] = dmutil.get_optional('version', j)
    p['notes'] = dmutil.get_optional('notes', j, [])
    p['input_conditions'] = dmutil.get_optional('input_conditions', j, [])
    p['input_files'] = dmutil.get_optional('input_files', j, [])
    p['output_conditions'] = dmutil.get_optional('output_conditions', j, [])
    p['output_files'] = dmutil.get_optional('output_files', j, [])
    p['runs'] = dmutil.get_optional('runs', j, [])
    p['citations'] = dmutil.get_optional('citations', j, [])
    p['status'] = dmutil.get_optional('status', j)
    id = dmutil.insert_entry('processes', p)
    return dmutil.jsoner_id(id)

@app.route('/process/update/<path:processid>', methods=['PUT'])
@apikey
@crossdomain(origin='*')
def update_process(processid):
    rv = r.table('processes').get(processid).update(request.json).run(g.conn)
    if (rv['replaced'] == 1 or rv['unchanged'] == 1):
        return ''
    else:
        error.update_conflict("Unable to update process: " + processid)
<commit_msg>Add @apikey call to get provenance process relationships. Fix up PEP-8 issues.<commit_after>from mcapp import app
from decorators import crossdomain, apikey, jsonp
from flask import request, g
import rethinkdb as r
import error
import dmutil
import access
import args

@app.route('/processes/project/<project_id>', methods=['GET'])
@apikey
def get_processes_for_project(project_id):
    processes = get_processes(project_id)
    return dmutil.jsoner(processes)

def get_processes(project_id):
    processes = list(r.table("processes")
                     .get_all(project_id, index="project_id")
                     .run(g.conn, time_format="raw"))
    # For fast lookup put processes in a map
    by_ids = {p['id']: p for p in processes}

    # Add additional fields to fill in for the properties
    # and upstream/downstream processes
    for id, process in by_ids.iteritems():
        process['inputs'] = {}
        process['outputs'] = {}
        process['upstream'] = {}
        process['downstream'] = {}

    all_property_sets = get_property_sets(by_ids)
    add_process_properties(all_property_sets, by_ids)
    connect_feeders(all_property_sets, by_ids)
    return processes

def get_property_sets(processes):
    # Get all properties for all of the processes
    all = list(r.table("property_sets")
               .get_all(*processes.keys(), index="item_id")
               .eq_join("id", r.table("properties"), index="item_id")
               .map(lambda row: row.merge({
                   "left": {
                       "ps_id": row["left"]["id"],
                       "ps_name": row["left"]["name"],
                       "ps_process_id": row["left"]["item_id"],
                       "ps_item_type": row["left"]["item_type"]
                   }
               }))
               .without({
                   "left": {
                       "id": True,
                       "name": True,
                       "item_id": True,
                       "item_type": True
                   }
               })
               .zip().run(g.conn, time_format="raw"))
    return all

def add_process_properties(property_sets, processes):
    # Now go through each property set and add the property set
    # and its properties to the relevant process
    for property_set in property_sets:
        process_id = property_set["ps_process_id"]
        process = processes[process_id]
        add_property_set(property_set, process)

def add_property_set(property_set, process):
    ps_name = property_set["ps_name"]
    stype = property_set["stype"]
    if ps_name not in process[stype]:
        process[stype][ps_name] = []
    process[stype][ps_name].append({
        "unit": property_set["unit"],
        "other": property_set["other"],
        "display_name": property_set["display_name"],
        "name": property_set["name"],
        "ptype": property_set["ptype"],
        "value": property_set["value"]
    })

def connect_feeders(property_sets, processes):
    # Get the ids of all input/output properties that
    # could go into or come from another process
    inputs2processes, outputs2processes = get_feeders(property_sets)
    attach_feeders(inputs2processes, outputs2processes, processes)

def get_feeders(property_sets):
    feeder_ids = [prop['value'] for prop in property_sets
                  if prop["ptype"] == "file" or prop["ptype"] == "sample"]
    query = r.table("properties")\
             .get_all(*feeder_ids, index="value")\
             .eq_join("item_id", r.table("property_sets"))\
             .zip()\
             .eq_join("item_id", r.table("processes"))\
             .zip()
    i2p = {}
    o2p = {}
    for p in query.run(g.conn):
        if p["stype"] == "inputs":
            use = i2p
        else:
            use = o2p
        id = p["value"]
        if p["value"] not in use:
            use[id] = {
                "ptype": p["ptype"],
                "name": p["other"]["name"],
                "processes": {}
            }
        pid = p["id"]
        use[id]["processes"][pid] = p["name"]
    return i2p, o2p

def attach_feeders(inputs2processes, outputs2processes, processes):
    attach_feeders_for_type(outputs2processes, "downstream", "inputs",
                            processes)
    attach_feeders_for_type(inputs2processes, "upstream", "outputs",
                            processes)

def attach_feeders_for_type(io2process, feeder_name, io_name, processes):
    for key, process in processes.iteritems():
        for key, io_item in process[io_name].iteritems():
            if key != "files" and key != "sample":
                continue
            for item in io_item:
                item_id = item["value"]
                if item_id in io2process:
                    for process_id in io2process[item_id]["processes"]:
                        if process_id != process["id"]:
                            p_item = io2process[item_id]
                            if process_id not in process[feeder_name]:
                                process[feeder_name][process_id] = {
                                    "name": p_item["processes"][process_id],
                                    "items": []
                                }
                            process[feeder_name][process_id]["items"].append({
                                "item_name": p_item["name"],
                                "item_type": p_item["ptype"],
                            })

@app.route('/processes/<process_id>', methods=['GET'])
@apikey(shared=True)
@jsonp
def get_process(process_id):
    process = dmutil.get_single_from_table('processes', process_id, raw=True)
    if process is None:
        return error.bad_request("Unknown process")
    result = build_sample_file_objects(process, '')
    return args.json_as_format_arg(result)

@app.route('/processes/template/<template_id>', methods=['GET'])
@apikey
@jsonp
def get_all_processes_for_template(template_id):
    rr = r.table('processes').filter({'template': template_id})
    selection = list(rr.run(g.conn, time_format='raw'))
    return args.json_as_format_arg(selection)

@app.route('/processes/sample/<sample_id>', methods=['GET'])
@apikey
@jsonp
def get_processes_by_sample(sample_id):
    rr = r.table('samples_processes_denorm').get_all(sample_id,
                                                     index='sample_id')
    selection = list(rr.run(g.conn, time_format='raw'))
    return args.json_as_format_arg(selection)

@app.route('/processes/file/<file_id>', methods=['GET'])
@jsonp
def get_processes_by_file(file_id):
    rv = r.table('datafiles_denorm')\
          .filter({'df_id': file_id})\
          .eq_join('process_id', r.table('processes')).zip()
    selection = list(rv.run(g.conn, time_format='raw'))
    result = build_sample_file_objects(selection, 'array')
    return args.json_as_format_arg(result)

def build_sample_file_objects(selection, type):
    sample_list = []
    file_list = []
    file_objs = []
    if type == "array":
        for s in selection:
            inputs = s['inputs']
            outputs = s['outputs']
            for i in inputs:
                if i['attribute'] == 'sample':
                    sample_list.append(i['properties']['id']['value'])
                elif i['attribute'] == 'file':
                    file_list.append(i['properties']['id']['value'])
            for o in outputs:
                if o['attribute'] == 'sample':
                    sample_list.append(o['properties']['id']['value'])
                elif o['attribute'] == 'file':
                    file_list.append(o['properties']['id']['value'])
        if len(sample_list) != 0:
            rr = r.table('samples').get_all(*sample_list)
            sample_objs = list(rr.run(g.conn, time_format='raw'))
        if len(file_list) != 0:
            rr = r.table('datafiles').get_all(*file_list)
            file_objs = list(rr.run(g.conn, time_format='raw'))
        # stick the objects
        for s in selection:
            # get_all_sample_ids
            for i in inputs:
                item = i["properties"]["id"]["value"]
                if i['attribute'] == 'sample':
                    i['properties']['obj'] = get_an_item(item, sample_objs)
                elif i['attribute'] == 'file':
                    file_list.append(item)
                    i['properties']['obj'] = get_an_item(item, file_objs)
            for o in outputs:
                item = o["properties"]["id"]["value"]
                if o['attribute'] == 'sample':
                    o['properties']['obj'] = get_an_item(item, sample_objs)
                elif o['attribute'] == 'file':
                    file_list.append(item)
                    o['properties']['obj'] = get_an_item(item, file_objs)
        return selection
    else:
        s = selection
        inputs = s['inputs']
        outputs = s['outputs']
        for i in inputs:
            if i['attribute'] == 'sample':
                sample_list.append(i['properties']['id']['value'])
            elif i['attribute'] == 'file':
                    file_list.append(i['properties']['id']['value'])
            for o in outputs:
                if o['attribute'] == 'sample':
                    sample_list.append(o['properties']['id']['value'])
                elif o['attribute'] == 'file':
                    file_list.append(o['properties']['id']['value'])
        if len(sample_list) != 0:
            rr = r.table('samples').get_all(*sample_list)
            sample_objs = list(rr.run(g.conn, time_format='raw'))
        if len(file_list) != 0:
            rr = r.table('datafiles').get_all(*file_list)
            file_objs = list(rr.run(g.conn, time_format='raw'))
        # stick the objects
        for s in selection:
            # get_all_sample_ids
            for i in inputs:
                item = i["properties"]["id"]["value"]
                if i['attribute'] == 'sample':
                    i['properties']['obj'] = get_an_item(item, sample_objs)
                elif i['attribute'] == 'file':
                    file_list.append(i['properties']['id']['value'])
                    i['properties']['obj'] = get_an_item(item, file_objs)
            for o in outputs:
                item = o["properties"]["id"]["value"]
                if o['attribute'] == 'sample':
                    o['properties']['obj'] = get_an_item(item, sample_objs)
                elif o['attribute'] == 'file':
                    file_list.append(item)
                    o['properties']['obj'] = get_an_item(item, file_objs)
        return selection

def get_an_item(id, obj_list):
    for obj in obj_list:
        if obj['id'] == id:
            return obj

@app.route('/processes/new', methods=['POST'])
@apikey
@crossdomain(origin='*')
def create_process():
    p = dict()
    j = request.get_json()
    p['name'] = dmutil.get_required('name', j)
    user = access.get_user()
    p['owner'] = user
    p['project'] = dmutil.get_required('project', j)
    if not dmutil.item_exists('projects', p['project']):
        return error.not_acceptable("Unknown project id: %s" % (p['project']))
    p['birthtime'] = r.now()
    p['mtime'] = p['birthtime']
    p['machine'] = dmutil.get_optional('machine', j)
    p['process_type'] = dmutil.get_required('process_type', j)
    p['description'] = dmutil.get_required('description', j)
    p['version'] = dmutil.get_optional('version', j)
    p['notes'] = dmutil.get_optional('notes', j, [])
    p['input_conditions'] = dmutil.get_optional('input_conditions', j, [])
    p['input_files'] = dmutil.get_optional('input_files', j, [])
    p['output_conditions'] = dmutil.get_optional('output_conditions', j, [])
    p['output_files'] = dmutil.get_optional('output_files', j, [])
    p['runs'] = dmutil.get_optional('runs', j, [])
    p['citations'] = dmutil.get_optional('citations', j, [])
    p['status'] = dmutil.get_optional('status', j)
    id = dmutil.insert_entry('processes', p)
    return dmutil.jsoner_id(id)

@app.route('/process/update/<path:processid>', methods=['PUT'])
@apikey
@crossdomain(origin='*')
def update_process(processid):
    rv = r.table('processes').get(processid).update(request.json).run(g.conn)
    if (rv['replaced'] == 1 or rv['unchanged'] == 1):
        return ''
    else:
        error.update_conflict("Unable to update process: " + processid)
<|endoftext|>


<commit_before>import socket
import sys
import threading
import time
import traceback

from processor import Processor
from utils import Hash, print_log
from version import VERSION

class IrcThread(threading.Thread):

    def __init__(self, processor, config):
        threading.Thread.__init__(self)

        self.processor = processor
        self.daemon = True
        self.stratum_tcp_port = config.get('server', 'stratum_tcp_port')
        self.stratum_http_port = config.get('server', 'stratum_http_port')
        self.stratum_tcp_ssl_port = config.get('server', 'stratum_tcp_ssl_port')
        self.stratum_http_ssl_port = config.get('server', 'stratum_http_ssl_port')
        self.report_stratum_tcp_port = config.get('server', 'report_stratum_tcp_port')
        self.report_stratum_http_port = config.get('server', 'report_stratum_http_port')
        self.report_stratum_tcp_ssl_port = config.get('server', 'report_stratum_tcp_ssl_port')
        self.report_stratum_http_ssl_port = config.get('server', 'report_stratum_http_ssl_port')
        self.peers = {}
        self.host = config.get('server', 'host')
        self.report_host = config.get('server', 'report_host')
        self.nick = config.get('server', 'irc_nick')
        if self.report_stratum_tcp_port:
            self.stratum_tcp_port = self.report_stratum_tcp_port
        if self.report_stratum_http_port:
            self.stratum_http_port = self.report_stratum_http_port
        if self.report_stratum_tcp_ssl_port:
            self.stratum_tcp_ssl_port = self.report_stratum_tcp_ssl_port
        if self.report_stratum_http_ssl_port:
            self.stratum_http_ssl_port = self.report_stratum_http_ssl_port
        if self.report_host:
            self.host = self.report_host
        if not self.nick:
            self.nick = Hash(self.host)[:5].encode("hex")
        self.prepend = 'E_'
        if config.get('server', 'coin') == 'litecoin':
            self.prepend = 'EL_'
        self.pruning = config.get('server', 'backend') == 'leveldb'
        if self.pruning:
            self.pruning_limit = config.get('leveldb', 'pruning_limit')
        self.nick = self.prepend + self.nick

    def get_peers(self):
        return self.peers.values()

    def getname(self):
        s = 'v' + VERSION + ' '
        if self.pruning:
            s += 'p' + self.pruning_limit + ' '

        def add_port(letter, number):
            DEFAULT_PORTS = {'t':'50001', 's':'50002', 'h':'8081', 'g':'8082'}
            if not number: return ''
            if DEFAULT_PORTS[letter] == number:
                return letter + ' '
            else:
                return letter + number + ' '

        s += add_port('t',self.stratum_tcp_port)
        s += add_port('h',self.stratum_http_port)
        s += add_port('s',self.stratum_tcp_ssl_port)
        s += add_port('g',self.stratum_http_ssl_port)
        return s

    def run(self):
        ircname = self.getname()
        print_log("joining IRC")

        while not self.processor.shared.stopped():
            try:
                s = socket.socket()
                s.connect(('irc.freenode.net', 6667))
                s.settimeout(0.1)
            except:
                s.close()
                print_log("IRC: reconnect in 10 s")
                time.sleep(10)
                continue

            self.message = ''
            out_msg = []

            try:
                s.send('USER electrum 0 * :' + self.host + ' ' + ircname + '\n')
                s.send('NICK ' + self.nick + '\n')
                s.send('JOIN #electrum\n')
                t = 0

                while not self.processor.shared.stopped():
                    try:
                        data = s.recv(2048)
                    except socket.timeout, e:
                        if out_msg:
                            m = out_msg.pop(0)
                            s.send(m)
                        continue
                    except:
                        print_log( "irc: socket error" )
                        time.sleep(1)
                        break

                    self.message += data

                    while self.message.find('\n') != -1:
                        pos = self.message.find('\n')
                        line = self.message[0:pos]
                        self.message = self.message[pos+1:]
                        line = line.strip('\r')
                        if not line:
                            continue
                        # print_log("<--", line)
                        line = line.split()
                        if line[0] == 'PING':
                            out_msg.append('PONG ' + line[1] + '\n')
                        elif '353' in line:  # answer to /names
                            k = line.index('353')
                            for item in line[k+1:]:
                                if item.startswith(self.prepend):
                                    out_msg.append('WHO %s\n' % item)
                        elif '352' in line:  # answer to /who
                            # warning: this is a horrible hack which apparently works
                            k = line.index('352')
                            try:
                                ip = socket.gethostbyname(line[k+4])
                            except:
                                print_log("gethostbyname error", line[k+4])
                                continue
                            name = line[k+6]
                            host = line[k+9]
                            ports = line[k+10:]
                            self.peers[name] = (ip, host, ports)

                    if time.time() - t > 5*60:
                        #self.processor.push_response({'method': 'server.peers', 'params': [self.get_peers()]})
                        s.send('NAMES #electrum\n')
                        t = time.time()
                        self.peers = {}
            except:
                traceback.print_exc(file=sys.stdout)
                time.sleep(1)
            finally:
                s.close()

        print_log("quitting IRC")

class ServerProcessor(Processor):

    def __init__(self, config):
        Processor.__init__(self)
        self.daemon = True
        self.banner = config.get('server', 'banner')

        if config.get('server', 'irc') == 'yes':
            self.irc = IrcThread(self, config)
        else:
            self.irc = None

    def get_peers(self):
        if self.irc:
            return self.irc.get_peers()
        else:
            return []

    def run(self):
        if self.irc:
            self.irc.start()
        Processor.run(self)

    def process(self, session, request):
        method = request['method']
        params = request['params']
        result = None

        if method == 'server.banner':
            result = self.banner.replace('\\n', '\n')

        elif method == 'server.peers.subscribe':
            result = self.get_peers()

        elif method == 'server.version':
            result = VERSION

        else:
            print_log("unknown method", method)

        if result != '':
            self.push_response(session, {'id': request['id'], 'result': result})
<commit_msg>Log KICK messages from IRC<commit_after>import socket
import sys
import threading
import time
import traceback

from processor import Processor
from utils import Hash, print_log
from version import VERSION

class IrcThread(threading.Thread):

    def __init__(self, processor, config):
        threading.Thread.__init__(self)

        self.processor = processor
        self.daemon = True
        self.stratum_tcp_port = config.get('server', 'stratum_tcp_port')
        self.stratum_http_port = config.get('server', 'stratum_http_port')
        self.stratum_tcp_ssl_port = config.get('server', 'stratum_tcp_ssl_port')
        self.stratum_http_ssl_port = config.get('server', 'stratum_http_ssl_port')
        self.report_stratum_tcp_port = config.get('server', 'report_stratum_tcp_port')
        self.report_stratum_http_port = config.get('server', 'report_stratum_http_port')
        self.report_stratum_tcp_ssl_port = config.get('server', 'report_stratum_tcp_ssl_port')
        self.report_stratum_http_ssl_port = config.get('server', 'report_stratum_http_ssl_port')
        self.peers = {}
        self.host = config.get('server', 'host')
        self.report_host = config.get('server', 'report_host')
        self.nick = config.get('server', 'irc_nick')
        if self.report_stratum_tcp_port:
            self.stratum_tcp_port = self.report_stratum_tcp_port
        if self.report_stratum_http_port:
            self.stratum_http_port = self.report_stratum_http_port
        if self.report_stratum_tcp_ssl_port:
            self.stratum_tcp_ssl_port = self.report_stratum_tcp_ssl_port
        if self.report_stratum_http_ssl_port:
            self.stratum_http_ssl_port = self.report_stratum_http_ssl_port
        if self.report_host:
            self.host = self.report_host
        if not self.nick:
            self.nick = Hash(self.host)[:5].encode("hex")
        self.prepend = 'E_'
        if config.get('server', 'coin') == 'litecoin':
            self.prepend = 'EL_'
        self.pruning = config.get('server', 'backend') == 'leveldb'
        if self.pruning:
            self.pruning_limit = config.get('leveldb', 'pruning_limit')
        self.nick = self.prepend + self.nick

    def get_peers(self):
        return self.peers.values()

    def getname(self):
        s = 'v' + VERSION + ' '
        if self.pruning:
            s += 'p' + self.pruning_limit + ' '

        def add_port(letter, number):
            DEFAULT_PORTS = {'t':'50001', 's':'50002', 'h':'8081', 'g':'8082'}
            if not number: return ''
            if DEFAULT_PORTS[letter] == number:
                return letter + ' '
            else:
                return letter + number + ' '

        s += add_port('t',self.stratum_tcp_port)
        s += add_port('h',self.stratum_http_port)
        s += add_port('s',self.stratum_tcp_ssl_port)
        s += add_port('g',self.stratum_http_ssl_port)
        return s

    def run(self):
        ircname = self.getname()
        print_log("joining IRC")

        while not self.processor.shared.stopped():
            try:
                s = socket.socket()
                s.connect(('irc.freenode.net', 6667))
                s.settimeout(0.1)
            except:
                s.close()
                print_log("IRC: reconnect in 10 s")
                time.sleep(10)
                continue

            self.message = ''
            out_msg = []

            try:
                s.send('USER electrum 0 * :' + self.host + ' ' + ircname + '\n')
                s.send('NICK ' + self.nick + '\n')
                s.send('JOIN #electrum\n')
                t = 0

                while not self.processor.shared.stopped():
                    try:
                        data = s.recv(2048)
                    except socket.timeout, e:
                        if out_msg:
                            m = out_msg.pop(0)
                            s.send(m)
                        continue
                    except:
                        print_log( "irc: socket error" )
                        time.sleep(1)
                        break

                    self.message += data

                    while self.message.find('\n') != -1:
                        pos = self.message.find('\n')
                        line = self.message[0:pos]
                        self.message = self.message[pos+1:]
                        line = line.strip('\r')
                        if not line:
                            continue
                        # print_log("<--", line)
                        line = line.split()
                        if line[0] == 'PING':
                            out_msg.append('PONG ' + line[1] + '\n')
                        elif '353' in line:  # answer to /names
                            k = line.index('353')
                            for item in line[k+1:]:
                                if item.startswith(self.prepend):
                                    out_msg.append('WHO %s\n' % item)
                        elif '352' in line:  # answer to /who
                            # warning: this is a horrible hack which apparently works
                            k = line.index('352')
                            try:
                                ip = socket.gethostbyname(line[k+4])
                            except:
                                print_log("gethostbyname error", line[k+4])
                                continue
                            name = line[k+6]
                            host = line[k+9]
                            ports = line[k+10:]
                            self.peers[name] = (ip, host, ports)
                        elif 'KICK' in line:
                            try:
                                print_log("KICK", line[3] + line[4])
                            except:
                                print_log("KICK", "error")

                    if time.time() - t > 5*60:
                        #self.processor.push_response({'method': 'server.peers', 'params': [self.get_peers()]})
                        s.send('NAMES #electrum\n')
                        t = time.time()
                        self.peers = {}
            except:
                traceback.print_exc(file=sys.stdout)
                time.sleep(1)
            finally:
                s.close()

        print_log("quitting IRC")

class ServerProcessor(Processor):

    def __init__(self, config):
        Processor.__init__(self)
        self.daemon = True
        self.banner = config.get('server', 'banner')

        if config.get('server', 'irc') == 'yes':
            self.irc = IrcThread(self, config)
        else:
            self.irc = None

    def get_peers(self):
        if self.irc:
            return self.irc.get_peers()
        else:
            return []

    def run(self):
        if self.irc:
            self.irc.start()
        Processor.run(self)

    def process(self, session, request):
        method = request['method']
        params = request['params']
        result = None

        if method == 'server.banner':
            result = self.banner.replace('\\n', '\n')

        elif method == 'server.peers.subscribe':
            result = self.get_peers()

        elif method == 'server.version':
            result = VERSION

        else:
            print_log("unknown method", method)

        if result != '':
            self.push_response(session, {'id': request['id'], 'result': result})
<|endoftext|>


<commit_before>__version__ = "2.8.2"
<commit_msg>bump version<commit_after>__version__ = "2.8.3"
<|endoftext|>