<commit_before>import png
import sys

##GLOBALS
ELEMENT_WIDTH = 50
REGION_WIDTH_IN_ELEMENTS = 200
REGION_WIDTH = ELEMENT_WIDTH * REGION_WIDTH_IN_ELEMENTS

## Identifies all regions in a given square
## Returns list of coordinates of lower left corner of all regions
def findRegions(x,y,w):
    xll = x/REGION_WIDTH * REGION_WIDTH
    yll = y/REGION_WIDTH * REGION_WIDTH
    xregions = []
    yregions = []
    ##populate x
    i = xll
    while i <= x+w:
        xregions.append(str(i))
        i += REGION_WIDTH
    ##populate y
    i = yll
    while i <= y+w:
        yregions.append(str(i))
        i += REGION_WIDTH
    output = []
    for xtemp in xregions:
        for ytemp in yregions:
            output.append((str(xtemp),str(ytemp)))
    return output

def makeFilename(x,y):
    return 'data\\' + str(x) + '-' + str(y) + '.asc'

## build 2d array for a region
def makeArrayForRegion(regionCoord):
    output = []
    filename = makeFilename(regionCoord[0],regionCoord[1])

    with open(filename,'r') as f:
        ncols = int(f.readline()[6:])
        nrows = int(f.readline()[6:])
        xllcorner = int(f.readline()[10:])
        yllcorner = int(f.readline()[10:])
        cellsize = int(f.readline()[9:])

        for i in range(nrows):
            newRow = [int(round((float(k)+100)*50)) for k in f.readline().split()]
            output.append(newRow)
            newRow = []

    return output

def appendMatrixLR(left,right):
    if len(left) != len(right):
        print "Matrix Dimension error"
        return False
    output = []
    for i in range(len(left)):
        newRow = left[i] + right[i]
        output.append(newRow)
    return output

def appendMatrixTB(top,bottom):
    ## basic check, not fully safe
    if len(top[0]) != len(bottom[0]):
        print "Matrix dimension error"
        return False
    output = top + bottom
    return output

def combineToColumns(regions,coords):
    output = []
    currentX = '-1'
    for i in range(len(regions)):
        ## if equal to last one then append
        if coords[i][0] == str(currentX):
            output[len(output)-1] = appendMatrixTB(regions[i],output[len(output)-1])
        else:
            output.append(regions[i])
            currentX = coords[i][0]
    return output

def combineToSquare(regions):
    output = regions[0]
    for i in range(1,len(regions)):
        output = appendMatrixLR(output,regions[i])
    return output

## input list of coordinates
## outputs 2d array of the combination of regions
def createDataArray(regionsNeeded):
    ## first construct a 2d array for each region and place in new list
    ## this list can be referenced using the regionsNeeded list
    regions = []
    for i in regionsNeeded:
        regions.append(makeArrayForRegion(i))

    ## combine regions into columns
    regions = combineToColumns(regions,regionsNeeded)

    ## combine columns into square
    regions = combineToSquare(regions)

    return regions

def refineDataArray(dataArray, xll, yll, width):
    ## the ll coord of the whole input square
    squareXll = xll/REGION_WIDTH * REGION_WIDTH
    squareYll = yll/REGION_WIDTH * REGION_WIDTH
    ## the ll coord of the desired region in terms of number of elements
    xStart = (xll-squareXll)/ELEMENT_WIDTH
    yStart = (yll-squareYll)/ELEMENT_WIDTH
    ## length of side of desired region in terms of number of elements
    length = width/ELEMENT_WIDTH
    output = []
    i = yStart
    while i < yStart+length:
        newRow = dataArray[i][xStart:xStart+length]
        output.append(newRow)
        i+=1

    return output

## takes an array and a float index
## returns float vaule of linear estimation of index
def advancedIndex(array,index):
    left = int(index)
    right = left + 1
    fraction = index - left
    if right == len(array):
        return array[left]
    difference = array[right]-array[left]
    return array[left] + difference*fraction

## lengthens the given 1D array using value estimation
def lengthenArray(arrayIn, outputLength):
    step = (len(arrayIn)-1)/float(outputLength-1)
    output = []
    k = 0
    for i in range(outputLength):
        output.append(k)
        k += step
    for i in range(len(output)):
        output[i] = int(round(advancedIndex(arrayIn,output[i])))
    return output

## transposes 2D array
def transpose(array):
    output = []
    for i in range(len(array[0])):
        newRow = []
        for j in range(len(array)):
            newRow.append(array[j][i])
        output.append(newRow)
    return output

def resizeArray(array, width):
    temp = transpose(array)
    temp2 = []
    output = []
    for col in temp:
        temp2.append(lengthenArray(col,width))
    temp2 = transpose(temp2)
    for row in temp2:
        output.append(lengthenArray(row,width))
    return output

def main():
    ##get the region that needs to be created
    xll = 400000
    yll = 200000
    width = 10000

    desiredSize = 400

    ## build list of coordinates where the data lies
    regionsNeeded = findRegions(xll,yll,width)

    ## build the 2d array of all squraes where the requested data exists
    dataArray = createDataArray(regionsNeeded)

    ## refine array to just data included in the request
    dataArray = refineDataArray(dataArray, xll, yll, width)

    ## resize the array to the output size
    dataArray = resizeArray(dataArray,desiredSize)

    ## print the png
    f = open('output.png', 'wb')
    w = png.Writer(desiredSize, desiredSize, greyscale=True, bitdepth=16)
    w.write(f, dataArray)
    f.close()

main()
<commit_msg>Added contrasting<commit_after>import png
import sys

##GLOBALS
ELEMENT_WIDTH = 50
REGION_WIDTH_IN_ELEMENTS = 200
REGION_WIDTH = ELEMENT_WIDTH * REGION_WIDTH_IN_ELEMENTS

## Identifies all regions in a given square
## Returns list of coordinates of lower left corner of all regions
def findRegions(x,y,w):
    xll = x/REGION_WIDTH * REGION_WIDTH
    yll = y/REGION_WIDTH * REGION_WIDTH
    xregions = []
    yregions = []
    ##populate x
    i = xll
    while i <= x+w:
        xregions.append(str(i))
        i += REGION_WIDTH
    ##populate y
    i = yll
    while i <= y+w:
        yregions.append(str(i))
        i += REGION_WIDTH
    output = []
    for xtemp in xregions:
        for ytemp in yregions:
            output.append((str(xtemp),str(ytemp)))
    return output

def makeFilename(x,y):
    return 'data\\' + str(x) + '-' + str(y) + '.asc'

## build 2d array for a region
def makeArrayForRegion(regionCoord):
    output = []
    filename = makeFilename(regionCoord[0],regionCoord[1])

    with open(filename,'r') as f:
        ncols = int(f.readline()[6:])
        nrows = int(f.readline()[6:])
        xllcorner = int(f.readline()[10:])
        yllcorner = int(f.readline()[10:])
        cellsize = int(f.readline()[9:])

        for i in range(nrows):
            newRow = [int(round((float(k)+100)*50)) for k in f.readline().split()]
            output.append(newRow)
            newRow = []

    return output

def appendMatrixLR(left,right):
    if len(left) != len(right):
        print "Matrix Dimension error"
        return False
    output = []
    for i in range(len(left)):
        newRow = left[i] + right[i]
        output.append(newRow)
    return output

def appendMatrixTB(top,bottom):
    ## basic check, not fully safe
    if len(top[0]) != len(bottom[0]):
        print "Matrix dimension error"
        return False
    output = top + bottom
    return output

def combineToColumns(regions,coords):
    output = []
    currentX = '-1'
    for i in range(len(regions)):
        ## if equal to last one then append
        if coords[i][0] == str(currentX):
            output[len(output)-1] = appendMatrixTB(regions[i],output[len(output)-1])
        else:
            output.append(regions[i])
            currentX = coords[i][0]
    return output

def combineToSquare(regions):
    output = regions[0]
    for i in range(1,len(regions)):
        output = appendMatrixLR(output,regions[i])
    return output

## input list of coordinates
## outputs 2d array of the combination of regions
def createDataArray(regionsNeeded):
    ## first construct a 2d array for each region and place in new list
    ## this list can be referenced using the regionsNeeded list
    regions = []
    for i in regionsNeeded:
        regions.append(makeArrayForRegion(i))

    ## combine regions into columns
    regions = combineToColumns(regions,regionsNeeded)

    ## combine columns into square
    regions = combineToSquare(regions)

    return regions

def refineDataArray(dataArray, xll, yll, width):
    ## the ll coord of the whole input square
    squareXll = xll/REGION_WIDTH * REGION_WIDTH
    squareYll = yll/REGION_WIDTH * REGION_WIDTH
    ## the ll coord of the desired region in terms of number of elements
    xStart = (xll-squareXll)/ELEMENT_WIDTH
    yStart = (yll-squareYll)/ELEMENT_WIDTH
    ## length of side of desired region in terms of number of elements
    length = width/ELEMENT_WIDTH
    output = []
    i = yStart
    while i < yStart+length:
        newRow = dataArray[i][xStart:xStart+length]
        output.append(newRow)
        i+=1

    return output

## takes an array and a float index
## returns float vaule of linear estimation of index
def advancedIndex(array,index):
    left = int(index)
    right = left + 1
    fraction = index - left
    if right == len(array):
        return array[left]
    difference = array[right]-array[left]
    return array[left] + difference*fraction

## lengthens the given 1D array using value estimation
def lengthenArray(arrayIn, outputLength):
    step = (len(arrayIn)-1)/float(outputLength-1)
    output = []
    k = 0
    for i in range(outputLength):
        output.append(k)
        k += step
    for i in range(len(output)):
        output[i] = int(round(advancedIndex(arrayIn,output[i])))
    return output

## transposes 2D array
def transpose(array):
    output = []
    for i in range(len(array[0])):
        newRow = []
        for j in range(len(array)):
            newRow.append(array[j][i])
        output.append(newRow)
    return output

## makes an array bigger
def resizeArray(array, width):
    temp = transpose(array)
    temp2 = []
    output = []
    for col in temp:
        temp2.append(lengthenArray(col,width))
    temp2 = transpose(temp2)
    for row in temp2:
        output.append(lengthenArray(row,width))
    return output

## funcion to apply full contrast to a store array
def applyContrast(data):
    maximum = 0
    minimum = 2**16
    for row in data:
        a = max(row)
        b = min(row)
        if a > maximum:
            maximum = a
        if b < minimum:
            minimum = b
    print minimum,maximum
    factor = (2**16-1)/(maximum-minimum)
    output = []
    for row in data:
        newRow = [int(round((x-minimum)*factor)) for x in row]
        output.append(newRow)
    return output

def main():
    ##get the region that needs to be created
    xll = 400000
    yll = 200000
    width = 10000

    desiredSize = 1081

    ## build list of coordinates where the data lies
    regionsNeeded = findRegions(xll,yll,width)

    ## build the 2d array of all squraes where the requested data exists
    dataArray = createDataArray(regionsNeeded)

    ## refine array to just data included in the request
    dataArray = refineDataArray(dataArray, xll, yll, width)

    ## resize the array to the output size
    dataArray = resizeArray(dataArray,desiredSize)

    ## apply contrast
    dataArray = applyContrast(dataArray)

    ## print the png
    f = open('output.png', 'wb')
    w = png.Writer(desiredSize, desiredSize, greyscale=True, bitdepth=16)
    w.write(f, dataArray)
    f.close()

main()
<|endoftext|>


<commit_before>__author__ = 'davidvollum'
from evernote.api.client import EvernoteClient
import evernote.edam.type.ttypes as Types
import os
import hashlib
import evernote.edam.error.ttypes as Errors
import sys
import logging

#Constants

#Must be a legal directory with only pdfs and text files in it; must end in a "/"/Users/davidvollum/Documents/testing/
directoryPath="/Users/davidvollum/Documents/testing/files"

#where the actual python sctipt is stored
infoPath = "/Users/davidvollum/PycharmProjects/Evernote/"

#determines weather to use manifests to prevent duplicate uploads Normal = True
useManifest=True

#badFiles contains a list of special files used by the OS that should not be uploaded or messed with:
badFiles=[".DS_Store"]

logging.basicConfig(level=logging.INFO)

#logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', filename=infoPath +'evernote.log',level=logging.INFO)
#logging.debug('This message should go to the log file')
#logging.info('So should this')
#logging.warning('And this, too')

dev_token = "S=s1:U=9342a:E=161009f0c1b:C=159a8eddcf0:P=1cd:A=en-devtoken:V=2:H=4ef27d10f138f1d2898b3ea25ff1d531"
client = EvernoteClient(token=dev_token, sandbox=True)
userStore = client.get_user_store()
user = userStore.getUser()

logging.info("Starting upload for user: " +  user.username)
#logging.info("")
noteStore = client.get_note_store()
notebooks = noteStore.listNotebooks()

def readfile(path):
    file = open(path)
    data=file.read()
    file.close()
    return data

class TheFile:
    def __init__(self, fileName, filePath, fileType, notebook, data=None):
        self.fileName = fileName
        self.filePath = filePath
        self.fileType = fileType
        self.notebook = notebook
        self.dateModified = str(os.path.getatime(filePath))
        self.data = data
    def getNoteName(self):
        noteName=self.fileName.split(".")
        return noteName[0]
    def getData(self):
        #print self.fileType
        if self.data:
            return self.data
        elif self.fileType == ".txt":
            data = readfile(self.filePath)
            #print data + "that data though"
            return data
        else:
            return "no data so you get this instead"

# **noteStore.createNote
# for n in notebooks:
#   print n.name
# os.path.getatime(/volumes)
# os.listdir()

def createNote(authToken, noteStore, noteTitle, noteBody, fileType, pathToFile=None, notebook=None):
        # Create the new note
        note = Types.Note()
        note.title = noteTitle
        if notebook:
            note.notebookGuid=notebook.guid
        note.content = '<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE en-note SYSTEM "http://xml.evernote.com/pub/enml2.dtd">'
        note.content += '<en-note>'
        note.content += noteBody
        note.content += '<br/>'

        # Calculate the md5 hash of the pdf
        if fileType == ".pdf":
            md5 = hashlib.md5()
            pdfFile = open(pathToFile,'rb')
            pdf_bytes = pdfFile.read()
            md5.update(pdf_bytes)
            md5hash = md5.hexdigest()

            # Create the Data type for evernote that goes into a resource
            pdf_data = Types.Data()
            pdf_data.bodyHash = md5hash
            pdf_data.size = len(pdf_bytes)
            pdf_data.body = pdf_bytes

            # Add a link in the evernote boy for this content
            link = '<en-media type="application/pdf" hash="%s"/>' % md5hash
            note.content += link

            # Create a resource for the note that contains the pdf
            pdf_resource = Types.Resource()
            pdf_resource.data = pdf_data
            #print pdf_resource.mime
            pdf_resource.mime = "application/pdf"

            # Create a resource list to hold the pdf resource
            resource_list = []
            resource_list.append(pdf_resource)

            # Set the note's resource list
            note.resources = resource_list
        note.content += '</en-note>'

        try:
            theNote = noteStore.createNote(authToken, note)
        except Errors.EDAMUserException, edue:
            ## Something was wrong with the note data
            ## See EDAMErrorCode enumeration for error code explanation
            ## http://dev.evernote.com/documentation/reference/Errors.html#Enum_EDAMErrorCode
            logging.error("EDAMUserException:" + str(edue))
            return None
        except Errors.EDAMNotFoundException, ednfe:
            ## Parent Notebook GUID doesn't correspond to an actual notebook
            logging.error("EDAMNotFoundException: Invalid parent notebook GUID")
            return None
        ## Return created note object
        return note

def getNotebook(notebookName, noteStore, authToken,  stack=None):
        notebooks = noteStore.listNotebooks()
        logging.debug(notebooks)
        notebookNames={}
        for theNotebook in notebooks:
            notebookNames[theNotebook.name]=theNotebook
        if notebookName in notebookNames.keys():
            logging.debug("found something")
            # Existing notebook, so just update the stack if needed
            notebook = notebookNames[notebookName]
            if stack:
                notebook.stack = stack
                noteStore.updateNotebook(notebook)
            return notebook
        else:
            # Need to create a new notebook
            notebook = Types.Notebook()
            notebook.name = notebookName
            logging.info("manifestPath " + notebook.name + "Notebook name " + notebookName)
            if stack:
                notebook.stack = stack
            notebook = noteStore.createNotebook(authToken, notebook)
            return notebook

def inManifest(path, fileName):
    filePath=path+fileName
    #print filePath + ":filepath"
    #dateModified=os.path.getatime(filePath)
    #print dateModified
    try:
        manifestFile = open(infoPath +"manifest.txt", "r+")
    except IOError:
        logging.info("could not find manifest, creating a new one")
        manifestFile = open(infoPath + "manifest.txt", "w+")
    filesInManifest=manifestFile.read()
    filesInManifest = filesInManifest.split("|")
    for line in filesInManifest:
        line=line.strip()
        #print "comparing line "+line+" filname "+fileName
        if line == fileName:
            #print("found match")
            return useManifest
    manifestFile.write(fileName+"|")
    manifestFile.close()
    return False

#returns a list of tuples. tuples are filename, subfolder, filetype
def getFilesToUpload(path, manifestPath):
    files=os.listdir(path)
    #print str(files) + "uggggg"
    filesList=[]
    for fileName in files:
        filePath=path+fileName
        fileType = os.path.splitext(filePath)[-1].lower()
        #dateModifyed=os.path.getatime(filePath)
        logging.debug(fileType)
        logging.debug("file: " + fileName + " is a .txt or .pdf " + str(".pdf" == fileType or ".txt" == fileType))

        if fileType=="" and fileName not in badFiles:
            logging.debug("found subfolder")
            logging.debug(filesList)
            filesList = filesList+getFilesToUpload(filePath+"/", manifestPath)
        elif ".pdf" == fileType or ".txt" == fileType and fileName not in badFiles:
            if inManifest(manifestPath,fileName)==False:
                theNotebook = getNotebook(path.split("/")[-2], noteStore, dev_token)
                logging.info("found file: " + fileName + " in folder: " + path.split("/")[-2] + " with fileType: " + fileType)
                logging.debug(theNotebook)
                logging.info("Preparing: " + fileName + " for upload")
                filesList.append(TheFile(fileName,filePath,fileType,theNotebook))
            else:
                logging.info("skipping: " + fileName)
        else:
            logging.warning("Invalid Path or File type for file: " + fileName)
    return filesList

def readDirectory(path):
    filesToUpload = getFilesToUpload(path, path)
    for fileToUpload in filesToUpload:
        logging.info("found " + fileToUpload.fileType + " file: " + fileToUpload.fileName + " which was last modified: " + fileToUpload.dateModified)
        #data=readfile(filePath)
        #print(theNotebook)
        createNote(dev_token, noteStore, fileToUpload.getNoteName(), fileToUpload.getData(), fileToUpload.fileType, fileToUpload.filePath, notebook=fileToUpload.notebook)
        logging.info("uploading: " + fileToUpload.fileName)

args = sys.argv
if "-noManifest" in args:
    print "Not using the Manifest: THIS IS NOT NORMAL"
    useManifest = False

if "-generateManifestOnly" in args:
    print "Gennerating manifest only"
    getFilesToUpload(directoryPath,directoryPath)
#new feature not in final
if "-path:" in args:
    index=0
    for arg in args:
        if arg == "-path:":
            directoryPath=args[index+1]
        else:
            index+=1
    readDirectory(directoryPath)
else:
    readDirectory(directoryPath)
#getFilesToUpload(directoryPath, directoryPath)<commit_msg>Delete main.py<commit_after><|endoftext|>


<commit_before>from machine import Pin, PWM
from encoder import Encoder
from ledstripcontroller import LedStripController
from uasyncio import get_event_loop

def main():
    enc = Encoder(pin_clk=21, pin_dt=22, pin_mode=Pin.PULL_UP,
                  min_val=40, max_val=1020, clicks=1, accel=5)
    enc._value = 1020

    fader_pins = [PWM(Pin(pin_num), freq=50000) for pin_num in (19,23,18)]
    button = Pin(16, Pin.IN, Pin.PULL_UP)
    controller = LedStripController(enc, button_pin=button, fader_pins=fader_pins)
    button.irq(trigger=Pin.IRQ_FALLING, handler=lambda _: controller.toggle_led_state())

    loop = get_event_loop()
    loop.create_task(controller.switch_loop(controller.enc))
    loop.create_task(controller.encoder_loop(controller.enc))
    loop.create_task(controller.fader_loop())
    loop.run_forever()
    controller.enc.close()

if __name__ == '__main__':
    main()
<commit_msg>Generalising initialisation to depend on the type of board detected<commit_after>from machine import Pin, PWM
from encoder import Encoder
from ledstripcontroller import LedStripController
from uasyncio import get_event_loop
from sys import platform

def main():
    dev_type = platform
    if dev_type == 'esp8266': # WEMOS D1 Mini (or similar)
        ENC_PINS = {'clk':4, 'dt':5, 'button':2}
        FADER_PINS = (12, 13, 14)
        FADER_FREQ = 1000
    elif dev_type == 'esp32': # TTGO
        ENC_PINS = {'clk':21, 'dt':22, 'button':16}
        FADER_PINS = (19, 23, 18)
        FADER_FREQ = 50000
    else: # Defaults
        ENC_PINS = {'clk':None, 'dt':None, 'button':None}
        FADER_PINS = (None, None, None)
        FADER_FREQ = None

    enc = Encoder(pin_clk=ENC_PINS['clk'], pin_dt=ENC_PINS['dt'], pin_mode=Pin.PULL_UP,
                  min_val=40, max_val=1020, clicks=1, accel=5)
    enc._value = 0

    fader_pins = [PWM(Pin(pin_num), freq=FADER_FREQ) for pin_num in FADER_PINS]
    button = Pin(ENC_PINS['button'], Pin.IN, Pin.PULL_UP)
    controller = LedStripController(enc, button_pin=button, fader_pins=fader_pins)
    button.irq(trigger=Pin.IRQ_FALLING, handler=lambda _: controller.toggle_led_state())

    loop = get_event_loop()
    loop.create_task(controller.switch_loop(controller.enc))
    loop.create_task(controller.encoder_loop(controller.enc))
    loop.create_task(controller.fader_loop())
    loop.run_forever()
    controller.enc.close()

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>from flask import Flask, request, redirect, render_template, abort
from urlparse import urlparse
import links, json

app = Flask(__name__)

@app.route('/', methods=['GET', 'POST'])
def index():
    if request.method == 'POST':
        shortlink = make_shortlink(request.form['link'])
        #print('SHORTLINK: %s' % shortlink)
        if shortlink:
            return render_template('/index.html', new_link=shortlink)
        else:
            return render_template('/index.html', error='URL must be valid!')
    else:
        return render_template('/index.html')

@app.route('/<link_id>')
def link_redirect(link_id):
    redir_link = links.click_link(link_id)
    #print('redir: %s' % redir_link)
    if redir_link:
        return redirect(redir_link['uri'])
    else:
        return abort(404), 404

@app.route('/stats', methods=['GET', 'POST'])
def stats():
    if request.method == 'POST':
        link = request.form['link']
        if 'https://fastl.ink' in link:
            link_id = link.split('/')[-1]
            if len(link_id) == 6:
                link_obj = links.find_link(link_id)
                return render_template('/stats.html', stats_link=link_obj)
            else:
                return render_template('/stats.html', error="Looks like that isn't a valid fastlink!")
        else:
            return render_template('/stats.html', error="Looks like that isn't a valid fastlink!")
    elif request.method == 'GET':
        return render_template('/stats.html')

@app.route('/stats/<link_id>')
def stats_by_id(link_id):
    link_obj = links.find_link(link_id)
    return render_template('/stats.html', stats_link=link_obj)

@app.route('/api/new', methods=['POST'])
def api_create():
    req_body = request.get_json(force=True)
    try:
        req_url = req_body['url']
    except KeyError:
        return abort(400), 400
    return ( 'https://fastl.ink/' + make_shortlink(req_url) )

@app.errorhandler(404)
def page_not_found(e):
    return render_template('404.html'), 404

def make_shortlink(link_str):
    if verify_url(link_str):
        result = links.new_link(link_str)
        return result
    else:
        return None

def verify_url(x):
    print('checking %s...' % x)
    result = urlparse(x)
    if result.scheme == '':
        print('  invalid URI %s!' % str(result))
        return False
    elif result.scheme in ['http', 'https']:
        if result.netloc == '':
            print('  invalid http/s: %s!' % str(result))
            return False
        else:
            print('  valid URI!', [result.scheme, result.netloc, result.path])
            return True
    else:
        if result.path == '':
            print('  invalid URI %s!' % str(result))
            return False
        else:
            print('  valid URI!', [result.scheme, result.netloc, result.path])
            return True

if __name__ == '__main__':
    app.run()
<commit_msg>forgot to change the urllib back<commit_after>from flask import Flask, request, redirect, render_template, abort
from urllib.parse import urlparse
import links, json

app = Flask(__name__)

@app.route('/', methods=['GET', 'POST'])
def index():
    if request.method == 'POST':
        shortlink = make_shortlink(request.form['link'])
        #print('SHORTLINK: %s' % shortlink)
        if shortlink:
            return render_template('/index.html', new_link=shortlink)
        else:
            return render_template('/index.html', error='URL must be valid!')
    else:
        return render_template('/index.html')

@app.route('/<link_id>')
def link_redirect(link_id):
    redir_link = links.click_link(link_id)
    #print('redir: %s' % redir_link)
    if redir_link:
        return redirect(redir_link['uri'])
    else:
        return abort(404), 404

@app.route('/stats', methods=['GET', 'POST'])
def stats():
    if request.method == 'POST':
        link = request.form['link']
        if 'https://fastl.ink' in link:
            link_id = link.split('/')[-1]
            if len(link_id) == 6:
                link_obj = links.find_link(link_id)
                return render_template('/stats.html', stats_link=link_obj)
            else:
                return render_template('/stats.html', error="Looks like that isn't a valid fastlink!")
        else:
            return render_template('/stats.html', error="Looks like that isn't a valid fastlink!")
    elif request.method == 'GET':
        return render_template('/stats.html')

@app.route('/stats/<link_id>')
def stats_by_id(link_id):
    link_obj = links.find_link(link_id)
    return render_template('/stats.html', stats_link=link_obj)

@app.route('/api/new', methods=['POST'])
def api_create():
    req_body = request.get_json(force=True)
    try:
        req_url = req_body['url']
    except KeyError:
        return abort(400), 400
    return ( 'https://fastl.ink/' + make_shortlink(req_url) )

@app.errorhandler(404)
def page_not_found(e):
    return render_template('404.html'), 404

def make_shortlink(link_str):
    if verify_url(link_str):
        result = links.new_link(link_str)
        return result
    else:
        return None

def verify_url(x):
    print('checking %s...' % x)
    result = urlparse(x)
    if result.scheme == '':
        print('  invalid URI %s!' % str(result))
        return False
    elif result.scheme in ['http', 'https']:
        if result.netloc == '':
            print('  invalid http/s: %s!' % str(result))
            return False
        else:
            print('  valid URI!', [result.scheme, result.netloc, result.path])
            return True
    else:
        if result.path == '':
            print('  invalid URI %s!' % str(result))
            return False
        else:
            print('  valid URI!', [result.scheme, result.netloc, result.path])
            return True

if __name__ == '__main__':
    app.run()
<|endoftext|>


<commit_before>import os, sys
import argparse
try:
    from configparser import ConfigParser
except ImportError:
    from ConfigParser import SafeConfigParser as ConfigParser
import fnmatch
import glob
import logging
import shlex
import subprocess

try:
    basestring
except NameError:
    basestring = str

logger = logging.getLogger(os.path.basename(__file__))

def _check_output(*args, **kwargs):
    #
    # The output from subprocess.check_output is bytes
    # in Python 3.x, str in 2.x. Decoding per sys.stdout
    # will hopefully result in a consistent unicode result.
    #
    return subprocess.check_output(*args, **kwargs).strip().decode(sys.stdout.encoding)

def _file_operation(pattern, start_from=".", recursive=False):
    """Common file-matching semantics:

    Search for files whose name matches [pattern].
    By default, search the current directory but a specific directory or iterable
    of directories can be specified, in which case it and (optionally) all its
    children will be searched.
    """
    if isinstance(start_from, basestring):
        paths = [start_from]
    else:
        paths = start_from

    for path in paths:
        if not os.path.exists(path):
            continue
        if recursive:
            for dirpath, dirnames, filenames in os.walk(path):
                for filename in filenames:
                    if fnmatch.fnmatch(filename, pattern):
                        yield os.path.join(dirpath, filename)
        else:
            for filepath in glob.glob(os.path.join(path, pattern)):
                yield filepath

def _where(pattern, start_from=None, recursive=False):
    """Search for a file whose name matches [pattern].

    By default, search the current directory and the PATH but a
    specific directory can be specified, in which case it and (optionally)
    all its children will be searched.
    """
    logger.info("Looking for %s in %s", pattern, start_from)
    if start_from is None:
        start_from = ["."] + [p for p in os.environ["PATH"].split(";")]
    for filepath in _file_operation(pattern, start_from, recursive):
        return filepath

def _delete(pattern, start_from=".", recursive=False, callback=None):
    """delete files who names match [pattern].

    By default, search the current directory but a specific directory
    can be specified. Recursion may be used but, by default, is not.
    """
    logger.info("About to delete %s from %s", pattern, start_from or ".")
    for filepath in _file_operation(pattern, start_from, recursive):
        os.unlink(filepath)
        if callback:
            callback(filepath)

def _rmdir(start_from, callback=None):
    """Remove a directory and everything underneath it
    """
    logger.info("About to rmdir %s", start_from)
    for dirpath, _, filenames in os.walk(start_from, topdown=False):
        for filename in filenames:
            filepath = os.path.join(dirpath, filename)
            if callback:
                callback.callback(filepath)
            os.unlink(filepath)
        os.rmdir(dirpath)

class Config(object):

    #
    # Defaults
    #
    configuration = "Debug"
    platform = "Win32"
    interactive = True
    externals = True

    def __init__(self):
        self.root = os.getcwd()
        self.here = os.path.dirname(os.path.abspath(__file__))
        try:
            self.branch = _check_output("hg branch")
        except subprocess.CalledProcessError:
            logger.warn("Can't determine hg branch; using default")
            self.branch = "default"
        parser = ConfigParser()
        parser.read([
            os.path.join(self.here, "configure.ini"),
            os.path.join(self.here, "configure.%s.ini" % self.branch),
            os.path.join(self.root, "configure.ini"),
        ])
        self.config = parser.get("configure", "configuration", fallback=self.__class__.configuration)
        self.platform = parser.get("configure", "platform", fallback=self.__class__.platform)
        self.visual_studio = os.path.abspath(os.environ[parser.get("configure", "envvar")])
        self.svnroot = parser.get("locations", "svnroot")
        self.externals = dict(parser.items("externals"))
        self.run_tests = shlex.split(parser.get("commands", "run_tests"))

        self.pcbuild = os.path.join(self.root, "PCBuild")
        self.externals_dir = os.path.abspath(os.path.join(self.pcbuild, "../.."))
        if self.platform == "x64":
            self.vcvarsarg = "x86_amd64"
            self.pybuilddir = os.path.join(self.pcbuild, "amd64")
        else:
            self.vcvarsarg = "x86"
            self.pybuilddir = self.pcbuild
        pybuildextension = "_d.exe" if self.configuration == "Debug" else ".exe"
        self.python_exe = os.path.join(self.pybuilddir, "python" + pybuildextension)

    def dumped(self):
        content = []
        content.append("{")
        for name in sorted(dir(self)):
            if name.startswith("_"):
                continue
            attr = getattr(self, name)
            if callable(attr):
                continue
            content.append("  %s => %r" % (name, attr))
        content.append("}")
        return "\n".join(content)

class Build(object):

    def __init__(self, config, stdout):
        self.config = config
        self.stdout = stdout

    @classmethod
    def valid_targets(cls):
        return set(m[3:] for m in dir(cls) if m.startswith("do_"))

    def _run_command(self, command, *args, **kwargs):
        p = subprocess.Popen(command, *args, stdout=self.stdout, stderr=subprocess.STDOUT, **kwargs)
        if p.wait() != 0:
            raise RuntimeError("Problem with command")

    def _find_interpreter(self):
        """Attempt to find an interpreter:

        * If a PYTHON3 env var is set, return its value
        * If a python.bat exists in the current directory, return that
        * If a python*.exe exists in the PCBuild directory, return that
        * If a py.exe exists any_where on the path, return that
        * Otherwise, return "python" and hope for the best

        NB This should not be run and cached, as the interpreter may not
        exist until it has been built.
        """
        if "PYTHON3" in os.environ:
            return os.environ["PYTHON3"]
        if os.path.exists(os.path.join(self.config.root, "python.bat")):
            return "python.bat"
        pcbuild_exe = _where("python*.exe", self.config.pcbuild, True)
        if pcbuild_exe:
            return pcbuild_exe
        py_exe = _where("py.exe")
        if py_exe:
            return py_exe
        return "python"

    def msbuild(self, project, target, **kwargs):
        vcvarsall = os.path.join(self.config.visual_studio, r"..\..\VC\vcvarsall.bat")
        params = ["/p:%s=%s" % (k, v) for (k, v) in kwargs.items()]
        msbuild = ["msbuild", "/target:%s" % target, project] + params
        logger.debug(msbuild)
        logger.info("Running msbuild %s", target)
        return self._run_command(["call", vcvarsall, self.config.vcvarsarg, "&&"] + msbuild, shell=True)

    def build_solution(self, target, **kwargs):
        return self.msbuild(r"PCbuild\pcbuild.sln", target, **kwargs)

    def check_externals(self):
        for name, version in self.config.externals.items():
            target_dirpath = os.path.abspath(os.path.join(self.config.externals_dir, version))
            if not os.path.exists(target_dirpath):
                logger.warn("External %s is not in place", version)

    def do_all(self):
        self.do_clean()
        self.do_externals()
        self.do_build()
        self.do_test()

    def do_build(self):
        """Build CPython"""
        logger.info("Build & run kill_python")
        self.msbuild(
            r"PCbuild\kill_python.vcxproj",
            "Build",
            Config="Debug",
            PlatformTarget=self.config.platform
        )
        self._run_command([os.path.join(self.config.pybuilddir, "kill_python_d.exe")])
        _delete(self.config.python_exe)
        logger.info("Check externals")
        self.check_externals()
        logger.info("Build Python")
        self.build_solution(
            "Build",
            Config=self.config.configuration,
            Platform=self.config.platform
        )

    def do_clean(self):
        """Clean out build artifacts for current configuration"""
        logger.info("Deleting .pyc / .pyo files")
        _delete("*.pyc", self.config.root, True, callback=logger.debug)
        _delete("*.pyo", self.config.root, True, callback=logger.debug)
        _rmdir(os.path.join(self.config.root, "build"), callback=logger.debug)
        for configuration in "Release", "Debug":
            self.build_solution("clean", Config=configuration, Platform=self.config.platform)
        _delete("python.bat", callback=logger.debug)

    def do_clobber(self):
        """Clean out all build artifacts and remove externals"""
        self.do_clean()
        _delete("tcl*.dll", "PCbuild", recursive=True)
        _delete("tk*.dll", "PCbuild", recursive=True)
        for name, version in self.config.externals.items():
            target_dirpath = os.path.abspath(os.path.join(self.config.externals_dir, version))
            _rmdir(target_dirpath, callback=logger.debug)

    def do_externals(self):
        """Fetch external libraries in preparation for building"""
        for name, version in self.config.externals.items():
            target_dirpath = os.path.join(self.config.externals_dir, version)
            if os.path.exists(target_dirpath):
                logger.warn("Not fetching %s; already exists", version)
            else:
                logger.info("Fetching %s into %s", version, target_dirpath)
                self._run_command(["svn", "export", "%s/%s" % (self.config.svnroot, version), target_dirpath])

    def do_importlib(self):
        """Build and run the _freeze_importlib project"""
        already_exists = set(glob.glob(r"PCBuild\python*.exe"))
        self.msbuild(
            r"PCbuild\_freeze_importlib.vcxproj",
            "Build",
            Config="Release",
            PlatformTarget=self.config.platform
        )
        for python_exe in glob.glob(r"PCBuild\python*.exe"):
            if python_exe not in already_exists:
                logger.warn("Removing %s because build artefact", python_exe)
                os.unlink(python_exe)

    def do_patchcheck(self):
        r"""Run Tools\scripts\patchcheck.py"""
        python_exe = os.path.abspath(self._find_interpreter())
        self._run_command([python_exe, "tools/scripts/patchcheck.py"])

    def do_test(self):
        """Test Python"""
        python_exe = os.path.abspath(self._find_interpreter())
        self._run_command([python_exe] + self.config.run_tests)

    def run_from_args(self, args):
        if args:
            targets = list(args)
        else:
            targets = ["build"]
        for target in targets:
            logger.info("Executing %s", target)
            function = getattr(self, "do_" + target, None)
            if function:
                function()
            else:
                logger.warn("Unknown target: %s; ignoring...", target)

def main(*args):
    output_stream = sys.stdout
    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
    logger.setLevel(level=logging.INFO)
    screen = logging.StreamHandler(output_stream)
    screen.setFormatter(formatter)
    logger.addHandler(screen)

    config = Config()
    logger.info(config.dumped())
    Build(config, output_stream).run_from_args(args)

if __name__ == '__main__':
    main(*sys.argv[1:])
<commit_msg>Allow commands to consume further parameters (specifically for test to start with)<commit_after>import os, sys
import argparse
try:
    from configparser import ConfigParser
except ImportError:
    from ConfigParser import SafeConfigParser as ConfigParser
import fnmatch
import glob
import logging
import shlex
import subprocess

try:
    basestring
except NameError:
    basestring = str

logger = logging.getLogger(os.path.basename(__file__))

def _check_output(*args, **kwargs):
    #
    # The output from subprocess.check_output is bytes
    # in Python 3.x, str in 2.x. Decoding per sys.stdout
    # will hopefully result in a consistent unicode result.
    #
    return subprocess.check_output(*args, **kwargs).strip().decode(sys.stdout.encoding)

def _file_operation(pattern, start_from=".", recursive=False):
    """Common file-matching semantics:

    Search for files whose name matches [pattern].
    By default, search the current directory but a specific directory or iterable
    of directories can be specified, in which case it and (optionally) all its
    children will be searched.
    """
    if isinstance(start_from, basestring):
        paths = [start_from]
    else:
        paths = start_from

    for path in paths:
        if not os.path.exists(path):
            continue
        if recursive:
            for dirpath, dirnames, filenames in os.walk(path):
                for filename in filenames:
                    if fnmatch.fnmatch(filename, pattern):
                        yield os.path.join(dirpath, filename)
        else:
            for filepath in glob.glob(os.path.join(path, pattern)):
                yield filepath

def _where(pattern, start_from=None, recursive=False):
    """Search for a file whose name matches [pattern].

    By default, search the current directory and the PATH but a
    specific directory can be specified, in which case it and (optionally)
    all its children will be searched.
    """
    logger.info("Looking for %s in %s", pattern, start_from)
    if start_from is None:
        start_from = ["."] + [p for p in os.environ["PATH"].split(";")]
    for filepath in _file_operation(pattern, start_from, recursive):
        return filepath

def _delete(pattern, start_from=".", recursive=False, callback=None):
    """delete files who names match [pattern].

    By default, search the current directory but a specific directory
    can be specified. Recursion may be used but, by default, is not.
    """
    logger.info("About to delete %s from %s", pattern, start_from or ".")
    for filepath in _file_operation(pattern, start_from, recursive):
        os.unlink(filepath)
        if callback:
            callback(filepath)

def _rmdir(start_from, callback=None):
    """Remove a directory and everything underneath it
    """
    logger.info("About to rmdir %s", start_from)
    for dirpath, _, filenames in os.walk(start_from, topdown=False):
        for filename in filenames:
            filepath = os.path.join(dirpath, filename)
            if callback:
                callback.callback(filepath)
            os.unlink(filepath)
        os.rmdir(dirpath)

class Config(object):

    #
    # Defaults
    #
    configuration = "Debug"
    platform = "Win32"
    interactive = True
    externals = True

    def __init__(self):
        self.root = os.getcwd()
        self.here = os.path.dirname(os.path.abspath(__file__))
        try:
            self.branch = _check_output("hg branch")
        except subprocess.CalledProcessError:
            logger.warn("Can't determine hg branch; using default")
            self.branch = "default"
        parser = ConfigParser()
        parser.read([
            os.path.join(self.here, "configure.ini"),
            os.path.join(self.here, "configure.%s.ini" % self.branch),
            os.path.join(self.root, "configure.ini"),
        ])
        self.config = parser.get("configure", "configuration", fallback=self.__class__.configuration)
        self.platform = parser.get("configure", "platform", fallback=self.__class__.platform)
        self.visual_studio = os.path.abspath(os.environ[parser.get("configure", "envvar")])
        self.svnroot = parser.get("locations", "svnroot")
        self.externals = dict(parser.items("externals"))
        self.run_tests = shlex.split(parser.get("commands", "run_tests"))

        self.pcbuild = os.path.join(self.root, "PCBuild")
        self.externals_dir = os.path.abspath(os.path.join(self.pcbuild, "../.."))
        if self.platform == "x64":
            self.vcvarsarg = "x86_amd64"
            self.pybuilddir = os.path.join(self.pcbuild, "amd64")
        else:
            self.vcvarsarg = "x86"
            self.pybuilddir = self.pcbuild
        pybuildextension = "_d.exe" if self.configuration == "Debug" else ".exe"
        self.python_exe = os.path.join(self.pybuilddir, "python" + pybuildextension)

    def dumped(self):
        content = []
        content.append("{")
        for name in sorted(dir(self)):
            if name.startswith("_"):
                continue
            attr = getattr(self, name)
            if callable(attr):
                continue
            content.append("  %s => %r" % (name, attr))
        content.append("}")
        return "\n".join(content)

class Build(object):

    def __init__(self, config, stdout):
        self.config = config
        self.stdout = stdout

    @classmethod
    def valid_targets(cls):
        return set(m[3:] for m in dir(cls) if m.startswith("do_"))

    def _run_command(self, command, *args, **kwargs):
        p = subprocess.Popen(command, *args, stdout=self.stdout, stderr=subprocess.STDOUT, **kwargs)
        if p.wait() != 0:
            raise RuntimeError("Problem with command")

    def _find_interpreter(self):
        """Attempt to find an interpreter:

        * If a PYTHON3 env var is set, return its value
        * If a python.bat exists in the current directory, return that
        * If a python*.exe exists in the PCBuild directory, return that
        * If a py.exe exists any_where on the path, return that
        * Otherwise, return "python" and hope for the best

        NB This should not be run and cached, as the interpreter may not
        exist until it has been built.
        """
        if "PYTHON3" in os.environ:
            return os.environ["PYTHON3"]
        if os.path.exists(os.path.join(self.config.root, "python.bat")):
            return "python.bat"
        pcbuild_exe = _where("python*.exe", self.config.pcbuild, True)
        if pcbuild_exe:
            return pcbuild_exe
        py_exe = _where("py.exe")
        if py_exe:
            return py_exe
        return "python"

    def msbuild(self, project, target, **kwargs):
        vcvarsall = os.path.join(self.config.visual_studio, r"..\..\VC\vcvarsall.bat")
        params = ["/p:%s=%s" % (k, v) for (k, v) in kwargs.items()]
        msbuild = ["msbuild", "/target:%s" % target, project] + params
        logger.debug(msbuild)
        logger.info("Running msbuild %s", target)
        return self._run_command(["call", vcvarsall, self.config.vcvarsarg, "&&"] + msbuild, shell=True)

    def build_solution(self, target, **kwargs):
        return self.msbuild(r"PCbuild\pcbuild.sln", target, **kwargs)

    def check_externals(self):
        for name, version in self.config.externals.items():
            target_dirpath = os.path.abspath(os.path.join(self.config.externals_dir, version))
            if not os.path.exists(target_dirpath):
                logger.warn("External %s is not in place", version)

    def do_all(self):
        self.do_test(self.do_build(self.do_externals(self.do_clean())))

    def do_build(self, args):
        """Build CPython"""
        logger.info("Build & run kill_python")
        self.msbuild(
            r"PCbuild\kill_python.vcxproj",
            "Build",
            Config="Debug",
            PlatformTarget=self.config.platform
        )
        self._run_command([os.path.join(self.config.pybuilddir, "kill_python_d.exe")])
        _delete(self.config.python_exe)
        logger.info("Check externals")
        self.check_externals()
        logger.info("Build Python")
        self.build_solution(
            "Build",
            Config=self.config.configuration,
            Platform=self.config.platform
        )
        return args

    def do_clean(self, args):
        """Clean out build artifacts for current configuration"""
        logger.info("Deleting .pyc / .pyo files")
        _delete("*.pyc", self.config.root, True, callback=logger.debug)
        _delete("*.pyo", self.config.root, True, callback=logger.debug)
        _rmdir(os.path.join(self.config.root, "build"), callback=logger.debug)
        for configuration in "Release", "Debug":
            self.build_solution("clean", Config=configuration, Platform=self.config.platform)
        _delete("python.bat", callback=logger.debug)
        return args

    def do_clobber(self, args):
        """Clean out all build artifacts and remove externals"""
        self.do_clean()
        _delete("tcl*.dll", "PCbuild", recursive=True)
        _delete("tk*.dll", "PCbuild", recursive=True)
        for name, version in self.config.externals.items():
            target_dirpath = os.path.abspath(os.path.join(self.config.externals_dir, version))
            _rmdir(target_dirpath, callback=logger.debug)
        return args

    def do_externals(self, args):
        """Fetch external libraries in preparation for building"""
        for name, version in self.config.externals.items():
            target_dirpath = os.path.join(self.config.externals_dir, version)
            if os.path.exists(target_dirpath):
                logger.warn("Not fetching %s; already exists", version)
            else:
                logger.info("Fetching %s into %s", version, target_dirpath)
                self._run_command(["svn", "export", "%s/%s" % (self.config.svnroot, version), target_dirpath])
        return args

    def do_importlib(self, args):
        """Build and run the _freeze_importlib project"""
        already_exists = set(glob.glob(r"PCBuild\python*.exe"))
        self.msbuild(
            r"PCbuild\_freeze_importlib.vcxproj",
            "Build",
            Config="Release",
            PlatformTarget=self.config.platform
        )
        for python_exe in glob.glob(r"PCBuild\python*.exe"):
            if python_exe not in already_exists:
                logger.warn("Removing %s because build artefact", python_exe)
                os.unlink(python_exe)
        return args

    def do_patchcheck(self, args):
        r"""Run Tools\scripts\patchcheck.py"""
        python_exe = os.path.abspath(self._find_interpreter())
        self._run_command([python_exe, "tools/scripts/patchcheck.py"])
        return args

    def do_test(self, args):
        """Test Python"""
        python_exe = os.path.abspath(self._find_interpreter())
        #
        # Consume all remaining test args
        #
        self._run_command([python_exe] + self.config.run_tests + args)
        return []

    def run_from_args(self, args):
        if args:
            targets = list(args)
        else:
            targets = ["build"]

        while targets:
            target = targets.pop(0)
            logger.info("Executing %s", target)
            function = getattr(self, "do_" + target, None)
            if function:
                targets = function(targets)
            else:
                logger.warn("Unknown target: %s; ignoring...", target)

def main(*args):
    output_stream = sys.stdout
    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
    logger.setLevel(level=logging.INFO)
    screen = logging.StreamHandler(output_stream)
    screen.setFormatter(formatter)
    logger.addHandler(screen)

    config = Config()
    logger.info(config.dumped())
    Build(config, output_stream).run_from_args(args)

if __name__ == '__main__':
    main(*sys.argv[1:])
<|endoftext|>


<commit_before>import __builtin__
import maps
import mock
from mock import patch
import pytest

import etcd
from etcd import Client

from tendrl.commons.flows.configure_monitoring import ConfigureMonitoring
from tendrl.commons.flows.exceptions import FlowExecutionFailedError
from tendrl.commons.utils import etcd_utils
from tendrl.commons.objects.cluster import Cluster

# basic flow attributes from test_flows_init.py
def get_obj_definition(*args, **kwargs):
    def_obj = maps.NamedDict(
        {
            'attrs': {
                'integration_id': {
                    'type': 'String',
                    'help': 'Tendrl managed/generated cluster id for the sds '
                            'being managed by Tendrl'},
                'cluster_name': {
                    'type': 'String',
                    'help': 'Name of the cluster'},
                'node_id': {
                    'type': 'String',
                    'help': 'Tendrl ID for the managed node'},
                'cluster_id': {
                    'type': 'String',
                    'help': 'UUID of the cluster'},
                'sds_version': {
                    'type': 'String',
                    'help': "Version of the Tendrl managed sds, eg: '3.2.1'"},
                'sds_name': {
                    'type': 'String',
                    'help': "Name of the Tendrl managed sds, eg: 'gluster'"}},
            'help': 'Tendrl context',
            'obj_list': '',
            'enabled': True,
            'obj_value': 'nodes/$NodeContext.node_id/TendrlContext',
            'flows': {},
            'atoms': {}})
    def_obj.flows["ImportCluster"] = {
        'help': 'Tendrl context',
        'enabled': True,
        'type': 'test_type',
        'flows': {},
        'atoms': {},
        'inputs': 'test_input',
        'uuid': 'test_uuid'}
    return def_obj

@patch.object(Cluster, 'load')		# simulates 'load' method for 'Cluster' object
@patch.object(etcd_utils, 'read')	# simulates 'read' method from 'etcd_utils'
def test_run_fail(patch_cluster_load, patch_etcd_utils_read):
	setattr(__builtin__, "NS", maps.NamedDict())
	setattr(NS, "_int", maps.NamedDict())
	NS.publisher_id = "node_context"
	NS.tendrl = maps.NamedDict()
	NS.tendrl_context = maps.NamedDict()
	NS.tendrl_context.integration_id = "test_integration_id"
	NS.tendrl.objects = maps.NamedDict()
	NS.tendrl.objects.Cluster = Cluster

	configure_obj = ConfigureMonitoring()

	# should fail: unmanaged cluster
	with pytest.raises(FlowExecutionFailedError):
		configure_obj.run()

@patch.object(Cluster, 'load')		# simulates 'load' method for 'Cluster' object
@patch.object(etcd_utils, 'read')	# simulates 'read' method from 'etcd_utils'
def test_run_success(patch_cluster_load, patch_etcd_utils_read):
	setattr(__builtin__, "NS", maps.NamedDict())
	setattr(NS, "_int", maps.NamedDict())
	NS.publisher_id = "node_context"
	NS.tendrl = maps.NamedDict()
	NS.tendrl_context = maps.NamedDict()
	NS.tendrl_context.integration_id = "test_integration_id"
	NS.tendrl.objects = maps.NamedDict()
	NS.tendrl.objects.Cluster = Cluster

	# simulates 'read' op to force 'is_managed' branch
	patch_etcd_utils_read.return_value = maps.NamedDict(
		integration_id = "test_integration_id",
		is_managed = "yes"
	)

	configure_obj = ConfigureMonitoring()
	configure_obj._defs = get_obj_definition()	# object cannot do final initialization without this

	# should succeed: cluster is managed
	configure_obj.run()

# import __builtin__
# import maps
# import mock
# from mock import patch
# import pytest

# import etcd
# from etcd import Client

# from tendrl.commons.flows.configure_monitoring import ConfigureMonitoring
# from tendrl.commons.flows.exceptions import FlowExecutionFailedError
# from tendrl.commons.utils import etcd_utils
# from tendrl.commons.objects.cluster import Cluster

# def get_obj_definition(*args, **kwargs):
#     def_obj = maps.NamedDict(
#         {
#             'attrs': {
#                 'integration_id': {
#                     'type': 'String',
#                     'help': 'Tendrl managed/generated cluster id for the sds '
#                             'being managed by Tendrl'},
#                 'cluster_name': {
#                     'type': 'String',
#                     'help': 'Name of the cluster'},
#                 'node_id': {
#                     'type': 'String',
#                     'help': 'Tendrl ID for the managed node'},
#                 'cluster_id': {
#                     'type': 'String',
#                     'help': 'UUID of the cluster'},
#                 'sds_version': {
#                     'type': 'String',
#                     'help': "Version of the Tendrl managed sds, eg: '3.2.1'"},
#                 'sds_name': {
#                     'type': 'String',
#                     'help': "Name of the Tendrl managed sds, eg: 'gluster'"}},
#             'help': 'Tendrl context',
#             'obj_list': '',
#             'enabled': True,
#             'obj_value': 'nodes/$NodeContext.node_id/TendrlContext',
#             'flows': {},
#             'atoms': {}})
#     def_obj.flows["ImportCluster"] = {
#         'help': 'Tendrl context',
#         'enabled': True,
#         'type': 'test_type',
#         'flows': {},
#         'atoms': {},
#         'inputs': 'test_input',
#         'uuid': 'test_uuid'}
#     return def_obj

# @patch.object(Cluster, 'load')
# @patch.object(etcd_utils, 'read')
# def test_run_fail(patch_cluster_load, patch_etcd_utils_read):
# 	setattr(__builtin__, "NS", maps.NamedDict())
# 	setattr(NS, "_int", maps.NamedDict())
# 	NS.publisher_id = "node_context"
# 	NS.tendrl = maps.NamedDict()
# 	NS.tendrl_context = maps.NamedDict()
# 	NS.tendrl_context.integration_id = "test_integration_id"
# 	NS.tendrl.objects = maps.NamedDict()
# 	NS.tendrl.objects.Cluster = Cluster

# 	configure_obj = ConfigureMonitoring()

# 	# should fail: unmanaged cluster
# 	with pytest.raises(FlowExecutionFailedError):
# 		configure_obj.run()

# @patch.object(Cluster, 'load')
# @patch.object(etcd_utils, 'read')
# def test_run_success(patch_cluster_load, patch_etcd_utils_read):
# 	setattr(__builtin__, "NS", maps.NamedDict())
# 	setattr(NS, "_int", maps.NamedDict())
# 	NS.publisher_id = "node_context"
# 	NS.tendrl = maps.NamedDict()
# 	NS.tendrl_context = maps.NamedDict()
# 	NS.tendrl_context.integration_id = "test_integration_id"
# 	NS.tendrl.objects = maps.NamedDict()
# 	NS.tendrl.objects.Cluster = Cluster

# 	patch_etcd_utils_read.return_value = maps.NamedDict(
# 		# value = '{"current_job": '
# 		# 		'{"status": "finished",'
# 		# 		'"job_id": "test_job_id",'
# 		# 		'"job_name": "ImportCluster"'
# 		# 		'},'
# 				# '{"status": "",'
# 				# '"short_name": "t2",'
# 				# '"volume_profiling_flag": "enable",'
# 				# '"conf_overrides": "",'
# 		integration_id = "test_integration_id",
# 				# '"errors": "[]",'
# 				# '"node_configuration": "",'
# 				# '"locked_by": {},'
# 				# '"last_sync": "2018-06-07 13:25:25.943914+00:00",'
# 				# '"volume_profiling_state": "enabled",'
# 				# '"public_network": "",'
# 		is_managed = "yes"
# 				# '"node_identifier": "[]",'
# 				# '"cluster_network": "127.0.0.1/24"}'
# 	)

# 	configure_obj = ConfigureMonitoring()
# 	configure_obj._defs = get_obj_definition()

# 	# should succeed: cluster is managed
# 	configure_obj.run()
<commit_msg>Cleaned up file<commit_after>import __builtin__
import maps
import mock
from mock import patch
import pytest

import etcd
from etcd import Client

from tendrl.commons.flows.configure_monitoring import ConfigureMonitoring
from tendrl.commons.flows.exceptions import FlowExecutionFailedError
from tendrl.commons.utils import etcd_utils
from tendrl.commons.objects.cluster import Cluster

# basic flow attributes from test_flows_init.py
def get_obj_definition(*args, **kwargs):
    def_obj = maps.NamedDict(
        {
            'attrs': {
                'integration_id': {
                    'type': 'String',
                    'help': 'Tendrl managed/generated cluster id for the sds '
                            'being managed by Tendrl'},
                'cluster_name': {
                    'type': 'String',
                    'help': 'Name of the cluster'},
                'node_id': {
                    'type': 'String',
                    'help': 'Tendrl ID for the managed node'},
                'cluster_id': {
                    'type': 'String',
                    'help': 'UUID of the cluster'},
                'sds_version': {
                    'type': 'String',
                    'help': "Version of the Tendrl managed sds, eg: '3.2.1'"},
                'sds_name': {
                    'type': 'String',
                    'help': "Name of the Tendrl managed sds, eg: 'gluster'"}},
            'help': 'Tendrl context',
            'obj_list': '',
            'enabled': True,
            'obj_value': 'nodes/$NodeContext.node_id/TendrlContext',
            'flows': {},
            'atoms': {}})
    def_obj.flows["ImportCluster"] = {
        'help': 'Tendrl context',
        'enabled': True,
        'type': 'test_type',
        'flows': {},
        'atoms': {},
        'inputs': 'test_input',
        'uuid': 'test_uuid'}
    return def_obj

@patch.object(Cluster, 'load')		# simulates 'load' method for 'Cluster' object
@patch.object(etcd_utils, 'read')	# simulates 'read' method from 'etcd_utils'
def test_run_fail(patch_cluster_load, patch_etcd_utils_read):
	setattr(__builtin__, "NS", maps.NamedDict())
	setattr(NS, "_int", maps.NamedDict())
	NS.publisher_id = "node_context"
	NS.tendrl = maps.NamedDict()
	NS.tendrl_context = maps.NamedDict()
	NS.tendrl_context.integration_id = "test_integration_id"
	NS.tendrl.objects = maps.NamedDict()
	NS.tendrl.objects.Cluster = Cluster

	configure_obj = ConfigureMonitoring()

	# should fail: unmanaged cluster
	with pytest.raises(FlowExecutionFailedError):
		configure_obj.run()

@patch.object(Cluster, 'load')		# simulates 'load' method for 'Cluster' object
@patch.object(etcd_utils, 'read')	# simulates 'read' method from 'etcd_utils'
def test_run_success(patch_cluster_load, patch_etcd_utils_read):
	setattr(__builtin__, "NS", maps.NamedDict())
	setattr(NS, "_int", maps.NamedDict())
	NS.publisher_id = "node_context"
	NS.tendrl = maps.NamedDict()
	NS.tendrl_context = maps.NamedDict()
	NS.tendrl_context.integration_id = "test_integration_id"
	NS.tendrl.objects = maps.NamedDict()
	NS.tendrl.objects.Cluster = Cluster

	# simulates 'read' op to force 'is_managed' branch
	patch_etcd_utils_read.return_value = maps.NamedDict(
		integration_id = "test_integration_id",
		is_managed = "yes"
	)

	configure_obj = ConfigureMonitoring()
	configure_obj._defs = get_obj_definition()	# object cannot do final initialization without this

	# should succeed: cluster is managed
	configure_obj.run()<|endoftext|>


<commit_before>import ConfigParser
from gevent import socket
import os
import shlex
import subprocess
from subprocess import Popen
import time
import traceback

import collectd

from tendrl.commons.utils import ini2json

GLUSTERD_ERROR_MSG = 'Connection failed. '\
    'Please check if gluster daemon is operational.'

def get_brick_state_mapping(status):
    return {
        'Started': 0,
        'Stopped': 2
    }.get(status, 1)

def get_volume_state_mapping(status):
    return {
        'Started': 0,
        'Stopped': 2
    }.get(status, 1)

def exec_command(cmd_str):
    try:
        cmd = Popen(
            shlex.split(
                cmd_str
            ),
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            stdin=open(os.devnull, "r"),
            close_fds=True
        )
        stdout, stderr = cmd.communicate()
        if stderr:
            return None, stderr
        return stdout, None
    except (subprocess.CalledProcessError, ValueError):
        except_traceback = traceback.format_exc()
        return None, except_traceback

def get_gluster_state_dump():
    global GLUSTERD_ERROR_MSG
    ret_val = {}
    try:
        gluster_state_dump_op, gluster_state_dump_err = exec_command(
            'gluster get-state glusterd odir /var/run file collectd_gstate'
        )
        if (
            gluster_state_dump_err or
            GLUSTERD_ERROR_MSG in gluster_state_dump_op
        ):
            return ret_val, gluster_state_dump_err
        gluster_state_dump = ini2json.ini_to_dict(
            '/var/run/collectd_gstate'
        )
        rm_state_dump, rm_state_dump_err = exec_command(
            'rm -rf /var/run/collectd_gstate'
        )
        if rm_state_dump_err:
            return ret_val, rm_state_dump_err
        return gluster_state_dump, None
    except (
        IOError,
        AttributeError,
        ConfigParser.MissingSectionHeaderError,
        ConfigParser.ParsingError,
        ValueError
    ):
        return ret_val, traceback.format_exc()

def parse_get_state(get_state_json):
    cluster_topology = {}
    if "Peers" in get_state_json:
        cluster_peers = []
        processed_peer_indexes = []
        peers = get_state_json['Peers']
        for key, value in peers.iteritems():
            peer_index = key.split('.')[0].split('peer')[1]
            if peer_index not in processed_peer_indexes:
                cluster_peers.append({
                    'host_name': peers[
                        'peer%s.primary_hostname' % peer_index
                    ],
                    'uuid': peers[
                        'peer%s.uuid' % peer_index
                    ],
                    'state': peers[
                        'peer%s.state' % peer_index
                    ],
                    'connected': peers[
                        'peer%s.connected' % peer_index
                    ]
                })
                processed_peer_indexes.append(peer_index)
                cluster_topology['peers'] = cluster_peers
    if "Volumes" in get_state_json:
        processed_vol_indexes = []
        vols = []
        volumes = get_state_json['Volumes']
        for key, value in volumes.iteritems():
            vol_index = key.split('.')[0].split('volume')[1]
            if vol_index not in processed_vol_indexes:
                volume = {
                    'name': volumes['volume%s.name' % vol_index],
                    'id': volumes['volume%s.id' % vol_index],
                    'type': volumes['volume%s.type' % vol_index],
                    'transport_type': volumes[
                        'volume%s.transport_type' % vol_index
                    ],
                    'status': volumes['volume%s.status' % vol_index],
                    'brick_count': volumes['volume%s.brickcount' % vol_index],
                    'snap_count': volumes['volume%s.snap_count' % vol_index],
                    'stripe_count': volumes[
                        'volume%s.stripe_count' % vol_index
                    ],
                    'replica_count': volumes[
                        'volume%s.replica_count' % vol_index
                    ],
                    'subvol_count': volumes[
                        'volume%s.subvol_count' % vol_index
                    ],
                    'arbiter_count': volumes[
                        'volume%s.arbiter_count' % vol_index
                    ],
                    'disperse_count': volumes[
                        'volume%s.disperse_count' % vol_index
                    ],
                    'redundancy_count': volumes[
                        'volume%s.redundancy_count' % vol_index
                    ],
                    'quorum_status': volumes[
                        'volume%s.quorum_status' % vol_index
                    ],
                    'rebalance_status': volumes[
                        'volume%s.rebalance.status' % vol_index
                    ],
                    'rebalance_failures': volumes[
                        'volume%s.rebalance.failures' % vol_index
                    ],
                    'rebalance_skipped': volumes[
                        'volume%s.rebalance.skipped' % vol_index
                    ],
                    'rebalance_files': volumes[
                        'volume%s.rebalance.files' % vol_index
                    ],
                    'rebalance_data': volumes[
                        'volume%s.rebalance.data' % vol_index
                    ]
                }
                sub_volumes = {}
                no_of_bricks = int(volume['brick_count'])
                for brick_index in range(1, no_of_bricks + 1):
                    brick = {
                        'hostname': volumes[
                            'volume%s.brick%s.hostname' % (
                                vol_index,
                                brick_index
                            )
                        ],
                        'path': volumes[
                            'volume%s.brick%s.path' % (
                                vol_index,
                                brick_index
                            )
                        ].split(':')[1],
                    }
                    brick_status_key = 'Volume%s.Brick%s.status' % (
                        vol_index,
                        brick_index
                    )
                    if brick_status_key in volumes:
                        brick['status'] = volumes[brick_status_key]
                    sub_vol_index = (
                        (brick_index - 1) * int(volume['subvol_count'])
                    ) / no_of_bricks
                    sub_vol_bricks = sub_volumes.get(str(sub_vol_index), [])
                    sub_vol_bricks.append(brick)
                    sub_volumes[str(sub_vol_index)] = sub_vol_bricks
                volume['bricks'] = sub_volumes
                vols.append(volume)
                cluster_topology['volumes'] = vols
                processed_vol_indexes.append(vol_index)
    return cluster_topology

def get_gluster_cluster_topology():
    gluster_state_dump_json, gluster_state_dump_err = get_gluster_state_dump()
    if gluster_state_dump_err:
        collectd.error(
            'Failed to fetch cluster topology. Error %s' % (
                gluster_state_dump_err
            )
        )
        return {}
    try:
        cluster_topology = parse_get_state(
            gluster_state_dump_json
        )
        return cluster_topology
    except (KeyError, AttributeError):
        collectd.error(
            'Failed to fetch cluster topology. Error %s' % (
                traceback.format_exc()
            )
        )
        return {}

# The below along with write_graphite function have to go into a custom
# write plugin to overcome shortcomings of write_graphite plugin
# And calls to write_grapite have to be replaced with collectd#Values
# i.e, send_metric need to be used..
prefix = 'tendrl'
delimeter = '.'

def write_graphite(path, value, graphite_host, graphite_port):
    try:
        graphite_sock = socket.socket()
        graphite_sock.connect(
            (
                graphite_host,
                int(graphite_port)
            )
        )
        message = '%s%s%s %s %d\n' % (
            prefix,
            delimeter,
            path,
            str(value),
            int(time.time())
        )
        graphite_sock.sendall(message)
        graphite_sock.close()
    except (
        socket.error,
        socket.gaierror
    ):
        collectd.error(
            'Failed to push brick stat for %s.Error %s' % (
                path,
                traceback.format_exc()
            )
        )

def send_metric(
    plugin_name,
    metric_type,
    instance_name,
    value,
    integration_id,
    plugin_instance=None
):
    metric = collectd.Values()
    metric.plugin = plugin_name
    metric.host = "cluster_%s" % integration_id
    metric.type = metric_type
    metric.values = [value]
    metric.type_instance = instance_name
    if plugin_instance:
        metric.plugin_instance = plugin_instance
    metric.dispatch()
<commit_msg>Fix get-state output race condition<commit_after>import ConfigParser
from gevent import socket
import os
import shlex
import subprocess
from subprocess import Popen
import time
import traceback
import uuid

import collectd

from tendrl.commons.utils import ini2json

GLUSTERD_ERROR_MSG = 'Connection failed. '\
    'Please check if gluster daemon is operational.'

def get_brick_state_mapping(status):
    return {
        'Started': 0,
        'Stopped': 2
    }.get(status, 1)

def get_volume_state_mapping(status):
    return {
        'Started': 0,
        'Stopped': 2
    }.get(status, 1)

def exec_command(cmd_str):
    try:
        cmd = Popen(
            shlex.split(
                cmd_str
            ),
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE,
            stdin=open(os.devnull, "r"),
            close_fds=True
        )
        stdout, stderr = cmd.communicate()
        if stderr:
            return None, stderr
        return stdout, None
    except (subprocess.CalledProcessError, ValueError):
        except_traceback = traceback.format_exc()
        return None, except_traceback

def get_gluster_state_dump():
    global GLUSTERD_ERROR_MSG
    ret_val = {}
    try:
        file_name = "collectd_gstate_%s" % str(uuid.uuid4())
        gluster_state_dump_op, gluster_state_dump_err = exec_command(
            'gluster get-state glusterd odir /var/run file %s' % file_name
        )
        if (
            gluster_state_dump_err or
            GLUSTERD_ERROR_MSG in gluster_state_dump_op
        ):
            return ret_val, gluster_state_dump_err
        gluster_state_dump = ini2json.ini_to_dict(
            '/var/run/%s' % file_name
        )
        rm_state_dump, rm_state_dump_err = exec_command(
            'rm -rf /var/run/%s' % file_name
        )
        if rm_state_dump_err:
            return ret_val, rm_state_dump_err
        return gluster_state_dump, None
    except (
        IOError,
        AttributeError,
        ConfigParser.MissingSectionHeaderError,
        ConfigParser.ParsingError,
        ValueError
    ):
        return ret_val, traceback.format_exc()

def parse_get_state(get_state_json):
    cluster_topology = {}
    if "Peers" in get_state_json:
        cluster_peers = []
        processed_peer_indexes = []
        peers = get_state_json['Peers']
        for key, value in peers.iteritems():
            peer_index = key.split('.')[0].split('peer')[1]
            if peer_index not in processed_peer_indexes:
                cluster_peers.append({
                    'host_name': peers[
                        'peer%s.primary_hostname' % peer_index
                    ],
                    'uuid': peers[
                        'peer%s.uuid' % peer_index
                    ],
                    'state': peers[
                        'peer%s.state' % peer_index
                    ],
                    'connected': peers[
                        'peer%s.connected' % peer_index
                    ]
                })
                processed_peer_indexes.append(peer_index)
                cluster_topology['peers'] = cluster_peers
    if "Volumes" in get_state_json:
        processed_vol_indexes = []
        vols = []
        volumes = get_state_json['Volumes']
        for key, value in volumes.iteritems():
            vol_index = key.split('.')[0].split('volume')[1]
            if vol_index not in processed_vol_indexes:
                volume = {
                    'name': volumes['volume%s.name' % vol_index],
                    'id': volumes['volume%s.id' % vol_index],
                    'type': volumes['volume%s.type' % vol_index],
                    'transport_type': volumes[
                        'volume%s.transport_type' % vol_index
                    ],
                    'status': volumes['volume%s.status' % vol_index],
                    'brick_count': volumes['volume%s.brickcount' % vol_index],
                    'snap_count': volumes['volume%s.snap_count' % vol_index],
                    'stripe_count': volumes[
                        'volume%s.stripe_count' % vol_index
                    ],
                    'replica_count': volumes[
                        'volume%s.replica_count' % vol_index
                    ],
                    'subvol_count': volumes[
                        'volume%s.subvol_count' % vol_index
                    ],
                    'arbiter_count': volumes[
                        'volume%s.arbiter_count' % vol_index
                    ],
                    'disperse_count': volumes[
                        'volume%s.disperse_count' % vol_index
                    ],
                    'redundancy_count': volumes[
                        'volume%s.redundancy_count' % vol_index
                    ],
                    'quorum_status': volumes[
                        'volume%s.quorum_status' % vol_index
                    ],
                    'rebalance_status': volumes[
                        'volume%s.rebalance.status' % vol_index
                    ],
                    'rebalance_failures': volumes[
                        'volume%s.rebalance.failures' % vol_index
                    ],
                    'rebalance_skipped': volumes[
                        'volume%s.rebalance.skipped' % vol_index
                    ],
                    'rebalance_files': volumes[
                        'volume%s.rebalance.files' % vol_index
                    ],
                    'rebalance_data': volumes[
                        'volume%s.rebalance.data' % vol_index
                    ]
                }
                sub_volumes = {}
                no_of_bricks = int(volume['brick_count'])
                for brick_index in range(1, no_of_bricks + 1):
                    brick = {
                        'hostname': volumes[
                            'volume%s.brick%s.hostname' % (
                                vol_index,
                                brick_index
                            )
                        ],
                        'path': volumes[
                            'volume%s.brick%s.path' % (
                                vol_index,
                                brick_index
                            )
                        ].split(':')[1],
                    }
                    brick_status_key = 'Volume%s.Brick%s.status' % (
                        vol_index,
                        brick_index
                    )
                    if brick_status_key in volumes:
                        brick['status'] = volumes[brick_status_key]
                    sub_vol_index = (
                        (brick_index - 1) * int(volume['subvol_count'])
                    ) / no_of_bricks
                    sub_vol_bricks = sub_volumes.get(str(sub_vol_index), [])
                    sub_vol_bricks.append(brick)
                    sub_volumes[str(sub_vol_index)] = sub_vol_bricks
                volume['bricks'] = sub_volumes
                vols.append(volume)
                cluster_topology['volumes'] = vols
                processed_vol_indexes.append(vol_index)
    return cluster_topology

def get_gluster_cluster_topology():
    gluster_state_dump_json, gluster_state_dump_err = get_gluster_state_dump()
    if gluster_state_dump_err:
        collectd.error(
            'Failed to fetch cluster topology. Error %s' % (
                gluster_state_dump_err
            )
        )
        return {}
    try:
        cluster_topology = parse_get_state(
            gluster_state_dump_json
        )
        return cluster_topology
    except (KeyError, AttributeError):
        collectd.error(
            'Failed to fetch cluster topology. Error %s' % (
                traceback.format_exc()
            )
        )
        return {}

# The below along with write_graphite function have to go into a custom
# write plugin to overcome shortcomings of write_graphite plugin
# And calls to write_grapite have to be replaced with collectd#Values
# i.e, send_metric need to be used..
prefix = 'tendrl'
delimeter = '.'

def write_graphite(path, value, graphite_host, graphite_port):
    try:
        graphite_sock = socket.socket()
        graphite_sock.connect(
            (
                graphite_host,
                int(graphite_port)
            )
        )
        message = '%s%s%s %s %d\n' % (
            prefix,
            delimeter,
            path,
            str(value),
            int(time.time())
        )
        graphite_sock.sendall(message)
        graphite_sock.close()
    except (
        socket.error,
        socket.gaierror
    ):
        collectd.error(
            'Failed to push brick stat for %s.Error %s' % (
                path,
                traceback.format_exc()
            )
        )

def send_metric(
    plugin_name,
    metric_type,
    instance_name,
    value,
    integration_id,
    plugin_instance=None
):
    metric = collectd.Values()
    metric.plugin = plugin_name
    metric.host = "cluster_%s" % integration_id
    metric.type = metric_type
    metric.values = [value]
    metric.type_instance = instance_name
    if plugin_instance:
        metric.plugin_instance = plugin_instance
    metric.dispatch()
<|endoftext|>


<commit_before>"""
This module is the add command of bowl.

Created on 26 May 2014
@author: Charlie Lewis
"""
import ast
import os

from bowl.cli_opts import link

class Object(object):
    pass

class add(object):
    """
    This class is responsible for the add command of the cli.
    """
    @classmethod
    def main(self, args):
        if args.repository:
            # !! TODO connect to api of repository
            print args.repository
            print "TODO"
        else:
            link_args = Object()
            link_args.z = True
            link_args.metadata_path = args.metadata_path
            link_args.SERVICE_HOST = "localhost"
            if link.link.main(link_args):
                directory = os.path.join(args.metadata_path, "services")
                directory = os.path.expanduser(directory)
                if not os.path.exists(directory):
                    os.makedirs(directory)
                directory_os = os.path.join(directory, args.OS)
                if not os.path.exists(directory_os):
                    os.makedirs(directory_os)
                directory_version = os.path.join(directory_os, args.VERSION)
                if not os.path.exists(directory_version):
                    os.makedirs(directory_version)
                directory_type = os.path.join(directory_version, args.TYPE)
                if not os.path.exists(directory_type):
                    os.makedirs(directory_type)
                directory_docker = os.path.join(directory_type, "dockerfiles")
                if not os.path.exists(directory_docker):
                    os.makedirs(directory_docker)
                directory_name = os.path.join(directory_docker, args.NAME)
                if not os.path.exists(directory_name):
                    os.makedirs(directory_name)

                # !! TODO !!!!!
                try:
                    with open(os.path.join(directory, "services"), 'a') as f:
                        f.write("{" +
                                "'title': 'localhost'," +
                                " 'type': 'choice_menu'" +
                                "}\n")
                except:
                    print "unable to add service"
            else:
                print "unable to link localhost as a repository"

        # !! TODO
        # check JSON for path
        # check JSON for "" of key/values
        # check JSON for '' of key/values
        # check JSON for no quoting of key/values
        # check all necessary fields in JSON are there
        # check JSON dependencies like background_cmd only if combine_cmd is true, etc.

        # !! TODO
        # get DOckerfile at path, but also everything in the context of that directory

        # !! TODO
        # add to new container directory (not .default)
        # ask the user if they would like to use .default for default services
        # ask the user if they would like remove .default from their services

        # !! TODO
        # mkdir ~/.bowl/services
        # contains a file that says what all of the service directories are
        #    most likely .default and/or ~/.bowl/services/
        # contains added services, unless specified to be somewhere else

        # !! TODO
        # add service to another repository ...

        # !! TODO
        print args
<commit_msg>cleaning up add, more to do<commit_after>"""
This module is the add command of bowl.

Created on 26 May 2014
@author: Charlie Lewis
"""
import ast
import fileinput
import os
import sys

from bowl.cli_opts import link

class Object(object):
    pass

class add(object):
    """
    This class is responsible for the add command of the cli.
    """
    @classmethod
    def main(self, args):
        if args.repository:
            # !! TODO connect to api of repository
            print args.repository
            print "TODO"
        else:
            link_args = Object()
            link_args.z = True
            link_args.metadata_path = args.metadata_path
            link_args.SERVICE_HOST = "localhost"
            if link.link.main(link_args):
                directory = os.path.join(args.metadata_path, "services")
                directory = os.path.expanduser(directory)
                if not os.path.exists(directory):
                    os.makedirs(directory)
                directory_os = os.path.join(directory, args.OS)
                if not os.path.exists(directory_os):
                    os.makedirs(directory_os)
                directory_version = os.path.join(directory_os, args.VERSION)
                if not os.path.exists(directory_version):
                    os.makedirs(directory_version)
                directory_databases = os.path.join(directory_version, "databases")
                if not os.path.exists(directory_databases):
                    os.makedirs(directory_databases)
                directory_docker = os.path.join(directory_databases, "dockerfiles")
                if not os.path.exists(directory_docker):
                    os.makedirs(directory_docker)
                directory_environment = os.path.join(directory_version, "environment")
                if not os.path.exists(directory_environment):
                    os.makedirs(directory_environment)
                directory_docker = os.path.join(directory_environment, "dockerfiles")
                if not os.path.exists(directory_docker):
                    os.makedirs(directory_docker)
                directory_services = os.path.join(directory_version, "services")
                if not os.path.exists(directory_services):
                    os.makedirs(directory_services)
                directory_docker = os.path.join(directory_services, "dockerfiles")
                if not os.path.exists(directory_docker):
                    os.makedirs(directory_docker)
                directory_tools = os.path.join(directory_version, "tools")
                if not os.path.exists(directory_tools):
                    os.makedirs(directory_tools)
                directory_docker = os.path.join(directory_tools, "dockerfiles")
                if not os.path.exists(directory_docker):
                    os.makedirs(directory_docker)
                directory_name = os.path.join(directory_docker, args.NAME)
                if not os.path.exists(directory_name):
                    os.makedirs(directory_name)

                try:
                    if os.path.exists(os.path.join(directory, "oses")):
                        os_name = "\""+args.OS+"\":"
                        if not os_name in open(os.path.join(directory, "oses")).read():
                            num_lines = sum(1 for line in open(os.path.join(directory, "oses")))
                            for line_number, line in enumerate(fileinput.input(os.path.join(directory, "oses"), inplace=1)):
                                try:
                                    if line_number == num_lines-2:
                                        sys.stdout.write(line.rstrip('\n')+",\n")
                                    elif line_number == num_lines-1:
                                        sys.stdout.write("\n")
                                    else:
                                        sys.stdout.write(line)
                                except:
                                    print "Malformed oses file, not enough lines"
                            with open(os.path.join(directory, "oses"), 'a') as f:
                                f.write(" \""+args.OS+"\":\n" +
                                        " {\n" +
                                        "  \"title\": \""+args.OS.capitalize()+"\",\n" +
                                        "  \"type\": \"menu\",\n" +
                                        "  \"subtitle\": \"Please select a version...\",\n" +
                                        "  \"object\": \"os\",\n" +
                                        "  \"options\": []\n" +
                                        " }\n" +
                                        "}")
                    else:
                        with open(os.path.join(directory, "oses"), 'a') as f:
                            f.write("{\n" +
                                    " \""+args.OS+"\":\n" +
                                    " {\n" +
                                    "  \"title\": \""+args.OS.capitalize()+"\",\n" +
                                    "  \"type\": \"menu\",\n" +
                                    "  \"subtitle\": \"Please select a version...\",\n" +
                                    "  \"object\": \"os\",\n" +
                                    "  \"options\": []\n" +
                                    " }\n" +
                                    "}")
                except:
                    print "unable to add operating systems"
                try:
                    directory = os.path.join(directory, args.OS)
                    if os.path.exists(os.path.join(directory, "versions")):
                        version_name = "\""+args.VERSION+"\":"
                        if not version_name in open(os.path.join(directory, "versions")).read():
                            num_lines = sum(1 for line in open(os.path.join(directory, "versions")))
                            for line_number, line in enumerate(fileinput.input(os.path.join(directory, "versions"), inplace=1)):
                                try:
                                    if line_number == num_lines-2:
                                        sys.stdout.write(line.rstrip('\n')+",\n")
                                    elif line_number == num_lines-1:
                                        sys.stdout.write("\n")
                                    else:
                                        sys.stdout.write(line)
                                except:
                                    print "Malformed version file, not enough lines"
                            with open(os.path.join(directory, "versions"), 'a') as f:
                                f.write(" \""+args.VERSION+"\":\n" +
                                        " {\n" +
                                        "  \"title\": \""+args.VERSION.capitalize()+"\",\n" +
                                        "  \"type\": \"menu\",\n" +
                                        "  \"subtitle\": \"Please select services...\",\n" +
                                        "  \"object\": \"version\",\n" +
                                        "  \"options\": []\n" +
                                        " }\n" +
                                        "}")
                    else:
                        with open(os.path.join(directory, "versions"), 'a') as f:
                            f.write("{\n" +
                                    " \""+args.VERSION+"\":\n" +
                                    " {\n" +
                                    "  \"title\": \""+args.VERSION.capitalize()+"\",\n" +
                                    "  \"type\": \"menu\",\n" +
                                    "  \"subtitle\": \"Please select services...\",\n" +
                                    "  \"object\": \"version\",\n" +
                                    "  \"options\": []\n" +
                                    " }\n" +
                                    "}")
                except:
                    print "unable to add versions"
                # !! TODO !!!!!
                if args.TYPE == 'database':
                    try:
                        if os.path.exists(os.path.join(directory_databases, "databases")):
                            db_name = "\""+args.NAME+"\":"
                            if not db_name in open(os.path.join(directory_databases, "databases")).read():
                                num_lines = sum(1 for line in open(os.path.join(directory_databases, "databases")))
                                for line_number, line in enumerate(fileinput.input(os.path.join(directory_databases, "databases"), inplace=1)):
                                    try:
                                        if line_number == num_lines-2:
                                            sys.stdout.write(line.rstrip('\n')+",\n")
                                        elif line_number == num_lines-1:
                                            sys.stdout.write("\n")
                                        else:
                                            sys.stdout.write(line)
                                    except:
                                        print "Malformed databases file, not enough lines"
                                with open(os.path.join(directory_databases, "databases"), 'a') as f:
                                    # !! TODO edd in args.JSON
                                    f.write(" \""+args.NAME+"\":\n" +
                                            " {\n" +
                                            "  \"title\": \""+args.NAME+"\",\n" +
                                            "  \"type\": \"choice_menu\",\n" +
                                            "  \"object\": \"database\",\n" +
                                            " }\n" +
                                            "}")
                        else:
                            with open(os.path.join(directory_databases, "databases"), 'a') as f:
                                # !! TODO edd in args.JSON
                                f.write("{\n" +
                                        " \""+args.NAME+"\":\n" +
                                        " {\n" +
                                        "  \"title\": \""+args.NAME+"\",\n" +
                                        "  \"type\": \"choice_menu\",\n" +
                                        "  \"object\": \"database\",\n" +
                                        " }\n" +
                                        "}")
                    except:
                        print "unable to add databases"
                else:
                    try:
                        if !os.path.exists(os.path.join(directory_databases, "databases")):
                            with open(os.path.join(directory_databases, "databases")) as f:
                                f.write("{\n}")
                    except:
                        print "unable to add databases file for that version"
            else:
                print "unable to link localhost as a repository"

        # !! TODO
        # check JSON for path
        # check JSON for "" of key/values
        # check JSON for '' of key/values
        # check JSON for no quoting of key/values
        # check all necessary fields in JSON are there
        # check JSON dependencies like background_cmd only if combine_cmd is true, etc.

        # !! TODO
        # get DOckerfile at path, but also everything in the context of that directory

        # !! TODO
        # add to new container directory (not .default)
        # ask the user if they would like to use .default for default services
        # ask the user if they would like remove .default from their services

        # !! TODO
        # mkdir ~/.bowl/services
        # contains a file that says what all of the service directories are
        #    most likely .default and/or ~/.bowl/services/
        # contains added services, unless specified to be somewhere else

        # !! TODO
        # add service to another repository ...

        # !! TODO
        print args
<|endoftext|>