<commit_before>from logging import StreamHandler
import os.path
from os.path import isdir
from sys import stderr
from time import time
from urllib import quote_plus

from flask import (Blueprint, Flask, send_from_directory, current_app,
                   send_file, request, redirect, jsonify, render_template)

from dxr.query import Query
from dxr.server_utils import connect_db
from dxr.utils import non_negative_int, search_url, sqlite3  # Make sure we load trilite before possibly importing the wrong version of sqlite3.

# Look in the 'dxr' package for static files, templates, etc.:
dxr_blueprint = Blueprint('dxr_blueprint', 'dxr')

def make_app(instance_path):
    """Return a DXR application which looks in the given folder for
    configuration.

    Also set up the static and template folder according to the configured
    template.

    """
    # TODO: Actually obey the template selection in the config file by passing
    # a different static_folder and template_folder to Flask().
    app = Flask('dxr', instance_path=instance_path)
    app.register_blueprint(dxr_blueprint)

    # Load the special config file generated by dxr-build:
    app.config.from_pyfile(os.path.join(app.instance_path, 'config.py'))

    # Log to Apache's error log in production:
    app.logger.addHandler(StreamHandler(stderr))
    return app

@dxr_blueprint.route('/')
def index():
    return send_file(current_app.open_instance_resource('trees/index.html'))

@dxr_blueprint.route('/<tree>/search')
def search(tree):
    """Search by regex, caller, superclass, or whatever."""
    # TODO: This function still does too much.
    querystring = request.values

    offset = non_negative_int(querystring.get('offset'), 0)
    limit = max(non_negative_int(querystring.get('limit'), 100), 1000)

    config = current_app.config

    # Arguments for the template:
    arguments = {
        # Common template variables
        'wwwroot': config['WWW_ROOT'],
        'tree': config['TREES'][0],
        'trees': config['TREES'],
        'config': config['TEMPLATE_PARAMETERS'],
        'generated_date': config['GENERATED_DATE']}

    error = warning = ''
    status_code = None

    if tree in config['TREES']:
        arguments['tree'] = tree

        # Connect to database
        conn = connect_db(tree, current_app.instance_path)
        if conn:
            # Parse the search query
            qtext = querystring.get('q', '')
            is_case_sensitive = querystring.get('case') == 'true'
            q = Query(conn,
                      qtext,
                      should_explain='explain' in querystring,
                      is_case_sensitive=is_case_sensitive)

            # Try for a direct result:
            if querystring.get('redirect') == 'true':
                result = q.direct_result()
                if result:
                    path, line = result
                    # TODO: Does this escape qtext properly?
                    return redirect(
                        '%s/%s/source/%s?from=%s%s#%i' %
                        (config['WWW_ROOT'],
                         tree,
                         path,
                         qtext,
                         '&case=true' if is_case_sensitive else '', line))

            # Return multiple results:
            template = 'search.html'
            start = time()
            try:
                results = list(q.results(offset, limit))
            except sqlite3.OperationalError as e:
                if e.message.startswith('REGEXP:'):
                    # Malformed regex
                    warning = e.message[7:]
                    results = []
                elif e.message.startswith('QUERY:'):
                    warning = e.message[6:]
                    results = []
                else:
                    error = 'Database error: %s' % e.message
            if not error:
                # Search template variables:
                arguments['time'] = time() - start
                arguments['query'] = qtext
                arguments['search_url'] = search_url(arguments['wwwroot'],
                                                     arguments['tree'],
                                                     qtext,
                                                     redirect=False)
                arguments['results'] = results
                arguments['offset'] = offset
                arguments['limit'] = limit
                arguments['is_case_sensitive'] = is_case_sensitive
        else:
            error = 'Failed to establish database connection.'
    else:
        error = "Tree '%s' is not a valid tree." % tree
        status_code = 404

    if warning or error:
        arguments['error'] = error or warning

    if querystring.get('format') == 'json':
        if error:
            # Return a non-OK code so the live search doesn't try to replace
            # the results with our empty ones:
            return jsonify(arguments), status_code or 500

        # Tuples are encoded as lists in JSON, and these are not real
        # easy to unpack or read in Javascript. So for ease of use, we
        # convert to dictionaries before returning the json results.
        # If further discrepancies are introduced, please document them in
        # templating.mkd.
        arguments['results'] = [
            {'icon': icon,
             'path': path,
             'lines': [{'line_number': nb, 'line': l} for nb, l in lines]}
                for icon, path, lines in arguments['results']]
        return jsonify(arguments)

    if error:
        return render_template('error.html', **arguments), status_code or 500
    else:
        return render_template('search.html', **arguments)

@dxr_blueprint.route('/<tree>/source/')
@dxr_blueprint.route('/<tree>/source/<path:path>')
def browse(tree, path=''):
    """Show a directory listing or a single file from one of the trees."""
    tree_folder = os.path.join(current_app.instance_path, 'trees', tree)

    if isdir(os.path.join(tree_folder, path)):
        # It's a bare directory. Add the index file to the end:
        path = os.path.join(path, current_app.config['DIRECTORY_INDEX'])
    else:
        # It's a file. Add the .html extension:
        path += '.html'

    return send_from_directory(tree_folder, path)
<commit_msg>Oops: min, not max<commit_after>from logging import StreamHandler
import os.path
from os.path import isdir
from sys import stderr
from time import time
from urllib import quote_plus

from flask import (Blueprint, Flask, send_from_directory, current_app,
                   send_file, request, redirect, jsonify, render_template)

from dxr.query import Query
from dxr.server_utils import connect_db
from dxr.utils import non_negative_int, search_url, sqlite3  # Make sure we load trilite before possibly importing the wrong version of sqlite3.

# Look in the 'dxr' package for static files, templates, etc.:
dxr_blueprint = Blueprint('dxr_blueprint', 'dxr')

def make_app(instance_path):
    """Return a DXR application which looks in the given folder for
    configuration.

    Also set up the static and template folder according to the configured
    template.

    """
    # TODO: Actually obey the template selection in the config file by passing
    # a different static_folder and template_folder to Flask().
    app = Flask('dxr', instance_path=instance_path)
    app.register_blueprint(dxr_blueprint)

    # Load the special config file generated by dxr-build:
    app.config.from_pyfile(os.path.join(app.instance_path, 'config.py'))

    # Log to Apache's error log in production:
    app.logger.addHandler(StreamHandler(stderr))
    return app

@dxr_blueprint.route('/')
def index():
    return send_file(current_app.open_instance_resource('trees/index.html'))

@dxr_blueprint.route('/<tree>/search')
def search(tree):
    """Search by regex, caller, superclass, or whatever."""
    # TODO: This function still does too much.
    querystring = request.values

    offset = non_negative_int(querystring.get('offset'), 0)
    limit = min(non_negative_int(querystring.get('limit'), 100), 1000)

    config = current_app.config

    # Arguments for the template:
    arguments = {
        # Common template variables
        'wwwroot': config['WWW_ROOT'],
        'tree': config['TREES'][0],
        'trees': config['TREES'],
        'config': config['TEMPLATE_PARAMETERS'],
        'generated_date': config['GENERATED_DATE']}

    error = warning = ''
    status_code = None

    if tree in config['TREES']:
        arguments['tree'] = tree

        # Connect to database
        conn = connect_db(tree, current_app.instance_path)
        if conn:
            # Parse the search query
            qtext = querystring.get('q', '')
            is_case_sensitive = querystring.get('case') == 'true'
            q = Query(conn,
                      qtext,
                      should_explain='explain' in querystring,
                      is_case_sensitive=is_case_sensitive)

            # Try for a direct result:
            if querystring.get('redirect') == 'true':
                result = q.direct_result()
                if result:
                    path, line = result
                    # TODO: Does this escape qtext properly?
                    return redirect(
                        '%s/%s/source/%s?from=%s%s#%i' %
                        (config['WWW_ROOT'],
                         tree,
                         path,
                         qtext,
                         '&case=true' if is_case_sensitive else '', line))

            # Return multiple results:
            template = 'search.html'
            start = time()
            try:
                results = list(q.results(offset, limit))
            except sqlite3.OperationalError as e:
                if e.message.startswith('REGEXP:'):
                    # Malformed regex
                    warning = e.message[7:]
                    results = []
                elif e.message.startswith('QUERY:'):
                    warning = e.message[6:]
                    results = []
                else:
                    error = 'Database error: %s' % e.message
            if not error:
                # Search template variables:
                arguments['time'] = time() - start
                arguments['query'] = qtext
                arguments['search_url'] = search_url(arguments['wwwroot'],
                                                     arguments['tree'],
                                                     qtext,
                                                     redirect=False)
                arguments['results'] = results
                arguments['offset'] = offset
                arguments['limit'] = limit
                arguments['is_case_sensitive'] = is_case_sensitive
        else:
            error = 'Failed to establish database connection.'
    else:
        error = "Tree '%s' is not a valid tree." % tree
        status_code = 404

    if warning or error:
        arguments['error'] = error or warning

    if querystring.get('format') == 'json':
        if error:
            # Return a non-OK code so the live search doesn't try to replace
            # the results with our empty ones:
            return jsonify(arguments), status_code or 500

        # Tuples are encoded as lists in JSON, and these are not real
        # easy to unpack or read in Javascript. So for ease of use, we
        # convert to dictionaries before returning the json results.
        # If further discrepancies are introduced, please document them in
        # templating.mkd.
        arguments['results'] = [
            {'icon': icon,
             'path': path,
             'lines': [{'line_number': nb, 'line': l} for nb, l in lines]}
                for icon, path, lines in arguments['results']]
        return jsonify(arguments)

    if error:
        return render_template('error.html', **arguments), status_code or 500
    else:
        return render_template('search.html', **arguments)

@dxr_blueprint.route('/<tree>/source/')
@dxr_blueprint.route('/<tree>/source/<path:path>')
def browse(tree, path=''):
    """Show a directory listing or a single file from one of the trees."""
    tree_folder = os.path.join(current_app.instance_path, 'trees', tree)

    if isdir(os.path.join(tree_folder, path)):
        # It's a bare directory. Add the index file to the end:
        path = os.path.join(path, current_app.config['DIRECTORY_INDEX'])
    else:
        # It's a file. Add the .html extension:
        path += '.html'

    return send_from_directory(tree_folder, path)
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
import threading
import time
import math
import random
import json

import mesos.interface
from mesos.interface import mesos_pb2
import mesos.native

#import magic_client

class MyExecutor(mesos.interface.Executor):
    def registered(self, driver, executorInfo, frameworkInfo, slaveInfo):
        print "Executor registered"

    def disconnected(self, driver):
        print "Executor disconnected"

    def launchTask(self, driver, task):
        # Create a thread to run the task. Tasks should always be run in new
        # threads or processes, rather than inside launchTask itself.
        def run_task():
            print "Running task %s" % task.task_id.value
            update = mesos_pb2.TaskStatus()
            update.task_id.value = task.task_id.value
            update.state = mesos_pb2.TASK_RUNNING
            update.data = 'task running'
            driver.sendStatusUpdate(update)
            print task.data

            parsed_json = json.loads(task.data)
            uid = parsed_json['uid'] #just echo
            location = parsed_json['location'] #starting search location
            temperature = parsed_json['temperature'] #starting temperature, decreases
            cooling_rate = parsed_json['cooling_rate'] #final
            num_mutations = parsed_json['num_mutations'] #final

            print "start: uid is: ".format(uid)
            print "start: location is: ".format(location)
            print "start: temperature is: ".format(temperature)
            print "start: cooling rate is: ".format(cooling_rate)
            print "start: num_mutations is: ".format(num_mutations)

            e = math.e

            best_fitness = parent_fitness = None
            best_key = location
            T = temperature
            while T >= 0:
                print("Temperature: {}".format(T))
                print("Cooling rate: {}".format(cooling_rate))
                print("Fitness: {}".format(parent_fitness))
                print("Key: {}".format(parent_key))
                start = time()
                for i in xrange(num_mutations):
                    new_key = magic_mutation(parent_key) #from client library
                    fitness = magic_fitness_score(new_key) #from client library
                    if parent_fitness is None:
                        parent_fitness = fitness
                        parent_key = new_key
                        continue
                    dF = fitness - parent_fitness
                    if dF > 0.0:
                        best_fitness = parent_fitness = fitness
                        best_key = parent_key = new_key
                    else:
                        prob = e**(dF/T)
                        if prob > random.uniform(0, 1):
                            parent_fitness = fitness
                            parent_key = new_key
                print("Keys/s: {:.2f}".format(num_mutations/(time() - start)))
                print("")
                T -= cooling_rate

            print "Best Fitness: {}".format(best_fitness)
            print "Best Key: {}".format(best_key)
            print "end: uid is: ".format(uid)
            print "end: location is: ".format(location)
            print "end: temperature is: ".format(temperature)
            print "end: cooling rate is: ".format(cooling_rate)
            print "end: num_mutations is: ".format(num_mutations)

            update = mesos_pb2.TaskStatus()
            update.task_id.value = task.task_id.value
            update.state = mesos_pb2.TASK_FINISHED
            update.data = json.dumps({"uid": uid},{"best_location": best_key},{"fitness_score": best_fitness})
            driver.sendStatusUpdate(update)
            return parent_key

        thread = threading.Thread(target=run_task)
        thread.start()

    def frameworkMessage(self, driver, message):
        # Send it back to the scheduler.
        driver.sendFrameworkMessage(message)

def main():
    print "Starting executor"
    driver = mesos.native.MesosExecutorDriver(MyExecutor())
    sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)

if __name__ == "__main__":
    main()<commit_msg>Momentarily adding TSPSA specifics<commit_after>#!/usr/bin/env python

import sys
import threading
from time import time
import math
import random
import json

import mesos.interface
from mesos.interface import mesos_pb2
import mesos.native

# XXX Hardcode client for testing
sys.path.append("/home/vagrant/traveling_sailor")
import traveling_sailor
# latitude and longitude for the twenty largest U.S. cities
cities = {
    'New York City': (40.72, 74.00),
    'Los Angeles': (34.05, 118.25),
    'Chicago': (41.88, 87.63),
    'Houston': (29.77, 95.38),
    'Phoenix': (33.45, 112.07),
    'Philadelphia': (39.95, 75.17),
    'San Antonio': (29.53, 98.47),
    'Dallas': (32.78, 96.80),
    'San Diego': (32.78, 117.15),
    'San Jose': (37.30, 121.87),
    'Detroit': (42.33, 83.05),
    'San Francisco': (37.78, 122.42),
    'Jacksonville': (30.32, 81.70),
    'Indianapolis': (39.78, 86.15),
    'Austin': (30.27, 97.77),
    'Columbus': (39.98, 82.98),
    'Fort Worth': (32.75, 97.33),
    'Charlotte': (35.23, 80.85),
    'Memphis': (35.12, 89.97),
    'Baltimore': (39.28, 76.62)
}

class MyExecutor(mesos.interface.Executor):
    def registered(self, driver, executorInfo, frameworkInfo, slaveInfo):
        print "Executor registered"

    def disconnected(self, driver):
        print "Executor disconnected"

    def launchTask(self, driver, task):
        # Create a thread to run the task. Tasks should always be run in new
        # threads or processes, rather than inside launchTask itself.
        def run_task():
            print "Running task %s" % task.task_id.value
            update = mesos_pb2.TaskStatus()
            update.task_id.value = task.task_id.value
            update.state = mesos_pb2.TASK_RUNNING
            update.data = 'task running'
            driver.sendStatusUpdate(update)
            print task.data

            parsed_json = json.loads(task.data)
            uid = parsed_json['uid'] #just echo
            location = parsed_json['location'] #starting search location
            temperature = parsed_json['temperature'] #starting temperature, decreases
            cooling_rate = parsed_json['cooling_rate'] #final
            num_mutations = parsed_json['num_mutations'] #final

            print "start: uid is: ".format(uid)
            print "start: location is: ".format(location)
            print "start: temperature is: ".format(temperature)
            print "start: cooling rate is: ".format(cooling_rate)
            print "start: num_mutations is: ".format(num_mutations)

            # XXX
            problem = traveling_sailor.TSPSA()
            problem.init(cities)

            e = math.e
            best_fitness = parent_fitness = None
            best_key = location
            T = temperature
            while T >= 0:
                print("Temperature: {}".format(T))
                print("Cooling rate: {}".format(cooling_rate))
                print("Fitness: {}".format(parent_fitness))
                print("Key: {}".format(parent_key))
                start = time()
                for i in xrange(num_mutations):
                    new_key = problem.mutation(parent_key)
                    fitness = problem.fitness_score(new_key)
                    if parent_fitness is None:
                        parent_fitness = fitness
                        parent_key = new_key
                        continue
                    dF = fitness - parent_fitness
                    if dF > 0.0:
                        best_fitness = parent_fitness = fitness
                        best_key = parent_key = new_key
                    else:
                        prob = e**(dF/T)
                        if prob > random.uniform(0, 1):
                            parent_fitness = fitness
                            parent_key = new_key
                print("Keys/s: {:.2f}".format(num_mutations/(time() - start)))
                print("")
                T -= cooling_rate

            print "Best Fitness: {}".format(best_fitness)
            print "Best Key: {}".format(best_key)
            print "end: uid is: ".format(uid)
            print "end: location is: ".format(location)
            print "end: temperature is: ".format(temperature)
            print "end: cooling rate is: ".format(cooling_rate)
            print "end: num_mutations is: ".format(num_mutations)

            update = mesos_pb2.TaskStatus()
            update.task_id.value = task.task_id.value
            update.state = mesos_pb2.TASK_FINISHED
            update.data = json.dumps({"uid": uid},{"best_location": best_key},{"fitness_score": best_fitness})
            driver.sendStatusUpdate(update)
            return parent_key

        thread = threading.Thread(target=run_task)
        thread.start()

    def frameworkMessage(self, driver, message):
        # Send it back to the scheduler.
        driver.sendFrameworkMessage(message)

def main():
    print "Starting executor"
    driver = mesos.native.MesosExecutorDriver(MyExecutor())
    sys.exit(0 if driver.run() == mesos_pb2.DRIVER_STOPPED else 1)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before><commit_msg>initial commit of trancscribing efvents<commit_after>
from Bio.Seq import Seq
from Bio.SeqRecord import SeqRecord
from Bio import SeqIO
from Bio.Alphabet import generic_dna
import six

hg19_fasta = '/projects/ps-yeolab/genomes/hg19/gencode/v19/GRCh37.p13.genome.fa'

def overlap(x, y):
    return not ((x.start > y.stop) or (x.stop < y.start))

seqrecords = []

splice_type_isoform1_exons = {'SE': ('exon1', 'exon3'), 'MXE': ('exon1', 'exon3', 'exon4')}
splice_type_isoform2_exons = {'SE': ('exon1', 'exon2','exon3'), 'MXE': ('exon1', 'exon2', 'exon4')}

for i, (event_id, row) in enumerate(splicing_feature_data.iterrows()):
    if (i+1) % 1000 == 0:
        six.print_('\t Features completed', i+1)

    isoform1_exons = splice_type_isoform1_exons[row.splice_type]
    isoform2_exons = splice_type_isoform2_exons[row.splice_type]

    isoform_to_exons = {'isoform1': map(lambda x: v19db[row[x]], isoform1_exons),
                        'isoform2': map(lambda x: v19db[row[x]], isoform2_exons)}

    isoform_to_seq = {'isoform1': [], 'isoform2': []}

    reverse = isoform_to_exons['isoform1'][0].strand == '-'

    for isoform, exons in isoform_to_exons.items():
        if reverse:
            sequence = Seq(''.join(exon.sequence(hg19_fasta)[::-1] for exon in exons), generic_dna).complement()
        else:
            sequence = Seq(''.join(exon.sequence(hg19_fasta) for exon in exons), generic_dna)
        transcribed = sequence.transcribe()
        seqrecord = SeqRecord(transcribed, id='{0}|{1}'.format(event_id, isoform))
        seqrecords.append(seqrecord)

with open(transcribed_fasta, 'w') as f:
    SeqIO.write(seqrecords, f, 'fasta')
<|endoftext|>


<commit_before>#---import modules---
from astropy.io import fits
import numpy as np
import matplotlib
matplotlib.use('PDF')
import pylab as plt
import sys
sys.path.append('/research/astro/fir/HELP/XID_plus/')
import XIDp_mod_beta
import pickle

#---Read in DESPHOT catalogue---
folder='/research/astro/fir/HELP/DESPHOT/'
hdulist=fits.open(folder+'cosmos_itermap_lacey_07012015_simulated_observation_w_noise__PSWXID_S100_50mic.fits')
fcat=hdulist[1].data
ind=fcat['xid'] >= 0
fcat=fcat[ind]
hdulist.close()

#---Read in truth catalogue---
folder='/research/astro/fir/cclarke/lacey/released/'
hdulist=fits.open(folder+'lacey_07012015_MillGas.ALLVOLS_cat_PSW_COSMOS.fits')
fcat_sim=hdulist[1].data
hdulist.close()

#---Read in XID+ catalogue---
folder='/research/astro/fir/HELP/XID_plus_output/100micron/log_uniform_prior/'
hdulist=fits.open(folder+'Tiled_SPIRE_cat_flux_notlog.fits')
fcat_xidp=hdulist[1].data
hdulist.close()

#---match DESPHOT and real catalogues---
from astropy.coordinates import SkyCoord
from astropy import units as u
c= SkyCoord(ra=fcat['INRA']*u.degree,dec=fcat['INDEC']*u.degree)
c1=SkyCoord(ra=fcat_sim['RA']*u.degree,dec=fcat_sim['DEC']*u.degree)
idx,d2d,d3d,= c.match_to_catalog_sky(c1)

#---match XID+ and real catalogues---
c= SkyCoord(ra=fcat_xidp['ra']*u.degree,dec=fcat_xidp['dec']*u.degree)
c1=SkyCoord(ra=fcat_sim['RA']*u.degree,dec=fcat_sim['DEC']*u.degree)
idx_xidp,d2d,d3d,= c.match_to_catalog_sky(c1)

#----output folder-----------------
output_folder='/research/astro/fir/HELP/XID_plus_output/100micron/log_uniform_prior/'

infile=output_folder+'Tiled_master_Lacey_notlog_flux.pkl'
with open(infile, "rb") as f:
    obj = pickle.load(f)
prior250=obj['psw']
prior350=obj['pmw']
prior500=obj['plw']

posterior=obj['posterior']

samples,chains,params=posterior.stan_fit.shape
flattened_post=posterior.stan_fit.reshape(samples*chains,params)

with open(output_folder+'Tiling_info.pkl', "rb") as f:
        obj = pickle.load(f)

tiles=obj['tiles']
tiling_list=obj['tiling_list']

import triangle
sources=[34477,50290]

print tiling_list[sources,0],tiling_list[sources,1],
truths=fcat_sim['S250'][idx_xidp]
print truths[sources]
print fcat_xidp[sources]
figure = triangle.corner(flattened_post[:,sources],truths=truths[sources])
figure.savefig("triangle.pdf")
<commit_msg>posterior comp codes<commit_after>#---import modules---
from astropy.io import fits
import numpy as np
import matplotlib
matplotlib.use('PDF')
import pylab as plt
import sys
sys.path.append('/research/astro/fir/HELP/XID_plus/')
import XIDp_mod_beta
import pickle

#---Read in DESPHOT catalogue---
folder='/research/astro/fir/HELP/DESPHOT/'
hdulist=fits.open(folder+'cosmos_itermap_lacey_07012015_simulated_observation_w_noise__PSWXID_S100_50mic.fits')
fcat=hdulist[1].data
ind=fcat['xid'] >= 0
fcat=fcat[ind]
hdulist.close()

#---Read in truth catalogue---
folder='/research/astro/fir/cclarke/lacey/released/'
hdulist=fits.open(folder+'lacey_07012015_MillGas.ALLVOLS_cat_PSW_COSMOS.fits')
fcat_sim=hdulist[1].data
hdulist.close()

#---Read in XID+ catalogue---
folder='/research/astro/fir/HELP/XID_plus_output/100micron/log_uniform_prior/'
hdulist=fits.open(folder+'Tiled_SPIRE_cat_flux_notlog.fits')
fcat_xidp=hdulist[1].data
hdulist.close()

#---match DESPHOT and real catalogues---
from astropy.coordinates import SkyCoord
from astropy import units as u
c= SkyCoord(ra=fcat['INRA']*u.degree,dec=fcat['INDEC']*u.degree)
c1=SkyCoord(ra=fcat_sim['RA']*u.degree,dec=fcat_sim['DEC']*u.degree)
idx,d2d,d3d,= c.match_to_catalog_sky(c1)

#---match XID+ and real catalogues---
c= SkyCoord(ra=fcat_xidp['ra']*u.degree,dec=fcat_xidp['dec']*u.degree)
c1=SkyCoord(ra=fcat_sim['RA']*u.degree,dec=fcat_sim['DEC']*u.degree)
idx_xidp,d2d,d3d,= c.match_to_catalog_sky(c1)

#----output folder-----------------
output_folder='/research/astro/fir/HELP/XID_plus_output/100micron/log_uniform_prior/'

infile=output_folder+'Tiled_master_Lacey_notlog_flux.pkl'
with open(infile, "rb") as f:
    obj = pickle.load(f)
prior250=obj['psw']
prior350=obj['pmw']
prior500=obj['plw']

posterior=obj['posterior']

samples,chains,params=posterior.stan_fit.shape
flattened_post=np.log10(posterior.stan_fit.reshape(samples*chains,params))

with open(output_folder+'Tiling_info.pkl', "rb") as f:
        obj = pickle.load(f)

tiles=obj['tiles']
tiling_list=obj['tiling_list']

import triangle
sources=[34477,50290]

print tiling_list[sources,0],tiling_list[sources,1],
truths=np.log10(fcat_sim['S250'][idx_xidp])
print truths[sources]
print fcat_xidp[sources]
figure = triangle.corner(flattened_post[:,sources],truths=truths[sources])
figure.savefig("triangle.pdf")
<|endoftext|>


<commit_before>import os
from datetime import timedelta
from weakref import WeakSet

import sqlalchemy as sa
from sqlalchemy.dialects.postgresql import array, HSTORE, INET

HERE = os.path.dirname(os.path.abspath(__file__))
cached_statements = {}

class ImproperlyConfigured(Exception):
    pass

class StatementExecutor(object):
    def __init__(self, stmt):
        self.stmt = stmt

    def __call__(self, target, bind, **kwargs):
        tx = bind.begin()
        bind.execute(self.stmt)
        tx.commit()

def read_file(file_):
    with open(os.path.join(HERE, file_)) as f:
        s = f.read()
    return s

def assign_actor(base, cls, actor_cls):
    if hasattr(cls, 'actor_id'):
        return
    if actor_cls:
        primary_key = sa.inspect(actor_cls).primary_key[0]

        cls.actor_id = sa.Column('actor_id', primary_key.type)
        cls.actor = sa.orm.relationship(
            actor_cls,
            primaryjoin=cls.actor_id == (
                getattr(
                    actor_cls,
                    primary_key.name
                )
            ),
            foreign_keys=[cls.actor_id]
        )
    else:
        cls.actor_id = sa.Column(sa.Text)

def audit_table(table, exclude_columns):
    args = [table.name]
    if exclude_columns:
        for column in exclude_columns:
            if column not in table.c:
                raise ImproperlyConfigured(
                    "Could not configure versioning. Table '{}'' does "
                    "not have a column named '{}'.".format(
                        table.name, column
                    )
                )
        args.append(array(exclude_columns))

    query = sa.select(
        [sa.func.audit.audit_table(*args)]
    )
    if query not in cached_statements:
        cached_statements[query] = StatementExecutor(query)
    listener = (table, 'after_create', cached_statements[query])
    if not sa.event.contains(*listener):
        sa.event.listen(*listener)

def activity_base(base):
    class ActivityBase(base):
        __abstract__ = True
        id = sa.Column(sa.BigInteger, primary_key=True)
        schema_name = sa.Column(sa.Text)
        table_name = sa.Column(sa.Text)
        relid = sa.Column(sa.Integer)
        issued_at = sa.Column(sa.DateTime)
        transaction_id = sa.Column(sa.BigInteger)
        client_addr = sa.Column(INET)
        client_port = sa.Column(sa.Integer)
        verb = sa.Column(sa.Text)
        object_id = sa.Column(sa.Text)
        target_id = sa.Column(sa.Text)
        row_data = sa.Column(HSTORE)
        changed_fields = sa.Column(HSTORE)

        def __repr__(self):
            return (
                '<{cls} table_name={table_name!r} '
                'id={id!r}>'
            ).format(
                cls=self.__class__.__name__,
                table_name=self.table_name,
                id=self.id
            )
    return ActivityBase

def convert_callables(values):
    result = {}
    for key, value in values.items():
        if callable(value):
            result[key] = value()
        else:
            result[key] = value
    return result

class VersioningManager(object):
    table_listeners = [
        (
            'before_create',
            sa.schema.DDL(read_file('schema.sql')),
        ),
        (
            'after_create',
            sa.schema.DDL(
                read_file('create_activity.sql').replace('%', '%%') +
                read_file('audit_table.sql').replace('%', '%%')
            )
        ),
        (
            'after_drop',
            sa.schema.DDL('DROP SCHEMA audit CASCADE')
        )
    ]

    def __init__(self, actor_cls=None):
        self.values = {}
        self._actor_cls = actor_cls
        self.listeners = (
            (
                sa.orm.mapper,
                'instrument_class',
                self.instrument_versioned_classes
            ),
            (
                sa.orm.mapper,
                'after_configured',
                self.configure_versioned_classes
            ),
            (
                sa.orm.session.Session,
                'after_flush',
                self.receive_after_flush,
            ),
        )
        self.pending_classes = WeakSet()
        self.cached_ddls = {}

    @property
    def transaction_values(self):
        return self.values

    def set_activity_values(self, session):
        table = self.activity_cls.__table__
        if self.values:
            values = convert_callables(self.transaction_values)
            stmt = (
                table
                .update()
                .values(**values)
                .where(
                    sa.and_(
                        table.c.transaction_id == sa.func.txid_current(),
                        table.c.issued_at > (
                            sa.func.now() - timedelta(days=1)
                        )
                    )
                )
            )
            session.execute(stmt)

    def receive_after_flush(self, session, flush_context):
        self.set_activity_values(session)

    def instrument_versioned_classes(self, mapper, cls):
        """
        Collect versioned class and add it to pending_classes list.

        :mapper mapper: SQLAlchemy mapper object
        :cls cls: SQLAlchemy declarative class
        """
        if hasattr(cls, '__versioned__') and cls not in self.pending_classes:
            self.pending_classes.add(cls)

    def configure_versioned_classes(self):
        """
        Configures all versioned classes that were collected during
        instrumentation process.
        """
        for cls in self.pending_classes:
            audit_table(cls.__table__, cls.__versioned__.get('exclude'))
        assign_actor(self.base, self.activity_cls, self.actor_cls)

    def attach_table_listeners(self):
        for values in self.table_listeners:
            sa.event.listen(self.activity_cls.__table__, *values)

    def remove_table_listeners(self):
        for values in self.table_listeners:
            sa.event.remove(self.activity_cls.__table__, *values)

    @property
    def actor_cls(self):
        if isinstance(self._actor_cls, str):
            if not self.base:
                raise ImproperlyConfigured(
                    'This manager does not have declarative base set up yet. '
                    'Call init method to set up this manager.'
                )
            registry = self.base._decl_class_registry
            try:
                return registry[self._actor_cls]
            except KeyError:
                raise ImproperlyConfigured(
                    'Could not build relationship between Activity'
                    ' and %s. %s was not found in declarative class '
                    'registry. Either configure VersioningManager to '
                    'use different actor class or disable this '
                    'relationship by setting it to None.' % (
                        self._actor_cls,
                        self._actor_cls
                    )
                )
        return self._actor_cls

    def attach_listeners(self):
        self.attach_table_listeners()
        for listener in self.listeners:
            sa.event.listen(*listener)

    def remove_listeners(self):
        self.remove_table_listeners()
        for listener in self.listeners:
            sa.event.remove(*listener)

    def activity_model_factory(self, base):
        class Activity(activity_base(base)):
            __tablename__ = 'activity'
            __table_args__ = {'schema': 'audit'}

        return Activity

    def init(self, base):
        self.base = base
        self.activity_cls = self.activity_model_factory(base)
        self.attach_listeners()

versioning_manager = VersioningManager()
<commit_msg>Fix imports<commit_after>import os
from datetime import timedelta
from weakref import WeakSet

import sqlalchemy as sa
from sqlalchemy import orm
from sqlalchemy.dialects.postgresql import array, HSTORE, INET

HERE = os.path.dirname(os.path.abspath(__file__))
cached_statements = {}

class ImproperlyConfigured(Exception):
    pass

class StatementExecutor(object):
    def __init__(self, stmt):
        self.stmt = stmt

    def __call__(self, target, bind, **kwargs):
        tx = bind.begin()
        bind.execute(self.stmt)
        tx.commit()

def read_file(file_):
    with open(os.path.join(HERE, file_)) as f:
        s = f.read()
    return s

def assign_actor(base, cls, actor_cls):
    if hasattr(cls, 'actor_id'):
        return
    if actor_cls:
        primary_key = sa.inspect(actor_cls).primary_key[0]

        cls.actor_id = sa.Column('actor_id', primary_key.type)
        cls.actor = orm.relationship(
            actor_cls,
            primaryjoin=cls.actor_id == (
                getattr(
                    actor_cls,
                    primary_key.name
                )
            ),
            foreign_keys=[cls.actor_id]
        )
    else:
        cls.actor_id = sa.Column(sa.Text)

def audit_table(table, exclude_columns):
    args = [table.name]
    if exclude_columns:
        for column in exclude_columns:
            if column not in table.c:
                raise ImproperlyConfigured(
                    "Could not configure versioning. Table '{}'' does "
                    "not have a column named '{}'.".format(
                        table.name, column
                    )
                )
        args.append(array(exclude_columns))

    query = sa.select(
        [sa.func.audit.audit_table(*args)]
    )
    if query not in cached_statements:
        cached_statements[query] = StatementExecutor(query)
    listener = (table, 'after_create', cached_statements[query])
    if not sa.event.contains(*listener):
        sa.event.listen(*listener)

def activity_base(base):
    class ActivityBase(base):
        __abstract__ = True
        id = sa.Column(sa.BigInteger, primary_key=True)
        schema_name = sa.Column(sa.Text)
        table_name = sa.Column(sa.Text)
        relid = sa.Column(sa.Integer)
        issued_at = sa.Column(sa.DateTime)
        transaction_id = sa.Column(sa.BigInteger)
        client_addr = sa.Column(INET)
        client_port = sa.Column(sa.Integer)
        verb = sa.Column(sa.Text)
        object_id = sa.Column(sa.Text)
        target_id = sa.Column(sa.Text)
        row_data = sa.Column(HSTORE)
        changed_fields = sa.Column(HSTORE)

        def __repr__(self):
            return (
                '<{cls} table_name={table_name!r} '
                'id={id!r}>'
            ).format(
                cls=self.__class__.__name__,
                table_name=self.table_name,
                id=self.id
            )
    return ActivityBase

def convert_callables(values):
    result = {}
    for key, value in values.items():
        if callable(value):
            result[key] = value()
        else:
            result[key] = value
    return result

class VersioningManager(object):
    table_listeners = [
        (
            'before_create',
            sa.schema.DDL(read_file('schema.sql')),
        ),
        (
            'after_create',
            sa.schema.DDL(
                read_file('create_activity.sql').replace('%', '%%') +
                read_file('audit_table.sql').replace('%', '%%')
            )
        ),
        (
            'after_drop',
            sa.schema.DDL('DROP SCHEMA audit CASCADE')
        )
    ]

    def __init__(self, actor_cls=None):
        self.values = {}
        self._actor_cls = actor_cls
        self.listeners = (
            (
                orm.mapper,
                'instrument_class',
                self.instrument_versioned_classes
            ),
            (
                orm.mapper,
                'after_configured',
                self.configure_versioned_classes
            ),
            (
                orm.session.Session,
                'after_flush',
                self.receive_after_flush,
            ),
        )
        self.pending_classes = WeakSet()
        self.cached_ddls = {}

    @property
    def transaction_values(self):
        return self.values

    def set_activity_values(self, session):
        table = self.activity_cls.__table__
        if self.values:
            values = convert_callables(self.transaction_values)
            stmt = (
                table
                .update()
                .values(**values)
                .where(
                    sa.and_(
                        table.c.transaction_id == sa.func.txid_current(),
                        table.c.issued_at > (
                            sa.func.now() - timedelta(days=1)
                        )
                    )
                )
            )
            session.execute(stmt)

    def receive_after_flush(self, session, flush_context):
        self.set_activity_values(session)

    def instrument_versioned_classes(self, mapper, cls):
        """
        Collect versioned class and add it to pending_classes list.

        :mapper mapper: SQLAlchemy mapper object
        :cls cls: SQLAlchemy declarative class
        """
        if hasattr(cls, '__versioned__') and cls not in self.pending_classes:
            self.pending_classes.add(cls)

    def configure_versioned_classes(self):
        """
        Configures all versioned classes that were collected during
        instrumentation process.
        """
        for cls in self.pending_classes:
            audit_table(cls.__table__, cls.__versioned__.get('exclude'))
        assign_actor(self.base, self.activity_cls, self.actor_cls)

    def attach_table_listeners(self):
        for values in self.table_listeners:
            sa.event.listen(self.activity_cls.__table__, *values)

    def remove_table_listeners(self):
        for values in self.table_listeners:
            sa.event.remove(self.activity_cls.__table__, *values)

    @property
    def actor_cls(self):
        if isinstance(self._actor_cls, str):
            if not self.base:
                raise ImproperlyConfigured(
                    'This manager does not have declarative base set up yet. '
                    'Call init method to set up this manager.'
                )
            registry = self.base._decl_class_registry
            try:
                return registry[self._actor_cls]
            except KeyError:
                raise ImproperlyConfigured(
                    'Could not build relationship between Activity'
                    ' and %s. %s was not found in declarative class '
                    'registry. Either configure VersioningManager to '
                    'use different actor class or disable this '
                    'relationship by setting it to None.' % (
                        self._actor_cls,
                        self._actor_cls
                    )
                )
        return self._actor_cls

    def attach_listeners(self):
        self.attach_table_listeners()
        for listener in self.listeners:
            sa.event.listen(*listener)

    def remove_listeners(self):
        self.remove_table_listeners()
        for listener in self.listeners:
            sa.event.remove(*listener)

    def activity_model_factory(self, base):
        class Activity(activity_base(base)):
            __tablename__ = 'activity'
            __table_args__ = {'schema': 'audit'}

        return Activity

    def init(self, base):
        self.base = base
        self.activity_cls = self.activity_model_factory(base)
        self.attach_listeners()

versioning_manager = VersioningManager()
<|endoftext|>


<commit_before># Copyright 2011 James McCauley
#
# This file is part of POX.
#
# POX is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# POX is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with POX.  If not, see <http://www.gnu.org/licenses/>.

from pox.lib.revent.revent import *
import libopenflow_01 as of
from openflow import *
from pox.core import core
from pox.topology.topology import *
from pox.openflow.discovery import *
from pox.lib.util import dpidToStr
from pox.lib.addresses import *

# After a switch disconnects, it has this many seconds to reconnect in
# order to reactivate the same OpenFlowSwitch object.  After this, if
# it reconnects, it will be a new switch object.
RECONNECT_TIMEOUT = 30

log = core.getLogger()

class OpenFlowTopology (EventMixin):
  """
  OpenFlow doesn't know anything about Topology, and Topology doesn't
  know anything about OpenFlow.  This class knows something about both,
  and hooks the two of them together
  """
  _wantComponents = set(['openflow','topology','openflow_discovery'])

  def _resolveComponents (self):
    if self._wantComponents == None or len(self._wantComponents) == 0:
      self._wantComponents = None
      return True

    got = set()
    for c in self._wantComponents:
      if core.hasComponent(c):
        setattr(self, c, getattr(core, c))
        self.listenTo(getattr(core, c), prefix=c)
        got.add(c)
      else:
        setattr(self, c, None)
    for c in got:
      self._wantComponents.remove(c)
    if len(self._wantComponents) == 0:
      self.wantComponents = None
      log.debug(self.__class__.__name__ + " ready")
      return True
    #log.debug(self.__class__.__name__ + " still wants: " + (', '.join(self._wantComponents)))
    return False

  def __init__ (self):
    super(EventMixin, self).__init__()
    if not self._resolveComponents():
      self.listenTo(core)

  def _handle_openflow_discovery_LinkEvent (self, event):
    if self.topology is None: return
    link = event.link
    sw1 = self.topology.getEntityByID(link.dpid1)
    sw2 = self.topology.getEntityByID(link.dpid2)
    if sw1 is None or sw2 is None: return
    if link.port1 not in sw1.ports or link.port2 not in sw2.ports: return
    if event.added:
      sw1.ports[link.port1].addEntity(sw2, single=True)
      sw2.ports[link.port2].addEntity(sw1, single=True)
    elif event.removed:
      sw1.ports[link.port1].entities.remove(sw2)
      sw2.ports[link.port2].entities.remove(sw1)

  def _handle_ComponentRegistered (self, event):
    if self._resolveComponents():
      return EventRemove

  def _handle_openflow_ConnectionUp (self, event):
    sw = self.topology.getEntityByID(event.dpid)
    add = False
    if sw is None:
      sw = OpenFlowSwitch(event.dpid)
      add = True
    else:
      if sw.connection is not None:
        log.warn("Switch %s connected, but... it's already connected!" %
                 (dpidToStr(event.dpid),))
    sw._setConnection(event.connection, event.ofp)
    log.info("Switch " + dpidToStr(event.dpid) + " connected")
    if add:
      self.topology.addEntity(sw)
      sw.raiseEvent(SwitchJoin, sw)

  def _handle_openflow_ConnectionDown (self, event):
    sw = self.topology.getEntityByID(event.dpid)
    if sw is None:
      log.warn("Switch %s disconnected, but... it doesn't exist!" %
               (dpidToStr(event.dpid),))
    else:
      if sw.connection is None:
        log.warn("Switch %s disconnected, but... it's wasn't connected!" %
                 (dpidToStr(event.dpid),))
      sw.connection = None
      log.info("Switch " + str(event.dpid) + " disconnected")

class OpenFlowPort (Port):
  def __init__ (self, ofp):
    # Passed an ofp_phy_port
    Port.__init__(self, ofp.port_no, ofp.hw_addr, ofp.name)
    self.isController = self.number == of.OFPP_CONTROLLER
    self._update(ofp)
    self.exists = True
    self.entities = set()

  def _update (self, ofp):
    assert self.name == ofp.name
    assert self.number == ofp.port_no
    self.hwAddr = EthAddr(ofp.hw_addr)
    self._config = ofp.config
    self._state = ofp.state

  def __contains__ (self, item):
    """ True if this port connects to the specified entity """
    return item in self.entities

  def addEntity (self, entity, single = False):
    if single:
      self.entities = set([entity])
    else:
      self.entities.add(entity)

  def __repr__ (self):
    return "<Port #" + str(self.number) + ">"

class OpenFlowSwitch (EventMixin, Switch):
  _eventMixin_events = set([
    SwitchJoin,
    SwitchLeave,

    PortStatus,
    FlowRemoved,
    PacketIn,
    BarrierIn,
  ])
  def __init__ (self, dpid):
    super(Switch, self).__init__(dpid)
    EventMixin.__init__(self)
    self.dpid = dpid
    self.ports = {}
    self.capabilities = 0
    self.connection = None
    self._listeners = []
    self._reconnectTimeout = None # Timer for reconnection

  def _setConnection (self, connection, ofp=None):
    if self.connection: self.connection.removeListeners(self._listeners)
    self._listeners = []
    self.connection = connection
    if self._reconnectTimeout is not None:
      self._reconnectTimeout.cancel()
      self._reconnectTimeout = None
    if connection is None:
      self._reconnectTimeout = Timer(RECONNECT_TIMEOUT, self._timer_ReconnectTimeout)
    if ofp is not None:
      self.capabilities = ofp.capabilities
      untouched = set(self.ports.keys())
      for p in ofp.ports:
        if p.port_no in self.ports:
          self.ports[p.port_no]._update(p)
          untouched.remove(p.port_no)
        else:
          self.ports[p.port_no] = OpenFlowPort(p)
      for p in untouched:
        self.ports[p].exists = False
        del self.ports[p]
    if connection is not None:
      self._listeners = self.listenTo(connection, prefix="con")

  def _timer_ReconnectTimeout (self):
    """ Called if we've been disconnected for RECONNECT_TIMEOUT seconds """
    self._reconnectTimeout = None
    core.topology.removeEntity(self)
    self.raiseEvent(SwitchLeave, self)

  def _handle_con_PortStatus (self, event):
    p = event.ofp.desc
    if event.ofp.reason == of.OFPPR_DELETE:
      if p.port_no in self.ports:
        self.ports[p.port_no].exists = False
        del self.ports[p.port_no]
    elif event.ofp.reason == of.OFPPR_MODIFY:
      self.ports[p.port_no]._update(p)
    else:
      assert event.ofp.reason == of.OFPPR_ADD
      assert p.port_no not in self.ports
      self.ports[p.port_no] = OpenFlowPort(p)
    self.raiseEvent(event)
    event.halt = False

  def _handle_con_ConnectionDown (self, event):
    self._setConnection(None)

  def _handle_con_PacketIn (self, event):
    self.raiseEvent(event)
    event.halt = False

  def _handle_con_BarrierIn (self, event):
    self.raiseEvent(event)
    event.halt = False

  def _handle_con_FlowRemoved (self, event):
    self.raiseEvent(event)
    event.halt = False

  def findPortForEntity (self, entity):
    for p in self.ports.itervalues():
      if entity in p:
        return p
    return None

  def __repr__ (self):
    return "<%s %s>" % (self.__class__.__name__, dpidToStr(self.dpid))

def launch ():
  if not core.hasComponent("openflow_topology"):
    core.register("openflow_topology", OpenFlowTopology())
<commit_msg>pox.openflow.topology: added a comment<commit_after># Copyright 2011 James McCauley
#
# This file is part of POX.
#
# POX is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# POX is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with POX.  If not, see <http://www.gnu.org/licenses/>.

"""
This module is an "adaptor" for translating between OpenFlow discovery and
pox.topology (which is protocol agnostitic?)
"""

from pox.lib.revent.revent import *
import libopenflow_01 as of
from openflow import *
from pox.core import core
from pox.topology.topology import *
from pox.openflow.discovery import *
from pox.lib.util import dpidToStr
from pox.lib.addresses import *

# After a switch disconnects, it has this many seconds to reconnect in
# order to reactivate the same OpenFlowSwitch object.  After this, if
# it reconnects, it will be a new switch object.
RECONNECT_TIMEOUT = 30

log = core.getLogger()

class OpenFlowTopology (EventMixin):
  """
  OpenFlow doesn't know anything about Topology, and Topology doesn't
  know anything about OpenFlow.  This class knows something about both,
  and hooks the two of them together
  """
  _wantComponents = set(['openflow','topology','openflow_discovery'])

  def _resolveComponents (self):
    if self._wantComponents == None or len(self._wantComponents) == 0:
      self._wantComponents = None
      return True

    got = set()
    for c in self._wantComponents:
      if core.hasComponent(c):
        setattr(self, c, getattr(core, c))
        self.listenTo(getattr(core, c), prefix=c)
        got.add(c)
      else:
        setattr(self, c, None)
    for c in got:
      self._wantComponents.remove(c)
    if len(self._wantComponents) == 0:
      self.wantComponents = None
      log.debug(self.__class__.__name__ + " ready")
      return True
    #log.debug(self.__class__.__name__ + " still wants: " + (', '.join(self._wantComponents)))
    return False

  def __init__ (self):
    super(EventMixin, self).__init__()
    if not self._resolveComponents():
      self.listenTo(core)

  def _handle_openflow_discovery_LinkEvent (self, event):
    if self.topology is None: return
    link = event.link
    sw1 = self.topology.getEntityByID(link.dpid1)
    sw2 = self.topology.getEntityByID(link.dpid2)
    if sw1 is None or sw2 is None: return
    if link.port1 not in sw1.ports or link.port2 not in sw2.ports: return
    if event.added:
      sw1.ports[link.port1].addEntity(sw2, single=True)
      sw2.ports[link.port2].addEntity(sw1, single=True)
    elif event.removed:
      sw1.ports[link.port1].entities.remove(sw2)
      sw2.ports[link.port2].entities.remove(sw1)

  def _handle_ComponentRegistered (self, event):
    if self._resolveComponents():
      return EventRemove

  def _handle_openflow_ConnectionUp (self, event):
    sw = self.topology.getEntityByID(event.dpid)
    add = False
    if sw is None:
      sw = OpenFlowSwitch(event.dpid)
      add = True
    else:
      if sw.connection is not None:
        log.warn("Switch %s connected, but... it's already connected!" %
                 (dpidToStr(event.dpid),))
    sw._setConnection(event.connection, event.ofp)
    log.info("Switch " + dpidToStr(event.dpid) + " connected")
    if add:
      self.topology.addEntity(sw)
      sw.raiseEvent(SwitchJoin, sw)

  def _handle_openflow_ConnectionDown (self, event):
    sw = self.topology.getEntityByID(event.dpid)
    if sw is None:
      log.warn("Switch %s disconnected, but... it doesn't exist!" %
               (dpidToStr(event.dpid),))
    else:
      if sw.connection is None:
        log.warn("Switch %s disconnected, but... it's wasn't connected!" %
                 (dpidToStr(event.dpid),))
      sw.connection = None
      log.info("Switch " + str(event.dpid) + " disconnected")

class OpenFlowPort (Port):
  def __init__ (self, ofp):
    # Passed an ofp_phy_port
    Port.__init__(self, ofp.port_no, ofp.hw_addr, ofp.name)
    self.isController = self.number == of.OFPP_CONTROLLER
    self._update(ofp)
    self.exists = True
    self.entities = set()

  def _update (self, ofp):
    assert self.name == ofp.name
    assert self.number == ofp.port_no
    self.hwAddr = EthAddr(ofp.hw_addr)
    self._config = ofp.config
    self._state = ofp.state

  def __contains__ (self, item):
    """ True if this port connects to the specified entity """
    return item in self.entities

  def addEntity (self, entity, single = False):
    if single:
      self.entities = set([entity])
    else:
      self.entities.add(entity)

  def __repr__ (self):
    return "<Port #" + str(self.number) + ">"

class OpenFlowSwitch (EventMixin, Switch):
  _eventMixin_events = set([
    SwitchJoin,
    SwitchLeave,

    PortStatus,
    FlowRemoved,
    PacketIn,
    BarrierIn,
  ])
  def __init__ (self, dpid):
    super(Switch, self).__init__(dpid)
    EventMixin.__init__(self)
    self.dpid = dpid
    self.ports = {}
    self.capabilities = 0
    self.connection = None
    self._listeners = []
    self._reconnectTimeout = None # Timer for reconnection

  def _setConnection (self, connection, ofp=None):
    if self.connection: self.connection.removeListeners(self._listeners)
    self._listeners = []
    self.connection = connection
    if self._reconnectTimeout is not None:
      self._reconnectTimeout.cancel()
      self._reconnectTimeout = None
    if connection is None:
      self._reconnectTimeout = Timer(RECONNECT_TIMEOUT, self._timer_ReconnectTimeout)
    if ofp is not None:
      self.capabilities = ofp.capabilities
      untouched = set(self.ports.keys())
      for p in ofp.ports:
        if p.port_no in self.ports:
          self.ports[p.port_no]._update(p)
          untouched.remove(p.port_no)
        else:
          self.ports[p.port_no] = OpenFlowPort(p)
      for p in untouched:
        self.ports[p].exists = False
        del self.ports[p]
    if connection is not None:
      self._listeners = self.listenTo(connection, prefix="con")

  def _timer_ReconnectTimeout (self):
    """ Called if we've been disconnected for RECONNECT_TIMEOUT seconds """
    self._reconnectTimeout = None
    core.topology.removeEntity(self)
    self.raiseEvent(SwitchLeave, self)

  def _handle_con_PortStatus (self, event):
    p = event.ofp.desc
    if event.ofp.reason == of.OFPPR_DELETE:
      if p.port_no in self.ports:
        self.ports[p.port_no].exists = False
        del self.ports[p.port_no]
    elif event.ofp.reason == of.OFPPR_MODIFY:
      self.ports[p.port_no]._update(p)
    else:
      assert event.ofp.reason == of.OFPPR_ADD
      assert p.port_no not in self.ports
      self.ports[p.port_no] = OpenFlowPort(p)
    self.raiseEvent(event)
    event.halt = False

  def _handle_con_ConnectionDown (self, event):
    self._setConnection(None)

  def _handle_con_PacketIn (self, event):
    self.raiseEvent(event)
    event.halt = False

  def _handle_con_BarrierIn (self, event):
    self.raiseEvent(event)
    event.halt = False

  def _handle_con_FlowRemoved (self, event):
    self.raiseEvent(event)
    event.halt = False

  def findPortForEntity (self, entity):
    for p in self.ports.itervalues():
      if entity in p:
        return p
    return None

  def __repr__ (self):
    return "<%s %s>" % (self.__class__.__name__, dpidToStr(self.dpid))

def launch ():
  if not core.hasComponent("openflow_topology"):
    core.register("openflow_topology", OpenFlowTopology())
<|endoftext|>


<commit_before># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyPytorchLightning(PythonPackage):
    """PyTorch Lightning is the lightweight PyTorch wrapper for ML researchers."""

    homepage = "https://github.com/PyTorchLightning/pytorch-lightning"
    pypi     = "pytorch-lightning/pytorch-lightning-1.2.10.tar.gz"

    version('1.5.0', sha256='5bc28022f749d6a7644027ca0428bc162c9ab401cdea78b264d14f5018430f3b')
    version('1.4.1', sha256='1d1128aeb5d0e523d2204c4d9399d65c4e5f41ff0370e96d694a823af5e8e6f3')
    version('1.4.0', sha256='6529cf064f9dc323c94f3ce84b56ee1a05db1b0ab17db77c4d15aa36e34da81f')
    version('1.3.8', sha256='60b0a3e464d394864dae4c8d251afa7aa453644a19bb7672f5ee400343cdf7b0')
    version('1.2.10', sha256='2d8365e30ded0c20e73ce6e5b6028478ae460b8fd33727df2275666df005a301')

    depends_on('python@3.6:', type=('build', 'run'))
    depends_on('py-setuptools', type='build')
    depends_on('py-numpy@1.17.2:', when='@1.3:', type=('build', 'run'))
    depends_on('py-numpy@1.16.6:', when='@:1.2', type=('build', 'run'))
    # TODO: add patch and remove +distributed once the following issue is resolved:
    # https://github.com/PyTorchLightning/pytorch-lightning/issues/10348
    depends_on('py-torch@1.6:+distributed', when='@1.5.0', type=('build', 'run'))
    depends_on('py-torch@1.6:', when='@1.4:', type=('build', 'run'))
    depends_on('py-torch@1.4:', when='@:1.3', type=('build', 'run'))
    depends_on('py-future@0.17.1:', type=('build', 'run'))
    depends_on('py-tqdm@4.41.0:', type=('build', 'run'))
    depends_on('py-pyyaml@5.1:', when='@1.4:', type=('build', 'run'))
    depends_on('py-pyyaml@5.1:5.4.1', when='@1.3', type=('build', 'run'))
    depends_on('py-pyyaml@5.1:5.3,5.5:', when='@:1.2', type=('build', 'run'))
    depends_on('py-fsspec@2021.05.0:2021.05,2021.06.1:+http', when='@1.3:', type=('build', 'run'))
    depends_on('py-fsspec@0.8.1:+http', when='@:1.2', type=('build', 'run'))
    depends_on('py-tensorboard@2.2.0:', when='@1.5:', type=('build', 'run'))
    depends_on('py-tensorboard@2.2.0:2.4,2.5.1:', when='@:1.4', type=('build', 'run'))
    depends_on('py-torchmetrics@0.4.1:', when='@1.5:', type=('build', 'run'))
    depends_on('py-torchmetrics@0.4.0:', when='@1.4:', type=('build', 'run'))
    depends_on('py-torchmetrics@0.2.0:', when='@1.3', type=('build', 'run'))
    depends_on('py-torchmetrics@0.2.0', when='@:1.2', type=('build', 'run'))
    depends_on('py-pydeprecate@0.3.1', when='@1.4:', type=('build', 'run'))
    depends_on('py-pydeprecate@0.3.0', when='@1.3', type=('build', 'run'))
    depends_on('py-packaging@17.0:', when='@1.3:', type=('build', 'run'))
    depends_on('py-packaging', type=('build', 'run'))
    depends_on('py-typing-extensions', when='@1.4:', type=('build', 'run'))
    depends_on('pil@:8.2,8.3.1:', when='@1.3', type=('build', 'run'))
<commit_msg>py-pytorch-lightning: add v1.5.3 (#27650)<commit_after># Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)

from spack import *

class PyPytorchLightning(PythonPackage):
    """PyTorch Lightning is the lightweight PyTorch wrapper for ML researchers."""

    homepage = "https://github.com/PyTorchLightning/pytorch-lightning"
    pypi     = "pytorch-lightning/pytorch-lightning-1.2.10.tar.gz"

    version('1.5.3', sha256='a206169a0c4356366a7edadb5ebd2f38e9a611ff78265ce93b767662682f5620')
    version('1.4.1', sha256='1d1128aeb5d0e523d2204c4d9399d65c4e5f41ff0370e96d694a823af5e8e6f3')
    version('1.4.0', sha256='6529cf064f9dc323c94f3ce84b56ee1a05db1b0ab17db77c4d15aa36e34da81f')
    version('1.3.8', sha256='60b0a3e464d394864dae4c8d251afa7aa453644a19bb7672f5ee400343cdf7b0')
    version('1.2.10', sha256='2d8365e30ded0c20e73ce6e5b6028478ae460b8fd33727df2275666df005a301')

    depends_on('python@3.6:', type=('build', 'run'))
    depends_on('py-setuptools', type='build')
    depends_on('py-numpy@1.17.2:', when='@1.3:', type=('build', 'run'))
    depends_on('py-numpy@1.16.6:', when='@:1.2', type=('build', 'run'))
    depends_on('py-torch@1.6:', when='@1.4:', type=('build', 'run'))
    depends_on('py-torch@1.4:', when='@:1.3', type=('build', 'run'))
    depends_on('py-future@0.17.1:', type=('build', 'run'))
    depends_on('py-tqdm@4.41.0:', type=('build', 'run'))
    depends_on('py-pyyaml@5.1:', when='@1.4:', type=('build', 'run'))
    depends_on('py-pyyaml@5.1:5.4.1', when='@1.3', type=('build', 'run'))
    depends_on('py-pyyaml@5.1:5.3,5.5:', when='@:1.2', type=('build', 'run'))
    depends_on('py-fsspec@2021.05.0:2021.05,2021.06.1:+http', when='@1.3:', type=('build', 'run'))
    depends_on('py-fsspec@0.8.1:+http', when='@:1.2', type=('build', 'run'))
    depends_on('py-tensorboard@2.2.0:', when='@1.5:', type=('build', 'run'))
    depends_on('py-tensorboard@2.2.0:2.4,2.5.1:', when='@:1.4', type=('build', 'run'))
    depends_on('py-torchmetrics@0.4.1:', when='@1.5:', type=('build', 'run'))
    depends_on('py-torchmetrics@0.4.0:', when='@1.4:', type=('build', 'run'))
    depends_on('py-torchmetrics@0.2.0:', when='@1.3', type=('build', 'run'))
    depends_on('py-torchmetrics@0.2.0', when='@:1.2', type=('build', 'run'))
    depends_on('py-pydeprecate@0.3.1', when='@1.4:', type=('build', 'run'))
    depends_on('py-pydeprecate@0.3.0', when='@1.3', type=('build', 'run'))
    depends_on('py-packaging@17.0:', when='@1.3:', type=('build', 'run'))
    depends_on('py-packaging', type=('build', 'run'))
    depends_on('py-typing-extensions', when='@1.4:', type=('build', 'run'))
    depends_on('pil@:8.2,8.3.1:', when='@1.3', type=('build', 'run'))
<|endoftext|>


<commit_before>import copy
import json
import os

from fabric.api import task, run, local, sudo, cd, env
from fabric.context_managers import settings

env.hosts = [
    os.environ['TWWEB_HOST'],
]

def virtualenv(command, user=None):
    run('source /var/www/envs/twweb/bin/activate && ' + command)

@task
def deploy(
    install='yes', build='yes', chown='no', refresh='yes', announce='yes'
):
    pubsub_message = {
        'data': {
            'should_refresh': True if refresh == 'yes' else False
        },
        'system': True,
    }
    pre_message = copy.copy(pubsub_message)
    pre_message['type'] = 'deploy_started'
    post_message = copy.copy(pubsub_message)
    post_message['type'] = 'deploy_finished'

    local('git push origin development')
    local('git checkout master')
    local('git merge development')
    local('git push origin master')
    local('git checkout development')

    run(
        "redis-cli -n 1 PUBLISH __general__ '%s'" % json.dumps(pre_message)
    )
    if chown == 'yes':
        sudo('/bin/chown -R www-data:www-data /var/www/twweb', shell=False)
    if build == 'yes':
        # Restart celery in case it's using a lot of memory -- which is
        # totally a possibility.
        sudo('/usr/sbin/service twweb-celery restart', shell=False)
    with cd('/var/www/twweb'):
        run('git fetch origin')
        run('git merge origin/master')
        if install == 'yes':
            run('bower install')
            run('npm install')
            virtualenv('pip install -r /var/www/twweb/requirements-frozen.txt')
        if build == 'yes':
            # Clear out vendor sourcemaps
            with settings(warn_only=True):
                run(
                    "grep -lr --include=*.js sourceMappingURL "
                    "bower_components/ | xargs sed -i 's/sourceMappingURL//g'"
                )
            run('ember build --environment=production')
        virtualenv('python manage.py collectstatic --noinput')
        virtualenv('python manage.py migrate')
    sudo('/bin/chown -R www-data:www-data /var/www/twweb/logs/', shell=False)
    run(
        "redis-cli -n 1 PUBLISH __general__ '%s'" % json.dumps(post_message)
    )
    sudo('/usr/sbin/service twweb restart', shell=False)
    sudo('/usr/sbin/service twweb-status restart', shell=False)
    sudo('/usr/sbin/service twweb-celery restart', shell=False)
    sudo('/usr/sbin/service twweb-sync-listener restart', shell=False)
    sudo('/usr/sbin/service twweb-log-consumer restart', shell=False)
    sudo('/bin/chown -R www-data:www-data /var/www/twweb/logs/', shell=False)

    if announce == 'yes':
        commit = local('git rev-parse HEAD', capture=True)
        virtualenv(
            "curl -d message='[Deploy completed successfully.]"
            "(https://github.com/coddingtonbear/inthe.am/commit/%s)' "
            "$GITTER_WEBHOOK_URL" % (
                commit,
            )
        )
<commit_msg>Try using --data-urlencode to make it format markdown properly.<commit_after>import copy
import json
import os

from fabric.api import task, run, local, sudo, cd, env
from fabric.context_managers import settings

env.hosts = [
    os.environ['TWWEB_HOST'],
]

def virtualenv(command, user=None):
    run('source /var/www/envs/twweb/bin/activate && ' + command)

@task
def deploy(
    install='yes', build='yes', chown='no', refresh='yes', announce='yes'
):
    pubsub_message = {
        'data': {
            'should_refresh': True if refresh == 'yes' else False
        },
        'system': True,
    }
    pre_message = copy.copy(pubsub_message)
    pre_message['type'] = 'deploy_started'
    post_message = copy.copy(pubsub_message)
    post_message['type'] = 'deploy_finished'

    local('git push origin development')
    local('git checkout master')
    local('git merge development')
    local('git push origin master')
    local('git checkout development')

    run(
        "redis-cli -n 1 PUBLISH __general__ '%s'" % json.dumps(pre_message)
    )
    if chown == 'yes':
        sudo('/bin/chown -R www-data:www-data /var/www/twweb', shell=False)
    if build == 'yes':
        # Restart celery in case it's using a lot of memory -- which is
        # totally a possibility.
        sudo('/usr/sbin/service twweb-celery restart', shell=False)
    with cd('/var/www/twweb'):
        run('git fetch origin')
        run('git merge origin/master')
        if install == 'yes':
            run('bower install')
            run('npm install')
            virtualenv('pip install -r /var/www/twweb/requirements-frozen.txt')
        if build == 'yes':
            # Clear out vendor sourcemaps
            with settings(warn_only=True):
                run(
                    "grep -lr --include=*.js sourceMappingURL "
                    "bower_components/ | xargs sed -i 's/sourceMappingURL//g'"
                )
            run('ember build --environment=production')
        virtualenv('python manage.py collectstatic --noinput')
        virtualenv('python manage.py migrate')
    sudo('/bin/chown -R www-data:www-data /var/www/twweb/logs/', shell=False)
    run(
        "redis-cli -n 1 PUBLISH __general__ '%s'" % json.dumps(post_message)
    )
    sudo('/usr/sbin/service twweb restart', shell=False)
    sudo('/usr/sbin/service twweb-status restart', shell=False)
    sudo('/usr/sbin/service twweb-celery restart', shell=False)
    sudo('/usr/sbin/service twweb-sync-listener restart', shell=False)
    sudo('/usr/sbin/service twweb-log-consumer restart', shell=False)
    sudo('/bin/chown -R www-data:www-data /var/www/twweb/logs/', shell=False)

    if announce == 'yes':
        commit = local('git rev-parse HEAD', capture=True)
        virtualenv(
            "curl --data-urlencode \"message=[Deploy completed successfully.]"
            "(https://github.com/coddingtonbear/inthe.am/commit/%s)\" "
            "$GITTER_WEBHOOK_URL" % (
                commit,
            )
        )
<|endoftext|>


<commit_before>from fabric.api import task
import sys

if '.' not in sys.path:
    sys.path.append('.')

KNOWN_LABS = ['g7-2-vts.yaml', 'g7-2-vpp.yaml', 'marahaika.yaml', 'c42top.yaml', 'i11tb3.yaml']

@task
def cmd():
    """fab cmd\t\t\t\tRun single command on lab device
    """
    from fabric.operations import prompt
    import time
    from lab.laboratory import Laboratory
    from lab.deployers.deployer_existing import DeployerExisting
    from lab.with_log import lab_logger

    def get_lab_nodes(cfg_path):
        l = Laboratory(config_path=cfg_path)
        return l, sorted(map(lambda node: node.get_node_id(), l.get_nodes_by_class()))

    def get_node_methodes(l, name):
        if name == 'cloud':
            d = DeployerExisting({'hardware-lab-config': lab_cfg_path}).execute([])
        elif device_name == 'lab':
            d = l
        else:
            d = l.get_node_by_id(name)
        return d, [x for x in dir(d) if not (x.startswith('_') or x[0].isupper())]

    def execute(d, name):
        d.log('executing method {} {}'.format(name, 10*':'))
        time.sleep(1)
        method_to_execute = getattr(d, name)
        parameters = method_to_execute.func_code.co_varnames[1:method_to_execute.func_code.co_argcount]
        arguments = []
        for parameter in parameters:
            argument = prompt(text='{p}=? '.format(p=parameter))
            if argument.startswith('['):
                argument = argument.strip('[]').split(',')
            elif argument in ['True', 'true', 'yes']:
                argument = True
            elif argument in ['False', 'false', 'no']:
                argument = False
            arguments.append(argument)
        try:
            results = method_to_execute(*arguments) if arguments else method_to_execute()
            d.log('{}() returns:\n\n {}\n'.format(name, results))
        except Exception as ex:
            lab_logger.exception('\n Exception: {0}'.format(ex))

    while True:
        lab_cfg_path = get_user_input(options_lst=KNOWN_LABS)
        lab, nodes = get_lab_nodes(lab_cfg_path)
        while True:
            device_name = get_user_input(owner=lab, options_lst=['lab', 'cloud'] + nodes)
            if device_name == 'level_up':
                break
            device, method_names = get_node_methodes(l=lab, name=device_name)
            while True:
                method_name = get_user_input(owner=device, options_lst=method_names)
                if method_name == 'level_up':
                    break
                execute(d=device, name=method_name)
                time.sleep(1)  # sleep to prevent fabric prompt clashing with
                prompt('continue? >')

@task
def ha(lab_cfg_path, test_regex, mode='run', is_noclean=False):
    """fab ha:g10,str\t\t\tRun all tests with 'str' in name on g10
        :param lab_cfg_path: which lab
        :param test_regex: regex to match some tc in $REPO/configs/ha
        :param mode: 'run' to run tests 'debug' to debug parallel infrastructure, 'check' to validate all HA test configs
        :param is_noclean: if True, do not cleanup objects created during test, leave them from post analysis
    """
    from lab.runners.runner_ha import RunnerHA

    possible_modes = [RunnerHA.MODE_CHECK, RunnerHA.MODE_DEBUG, RunnerHA.MODE_RUN]
    if mode not in possible_modes:
        raise ValueError('fabfile.py.ha(): mode "{}" is invalid, use one of {}'.format(mode, possible_modes))

    RunnerHA.run(lab_cfg_path=lab_cfg_path, test_regex=test_regex, mode=mode, is_noclean=is_noclean)

@task
def rally(lab, concurrency, max_vlans, task_yaml, rally_repo='https://git.openstack.org/openstack/rally.git', rally_patch=''):
    """fab rally:g10,2,0,200\t\tRun rally with 2 threads for 0-200 vlans
    :param lab: lab name - one of yaml in $REPO/configs
    :param concurrency: how many parallel threads
    :param max_vlans: right margin of vlan range
    :param task_yaml: specify task yaml, one from $REPO/configs/rally
    :param rally_repo: specify rally git repo if needed
    :param rally_patch: specify review if needed
    """
    from lab.with_config import WithConfig

    lab_confirmed = filter(lambda x: lab in x, WithConfig.ls_configs())

    if not lab_confirmed:
        raise ValueError('There is no hardware configuration for lab {0}'.format(lab))

    n_tenants = int(max_vlans) / 2
    with open('configs/rally/{0}.yaml'.format(task_yaml)) as f:
        task_body = f.read()
        task_body = task_body.replace('{n_times}', max_vlans)
        task_body = task_body.replace('{concurrency}', concurrency)
        task_body = task_body.replace('{n_tenants}', '{0}'.format(n_tenants))

    with WithConfig.open_artifact('task-rally.yaml', 'w') as f:
        f.write(task_body)

    with WithConfig.open_artifact('rally-runner.yaml', 'w') as f:
        f.write('deployer:  {lab.deployers.deployer_existing_osp7.DeployerExistingOSP7: {cloud: %s, hardware-lab-config: %s}}\n' % (lab, lab))
        f.write('runner:    {{lab.runners.runner_rally.RunnerRally: {{cloud: {lab}, hardware-lab-config: {lab}, task-yaml: artifacts/task-rally.yaml, rally-repo: "{repo}", rally-patch: "{patch}" }}}}\n'.format(
            lab=lab, repo=rally_repo, patch=rally_patch))
    run('artifacts/rally-runner.yaml')

@task
def run(config_path, version):
    """fab run:bxb-run-rally\t\tGeneral: run any job specified by yaml
        :param config_path: path to valid run specification, usually one of yaml from $REPO/configs/run
        :param version: specify a version of the product
    """
    from lab.base_lab import BaseLab

    l = BaseLab(yaml_name=config_path, version=version)
    return l.run()

@task
def conf():
    """fab conf\t\t\t\tTries to create lab configuration yaml
    """
    from lab.configurator import LabConfigurator

    LabConfigurator()

@task
def ansible():
    from collections import namedtuple
    from ansible.parsing.dataloader import DataLoader
    from ansible.vars import VariableManager
    from ansible.inventory import Inventory
    from ansible.playbook.play import Play
    from ansible.executor.task_queue_manager import TaskQueueManager
    from ansible.plugins.callback import CallbackBase
    from lab.with_log import lab_logger

    class ResultCallback(CallbackBase):
        def __init__(self):
            super(ResultCallback, self).__init__()

        def v2_runner_on_ok(self, result, **kwargs):
            lab_logger.info(result)

    variable_manager = VariableManager()
    loader = DataLoader()
    options = namedtuple('Options', ['connection', 'module_path', 'forks', 'become', 'become_method', 'become_user', 'check'])(connection='local',
                                                                                module_path=None,
                                                                                forks=100,
                                                                                become=None,
                                                                                become_method=None,
                                                                                become_user=None,
                                                                                check=False)
    passwords = dict(vault_pass='secret')

    # create inventory and pass to var manager
    inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=['10.23.221.142'])
    variable_manager.set_inventory(inventory)

    # create play with tasks
    play_source = dict(name="Ansible Play", hosts='10.23.221.142', gather_facts='no', tasks=[
                            dict(action=dict(module='shell', args='ls'), register='shell_out'),
                            dict(action=dict(module='debug', args=dict(msg='{{shell_out.stdout}}')))
                            ]
                       )

    play = Play().load(play_source, variable_manager=variable_manager, loader=loader)

    tqm = None
    try:
        tqm = TaskQueueManager(inventory=inventory, variable_manager=variable_manager, loader=loader, options=options, passwords=passwords, stdout_callback=ResultCallback())
        res = tqm.run(play)
        lab_logger.info('Ansible Result: {}'.format(res))
    finally:
        if tqm is not None:
            tqm.cleanup()

@task
def info(lab_config_path=None, regex=None):
    """fab info:g10,regex\t\t\tExec grep regex
    """
    from lab.laboratory import Laboratory
    from lab.deployers.deployer_existing import DeployerExisting

    lab_config_path = lab_config_path or get_user_input(options_lst=KNOWN_LABS)
    regex = regex or get_user_input(options_lst=['ERROR', 'error'])

    l = Laboratory(lab_config_path)
    d = DeployerExisting({'hardware-lab-config': lab_config_path})
    try:
        cloud = d.execute([])
        cloud.r_collect_information(regex=regex, comment='')
    except RuntimeError:
        pass  # it's ok if cloud is not yet deployed in the lab
    l.r_collect_information(regex=regex, comment='')

def get_user_input(options_lst, owner=None):
    from fabric.operations import prompt
    import sys

    sub_list = options_lst
    while True:
        choice = prompt('choose one of:\n{}\nq to quit{}>'.format('\n'.join(sorted(sub_list)), '\nu to level up\nfor {}'.format(owner) if owner else ''))
        if choice == 'q':
            sys.exit(2)
        if owner and choice == 'u':
            return 'level_up'
        sub_list = filter(lambda x: choice in x, sub_list)
        if len(sub_list) == 1:
            return sub_list[0]  # unique match , return it
        elif len(sub_list) == 0:
            sub_list = options_lst
            continue  # wrong input ask again with full list of options
        else:
            continue  # ask again with restricted list of options

@task
def bash():
    """fab bash\t\t\t\tDefine bash aliases for lab"""
    from lab.laboratory import Laboratory
    from fabric.api import local

    lab_cfg_path = get_user_input(options_lst=KNOWN_LABS)
    l = Laboratory(lab_cfg_path)
    file_name = 'tmp.aliases'
    local('rm -f ' + file_name)
    for node in l.get_nodes_by_class():
        cmds = node.get_ssh_for_bash()
        if type(cmds) is tuple:
            local('echo \'alias {}="{}"\' >> {}'.format(node.get_node_id(), cmds[0], file_name))
            local('echo \'alias cimc_{}="{}"\' >> {}'.format(node.get_node_id(), cmds[1], file_name))
        else:
            local('echo \'alias {}="{}"\' >> {}'.format(node.get_node_id(), cmds, file_name))
    local('echo \'PS1="({}) $PS1 "\' >> {}'.format(l, file_name))
<commit_msg>fab: make sure that empty string is considered as False in ha job<commit_after>from fabric.api import task
import sys

if '.' not in sys.path:
    sys.path.append('.')

KNOWN_LABS = ['g7-2-vts.yaml', 'g7-2-vpp.yaml', 'marahaika.yaml', 'c42top.yaml', 'i11tb3.yaml']

@task
def cmd():
    """fab cmd\t\t\t\tRun single command on lab device
    """
    from fabric.operations import prompt
    import time
    from lab.laboratory import Laboratory
    from lab.deployers.deployer_existing import DeployerExisting
    from lab.with_log import lab_logger

    def get_lab_nodes(cfg_path):
        l = Laboratory(config_path=cfg_path)
        return l, sorted(map(lambda node: node.get_node_id(), l.get_nodes_by_class()))

    def get_node_methodes(l, name):
        if name == 'cloud':
            d = DeployerExisting({'hardware-lab-config': lab_cfg_path}).execute([])
        elif device_name == 'lab':
            d = l
        else:
            d = l.get_node_by_id(name)
        return d, [x for x in dir(d) if not (x.startswith('_') or x[0].isupper())]

    def execute(d, name):
        d.log('executing method {} {}'.format(name, 10*':'))
        time.sleep(1)
        method_to_execute = getattr(d, name)
        parameters = method_to_execute.func_code.co_varnames[1:method_to_execute.func_code.co_argcount]
        arguments = []
        for parameter in parameters:
            argument = prompt(text='{p}=? '.format(p=parameter))
            if argument.startswith('['):
                argument = argument.strip('[]').split(',')
            elif argument in ['True', 'true', 'yes']:
                argument = True
            elif argument in ['False', 'false', 'no']:
                argument = False
            arguments.append(argument)
        try:
            results = method_to_execute(*arguments) if arguments else method_to_execute()
            d.log('{}() returns:\n\n {}\n'.format(name, results))
        except Exception as ex:
            lab_logger.exception('\n Exception: {0}'.format(ex))

    while True:
        lab_cfg_path = get_user_input(options_lst=KNOWN_LABS)
        lab, nodes = get_lab_nodes(lab_cfg_path)
        while True:
            device_name = get_user_input(owner=lab, options_lst=['lab', 'cloud'] + nodes)
            if device_name == 'level_up':
                break
            device, method_names = get_node_methodes(l=lab, name=device_name)
            while True:
                method_name = get_user_input(owner=device, options_lst=method_names)
                if method_name == 'level_up':
                    break
                execute(d=device, name=method_name)
                time.sleep(1)  # sleep to prevent fabric prompt clashing with
                prompt('continue? >')

@task
def ha(lab_cfg_path, test_regex, mode='run', is_noclean=False):
    """fab ha:g10,str\t\t\tRun all tests with 'str' in name on g10
        :param lab_cfg_path: which lab
        :param test_regex: regex to match some tc in $REPO/configs/ha
        :param mode: 'run' to run tests 'debug' to debug parallel infrastructure, 'check' to validate all HA test configs
        :param is_noclean: if True, do not cleanup objects created during test, leave them from post analysis
    """
    from lab.runners.runner_ha import RunnerHA

    possible_modes = [RunnerHA.MODE_CHECK, RunnerHA.MODE_DEBUG, RunnerHA.MODE_RUN]
    if mode not in possible_modes:
        raise ValueError('fabfile.py.ha(): mode "{}" is invalid, use one of {}'.format(mode, possible_modes))

    RunnerHA.run(lab_cfg_path=lab_cfg_path, test_regex=test_regex, mode=mode, is_noclean=str(is_noclean) in ['true', 'True', 'yes', 'Yes'])

@task
def rally(lab, concurrency, max_vlans, task_yaml, rally_repo='https://git.openstack.org/openstack/rally.git', rally_patch=''):
    """fab rally:g10,2,0,200\t\tRun rally with 2 threads for 0-200 vlans
    :param lab: lab name - one of yaml in $REPO/configs
    :param concurrency: how many parallel threads
    :param max_vlans: right margin of vlan range
    :param task_yaml: specify task yaml, one from $REPO/configs/rally
    :param rally_repo: specify rally git repo if needed
    :param rally_patch: specify review if needed
    """
    from lab.with_config import WithConfig

    lab_confirmed = filter(lambda x: lab in x, WithConfig.ls_configs())

    if not lab_confirmed:
        raise ValueError('There is no hardware configuration for lab {0}'.format(lab))

    n_tenants = int(max_vlans) / 2
    with open('configs/rally/{0}.yaml'.format(task_yaml)) as f:
        task_body = f.read()
        task_body = task_body.replace('{n_times}', max_vlans)
        task_body = task_body.replace('{concurrency}', concurrency)
        task_body = task_body.replace('{n_tenants}', '{0}'.format(n_tenants))

    with WithConfig.open_artifact('task-rally.yaml', 'w') as f:
        f.write(task_body)

    with WithConfig.open_artifact('rally-runner.yaml', 'w') as f:
        f.write('deployer:  {lab.deployers.deployer_existing_osp7.DeployerExistingOSP7: {cloud: %s, hardware-lab-config: %s}}\n' % (lab, lab))
        f.write('runner:    {{lab.runners.runner_rally.RunnerRally: {{cloud: {lab}, hardware-lab-config: {lab}, task-yaml: artifacts/task-rally.yaml, rally-repo: "{repo}", rally-patch: "{patch}" }}}}\n'.format(
            lab=lab, repo=rally_repo, patch=rally_patch))
    run('artifacts/rally-runner.yaml')

@task
def run(config_path, version):
    """fab run:bxb-run-rally\t\tGeneral: run any job specified by yaml
        :param config_path: path to valid run specification, usually one of yaml from $REPO/configs/run
        :param version: specify a version of the product
    """
    from lab.base_lab import BaseLab

    l = BaseLab(yaml_name=config_path, version=version)
    return l.run()

@task
def conf():
    """fab conf\t\t\t\tTries to create lab configuration yaml
    """
    from lab.configurator import LabConfigurator

    LabConfigurator()

@task
def ansible():
    from collections import namedtuple
    from ansible.parsing.dataloader import DataLoader
    from ansible.vars import VariableManager
    from ansible.inventory import Inventory
    from ansible.playbook.play import Play
    from ansible.executor.task_queue_manager import TaskQueueManager
    from ansible.plugins.callback import CallbackBase
    from lab.with_log import lab_logger

    class ResultCallback(CallbackBase):
        def __init__(self):
            super(ResultCallback, self).__init__()

        def v2_runner_on_ok(self, result, **kwargs):
            lab_logger.info(result)

    variable_manager = VariableManager()
    loader = DataLoader()
    options = namedtuple('Options', ['connection', 'module_path', 'forks', 'become', 'become_method', 'become_user', 'check'])(connection='local',
                                                                                module_path=None,
                                                                                forks=100,
                                                                                become=None,
                                                                                become_method=None,
                                                                                become_user=None,
                                                                                check=False)
    passwords = dict(vault_pass='secret')

    # create inventory and pass to var manager
    inventory = Inventory(loader=loader, variable_manager=variable_manager, host_list=['10.23.221.142'])
    variable_manager.set_inventory(inventory)

    # create play with tasks
    play_source = dict(name="Ansible Play", hosts='10.23.221.142', gather_facts='no', tasks=[
                            dict(action=dict(module='shell', args='ls'), register='shell_out'),
                            dict(action=dict(module='debug', args=dict(msg='{{shell_out.stdout}}')))
                            ]
                       )

    play = Play().load(play_source, variable_manager=variable_manager, loader=loader)

    tqm = None
    try:
        tqm = TaskQueueManager(inventory=inventory, variable_manager=variable_manager, loader=loader, options=options, passwords=passwords, stdout_callback=ResultCallback())
        res = tqm.run(play)
        lab_logger.info('Ansible Result: {}'.format(res))
    finally:
        if tqm is not None:
            tqm.cleanup()

@task
def info(lab_config_path=None, regex=None):
    """fab info:g10,regex\t\t\tExec grep regex
    """
    from lab.laboratory import Laboratory
    from lab.deployers.deployer_existing import DeployerExisting

    lab_config_path = lab_config_path or get_user_input(options_lst=KNOWN_LABS)
    regex = regex or get_user_input(options_lst=['ERROR', 'error'])

    l = Laboratory(lab_config_path)
    d = DeployerExisting({'hardware-lab-config': lab_config_path})
    try:
        cloud = d.execute([])
        cloud.r_collect_information(regex=regex, comment='')
    except RuntimeError:
        pass  # it's ok if cloud is not yet deployed in the lab
    l.r_collect_information(regex=regex, comment='')

def get_user_input(options_lst, owner=None):
    from fabric.operations import prompt
    import sys

    sub_list = options_lst
    while True:
        choice = prompt('choose one of:\n{}\nq to quit{}>'.format('\n'.join(sorted(sub_list)), '\nu to level up\nfor {}'.format(owner) if owner else ''))
        if choice == 'q':
            sys.exit(2)
        if owner and choice == 'u':
            return 'level_up'
        sub_list = filter(lambda x: choice in x, sub_list)
        if len(sub_list) == 1:
            return sub_list[0]  # unique match , return it
        elif len(sub_list) == 0:
            sub_list = options_lst
            continue  # wrong input ask again with full list of options
        else:
            continue  # ask again with restricted list of options

@task
def bash():
    """fab bash\t\t\t\tDefine bash aliases for lab"""
    from lab.laboratory import Laboratory
    from fabric.api import local

    lab_cfg_path = get_user_input(options_lst=KNOWN_LABS)
    l = Laboratory(lab_cfg_path)
    file_name = 'tmp.aliases'
    local('rm -f ' + file_name)
    for node in l.get_nodes_by_class():
        cmds = node.get_ssh_for_bash()
        if type(cmds) is tuple:
            local('echo \'alias {}="{}"\' >> {}'.format(node.get_node_id(), cmds[0], file_name))
            local('echo \'alias cimc_{}="{}"\' >> {}'.format(node.get_node_id(), cmds[1], file_name))
        else:
            local('echo \'alias {}="{}"\' >> {}'.format(node.get_node_id(), cmds, file_name))
    local('echo \'PS1="({}) $PS1 "\' >> {}'.format(l, file_name))
<|endoftext|>