<commit_before>#!/usr/bin/python

"""timecard.py

Records time usage for e.g. billable time and proof of work.
"""

import sys
import os
import logging
import signal
import subprocess
import time
import argparse
import datetime
import re
from dateutil import parser as dateparser
import gtk.gdk

def find_display(max_n=9):
    import Xlib.display, Xlib.error
    display_name = ""
    test_count = 0
    while display_name == "":
        test_name = ":%d.0" % (test_count)
        try:
            display_name = Xlib.display.Display(test_name).display.display_name
        except Xlib.error.DisplayConnectionError:
            if test_count > max_n:
                return None
            test_count += 1
            continue
    return display_name

def get_active_window():
    window_id = subprocess.check_output(['xprop', '-root', '-f', '_NET_ACTIVE_WINDOW', '0x', r' $0\n', '_NET_ACTIVE_WINDOW']).split()[1]
    window_title = ' '.join(subprocess.check_output(['xprop', '-id', str(window_id), '-f', '_NET_WM_NAME', '0s', r' $0\n', '_NET_WM_NAME']).split()[1:]).strip('"')
    return window_title

def get_lock(lockfilename):
    if not os.path.isfile(lockfilename):
        return None
    try:
        pid = int(open(lockfilename, 'r').read().strip())
    except ValueError:
        return None
    return pid

def lock_timecard(pid, lockfilename):
    if not get_lock(lockfilename):
        open(lockfilename, 'w').write(str(pid))
        return True
    else:
        return False

def release_lock(lockfilename):
    if get_lock(lockfilename) == os.getpid():
        os.remove(lockfilename)
        return True
    else:
        return False

def format_timestamp(dt, compact=False):
    if compact:
        return dt.strftime("%Y-%m-%d_%H:%M:%S")
    else:
        return dt.strftime("%H:%M:%S, %a %b %d, %Y")

def get_current_timestamp(compact=False):
    return format_timestamp(datetime.datetime.now(), compact=compact)

def parse_timerange(timerange):
	timerange = timerange.split('-')
	if len(timerange) == 1:
		timerange.append('now')
	timestamp_range = []
	for item in timerange:
		if item == 'now':
			timestamp_range.append(datetime.datetime.now())
		elif re.match(r'(\d+[wdh])+', item):
			quanta = filter(len, re.split(r'(\d+[wdh])', item))
			quanta = {q[-1]: int(q[:-1]) for q in quanta}
			for k in ('w', 'd', 'h'):
				if k not in quanta:
					quanta[k] = 0
			quanta['d'] = quanta['w']*7 + quanta['d']
			quanta['s'] = quanta['h']*24*60*60
			delta = datetime.timedelta(days=quanta['d'], seconds=quanta['s'])
			timestamp_range.append(datetime.datetime.now() - delta)
		else:
			try:
				timestamp_range.append(datetime.datetime.fromtimestamp(int(item)))
				continue
			except ValueError:
				pass
			try:
				timestamp_range.append(dateparser.parse(item))
			except ValueError:
				raise ValueError, "unknown date format"
	timestamp_range.sort()
	#print timestamp_range
	return timestamp_range

def start_log():
    global args
    f = open(args.file, 'a')
    print >>f, "-- Starting log at %s --" % (get_current_timestamp())
    logger.debug("-- Starting log at %s --", get_current_timestamp())
    f.close()

def write_note(note):
    global args
    f = open(args.file, 'a')
    print >>f, "%s: [Note] %s" % (get_current_timestamp(), note)
    f.close()

def monitor():
    global args
    f = open(args.file, 'a')
    print >>f, "%s: %s" % (get_current_timestamp(), get_active_window())
    logger.debug("%s: %s", get_current_timestamp(), get_active_window())
    f.close()

def close_log():
    global args
    f = open(args.file, 'a')
    print >>f, "-- Closing log at %s --" % (get_current_timestamp())
    logger.debug("-- Closing log at %s --", get_current_timestamp())
    f.close()

def stop_monitoring(signum, frame):
    if signum in (signal.SIGTERM, signal.SIGINT) and args.verbose >= 2:
        logger.debug("Got %s." % ("SIGTERM" if signum==signal.SIGTERM else "SIGINT"))
    if signum in (signal.SIGTERM, signal.SIGINT):
        close_log()
        if release_lock(args.lockfile):
            sys.exit(0)
        else:
            print >>sys.stderr, "Failed to release lock."
            sys.exit(1)

def take_screenshot():
    logger.debug("Taking screenshot.")
    w = gtk.gdk.get_default_root_window()
    sz = w.get_size()
    pb = gtk.gdk.Pixbuf(gtk.gdk.COLORSPACE_RGB,False,8,sz[0],sz[1])
    pb = pb.get_from_drawable(w,w.get_colormap(),0,0,0,0,sz[0],sz[1])
    #pb = pb.subpixbuf(0, 0, 1920, 1200)
    #pb = pb.scale_simple(int(1920/1.25), int(1200/1.25))
    pb = pb.scale_simple(int(sz[0]/1.25), int(sz[1]/1.25))
    if (pb != None):
        pb.save(os.path.join(args.screenshots, "%s.png" % (get_current_timestamp(compact=True))), "png", {'compression': '9'})
        return True
    else:
        return False

logger = logging.getLogger(__name__)

# If called from cron, find first display.
if not 'DISPLAY' in os.environ:
	os.environ['DISPLAY'] = find_display()

commands = ["start", "stop", "note", "list", "analyze", "test"]

argparser = argparse.ArgumentParser(description="Record or analyze time usage.")
argparser.add_argument('-v', '--verbose', action='count', default=0, help="Display debug messages. -vv will disable forking.")
argparser.add_argument('-f', '--file', metavar='file', default='timecard.log', help='Time log file.')
argparser.add_argument('-l', '--lockfile', metavar='lockfile', default='timecard.lock', help='Lock file name.')
argparser.add_argument('-s', '--screenshots', metavar='dir', nargs='?', default=None, const='screenshots', help='Take screenshots with every log entry. Optional: directory to store screenshots (default is screenshots/).')
argparser.add_argument('-i', '--interval', metavar='interval', type=int, default=300, help='Seconds between monitor reports.')
argparser.add_argument('command', choices=commands)
argparser.add_argument('-n', '--note', metavar='note', help='Add a note to this action.')
argparser.add_argument('timerange', nargs='?', help='Time range for list command.')
argparser.add_argument('note_arg', nargs='?', help='Note to be recorded with note command.')
args = argparser.parse_args()

debuglevel = {
    0: logging.ERROR,
    1: logging.INFO,
    2: logging.DEBUG
}

logger.setLevel(debuglevel[args.verbose])
ch = logging.StreamHandler()
ch.setLevel(debuglevel[args.verbose])
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)

logger.debug(args)

if args.command == 'start':

    if get_lock(args.lockfile):
        logger.error("Timecard is already locked.")
        sys.exit(1)

    if args.verbose >= 2:
        if not lock_timecard(os.getpid(), args.lockfile):
            logger.error("Unable to create lock file.")
            sys.exit(1)
        print "Clocked in at %s." % (datetime.datetime.now().strftime("%H:%M:%S, %a %b %d, %Y"))
    else:
        pid = os.fork()
        if pid > 0:
            logger.debug("Parent reports child pid=%d", pid)
            if not lock_timecard(pid, args.lockfile):
                os.kill(pid, signal.SIGTERM)
                logger.error("Unable to create lock file.")
                sys.exit(1)
            print "Clocked in at %s." % (datetime.datetime.now().strftime("%H:%M:%S, %a %b %d, %Y"))
            sys.exit(0)

    # Child process - this will do the monitoring
    # Give the parent a chance to do last checks and kill us if needed.
    logger.debug("Child started.")
    time.sleep(2)
    start_log()
    if args.note:
        write_note(args.note)
    time.sleep(5)
    signal.signal(signal.SIGTERM, stop_monitoring)
    if args.verbose >= 2:
        signal.signal(signal.SIGINT, stop_monitoring)
    while True:
        monitor()
        if args.screenshots:
            take_screenshot()
        logger.debug("Sleeping %d seconds.", args.interval)
        time.sleep(args.interval)

elif args.command == 'note':
    write_note(args.timerange) # Because only one optional positional arg works
    print "Note saved at %s." % (get_current_timestamp())

elif args.command == 'stop':
    pid = get_lock(args.lockfile)
    if not pid:
        logger.error("Could not get a valid PID from lock file.")
        sys.exit(1)
    if args.note:
    	write_note(args.note)
    logger.debug("Killing process %d.", pid)
    os.kill(pid, signal.SIGTERM)
    print "Clocked out at %s." % (datetime.datetime.now().strftime("%H:%M:%S, %a %b %d, %Y"))

elif args.command == 'list':
    total_log = map(lambda l: l.strip(), open(args.file, 'r').readlines())
    spans = []
    closed = True
    for line in total_log:
        if closed and not line.startswith("-- Starting"):
            continue
        elif line.startswith("-- Starting"):
            timestamp = dateparser.parse(line[len("-- Starting log at "):-3])
            spans.append([(timestamp, line)])
            closed = False
        elif line.startswith("-- Closing"):
            timestamp = dateparser.parse(line[len("-- Closing log at "):-3])
            spans[-1].append((timestamp, line))
            closed = True
        else:
            timestamp = dateparser.parse(':'.join(line.split(':')[:3]))
            spans[-1].append((timestamp, ':'.join(line.split(':')[3:])))
    if args.timerange:
        start_time, end_time = parse_timerange(args.timerange)
        spans = filter(lambda s: s[0][0]>start_time, spans)
    total_hours = 0.0
    for span in spans:
        st_time = span[0][0]
        e_time = span[-1][0]
        delta = e_time - st_time
        hours = delta.total_seconds()/3600
        total_hours += hours
        print "Worked from %s to %s\n  -- Total %.3f hours." % (format_timestamp(st_time), format_timestamp(e_time), hours)
    if args.timerange:
        print "\nTotal time worked from %s to %s:\n    %.3f hours" % (format_timestamp(start_time, True), format_timestamp(end_time, True), total_hours)
    else:
        print "\nTotal time worked from %s to %s:\n    %.3f hours" % (format_timestamp(spans[0][0][0], True), format_timestamp(spans[-1][-1][0], True), total_hours)

elif args.command == 'test':
    print args
    take_screenshot()

<commit_msg>timecard.py now uses modularized screenshot.py<commit_after>#!/usr/bin/python

"""timecard.py

Records time usage for e.g. billable time and proof of work.
"""

import sys
import os
import logging
import signal
import subprocess
import time
import argparse
import datetime
import re
from dateutil import parser as dateparser

def find_display(max_n=9):
    import Xlib.display, Xlib.error
    display_name = ""
    test_count = 0
    while display_name == "":
        test_name = ":%d.0" % (test_count)
        try:
            display_name = Xlib.display.Display(test_name).display.display_name
        except Xlib.error.DisplayConnectionError:
            if test_count > max_n:
                return None
            test_count += 1
            continue
    return display_name

def get_active_window():
    window_id = subprocess.check_output(['xprop', '-root', '-f', '_NET_ACTIVE_WINDOW', '0x', r' $0\n', '_NET_ACTIVE_WINDOW']).split()[1]
    window_title = ' '.join(subprocess.check_output(['xprop', '-id', str(window_id), '-f', '_NET_WM_NAME', '0s', r' $0\n', '_NET_WM_NAME']).split()[1:]).strip('"')
    return window_title

def get_lock(lockfilename):
    if not os.path.isfile(lockfilename):
        return None
    try:
        pid = int(open(lockfilename, 'r').read().strip())
    except ValueError:
        return None
    return pid

def lock_timecard(pid, lockfilename):
    if not get_lock(lockfilename):
        open(lockfilename, 'w').write(str(pid))
        return True
    else:
        return False

def release_lock(lockfilename):
    if get_lock(lockfilename) == os.getpid():
        os.remove(lockfilename)
        return True
    else:
        return False

def format_timestamp(dt, compact=False):
    if compact:
        return dt.strftime("%Y-%m-%d_%H:%M:%S")
    else:
        return dt.strftime("%H:%M:%S, %a %b %d, %Y")

def get_current_timestamp(compact=False):
    return format_timestamp(datetime.datetime.now(), compact=compact)

def parse_timerange(timerange):
	timerange = timerange.split('-')
	if len(timerange) == 1:
		timerange.append('now')
	timestamp_range = []
	for item in timerange:
		if item == 'now':
			timestamp_range.append(datetime.datetime.now())
		elif re.match(r'(\d+[wdh])+', item):
			quanta = filter(len, re.split(r'(\d+[wdh])', item))
			quanta = {q[-1]: int(q[:-1]) for q in quanta}
			for k in ('w', 'd', 'h'):
				if k not in quanta:
					quanta[k] = 0
			quanta['d'] = quanta['w']*7 + quanta['d']
			quanta['s'] = quanta['h']*24*60*60
			delta = datetime.timedelta(days=quanta['d'], seconds=quanta['s'])
			timestamp_range.append(datetime.datetime.now() - delta)
		else:
			try:
				timestamp_range.append(datetime.datetime.fromtimestamp(int(item)))
				continue
			except ValueError:
				pass
			try:
				timestamp_range.append(dateparser.parse(item))
			except ValueError:
				raise ValueError, "unknown date format"
	timestamp_range.sort()
	#print timestamp_range
	return timestamp_range

def start_log():
    global args
    f = open(args.file, 'a')
    print >>f, "-- Starting log at %s --" % (get_current_timestamp())
    logger.debug("-- Starting log at %s --", get_current_timestamp())
    f.close()

def write_note(note):
    global args
    f = open(args.file, 'a')
    print >>f, "%s: [Note] %s" % (get_current_timestamp(), note)
    f.close()

def monitor():
    global args
    f = open(args.file, 'a')
    print >>f, "%s: %s" % (get_current_timestamp(), get_active_window())
    logger.debug("%s: %s", get_current_timestamp(), get_active_window())
    f.close()

def close_log():
    global args
    f = open(args.file, 'a')
    print >>f, "-- Closing log at %s --" % (get_current_timestamp())
    logger.debug("-- Closing log at %s --", get_current_timestamp())
    f.close()

def stop_monitoring(signum, frame):
    if signum in (signal.SIGTERM, signal.SIGINT) and args.verbose >= 2:
        logger.debug("Got %s." % ("SIGTERM" if signum==signal.SIGTERM else "SIGINT"))
    if signum in (signal.SIGTERM, signal.SIGINT):
        close_log()
        if release_lock(args.lockfile):
            sys.exit(0)
        else:
            print >>sys.stderr, "Failed to release lock."
            sys.exit(1)

if __name__ == "__main__":
    # If called from cron, find first display.
    if not 'DISPLAY' in os.environ:
	    os.environ['DISPLAY'] = find_display()

    commands = ["start", "stop", "note", "list", "analyze", "test"]

    argparser = argparse.ArgumentParser(description="Record or analyze time usage.")
    argparser.add_argument('-v', '--verbose', action='count', default=0, help="Display debug messages. -vv will disable forking.")
    argparser.add_argument('-f', '--file', metavar='file', default='timecard.log', help='Time log file.')
    argparser.add_argument('-l', '--lockfile', metavar='lockfile', default='timecard.lock', help='Lock file name.')
    argparser.add_argument('-s', '--screenshots', metavar='dir', nargs='?', default=None, const='screenshots', help='Take screenshots with every log entry. Optional: directory to store screenshots (default is screenshots/).')
    argparser.add_argument('-i', '--interval', metavar='interval', type=int, default=300, help='Seconds between monitor reports.')
    argparser.add_argument('command', choices=commands)
    argparser.add_argument('-n', '--note', metavar='note', help='Add a note to this action.')
    argparser.add_argument('timerange', nargs='?', help='Time range for list command.')
    argparser.add_argument('note_arg', nargs='?', help='Note to be recorded with note command.')
    args = argparser.parse_args()

    debuglevel = {
        0: logging.ERROR,
        1: logging.INFO,
        2: logging.DEBUG
    }

    logger = logging.getLogger(__name__)
    logger.setLevel(debuglevel[args.verbose])
    ch = logging.StreamHandler()
    ch.setLevel(debuglevel[args.verbose])
    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
    ch.setFormatter(formatter)
    logger.addHandler(ch)

    logger.debug(args)

    if args.screenshots:
        import screenshot
        screenshot.logger = logger

    if args.command == 'start':

        if get_lock(args.lockfile):
            logger.error("Timecard is already locked.")
            sys.exit(1)

        if args.verbose >= 2:
            if not lock_timecard(os.getpid(), args.lockfile):
                logger.error("Unable to create lock file.")
                sys.exit(1)
            print "Clocked in at %s." % (datetime.datetime.now().strftime("%H:%M:%S, %a %b %d, %Y"))
        else:
            pid = os.fork()
            if pid > 0:
                logger.debug("Parent reports child pid=%d", pid)
                if not lock_timecard(pid, args.lockfile):
                    os.kill(pid, signal.SIGTERM)
                    logger.error("Unable to create lock file.")
                    sys.exit(1)
                print "Clocked in at %s." % (datetime.datetime.now().strftime("%H:%M:%S, %a %b %d, %Y"))
                sys.exit(0)

        # Child process - this will do the monitoring
        # Give the parent a chance to do last checks and kill us if needed.
        logger.debug("Child started.")
        time.sleep(2)
        start_log()
        if args.note:
            write_note(args.note)
        time.sleep(5)
        signal.signal(signal.SIGTERM, stop_monitoring)
        if args.verbose >= 2:
            signal.signal(signal.SIGINT, stop_monitoring)
        while True:
            monitor()
            if args.screenshots:
                screenshot.take_screenshot(os.path.join(args.screenshots, get_current_timestamp(True)))
            logger.debug("Sleeping %d seconds.", args.interval)
            time.sleep(args.interval)

    elif args.command == 'note':
        write_note(args.timerange) # Because only one optional positional arg works
        print "Note saved at %s." % (get_current_timestamp())

    elif args.command == 'stop':
        pid = get_lock(args.lockfile)
        if not pid:
            logger.error("Could not get a valid PID from lock file.")
            sys.exit(1)
        if args.note:
        	write_note(args.note)
        logger.debug("Killing process %d.", pid)
        os.kill(pid, signal.SIGTERM)
        print "Clocked out at %s." % (datetime.datetime.now().strftime("%H:%M:%S, %a %b %d, %Y"))

    elif args.command == 'list':
        total_log = map(lambda l: l.strip(), open(args.file, 'r').readlines())
        spans = []
        closed = True
        for line in total_log:
            if closed and not line.startswith("-- Starting"):
                continue
            elif line.startswith("-- Starting"):
                timestamp = dateparser.parse(line[len("-- Starting log at "):-3])
                spans.append([(timestamp, line)])
                closed = False
            elif line.startswith("-- Closing"):
                timestamp = dateparser.parse(line[len("-- Closing log at "):-3])
                spans[-1].append((timestamp, line))
                closed = True
            else:
                timestamp = dateparser.parse(':'.join(line.split(':')[:3]))
                spans[-1].append((timestamp, ':'.join(line.split(':')[3:])))
        if args.timerange:
            start_time, end_time = parse_timerange(args.timerange)
            spans = filter(lambda s: s[0][0]>start_time, spans)
        total_hours = 0.0
        for span in spans:
            st_time = span[0][0]
            e_time = span[-1][0]
            delta = e_time - st_time
            hours = delta.total_seconds()/3600
            total_hours += hours
            print "Worked from %s to %s\n  -- Total %.3f hours." % (format_timestamp(st_time), format_timestamp(e_time), hours)
        if args.timerange:
            print "\nTotal time worked from %s to %s:\n    %.3f hours" % (format_timestamp(start_time, True), format_timestamp(end_time, True), total_hours)
        else:
            print "\nTotal time worked from %s to %s:\n    %.3f hours" % (format_timestamp(spans[0][0][0], True), format_timestamp(spans[-1][-1][0], True), total_hours)

    elif args.command == 'test':
        print args
        import screenshot
        take_screenshot("tests/test.png", target=screenshot.ENTIRE_DESKTOP)

<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function
import numpy as np
import numpy.linalg as npl
import matplotlib.pyplot as plt
import nibabel as nib
import pandas as pd # new
import sys # instead of os
import scipy.stats
from scipy.stats import gamma
import os
import scipy.stats as stats

# Relative path to subject 1 data

project_path          = "../../"
path_to_data          = project_path+"data/ds009/"
location_of_images    = project_path+"images/"
location_of_functions = project_path+"code/utils/functions/"
final_data            = "../data/"
smooth_data           =  final_data + 'smooth/'
hrf_data              = final_data + 'hrf/'

#sys.path.append(os.path.join(os.path.dirname(__file__), "../functions/"))
sys.path.append(location_of_functions)

sub_list = os.listdir(path_to_data)[1:]

from glm import glm_multiple, glm_diagnostics
# iv. import image viewing tool
from Image_Visualizing import present_3d
from noise_correction import mean_underlying_noise, fourier_predict_underlying_noise,fourier_creation
from hypothesis import t_stat_mult_regression
# Progress bar
toolbar_width=len(sub_list)
sys.stdout.write("GLM, :  ")
sys.stdout.write("[%s]" % (" " * toolbar_width))
sys.stdout.flush()
sys.stdout.write("\b" * (toolbar_width+1)) # return to start of line, after '['

for i in sub_list[0:2]:

    img = nib.load(smooth_data+ i +"_bold_smoothed.nii")
    data = img.get_data()

    n_vols = data.shape[-1]
    convolve = np.loadtxt(hrf_data+i+"_hrf.txt")

    residual_final = np.zeros((data.shape))
    t_final = np.zeros((data.shape[:-1]))

    for j in range(data.shape[2]):

        data_slice = data[:,:,j,:]
        X = np.ones((n_vols,6))
        X[:,1] = np.ones(n_vols)#convolve[:,j]
        X[:,2]=np.linspace(-1,1,num=X.shape[0]) #drift
        X[:,3:]=fourier_creation(X.shape[0],3)[:,1:]

        beta,t,df,p = t_stat_mult_regression(data_slice, X)

        beta = beta.reshape((64,64,6))

        MRSS, fitted, residuals = glm_diagnostics(beta, X, data_slice)

        t_final[:,:,j] = t[1,:].reshape(data_slice.shape[:-1])

        residual_final[:,:,j,:] = residuals.reshape(data_slice.shape)

        np.save("../data/glm/t_stat/"+i+"_tstat.npy", t_final)
        np.save("../data/glm/residual/"+i+"_residual.npy", residual_final)

    sys.stdout.write("-")
    sys.stdout.flush()
sys.stdout.write("\n")

    <commit_msg>fixing glm.py<commit_after>from __future__ import absolute_import, division, print_function
import numpy as np
import numpy.linalg as npl
import matplotlib.pyplot as plt
import nibabel as nib
import pandas as pd # new
import sys # instead of os
import scipy.stats
from scipy.stats import gamma
import os
import scipy.stats as stats

# Relative path to subject 1 data

project_path          = "../../"
path_to_data          = project_path+"data/ds009/"
location_of_images    = project_path+"images/"
location_of_functions = project_path+"code/utils/functions/"
final_data            = "../data/"
smooth_data           =  final_data + 'smooth/'
hrf_data              = final_data + 'hrf/'

#sys.path.append(os.path.join(os.path.dirname(__file__), "../functions/"))
sys.path.append(location_of_functions)

sub_list = os.listdir(path_to_data)[1:]

from glm import glm_multiple, glm_diagnostics
# iv. import image viewing tool
from Image_Visualizing import present_3d
from noise_correction import mean_underlying_noise, fourier_predict_underlying_noise,fourier_creation
from hypothesis import t_stat_mult_regression
# Progress bar
toolbar_width=len(sub_list)
sys.stdout.write("GLM, :  ")
sys.stdout.write("[%s]" % (" " * toolbar_width))
sys.stdout.flush()
sys.stdout.write("\b" * (toolbar_width+1)) # return to start of line, after '['

for i in sub_list:

    img = nib.load(smooth_data+ i +"_bold_smoothed.nii")
    data = img.get_data()

    n_vols = data.shape[-1]
    convolve = np.loadtxt(hrf_data+i+"_hrf.txt")

    residual_final = np.zeros((data.shape))
    t_final = np.zeros((data.shape[:-1]))

    for j in range(data.shape[2]):

        data_slice = data[:,:,j,:]
        X = np.ones((n_vols,6))
        X[:,1] = np.ones(n_vols)#convolve[:,j]
        X[:,2]=np.linspace(-1,1,num=X.shape[0]) #drift
        X[:,3:]=fourier_creation(X.shape[0],3)[:,1:]

        beta,t,df,p = t_stat_mult_regression(data_slice, X)

        beta = beta.reshape((64,64,6))

        MRSS, fitted, residuals = glm_diagnostics(beta, X, data_slice)

        t_final[:,:,j] = t[1,:].reshape(data_slice.shape[:-1])

        residual_final[:,:,j,:] = residuals.reshape(data_slice.shape)

        np.save("../data/glm/t_stat/"+i+"_tstat.npy", t_final)
        np.save("../data/glm/residual/"+i+"_residual.npy", residual_final)

    sys.stdout.write("-")
    sys.stdout.flush()
sys.stdout.write("\n")

    <|endoftext|>


<commit_before>from .generate_structure import GenerateStructure
<commit_msg>Transforming each command to be generated by a factory<commit_after>from .generate_structure import GenerateStructure

class CommandFactory(object):
  pass
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import codecs
import csv
import cStringIO

# These classes are copied from http://docs.python.org/2/library/csv.html

class CsvUnicodeWriter(object):
    """
    A CSV writer which will write rows to CSV file "f",
    which is encoded in the given encoding.
    """

    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
        # Redirect output to a queue
        self.queue = cStringIO.StringIO()
        self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
        self.stream = f
        self.encoder = codecs.getincrementalencoder(encoding)()
        # Write BOM into file
        self.stream.write(codecs.BOM_UTF8)

    def cast_to_str(self, obj):
        if isinstance(obj, unicode):
            return obj.encode('utf-8')
        elif isinstance(obj, str):
            return obj
        elif hasattr(obj, '__unicode__'):
            return unicode(obj).encode('utf-8')
        elif hasattr(obj, '__str__'):
            return str(obj)
        else:
            raise TypeError('Expecting unicode, str, or object castable'
                            ' to unicode or string, got: %r' % type(obj))

    def writerow(self, row):
        self.writer.writerow([self.cast_to_str(s) for s in row])
        # Fetch UTF-8 output from the queue ...
        data = self.queue.getvalue()
        data = data.decode("utf-8")
        # ... and reencode it into the target encoding
        data = self.encoder.encode(data)
        # write to the target stream
        self.stream.write(data)
        # empty queue
        self.queue.truncate(0)

    def writerows(self, rows):
        for row in rows:
            self.writerow(row)

class UTF8Recoder(object):
    """
    Iterator that reads an encoded stream and reencodes the input to UTF-8
    """

    def __init__(self, f, encoding):
        self.reader = codecs.getreader(encoding)(f)

    def __iter__(self):
        return self

    def next(self):
        return self.reader.next().encode("utf-8")

class CsvUnicodeReader(object):
    """
    A CSV reader which will iterate over lines in the CSV file "f",
    which is encoded in the given encoding.
    """

    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwargs):
        f = UTF8Recoder(f, encoding)
        self.reader = csv.reader(f, dialect=dialect, **kwargs)

    def next(self):
        row = self.reader.next()
        return [unicode(s, "utf-8") for s in row]

    def __iter__(self):
        return self
<commit_msg>Remove BOM from CSV writer<commit_after># -*- coding: utf-8 -*-
import codecs
import csv
import cStringIO

# These classes are copied from http://docs.python.org/2/library/csv.html

class CsvUnicodeWriter(object):
    """
    A CSV writer which will write rows to CSV file "f",
    which is encoded in the given encoding.
    """

    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds):
        # Redirect output to a queue
        self.queue = cStringIO.StringIO()
        self.writer = csv.writer(self.queue, dialect=dialect, **kwds)
        self.stream = f
        self.encoder = codecs.getincrementalencoder(encoding)()

    def cast_to_str(self, obj):
        if isinstance(obj, unicode):
            return obj.encode('utf-8')
        elif isinstance(obj, str):
            return obj
        elif hasattr(obj, '__unicode__'):
            return unicode(obj).encode('utf-8')
        elif hasattr(obj, '__str__'):
            return str(obj)
        else:
            raise TypeError('Expecting unicode, str, or object castable'
                            ' to unicode or string, got: %r' % type(obj))

    def writerow(self, row):
        self.writer.writerow([self.cast_to_str(s) for s in row])
        # Fetch UTF-8 output from the queue ...
        data = self.queue.getvalue()
        data = data.decode("utf-8")
        # ... and reencode it into the target encoding
        data = self.encoder.encode(data)
        # write to the target stream
        self.stream.write(data)
        # empty queue
        self.queue.truncate(0)

    def writerows(self, rows):
        for row in rows:
            self.writerow(row)

class UTF8Recoder(object):
    """
    Iterator that reads an encoded stream and reencodes the input to UTF-8
    """

    def __init__(self, f, encoding):
        self.reader = codecs.getreader(encoding)(f)

    def __iter__(self):
        return self

    def next(self):
        return self.reader.next().encode("utf-8")

class CsvUnicodeReader(object):
    """
    A CSV reader which will iterate over lines in the CSV file "f",
    which is encoded in the given encoding.
    """

    def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwargs):
        f = UTF8Recoder(f, encoding)
        self.reader = csv.reader(f, dialect=dialect, **kwargs)

    def next(self):
        row = self.reader.next()
        return [unicode(s, "utf-8") for s in row]

    def __iter__(self):
        return self
<|endoftext|>


<commit_before># TODO: Temporal stripping
import datetime
import re
import flatland.exc as exc
from flatland.util import (
    Unspecified,
    as_mapping,
    lazy_property,
    )
from flatland import valid
from .base import FieldSchema, Element

__all__ = ('String', 'Integer', 'Long', 'Float', 'Boolean',
           'DateTime', 'Date', 'Time', 'Ref')

class _ScalarElement(Element):
    flattenable = True

    def __init__(self, schema, **kw):
        value = kw.pop('value', Unspecified)

        Element.__init__(self, schema, **kw)

        if value is not Unspecified:
            self.set(value)

    def set(self, value):
        """Assign the native and Unicode value.

        Attempts to adapt the given value and assigns this element's
        `.value` and `.u` attributes in tandem.  Returns True if the
        adaptation was successful.

        If adaptation succeeds, `.value` will contain the adapted
        native value and `.u` will contain a Unicode serialized
        version of it. A native value of None will be represented as
        u'' in `.u`.

        If adaptation fails, `.value` will be None and `.u` will
        contain `unicode(value)` or u'' for none.

        """

        try:
            # adapt and normalize the value, if possible
            value = self.value = self.schema.adapt(self, value)
        except exc.AdaptationError:
            self.value = None
            if value is None:
                self.u = u''
            elif isinstance(value, unicode):
                self.u = value
            else:
                try:
                    self.u = unicode(value)
                except UnicodeDecodeError:
                    self.u = unicode(value, errors='replace')
            return False

        # stringify it, possibly storing what we received verbatim or
        # a normalized version of it.
        if value is None:
            self.u = u''
        else:
            self.u = self.schema.serialize(self, value)
        return True

    def _index(self, name):
        raise IndexError(name)

    def _set_flat(self, pairs, sep):
        for key, value in pairs:
            if key == self.name:
                self.set(value)
                break

    def set_default(self):
        if self.schema.default is not Unspecified:
            self.set(self.schema.default)

    def __nonzero__(self):
        return True if self.u and self.value else False

    def __unicode__(self):
        return self.u

    def __repr__(self):
        return '<%s %r; value=%r>' % (
            type(self.schema).__name__, self.name, self.value)

class Scalar(FieldSchema):
    """The most common type, a single value such as a string or number.

    Scalar subclasses are responsible for translating most data types
    in and out of Python native form: strings, numbers, dates, times,
    Boolean values, etc.  Any data which is represented by a single
    key, value pair is a likely Scalar.

    Scalar subclasses have two responsibilities: provide a method to
    adapt a value to native Python form, and provide a method to
    serialize the native form to a Unicode string.

    Elements can be equality compared (==) to their Unicode
    representation, their native representation or other elements.

    """

    element_type = _ScalarElement

    def adapt(self, element, value):
        """Given any value, try to coerce it into native format.

        Returns the native format or raises AdaptationError on failure.

        """
        raise NotImplementedError()

    def serialize(self, element, value):
        """Given any value, coerce it into a Unicode representation.

        No special effort is made to coerce values not of native or
        a compatible type.

        *Must* return a Unicode object, always.

        """
        return unicode(value)

    def validate_element(self, element, state, descending):
        """Validates on the first, downward pass.

        See :meth:`FieldSchema.validate_element`.
        """
        if descending:
            return FieldSchema.validate_element(self, element, state, descending)
        else:
            return None

class _StringElement(_ScalarElement):
    """An Element type with string specific behavior."""

    @property
    def is_empty(self):
        """True if the string is blank or has no value."""
        return True if (not self.value and self.u == u'') else False

class String(Scalar):
    """A regular old Unicode string.

    :arg name: field name
    :arg strip: if ``True``, strip leading and trailing whitespace
                during normalization.

    """
    element_type = _StringElement

    def __init__(self, name, strip=True, **kw):
        super(String, self).__init__(name, **kw)
        self.strip = strip

    def adapt(self, element, value):
        """Return a Unicode representation.

        If ``strip=True``, leading and trailing whitespace will be
        removed.

        :returns: ``unicode(value)`` or ``None``

        """
        if value is None:
            return None
        elif self.strip:
            return unicode(value).strip()
        else:
            return unicode(value)

    def serialize(self, element, value):
        """Return a Unicode representation.

        If ``strip=True``, leading and trailing whitespace will be
        removed.

        :returns: ``unicode(value)`` or ``u'' if value == None``

        """
        if value is None:
            return u''
        elif self.strip:
            return unicode(value).strip()
        else:
            return unicode(value)

class Number(Scalar):
    """Base for numeric fields.

    :arg name: field name
    :arg signed: if ``False``, disallow negative numbers
    :arg format: override the class's default serialization format

    Subclasses provide :attr:`type_` and :attr:`format` attributes for
    :meth:`adapt` and :meth:`serialize`.

    """

    type_ = None
    format = u'%s'

    def __init__(self, name, signed=True, format=Unspecified, **kw):
        super(Number, self).__init__(name, **kw)
        self.signed = signed
        if format is not Unspecified:
            assert isinstance(format, unicode)
            self.format = format

    def adapt(self, element, value):
        """Generic numeric coercion.

        Attempt to convert *value* using the class's :attr:`type_` callable.

        """
        if value is None:
            return None
        try:
            native = self.type_(value)
        except (ValueError, TypeError):
            raise exc.AdaptationError()
        else:
            if not self.signed:
                if native < 0:
                    raise exc.AdaptationError()
            return native

    def serialize(self, element, value):
        """Generic numeric serialization.

        Converts *value* to a string using Python's string formatting
        function and the :attr:`format` as the template.  The *value*
        is provided to the format as a single, positional format
        argument.

        """
        if type(value) is self.type_:
            return self.format % value
        return unicode(value)

class Integer(Number):
    """Field type for Python's int."""
    type_ = int
    format = u'%i'

class Long(Number):
    """Field type for Python's long."""
    type_ = long
    format = u'%i'

class Float(Number):
    """Field type for Python's float."""
    type_ = float
    format = u'%f'

# TODO: Decimal

class Boolean(Scalar):
    """Field type for Python's bool.

    :arg name: field name
    :arg true: The Unicode serialization for True
    :arg false: The Unicode serialization for False

    """

    true_synonyms = (u'on', u'true', u'True', u'1')
    false_synonyms = (u'off', u'false', u'False', u'0', u'')

    def __init__(self, name, true=u'1', false=u'', **kw):
        super(Scalar, self).__init__(name, **kw)
        assert isinstance(true, unicode)
        assert isinstance(false, unicode)

        self.true = true
        self.false = false

    def adapt(self, element, value):
        """Coerce value to bool.

        If value is a string, returns True if the value is in
        :attr:`true_synonyms`, False if in :attr:`false_synonyms` and
        None otherwise.

        For non-string values, equivalent to ``bool(value)``.
        """

        if not isinstance(value, basestring):
            return bool(value)
        if value == self.true or value in self.true_synonyms:
            return True
        if value == self.false or value in self.false_synonyms:
            return False
        return None

    def serialize(self, element, value):
        """Convert bool(value) to a canonical string representation.

        Will return either :attr:`self.true` or :attr:`self.false`.

        """
        return self.true if value else self.false

class Enum(String):
    """Field type for one choice out of multiple valid strings.

    :valid_options: A sequence of valid strings. Can be defined on the element
    for dynamic validation.

    """
    valid_options = None

    def __init__(self, name, valid_options=(), **kw):
        super(Enum, self).__init__(name, **kw)
        self.valid_options = set(valid_options)

    def validate_element(self, element, state, descending):
        if not descending:
            return None

        is_valid = super(Enum, self).validate_element(element, state, descending)
        if hasattr(element, 'valid_options'):
            valid_options = element.valid_options
        else:
            valid_options = self.valid_options

        validator = valid.ValueIn(valid_options)
        return is_valid and validator.validate(element, state)

class Temporal(Scalar):
    """Base for datetime-based date and time fields."""

    type_ = None
    regex = None
    format = None
    used = None

    def __init__(self, name, strip=True, **kw):
        Scalar.__init__(self, name, **kw)
        self.strip = strip

    def adapt(self, element, value):
        """Coerces value to a native type.

        If *value* is an instance of :attr:`type_`, returns it
        unchanged.  If a string, attempts to parse it and construct a
        :attr:`type` as described in the attribute documentation.

        """
        if isinstance(value, self.type_):
            return value
        elif isinstance(value, basestring):
            if self.strip:
                value = value.strip()
            match = self.regex.match(value)
            if not match:
                raise exc.AdaptationError()
            try:
                args = [int(match.group(f)) for f in self.used]
                return self.type_(*args)
            except (TypeError, ValueError), ex:
                raise exc.AdaptationError()
        else:
            raise exc.AdaptationError()

    def serialize(self, element, value):
        """Serializes value to string.

        If *value* is an instance of :attr:`type`, formats it as
        described in the attribute documentation.  Otherwise returns
        ``unicode(value)``.

        """
        if isinstance(value, self.type_):
            return self.format % as_mapping(value)
        else:
            return unicode(value)

class DateTime(Temporal):
    """Field type for Python datetime.datetime.

    Serializes to and from YYYY-MM-DD HH:MM:SS format.

    """

    type_ = datetime.datetime
    regex = re.compile(
        ur'^(?P<year>\d{4})-(?P<month>\d{2})-(?P<day>\d{2}) '
        ur'(?P<hour>\d{2}):(?P<minute>\d{2}):(?P<second>\d{2})$')
    format = (u'%(year)04i-%(month)02i-%(day)02i '
              u'%(hour)02i:%(minute)02i:%(second)02i')
    used = ('year', 'month', 'day', 'hour', 'minute', 'second')

class Date(Temporal):
    """Field type for Python datetime.date.

    Serializes to and from YYYY-MM-DD format.

    """

    type_ = datetime.date
    regex = re.compile(
        ur'^(?P<year>\d{4})-(?P<month>\d{2})-(?P<day>\d{2})$')
    format = u'%(year)04i-%(month)02i-%(day)02i'
    used = ('year', 'month', 'day')

class Time(Temporal):
    """Field type for Python datetime.time.

    Serializes to and from HH:MM:SS format.

    """

    type_ = datetime.time
    regex = re.compile(
        ur'^(?P<hour>\d{2}):(?P<minute>\d{2}):(?P<second>\d{2})$')
    format = u'%(hour)02i:%(minute)02i:%(second)02i'
    used = ('hour', 'minute', 'second')

class _RefElement(_ScalarElement):
    flattenable = False

    @lazy_property
    def target(self):
        return self.root.el(self.schema.path)

    def _get_u(self):
        """The Unicode representation of the reference target."""
        return self.target.u

    def _set_u(self, ustr):
        if self.schema.writable == 'ignore':
            return
        elif self.schema.writable:
            self.target.u = ustr
        else:
            raise TypeError(u'Ref "%s" is not writable.' % self.name)

    u = property(_get_u, _set_u)
    del _get_u, _set_u

    def _get_value(self):
        """The native value representation of the reference target."""
        return self.target.value

    def _set_value(self, value):
        if self.schema.writable == 'ignore':
            return
        elif self.schema.writable:
            self.target.value = value
        else:
            raise TypeError(u'Ref "%s" is not writable.' % self.name)

    value = property(_get_value, _set_value)
    del _get_value, _set_value

class Ref(Scalar):
    """A functional reference to another element."""

    element_type = _RefElement

    def __init__(self, name, path, writable='ignore', sep='.', **kw):
        super(Ref, self).__init__(name, **kw)

        self.path = list(self.element_type._parse_element_path(path, sep))
        assert self.path is not None

        self.writable = writable

    def adapt(self, element, value):
        return element.target.schema.adapt(element, value)

    def serialize(self, element, value):
        return element.target.schema.serialize(element, value)
<commit_msg>Moved valid_options to the element for ease of access by templates.<commit_after># TODO: Temporal stripping
import datetime
import re
import flatland.exc as exc
from flatland.util import (
    Unspecified,
    as_mapping,
    lazy_property,
    )
from flatland import valid
from .base import FieldSchema, Element

__all__ = ('String', 'Integer', 'Long', 'Float', 'Boolean',
           'DateTime', 'Date', 'Time', 'Ref')

class _ScalarElement(Element):
    flattenable = True

    def __init__(self, schema, **kw):
        value = kw.pop('value', Unspecified)

        Element.__init__(self, schema, **kw)

        if value is not Unspecified:
            self.set(value)

    def set(self, value):
        """Assign the native and Unicode value.

        Attempts to adapt the given value and assigns this element's
        `.value` and `.u` attributes in tandem.  Returns True if the
        adaptation was successful.

        If adaptation succeeds, `.value` will contain the adapted
        native value and `.u` will contain a Unicode serialized
        version of it. A native value of None will be represented as
        u'' in `.u`.

        If adaptation fails, `.value` will be None and `.u` will
        contain `unicode(value)` or u'' for none.

        """

        try:
            # adapt and normalize the value, if possible
            value = self.value = self.schema.adapt(self, value)
        except exc.AdaptationError:
            self.value = None
            if value is None:
                self.u = u''
            elif isinstance(value, unicode):
                self.u = value
            else:
                try:
                    self.u = unicode(value)
                except UnicodeDecodeError:
                    self.u = unicode(value, errors='replace')
            return False

        # stringify it, possibly storing what we received verbatim or
        # a normalized version of it.
        if value is None:
            self.u = u''
        else:
            self.u = self.schema.serialize(self, value)
        return True

    def _index(self, name):
        raise IndexError(name)

    def _set_flat(self, pairs, sep):
        for key, value in pairs:
            if key == self.name:
                self.set(value)
                break

    def set_default(self):
        if self.schema.default is not Unspecified:
            self.set(self.schema.default)

    def __nonzero__(self):
        return True if self.u and self.value else False

    def __unicode__(self):
        return self.u

    def __repr__(self):
        return '<%s %r; value=%r>' % (
            type(self.schema).__name__, self.name, self.value)

class Scalar(FieldSchema):
    """The most common type, a single value such as a string or number.

    Scalar subclasses are responsible for translating most data types
    in and out of Python native form: strings, numbers, dates, times,
    Boolean values, etc.  Any data which is represented by a single
    key, value pair is a likely Scalar.

    Scalar subclasses have two responsibilities: provide a method to
    adapt a value to native Python form, and provide a method to
    serialize the native form to a Unicode string.

    Elements can be equality compared (==) to their Unicode
    representation, their native representation or other elements.

    """

    element_type = _ScalarElement

    def adapt(self, element, value):
        """Given any value, try to coerce it into native format.

        Returns the native format or raises AdaptationError on failure.

        """
        raise NotImplementedError()

    def serialize(self, element, value):
        """Given any value, coerce it into a Unicode representation.

        No special effort is made to coerce values not of native or
        a compatible type.

        *Must* return a Unicode object, always.

        """
        return unicode(value)

    def validate_element(self, element, state, descending):
        """Validates on the first, downward pass.

        See :meth:`FieldSchema.validate_element`.
        """
        if descending:
            return FieldSchema.validate_element(self, element, state, descending)
        else:
            return None

class _StringElement(_ScalarElement):
    """An Element type with string specific behavior."""

    @property
    def is_empty(self):
        """True if the string is blank or has no value."""
        return True if (not self.value and self.u == u'') else False

class String(Scalar):
    """A regular old Unicode string.

    :arg name: field name
    :arg strip: if ``True``, strip leading and trailing whitespace
                during normalization.

    """
    element_type = _StringElement

    def __init__(self, name, strip=True, **kw):
        super(String, self).__init__(name, **kw)
        self.strip = strip

    def adapt(self, element, value):
        """Return a Unicode representation.

        If ``strip=True``, leading and trailing whitespace will be
        removed.

        :returns: ``unicode(value)`` or ``None``

        """
        if value is None:
            return None
        elif self.strip:
            return unicode(value).strip()
        else:
            return unicode(value)

    def serialize(self, element, value):
        """Return a Unicode representation.

        If ``strip=True``, leading and trailing whitespace will be
        removed.

        :returns: ``unicode(value)`` or ``u'' if value == None``

        """
        if value is None:
            return u''
        elif self.strip:
            return unicode(value).strip()
        else:
            return unicode(value)

class Number(Scalar):
    """Base for numeric fields.

    :arg name: field name
    :arg signed: if ``False``, disallow negative numbers
    :arg format: override the class's default serialization format

    Subclasses provide :attr:`type_` and :attr:`format` attributes for
    :meth:`adapt` and :meth:`serialize`.

    """

    type_ = None
    format = u'%s'

    def __init__(self, name, signed=True, format=Unspecified, **kw):
        super(Number, self).__init__(name, **kw)
        self.signed = signed
        if format is not Unspecified:
            assert isinstance(format, unicode)
            self.format = format

    def adapt(self, element, value):
        """Generic numeric coercion.

        Attempt to convert *value* using the class's :attr:`type_` callable.

        """
        if value is None:
            return None
        try:
            native = self.type_(value)
        except (ValueError, TypeError):
            raise exc.AdaptationError()
        else:
            if not self.signed:
                if native < 0:
                    raise exc.AdaptationError()
            return native

    def serialize(self, element, value):
        """Generic numeric serialization.

        Converts *value* to a string using Python's string formatting
        function and the :attr:`format` as the template.  The *value*
        is provided to the format as a single, positional format
        argument.

        """
        if type(value) is self.type_:
            return self.format % value
        return unicode(value)

class Integer(Number):
    """Field type for Python's int."""
    type_ = int
    format = u'%i'

class Long(Number):
    """Field type for Python's long."""
    type_ = long
    format = u'%i'

class Float(Number):
    """Field type for Python's float."""
    type_ = float
    format = u'%f'

# TODO: Decimal

class Boolean(Scalar):
    """Field type for Python's bool.

    :arg name: field name
    :arg true: The Unicode serialization for True
    :arg false: The Unicode serialization for False

    """

    true_synonyms = (u'on', u'true', u'True', u'1')
    false_synonyms = (u'off', u'false', u'False', u'0', u'')

    def __init__(self, name, true=u'1', false=u'', **kw):
        super(Scalar, self).__init__(name, **kw)
        assert isinstance(true, unicode)
        assert isinstance(false, unicode)

        self.true = true
        self.false = false

    def adapt(self, element, value):
        """Coerce value to bool.

        If value is a string, returns True if the value is in
        :attr:`true_synonyms`, False if in :attr:`false_synonyms` and
        None otherwise.

        For non-string values, equivalent to ``bool(value)``.
        """

        if not isinstance(value, basestring):
            return bool(value)
        if value == self.true or value in self.true_synonyms:
            return True
        if value == self.false or value in self.false_synonyms:
            return False
        return None

    def serialize(self, element, value):
        """Convert bool(value) to a canonical string representation.

        Will return either :attr:`self.true` or :attr:`self.false`.

        """
        return self.true if value else self.false

class _EnumElement(_StringElement):
    _valid_options = None

    def get_valid_options(self):
        if self._valid_options is not None:
            return self._valid_options
        return self.schema.valid_options

    def set_valid_options(self, valid_options):
        self._valid_options = valid_options

    valid_options = property(get_valid_options, set_valid_options)

class Enum(String):
    """Field type for one choice out of multiple valid strings.

    :valid_options: A sequence of valid strings. Can be defined on the element
    for dynamic validation.

    """
    element_type = _EnumElement
    valid_options = None

    def __init__(self, name, valid_options=(), **kw):
        super(Enum, self).__init__(name, **kw)
        self.valid_options = set(valid_options)

    def validate_element(self, element, state, descending):
        if not descending:
            return None

        is_valid = super(Enum, self).validate_element(element, state, descending)

        validator = valid.ValueIn(element.valid_options)
        return is_valid and validator.validate(element, state)

class Temporal(Scalar):
    """Base for datetime-based date and time fields."""

    type_ = None
    regex = None
    format = None
    used = None

    def __init__(self, name, strip=True, **kw):
        Scalar.__init__(self, name, **kw)
        self.strip = strip

    def adapt(self, element, value):
        """Coerces value to a native type.

        If *value* is an instance of :attr:`type_`, returns it
        unchanged.  If a string, attempts to parse it and construct a
        :attr:`type` as described in the attribute documentation.

        """
        if isinstance(value, self.type_):
            return value
        elif isinstance(value, basestring):
            if self.strip:
                value = value.strip()
            match = self.regex.match(value)
            if not match:
                raise exc.AdaptationError()
            try:
                args = [int(match.group(f)) for f in self.used]
                return self.type_(*args)
            except (TypeError, ValueError), ex:
                raise exc.AdaptationError()
        else:
            raise exc.AdaptationError()

    def serialize(self, element, value):
        """Serializes value to string.

        If *value* is an instance of :attr:`type`, formats it as
        described in the attribute documentation.  Otherwise returns
        ``unicode(value)``.

        """
        if isinstance(value, self.type_):
            return self.format % as_mapping(value)
        else:
            return unicode(value)

class DateTime(Temporal):
    """Field type for Python datetime.datetime.

    Serializes to and from YYYY-MM-DD HH:MM:SS format.

    """

    type_ = datetime.datetime
    regex = re.compile(
        ur'^(?P<year>\d{4})-(?P<month>\d{2})-(?P<day>\d{2}) '
        ur'(?P<hour>\d{2}):(?P<minute>\d{2}):(?P<second>\d{2})$')
    format = (u'%(year)04i-%(month)02i-%(day)02i '
              u'%(hour)02i:%(minute)02i:%(second)02i')
    used = ('year', 'month', 'day', 'hour', 'minute', 'second')

class Date(Temporal):
    """Field type for Python datetime.date.

    Serializes to and from YYYY-MM-DD format.

    """

    type_ = datetime.date
    regex = re.compile(
        ur'^(?P<year>\d{4})-(?P<month>\d{2})-(?P<day>\d{2})$')
    format = u'%(year)04i-%(month)02i-%(day)02i'
    used = ('year', 'month', 'day')

class Time(Temporal):
    """Field type for Python datetime.time.

    Serializes to and from HH:MM:SS format.

    """

    type_ = datetime.time
    regex = re.compile(
        ur'^(?P<hour>\d{2}):(?P<minute>\d{2}):(?P<second>\d{2})$')
    format = u'%(hour)02i:%(minute)02i:%(second)02i'
    used = ('hour', 'minute', 'second')

class _RefElement(_ScalarElement):
    flattenable = False

    @lazy_property
    def target(self):
        return self.root.el(self.schema.path)

    def _get_u(self):
        """The Unicode representation of the reference target."""
        return self.target.u

    def _set_u(self, ustr):
        if self.schema.writable == 'ignore':
            return
        elif self.schema.writable:
            self.target.u = ustr
        else:
            raise TypeError(u'Ref "%s" is not writable.' % self.name)

    u = property(_get_u, _set_u)
    del _get_u, _set_u

    def _get_value(self):
        """The native value representation of the reference target."""
        return self.target.value

    def _set_value(self, value):
        if self.schema.writable == 'ignore':
            return
        elif self.schema.writable:
            self.target.value = value
        else:
            raise TypeError(u'Ref "%s" is not writable.' % self.name)

    value = property(_get_value, _set_value)
    del _get_value, _set_value

class Ref(Scalar):
    """A functional reference to another element."""

    element_type = _RefElement

    def __init__(self, name, path, writable='ignore', sep='.', **kw):
        super(Ref, self).__init__(name, **kw)

        self.path = list(self.element_type._parse_element_path(path, sep))
        assert self.path is not None

        self.writable = writable

    def adapt(self, element, value):
        return element.target.schema.adapt(element, value)

    def serialize(self, element, value):
        return element.target.schema.serialize(element, value)
<|endoftext|>


<commit_before># Copyright Hybrid Logic Ltd.  See LICENSE file for details.
"""
A HTTP REST API for controlling the Dataset Manager.
"""

import yaml
from uuid import uuid4

from pyrsistent import pmap, thaw

from twisted.python.filepath import FilePath
from twisted.web.http import CONFLICT, CREATED, NOT_FOUND
from twisted.web.server import Site
from twisted.web.resource import Resource
from twisted.application.internet import StreamServerEndpointService

from klein import Klein

from ..restapi import (
    EndpointResponse, structured, user_documentation, make_bad_request
)
from . import Dataset, Manifestation, Node, Deployment
from .. import __version__

SCHEMA_BASE = FilePath(__file__).parent().child(b'schema')
SCHEMAS = {
    b'/v1/types.json': yaml.safe_load(
        SCHEMA_BASE.child(b'types.yml').getContent()),
    b'/v1/endpoints.json': yaml.safe_load(
        SCHEMA_BASE.child(b'endpoints.yml').getContent()),
    }

DATASET_ID_COLLISION = make_bad_request(
    code=CONFLICT, description=u"The provided dataset_id is already in use.")
PRIMARY_NODE_NOT_FOUND = make_bad_request(
    description=u"The provided primary node is not part of the cluster.")
DATASET_NOT_FOUND = make_bad_request(
    code=NOT_FOUND, description=u"Dataset not found.")

class DatasetAPIUserV1(object):
    """
    A user accessing the API.

    The APIs exposed here typically operate on cluster configuration.  They
    frequently return success results when a configuration change has been made
    durable but has not yet been deployed onto the cluster.
    """
    app = Klein()

    def __init__(self, persistence_service, cluster_state_service):
        """
        :param ConfigurationPersistenceService persistence_service: Service
            for retrieving and setting desired configuration.

        :param ClusterStateService cluster_state_service: Service that
            knows about the current state of the cluster.
        """
        self.persistence_service = persistence_service
        self.cluster_state_service = cluster_state_service

    @app.route("/version", methods=['GET'])
    @user_documentation("""
        Get the version of Flocker being run.
        """, examples=[u"get version"])
    @structured(
        inputSchema={},
        outputSchema={'$ref': '/v1/endpoints.json#/definitions/versions'},
        schema_store=SCHEMAS
    )
    def version(self):
        """
        Return the ``flocker`` version string.
        """
        return {u"flocker":  __version__}

    @app.route("/configuration/datasets", methods=['GET'])
    @user_documentation(
        """
        Get the cluster's dataset configuration.
        """,
        examples=[u"get configured datasets"],
    )
    @structured(
        inputSchema={},
        outputSchema={
            '$ref': '/v1/endpoints.json#/definitions/datasets_array',
        },
        schema_store=SCHEMAS,
    )
    def get_dataset_configuration(self):
        """
        Get the configured datasets.

        :return: A ``list`` of ``dict`` representing each of dataset
            that is configured to exist anywhere on the cluster.
        """
        return list(datasets_from_deployment(self.persistence_service.get()))

    @app.route("/configuration/datasets", methods=['POST'])
    @user_documentation(
        """
        Create a new dataset.
        """,
        examples=[
            u"create dataset",
            u"create dataset with dataset_id",
            u"create dataset with duplicate dataset_id",
            u"create dataset with maximum_size",
            u"create dataset with metadata",
        ]
    )
    @structured(
        inputSchema={'$ref': '/v1/endpoints.json#/definitions/datasets'},
        outputSchema={'$ref': '/v1/endpoints.json#/definitions/datasets'},
        schema_store=SCHEMAS
    )
    def create_dataset_configuration(self, primary, dataset_id=None,
                                     maximum_size=None, metadata=None):
        """
        Create a new dataset in the cluster configuration.

        :param unicode primary: The address of the node on which the primary
            manifestation of the dataset will be created.

        :param unicode dataset_id: A unique identifier to assign to the
            dataset.  This is a string giving a UUID (per RFC 4122).  If no
            value is given, one will be generated and returned in the response.
            This is not for easy human use.  For human-friendly identifiers,
            use items in ``metadata``.

        :param int maximum_size: The maximum number of bytes the dataset will
            be capable of storing.  This may be optional or required depending
            on the dataset backend.

        :param dict metadata: A small collection of unicode key/value pairs to
            associate with the dataset.  These items are not interpreted.  They
            are only stored and made available for later retrieval.  Use this
            for things like human-friendly dataset naming, ownership
            information, etc.

        :return: A ``dict`` describing the dataset which has been added to the
            cluster configuration or giving error information if this is not
            possible.
        """
        if dataset_id is None:
            dataset_id = unicode(uuid4())
        dataset_id = dataset_id.lower()

        if metadata is None:
            metadata = {}

        # Use persistence_service to get a Deployment for the cluster
        # configuration.
        deployment = self.persistence_service.get()
        for node in deployment.nodes:
            for manifestation in node.manifestations():
                if manifestation.dataset.dataset_id == dataset_id:
                    raise DATASET_ID_COLLISION

        # XXX Check cluster state to determine if the given primary node
        # actually exists.  If not, raise PRIMARY_NODE_NOT_FOUND.
        # See FLOC-1278

        dataset = Dataset(
            dataset_id=dataset_id,
            maximum_size=maximum_size,
            metadata=pmap(metadata)
        )
        manifestation = Manifestation(dataset=dataset, primary=True)

        primary_nodes = list(
            node for node in deployment.nodes if primary == node.hostname
        )
        if len(primary_nodes) == 0:
            # The node wasn't found in the configuration so create a new node
            # to which a manifestation can be added.  FLOC-1278 will make sure
            # we're not creating nonsense configuration in this step.
            primary_node = Node(hostname=primary)
        else:
            # One was found.  Add the manifestation to it.
            (primary_node,) = primary_nodes

        new_node_config = Node(
            hostname=primary_node.hostname,
            applications=primary_node.applications,
            other_manifestations=(
                primary_node.other_manifestations | frozenset({manifestation})
            )
        )
        other_nodes = frozenset(
            node for node in deployment.nodes if node is not primary_node
        )
        new_deployment = Deployment(
            nodes=other_nodes | frozenset({new_node_config})
        )
        saving = self.persistence_service.save(new_deployment)

        def saved(ignored):
            result = {
                u"dataset_id": dataset_id,
                u"primary": primary,
                u"metadata": metadata,
            }
            if maximum_size is not None:
                result[u"maximum_size"] = maximum_size
            return EndpointResponse(CREATED, result)
        saving.addCallback(saved)
        return saving

    @app.route("/configuration/datasets/<dataset_id>", methods=['POST'])
    @user_documentation(
        """
        Update an existing dataset.
        """,
        examples=[
            u"update dataset with primary",
            u"update dataset with unknown dataset id",
        ]
    )
    @structured(
        inputSchema={'$ref': '/v1/endpoints.json#/definitions/datasets'},
        outputSchema={'$ref': '/v1/endpoints.json#/definitions/datasets'},
        schema_store=SCHEMAS
    )
    def update_dataset(self, dataset_id, primary=None):
        """
        Update an existing dataset in the cluster configuration.

        :param unicode dataset_id: The unique identifier of the dataset.  This
            is a string giving a UUID (per RFC 4122).

        :param unicode primary: The address of the node to which the dataset
            will be moved.

        :return: A ``dict`` describing the dataset which has been added to the
            cluster configuration or giving error information if this is not
            possible.
        """
        # Get the current configuration.
        deployment = self.persistence_service.get()

        # Find all manifestations of requested dataset.
        manifestations_and_nodes = list(
            other_manifestations_from_deployment(deployment, dataset_id)
        )

        # There are no manifestations containing the requested dataset.
        if not manifestations_and_nodes:
            raise DATASET_NOT_FOUND

        # Lookup the node that has a primary Manifestation (if any)
        primary_manifestation, origin_node = [
            (manifestation, node)
            for manifestation, node in manifestations_and_nodes
            if manifestation.primary
        ][0]

        if origin_node.hostname == primary:
            # Maybe return early here rather than bother the
            # persistence_service.  raise Exception('New primary is the same as
            # existing primary')
            pass

        # Now construct a new_deployment where the primary manifestation of the
        # dataset is on the requested primary node.
        new_origin_node = Node(
            hostname=origin_node.hostname,
            applications=origin_node.applications,
            other_manifestations=(
                origin_node.other_manifestations
                - frozenset({primary_manifestation})
            )
        )
        deployment = deployment.update_node(new_origin_node)

        primary_nodes = list(
            node for node in deployment.nodes if primary == node.hostname
        )
        if len(primary_nodes) == 0:
            # `primary` is not in cluster. Add it.
            # XXX Check cluster state to determine if the given primary node
            # actually exists.  If not, raise PRIMARY_NODE_NOT_FOUND.
            # See FLOC-1278
            new_target_node = Node(
                hostname=primary,
                other_manifestations=frozenset({primary_manifestation})
            )
        else:
            # There should only be one node with the requested primary
            # hostname. ``IndexError`` here if that's not the case.
            (target_node,) = primary_nodes
            new_target_node = Node(
                hostname=target_node.hostname,
                applications=target_node.applications,
                other_manifestations=(
                    target_node.other_manifestations
                    | frozenset({primary_manifestation})
                )
            )

        deployment = deployment.update_node(new_target_node)

        saving = self.persistence_service.save(deployment)

        # Construct the return dictionary from the supplied dataset_id, primary
        # and the found existing primary manifestation.
        # def saved(ignored):
        #     result = {
        #         u"dataset_id": dataset_id,
        #         u"primary": primary,
        #         u"metadata": metadata,
        #     }
        #     if maximum_size is not None:
        #         result[u"maximum_size"] = maximum_size
        #     return EndpointResponse(CREATED, result)
        # saving.addCallback(saved)
        # return saving
        return api_dataset_from_dataset_and_node(
            primary_manifestation.dataset,
            new_target_node.hostname
        )

    @app.route("/state/datasets", methods=['GET'])
    @user_documentation("""
        Get current cluster datasets.
        """, examples=[u"get state datasets"])
    @structured(
        inputSchema={},
        outputSchema={
            '$ref': '/v1/endpoints.json#/definitions/datasets_array'
            },
        schema_store=SCHEMAS
    )
    def state_datasets(self):
        """
        Return the current primary datasets in the cluster.

        :return: A ``list`` containing all datasets in the cluster.
        """
        deployment = self.cluster_state_service.as_deployment()
        return list(datasets_from_deployment(deployment))

def other_manifestations_from_deployment(deployment, dataset_id):
    """
    Extract all other manifestations of the supplied dataset_id from the
    supplied deployment.
    """
    for node in deployment.nodes:
        for manifestation in node.other_manifestations:
            if manifestation.dataset.dataset_id == dataset_id:
                yield manifestation, node
# This looks very similar to the function below. Refactor?

def datasets_from_deployment(deployment):
    """
    Extract the primary datasets from the supplied deployment instance.

    Currently does not support secondary datasets, but this info might be
    useful to provide.  For ZFS, for example, may show how up-to-date they
    are with respect to the primary.

    :param Deployment deployment: A ``Deployment`` describing the state
        of the cluster.

    :return: Iterable returning all datasets.
    """
    for node in deployment.nodes:
        for manifestation in node.manifestations():
            if manifestation.primary:
                # There may be multiple datasets marked as primary until we
                # implement consistency checking when state is reported by each
                # node.
                # See https://clusterhq.atlassian.net/browse/FLOC-1303
                yield api_dataset_from_dataset_and_node(
                    manifestation.dataset, node.hostname
                )

def api_dataset_from_dataset_and_node(dataset, node_hostname):
    """
    Return a dataset dict which conforms to
    ``/v1/endpoints.json#/definitions/datasets_array``

    :param Dataset dataset: A dataset present in the cluster.
    :param unicode node_hostname: Hostname of the primary node for the
        `dataset`.
    :return: A ``dict`` containing the dataset information and the
        hostname of the primary node, conforming to
        ``/v1/endpoints.json#/definitions/datasets_array``.
    """
    result = dict(
        dataset_id=dataset.dataset_id,
        primary=node_hostname,
        metadata=thaw(dataset.metadata)
    )
    if dataset.maximum_size is not None:
        result[u'maximum_size'] = dataset.maximum_size
    return result

def create_api_service(persistence_service, cluster_state_service, endpoint):
    """
    Create a Twisted Service that serves the API on the given endpoint.

    :param ConfigurationPersistenceService persistence_service: Service
        for retrieving and setting desired configuration.

    :param ClusterStateService cluster_state_service: Service that
        knows about the current state of the cluster.

    :param endpoint: Twisted endpoint to listen on.

    :return: Service that will listen on the endpoint using HTTP API server.
    """
    api_root = Resource()
    user = DatasetAPIUserV1(persistence_service, cluster_state_service)
    api_root.putChild('v1', user.app.resource())
    api_root._v1_user = user  # For unit testing purposes, alas
    return StreamServerEndpointService(endpoint, Site(api_root))
<commit_msg>Wait for the deployement to be saved before returning the result.<commit_after># Copyright Hybrid Logic Ltd.  See LICENSE file for details.
"""
A HTTP REST API for controlling the Dataset Manager.
"""

import yaml
from uuid import uuid4

from pyrsistent import pmap, thaw

from twisted.python.filepath import FilePath
from twisted.web.http import CONFLICT, CREATED, NOT_FOUND, OK
from twisted.web.server import Site
from twisted.web.resource import Resource
from twisted.application.internet import StreamServerEndpointService

from klein import Klein

from ..restapi import (
    EndpointResponse, structured, user_documentation, make_bad_request
)
from . import Dataset, Manifestation, Node, Deployment
from .. import __version__

SCHEMA_BASE = FilePath(__file__).parent().child(b'schema')
SCHEMAS = {
    b'/v1/types.json': yaml.safe_load(
        SCHEMA_BASE.child(b'types.yml').getContent()),
    b'/v1/endpoints.json': yaml.safe_load(
        SCHEMA_BASE.child(b'endpoints.yml').getContent()),
    }

DATASET_ID_COLLISION = make_bad_request(
    code=CONFLICT, description=u"The provided dataset_id is already in use.")
PRIMARY_NODE_NOT_FOUND = make_bad_request(
    description=u"The provided primary node is not part of the cluster.")
DATASET_NOT_FOUND = make_bad_request(
    code=NOT_FOUND, description=u"Dataset not found.")

class DatasetAPIUserV1(object):
    """
    A user accessing the API.

    The APIs exposed here typically operate on cluster configuration.  They
    frequently return success results when a configuration change has been made
    durable but has not yet been deployed onto the cluster.
    """
    app = Klein()

    def __init__(self, persistence_service, cluster_state_service):
        """
        :param ConfigurationPersistenceService persistence_service: Service
            for retrieving and setting desired configuration.

        :param ClusterStateService cluster_state_service: Service that
            knows about the current state of the cluster.
        """
        self.persistence_service = persistence_service
        self.cluster_state_service = cluster_state_service

    @app.route("/version", methods=['GET'])
    @user_documentation("""
        Get the version of Flocker being run.
        """, examples=[u"get version"])
    @structured(
        inputSchema={},
        outputSchema={'$ref': '/v1/endpoints.json#/definitions/versions'},
        schema_store=SCHEMAS
    )
    def version(self):
        """
        Return the ``flocker`` version string.
        """
        return {u"flocker":  __version__}

    @app.route("/configuration/datasets", methods=['GET'])
    @user_documentation(
        """
        Get the cluster's dataset configuration.
        """,
        examples=[u"get configured datasets"],
    )
    @structured(
        inputSchema={},
        outputSchema={
            '$ref': '/v1/endpoints.json#/definitions/datasets_array',
        },
        schema_store=SCHEMAS,
    )
    def get_dataset_configuration(self):
        """
        Get the configured datasets.

        :return: A ``list`` of ``dict`` representing each of dataset
            that is configured to exist anywhere on the cluster.
        """
        return list(datasets_from_deployment(self.persistence_service.get()))

    @app.route("/configuration/datasets", methods=['POST'])
    @user_documentation(
        """
        Create a new dataset.
        """,
        examples=[
            u"create dataset",
            u"create dataset with dataset_id",
            u"create dataset with duplicate dataset_id",
            u"create dataset with maximum_size",
            u"create dataset with metadata",
        ]
    )
    @structured(
        inputSchema={'$ref': '/v1/endpoints.json#/definitions/datasets'},
        outputSchema={'$ref': '/v1/endpoints.json#/definitions/datasets'},
        schema_store=SCHEMAS
    )
    def create_dataset_configuration(self, primary, dataset_id=None,
                                     maximum_size=None, metadata=None):
        """
        Create a new dataset in the cluster configuration.

        :param unicode primary: The address of the node on which the primary
            manifestation of the dataset will be created.

        :param unicode dataset_id: A unique identifier to assign to the
            dataset.  This is a string giving a UUID (per RFC 4122).  If no
            value is given, one will be generated and returned in the response.
            This is not for easy human use.  For human-friendly identifiers,
            use items in ``metadata``.

        :param int maximum_size: The maximum number of bytes the dataset will
            be capable of storing.  This may be optional or required depending
            on the dataset backend.

        :param dict metadata: A small collection of unicode key/value pairs to
            associate with the dataset.  These items are not interpreted.  They
            are only stored and made available for later retrieval.  Use this
            for things like human-friendly dataset naming, ownership
            information, etc.

        :return: A ``dict`` describing the dataset which has been added to the
            cluster configuration or giving error information if this is not
            possible.
        """
        if dataset_id is None:
            dataset_id = unicode(uuid4())
        dataset_id = dataset_id.lower()

        if metadata is None:
            metadata = {}

        # Use persistence_service to get a Deployment for the cluster
        # configuration.
        deployment = self.persistence_service.get()
        for node in deployment.nodes:
            for manifestation in node.manifestations():
                if manifestation.dataset.dataset_id == dataset_id:
                    raise DATASET_ID_COLLISION

        # XXX Check cluster state to determine if the given primary node
        # actually exists.  If not, raise PRIMARY_NODE_NOT_FOUND.
        # See FLOC-1278

        dataset = Dataset(
            dataset_id=dataset_id,
            maximum_size=maximum_size,
            metadata=pmap(metadata)
        )
        manifestation = Manifestation(dataset=dataset, primary=True)

        primary_nodes = list(
            node for node in deployment.nodes if primary == node.hostname
        )
        if len(primary_nodes) == 0:
            # The node wasn't found in the configuration so create a new node
            # to which a manifestation can be added.  FLOC-1278 will make sure
            # we're not creating nonsense configuration in this step.
            primary_node = Node(hostname=primary)
        else:
            # One was found.  Add the manifestation to it.
            (primary_node,) = primary_nodes

        new_node_config = Node(
            hostname=primary_node.hostname,
            applications=primary_node.applications,
            other_manifestations=(
                primary_node.other_manifestations | frozenset({manifestation})
            )
        )
        other_nodes = frozenset(
            node for node in deployment.nodes if node is not primary_node
        )
        new_deployment = Deployment(
            nodes=other_nodes | frozenset({new_node_config})
        )
        saving = self.persistence_service.save(new_deployment)

        def saved(ignored):
            result = {
                u"dataset_id": dataset_id,
                u"primary": primary,
                u"metadata": metadata,
            }
            if maximum_size is not None:
                result[u"maximum_size"] = maximum_size
            return EndpointResponse(CREATED, result)
        saving.addCallback(saved)
        return saving

    @app.route("/configuration/datasets/<dataset_id>", methods=['POST'])
    @user_documentation(
        """
        Update an existing dataset.
        """,
        examples=[
            u"update dataset with primary",
            u"update dataset with unknown dataset id",
        ]
    )
    @structured(
        inputSchema={'$ref': '/v1/endpoints.json#/definitions/datasets'},
        outputSchema={'$ref': '/v1/endpoints.json#/definitions/datasets'},
        schema_store=SCHEMAS
    )
    def update_dataset(self, dataset_id, primary=None):
        """
        Update an existing dataset in the cluster configuration.

        :param unicode dataset_id: The unique identifier of the dataset.  This
            is a string giving a UUID (per RFC 4122).

        :param unicode primary: The address of the node to which the dataset
            will be moved.

        :return: A ``dict`` describing the dataset which has been added to the
            cluster configuration or giving error information if this is not
            possible.
        """
        # Get the current configuration.
        deployment = self.persistence_service.get()

        # Find all manifestations of requested dataset.
        manifestations_and_nodes = list(
            other_manifestations_from_deployment(deployment, dataset_id)
        )

        # There are no manifestations containing the requested dataset.
        if not manifestations_and_nodes:
            raise DATASET_NOT_FOUND

        # Lookup the node that has a primary Manifestation (if any)
        primary_manifestation, origin_node = [
            (manifestation, node)
            for manifestation, node in manifestations_and_nodes
            if manifestation.primary
        ][0]

        if origin_node.hostname == primary:
            # Maybe return early here rather than bother the
            # persistence_service.  raise Exception('New primary is the same as
            # existing primary')
            pass

        # Now construct a new_deployment where the primary manifestation of the
        # dataset is on the requested primary node.
        new_origin_node = Node(
            hostname=origin_node.hostname,
            applications=origin_node.applications,
            other_manifestations=(
                origin_node.other_manifestations
                - frozenset({primary_manifestation})
            )
        )
        deployment = deployment.update_node(new_origin_node)

        primary_nodes = list(
            node for node in deployment.nodes if primary == node.hostname
        )
        if len(primary_nodes) == 0:
            # `primary` is not in cluster. Add it.
            # XXX Check cluster state to determine if the given primary node
            # actually exists.  If not, raise PRIMARY_NODE_NOT_FOUND.
            # See FLOC-1278
            new_target_node = Node(
                hostname=primary,
                other_manifestations=frozenset({primary_manifestation})
            )
        else:
            # There should only be one node with the requested primary
            # hostname. ``IndexError`` here if that's not the case.
            (target_node,) = primary_nodes
            new_target_node = Node(
                hostname=target_node.hostname,
                applications=target_node.applications,
                other_manifestations=(
                    target_node.other_manifestations
                    | frozenset({primary_manifestation})
                )
            )

        deployment = deployment.update_node(new_target_node)

        saving = self.persistence_service.save(deployment)

        # Return an API response dictionary containing the dataset with updated
        # primary address.
        def saved(ignored):
            result = api_dataset_from_dataset_and_node(
                primary_manifestation.dataset,
                new_target_node.hostname
            )
            return EndpointResponse(OK, result)
        saving.addCallback(saved)
        return saving

    @app.route("/state/datasets", methods=['GET'])
    @user_documentation("""
        Get current cluster datasets.
        """, examples=[u"get state datasets"])
    @structured(
        inputSchema={},
        outputSchema={
            '$ref': '/v1/endpoints.json#/definitions/datasets_array'
            },
        schema_store=SCHEMAS
    )
    def state_datasets(self):
        """
        Return the current primary datasets in the cluster.

        :return: A ``list`` containing all datasets in the cluster.
        """
        deployment = self.cluster_state_service.as_deployment()
        return list(datasets_from_deployment(deployment))

def other_manifestations_from_deployment(deployment, dataset_id):
    """
    Extract all other manifestations of the supplied dataset_id from the
    supplied deployment.
    """
    for node in deployment.nodes:
        for manifestation in node.other_manifestations:
            if manifestation.dataset.dataset_id == dataset_id:
                yield manifestation, node
# This looks very similar to the function below. Refactor?

def datasets_from_deployment(deployment):
    """
    Extract the primary datasets from the supplied deployment instance.

    Currently does not support secondary datasets, but this info might be
    useful to provide.  For ZFS, for example, may show how up-to-date they
    are with respect to the primary.

    :param Deployment deployment: A ``Deployment`` describing the state
        of the cluster.

    :return: Iterable returning all datasets.
    """
    for node in deployment.nodes:
        for manifestation in node.manifestations():
            if manifestation.primary:
                # There may be multiple datasets marked as primary until we
                # implement consistency checking when state is reported by each
                # node.
                # See https://clusterhq.atlassian.net/browse/FLOC-1303
                yield api_dataset_from_dataset_and_node(
                    manifestation.dataset, node.hostname
                )

def api_dataset_from_dataset_and_node(dataset, node_hostname):
    """
    Return a dataset dict which conforms to
    ``/v1/endpoints.json#/definitions/datasets_array``

    :param Dataset dataset: A dataset present in the cluster.
    :param unicode node_hostname: Hostname of the primary node for the
        `dataset`.
    :return: A ``dict`` containing the dataset information and the
        hostname of the primary node, conforming to
        ``/v1/endpoints.json#/definitions/datasets_array``.
    """
    result = dict(
        dataset_id=dataset.dataset_id,
        primary=node_hostname,
        metadata=thaw(dataset.metadata)
    )
    if dataset.maximum_size is not None:
        result[u'maximum_size'] = dataset.maximum_size
    return result

def create_api_service(persistence_service, cluster_state_service, endpoint):
    """
    Create a Twisted Service that serves the API on the given endpoint.

    :param ConfigurationPersistenceService persistence_service: Service
        for retrieving and setting desired configuration.

    :param ClusterStateService cluster_state_service: Service that
        knows about the current state of the cluster.

    :param endpoint: Twisted endpoint to listen on.

    :return: Service that will listen on the endpoint using HTTP API server.
    """
    api_root = Resource()
    user = DatasetAPIUserV1(persistence_service, cluster_state_service)
    api_root.putChild('v1', user.app.resource())
    api_root._v1_user = user  # For unit testing purposes, alas
    return StreamServerEndpointService(endpoint, Site(api_root))
<|endoftext|>


<commit_before>#
# Los Cocos: Multi-Menu Example
# http://code.google.com/p/los-cocos/
#

import sys
import os
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..'))

from pyglet import image
from pyglet.gl import *
from pyglet import font

from cocos.director import *
from cocos.menu import *
from cocos.scene import *
from cocos.layer import *

class MainMenu(Menu):
    def __init__( self ):

        # call superclass with the title
        super( MainMenu, self ).__init__("GROSSINI'S SISTERS" )

        # you can override the font that will be used for the title and the items
        font_title = {
            'font_name':'KonQa',
            'font_size':72,
            'bold':True,
            }
        self.font_title.update( font_title )

        font_item = {
            'font_name':'You Are Loved',
            'font_size': 32,
            }
        self.font_item.update( font_item )

        font_item = {
            'font_name':'You Are Loved',
            'font_size': 48,
            }
        self.font_item_selected.update( font_item )

        # example: menus can be vertical aligned and horizontal aligned
        self.menu_valign = CENTER
        self.menu_halign = CENTER

        # then add the items
        self.add_item( MenuItem('New Game', self.on_new_game ) )
        self.add_item( MenuItem('Options', self.on_options ) )
        self.add_item( MenuItem('Scores', self.on_scores ) )
        self.add_item( MenuItem('Quit', self.on_quit ) )

        # after adding all the items just call build_items()
        self.build_items()

    # Callbacks
    def on_new_game( self ):
#        director.set_scene( StartGame() )
        print "on_new_game()"

    def on_scores( self ):
        self.switch_to( 2 )

    def on_options( self ):
        self.switch_to( 1 )

    def on_quit( self ):
        print "on_quit()"
        sys.exit()

class OptionMenu(Menu):
    def __init__( self ):
        super( OptionMenu, self ).__init__("GROSSINI'S SISTERS" )

        font_title = {
            'font_name':'KonQa',
            'font_size':72,
            'bold':True,
            }
        self.font_title.update( font_title )

        font_item = {
            'font_name':'You Are Loved',
            'font_size': 32,
            }
        self.font_item.update( font_item )

        font_item = {
            'font_name':'You Are Loved',
            'font_size': 48,
            }
        self.font_item_selected.update( font_item )

        self.menu_valign = BOTTOM
        self.menu_halign = RIGHT

        self.add_item( MenuItem('Fullscreen', self.on_fullscreen) )
        self.add_item( ToggleMenuItem('Show FPS', True, self.on_show_fps) )
        self.add_item( MenuItem('OK', self.on_quit) )
        self.build_items()

    # Callbacks
    def on_fullscreen( self ):
        director.window.set_fullscreen( not director.window.fullscreen )

    def on_quit( self ):
        self.switch_to( 0 )

    def on_show_fps( self, value ):
        director.show_FPS = value

class ScoreMenu(Menu):
    def __init__( self ):
        super( ScoreMenu, self ).__init__("GROSSINI'S SISTERS" )

        font_title = {
            'font_name':'KonQa',
            'font_size':72,
            'bold':True,
            }
        self.font_title.update( font_title )

        font_item = {
            'font_name':'You Are Loved',
            'font_size': 32,
            }
        self.font_item.update( font_item )

        font_item = {
            'font_name':'You Are Loved',
            'font_size': 48,
            }
        self.font_item_selected.update( font_item )

        self.menu_valign = BOTTOM
        self.menu_halign = LEFT

        self.add_item( MenuItem('Go Back', self.on_quit) )
        self.build_items()

    def on_quit( self ):
        self.switch_to( 0 )

if __name__ == "__main__":

    pyglet.resource.path.append('.')
    pyglet.resource.reindex()
    pyglet.font.add_directory('.')

    director.init( resizable=True)
    director.run( Scene(
            MultiplexLayer( MainMenu(), OptionMenu(), ScoreMenu() )
            ) )
<commit_msg>remove test_menu.py. test directory includes tests for the menu<commit_after><|endoftext|>