<commit_before>import string

#untrusted meminfo size is taken from xenstore key, thus its size is limited
#so splits do not require excessive memory
def parse_meminfo(untrusted_meminfo):
    untrusted_dict = {}
#split meminfo contents into lines
    untrusted_lines = string.split(untrusted_meminfo,"\n")
    for untrusted_lines_iterator in untrusted_lines:
#split a single meminfo line into words
        untrusted_words = string.split(untrusted_lines_iterator)
        if len(untrusted_words) >= 2:
            untrusted_dict[string.rstrip(untrusted_words[0], ":")] = untrusted_words[1]

    return untrusted_dict

def is_meminfo_suspicious(domain, untrusted_meminfo):
    ret = False

#check whether the required keys exist and are not negative
    try:
        for i in ('MemTotal', 'MemFree', 'Buffers', 'Cached', 'SwapTotal', 'SwapFree'):
            val = int(untrusted_meminfo[i])*1024
            if (val < 0):
                ret = True
            untrusted_meminfo[i] = val
    except:
        ret = True

    if not ret and untrusted_meminfo['SwapTotal'] < untrusted_meminfo['SwapFree']:
        ret = True
    if not ret and untrusted_meminfo['MemTotal'] < untrusted_meminfo['MemFree'] + untrusted_meminfo['Cached'] + untrusted_meminfo['Buffers']:
        ret = True
#we could also impose some limits on all the above values
#but it has little purpose - all the domain can gain by passing e.g.
#very large SwapTotal is that it will be assigned all free Xen memory
#it can be achieved with legal values, too, and it will not allow to
#starve existing domains, by design
    if ret:
        print 'suspicious meminfo for domain', domain.id, 'mem actual', domain.memory_actual, untrusted_meminfo
    return ret

#called when a domain updates its 'meminfo' xenstore key
def refresh_meminfo_for_domain(domain, untrusted_xenstore_key):
    untrusted_meminfo = parse_meminfo(untrusted_xenstore_key)
    if untrusted_meminfo is None:
        domain.meminfo = None
        return
#sanitize start
    if is_meminfo_suspicious(domain, untrusted_meminfo):
#sanitize end
        domain.meminfo = None
        domain.mem_used = None
    else:
#sanitized, can assign
        domain.meminfo = untrusted_meminfo
        domain.mem_used =  domain.meminfo['MemTotal'] - domain.meminfo['MemFree'] - domain.meminfo['Cached'] - domain.meminfo['Buffers'] + domain.meminfo['SwapTotal'] - domain.meminfo['SwapFree']

def prefmem(domain):
    CACHE_FACTOR = 1.3
#dom0 is special, as it must have large cache, for vbds. Thus, give it a special boost
    if domain.id == '0':
        return domain.mem_used*CACHE_FACTOR + 350*1024*1024
    return min(domain.mem_used*CACHE_FACTOR, domain.memory_maximum)

def memory_needed(domain):
#do not change
#in balance(), "distribute total_available_memory proportionally to mempref" relies on this exact formula
    ret = prefmem(domain) - domain.memory_actual
    return ret

#prepare list of (domain, memory_target) pairs that need to be passed
#to "xm memset" equivalent in order to obtain "memsize" of memory
#return empty list when the request cannot be satisfied
def balloon(memsize, domain_dictionary):
    REQ_SAFETY_NET_FACTOR = 1.05
    donors = list()
    request = list()
    available = 0
    for i in domain_dictionary.keys():
        if domain_dictionary[i].meminfo is None:
            continue
        if domain_dictionary[i].no_progress:
            continue
        need = memory_needed(domain_dictionary[i])
        if need < 0:
            print 'balloon: dom' , i, 'has actual memory', domain_dictionary[i].memory_actual
            donors.append((i,-need))
            available-=need
    print 'req=', memsize, 'avail=', available, 'donors', donors
    if available<memsize:
        return ()
    scale = 1.0*memsize/available
    for donors_iter in donors:
        id, mem = donors_iter
        memborrowed = mem*scale*REQ_SAFETY_NET_FACTOR
        print 'borrow' , memborrowed, 'from', id
        memtarget = int(domain_dictionary[id].memory_actual - memborrowed)
        request.append((id, memtarget))
    return request
# REQ_SAFETY_NET_FACTOR is a bit greater that 1. So that if the domain yields a bit less than requested, due
# to e.g. rounding errors, we will not get stuck. The surplus will return to the VM during "balance" call.

#redistribute positive "total_available_memory" of memory between domains, proportionally to prefmem
def balance_when_enough_memory(domain_dictionary, xen_free_memory, total_mem_pref, total_available_memory):
    print 'balance_when_enough_memory(', xen_free_memory, total_mem_pref, total_available_memory, ')'
    target_memory = {}
    # memory not assigned because of static max
    left_memory = 0
    acceptors_count = 0
    for i in domain_dictionary.keys():
        if domain_dictionary[i].meminfo is None:
            continue
#distribute total_available_memory proportionally to mempref
        scale = 1.0*prefmem(domain_dictionary[i])/total_mem_pref
        target_nonint = prefmem(domain_dictionary[i]) + scale*total_available_memory
#prevent rounding errors
        target = int(0.999*target_nonint)
#do not try to give more memory than static max
        if target > domain_dictionary[i].memory_maximum:
            left_memory += target-domain_dictionary[i].memory_maximum
            target = domain_dictionary[i].memory_maximum
        else:
# count domains which can accept more memory
            acceptors_count += 1
        target_memory[i] = target
# distribute left memory across all acceptors
    while left_memory > 0 and acceptors_count > 0:
        print '  left_memory:', left_memory, 'acceptors_count:', acceptors_count
        new_left_memory = 0
        for i in target_memory.keys():
            target = target_memory[i]
            if target < domain_dictionary[i].memory_maximum:
                memory_bonus = int(0.999*(left_memory/acceptors_count))
                if target+memory_bonus >= domain_dictionary[i].memory_maximum:
                    new_left_memory += target+memory_bonus - domain_dictionary[i].memory_maximum
                    target = domain_dictionary[i].memory_maximum
                    acceptors_count -= 1
                else:
                    target += memory_bonus
            target_memory[i] = target
        left_memory = new_left_memory
# split target_memory dictionary to donors and acceptors
#  this is needed to first get memory from donors and only then give it to acceptors
    donors_rq = list()
    acceptors_rq = list()
    for i in target_memory.keys():
        target = target_memory[i]
        if (target < domain_dictionary[i].memory_actual):
            donors_rq.append((i, target))
        else:
            acceptors_rq.append((i, target))

#    print 'balance(enough): xen_free_memory=', xen_free_memory, 'requests:', donors_rq + acceptors_rq
    return donors_rq + acceptors_rq

#when not enough mem to make everyone be above prefmem, make donors be at prefmem, and
#redistribute anything left between acceptors
def balance_when_low_on_memory(domain_dictionary, xen_free_memory, total_mem_pref_acceptors, donors, acceptors):
    donors_rq = list()
    acceptors_rq = list()
    squeezed_mem = xen_free_memory
    for i in donors:
        avail = -memory_needed(domain_dictionary[i])
        if avail < 10*1024*1024:
            #probably we have already tried making it exactly at prefmem, give up
            continue
        squeezed_mem -= avail
        donors_rq.append((i, prefmem(domain_dictionary[i])))
#the below can happen if initially xen free memory is below 50M
    if squeezed_mem < 0:
        return donors_rq
    for i in acceptors:
        scale = 1.0*prefmem(domain_dictionary[i])/total_mem_pref_acceptors
        target_nonint = domain_dictionary[i].memory_actual + scale*squeezed_mem
#do not try to give more memory than static max
        target = min(int(0.999*target_nonint), domain_dictionary[i].memory_maximum)
        acceptors_rq.append((i, target))
#    print 'balance(low): xen_free_memory=', xen_free_memory, 'requests:', donors_rq + acceptors_rq
    return donors_rq + acceptors_rq

#redistribute memory across domains
#called when one of domains update its 'meminfo' xenstore key
#return the list of (domain, memory_target) pairs to be passed to
#"xm memset" equivalent
def balance(xen_free_memory, domain_dictionary):

#sum of all memory requirements - in other words, the difference between
#memory required to be added to domains (acceptors) to make them be at their
#preferred memory, and memory that can be taken from domains (donors) that
#can provide memory. So, it can be negative when plenty of memory.
    total_memory_needed = 0

#sum of memory preferences of all domains
    total_mem_pref = 0

#sum of memory preferences of all domains that require more memory
    total_mem_pref_acceptors = 0

    donors = list()	# domains that can yield memory
    acceptors = list()  # domains that require more memory
#pass 1: compute the above "total" values
    for i in domain_dictionary.keys():
        if domain_dictionary[i].meminfo is None:
            continue
        need = memory_needed(domain_dictionary[i])
#        print 'domain' , i, 'act/pref', domain_dictionary[i].memory_actual, prefmem(domain_dictionary[i]), 'need=', need
        if need < 0 or domain_dictionary[i].memory_actual >= domain_dictionary[i].memory_maximum:
            donors.append(i)
        else:
            acceptors.append(i)
            total_mem_pref_acceptors += prefmem(domain_dictionary[i])
        total_memory_needed += need
        total_mem_pref += prefmem(domain_dictionary[i])

    total_available_memory = xen_free_memory - total_memory_needed
    if total_available_memory > 0:
        return balance_when_enough_memory(domain_dictionary, xen_free_memory, total_mem_pref, total_available_memory)
    else:
        return balance_when_low_on_memory(domain_dictionary, xen_free_memory, total_mem_pref_acceptors, donors, acceptors)
<commit_msg>dom0/qmemman: Check for memory_maximum also for dom0<commit_after>import string

#untrusted meminfo size is taken from xenstore key, thus its size is limited
#so splits do not require excessive memory
def parse_meminfo(untrusted_meminfo):
    untrusted_dict = {}
#split meminfo contents into lines
    untrusted_lines = string.split(untrusted_meminfo,"\n")
    for untrusted_lines_iterator in untrusted_lines:
#split a single meminfo line into words
        untrusted_words = string.split(untrusted_lines_iterator)
        if len(untrusted_words) >= 2:
            untrusted_dict[string.rstrip(untrusted_words[0], ":")] = untrusted_words[1]

    return untrusted_dict

def is_meminfo_suspicious(domain, untrusted_meminfo):
    ret = False

#check whether the required keys exist and are not negative
    try:
        for i in ('MemTotal', 'MemFree', 'Buffers', 'Cached', 'SwapTotal', 'SwapFree'):
            val = int(untrusted_meminfo[i])*1024
            if (val < 0):
                ret = True
            untrusted_meminfo[i] = val
    except:
        ret = True

    if not ret and untrusted_meminfo['SwapTotal'] < untrusted_meminfo['SwapFree']:
        ret = True
    if not ret and untrusted_meminfo['MemTotal'] < untrusted_meminfo['MemFree'] + untrusted_meminfo['Cached'] + untrusted_meminfo['Buffers']:
        ret = True
#we could also impose some limits on all the above values
#but it has little purpose - all the domain can gain by passing e.g.
#very large SwapTotal is that it will be assigned all free Xen memory
#it can be achieved with legal values, too, and it will not allow to
#starve existing domains, by design
    if ret:
        print 'suspicious meminfo for domain', domain.id, 'mem actual', domain.memory_actual, untrusted_meminfo
    return ret

#called when a domain updates its 'meminfo' xenstore key
def refresh_meminfo_for_domain(domain, untrusted_xenstore_key):
    untrusted_meminfo = parse_meminfo(untrusted_xenstore_key)
    if untrusted_meminfo is None:
        domain.meminfo = None
        return
#sanitize start
    if is_meminfo_suspicious(domain, untrusted_meminfo):
#sanitize end
        domain.meminfo = None
        domain.mem_used = None
    else:
#sanitized, can assign
        domain.meminfo = untrusted_meminfo
        domain.mem_used =  domain.meminfo['MemTotal'] - domain.meminfo['MemFree'] - domain.meminfo['Cached'] - domain.meminfo['Buffers'] + domain.meminfo['SwapTotal'] - domain.meminfo['SwapFree']

def prefmem(domain):
    CACHE_FACTOR = 1.3
#dom0 is special, as it must have large cache, for vbds. Thus, give it a special boost
    if domain.id == '0':
        return min(domain.mem_used*CACHE_FACTOR + 350*1024*1024, domain.memory_maximum)
    return min(domain.mem_used*CACHE_FACTOR, domain.memory_maximum)

def memory_needed(domain):
#do not change
#in balance(), "distribute total_available_memory proportionally to mempref" relies on this exact formula
    ret = prefmem(domain) - domain.memory_actual
    return ret

#prepare list of (domain, memory_target) pairs that need to be passed
#to "xm memset" equivalent in order to obtain "memsize" of memory
#return empty list when the request cannot be satisfied
def balloon(memsize, domain_dictionary):
    REQ_SAFETY_NET_FACTOR = 1.05
    donors = list()
    request = list()
    available = 0
    for i in domain_dictionary.keys():
        if domain_dictionary[i].meminfo is None:
            continue
        if domain_dictionary[i].no_progress:
            continue
        need = memory_needed(domain_dictionary[i])
        if need < 0:
            print 'balloon: dom' , i, 'has actual memory', domain_dictionary[i].memory_actual
            donors.append((i,-need))
            available-=need
    print 'req=', memsize, 'avail=', available, 'donors', donors
    if available<memsize:
        return ()
    scale = 1.0*memsize/available
    for donors_iter in donors:
        id, mem = donors_iter
        memborrowed = mem*scale*REQ_SAFETY_NET_FACTOR
        print 'borrow' , memborrowed, 'from', id
        memtarget = int(domain_dictionary[id].memory_actual - memborrowed)
        request.append((id, memtarget))
    return request
# REQ_SAFETY_NET_FACTOR is a bit greater that 1. So that if the domain yields a bit less than requested, due
# to e.g. rounding errors, we will not get stuck. The surplus will return to the VM during "balance" call.

#redistribute positive "total_available_memory" of memory between domains, proportionally to prefmem
def balance_when_enough_memory(domain_dictionary, xen_free_memory, total_mem_pref, total_available_memory):
    print 'balance_when_enough_memory(', xen_free_memory, total_mem_pref, total_available_memory, ')'
    target_memory = {}
    # memory not assigned because of static max
    left_memory = 0
    acceptors_count = 0
    for i in domain_dictionary.keys():
        if domain_dictionary[i].meminfo is None:
            continue
#distribute total_available_memory proportionally to mempref
        scale = 1.0*prefmem(domain_dictionary[i])/total_mem_pref
        target_nonint = prefmem(domain_dictionary[i]) + scale*total_available_memory
#prevent rounding errors
        target = int(0.999*target_nonint)
#do not try to give more memory than static max
        if target > domain_dictionary[i].memory_maximum:
            left_memory += target-domain_dictionary[i].memory_maximum
            target = domain_dictionary[i].memory_maximum
        else:
# count domains which can accept more memory
            acceptors_count += 1
        target_memory[i] = target
# distribute left memory across all acceptors
    while left_memory > 0 and acceptors_count > 0:
        print '  left_memory:', left_memory, 'acceptors_count:', acceptors_count
        new_left_memory = 0
        for i in target_memory.keys():
            target = target_memory[i]
            if target < domain_dictionary[i].memory_maximum:
                memory_bonus = int(0.999*(left_memory/acceptors_count))
                if target+memory_bonus >= domain_dictionary[i].memory_maximum:
                    new_left_memory += target+memory_bonus - domain_dictionary[i].memory_maximum
                    target = domain_dictionary[i].memory_maximum
                    acceptors_count -= 1
                else:
                    target += memory_bonus
            target_memory[i] = target
        left_memory = new_left_memory
# split target_memory dictionary to donors and acceptors
#  this is needed to first get memory from donors and only then give it to acceptors
    donors_rq = list()
    acceptors_rq = list()
    for i in target_memory.keys():
        target = target_memory[i]
        if (target < domain_dictionary[i].memory_actual):
            donors_rq.append((i, target))
        else:
            acceptors_rq.append((i, target))

#    print 'balance(enough): xen_free_memory=', xen_free_memory, 'requests:', donors_rq + acceptors_rq
    return donors_rq + acceptors_rq

#when not enough mem to make everyone be above prefmem, make donors be at prefmem, and
#redistribute anything left between acceptors
def balance_when_low_on_memory(domain_dictionary, xen_free_memory, total_mem_pref_acceptors, donors, acceptors):
    donors_rq = list()
    acceptors_rq = list()
    squeezed_mem = xen_free_memory
    for i in donors:
        avail = -memory_needed(domain_dictionary[i])
        if avail < 10*1024*1024:
            #probably we have already tried making it exactly at prefmem, give up
            continue
        squeezed_mem -= avail
        donors_rq.append((i, prefmem(domain_dictionary[i])))
#the below can happen if initially xen free memory is below 50M
    if squeezed_mem < 0:
        return donors_rq
    for i in acceptors:
        scale = 1.0*prefmem(domain_dictionary[i])/total_mem_pref_acceptors
        target_nonint = domain_dictionary[i].memory_actual + scale*squeezed_mem
#do not try to give more memory than static max
        target = min(int(0.999*target_nonint), domain_dictionary[i].memory_maximum)
        acceptors_rq.append((i, target))
#    print 'balance(low): xen_free_memory=', xen_free_memory, 'requests:', donors_rq + acceptors_rq
    return donors_rq + acceptors_rq

#redistribute memory across domains
#called when one of domains update its 'meminfo' xenstore key
#return the list of (domain, memory_target) pairs to be passed to
#"xm memset" equivalent
def balance(xen_free_memory, domain_dictionary):

#sum of all memory requirements - in other words, the difference between
#memory required to be added to domains (acceptors) to make them be at their
#preferred memory, and memory that can be taken from domains (donors) that
#can provide memory. So, it can be negative when plenty of memory.
    total_memory_needed = 0

#sum of memory preferences of all domains
    total_mem_pref = 0

#sum of memory preferences of all domains that require more memory
    total_mem_pref_acceptors = 0

    donors = list()	# domains that can yield memory
    acceptors = list()  # domains that require more memory
#pass 1: compute the above "total" values
    for i in domain_dictionary.keys():
        if domain_dictionary[i].meminfo is None:
            continue
        need = memory_needed(domain_dictionary[i])
#        print 'domain' , i, 'act/pref', domain_dictionary[i].memory_actual, prefmem(domain_dictionary[i]), 'need=', need
        if need < 0 or domain_dictionary[i].memory_actual >= domain_dictionary[i].memory_maximum:
            donors.append(i)
        else:
            acceptors.append(i)
            total_mem_pref_acceptors += prefmem(domain_dictionary[i])
        total_memory_needed += need
        total_mem_pref += prefmem(domain_dictionary[i])

    total_available_memory = xen_free_memory - total_memory_needed
    if total_available_memory > 0:
        return balance_when_enough_memory(domain_dictionary, xen_free_memory, total_mem_pref, total_available_memory)
    else:
        return balance_when_low_on_memory(domain_dictionary, xen_free_memory, total_mem_pref_acceptors, donors, acceptors)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# Print some host NUMA node statistics
#
# Authors:
#   Michal Privoznik <mprivozn@redhat.com>

import libvirt
import sys
from xml.dom import minidom
import libxml2

def xpath_eval(ctxt, path):
    res = ctxt.xpathEval(path)
    if res is None or len(res) == 0:
        value = None
    else:
        value = res[0].content
    return value

try:
    conn = libvirt.openReadOnly(None)
except libvirt.libvirtError:
    print("Failed to connect to the hypervisor")
    sys.exit(1)

try:
    capsXML = conn.getCapabilities()
except libvirt.libvirtError:
    print("Failed to request capabilities")
    sys.exit(1)

caps = minidom.parseString(capsXML)
cells = caps.getElementsByTagName("cells")[0]

nodesIDs = [ int(proc.getAttribute("id"))
             for proc in cells.getElementsByTagName("cell") ]

nodesMem = [ conn.getMemoryStats(int(proc))
             for proc in nodesIDs]

doms = conn.listAllDomains(libvirt.VIR_CONNECT_LIST_DOMAINS_ACTIVE)

domsStrict = [ proc
               for proc in doms
               if proc.numaParameters()["numa_mode"] == libvirt.VIR_DOMAIN_NUMATUNE_MEM_STRICT ]

domsStrictCfg = {}
for dom in domsStrict:
    xmlStr = dom.XMLDesc()
    doc = libxml2.parseDoc(xmlStr)
    ctxt = doc.xpathNewContext()

    domsStrictCfg[dom] = {}

    pin = ctxt.xpathEval("string(/domain/numatune/memory/@nodeset)")
    memsize = ctxt.xpathEval("string(/domain/memory)")
    domsStrictCfg[dom]["memory"] = {"size": int(memsize), "pin": pin}

    for memnode in ctxt.xpathEval("/domain/numatune/memnode"):
        ctxt.setContextNode(memnode)
        cellid = xpath_eval(ctxt, "@cellid")
        nodeset = xpath_eval(ctxt, "@nodeset")

        nodesize = xpath_eval(ctxt, "/domain/cpu/numa/cell[@id='%s']/@memory" % cellid)
        domsStrictCfg[dom][cellid] = {"size": int(nodesize), "pin": nodeset}

print("NUMA stats")
print("NUMA nodes:\t" + "\t".join(str(node) for node in nodesIDs))
print("MemTotal:\t" + "\t".join(str(i.get("total") // 1024) for i in nodesMem))
print("MemFree:\t" + "\t".join(str(i.get("free") // 1024) for i in nodesMem))

for dom, v in domsStrictCfg.items():
    print("Domain '%s':\t" % dom.name())

    toPrint = "\tOverall memory: %d MiB" % (v["memory"]["size"] // 1024)
    if v["memory"]["pin"] is not None and v["memory"]["pin"] is not "":
        toPrint = toPrint + " nodes %s" % v["memory"]["pin"]
    print(toPrint)

    for k, node in sorted(v.items()):
        if k is "memory":
            continue
        toPrint = "\tNode %s:\t%d MiB" % (k, node["size"] // 1024)
        if node["pin"] is not None and node["pin"] is not "":
            toPrint = toPrint + " nodes %s" % node["pin"]
        print(toPrint)
<commit_msg>examples/nodestat: Fix None comparison<commit_after>#!/usr/bin/env python3
# Print some host NUMA node statistics
#
# Authors:
#   Michal Privoznik <mprivozn@redhat.com>

import libvirt
import sys
from xml.dom import minidom
import libxml2

def xpath_eval(ctxt, path):
    res = ctxt.xpathEval(path)
    if res is None or len(res) == 0:
        value = None
    else:
        value = res[0].content
    return value

try:
    conn = libvirt.openReadOnly(None)
except libvirt.libvirtError:
    print("Failed to connect to the hypervisor")
    sys.exit(1)

try:
    capsXML = conn.getCapabilities()
except libvirt.libvirtError:
    print("Failed to request capabilities")
    sys.exit(1)

caps = minidom.parseString(capsXML)
cells = caps.getElementsByTagName("cells")[0]

nodesIDs = [ int(proc.getAttribute("id"))
             for proc in cells.getElementsByTagName("cell") ]

nodesMem = [ conn.getMemoryStats(int(proc))
             for proc in nodesIDs]

doms = conn.listAllDomains(libvirt.VIR_CONNECT_LIST_DOMAINS_ACTIVE)

domsStrict = [ proc
               for proc in doms
               if proc.numaParameters()["numa_mode"] == libvirt.VIR_DOMAIN_NUMATUNE_MEM_STRICT ]

domsStrictCfg = {}
for dom in domsStrict:
    xmlStr = dom.XMLDesc()
    doc = libxml2.parseDoc(xmlStr)
    ctxt = doc.xpathNewContext()

    domsStrictCfg[dom] = {}

    pin = ctxt.xpathEval("string(/domain/numatune/memory/@nodeset)")
    memsize = ctxt.xpathEval("string(/domain/memory)")
    domsStrictCfg[dom]["memory"] = {"size": int(memsize), "pin": pin}

    for memnode in ctxt.xpathEval("/domain/numatune/memnode"):
        ctxt.setContextNode(memnode)
        cellid = xpath_eval(ctxt, "@cellid")
        nodeset = xpath_eval(ctxt, "@nodeset")

        nodesize = xpath_eval(ctxt, "/domain/cpu/numa/cell[@id='%s']/@memory" % cellid)
        domsStrictCfg[dom][cellid] = {"size": int(nodesize), "pin": nodeset}

print("NUMA stats")
print("NUMA nodes:\t" + "\t".join(str(node) for node in nodesIDs))
print("MemTotal:\t" + "\t".join(str(i.get("total") // 1024) for i in nodesMem))
print("MemFree:\t" + "\t".join(str(i.get("free") // 1024) for i in nodesMem))

for dom, v in domsStrictCfg.items():
    print("Domain '%s':\t" % dom.name())

    toPrint = "\tOverall memory: %d MiB" % (v["memory"]["size"] // 1024)
    if v["memory"]["pin"] is not None and v["memory"]["pin"] != "":
        toPrint = toPrint + " nodes %s" % v["memory"]["pin"]
    print(toPrint)

    for k, node in sorted(v.items()):
        if k == "memory":
            continue
        toPrint = "\tNode %s:\t%d MiB" % (k, node["size"] // 1024)
        if node["pin"] is not None and node["pin"] == "":
            toPrint = toPrint + " nodes %s" % node["pin"]
        print(toPrint)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import time
import datetime as dt

from doorman.plugins import AbstractLogsPlugin
from doorman.utils import extract_results, quote

class LogPlugin(AbstractLogsPlugin):
    def __init__(self, config):
        append = config.setdefault('DOORMAN_LOG_FILE_PLUGIN_APPEND', True)
        status_path = config.get('DOORMAN_LOG_FILE_PLUGIN_STATUS_LOG')
        result_path = config.get('DOORMAN_LOG_FILE_PLUGIN_RESULT_LOG')

        mode = 'a' if append else 'w'
        self.status = open(status_path, mode) if status_path else None
        self.result = open(result_path, mode) if result_path else None

    @property
    def name(self):
        return "file"

    def join_fields(self, fields):
        parts = []
        for name, val in fields.items():
            if not isinstance(val, str):
                val = str(val)
            parts.append(name + '=' + quote(val))

        return ', '.join(parts)

    def handle_status(self, data, **kwargs):
        if self.status is None:
            return

        # Write each status log on a different line
        for item in data.get('data', []):
            fields = {}
            fields.update(kwargs)
            fields.update({
                'line':     item.get('line', ''),
                'message':  item.get('message', ''),
                'severity': item.get('severity', ''),
                'filename': item.get('filename', ''),
                'version': item.get('version'),  # be null
            })

            if 'created' in item:
                fields['created'] = time.mktime(item['created'].timetuple())
            else:
                fields['created'] = time.mktime(dt.datetime.utcnow().timetuple())

            self.status.write(self.join_fields(fields) + '\n')

    def handle_result(self, data, **kwargs):
        if self.result is None:
            return

        # Process each result individually
        for item in extract_results(data):
            fields = {}
            fields.update(kwargs)

            if item.timestamp:
                timestamp = time.mktime(item.timestamp.timetuple())
            else:
                timestamp = time.mktime(dt.datetime.utcnow.timetuple())

            fields.update(name=item.name, timestamp=timestamp)

            base = self.join_fields(fields)

            # Write each added/removed entry on a different line
            curr_fields = {'result_type': item.action}
            for key, val in item.columns.items():
                curr_fields['_'.join([item.action, key])] = val

            self.result.write(base + ', ' + self.join_fields(curr_fields) + '\n')
<commit_msg>Properly flush the status / results file after handling<commit_after># -*- coding: utf-8 -*-
import os
import time
import datetime as dt

from doorman.plugins import AbstractLogsPlugin
from doorman.utils import extract_results, quote

class LogPlugin(AbstractLogsPlugin):
    def __init__(self, config):
        append = config.setdefault('DOORMAN_LOG_FILE_PLUGIN_APPEND', True)
        status_path = config.get('DOORMAN_LOG_FILE_PLUGIN_STATUS_LOG')
        result_path = config.get('DOORMAN_LOG_FILE_PLUGIN_RESULT_LOG')

        mode = 'a' if append else 'w'
        self.status = open(status_path, mode) if status_path else None
        self.result = open(result_path, mode) if result_path else None

    @property
    def name(self):
        return "file"

    def join_fields(self, fields):
        parts = []
        for name, val in fields.items():
            if not isinstance(val, str):
                val = str(val)
            parts.append(name + '=' + quote(val))

        return ', '.join(parts)

    def handle_status(self, data, **kwargs):
        if self.status is None:
            return

        # Write each status log on a different line
        try:
            for item in data.get('data', []):
                fields = {}
                fields.update(kwargs)
                fields.update({
                    'line':     item.get('line', ''),
                    'message':  item.get('message', ''),
                    'severity': item.get('severity', ''),
                    'filename': item.get('filename', ''),
                    'version': item.get('version'),  # be null
                })

                if 'created' in item:
                    fields['created'] = time.mktime(item['created'].timetuple())
                else:
                    fields['created'] = time.mktime(dt.datetime.utcnow().timetuple())

                self.status.write(self.join_fields(fields) + '\n')
        finally:
            self.status.flush()
            os.fsync(self.status.fileno())

    def handle_result(self, data, **kwargs):
        if self.result is None:
            return

        # Process each result individually
        try:
            for item in extract_results(data):
                fields = {}
                fields.update(kwargs)

                if item.timestamp:
                    timestamp = time.mktime(item.timestamp.timetuple())
                else:
                    timestamp = time.mktime(dt.datetime.utcnow.timetuple())

                fields.update(name=item.name, timestamp=timestamp)

                base = self.join_fields(fields)

                # Write each added/removed entry on a different line
                curr_fields = {'result_type': item.action}
                for key, val in item.columns.items():
                    curr_fields['_'.join([item.action, key])] = val

                self.result.write(base + ', ' + self.join_fields(curr_fields) + '\n')
        finally:
            self.result.flush()
            os.fsync(self.result.fileno())
<|endoftext|>


<commit_before>#Note to self - don't EVER disassemble the mast. Doesn't end well.
#.456 - maximum parallel mast
kMastBack = .300 #This is so the lift won't go up when the mast is back - DO NOT CHANGE IN TESTING
kMastBackLimit = .311
kMastForwardLimit = .482
kMastParallel = .416

#claw setpoints - tested 2/12/15
kOpen = .183
kClose = .890 #2/12/15 23.12: .771 2/13/15 16:30: .925 2/15/15 .070
kCan = .458
kTote = .537
kStall = 2 #This is for the current sensor.

#lift setpoints - untested
Tote=.104
kUp = .189 #This is so the mast won't tilt when the lift is up
kTop = .105 #was .651
kBottom = .725 # was .050
kDelta = (kTop-kBottom)
kAboveSecond = .555
kAboveFirst = .465
#diff of .066
lift_levels_reletive = [-0.0, 0.0779, 0.2323, 0.4044, 0.5691, 0.7338, 0.8911]
lift_level_setpoints = [(i*kDelta)+kBottom for i in lift_levels_reletive]
lift_step_setpoints = [i-.015 for i in lift_level_setpoints]
<commit_msg>More reletive.<commit_after>#Note to self - don't EVER disassemble the mast. Doesn't end well.
#.456 - maximum parallel mast
def scale_reletive(v, top, bottom):
    return v*(top-bottom)+bottom

kMastBack = .300 #This is so the lift won't go up when the mast is back - DO NOT CHANGE IN TESTING
kMastBackLimit = .311
kMastForwardLimit = .482
kMastParallel = .416

#claw setpoints - tested 2/12/15
kOpen = .183
kClose = .890 #2/12/15 23.12: .771 2/13/15 16:30: .925 2/15/15 .070
kCan = .458
kTote = .537
kStall = 2 #This is for the current sensor.

#lift setpoints - untested
kUp = .189 #This is so the mast won't tilt when the lift is up
kTop = .105 #was .651
kBottom = .725 # was .050
kDelta = (kTop-kBottom)
kAboveSecond = scale_reletive(0.1564, kTop, kBottom)
kAboveFirst = scale_reletive(0.2176, kTop, kBottom)
#diff of .066
lift_levels_reletive = [-0.0, 0.0779, 0.2323, 0.4044, 0.5691, 0.7338, 0.8911]
lift_level_setpoints = [scale_reletive(i, kTop, kBottom) for i in lift_levels_reletive]
lift_step_setpoints = [i-.015 for i in lift_level_setpoints]
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import optparse
import os
import sys
import tarfile

import buildbot_common

def main(args):
  parser = optparse.OptionParser()
  parser.add_option('--install-dir',
                    help='Install Directory',
                    dest='install_dir',
                    default='naclmono')
  parser.add_option('--tar-path',
                    help='Tarfile path',
                    dest='tar_path',
                    default='naclmono_%pepperrev%.bz2')
  parser.add_option('--upload-path',
                    help='Upload path (nativeclient-mirror/nacl/nacl_sdk/XXX)',
                    dest='upload_path',
                    default=None)
  parser.add_option('--pepper-revision',
                    help='Pepper revision',
                    dest='pepper_revision',
                    default=None)
  parser.add_option('--skip-upload',
                    help='Skips upload step',
                    action="store_true",
                    dest='skip_upload')
  (options, args) = parser.parse_args(args[1:])

  if not options.upload_path:
    buildbot_common.ErrorExit('--upload-path is required')
  if not options.pepper_revision:
    buildbot_common.ErrorExit('--pepper-revision is required')

  options.tar_path = options.tar_path.replace('%pepperrev%',
                                              options.pepper_revision)

  install_folders = ['bin', 'etc', 'include', 'lib', 'lib32', 'share']

  buildbot_common.BuildStep('Archive Build')
  tar_file = None
  buildbot_common.RemoveFile(options.tar_path)
  try:
    tar_file = tarfile.open(options.tar_path, mode='w:bz2')
    for subfolder in install_folders:
      tar_file.add(os.path.join(options.install_dir, subfolder),
                   arcname=subfolder)
  finally:
    if tar_file:
      tar_file.close()

  if not options.skip_upload:
    buildbot_common.Archive(os.path.basename(options.tar_path),
        'nativeclient-mirror/nacl/nacl_sdk/%s' % options.upload_path,
        cwd=os.path.dirname(os.path.abspath(options.tar_path)))

if __name__ == '__main__':
  sys.exit(main(sys.argv))
<commit_msg>[NaCl SDK] Dereference links before archiving naclmono, fixes Windows build problems<commit_after>#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import optparse
import os
import sys
import tarfile

import buildbot_common

def main(args):
  parser = optparse.OptionParser()
  parser.add_option('--install-dir',
                    help='Install Directory',
                    dest='install_dir',
                    default='naclmono')
  parser.add_option('--tar-path',
                    help='Tarfile path',
                    dest='tar_path',
                    default='naclmono_%pepperrev%.bz2')
  parser.add_option('--upload-path',
                    help='Upload path (nativeclient-mirror/nacl/nacl_sdk/XXX)',
                    dest='upload_path',
                    default=None)
  parser.add_option('--pepper-revision',
                    help='Pepper revision',
                    dest='pepper_revision',
                    default=None)
  parser.add_option('--skip-upload',
                    help='Skips upload step',
                    action="store_true",
                    dest='skip_upload')
  (options, args) = parser.parse_args(args[1:])

  if not options.upload_path:
    buildbot_common.ErrorExit('--upload-path is required')
  if not options.pepper_revision:
    buildbot_common.ErrorExit('--pepper-revision is required')

  options.tar_path = options.tar_path.replace('%pepperrev%',
                                              options.pepper_revision)

  install_folders = ['bin', 'etc', 'include', 'lib', 'lib32', 'share']

  buildbot_common.BuildStep('Archive Build')
  tar_file = None
  buildbot_common.RemoveFile(options.tar_path)
  try:
    tar_file = tarfile.open(options.tar_path, mode='w:bz2', dereference=True)
    for subfolder in install_folders:
      tar_file.add(os.path.join(options.install_dir, subfolder),
                   arcname=subfolder)
  finally:
    if tar_file:
      tar_file.close()

  if not options.skip_upload:
    buildbot_common.Archive(os.path.basename(options.tar_path),
        'nativeclient-mirror/nacl/nacl_sdk/%s' % options.upload_path,
        cwd=os.path.dirname(os.path.abspath(options.tar_path)))

if __name__ == '__main__':
  sys.exit(main(sys.argv))
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
import argparse
import Tools
import Align
import pdb
from  Bio import SeqIO

ap = argparse.ArgumentParser(description="Print gaps in a SAM file.")
ap.add_argument("genome", help="Genome file with a .fai")
ap.add_argument("sam", help="Sam file of alignment.", nargs="+")
ap.add_argument("--onTarget", help="Assume the query encodes the position of the aligned sequence, and make sure at least the chromosomes match.", default=False, action='store_true')
ap.add_argument("--gapFree", help="Print sequences without gaps.", default=None)
ap.add_argument("--minLength", help="Minimum gapLength.", default=30, type=int)
ap.add_argument("--outFile", help="Print output here, default= stdout", default=None)
ap.add_argument("--context", help="Print surrounding context", default=0, type=int)
ap.add_argument("--condense", help="Pack indels if the matches separating them is less than this value.", default=10, type=int)
ap.add_argument("--tsd", help="Attempt to find Target Site Duplications at most this length", default=20, type=int)
ap.add_argument("--outsam", help="Write the modified condensed sam to a file.", default=None)
ap.add_argument("--minq", help="Minimal mapping quality to consider (10)",default=10,type=int)
ap.add_argument("--qpos", help="Write query position of gaps", default=False,action='store_true')

args = ap.parse_args()

genome = file(args.genome, 'r')
handle = open(args.genome, "r")

if (args.outFile is None):
    outFile = sys.stdout
else:
    outFile = open(args.outFile, 'w')

if (args.gapFree is not None):
    gapFree = open(args.gapFree, 'w')

fai = Tools.ReadFAIFile(args.genome + ".fai")

#genomeDict = SeqIO.to_dict(SeqIO.parse(handle, "fasta"))

if (args.outsam is not None):
    outsam = open(args.outsam, 'w')

fai = Tools.ReadFAIFile(args.genome + ".fai")
genomeFile = open(args.genome, 'r')

M = 'M'
I = 'I'
D = 'D'
N = 'N'
S = 'S'
H = 'H'
P = 'P'
if (args.sam[0].find(".fofn") >= 0):
    fofnFile = open(args.sam[0])
    samFiles = [line.strip() for line in fofnFile.readlines()]
    args.sam = samFiles
lineNumber = 0
for samFileName in args.sam:
    samFile = open(samFileName)

    for line in samFile.readlines():
        lineNumber = lineNumber + 1
        if (line[0] == "@"):
            if (args.outsam is not None):
                outsam.write(line)
            continue
        if (len(line) <= 1):
            continue

        aln = Tools.SAMEntry(line)
        if (aln.title is None):
            continue

        if (args.onTarget == True):
            srcChrom = aln.title.split(":")[0]
            if (srcChrom != aln.tName):
                continue
        if (aln.flag & 256 != 0):
            continue
        if (aln.mapqv < args.minq):
            continue

        tPos = aln.tStart
        qPos = 0
        #
        # condense matches.
        #
        packedCigar = []
        i = 0
        i1 = 1
        niter = 0
        maxGap = 0
        maxGapType = 0
        packedOps = []
        packedLengths = []
        #print str(aln.ops)

        foundGap = False
        while (i < len(aln.lengths)):
            l = aln.lengths[i]
            op = aln.ops[i]
            j = i
            if (op == I or op == D):
                if (l > maxGap):
                    maxGap = l
                    maxGapType = op

            if (op == I or op == D and i < len(aln.ops) - 2 and aln.ops[i+2][0] == op):
                matchLen = 0
                gapLen   = 0
                while (j+2 < len(aln.ops) and aln.ops[j+2][0] == op and aln.ops[j+1][0] == M and aln.lengths[j+1] < args.condense):

                    matchLen += aln.lengths[j+1]
                    gapLen   += aln.lengths[j+2]
                    j+=2
                if (j > i):
                    newIndel = (op, l+gapLen)
                    newMatch = (M, matchLen)
                    packedOps.append(op)
                    packedLengths.append(l+gapLen)

                    packedOps.append(M)
                    packedLengths.append(matchLen)

                else:
                    packedLengths.append(l)
                    packedOps.append(op)

            else:
                packedLengths.append(l)
                packedOps.append(op)

            i = j + 1
            niter +=1
            if (niter > len(aln.ops)):
                print "ERROR! too many interations."
        for i in range(len(packedOps)):
            op = packedOps[i]
            l  = packedLengths[i]

            if (op == M or op == N or op == S):
                tPos += l
                qPos += l
            if (op == I):
                if (l > args.minLength):
                    #                    print "gap at " + str(i) + " " + str(op) + " " + str(l) + " " + str(qPos) + " qlen: " + str(len(aln.seq))
                    foundGap = True
                    chrName = aln.tName
                    gapSeq = aln.seq[max(0,qPos-args.context):min(qPos+l+args.context, len(aln.seq))]
                    tsd = "notsd"
                    if (len(gapSeq) == 0):
                        print "ERROR, gap seq is of zero length"
                    if (args.tsd):
                        # try and find the target site duplications, this may be on either side of the alignemnt
                        tsdSuffix = gapSeq[-args.tsd:]
                        tsdSuffix = tsdSuffix.upper()
                        tsdPrefix = gapSeq[0:args.tsd]
                        tsdPrefix = tsdPrefix.upper()
                        targetPrefix = Tools.ExtractSeq((chrName, tPos-args.tsd,tPos), genomeFile, fai)
#                        targetPrefix = genomeDict[chrName].seq[tPos-args.tsd:tPos]

                        targetPrefix = targetPrefix.upper()
                        #targetSuffix = genomeDict[chrName].seq[tPos:tPos+args.tsd]
                        targetSuffix = Tools.ExtractSeq((chrName, tPos,tPos+args.tsd), genomeFile, fai)
                        targetSuffix = targetSuffix.upper()
                        (sp, ss, sScore) = Align.TSDAlign(tsdSuffix, targetPrefix, 'suffix')
                        (pp, ps, pScore) = Align.TSDAlign(tsdPrefix, targetSuffix, 'prefix')
                        if (sScore > pScore ):
                            tsd = ss
                        elif (pScore > sScore ):
                            tsd = ps
                        #print "sp ss: " + str(sScore) + " " + sp + " " + ss + " " + tsdSuffix + " " + targetPrefix
                        #print "pp ps: " + str(pScore) + " " + pp + " " + ps + " " + tsdPrefix + " " + targetSuffix

                    nucs = ['A', 'C', 'G', 'T']
                    fracs = [float(gapSeq.count(n))/(len(gapSeq)+1) for n in nucs]
                    doPrint = True
                    for frac in fracs:
                        if (frac > 0.85):
                            doPrint = False
                    if (doPrint):
                        outFile.write("{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}".format(chrName, tPos, tPos + l, "insertion", l, gapSeq, tsd, aln.title))
                        if (args.qpos):
                            outFile.write("\t{}\t{}".format(qPos, qPos + len(gapSeq)))
                        outFile.write("\n")

                qPos += l
            if (op == D):
                if (l > args.minLength):
                    foundGap = True
                    chrName = aln.tName
                    if (tPos > fai[chrName][0]):
                        print "ERROR! tpos is past the genome end." + str(tPos) + " " + str(fai[chrName][0])
                    delStart = max(tPos - args.context, 0)
                    delEnd   = min(tPos + args.context + l, fai[chrName][0])
                    #delSeq = genomeDict[chrName].seq[delStart:delEnd].tostring()
                    delSeq = Tools.ExtractSeq([chrName, delStart, delEnd], genomeFile, fai)
                    outFile.write("{}\t{}\t{}\t{}\t{}\t{}\tno_tsd\t{}".format(chrName, tPos, tPos + l, "deletion", l, delSeq, aln.title))
                    if (args.qpos):
                        outFile.write("\t{}\t{}".format(qPos, qPos))
                    outFile.write("\n")

                tPos += l
            if (op == H):
                pass
        if (foundGap == False and args.gapFree is not None):
            gapFree.write(aln.tName + "\t" + str(aln.tStart) + "\t" + str(aln.tEnd) + "\t" + aln.title + "\n")

        if (args.outsam is not None):
            packedCigar= ''.join([str(v[0]) + str(v[1]) for v in zip(packedLengths, packedOps)])
            vals = line.split()
            packedLine = '\t'.join(vals[0:5]) + "\t" + packedCigar + '\t'.join(vals[6:]) + "\n"
            outsam.write(packedLine)

if (args.gapFree is not None):
    gapFree.close()

outFile.close()
if (args.outsam is not None):
    outsam.close()
<commit_msg>added TODO<commit_after>#!/usr/bin/env python

import sys
import argparse
import Tools
import Align
import pdb
from  Bio import SeqIO

ap = argparse.ArgumentParser(description="Print gaps in a SAM file.")
ap.add_argument("genome", help="Genome file with a .fai")
ap.add_argument("sam", help="Sam file of alignment.", nargs="+")
ap.add_argument("--onTarget", help="Assume the query encodes the position of the aligned sequence, and make sure at least the chromosomes match.", default=False, action='store_true')
ap.add_argument("--gapFree", help="Print sequences without gaps.", default=None)
ap.add_argument("--minLength", help="Minimum gapLength.", default=30, type=int)
ap.add_argument("--outFile", help="Print output here, default= stdout", default=None)
ap.add_argument("--context", help="Print surrounding context", default=0, type=int)
ap.add_argument("--condense", help="Pack indels if the matches separating them is less than this value.", default=10, type=int)
ap.add_argument("--tsd", help="Attempt to find Target Site Duplications at most this length", default=20, type=int)
ap.add_argument("--outsam", help="Write the modified condensed sam to a file.", default=None)
ap.add_argument("--minq", help="Minimal mapping quality to consider (10)",default=10,type=int)
ap.add_argument("--qpos", help="Write query position of gaps", default=False,action='store_true')

args = ap.parse_args()

genome = file(args.genome, 'r')
handle = open(args.genome, "r")

if (args.outFile is None):
    outFile = sys.stdout
else:
    outFile = open(args.outFile, 'w')

if (args.gapFree is not None):
    gapFree = open(args.gapFree, 'w')

fai = Tools.ReadFAIFile(args.genome + ".fai")

#genomeDict = SeqIO.to_dict(SeqIO.parse(handle, "fasta"))

if (args.outsam is not None):
    outsam = open(args.outsam, 'w')

fai = Tools.ReadFAIFile(args.genome + ".fai")
genomeFile = open(args.genome, 'r')

M = 'M'
I = 'I'
D = 'D'
N = 'N'
S = 'S'
H = 'H'
P = 'P'
if (args.sam[0].find(".fofn") >= 0):
    fofnFile = open(args.sam[0])
    samFiles = [line.strip() for line in fofnFile.readlines()]
    args.sam = samFiles
lineNumber = 0
for samFileName in args.sam:
    samFile = open(samFileName)

    for line in samFile.readlines():
        lineNumber = lineNumber + 1
        if (line[0] == "@"):
            if (args.outsam is not None):
                outsam.write(line)
            continue
        if (len(line) <= 1):
            continue

        aln = Tools.SAMEntry(line)
        if (aln.title is None):
            continue

        if (args.onTarget == True):
            srcChrom = aln.title.split(":")[0]
            if (srcChrom != aln.tName):
                continue
        if (aln.flag & 256 != 0):
            continue
        if (aln.mapqv < args.minq):
            continue

        tPos = aln.tStart
        qPos = 0
        #
        # condense matches.
        #
        packedCigar = []
        i = 0
        i1 = 1
        niter = 0
        maxGap = 0
        maxGapType = 0
        packedOps = []
        packedLengths = []
        #print str(aln.ops)

        foundGap = False
        while (i < len(aln.lengths)):
            l = aln.lengths[i]
            op = aln.ops[i]
            j = i
            if (op == I or op == D):
                if (l > maxGap):
                    maxGap = l
                    maxGapType = op

            if (op == I or op == D and i < len(aln.ops) - 2 and aln.ops[i+2][0] == op):
                matchLen = 0
                gapLen   = 0
                while (j+2 < len(aln.ops) and aln.ops[j+2][0] == op and aln.ops[j+1][0] == M and aln.lengths[j+1] < args.condense):

                    matchLen += aln.lengths[j+1]
                    gapLen   += aln.lengths[j+2]
                    j+=2
                if (j > i):
                    newIndel = (op, l+gapLen)
                    newMatch = (M, matchLen)
                    packedOps.append(op)
                    packedLengths.append(l+gapLen)

                    packedOps.append(M)
                    packedLengths.append(matchLen)

                else:
                    packedLengths.append(l)
                    packedOps.append(op)

            else:
                packedLengths.append(l)
                packedOps.append(op)

            i = j + 1
            niter +=1
            if (niter > len(aln.ops)):
                print "ERROR! too many interations."
        for i in range(len(packedOps)):
            op = packedOps[i]
            l  = packedLengths[i]

            if (op == M or op == N or op == S):
                tPos += l
                qPos += l
            if (op == I):
                if (l > args.minLength):
                    #                    print "gap at " + str(i) + " " + str(op) + " " + str(l) + " " + str(qPos) + " qlen: " + str(len(aln.seq))
                    foundGap = True
                    chrName = aln.tName
                    gapSeq = aln.seq[max(0,qPos-args.context):min(qPos+l+args.context, len(aln.seq))]
                    tsd = "notsd"
                    if (len(gapSeq) == 0):
                        print "ERROR, gap seq is of zero length"
                    if (args.tsd):
                        # try and find the target site duplications, this may be on either side of the alignemnt
                        tsdSuffix = gapSeq[-args.tsd:]
                        tsdSuffix = tsdSuffix.upper()
                        tsdPrefix = gapSeq[0:args.tsd]
                        tsdPrefix = tsdPrefix.upper()
                        targetPrefix = Tools.ExtractSeq((chrName, tPos-args.tsd,tPos), genomeFile, fai)
#                        targetPrefix = genomeDict[chrName].seq[tPos-args.tsd:tPos]

                        targetPrefix = targetPrefix.upper()
                        #targetSuffix = genomeDict[chrName].seq[tPos:tPos+args.tsd]
                        targetSuffix = Tools.ExtractSeq((chrName, tPos,tPos+args.tsd), genomeFile, fai)
                        targetSuffix = targetSuffix.upper()
                        (sp, ss, sScore) = Align.TSDAlign(tsdSuffix, targetPrefix, 'suffix')
                        (pp, ps, pScore) = Align.TSDAlign(tsdPrefix, targetSuffix, 'prefix')
                        if (sScore > pScore ):
                            tsd = ss
                        elif (pScore > sScore ):
                            tsd = ps
                        #print "sp ss: " + str(sScore) + " " + sp + " " + ss + " " + tsdSuffix + " " + targetPrefix
                        #print "pp ps: " + str(pScore) + " " + pp + " " + ps + " " + tsdPrefix + " " + targetSuffix

                    nucs = ['A', 'C', 'G', 'T']
                    fracs = [float(gapSeq.count(n))/(len(gapSeq)+1) for n in nucs]
                    doPrint = True
                    for frac in fracs:
                        # TODO: Define or replace magic number.
                        if (frac > 0.85):
                            doPrint = False
                    if (doPrint):
                        outFile.write("{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}".format(chrName, tPos, tPos + l, "insertion", l, gapSeq, tsd, aln.title))
                        if (args.qpos):
                            outFile.write("\t{}\t{}".format(qPos, qPos + len(gapSeq)))
                        outFile.write("\n")

                qPos += l
            if (op == D):
                if (l > args.minLength):
                    foundGap = True
                    chrName = aln.tName
                    if (tPos > fai[chrName][0]):
                        print "ERROR! tpos is past the genome end." + str(tPos) + " " + str(fai[chrName][0])
                    delStart = max(tPos - args.context, 0)
                    delEnd   = min(tPos + args.context + l, fai[chrName][0])
                    #delSeq = genomeDict[chrName].seq[delStart:delEnd].tostring()
                    delSeq = Tools.ExtractSeq([chrName, delStart, delEnd], genomeFile, fai)
                    outFile.write("{}\t{}\t{}\t{}\t{}\t{}\tno_tsd\t{}".format(chrName, tPos, tPos + l, "deletion", l, delSeq, aln.title))
                    if (args.qpos):
                        outFile.write("\t{}\t{}".format(qPos, qPos))
                    outFile.write("\n")

                tPos += l
            if (op == H):
                pass
        if (foundGap == False and args.gapFree is not None):
            gapFree.write(aln.tName + "\t" + str(aln.tStart) + "\t" + str(aln.tEnd) + "\t" + aln.title + "\n")

        if (args.outsam is not None):
            packedCigar= ''.join([str(v[0]) + str(v[1]) for v in zip(packedLengths, packedOps)])
            vals = line.split()
            packedLine = '\t'.join(vals[0:5]) + "\t" + packedCigar + '\t'.join(vals[6:]) + "\n"
            outsam.write(packedLine)

if (args.gapFree is not None):
    gapFree.close()

outFile.close()
if (args.outsam is not None):
    outsam.close()
<|endoftext|>


<commit_before># Copyright (c) 2009,2010 Dennis Kaarsemaker <dennis@kaarsemaker.net>
# Small piece of middleware to be able to access authentication data from
# everywhere in the django code.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#     1. Redistributions of source code must retain the above copyright notice,
#        this list of conditions and the following disclaimer.
#
#     2. Redistributions in binary form must reproduce the above copyright
#        notice, this list of conditions and the following disclaimer in the
#        documentation and/or other materials provided with the distribution.
#
#     3. Neither the name of Django nor the names of its contributors may be used
#        to endorse or promote products derived from this software without
#        specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

from django.conf import settings
from django.contrib.auth.models import User
from django.db.models.fields.related import ForeignKey, ManyToOneRel

from echelon.middleware import EchelonMiddleware

if 'echelon' not in settings.INSTALLED_APPS:
    raise ValueError("Echelon middleware is not enabled")

class CurrentUserField(ForeignKey):
    def __init__(self, to_field=None, rel_class=ManyToOneRel, **kwargs):
        to = User
        kwargs['editable'] = False
        self.add_only = kwargs.pop('add_only', False)
        super(CurrentUserField, self).__init__(to, to_field, rel_class, **kwargs)

    def pre_save(self, model_instance, add):
        if add or not self.add_only:
            user = EchelonMiddleware.get_user()
            if user:
                setattr(model_instance, self.attname, user.pk)
                return user.pk
        return super(CurrentUserField, self).pre_save(model_instance, add)
<commit_msg>South compatibility<commit_after># Copyright (c) 2009,2010 Dennis Kaarsemaker <dennis@kaarsemaker.net>
# Small piece of middleware to be able to access authentication data from
# everywhere in the django code.
#
# Redistribution and use in source and binary forms, with or without modification,
# are permitted provided that the following conditions are met:
#
#     1. Redistributions of source code must retain the above copyright notice,
#        this list of conditions and the following disclaimer.
#
#     2. Redistributions in binary form must reproduce the above copyright
#        notice, this list of conditions and the following disclaimer in the
#        documentation and/or other materials provided with the distribution.
#
#     3. Neither the name of Django nor the names of its contributors may be used
#        to endorse or promote products derived from this software without
#        specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

from django.conf import settings
from django.contrib.auth.models import User
from django.db.models.fields.related import ForeignKey, ManyToOneRel
from echelon.middleware import EchelonMiddleware

if 'echelon' not in settings.INSTALLED_APPS:
    raise ValueError("Echelon middleware is not enabled")

# Register OrderingField with south
if 'south' in settings.INSTALLED_APPS:
    from south.modelsinspector import add_introspection_rules
    add_introspection_rules([], ["^echelon\.fields\.CurrentUserField"])

class CurrentUserField(ForeignKey):
    def __init__(self, to_field=None, rel_class=ManyToOneRel, **kwargs):
        to = User
        kwargs['editable'] = False
        self.add_only = kwargs.pop('add_only', False)
        super(CurrentUserField, self).__init__(to, to_field, rel_class, **kwargs)

    def pre_save(self, model_instance, add):
        if add or not self.add_only:
            user = EchelonMiddleware.get_user()
            if user:
                setattr(model_instance, self.attname, user.pk)
                return user.pk
        return super(CurrentUserField, self).pre_save(model_instance, add)
<|endoftext|>


<commit_before># pvtol_lqr.m - LQR design for vectored thrust aircraft
# RMM, 14 Jan 03
#
# This file works through an LQR based design problem, using the
# planar vertical takeoff and landing (PVTOL) aircraft example from
# Astrom and Murray, Chapter 5.  It is intended to demonstrate the
# basic functionality of the python-control package.
#

import os
import numpy as np
import matplotlib.pyplot as plt  # MATLAB plotting functions
from control.matlab import *  # MATLAB-like functions

#
# System dynamics
#
# These are the dynamics for the PVTOL system, written in state space
# form.
#

# System parameters
m = 4       # mass of aircraft
J = 0.0475  # inertia around pitch axis
r = 0.25    # distance to center of force
g = 9.8     # gravitational constant
c = 0.05    # damping factor (estimated)

# State space dynamics
xe = [0, 0, 0, 0, 0, 0]  # equilibrium point of interest
ue = [0, m*g]  # (note these are lists, not matrices)

# TODO: The following objects need converting from np.matrix to np.array
# This will involve re-working the subsequent equations as the shapes
# See below.

# Dynamics matrix (use matrix type so that * works for multiplication)
A = np.matrix(
    [[0, 0, 0, 1, 0, 0],
     [0, 0, 0, 0, 1, 0],
     [0, 0, 0, 0, 0, 1],
     [0, 0, (-ue[0]*np.sin(xe[2]) - ue[1]*np.cos(xe[2]))/m, -c/m, 0, 0],
     [0, 0, (ue[0]*np.cos(xe[2]) - ue[1]*np.sin(xe[2]))/m, 0, -c/m, 0],
     [0, 0, 0, 0, 0, 0]]
)

# Input matrix
B = np.matrix(
    [[0, 0], [0, 0], [0, 0],
     [np.cos(xe[2])/m, -np.sin(xe[2])/m],
     [np.sin(xe[2])/m, np.cos(xe[2])/m],
     [r/J, 0]]
)

# Output matrix
C = np.matrix([[1, 0, 0, 0, 0, 0], [0, 1, 0, 0, 0, 0]])
D = np.matrix([[0, 0], [0, 0]])

#
# Construct inputs and outputs corresponding to steps in xy position
#
# The vectors xd and yd correspond to the states that are the desired
# equilibrium states for the system.  The matrices Cx and Cy are the
# corresponding outputs.
#
# The way these vectors are used is to compute the closed loop system
# dynamics as
#
#   xdot = Ax + B u  =>  xdot = (A-BK)x + K xd
#      u = -K(x - xd)       y = Cx
#
# The closed loop dynamics can be simulated using the "step" command,
# with K*xd as the input vector (assumes that the "input" is unit size,
# so that xd corresponds to the desired steady state.
#

xd = np.matrix([[1], [0], [0], [0], [0], [0]])
yd = np.matrix([[0], [1], [0], [0], [0], [0]])

#
# Extract the relevant dynamics for use with SISO library
#
# The current python-control library only supports SISO transfer
# functions, so we have to modify some parts of the original MATLAB
# code to extract out SISO systems.  To do this, we define the 'lat' and
# 'alt' index vectors to consist of the states that are are relevant
# to the lateral (x) and vertical (y) dynamics.
#

# Indices for the parts of the state that we want
lat = (0, 2, 3, 5)
alt = (1, 4)

# Decoupled dynamics
Ax = (A[lat, :])[:, lat]  # ! not sure why I have to do it this way
Bx = B[lat, 0]
Cx = C[0, lat]
Dx = D[0, 0]

Ay = (A[alt, :])[:, alt]  # ! not sure why I have to do it this way
By = B[alt, 1]
Cy = C[1, alt]
Dy = D[1, 1]

# Label the plot
plt.clf()
plt.suptitle("LQR controllers for vectored thrust aircraft (pvtol-lqr)")

#
# LQR design
#

# Start with a diagonal weighting
Qx1 = np.diag([1, 1, 1, 1, 1, 1])
Qu1a = np.diag([1, 1])
K, X, E = lqr(A, B, Qx1, Qu1a)
K1a = np.matrix(K)

# Close the loop: xdot = Ax - B K (x-xd)
# Note: python-control requires we do this 1 input at a time
# H1a = ss(A-B*K1a, B*K1a*concatenate((xd, yd), axis=1), C, D);
# (T, Y) = step(H1a, T=np.linspace(0,10,100));

# TODO: The following equations will need modifying when converting from np.matrix to np.array
# because the results and even intermediate calculations will be different with numpy arrays
# For example:
# Bx = B[lat, 0]
# Will need to be changed to:
# Bx = B[lat, 0].reshape(-1, 1)
# (if we want it to have the same shape as before)

# For reference, here is a list of the correct shapes of these objects:
# A: (6, 6)
# B: (6, 2)
# C: (2, 6)
# D: (2, 2)
# xd: (6, 1)
# yd: (6, 1)
# Ax: (4, 4)
# Bx: (4, 1)
# Cx: (1, 4)
# Dx: ()
# Ay: (2, 2)
# By: (2, 1)
# Cy: (1, 2)

# Step response for the first input
H1ax = ss(Ax - Bx*K1a[0, lat], Bx*K1a[0, lat]*xd[lat, :], Cx, Dx)
Yx, Tx = step(H1ax, T=np.linspace(0, 10, 100))

# Step response for the second input
H1ay = ss(Ay - By*K1a[1, alt], By*K1a[1, alt]*yd[alt, :], Cy, Dy)
Yy, Ty = step(H1ay, T=np.linspace(0, 10, 100))

plt.subplot(221)
plt.title("Identity weights")
# plt.plot(T, Y[:,1, 1], '-', T, Y[:,2, 2], '--')
plt.plot(Tx.T, Yx.T, '-', Ty.T, Yy.T, '--')
plt.plot([0, 10], [1, 1], 'k-')

plt.axis([0, 10, -0.1, 1.4])
plt.ylabel('position')
plt.legend(('x', 'y'), loc='lower right')

# Look at different input weightings
Qu1a = np.diag([1, 1])
K1a, X, E = lqr(A, B, Qx1, Qu1a)
H1ax = ss(Ax - Bx*K1a[0, lat], Bx*K1a[0, lat]*xd[lat, :], Cx, Dx)

Qu1b = (40 ** 2)*np.diag([1, 1])
K1b, X, E = lqr(A, B, Qx1, Qu1b)
H1bx = ss(Ax - Bx*K1b[0, lat], Bx*K1b[0, lat]*xd[lat, :], Cx, Dx)

Qu1c = (200 ** 2)*np.diag([1, 1])
K1c, X, E = lqr(A, B, Qx1, Qu1c)
H1cx = ss(Ax - Bx*K1c[0, lat], Bx*K1c[0, lat]*xd[lat, :], Cx, Dx)

[Y1, T1] = step(H1ax, T=np.linspace(0, 10, 100))
[Y2, T2] = step(H1bx, T=np.linspace(0, 10, 100))
[Y3, T3] = step(H1cx, T=np.linspace(0, 10, 100))

plt.subplot(222)
plt.title("Effect of input weights")
plt.plot(T1.T, Y1.T, 'b-')
plt.plot(T2.T, Y2.T, 'b-')
plt.plot(T3.T, Y3.T, 'b-')
plt.plot([0, 10], [1, 1], 'k-')

plt.axis([0, 10, -0.1, 1.4])

# arcarrow([1.3, 0.8], [5, 0.45], -6)
plt.text(5.3, 0.4, 'rho')

# Output weighting - change Qx to use outputs
Qx2 = C.T*C
Qu2 = 0.1*np.diag([1, 1])
K, X, E = lqr(A, B, Qx2, Qu2)
K2 = np.matrix(K)

H2x = ss(Ax - Bx*K2[0, lat], Bx*K2[0, lat]*xd[lat, :], Cx, Dx)
H2y = ss(Ay - By*K2[1, alt], By*K2[1, alt]*yd[alt, :], Cy, Dy)

plt.subplot(223)
plt.title("Output weighting")
[Y2x, T2x] = step(H2x, T=np.linspace(0, 10, 100))
[Y2y, T2y] = step(H2y, T=np.linspace(0, 10, 100))
plt.plot(T2x.T, Y2x.T, T2y.T, Y2y.T)
plt.ylabel('position')
plt.xlabel('time')
plt.ylabel('position')
plt.legend(('x', 'y'), loc='lower right')

#
# Physically motivated weighting
#
# Shoot for 1 cm error in x, 10 cm error in y.  Try to keep the angle
# less than 5 degrees in making the adjustments.  Penalize side forces
# due to loss in efficiency.
#

Qx3 = np.diag([100, 10, 2*np.pi/5, 0, 0, 0])
Qu3 = 0.1*np.diag([1, 10])
(K, X, E) = lqr(A, B, Qx3, Qu3)
K3 = np.matrix(K)

H3x = ss(Ax - Bx*K3[0, lat], Bx*K3[0, lat]*xd[lat, :], Cx, Dx)
H3y = ss(Ay - By*K3[1, alt], By*K3[1, alt]*yd[alt, :], Cy, Dy)
plt.subplot(224)
# step(H3x, H3y, 10)
[Y3x, T3x] = step(H3x, T=np.linspace(0, 10, 100))
[Y3y, T3y] = step(H3y, T=np.linspace(0, 10, 100))
plt.plot(T3x.T, Y3x.T, T3y.T, Y3y.T)
plt.title("Physically motivated weights")
plt.xlabel('time')
plt.legend(('x', 'y'), loc='lower right')

if 'PYCONTROL_TEST_EXAMPLES' not in os.environ:
    plt.show()
<commit_msg>Alternative array indexing<commit_after># pvtol_lqr.m - LQR design for vectored thrust aircraft
# RMM, 14 Jan 03
#
# This file works through an LQR based design problem, using the
# planar vertical takeoff and landing (PVTOL) aircraft example from
# Astrom and Murray, Chapter 5.  It is intended to demonstrate the
# basic functionality of the python-control package.
#

import os
import numpy as np
import matplotlib.pyplot as plt  # MATLAB plotting functions
from control.matlab import *  # MATLAB-like functions

#
# System dynamics
#
# These are the dynamics for the PVTOL system, written in state space
# form.
#

# System parameters
m = 4       # mass of aircraft
J = 0.0475  # inertia around pitch axis
r = 0.25    # distance to center of force
g = 9.8     # gravitational constant
c = 0.05    # damping factor (estimated)

# State space dynamics
xe = [0, 0, 0, 0, 0, 0]  # equilibrium point of interest
ue = [0, m*g]  # (note these are lists, not matrices)

# TODO: The following objects need converting from np.matrix to np.array
# This will involve re-working the subsequent equations as the shapes
# See below.

# Dynamics matrix (use matrix type so that * works for multiplication)
A = np.matrix(
    [[0, 0, 0, 1, 0, 0],
     [0, 0, 0, 0, 1, 0],
     [0, 0, 0, 0, 0, 1],
     [0, 0, (-ue[0]*np.sin(xe[2]) - ue[1]*np.cos(xe[2]))/m, -c/m, 0, 0],
     [0, 0, (ue[0]*np.cos(xe[2]) - ue[1]*np.sin(xe[2]))/m, 0, -c/m, 0],
     [0, 0, 0, 0, 0, 0]]
)

# Input matrix
B = np.matrix(
    [[0, 0], [0, 0], [0, 0],
     [np.cos(xe[2])/m, -np.sin(xe[2])/m],
     [np.sin(xe[2])/m, np.cos(xe[2])/m],
     [r/J, 0]]
)

# Output matrix
C = np.matrix([[1, 0, 0, 0, 0, 0], [0, 1, 0, 0, 0, 0]])
D = np.matrix([[0, 0], [0, 0]])

#
# Construct inputs and outputs corresponding to steps in xy position
#
# The vectors xd and yd correspond to the states that are the desired
# equilibrium states for the system.  The matrices Cx and Cy are the
# corresponding outputs.
#
# The way these vectors are used is to compute the closed loop system
# dynamics as
#
#   xdot = Ax + B u  =>  xdot = (A-BK)x + K xd
#      u = -K(x - xd)       y = Cx
#
# The closed loop dynamics can be simulated using the "step" command,
# with K*xd as the input vector (assumes that the "input" is unit size,
# so that xd corresponds to the desired steady state.
#

xd = np.matrix([[1], [0], [0], [0], [0], [0]])
yd = np.matrix([[0], [1], [0], [0], [0], [0]])

#
# Extract the relevant dynamics for use with SISO library
#
# The current python-control library only supports SISO transfer
# functions, so we have to modify some parts of the original MATLAB
# code to extract out SISO systems.  To do this, we define the 'lat' and
# 'alt' index vectors to consist of the states that are are relevant
# to the lateral (x) and vertical (y) dynamics.
#

# Indices for the parts of the state that we want
lat = (0, 2, 3, 5)
alt = (1, 4)

# Decoupled dynamics
Ax = A[np.ix_(lat, lat)]
Bx = B[lat, 0]
Cx = C[0, lat]
Dx = D[0, 0]

Ay = A[np.ix_(alt, alt)]
By = B[alt, 1]
Cy = C[1, alt]
Dy = D[1, 1]

# Label the plot
plt.clf()
plt.suptitle("LQR controllers for vectored thrust aircraft (pvtol-lqr)")

#
# LQR design
#

# Start with a diagonal weighting
Qx1 = np.diag([1, 1, 1, 1, 1, 1])
Qu1a = np.diag([1, 1])
K, X, E = lqr(A, B, Qx1, Qu1a)
K1a = np.matrix(K)

# Close the loop: xdot = Ax - B K (x-xd)
# Note: python-control requires we do this 1 input at a time
# H1a = ss(A-B*K1a, B*K1a*concatenate((xd, yd), axis=1), C, D);
# (T, Y) = step(H1a, T=np.linspace(0,10,100));

# TODO: The following equations will need modifying when converting from np.matrix to np.array
# because the results and even intermediate calculations will be different with numpy arrays
# For example:
# Bx = B[lat, 0]
# Will need to be changed to:
# Bx = B[lat, 0].reshape(-1, 1)
# (if we want it to have the same shape as before)

# For reference, here is a list of the correct shapes of these objects:
# A: (6, 6)
# B: (6, 2)
# C: (2, 6)
# D: (2, 2)
# xd: (6, 1)
# yd: (6, 1)
# Ax: (4, 4)
# Bx: (4, 1)
# Cx: (1, 4)
# Dx: ()
# Ay: (2, 2)
# By: (2, 1)
# Cy: (1, 2)

# Step response for the first input
H1ax = ss(Ax - Bx*K1a[0, lat], Bx*K1a[0, lat]*xd[lat, :], Cx, Dx)
Yx, Tx = step(H1ax, T=np.linspace(0, 10, 100))

# Step response for the second input
H1ay = ss(Ay - By*K1a[1, alt], By*K1a[1, alt]*yd[alt, :], Cy, Dy)
Yy, Ty = step(H1ay, T=np.linspace(0, 10, 100))

plt.subplot(221)
plt.title("Identity weights")
# plt.plot(T, Y[:,1, 1], '-', T, Y[:,2, 2], '--')
plt.plot(Tx.T, Yx.T, '-', Ty.T, Yy.T, '--')
plt.plot([0, 10], [1, 1], 'k-')

plt.axis([0, 10, -0.1, 1.4])
plt.ylabel('position')
plt.legend(('x', 'y'), loc='lower right')

# Look at different input weightings
Qu1a = np.diag([1, 1])
K1a, X, E = lqr(A, B, Qx1, Qu1a)
H1ax = ss(Ax - Bx*K1a[0, lat], Bx*K1a[0, lat]*xd[lat, :], Cx, Dx)

Qu1b = (40 ** 2)*np.diag([1, 1])
K1b, X, E = lqr(A, B, Qx1, Qu1b)
H1bx = ss(Ax - Bx*K1b[0, lat], Bx*K1b[0, lat]*xd[lat, :], Cx, Dx)

Qu1c = (200 ** 2)*np.diag([1, 1])
K1c, X, E = lqr(A, B, Qx1, Qu1c)
H1cx = ss(Ax - Bx*K1c[0, lat], Bx*K1c[0, lat]*xd[lat, :], Cx, Dx)

[Y1, T1] = step(H1ax, T=np.linspace(0, 10, 100))
[Y2, T2] = step(H1bx, T=np.linspace(0, 10, 100))
[Y3, T3] = step(H1cx, T=np.linspace(0, 10, 100))

plt.subplot(222)
plt.title("Effect of input weights")
plt.plot(T1.T, Y1.T, 'b-')
plt.plot(T2.T, Y2.T, 'b-')
plt.plot(T3.T, Y3.T, 'b-')
plt.plot([0, 10], [1, 1], 'k-')

plt.axis([0, 10, -0.1, 1.4])

# arcarrow([1.3, 0.8], [5, 0.45], -6)
plt.text(5.3, 0.4, 'rho')

# Output weighting - change Qx to use outputs
Qx2 = C.T*C
Qu2 = 0.1*np.diag([1, 1])
K, X, E = lqr(A, B, Qx2, Qu2)
K2 = np.matrix(K)

H2x = ss(Ax - Bx*K2[0, lat], Bx*K2[0, lat]*xd[lat, :], Cx, Dx)
H2y = ss(Ay - By*K2[1, alt], By*K2[1, alt]*yd[alt, :], Cy, Dy)

plt.subplot(223)
plt.title("Output weighting")
[Y2x, T2x] = step(H2x, T=np.linspace(0, 10, 100))
[Y2y, T2y] = step(H2y, T=np.linspace(0, 10, 100))
plt.plot(T2x.T, Y2x.T, T2y.T, Y2y.T)
plt.ylabel('position')
plt.xlabel('time')
plt.ylabel('position')
plt.legend(('x', 'y'), loc='lower right')

#
# Physically motivated weighting
#
# Shoot for 1 cm error in x, 10 cm error in y.  Try to keep the angle
# less than 5 degrees in making the adjustments.  Penalize side forces
# due to loss in efficiency.
#

Qx3 = np.diag([100, 10, 2*np.pi/5, 0, 0, 0])
Qu3 = 0.1*np.diag([1, 10])
(K, X, E) = lqr(A, B, Qx3, Qu3)
K3 = np.matrix(K)

H3x = ss(Ax - Bx*K3[0, lat], Bx*K3[0, lat]*xd[lat, :], Cx, Dx)
H3y = ss(Ay - By*K3[1, alt], By*K3[1, alt]*yd[alt, :], Cy, Dy)
plt.subplot(224)
# step(H3x, H3y, 10)
[Y3x, T3x] = step(H3x, T=np.linspace(0, 10, 100))
[Y3y, T3y] = step(H3y, T=np.linspace(0, 10, 100))
plt.plot(T3x.T, Y3x.T, T3y.T, Y3y.T)
plt.title("Physically motivated weights")
plt.xlabel('time')
plt.legend(('x', 'y'), loc='lower right')

if 'PYCONTROL_TEST_EXAMPLES' not in os.environ:
    plt.show()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

from anormbookmarker.test.test_enviroment import *

ed = Word.construct(session=SESSION, word='Eucalyptus deglupta')
SESSION.commit()

assert False #shouldnt get here

db_result = [('select COUNT(*) from alias;', 0),
             ('select COUNT(*) from aliasword;', 0),
             ('select COUNT(*) from bookmark;', 0),
             ('select COUNT(*) from filename;', 0),
             ('select COUNT(*) from tag;', 0),
             ('select COUNT(*) from tag_relationship;', 0),
             ('select COUNT(*) from tagbookmarks;', 0),
             ('select COUNT(*) from tagword;', 0),
             ('select COUNT(*) from word;', 1),
             ('select COUNT(*) from wordmisspelling;', 0)]

check_db_result(config=CONFIG, db_result=db_result, session=SESSION)
<commit_msg>auto-commit<commit_after>#!/usr/bin/env python3

from anormbookmarker.test.test_enviroment import *
from sqlalchemy.exc import IntegrityError

try:
    ed = Word.construct(session=SESSION, word='Eucalyptus deglupta')
    SESSION.commit()
except IntegrityError:
    print("Correctly raises IntegrityError")

db_result = [('select COUNT(*) from alias;', 0),
             ('select COUNT(*) from aliasword;', 0),
             ('select COUNT(*) from bookmark;', 0),
             ('select COUNT(*) from filename;', 0),
             ('select COUNT(*) from tag;', 0),
             ('select COUNT(*) from tag_relationship;', 0),
             ('select COUNT(*) from tagbookmarks;', 0),
             ('select COUNT(*) from tagword;', 0),
             ('select COUNT(*) from word;', 1),
             ('select COUNT(*) from wordmisspelling;', 0)]

check_db_result(config=CONFIG, db_result=db_result, session=SESSION)
<|endoftext|>


<commit_before>import flask

from datetime import datetime  # ugh

from ruddock.resources import Permissions
from ruddock.decorators import login_required, get_args_from_form
from ruddock.modules.budget import blueprint, helpers
from .helpers import PaymentType

@blueprint.route('/')
@login_required(Permissions.BUDGET)
def route_portal():
  """Displays the budget portal."""
  return flask.render_template('budget_portal.html')

@blueprint.route('/summary')
@login_required(Permissions.BUDGET)
def route_summary():
  """Displays account and budget summaries."""

  fyear_dict = {r["fyear_num"]: r["fyear_id"] for r in helpers.get_fyears()}

  fyear_num = flask.request.args.get("fyear", None)

  if fyear_num is not None:
    fyear_id = fyear_dict.get(int(fyear_num), None)
  else:
    fyear_id = helpers.get_current_fyear()["fyear_id"]
    fyear_num = "Current"

  a_summary = helpers.get_account_summary()
  b_summary = helpers.get_budget_summary(fyear_id)

  fyear_options = [
    (num, fyear_dict[num])
    for num in reversed(sorted(fyear_dict.keys()))
  ]

  return flask.render_template('summary.html',
    a_summary=a_summary,
    b_summary=b_summary,
    fyear_num=fyear_num,
    fyear_options=fyear_options)

@blueprint.route('/expenses')
@login_required(Permissions.BUDGET)
def route_expenses():
  """Displays list of expenses."""
  return flask.render_template('expenses.html',
    expenses=helpers.get_transactions(),
    ptypes=helpers.get_payment_types())

@blueprint.route('/payments')
@login_required(Permissions.BUDGET)
def route_payments():
  """Displays list of payments."""
  return flask.render_template('payments.html',
    payments=helpers.get_payments(),
    ptypes=helpers.get_payment_types())

@blueprint.route('/add_expense')
@login_required(Permissions.BUDGET)
def route_add_expense():
  """Provides an interface for submitting an expense."""

  # TODO allow for multiple years

  # Get the lists for the dropdown menus
  current_fyear_id = helpers.get_current_fyear()["fyear_id"]
  budgets_list = helpers.get_budget_list(current_fyear_id)
  payment_types = helpers.get_payment_types()
  accounts = helpers.get_accounts()
  payees = helpers.get_payees()

  return flask.render_template('add_expense.html',
    budgets=budgets_list,
    payment_types=payment_types,
    accounts=accounts,
    payees=payees)

@blueprint.route('/add_expense/submit', methods=['POST'])
@login_required(Permissions.BUDGET)
@get_args_from_form()
def route_submit_expense(budget_id, date_incurred, amount, description,
    payee_id, new_payee, payment_type, account_id, check_no, defer_payment):
  """Sends the expense to the database."""

  # Checkboxes aren't sent as bools... :(
  defer_payment = defer_payment is not None

  # Server side validation
  valid_expense = helpers.validate_expense(budget_id, date_incurred, amount,
      description)
  valid_payment = helpers.validate_payment(payment_type,
      account_id, check_no)
  valid_payee = helpers.validate_payee(payee_id, new_payee)

  errs = helpers.test_predicates((
    (valid_expense, True,              "Invalid expense."),
    (valid_payment, not defer_payment, "Invalid payment."),
    (valid_payee,   defer_payment,     "Invalid payee."  )
  ))

  if errs:
    return flask.redirect(flask.url_for("budget.route_add_expense"))

  transaction = flask.g.db.begin()
  try:
    # This next part depends on whether we are deferring payment or not.
    # If so, then we leave the payment ID null, cause it the corresponding
    # payment hasn't been made yet. However, we require the payee. If it's a new
    # payee, we have to insert it into the database first.
    # If not, then we need to make a new payment, and use its ID. Payee ID is
    # not needed.
    if defer_payment:
      payee_id = payee_id or helpers.record_new_payee(new_payee)
      payment_id = None
    else:
      date_written = date_incurred  # non-deferred payments are instant
      date_posted = None if payment_type == PaymentType.CHECK else date_written

      payee_id = None
      payment_id = helpers.record_payment(
        account_id, payment_type, amount, date_written, date_posted,
        payee_id, check_no)

    # Either way, record the expense
    helpers.record_expense(
        budget_id, date_incurred, description, amount, payment_id, payee_id)
    transaction.commit()
    flask.flash("Expense recorded successfully!")

  except Exception:
      transaction.rollback()
      flask.flash("An unexpected error occurred. Please find an IMSS rep.")

  return flask.redirect(flask.url_for("budget.route_add_expense"))

@blueprint.route('/unpaid')
@login_required(Permissions.BUDGET)
def route_unpaid():
  """
  Displays unpaid expenses, and allows the user to create payments for them.
  """

  payment_types = helpers.get_payment_types()
  accounts = helpers.get_accounts()
  expense_groups=helpers.get_unpaid_expenses()
  today = datetime.now().strftime("%Y-%m-%d")

  return flask.render_template('unpaid.html',
    payment_types=payment_types,
    accounts=accounts,
    expense_groups=expense_groups,
    today=today)

@blueprint.route('/unpaid/submit', methods=['POST'])
@login_required(Permissions.BUDGET)
@get_args_from_form()
def route_submit_unpaid(payee_id, payment_type, account_id, check_no,
    date_written):
  """Sends the payment to the database."""

  # The date posted is the same as the date written unless we're using a check
  date_posted = None if (payment_type != PaymentType.CHECK) else date_written

  # Server side validation
  total = helpers.get_unpaid_amount(payee_id)
  valid_payment = helpers.validate_payment(payment_type, account_id, check_no)
  has_expenses = (total is not None)

  errs = helpers.test_predicates((
    (valid_payment, True, "Invalid payment."),
    (has_expenses,  True, "This payee has no expenses to reimburse."),
  ))

  if errs:
    return flask.redirect(flask.url_for("budget.route_unpaid"))

  # We use a transaction to make sure we don't submit halfway.
  transaction = flask.g.db.begin()
  try:
    payment_id = helpers.record_payment(
        account_id, payment_type, total, date_written, date_posted,
        payee_id, check_no)
    helpers.mark_as_paid(payee_id, payment_id)
    transaction.commit()
    flask.flash("Payment recorded successfully!")

  except Exception:
    transaction.rollback()
    flask.flash("An unexpected error occurred. Please find an IMSS rep.")

  return flask.redirect(flask.url_for("budget.route_unpaid"))

@blueprint.route('/checks')
@login_required(Permissions.BUDGET)
def route_checks():
  """Displays all undeposited checks."""
  return flask.render_template('checks.html',
    checks=helpers.get_unposted_payments(),
    ptypes=helpers.get_payment_types())

@blueprint.route('/checks/submit', methods=['POST'])
@login_required(Permissions.BUDGET)
@get_args_from_form()
def route_process_check(payment_id, date_posted, action):
  """Records a check as deposited."""

  # Server side validation
  unposted_ids = [str(x["payment_id"]) for x in helpers.get_unposted_payments()]

  errs = helpers.test_predicates((
    (payment_id in unposted_ids, True,             "Not a valid payment ID!"),
    (date_posted != "",          action == "Post", "No date entered!"       )
  ))

  # If any of the validation failed
  if errs:
    return flask.redirect(flask.url_for("budget.route_checks"))

  # Decide what to do
  if action == "Post":
    helpers.post_payment(payment_id, date_posted)
    flask.flash("Payment successfully posted!")
  elif action == "Void":
    helpers.void_payment(payment_id)
    flask.flash("Payment successfully voided!")
  else:
    flask.flash("Not a legitimate action!")

  return flask.redirect(flask.url_for("budget.route_checks"))
<commit_msg>Change text label in add expense<commit_after>import flask

from datetime import datetime  # ugh

from ruddock.resources import Permissions
from ruddock.decorators import login_required, get_args_from_form
from ruddock.modules.budget import blueprint, helpers
from .helpers import PaymentType

@blueprint.route('/')
@login_required(Permissions.BUDGET)
def route_portal():
  """Displays the budget portal."""
  return flask.render_template('budget_portal.html')

@blueprint.route('/summary')
@login_required(Permissions.BUDGET)
def route_summary():
  """Displays account and budget summaries."""

  fyear_dict = {r["fyear_num"]: r["fyear_id"] for r in helpers.get_fyears()}

  fyear_num = flask.request.args.get("fyear", None)

  if fyear_num is not None:
    fyear_id = fyear_dict.get(int(fyear_num), None)
  else:
    fyear_id = helpers.get_current_fyear()["fyear_id"]
    fyear_num = "Current"

  a_summary = helpers.get_account_summary()
  b_summary = helpers.get_budget_summary(fyear_id)

  fyear_options = [
    (num, fyear_dict[num])
    for num in reversed(sorted(fyear_dict.keys()))
  ]

  return flask.render_template('summary.html',
    a_summary=a_summary,
    b_summary=b_summary,
    fyear_num=fyear_num,
    fyear_options=fyear_options)

@blueprint.route('/expenses')
@login_required(Permissions.BUDGET)
def route_expenses():
  """Displays list of expenses."""
  return flask.render_template('expenses.html',
    expenses=helpers.get_transactions(),
    ptypes=helpers.get_payment_types())

@blueprint.route('/payments')
@login_required(Permissions.BUDGET)
def route_payments():
  """Displays list of payments."""
  return flask.render_template('payments.html',
    payments=helpers.get_payments(),
    ptypes=helpers.get_payment_types())

@blueprint.route('/add_expense')
@login_required(Permissions.BUDGET)
def route_add_expense():
  """Provides an interface for submitting an expense."""

  # TODO allow for multiple years

  # Get the lists for the dropdown menus
  current_fyear = helpers.get_current_fyear()
  budgets_list = helpers.get_budget_list(current_fyear["fyear_id"])
  payment_types = helpers.get_payment_types()
  accounts = helpers.get_accounts()
  payees = helpers.get_payees()

  return flask.render_template('add_expense.html',
    fyear_num=current_fyear["fyear_num"],
    budgets=budgets_list,
    payment_types=payment_types,
    accounts=accounts,
    payees=payees)

@blueprint.route('/add_expense/submit', methods=['POST'])
@login_required(Permissions.BUDGET)
@get_args_from_form()
def route_submit_expense(budget_id, date_incurred, amount, description,
    payee_id, new_payee, payment_type, account_id, check_no, defer_payment):
  """Sends the expense to the database."""

  # Checkboxes aren't sent as bools... :(
  defer_payment = defer_payment is not None

  # Server side validation
  valid_expense = helpers.validate_expense(budget_id, date_incurred, amount,
      description)
  valid_payment = helpers.validate_payment(payment_type,
      account_id, check_no)
  valid_payee = helpers.validate_payee(payee_id, new_payee)

  errs = helpers.test_predicates((
    (valid_expense, True,              "Invalid expense."),
    (valid_payment, not defer_payment, "Invalid payment."),
    (valid_payee,   defer_payment,     "Invalid payee."  )
  ))

  if errs:
    return flask.redirect(flask.url_for("budget.route_add_expense"))

  transaction = flask.g.db.begin()
  try:
    # This next part depends on whether we are deferring payment or not.
    # If so, then we leave the payment ID null, cause it the corresponding
    # payment hasn't been made yet. However, we require the payee. If it's a new
    # payee, we have to insert it into the database first.
    # If not, then we need to make a new payment, and use its ID. Payee ID is
    # not needed.
    if defer_payment:
      payee_id = payee_id or helpers.record_new_payee(new_payee)
      payment_id = None
    else:
      date_written = date_incurred  # non-deferred payments are instant
      date_posted = None if payment_type == PaymentType.CHECK else date_written

      payee_id = None
      payment_id = helpers.record_payment(
        account_id, payment_type, amount, date_written, date_posted,
        payee_id, check_no)

    # Either way, record the expense
    helpers.record_expense(
        budget_id, date_incurred, description, amount, payment_id, payee_id)
    transaction.commit()
    flask.flash("Expense recorded successfully!")

  except Exception:
      transaction.rollback()
      flask.flash("An unexpected error occurred. Please find an IMSS rep.")

  return flask.redirect(flask.url_for("budget.route_add_expense"))

@blueprint.route('/unpaid')
@login_required(Permissions.BUDGET)
def route_unpaid():
  """
  Displays unpaid expenses, and allows the user to create payments for them.
  """

  payment_types = helpers.get_payment_types()
  accounts = helpers.get_accounts()
  expense_groups=helpers.get_unpaid_expenses()
  today = datetime.now().strftime("%Y-%m-%d")

  return flask.render_template('unpaid.html',
    payment_types=payment_types,
    accounts=accounts,
    expense_groups=expense_groups,
    today=today)

@blueprint.route('/unpaid/submit', methods=['POST'])
@login_required(Permissions.BUDGET)
@get_args_from_form()
def route_submit_unpaid(payee_id, payment_type, account_id, check_no,
    date_written):
  """Sends the payment to the database."""

  # The date posted is the same as the date written unless we're using a check
  date_posted = None if (payment_type != PaymentType.CHECK) else date_written

  # Server side validation
  total = helpers.get_unpaid_amount(payee_id)
  valid_payment = helpers.validate_payment(payment_type, account_id, check_no)
  has_expenses = (total is not None)

  errs = helpers.test_predicates((
    (valid_payment, True, "Invalid payment."),
    (has_expenses,  True, "This payee has no expenses to reimburse."),
  ))

  if errs:
    return flask.redirect(flask.url_for("budget.route_unpaid"))

  # We use a transaction to make sure we don't submit halfway.
  transaction = flask.g.db.begin()
  try:
    payment_id = helpers.record_payment(
        account_id, payment_type, total, date_written, date_posted,
        payee_id, check_no)
    helpers.mark_as_paid(payee_id, payment_id)
    transaction.commit()
    flask.flash("Payment recorded successfully!")

  except Exception:
    transaction.rollback()
    flask.flash("An unexpected error occurred. Please find an IMSS rep.")

  return flask.redirect(flask.url_for("budget.route_unpaid"))

@blueprint.route('/checks')
@login_required(Permissions.BUDGET)
def route_checks():
  """Displays all undeposited checks."""
  return flask.render_template('checks.html',
    checks=helpers.get_unposted_payments(),
    ptypes=helpers.get_payment_types())

@blueprint.route('/checks/submit', methods=['POST'])
@login_required(Permissions.BUDGET)
@get_args_from_form()
def route_process_check(payment_id, date_posted, action):
  """Records a check as deposited."""

  # Server side validation
  unposted_ids = [str(x["payment_id"]) for x in helpers.get_unposted_payments()]

  errs = helpers.test_predicates((
    (payment_id in unposted_ids, True,             "Not a valid payment ID!"),
    (date_posted != "",          action == "Post", "No date entered!"       )
  ))

  # If any of the validation failed
  if errs:
    return flask.redirect(flask.url_for("budget.route_checks"))

  # Decide what to do
  if action == "Post":
    helpers.post_payment(payment_id, date_posted)
    flask.flash("Payment successfully posted!")
  elif action == "Void":
    helpers.void_payment(payment_id)
    flask.flash("Payment successfully voided!")
  else:
    flask.flash("Not a legitimate action!")

  return flask.redirect(flask.url_for("budget.route_checks"))
<|endoftext|>