<commit_before>#!/usr/bin/env python

from subprocess import call
import sys
import subprocess
import dbus
import string
import os
import fcntl
import time
import pexpect
import glib
import gobject
import dbus.service
import dbus.mainloop.glib

DBUS_NAME = 'org.openbmc.UserManager'
INTF_NAME = 'org.openbmc.Enrol'
OBJ_NAME_GROUPS = '/org/openbmc/UserManager/Groups'
OBJ_NAME_GROUP = '/org/openbmc/UserManager/Group'
OBJ_NAME_USERS = '/org/openbmc/UserManager/Users'
OBJ_NAME_USER = '/org/openbmc/UserManager/User'

'''
    Object Path > /org/openbmc/UserManager/Groups
        Interface:Method > org.openbmc.Enrol.GroupAddSys string:"groupname"
        Interface:Method > org.openbmc.Enrol.GroupAddUsr string:"groupname"
        Interface:Method > org.openbmc.Enrol.GroupListUsr
        Interface:Method > org.openbmc.Enrol.GroupListSys
    Object Path > /org/openbmc/UserManager/Group
        Interface:Method > org.openbmc.Enrol.GroupDel string:"groupname"
    Object Path > /org/openbmc/UserManager/Users
       Interface:Method > org.openbmc.Enrol.UserAdd string:"comment" string:"username" string:"groupname" string:"passwd"
        Interface:Method > org.openbmc.Enrol.UserList
    Object Path > /org/openbmc/UserManager/User
        Interface:Method > org.openbmc.Enrol.UserDel string:"username"
        Interface:Method > org.openbmc.Enrol.Passswd string:"username" string:"passwd"
'''

userman_providers = {
    'pam' : {
        'adduser' : 'user add',
    },
    'ldap' : {
        'adduser' : 'ldap command to add user',
    },
}

class UserManGroups (dbus.service.Object):
    def __init__(self, bus, name):
        self.bus = bus
        self.name = name
        dbus.service.Object.__init__(self,bus,name)

    def setUsermanProvider(self, provider):
        self.provider = provider

    @dbus.service.method(INTF_NAME, "", "")
    def test(self):
        print("TEST")

    @dbus.service.method(INTF_NAME, "s", "x")
    def GroupAddUsr (self, groupname):
        if not groupname : raise ValueError("Invalid Groupname")

        groups = self.GroupListAll ()
        if groupname in groups: raise ValueError("Group ", groupname, " Exists")

        r = call (["addgroup", groupname])
        return r

    #@dbus.service.method(INTF_NAME, "s", "x")
    def GroupAddSys (self, groupname):
        if not groupname : raise ValueError("Invalid Groupname")

        groups = self.GroupListAll ()
        if groupname in groups: raise ValueError("Group ", groupname, " Exists")

        r = call (["addgroup", "-S", groupname])
        return r

    @dbus.service.method(INTF_NAME, "", "as")
    def GroupListUsr (self):
        groupList = []
        with open("/etc/group", "r") as f:
            for grent in f:
                groupParams = grent.split (":")
                if (int(groupParams[2]) >= 1000 and int(groupParams[2]) != 65534):
                    groupList.append(groupParams[0])
        return groupList

    @dbus.service.method(INTF_NAME, "", "as")
    def GroupListSys (self):
        groupList = []
        with open("/etc/group", "r") as f:
            for grent in f:
                groupParams = grent.split (":")
                if (int(groupParams[2]) > 100 and int(groupParams[2]) < 1000): groupList.append(groupParams[0])
        return groupList

    def GroupListAll (self):
        groupList = []
        with open("/etc/group", "r") as f:
            for grent in f:
                groupParams = grent.split (":")
                groupList.append(groupParams[0])
        return groupList

class UserManGroup (dbus.service.Object):
    def __init__(self, bus, name):
        self.bus = bus
        self.name = name
        dbus.service.Object.__init__(self,bus,name)

    def setUsermanProvider(self, provider):
        self.provider = provider

    @dbus.service.method(INTF_NAME, "", "")
    def test(self):
        print("TEST")

    @dbus.service.method(INTF_NAME, "", "x")
    def GroupDel (self, groupname):
        if not groupname : raise ValueError("Invalid Groupname")

        groups = Groupsobj.GroupListAll ()
        if groupname not in groups: raise ValueError("No such Group: ", groupname)

        r = call (["delgroup", groupname])
        return r

class UserManUsers (dbus.service.Object):
    def __init__(self, bus, name):
        self.bus = bus
        self.name = name
        dbus.service.Object.__init__(self,bus,name)

    def setUsermanProvider(self, provider):
        self.provider = provider

    @dbus.service.method(INTF_NAME, "", "")
    def test(self):
        print("TEST")

    @dbus.service.method(INTF_NAME, "ssss", "x")
    def UserAdd (self, gecos, username, groupname, passwd):
        if not username : raise ValueError("Invalid Username")

        users = self.UserListAll ()
        if username in users : raise ValueError("User ", username, " Exists")

        if groupname:
            groups = Groupsobj.GroupListAll ()
            if groupname not in groups: raise ValueError("No such Group: ", groupname)

        opts = ""
        if gecos: opts = " -g " + '"' + gecos + '"'

        if groupname:
            cmd = "adduser "  + opts + " " + " -G " + groupname + " " + username
        else:
            cmd = "adduser "  + opts + " " + username

        prompts = ['New password: ', 'Retype password: ', 'Re-enter new password: ']
        proc = pexpect.spawn (cmd)
        proc.expect (prompts)
        proc.sendline (passwd)
        proc.expect (prompts)
        proc.sendline (passwd)

        if proc.expect(prompts + [pexpect.EOF]) != len(prompts):
            proc.sendline (passwd)

        r = proc.wait()
        return r if r else 0

    @dbus.service.method(INTF_NAME, "", "as")
    def UserList (self):
        userList = []
        with open("/etc/passwd", "r") as f:
            for usent in f:
                userParams = usent.split (":")
                if (int(userParams[2]) >= 1000 and int(userParams[2]) != 65534):
                    userList.append(userParams[0])
        return userList

    def UserListAll (self):
        userList = []
        with open("/etc/passwd", "r") as f:
            for usent in f:
                userParams = usent.split (":")
                userList.append(userParams[0])
        return userList

class UserManUser (dbus.service.Object):
    def __init__(self, bus, name):
        self.bus = bus
        self.name = name
        dbus.service.Object.__init__(self,bus,name)

    @dbus.service.method(INTF_NAME, "", "")
    def test(self):
        print("TEST")

    def setUsermanProvider(self, provider):
        self.provider = provider

    @dbus.service.method(INTF_NAME, "s", "x")
    def UserDel (self, username):
        if not username : raise ValueError("Invalid Username")

        users = Usersobj.UserList ()
        if username not in users : raise ValueError("No such User: ", username)

        r = call (["deluser", username])
        return r

    @dbus.service.method(INTF_NAME, "ss", "x")
    def Passwd (self, username, passwd):
        if not username : raise ValueError("Invalid Username")

        users = Usersobj.UserList ()
        if username not in users : raise ValueError("No such User: ", username)

        cmd = "passwd" + " " + username
        proc = pexpect.spawn (cmd)
        proc.expect (['New password: ', 'Retype password: '])
        proc.sendline (passwd)
        proc.expect (['New password: ', 'Retype password: '])
        proc.sendline (passwd)

        proc.wait()
        return 0

def main():
    dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
    bus = dbus.SystemBus()
    name = dbus.service.BusName(DBUS_NAME, bus)

    global Groupsobj
    global Groupobj
    global Usersobj
    global Userobj

    Groupsobj   = UserManGroups (bus, OBJ_NAME_GROUPS)
    Groupobj    = UserManGroup  (bus, OBJ_NAME_GROUP)
    Usersobj    = UserManUsers  (bus, OBJ_NAME_USERS)
    Userobj     = UserManUser   (bus, OBJ_NAME_USER)

    Groupsobj.setUsermanProvider ("pam")
    Groupobj.setUsermanProvider ("pam")
    Usersobj.setUsermanProvider ("pam")
    Userobj.setUsermanProvider ("pam")

    mainloop = gobject.MainLoop()
    print("Started")
    mainloop.run()

if __name__ == '__main__':
    sys.exit(main())

<commit_msg>userman interface to add user and change password pexpect fix<commit_after>#!/usr/bin/env python

from subprocess import call
import sys
import subprocess
import dbus
import string
import os
import fcntl
import time
import pexpect
import glib
import gobject
import dbus.service
import dbus.mainloop.glib

DBUS_NAME = 'org.openbmc.UserManager'
INTF_NAME = 'org.openbmc.Enrol'
OBJ_NAME_GROUPS = '/org/openbmc/UserManager/Groups'
OBJ_NAME_GROUP = '/org/openbmc/UserManager/Group'
OBJ_NAME_USERS = '/org/openbmc/UserManager/Users'
OBJ_NAME_USER = '/org/openbmc/UserManager/User'

'''
    Object Path > /org/openbmc/UserManager/Groups
        Interface:Method > org.openbmc.Enrol.GroupAddSys string:"groupname"
        Interface:Method > org.openbmc.Enrol.GroupAddUsr string:"groupname"
        Interface:Method > org.openbmc.Enrol.GroupListUsr
        Interface:Method > org.openbmc.Enrol.GroupListSys
    Object Path > /org/openbmc/UserManager/Group
        Interface:Method > org.openbmc.Enrol.GroupDel string:"groupname"
    Object Path > /org/openbmc/UserManager/Users
       Interface:Method > org.openbmc.Enrol.UserAdd string:"comment" string:"username" string:"groupname" string:"passwd"
        Interface:Method > org.openbmc.Enrol.UserList
    Object Path > /org/openbmc/UserManager/User
        Interface:Method > org.openbmc.Enrol.UserDel string:"username"
        Interface:Method > org.openbmc.Enrol.Passswd string:"username" string:"passwd"
'''

userman_providers = {
    'pam' : {
        'adduser' : 'user add',
    },
    'ldap' : {
        'adduser' : 'ldap command to add user',
    },
}

class UserManGroups (dbus.service.Object):
    def __init__(self, bus, name):
        self.bus = bus
        self.name = name
        dbus.service.Object.__init__(self,bus,name)

    def setUsermanProvider(self, provider):
        self.provider = provider

    @dbus.service.method(INTF_NAME, "", "")
    def test(self):
        print("TEST")

    @dbus.service.method(INTF_NAME, "s", "x")
    def GroupAddUsr (self, groupname):
        if not groupname : raise ValueError("Invalid Groupname")

        groups = self.GroupListAll ()
        if groupname in groups: raise ValueError("Group ", groupname, " Exists")

        r = call (["addgroup", groupname])
        return r

    #@dbus.service.method(INTF_NAME, "s", "x")
    def GroupAddSys (self, groupname):
        if not groupname : raise ValueError("Invalid Groupname")

        groups = self.GroupListAll ()
        if groupname in groups: raise ValueError("Group ", groupname, " Exists")

        r = call (["addgroup", "-S", groupname])
        return r

    @dbus.service.method(INTF_NAME, "", "as")
    def GroupListUsr (self):
        groupList = []
        with open("/etc/group", "r") as f:
            for grent in f:
                groupParams = grent.split (":")
                if (int(groupParams[2]) >= 1000 and int(groupParams[2]) != 65534):
                    groupList.append(groupParams[0])
        return groupList

    @dbus.service.method(INTF_NAME, "", "as")
    def GroupListSys (self):
        groupList = []
        with open("/etc/group", "r") as f:
            for grent in f:
                groupParams = grent.split (":")
                if (int(groupParams[2]) > 100 and int(groupParams[2]) < 1000): groupList.append(groupParams[0])
        return groupList

    def GroupListAll (self):
        groupList = []
        with open("/etc/group", "r") as f:
            for grent in f:
                groupParams = grent.split (":")
                groupList.append(groupParams[0])
        return groupList

class UserManGroup (dbus.service.Object):
    def __init__(self, bus, name):
        self.bus = bus
        self.name = name
        dbus.service.Object.__init__(self,bus,name)

    def setUsermanProvider(self, provider):
        self.provider = provider

    @dbus.service.method(INTF_NAME, "", "")
    def test(self):
        print("TEST")

    @dbus.service.method(INTF_NAME, "", "x")
    def GroupDel (self, groupname):
        if not groupname : raise ValueError("Invalid Groupname")

        groups = Groupsobj.GroupListAll ()
        if groupname not in groups: raise ValueError("No such Group: ", groupname)

        r = call (["delgroup", groupname])
        return r

class UserManUsers (dbus.service.Object):
    def __init__(self, bus, name):
        self.bus = bus
        self.name = name
        dbus.service.Object.__init__(self,bus,name)

    def setUsermanProvider(self, provider):
        self.provider = provider

    @dbus.service.method(INTF_NAME, "", "")
    def test(self):
        print("TEST")

    @dbus.service.method(INTF_NAME, "ssss", "x")
    def UserAdd (self, gecos, username, groupname, passwd):
        if not username : raise ValueError("Invalid Username")

        users = self.UserListAll ()
        if username in users : raise ValueError("User ", username, " Exists")

        if groupname:
            groups = Groupsobj.GroupListAll ()
            if groupname not in groups: raise ValueError("No such Group: ", groupname)

        opts = ""
        if gecos: opts = " -g " + '"' + gecos + '"'

        if groupname:
            cmd = "adduser "  + opts + " " + " -G " + groupname + " " + "-s /bin/sh" + " " + username
        else:
            cmd = "adduser "  + opts + " " + "-s /bin/sh" + " " + username

        prompts = ['New password: ', 'Retype password: ', 'Re-enter new password: ']
        proc = pexpect.spawn (cmd)
        proc.expect (prompts)
        proc.sendline (passwd)
        proc.expect (prompts)
        proc.sendline (passwd)

        if proc.expect(prompts + [pexpect.EOF]) != len(prompts):
            proc.sendline (passwd)

        r = proc.wait()
        return r if r else 0

    @dbus.service.method(INTF_NAME, "", "as")
    def UserList (self):
        userList = []
        with open("/etc/passwd", "r") as f:
            for usent in f:
                userParams = usent.split (":")
                if (int(userParams[2]) >= 1000 and int(userParams[2]) != 65534):
                    userList.append(userParams[0])
        return userList

    def UserListAll (self):
        userList = []
        with open("/etc/passwd", "r") as f:
            for usent in f:
                userParams = usent.split (":")
                userList.append(userParams[0])
        return userList

class UserManUser (dbus.service.Object):
    def __init__(self, bus, name):
        self.bus = bus
        self.name = name
        dbus.service.Object.__init__(self,bus,name)

    @dbus.service.method(INTF_NAME, "", "")
    def test(self):
        print("TEST")

    def setUsermanProvider(self, provider):
        self.provider = provider

    @dbus.service.method(INTF_NAME, "s", "x")
    def UserDel (self, username):
        if not username : raise ValueError("Invalid Username")

        users = Usersobj.UserList ()
        if username not in users : raise ValueError("No such User: ", username)

        r = call (["deluser", username])
        return r

    @dbus.service.method(INTF_NAME, "ss", "x")
    def Passwd (self, username, passwd):
        if not username : raise ValueError("Invalid Username")

        users = Usersobj.UserList ()
        if username not in users : raise ValueError("No such User: ", username)

        cmd = "passwd" + " " + username
        prompts = ['New password: ', 'Retype password: ', 'Re-enter new password: ']
        proc = pexpect.spawn (cmd)
        proc.expect (prompts)
        proc.sendline (passwd)
        proc.expect (prompts)
        proc.sendline (passwd)

        if proc.expect(prompts + [pexpect.EOF]) != len(prompts):
            proc.sendline (passwd)

        r = proc.wait()
        return r if r else 0

def main():
    dbus.mainloop.glib.DBusGMainLoop(set_as_default=True)
    bus = dbus.SystemBus()
    name = dbus.service.BusName(DBUS_NAME, bus)

    global Groupsobj
    global Groupobj
    global Usersobj
    global Userobj

    Groupsobj   = UserManGroups (bus, OBJ_NAME_GROUPS)
    Groupobj    = UserManGroup  (bus, OBJ_NAME_GROUP)
    Usersobj    = UserManUsers  (bus, OBJ_NAME_USERS)
    Userobj     = UserManUser   (bus, OBJ_NAME_USER)

    Groupsobj.setUsermanProvider ("pam")
    Groupobj.setUsermanProvider ("pam")
    Usersobj.setUsermanProvider ("pam")
    Userobj.setUsermanProvider ("pam")

    mainloop = gobject.MainLoop()
    print("Started")
    mainloop.run()

if __name__ == '__main__':
    sys.exit(main())

<|endoftext|>


<commit_before>#
# uwhoisd
# by Keith Gaughan <http://stereochro.me/>
#
# Copyright (c) Keith Gaughan, 2012
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#

"""
A 'universal' WHOIS proxy server.
"""

import diesel
import re
import os.path
import sys
from ConfigParser import SafeConfigParser

__version__ = '0.0.1'
__author__ = 'Keith Gaughan'
__email__ = 'k@stereochro.me'

USAGE = "Usage: %s <config>"

def split_fqdn(fqdn):
    """
    Splits an FQDN into the domain name and zone.

    >>> split_fqdn('stereochro.me')
    ['stereochro', 'me']
    >>> split_fqdn('stereochro.me.')
    ['stereochro', 'me']
    >>> split_fqdn('stereochrome')
    ['stereochrome']
    >>> split_fqdn('keithgaughan.co.uk')
    ['keithgaughan', 'co.uk']
    """
    return fqdn.rstrip('.').split('.', 1)

def get_whois_server(suffix, overrides, zone):
    """
    Returns the WHOIS server hostname for a given zone.
    """
    return overrides[zone] if zone in overrides else zone + '.' + suffix

class WhoisClient(diesel.Client):
    """
    A WHOIS client for diesel.
    """

    __slots__ = ()

    @diesel.call
    def whois(self, query):
        """
        Perform a query against the server. Returns either the server's
        response, or `None` if the connection timed out.
        """
        diesel.send(query + "\r\n")
        result = []
        try:
            while True:
                evt, data = diesel.first(sleep=5, receive=8192)
                if evt == 'sleep':
                    return None
                result.append(data)
        except diesel.ConnectionClosed, ex:
            if ex.buffer:
                result.append(ex.buffer)
        return ''.join(result)

def whois(server, query):
    """
    Helper function for using `WhoisClient`.
    """
    with WhoisClient(server, 43) as client:
        return client.whois(query)

def read_config(parser):
    """
    Extract the configuration from a parsed configuration file, checking that
    the data contained within is valid.
    """
    # Ensure all sections are present, even if empty.
    for section in ('uwhoisd', 'overrides', 'prefixes', 'recursion_patterns'):
        if not parser.has_section(section):
            parser.add_section(section)

    def safe_get(section, option, default):
        """Get a configuration option safely."""
        if parser.has_option(section, option):
            return parser.get(section, option)
        return default

    iface = safe_get('uwhoisd', 'iface', '0.0.0.0')
    port = int(safe_get('uwhoisd', 'port', 4243))
    suffix = safe_get('uwhoisd', 'suffix', 'whois-servers.net')

    # TODO: Check all hostnames and zone names are well-formed.
    overrides = dict(parser.items('overrides'))
    prefixes = dict(parser.items('prefixes'))

    # We could use a compilation cache for these things, but the re module does
    # that anyway, so why bother.
    recursion_patterns = {}
    for zone, pattern in parser.items('recursion_patterns'):
        recursion_patterns[zone] = re.compile(pattern)

    return (iface, port, suffix, overrides, prefixes, recursion_patterns)

class Responder(object):

    __slots__ = ('suffix', 'overrides', 'prefixes', 'recursion_patterns')

    def __init__(self, suffix, overrides, prefixes, recursion_patterns):
        super(Responder, self).__init__()
        self.suffix = suffix
        self.overrides = overrides
        self.prefixes = prefixes
        self.recursion_patterns = recursion_patterns

    def respond(self, _addr):
        query = diesel.until_eol()
        diesel.send("You sent: [%s]\n" % query.strip())

def main():
    """
    Execute the daemon.
    """
    if len(sys.argv) != 2:
        print >> sys.stderr, USAGE % os.path.basename(sys.argv[0])
        return 1

    parser = SafeConfigParser(allow_no_value=True)
    parser.read(sys.argv[1])
    iface, port, suffix, overrides, prefixes, recursion_patterns = \
        read_config(parser)

    responder = Responder(suffix, overrides, prefixes, recursion_patterns)
    diesel.quickstart(diesel.Service(responder.respond, port, iface))

if __name__ == '__main__':
    main()
<commit_msg>Proper implementation of the request responder.<commit_after>#
# uwhoisd
# by Keith Gaughan <http://stereochro.me/>
#
# Copyright (c) Keith Gaughan, 2012
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
#

"""
A 'universal' WHOIS proxy server.
"""

import diesel
import re
import os.path
import sys
from ConfigParser import SafeConfigParser

__version__ = '0.0.1'
__author__ = 'Keith Gaughan'
__email__ = 'k@stereochro.me'

USAGE = "Usage: %s <config>"

def split_fqdn(fqdn):
    """
    Splits an FQDN into the domain name and zone.

    >>> split_fqdn('stereochro.me')
    ['stereochro', 'me']
    >>> split_fqdn('stereochro.me.')
    ['stereochro', 'me']
    >>> split_fqdn('stereochrome')
    ['stereochrome']
    >>> split_fqdn('keithgaughan.co.uk')
    ['keithgaughan', 'co.uk']
    """
    return fqdn.rstrip('.').split('.', 1)

def get_whois_server(suffix, overrides, zone):
    """
    Returns the WHOIS server hostname for a given zone.
    """
    return overrides[zone] if zone in overrides else zone + '.' + suffix

class WhoisClient(diesel.Client):
    """
    A WHOIS client for diesel.
    """

    __slots__ = ()

    @diesel.call
    def whois(self, query):
        """
        Perform a query against the server. Returns either the server's
        response, or `None` if the connection timed out.
        """
        diesel.send(query + "\r\n")
        result = []
        try:
            while True:
                evt, data = diesel.first(sleep=5, receive=8192)
                if evt == 'sleep':
                    return None
                result.append(data)
        except diesel.ConnectionClosed, ex:
            if ex.buffer:
                result.append(ex.buffer)
        return ''.join(result)

def whois(server, query):
    """
    Helper function for using `WhoisClient`.
    """
    with WhoisClient(server, 43) as client:
        return client.whois(query)

def read_config(parser):
    """
    Extract the configuration from a parsed configuration file, checking that
    the data contained within is valid.
    """
    # Ensure all sections are present, even if empty.
    for section in ('uwhoisd', 'overrides', 'prefixes', 'recursion_patterns'):
        if not parser.has_section(section):
            parser.add_section(section)

    def safe_get(section, option, default):
        """Get a configuration option safely."""
        if parser.has_option(section, option):
            return parser.get(section, option)
        return default

    iface = safe_get('uwhoisd', 'iface', '0.0.0.0')
    port = int(safe_get('uwhoisd', 'port', 4243))
    suffix = safe_get('uwhoisd', 'suffix', 'whois-servers.net')

    # TODO: Check all hostnames and zone names are well-formed.
    overrides = dict(parser.items('overrides'))
    prefixes = dict(parser.items('prefixes'))

    # We could use a compilation cache for these things, but the re module does
    # that anyway, so why bother.
    recursion_patterns = {}
    for zone, pattern in parser.items('recursion_patterns'):
        recursion_patterns[zone] = re.compile(pattern)

    return (iface, port, suffix, overrides, prefixes, recursion_patterns)

class Responder(object):
    """
    Responds to requests.
    """

    __slots__ = ('suffix', 'overrides', 'prefixes', 'recursion_patterns')

    def __init__(self, suffix, overrides, prefixes, recursion_patterns):
        super(Responder, self).__init__()
        self.suffix = suffix
        self.overrides = overrides
        self.prefixes = prefixes
        self.recursion_patterns = recursion_patterns

    def get_whois_server(self, zone):
        """
        Get the WHOIS server for the given zone.
        """
        return get_whois_server(self.suffix, self.overrides, zone)

    def get_prefix(self, zone):
        """
        Gets the prefix required when querying the servers for the given zone.
        """
        return self.prefixes[zone] if zone in self.prefixes else ''

    def respond(self, _addr):
        """
        Respond to a single request.
        """
        query = diesel.until_eol().rstrip("\r\n").lower()
        try:
            _, zone = split_fqdn(query)
        except ValueError:
            diesel.send("; Bad request: '%s'\r\n" % query)
            return

        # Query the registry's WHOIS server.
        server = self.get_whois_server(zone)
        response = whois(server, self.get_prefix(zone) + query)
        if response is None:
            diesel.send("; Slow response from registry WHOIS server.\r\n")
            return

        # Thin registry? Query the registrar's WHOIS server.
        if zone in self.recursion_patterns:
            matches = self.recursion_patterns[zone].search(response)
            if matches is None:
                diesel.send("; Registrar WHOIS server unknown.\r\n")
                return
            response = whois(matches.group('server'), query)
            if response is None:
                diesel.send("; Slow response from registrar WHOIS server.\r\n")
                return

        diesel.send(response)

def main():
    """
    Execute the daemon.
    """
    if len(sys.argv) != 2:
        print >> sys.stderr, USAGE % os.path.basename(sys.argv[0])
        return 1

    parser = SafeConfigParser(allow_no_value=True)
    parser.read(sys.argv[1])
    iface, port, suffix, overrides, prefixes, recursion_patterns = \
        read_config(parser)

    responder = Responder(suffix, overrides, prefixes, recursion_patterns)
    diesel.quickstart(diesel.Service(responder.respond, port, iface))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python

### Class to control the RGB LED Indicator ###

import sys
import signal
sys.path.append("/home/debian/pixy/build/pantilt_in_python")
from pixy import *
from ctypes import *

PIXY_MIN_X             =    0
PIXY_MAX_X             =  319

PIXY_X_CENTER          =  ((PIXY_MAX_X-PIXY_MIN_X) / 2)
PIXY_RCS_MIN_POS       =    0
PIXY_RCS_MAX_POS       = 1000
PIXY_RCS_CENTER_POS    =  ((PIXY_RCS_MAX_POS-PIXY_RCS_MIN_POS) / 2)

PIXY_RCS_PAN_CHANNEL   =    0

PAN_PROPORTIONAL_GAIN  =  400
PAN_DERIVATIVE_GAIN    =  800

BLOCK_BUFFER_SIZE      =    1

class Blocks (Structure):
  _fields_ = [ ("type", c_uint),
               ("signature", c_uint),
               ("x", c_uint),
               ("y", c_uint),
               ("width", c_uint),
               ("height", c_uint),
               ("angle", c_uint) ]

class Gimbal ():
  _fields_ = [ ("position", c_uint),
               ("first_update", bool),
               ("previous_error", c_uint),
               ("proportional_gain", c_uint),
               ("derivative_gain", c_uint) ]

  def __init__(self, start_position, proportional_gain, derivative_gain):
    self.position          = start_position
    self.proportional_gain = proportional_gain
    self.derivative_gain   = derivative_gain
    self.previous_error    = 0
    self.first_update      = True

  def update(self, error):
    if self.first_update == False:
      error_delta = error - self.previous_error
      P_gain      = self.proportional_gain;
      D_gain      = self.derivative_gain;

      # Using the proportional and derivative gain for the gimbal #
      # calculate the change to the position                      #
      velocity = (error * P_gain + error_delta * D_gain) / 1024;

      self.position += velocity;

      if self.position > PIXY_RCS_MAX_POS:
        self.position = PIXY_RCS_MAX_POS
      elif self.position < PIXY_RCS_MIN_POS:
        self.position = PIXY_RCS_MIN_POS
    else:
      self.first_update = False

    self.previous_error = error

class pixyController(Sensor):
  def __init__(self, inName, inDefaultPriority=3, **kwargs):
    # boilerplate bookkeeping
    Sensor.__init__(self, inName, inSensorThreadProc=self.__threadProc,
                    inSensorThreadProcKWArgs=kwargs, inDefaultPriority=3)

    # Initialize Pixy Interpreter thread #
    self.pixy_init_status = pixy_init()

    if self. pixy_init_status != 0:
      pixy_error(pixy_init_status)
      return

    #  Initialize Gimbals #
    self.pan_gimbal = Gimbal(PIXY_RCS_CENTER_POS, PAN_PROPORTIONAL_GAIN, PAN_DERIVATIVE_GAIN)

    # Initialize block #
    self.block = Block()

  # service the sensor, post the value
  def __threadProc(self, **kwargs):
    myReading = self.initialVal_
    while not self.quitEvent_.isSet():                     # should we quit?

      # Grab a block #
      self.count = pixy_get_blocks(BLOCK_BUFFER_SIZE, self.block)

      # Was there an error? #
      if self.count < 0:
        print 'Error: pixy_get_blocks()'
        pixy_error(self.count)
        sys.exit(1)

      if self.count > 0:
        # We found a block #

        # Calculate the difference between Pixy's center of focus #
        # and the target.
        #print "Pixy Center: {} -- X Block: {}".format(PIXY_X_CENTER, block.x)                                         #
        self.pan_error  = PIXY_X_CENTER - self.block.x

        # Apply corrections to the pan/tilt gimbals with the goal #
        # of putting the target in the center of Pixy's focus.    #
        pan_gimbal.update(self.pan_error)
        self.servo_position = (1024-pan_gimbal.position)*(180/1024.0)-90
        #print "Pixy object found at {} degrees".format(self.servo_position)

        set_position_result = pixy_rcs_set_position(PIXY_RCS_PAN_CHANNEL, pan_gimbal.position)

        if set_position_result < 0:
          print 'Error: pixy_rcs_set_position() '
          pixy_error(result)
          sys.exit(2)

      myReading = (1024-pan_gimbal.position)*(180/1024.0)-90)                # ZarkonSensor takes one parameter, incrementBy
      print "\Pixy %s SPEAKS: %s" % (self.name_, self.servo_position)
      self.postReading(myReading, 1)                       # "high" priority
      time.sleep(1)
      while self.sleepEvent_.isSet():                      # should we sleep?
        time.sleep(0.5)
<commit_msg>updated pixyTracker<commit_after>#!/usr/bin/env python

### Class to control the RGB LED Indicator ###

import sys
import signal
import sensorState
sys.path.append("/home/debian/pixy/build/pantilt_in_python")
from pixy import *
from ctypes import *

PIXY_MIN_X             =    0
PIXY_MAX_X             =  319

PIXY_X_CENTER          =  ((PIXY_MAX_X-PIXY_MIN_X) / 2)
PIXY_RCS_MIN_POS       =    0
PIXY_RCS_MAX_POS       = 1000
PIXY_RCS_CENTER_POS    =  ((PIXY_RCS_MAX_POS-PIXY_RCS_MIN_POS) / 2)

PIXY_RCS_PAN_CHANNEL   =    0

PAN_PROPORTIONAL_GAIN  =  400
PAN_DERIVATIVE_GAIN    =  800

BLOCK_BUFFER_SIZE      =    1

class Blocks (Structure):
  _fields_ = [ ("type", c_uint),
               ("signature", c_uint),
               ("x", c_uint),
               ("y", c_uint),
               ("width", c_uint),
               ("height", c_uint),
               ("angle", c_uint) ]

class Gimbal ():
  _fields_ = [ ("position", c_uint),
               ("first_update", bool),
               ("previous_error", c_uint),
               ("proportional_gain", c_uint),
               ("derivative_gain", c_uint) ]

  def __init__(self, start_position, proportional_gain, derivative_gain):
    self.position          = start_position
    self.proportional_gain = proportional_gain
    self.derivative_gain   = derivative_gain
    self.previous_error    = 0
    self.first_update      = True

  def update(self, error):
    if self.first_update == False:
      error_delta = error - self.previous_error
      P_gain      = self.proportional_gain;
      D_gain      = self.derivative_gain;

      # Using the proportional and derivative gain for the gimbal #
      # calculate the change to the position                      #
      velocity = (error * P_gain + error_delta * D_gain) / 1024;

      self.position += velocity;

      if self.position > PIXY_RCS_MAX_POS:
        self.position = PIXY_RCS_MAX_POS
      elif self.position < PIXY_RCS_MIN_POS:
        self.position = PIXY_RCS_MIN_POS
    else:
      self.first_update = False

    self.previous_error = error

class pixyController(sensorState.Sensor):
  def __init__(self, inName, inDefaultPriority=3, **kwargs):
    # boilerplate bookkeeping
    Sensor.__init__(self, inName, inSensorThreadProc=self.__threadProc,
                    inSensorThreadProcKWArgs=kwargs, inDefaultPriority=3)

    # Initialize Pixy Interpreter thread #
    self.pixy_init_status = pixy_init()

    if self. pixy_init_status != 0:
      pixy_error(pixy_init_status)
      return

    #  Initialize Gimbals #
    self.pan_gimbal = Gimbal(PIXY_RCS_CENTER_POS, PAN_PROPORTIONAL_GAIN, PAN_DERIVATIVE_GAIN)

    # Initialize block #
    self.block = Block()

  # service the sensor, post the value
  def __threadProc(self, **kwargs):
    myReading = self.initialVal_
    while not self.quitEvent_.isSet():                     # should we quit?

      # Grab a block #
      self.count = pixy_get_blocks(BLOCK_BUFFER_SIZE, self.block)

      # Was there an error? #
      if self.count < 0:
        print 'Error: pixy_get_blocks()'
        pixy_error(self.count)
        sys.exit(1)

      if self.count > 0:
        # We found a block #

        # Calculate the difference between Pixy's center of focus #
        # and the target.
        #print "Pixy Center: {} -- X Block: {}".format(PIXY_X_CENTER, block.x)                                         #
        self.pan_error  = PIXY_X_CENTER - self.block.x

        # Apply corrections to the pan/tilt gimbals with the goal #
        # of putting the target in the center of Pixy's focus.    #
        pan_gimbal.update(self.pan_error)
        self.servo_position = (1024-pan_gimbal.position)*(180/1024.0)-90
        #print "Pixy object found at {} degrees".format(self.servo_position)

        set_position_result = pixy_rcs_set_position(PIXY_RCS_PAN_CHANNEL, pan_gimbal.position)

        if set_position_result < 0:
          print 'Error: pixy_rcs_set_position() '
          pixy_error(result)
          sys.exit(2)

      myReading = (1024-pan_gimbal.position)*(180/1024.0)-90)                # ZarkonSensor takes one parameter, incrementBy
      print "\Pixy %s SPEAKS: %s" % (self.name_, self.servo_position)
      self.postReading(myReading, 1)                       # "high" priority
      time.sleep(1)
      while self.sleepEvent_.isSet():                      # should we sleep?
        time.sleep(0.5)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- python -*-
#-----------------------------------------------------------------------
# XALT: A tool that tracks users jobs and environments on a cluster.
# Copyright (C) 2013-2014 University of Texas at Austin
# Copyright (C) 2013-2014 University of Tennessee
#
# This library is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as
# published by the Free Software Foundation; either version 2.1 of
# the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser  General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free
# Software Foundation, Inc., 59 Temple Place, Suite 330,
# Boston, MA 02111-1307 USA
#-----------------------------------------------------------------------

from __future__ import print_function, division
import os, sys, re

def getTerminalSize():
  """
  Finds the terminal size if possible otherwise it assumes 25, 80
  @returns: lines  the number of rows
  @returns: cols   the number of columns
  """
  import os, struct
  def ioctl_GWINSZ(fd):
      import fcntl, termios
      return struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234"))
  # try stdin, stdout, stderr
  if (not sys.stdout.isatty()):
    return (25, 80)

  fd = 1
  try:
    return ioctl_GWINSZ(fd)
  except:
    pass
  # try os.ctermid()
  try:
    fd = os.open(os.ctermid(), os.O_RDONLY)
    try:
      return ioctl_GWINSZ(fd)
    finally:
      os.close(fd)
  except:
    pass
  # try `stty size`
  try:
    return tuple(int(x) for x in os.popen("stty size", "r").read().split())
  except:
    pass
  # try environment variables
  try:
    return tuple(int(os.getenv(var)) for var in ("LINES", "COLUMNS"))
  except:
    pass
  # i give up. return default.
  return (25, 80)

class ProgressBar(object):
  """ A progress bar display class """

  def __init__(self, termWidth=None, barWidth=None, maxVal=None, ttyOnly=False, fd=sys.stderr):
    """
    Ctor that figures out the range of the progress bar.
    @param termWidth: The terminal width
    @param barWidth:  The number of character that make up the bar
    @param maxVal:    The maximum value (required)
    @param ttyOnly:   If true then only print progress bar element if connected to a terminal.
    @param fd:        The output stream.
    """
    if (not maxVal):
      ValueError('Must specify maxVal')

    self.__maxVal = maxVal
    if (not barWidth):
      barWidth = termWidth
      if (not barWidth):
        r, barWidth = getTerminalSize()
        if (barWidth < 40):
          barWidth = 40

    barWidth = min(barWidth, self.__maxVal)

    self.__active   = True
    if (ttyOnly and not sys.stdout.isatty()):
      self.__active = False

    self.__fd       = fd
    self.__barWidth = barWidth

    self.__unit     = 100/self.__barWidth
    self.__fence    = self.__unit
    self.__mark     = 10
    self.__count    = -1
    self.__symbolT  = [ '+', '+', '+', '+', '|', '+', '+', '+', '+', '|' ]

  def update(self, i):
    """
    Update progress bar.
    @param i:  input value
    """

    if (not self.__active):
      return

    j = 100*i//self.__maxVal
    k = 100*(i+1)//self.__maxVal

    #print("i: ", i, "j: ",j,"k:",k, "fence: ", self.__fence,"mark:",self.__mark)

    if (j >= self.__fence):
      symbol = "-"
      if (( j <= self.__mark and k >= self.__mark) or
          ( j == k and j == self.__mark)         or
          (self.__fence > self.__mark)):

        self.__count +=  1
        self.__mark  += 10
        symbol = self.__symbolT[self.__count]
      self.__fd.write(symbol)
      self.__fence += self.__unit
  def fini(self):
    """ Finish progress bar output. """
    if (not self.__active):
      return
    self.__fd.write("\n")

def main():
  """
  Test program for progress bar.
  """

  num = 200
  pbar = ProgressBar(maxVal=num, ttyOnly=False)
  for i in range(num):
    pbar.update(i+1)
  pbar.fini()

if ( __name__ == '__main__'): main()

<commit_msg>force flush to file descriptor for progressBar<commit_after>#!/usr/bin/env python
# -*- python -*-
#-----------------------------------------------------------------------
# XALT: A tool that tracks users jobs and environments on a cluster.
# Copyright (C) 2013-2014 University of Texas at Austin
# Copyright (C) 2013-2014 University of Tennessee
#
# This library is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as
# published by the Free Software Foundation; either version 2.1 of
# the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser  General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free
# Software Foundation, Inc., 59 Temple Place, Suite 330,
# Boston, MA 02111-1307 USA
#-----------------------------------------------------------------------

from __future__ import print_function, division
import os, sys, re

def getTerminalSize():
  """
  Finds the terminal size if possible otherwise it assumes 25, 80
  @returns: lines  the number of rows
  @returns: cols   the number of columns
  """
  import os, struct
  def ioctl_GWINSZ(fd):
      import fcntl, termios
      return struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234"))
  # try stdin, stdout, stderr
  if (not sys.stdout.isatty()):
    return (25, 80)

  fd = 1
  try:
    return ioctl_GWINSZ(fd)
  except:
    pass
  # try os.ctermid()
  try:
    fd = os.open(os.ctermid(), os.O_RDONLY)
    try:
      return ioctl_GWINSZ(fd)
    finally:
      os.close(fd)
  except:
    pass
  # try `stty size`
  try:
    return tuple(int(x) for x in os.popen("stty size", "r").read().split())
  except:
    pass
  # try environment variables
  try:
    return tuple(int(os.getenv(var)) for var in ("LINES", "COLUMNS"))
  except:
    pass
  # i give up. return default.
  return (25, 80)

class ProgressBar(object):
  """ A progress bar display class """

  def __init__(self, termWidth=None, barWidth=None, maxVal=None, ttyOnly=False, fd=sys.stderr):
    """
    Ctor that figures out the range of the progress bar.
    @param termWidth: The terminal width
    @param barWidth:  The number of character that make up the bar
    @param maxVal:    The maximum value (required)
    @param ttyOnly:   If true then only print progress bar element if connected to a terminal.
    @param fd:        The output stream.
    """
    if (not maxVal):
      ValueError('Must specify maxVal')

    self.__maxVal = maxVal
    if (not barWidth):
      barWidth = termWidth
      if (not barWidth):
        r, barWidth = getTerminalSize()
        if (barWidth < 40):
          barWidth = 40

    barWidth = min(barWidth, self.__maxVal)

    self.__active   = True
    if (ttyOnly and not sys.stdout.isatty()):
      self.__active = False

    self.__fd       = fd
    self.__barWidth = barWidth

    self.__unit     = 100/self.__barWidth
    self.__fence    = self.__unit
    self.__mark     = 10
    self.__count    = -1
    self.__symbolT  = [ '+', '+', '+', '+', '|', '+', '+', '+', '+', '|' ]

  def update(self, i):
    """
    Update progress bar.
    @param i:  input value
    """

    if (not self.__active):
      return

    j = 100*i//self.__maxVal
    k = 100*(i+1)//self.__maxVal

    #print("i: ", i, "j: ",j,"k:",k, "fence: ", self.__fence,"mark:",self.__mark)

    if (j >= self.__fence):
      symbol = "-"
      if (( j <= self.__mark and k >= self.__mark) or
          ( j == k and j == self.__mark)         or
          (self.__fence > self.__mark)):

        self.__count +=  1
        self.__mark  += 10
        symbol = self.__symbolT[self.__count]
      self.__fd.write(symbol); self._fd.flush()
      self.__fence += self.__unit
  def fini(self):
    """ Finish progress bar output. """
    if (not self.__active):
      return
    self.__fd.write("\n")

def main():
  """
  Test program for progress bar.
  """

  num = 200
  pbar = ProgressBar(maxVal=num, ttyOnly=False)
  for i in range(num):
    pbar.update(i+1)
  pbar.fini()

if ( __name__ == '__main__'): main()

<|endoftext|>


<commit_before>#!/usr/bin/python

import shutil
import sys, argparse, os
import svn.remote
from git import Repo, RemoteProgress
from subprocess import call

GITBENCHMARK = 'git-benchmark.sh'
SVNBENCHMARK = 'svn-benchmark.sh'
GIT_DIR = 'git-work'
SVN_DIR = 'svn-work'

commitslist = []

class CloneProgressPrinter(RemoteProgress):
    def update(self, op_code, cur_count, max_count=None, message=''):
        if cur_count / (max_count or 100.0) % 10:
            sys.stdout.write('.')
            sys.stdout.flush()

class IncorrectCommitOrder(Exception):
    def __init__(self, start, end):
        self.start = str(start)
        self.end = str(end)
    def __str__(self):
        return self.start + '->' + self.end

class CommitNotFound(Exception):
    def __init__(self, commit):
        self.commit = commit
    def __str__(self):
        return 'Commit ' + str(self.commit) + ' cannot be found.'

def parse_args():
    parser = argparse.ArgumentParser(description='Version Control Benchmark')
    parser.add_argument('vc', help='The version control to benchmark',
            choices=['git', 'svn'])
    parser.add_argument('-s', '--start', help="""The commit at which to start.\
            If omitted, the first commit will be used.""")
    parser.add_argument('-e', '--end', help="""The commit at which to end. If \
            omitted, the latest commit will be used.""")
    parser.add_argument('repo', help='The repository location')
    parser.add_argument('-pre', help='Instruction to run before benchmark')
    parser.add_argument('-post', help='Instruction to run after benchmark')
    return parser.parse_args()

def find_git_start(repo):
    commits = repo.iter_commits()
    commit = ''
    for commit in commits:
        pass

    return commit

def find_git_end(repo):
    commits = repo.iter_commits()
    for commit in commits:
        return commit

def verify_git_revs(start, end, repo):
    start_found = False
    end_found = False

    for commit in repo.iter_commits():
        if commit == start:
            start_found = True
        elif commit == end:
            end_found = True

        if not end_found and start_found:
            raise IncorrectCommitOrder(start.name_rev[0:40], end.name_rev[0:40])

        if end_found:
            commitslist.append(commit.name_rev[0:40])

        if start_found:
            break

    commitslist.reverse()
    return

def benchmark_git(repourl, start, end, preInst, postInst):
    if os.path.exists(GIT_DIR):
        shutil.rmtree(GIT_DIR)

    print 'Benchmarking git repo ' + repourl
    sys.stdout.write('Cloning')
    sys.stdout.flush()
    repo = Repo.clone_from(repourl, GIT_DIR, progress=CloneProgressPrinter())
    print 'Done'

    if not start:
        startcommit = find_git_start(repo)
    else:
        startcommit = repo.commit(start)

    if not end:
        endcommit = find_git_end(repo)
    else:
        endcommit = repo.commit(end)

    verify_git_revs(startcommit, endcommit, repo)

    fo = open(GITBENCHMARK, 'w')
    print 'Outputting benchmark'
    fo.write("#!/usr/bin/sh\n")
    if preInst:
        fo.write(preInst + "\n")
    fo.write("git checkout " + commitslist[0] + "\n")
    for commit in commitslist[1:]:
        fo.write("git merge -m \"benchmark\" " + commit)
    if postInst:
        fo.write(postInst + "\n")
    print 'Done.'

    return

def find_svn_start(client):
    end = 0
    for commit in client.log_default():
        end = commit

    return end.revision

def find_svn_end(client):
    for commit in client.log_default():
        return commit.revision

def verify_svn_revs(startcommit, endcommit, client):
    start_found = False
    end_found = False

    for commit in client.log_default():
        if commit.revision == startcommit:
            start_found = True
        elif commit.revision == endcommit:
            end_found = True

        if end_found:
            commitslist.append(commit.revision)

        if start_found:
            break

    if not start_found:
        raise CommitNotFound(startcommit)
    elif not end_found:
        raise CommitNotFound(endcommit)
    elif startcommit > endcommit:
        raise IncorrectCommitOrder(startcommit, endcommit)

    commitslist.reverse()
    return

def benchmark_svn(repourl, start, end, preInst, postInst):
    if os.path.exists(SVN_DIR):
        shutil.rmtree(SVN_DIR)

    fo = open(SVNBENCHMARK, 'w')
    client = svn.remote.RemoteClient(repourl)
    sys.stdout.write('Checking out...')
    sys.stdout.flush()
    revision = client.checkout(SVN_DIR)
    print 'Done'

    if not start:
        startcommit = find_svn_start(client)
    else:
        startcommit = start

    if not end:
        endcommit = find_svn_end(client)
    else:
        endcommit = end

    verify_svn_revs(startcommit, endcommit, client)
    fo.write('#!/usr/bin/sh\n')
    fo.write('cd ' + SVN_DIR + '\n')
    if preInst:
        fo.write(preInst + "\n")
    fo.write('svn up -r' + str(commitslist[0]) + '\n')
    for commit in commitslist[1:]:
        fo.write("svn merge -c " + str(commit) + " .\n")
    if postInst:
        fo.write(postInst + "\n")

    return

def main():
    args = parse_args()
    try:
        if args.vc == 'git':
            benchmark_git(args.repo, args.start, args.end, args.pre, args.post)
        elif args.vc == 'svn':
            benchmark_svn(args.repo, args.start, args.end, args.pre, args.post)
    except IncorrectCommitOrder as ico:
        sys.stdout.write("Commit " + ico.start)
        sys.stdout.write(" comes before " + ico.end)
        sys.stdout.write("\n")
        sys.stdout.flush()
    except CommitNotFound as cnf:
        print(cnf)

    return

if __name__ == "__main__":
    main()
<commit_msg>fixing another git bug<commit_after>#!/usr/bin/python

import shutil
import sys, argparse, os
import svn.remote
from git import Repo, RemoteProgress
from subprocess import call

GITBENCHMARK = 'git-benchmark.sh'
SVNBENCHMARK = 'svn-benchmark.sh'
GIT_DIR = 'git-work'
SVN_DIR = 'svn-work'

commitslist = []

class CloneProgressPrinter(RemoteProgress):
    def update(self, op_code, cur_count, max_count=None, message=''):
        if cur_count / (max_count or 100.0) % 10:
            sys.stdout.write('.')
            sys.stdout.flush()

class IncorrectCommitOrder(Exception):
    def __init__(self, start, end):
        self.start = str(start)
        self.end = str(end)
    def __str__(self):
        return self.start + '->' + self.end

class CommitNotFound(Exception):
    def __init__(self, commit):
        self.commit = commit
    def __str__(self):
        return 'Commit ' + str(self.commit) + ' cannot be found.'

def parse_args():
    parser = argparse.ArgumentParser(description='Version Control Benchmark')
    parser.add_argument('vc', help='The version control to benchmark',
            choices=['git', 'svn'])
    parser.add_argument('-s', '--start', help="""The commit at which to start.\
            If omitted, the first commit will be used.""")
    parser.add_argument('-e', '--end', help="""The commit at which to end. If \
            omitted, the latest commit will be used.""")
    parser.add_argument('repo', help='The repository location')
    parser.add_argument('-pre', help='Instruction to run before benchmark')
    parser.add_argument('-post', help='Instruction to run after benchmark')
    return parser.parse_args()

def find_git_start(repo):
    commits = repo.iter_commits()
    commit = ''
    for commit in commits:
        pass

    return commit

def find_git_end(repo):
    commits = repo.iter_commits()
    for commit in commits:
        return commit

def verify_git_revs(start, end, repo):
    start_found = False
    end_found = False

    for commit in repo.iter_commits():
        if commit == start:
            start_found = True
        elif commit == end:
            end_found = True

        if not end_found and start_found:
            raise IncorrectCommitOrder(start.name_rev[0:40], end.name_rev[0:40])

        if end_found:
            commitslist.append(commit.name_rev[0:40])

        if start_found:
            break

    commitslist.reverse()
    return

def benchmark_git(repourl, start, end, preInst, postInst):
    if os.path.exists(GIT_DIR):
        shutil.rmtree(GIT_DIR)

    print 'Benchmarking git repo ' + repourl
    sys.stdout.write('Cloning')
    sys.stdout.flush()
    repo = Repo.clone_from(repourl, GIT_DIR, progress=CloneProgressPrinter())
    print 'Done'

    if not start:
        startcommit = find_git_start(repo)
    else:
        startcommit = repo.commit(start)

    if not end:
        endcommit = find_git_end(repo)
    else:
        endcommit = repo.commit(end)

    verify_git_revs(startcommit, endcommit, repo)

    fo = open(GITBENCHMARK, 'w')
    print 'Outputting benchmark'
    fo.write("#!/usr/bin/sh\n")
    fo.write("cd " + GIT_DIR + "\n")
    if preInst:
        fo.write(preInst + "\n")
    fo.write("git checkout " + commitslist[0] + "\n")
    for commit in commitslist[1:]:
        fo.write("git merge -m \"benchmark\" " + commit + "\n")
    if postInst:
        fo.write(postInst + "\n")
    print 'Done.'

    return

def find_svn_start(client):
    end = 0
    for commit in client.log_default():
        end = commit

    return end.revision

def find_svn_end(client):
    for commit in client.log_default():
        return commit.revision

def verify_svn_revs(startcommit, endcommit, client):
    start_found = False
    end_found = False

    for commit in client.log_default():
        if commit.revision == startcommit:
            start_found = True
        elif commit.revision == endcommit:
            end_found = True

        if end_found:
            commitslist.append(commit.revision)

        if start_found:
            break

    if not start_found:
        raise CommitNotFound(startcommit)
    elif not end_found:
        raise CommitNotFound(endcommit)
    elif startcommit > endcommit:
        raise IncorrectCommitOrder(startcommit, endcommit)

    commitslist.reverse()
    return

def benchmark_svn(repourl, start, end, preInst, postInst):
    if os.path.exists(SVN_DIR):
        shutil.rmtree(SVN_DIR)

    fo = open(SVNBENCHMARK, 'w')
    client = svn.remote.RemoteClient(repourl)
    sys.stdout.write('Checking out...')
    sys.stdout.flush()
    revision = client.checkout(SVN_DIR)
    print 'Done'

    if not start:
        startcommit = find_svn_start(client)
    else:
        startcommit = start

    if not end:
        endcommit = find_svn_end(client)
    else:
        endcommit = end

    verify_svn_revs(startcommit, endcommit, client)
    fo.write('#!/usr/bin/sh\n')
    fo.write('cd ' + SVN_DIR + '\n')
    if preInst:
        fo.write(preInst + "\n")
    fo.write('svn up -r' + str(commitslist[0]) + '\n')
    for commit in commitslist[1:]:
        fo.write("svn merge -c " + str(commit) + " .\n")
    if postInst:
        fo.write(postInst + "\n")

    return

def main():
    args = parse_args()
    try:
        if args.vc == 'git':
            benchmark_git(args.repo, args.start, args.end, args.pre, args.post)
        elif args.vc == 'svn':
            benchmark_svn(args.repo, args.start, args.end, args.pre, args.post)
    except IncorrectCommitOrder as ico:
        sys.stdout.write("Commit " + ico.start)
        sys.stdout.write(" comes before " + ico.end)
        sys.stdout.write("\n")
        sys.stdout.flush()
    except CommitNotFound as cnf:
        print(cnf)

    return

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3

#
# vdbtest.py - VDbench Test Script
#
# Author: Ramon A. Lovato (ramonalovato.com)
# For: DeepStorage, LLC (deepstorage.net)
#

import argparse
import os.path
import os
import re
from vdbconfig import vdbconfig
from NetJobs import NetJobs

DEFAULT_RUNS = 5
DEFAULT_TIMEOUT = 0
ARCHIVE_DIR_FORMAT = "__{content}_{testID}__"
ARCHIVE_DIR_REGEX = ARCHIVE_DIR_FORMAT.format(content="\w+", testID="\d+")
ID_SEP = "_##"
DEFAULT_SUCCESS_MULTIPLIER = 5.0
DEFAULT_FAILURE_MULTIPLIER = 0.3
DEFAULT_CONSECUTIVE_FAILURES = 2

# Get CLI arguments.
def getArgs():
    parser = argparse.ArgumentParser(description="Run VDbench tests to match "
        "IO response across multiple machines against target latency.")
    # Positional.
    parser.add_argument("configFile", type=str,
        help="path to the configuration file")
    parser.add_argument("configDir", type=str,
        help="directory containing VDbench config files")
    parser.add_argument("outputParent", type=str,
        help="the parent directory of the directories containing output files")
    parser.add_argument("workFolder", type=str,
        help="path to the work folder for intermediate file storage")
    parser.add_argument("targetLatency", type=float,
        help="target latency we're trying for (in ms)")

    # Optional.
    parser.add_argument("-m", "--max-runs", type=int, default=DEFAULT_RUNS,
        help="override maximum number of runs before aborting (default {})"
        .format(DEFAULT_RUNS))
    parser.add_argument("-t", "--timeout", type=int, default=DEFAULT_TIMEOUT,
        help="set a timeout in seconds for the scheduler (default {})"
        .format(DEFAULT_TIMEOUT))
    parser.add_argument("--success-multiplier", type=float,
        default=DEFAULT_SUCCESS_MULTIPLIER,
        help="override IO rate multiplier when target is below target latency (default {})"
        .format(DEFAULT_SUCCESS_MULTIPLIER))
    parser.add_argument("--failure-multiplier", type=float,
        default=DEFAULT_FAILURE_MULTIPLIER,
        help="override IO rate multiplier when target is above target latency (default {})"
        .format(DEFAULT_FAILURE_MULTIPLIER))
    parser.add_argument("--consecutive-failures", type=int,
        default=DEFAULT_CONSECUTIVE_FAILURES,
        help="terminate after n consecutive failures (default {})"
        .format(DEFAULT_CONSECUTIVE_FAILURES))
    parser.add_argument("-v", "--verbose", action="store_true",
        help="enable verbose mode")

    args = parser.parse_args()

    args.configFile = os.path.realpath(args.configFile)
    args.configDir = os.path.realpath(args.configDir)
    args.outputParent = os.path.realpath(args.outputParent)
    args.workFolder = os.path.realpath(args.workFolder)

    if args.max_runs < 1:
        print("Warning: max_runs < 1. Using default ({}).".format(DEFAULT_RUNS))
        args.max_runs = DEFAULT_RUNS

    return args

# Read vdbtest config file.
def readConfig(configFile):
    config = {
        "targets": [],
        "command": None,
    }

    try:
        with open(configFile, "r") as f:
            targetsReached = False

            # A simple state machine for parsing the config file.
            for realLine in f:
                line = realLine.strip()

                # Comment line or blank.
                if not line or line == "" or line.startswith("#"):
                    continue

                partials = re.split(":", line, maxsplit=1)
                key = partials[0].strip().lower()

                if not targetsReached:
                    if key == "targets":
                        targetsReached = True
                        continue
                    elif len(partials) < 2:
                        raise Exception(
                            "Error: configuration file {configFile} --- unrecognized line {line}.".format(
                                line=line, configFile=configFile))

                if targetsReached:
                    config["targets"].append(line)
                else:
                    value = partials[1].strip()

                    # Command string.
                    if key == "command":
                        config["command"] = value

                    # No idea.
                    else:
                        raise Exception(
                            "Error: configuration file {configFile} --- unrecognized line {line}.".format(
                                configFile=configFile, line=line))

    except Exception as e:
        raise e

    if len(config["targets"]) == 0:
        raise Exception(
            "Error: configuration file {} --- no targets specified.".format(
                configFile))
    if not config["command"]:
        raise Exception(
            "Error: configuration file {} --- no command specified.".format(
                configFile))

    return config

# Archive everything in the specified directory that isn't itself an archive
# directory.
def archiveContents(parentDir, testID):
    candidates = getNonArchiveDirs(parentDir)
    for c in candidates:
        archiveFile(c, testID)

# Archive the specified file. Return the new archive path.
def archiveFile(oldPath, testID):
    parentDir = os.path.dirname(oldPath)
    content = os.path.split(parentDir)[-1]
    archDir = os.path.join(parentDir,
        ARCHIVE_DIR_FORMAT.format(content=content, testID=testID))
    if not os.path.isdir(archDir):
        os.makedirs(archDir)
    newPath = os.path.join(archDir, os.path.basename(oldPath))
    os.rename(oldPath, newPath)

    return newPath

# Gets a list of directories in the specified parent directory, excluding
# archives.
def getNonArchiveDirs(parentDir):
    names = filter(lambda d: not re.match(ARCHIVE_DIR_REGEX, d),
        os.listdir(parentDir))
    return [os.path.join(parentDir, p) for p in names]

# Reads test results from totals.html in the specified directory.
def getTestResults(parentDir):
    try:
        totalsFile = findTotalsFile(parentDir)
    except Exception as e:
        raise e

    try:
        with open(totalsFile, "r") as f:
            # Line we care about is the last in the file.
            lines = f.readlines()
            # Easier to hard-code these, since they shouldn't change.
            keys = ["date/time",
                    "interval",
                    "i/o rate",
                    "MB/sec 1024**2",
                    "bytes i/o",
                    "read pct",
                    "resp time",
                    "read resp",
                    "write resp",
                    "resp max",
                    "resp stddev",
                    "queue depth",
                    "cpu% sys+u",
                    "cpu% sys"]
            values = re.split("\s+", lines[-1])
            results = dict(zip(keys, values))
            return results
    except IOError as e:
        raise e

# Find absolute path to totals.html file in specified directory.
def findTotalsFile(parentDir):
    for f in os.listdir(parentDir):
        if os.path.basename(f) == "totals.html":
            return os.path.join(parentDir, f)
    # Didn't find.
    raise Exception(
        "Error: directory {} does not contain the file totals.html.".format(
            parentDir))

# Get all test results from the directories within the output directory.
def getAllTestResults(outputDir):
    allResults = {}
    for f in getNonArchiveDirs(outputDir):
        allResults[os.path.basename(f)] = getTestResults(f)
    return allResults

# Given a results dictionary from getAllTestResults and the target latency,
# returns true if ALL test results were below the target latency, else false.
def compareResultLatencies(allResults, targetLatency):
    for r in allResults.values():
        try:
            responseTime = float(r["resp time"])
        except ValueError as e:
            raise e

        if responseTime > targetLatency:
            return False

    return True

# Make a new VDbench configuration file.
def makeNewVDBConfig(oldConfig, newConfig, newIORate):
    try:
        vdbconfig.makeNewConfig(oldConfig, newConfig, newIORate)
    except IOError as e:
        raise e

# Helper for makeNewVDBConfig that tries to remove the test ID from a config
# file name.
def stripIDFromFile(filename):
    partials = re.split(ID_SEP, os.path.splitext(os.path.basename(filename))[0])
    return partials[-2] if len(partials) > 1 else partials[-1]

# Helper for makeAllNewConfigs that extracts just the test ID from an existing
# config file.
def getTestIDFromConfig(filename):
    partials = re.split(ID_SEP, os.path.splitext(os.path.basename(filename))[0])
    return int(partials[-1]) if len(partials) > 1 else 0

# Make new configuration file for NetJobs.
def makeNetJobsConfig(workFolder, timeout, targets, command, configFile):
    identifier = os.path.splitext(os.path.basename(configFile))[0]
    fileID = 0

    nj_path = os.path.join(workFolder,
        "vdbtest_{identifier}.netjobsconfig".format(identifier=identifier))

    while os.path.exists(nj_path):
        fileID += 1
        nj_path = os.path.join(workFolder,
        "vdbtest_{identifier} ({fileID}).netjobsconfig".format(
            identifier=identifier, fileID=fileID))

    try:
        with open(nj_path, "w") as f:
            # Test label, timeout/duration.
            f.write("{}:\n".format(identifier.replace(":", "_")))
            f.write("-generaltimeout: {timeout}s\n".format(timeout=timeout))
            # Target specs.
            for t in targets:
                f.write("{target}: {command}\n".format(
                    target=t, command=command))
            f.write("end")
    except Exception as e:
        raise e

    print("NetJobs config saved as: {}".format(nj_path))

    return nj_path

# Run NetJobs once.
def startNetJobs(njconfig, verbose=False):
    try:
        if verbose:
            njargs = ("-l", "-v", njconfig)
        else:
            njargs = ("-l", njconfig)
        NetJobs.main(njargs)
    except Exception as e:
        raise e

# Calculate the new IO rate based on the given config file and the allPassed status.
def calculateNewIORate(configFile, args, allPassed):
    rate = getOldIORate(configFile)
    rate *= args.success_multiplier if allPassed else args.failure_multiplier
    return int(round(rate))

# Get the old IO rate based on the given config file.
def getOldIORate(configFile):
    try:
        with open(configFile, "r") as inFile:
            for line in inFile:
                if not re.match(r"[\/\#\*].*", line):
                    tokens = [t.split("=") for t in line.split(",")]
                    for i in range(len(tokens)):
                        token = tokens[i]
                        if len(token) > 1:
                            key = token[0]
                            value = token[1]

                            if key == "iorate":
                                return int(value)
    except (IOError, ValueError) as e:
        raise e
    except (IsADirectoryError) as e:
        print("Warning: {} is a directory, not a file.".format(configFile))
    # If we got here, the config file doesn't contain an iorate, so there's
    # something wrong.
    raise Exception("Error: config file {} malformed --- no \"iorate\" specified.".format(
        configFile))

# Update all config files and archive the old ones.
def updateAndArchiveConfigs(args, allPassed, testID):
    for f in getNonArchiveDirs(args.configDir):
        name = os.path.join(args.configDir, f)
        oldName = name
        oldFile = archiveFile(name, testID)
        newIORate = calculateNewIORate(oldFile, args, allPassed)
        makeNewVDBConfig(oldFile, oldName, newIORate)

# Start the main run.
def run(args, config, njconfig, verbose=False):
    consecutiveFailures = 0

    # Main loop.
    for run in range(args.max_runs):
        startNetJobs(njconfig)
        allResults = getAllTestResults(args.outputParent)
        allPassed = compareResultLatencies(allResults, args.targetLatency)

        if allPassed:
            consecutiveFailures = 0
        else:
            consecutiveFailures += 1
            if consecutiveFailures >= args.consecutive_failures:
                print("Notice: VDbench failed to achieve the target latency "
                    "{} consecutive time(s). Aborting run.".format(
                        args.consecutive_failures))
                return

        if not run == args.max_runs - 1:
            archiveContents(args.outputParent, run)
            updateAndArchiveConfigs(args, allPassed, run)

# Main.
def main():
    args = getArgs()
    config = readConfig(args.configFile)
    njconfig = makeNetJobsConfig(args.workFolder, args.timeout,
        config["targets"], config["command"], args.configFile)

    # Done with setup.
    run(args, config, njconfig, verbose=args.verbose)

if __name__ == "__main__":
    main()<commit_msg>Fixed a bad indent block<commit_after>#!/usr/bin/env python3

#
# vdbtest.py - VDbench Test Script
#
# Author: Ramon A. Lovato (ramonalovato.com)
# For: DeepStorage, LLC (deepstorage.net)
#

import argparse
import os.path
import os
import re
from vdbconfig import vdbconfig
from NetJobs import NetJobs

DEFAULT_RUNS = 5
DEFAULT_TIMEOUT = 0
ARCHIVE_DIR_FORMAT = "__{content}_{testID}__"
ARCHIVE_DIR_REGEX = ARCHIVE_DIR_FORMAT.format(content="\w+", testID="\d+")
ID_SEP = "_##"
DEFAULT_SUCCESS_MULTIPLIER = 5.0
DEFAULT_FAILURE_MULTIPLIER = 0.3
DEFAULT_CONSECUTIVE_FAILURES = 2

# Get CLI arguments.
def getArgs():
    parser = argparse.ArgumentParser(description="Run VDbench tests to match "
        "IO response across multiple machines against target latency.")
    # Positional.
    parser.add_argument("configFile", type=str,
        help="path to the configuration file")
    parser.add_argument("configDir", type=str,
        help="directory containing VDbench config files")
    parser.add_argument("outputParent", type=str,
        help="the parent directory of the directories containing output files")
    parser.add_argument("workFolder", type=str,
        help="path to the work folder for intermediate file storage")
    parser.add_argument("targetLatency", type=float,
        help="target latency we're trying for (in ms)")

    # Optional.
    parser.add_argument("-m", "--max-runs", type=int, default=DEFAULT_RUNS,
        help="override maximum number of runs before aborting (default {})"
        .format(DEFAULT_RUNS))
    parser.add_argument("-t", "--timeout", type=int, default=DEFAULT_TIMEOUT,
        help="set a timeout in seconds for the scheduler (default {})"
        .format(DEFAULT_TIMEOUT))
    parser.add_argument("--success-multiplier", type=float,
        default=DEFAULT_SUCCESS_MULTIPLIER,
        help="override IO rate multiplier when target is below target latency (default {})"
        .format(DEFAULT_SUCCESS_MULTIPLIER))
    parser.add_argument("--failure-multiplier", type=float,
        default=DEFAULT_FAILURE_MULTIPLIER,
        help="override IO rate multiplier when target is above target latency (default {})"
        .format(DEFAULT_FAILURE_MULTIPLIER))
    parser.add_argument("--consecutive-failures", type=int,
        default=DEFAULT_CONSECUTIVE_FAILURES,
        help="terminate after n consecutive failures (default {})"
        .format(DEFAULT_CONSECUTIVE_FAILURES))
    parser.add_argument("-v", "--verbose", action="store_true",
        help="enable verbose mode")

    args = parser.parse_args()

    args.configFile = os.path.realpath(args.configFile)
    args.configDir = os.path.realpath(args.configDir)
    args.outputParent = os.path.realpath(args.outputParent)
    args.workFolder = os.path.realpath(args.workFolder)

    if args.max_runs < 1:
        print("Warning: max_runs < 1. Using default ({}).".format(DEFAULT_RUNS))
        args.max_runs = DEFAULT_RUNS

    return args

# Read vdbtest config file.
def readConfig(configFile):
    config = {
        "targets": [],
        "command": None,
    }

    try:
        with open(configFile, "r") as f:
            targetsReached = False

            # A simple state machine for parsing the config file.
            for realLine in f:
                line = realLine.strip()

                # Comment line or blank.
                if not line or line == "" or line.startswith("#"):
                    continue

                partials = re.split(":", line, maxsplit=1)
                key = partials[0].strip().lower()

                if not targetsReached:
                    if key == "targets":
                        targetsReached = True
                        continue
                    elif len(partials) < 2:
                        raise Exception(
                            "Error: configuration file {configFile} --- unrecognized line {line}.".format(
                                line=line, configFile=configFile))

                if targetsReached:
                    config["targets"].append(line)
                else:
                    value = partials[1].strip()

                    # Command string.
                    if key == "command":
                        config["command"] = value

                    # No idea.
                    else:
                        raise Exception(
                            "Error: configuration file {configFile} --- unrecognized line {line}.".format(
                                configFile=configFile, line=line))

    except Exception as e:
        raise e

    if len(config["targets"]) == 0:
        raise Exception(
            "Error: configuration file {} --- no targets specified.".format(
                configFile))
    if not config["command"]:
        raise Exception(
            "Error: configuration file {} --- no command specified.".format(
                configFile))

    return config

# Archive everything in the specified directory that isn't itself an archive
# directory.
def archiveContents(parentDir, testID):
    candidates = getNonArchiveDirs(parentDir)
    for c in candidates:
        archiveFile(c, testID)

# Archive the specified file. Return the new archive path.
def archiveFile(oldPath, testID):
    parentDir = os.path.dirname(oldPath)
    content = os.path.split(parentDir)[-1]
    archDir = os.path.join(parentDir,
        ARCHIVE_DIR_FORMAT.format(content=content, testID=testID))
    if not os.path.isdir(archDir):
        os.makedirs(archDir)
    newPath = os.path.join(archDir, os.path.basename(oldPath))
    os.rename(oldPath, newPath)

    return newPath

# Gets a list of directories in the specified parent directory, excluding
# archives.
def getNonArchiveDirs(parentDir):
    names = filter(lambda d: not re.match(ARCHIVE_DIR_REGEX, d),
        os.listdir(parentDir))
    return [os.path.join(parentDir, p) for p in names]

# Reads test results from totals.html in the specified directory.
def getTestResults(parentDir):
    try:
        totalsFile = findTotalsFile(parentDir)
    except Exception as e:
        raise e

    try:
        with open(totalsFile, "r") as f:
            # Line we care about is the last in the file.
            lines = f.readlines()
            # Easier to hard-code these, since they shouldn't change.
            keys = ["date/time",
                    "interval",
                    "i/o rate",
                    "MB/sec 1024**2",
                    "bytes i/o",
                    "read pct",
                    "resp time",
                    "read resp",
                    "write resp",
                    "resp max",
                    "resp stddev",
                    "queue depth",
                    "cpu% sys+u",
                    "cpu% sys"]
            values = re.split("\s+", lines[-1])
            results = dict(zip(keys, values))
            return results
    except IOError as e:
        raise e

# Find absolute path to totals.html file in specified directory.
def findTotalsFile(parentDir):
    for f in os.listdir(parentDir):
        if os.path.basename(f) == "totals.html":
            return os.path.join(parentDir, f)
    # Didn't find.
    raise Exception(
        "Error: directory {} does not contain the file totals.html.".format(
            parentDir))

# Get all test results from the directories within the output directory.
def getAllTestResults(outputDir):
    allResults = {}
    for f in getNonArchiveDirs(outputDir):
        allResults[os.path.basename(f)] = getTestResults(f)
    return allResults

# Given a results dictionary from getAllTestResults and the target latency,
# returns true if ALL test results were below the target latency, else false.
def compareResultLatencies(allResults, targetLatency):
    for r in allResults.values():
        try:
            responseTime = float(r["resp time"])
        except ValueError as e:
            raise e

        if responseTime > targetLatency:
            return False

    return True

# Make a new VDbench configuration file.
def makeNewVDBConfig(oldConfig, newConfig, newIORate):
    try:
        vdbconfig.makeNewConfig(oldConfig, newConfig, newIORate)
    except IOError as e:
        raise e

# Helper for makeNewVDBConfig that tries to remove the test ID from a config
# file name.
def stripIDFromFile(filename):
    partials = re.split(ID_SEP, os.path.splitext(os.path.basename(filename))[0])
    return partials[-2] if len(partials) > 1 else partials[-1]

# Helper for makeAllNewConfigs that extracts just the test ID from an existing
# config file.
def getTestIDFromConfig(filename):
    partials = re.split(ID_SEP, os.path.splitext(os.path.basename(filename))[0])
    return int(partials[-1]) if len(partials) > 1 else 0

# Make new configuration file for NetJobs.
def makeNetJobsConfig(workFolder, timeout, targets, command, configFile):
    identifier = os.path.splitext(os.path.basename(configFile))[0]
    fileID = 0

    nj_path = os.path.join(workFolder,
        "vdbtest_{identifier}.netjobsconfig".format(identifier=identifier))

    while os.path.exists(nj_path):
        fileID += 1
        nj_path = os.path.join(workFolder,
        "vdbtest_{identifier} ({fileID}).netjobsconfig".format(
            identifier=identifier, fileID=fileID))

    try:
        with open(nj_path, "w") as f:
            # Test label, timeout/duration.
            f.write("{}:\n".format(identifier.replace(":", "_")))
            f.write("-generaltimeout: {timeout}s\n".format(timeout=timeout))
            # Target specs.
            for t in targets:
                f.write("{target}: {command}\n".format(
                    target=t, command=command))
            f.write("end")
    except Exception as e:
        raise e

    print("NetJobs config saved as: {}".format(nj_path))

    return nj_path

# Run NetJobs once.
def startNetJobs(njconfig, verbose=False):
    try:
        if verbose:
            njargs = ("-l", "-v", njconfig)
        else:
            njargs = ("-l", njconfig)
        NetJobs.main(njargs)
    except Exception as e:
        raise e

# Calculate the new IO rate based on the given config file and the allPassed status.
def calculateNewIORate(configFile, args, allPassed):
    rate = getOldIORate(configFile)
    rate *= args.success_multiplier if allPassed else args.failure_multiplier
    return int(round(rate))

# Get the old IO rate based on the given config file.
def getOldIORate(configFile):
    try:
        with open(configFile, "r") as inFile:
            for line in inFile:
                if not re.match(r"[\/\#\*].*", line):
                    tokens = [t.split("=") for t in line.split(",")]
                    for i in range(len(tokens)):
                        token = tokens[i]
                        if len(token) > 1:
                            key = token[0]
                            value = token[1]

                            if key == "iorate":
                                return int(value)
    except (IOError, ValueError) as e:
        raise e
    except (IsADirectoryError) as e:
        print("Warning: {} is a directory, not a file.".format(configFile))
    # If we got here, the config file doesn't contain an iorate, so there's
    # something wrong.
    raise Exception("Error: config file {} malformed --- no \"iorate\" specified.".format(
        configFile))

# Update all config files and archive the old ones.
def updateAndArchiveConfigs(args, allPassed, testID):
    for f in getNonArchiveDirs(args.configDir):
        name = os.path.join(args.configDir, f)
        oldName = name
        oldFile = archiveFile(name, testID)
        newIORate = calculateNewIORate(oldFile, args, allPassed)
        makeNewVDBConfig(oldFile, oldName, newIORate)

# Start the main run.
def run(args, config, njconfig, verbose=False):
    consecutiveFailures = 0

    # Main loop.
    for run in range(args.max_runs):
        startNetJobs(njconfig)
        allResults = getAllTestResults(args.outputParent)
        allPassed = compareResultLatencies(allResults, args.targetLatency)

        if allPassed:
            consecutiveFailures = 0
        else:
            consecutiveFailures += 1
            if consecutiveFailures >= args.consecutive_failures:
                print("Notice: VDbench failed to achieve the target latency "
                    "{} consecutive time(s). Aborting run.".format(
                        args.consecutive_failures))
                return

            if not run == args.max_runs - 1:
                archiveContents(args.outputParent, run)
                updateAndArchiveConfigs(args, allPassed, run)

# Main.
def main():
    args = getArgs()
    config = readConfig(args.configFile)
    njconfig = makeNetJobsConfig(args.workFolder, args.timeout,
        config["targets"], config["command"], args.configFile)

    # Done with setup.
    run(args, config, njconfig, verbose=args.verbose)

if __name__ == "__main__":
    main()<|endoftext|>


<commit_before>from django.core.exceptions import ValidationError
from django.forms.forms import BaseForm
from django.forms.formsets import BaseFormSet
from django.forms.models import BaseInlineFormSet
from django.forms.util import ErrorList

from rebar.validators import StateValidatorFormMixin

class Unspecified(object):
    """Unspecified Value."""
Unspecified = Unspecified()

class FormGroup(object):
    """Form-like wrapper for a heterogenous collection of Forms.

    A FormGroup collects an ordered set of Forms and/or FormSets,
    and provides convenience methods for validating the group as a
    whole.

    """

    def __init__(self,
                 data=None,
                 files=None,
                 auto_id='id_%s',
                 prefix=None,
                 initial=None,
                 label_suffix=':',
                 instance=Unspecified,
                 error_class=None,
                 member_kwargs=None):

        self.is_bound = data is not None or files is not None
        self.data = data or {}
        self.files = files or {}
        self.initial = initial or {}
        self.label_suffix = label_suffix
        self.instance = instance
        self.auto_id = auto_id
        self.error_class = error_class or ErrorList
        self._errors = None
        self._group_errors = None

        self.prefix = prefix or self.get_default_prefix()

        self.member_kwargs = member_kwargs or {}

        # instantiate the members
        self._forms = []
        self.named_forms = {}

        for member_class in self.form_classes:
            if isinstance(member_class, tuple):
                member_class, name = member_class
            else:
                name = member_class.__name__.lower()
                if name.endswith('form'):
                    name = name[:-4]

            kwargs = dict(
                prefix=self.add_prefix(name),
                data=data,
                files=files,
            )

            if issubclass(member_class, BaseForm):
                kwargs.update(dict(
                    auto_id=self.auto_id,
                    error_class=self.error_class,
                    label_suffix=self.label_suffix,
                    initial=self.initial,
                ))

            elif issubclass(member_class, BaseInlineFormSet):
                # inline formsets do not take additional kwargs
                pass

            elif issubclass(member_class, BaseFormSet):
                kwargs.update(dict(
                    auto_id=self.auto_id,
                    error_class=self.error_class,
                ))
            extra_kwargs = self.member_kwargs.get(name, {})
            if self.instance is not Unspecified:
                extra_kwargs['instance'] = self.instance
            kwargs.update(extra_kwargs)

            new_form = member_class(
                **kwargs
            )
            self.named_forms[name] = new_form
            self._forms.append(new_form)

    @property
    def forms(self):
        return self._forms

    def __len__(self):

        return len(self.forms)

    def __getitem__(self, index):

        return self.forms[index]

    def __getattr__(self, name):

        try:
            return self.named_forms[name]
        except KeyError:
            raise AttributeError

    def _apply(self, method_name, *args, **kwargs):
        """Call ``method_name`` with args and kwargs on each member.

        Returns a sequence of return values.

        """

        return [
            getattr(member, method_name)(*args, **kwargs)
            for member in self.forms
        ]

    def get_default_prefix(self):

        return 'group'

    def add_prefix(self, field_name):
        """Return the field name with a prefix prepended."""

        return '%s-%s' % (self.prefix, field_name)

    def html_id(self, field_name, form=None):
        """Return the html ID for the given field_name."""

        if form is None:
            form = self

        return form.auto_id % (form.add_prefix(field_name),)

    def _full_clean(self):

        self._errors = []
        if not self.is_bound:
            return

        self._errors = [
            f.errors
            for f in self.forms
        ]

        try:
            self.clean()
        except ValidationError as e:
            self._group_errors = self.error_class(e.messages)

    def clean(self):
        """
        Hook for doing formgroup-wide cleaning/validation.

        Subclasses can override this to perform validation after
        .clean() has been called on every member.

        Any ValidationError raised by this method will be accessible
        via formgroup.group_errors().()

        """

    def is_valid(self):

        if not self.is_bound:
            return False

        self._full_clean()

        forms_valid = [
            f.is_valid()
            for f in self.forms
        ]

        return all(forms_valid)

    @property
    def errors(self):

        if self._errors is None:
            self._full_clean()

        return self._errors

    def group_errors(self):
        """
        Return the group level validation errors.

        Returns an ErrorList of errors that aren't associated with a
        particular form. Returns an empty ErrorList if there are none.

        """

        if self._group_errors is not None:
            return self._group_errors

        return self.error_class()

    def save(self):
        """Save the changes to the instance and any related objects."""

        # first call save with commit=False for all Forms
        for form in self._forms:
            if isinstance(form, BaseForm):
                form.save(commit=False)

        # call save on the instance
        self.instance.save()

        # call any post-commit hooks that have been stashed on Forms
        for form in self.forms:
            if isinstance(form, BaseForm):
                if hasattr(form, 'save_m2m'):
                    form.save_m2m()
                if hasattr(form, 'save_related'):
                    form.save_related()

        # call save on any formsets
        for form in self._forms:
            if isinstance(form, BaseFormSet):
                form.save(commit=True)

        return self.instance

    @property
    def media(self):

        group_media = self.forms[0].media

        for form in self.forms[1:]:
            group_media += form.media

        return group_media

class StateValidatorFormGroup(StateValidatorFormMixin, FormGroup):
    """

    Subclasses are expected to define the state_validators property,
    which is a mapping of states to StateValidator objects."""

    def get_errors(self, *states):

        return [form.get_errors(*states)
                   for form in self.forms
                   if isinstance(form, StateValidatorFormMixin)] + \
               [self.state_validators[state].errors(self)
                   for state in states]

    def is_valid(self, *states):
        """Returns True if no errors are thrown for the specified state."""

        # if no state is specified, fallback to the base FormGroup's is_valid
        if not states:
            return super(StateValidatorFormGroup, self).is_valid()

        # see if the states pass for all forms that define state_validators
        return all(form.is_valid(*states)
                   for form in self.forms
                   if isinstance(form, StateValidatorFormMixin)) and \
               all(self.state_validators[state].is_valid(self)
                   for state in states)

def formgroup_factory(form_classes,
                      formgroup=None,
                      state_validators=None,
                      ):
    """Return a FormGroup class for the given form[set] form_classes.

    """

    base_class = formgroup or FormGroup
    if state_validators is not None:
        base_class = StateValidatorFormGroup

    if not issubclass(base_class, FormGroup):
        raise TypeError("Base formgroup class must subclass FormGroup.")

    return type(
        'FormGroup',
        (base_class,),
        dict(
            form_classes=form_classes,
            state_validators=state_validators,
        ),
    )
<commit_msg>copying initial<commit_after>from django.core.exceptions import ValidationError
from django.forms.forms import BaseForm
from django.forms.formsets import BaseFormSet
from django.forms.models import BaseInlineFormSet
from django.forms.util import ErrorList

from rebar.validators import StateValidatorFormMixin

class Unspecified(object):
    """Unspecified Value."""
Unspecified = Unspecified()

class FormGroup(object):
    """Form-like wrapper for a heterogenous collection of Forms.

    A FormGroup collects an ordered set of Forms and/or FormSets,
    and provides convenience methods for validating the group as a
    whole.

    """

    def __init__(self,
                 data=None,
                 files=None,
                 auto_id='id_%s',
                 prefix=None,
                 initial=None,
                 label_suffix=':',
                 instance=Unspecified,
                 error_class=None,
                 member_kwargs=None):

        self.is_bound = data is not None or files is not None
        self.data = data or {}
        self.files = files or {}
        self.initial = initial or {}
        self.label_suffix = label_suffix
        self.instance = instance
        self.auto_id = auto_id
        self.error_class = error_class or ErrorList
        self._errors = None
        self._group_errors = None

        self.prefix = prefix or self.get_default_prefix()

        self.member_kwargs = member_kwargs or {}

        # instantiate the members
        self._forms = []
        self.named_forms = {}

        for member_class in self.form_classes:
            if isinstance(member_class, tuple):
                member_class, name = member_class
            else:
                name = member_class.__name__.lower()
                if name.endswith('form'):
                    name = name[:-4]

            kwargs = dict(
                prefix=self.add_prefix(name),
                data=data,
                files=files,
            )

            if issubclass(member_class, BaseForm):
                kwargs.update(dict(
                    auto_id=self.auto_id,
                    error_class=self.error_class,
                    label_suffix=self.label_suffix,
                    initial=self.initial.copy(),
                ))

            elif issubclass(member_class, BaseInlineFormSet):
                # inline formsets do not take additional kwargs
                pass

            elif issubclass(member_class, BaseFormSet):
                kwargs.update(dict(
                    auto_id=self.auto_id,
                    error_class=self.error_class,
                ))
            extra_kwargs = self.member_kwargs.get(name, {})
            if self.instance is not Unspecified:
                extra_kwargs['instance'] = self.instance
            kwargs.update(extra_kwargs)

            new_form = member_class(
                **kwargs
            )
            self.named_forms[name] = new_form
            self._forms.append(new_form)

    @property
    def forms(self):
        return self._forms

    def __len__(self):

        return len(self.forms)

    def __getitem__(self, index):

        return self.forms[index]

    def __getattr__(self, name):

        try:
            return self.named_forms[name]
        except KeyError:
            raise AttributeError

    def _apply(self, method_name, *args, **kwargs):
        """Call ``method_name`` with args and kwargs on each member.

        Returns a sequence of return values.

        """

        return [
            getattr(member, method_name)(*args, **kwargs)
            for member in self.forms
        ]

    def get_default_prefix(self):

        return 'group'

    def add_prefix(self, field_name):
        """Return the field name with a prefix prepended."""

        return '%s-%s' % (self.prefix, field_name)

    def html_id(self, field_name, form=None):
        """Return the html ID for the given field_name."""

        if form is None:
            form = self

        return form.auto_id % (form.add_prefix(field_name),)

    def _full_clean(self):

        self._errors = []
        if not self.is_bound:
            return

        self._errors = [
            f.errors
            for f in self.forms
        ]

        try:
            self.clean()
        except ValidationError as e:
            self._group_errors = self.error_class(e.messages)

    def clean(self):
        """
        Hook for doing formgroup-wide cleaning/validation.

        Subclasses can override this to perform validation after
        .clean() has been called on every member.

        Any ValidationError raised by this method will be accessible
        via formgroup.group_errors().()

        """

    def is_valid(self):

        if not self.is_bound:
            return False

        self._full_clean()

        forms_valid = [
            f.is_valid()
            for f in self.forms
        ]

        return all(forms_valid)

    @property
    def errors(self):

        if self._errors is None:
            self._full_clean()

        return self._errors

    def group_errors(self):
        """
        Return the group level validation errors.

        Returns an ErrorList of errors that aren't associated with a
        particular form. Returns an empty ErrorList if there are none.

        """

        if self._group_errors is not None:
            return self._group_errors

        return self.error_class()

    def save(self):
        """Save the changes to the instance and any related objects."""

        # first call save with commit=False for all Forms
        for form in self._forms:
            if isinstance(form, BaseForm):
                form.save(commit=False)

        # call save on the instance
        self.instance.save()

        # call any post-commit hooks that have been stashed on Forms
        for form in self.forms:
            if isinstance(form, BaseForm):
                if hasattr(form, 'save_m2m'):
                    form.save_m2m()
                if hasattr(form, 'save_related'):
                    form.save_related()

        # call save on any formsets
        for form in self._forms:
            if isinstance(form, BaseFormSet):
                form.save(commit=True)

        return self.instance

    @property
    def media(self):

        group_media = self.forms[0].media

        for form in self.forms[1:]:
            group_media += form.media

        return group_media

class StateValidatorFormGroup(StateValidatorFormMixin, FormGroup):
    """

    Subclasses are expected to define the state_validators property,
    which is a mapping of states to StateValidator objects."""

    def get_errors(self, *states):

        return [form.get_errors(*states)
                   for form in self.forms
                   if isinstance(form, StateValidatorFormMixin)] + \
               [self.state_validators[state].errors(self)
                   for state in states]

    def is_valid(self, *states):
        """Returns True if no errors are thrown for the specified state."""

        # if no state is specified, fallback to the base FormGroup's is_valid
        if not states:
            return super(StateValidatorFormGroup, self).is_valid()

        # see if the states pass for all forms that define state_validators
        return all(form.is_valid(*states)
                   for form in self.forms
                   if isinstance(form, StateValidatorFormMixin)) and \
               all(self.state_validators[state].is_valid(self)
                   for state in states)

def formgroup_factory(form_classes,
                      formgroup=None,
                      state_validators=None,
                      ):
    """Return a FormGroup class for the given form[set] form_classes.

    """

    base_class = formgroup or FormGroup
    if state_validators is not None:
        base_class = StateValidatorFormGroup

    if not issubclass(base_class, FormGroup):
        raise TypeError("Base formgroup class must subclass FormGroup.")

    return type(
        'FormGroup',
        (base_class,),
        dict(
            form_classes=form_classes,
            state_validators=state_validators,
        ),
    )
<|endoftext|>