<commit_before>from ctypes import *

mylib = cdll.LoadLibrary("osvrClientKit")

def OSVR_ClientContext(Structure):
    pass
def OSVR_ReturnCode(Structure):
    pass
def OSVR_ClientInterface(Structure):
    pass
def OSVR_DisplayConfigObject(Structure):
    pass
def OSVR_DisplayDimension(Structure):
    pass
def OSVR_ViewerCount(Structure):
    pass
def OSVR_Pose3(Structure):
    pass
def OSVR_EyeCount(Structure):
    pass
def OSVR_SurfaceCount(Structure):
    pass
def OSVR_RadialDistortionParameters(Structure):
    pass
#does CBool need to be redefined?

# ContextC.h functions

def osvrClientInit(applicationIdentifier, flags):
    mylib.osvrClientInit.argtypes = [c_char_p, c_uint32]
    mylib.osvrClientInit.restype = OSVR_ClientContext
    return mylib.osvrClientInit(c_char_p(applicationIdentifier), c_uint32(flags))

def osvrClientUpdate(ctx):
    mylib.osvrClientUpdate.argtypes = [OSVR_ClientContext]
    mylib.osvrClientUpdate.restype = OSVR_ReturnCode
    return mylib.osvrClientUpdate(ctx)

def osvrClientCheckStatus(ctx):
    mylib.osvrClientCheckStatus.argtypes = [OSVR_ClientContext]
    mylib.osvrClientCheckStatus.restype = OSVR_ReturnCode
    return mylib.osvrClientCheckStatus(ctx)

def osvrClientShutdown(ctx):
    mylib.osvrClientShutdown.argtypes = [OSVR_ClientContext]
    mylib.osvrClientShutdown.restype = OSVR_ReturnCode
    return mylib.osvrClientShutdown(ctx)

# DisplayC.h functions

def osvrClientGetDisplay(ctx, disp):
    mylib.osvrClientGetDisplay.argtypes = [OSVR_ClientContext, pointer(OSVR_DisplayConfig)]
    mylib.osvrClientGetDisplay.restype = OSVR_ReturnCode
    return mylib.osvrClientGetDisplay(ctx, disp)

def osvrClientFreeDisplay(disp):
    mylib.osvrClientFreeDisplay.argtypes = [OSVR_DisplayConfig]
    mylib.osvrClientFreeDisplay.restype = OSVR_ReturnCode
    return mylib.osvrClientFreeDisplay(disp)

def osvrClientCheckDisplayStartup(disp):
    mylib.osvrClientCheckDisplayStartup.argtypes = [OSVR_DisplayConfig]
    mylib.osvrClientCheckDisplayStartup.restype = OSVR_ReturnCode
    return mylib.osvrClientCheckDisplayStartup(disp)

def osvrClientGetNumDisplayInputs(disp, numDisplayInputs):
    mylib.osvrClientGetNumDisplayInputs.argtypes = [OSVR_DisplayConfig, pointer(OSVR_DisplayInputCount)]
    mylib.osvrClientGetNumDisplayInputs.restype = OSVR_ReturnCode
    return mylib.osvrClientGetNumDisplayInputs(disp, numDisplayInputs)

def osvrClientGetDisplayDimensions(disp, displayInputIndex, width, height):
    mylib.osvrClientGetDisplayDimensions.argtypes = [OSVR_DisplayConfig, OSVR_DisplayInputCount, pointer(OSVR_DisplayDimension), pointer(OSVR_DisplayDimension)]
    mylib.osvrClientGetDisplayDimensions.restype = OSVR_ReturnCode
    return mylib.osvrClientGetDisplayDimensions(disp, displayInputIndex, width, height)

def osvrClientGetNumViewers(disp, viewers):
    mylib.osvrClientGetNumViewers.argtypes = [OSVR_DisplayConfig, pointer(OSVR_ViewerCount)]
    mylib.osvrClientGetNumViewers.restype = OSVR_ReturnCode
    return mylib.osvrClientGetNumViewers(disp, pointer(viewers))

def osvrClientGetViewerPose(disp, viewer, pose):
    mylib.osvrClientGetViewerPose.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, pointer(OSVR_Pose3)]
    mylib.osvrClientGetViewerPose.restype = OSVR_ReturnCode
    return mylib.osvrClientGetViewerPose(disp, viewer, pointer(pose))

def osvrClientGetNumEyesForViewer(disp, viewer, eyes):
    mylib.osvrClientGetNumEyesForViewer.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, pointer(OSVR_EyeCount)]
    mylib.osvrClientGetNumEyesForViewer.restype = OSVR_ReturnCode
    return mylib.osvrClientGetNumEyesForViewer(disp, viewer, pointer(eyes))

def osvrClientGetViewerEyePose(disp, viewer, eye, pose):
    mylib.osvrClientGetViewerEyePose.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, OSVR_EyeCount, pointer(OSVR_Pose3)]
    mylib.osvrClientGetViewerEyePose.restype = OSVR_ReturnCode
    return mylib.osvrClientGetViewerPose(disp, viewer, eye, pointer(pose))

def osvrClientGetViewerEyeViewMatrixd(disp, viewer, eye, flags, mat):
    mylib.osvrClientGetViewerEyeViewMatrixd.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, OSVR_EyeCount, OSVR_MatrixConventions, pointer(c_double)]
    mylib.osvrClientGetViewerEyeViewMatrixd.restype = OSVR_ReturnCode
    return mylib.osvrClientGetViewerEyeViewMatrixd(disp, viewer, eye, flags, pointer(c_double(mat)))

def osvrClientGetViewerEyeViewMatrixf(disp, viewer, eye, flags, mat):
    mylib.osvrClientGetViewerEyeViewMatrixf.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, OSVR_EyeCount,rOSVR_MatrixConventions, pointer(c_float)]
    mylib.osvrClientGetViewerEyeViewMatrixf.restype = OSVR_ReturnCode
    return mylib.osvrClientGetViewerEyeViewMatrixd(disp, viewer, eye, flags, pointer(c_float(mat)))

def osvrClientGetNumSurfacesForViewerEye(disp, viewer, eye, surfaces):
    mylib.osvrClientGetNumSurfacesForViewerEye.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, OSVR_EyeCount, pointer(OSVR_SurfaceCount)]
    mylib.osvrClientGetNumSurfacesForViewerEye.restype = OSVR_ReturnCode
    return mylib.osvrClientGetNumSurfacesForViewerEye(disp, viewer, eye, pointer(surfaces))

def osvrClientGetRelativeViewportForViewerEyeSurface(disp, viewer, eye, surface, left, bottom, width, height):
    mylib.osvrClientGetRelativeViewportForViewerEyeSurface.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, OSVR_EyeCount, OSVR_SurfaceCount, pointer(OSVR_ViewportDimension), pointer(OSVR_ViewportDimension), pointer(OSVR_ViewportDimension), pointer(OSVR_ViewportDimension)]
    mylib.osvrClientGetRelativeViewportForViewerEyeSurface.restype = OSVR_ReturnCode
    return mylib.osvrClientGetRelativeViewportForViewerEyeSurface(disp, viewer, eye, surface, pointer(left), pointer(bottom), pointer(width), pointer(height))

def osvrClientGetViewerEyeSurfaceDisplayInputIndex(disp, viewer, eye, surface, displayInput):
    mylib.osvrClientGetViewerEyeSurfaceDisplayInputIndex.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, OSVR_EyeCount, OSVR_SurfaceCount, pointer(OSVR_DisplayInputCount)]
    mylib.osvrClientGetViewerEyeSurfaceDisplayInputIndex.restype = OSVR_ReturnCode
    return osvrClientGetViewerEyeSurfaceDisplayInputIndex(disp, viewer, eye, surface, pointer(displayInput))

def osvrClientGetViewerEyeSurfaceProjectionMatrixd(disp, viewer, eye, surface, near, far, flags, matrix):
    mylib.osvrClientGetViewerEyeSurfaceProjectionMatrixd.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, OSVR_EyeCount, OSVR_SurfaceCount, c_double, c_double, OSVR_MatrixConventions, pointer(c_double)]
    mylib.osvrClientGetViewerEyeSurfaceProjectionMatrixd.restype = OSVR_ReturnCode
    return mylib.osvrClientGetViewerEyeSurfaceProjectionMatrixd(disp, viewer, eye, surface, near, far, flags, pointer(matrix))

def osvrClientGetViewerEyeSurfaceProjectionMatrixf(disp, viewer, eye, surface, near, far, flags, matrix):
    mylib.osvrClientGetViewerEyeSurfaceProjectionMatrixf.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, OSVR_EyeCount, OSVR_SurfaceCount, c_double, c_double, OSVR_MatrixConventions, pointer(c_float)]
    mylib.osvrClientGetViewerEyeSurfaceProjectionMatrixf.restype = OSVR_ReturnCode
    return mylib.osvrClientGetViewerEyeSurfaceProjectionMatrixf(disp, viewer, eye, surface, near, far, flags, pointer(matrix))

def osvrClientGetViewerEyeSurfaceProjectionClippingPlanes(disp, viewer, eye, surface, left, right, bottom, top):
    mylib.osvrClientGetViewerEyeSurfaceProjectionClippingPlanes.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, OSVR_SurfaceCount, pointer(c_double), pointer(c_double), pointer(c_double), pointer(c_double)]
    mylib.osvrClientGetViewerEyeSurfaceProjectionClippingPlanes.restype = OSVR_ReturnCode
    return mylib.osvrClientGetViewerEyeSurfaceProjectionClippingPlanes(disp, viewer, eye, surface, pointer(left), pointer(right), pointer(bottom), pointer(top))

def osvrClientDoesViewerEyeSurfaceWantDistortion(disp, viewer, eye, surface, distortionRequested):
    mylib.osvrClientDoesViewerEyeSurfaceWantDistortion.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, OSVR_SurfaceCount, pointer(uint8_t)]
    mylib.osvrClientDoesViewerEyeSurfaceWantDistortion.restype = OSVR_ReturnCode
    return mylib.osvrClientDoesViewerEyeSurfaceWantDistortion(disp, viewer, eye, surface, pointer(distortionRequested))

def osvrClientGetViewerEyeSurfaceRadialDistortionPriority(disp, viewer, eye, surface, priority):
    mylib.osvrClientGetViewerEyeSurfaceRadialDistortionPriority.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, OSVR_SurfaceCount, pointer(uint32_t)]
    mylib.osvrClientGetViewerEyeSurfaceRadialDistortionPriority.restype = OSVR_ReturnCode
    return mylib.osvrClientGetViewerEyeSurfaceRadialDistortionPriority(disp, viewer, eye, surface, pointer(priority))

def osvrClientGetViewerEyeSurfaceRadialDistortion(disp, viewer, eye, surface, params):
    mylib.osvrClientGetViewerEyeSurfaceRadialDistortion.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, OSVR_EyeCount, OSVR_SurfaceCount, pointer(OSVR_RadialDistortionParameters)]
    mylib.osvrClientGetViewerEyeSurfaceRadialDistortion.restype = OSVR_ReturnCode
    return mylib.osvrClientGetViewerEyeSurfaceRadialDistortion(disp, viewer, eye, surface, pointer(params))

# ImagingC.h functions

def osvrClientFreeImage(ctx, buf):
    mylib.osvrClientFreeImage.argtypes = [OSVR_ClientContext, pointer(OSVR_ImageBufferElement)]
    mylib.osvrClientFreeImage.restype = OSVR_ReturnCode
    return mylib.osvrClientFreeImage(ctx, pointer(buf))

# InterfaceC.h functions

def osvrClientGetInterface(ctx, path, iface):
    mylib.osvrClientGetInterface.argtypes = [OSVR_ClientContext, c_char_p, pointer(OSVR_ClientInterface)]
    mylib.osvrClientGetInterace.restype = OSVR_ReturnCode
    return mylib.osvrClientGetInterface(ctx, c_char_p(path), iface)

def osvrClientFreeInterface(ctx, iface):
    mylib.osvrClientFreeInterface.argtypes = [OSVR_ClientContext, OSVR_ClientInterface]
    mylib.osvrClientFreeInterface.restype = OSVR_ReturnCode
    return mylib.osvrClientFreeInterface(ctx, iface)

# ParametersC.h functions

def osvrClientGetStringParameterLength(ctx, path, len):
    mylib.osvrClientGetStringParameterLength.argtypes = [OSVR_ClientContext, c_char_p, c_size_t]
    mylib.osvrClientGetStringParameterLength.restype = OSVR_ReturnCode
    return mylib.osvrGetStringParameterLength(ctx, c_char_p(path), c_size_t(len))

def osvrClientGetStringParameter(ctx, path, buf, len):
    mylib.osvrClientGetStringParameterLength.argtypes = [OSVR_ClientContext, c_char_p, c_char_p, c_size_t]
    mylib.osvrClientGetStringParameterLength.restype = OSVR_ReturnCode
    return mylib.osvrClientGetStringParameter(ctx, c_char_p(path), c_char_p(buf), c_size_t(len))
<commit_msg>added return_code checking<commit_after>from ctypes import *

mylib = cdll.LoadLibrary("osvrClientKit")

class OSVR_ClientContext(Structure):
    pass
class OSVR_ReturnCode(Structure):
    pass
class OSVR_Return_Failure(Structure):
    pass
class OSVR_ClientInterface(Structure):
    pass
class OSVR_DisplayConfigObject(Structure):
    pass
class OSVR_DisplayDimension(Structure):
    pass
class OSVR_ViewerCount(Structure):
    pass
class OSVR_Pose3(Structure):
    pass
class OSVR_EyeCount(Structure):
    pass
class OSVR_SurfaceCount(Structure):
    pass
class OSVR_RadialDistortionParameters(Structure):
    pass
#does CBool need to be redefined?

# Error checking

class ReturnError(Exception):
    def __init__(self, value, function):
        self.value = value
        self.function = function
    def __str__(self):
        return repr(self.function)

def checkReturn(returnValue, function):
    if returnValue == OSVR_Return_Failure:
        raise ReturnError(returnValue, function)

# ContextC.h functions

def osvrClientInit(applicationIdentifier, flags):
    mylib.osvrClientInit.argtypes = [c_char_p, c_uint32]
    mylib.osvrClientInit.restype = OSVR_ClientContext
    return mylib.osvrClientInit(c_char_p(applicationIdentifier), c_uint32(flags))

def osvrClientUpdate(ctx):
    mylib.osvrClientUpdate.argtypes = [OSVR_ClientContext]
    mylib.osvrClientUpdate.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientUpdate(ctx)
    checkReturn(returnvalue, 'osvrClientUpdate')
    return returnvalue

def osvrClientCheckStatus(ctx):
    mylib.osvrClientCheckStatus.argtypes = [OSVR_ClientContext]
    mylib.osvrClientCheckStatus.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientCheckStatus(ctx)
    checkReturn(returnvalue, 'osvrClientCheckStatus')
    return returnvalue

def osvrClientShutdown(ctx):
    mylib.osvrClientShutdown.argtypes = [OSVR_ClientContext]
    mylib.osvrClientShutdown.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientShutdown(ctx)
    checkReturn(returnvalue, 'osvrClientShutdown')
    return returnvalue

# DisplayC.h functions

def osvrClientGetDisplay(ctx, disp):
    mylib.osvrClientGetDisplay.argtypes = [OSVR_ClientContext, pointer(OSVR_DisplayConfig)]
    mylib.osvrClientGetDisplay.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientGetDisplay(ctx, pointer(disp))
    checkReturn(returnvalue, 'osvrClientGetDisplay')
    return returnvalue

def osvrClientFreeDisplay(disp):
    mylib.osvrClientFreeDisplay.argtypes = [OSVR_DisplayConfig]
    mylib.osvrClientFreeDisplay.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientFreeDisplay(disp)
    checkReturn(returnvalue, 'osvrClientFreeDisplay')
    return returnvalue

def osvrClientCheckDisplayStartup(disp):
    mylib.osvrClientCheckDisplayStartup.argtypes = [OSVR_DisplayConfig]
    mylib.osvrClientCheckDisplayStartup.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientCheckDisplayStartup(disp)
    checkReturn(returnvalue, 'osvrClientCheckDisplayStartup')
    return returnvalue

def osvrClientGetNumDisplayInputs(disp, numDisplayInputs):
    mylib.osvrClientGetNumDisplayInputs.argtypes = [OSVR_DisplayConfig, pointer(OSVR_DisplayInputCount)]
    mylib.osvrClientGetNumDisplayInputs.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientGetNumDisplayInputs(disp, numDisplayInputs)
    checkReturn(returnvalue, 'osvrClientGetNumDisplayInputs')
    return returnvalue

def osvrClientGetDisplayDimensions(disp, displayInputIndex, width, height):
    mylib.osvrClientGetDisplayDimensions.argtypes = [OSVR_DisplayConfig, OSVR_DisplayInputCount, pointer(OSVR_DisplayDimension), pointer(OSVR_DisplayDimension)]
    mylib.osvrClientGetDisplayDimensions.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientGetDisplayDimensions(disp, displayInputIndex, pointer(width), pointer(height))
    checkReturn(returnvalue, 'osvrClientGetDisplayDimensions')
    return returnvalue

def osvrClientGetNumViewers(disp, viewers):
    mylib.osvrClientGetNumViewers.argtypes = [OSVR_DisplayConfig, pointer(OSVR_ViewerCount)]
    mylib.osvrClientGetNumViewers.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientGetNumViewers(disp, pointer(viewers))
    checkReturn(returnvalue, 'osvrClientGetNumViewers')
    return returnvalue

def osvrClientGetViewerPose(disp, viewer, pose):
    mylib.osvrClientGetViewerPose.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, pointer(OSVR_Pose3)]
    mylib.osvrClientGetViewerPose.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientGetViewerPose(disp, viewer, pointer(pose))
    checkReturn(returnvalue, 'osvrClientGetViewerPose')
    return returnvalue

def osvrClientGetNumEyesForViewer(disp, viewer, eyes):
    mylib.osvrClientGetNumEyesForViewer.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, pointer(OSVR_EyeCount)]
    mylib.osvrClientGetNumEyesForViewer.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientGetNumEyesForViewer(disp, viewer, pointer(eyes))
    checkReturn(returnvalue, 'osvrClientGetNumEyesForViewer')
    return returnvalue

def osvrClientGetViewerEyePose(disp, viewer, eye, pose):
    mylib.osvrClientGetViewerEyePose.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, OSVR_EyeCount, pointer(OSVR_Pose3)]
    mylib.osvrClientGetViewerEyePose.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientGetViewerPose(disp, viewer, eye, pointer(pose))
    checkReturn(returnvalue, 'osvrClientGetViewerEyePose')
    return returnvalue

def osvrClientGetViewerEyeViewMatrixd(disp, viewer, eye, flags, mat):
    mylib.osvrClientGetViewerEyeViewMatrixd.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, OSVR_EyeCount, OSVR_MatrixConventions, pointer(c_double)]
    mylib.osvrClientGetViewerEyeViewMatrixd.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientGetViewerEyeViewMatrixd(disp, viewer, eye, flags, pointer(c_double(mat)))
    checkReturn(returnvalue, 'osvrClientGetViewerEyeViewMatrixd')
    return returnvalue

def osvrClientGetViewerEyeViewMatrixf(disp, viewer, eye, flags, mat):
    mylib.osvrClientGetViewerEyeViewMatrixf.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, OSVR_EyeCount,rOSVR_MatrixConventions, pointer(c_float)]
    mylib.osvrClientGetViewerEyeViewMatrixf.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientGetViewerEyeViewMatrixd(disp, viewer, eye, flags, pointer(c_float(mat)))
    checkReturn(returnvalue, 'osvrClientGetViewerEyeViewMatrixf')
    return returnvalue

def osvrClientGetNumSurfacesForViewerEye(disp, viewer, eye, surfaces):
    mylib.osvrClientGetNumSurfacesForViewerEye.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, OSVR_EyeCount, pointer(OSVR_SurfaceCount)]
    mylib.osvrClientGetNumSurfacesForViewerEye.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientGetNumSurfacesForViewerEye(disp, viewer, eye, pointer(surfaces))
    checkReturn(returnvalue, 'osvrClientGetNumSurfacesForViewerEye')
    return returnvalue

def osvrClientGetRelativeViewportForViewerEyeSurface(disp, viewer, eye, surface, left, bottom, width, height):
    mylib.osvrClientGetRelativeViewportForViewerEyeSurface.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, OSVR_EyeCount, OSVR_SurfaceCount, pointer(OSVR_ViewportDimension), pointer(OSVR_ViewportDimension), pointer(OSVR_ViewportDimension), pointer(OSVR_ViewportDimension)]
    mylib.osvrClientGetRelativeViewportForViewerEyeSurface.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientGetRelativeViewportForViewerEyeSurface(disp, viewer, eye, surface, pointer(left), pointer(bottom), pointer(width), pointer(height))
    checkReturn(returnvalue, 'osvrClientGetRelativeViewportForViewerEyeSurface')
    return returnvalue

def osvrClientGetViewerEyeSurfaceDisplayInputIndex(disp, viewer, eye, surface, displayInput):
    mylib.osvrClientGetViewerEyeSurfaceDisplayInputIndex.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, OSVR_EyeCount, OSVR_SurfaceCount, pointer(OSVR_DisplayInputCount)]
    mylib.osvrClientGetViewerEyeSurfaceDisplayInputIndex.restype = OSVR_ReturnCode
    returnvalue = osvrClientGetViewerEyeSurfaceDisplayInputIndex(disp, viewer, eye, surface, pointer(displayInput))
    checkReturn(returnvalue, 'osvrClientGetRelativeViewportEyeSurfaceDisplayInputIndex')
    return returnvalue

def osvrClientGetViewerEyeSurfaceProjectionMatrixd(disp, viewer, eye, surface, near, far, flags, matrix):
    mylib.osvrClientGetViewerEyeSurfaceProjectionMatrixd.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, OSVR_EyeCount, OSVR_SurfaceCount, c_double, c_double, OSVR_MatrixConventions, pointer(c_double)]
    mylib.osvrClientGetViewerEyeSurfaceProjectionMatrixd.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientGetViewerEyeSurfaceProjectionMatrixd(disp, viewer, eye, surface, near, far, flags, pointer(matrix))
    checkReturn(returnvalue, 'osvrClientGetViewerEyeSurfaceProjectionMatrixd')
    return returnvalue

def osvrClientGetViewerEyeSurfaceProjectionMatrixf(disp, viewer, eye, surface, near, far, flags, matrix):
    mylib.osvrClientGetViewerEyeSurfaceProjectionMatrixf.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, OSVR_EyeCount, OSVR_SurfaceCount, c_double, c_double, OSVR_MatrixConventions, pointer(c_float)]
    mylib.osvrClientGetViewerEyeSurfaceProjectionMatrixf.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientGetViewerEyeSurfaceProjectionMatrixf(disp, viewer, eye, surface, near, far, flags, pointer(matrix))
    checkReturn(returnvalue, 'osvrClientGetViewerEyeSurfaceProjectionMatrixf')
    return returnvalue

def osvrClientGetViewerEyeSurfaceProjectionClippingPlanes(disp, viewer, eye, surface, left, right, bottom, top):
    mylib.osvrClientGetViewerEyeSurfaceProjectionClippingPlanes.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, OSVR_SurfaceCount, pointer(c_double), pointer(c_double), pointer(c_double), pointer(c_double)]
    mylib.osvrClientGetViewerEyeSurfaceProjectionClippingPlanes.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientGetViewerEyeSurfaceProjectionClippingPlanes(disp, viewer, eye, surface, pointer(left), pointer(right), pointer(bottom), pointer(top))
    checkReturn(returnvalue, 'osvrClientGetViewerEyeSurfaceProjectionClippingPlanes')
    return returnvalue

def osvrClientDoesViewerEyeSurfaceWantDistortion(disp, viewer, eye, surface, distortionRequested):
    mylib.osvrClientDoesViewerEyeSurfaceWantDistortion.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, OSVR_SurfaceCount, pointer(uint8_t)]
    mylib.osvrClientDoesViewerEyeSurfaceWantDistortion.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientDoesViewerEyeSurfaceWantDistortion(disp, viewer, eye, surface, pointer(distortionRequested))
    checkReturn(returnvalue, 'osvrClientDoesViewerEyeSurfaceWantDistortion')
    return returnvalue

def osvrClientGetViewerEyeSurfaceRadialDistortionPriority(disp, viewer, eye, surface, priority):
    mylib.osvrClientGetViewerEyeSurfaceRadialDistortionPriority.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, OSVR_SurfaceCount, pointer(uint32_t)]
    mylib.osvrClientGetViewerEyeSurfaceRadialDistortionPriority.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientGetViewerEyeSurfaceRadialDistortionPriority(disp, viewer, eye, surface, pointer(priority))
    checkReturn(returnvalue, 'osvrClientGetViewerEyeSurfaceRadialDistortionPriority')
    return returnvalue

def osvrClientGetViewerEyeSurfaceRadialDistortion(disp, viewer, eye, surface, params):
    mylib.osvrClientGetViewerEyeSurfaceRadialDistortion.argtypes = [OSVR_DisplayConfig, OSVR_ViewerCount, OSVR_EyeCount, OSVR_SurfaceCount, pointer(OSVR_RadialDistortionParameters)]
    mylib.osvrClientGetViewerEyeSurfaceRadialDistortion.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientGetViewerEyeSurfaceRadialDistortion(disp, viewer, eye, surface, pointer(params))
    checkReturn(returnvalue, 'osvrClientGetViewerEyeSurfaceRadialDistortion')
    return returnvalue

# ImagingC.h functions

def osvrClientFreeImage(ctx, buf):
    mylib.osvrClientFreeImage.argtypes = [OSVR_ClientContext, pointer(OSVR_ImageBufferElement)]
    mylib.osvrClientFreeImage.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientFreeImage(ctx, pointer(buf))
    checkReturn(returnvalue, 'osvrClientFreeImage')
    return returnvalue

# InterfaceC.h functions

def osvrClientGetInterface(ctx, path, iface):
    mylib.osvrClientGetInterface.argtypes = [OSVR_ClientContext, c_char_p, pointer(OSVR_ClientInterface)]
    mylib.osvrClientGetInterace.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientGetInterface(ctx, c_char_p(path), iface)
    checkReturn(returnvalue, 'osvrClientGetInterface')
    return returnvalue

def osvrClientFreeInterface(ctx, iface):
    mylib.osvrClientFreeInterface.argtypes = [OSVR_ClientContext, OSVR_ClientInterface]
    mylib.osvrClientFreeInterface.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientFreeInterface(ctx, iface)
    checkReturn(returnvalue, 'osvrClientFreeInterface')
    return returnvalue

# ParametersC.h functions

def osvrClientGetStringParameterLength(ctx, path, len):
    mylib.osvrClientGetStringParameterLength.argtypes = [OSVR_ClientContext, c_char_p, c_size_t]
    mylib.osvrClientGetStringParameterLength.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrGetStringParameterLength(ctx, c_char_p(path), c_size_t(len))
    checkReturn(returnvalue, 'osvrClientGetStringParameterLength')
    return returnvalue

def osvrClientGetStringParameter(ctx, path, buf, len):
    mylib.osvrClientGetStringParameterLength.argtypes = [OSVR_ClientContext, c_char_p, c_char_p, c_size_t]
    mylib.osvrClientGetStringParameterLength.restype = OSVR_ReturnCode
    returnvalue = mylib.osvrClientGetStringParameter(ctx, c_char_p(path), c_char_p(buf), c_size_t(len))
    checkReturn(returnvalue, 'osvrClientGetStringParameter')
    return returnvalue
<|endoftext|>


<commit_before>'''fit many models on the real estate data

INVOCATION
  python fit.py DATA MODEL LASTMONTH NIEHGBORHOOD
where
 DATA         in {train, all} specifies which data in Working/samples2 to use
 MODEL        in {en, gb, rf} specified which model to use
 LASTMONTH    like YYYYMM specfies the last month of training data
 NEIGHBORHOOD in {all, city_name} specifies whether to train a model on all cities or just the specified city

INPUTS
 WORKING/samples2/train.csv or
 WORKING/samples2/all.csv

OUTPUTS
 WORKING/fit[-test]/DATA-MODEL-LASTMONTH-NEIGHBORHOOD/<filename>.pickle
where
 <filename> is the hyperparameters for the model
 the pickle files contains either
  (True, <fitted-model>)
  (False, <error message explaining why model could not be fitted)
'''

from __future__ import division

import argparse
import cPickle as pickle
import datetime
import gc
import os
import pandas as pd
import pdb
from pprint import pprint
import random
import sklearn
import sys
import time

from Bunch import Bunch
import dirutility
from Features import Features
import HPs
import layout_transactions
from Logger import Logger
from Month import Month
from Path import Path
from Timer import Timer

def make_control(argv):
    'return a Bunch'

    print argv
    parser = argparse.ArgumentParser()
    parser.add_argument('invocation')
    parser.add_argument('data', choices=['all', 'train'])
    parser.add_argument('model', choices=['en', 'gb', 'rf'])
    parser.add_argument('last_month')
    parser.add_argument('neighborhood')
    parser.add_argument('--test', action='store_true')
    parser.add_argument('--trace', action='store_true')
    arg = parser.parse_args(argv)
    arg.me = arg.invocation.split('.')[0]

    if arg.trace:
        pdb.set_trace()

    arg.last = Month(arg.last_month)  # convert to Month and validate value

    # convert arg.neighborhood into arg.all and arg.city
    arg.city = (
        None if arg.neighborhood == 'all' else
        arg.neighborhood.replace('_', ' ')
    )

    random_seed = 123
    random.seed(random_seed)

    dir_working = Path().dir_working()
    fit_dir = (
        os.path.join(dir_working, arg.me + '-test') if arg.test else
        os.path.join(dir_working, arg.me)
    )
    last_dir = '%s-%s-%s-%s' % (arg.data, arg.model, arg.last_month, arg.neighborhood)
    path_out_dir = os.path.join(fit_dir, last_dir, '')
    dirutility.assure_exists(path_out_dir)

    return Bunch(
        arg=arg,
        path_in_dir=os.path.join(dir_working, 'samples2', ''),
        path_out_dir=path_out_dir,
        path_out_log=os.path.join(path_out_dir, '0log.txt'),
        random_seed=random_seed,
        timer=Timer(),
    )

def select_in_time_period(df, last_month_str, n_months_back):
    'return subset of DataFrame df that are in the time period'

    first_date_float = float(Month(last_month_str).decrement(n_months_back).as_int() * 100 + 1)
    next_month = Month(last_month_str).increment()
    last_date = datetime.date(next_month.year, next_month.month, 1) - datetime.timedelta(1)
    last_date_float = last_date.year * 10000.0 + last_date.month * 100.0 + last_date.day

    sale_date_column = layout_transactions.sale_date
    sale_dates_float = df[sale_date_column]  # type is float
    assert isinstance(sale_dates_float.iloc[0], float)

    mask1 = sale_dates_float >= first_date_float
    mask2 = sale_dates_float <= last_date_float
    mask_in_range = mask1 & mask2
    df_in_range = df.loc[mask_in_range]
    return df_in_range

def select_in_city(df, city):
    'return subset of DataFrame df that are in the city'
    cities = df[layout_transactions.city]
    mask = cities == city
    df_in_city = df.loc[mask]
    return df_in_city

def select_in_time_period_and_in_city(df, last_month, n_months_back, city):
    'return new df with the specified training data'
    verbose = False
    in_time_period = select_in_time_period(
        df.copy(),
        last_month,
        n_months_back,
    )
    in_neighborhood = (
        in_time_period if city is None else
        select_in_city(in_time_period, city)
    )
    if verbose:
        print 'neighborhood %s: %d in time period, %d also in neighborhood' % (
            'all' if city is None else city,
            len(in_time_period),
            len(in_neighborhood),
        )
    return in_neighborhood

def fit_enOLD(dir_out, x, y, hps, random_seed, timer):
    'write one fitted model'
    verbose = False
    start_time = time.clock()
    assert len(hps) == 5
    model = sklearn.linear_model.ElasticNet(
        alpha=hps['alpha'],
        l1_ratio=hps['l1_ratio'],
        random_state=random_seed,
        # all these parameters are at the default value format skikit-learn version 0.18.1
        fit_intercept=True,
        normalize=False,
        max_iter=1000,
        copy_X=False,
        tol=0.0001,
        warm_start=False,
        selection='cyclic',
    )
    fitted = model.fit(x, y)
    out_filename = HPs.to_str(hps)
    path_out = os.path.join(dir_out, out_filename + '.pickle')
    with open(path_out, 'w') as f:
        obj = (True, fitted)  # True ==> fitted successfully
        pickle.dump(obj, f)
    if verbose:
        print 'fit and write en %s in wallclock secs %s' % (
            out_filename,
            time.clock() - start_time,
        )

def fit_en(X, y, hps, random_seed):
    'return fitted ElastNet model'
    assert len(hps) == 5
    model = sklearn.linear_model.ElasticNet(
        alpha=hps['alpha'],
        l1_ratio=hps['l1_ratio'],
        random_state=random_seed,
        # all these parameters are at the default value format skikit-learn version 0.18.1
        fit_intercept=True,
        normalize=False,
        max_iter=1000,
        copy_X=False,
        tol=0.0001,
        warm_start=False,
        selection='cyclic',
    )
    fitted = model.fit(X, y)
    return fitted

def fit_gb(X, y, hps, random_seed):
    'return fitted GradientBoostingRegressor model'
    assert len(hps) == 7
    model = sklearn.ensemble.GradientBoostingRegressor(
        learning_rate=hps['learning_rate'],
        n_estimators=hps['n_estimators'],
        max_depth=hps['max_depth'],
        max_features=hps['max_features'],
        random_state=random_seed,
        # all these parameters are at the default value format skikit-learn version 0.18.1
        loss='ls',
        criterion='friedman_mse',
        min_samples_split=2,
        min_samples_leaf=1,
        min_weight_fraction_leaf=0,
        subsample=1.0,
        max_leaf_nodes=None,
        min_impurity_split=1e-7,
        alpha=0.9,
        init=None,
        verbose=0,
        presort='auto',
    )
    fitted = model.fit(X, y)
    return fitted

def fit_rf(X, y, hps, random_seed):
    'return fitted RandomForestRegressor model'
    assert len(hps) == 6
    model = sklearn.ensemble.RandomForestRegressor(
        n_estimators=hps['n_estimators'],
        max_features=hps['max_features'],
        max_depth=hps['max_depth'],
        random_state=random_seed,
        # all these parameters are at the default value format skikit-learn version 0.18.1
        criterion='mse',
        min_samples_split=2,
        min_samples_leaf=1,
        min_weight_fraction_leaf=0,
        max_leaf_nodes=None,
        min_impurity_split=1e-7,
        bootstrap=True,
        oob_score=False,
        n_jobs=1,
        verbose=0,
        warm_start=False,
    )
    fitted = model.fit(X, y)
    return fitted

def num_hps(model):
    'return number of hyperparameters'
    count = 0
    for hps in HPs.iter_hps_model(model):
        count += 1
    return count

def do_work(control):
    'write fitted models to file system'
    path_in = os.path.join(control.path_in_dir, control.arg.data + '.csv')
    training_data = pd.read_csv(
        path_in,
        nrows=10 if control.arg.test else None,
        usecols=None,  # TODO: change to columns we actually use
        low_memory=False,
    )
    print 'read %d rows of training data from file %s' % (len(training_data), path_in)
    n_hps = num_hps(control.arg.model)
    count_fitted = 0
    for hps in HPs.iter_hps_model(control.arg.model):
        start_time = time.clock()
        relevant = select_in_time_period_and_in_city(
            training_data,
            control.arg.last_month,
            hps['n_months_back'],
            control.arg.city,
        )
        if len(relevant) == 0:
            print 'skipping fitting of model, because no training samples for those hyperparameters'
            continue

        X, y = Features().extract_and_transform(
            relevant,
            hps['units_X'],
            hps['units_y'],
        )

        fitted = (
            fit_en(X, y, hps, control.random_seed) if control.arg.model == 'en' else
            fit_gb(X, y, hps, control.random_seed) if control.arg.model == 'gb' else
            fit_rf(X, y, hps, control.random_seed)
        )
        with open(os.path.join(control.path_out_dir, HPs.to_str(hps) + '.pickle'), 'w') as f:
            obj = (
                (False, fitted) if isinstance(fitted, str) else
                (True, fitted)  # not an error message
            )
            pickle.dump(obj, f)

        count_fitted += 1
        print 'fitted #%4d/%4d on:%6d in: %6.2f %s %s %s %s hps: %s ' % (
            count_fitted,
            n_hps,
            len(relevant),
            time.clock() - start_time,            control.arg.data,
            control.arg.model,
            control.arg.last_month,
            control.arg.neighborhood,
            HPs.to_str(hps),

        )
        # gc.set_debug(gc.DEBUG_STATS + gc.DEBUG_UNCOLLECTABLE)
        # collect to get memory usage stable
        # this enables multiprocessing
        unreachable = gc.collect()
        if False and unreachable != 0:
            print 'gc reports %d unreachable objects' % unreachable
            pdb.set_trace()
        if control.arg.test and count_fitted == 5:
            print 'breaking because we are testing'
            break

def main(argv):
    control = make_control(argv)
    sys.stdout = Logger(control.path_out_log)  # now print statements also write to the log file
    print control
    lap = control.timer.lap

    do_work(control)

    lap('work completed')
    if control.arg.test:
        print 'DISCARD OUTPUT: test'
    print control
    print 'done'
    return

if __name__ == '__main__':
    if False:
        # avoid pyflakes warnings
        pdb.set_trace()
        pprint()

    main(sys.argv)
<commit_msg>add --dry invocation option<commit_after>'''fit many models on the real estate data

INVOCATION
  python fit.py DATA MODEL LASTMONTH NIEHGBORHOOD
where
 DATA         in {train, all} specifies which data in Working/samples2 to use
 MODEL        in {en, gb, rf} specified which model to use
 LASTMONTH    like YYYYMM specfies the last month of training data
 NEIGHBORHOOD in {all, city_name} specifies whether to train a model on all cities or just the specified city

INPUTS
 WORKING/samples2/train.csv or
 WORKING/samples2/all.csv

OUTPUTS
 WORKING/fit[-test]/DATA-MODEL-LASTMONTH-NEIGHBORHOOD/<filename>.pickle
where
 <filename> is the hyperparameters for the model
 the pickle files contains either
  (True, <fitted-model>)
  (False, <error message explaining why model could not be fitted)
'''

from __future__ import division

import argparse
import cPickle as pickle
import datetime
import gc
import os
import pandas as pd
import pdb
from pprint import pprint
import random
import sklearn
import sklearn.ensemble
import sklearn.linear_model
import sys
import time

from Bunch import Bunch
import dirutility
from Features import Features
import HPs
import layout_transactions
from Logger import Logger
from Month import Month
from Path import Path
from Timer import Timer

def make_control(argv):
    'return a Bunch'

    print argv
    parser = argparse.ArgumentParser()
    parser.add_argument('invocation')
    parser.add_argument('data', choices=['all', 'train'])
    parser.add_argument('model', choices=['en', 'gb', 'rf'])
    parser.add_argument('last_month')
    parser.add_argument('neighborhood')
    parser.add_argument('--test', action='store_true')
    parser.add_argument('--trace', action='store_true')
    parser.add_argument('--dry', action='store_true')     # don't write output
    arg = parser.parse_args(argv)
    arg.me = arg.invocation.split('.')[0]

    if arg.trace:
        pdb.set_trace()

    arg.last = Month(arg.last_month)  # convert to Month and validate value

    # convert arg.neighborhood into arg.all and arg.city
    arg.city = (
        None if arg.neighborhood == 'all' else
        arg.neighborhood.replace('_', ' ')
    )

    random_seed = 123
    random.seed(random_seed)

    dir_working = Path().dir_working()
    fit_dir = (
        os.path.join(dir_working, arg.me + '-test') if arg.test else
        os.path.join(dir_working, arg.me)
    )
    last_dir = '%s-%s-%s-%s' % (arg.data, arg.model, arg.last_month, arg.neighborhood)
    path_out_dir = os.path.join(fit_dir, last_dir, '')
    dirutility.assure_exists(path_out_dir)

    return Bunch(
        arg=arg,
        path_in_dir=os.path.join(dir_working, 'samples2', ''),
        path_out_dir=path_out_dir,
        path_out_log=os.path.join(path_out_dir, '0log.txt'),
        random_seed=random_seed,
        timer=Timer(),
    )

def select_in_time_period(df, last_month_str, n_months_back):
    'return subset of DataFrame df that are in the time period'

    first_date_float = float(Month(last_month_str).decrement(n_months_back).as_int() * 100 + 1)
    next_month = Month(last_month_str).increment()
    last_date = datetime.date(next_month.year, next_month.month, 1) - datetime.timedelta(1)
    last_date_float = last_date.year * 10000.0 + last_date.month * 100.0 + last_date.day

    sale_date_column = layout_transactions.sale_date
    sale_dates_float = df[sale_date_column]  # type is float
    assert isinstance(sale_dates_float.iloc[0], float)

    mask1 = sale_dates_float >= first_date_float
    mask2 = sale_dates_float <= last_date_float
    mask_in_range = mask1 & mask2
    df_in_range = df.loc[mask_in_range]
    return df_in_range

def select_in_city(df, city):
    'return subset of DataFrame df that are in the city'
    cities = df[layout_transactions.city]
    mask = cities == city
    df_in_city = df.loc[mask]
    return df_in_city

def select_in_time_period_and_in_city(df, last_month, n_months_back, city):
    'return new df with the specified training data'
    verbose = False
    in_time_period = select_in_time_period(
        df.copy(),
        last_month,
        n_months_back,
    )
    in_neighborhood = (
        in_time_period if city is None else
        select_in_city(in_time_period, city)
    )
    if verbose:
        print 'neighborhood %s: %d in time period, %d also in neighborhood' % (
            'all' if city is None else city,
            len(in_time_period),
            len(in_neighborhood),
        )
    return in_neighborhood

def fit_enOLD(dir_out, x, y, hps, random_seed, timer):
    'write one fitted model'
    verbose = False
    start_time = time.clock()
    assert len(hps) == 5
    model = sklearn.linear_model.ElasticNet(
        alpha=hps['alpha'],
        l1_ratio=hps['l1_ratio'],
        random_state=random_seed,
        # all these parameters are at the default value format skikit-learn version 0.18.1
        fit_intercept=True,
        normalize=False,
        max_iter=1000,
        copy_X=False,
        tol=0.0001,
        warm_start=False,
        selection='cyclic',
    )
    fitted = model.fit(x, y)
    out_filename = HPs.to_str(hps)
    path_out = os.path.join(dir_out, out_filename + '.pickle')
    with open(path_out, 'w') as f:
        obj = (True, fitted)  # True ==> fitted successfully
        pickle.dump(obj, f)
    if verbose:
        print 'fit and write en %s in wallclock secs %s' % (
            out_filename,
            time.clock() - start_time,
        )

def fit_en(X, y, hps, random_seed):
    'return fitted ElastNet model'
    assert len(hps) == 5
    model = sklearn.linear_model.ElasticNet(
        alpha=hps['alpha'],
        l1_ratio=hps['l1_ratio'],
        random_state=random_seed,
        # all these parameters are at the default value format skikit-learn version 0.18.1
        fit_intercept=True,
        normalize=False,
        max_iter=1000,
        copy_X=False,
        tol=0.0001,
        warm_start=False,
        selection='cyclic',
    )
    fitted = model.fit(X, y)
    return fitted

def fit_gb(X, y, hps, random_seed):
    'return fitted GradientBoostingRegressor model'
    assert len(hps) == 7
    model = sklearn.ensemble.GradientBoostingRegressor(
        learning_rate=hps['learning_rate'],
        n_estimators=hps['n_estimators'],
        max_depth=hps['max_depth'],
        max_features=hps['max_features'],
        random_state=random_seed,
        # all these parameters are at the default value format skikit-learn version 0.18.1
        loss='ls',
        criterion='friedman_mse',
        min_samples_split=2,
        min_samples_leaf=1,
        min_weight_fraction_leaf=0,
        subsample=1.0,
        max_leaf_nodes=None,
        min_impurity_split=1e-7,
        alpha=0.9,
        init=None,
        verbose=0,
        presort='auto',
    )
    fitted = model.fit(X, y)
    return fitted

def fit_rf(X, y, hps, random_seed):
    'return fitted RandomForestRegressor model'
    assert len(hps) == 6
    model = sklearn.ensemble.RandomForestRegressor(
        n_estimators=hps['n_estimators'],
        max_features=hps['max_features'],
        max_depth=hps['max_depth'],
        random_state=random_seed,
        # all these parameters are at the default value format skikit-learn version 0.18.1
        criterion='mse',
        min_samples_split=2,
        min_samples_leaf=1,
        min_weight_fraction_leaf=0,
        max_leaf_nodes=None,
        min_impurity_split=1e-7,
        bootstrap=True,
        oob_score=False,
        n_jobs=1,
        verbose=0,
        warm_start=False,
    )
    fitted = model.fit(X, y)
    return fitted

def num_hps(model):
    'return number of hyperparameters'
    count = 0
    for hps in HPs.iter_hps_model(model):
        count += 1
    return count

def do_work(control):
    'write fitted models to file system'
    path_in = os.path.join(control.path_in_dir, control.arg.data + '.csv')
    training_data = pd.read_csv(
        path_in,
        nrows=10 if control.arg.test else None,
        usecols=None,  # TODO: change to columns we actually use
        low_memory=False,
    )
    print 'read %d rows of training data from file %s' % (len(training_data), path_in)
    n_hps = num_hps(control.arg.model)
    count_fitted = 0
    for hps in HPs.iter_hps_model(control.arg.model):
        start_time = time.clock()
        relevant = select_in_time_period_and_in_city(
            training_data,
            control.arg.last_month,
            hps['n_months_back'],
            control.arg.city,
        )
        if len(relevant) == 0:
            print 'skipping fitting of model, because no training samples for those hyperparameters'
            continue

        X, y = Features().extract_and_transform(
            relevant,
            hps['units_X'],
            hps['units_y'],
        )

        fitted = (
            fit_en(X, y, hps, control.random_seed) if control.arg.model == 'en' else
            fit_gb(X, y, hps, control.random_seed) if control.arg.model == 'gb' else
            fit_rf(X, y, hps, control.random_seed)
        )
        if not control.arg.dry:
            with open(os.path.join(control.path_out_dir, HPs.to_str(hps) + '.pickle'), 'w') as f:
                obj = (
                    (False, fitted) if isinstance(fitted, str) else
                    (True, fitted)  # not an error message
                )
                pickle.dump(obj, f)

        count_fitted += 1
        print 'fitted #%4d/%4d on:%6d in: %6.2f %s %s %s %s hps: %s ' % (
            count_fitted,
            n_hps,
            len(relevant),
            time.clock() - start_time,            control.arg.data,
            control.arg.model,
            control.arg.last_month,
            control.arg.neighborhood,
            HPs.to_str(hps),

        )
        # gc.set_debug(gc.DEBUG_STATS + gc.DEBUG_UNCOLLECTABLE)
        # collect to get memory usage stable
        # this enables multiprocessing
        unreachable = gc.collect()
        if False and unreachable != 0:
            print 'gc reports %d unreachable objects' % unreachable
            pdb.set_trace()
        if control.arg.test and count_fitted == 5:
            print 'breaking because we are testing'
            break

def main(argv):
    control = make_control(argv)
    sys.stdout = Logger(control.path_out_log)  # now print statements also write to the log file
    print control
    lap = control.timer.lap

    do_work(control)

    lap('work completed')
    if control.arg.test:
        print 'DISCARD OUTPUT: test'
    print control
    print 'done'
    return

if __name__ == '__main__':
    if False:
        # avoid pyflakes warnings
        pdb.set_trace()
        pprint()

    main(sys.argv)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Designed for use with boofuzz v0.0.1-dev3
from boofuzz import *
import click

@click.group()
def cli():
    pass

@click.command()
@click.option('--target-host', help='Host or IP address of target', prompt=True)
@click.option('--target-port', type=int, default=21, help='Network port of target', prompt=True)
@click.option('--username', help='FTP username', prompt=True)
@click.option('--password', help='FTP password', prompt=True)
@click.option('--test-case-index', help='Test case index', type=int)
@click.option('--test-case-name', help='Name of node or specific test case')
@click.option('--csv-out', help='Output to CSV file')
@click.option('--sleep-between-cases', help='Wait time between test cases (floating point)', type=float, default=0)
def fuzz(target_host, target_port, username, password, test_case_index, test_case_name, csv_out, sleep_between_cases):
    fuzz_loggers = [FuzzLoggerText()]
    if csv_out is not None:
        f = open('ftp-fuzz.csv', 'wb')
        fuzz_loggers.append(FuzzLoggerCsv(file_handle=f))

    session = Session(
        target=Target(
            connection=SocketConnection(target_host, target_port, proto='tcp')),
        fuzz_data_logger=FuzzLogger(fuzz_loggers=fuzz_loggers),
        sleep_time=sleep_between_cases
    )

    initialize_ftp(session, username, password)

    if test_case_index is not None:
        session.fuzz_single_case(mutant_index=test_case_index)
    elif test_case_name is not None:
        session.fuzz_by_name(test_case_name)
    else:
        session.fuzz()

def initialize_ftp(session, username, password):
    s_initialize("user")
    s_string("USER")
    s_delim(" ")
    s_string(username.encode('ascii'))
    s_static("\r\n")

    s_initialize("pass")
    s_string("PASS")
    s_delim(" ")
    s_string(password.encode('ascii'))
    s_static("\r\n")

    s_initialize("stor")
    s_string("STOR")
    s_delim(" ")
    s_string("AAAA")
    s_static("\r\n")

    s_initialize("retr")
    s_string("RETR")
    s_delim(" ")
    s_string("AAAA")
    s_static("\r\n")

    session.connect(s_get("user"))
    session.connect(s_get("user"), s_get("pass"))
    session.connect(s_get("pass"), s_get("stor"))
    session.connect(s_get("pass"), s_get("retr"))

cli.add_command(fuzz)

if __name__ == "__main__":
    cli()
<commit_msg>Add procmon options<commit_after>#!/usr/bin/env python
# Designed for use with boofuzz v0.0.1-dev3
from boofuzz import *
import click

@click.group()
def cli():
    pass

@click.command()
@click.option('--target-host', help='Host or IP address of target', prompt=True)
@click.option('--target-port', type=int, default=21, help='Network port of target', prompt=True)
@click.option('--username', help='FTP username', prompt=True)
@click.option('--password', help='FTP password', prompt=True)
@click.option('--test-case-index', help='Test case index', type=int)
@click.option('--test-case-name', help='Name of node or specific test case')
@click.option('--csv-out', help='Output to CSV file')
@click.option('--sleep-between-cases', help='Wait time between test cases (floating point)', type=float, default=0)
@click.option('--procmon-host', help='Process monitor port host or IP')
@click.option('--procmon-port', type=int, default=26002, help='Process monitor port')
@click.option('--procmon-start', help='Process monitor start command')
def fuzz(target_host, target_port, username, password, test_case_index, test_case_name, csv_out, sleep_between_cases,
         procmon_host, procmon_port, procmon_start):
    fuzz_loggers = [FuzzLoggerText()]
    if csv_out is not None:
        f = open('ftp-fuzz.csv', 'wb')
        fuzz_loggers.append(FuzzLoggerCsv(file_handle=f))

    if procmon_host is not None:
        procmon = pedrpc.Client(procmon_host, procmon_port)
    else:
        procmon = None

    procmon_options = {}
    if procmon_start is not None:
        procmon_options['start_commands'] = [procmon_start]

    session = Session(
        target=Target(
            connection=SocketConnection(target_host, target_port, proto='tcp'),
            procmon=procmon,
            procmon_options=procmon_options,
        ),
        fuzz_data_logger=FuzzLogger(fuzz_loggers=fuzz_loggers),
        sleep_time=sleep_between_cases
    )

    initialize_ftp(session, username, password)

    if test_case_index is not None:
        session.fuzz_single_case(mutant_index=test_case_index)
    elif test_case_name is not None:
        session.fuzz_by_name(test_case_name)
    else:
        session.fuzz()

def initialize_ftp(session, username, password):
    s_initialize("user")
    s_string("USER")
    s_delim(" ")
    s_string(username.encode('ascii'))
    s_static("\r\n")

    s_initialize("pass")
    s_string("PASS")
    s_delim(" ")
    s_string(password.encode('ascii'))
    s_static("\r\n")

    s_initialize("stor")
    s_string("STOR")
    s_delim(" ")
    s_string("AAAA")
    s_static("\r\n")

    s_initialize("retr")
    s_string("RETR")
    s_delim(" ")
    s_string("AAAA")
    s_static("\r\n")

    session.connect(s_get("user"))
    session.connect(s_get("user"), s_get("pass"))
    session.connect(s_get("pass"), s_get("stor"))
    session.connect(s_get("pass"), s_get("retr"))

cli.add_command(fuzz)

if __name__ == "__main__":
    cli()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# GAme Format Parser

class ParseError(Exception):
    def __init__(self, message):
        self.message = message

class CharFile:
    def __init__(self, f):
        self.data = f.read()
        self.index = 0

    def read(self):
        c = self.data[self.index]
        self.index += 1
        return c

    def peek(self):
        return self.data[self.index]

def read_char(f):
    return f.read()

def peek_char(f):
    return f.peek()

def is_space(ch):
    if ch == ' ':
        return True
    if ch == '\n':
        return True
    if ch == '\r':
        return True
    if ch == '\t':
        return True
    return False

def is_ident(first, ch):
    if 'a' <= ch <= 'z':
        return True
    if 'A' <= ch <= 'Z':
        return True
    if ch == '_':
        return True
    if first is False:
        if '0' <= ch <= '9':
            return True
    return False

def is_valid_type(ty):
    if ty == 'int8':
        return True
    if ty == 'int16':
        return True
    if ty == 'int32':
        return True
    if ty == 'int64':
        return True
    if ty == 'float':
        return True
    if ty == 'double':
        return True
    if ty == 'byte':
        return True
    return False

def read_spaces(f):
    while is_space(peek_char(f)):
        read_char(f)

def read_ident(f):
    read_spaces(f)
    ident = ''
    first = True
    while is_ident(first, peek_char(f)):
        ident += read_char(f)
        first = False
    if len(ident) == 0:
        raise ParseError('expecting ident but found {}'.format(peek_char(f)))
    return ident

def read_single_char(f, ch):
    read_spaces(f)
    r = read_char(f)
    if r == ch:
        pass
    else:
        raise ParseError('expecting char {c}, but found {r}'.format(c=ch, r=r))

def read_struct(f):
    struct = read_ident(f)
    if struct != 'struct':
        raise ParseError('expected struct found ident {}'.format(struct))
    typename = read_ident(f)
    read_single_char(f, '{')
    while peek_char(f) != '}':
        ty = read_ident(f)
        if is_valid_type(ty) is False:
            raise ParseError('Inavlid type {}'.format(ty))
        name = read_ident(f)
        # todo: add default value
        read_single_char(f, ';')
        read_spaces(f)
    read_single_char(f, '}')

def on_generate_command(args):
    read_struct(CharFile(args.file))

def main():
    import argparse
    parser = argparse.ArgumentParser(description='GAme Format parser')
    sub = parser.add_subparsers(help='sub-command help')

    gen_parser = sub.add_parser('generate', help='generate a game format parser', aliases=['gen'])
    gen_parser.add_argument('file', type=argparse.FileType('r'))
    gen_parser.set_defaults(func=on_generate_command)

    args = parser.parse_args()
    args.func(args)

if __name__ == "__main__":
    main()
<commit_msg>returning struct<commit_after>#!/usr/bin/env python3
# GAme Format Parser

class ParseError(Exception):
    def __init__(self, message):
        self.message = message

class CharFile:
    def __init__(self, f):
        self.data = f.read()
        self.index = 0

    def read(self):
        c = self.data[self.index]
        self.index += 1
        return c

    def peek(self):
        return self.data[self.index]

def read_char(f):
    return f.read()

def peek_char(f):
    return f.peek()

def is_space(ch):
    if ch == ' ':
        return True
    if ch == '\n':
        return True
    if ch == '\r':
        return True
    if ch == '\t':
        return True
    return False

def is_ident(first, ch):
    if 'a' <= ch <= 'z':
        return True
    if 'A' <= ch <= 'Z':
        return True
    if ch == '_':
        return True
    if first is False:
        if '0' <= ch <= '9':
            return True
    return False

def is_valid_type(ty):
    if ty == 'int8':
        return True
    if ty == 'int16':
        return True
    if ty == 'int32':
        return True
    if ty == 'int64':
        return True
    if ty == 'float':
        return True
    if ty == 'double':
        return True
    if ty == 'byte':
        return True
    return False

def read_spaces(f):
    while is_space(peek_char(f)):
        read_char(f)

def read_ident(f):
    read_spaces(f)
    ident = ''
    first = True
    while is_ident(first, peek_char(f)):
        ident += read_char(f)
        first = False
    if len(ident) == 0:
        raise ParseError('expecting ident but found {}'.format(peek_char(f)))
    return ident

def read_single_char(f, ch):
    read_spaces(f)
    r = read_char(f)
    if r == ch:
        pass
    else:
        raise ParseError('expecting char {c}, but found {r}'.format(c=ch, r=r))

class Member:
    def __init__(self, name, typename):
        self.name = name
        self.typename = typename

class Struct:
    def __init__(self, name):
        self.name = name
        self.members = []

    def add_member(self, member):
        self.members.append(member)

def read_struct(f):
    struct_keyword = read_ident(f)
    if struct_keyword != 'struct':
        raise ParseError('expected struct found ident {}'.format(struct_keyword))
    struct_name = read_ident(f)
    struct = Struct(struct_name)
    read_single_char(f, '{')
    while peek_char(f) != '}':
        ty = read_ident(f)
        if is_valid_type(ty) is False:
            raise ParseError('Inavlid type {}'.format(ty))
        name = read_ident(f)
        mem = Member(name, ty)
        # todo: add default value
        read_single_char(f, ';')
        struct.add_member(mem)
        read_spaces(f)
    read_single_char(f, '}')
    return struct

def on_generate_command(args):
    s = read_struct(CharFile(args.file))
    print(s)

def main():
    import argparse
    parser = argparse.ArgumentParser(description='GAme Format parser')
    sub = parser.add_subparsers(help='sub-command help')

    gen_parser = sub.add_parser('generate', help='generate a game format parser', aliases=['gen'])
    gen_parser.add_argument('file', type=argparse.FileType('r'))
    gen_parser.set_defaults(func=on_generate_command)

    args = parser.parse_args()
    args.func(args)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Generator for model"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from builtins import range

import argparse
import os
import sys
import random
import pickle
from math import exp

import torch
from torch.autograd import Variable
# Import my own cleaning lib, use jieba for other user
try:
    from purewords import clean_sentence as clean
except ImportError:
    from jieba import lcut as clean
import model
import utils
from utils import check_cuda_for_var, check_directory

parser = argparse.ArgumentParser(description=\
        "Generator for HRNN/Seq2seq")
parser.add_argument('--data', type=str,
        help="location of the data corpus(json file)")
parser.add_argument('--type', type=str,
        help="generate dialog with hrnn/seq2seq model")
parser.add_argument('--save', type=str, default='model/',
        help='path to load the final model\'s directory')
parser.add_argument('--seed', type=int, default=55665566,
        help='random seed')
parser.add_argument('--beam', type=int, default=1,
        help='beam size for beam search(default 1 will be greedy search)')
parser.add_argument('--eodlong', type=int, default=0,
        help='whether force model to gen a longer dialog (1 for on, 0 for off, default = 0)')
parser.add_argument('--nosr', type=int, default=0,
        help='whether force model don\'t self repeat (1 for on, 0 for off, default = 0)')
args = parser.parse_args()

torch.manual_seed(args.seed)
random.seed(args.seed)

DEBUG = False

if args.type != "hrnn" and args.type != "seq2seq":
    raise ValueError("args.type should be hrnn or seq2seq, but got %s" % (args.type))
if args.beam <= 0:
    raise ValueError("args.beam should be at least 1 or larger number")
if not os.path.isfile('dict.pkl'):
    my_lang, _ = utils.build_lang(args.data)
    with open('dict.pkl', 'wb') as filename:
        pickle.dump(my_lang, filename)
else:
    print("Load dict.pkl")
    with open('dict.pkl', 'rb') as filename:
        my_lang = pickle.load(filename)
if args.type == "hrnn":
    # Load last HRNN model
    number = torch.load(os.path.join(args.save, 'checkpoint.pt'))
    encoder = torch.load(os.path.join(args.save, 'encoder'+str(number)+'.pt'))
    context = torch.load(os.path.join(args.save, 'context'+str(number)+'.pt'))
    decoder = torch.load(os.path.join(args.save, 'decoder'+str(number)+'.pt'))
    if torch.cuda.is_available():
        encoder = encoder.cuda()
        context = context.cuda()
        decoder = decoder.cuda()
    def gen(sentence):
        encoder.eval()
        context.eval()
        decoder.eval()

        # Inference
        gen_sentence = []
        talking_history = []
        context_hidden = context.init_hidden()
        max_dialog_len = 20
        max_sentence_len = 15
        beam_size = args.beam
        for _ in range(max_dialog_len):
            decoder_input = Variable(torch.LongTensor([[my_lang.word2index["SOS"]]]))
            decoder_input = check_cuda_for_var(decoder_input)
            encoder_hidden = encoder.init_hidden()
            decoder_hidden = decoder.init_hidden()
            if len(gen_sentence) > 0:
                for ei in range(len(gen_sentence)):
                    _, encoder_hidden = encoder(gen_sentence[ei], encoder_hidden)
                # Clean generated sentence list
                gen_sentence = []
            else:
                for ei in range(len(sentence)):
                    _, encoder_hidden = encoder(sentence[ei], encoder_hidden)
            context_output, context_hidden = context(encoder_hidden, context_hidden)
            # Beam search
            index2state = {}
            for index in range(beam_size):
                index2state[index] = [decoder_input, decoder_hidden, [decoder_input.data[0][0]], 0.0]
            # One step to get beam_size candidates
            decoder_output, decoder_hidden = decoder(context_hidden,\
                    decoder_input, decoder_hidden)
            scores, topi = decoder_output.data.topk(beam_size)
            for index in range(beam_size):
                ni = topi[0][index]
                index2state[index][0] = check_cuda_for_var(Variable(torch.LongTensor([[ni]])))
                index2state[index][1] = decoder_hidden
                index2state[index][2].append(ni)
                index2state[index][3] = scores[0][index]
            for sentence_pointer in range(max_sentence_len):
                current_scores = []
                current2state = {}
                # Init current2state
                for index in range(beam_size):
                    for jndex in range(beam_size):
                        current2state[index * beam_size + jndex] = [0, 0, 0, 0]
                for index in range(beam_size):
                    output, hidden = decoder(context_hidden, \
                            index2state[index][0], index2state[index][1])
                    tops, topi = output.data.topk(beam_size)
                    for jndex in range(beam_size):
                        ni = topi[0][jndex]
                        current_map = current2state[index * beam_size + jndex]
                        current_map[0] = check_cuda_for_var(Variable(torch.LongTensor([[ni]])))
                        current_map[1] = hidden
                        current_map[2] = index2state[index][2][:]
                        current_map[2].append(ni)
                        current_map[3] = tops[0][jndex] + index2state[index][3]
                        if args.eodlong == 1 and my_lang.word2index["EOD"] in current_map[2]:
                            current_map[3] *= exp(max_sentence_len - 12 - sentence_pointer)
                        current_scores.append(current_map[3])

                _, top_of_beamsize2 = torch.FloatTensor(current_scores).topk(beam_size)
                # Top beam's output is eos, break and output the top beam
                if current2state[top_of_beamsize2[0]][2][-1] == my_lang.word2index["EOS"]:
                    if args.nosr == 1 and current2state[top_of_beamsize2[0]][2] in talking_history:
                        # Don't repeat itself
                        # Soft verion
                        current2state[top_of_beamsize2[0]][3] *= 2
                        # Hard version
                        #current2state[top_of_beamsize2[0][3]] *= 100000.0
                    else:
                        first_eos = current2state[top_of_beamsize2[0]][2].index(my_lang.word2index["EOS"])
                        gen_sentence = current2state[top_of_beamsize2[0]][2][:first_eos+1]
                        break
                after_beam_dict = {}
                for index, candidate in enumerate(top_of_beamsize2):
                    after_beam_dict[index] = current2state[candidate]
                index2state = after_beam_dict
            # Beam Search a good sentence and assign to gen_sentence
            talking_history.append(gen_sentence)
            gen_sentence = Variable(torch.LongTensor(gen_sentence))
            gen_sentence = check_cuda_for_var(gen_sentence)
            try:
                string = ' '.join([my_lang.index2word[word.data[0]] for word in gen_sentence])
                print(string)
                if "EOD" in string:
                    break
            except RuntimeError:
                break
        return talking_history
else:
    # Load last Seq2seq model
    number = torch.load(os.path.join(args.save, 'checkpoint.pt'))
    encoder = torch.load(os.path.join(args.save, 'encoder'+str(number)+'.pt'))
    decoder = torch.load(os.path.join(args.save, 'decoder'+str(number)+'.pt'))
    if torch.cuda.is_available():
        encoder = encoder.cuda()
        decoder = decoder.cuda()
    def gen(sentence):
        max_length = 20
        encoder.eval()
        decoder.eval()
        talking_history = []
        gen_sentence = []
        counter = 0
        while counter < 10:
            encoder_hidden = encoder.init_hidden()
            encoder_outputs = Variable(torch.zeros(max_length, encoder.hidden_size))
            decoder_input = Variable(torch.LongTensor([[my_lang.word2index["SOS"]]]))
            encoder_outputs = check_cuda_for_var(encoder_outputs)
            decoder_input = check_cuda_for_var(decoder_input)
            if len(gen_sentence) > 0:
                for ei in range(len(gen_sentence)):
                    encoder_output, encoder_hidden = encoder(gen_sentence[ei], encoder_hidden)
                    encoder_outputs[ei] = encoder_output[0][0]
                    # Clean generated sentence list
                gen_sentence = []
            else:
                for ei in range(len(sentence)):
                    encoder_output, encoder_hidden = encoder(sentence[ei], encoder_hidden)
                    encoder_outputs[ei] = encoder_output[0][0]
            decoder_hidden = encoder_hidden
            while True:
                if DEBUG:
                    print("[Debug] ", decoder_input.data)
                gen_sentence.append(decoder_input.data[0][0])
                if gen_sentence[-1] == my_lang.word2index["EOS"] or len(gen_sentence) >= max_length - 1:
                    break
                decoder_output, decoder_hidden, decoder_attention = decoder(decoder_input, decoder_hidden, \
                        encoder_outputs)
                _, topi = decoder_output.data.topk(1)
                ni = topi[0][0]
                decoder_input = Variable(torch.LongTensor([[ni]]))
                decoder_input = check_cuda_for_var(decoder_input)
            gen_sentence = Variable(torch.LongTensor(gen_sentence))
            gen_sentence = check_cuda_for_var(gen_sentence)
            string = ' '.join([my_lang.index2word[word.data[0]] for word in gen_sentence])
            print(string)
            talking_history.append(string)
            if "EOD" in string:
                break
            counter += 1
        return talking_history
# Generating string
try:
    while True:
        start = input("[%s] >>> " % (args.type.upper()))
        clean_sentence = clean(start)
        clean_sentence_idx = my_lang.sentence2index(clean_sentence)
        clean_sentence_idx = Variable(torch.LongTensor(clean_sentence_idx))
        clean_sentence_idx = check_cuda_for_var(clean_sentence_idx)
        gen(clean_sentence_idx)
except KeyboardInterrupt:
    print()
<commit_msg>[Feature] Make gen.py can gen sentence one by one<commit_after># -*- coding: utf-8 -*-
"""Generator for model"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from builtins import range

import argparse
import os
import sys
import random
import pickle
from math import exp

import torch
from torch.autograd import Variable
# Import my own cleaning lib, use jieba for other user
try:
    from purewords import clean_sentence as clean
except ImportError:
    from jieba import lcut as clean
import model
import utils
from utils import check_cuda_for_var, check_directory

parser = argparse.ArgumentParser(description=\
        "Generator for HRNN/Seq2seq")
parser.add_argument('--data', type=str,
        help="location of the data corpus(json file)")
parser.add_argument('--type', type=str,
        help="generate dialog with hrnn/seq2seq model")
parser.add_argument('--save', type=str, default='model/',
        help='path to load the final model\'s directory')
parser.add_argument('--seed', type=int, default=55665566,
        help='random seed')
parser.add_argument('--beam', type=int, default=1,
        help='beam size for beam search(default 1 will be greedy search)')
parser.add_argument('--eodlong', type=int, default=0,
        help='whether force model to gen a longer dialog (1 for on, 0 for off, default = 0)')
parser.add_argument('--nosr', type=int, default=0,
        help='whether force model don\'t self repeat (1 for on, 0 for off, default = 0)')
parser.add_argument('--number', type=int, default=0,
        help='model number to restore')
parser.add_argument('--sbs', type=int, default=0,
        help='Generate sentence by sentence (1 for on, 0 for off, default = 0)')
args = parser.parse_args()

torch.manual_seed(args.seed)
random.seed(args.seed)

DEBUG = False

if args.type != "hrnn" and args.type != "seq2seq":
    raise ValueError("args.type should be hrnn or seq2seq, but got %s" % (args.type))
if args.beam <= 0:
    raise ValueError("args.beam should be at least 1 or larger number")
if not os.path.isfile('dict.pkl'):
    my_lang, _ = utils.build_lang(args.data)
    with open('dict.pkl', 'wb') as filename:
        pickle.dump(my_lang, filename)
else:
    print("Load dict.pkl")
    with open('dict.pkl', 'rb') as filename:
        my_lang = pickle.load(filename)
if args.type == "hrnn":
    # Load last HRNN model
    if args.number == 0:
        number = torch.load(os.path.join(args.save, 'checkpoint.pt'))
    else:
        number = args.number
    encoder = torch.load(os.path.join(args.save, 'encoder'+str(number)+'.pt'))
    context = torch.load(os.path.join(args.save, 'context'+str(number)+'.pt'))
    decoder = torch.load(os.path.join(args.save, 'decoder'+str(number)+'.pt'))
    if torch.cuda.is_available():
        encoder = encoder.cuda()
        context = context.cuda()
        decoder = decoder.cuda()
    def gen(sentence):
        encoder.eval()
        context.eval()
        decoder.eval()

        # Inference
        gen_sentence = []
        talking_history = []
        context_hidden = context.init_hidden()
        max_dialog_len = 20
        max_sentence_len = 15
        beam_size = args.beam
        for _ in range(max_dialog_len):
            decoder_input = Variable(torch.LongTensor([[my_lang.word2index["SOS"]]]))
            decoder_input = check_cuda_for_var(decoder_input)
            encoder_hidden = encoder.init_hidden()
            decoder_hidden = decoder.init_hidden()
            if len(gen_sentence) > 0:
                for ei in range(len(gen_sentence)):
                    _, encoder_hidden = encoder(gen_sentence[ei], encoder_hidden)
                # Clean generated sentence list
                gen_sentence = []
            else:
                for ei in range(len(sentence)):
                    _, encoder_hidden = encoder(sentence[ei], encoder_hidden)
            context_output, context_hidden = context(encoder_hidden, context_hidden)
            # Beam search
            index2state = {}
            for index in range(beam_size):
                index2state[index] = [decoder_input, decoder_hidden, [decoder_input.data[0][0]], 0.0]
            # One step to get beam_size candidates
            decoder_output, decoder_hidden = decoder(context_hidden,\
                    decoder_input, decoder_hidden)
            scores, topi = decoder_output.data.topk(beam_size)
            for index in range(beam_size):
                ni = topi[0][index]
                index2state[index][0] = check_cuda_for_var(Variable(torch.LongTensor([[ni]])))
                index2state[index][1] = decoder_hidden
                index2state[index][2].append(ni)
                index2state[index][3] = scores[0][index]
            for sentence_pointer in range(max_sentence_len):
                current_scores = []
                current2state = {}
                # Init current2state
                for index in range(beam_size):
                    for jndex in range(beam_size):
                        current2state[index * beam_size + jndex] = [0, 0, 0, 0]
                for index in range(beam_size):
                    output, hidden = decoder(context_hidden, \
                            index2state[index][0], index2state[index][1])
                    tops, topi = output.data.topk(beam_size)
                    for jndex in range(beam_size):
                        ni = topi[0][jndex]
                        current_map = current2state[index * beam_size + jndex]
                        current_map[0] = check_cuda_for_var(Variable(torch.LongTensor([[ni]])))
                        current_map[1] = hidden
                        current_map[2] = index2state[index][2][:]
                        current_map[2].append(ni)
                        current_map[3] = tops[0][jndex] + index2state[index][3]
                        if args.eodlong == 1 and my_lang.word2index["EOD"] in current_map[2]:
                            current_map[3] *= exp(max_sentence_len - 12 - sentence_pointer)
                        current_scores.append(current_map[3])

                _, top_of_beamsize2 = torch.FloatTensor(current_scores).topk(beam_size)
                # Top beam's output is eos, break and output the top beam
                if current2state[top_of_beamsize2[0]][2][-1] == my_lang.word2index["EOS"]:
                    if args.nosr == 1 and current2state[top_of_beamsize2[0]][2] in talking_history:
                        # Don't repeat itself
                        # Soft verion
                        current2state[top_of_beamsize2[0]][3] *= 2
                        # Hard version
                        #current2state[top_of_beamsize2[0][3]] *= 100000.0
                    else:
                        first_eos = current2state[top_of_beamsize2[0]][2].index(my_lang.word2index["EOS"])
                        gen_sentence = current2state[top_of_beamsize2[0]][2][:first_eos+1]
                        break
                after_beam_dict = {}
                for index, candidate in enumerate(top_of_beamsize2):
                    after_beam_dict[index] = current2state[candidate]
                index2state = after_beam_dict
            # Beam Search a good sentence and assign to gen_sentence
            talking_history.append(gen_sentence)
            gen_sentence = Variable(torch.LongTensor(gen_sentence))
            gen_sentence = check_cuda_for_var(gen_sentence)
            try:
                string = ' '.join([my_lang.index2word[word.data[0]] for word in gen_sentence])
                print(string)
                if "EOD" in string:
                    break
            except RuntimeError:
                break
        return talking_history
    def genSbyS():
        try:
            encoder.eval()
            context.eval()
            decoder.eval()

            context_hidden = context.init_hidden()
            max_sentence_len = 15
            beam_size = args.beam
            talking_history = []

            while True:
                start = input("[%s] >>> " % (args.type.upper()))
                if start == 'reset':
                    context_hidden = context.init_hidden()
                    talking_history = []
                    continue
                clean_sentence = clean(start)
                clean_sentence_idx = my_lang.sentence2index(clean_sentence)
                if len(clean_sentence_idx) == 0:
                    continue
                clean_sentence_idx = Variable(torch.LongTensor(clean_sentence_idx))
                clean_sentence_idx = check_cuda_for_var(clean_sentence_idx)
                sentence = clean_sentence_idx

                decoder_input = Variable(torch.LongTensor([[my_lang.word2index["SOS"]]]))
                decoder_input = check_cuda_for_var(decoder_input)
                encoder_hidden = encoder.init_hidden()
                decoder_hidden = decoder.init_hidden()

                for ei in range(len(sentence)):
                    _, encoder_hidden = encoder(sentence[ei], encoder_hidden)

                context_output, context_hidden = context(encoder_hidden, context_hidden)
                # Beam search
                index2state = {}
                for index in range(beam_size):
                    index2state[index] = [decoder_input, decoder_hidden, [decoder_input.data[0][0]], 0.0]
                # One step to get beam_size candidates
                decoder_output, decoder_hidden = decoder(context_hidden,\
                        decoder_input, decoder_hidden)
                scores, topi = decoder_output.data.topk(beam_size)
                for index in range(beam_size):
                    ni = topi[0][index]
                    index2state[index][0] = check_cuda_for_var(Variable(torch.LongTensor([[ni]])))
                    index2state[index][1] = decoder_hidden
                    index2state[index][2].append(ni)
                    index2state[index][3] = scores[0][index]
                for sentence_pointer in range(max_sentence_len):
                    current_scores = []
                    current2state = {}
                    # Init current2state
                    for index in range(beam_size):
                        for jndex in range(beam_size):
                            current2state[index * beam_size + jndex] = [0, 0, 0, 0]
                    for index in range(beam_size):
                        output, hidden = decoder(context_hidden, \
                                index2state[index][0], index2state[index][1])
                        tops, topi = output.data.topk(beam_size)
                        for jndex in range(beam_size):
                            ni = topi[0][jndex]
                            current_map = current2state[index * beam_size + jndex]
                            current_map[0] = check_cuda_for_var(Variable(torch.LongTensor([[ni]])))
                            current_map[1] = hidden
                            current_map[2] = index2state[index][2][:]
                            current_map[2].append(ni)
                            current_map[3] = tops[0][jndex] + index2state[index][3]
                            if args.eodlong == 1 and my_lang.word2index["EOD"] in current_map[2]:
                                current_map[3] *= exp(max_sentence_len - 12 - sentence_pointer)
                            current_scores.append(current_map[3])

                    _, top_of_beamsize2 = torch.FloatTensor(current_scores).topk(beam_size)
                    # Top beam's output is eos, break and output the top beam
                    if current2state[top_of_beamsize2[0]][2][-1] == my_lang.word2index["EOS"]:
                        if args.nosr == 1 and current2state[top_of_beamsize2[0]][2] in talking_history:
                            # Don't repeat itself
                            # Soft verion
                            current2state[top_of_beamsize2[0]][3] *= 2
                            # Hard version
                            #current2state[top_of_beamsize2[0][3]] *= 100000.0
                        else:
                            first_eos = current2state[top_of_beamsize2[0]][2].index(my_lang.word2index["EOS"])
                            gen_sentence = current2state[top_of_beamsize2[0]][2][:first_eos+1]
                            break
                    after_beam_dict = {}
                    for index, candidate in enumerate(top_of_beamsize2):
                        after_beam_dict[index] = current2state[candidate]
                    index2state = after_beam_dict
                # Beam Search a good sentence and assign to gen_sentence
                talking_history.append(gen_sentence)
                gen_sentence = Variable(torch.LongTensor(gen_sentence))
                gen_sentence = check_cuda_for_var(gen_sentence)
                string = ' '.join([my_lang.index2word[word.data[0]] for word in gen_sentence])
                print(string)
                if "EOD" in string:
                    break

                decoder_input = Variable(torch.LongTensor([[my_lang.word2index["SOS"]]]))
                decoder_input = check_cuda_for_var(decoder_input)
                encoder_hidden = encoder.init_hidden()
                decoder_hidden = decoder.init_hidden()

                for ei in range(len(gen_sentence)):
                    _, encoder_hidden = encoder(gen_sentence[ei], encoder_hidden)

                context_output, context_hidden = context(encoder_hidden, context_hidden)
        except KeyboardInterrupt:
            print()
else:
    # Load last Seq2seq model
    number = torch.load(os.path.join(args.save, 'checkpoint.pt'))
    encoder = torch.load(os.path.join(args.save, 'encoder'+str(number)+'.pt'))
    decoder = torch.load(os.path.join(args.save, 'decoder'+str(number)+'.pt'))
    if torch.cuda.is_available():
        encoder = encoder.cuda()
        decoder = decoder.cuda()
    def gen(sentence):
        max_length = 20
        encoder.eval()
        decoder.eval()
        talking_history = []
        gen_sentence = []
        counter = 0
        while counter < 10:
            encoder_hidden = encoder.init_hidden()
            encoder_outputs = Variable(torch.zeros(max_length, encoder.hidden_size))
            decoder_input = Variable(torch.LongTensor([[my_lang.word2index["SOS"]]]))
            encoder_outputs = check_cuda_for_var(encoder_outputs)
            decoder_input = check_cuda_for_var(decoder_input)
            if len(gen_sentence) > 0:
                for ei in range(len(gen_sentence)):
                    encoder_output, encoder_hidden = encoder(gen_sentence[ei], encoder_hidden)
                    encoder_outputs[ei] = encoder_output[0][0]
                    # Clean generated sentence list
                gen_sentence = []
            else:
                for ei in range(len(sentence)):
                    encoder_output, encoder_hidden = encoder(sentence[ei], encoder_hidden)
                    encoder_outputs[ei] = encoder_output[0][0]
            decoder_hidden = encoder_hidden
            while True:
                if DEBUG:
                    print("[Debug] ", decoder_input.data)
                gen_sentence.append(decoder_input.data[0][0])
                if gen_sentence[-1] == my_lang.word2index["EOS"] or len(gen_sentence) >= max_length - 1:
                    break
                decoder_output, decoder_hidden, decoder_attention = decoder(decoder_input, decoder_hidden, \
                        encoder_outputs)
                _, topi = decoder_output.data.topk(1)
                ni = topi[0][0]
                decoder_input = Variable(torch.LongTensor([[ni]]))
                decoder_input = check_cuda_for_var(decoder_input)
            gen_sentence = Variable(torch.LongTensor(gen_sentence))
            gen_sentence = check_cuda_for_var(gen_sentence)
            string = ' '.join([my_lang.index2word[word.data[0]] for word in gen_sentence])
            print(string)
            talking_history.append(string)
            if "EOD" in string:
                break
            counter += 1
        return talking_history
# Generating string
try:
    if args.sbs == 0:
        while True:
            start = input("[%s] >>> " % (args.type.upper()))
            clean_sentence = clean(start)
            clean_sentence_idx = my_lang.sentence2index(clean_sentence)
            clean_sentence_idx = Variable(torch.LongTensor(clean_sentence_idx))
            clean_sentence_idx = check_cuda_for_var(clean_sentence_idx)
            gen(clean_sentence_idx)
    else:
        genSbyS()
except KeyboardInterrupt:
    print()
<|endoftext|>


<commit_before>import random
import string
import json

num_players = 5
num_matches = 30

class IntegerUniformDistribution:

  def __init__(self, low, high):
    self.low = low
    self.high = high

  def value(self):
    return random.randrange(self.low, self.high + 1)

class NameGenerator:

  def __init__(self, seed=None):
    if seed != None:
      random.seed(seed)

  def value(self):
    parts = []
    for i in range(2):
      n = random.randrange(4, 9)
      part = ''.join(random.choice(string.ascii_lowercase) for i in range(n))
      part = part[0].upper() + part[1:]
      parts.append(part)
    return ' '.join(parts)

class Player:

  def __init__(self, name, eagerness, skill):
    self.name = name
    self.eagerness = eagerness
    self.skill = skill
    self.matches = []
    self.defeated = {}
    self.defeated_by = {}

  def __str__(self):
    return '%s, eagerness %d, skill %d' % (self.name, self.eagerness,
        self.skill)

  def dictionary(self):
    d = {}
    d['name'] = self.name
    d['defeated'] = {}
    for opponent, matches in self.defeated.items():
      d['defeated'][opponent.name] = [ str(match) for match in matches ]
    d['defeated_by'] = {}
    for opponent, matches in self.defeated_by.items():
      d['defeated'][opponent.name] = [ str(match) for match in matches ]
    return d

class Match:

  def __init__(self, winner, loser):
    self.winner = winner
    self.loser = loser

  def __str__(self):
    return '%s defeats %s' % (self.winner.name, self.loser.name)

def decide_outcome(a, b):
  if random.randrange(a.skill + b.skill) < a.skill:
    return a, b
  return b, a

eagerness_distribution = IntegerUniformDistribution(2, 30)
skill_distribution = IntegerUniformDistribution(5, 25)
name_generator = NameGenerator()

players = num_players * [ None ]
eagerness_total = 0
for i in range(num_players):
  player = Player(name_generator.value(),
      eagerness_distribution.value(),
      skill_distribution.value())
  players[i] = player
  eagerness_total += player.eagerness

weighted_players = eagerness_total * [ None ]
pos = 0
for player in players:
  for i in range(player.eagerness):
    weighted_players[pos] = player
    pos += 1

def choose_match_player():
  return random.choice(weighted_players)

for i in range(num_matches):
  a = choose_match_player()
  while True:
    b = choose_match_player()
    if b != a:
      break
  a, b = sorted([a, b], key=lambda player: player.name)
  winner, loser = decide_outcome(a, b)
  match = Match(winner, loser)
  for player in [winner, loser]:
    player.matches.append(match)
  winner.defeated.setdefault(loser, []).append(match)
  loser.defeated_by.setdefault(winner, []).append(match)

for player in sorted(players, key=lambda player: -player.skill):
  print(str(player))
  print('  defeated:')
  for opponent, matches in player.defeated.items():
    print('    %d times: %s' % (len(matches), opponent.name))
  print('  defeated by:')
  for opponent, matches in player.defeated_by.items():
    print('    %d times: %s' % (len(matches), opponent.name))
  print(json.dumps(player.dictionary()))
  print('')
<commit_msg>Dump players to JSON<commit_after>import random
import string
import json

num_players = 5
num_matches = 30

class IntegerUniformDistribution:

  def __init__(self, low, high):
    self.low = low
    self.high = high

  def value(self):
    return random.randrange(self.low, self.high + 1)

class NameGenerator:

  def __init__(self, seed=None):
    if seed != None:
      random.seed(seed)

  def value(self):
    parts = []
    for i in range(2):
      n = random.randrange(4, 9)
      part = ''.join(random.choice(string.ascii_lowercase) for i in range(n))
      part = part[0].upper() + part[1:]
      parts.append(part)
    return ' '.join(parts)

class Player:

  def __init__(self, name, eagerness, skill):
    self.name = name
    self.eagerness = eagerness
    self.skill = skill
    self.matches = []
    self.defeated = {}
    self.defeated_by = {}

  def __str__(self):
    return '%s, eagerness %d, skill %d' % (self.name, self.eagerness,
        self.skill)

  def to_dict(self):
    d = {}
    d['name'] = self.name
    d['defeated'] = {}
    for opponent, matches in self.defeated.items():
      d['defeated'][opponent.name] = [ match.to_dict() for match in matches ]
    d['defeated_by'] = {}
    for opponent, matches in self.defeated_by.items():
      d['defeated'][opponent.name] = [ match.to_dict() for match in matches ]
    return d

class Match:

  def __init__(self, id, event_id, winner, loser):
    self.id = id
    self.event_id = event_id
    self.winner = winner
    self.loser = loser

  def __str__(self):
    return '%d, %d, %s defeats %s' % (id, event_id,
        self.winner.name, self.loser.name)

  def to_dict(self):
    return {
        'id': self.id, 'event_id': self.event_id,
        'winner': str(self.winner), 'loser': str(self.loser)
    }

def decide_outcome(a, b):
  if random.randrange(a.skill + b.skill) < a.skill:
    return a, b
  return b, a

eagerness_distribution = IntegerUniformDistribution(2, 30)
skill_distribution = IntegerUniformDistribution(5, 25)
name_generator = NameGenerator()

players = num_players * [ None ]
eagerness_total = 0
for i in range(num_players):
  player = Player(name_generator.value(),
      eagerness_distribution.value(),
      skill_distribution.value())
  players[i] = player
  eagerness_total += player.eagerness

weighted_players = eagerness_total * [ None ]
pos = 0
for player in players:
  for i in range(player.eagerness):
    weighted_players[pos] = player
    pos += 1

def choose_match_player():
  return random.choice(weighted_players)

for i in range(num_matches):
  a = choose_match_player()
  while True:
    b = choose_match_player()
    if b != a:
      break
  a, b = sorted([a, b], key=lambda player: player.name)
  winner, loser = decide_outcome(a, b)
  match_id = i + 1
  event_id = (i % 3) + 1
  match = Match(match_id, event_id, winner, loser)
  for player in [winner, loser]:
    player.matches.append(match)
  winner.defeated.setdefault(loser, []).append(match)
  loser.defeated_by.setdefault(winner, []).append(match)

sorted_players = sorted(players, key=lambda player: -player.skill)
if False:
  for player in sorted_players:
    print(str(player))
    print('  defeated:')
    for opponent, matches in player.defeated.items():
      print('    %d times: %s' % (len(matches), opponent.name))
    print('  defeated by:')
    for opponent, matches in player.defeated_by.items():
      print('    %d times: %s' % (len(matches), opponent.name))
    print('')
else:
  array = [ player.to_dict() for player in sorted_players ]
  print(json.dumps(array))

<|endoftext|>