<commit_before>"""
Force10 Backend.

This backend will only work with SSH version 2 capable Force10 switches.

This excludes most, if not all, of the etherscale series.

The backend has been developed for the E series.
The backend has been developed and tested on a Terascale E300 switch.

The switch (or router, depending on your level off pedanticness) is configured
by the backend logging via ssh, requesting a cli, and firing the necessary
command for configuring a VLAN. This approach was choosen over netconf / XML,
as a fairly reliable source said that not all the necessary functionality
needed was available via the previously mentioned interfaces.

Currently the backend does support VLAN rewriting, and I am not sure if/how it
is supported.

Configuration:

To setup a VLAN connection:

configure
interface vlan $vlan_id
name $name
description $description
no shut
tagged $source_port
tagged $dest_port
end

Teardown:

configure
no interface vlan $vlan_id
end

Ensure that the interfaces are configure to be layer 2.

Ralph developed a backend for etherscale, where a lot of the input from this
backend comes from.

Authors: Henrik Thostrup Jensen <htj@nordu.net>
         Ralph Koning <R.Koning@uva.nl>

Copyright: NORDUnet (2011-2013)
"""

import string
import random

from twisted.python import log
from twisted.internet import defer

from opennsa import constants as cnt, config
from opennsa.backends.common import ssh, genericbackend

LOG_SYSTEM = 'opennsa.force10'

COMMAND_CONFIGURE       = 'configure'
COMMAND_END             = 'end'
COMMAND_EXIT            = 'exit'
COMMAND_WRITE           = 'write'       # writes config

COMMAND_INTERFACE_VLAN  = 'interface vlan %(vlan)i'
COMMAND_NAME            = 'name %(name)s'
COMMAND_NO_SHUTDOWN     = 'no shutdown'
COMMAND_TAGGED          = 'tagged %(interface)s'

COMMAND_NO_INTERFACE    = 'no interface vlan %(vlan)i'

def _portToInterfaceVLAN(nrm_port):

    interface, vlan = nrm_port.rsplit('.')
    vlan = int(vlan)
    return interface, vlan

def _createSetupCommands(source_nrm_port, dest_nrm_port):

    s_interface, s_vlan = _portToInterfaceVLAN(source_nrm_port)
    d_interface, d_vlan = _portToInterfaceVLAN(dest_nrm_port)

    assert s_vlan == d_vlan, 'Source and destination VLANs differ, unpossible!'

    name = 'opennsa-%i' % s_vlan

    cmd_vlan    = COMMAND_INTERFACE_VLAN    % { 'vlan' : s_vlan }
    cmd_name    = COMMAND_NAME              % { 'name' : name   }
    cmd_s_intf  = COMMAND_TAGGED            % { 'interface' : s_interface }
    cmd_d_intf  = COMMAND_TAGGED            % { 'interface' : d_interface }

    commands = [ cmd_vlan, cmd_name, cmd_s_intf, cmd_d_intf, COMMAND_NO_SHUTDOWN, COMMAND_END ]
    return commands

def _createTeardownCommands(source_nrm_port, dest_nrm_port):

    _, s_vlan = _portToInterfaceVLAN(source_nrm_port)
    _, d_vlan = _portToInterfaceVLAN(dest_nrm_port)

    assert s_vlan == d_vlan, 'Source and destination VLANs differ, unpossible!'

    cmd_no_intf = COMMAND_NO_INTERFACE % { 'vlan' : s_vlan }

    commands = [ cmd_no_intf, COMMAND_END ]
    return commands

class SSHChannel(ssh.SSHChannel):

    name = 'session'

    def __init__(self, conn):
        ssh.SSHChannel.__init__(self, conn=conn)

        self.data = ''

        self.wait_defer = None
        self.wait_data  = None

    @defer.inlineCallbacks
    def sendCommands(self, commands):
        LT = '\r' # line termination

        try:
            log.msg('Requesting shell for sending commands', debug=True, system=LOG_SYSTEM)
            yield self.conn.sendRequest(self, 'shell', '', wantReply=1)

            d = self.waitForData('#')
            self.write(COMMAND_CONFIGURE + LT)
            yield d
            log.msg('Entered configure mode', debug=True, system=LOG_SYSTEM)

            for cmd in commands:
                log.msg('CMD> %s' % cmd, debug=True, system=LOG_SYSTEM)
                d = self.waitForData('#')
                self.write(cmd + LT)
                yield d

            # not quite sure how to handle failure here
            log.msg('Commands send, sending end command.', debug=True, system=LOG_SYSTEM)
            d = self.waitForData('#')
            self.write(COMMAND_END + LT)
            yield d

            log.msg('Configuration done, writing configuration.', debug=True, system=LOG_SYSTEM)
            d = self.waitForData('#')
            self.write(COMMAND_WRITE + LT)
            yield d

            log.msg('Configuration written. Exiting.', debug=True, system=LOG_SYSTEM)
            d = self.waitForData('#')
            self.write(COMMAND_EXIT + LT)
            yield d

        except Exception, e:
            log.msg('Error sending commands: %s' % str(e))
            raise e

        log.msg('Commands successfully send', system=LOG_SYSTEM)
        self.sendEOF()
        self.closeIt()

    def waitForData(self, data):
        self.wait_data  = data
        self.wait_defer = defer.Deferred()
        return self.wait_defer

    def dataReceived(self, data):
        if len(data) == 0:
            pass
        else:
            self.data += data
            if self.wait_data and self.wait_data in self.data:
                d = self.wait_defer
                self.data       = ''
                self.wait_data  = None
                self.wait_defer = None
                d.callback(self)

class Force10CommandSender:

    def __init__(self, host, port, ssh_host_fingerprint, user, ssh_public_key_path, ssh_private_key_path):

        self.ssh_connection_creator = \
             ssh.SSHConnectionCreator(host, port, [ ssh_host_fingerprint ], user, ssh_public_key_path, ssh_private_key_path)

    @defer.inlineCallbacks
    def sendCommands(self, commands):

        # Note: FTOS does not allow multiple channels in an SSH connection,
        # so we open a connection for each request. Party like it is 1988.

        # The "correct" solution for this would be to create a connection pool,
        # but that won't happen just now.

        log.msg('Creating new SSH connection', debug=True, system=LOG_SYSTEM)
        ssh_connection = yield self.ssh_connection_creator.getSSHConnection()

        try:
            channel = SSHChannel(conn=ssh_connection)
            ssh_connection.openChannel(channel)

            yield channel.channel_open
            yield channel.sendCommands(commands)

        finally:
            ssh_connection.transport.loseConnection()

class Force10ConnectionManager:

    def __init__(self, log_system, port_map, cfg):
        self.log_system = log_system
        self.port_map   = port_map

        host             = cfg[config.FORCE10_HOST]
        port             = cfg.get(config.FORCE10_PORT, 22)
        host_fingerprint = cfg[config.FORCE10_HOST_FINGERPRINT]
        user             = cfg[config.FORCE11_USER]
        ssh_public_key   = cfg[config.FORCE10_SSH_PUBLIC_KEY]
        ssh_private_key  = cfg[config.FORCE10_SSH_PRIVATE_KEY]

        self.command_sender = Force10CommandSender(host, port, host_fingerprint, user, ssh_public_key, ssh_private_key)

    def getResource(self, port, label_type, label_value):
        assert label_type == cnt.ETHERNET_VLAN, 'Label type must be ethernet-vlan'
        return str(label_value)

    def getTarget(self, port, label_type, label_value):
        return self.port_map[port] + '.' + label_value

    def createConnectionId(self, source_target, dest_target):
        return 'F10-' + ''.join( [ random.choice(string.hexdigits[:16]) for _ in range(10) ] )

    def canSwapLabel(self, label_type):
        return False

    def setupLink(self, connection_id, source_target, dest_target, bandwidth):

        def linkUp(pt):
            log.msg('Link %s -> %s up' % (source_target, dest_target), system=self.log_system)
            return pt

        commands = _createSetupCommands(source_target, dest_target)
        d = self.command_sender.sendCommands(commands)
        d.addCallback(linkUp)
        return d

    def teardownLink(self, connection_id, source_target, dest_target, bandwidth):

        def linkDown(pt):
            log.msg('Link %s -> %s down' % (source_target, dest_target), system=self.log_system)
            return pt

        commands = _createTeardownCommands(source_target, dest_target)
        d = self.command_sender.sendCommands(commands)
        d.addCallback(linkDown)
        return d

def Force10Backend(network_name, network_topology, parent_requester, port_map, configuration):
    name = 'Force10 %s' % network_name
    cm = Force10ConnectionManager(name, port_map, configuration)
    return genericbackend.GenericBackend(network_name, network_topology, cm, parent_requester, name)
<commit_msg>fix small mistake in force10 backend<commit_after>"""
Force10 Backend.

This backend will only work with SSH version 2 capable Force10 switches.

This excludes most, if not all, of the etherscale series.

The backend has been developed for the E series.
The backend has been developed and tested on a Terascale E300 switch.

The switch (or router, depending on your level off pedanticness) is configured
by the backend logging via ssh, requesting a cli, and firing the necessary
command for configuring a VLAN. This approach was choosen over netconf / XML,
as a fairly reliable source said that not all the necessary functionality
needed was available via the previously mentioned interfaces.

Currently the backend does support VLAN rewriting, and I am not sure if/how it
is supported.

Configuration:

To setup a VLAN connection:

configure
interface vlan $vlan_id
name $name
description $description
no shut
tagged $source_port
tagged $dest_port
end

Teardown:

configure
no interface vlan $vlan_id
end

Ensure that the interfaces are configure to be layer 2.

Ralph developed a backend for etherscale, where a lot of the input from this
backend comes from.

Authors: Henrik Thostrup Jensen <htj@nordu.net>
         Ralph Koning <R.Koning@uva.nl>

Copyright: NORDUnet (2011-2013)
"""

import string
import random

from twisted.python import log
from twisted.internet import defer

from opennsa import constants as cnt, config
from opennsa.backends.common import ssh, genericbackend

LOG_SYSTEM = 'opennsa.force10'

COMMAND_CONFIGURE       = 'configure'
COMMAND_END             = 'end'
COMMAND_EXIT            = 'exit'
COMMAND_WRITE           = 'write'       # writes config

COMMAND_INTERFACE_VLAN  = 'interface vlan %(vlan)i'
COMMAND_NAME            = 'name %(name)s'
COMMAND_NO_SHUTDOWN     = 'no shutdown'
COMMAND_TAGGED          = 'tagged %(interface)s'

COMMAND_NO_INTERFACE    = 'no interface vlan %(vlan)i'

def _portToInterfaceVLAN(nrm_port):

    interface, vlan = nrm_port.rsplit('.')
    vlan = int(vlan)
    return interface, vlan

def _createSetupCommands(source_nrm_port, dest_nrm_port):

    s_interface, s_vlan = _portToInterfaceVLAN(source_nrm_port)
    d_interface, d_vlan = _portToInterfaceVLAN(dest_nrm_port)

    assert s_vlan == d_vlan, 'Source and destination VLANs differ, unpossible!'

    name = 'opennsa-%i' % s_vlan

    cmd_vlan    = COMMAND_INTERFACE_VLAN    % { 'vlan' : s_vlan }
    cmd_name    = COMMAND_NAME              % { 'name' : name   }
    cmd_s_intf  = COMMAND_TAGGED            % { 'interface' : s_interface }
    cmd_d_intf  = COMMAND_TAGGED            % { 'interface' : d_interface }

    commands = [ cmd_vlan, cmd_name, cmd_s_intf, cmd_d_intf, COMMAND_NO_SHUTDOWN, COMMAND_END ]
    return commands

def _createTeardownCommands(source_nrm_port, dest_nrm_port):

    _, s_vlan = _portToInterfaceVLAN(source_nrm_port)
    _, d_vlan = _portToInterfaceVLAN(dest_nrm_port)

    assert s_vlan == d_vlan, 'Source and destination VLANs differ, unpossible!'

    cmd_no_intf = COMMAND_NO_INTERFACE % { 'vlan' : s_vlan }

    commands = [ cmd_no_intf, COMMAND_END ]
    return commands

class SSHChannel(ssh.SSHChannel):

    name = 'session'

    def __init__(self, conn):
        ssh.SSHChannel.__init__(self, conn=conn)

        self.data = ''

        self.wait_defer = None
        self.wait_data  = None

    @defer.inlineCallbacks
    def sendCommands(self, commands):
        LT = '\r' # line termination

        try:
            log.msg('Requesting shell for sending commands', debug=True, system=LOG_SYSTEM)
            yield self.conn.sendRequest(self, 'shell', '', wantReply=1)

            d = self.waitForData('#')
            self.write(COMMAND_CONFIGURE + LT)
            yield d
            log.msg('Entered configure mode', debug=True, system=LOG_SYSTEM)

            for cmd in commands:
                log.msg('CMD> %s' % cmd, debug=True, system=LOG_SYSTEM)
                d = self.waitForData('#')
                self.write(cmd + LT)
                yield d

            # not quite sure how to handle failure here
            log.msg('Commands send, sending end command.', debug=True, system=LOG_SYSTEM)
            d = self.waitForData('#')
            self.write(COMMAND_END + LT)
            yield d

            log.msg('Configuration done, writing configuration.', debug=True, system=LOG_SYSTEM)
            d = self.waitForData('#')
            self.write(COMMAND_WRITE + LT)
            yield d

            log.msg('Configuration written. Exiting.', debug=True, system=LOG_SYSTEM)
            d = self.waitForData('#')
            self.write(COMMAND_EXIT + LT)
            yield d

        except Exception, e:
            log.msg('Error sending commands: %s' % str(e))
            raise e

        log.msg('Commands successfully send', system=LOG_SYSTEM)
        self.sendEOF()
        self.closeIt()

    def waitForData(self, data):
        self.wait_data  = data
        self.wait_defer = defer.Deferred()
        return self.wait_defer

    def dataReceived(self, data):
        if len(data) == 0:
            pass
        else:
            self.data += data
            if self.wait_data and self.wait_data in self.data:
                d = self.wait_defer
                self.data       = ''
                self.wait_data  = None
                self.wait_defer = None
                d.callback(self)

class Force10CommandSender:

    def __init__(self, host, port, ssh_host_fingerprint, user, ssh_public_key_path, ssh_private_key_path):

        self.ssh_connection_creator = \
             ssh.SSHConnectionCreator(host, port, [ ssh_host_fingerprint ], user, ssh_public_key_path, ssh_private_key_path)

    @defer.inlineCallbacks
    def sendCommands(self, commands):

        # Note: FTOS does not allow multiple channels in an SSH connection,
        # so we open a connection for each request. Party like it is 1988.

        # The "correct" solution for this would be to create a connection pool,
        # but that won't happen just now.

        log.msg('Creating new SSH connection', debug=True, system=LOG_SYSTEM)
        ssh_connection = yield self.ssh_connection_creator.getSSHConnection()

        try:
            channel = SSHChannel(conn=ssh_connection)
            ssh_connection.openChannel(channel)

            yield channel.channel_open
            yield channel.sendCommands(commands)

        finally:
            ssh_connection.transport.loseConnection()

class Force10ConnectionManager:

    def __init__(self, log_system, port_map, cfg):
        self.log_system = log_system
        self.port_map   = port_map

        host             = cfg[config.FORCE10_HOST]
        port             = cfg.get(config.FORCE10_PORT, 22)
        host_fingerprint = cfg[config.FORCE10_HOST_FINGERPRINT]
        user             = cfg[config.FORCE10_USER]
        ssh_public_key   = cfg[config.FORCE10_SSH_PUBLIC_KEY]
        ssh_private_key  = cfg[config.FORCE10_SSH_PRIVATE_KEY]

        self.command_sender = Force10CommandSender(host, port, host_fingerprint, user, ssh_public_key, ssh_private_key)

    def getResource(self, port, label_type, label_value):
        assert label_type == cnt.ETHERNET_VLAN, 'Label type must be ethernet-vlan'
        return str(label_value)

    def getTarget(self, port, label_type, label_value):
        return self.port_map[port] + '.' + label_value

    def createConnectionId(self, source_target, dest_target):
        return 'F10-' + ''.join( [ random.choice(string.hexdigits[:16]) for _ in range(10) ] )

    def canSwapLabel(self, label_type):
        return False

    def setupLink(self, connection_id, source_target, dest_target, bandwidth):

        def linkUp(pt):
            log.msg('Link %s -> %s up' % (source_target, dest_target), system=self.log_system)
            return pt

        commands = _createSetupCommands(source_target, dest_target)
        d = self.command_sender.sendCommands(commands)
        d.addCallback(linkUp)
        return d

    def teardownLink(self, connection_id, source_target, dest_target, bandwidth):

        def linkDown(pt):
            log.msg('Link %s -> %s down' % (source_target, dest_target), system=self.log_system)
            return pt

        commands = _createTeardownCommands(source_target, dest_target)
        d = self.command_sender.sendCommands(commands)
        d.addCallback(linkDown)
        return d

def Force10Backend(network_name, network_topology, parent_requester, port_map, configuration):
    name = 'Force10 %s' % network_name
    cm = Force10ConnectionManager(name, port_map, configuration)
    return genericbackend.GenericBackend(network_name, network_topology, cm, parent_requester, name)
<|endoftext|>


<commit_before>import pymel.core as pm

imgOp = 0
imgDep = 0
curCam = ''
ImgPln = []
fileNm = ''
pLoc = ''
bmCam = ''
#create image plane and set up some attrs
def createImgPln():
    global imgOp
    global imgDep
    global curCam
    global ImgPln
    global fileNm
    global pLoc
    #image plane opacty and offset from camera
    imgOp = 0.3
    imgDep = 10
    #get current camera
    curCam = pm.modelPanel(pm.getPanel(wf=True),q=True,cam=True)
    #select image and creat imagePlane and setup
    fileNm = pm.fileDialog2(ds=0,fm=1,cap='open',okc='Select Image')
    ImgPln = pm.imagePlane(fn=fileNm[0],lookThrough=curCam,maintainRatio=1)
    pm.setAttr(ImgPln[1]+'.displayOnlyIfCurrent',True)
    pm.setAttr(ImgPln[0]+'.translateZ',-pm.getAttr(curCam+'.translateZ')/3+-imgDep)
    pm.setAttr(ImgPln[1]+'.alphaGain',imgOp)
    pm.setAttr(ImgPln[1]+'.textureFilter',1)

    #aligh to the camera
    #create locator to be the parent and then create parent constraint
    pLoc = pm.spaceLocator(name='ImagePlane_Parent_Loc')
    pm.parent(ImgPln[0],pLoc)
    pm.parentConstraint(curCam,pLoc)

#Toggle image plane visibility
def imgPlnVisTgl():
    if(pm.getAttr(ImgPln[1]+'.visibility')):
        pm.setAttr(ImgPln[1]+'.visibility',0)
    else:
        pm.setAttr(ImgPln[1]+'.visibility',1)

#Toggle lock current camera
def camLockTgl():
    if(pm.getAttr(curCam+'.t',lock=True)):
        pm.setAttr(curCam+'.t',lock=False)
        pm.setAttr(curCam+'.r',lock=False)
    else:
        pm.setAttr(curCam+'.t',lock=True)
        pm.setAttr(curCam+'.r',lock=True)
        addImgBookMark()

#Select image plane
def selImgPln():
    pm.select(ImgPln[0],r=True)
    if(pm.getAttr(ImgPln[0]+'.t',lock=True)):
        pm.setAttr(ImgPln[0]+'.t',lock=False)
        pm.setAttr(ImgPln[0]+'.r',lock=False)
        pm.setAttr(ImgPln[0]+'.s',lock=False)
        pm.toggle(ImgPln[0],state=False,template=True)

'''
#Toggle lock image plane's translate rotate and scale
def imgPlnLockTgl():
    if(pm.getAttr(ImgPln[0]+'.t',lock=True)):
        pm.setAttr(ImgPln[0]+'.t',lock=False)
        pm.setAttr(ImgPln[0]+'.r',lock=False)
        pm.setAttr(ImgPln[0]+'.s',lock=False)
        pm.toggle(ImgPln[0],state=False,template=True)
        pm.select(ImgPln,r=True)
    else:
        pm.setAttr(ImgPln[0]+'.t',lock=True)
        pm.setAttr(ImgPln[0]+'.r',lock=True)
        pm.setAttr(ImgPln[0]+'.s',lock=True)
        pm.toggle(ImgPln[0],state=True,template=True)
        pm.select(ImgPln,d=True)
'''
#Lock the image plane
def imgPlnLock():
    if(pm.getAttr(ImgPln[0]+'.t',lock=False)):
        pm.setAttr(ImgPln[0]+'.t',lock=True)
        pm.setAttr(ImgPln[0]+'.r',lock=True)
        pm.setAttr(ImgPln[0]+'.s',lock=True)
        pm.toggle(ImgPln[0],state=True,template=True)
        pm.select(ImgPln,d=True)

#remove image plane and cleanup
def cleanupImgPln():
    pm.delete(pLoc)

#bookmark camera position
def addImgBookMark():
    global bmCam
    bmCam = pm.modelPanel(pm.getPanel(wf=True),q=True,cam=True)
    #remove current bookmark
    cmvList = pm.ls(type='cameraView')
    if len(cmvList)!=0:
        for cmv in cmvList:
            if cmv == 'imageView_bookmark':
                pm.cameraView(cmv,c=bmCam,e=True,rb=True)
                pm.delete(cmv)
    #add bookmark
    addBM = pm.cameraView(c=bmCam,ab=True,n='imageView_bookmark')

#Restore from bookmark
def restoreImgBM():
    pm.cameraView('imageView_bookmark',e=True,c=bmCam,setCamera=True)

<commit_msg>Update afImgPanel.py<commit_after>import pymel.core as pm

imgOp = 0
imgDep = 0
curCam = ''
ImgPln = []
fileNm = ''
pLoc = ''
bmCam = ''
#create image plane and set up some attrs
def createImgPln():
    global imgOp
    global imgDep
    global curCam
    global ImgPln
    global fileNm
    global pLoc
    #image plane opacty and offset from camera
    imgOp = 0.3
    imgDep = 10
    #get current camera
    curCam = pm.modelPanel(pm.getPanel(wf=True),q=True,cam=True)
    #select image and creat imagePlane and setup
    fileNm = pm.fileDialog2(ds=0,fm=1,cap='open',okc='Select Image')
    ImgPln = pm.imagePlane(fn=fileNm[0],lookThrough=curCam,maintainRatio=1)
    pm.setAttr(ImgPln[1]+'.displayOnlyIfCurrent',True)
    pm.setAttr(ImgPln[0]+'.translateZ',-pm.getAttr(curCam+'.translateZ')/3+-imgDep)
    pm.setAttr(ImgPln[1]+'.alphaGain',imgOp)
    pm.setAttr(ImgPln[1]+'.textureFilter',1)

    #aligh to the camera
    #create locator to be the parent and then create parent constraint
    pLoc = pm.spaceLocator(name='ImagePlane_Parent_Loc')
    pm.parent(ImgPln[0],pLoc)
    pm.parentConstraint(curCam,pLoc)

#Toggle image plane visibility
def imgPlnVisTgl():
    if(pm.getAttr(ImgPln[1]+'.visibility')):
        pm.setAttr(ImgPln[1]+'.visibility',0)
    else:
        pm.setAttr(ImgPln[1]+'.visibility',1)

#Toggle lock current camera
def camLockTgl():
    if(pm.getAttr(curCam+'.t',lock=True)):
        pm.setAttr(curCam+'.t',lock=False)
        pm.setAttr(curCam+'.r',lock=False)
    else:
        pm.setAttr(curCam+'.t',lock=True)
        pm.setAttr(curCam+'.r',lock=True)
        addImgBookMark()

#Select image plane
def selImgPln():
    pm.select(ImgPln[0],r=True)
    if(pm.getAttr(ImgPln[0]+'.t',lock=True)):
        pm.setAttr(ImgPln[0]+'.t',lock=False)
        pm.setAttr(ImgPln[0]+'.r',lock=False)
        pm.setAttr(ImgPln[0]+'.s',lock=False)
        pm.toggle(ImgPln[0],state=False,template=True)

#Lock the image plane
def imgPlnLock():
    if(pm.getAttr(ImgPln[0]+'.t',lock=False)):
        pm.setAttr(ImgPln[0]+'.t',lock=True)
        pm.setAttr(ImgPln[0]+'.r',lock=True)
        pm.setAttr(ImgPln[0]+'.s',lock=True)
        pm.toggle(ImgPln[0],state=True,template=True)
        pm.select(ImgPln,d=True)

#remove image plane and cleanup
def cleanupImgPln():
    allTransNd = pm.ls(type='transform',fl=True)
    for trans in allTransNd:
        if trans == 'ImagePlane_Parent_Loc':
            pm.delete(pLoc)
    #unlock the camera
    if(pm.getAttr(curCam+'.t',lock=True)):
        pm.setAttr(curCam+'.t',lock=False)
        pm.setAttr(curCam+'.r',lock=False)

#bookmark camera position
def addImgBookMark():
    global bmCam
    bmCam = pm.modelPanel(pm.getPanel(wf=True),q=True,cam=True)
    #remove current bookmark
    cmvList = pm.ls(type='cameraView')
    if len(cmvList)!=0:
        for cmv in cmvList:
            if cmv == 'imageView_bookmark':
                pm.cameraView(cmv,c=bmCam,e=True,rb=True)
                pm.delete(cmv)
    #add bookmark
    addBM = pm.cameraView(c=bmCam,ab=True,n='imageView_bookmark')

#Restore from bookmark
def restoreImgBM():
    pm.cameraView('imageView_bookmark',e=True,c=bmCam,setCamera=True)

<|endoftext|>


<commit_before># coding: utf-8

""" Check autocorrelation times """

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Standard library
import os, sys
import glob

# Third-party
from emcee import autocorr
import matplotlib.pyplot as plt
import numpy as np
import h5py

# Project
from streams.inference import StreamModel
import streams.io as io

def main(config_file):
    # read configuration from a YAML file
    config = io.read_config(config_file)
    output_path = os.path.join(config["output_path"], "diagnostics")
    cache_path = os.path.join(config["output_path"],"cache")

    if not os.path.exists(output_path):
        os.mkdir(output_path)

    for filename in glob.glob(os.path.join(cache_path,"*.hdf5")):
        with h5py.File(filename, "r") as f:
            try:
                chain = np.hstack((chain,f["chain"].value))
            except NameError:
                chain = f["chain"].value
            accfr = f["acceptance_fraction"].value

            _a = (np.min(accfr),np.median(accfr),np.max(accfr))
            print("min, median, max: {}, {}, {}".format(*_a))

    acf = autocorr.function(np.mean(chain, axis=0), axis=0)

    plt.clf()
    for ii in range(4):
        plt.plot(acf[:,ii], marker=None, alpha=0.75)
    plt.savefig(os.path.join(output_path, "acf.png"))

    # windows = np.logspace(1, 4, 25)
    # acors = []
    # for window in windows:
    #     _acor = autocorr.integrated_time(np.mean(chain, axis=0),
    #                                      axis=0, window=window) # 50 comes from emcee
    #     acors.append(np.median(_acor))

    # plt.clf()
    # v = plt.loglog(windows, acors, linestyle='none', marker='o')
    # plt.ylim(1,1e4)
    # plt.savefig(os.path.join(output_path, "acor_windows.png"))

if __name__ == "__main__":
    from argparse import ArgumentParser

    parser = ArgumentParser(description="")
    parser.add_argument("-f", "--file", dest="file", required=True,
                        help="Path to the configuration file to run with.")
    args = parser.parse_args()

    main(args.file)<commit_msg>only files start with inference<commit_after># coding: utf-8

""" Check autocorrelation times """

from __future__ import division, print_function

__author__ = "adrn <adrn@astro.columbia.edu>"

# Standard library
import os, sys
import glob

# Third-party
from emcee import autocorr
import matplotlib.pyplot as plt
import numpy as np
import h5py

# Project
from streams.inference import StreamModel
import streams.io as io

def main(config_file):
    # read configuration from a YAML file
    config = io.read_config(config_file)
    output_path = os.path.join(config["output_path"], "diagnostics")
    cache_path = os.path.join(config["output_path"],"cache")

    if not os.path.exists(output_path):
        os.mkdir(output_path)

    for filename in glob.glob(os.path.join(cache_path,"inference_*.hdf5")):
        with h5py.File(filename, "r") as f:
            try:
                chain = np.hstack((chain,f["chain"].value))
            except NameError:
                chain = f["chain"].value
            accfr = f["acceptance_fraction"].value

            _a = (np.min(accfr),np.median(accfr),np.max(accfr))
            print("min, median, max: {}, {}, {}".format(*_a))

    acf = autocorr.function(np.mean(chain, axis=0), axis=0)

    plt.clf()
    for ii in range(4):
        plt.plot(acf[:,ii], marker=None, alpha=0.75)
    plt.savefig(os.path.join(output_path, "acf.png"))

    # windows = np.logspace(1, 4, 25)
    # acors = []
    # for window in windows:
    #     _acor = autocorr.integrated_time(np.mean(chain, axis=0),
    #                                      axis=0, window=window) # 50 comes from emcee
    #     acors.append(np.median(_acor))

    # plt.clf()
    # v = plt.loglog(windows, acors, linestyle='none', marker='o')
    # plt.ylim(1,1e4)
    # plt.savefig(os.path.join(output_path, "acor_windows.png"))

if __name__ == "__main__":
    from argparse import ArgumentParser

    parser = ArgumentParser(description="")
    parser.add_argument("-f", "--file", dest="file", required=True,
                        help="Path to the configuration file to run with.")
    args = parser.parse_args()

    main(args.file)
<|endoftext|>


<commit_before>#!/usr/bin/env python
import argparse
import re

import pandas as pd

from isochrones.cluster import StarClusterModel, StarCatalog
from isochrones import get_ichrone
from isochrones.priors import FehPrior

try:
    from mpi4py import MPI
    comm = MPI.COMM_WORLD
    rank = comm.Get_rank()
except ImportError:
    rank = 0

parser = argparse.ArgumentParser()
parser.add_argument('starfile', type=str, help='File with star cluster data.')
parser.add_argument('--bands', nargs='*', help='bands to use (vals, uncs must be in table)')
parser.add_argument('--props', nargs='*', help='properties to use (valus, uncs must be in table)')
parser.add_argument('--models', type=str, default='mist')
parser.add_argument('--max_distance', type=float, default=1000)
parser.add_argument('--mineep', type=int, default=202)
parser.add_argument('--maxeep', type=int, default=800)
parser.add_argument('--maxAV', type=float, default=0.1)
parser.add_argument('--overwrite', '-o', action='store_true')
parser.add_argument('--nlive', type=int, default=1000)

args = parser.parse_args()

if rank==0:
    ic = get_ichrone(args.models)

    if re.search('.h5', args.starfile) or re.search('.hdf', args.starfile):
        stars = pd.read_hdf(args.starfile)
    else:
        stars = pd.read_csv(args.starfile)

    cat = StarCatalog(stars, props=args.props)

    model = StarClusterModel(ic, cat, eep_bounds=(args.mineep, args.maxeep),
                             max_distance=args.max_distance, max_AV=args.maxAV, name=args.name)
    model.set_prior('feh', FehPrior(halo_fraction=0.5))

else:
    model = None

model = comm.bcast(model, root=0)
model.fit(overwrite=args.overwrite, n_live_points=args.nlive, init_MPI=False)
<commit_msg>add name arg<commit_after>#!/usr/bin/env python
import argparse
import re

import pandas as pd

from isochrones.cluster import StarClusterModel, StarCatalog
from isochrones import get_ichrone
from isochrones.priors import FehPrior

try:
    from mpi4py import MPI
    comm = MPI.COMM_WORLD
    rank = comm.Get_rank()
except ImportError:
    rank = 0

parser = argparse.ArgumentParser()
parser.add_argument('starfile', type=str, help='File with star cluster data.')
parser.add_argument('--bands', nargs='*', help='bands to use (vals, uncs must be in table)')
parser.add_argument('--props', nargs='*', help='properties to use (valus, uncs must be in table)')
parser.add_argument('--models', type=str, default='mist')
parser.add_argument('--max_distance', type=float, default=1000)
parser.add_argument('--mineep', type=int, default=202)
parser.add_argument('--maxeep', type=int, default=800)
parser.add_argument('--maxAV', type=float, default=0.1)
parser.add_argument('--overwrite', '-o', action='store_true')
parser.add_argument('--nlive', type=int, default=1000)
parser.add_argument('--name', type=str, default='')

args = parser.parse_args()

if rank==0:
    ic = get_ichrone(args.models)

    if re.search('.h5', args.starfile) or re.search('.hdf', args.starfile):
        stars = pd.read_hdf(args.starfile)
    else:
        stars = pd.read_csv(args.starfile)

    cat = StarCatalog(stars, props=args.props)

    model = StarClusterModel(ic, cat, eep_bounds=(args.mineep, args.maxeep),
                             max_distance=args.max_distance, max_AV=args.maxAV, name=args.name)
    model.set_prior('feh', FehPrior(halo_fraction=0.5))

else:
    model = None

model = comm.bcast(model, root=0)
model.fit(overwrite=args.overwrite, n_live_points=args.nlive, init_MPI=False)
<|endoftext|>


<commit_before>#!/usr/bin/env python
__author__ = 'daniela'
from scipy.spatial.distance import cosine
from corpora.scikit import ScikitLda
import zipfile
import os
import tarfile
import gzip
import zlib
if __name__ == '__main__':

    dirname = "./data/data/enron_out_0.1/"

    for subdir in os.listdir(dirname):
        # print("subdir", subdir)
        for file in os.listdir(dirname+subdir):
            if file.endswith('pkl'):
                print("attempting... ", file)

                lda = ScikitLda.load(dirname+subdir+"/"+file)
                topics = []
                print "TOPICS in file %s " %file
                print lda.topics

                for topic in lda.topics:
                    topics.append(topic / topic.sum())

                for i in range(len(topics)):
                    for j in range(i + 1, len(topics)):
                            print 'Topic#%d Topic#%d %f' % (i, j, cosine(topics[i], topics[j]))

<commit_msg>use only subdirectories in dirname<commit_after>#!/usr/bin/env python
__author__ = 'daniela'
from scipy.spatial.distance import cosine
from corpora.scikit import ScikitLda
import zipfile
import os
import tarfile
import gzip
import zlib
if __name__ == '__main__':

    dirname = "./data/data/enron_out_0.1/"
    for subdir in [x[0] for x in os.walk(dirname)][1:]:
        # print("subdir", subdir)
        for file in os.listdir(subdir):
            if file.endswith('pkl'):
                print("attempting... ", file)

                lda = ScikitLda.load(subdir+"/"+file)
                topics = []
                print "TOPICS in file %s " %file
                print lda.topics

                for topic in lda.topics:
                    topics.append(topic / topic.sum())

                for i in range(len(topics)):
                    for j in range(i + 1, len(topics)):
                            print 'Topic#%d Topic#%d %f' % (i, j, cosine(topics[i], topics[j]))

<|endoftext|>


<commit_before>#!/usr/bin/env python

import argparse
import sys
import logging
import os.path as pt
import pandas as pd
import numpy as np

__dir = pt.dirname(pt.realpath(__file__))
# sys.path.insert(0, pt.join(__dir, '../module'))

class App(object):

    def run(self, args):
        name = pt.basename(args[0])
        parser = self.create_parser(name)
        opts = parser.parse_args(args[1:])
        return self.main(name, opts)

    def create_parser(self, name):
        p = argparse.ArgumentParser(
            prog=name,
            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
            description='Converts bismark coverage file to bed.')
        p.add_argument(
            'in_file',
            help='Input coverage file',
            nargs='?')
        p.add_argument(
            '-o', '--out_file',
            help='Output file')
        p.add_argument(
            '--verbose',
            help='More detailed log messages',
            action='store_true')
        p.add_argument(
            '--log_file',
            help='Write log messages to file')
        return p

    def main(self, name, opts):
        logging.basicConfig(filename=opts.log_file,
                            format='%(levelname)s (%(asctime)s): %(message)s')
        log = logging.getLogger(name)
        if opts.verbose:
            log.setLevel(logging.DEBUG)
        else:
            log.setLevel(logging.INFO)
        log.debug(opts)

        in_file = opts.in_file if opts.in_file else sys.stdin
        d = pd.read_table(in_file, header=None, usecols=[0, 1, 4, 5],
                          dtype={0: np.str})
        d.columns = ['chromo', 'pos', 'npos', 'nneg']
        d['rate'] = d.npos / (d.npos + d.nneg)
        d = d.loc[:, ['chromo', 'pos', 'rate']]
        s = d.to_csv(opts.out_file, sep='\t', header=None, index=False)
        if s is not None:
            print(s, end='')

        return 0

if __name__ == '__main__':
    app = App()
    app.run(sys.argv)
<commit_msg>Minor<commit_after>#!/usr/bin/env python

import argparse
import sys
import logging
import os.path as pt
import pandas as pd
import numpy as np

__dir = pt.dirname(pt.realpath(__file__))
# sys.path.insert(0, pt.join(__dir, '../module'))

class App(object):

    def run(self, args):
        name = pt.basename(args[0])
        parser = self.create_parser(name)
        opts = parser.parse_args(args[1:])
        return self.main(name, opts)

    def create_parser(self, name):
        p = argparse.ArgumentParser(
            prog=name,
            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
            description='Converts bismark coverage file to bed.')
        p.add_argument(
            'in_file',
            help='Input coverage file',
            nargs='?')
        p.add_argument(
            '-o', '--out_file',
            help='Output file')
        p.add_argument(
            '--rate',
            help='Compute rate instead of binary CpG state',
            action='store_true')
        p.add_argument(
            '--nrows',
            help='Read only that many rows',
            type=int)
        p.add_argument(
            '--verbose',
            help='More detailed log messages',
            action='store_true')
        p.add_argument(
            '--log_file',
            help='Write log messages to file')
        return p

    def main(self, name, opts):
        logging.basicConfig(filename=opts.log_file,
                            format='%(levelname)s (%(asctime)s): %(message)s')
        log = logging.getLogger(name)
        if opts.verbose:
            log.setLevel(logging.DEBUG)
        else:
            log.setLevel(logging.INFO)
        log.debug(opts)

        in_file = opts.in_file if opts.in_file else sys.stdin
        d = pd.read_table(in_file, header=None, usecols=[0, 1, 4, 5],
                          dtype={0: np.str}, nrows=opts.nrows)
        d.columns = ['chromo', 'pos', 'npos', 'nneg']
        d['rate'] = d.npos / (d.npos + d.nneg)
        if not opts.rate:
            d['rate'] = d['rate'].round()
        d = d.loc[:, ['chromo', 'pos', 'rate']]
        s = d.to_csv(opts.out_file, sep='\t', header=None, index=False)
        if s is not None:
            print(s, end='')

        return 0

if __name__ == '__main__':
    app = App()
    app.run(sys.argv)
<|endoftext|>


<commit_before>import json
import sys

HEADER = u"digraph finite_state_machine {\n\tdpi=100;\n\trankdir=LR;\n"
EXCLUDE = ("punct")

def dep_to_dot(deps, fn):
    edges = [
        (d['dep']['lemma'], d['type'], d['gov']['lemma']) for d in deps
        if d['type'] not in EXCLUDE]
    words = set([e[0] for e in edges] + [e[2] for e in edges])
    lines = []
    for word in words:
        lines.append(u'\t{0} [shape=rectangle, label="{0}"];'.format(word))
    for dep, dtype, gov in edges:
        lines.append(u'\t{0} -> {1} [label="{2}"];'.format(dep, gov, dtype))
    with open(fn, 'w') as f:
        f.write(HEADER.encode("utf-8"))
        f.write(u"\n".join(lines).encode("utf-8"))
        f.write("}\n")

def main():
    data = json.load(open(sys.argv[1]))
    i = 0 if len(sys.argv) == 3 else int(sys.argv[3])
    sen = data['deps'][i]
    dep_to_dot(sen, sys.argv[2])

if __name__ == "__main__":
    main()
<commit_msg>now works with dict files too<commit_after>import json
import sys

HEADER = u"digraph finite_state_machine {\n\tdpi=100;\n\trankdir=LR;\n"
EXCLUDE = ("punct")

def dep_to_dot(deps, fn):
    edges = [
        (d['dep']['lemma'], d['type'], d['gov']['lemma']) for d in deps
        if d['type'] not in EXCLUDE]
    words = set([e[0] for e in edges] + [e[2] for e in edges])
    lines = []
    for word in words:
        lines.append(u'\t{0} [shape=rectangle, label="{0}"];'.format(word))
    for dep, dtype, gov in edges:
        lines.append(u'\t{0} -> {1} [label="{2}"];'.format(dep, gov, dtype))
    with open(fn, 'w') as f:
        f.write(HEADER.encode("utf-8"))
        f.write(u"\n".join(lines).encode("utf-8"))
        f.write("}\n")

def main():
    data = json.load(open(sys.argv[1]))
    if 'deps' in data:
        i = 0 if len(sys.argv) == 3 else int(sys.argv[3])
        sen = data['deps'][i]
        dep_to_dot(sen, sys.argv[2])
    else:
        for word, entry in data.iteritems():
            sen = entry['senses'][0]['definition']['deps']
            fn = "{0}/{1}.dot".format(sys.argv[2], word)
            dep_to_dot(sen, fn)

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>import os
import re
import sys
from bf_utilities import run

def execute(command):
    print(command)
    run(command, r"D:\temp")
    print("Finished")
    print()

class Experiment:
    def __init__(self,
                 bf_executables,
                 mg4j_repo,
                 pef_executables,
                 index_root,
                 basename,
                 chunk_dir,
                 chunk_pattern,
                 queries):

        self.bf_executable = bf_executables
        self.mg4j_repo = mg4j_repo
        self.pef_executable = pef_executables

        self.index_root = index_root

        self.chunk_dir = chunk_dir
        self.chunk_pattern = chunk_pattern
        self.basename = basename
        self.queries = queries;

        self.thread_count = 8
        self.pef_index_type = 'opt'

        self.update()

    # update() recalculates properties derived from constructor parameters.
    # For convenience, one can change any of the members directly initialized
    # in the constructor and then call update() to regenerate the derived
    # members.
    def update(self):
        # TODO: Consider copying manifest into root.
        # TODO: What if manifest already exists? Consider pre-verification step that catches errors up front.
        # TODO: Consider basing root name off of manifest name.
        self.root = os.path.join(self.index_root, self.basename)

        self.bf_index_path = os.path.join(self.root, "bitfunnel")
        self.mg4j_index_path = os.path.join(self.root, "mg4j")
        self.pef_index_path = os.path.join(self.root, "pef")

        self.manifest = os.path.join(self.root, self.basename + "-manifest.txt")

        # BitFunnel variables
        self.bf_repl_script = os.path.join(self.bf_index_path, self.basename + "-repl.script")
        self.bf_shard_definition = os.path.join(self.bf_index_path, "ShardDefinition.csv")

        # mg4j variables
        self.mg4j_classpath = os.path.join(self.mg4j_repo, "target", "mg4j-1.0-SNAPSHOT-jar-with-dependencies.jar")
        self.mg4j_basename = os.path.join(self.mg4j_index_path, self.basename)

        # Partitioned ELias-Fano variables
        self.pef_basename = os.path.join(self.pef_index_path, self.basename)
        self.pef_collection = os.path.join(self.pef_index_path, self.basename)
        # TODO: don't hard-code opt
        self.pef_index_file = os.path.join(self.pef_index_path, self.basename + ".index." + self.pef_index_type)
        self.pef_creator = os.path.join(self.pef_executable, "create_freq_index")
        self.pef_runner = os.path.join(self.pef_executable, "Runner")

        # TODO: mapping to filtered query file is in Java right now. Can this be moved here?
        self.queries_basename = os.path.basename(self.queries)
        self.pef_query_file = os.path.join(self.pef_index_path, self.queries_basename + "-ints.txt")
        self.filtered_query_file = os.path.join(self.pef_index_path, self.queries_basename + "-filtered.txt")

        self.pef_results_file = os.path.join(self.pef_index_path, self.queries_basename + "-results.csv")
        self.mg4j_results_file = os.path.join(self.mg4j_index_path, self.queries_basename + "-results.csv")

    def build_mg4j_index(self):
        args = ("java -cp {0} "
                "it.unimi.di.big.mg4j.tool.IndexBuilder "
                "-o org.bitfunnel.reproducibility.ChunkManifestDocumentSequence({1}) "
                "{2}").format(self.mg4j_classpath, self.manifest, self.mg4j_basename)
        if not os.path.exists(self.mg4j_index_path):
            os.makedirs(self.mg4j_index_path)
        execute(args)

    def run_mg4j_queries(self):
        args = ("java -cp {0} "
                "org.bitfunnel.reproducibility.QueryLogRunner "
                "-t {1} {2} {3} {4}").format(self.mg4j_classpath,
                                             self.thread_count,
                                             self.mg4j_basename,
                                             self.filtered_query_file,
                                             self.mg4j_results_file)
        execute(args)

    # TODO: test this method.
    def build_pef_collection(self):
        if not os.path.exists(self.pef_index_path):
            os.makedirs(self.pef_index_path)
        if (self.queries is None):
            args = ("java -cp {0} "
                    "org.bitfunnel.reproducibility.IndexExporter "
                    "{1} {2} --index").format(self.mg4j_classpath, self.mg4j_basename, self.pef_basename);
        else:
            args = ("java -cp {0} "
                    "org.bitfunnel.reproducibility.IndexExporter "
                    "{1} {2} --index "
                    "--queries {3}").format(self.mg4j_classpath, self.mg4j_basename, self.pef_basename, self.queries);
        execute(args)

    # TODO: test this method.
    def build_pef_index(self):
        args = ("{0} {1} {2} {3}").format(self.pef_creator,
                                          self.pef_index_type,
                                          self.pef_collection,
                                          self.pef_index_file)
        execute(args)

    # TODO: test this method.
    def pef_index_from_mg4j_index(params):
        params.build_pef_collection()
        params.build_pef_index()

    # TODO: test this method.
    def run_pef_queries(self):
        args = ("{0}{1} {2} {3} {4} {5}").format(self.pef_runner,
                                               self.pef_index_type,
                                               self.pef_index_file,
                                               self.pef_query_file,
                                               self.thread_count,
                                               self.pef_results_file)
        execute(args)

    def build_bf_index(self):
        if not os.path.exists(self.bf_index_path):
            os.makedirs(self.bf_index_path)

        # We're currently restricted to a single shard,
        # so create an empty ShardDefinition file.
        # TODO: reinstate following line.
        open(self.bf_shard_definition, "w").close()

        # Run statistics builder
        args = ("{0} statistics {1} {2}").format(self.bf_executable,
                                             self.manifest,
                                             self.bf_index_path)
        execute(args)

        # Run termtable builder
        # TODO: don't hard code density.
        # TODO: don't hard code Optimal.
        # TODO: don't hard code SNR.
        args = ("{0} termtable {1} 0.1 Optimal").format(self.bf_executable,
                                                        self.bf_index_path)
        execute(args)

    # TODO: test this method.
    def run_bf_queries(self):
        # Create script file
        # TODO: reinstate following lines.
        with open(self.bf_repl_script, "w") as file:
        # file = sys.stdout
            for i in range(0,1):
                file.write("load manifest {0}\n".format(self.manifest));
                file.write("compiler\n");
                for t in range(1, self.thread_count + 1):
                    results_dir = os.path.join(self.bf_index_path, "results-{0}".format(t))
                    if not os.path.exists(results_dir):
                        print("mkdir " + results_dir)
                        os.makedirs(results_dir)
                    file.write("threads {0}\n".format(t))
                    file.write("cd {0}\n".format(results_dir))
                    file.write("query log {0}\n".format(self.filtered_query_file))
                    file.write("quit\n")

        # Start BitFunnel repl
        args = ("{0} repl {1} -script {2}").format(self.bf_executable,
                                                   self.bf_index_path,
                                                   self.bf_repl_script)
        execute(args)

    # TODO: test this method.
    def build_lucene_index(self):
        print("TODO: Implement build_lucene_index")

    # TODO: test this method.
    def run_lucene_queries(self):
        print("TODO: Implement run_lucene_queries")

    def build_chunk_manifest(self):
        regex = re.compile(self.chunk_pattern)
        chunks = [os.path.join(self.chunk_dir, f)
                  for root, dirs, files in os.walk(self.chunk_dir)
                  for f in files
                  if regex.search(f) is not None]

        for chunk in chunks:
            print(chunk)

        if not os.path.exists(self.root):
            os.makedirs(self.root)
        # dir = os.path.dirname(manifest)
        # if not os.path.exists(dir):
        #     os.makedirs(dir)

        print("Writing manifest {0}".format(self.manifest))
        with open(self.manifest, 'w') as file:
            for chunk in chunks:
                file.write(chunk + '\n')

manifest = r"D:\temp\index-273-100-150\index-273-1000-1500.txt"

experiment = Experiment(
    # Paths to tools
    r"D:\git\BitFunnel\build-msvc\tools\BitFunnel\src\Release\BitFunnel.exe",
    r"D:\git\mg4j-workbench",
    r"/home/mhop/git/partitioned_elias_fano/bin",

    # The directory containing all indexes and the basename for this index
    r"D:\temp\indexes",
    r"273-150-100",

    # The directory with the gov2 chunks and the regular expression pattern
    # used to determine which chunks will be used for this experiment.
    r"d:\sigir\chunks-100-150",
    r"GX.*",  # Use all chunks

    # The query log to be used for this experiment.
    r"D:\git\mg4j-workbench\data\trec-terabyte\06.efficiency_topics.all"
)

experiment.build_chunk_manifest()
experiment.build_mg4j_index()

def tested():
    experiment.build_mg4j_index()
    experiment.run_mg4j_queries()

    experiment.build_bf_index()
    experiment.run_bf_queries()

# Partially tested. Still need to test PEF part.
# experiment.pef_index_from_mg4j_index()

# Untested
# experiment.run_pef_queries()
#
#
# experiment.build_lucene_index()
# experiment.run_lucene_queries()

<commit_msg>Fixed bug where BitFunnel repl would exit after one run.<commit_after>import os
import re
import sys
from bf_utilities import run

def execute(command):
    print(command)
    run(command, r"D:\temp")
    print("Finished")
    print()

class Experiment:
    def __init__(self,
                 bf_executables,
                 mg4j_repo,
                 pef_executables,
                 index_root,
                 basename,
                 chunk_dir,
                 chunk_pattern,
                 queries):

        self.bf_executable = bf_executables
        self.mg4j_repo = mg4j_repo
        self.pef_executable = pef_executables

        self.index_root = index_root

        self.chunk_dir = chunk_dir
        self.chunk_pattern = chunk_pattern
        self.basename = basename
        self.queries = queries;

        self.thread_count = 8
        self.pef_index_type = 'opt'

        self.update()

    # update() recalculates properties derived from constructor parameters.
    # For convenience, one can change any of the members directly initialized
    # in the constructor and then call update() to regenerate the derived
    # members.
    def update(self):
        # TODO: Consider copying manifest into root.
        # TODO: What if manifest already exists? Consider pre-verification step that catches errors up front.
        # TODO: Consider basing root name off of manifest name.
        self.root = os.path.join(self.index_root, self.basename)

        self.bf_index_path = os.path.join(self.root, "bitfunnel")
        self.mg4j_index_path = os.path.join(self.root, "mg4j")
        self.pef_index_path = os.path.join(self.root, "pef")

        self.manifest = os.path.join(self.root, self.basename + "-manifest.txt")

        # BitFunnel variables
        self.bf_repl_script = os.path.join(self.bf_index_path, self.basename + "-repl.script")
        self.bf_shard_definition = os.path.join(self.bf_index_path, "ShardDefinition.csv")

        # mg4j variables
        self.mg4j_classpath = os.path.join(self.mg4j_repo, "target", "mg4j-1.0-SNAPSHOT-jar-with-dependencies.jar")
        self.mg4j_basename = os.path.join(self.mg4j_index_path, self.basename)

        # Partitioned ELias-Fano variables
        self.pef_basename = os.path.join(self.pef_index_path, self.basename)
        self.pef_collection = os.path.join(self.pef_index_path, self.basename)
        # TODO: don't hard-code opt
        self.pef_index_file = os.path.join(self.pef_index_path, self.basename + ".index." + self.pef_index_type)
        self.pef_creator = os.path.join(self.pef_executable, "create_freq_index")
        self.pef_runner = os.path.join(self.pef_executable, "Runner")

        # TODO: mapping to filtered query file is in Java right now. Can this be moved here?
        self.queries_basename = os.path.basename(self.queries)
        self.pef_query_file = os.path.join(self.pef_index_path, self.queries_basename + "-ints.txt")
        self.filtered_query_file = os.path.join(self.pef_index_path, self.queries_basename + "-filtered.txt")

        self.pef_results_file = os.path.join(self.pef_index_path, self.queries_basename + "-results.csv")
        self.mg4j_results_file = os.path.join(self.mg4j_index_path, self.queries_basename + "-results.csv")

    def build_mg4j_index(self):
        args = ("java -cp {0} "
                "it.unimi.di.big.mg4j.tool.IndexBuilder "
                "-o org.bitfunnel.reproducibility.ChunkManifestDocumentSequence({1}) "
                "{2}").format(self.mg4j_classpath, self.manifest, self.mg4j_basename)
        if not os.path.exists(self.mg4j_index_path):
            os.makedirs(self.mg4j_index_path)
        execute(args)

    def run_mg4j_queries(self):
        args = ("java -cp {0} "
                "org.bitfunnel.reproducibility.QueryLogRunner "
                "-t {1} {2} {3} {4}").format(self.mg4j_classpath,
                                             self.thread_count,
                                             self.mg4j_basename,
                                             self.filtered_query_file,
                                             self.mg4j_results_file)
        execute(args)

    # TODO: test this method.
    def build_pef_collection(self):
        if not os.path.exists(self.pef_index_path):
            os.makedirs(self.pef_index_path)
        if (self.queries is None):
            args = ("java -cp {0} "
                    "org.bitfunnel.reproducibility.IndexExporter "
                    "{1} {2} --index").format(self.mg4j_classpath, self.mg4j_basename, self.pef_basename);
        else:
            args = ("java -cp {0} "
                    "org.bitfunnel.reproducibility.IndexExporter "
                    "{1} {2} --index "
                    "--queries {3}").format(self.mg4j_classpath, self.mg4j_basename, self.pef_basename, self.queries);
        execute(args)

    # TODO: test this method.
    def build_pef_index(self):
        args = ("{0} {1} {2} {3}").format(self.pef_creator,
                                          self.pef_index_type,
                                          self.pef_collection,
                                          self.pef_index_file)
        execute(args)

    # TODO: test this method.
    def pef_index_from_mg4j_index(params):
        params.build_pef_collection()
        params.build_pef_index()

    # TODO: test this method.
    def run_pef_queries(self):
        args = ("{0}{1} {2} {3} {4} {5}").format(self.pef_runner,
                                               self.pef_index_type,
                                               self.pef_index_file,
                                               self.pef_query_file,
                                               self.thread_count,
                                               self.pef_results_file)
        execute(args)

    def build_bf_index(self):
        if not os.path.exists(self.bf_index_path):
            os.makedirs(self.bf_index_path)

        # We're currently restricted to a single shard,
        # so create an empty ShardDefinition file.
        # TODO: reinstate following line.
        open(self.bf_shard_definition, "w").close()

        # Run statistics builder
        args = ("{0} statistics {1} {2}").format(self.bf_executable,
                                             self.manifest,
                                             self.bf_index_path)
        execute(args)

        # Run termtable builder
        # TODO: don't hard code density.
        # TODO: don't hard code Optimal.
        # TODO: don't hard code SNR.
        args = ("{0} termtable {1} 0.1 Optimal").format(self.bf_executable,
                                                        self.bf_index_path)
        execute(args)

    # TODO: test this method.
    def run_bf_queries(self):
        # Create script file
        # TODO: reinstate following lines.
        with open(self.bf_repl_script, "w") as file:
        # file = sys.stdout
            for i in range(0,1):
                file.write("load manifest {0}\n".format(self.manifest));
                file.write("compiler\n");
                for t in range(1, self.thread_count + 1):
                    results_dir = os.path.join(self.bf_index_path, "results-{0}".format(t))
                    if not os.path.exists(results_dir):
                        print("mkdir " + results_dir)
                        os.makedirs(results_dir)
                    file.write("threads {0}\n".format(t))
                    file.write("cd {0}\n".format(results_dir))
                    file.write("query log {0}\n".format(self.filtered_query_file))

                file.write("quit\n")

        # Start BitFunnel repl
        args = ("{0} repl {1} -script {2}").format(self.bf_executable,
                                                   self.bf_index_path,
                                                   self.bf_repl_script)
        execute(args)

    # TODO: test this method.
    def build_lucene_index(self):
        print("TODO: Implement build_lucene_index")

    # TODO: test this method.
    def run_lucene_queries(self):
        print("TODO: Implement run_lucene_queries")

    def build_chunk_manifest(self):
        regex = re.compile(self.chunk_pattern)
        chunks = [os.path.join(self.chunk_dir, f)
                  for root, dirs, files in os.walk(self.chunk_dir)
                  for f in files
                  if regex.search(f) is not None]

        for chunk in chunks:
            print(chunk)

        if not os.path.exists(self.root):
            os.makedirs(self.root)
        # dir = os.path.dirname(manifest)
        # if not os.path.exists(dir):
        #     os.makedirs(dir)

        print("Writing manifest {0}".format(self.manifest))
        with open(self.manifest, 'w') as file:
            for chunk in chunks:
                file.write(chunk + '\n')

# manifest = r"D:\temp\index-273-100-150\index-273-1000-1500.txt"

experiment = Experiment(
    # Paths to tools
    r"D:\git\BitFunnel\build-msvc\tools\BitFunnel\src\Release\BitFunnel.exe",
    r"D:\git\mg4j-workbench",
    r"/home/mhop/git/partitioned_elias_fano/bin",

    # The directory containing all indexes and the basename for this index
    r"D:\temp\indexes",
    r"273-1000-1500",

    # The directory with the gov2 chunks and the regular expression pattern
    # used to determine which chunks will be used for this experiment.
    r"d:\sigir\chunks-1000-1500",
    r"GX.*",  # Use all chunks

    # The query log to be used for this experiment.
    r"D:\git\mg4j-workbench\data\trec-terabyte\06.efficiency_topics.all"
)

# experiment.build_chunk_manifest()
# experiment.build_mg4j_index()
# experiment.run_mg4j_queries()
# experiment.build_bf_index()
experiment.run_bf_queries()

def tested():
    experiment.build_mg4j_index()
    experiment.run_mg4j_queries()

    experiment.build_bf_index()
    experiment.run_bf_queries()

# Partially tested. Still need to test PEF part.
# experiment.pef_index_from_mg4j_index()

# Untested
# experiment.run_pef_queries()
#
#
# experiment.build_lucene_index()
# experiment.run_lucene_queries()

<|endoftext|>


<commit_before>import os
import re
import time
import config
import plotter
import subprocess
import matplotlib.mlab as mlab
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages

class experiment:
        def __init__(self, task, outdir, trials = 10):
                self.cfg = config.config()
                self.task = task
                self.dir = outdir
                self.models = []
                self.trainers = []
                self.enhancers = []
                self.losses = []
                self.trials = trials

        def log(self, *messages):
                print(time.strftime("[%Y-%m-%d %H:%M:%S]"), ' '.join(messages))

        def add_model(self, name, parameters):
                self.models.append([name, parameters])

        def add_trainer(self, name, parameters, config_name = None):
                self.trainers.append([config_name if config_name else name, self.cfg.config_trainer(name, parameters)])

        def add_loss(self, name, parameters = "", config_name = None):
                self.losses.append([config_name if config_name else name, self.cfg.config_loss(name, parameters)])

        def add_enhancer(self, name, parameters = "", config_name = None):
                self.enhancers.append([config_name if config_name else name, self.cfg.config_enhancer(name, parameters)])

        def path(self, trial, mname, tname, ename, lname, extension):
                basepath = self.dir
                basepath += "trial{}".format(trial) if not (trial is None) else "result"
                basepath += "_M" + mname if mname else ""
                basepath += "_T" + tname if tname else ""
                basepath += "_E" + ename if ename else ""
                basepath += "_L" + lname if lname else ""
                basepath += extension
                return basepath

        def name(self, name_config):
                return name_config[0]

        def config(self, name_config):
                return name_config[1]

        def names(self, names_configs, name_reg = None):
                names = []
                for name_config in names_configs:
                        name = self.name(name_config)
                        if name_reg == None or re.match(name_reg, name):
                                names.append(name)
                return names

        def filter_names(self, mname_reg, tname_reg, ename_reg, lname_reg):
                mnames = self.names(self.models, mname_reg)
                tnames = self.names(self.trainers, tname_reg)
                enames = self.names(self.enhancers, ename_reg)
                lnames = self.names(self.losses, lname_reg)
                return mnames, tnames, enames, lnames

        def filter_paths(self, trial, mname_reg, tname_reg, ename_reg, lname_reg, extension):
                paths = []
                mnames, tnames, enames, lnames = self.filter_names(mname_reg, tname_reg, ename_reg, lname_reg)
                for mname, tname, ename, lname in [(u, x, y, z) for u in mnames for x in tnames for y in enames for z in lnames]:
                        paths.append(self.path(trial, mname, tname, ename, lname, extension))
                return paths

        def reg2str(self, reg):
                return reg.replace(".*", "all").replace("*", "").replace(".", "") if reg else "all"

        def train_one(self, param, lpath):
                with open(lpath, "w") as lfile:
                        self.log("running <", param, ">...")
                        subprocess.check_call((self.cfg.app_train + " " + param).split(), stdout = lfile)
                self.log("|--->training done, see <", lpath, ">")

        def plot_one(self, spath, ppath):
                plotter.plot_state_one(spath, ppath)
                self.log("|--->plotting done, see <", ppath, ">")

        def plot_many(self, spaths, ppath):
                plotter.plot_state_many(spaths, ppath)
                self.log("|--->plotting done, see <", ppath, ">")

        def plot_trial(self, spaths, ppath, names):
                plotter.plot_trial_many(spaths, ppath, names)
                self.log("|--->plotting done, see <", ppath, ">")

        def tabulate(self, cpath, lpath):
                with open(lpath, "w") as lfile:
                        subprocess.check_call((self.cfg.app_tabulate + " -i " + cpath + " -d \";\"").split(), stdout = lfile)

                with open(lpath, "r") as lfile:
                        self.log()
                        print(lfile.read())

        def summarize_by_models(self, mname_reg = ".*"):
                mname = self.reg2str(mname_reg)

        def train_trials(self, mname, mparam, tname, tparam, ename, eparam, lname, lparam):
                # train the given configuration for multiple trials
                for trial in range(self.trials):
                        os.makedirs(self.dir, exist_ok = True)
                        mpath = self.path(trial, mname, tname, ename, lname, ".model")
                        spath = self.path(trial, mname, tname, ename, lname, ".state")
                        lpath = self.path(trial, mname, tname, ename, lname, ".log")
                        ppath = self.path(trial, mname, tname, ename, lname, ".pdf")

                        param = self.task + " " + mparam + " " + tparam + " " + eparam + " " + lparam + " --model-file " + mpath
                        self.train_one(param, lpath)
                        self.plot_one(spath, ppath)
                        self.log()

                # export the results from multiple trials as csv
                cpath = self.path(None, mname, tname, ename, lname, ".csv")
                spath = self.path(None, mname, tname, ename, lname, ".stats")
                with open(cpath, "w") as cfile, open(spath, "w") as sfile:
                        print(self.get_csv_header(), file = cfile)
                        print(self.get_csv_header(), file = sfile)
                        values, errors, epochs, speeds, deltas = self.get_logs( mname, tname, ename, lname)
                        for value, error, epoch, speed, delta in zip(values, errors, epochs, speeds, deltas):
                                print(self.get_csv_row(mname, tname, ename, lname, value, error, epoch, speed, delta), file = cfile)
                        value_stats, error_stats, epoch_stats, speed_stats, delta_stats = self.get_log_stats(values, errors, epochs, speeds, deltas)
                        print(self.get_csv_row(mname, tname, ename, lname, value_stats, error_stats, epoch_stats, speed_stats, delta_stats), file = sfile)

                # display basic statistics for the results from multiple trials
                lpath = self.path(None, mname, tname, ename, lname, ".log")
                self.tabulate(spath, lpath)

        def train_all(self):
                # run all possible configurations
                mdatas = self.models
                tdatas = self.trainers
                edatas = self.enhancers
                ldatas = self.losses
                for mdata, tdata, edata, ldata in [(u, x, y, z) for u in mdatas for x in tdatas for y in edatas for z in ldatas]:
                        self.train_trials(
                                self.name(mdata), self.config(mdata),
                                self.name(tdata), self.config(tdata),
                                self.name(edata), self.config(edata),
                                self.name(ldata), self.config(ldata))

        def summarize_trials(self, mname, mname_reg, tname, tname_reg, ename, ename_reg, lname, lname_reg, names):
                # compare configurations for each trial: plot the training state evaluation
                for trial in range(self.trials):
                        self.plot_many(
                                self.filter_paths(trial, mname_reg, tname_reg, ename_reg, lname_reg, ".state"),
                                self.path(trial, mname, tname, ename, lname, ".pdf"))

                # compare configurations across all trials: boxplot the results
                ppath = self.path(None, mname, tname, ename, lname, ".pdf")

                self.plot_trial(
                        self.filter_paths(None, mname_reg, tname_reg, ename_reg, lname_reg, ".csv"),
                        ppath, names)

                # compare configurations across all trials: display basic statistics
                spath = self.path(None, mname, tname, ename, lname, ".stats")
                with open(spath, "w") as sfile:
                        print(self.get_csv_header(), file = sfile)
                        mnames, tnames, enames, lnames = self.filter_names(mname_reg, tname_reg, ename_reg, lname_reg)
                        for mname, tname, ename, lname in [(u, x, y, z) for u in mnames for x in tnames for y in enames for z in lnames]:
                                values, errors, epochs, speeds, deltas = self.get_logs(mname, tname, ename, lname)
                                value_stats, error_stats, epoch_stats, speed_stats, delta_stats = self.get_log_stats(values, errors, epochs, speeds, deltas)
                                print(self.get_csv_row(mname, tname, ename, lname, value_stats, error_stats, epoch_stats, speed_stats, delta_stats), file = sfile)

                lpath = self.path(None, mname, tname, ename, lname, ".log")
                self.tabulate(spath, lpath)

        def summarize_by_models(self, mname_reg = ".*"):
                mname = self.reg2str(mname_reg)
                mnames, tnames, enames, lnames = self.filter_names(mname_reg, None, None, None)
                for tname, ename, lname in [(x, y, z) for x in tnames for y in enames for z in lnames]:
                        self.summarize_trials(mname, mname_reg, tname, tname, ename, ename, lname, lname, mnames)

        def summarize_by_trainers(self, tname_reg = ".*"):
                tname = self.reg2str(tname_reg)
                mnames, tnames, enames, lnames = self.filter_names(None, tname_reg, None, None)
                for mname, ename, lname in [(x, y, z) for x in mnames for y in enames for z in lnames]:
                        self.summarize_trials(mname, mname, tname, tname_reg, ename, ename, lname, lname, tnames)

        def summarize_by_enhancers(self, ename_reg = ".*"):
                ename = self.reg2str(ename_reg)
                mnames, tnames, enames, lnames = self.filter_names(None, None, ename_reg, None)
                for mname, tname, lname in [(x, y, z) for x in mnames for y in tnames for z in lnames]:
                        self.summarize_trials(mname, mname, tname, tname, ename, ename_reg, lname, lname, enames)

        def summarize_by_losses(self, lname_reg = ".*"):
                lname = self.reg2str(lname_reg)
                mnames, tnames, enames, lnames = self.filter_names(None, None, None, lname_reg)
                for mname, tname, ename in [(x, y, z) for x in mnames for y in tnames for z in enames]:
                        self.summarize_trials(mname, mname, tname, tname, ename, ename, lname, lname_reg, lnames)

        def get_token(self, line, begin_delim, end_delim, start = 0):
                begin = line.find(begin_delim, start) + len(begin_delim)
                end = line.find(end_delim, begin)
                return line[begin : end], end

        def get_seconds(self, delta):
                delta = delta.replace("ms", "/1000")
                delta = delta.replace("s:", "+")
                delta = delta.replace("m:", "*60*60+")
                delta = delta.replace("h:", "*60*60*60+")
                delta = delta.replace("d:", "*24*60*60*60+")
                while (len(delta) > 0) and (delta[0] == '0'):
                        delta = delta[1:]
                delta = delta.replace("+00*", "1*")
                delta = delta.replace("+00", "")
                delta = delta.replace("+0", "+")
                return str(eval(delta))

        def get_log(self, lpath):
                with open(lpath, "r") as lfile:
                        for line in lfile:
                                if line.find("speed=") < 0:
                                        continue
                                # search for the test loss value
                                value, index = self.get_token(line, "test=", "|", 0)
                                # search for the test error
                                error, index = self.get_token(line, "|", ",", index)
                                # search for the optimum number of epochs
                                epoch, index = self.get_token(line, "epoch=", ",", index)
                                # search for the convergence speed
                                speed, index = self.get_token(line, "speed=", "/s", index)
                                # duration
                                delta, index = self.get_token(line, "time=", ".", index)
                                return value, error, epoch, speed, self.get_seconds(delta)
                print("invalid log file <", lpath, ">")

        def get_logs(self, mname, tname, ename, lname):
                values = []
                errors = []
                epochs = []
                speeds = []
                deltas = []
                for trial in range(self.trials):
                        lpath = self.path(trial, mname, tname, ename, lname, ".log")
                        value, error, epoch, speed, delta = self.get_log(lpath)
                        values.append(value)
                        errors.append(error)
                        epochs.append(epoch)
                        speeds.append(speed)
                        deltas.append(delta)
                return values, errors, epochs, speeds, deltas

        def get_log_stats(self, values, errors, epochs, speeds, deltas):
                cmdline = self.cfg.app_stats + " -p 4"
                value_stats = subprocess.check_output(cmdline.split() + values).decode('utf-8').strip()
                error_stats = subprocess.check_output(cmdline.split() + errors).decode('utf-8').strip()
                epoch_stats = subprocess.check_output(cmdline.split() + epochs).decode('utf-8').strip()
                speed_stats = subprocess.check_output(cmdline.split() + speeds).decode('utf-8').strip()
                delta_stats = subprocess.check_output(cmdline.split() + deltas).decode('utf-8').strip()
                return value_stats, error_stats, epoch_stats, speed_stats, delta_stats

        def get_csv_header(self, delim = ";"):
                return delim.join(["model", "trainer", "enhancer", "loss", "test value", "test error", "epochs", "convergence speed", "duration (sec)"])

        def get_csv_row(self, mname, tname, ename, lname, value, error, epoch, speed, delta, delim = ";"):
                return delim.join([mname, tname, ename, lname, value, error, epoch, speed, delta])
<commit_msg>draft of separating an experimentation run into its own class<commit_after>import os
import re
import time
import config
import plotter
import subprocess
import matplotlib.mlab as mlab
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages

class experiment_run:
        """ an experiment run consists of:
                - a trial (fold index or None if cumulating results from multiple trials)
                - a model (name + command line parameters to apps/builder)
                - a trainer (name + json parameters)
                - an enhancer (name + json parameters)
                - a loss function (name)
        """
        def __init__(self, trial, mname, mparam, tname, tparam, ename, eparam, lname, lparam, dirname):
                self.trial = trial
                self.mname = mname
                self.tname = tname
                self.ename = ename
                self.lname = lname
                self.mparam = mparam
                self.tparam = tparam
                self.eparam = eparam
                self.lparam = lparam
                self.dirname = os.path.join(dirname, "/trial{}/".format(trial) if not (trial is None) else "/result/")
                os.makedirs(self.dirname, exist_ok = True)

        def path(self, extension):
                return os.path.join(self.dirname, "{}{}{}{}".format(
                        "_M" + mname if mname else "",
                        "_T" + tname if tname else "",
                        "_E" + ename if ename else "",
                        "_L" + lname if lname else ""), extension)

        def mpath(self):
                return self.path(".model")

        def spath(self):
                return self.path(".state")

        def lpath(self):
                return self.path(".log")

        def ppath(self):
                return self.path(".pdf")

        def filter(self, trial, mname_reg, tname_reg, ename_reg, lname_reg):
                return  self.trial == trial and
                        re.match(mname_reg, self.mname) and
                        re.match(tname_reg, self.tname) and
                        re.match(ename_reg, self.ename) and
                        re.match(lname_reg, self.lname)

class experiment:
        """ """
        def __init__(self, task, outdir, trials = 10):
                self.cfg = config.config()
                self.task = task
                self.dir = outdir
                self.trials = trials
                self.runs = []
                self.losses = []
                self.models = []
                self.trainers = []
                self.enhancers = []

        def log(self, *messages):
                print(time.strftime("[%Y-%m-%d %H:%M:%S]"), ' '.join(messages))

        def add_model(self, name, parameters):
                self.models.append([name, parameters])

        def add_trainer(self, name, parameters, config_name = None):
                self.trainers.append([config_name if config_name else name, parameters])

        def add_loss(self, name, parameters = "", config_name = None):
                self.losses.append([config_name if config_name else name, parameters])

        def add_enhancer(self, name, parameters = "", config_name = None):
                self.enhancers.append([config_name if config_name else name, parameters])

        def path(self, trial, mname, tname, ename, lname, extension):
                basepath = self.dir
                basepath += "trial{}".format(trial) if not (trial is None) else "result"
                basepath += "_M" + mname if mname else ""
                basepath += "_T" + tname if tname else ""
                basepath += "_E" + ename if ename else ""
                basepath += "_L" + lname if lname else ""
                basepath += extension
                return basepath

        def name(self, name_config):
                return name_config[0]

        def config(self, name_config):
                return name_config[1]

        def names(self, names_configs, name_reg = None):
                names = []
                for name_config in names_configs:
                        name = self.name(name_config)
                        if name_reg == None or re.match(name_reg, name):
                                names.append(name)
                return names

        def filter_names(self, mname_reg, tname_reg, ename_reg, lname_reg):
                mnames = self.names(self.models, mname_reg)
                tnames = self.names(self.trainers, tname_reg)
                enames = self.names(self.enhancers, ename_reg)
                lnames = self.names(self.losses, lname_reg)
                return mnames, tnames, enames, lnames

        def filter_paths(self, trial, mname_reg, tname_reg, ename_reg, lname_reg, extension):
                paths = []
                mnames, tnames, enames, lnames = self.filter_names(mname_reg, tname_reg, ename_reg, lname_reg)
                for mname, tname, ename, lname in [(u, x, y, z) for u in mnames for x in tnames for y in enames for z in lnames]:
                        paths.append(self.path(trial, mname, tname, ename, lname, extension))
                return paths

        def reg2str(self, reg):
                return reg.replace(".*", "all").replace("*", "").replace(".", "") if reg else "all"

        def train_one(self, param, lpath):
                with open(lpath, "w") as lfile:
                        self.log("running <", param, ">...")
                        subprocess.check_call((self.cfg.app_train + " " + param).split(), stdout = lfile)
                self.log("|--->training done, see <", lpath, ">")

        def plot_one(self, spath, ppath):
                plotter.plot_state_one(spath, ppath)
                self.log("|--->plotting done, see <", ppath, ">")

        def plot_many(self, spaths, ppath):
                plotter.plot_state_many(spaths, ppath)
                self.log("|--->plotting done, see <", ppath, ">")

        def plot_trial(self, spaths, ppath, names):
                plotter.plot_trial_many(spaths, ppath, names)
                self.log("|--->plotting done, see <", ppath, ">")

        def tabulate(self, cpath, lpath):
                with open(lpath, "w") as lfile:
                        subprocess.check_call((self.cfg.app_tabulate + " -i " + cpath + " -d \";\"").split(), stdout = lfile)

                with open(lpath, "r") as lfile:
                        self.log()
                        print(lfile.read())

        def summarize_by_models(self, mname_reg = ".*"):
                mname = self.reg2str(mname_reg)

        def train_trials(self, mname, mparam, tname, tparam, ename, eparam, lname, lparam):
                # train the given configuration for multiple trials
                for trial in range(self.trials):
                        os.makedirs(self.dir, exist_ok = True)
                        mpath = self.path(trial, mname, tname, ename, lname, ".model")
                        spath = self.path(trial, mname, tname, ename, lname, ".state")
                        lpath = self.path(trial, mname, tname, ename, lname, ".log")
                        ppath = self.path(trial, mname, tname, ename, lname, ".pdf")

                        param = self.task + " " + mparam + " " + tparam + " " + eparam + " " + lparam + " --model-file " + mpath
                        self.train_one(param, lpath)
                        self.plot_one(spath, ppath)
                        self.log()

                # export the results from multiple trials as csv
                cpath = self.path(None, mname, tname, ename, lname, ".csv")
                spath = self.path(None, mname, tname, ename, lname, ".stats")
                with open(cpath, "w") as cfile, open(spath, "w") as sfile:
                        print(self.get_csv_header(), file = cfile)
                        print(self.get_csv_header(), file = sfile)
                        values, errors, epochs, speeds, deltas = self.get_logs( mname, tname, ename, lname)
                        for value, error, epoch, speed, delta in zip(values, errors, epochs, speeds, deltas):
                                print(self.get_csv_row(mname, tname, ename, lname, value, error, epoch, speed, delta), file = cfile)
                        value_stats, error_stats, epoch_stats, speed_stats, delta_stats = self.get_log_stats(values, errors, epochs, speeds, deltas)
                        print(self.get_csv_row(mname, tname, ename, lname, value_stats, error_stats, epoch_stats, speed_stats, delta_stats), file = sfile)

                # display basic statistics for the results from multiple trials
                lpath = self.path(None, mname, tname, ename, lname, ".log")
                self.tabulate(spath, lpath)

        def train_all(self):
                # run all possible configurations
                mdatas = self.models
                tdatas = self.trainers
                edatas = self.enhancers
                ldatas = self.losses
                for mdata, tdata, edata, ldata in [(u, x, y, z) for u in mdatas for x in tdatas for y in edatas for z in ldatas]:
                        self.train_trials(
                                self.name(mdata), self.config(mdata),
                                self.name(tdata), self.config(tdata),
                                self.name(edata), self.config(edata),
                                self.name(ldata), self.config(ldata))

        def summarize_trials(self, mname, mname_reg, tname, tname_reg, ename, ename_reg, lname, lname_reg, names):
                # compare configurations for each trial: plot the training state evaluation
                for trial in range(self.trials):
                        self.plot_many(
                                self.filter_paths(trial, mname_reg, tname_reg, ename_reg, lname_reg, ".state"),
                                self.path(trial, mname, tname, ename, lname, ".pdf"))

                # compare configurations across all trials: boxplot the results
                ppath = self.path(None, mname, tname, ename, lname, ".pdf")

                self.plot_trial(
                        self.filter_paths(None, mname_reg, tname_reg, ename_reg, lname_reg, ".csv"),
                        ppath, names)

                # compare configurations across all trials: display basic statistics
                spath = self.path(None, mname, tname, ename, lname, ".stats")
                with open(spath, "w") as sfile:
                        print(self.get_csv_header(), file = sfile)
                        mnames, tnames, enames, lnames = self.filter_names(mname_reg, tname_reg, ename_reg, lname_reg)
                        for mname, tname, ename, lname in [(u, x, y, z) for u in mnames for x in tnames for y in enames for z in lnames]:
                                values, errors, epochs, speeds, deltas = self.get_logs(mname, tname, ename, lname)
                                value_stats, error_stats, epoch_stats, speed_stats, delta_stats = self.get_log_stats(values, errors, epochs, speeds, deltas)
                                print(self.get_csv_row(mname, tname, ename, lname, value_stats, error_stats, epoch_stats, speed_stats, delta_stats), file = sfile)

                lpath = self.path(None, mname, tname, ename, lname, ".log")
                self.tabulate(spath, lpath)

        def summarize_by_models(self, mname_reg = ".*"):
                mname = self.reg2str(mname_reg)
                mnames, tnames, enames, lnames = self.filter_names(mname_reg, None, None, None)
                for tname, ename, lname in [(x, y, z) for x in tnames for y in enames for z in lnames]:
                        self.summarize_trials(mname, mname_reg, tname, tname, ename, ename, lname, lname, mnames)

        def summarize_by_trainers(self, tname_reg = ".*"):
                tname = self.reg2str(tname_reg)
                mnames, tnames, enames, lnames = self.filter_names(None, tname_reg, None, None)
                for mname, ename, lname in [(x, y, z) for x in mnames for y in enames for z in lnames]:
                        self.summarize_trials(mname, mname, tname, tname_reg, ename, ename, lname, lname, tnames)

        def summarize_by_enhancers(self, ename_reg = ".*"):
                ename = self.reg2str(ename_reg)
                mnames, tnames, enames, lnames = self.filter_names(None, None, ename_reg, None)
                for mname, tname, lname in [(x, y, z) for x in mnames for y in tnames for z in lnames]:
                        self.summarize_trials(mname, mname, tname, tname, ename, ename_reg, lname, lname, enames)

        def summarize_by_losses(self, lname_reg = ".*"):
                lname = self.reg2str(lname_reg)
                mnames, tnames, enames, lnames = self.filter_names(None, None, None, lname_reg)
                for mname, tname, ename in [(x, y, z) for x in mnames for y in tnames for z in enames]:
                        self.summarize_trials(mname, mname, tname, tname, ename, ename, lname, lname_reg, lnames)

        def get_token(self, line, begin_delim, end_delim, start = 0):
                begin = line.find(begin_delim, start) + len(begin_delim)
                end = line.find(end_delim, begin)
                return line[begin : end], end

        def get_seconds(self, delta):
                delta = delta.replace("ms", "/1000")
                delta = delta.replace("s:", "+")
                delta = delta.replace("m:", "*60*60+")
                delta = delta.replace("h:", "*60*60*60+")
                delta = delta.replace("d:", "*24*60*60*60+")
                while (len(delta) > 0) and (delta[0] == '0'):
                        delta = delta[1:]
                delta = delta.replace("+00*", "1*")
                delta = delta.replace("+00", "")
                delta = delta.replace("+0", "+")
                return str(eval(delta))

        def get_log(self, lpath):
                with open(lpath, "r") as lfile:
                        for line in lfile:
                                if line.find("speed=") < 0:
                                        continue
                                # search for the test loss value
                                value, index = self.get_token(line, "test=", "|", 0)
                                # search for the test error
                                error, index = self.get_token(line, "|", ",", index)
                                # search for the optimum number of epochs
                                epoch, index = self.get_token(line, "epoch=", ",", index)
                                # search for the convergence speed
                                speed, index = self.get_token(line, "speed=", "/s", index)
                                # duration
                                delta, index = self.get_token(line, "time=", ".", index)
                                return value, error, epoch, speed, self.get_seconds(delta)
                print("invalid log file <", lpath, ">")

        def get_logs(self, mname, tname, ename, lname):
                values = []
                errors = []
                epochs = []
                speeds = []
                deltas = []
                for trial in range(self.trials):
                        lpath = self.path(trial, mname, tname, ename, lname, ".log")
                        value, error, epoch, speed, delta = self.get_log(lpath)
                        values.append(value)
                        errors.append(error)
                        epochs.append(epoch)
                        speeds.append(speed)
                        deltas.append(delta)
                return values, errors, epochs, speeds, deltas

        def get_log_stats(self, values, errors, epochs, speeds, deltas):
                cmdline = self.cfg.app_stats + " -p 4"
                value_stats = subprocess.check_output(cmdline.split() + values).decode('utf-8').strip()
                error_stats = subprocess.check_output(cmdline.split() + errors).decode('utf-8').strip()
                epoch_stats = subprocess.check_output(cmdline.split() + epochs).decode('utf-8').strip()
                speed_stats = subprocess.check_output(cmdline.split() + speeds).decode('utf-8').strip()
                delta_stats = subprocess.check_output(cmdline.split() + deltas).decode('utf-8').strip()
                return value_stats, error_stats, epoch_stats, speed_stats, delta_stats

        def get_csv_header(self, delim = ";"):
                return delim.join(["model", "trainer", "enhancer", "loss", "test value", "test error", "epochs", "convergence speed", "duration (sec)"])

        def get_csv_row(self, mname, tname, ename, lname, value, error, epoch, speed, delta, delim = ";"):
                return delim.join([mname, tname, ename, lname, value, error, epoch, speed, delta])
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Manages production plants initialization
"""

import erppeek
import datetime
from consolemsg import step, success, warn, error, fail
import dbconfig
from yamlns import namespace as ns

import click

erp = erppeek.Client(**dbconfig.erppeek)
Mix = erp.GenerationkwhProductionAggregator
Plant = erp.GenerationkwhProductionPlant
Meter = erp.GenerationkwhProductionMeter

@click.group()
def mtc():
    """
    Retrieves time series from mongo.
    """
    privateconfig = ns(dbconfig.erppeek)
    del privateconfig.password
    warn("Using the following configuration {}:\n\n{}\n", dbconfig.__file__, privateconfig.dump())

from plantmeter.isodates import localisodate
from plantmeter.isodates import addDays
from plantmeter.mongotimecurve import MongoTimeCurve
import pymongo

sources = ns.loads("""
    production:
        collection: tm_profile
        datafield: ae
        timefield: utc_gkwh_timestamp
        creationfield: create_date
    production_old: # deprecated
        collection: generationkwh.production.measurement
        datafield: ae
        timefield: datetime
        creationfield: create_at
    rightspershare:
        collection: rightspershare
        datafield: rights_kwh
        timefield: datetime
        creationfield: create_at
    memberrightusage:
        collection: memberrightusage
        datafield: usage_kwh
        timefield: datetime
        creationfield: create_at
        intname: true
""")

@mtc.command()
@click.argument('type', type=click.Choice(sources.keys()))
@click.argument('name', required=False)
@click.option('--database', '-d', default='somenergia')
@click.option('--from','-f', type=localisodate, default="2016-05-01")
@click.option('--to','-t', type=localisodate, default=str(datetime.date.today()))
def curve(database, type, name, **args):
    """
    Outputs in a tabular format a mongo time curve

    $ scripts/genkwh_production.py curve gisce 501600324

    $ scripts/genkwh_production.py curve production 1
    """

    source = sources[type]
    c = pymongo.MongoClient(**dbconfig.mongo)
    mongodb = c[database]
    mtc = MongoTimeCurve(
        mongodb,
        source.collection,
        source.timefield,
        source.creationfield,
        )
    curve = mtc.get(
        start=args.get('from',None),
        stop=args.get('to',None),
        filter=None if name is None else long(name) if source.get('intname',False) else name,
        field=source.datafield,
        )
    import numpy
    for day, measures in enumerate(numpy.reshape(curve,(-1,25))):
        print addDays(args['from'],day).date(),
        for x in measures:
            print format(x, ' 5d'),
        print format(sum(measures), ' 7d')

    print "Total", sum(curve)

if __name__ == '__main__':
    mtc(obj={})

# vim: et ts=4 sw=4
<commit_msg>added new curve rightscorrection<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Manages production plants initialization
"""

import erppeek
import datetime
from consolemsg import step, success, warn, error, fail
import dbconfig
from yamlns import namespace as ns

import click

erp = erppeek.Client(**dbconfig.erppeek)
Mix = erp.GenerationkwhProductionAggregator
Plant = erp.GenerationkwhProductionPlant
Meter = erp.GenerationkwhProductionMeter

@click.group()
def mtc():
    """
    Retrieves time series from mongo.
    """
    privateconfig = ns(dbconfig.erppeek)
    del privateconfig.password
    warn("Using the following configuration {}:\n\n{}\n", dbconfig.__file__, privateconfig.dump())

from plantmeter.isodates import localisodate
from plantmeter.isodates import addDays
from plantmeter.mongotimecurve import MongoTimeCurve
import pymongo

sources = ns.loads("""
    production:
        collection: tm_profile
        datafield: ae
        timefield: utc_gkwh_timestamp
        creationfield: create_date
    production_old: # deprecated
        collection: generationkwh.production.measurement
        datafield: ae
        timefield: datetime
        creationfield: create_at
    rightspershare:
        collection: rightspershare
        datafield: rights_kwh
        timefield: datetime
        creationfield: create_at
    memberrightusage:
        collection: memberrightusage
        datafield: usage_kwh
        timefield: datetime
        creationfield: create_at
        intname: true
    rightscorrection:
        collection: generationkwh_rightscorrection
        datafield: rights_kwh
        timefield: datetime
        creationfield: create_at
""")

@mtc.command()
@click.argument('type', type=click.Choice(sources.keys()))
@click.argument('name', required=False)
@click.option('--database', '-d', default='somenergia')
@click.option('--from','-f', type=localisodate, default="2016-05-01")
@click.option('--to','-t', type=localisodate, default=str(datetime.date.today()))
def curve(database, type, name, **args):
    """
    Outputs in a tabular format a mongo time curve

    $ scripts/genkwh_production.py curve gisce 501600324

    $ scripts/genkwh_production.py curve production 1
    """

    source = sources[type]
    c = pymongo.MongoClient(**dbconfig.mongo)
    mongodb = c[database]
    mtc = MongoTimeCurve(
        mongodb,
        source.collection,
        source.timefield,
        source.creationfield,
        )
    curve = mtc.get(
        start=args.get('from',None),
        stop=args.get('to',None),
        filter=None if name is None else long(name) if source.get('intname',False) else name,
        field=source.datafield,
        )
    import numpy
    for day, measures in enumerate(numpy.reshape(curve,(-1,25))):
        print addDays(args['from'],day).date(),
        for x in measures:
            print format(x, ' 5d'),
        print format(sum(measures), ' 7d')

    print "Total", sum(curve)

if __name__ == '__main__':
    mtc(obj={})

# vim: et ts=4 sw=4
<|endoftext|>