<commit_before>#-*- coding: utf-8 -*-
#!/usr/bin/env python
import visa
import time
import datetime
import os

import serial2i2c
try:
    import uli
    import usbio.usbio as usbio
except:
    import dummyULI as uli
    import dummyUSB as usbio

class powersupply(object):
    PSU=False
    rm=False
    setvolt=3.7
    def __init__(self,resourcemanager,gpib):
        try:
            self.rm=resourcemanager
            self.PSU = self.rm.get_instrument("GPIB0::"+gpib+"::INSTR")
            self.PSU.write('INST P6V')
            self.PSU.write('CURR 1e0;:VOLT '+"3.7"+';')
        except:
            pass

    def SetVoltage(self,volt):
        try:
            self.PSU.write('VOLT '+volt)
            self.setvolt=volt
            return True
        except:
            pass

    def SetCurrent(self,current):
        try:
            self.PSU.write('CURR '+current)
            return True
        except:
            pass

    def disconnect(self,visalib):
        try:
            visa.VisaLibrary.gpib_control_ren(visalib, self.PSU.session,6)
            visa.VisaLibrary.close(visalib, self.PSU.session)
        except:
            pass

    def output(self,on=False):
        if on:
            self.PSU.write('OUTPut 1')
        else:
            self.PSU.write('OUTPut 0')

    def read(self):
        return self.setvolt

class sourcemeter(object):
    SMU=False
    rm=False
    setvolt=3.7
    setcurrent=0.0
    def __init__(self,resourcemanager,gpib):
        try:
            self.rm=resourcemanager
            self.SMU = self.rm.get_instrument("GPIB0::"+gpib+"::INSTR")
            self.reset()
            self.SMU.write(':DISP:DIG %d' % int(digits) )
        except:
            pass

    def disconnect(self,visalib):
        try:
            visa.VisaLibrary.gpib_control_ren(visalib, self.SMU.session,6)
            visa.VisaLibrary.close(visalib, self.SMU.session)
        except:
            pass

    def reset (self):
        self.SMU.write('*RST')

    def output_enable (self, channel = 1):
        if (channel == 1):
            self.SMU.write(':OUTP:STAT 1;:INIT')
        else:
            raise Exception("Invalid channel selection CH{0}" .format(channel))

    def output_disable (self, channel = 1):
        self.SMU.write(':OUTP:STAT 0')
        if (channel != 1):
            raise Exception("Invalid channel selection CH{0}" .format(channel))

    def output(self,on=False):
        if on:
            self.output_enable(1)
        else:
            self.output_disable(1)

    # Set Out V
    def SetVoltage (self, voltage, channel = 1):
        if (channel == 1):
            self.SMU.write(":SOUR:FUNC VOLT")
            self.SMU.write(":SOUR:VOLT:MODE FIX")
            self.SMU.write(":SOUR:VOLT %f"%(voltage))
        else:
            raise Exception("Invalid channel selection CH{0}" .format(channel))

    # Set Out I
    def SetCurrent (self, current, channel = 1):
        if (channel == 1):
            self.SMU.write(":SOUR:FUNC CURR")
            self.SMU.write(":SOUR:CURR:MODE FIX")
            self.SMU.write(":SOUR:CURR %f"%(current))
        else:
            raise Exception("Invalid channel selection CH{0}" .format(channel))

    # Set Out V Limit
    def SetVoltageLimit (self, voltage, channel = 1):
        if (channel == 1):
            self.SMU.write(":SOUR:FUNC CURR")
            self.SMU.write(":SOUR:CURR:MODE FIX")
            self.SMU.write(":SENS:VOLT:DC:PROT:LEV %f"%(voltage))
        else:
            raise Exception("Invalid channel selection CH{0}" .format(channel))

    # Set Out I Limit
    def SetCurrentLimit (self, current, channel = 1):
        if (channel == 1):
            self.SMU.write(":SOUR:FUNC VOLT")
            self.SMU.write(":SOUR:VOLT:MODE FIX")
            self.SMU.write(":SENS:CURR:DC:PROT:LEV %f"%(current) )
        else:
            raise Exception("Invalid channel selection CH{0}" .format(channel))

    # Set V Range
    def SetVoltageRange (self, voltage, channel = 1, mode = 'source'):
        if (channel == 1):
            if (mode == 'sense'):
                if (voltage != 'auto'):
                    self.SMU.write(':SENS:VOLT:RANG %f' % voltage)
                else:
                    self.SMU.write(':SENS:VOLT:DC:RANGE:AUTO 1')

            elif (mode == 'source'):
                if (voltage != 'auto'):
                    self.SMU.write(":SOUR:FUNC VOLT")
                    self.SMU.write(":SOUR:VOLT:MODE FIX")
                    self.SMU.write(":SOUR:VOLT:RANG %f"%(voltage))
                else:
                    self.SMU.write(':SOUR:VOLT:DC:RANGE:AUTO 1')

            else:
                raise Exception('Invalid Mode Selection ("source" or "sense")')

        else:
            raise Exception("Invalid channel selection CH{0}" .format(channel))

    # Set I Range
    def SetCurrentRange (self, current, channel = 1, mode = 'source'):
        if (channel == 1):
            if (mode == 'sense'):
                if (current != 'auto'):
                    self.SMU.write(':SENS:CURR:RANG %f' % current)
                else:
                    self.SMU.write(':SENS:CURR:DC:RANGE:AUTO 1')

            elif (mode == 'source'):
                if (current != 'auto'):
                    self.SMU.write(":SOUR:FUNC CURR")
                    self.SMU.write(":SOUR:CURR:MODE FIX")
                    self.SMU.write(":SOUR:CURR:RANG %f"%(current) )
                else:
                    self.SMU.write(':SOUR:CURR:DC:RANGE:AUTO 1')

            else:
                raise Exception('Invalid Mode Selection ("source" or "sense")')

        else:
            raise Exception("Invalid channel selection CH{0}" .format(channel))

    def sample(self,mode="voltage"):
        if mode == "voltage":
            pass
        elif mode == "current":
            pass

    # Read V
    def ReadVoltage (self, samples = 1, channel = 1, avg_mode = 'MOV', filt = 'ON', nplc = 1):
        if (channel == 1):
            self.SMU.write("*CLS")
            self.SMU.write(":ABOR")
            self.SMU.write(":ARM:COUN 1;SOUR TIM")
            self.SMU.write(":FUNC 'VOLT:DC';:VOLT:NPLC %f;:AVER:TCON %s;COUN %d;STAT %s"%(nplc, avg_mode, samples, flit))
            self.SMU.write(":FORM:ELEM:SENS VOLT")
            self.SMU.write(":INIT")

            self.SMU.ask('*OPC?')

            return float(self.SMU.ask(':SENSE:DATA:LAT?'))
        else:
            raise Exception("Invalid channel selection CH{0}" .format(channel))

    # Read I
    def ReadCurrent (self, samples = 1, channel = 1, avg_mode = 'MOV', filt = 'ON', nplc = 1):
        if (channel == 1):
            self.SMU.write("*CLS")
            self.SMU.write(":ABOR")
            self.SMU.write(":ARM:COUN 1;SOUR TIM")
            self.SMU.write(":FUNC 'CURR:DC';:CURR:NPLC %f;:AVER:TCON %s;COUN %d;STAT %s"%(nplc, avg_mode, samples, filt))
            self.SMU.write(":FORMAT:ELEMENTS:SENSE CURR")
            self.SMU.write(":INIT")

            self.SMU.ask('*OPC?')

            return float(self.SMU.ask(':SENSE:DATA:LAT?'))

        else:
            raise Exception("Invalid channel selection CH{0}" .format(channel))

    #Set I range for the display
    def set_range_display (self, digits = 7, channel = 1):
        if (channel == 1):
            self.SMU.write(':DISP:DIG %d' % int(digits) )

            return

        else:
            raise Exception("Invalid channel selection CH{0}" .format(channel))

    def error_query(self):
        ''' the error queue hold error and status messages
        '''
        print self.SMU.ask('SYST:ERR?')

class multimeter(object):
    VOLTMETER=""
    rm=""
    def __init__(self,resourcemanager,gpib,isUSB=False):
        self.rm=resourcemanager
        if isUSB==False:
            self.VOLTMETER = self.rm.get_instrument("GPIB0::"+gpib+"::INSTR")
        else:
            self.VOLTMETER = self.rm.get_instrument("USB0::0x0957::0x1A07::"+gpib+"::INSTR")#USB0::0x0957::0x1A07::SN????????::INSTR

            self.VOLTMETER.write("INP:IMP:AUTO ON")

    def sample(self):
        read=self.VOLTMETER.ask('READ?')
        return read

    def disconnect(self,visalib):
        visa.VisaLibrary.gpib_control_ren(visalib, self.VOLTMETER.session,6)
        visa.VisaLibrary.close(visalib, self.VOLTMETER.session)
        return True

class multimeter2(object):
    VOLTMETER2=""
    channels="(@101)"
    rm=""
    isUse=False
    def __init__(self,resourcemanager,gpib=10,isuse=False):
        self.isUse=isuse
        self.rm=resourcemanager
        if self.isUse == True:
            self.VOLTMETER2 = self.rm.get_instrument("GPIB0::"+gpib+"::INSTR")
            read=self.VOLTMETER2.ask('*IDN?')
            self.VOLTMETER2.write('*RST')

    def sample(self):
        if self.isUse == True:
            self.VOLTMETER2.write('SENS:FUNC "VOLT:DC", '+self.channels    )
            self.VOLTMETER2.write('INP:IMP:AUTO ON,'+self.channels    )#

            self.VOLTMETER2.write('SENS:VOLT:DC:NPLC 2, '+self.channels    )
            self.VOLTMETER2.write('SENS:VOLT:DC:RANG:AUTO ON, '+self.channels    )
            self.VOLTMETER2.write('TRIG:SOUR IMM')
            self.VOLTMETER2.write('TRIG:COUNT 1.0e0')
            self.VOLTMETER2.write('INIT')
            read=self.VOLTMETER2.ask("*OPC?")
            read=self.VOLTMETER2.ask('READ?')#
            return read
        return 0

    def setChannel(self,chanlist):
        if self.isUse == True:
            self.channels = chanlist
            self.VOLTMETER2.write('CONF:VOLT:DC 10,1.0e-5,'+chanlist)
            self.VOLTMETER2.write('ROUT:SCAN '+chanlist)
            self.VOLTMETER2.write('ROUT:CHAN:DEL:AUTO OFF, '+chanlist    )
            self.VOLTMETER2.write('ROUT:CHAN:DEL 2e-1, '+chanlist    )
            self.VOLTMETER2.write('FORM:READ:TIME:TYPE ABS')
            self.VOLTMETER2.write('FORM:READ:CHAN ON')
            self.VOLTMETER2.write('SENS:FUNC "VOLT:DC", '+self.channels    )
            self.VOLTMETER2.write('INP:IMP:AUTO ON,'+self.channels    )#
        return True

    def disconnect(self,visalib):
        if self.isUse == True:
            visa.VisaLibrary.gpib_control_ren(visalib, self.VOLTMETER2.session,6)
            visa.VisaLibrary.close(visalib, self.VOLTMETER2.session)
        return True

class chamber(object):
    isCtrl=False
    CHAMBER=""
    def __init__(self,resourcemanager,gpib=16,isctrl=False):
        isCtrl=isctrl
        self.rm=resourcemanager
        self.CHAMBER = self.rm.get_instrument("GPIB0::"+gpib+"::INSTR")
        self.CHAMBER.ask('ROM?')
        self.CHAMBER.ask("MODE, STANDBY")
        if isCtrl == True:
            self.CHAMBER.ask("MODE, CONSTANT")

    def getTemp(self):
        read=self.CHAMBER.ask("TEMP?")
        return read.split(',')

    def setTemp(self,temp):
        read=self.CHAMBER.ask("TEMP,s"+temp)
        return read

    def close(self):
        self.CHAMBER.ask("MODE, STANDBY")
        return True

    def disconnect(self,visalib):
        self.close()
        visa.VisaLibrary.gpib_control_ren(visalib, self.CHAMBER.session,6)
        visa.VisaLibrary.close(visalib, self.CHAMBER.session)
        return True

class serial_i2c(object):
    isUse=False
    Channel=0
    Slave=[0x90,0x90,0x90,0x90]
    I2C=0
    def __init__(self, port='com3', baud='115200', isuse=False):
        self.isUse = isuse

        if self.isUse == True:
            self.I2C=serial2i2c(port,baud)
        pass

    def setBase(self, channel=0, base=0x90):
        if self.isUse == True:
            self.Slave[channel]=base
        pass

    def regWrite(self, reg=0x00, data=0x00):
        if self.isUse == True:
            slave=self.I2C.convert_hex_to_ascii2(self.Slave[self.Channel],mask=0xa0)
            reg=self.I2C.convert_hex_to_ascii2(reg,mask=0xb0)
            data=self.I2C.convert_hex_to_ascii2(data,mask=0xc0)
            length=self.I2C.convert_hex_to_ascii2(len(reg)/2+len(data)/2,mask=0xd0)

            self.I2C.start()
            for hoge in slave:
                self.I2C.raw_write()
            for hoge in length:
                self.I2C.raw_write()
            for hoge in reg:
                self.I2C.raw_write()
            for hoge in data:
                self.I2C.raw_write()
            self.I2C.stop()
        pass

    def setChannel(self, channel=0):
        if self.isUse == True:
            self.Channel=channel
            self.I2C.setChannel(channel)
        pass

    def getChannel(self):
        channel=-1
        if self.isUse == True:
            channel = self.Channel
        return channel

class dummy(object):
    def __init__(self):
        pass

    def SetVoltage(self,volt):
        pass

    def SetCurrent(self,current):
        pass

    def output(self,on=False):
        pass

    def disconnect(self,visalib):
        pass

    def read(self):
        return -3640

    def sample(self):
        return -34401

    def setChannel(self,chanlist):
        pass

    def getChannel(self):
        return -1

    def setBase(self,channel,base):
        pass

    def regWrite(self,reg,data):
        pass

    def getTemp(self):
        return ['-241','-241','limit+','limit-']

    def setTemp(self,temp):
        return "dummy"

    def reset (self):
        pass

#    def sense_4W(self,on=False):
#        pass

    def SetVoltageLimit (self, voltage, channel = 1):
        pass

    def SetCurrentLimit (self, current, channel = 1):
        pass

    def SetVoltageRange (self, voltage, channel = 1, mode = 'source'):
        pass

    def SetCurrentRange (self, current, channel = 1, mode = 'source'):
        pass

    def ReadVoltage (self, samples = 1, channel = 1, avg_mode = 'MOV', filt = 'ON', nplc = 1):
        return "dummy"

    def ReadCurrent (self, samples = 1, channel = 1, avg_mode = 'MOV', filt = 'ON', nplc = 1):
        return "dummy"

    def SetVoltage (self, voltage, channel = 1):
        pass

    def SetCurrent (self, current, channel = 1):
        pass

<commit_msg>import statement removed<commit_after>#-*- coding: utf-8 -*-
#!/usr/bin/env python
import visa
import time
import datetime
import os

import serial2i2c
#try:
#    import uli
#    import usbio.usbio as usbio
#except:
#    import dummyULI as uli
#    import dummyUSB as usbio

class powersupply(object):
    PSU=False
    rm=False
    setvolt=3.7
    def __init__(self,resourcemanager,gpib):
        try:
            self.rm=resourcemanager
            self.PSU = self.rm.get_instrument("GPIB0::"+gpib+"::INSTR")
            self.PSU.write('INST P6V')
            self.PSU.write('CURR 1e0;:VOLT '+"3.7"+';')
        except:
            pass

    def SetVoltage(self,volt):
        try:
            self.PSU.write('VOLT '+volt)
            self.setvolt=volt
            return True
        except:
            pass

    def SetCurrent(self,current):
        try:
            self.PSU.write('CURR '+current)
            return True
        except:
            pass

    def disconnect(self,visalib):
        try:
            visa.VisaLibrary.gpib_control_ren(visalib, self.PSU.session,6)
            visa.VisaLibrary.close(visalib, self.PSU.session)
        except:
            pass

    def output(self,on=False):
        if on:
            self.PSU.write('OUTPut 1')
        else:
            self.PSU.write('OUTPut 0')

    def read(self):
        return self.setvolt

class sourcemeter(object):
    SMU=False
    rm=False
    setvolt=3.7
    setcurrent=0.0
    def __init__(self,resourcemanager,gpib):
        try:
            self.rm=resourcemanager
            self.SMU = self.rm.get_instrument("GPIB0::"+gpib+"::INSTR")
            self.reset()
            self.SMU.write(':DISP:DIG %d' % int(digits) )
        except:
            pass

    def disconnect(self,visalib):
        try:
            visa.VisaLibrary.gpib_control_ren(visalib, self.SMU.session,6)
            visa.VisaLibrary.close(visalib, self.SMU.session)
        except:
            pass

    def reset (self):
        self.SMU.write('*RST')

    def output_enable (self, channel = 1):
        if (channel == 1):
            self.SMU.write(':OUTP:STAT 1;:INIT')
        else:
            raise Exception("Invalid channel selection CH{0}" .format(channel))

    def output_disable (self, channel = 1):
        self.SMU.write(':OUTP:STAT 0')
        if (channel != 1):
            raise Exception("Invalid channel selection CH{0}" .format(channel))

    def output(self,on=False):
        if on:
            self.output_enable(1)
        else:
            self.output_disable(1)

    # Set Out V
    def SetVoltage (self, voltage, channel = 1):
        if (channel == 1):
            self.SMU.write(":SOUR:FUNC VOLT")
            self.SMU.write(":SOUR:VOLT:MODE FIX")
            self.SMU.write(":SOUR:VOLT %f"%(voltage))
        else:
            raise Exception("Invalid channel selection CH{0}" .format(channel))

    # Set Out I
    def SetCurrent (self, current, channel = 1):
        if (channel == 1):
            self.SMU.write(":SOUR:FUNC CURR")
            self.SMU.write(":SOUR:CURR:MODE FIX")
            self.SMU.write(":SOUR:CURR %f"%(current))
        else:
            raise Exception("Invalid channel selection CH{0}" .format(channel))

    # Set Out V Limit
    def SetVoltageLimit (self, voltage, channel = 1):
        if (channel == 1):
            self.SMU.write(":SOUR:FUNC CURR")
            self.SMU.write(":SOUR:CURR:MODE FIX")
            self.SMU.write(":SENS:VOLT:DC:PROT:LEV %f"%(voltage))
        else:
            raise Exception("Invalid channel selection CH{0}" .format(channel))

    # Set Out I Limit
    def SetCurrentLimit (self, current, channel = 1):
        if (channel == 1):
            self.SMU.write(":SOUR:FUNC VOLT")
            self.SMU.write(":SOUR:VOLT:MODE FIX")
            self.SMU.write(":SENS:CURR:DC:PROT:LEV %f"%(current) )
        else:
            raise Exception("Invalid channel selection CH{0}" .format(channel))

    # Set V Range
    def SetVoltageRange (self, voltage, channel = 1, mode = 'source'):
        if (channel == 1):
            if (mode == 'sense'):
                if (voltage != 'auto'):
                    self.SMU.write(':SENS:VOLT:RANG %f' % voltage)
                else:
                    self.SMU.write(':SENS:VOLT:DC:RANGE:AUTO 1')

            elif (mode == 'source'):
                if (voltage != 'auto'):
                    self.SMU.write(":SOUR:FUNC VOLT")
                    self.SMU.write(":SOUR:VOLT:MODE FIX")
                    self.SMU.write(":SOUR:VOLT:RANG %f"%(voltage))
                else:
                    self.SMU.write(':SOUR:VOLT:DC:RANGE:AUTO 1')

            else:
                raise Exception('Invalid Mode Selection ("source" or "sense")')

        else:
            raise Exception("Invalid channel selection CH{0}" .format(channel))

    # Set I Range
    def SetCurrentRange (self, current, channel = 1, mode = 'source'):
        if (channel == 1):
            if (mode == 'sense'):
                if (current != 'auto'):
                    self.SMU.write(':SENS:CURR:RANG %f' % current)
                else:
                    self.SMU.write(':SENS:CURR:DC:RANGE:AUTO 1')

            elif (mode == 'source'):
                if (current != 'auto'):
                    self.SMU.write(":SOUR:FUNC CURR")
                    self.SMU.write(":SOUR:CURR:MODE FIX")
                    self.SMU.write(":SOUR:CURR:RANG %f"%(current) )
                else:
                    self.SMU.write(':SOUR:CURR:DC:RANGE:AUTO 1')

            else:
                raise Exception('Invalid Mode Selection ("source" or "sense")')

        else:
            raise Exception("Invalid channel selection CH{0}" .format(channel))

    def sample(self,mode="voltage"):
        if mode == "voltage":
            pass
        elif mode == "current":
            pass

    # Read V
    def ReadVoltage (self, samples = 1, channel = 1, avg_mode = 'MOV', filt = 'ON', nplc = 1):
        if (channel == 1):
            self.SMU.write("*CLS")
            self.SMU.write(":ABOR")
            self.SMU.write(":ARM:COUN 1;SOUR TIM")
            self.SMU.write(":FUNC 'VOLT:DC';:VOLT:NPLC %f;:AVER:TCON %s;COUN %d;STAT %s"%(nplc, avg_mode, samples, flit))
            self.SMU.write(":FORM:ELEM:SENS VOLT")
            self.SMU.write(":INIT")

            self.SMU.ask('*OPC?')

            return float(self.SMU.ask(':SENSE:DATA:LAT?'))
        else:
            raise Exception("Invalid channel selection CH{0}" .format(channel))

    # Read I
    def ReadCurrent (self, samples = 1, channel = 1, avg_mode = 'MOV', filt = 'ON', nplc = 1):
        if (channel == 1):
            self.SMU.write("*CLS")
            self.SMU.write(":ABOR")
            self.SMU.write(":ARM:COUN 1;SOUR TIM")
            self.SMU.write(":FUNC 'CURR:DC';:CURR:NPLC %f;:AVER:TCON %s;COUN %d;STAT %s"%(nplc, avg_mode, samples, filt))
            self.SMU.write(":FORMAT:ELEMENTS:SENSE CURR")
            self.SMU.write(":INIT")

            self.SMU.ask('*OPC?')

            return float(self.SMU.ask(':SENSE:DATA:LAT?'))

        else:
            raise Exception("Invalid channel selection CH{0}" .format(channel))

    #Set I range for the display
    def set_range_display (self, digits = 7, channel = 1):
        if (channel == 1):
            self.SMU.write(':DISP:DIG %d' % int(digits) )

            return

        else:
            raise Exception("Invalid channel selection CH{0}" .format(channel))

    def error_query(self):
        ''' the error queue hold error and status messages
        '''
        print self.SMU.ask('SYST:ERR?')

class multimeter(object):
    VOLTMETER=""
    rm=""
    def __init__(self,resourcemanager,gpib,isUSB=False):
        self.rm=resourcemanager
        if isUSB==False:
            self.VOLTMETER = self.rm.get_instrument("GPIB0::"+gpib+"::INSTR")
        else:
            self.VOLTMETER = self.rm.get_instrument("USB0::0x0957::0x1A07::"+gpib+"::INSTR")#USB0::0x0957::0x1A07::SN????????::INSTR

            self.VOLTMETER.write("INP:IMP:AUTO ON")

    def sample(self):
        read=self.VOLTMETER.ask('READ?')
        return read

    def disconnect(self,visalib):
        visa.VisaLibrary.gpib_control_ren(visalib, self.VOLTMETER.session,6)
        visa.VisaLibrary.close(visalib, self.VOLTMETER.session)
        return True

class multimeter2(object):
    VOLTMETER2=""
    channels="(@101)"
    rm=""
    isUse=False
    def __init__(self,resourcemanager,gpib=10,isuse=False):
        self.isUse=isuse
        self.rm=resourcemanager
        if self.isUse == True:
            self.VOLTMETER2 = self.rm.get_instrument("GPIB0::"+gpib+"::INSTR")
            read=self.VOLTMETER2.ask('*IDN?')
            self.VOLTMETER2.write('*RST')

    def sample(self):
        if self.isUse == True:
            self.VOLTMETER2.write('SENS:FUNC "VOLT:DC", '+self.channels    )
            self.VOLTMETER2.write('INP:IMP:AUTO ON,'+self.channels    )#

            self.VOLTMETER2.write('SENS:VOLT:DC:NPLC 2, '+self.channels    )
            self.VOLTMETER2.write('SENS:VOLT:DC:RANG:AUTO ON, '+self.channels    )
            self.VOLTMETER2.write('TRIG:SOUR IMM')
            self.VOLTMETER2.write('TRIG:COUNT 1.0e0')
            self.VOLTMETER2.write('INIT')
            read=self.VOLTMETER2.ask("*OPC?")
            read=self.VOLTMETER2.ask('READ?')#
            return read
        return 0

    def setChannel(self,chanlist):
        if self.isUse == True:
            self.channels = chanlist
            self.VOLTMETER2.write('CONF:VOLT:DC 10,1.0e-5,'+chanlist)
            self.VOLTMETER2.write('ROUT:SCAN '+chanlist)
            self.VOLTMETER2.write('ROUT:CHAN:DEL:AUTO OFF, '+chanlist    )
            self.VOLTMETER2.write('ROUT:CHAN:DEL 2e-1, '+chanlist    )
            self.VOLTMETER2.write('FORM:READ:TIME:TYPE ABS')
            self.VOLTMETER2.write('FORM:READ:CHAN ON')
            self.VOLTMETER2.write('SENS:FUNC "VOLT:DC", '+self.channels    )
            self.VOLTMETER2.write('INP:IMP:AUTO ON,'+self.channels    )#
        return True

    def disconnect(self,visalib):
        if self.isUse == True:
            visa.VisaLibrary.gpib_control_ren(visalib, self.VOLTMETER2.session,6)
            visa.VisaLibrary.close(visalib, self.VOLTMETER2.session)
        return True

class chamber(object):
    isCtrl=False
    CHAMBER=""
    def __init__(self,resourcemanager,gpib=16,isctrl=False):
        isCtrl=isctrl
        self.rm=resourcemanager
        self.CHAMBER = self.rm.get_instrument("GPIB0::"+gpib+"::INSTR")
        self.CHAMBER.ask('ROM?')
        self.CHAMBER.ask("MODE, STANDBY")
        if isCtrl == True:
            self.CHAMBER.ask("MODE, CONSTANT")

    def getTemp(self):
        read=self.CHAMBER.ask("TEMP?")
        return read.split(',')

    def setTemp(self,temp):
        read=self.CHAMBER.ask("TEMP,s"+temp)
        return read

    def close(self):
        self.CHAMBER.ask("MODE, STANDBY")
        return True

    def disconnect(self,visalib):
        self.close()
        visa.VisaLibrary.gpib_control_ren(visalib, self.CHAMBER.session,6)
        visa.VisaLibrary.close(visalib, self.CHAMBER.session)
        return True

class serial_i2c(object):
    isUse=False
    Channel=0
    Slave=[0x90,0x90,0x90,0x90]
    I2C=0
    def __init__(self, port='com3', baud='115200', isuse=False):
        self.isUse = isuse

        if self.isUse == True:
            self.I2C=serial2i2c(port,baud)
        pass

    def setBase(self, channel=0, base=0x90):
        if self.isUse == True:
            self.Slave[channel]=base
        pass

    def regWrite(self, reg=0x00, data=0x00):
        if self.isUse == True:
            slave=self.I2C.convert_hex_to_ascii2(self.Slave[self.Channel],mask=0xa0)
            reg=self.I2C.convert_hex_to_ascii2(reg,mask=0xb0)
            data=self.I2C.convert_hex_to_ascii2(data,mask=0xc0)
            length=self.I2C.convert_hex_to_ascii2(len(reg)/2+len(data)/2,mask=0xd0)

            self.I2C.start()
            for hoge in slave:
                self.I2C.raw_write()
            for hoge in length:
                self.I2C.raw_write()
            for hoge in reg:
                self.I2C.raw_write()
            for hoge in data:
                self.I2C.raw_write()
            self.I2C.stop()
        pass

    def setChannel(self, channel=0):
        if self.isUse == True:
            self.Channel=channel
            self.I2C.setChannel(channel)
        pass

    def getChannel(self):
        channel=-1
        if self.isUse == True:
            channel = self.Channel
        return channel

class dummy(object):
    def __init__(self):
        pass

    def SetVoltage(self,volt):
        pass

    def SetCurrent(self,current):
        pass

    def output(self,on=False):
        pass

    def disconnect(self,visalib):
        pass

    def read(self):
        return -3640

    def sample(self):
        return -34401

    def setChannel(self,chanlist):
        pass

    def getChannel(self):
        return -1

    def setBase(self,channel,base):
        pass

    def regWrite(self,reg,data):
        pass

    def getTemp(self):
        return ['-241','-241','limit+','limit-']

    def setTemp(self,temp):
        return "dummy"

    def reset (self):
        pass

#    def sense_4W(self,on=False):
#        pass

    def SetVoltageLimit (self, voltage, channel = 1):
        pass

    def SetCurrentLimit (self, current, channel = 1):
        pass

    def SetVoltageRange (self, voltage, channel = 1, mode = 'source'):
        pass

    def SetCurrentRange (self, current, channel = 1, mode = 'source'):
        pass

    def ReadVoltage (self, samples = 1, channel = 1, avg_mode = 'MOV', filt = 'ON', nplc = 1):
        return "dummy"

    def ReadCurrent (self, samples = 1, channel = 1, avg_mode = 'MOV', filt = 'ON', nplc = 1):
        return "dummy"

    def SetVoltage (self, voltage, channel = 1):
        pass

    def SetCurrent (self, current, channel = 1):
        pass

<|endoftext|>


<commit_before>"""
Encapsulating some of the 2D particle tracking from
currents-figures-netcdf-animation-v03.ipynb
"""
from __future__ import print_function

import matplotlib.pyplot as plt
import numpy as np
import os
import glob
import six
import logging

import matplotlib.dates as mdates
from shapely import geometry

import xarray as xr
from stompy import utils
from stompy.plot import (plot_utils, cmap, plot_wkb)
from stompy.grid import unstructured_grid
from stompy.spatial import (wkb2shp,field)

##
class UgridParticles(object):
    """
    time: all timestamps are stored as 64-bit double, in seconds
    since the epoch.

    This is to hedge between the various desires on time - data is
    rarely at a resolution finer than 1s.  A unix timestamp in 64-bit
    double is exact to the second for over 280 million years.
    """
    # x: horizontal location
    # c: cell containing particle
    # j_last: last edge the particle crossed
    part_dtype = [ ('x',(np.float64,2)),
                   ('c',np.int32),
                   ('u',(np.float64,2)),
                   ('j_last',np.int32) ]

    def __init__(self,ncs,grid=None):
        self.log=logging.getLogger(self.__class__.__name__)
        self.ncs=ncs
        self.scan_ncs()

        self.load_grid(grid=grid)

        self.init_particles()
        self.t_unix=None
        self.current_nc_idx=-1

        self.U=None

    def scan_ncs(self):
        self.nc_catalog=np.zeros( len(self.ncs),
                                  dtype=[('start_t','f8'),
                                         ('end_t','f8')] )
        for nc_i,nc in enumerate(self.ncs):
            self.nc_catalog['start_t'][nc_i] = utils.to_unix(nc.time[0])
            self.nc_catalog['end_t'][nc_i]   = utils.to_unix(nc.time[-1])

    def set_time(self,t):
        self.t_unix=t
        self.update_velocity()

    def load_grid(self,grid=None):
        if grid is None:
            self.g=unstructured_grid.UnstructuredGrid.from_ugrid(self.ncs[0])
        else:
            self.g=grid
        self.g.edge_to_cells()

        self.edge_norm=self.g.edges_normals()

    def set_current_nc(self,nc_i):
        self.current_nc_idx=nc_i
        self.current_nc=self.ncs[self.current_nc_idx]
        self.nc_t_unix=utils.to_unix(self.current_nc.time.values)
        self.nc_time_i=-999

    def update_velocity(self):
        changed=False
        nc_i = np.searchsorted(self.nc_catalog['start_t'],self.t_unix,side='right') - 1

        if nc_i != self.current_nc_idx:
            self.set_current_nc(nc_i)
            changed=True

        new_time_i=np.searchsorted(self.nc_t_unix, self.t_unix,side='right')-1
        if new_time_i!=self.nc_time_i:
            changed=True
        self.nc_time_i=new_time_i

        # find out how long this velocity is good for
        next_step=np.inf
        if self.nc_time_i+1 < len(self.nc_t_unix):
            next_step=self.nc_t_unix[self.nc_time_i+1]
        if self.current_nc_idx+1<len(self.ncs):
            next_step=min(next_step,
                          self.nc_catalog['start_t'][self.current_nc_idx+1] )

        assert np.isfinite(next_step) # did we integrate off the end of the avail. data?

        self.velocity_valid_time=[self.nc_t_unix[self.nc_time_i],
                                  next_step ]
        # This is split off here because it's a point where basic
        # and KSG differ
        self.update_particle_velocity_for_new_step()

    def update_particle_velocity_for_new_step(self):
        # face, layer, time.
        # assumes 2D here.
        u=self.current_nc.cell_east_velocity.values[:,0,self.nc_time_i]
        v=self.current_nc.cell_north_velocity.values[:,0,self.nc_time_i]

        self.U=np.array( [u,v] ).T # again assume 2D

        # A little dicey - this overwrites any memory of convergent edges.
        # so every input interval, it's going to forget
        self.P['u']=self.U[ self.P['c'] ]

    def init_particles(self):
        self.P=np.zeros( 0, self.part_dtype )

    def add_particles(self,**kw):
        fields=list(kw.keys())

        Nold=len(self.P)
        Nnew=np.atleast_1d( kw[fields[0]] ).shape[0]

        recs=np.zeros( Nnew, dtype=self.part_dtype)

        slc=slice(Nold,Nold+Nnew)

        self.P=utils.array_concatenate( [self.P,recs] )

        for k,v in six.iteritems(kw):
            self.P[k][slc]=v

        # figure out which cell they are in
        for i in range(Nold,Nold+Nnew):
            c=self.g.select_cells_nearest(self.P['x'][i],inside=True)
            self.P['c'][i]=c
            self.P['j_last'][i]=-999

            # if the velocity fields were continuous, then we could
            # skip this part, since we wouldn't really need to store
            # velocity.
            self.P['u'][i] = np.nan # signal that it needs to be set

    record_dense=False

    def integrate(self,output_times_unix):
        next_out_idx=0
        next_out_time=output_times_unix[next_out_idx]

        self.output=[]
        self.append_state(self.output)
        if self.record_dense:
            self.dense=[]
            self.append_state(self.dense)

        next_vel_time=self.velocity_valid_time[1]

        assert self.t_unix>=self.velocity_valid_time[0]
        assert self.t_unix<=self.velocity_valid_time[1]
        assert next_out_time>=self.t_unix

        while next_out_time is not None: # main loop
            # the max time step we can take is the minimum of
            # (i) time to next output interval
            # (ii) time to next update of input velocities
            # (iii) time to cross into a new cell
            #[(iv) eventually, time until update behavior]

            t_next=min(next_out_time,next_vel_time)

            self.move_particles(t_next)

            self.t_unix=t_next
            if t_next==next_out_time:
                self.log.info('Output %d / %d'%(next_out_idx,len(output_times_unix)))
                self.append_state(self.output)
                next_out_idx+=1
                if next_out_idx<len(output_times_unix):
                    next_out_time=output_times_unix[next_out_idx]
                else:
                    next_out_time=None
            if t_next==next_vel_time:
                self.update_velocity()
                next_vel_time=self.velocity_valid_time[1]
                self.P['j_last']=-999 # okay to cross back if the velocities changed.

    def append_state(self,A):
        A.append( (self.P['x'].copy(), self.t_unix ))

    def move_particles(self,stop_t):
        """
        Advance each particle to the correct state at stop_t.
        Assumes that no input (updating velocities) or output
        is needed between self.t_unix and stop_t.

        Caller is responsible for updating self.t_unix
        """
        g=self.g

        for i,p in enumerate(self.P):

            # advance each particle to the correct state at stop_t
            part_t=self.t_unix

            if np.isnan( p['u'][0] ):
                # probably first time this particle has been moved.
                self.P['u'][i]=self.U[ self.P['c'][i] ]

            while part_t<stop_t:
                dt_max_edge=np.inf
                j_cross=None
                j_cross_normal=None

                for j in g.cell_to_edges(p['c']):
                    if j==p['j_last']:
                        continue # don't cross back
                    normal=self.edge_norm[j]
                    if g.edges['cells'][j,1]==p['c']: # ~checked
                        normal=-normal
                    # vector from xy to a point on the edge
                    d_xy_n = g.nodes['x'][g.edges['nodes'][j,0]] - p['x']
                    # perpendicular distance
                    dp_xy_n=d_xy_n[0] * normal[0] + d_xy_n[1]*normal[1]
                    assert dp_xy_n>=0 #otherwise sgn probably wrong above

                    #closing=u*normal[0] + v*normal[1]
                    closing=self.P['u'][i,0]*normal[0] + self.P['u'][i,1]*normal[1]

                    if closing<0:
                        continue
                    else:
                        dt_j=dp_xy_n/closing
                        if dt_j>0 and dt_j<dt_max_edge:
                            j_cross=j
                            dt_max_edge=dt_j
                            j_cross_normal=normal

                t_max_edge=part_t+dt_max_edge
                if t_max_edge>stop_t:
                    # don't make it to the edge
                    dt=stop_t-part_t
                    part_t=stop_t
                    j_cross=None
                else:
                    dt=dt_max_edge
                    part_t=t_max_edge

                # Take the step
                self.P['x'][i] += self.P['u'][i]*dt

                if j_cross is not None:
                    # cross edge j, update time.  careful that j isn't boundary
                    # or start sliding on boundary.
                    # print "Cross edge"
                    cells=g.edges['cells'][j_cross]
                    if cells[0]==p['c']:
                        new_c=cells[1]
                    elif cells[1]==p['c']:
                        new_c=cells[0]
                    else:
                        assert False

                    # More scrutiny on the edge crossing -
                    # would it take us out of the domain? then bounce and frown.
                    # would it take us to a convergent edge? then bounce and frown.
                    bounce=False

                    if new_c<0:
                        bounce=True
                    else:
                        recross= ( self.U[new_c,0]*j_cross_normal[0] +
                                   self.U[new_c,1]*j_cross_normal[1] )
                        if recross<=0:
                            bounce=True

                    if bounce:
                        closing= (self.P['u'][i,0]*j_cross_normal[0] +
                                  self.P['u'][i,1]*j_cross_normal[1] )
                        # slightly over-compensate, pushing away from problematic
                        # edge
                        print("BOUNCE")

                        self.P['u'][i] -= 1.1 * j_cross_normal*closing
                        self.P['j_last'][i]=j_cross
                    else:
                        self.P['c'][i]=new_c
                        self.P['j_last'][i]=j_cross
                        self.P['u'][i] = self.U[new_c]
                        # HERE: need to check whether the new cell's
                        # velocity is going to push us back, in which case
                        # we should instead scoot along the tangent and
                        # hide our faces.
                        # actually, better solution is to handle *before*
                        # the particle hits the edge.  If we handle it after,
                        # two particles converging on this edge will cross
                        # paths, and that really doesn't pass the sniff test.
                        # this is not good, but will let the sim complete

                if self.record_dense:
                    self.append_state(self.dense)

    def save_tracks(self,fn,overwrite=True):
        if os.path.exists(fn):
            if overwrite:
                os.unlink(fn)
            else:
                raise Exception("Output %s already exists"%fn)

        # archive the grid for good measure:
        self.g.write_ugrid(fn)
        ds=xr.open_dataset(fn).copy(deep=True)

        ds['particle']=( ('particle',), np.arange(len(self.P)) )
        times=np.array([out[1] for out in self.output])

        ds['time'] = ( ('time',), times )
        ds.time.attrs['units']="seconds since 1970-01-01T00:00:00"

        X=np.array( [out[0] for out in self.output] )

        ds['x'] = ( ('time','particle'), X[:,:,0] )
        ds['y'] = ( ('time','particle'), X[:,:,1] )

        os.unlink(fn)
        ds.to_netcdf(fn)
        return ds

#    Edges and incompatible velocities
#    ---------------------------------
#
#    With a discontinuous velocity field, sooner or later there
#    is a particle which is one cell, pushed to the edge, and
#    enters a new cell which also wants to push the particle to
#    the edge.
#
#    Solutions:
#     1. Fix the velocity field.  The 0th order in space (constant
#        velocity within a cell) approach cannot be fixed in this way.
#        Going to a reconstruction like in Gerard's paper would be a
#        solution, though it doesn't extend beyond triangles.  Postma
#        I think would be sufficient to get triangles and quads, but
#        still no good on pentagons, hexes, etc.
#     2. Adjust the velocity when this happens.  The adjustment might
#        be to take the average of the two cells, reach back and use
#        an edge velocity to figure it out, force a tangent trajectory,
#        or other approaches.
#
#        Do any of these approaches not break the reversibility of
#        the method?  Seems that any approach which "reacts" to finding
#        a convergent edge is not reversible, since that edge would be
#        divergent in reverse.
#
#        I think for now I have to give up on reversibility, but at least
#        preserve invariance w.r.t. to output time stepping.  I.e. changing the
#        output timestep shouldn't alter any results.

# Method of Ketefian, Gross, Stelling
# - need a 2x2 matrix Ai and vector Bi per cell
#   u is linear in x,y
#   they include closed solutions for a trajectory

# The full analytic solution is pretty messy, but the bulk of the mess
# is due to the analytical integration of the linear velocity field.
# an intermediate step would be to go for the velocity field reconstruction,
# but go back to a numerical integration.
<commit_msg>Whitespace and parameter setting<commit_after>"""
Encapsulating some of the 2D particle tracking from
currents-figures-netcdf-animation-v03.ipynb
"""
from __future__ import print_function

import matplotlib.pyplot as plt
import numpy as np
import os
import glob
import six
import logging

import matplotlib.dates as mdates
from shapely import geometry

import xarray as xr
from stompy import utils
from stompy.plot import (plot_utils, cmap, plot_wkb)
from stompy.grid import unstructured_grid
from stompy.spatial import (wkb2shp,field)

##
class UgridParticles(object):
    """
    time: all timestamps are stored as 64-bit double, in seconds
    since the epoch.

    This is to hedge between the various desires on time - data is
    rarely at a resolution finer than 1s.  A unix timestamp in 64-bit
    double is exact to the second for over 280 million years.
    """
    # x: horizontal location
    # c: cell containing particle
    # j_last: last edge the particle crossed
    part_dtype = [ ('x',(np.float64,2)),
                   ('c',np.int32),
                   ('u',(np.float64,2)),
                   ('j_last',np.int32) ]

    def __init__(self,ncs,grid=None,**kw):
        self.__dict__.update(kw)
        self.log=logging.getLogger(self.__class__.__name__)
        self.ncs=ncs
        self.scan_ncs()

        self.load_grid(grid=grid)

        self.init_particles()
        self.t_unix=None
        self.current_nc_idx=-1

        self.U=None

    def scan_ncs(self):
        self.nc_catalog=np.zeros( len(self.ncs),
                                  dtype=[('start_t','f8'),
                                         ('end_t','f8')] )
        for nc_i,nc in enumerate(self.ncs):
            self.nc_catalog['start_t'][nc_i] = utils.to_unix(nc.time[0])
            self.nc_catalog['end_t'][nc_i]   = utils.to_unix(nc.time[-1])

    def set_time(self,t):
        self.t_unix=t
        self.update_velocity()

    def load_grid(self,grid=None):
        if grid is None:
            self.g=unstructured_grid.UnstructuredGrid.from_ugrid(self.ncs[0])
        else:
            self.g=grid
        self.g.edge_to_cells()

        self.edge_norm=self.g.edges_normals()

    def set_current_nc(self,nc_i):
        self.current_nc_idx=nc_i
        self.current_nc=self.ncs[self.current_nc_idx]
        self.nc_t_unix=utils.to_unix(self.current_nc.time.values)
        self.nc_time_i=-999

    def update_velocity(self):
        changed=False
        nc_i = np.searchsorted(self.nc_catalog['start_t'],self.t_unix,side='right') - 1

        if nc_i != self.current_nc_idx:
            self.set_current_nc(nc_i)
            changed=True

        new_time_i=np.searchsorted(self.nc_t_unix, self.t_unix,side='right')-1
        if new_time_i!=self.nc_time_i:
            changed=True
        self.nc_time_i=new_time_i

        # find out how long this velocity is good for
        next_step=np.inf
        if self.nc_time_i+1 < len(self.nc_t_unix):
            next_step=self.nc_t_unix[self.nc_time_i+1]
        if self.current_nc_idx+1<len(self.ncs):
            next_step=min(next_step,
                          self.nc_catalog['start_t'][self.current_nc_idx+1] )

        assert np.isfinite(next_step) # did we integrate off the end of the avail. data?

        self.velocity_valid_time=[self.nc_t_unix[self.nc_time_i],
                                  next_step ]
        # This is split off here because it's a point where basic
        # and KSG differ
        self.update_particle_velocity_for_new_step()

    def update_particle_velocity_for_new_step(self):
        # face, layer, time.
        # assumes 2D here.
        u=self.current_nc.cell_east_velocity.values[:,0,self.nc_time_i]
        v=self.current_nc.cell_north_velocity.values[:,0,self.nc_time_i]

        self.U=np.array( [u,v] ).T # again assume 2D

        # A little dicey - this overwrites any memory of convergent edges.
        # so every input interval, it's going to forget
        self.P['u']=self.U[ self.P['c'] ]

    def init_particles(self):
        self.P=np.zeros( 0, self.part_dtype )

    def add_particles(self,**kw):
        fields=list(kw.keys())

        Nold=len(self.P)
        Nnew=np.atleast_1d( kw[fields[0]] ).shape[0]

        recs=np.zeros( Nnew, dtype=self.part_dtype)

        slc=slice(Nold,Nold+Nnew)

        self.P=utils.array_concatenate( [self.P,recs] )

        for k,v in six.iteritems(kw):
            self.P[k][slc]=v

        # figure out which cell they are in
        for i in range(Nold,Nold+Nnew):
            c=self.g.select_cells_nearest(self.P['x'][i],inside=True)
            self.P['c'][i]=c
            self.P['j_last'][i]=-999

            # if the velocity fields were continuous, then we could
            # skip this part, since we wouldn't really need to store
            # velocity.
            self.P['u'][i] = np.nan # signal that it needs to be set

    record_dense=False

    def integrate(self,output_times_unix):
        next_out_idx=0
        next_out_time=output_times_unix[next_out_idx]

        self.output=[]
        self.append_state(self.output)
        if self.record_dense:
            self.dense=[]
            self.append_state(self.dense)

        next_vel_time=self.velocity_valid_time[1]

        assert self.t_unix>=self.velocity_valid_time[0]
        assert self.t_unix<=self.velocity_valid_time[1]
        assert next_out_time>=self.t_unix

        while next_out_time is not None: # main loop
            # the max time step we can take is the minimum of
            # (i) time to next output interval
            # (ii) time to next update of input velocities
            # (iii) time to cross into a new cell
            #[(iv) eventually, time until update behavior]

            t_next=min(next_out_time,next_vel_time)

            self.move_particles(t_next)

            self.t_unix=t_next
            if t_next==next_out_time:
                self.log.info('Output %d / %d'%(next_out_idx,len(output_times_unix)))
                self.append_state(self.output)
                next_out_idx+=1
                if next_out_idx<len(output_times_unix):
                    next_out_time=output_times_unix[next_out_idx]
                else:
                    next_out_time=None
            if t_next==next_vel_time:
                self.update_velocity()
                next_vel_time=self.velocity_valid_time[1]
                self.P['j_last']=-999 # okay to cross back if the velocities changed.

    def append_state(self,A):
        A.append( (self.P['x'].copy(), self.t_unix ))

    def move_particles(self,stop_t):
        """
        Advance each particle to the correct state at stop_t.
        Assumes that no input (updating velocities) or output
        is needed between self.t_unix and stop_t.

        Caller is responsible for updating self.t_unix
        """
        g=self.g

        for i,p in enumerate(self.P):
            # advance each particle to the correct state at stop_t
            part_t=self.t_unix

            if np.isnan( p['u'][0] ):
                # probably first time this particle has been moved.
                self.P['u'][i]=self.U[ self.P['c'][i] ]

            while part_t<stop_t:
                dt_max_edge=np.inf
                j_cross=None
                j_cross_normal=None

                for j in g.cell_to_edges(p['c']):
                    if j==p['j_last']:
                        continue # don't cross back
                    normal=self.edge_norm[j]
                    if g.edges['cells'][j,1]==p['c']: # ~checked
                        normal=-normal
                    # vector from xy to a point on the edge
                    d_xy_n = g.nodes['x'][g.edges['nodes'][j,0]] - p['x']
                    # perpendicular distance
                    dp_xy_n=d_xy_n[0] * normal[0] + d_xy_n[1]*normal[1]
                    assert dp_xy_n>=0 #otherwise sgn probably wrong above

                    #closing=u*normal[0] + v*normal[1]
                    closing=self.P['u'][i,0]*normal[0] + self.P['u'][i,1]*normal[1]

                    if closing<0:
                        continue
                    else:
                        dt_j=dp_xy_n/closing
                        if dt_j>0 and dt_j<dt_max_edge:
                            j_cross=j
                            dt_max_edge=dt_j
                            j_cross_normal=normal

                t_max_edge=part_t+dt_max_edge
                if t_max_edge>stop_t:
                    # don't make it to the edge
                    dt=stop_t-part_t
                    part_t=stop_t
                    j_cross=None
                else:
                    dt=dt_max_edge
                    part_t=t_max_edge

                # Take the step
                self.P['x'][i] += self.P['u'][i]*dt

                if j_cross is not None:
                    # cross edge j, update time.  careful that j isn't boundary
                    # or start sliding on boundary.
                    # print "Cross edge"
                    cells=g.edges['cells'][j_cross]
                    if cells[0]==p['c']:
                        new_c=cells[1]
                    elif cells[1]==p['c']:
                        new_c=cells[0]
                    else:
                        assert False

                    # More scrutiny on the edge crossing -
                    # would it take us out of the domain? then bounce and frown.
                    # would it take us to a convergent edge? then bounce and frown.
                    bounce=False

                    if new_c<0:
                        bounce=True
                    else:
                        recross= ( self.U[new_c,0]*j_cross_normal[0] +
                                   self.U[new_c,1]*j_cross_normal[1] )
                        if recross<=0:
                            bounce=True

                    if bounce:
                        closing= (self.P['u'][i,0]*j_cross_normal[0] +
                                  self.P['u'][i,1]*j_cross_normal[1] )
                        # slightly over-compensate, pushing away from problematic
                        # edge
                        print("BOUNCE")

                        self.P['u'][i] -= 1.1 * j_cross_normal*closing
                        self.P['j_last'][i]=j_cross
                    else:
                        self.P['c'][i]=new_c
                        self.P['j_last'][i]=j_cross
                        self.P['u'][i] = self.U[new_c]
                        # HERE: need to check whether the new cell's
                        # velocity is going to push us back, in which case
                        # we should instead scoot along the tangent and
                        # hide our faces.
                        # actually, better solution is to handle *before*
                        # the particle hits the edge.  If we handle it after,
                        # two particles converging on this edge will cross
                        # paths, and that really doesn't pass the sniff test.
                        # this is not good, but will let the sim complete

                    if self.record_dense:
                        self.append_state(self.dense)

    def save_tracks(self,fn,overwrite=True):
        if os.path.exists(fn):
            if overwrite:
                os.unlink(fn)
            else:
                raise Exception("Output %s already exists"%fn)

        # archive the grid for good measure:
        self.g.write_ugrid(fn)
        ds=xr.open_dataset(fn).copy(deep=True)

        ds['particle']=( ('particle',), np.arange(len(self.P)) )
        times=np.array([out[1] for out in self.output])

        ds['time'] = ( ('time',), times )
        ds.time.attrs['units']="seconds since 1970-01-01T00:00:00"

        X=np.array( [out[0] for out in self.output] )

        ds['x'] = ( ('time','particle'), X[:,:,0] )
        ds['y'] = ( ('time','particle'), X[:,:,1] )

        os.unlink(fn)
        ds.to_netcdf(fn)
        return ds

#    Edges and incompatible velocities
#    ---------------------------------
#
#    With a discontinuous velocity field, sooner or later there
#    is a particle which is one cell, pushed to the edge, and
#    enters a new cell which also wants to push the particle to
#    the edge.
#
#    Solutions:
#     1. Fix the velocity field.  The 0th order in space (constant
#        velocity within a cell) approach cannot be fixed in this way.
#        Going to a reconstruction like in Gerard's paper would be a
#        solution, though it doesn't extend beyond triangles.  Postma
#        I think would be sufficient to get triangles and quads, but
#        still no good on pentagons, hexes, etc.
#     2. Adjust the velocity when this happens.  The adjustment might
#        be to take the average of the two cells, reach back and use
#        an edge velocity to figure it out, force a tangent trajectory,
#        or other approaches.
#
#        Do any of these approaches not break the reversibility of
#        the method?  Seems that any approach which "reacts" to finding
#        a convergent edge is not reversible, since that edge would be
#        divergent in reverse.
#
#        I think for now I have to give up on reversibility, but at least
#        preserve invariance w.r.t. to output time stepping.  I.e. changing the
#        output timestep shouldn't alter any results.

# Method of Ketefian, Gross, Stelling
# - need a 2x2 matrix Ai and vector Bi per cell
#   u is linear in x,y
#   they include closed solutions for a trajectory

# The full analytic solution is pretty messy, but the bulk of the mess
# is due to the analytical integration of the linear velocity field.
# an intermediate step would be to go for the velocity field reconstruction,
# but go back to a numerical integration.
<|endoftext|>


<commit_before>#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by James Hester
# Copyright (c) 2014 James Hester
#
# License: MIT
#

"""This module exports the R plugin class."""

from SublimeLinter.lint import Linter, util

class R(Linter):

    """Provides an interface to R."""

    syntax = 'r'
    executable = 'R'
    version_args = '--slave --restore --no-save -e "packageVersion(\\\"lintr\\\")"'
    version_re = r'(?P<version>\d+\.\d+\.\d+)'
    version_requirement = '>= 0.1.0'
    defaults = {
        'linters': 'default_linters'
    }
    regex = (
        r'^.+?:(?P<line>\d+):(?P<col>\d+): '
        r'(?:(?P<error>error)|(?P<warning>warning|style)): '
        r'(?P<message>.+)'
    )
    multiline = False
    line_col_base = (1,
1)
    tempfile_suffix = None
    error_stream = util.STREAM_BOTH
    selectors = {}
    word_re = None
    inline_settings = None
    inline_overrides = None
    comment_re = r'#'
    tempfile_suffix = 'lintr'

    def cmd(self):
        """Return a list with the command line to execute."""

        settings = self.get_view_settings()
        return [self.executable_path,
                    '--slave',
                    '--restore',
                    '--no-save',
                    '-e',
                     'library(lintr);lint(commandArgs(TRUE), {0})'.format(settings['linters']),
                    '--args',
                    '@']<commit_msg>Add cache argument<commit_after>#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by James Hester
# Copyright (c) 2014 James Hester
#
# License: MIT
#

"""This module exports the R plugin class."""

from SublimeLinter.lint import Linter, util

class R(Linter):

    """Provides an interface to R."""

    syntax = 'r'
    executable = 'R'
    version_args = '--slave --restore --no-save -e "packageVersion(\\\"lintr\\\")"'
    version_re = r'(?P<version>\d+\.\d+\.\d+)'
    version_requirement = '>= 0.1.0'
    defaults = {
        'linters': 'default_linters',
        'cache': 'TRUE'
    }
    regex = (
        r'^.+?:(?P<line>\d+):(?P<col>\d+): '
        r'(?:(?P<error>error)|(?P<warning>warning|style)): '
        r'(?P<message>.+)'
    )
    multiline = False
    line_col_base = (1,1)
    tempfile_suffix = None
    error_stream = util.STREAM_BOTH
    selectors = {}
    word_re = None
    inline_settings = None
    inline_overrides = None
    comment_re = r'#'
    tempfile_suffix = 'lintr'

    def cmd(self):
        """Return a list with the command line to execute."""

        settings = self.get_view_settings()
        return [self.executable_path,
                    '--slave',
                    '--restore',
                    '--no-save',
                    '-e',
                     'library(lintr);lint(cache = {0}, commandArgs(TRUE), {1})'.format(settings['cache'], settings['linters']),
                    '--args',
                    '@']<|endoftext|>


<commit_before>#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by Jon Surrell
# Copyright (c) 2013 Jon Surrell
#
# License: MIT
#

"""This module exports the Ghc plugin class."""

from SublimeLinter.lint import Linter, util

class Ghc(Linter):

    """Provides an interface to ghc."""

    syntax = 'haskell'
    cmd = ('ghc', '-fno-code', '-Wall', '-Wwarn', '-fno-helpful-errors')
    regex = r'^.+:(?P<line>\d+):(?P<col>\d+):\s+(?P<warning>Warning:\s+)?(?P<message>.+)$'
    multiline = True

    # No stdin
    tempfile_suffix = 'hs'

    # ghc writes errors to STDERR
    error_stream = util.STREAM_STDERR

    # @todo allow some settings
    defaults = {}
    inline_settings = None
    inline_overrides = None
    comment_re = None
<commit_msg>Add support for sublimehaskell<commit_after>#
# linter.py
# Linter for SublimeLinter3, a code checking framework for Sublime Text 3
#
# Written by Jon Surrell
# Copyright (c) 2013 Jon Surrell
#
# License: MIT
#

"""This module exports the Ghc plugin class."""

from SublimeLinter.lint import Linter, util

class Ghc(Linter):

    """Provides an interface to ghc."""

    syntax = ('haskell', 'haskell-sublimehaskell')
    cmd = ('ghc', '-fno-code', '-Wall', '-Wwarn', '-fno-helpful-errors')
    regex = r'^.+:(?P<line>\d+):(?P<col>\d+):\s+(?P<warning>Warning:\s+)?(?P<message>.+)$'
    multiline = True

    # No stdin
    tempfile_suffix = 'hs'

    # ghc writes errors to STDERR
    error_stream = util.STREAM_STDERR

    # @todo allow some settings
    defaults = {}
    inline_settings = None
    inline_overrides = None
    comment_re = None
<|endoftext|>


<commit_before>#!/usr/bin/env python

import time

import numpy as np

from sklearn.metrics import r2_score
import tensorflow as tf

class RNN:
    def __init__(self):
        pass

    def build(self, job):
        ###################################################
        # model parameters and placeholders
        ###################################################
        self.set_params(job)

        self.inputs = tf.placeholder(tf.float32, shape=(self.batch_size, self.batch_length, self.seq_depth))
        self.targets = tf.placeholder(tf.float32, shape=(self.batch_size, self.batch_length, self.num_targets))

        ###################################################
        # convolution layers
        ###################################################
        if self.cnn_layers > 0:
            seq_length = self.batch_length
            seq_depth = self.seq_depth

            # reshape
            cinput = tf.reshape(self.inputs, [self.batch_size, 1, seq_length, seq_depth])

            for li in range(self.cnn_layers):
                with tf.variable_scope('cnn%d' % li) as vs:
                    # convolution params
                    stdev = 1./np.sqrt(self.cnn_filters[li]*seq_depth)
                    kernel = tf.Variable(tf.random_uniform([1, self.cnn_filter_sizes[li], seq_depth, self.cnn_filters[li]], minval=-stdev, maxval=stdev), name='kernel')
                    biases = tf.Variable(tf.zeros([self.cnn_filters[li]]), name='bias')

                    # convolution
                    conv = tf.nn.conv2d(cinput, kernel, [1, 1, 1, 1], padding='SAME')
                    convf = tf.nn.relu(tf.nn.bias_add(conv, biases), name='conv%d'%li)

                    if self.cnn_pool[li] == 1:
                        cinput = convf
                    else:
                        # pool
                        pool = tf.nn.max_pool(convf, ksize=[1,1,self.cnn_pool[li],1], strides=[1,1,self.cnn_pool[li],1], padding='SAME', name='pool%d'%li)
                        cinput = pool

                    # updates size variables
                    seq_length = seq_length // self.cnn_pool[li]
                    seq_depth = self.cnn_filters[li]

            # reshape for RNN
            rinput = tf.reshape(cinput, [self.batch_size, seq_length, seq_depth])

        else:
            rinput = self.inputs

        ###################################################
        # recurrent layers
        ###################################################
        # tf needs batch_length in the front as a list
        rinput = tf.unpack(tf.transpose(rinput, [1, 0, 2]))

        for li in range(self.rnn_layers):
            with tf.variable_scope('rnn%d' % li) as vs:
                # determine cell
                if self.cell == 'rnn':
                    cell = tf.nn.rnn_cell.BasicRNNCell(self.rnn_units[li])
                elif self.cell == 'gru':
                    cell = tf.nn.rnn_cell.GRUCell(self.rnn_units[li])
                elif self.cell == 'lstm':
                    cell = tf.nn.rnn_cell.LSTMCell(self.rnn_units[li], state_is_tuple=True, initializer=tf.contrib.layers.xavier_initializer(uniform=True))
                else:
                    print('Cannot recognize RNN cell type %s' % self.cell)
                    exit(1)

                # dropout
                if li < len(self.rnn_dropout) and self.rnn_dropout[li] > 0:
                    cell = tf.nn.rnn_cell.DropoutWrapper(cell, output_keep_prob=(1.0-self.rnn_dropout[li]))

                # run bidirectional
                outputs, _, _ = tf.nn.bidirectional_rnn(cell, cell, rinput, dtype=tf.float32)

                # outputs become input to next layer
                rinput = outputs

        ###################################################
        # output layers
        ###################################################
        with tf.variable_scope('out'):
            out_weights = tf.get_variable(name='weights', shape=[2*self.rnn_units[-1], self.num_targets], dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer(uniform=True))
            out_biases = tf.Variable(tf.zeros(self.num_targets), name='bias')

        # make final predictions
        preds_length = []
        for li in range(self.batch_buffer, self.batch_length-self.batch_buffer):
            preds_length.append(tf.matmul(outputs[li], out_weights) + out_biases)

        # convert list to tensor
        preds = tf.pack(preds_length)

        # transpose back to batches in front
        self.preds = tf.transpose(preds, [1, 0, 2])

        for v in tf.all_variables():
            print(v.name, v.get_shape())

        ###################################################
        # loss and optimization
        ###################################################
        # take square difference
        sq_diff = tf.squared_difference(self.preds, self.targets[:,self.batch_buffer:self.batch_length-self.batch_buffer,:])

        # set any NaN's to zero
        nan_indexes = tf.is_nan(sq_diff)
        tens0 = tf.zeros_like(sq_diff)
        sq_diff = tf.select(nan_indexes, tens0, sq_diff)

        # take the mean
        self.loss_op = tf.reduce_mean(sq_diff)

        # define optimization
        if self.optimization == 'adam':
            opt = tf.train.AdamOptimizer(self.learning_rate, self.adam_beta1, self.adam_beta2)
        else:
            print('Cannot recognize optimization algorithm %s' % self.optimization)
            exit(1)

        # clip gradients
        gvs = opt.compute_gradients(self.loss_op)
        if self.grad_clip is None:
            clip_gvs = gvs
        else:
            clip_gvs = [(tf.clip_by_value(g, -self.grad_clip, self.grad_clip), v) for g, v in gvs]
        self.step_op = opt.apply_gradients(clip_gvs)

    def load(self):
        pass

    def set_params(self, job):
        ''' Set RNN parameters. '''

        ###################################################
        # data attributes
        ###################################################
        self.seq_depth = job.get('seq_depth', 4)
        self.num_targets = job['num_targets']

        ###################################################
        # batching
        ###################################################
        self.batch_size = job.get('batch_size', 4)
        self.batch_length = job.get('batch_length', 1024)
        self.batch_buffer = job.get('batch_buffer', 8)

        ###################################################
        # training
        ###################################################
        self.learning_rate = job.get('learning_rate', 0.004)
        self.adam_beta1 = job.get('adam_beta1', 0.9)
        self.adam_beta2 = job.get('adam_beta2', 0.99)
        self.optimization = job.get('optimization', 'adam').lower()
        self.grad_clip = job.get('grad_clip', 4)

        ###################################################
        # CNN params
        ###################################################
        self.cnn_filters = np.atleast_1d(job.get('cnn_filters', []))
        self.cnn_filter_sizes = np.atleast_1d(job.get('cnn_filter_sizes', []))
        self.cnn_layers = len(self.cnn_filters)
        self.cnn_pool = layer_extend(job.get('cnn_pool', []), 1, self.cnn_layers)

        ###################################################
        # RNN params
        ###################################################
        self.rnn_units = np.atleast_1d(job.get('rnn_units', [100]))
        self.rnn_layers = len(self.rnn_units)
        self.cell = job.get('cell', 'lstm').lower()

        ###################################################
        # regularization
        ###################################################
        self.cnn_dropout = layer_extend(job.get('cnn_dropout', []), 0, self.cnn_layers)
        self.rnn_dropout = layer_extend(job.get('rnn_dropout', []), 0, self.rnn_layers)

        # batch normalization?

    def test(self, sess, batcher):
        ''' Compute model accuracy on a test set. '''

        batch_losses = []
        preds = []
        targets = []

        # get first batch
        Xb, Yb = batcher.next()

        while Xb is not None:
            # measure batch loss
            preds_batch, loss_batch = sess.run([self.preds, self.loss_op], feed_dict={self.inputs:Xb, self.targets:Yb})

            # accumulate loss
            batch_losses.append(loss_batch)

            # accumulate predictions and targets
            preds.append(preds_batch)
            targets.append(Yb[:,self.batch_buffer:self.batch_length-self.batch_buffer,:])

            # next batch
            Xb, Yb = batcher.next()

        # reset batcher
        batcher.reset()

        # accumulate predictions and targets
        preds = np.vstack(preds)
        targets = np.vstack(targets)

        # compute R2 per target
        r2 = np.zeros(self.num_targets)
        for ti in range(self.num_targets):
            # flatten
            preds_ti = preds[:,:,ti].flatten()
            targets_ti = targets[:,:,ti].flatten()

            # remove NaN's
            valid_indexes = np.logical_not(np.isnan(targets_ti))
            preds_ti = preds_ti[valid_indexes]
            targets_ti = targets_ti[valid_indexes]

            # compute R2
            tmean = targets_ti.mean(dtype='float64')
            tvar = (targets_ti-tmean).var(dtype='float64')
            pvar = (targets_ti-preds_ti).var(dtype='float64')
            r2[ti] = 1.0 - pvar/tvar
            # print('%d %f %f %f %f' % (ti, tmean, tvar, pvar, r2[ti]))

        return np.mean(batch_losses), np.mean(r2)

    def train_epoch(self, sess, batcher):
        ''' Execute one training epoch '''

        # initialize training loss
        train_loss = []

        # get first batch
        Xb, Yb = batcher.next()

        while Xb is not None:
            # run step
            loss_batch, _ = sess.run([self.loss_op, self.step_op], feed_dict={self.inputs:Xb, self.targets:Yb})

            # accumulate loss
            train_loss.append(loss_batch)

            # next batch
            Xb, Yb = batcher.next()

        # reset training batcher
        batcher.reset()

        return np.mean(train_loss)

def layer_extend(var, default, layers):
    ''' Process job input to extend for the
         proper number of layers. '''

    # if it's a number
    if type(var) != list:
        # change the default to that number
        default = var

        # make it a list
        var = [var]

    # extend for each layer
    while len(var) < layers:
        var.append(default)

    return var

<commit_msg>weight tied biRNN; activations; norm_stabilizer<commit_after>#!/usr/bin/env python

import time

import numpy as np

from sklearn.metrics import r2_score
import tensorflow as tf

class RNN:
    def __init__(self):
        pass

    def build(self, job):
        ###################################################
        # model parameters and placeholders
        ###################################################
        self.set_params(job)

        self.inputs = tf.placeholder(tf.float32, shape=(self.batch_size, self.batch_length, self.seq_depth))
        self.targets = tf.placeholder(tf.float32, shape=(self.batch_size, self.batch_length, self.num_targets))

        ###################################################
        # convolution layers
        ###################################################
        seq_length = self.batch_length
        seq_depth = self.seq_depth

        if self.cnn_layers > 0:
            # reshape
            cinput = tf.reshape(self.inputs, [self.batch_size, 1, seq_length, seq_depth])

            for li in range(self.cnn_layers):
                with tf.variable_scope('cnn%d' % li) as vs:
                    # convolution params
                    stdev = 1./np.sqrt(self.cnn_filters[li]*seq_depth)
                    kernel = tf.Variable(tf.random_uniform([1, self.cnn_filter_sizes[li], seq_depth, self.cnn_filters[li]], minval=-stdev, maxval=stdev), name='kernel')
                    biases = tf.Variable(tf.zeros([self.cnn_filters[li]]), name='bias')

                    # convolution
                    conv = tf.nn.conv2d(cinput, kernel, [1, 1, 1, 1], padding='SAME')
                    convf = tf.nn.relu(tf.nn.bias_add(conv, biases), name='conv%d'%li)

                    if self.cnn_pool[li] == 1:
                        cinput = convf
                    else:
                        # pool
                        pool = tf.nn.max_pool(convf, ksize=[1,1,self.cnn_pool[li],1], strides=[1,1,self.cnn_pool[li],1], padding='SAME', name='pool%d'%li)
                        cinput = pool

                    # updates size variables
                    seq_length = seq_length // self.cnn_pool[li]
                    seq_depth = self.cnn_filters[li]

            # reshape for RNN
            rinput = tf.reshape(cinput, [self.batch_size, seq_length, seq_depth])

        else:
            rinput = self.inputs

        ###################################################
        # recurrent layers
        ###################################################
        # initialize norm stabilizer term
        norm_stabilizer = 0

        # move batch_length to the front as a list
        rinput = tf.unpack(tf.transpose(rinput, [1, 0, 2]))

        for li in range(self.rnn_layers):
            with tf.variable_scope('rnn%d' % li) as vs:
                # determine cell
                if self.cell == 'rnn':
                    cell = tf.nn.rnn_cell.BasicRNNCell(self.rnn_units[li], activation=self.activation)
                elif self.cell == 'gru':
                    cell = tf.nn.rnn_cell.GRUCell(self.rnn_units[li], activation=self.activation)
                elif self.cell == 'lstm':
                    # cell = tf.nn.rnn_cell.LSTMCell(self.rnn_units[li], state_is_tuple=True, initializer=tf.contrib.layers.xavier_initializer(uniform=True))
                    cell = tf.nn.rnn_cell.LSTMCell(self.rnn_units[li], state_is_tuple=True, initializer=tf.contrib.layers.xavier_initializer(uniform=True), activation=self.activation)
                else:
                    print('Cannot recognize RNN cell type %s' % self.cell)
                    exit(1)

                # dropout
                if li < len(self.rnn_dropout) and self.rnn_dropout[li] > 0:
                    cell = tf.nn.rnn_cell.DropoutWrapper(cell, output_keep_prob=(1.0-self.rnn_dropout[li]))

                # run bidirectional
                if self.cnn_layers == 0 and li == 0:
                    outputs, _, _ = tf.nn.bidirectional_rnn(cell, cell, rinput, dtype=tf.float32)
                else:
                    outputs, _, _ = bidirectional_rnn_tied(cell, cell, rinput, dtype=tf.float32)

                # accumulate norm stablizer
                if self.norm_stabilizer[li] > 0:
                    output_norms = tf.sqrt(tf.reduce_sum(tf.square(outputs), reduction_indices=1))
                    print(output_norms.get_shape())
                    tmp = tf.squared_difference(output_norms[1:,:], output_norms[:seq_length-1,:])
                    norm_stabilizer += self.norm_stabilizer[li]*tf.reduce_mean(tmp)

                # outputs become input to next layer
                rinput = outputs

        ###################################################
        # output layers
        ###################################################
        with tf.variable_scope('out'):
            out_weights = tf.get_variable(name='weights', shape=[2*self.rnn_units[-1], self.num_targets], dtype=tf.float32, initializer=tf.contrib.layers.xavier_initializer(uniform=True))
            out_biases = tf.Variable(tf.zeros(self.num_targets), name='bias')

        # make final predictions
        preds_length = []
        for li in range(self.batch_buffer, self.batch_length-self.batch_buffer):
            preds_length.append(tf.matmul(outputs[li], out_weights) + out_biases)

        # convert list to tensor
        preds = tf.pack(preds_length)

        # transpose back to batches in front
        self.preds = tf.transpose(preds, [1, 0, 2])

        for v in tf.all_variables():
            print(v.name, v.get_shape())

        ###################################################
        # loss and optimization
        ###################################################
        # take square difference
        sq_diff = tf.squared_difference(self.preds, self.targets[:,self.batch_buffer:self.batch_length-self.batch_buffer,:])

        # set any NaN's to zero
        # nan_indexes = tf.is_nan(sq_diff)
        # tens0 = tf.zeros_like(sq_diff)
        # sq_diff = tf.select(nan_indexes, tens0, sq_diff)

        # take the mean
        self.loss_op = tf.reduce_mean(sq_diff) + norm_stabilizer

        # define optimization
        if self.optimization == 'adam':
            opt = tf.train.AdamOptimizer(self.learning_rate, self.adam_beta1, self.adam_beta2)
        else:
            print('Cannot recognize optimization algorithm %s' % self.optimization)
            exit(1)

        # clip gradients
        gvs = opt.compute_gradients(self.loss_op)
        if self.grad_clip is None:
            clip_gvs = gvs
        else:
            clip_gvs = [(tf.clip_by_value(g, -self.grad_clip, self.grad_clip), v) for g, v in gvs]
        self.step_op = opt.apply_gradients(clip_gvs)

    def load(self):
        pass

    def set_params(self, job):
        ''' Set RNN parameters. '''

        ###################################################
        # data attributes
        ###################################################
        self.seq_depth = job.get('seq_depth', 4)
        self.num_targets = job['num_targets']

        ###################################################
        # batching
        ###################################################
        self.batch_size = job.get('batch_size', 4)
        self.batch_length = job.get('batch_length', 1024)
        self.batch_buffer = job.get('batch_buffer', 8)

        ###################################################
        # training
        ###################################################
        self.learning_rate = job.get('learning_rate', 0.004)
        self.adam_beta1 = job.get('adam_beta1', 0.9)
        self.adam_beta2 = job.get('adam_beta2', 0.99)
        self.optimization = job.get('optimization', 'adam').lower()
        self.grad_clip = job.get('grad_clip', 4)

        ###################################################
        # CNN params
        ###################################################
        self.cnn_filters = np.atleast_1d(job.get('cnn_filters', []))
        self.cnn_filter_sizes = np.atleast_1d(job.get('cnn_filter_sizes', []))
        self.cnn_layers = len(self.cnn_filters)
        self.cnn_pool = layer_extend(job.get('cnn_pool', []), 1, self.cnn_layers)

        ###################################################
        # RNN params
        ###################################################
        self.rnn_units = np.atleast_1d(job.get('rnn_units', [100]))
        self.rnn_layers = len(self.rnn_units)
        self.cell = job.get('cell', 'lstm').lower()
        self.activation = tf.tanh
        if job.get('activation').lower() == 'relu':
            self.activation = tf.nn.relu

        ###################################################
        # regularization
        ###################################################
        self.cnn_dropout = layer_extend(job.get('cnn_dropout', []), 0, self.cnn_layers)

        self.rnn_dropout = layer_extend(job.get('rnn_dropout', []), 0, self.rnn_layers)
        self.norm_stabilizer = layer_extend(job.get('norm_stabilizer', []), 0, self.rnn_layers)

        # batch normalization?

    def test(self, sess, batcher):
        ''' Compute model accuracy on a test set. '''

        batch_losses = []
        preds = []
        targets = []

        # get first batch
        Xb, Yb = batcher.next()

        while Xb is not None:
            # measure batch loss
            preds_batch, loss_batch = sess.run([self.preds, self.loss_op], feed_dict={self.inputs:Xb, self.targets:Yb})

            # accumulate loss
            batch_losses.append(loss_batch)

            # accumulate predictions and targets
            preds.append(preds_batch)
            targets.append(Yb[:,self.batch_buffer:self.batch_length-self.batch_buffer,:])

            # next batch
            Xb, Yb = batcher.next()

        # reset batcher
        batcher.reset()

        # accumulate predictions and targets
        preds = np.vstack(preds)
        targets = np.vstack(targets)

        # compute R2 per target
        r2 = np.zeros(self.num_targets)
        for ti in range(self.num_targets):
            # flatten
            preds_ti = preds[:,:,ti].flatten()
            targets_ti = targets[:,:,ti].flatten()

            # remove NaN's
            #valid_indexes = np.logical_not(np.isnan(targets_ti))
            #preds_ti = preds_ti[valid_indexes]
            #targets_ti = targets_ti[valid_indexes]

            # compute R2
            tmean = targets_ti.mean(dtype='float64')
            tvar = (targets_ti-tmean).var(dtype='float64')
            pvar = (targets_ti-preds_ti).var(dtype='float64')
            r2[ti] = 1.0 - pvar/tvar
            # print('%d %f %f %f %f' % (ti, tmean, tvar, pvar, r2[ti]))

        return np.mean(batch_losses), np.mean(r2)

    def train_epoch(self, sess, batcher):
        ''' Execute one training epoch '''

        # initialize training loss
        train_loss = []

        # get first batch
        Xb, Yb = batcher.next()

        while Xb is not None:
            # run step
            loss_batch, _ = sess.run([self.loss_op, self.step_op], feed_dict={self.inputs:Xb, self.targets:Yb})

            # accumulate loss
            train_loss.append(loss_batch)

            # next batch
            Xb, Yb = batcher.next()

        # reset training batcher
        batcher.reset()

        return np.mean(train_loss)

def layer_extend(var, default, layers):
    ''' Process job input to extend for the
         proper number of layers. '''

    # if it's a number
    if type(var) != list:
        # change the default to that number
        default = var

        # make it a list
        var = [var]

    # extend for each layer
    while len(var) < layers:
        var.append(default)

    return var

from tensorflow.python.ops import variable_scope as vs
from tensorflow.python.ops import array_ops
from tensorflow.python.ops.rnn import _reverse_seq
def bidirectional_rnn_tied(cell_fw, cell_bw, inputs,
                      initial_state_fw=None, initial_state_bw=None,
                      dtype=None, sequence_length=None, scope=None):
    name = scope or "BiRNN"
    with vs.variable_scope(name) as fw_scope:
        # Forward direction
        output_fw, output_state_fw = tf.nn.rnn(cell_fw, inputs, initial_state_fw, dtype, sequence_length, scope=fw_scope)

    with vs.variable_scope(name, reuse=True) as bw_scope:
        # Backward direction
        tmp, output_state_bw = tf.nn.rnn(cell_bw, _reverse_seq(inputs, sequence_length),
                 initial_state_bw, dtype, sequence_length, scope=bw_scope)

    output_bw = _reverse_seq(tmp, sequence_length)

    # Concat each of the forward/backward outputs
    outputs = [array_ops.concat(1, [fw, bw]) for fw, bw in zip(output_fw, output_bw)]

    return (outputs, output_state_fw, output_state_bw)

<|endoftext|>


<commit_before>#!/usr/bin/env python

import os, optparse, multiprocessing, sys;
from multiprocessing import Pool

def job_func(cmd):
    os.system(cmd);
    return cmd;

class Usage(Exception):
    def __init__(self,msg):
        self.msg = msg

def main():
    try:
        try:
            usage = "phosolve.py [--help][--threads N][--level N] ptk-url "
            parser = optparse.OptionParser(usage=usage);
            parser.set_defaults(threads=10)
            parser.set_defaults(level=-1)
            parser.set_defaults(iterations=100)
            parser.add_option("-t", "--threads", dest="threads",
                              help="Number of threads to use",type="int")
            parser.add_option("-l", "--level", dest="level",
                              help="Lowest level to process at",type="int")
            parser.add_option("-i", "--iter", dest="iterations",
                              help="Max iterations to perform",type="int")

            (options, args) = parser.parse_args()

            if not args: parser.error("missing files")

        except optparse.OptionError, msg:
            raise Usage(msg)

        pool = Pool(processes=options.threads)

        for iteration in range(options.iterations):
            # Perform Albedo
            albedo_cmd = []
            for i in range(2*options.threads):
                cmd = "phoitalbedo "
                if ( options.level > 0 ):
                    cmd = cmd + "-l "+str(options.level)+" "
                cmd = cmd + "-j "+str(i)+" -n "+str(2*options.threads)+" "+args[0]
                albedo_cmd.append( cmd )
            results = [pool.apply_async(job_func, (cmd,)) for cmd in albedo_cmd]
            for result in results:
                result.get()

            # Mipmap up the levels
            os.system("mipmap pf://index/Albedo.plate")

            # Update the Time Estimate
            time_cmd = []
            for i in range(options.threads):
                cmd = "phoittime "
                if ( options.level > 0 ):
                    cmd = cmd + "-l "+str(options.level-2)+" "
                cmd = cmd + "-j "+str(i)+" -n "+str(options.threads)+" "+args[0]
                time_cmd.append(cmd)
            results = [pool.apply_async(job_func, (cmd,)) for cmd in time_cmd]
            for result in results:
                result.get()

    except Usage, err:
        print >>sys.stderr, err.msg
        return 2

    except Exception, err:
        sys.stderr.write( str(err) + '\n' )
        return 1

if __name__ == "__main__":
    sys.exit(main())

<commit_msg>removed hard coded url<commit_after>#!/usr/bin/env python2.6

import os, optparse, multiprocessing, sys;
from multiprocessing import Pool

def job_func(cmd):
    os.system(cmd);
    return cmd;

class Usage(Exception):
    def __init__(self,msg):
        self.msg = msg

def main():
    try:
        try:
            usage = "phosolve.py [--help][--threads N][--level N] ptk-url "
            parser = optparse.OptionParser(usage=usage);
            parser.set_defaults(threads=10)
            parser.set_defaults(level=-1)
            parser.set_defaults(iterations=100)
            parser.add_option("-t", "--threads", dest="threads",
                              help="Number of threads to use",type="int")
            parser.add_option("-l", "--level", dest="level",
                              help="Lowest level to process at",type="int")
            parser.add_option("-i", "--iter", dest="iterations",
                              help="Max iterations to perform",type="int")

            (options, args) = parser.parse_args()

            if not args: parser.error("missing files")

        except optparse.OptionError, msg:
            raise Usage(msg)

        pool = Pool(processes=options.threads)

        # Finding URL for Albedo
        albedo_url = args[0][:args[0].find("/ptk/")]+"/index/Albedo.plate"

        for iteration in range(options.iterations):
            # Perform Albedo
            print " --- ALBEDO ---"
            albedo_cmd = []
            for i in range(2*options.threads):
                cmd = "phoitalbedo "
                if ( options.level > 0 ):
                    cmd = cmd + "-l "+str(options.level)+" "
                cmd = cmd + "-j "+str(i)+" -n "+str(2*options.threads)+" "+args[0]
                albedo_cmd.append( cmd )
            results = [pool.apply_async(job_func, (cmd,)) for cmd in albedo_cmd]
            for result in results:
                result.get()

            # Mipmap up the levels
            print " --- MIPMAP ---"
            os.system("mipmap "+albedo_url)

            # Update the Time Estimate
            time_cmd = []
            print " --- TIME ---"
            for i in range(options.threads):
                cmd = "phoittime "
                if ( options.level > 0 ):
                    cmd = cmd + "-l "+str(options.level-2)+" "
                cmd = cmd + "-j "+str(i)+" -n "+str(options.threads)+" "+args[0]
                time_cmd.append(cmd)
            results = [pool.apply_async(job_func, (cmd,)) for cmd in time_cmd]
            for result in results:
                result.get()

    except Usage, err:
        print >>sys.stderr, err.msg
        return 2

    except Exception, err:
        sys.stderr.write( str(err) + '\n' )
        return 1

if __name__ == "__main__":
    sys.exit(main())

<|endoftext|>


<commit_before>from django.contrib.gis import admin
from django.contrib import admin as oldAdmin
from fixcity.bmabr.models import Rack, Comment, Neighborhoods, CommunityBoard, SubwayStations
from fixcity.bmabr.models import  StatementOfSupport

#class PhotoAdmin(admin.GeoModelAdmin):
#    list_display  = ('rack','email')
#admin.site.register(Photo,PhotoAdmin)

class StatementOfSupportAdmin(admin.GeoModelAdmin):
    list_display = ('s_rack','email')
admin.site.register(StatementOfSupport,StatementOfSupportAdmin)

class StatementInline(oldAdmin.StackedInline):
    model = StatementOfSupport

class SubwayAdmin(admin.GeoModelAdmin):
    list_display = ('name','borough')
    search_fields = ('name','borough')
admin.site.register(SubwayStations,SubwayAdmin)

class CommentAdmin(admin.GeoModelAdmin):
    list_display = ('rack','email')
admin.site.register(Comment,CommentAdmin)

class RackAdmin(admin.GeoModelAdmin):
    list_display = ('address','location')
admin.site.register(Rack, RackAdmin)

class NeighborhoodsAdmin(admin.GeoModelAdmin):
    list_display = ('name','county')
admin.site.register(Neighborhoods,NeighborhoodsAdmin)

class CommunityBoardAdmin(admin.GeoModelAdmin):
    list_display = ('name','gid')
admin.site.register(CommunityBoard,CommunityBoardAdmin)
<commit_msg>register sources with django admin<commit_after>from django.contrib.gis import admin
from django.contrib import admin as oldAdmin
from fixcity.bmabr.models import Rack, Comment, Neighborhoods, CommunityBoard, SubwayStations
from fixcity.bmabr.models import SeeClickFixSource
from fixcity.bmabr.models import Source
from fixcity.bmabr.models import EmailSource
from fixcity.bmabr.models import TwitterSource
from fixcity.bmabr.models import  StatementOfSupport

#class PhotoAdmin(admin.GeoModelAdmin):
#    list_display  = ('rack','email')
#admin.site.register(Photo,PhotoAdmin)

class StatementOfSupportAdmin(admin.GeoModelAdmin):
    list_display = ('s_rack','email')
admin.site.register(StatementOfSupport,StatementOfSupportAdmin)

class StatementInline(oldAdmin.StackedInline):
    model = StatementOfSupport

class SubwayAdmin(admin.GeoModelAdmin):
    list_display = ('name','borough')
    search_fields = ('name','borough')
admin.site.register(SubwayStations,SubwayAdmin)

class CommentAdmin(admin.GeoModelAdmin):
    list_display = ('rack','email')
admin.site.register(Comment,CommentAdmin)

class RackAdmin(admin.GeoModelAdmin):
    list_display = ('address','location')
admin.site.register(Rack, RackAdmin)

class NeighborhoodsAdmin(admin.GeoModelAdmin):
    list_display = ('name','county')
admin.site.register(Neighborhoods,NeighborhoodsAdmin)

class CommunityBoardAdmin(admin.GeoModelAdmin):
    list_display = ('name','gid')
admin.site.register(CommunityBoard,CommunityBoardAdmin)

admin.site.register(Source)
admin.site.register(SeeClickFixSource)
admin.site.register(EmailSource)
admin.site.register(TwitterSource)
<|endoftext|>