<commit_before>from jtagDeviceDescription import JTAGDeviceDescription

import time
from bitarray import bitarray

def gc(addr):
    return (addr>>1)^addr

def pstatus(resflags):
    print "STATUS: "+("" if (ord(resflags)&0b11000011==1) else "INVALID_STATUS ")+\
        ("ISCDIS " if ord(resflags)&32 else "")+("ISCEN " if ord(resflags)&16 else "")+\
        ("SECURE " if ord(resflags)&8 else "")+("DONE " if ord(resflags)&4 else "")

def build_byte_align_buff(bit_count):
    bitmod = bit_count%8
    if bitmod == 0:
        rdiff = bitarray()
        print "NO BUFF NEEDED", rdiff
    else:
        rdiff = bitarray(8-bitmod)
        rdiff.setall(False)
    return rdiff

class JTAGStateMachine(object):
    STATE_TLR = 0
    STATE_IDLE = 1
    STATE_SELECT_DR = 2
    STATE_CAPTURE_DR = 3
    STATE_SHIFT_DR = 4
    STATE_EXIT1_DR = 5
    STATE_PAUSE_DR = 6
    STATE_EXIT2_DR = 7
    STATE_UPDATE_DR = 8
    STATE_SELECT_IR = 9
    STATE_CAPTURE_IR = 10
    STATE_SHIFT_IR = 11
    STATE_EXIT1_IR = 12
    STATE_PAUSE_IR = 13
    STATE_EXIT2_IR = 14
    STATE_UPDATE_IR = 15

    STATE_NAMES = {0:'TLR',
                   1:'IDLE',
                   2:'SELECT_DR',
                   3:'CAPTURE_DR',
                   4:'SHIFT_DR',
                   5:'EXIT1_DR',
                   6:'PAUSE_DR',
                   7:'EXIT2_DR',
                   8:'UPDATE_DR' ,
                   9:'SELECT_IR' ,
                   10:'CAPTURE_IR',
                   11:'SHIFT_IR',
                   12:'EXIT1_IR',
                   13:'PAUSE_IR',
                   14:'EXIT2_IR',
                   15:'UPDATE_IR'}

    def __init__(self):
        self._state = self.STATE_TLR

    def transition(self, bit):
        if self._state == self.STATE_TLR:
            if not bit: self._state = self.STATE_IDLE
        elif self._state == self.STATE_IDLE:
            if bit:     self._state = self.STATE_SELECT_DR
        elif self._state == self.STATE_SELECT_DR:
            if bit:     self._state = self.STATE_SELECT_IR
            else:       self._state = self.STATE_CAPTURE_DR
        elif self._state == self.STATE_CAPTURE_DR:
            if bit:     self._state = self.STATE_EXIT1_DR
            else:       self._state = self.STATE_SHIFT_DR
        elif self._state == self.STATE_SHIFT_DR:
            if bit:     self._state = self.STATE_EXIT1_DR
        elif self._state == self.STATE_EXIT1_DR:
            if bit:     self._state = self.STATE_UPDATE_DR
            else:       self._state = self.STATE_PAUSE_DR
        elif self._state == self.STATE_PAUSE_DR:
            if bit:     self._state = self.STATE_EXIT2_DR
        elif self._state == self.STATE_EXIT2_DR:
            if bit:     self._state = self.STATE_UPDATE_DR
            else:       self._state = self.STATE_SHIFT_DR
        elif self._state == self.STATE_UPDATE_DR:
            if bit:     self._state = self.STATE_SELECT_DR
            else:       self._state = self.STATE_IDLE
        elif self._state == self.STATE_SELECT_IR:
            if bit:     self._state = self.STATE_TLR
            else:       self._state = self.STATE_CAPTURE_IR
        elif self._state == self.STATE_CAPTURE_IR:
            if bit:     self._state = self.STATE_EXIT1_IR
            else:       self._state = self.STATE_SHIFT_IR
        elif self._state == self.STATE_SHIFT_IR:
            if bit:     self._state = self.STATE_EXIT1_IR
        elif self._state == self.STATE_EXIT1_IR:
            if bit:     self._state = self.STATE_UPDATE_IR
            else:       self._state = self.STATE_PAUSE_IR
        elif self._state == self.STATE_PAUSE_IR:
            if bit:     self._state = self.STATE_EXIT2_IR
        elif self._state == self.STATE_EXIT2_IR:
            if bit:     self._state = self.UPDATE_IR
            else:       self._state = self.SHIFT_IR
        elif self._state == self.STATE_UPDATE_IR:
            if bit:     self._state = self.STATE_SELECT_DR
            else:       self._state = self.STATE_IDLE
        #print "STATE:", self.state

    @property
    def state(self):
        return self.STATE_NAMES.get(self._state, "UNKNOWN STATE!")

class JTAGDevice(object):
    def __init__(self, chain, idcode):
        #print "IDCODE", idcode.__repr__()
        if not isinstance(chain, JTAGScanChain):
            raise ValueError("JTAGDevice requires an instnace of JTAGScanChain")
        self._chain = chain

        if isinstance(idcode, int):
            self._id = idcode
        elif isinstance(idcode, str):
            if len(idcode)==4:
                self._id = (ord(idcode[0])<<24)|(ord(idcode[1])<<16)|\
                    (ord(idcode[2])<<8)|ord(idcode[3])
            else:
                raise ValueError("JTAGDevice idcode parameter must be an "
                                 "int or string of length 4. (%s)"%self._id)
        else:
            raise ValueError("JTAGDevice idcode parameter must be an int or "
                             "string of length 4. (Invalid Type: %s)"%type(idcode))

        self.desc = JTAGDeviceDescription.get_descriptor_for_idcode(self._id)

    def erase(self):
        #import ipdb
        con = self._chain._controller
        self._chain._jtagEnable()
        #print self.desc._instructions

        cmd_isce = self.desc._instructions['ISC_ENABLE']
        cmd_iscd = self.desc._instructions['ISC_DISABLE']
        cmd_iscerase = self.desc._instructions['ISC_ERASE']
        print cmd_isce, cmd_iscd, cmd_iscerase

        #ipdb.set_trace()
        con.writeTMSBits('\x00\xDF', 10) #Get to ShiftIR
        pstatus(con.writeTDIBits('\x68', 7, return_tdo=True))
        con.writeTDIBits('\x01', 1, TMS=True) #ISC_ENABLE
        con.writeTMSBits('\x01', 8) #RUN
        time.sleep(0.01)

        #ipdb.set_trace()
        con.writeTMSBits('\x03', 4) #Get to ShiftIR
        pstatus(con.writeTDIBits('\x6D', 7, return_tdo=True))
        con.writeTDIBits('\x01', 1, TMS=True) #ISC_ERASE
        con.writeTMSBits('\x01', 8) #RUN
        time.sleep(0.01)

        #ipdb.set_trace()
        con.writeTMSBits('\x03', 4) #Get to ShiftIR
        pstatus(con.writeTDIBits('\xF0', 7, return_tdo=True))
        con.writeTDIBits('\x01', 1, TMS=True) #ISC_DISCHARGE
        con.writeTMSBits('\x01', 8) #RUN
        time.sleep(0.01)

        #ipdb.set_trace()
        con.writeTMSBits('\x03', 4) #Get to ShiftIR
        pstatus(con.writeTDIBits('\xF0', 7, return_tdo=True))
        con.writeTDIBits('\x01', 1, TMS=True) #ISC_INIT
        con.writeTMSBits('\x1b', 5) #init pulse
        con.writeTMSBits('\x00', 8) #RUN
        time.sleep(0.01)

        #ipdb.set_trace()
        con.writeTMSBits('\x03', 4) #Get to ShiftIR
        pstatus(con.writeTDIBits('\x40', 7, return_tdo=True))
        con.writeTDIBits('\x01', 1, TMS=True) #ISC_DISABLE
        con.writeTMSBits('\x01', 8) #RUN
        time.sleep(0.01)

        #ipdb.set_trace()
        con.writeTMSBits('\x03', 4) #Get to ShiftIR
        pstatus(con.writeTDIBits('\xff', 7, return_tdo=True))
        con.writeTDIBits('\x01', 1, TMS=True) #BYPASS
        con.writeTMSBits('\xFF', 8) #RUN
        time.sleep(0.01)

        #time.sleep(2)
        self._chain._jtagDisable()

    def program(self, data_array):
        import ipdb
        con = self._chain._controller
        self._chain._jtagEnable()

        #ipdb.set_trace()
        con.writeTMSBits('\x00\xDF', 10) #Get to ShiftIR
        pstatus(con.writeTDIBits('\x68', 7, return_tdo=True))
        con.writeTDIBits('\x01', 1, TMS=True) #ISC_ENABLE
        con.writeTMSBits('\x01', 8) #RUN
        time.sleep(0.01)

        for i,r in enumerate(data_array):
            addr = bitarray(bin(gc(i))[2:].zfill(7))
            addr.reverse()

            #print "\nLOADING(%s:%s) ="%(i,addr), r
            con.writeTMSBits('\x03', 4) #Get to ShiftIR
            pstatus(con.writeTDIBits('\x6A', 7, return_tdo=True))#first ins bits
            con.writeTDIBits('\x01', 1, TMS=True) #ISC_PROGRAM
            con.writeTMSBits('\x03', 4) #Get to ShiftDR without running

            buffered_r = build_byte_align_buff(len(r)) + r
            buffered_addr = build_byte_align_buff(len(addr)) + addr

            con.writeTDIBits(buffered_r.tobytes(), len(r), return_tdo=True).__repr__()
            con.writeTDIBits(buffered_addr.tobytes(), len(addr)-1, return_tdo=True) #ISC_PROGRAM
            con.writeTDIBits(chr(addr[0]), 1, TMS=True, return_tdo=True) #ISC_PROGRAM

            con.writeTMSBits('\x01', 8) #RUN
            time.sleep(0.01)

        #ipdb.set_trace()
        con.writeTMSBits('\x03', 4) #Get to ShiftIR
        pstatus(con.writeTDIBits('\xF0', 7, return_tdo=True))
        con.writeTDIBits('\x01', 1, TMS=True) #ISC_DISCHARGE
        con.writeTMSBits('\x01', 8) #RUN
        time.sleep(0.01)

        #ipdb.set_trace()
        con.writeTMSBits('\x03', 4) #Get to ShiftIR
        pstatus(con.writeTDIBits('\xF0', 7, return_tdo=True))
        con.writeTDIBits('\x01', 1, TMS=True) #ISC_INIT
        con.writeTMSBits('\x1b', 5) #init pulse
        con.writeTMSBits('\x00', 8) #RUN
        time.sleep(0.01)

        #ipdb.set_trace()
        con.writeTMSBits('\x03', 4) #Get to ShiftIR
        con.writeTDIBits('\x40', 7, return_tdo=True).__repr__()
        con.writeTDIBits('\x01', 1, TMS=True) #ISC_DISABLE
        con.writeTMSBits('\x01', 8) #RUN
        time.sleep(0.01)

        #ipdb.set_trace()
        con.writeTMSBits('\x03', 4) #Get to ShiftIR
        pstatus(con.writeTDIBits('\xff', 7, return_tdo=True))
        con.writeTDIBits('\x01', 1, TMS=True) #BYPASS
        con.writeTMSBits('\xFF', 8) #RUN
        time.sleep(0.01)

        self._chain._jtagDisable()

class JTAGScanChain(object):
    def __init__(self, controller):
        self._controller = controller
        self._controller._scanchain = self #This might necessitate a factory
        self._sm = JTAGStateMachine()
        self._hasinit = False
        self._devices = []

    def init_chain(self):
        if not self._hasinit:
            #Set state to SHIFT_DR through TLR
            self._controller.jtagEnable()
            self._controller.writeTMSBits('\x00\x5F', 9)

            self._devices = []
            idcode_str = self._controller.readTDOBits(32)
            while idcode_str != '\x00\x00\x00\x00':
                Jdev = JTAGDevice(self, idcode_str)
                self._devices.append(Jdev)
                idcode_str = self._controller.readTDOBits(32)
            self._controller.jtagDisable()

            #The chain comes out last first. Reverse it to get order.
            self._devices.reverse()

    def _jtagDisable(self):
        self._controller.jtagDisable()

    def _jtagEnable(self):
        self._controller.jtagEnable()

    def _tapTransition(self, bits):
        #print "Bits:", bits
        statetrans = [self._sm.state]
        for bit in bits[::-1]:
            #print "Transitioning TMS->%s"%bit
            self._sm.transition(bit)
            #if statetrans[-1]!=self._sm.state:
            statetrans.append(self._sm.state)
        #print "TAP:",self._sm.state
        #print "TAP State Change:", "->".join(statetrans)

if __name__ == "__main__":
    from digilentdriver import JTAGController
    controllers = JTAGController.getAttachedControllers()
    print "USB Controllers:"
    for i, c in enumerate(controllers):
        print "  %d %s"%(i, c)
        chain = JTAGScanChain(c)
        chain.init_chain()

<commit_msg>Fixed issue with PC1 cable detecting forever when not attached to board.<commit_after>from jtagDeviceDescription import JTAGDeviceDescription

import time
from bitarray import bitarray

def gc(addr):
    return (addr>>1)^addr

def pstatus(resflags):
    print "STATUS: "+("" if (ord(resflags)&0b11000011==1) else "INVALID_STATUS ")+\
        ("ISCDIS " if ord(resflags)&32 else "")+("ISCEN " if ord(resflags)&16 else "")+\
        ("SECURE " if ord(resflags)&8 else "")+("DONE " if ord(resflags)&4 else "")

def build_byte_align_buff(bit_count):
    bitmod = bit_count%8
    if bitmod == 0:
        rdiff = bitarray()
        print "NO BUFF NEEDED", rdiff
    else:
        rdiff = bitarray(8-bitmod)
        rdiff.setall(False)
    return rdiff

class JTAGStateMachine(object):
    STATE_TLR = 0
    STATE_IDLE = 1
    STATE_SELECT_DR = 2
    STATE_CAPTURE_DR = 3
    STATE_SHIFT_DR = 4
    STATE_EXIT1_DR = 5
    STATE_PAUSE_DR = 6
    STATE_EXIT2_DR = 7
    STATE_UPDATE_DR = 8
    STATE_SELECT_IR = 9
    STATE_CAPTURE_IR = 10
    STATE_SHIFT_IR = 11
    STATE_EXIT1_IR = 12
    STATE_PAUSE_IR = 13
    STATE_EXIT2_IR = 14
    STATE_UPDATE_IR = 15

    STATE_NAMES = {0:'TLR',
                   1:'IDLE',
                   2:'SELECT_DR',
                   3:'CAPTURE_DR',
                   4:'SHIFT_DR',
                   5:'EXIT1_DR',
                   6:'PAUSE_DR',
                   7:'EXIT2_DR',
                   8:'UPDATE_DR' ,
                   9:'SELECT_IR' ,
                   10:'CAPTURE_IR',
                   11:'SHIFT_IR',
                   12:'EXIT1_IR',
                   13:'PAUSE_IR',
                   14:'EXIT2_IR',
                   15:'UPDATE_IR'}

    def __init__(self):
        self._state = self.STATE_TLR

    def transition(self, bit):
        if self._state == self.STATE_TLR:
            if not bit: self._state = self.STATE_IDLE
        elif self._state == self.STATE_IDLE:
            if bit:     self._state = self.STATE_SELECT_DR
        elif self._state == self.STATE_SELECT_DR:
            if bit:     self._state = self.STATE_SELECT_IR
            else:       self._state = self.STATE_CAPTURE_DR
        elif self._state == self.STATE_CAPTURE_DR:
            if bit:     self._state = self.STATE_EXIT1_DR
            else:       self._state = self.STATE_SHIFT_DR
        elif self._state == self.STATE_SHIFT_DR:
            if bit:     self._state = self.STATE_EXIT1_DR
        elif self._state == self.STATE_EXIT1_DR:
            if bit:     self._state = self.STATE_UPDATE_DR
            else:       self._state = self.STATE_PAUSE_DR
        elif self._state == self.STATE_PAUSE_DR:
            if bit:     self._state = self.STATE_EXIT2_DR
        elif self._state == self.STATE_EXIT2_DR:
            if bit:     self._state = self.STATE_UPDATE_DR
            else:       self._state = self.STATE_SHIFT_DR
        elif self._state == self.STATE_UPDATE_DR:
            if bit:     self._state = self.STATE_SELECT_DR
            else:       self._state = self.STATE_IDLE
        elif self._state == self.STATE_SELECT_IR:
            if bit:     self._state = self.STATE_TLR
            else:       self._state = self.STATE_CAPTURE_IR
        elif self._state == self.STATE_CAPTURE_IR:
            if bit:     self._state = self.STATE_EXIT1_IR
            else:       self._state = self.STATE_SHIFT_IR
        elif self._state == self.STATE_SHIFT_IR:
            if bit:     self._state = self.STATE_EXIT1_IR
        elif self._state == self.STATE_EXIT1_IR:
            if bit:     self._state = self.STATE_UPDATE_IR
            else:       self._state = self.STATE_PAUSE_IR
        elif self._state == self.STATE_PAUSE_IR:
            if bit:     self._state = self.STATE_EXIT2_IR
        elif self._state == self.STATE_EXIT2_IR:
            if bit:     self._state = self.UPDATE_IR
            else:       self._state = self.SHIFT_IR
        elif self._state == self.STATE_UPDATE_IR:
            if bit:     self._state = self.STATE_SELECT_DR
            else:       self._state = self.STATE_IDLE
        #print "STATE:", self.state

    @property
    def state(self):
        return self.STATE_NAMES.get(self._state, "UNKNOWN STATE!")

class JTAGDevice(object):
    def __init__(self, chain, idcode):
        #print "IDCODE", idcode.__repr__()
        if not isinstance(chain, JTAGScanChain):
            raise ValueError("JTAGDevice requires an instnace of JTAGScanChain")
        self._chain = chain

        if isinstance(idcode, int):
            self._id = idcode
        elif isinstance(idcode, str):
            if len(idcode)==4:
                self._id = (ord(idcode[0])<<24)|(ord(idcode[1])<<16)|\
                    (ord(idcode[2])<<8)|ord(idcode[3])
            else:
                raise ValueError("JTAGDevice idcode parameter must be an "
                                 "int or string of length 4. (%s)"%self._id)
        else:
            raise ValueError("JTAGDevice idcode parameter must be an int or "
                             "string of length 4. (Invalid Type: %s)"%type(idcode))

        self.desc = JTAGDeviceDescription.get_descriptor_for_idcode(self._id)

    def erase(self):
        #import ipdb
        con = self._chain._controller
        self._chain._jtagEnable()
        #print self.desc._instructions

        cmd_isce = self.desc._instructions['ISC_ENABLE']
        cmd_iscd = self.desc._instructions['ISC_DISABLE']
        cmd_iscerase = self.desc._instructions['ISC_ERASE']
        print cmd_isce, cmd_iscd, cmd_iscerase

        #ipdb.set_trace()
        con.writeTMSBits('\x00\xDF', 10) #Get to ShiftIR
        pstatus(con.writeTDIBits('\x68', 7, return_tdo=True))
        con.writeTDIBits('\x01', 1, TMS=True) #ISC_ENABLE
        con.writeTMSBits('\x01', 8) #RUN
        time.sleep(0.01)

        #ipdb.set_trace()
        con.writeTMSBits('\x03', 4) #Get to ShiftIR
        pstatus(con.writeTDIBits('\x6D', 7, return_tdo=True))
        con.writeTDIBits('\x01', 1, TMS=True) #ISC_ERASE
        con.writeTMSBits('\x01', 8) #RUN
        time.sleep(0.01)

        #ipdb.set_trace()
        con.writeTMSBits('\x03', 4) #Get to ShiftIR
        pstatus(con.writeTDIBits('\xF0', 7, return_tdo=True))
        con.writeTDIBits('\x01', 1, TMS=True) #ISC_DISCHARGE
        con.writeTMSBits('\x01', 8) #RUN
        time.sleep(0.01)

        #ipdb.set_trace()
        con.writeTMSBits('\x03', 4) #Get to ShiftIR
        pstatus(con.writeTDIBits('\xF0', 7, return_tdo=True))
        con.writeTDIBits('\x01', 1, TMS=True) #ISC_INIT
        con.writeTMSBits('\x1b', 5) #init pulse
        con.writeTMSBits('\x00', 8) #RUN
        time.sleep(0.01)

        #ipdb.set_trace()
        con.writeTMSBits('\x03', 4) #Get to ShiftIR
        pstatus(con.writeTDIBits('\x40', 7, return_tdo=True))
        con.writeTDIBits('\x01', 1, TMS=True) #ISC_DISABLE
        con.writeTMSBits('\x01', 8) #RUN
        time.sleep(0.01)

        #ipdb.set_trace()
        con.writeTMSBits('\x03', 4) #Get to ShiftIR
        pstatus(con.writeTDIBits('\xff', 7, return_tdo=True))
        con.writeTDIBits('\x01', 1, TMS=True) #BYPASS
        con.writeTMSBits('\xFF', 8) #RUN
        time.sleep(0.01)

        #time.sleep(2)
        self._chain._jtagDisable()

    def program(self, data_array):
        import ipdb
        con = self._chain._controller
        self._chain._jtagEnable()

        #ipdb.set_trace()
        con.writeTMSBits('\x00\xDF', 10) #Get to ShiftIR
        pstatus(con.writeTDIBits('\x68', 7, return_tdo=True))
        con.writeTDIBits('\x01', 1, TMS=True) #ISC_ENABLE
        con.writeTMSBits('\x01', 8) #RUN
        time.sleep(0.01)

        for i,r in enumerate(data_array):
            addr = bitarray(bin(gc(i))[2:].zfill(7))
            addr.reverse()

            #print "\nLOADING(%s:%s) ="%(i,addr), r
            con.writeTMSBits('\x03', 4) #Get to ShiftIR
            pstatus(con.writeTDIBits('\x6A', 7, return_tdo=True))#first ins bits
            con.writeTDIBits('\x01', 1, TMS=True) #ISC_PROGRAM
            con.writeTMSBits('\x03', 4) #Get to ShiftDR without running

            buffered_r = build_byte_align_buff(len(r)) + r
            buffered_addr = build_byte_align_buff(len(addr)) + addr

            con.writeTDIBits(buffered_r.tobytes(), len(r), return_tdo=True).__repr__()
            con.writeTDIBits(buffered_addr.tobytes(), len(addr)-1, return_tdo=True) #ISC_PROGRAM
            con.writeTDIBits(chr(addr[0]), 1, TMS=True, return_tdo=True) #ISC_PROGRAM

            con.writeTMSBits('\x01', 8) #RUN
            time.sleep(0.01)

        #ipdb.set_trace()
        con.writeTMSBits('\x03', 4) #Get to ShiftIR
        pstatus(con.writeTDIBits('\xF0', 7, return_tdo=True))
        con.writeTDIBits('\x01', 1, TMS=True) #ISC_DISCHARGE
        con.writeTMSBits('\x01', 8) #RUN
        time.sleep(0.01)

        #ipdb.set_trace()
        con.writeTMSBits('\x03', 4) #Get to ShiftIR
        pstatus(con.writeTDIBits('\xF0', 7, return_tdo=True))
        con.writeTDIBits('\x01', 1, TMS=True) #ISC_INIT
        con.writeTMSBits('\x1b', 5) #init pulse
        con.writeTMSBits('\x00', 8) #RUN
        time.sleep(0.01)

        #ipdb.set_trace()
        con.writeTMSBits('\x03', 4) #Get to ShiftIR
        con.writeTDIBits('\x40', 7, return_tdo=True).__repr__()
        con.writeTDIBits('\x01', 1, TMS=True) #ISC_DISABLE
        con.writeTMSBits('\x01', 8) #RUN
        time.sleep(0.01)

        #ipdb.set_trace()
        con.writeTMSBits('\x03', 4) #Get to ShiftIR
        pstatus(con.writeTDIBits('\xff', 7, return_tdo=True))
        con.writeTDIBits('\x01', 1, TMS=True) #BYPASS
        con.writeTMSBits('\xFF', 8) #RUN
        time.sleep(0.01)

        self._chain._jtagDisable()

class JTAGScanChain(object):
    def __init__(self, controller):
        self._controller = controller
        self._controller._scanchain = self #This might necessitate a factory
        self._sm = JTAGStateMachine()
        self._hasinit = False
        self._devices = []

    def init_chain(self):
        if not self._hasinit:
            #Set state to SHIFT_DR through TLR
            self._controller.jtagEnable()
            self._controller.writeTMSBits('\x00\x5F', 9)

            self._devices = []
            idcode_str = self._controller.readTDOBits(32)
            while idcode_str != '\x00\x00\x00\x00' and\
                    idcode_str != '\xff\xff\xff\xff':
                Jdev = JTAGDevice(self, idcode_str)
                self._devices.append(Jdev)
                idcode_str = self._controller.readTDOBits(32)
            self._controller.jtagDisable()

            #The chain comes out last first. Reverse it to get order.
            self._devices.reverse()

    def _jtagDisable(self):
        self._controller.jtagDisable()

    def _jtagEnable(self):
        self._controller.jtagEnable()

    def _tapTransition(self, bits):
        #print "Bits:", bits
        statetrans = [self._sm.state]
        for bit in bits[::-1]:
            #print "Transitioning TMS->%s"%bit
            self._sm.transition(bit)
            #if statetrans[-1]!=self._sm.state:
            statetrans.append(self._sm.state)
        #print "TAP:",self._sm.state
        #print "TAP State Change:", "->".join(statetrans)

if __name__ == "__main__":
    from digilentdriver import JTAGController
    controllers = JTAGController.getAttachedControllers()
    print "USB Controllers:"
    for i, c in enumerate(controllers):
        print "  %d %s"%(i, c)
        chain = JTAGScanChain(c)
        chain.init_chain()

<|endoftext|>


<commit_before>import hashlib
import os
import random
import shutil

import errno
import fcntl
# import data_operations.data_generators
import sys

from utils import shell_utils

sys.path.append('/qa/dynamo')
from config import error_codes

__author__ = "samuels"

MAX_DIR_SIZE = 128 * 1024
INLINE = INLINE_MAX_SIZE = 3499
KB1 = 1024
KB4 = KB1 * 4
MB1 = (1024 * 1024)
GB1 = (1024 * 1024 * 1024)
TB1 = (1024 * 1024 * 1024 * 1024)
MB512 = (MB1 * 512)  # level 1 can map up to 512MB
GB256 = (GB1 * 256)  # level 2 can map up to 256GB
GB512 = (GB1 * 512)  # level 2 can map up to 256GB
TB128 = (TB1 * 128)  # level 3 can map up to 128TB
ZERO_PADDING_START = 128 * MB1  # 128MB
MAX_FILE_SIZE = TB1 + ZERO_PADDING_START
DATA_PATTERN_A = {'pattern': 'A', 'repeats': 1}
DATA_PATTERN_B = {'pattern': 'B', 'repeats': 3}
DATA_PATTERN_C = {'pattern': 'C', 'repeats': 17}
DATA_PATTERN_D = {'pattern': 'D', 'repeats': 33}
DATA_PATTERN_E = {'pattern': 'E', 'repeats': 65}
DATA_PATTERN_F = {'pattern': 'F', 'repeats': 129}
DATA_PATTERN_G = {'pattern': 'G', 'repeats': 257}
DATA_PATTERN_H = {'pattern': 'H', 'repeats': 1025}
DATA_PATTERN_I = {'pattern': 'I', 'repeats': 128 * KB1 + 1}
DATA_PATTERN_J = {'pattern': 'J', 'repeats': 64 * KB1 + 1}

PADDING = [0, ZERO_PADDING_START]
OFFSETS_LIST = [0, INLINE, KB1, KB4, MB1, MB512, GB1, GB256, GB512, TB1]
DATA_PATTERNS_LIST = [DATA_PATTERN_A, DATA_PATTERN_B, DATA_PATTERN_C, DATA_PATTERN_D, DATA_PATTERN_E, DATA_PATTERN_F,
                      DATA_PATTERN_G, DATA_PATTERN_H, DATA_PATTERN_I, DATA_PATTERN_J]

class DynamoException(EnvironmentError):
    pass

class DataPatterns:
    def __init__(self):
        self.data_patterns_dict = {}

        for _ in range(1000):
            pass

def response_action(action, mount_point, incoming_data, **kwargs):
    return {
        "mkdir": mkdir,
        "list": list_dir,
        "delete": delete,
        "touch": touch,
        "stat": stat,
        "read": read,
        "write": write,
        "rename": rename,
        "rename_exist": rename_exist,
        "truncate": truncate
    }[action](mount_point, incoming_data, **kwargs)

def mkdir(mount_point, incoming_data, **kwargs):
    outgoing_data = {}
    os.mkdir("{0}/{1}".format(mount_point, incoming_data['target']))
    outgoing_data['dirsize'] = os.stat("{0}/{1}".format(mount_point, incoming_data['target'])).st_size
    return outgoing_data

def list_dir(mount_point, incoming_data, **kwargs):
    os.listdir('{0}/{1}'.format(mount_point, incoming_data['target']))

def delete(mount_point, incoming_data, **kwargs):
    outgoing_data = {}
    dirpath = incoming_data['target'].split('/')[1]
    fname = incoming_data['target'].split('/')[2]
    os.remove('{0}/{1}/{2}'.format(mount_point, dirpath, fname))
    outgoing_data['uuid'] = incoming_data['uuid']

def touch(mount_point, incoming_data, **kwargs):
    outgoing_data = {}
    dirsize = os.stat("{0}/{1}".format(mount_point, incoming_data['target'].split('/')[1])).st_size
    if dirsize > MAX_DIR_SIZE:  # if Directory entry size > 128K, we'll stop writing new files
        outgoing_data['target_path'] = incoming_data['target']
        raise DynamoException(error_codes.MAX_DIR_SIZE, "Directory Entry reached {0} size limit".format(MAX_DIR_SIZE),
                              incoming_data['target'])
    # File will be only created if not exists otherwise EEXIST error returned
    flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY
    fd = os.open('{0}{1}'.format(mount_point, incoming_data['target']), flags)
    os.fsync(fd)
    os.close(fd)
    outgoing_data['dirsize'] = os.stat("{0}/{1}".format(mount_point, incoming_data['target'].split('/')[1])).st_size
    # outgoing_data['uuid'] = incoming_data['uuid']
    return outgoing_data

def stat(mount_point, incoming_data, **kwargs):
    outgoing_data = {}
    os.stat("{0}{1}".format(mount_point, incoming_data['target']))
    outgoing_data['uuid'] = incoming_data['uuid']
    return outgoing_data

def read(mount_point, incoming_data, **kwargs):
    outgoing_data = {}
    with open("{0}{1}".format(mount_point, incoming_data['target']), 'r') as f:
        f.seek(incoming_data['offset'])
        buf = f.read(incoming_data['repeats'])
        hasher = hashlib.md5()
        hasher.update(buf)
        outgoing_data['hash'] = hasher.hexdigest()
        outgoing_data['offset'] = incoming_data['offset']
        outgoing_data['chunk_size'] = incoming_data['repeats']
        outgoing_data['uuid'] = incoming_data['uuid']
        return outgoing_data

def write(mount_point, incoming_data, **kwargs):
    outgoing_data = {}
    hasher = hashlib.md5()
    fp = None
    if incoming_data['io_type'] == 'sequential':
        offset = incoming_data['offset'] + incoming_data['data_pattern_len']
    else:
        padding = random.choice(PADDING)
        base_offset = random.choice(OFFSETS_LIST) + padding
        offset = base_offset + random.randint(base_offset, MAX_FILE_SIZE)
    data_pattern = random.choice(DATA_PATTERNS_LIST)
    pattern_to_write = data_pattern['pattern'] * data_pattern['repeats']
    hasher.update(pattern_to_write)
    data_hash = hasher.hexdigest()
    try:
        fp = open("{0}{1}".format(mount_point, incoming_data['target']), 'r+')
        fcntl.lockf(fp.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB, data_pattern['repeats'], offset, 0)
        fp.seek(offset)
        fp.write(pattern_to_write)
        fp.flush()
        os.fsync(fp.fileno())
        #  Checking if original data pattern and pattern on disk are the same
        fp.seek(offset)
        buf = fp.read(data_pattern['repeats'])
        hasher = hashlib.md5()
        hasher.update(buf)
        read_hash = hasher.hexdigest()
        if read_hash != data_hash:
            outgoing_data['dynamo_error'] = error_codes.HASHERR
            outgoing_data['bad_hash'] = read_hash
        fcntl.lockf(fp.fileno(), fcntl.LOCK_UN)
        fp.close()
    except (IOError, OSError) as env_error:
        fcntl.lockf(fp.fileno(), fcntl.LOCK_UN)
        fp.close()
        raise env_error
    outgoing_data['data_pattern'] = data_pattern['pattern']
    outgoing_data['chunk_size'] = data_pattern['repeats']
    outgoing_data['hash'] = data_hash
    outgoing_data['offset'] = offset
    outgoing_data['uuid'] = incoming_data['uuid']
    outgoing_data['io_type'] = incoming_data['io_type']
    return outgoing_data

def rename(mount_point, incoming_data, **kwargs):
    outgoing_data = {}
    fullpath = incoming_data['target'].split('/')[1:]
    dirpath = fullpath[0]
    fname = fullpath[1]
    dst_mount_point = kwargs['dst_mount_point']
    outgoing_data['rename_dest'] = incoming_data['rename_dest']
    shutil.move("{0}/{1}/{2}".format(mount_point, dirpath, fname),
                "{0}/{1}/{2}".format(dst_mount_point, dirpath, incoming_data['rename_dest']))
    outgoing_data['uuid'] = incoming_data['uuid']
    return outgoing_data

def rename_exist(mount_point, incoming_data, **kwargs):
    outgoing_data = {}
    src_path = incoming_data['rename_source']
    dst_path = incoming_data['rename_dest']
    src_dirpath = src_path.split('/')[1]
    src_fname = src_path.split('/')[2]
    dst_dirpath = dst_path.split('/')[1]
    dst_fname = dst_path.split('/')[2]
    if src_fname == dst_fname:
        raise DynamoException(error_codes.SAMEFILE, "Error: Trying to move file into itself.", src_path)
    dst_mount_point = kwargs['dst_mount_point']
    shutil.move("{0}/{1}/{2}".format(mount_point, src_dirpath, src_fname),
                "{0}/{1}/{2}".format(dst_mount_point, dst_dirpath, dst_fname))
    outgoing_data['rename_source'] = src_path
    outgoing_data['rename_dest'] = dst_path
    outgoing_data['uuid'] = incoming_data['uuid']
    return outgoing_data

def truncate(mount_point, incoming_data, **kwargs):
    outgoing_data = {}
    padding = random.choice(PADDING)
    offset = random.choice(OFFSETS_LIST) + padding
    fp = None
    try:
        fp = open("{0}{1}".format(mount_point, incoming_data['target']), 'r+')
        fcntl.lockf(fp.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
        fp.truncate(offset)
        fp.flush()
        os.fsync(fp.fileno())
        fcntl.lockf(fp.fileno(), fcntl.LOCK_UN)
        fp.close()
    except (IOError, OSError) as env_error:
        fcntl.lockf(fp.fileno(), fcntl.LOCK_UN)
        fp.close()
        raise env_error
    outgoing_data['size'] = offset
    outgoing_data['uuid'] = incoming_data['uuid']
    return outgoing_data
<commit_msg>Fix: unlock file in case of exception  in write and truncate methods<commit_after>import hashlib
import os
import random
import shutil

import errno
import fcntl
# import data_operations.data_generators
import sys

from utils import shell_utils

sys.path.append('/qa/dynamo')
from config import error_codes

__author__ = "samuels"

MAX_DIR_SIZE = 128 * 1024
INLINE = INLINE_MAX_SIZE = 3499
KB1 = 1024
KB4 = KB1 * 4
MB1 = (1024 * 1024)
GB1 = (1024 * 1024 * 1024)
TB1 = (1024 * 1024 * 1024 * 1024)
MB512 = (MB1 * 512)  # level 1 can map up to 512MB
GB256 = (GB1 * 256)  # level 2 can map up to 256GB
GB512 = (GB1 * 512)  # level 2 can map up to 256GB
TB128 = (TB1 * 128)  # level 3 can map up to 128TB
ZERO_PADDING_START = 128 * MB1  # 128MB
MAX_FILE_SIZE = TB1 + ZERO_PADDING_START
DATA_PATTERN_A = {'pattern': 'A', 'repeats': 1}
DATA_PATTERN_B = {'pattern': 'B', 'repeats': 3}
DATA_PATTERN_C = {'pattern': 'C', 'repeats': 17}
DATA_PATTERN_D = {'pattern': 'D', 'repeats': 33}
DATA_PATTERN_E = {'pattern': 'E', 'repeats': 65}
DATA_PATTERN_F = {'pattern': 'F', 'repeats': 129}
DATA_PATTERN_G = {'pattern': 'G', 'repeats': 257}
DATA_PATTERN_H = {'pattern': 'H', 'repeats': 1025}
DATA_PATTERN_I = {'pattern': 'I', 'repeats': 128 * KB1 + 1}
DATA_PATTERN_J = {'pattern': 'J', 'repeats': 64 * KB1 + 1}

PADDING = [0, ZERO_PADDING_START]
OFFSETS_LIST = [0, INLINE, KB1, KB4, MB1, MB512, GB1, GB256, GB512, TB1]
DATA_PATTERNS_LIST = [DATA_PATTERN_A, DATA_PATTERN_B, DATA_PATTERN_C, DATA_PATTERN_D, DATA_PATTERN_E, DATA_PATTERN_F,
                      DATA_PATTERN_G, DATA_PATTERN_H, DATA_PATTERN_I, DATA_PATTERN_J]

class DynamoException(EnvironmentError):
    pass

class DataPatterns:
    def __init__(self):
        self.data_patterns_dict = {}

        for _ in range(1000):
            pass

def response_action(action, mount_point, incoming_data, **kwargs):
    return {
        "mkdir": mkdir,
        "list": list_dir,
        "delete": delete,
        "touch": touch,
        "stat": stat,
        "read": read,
        "write": write,
        "rename": rename,
        "rename_exist": rename_exist,
        "truncate": truncate
    }[action](mount_point, incoming_data, **kwargs)

def mkdir(mount_point, incoming_data, **kwargs):
    outgoing_data = {}
    os.mkdir("{0}/{1}".format(mount_point, incoming_data['target']))
    outgoing_data['dirsize'] = os.stat("{0}/{1}".format(mount_point, incoming_data['target'])).st_size
    return outgoing_data

def list_dir(mount_point, incoming_data, **kwargs):
    os.listdir('{0}/{1}'.format(mount_point, incoming_data['target']))

def delete(mount_point, incoming_data, **kwargs):
    outgoing_data = {}
    dirpath = incoming_data['target'].split('/')[1]
    fname = incoming_data['target'].split('/')[2]
    os.remove('{0}/{1}/{2}'.format(mount_point, dirpath, fname))
    outgoing_data['uuid'] = incoming_data['uuid']

def touch(mount_point, incoming_data, **kwargs):
    outgoing_data = {}
    dirsize = os.stat("{0}/{1}".format(mount_point, incoming_data['target'].split('/')[1])).st_size
    if dirsize > MAX_DIR_SIZE:  # if Directory entry size > 128K, we'll stop writing new files
        outgoing_data['target_path'] = incoming_data['target']
        raise DynamoException(error_codes.MAX_DIR_SIZE, "Directory Entry reached {0} size limit".format(MAX_DIR_SIZE),
                              incoming_data['target'])
    # File will be only created if not exists otherwise EEXIST error returned
    flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY
    fd = os.open('{0}{1}'.format(mount_point, incoming_data['target']), flags)
    os.fsync(fd)
    os.close(fd)
    outgoing_data['dirsize'] = os.stat("{0}/{1}".format(mount_point, incoming_data['target'].split('/')[1])).st_size
    # outgoing_data['uuid'] = incoming_data['uuid']
    return outgoing_data

def stat(mount_point, incoming_data, **kwargs):
    outgoing_data = {}
    os.stat("{0}{1}".format(mount_point, incoming_data['target']))
    outgoing_data['uuid'] = incoming_data['uuid']
    return outgoing_data

def read(mount_point, incoming_data, **kwargs):
    outgoing_data = {}
    with open("{0}{1}".format(mount_point, incoming_data['target']), 'r') as f:
        f.seek(incoming_data['offset'])
        buf = f.read(incoming_data['repeats'])
        hasher = hashlib.md5()
        hasher.update(buf)
        outgoing_data['hash'] = hasher.hexdigest()
        outgoing_data['offset'] = incoming_data['offset']
        outgoing_data['chunk_size'] = incoming_data['repeats']
        outgoing_data['uuid'] = incoming_data['uuid']
        return outgoing_data

def write(mount_point, incoming_data, **kwargs):
    outgoing_data = {}
    hasher = hashlib.md5()
    fp = None
    if incoming_data['io_type'] == 'sequential':
        offset = incoming_data['offset'] + incoming_data['data_pattern_len']
    else:
        padding = random.choice(PADDING)
        base_offset = random.choice(OFFSETS_LIST) + padding
        offset = base_offset + random.randint(base_offset, MAX_FILE_SIZE)
    data_pattern = random.choice(DATA_PATTERNS_LIST)
    pattern_to_write = data_pattern['pattern'] * data_pattern['repeats']
    hasher.update(pattern_to_write)
    data_hash = hasher.hexdigest()
    try:
        fp = open("{0}{1}".format(mount_point, incoming_data['target']), 'r+')
        fcntl.lockf(fp.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB, data_pattern['repeats'], offset, 0)
        fp.seek(offset)
        fp.write(pattern_to_write)
        fp.flush()
        os.fsync(fp.fileno())
        #  Checking if original data pattern and pattern on disk are the same
        fp.seek(offset)
        buf = fp.read(data_pattern['repeats'])
        hasher = hashlib.md5()
        hasher.update(buf)
        read_hash = hasher.hexdigest()
        if read_hash != data_hash:
            outgoing_data['dynamo_error'] = error_codes.HASHERR
            outgoing_data['bad_hash'] = read_hash
        fcntl.lockf(fp.fileno(), fcntl.LOCK_UN)
        fp.close()
    except (IOError, OSError) as env_error:
        if fp:
            fcntl.lockf(fp.fileno(), fcntl.LOCK_UN)
            fp.close()
        raise env_error
    outgoing_data['data_pattern'] = data_pattern['pattern']
    outgoing_data['chunk_size'] = data_pattern['repeats']
    outgoing_data['hash'] = data_hash
    outgoing_data['offset'] = offset
    outgoing_data['uuid'] = incoming_data['uuid']
    outgoing_data['io_type'] = incoming_data['io_type']
    return outgoing_data

def rename(mount_point, incoming_data, **kwargs):
    outgoing_data = {}
    fullpath = incoming_data['target'].split('/')[1:]
    dirpath = fullpath[0]
    fname = fullpath[1]
    dst_mount_point = kwargs['dst_mount_point']
    outgoing_data['rename_dest'] = incoming_data['rename_dest']
    shutil.move("{0}/{1}/{2}".format(mount_point, dirpath, fname),
                "{0}/{1}/{2}".format(dst_mount_point, dirpath, incoming_data['rename_dest']))
    outgoing_data['uuid'] = incoming_data['uuid']
    return outgoing_data

def rename_exist(mount_point, incoming_data, **kwargs):
    outgoing_data = {}
    src_path = incoming_data['rename_source']
    dst_path = incoming_data['rename_dest']
    src_dirpath = src_path.split('/')[1]
    src_fname = src_path.split('/')[2]
    dst_dirpath = dst_path.split('/')[1]
    dst_fname = dst_path.split('/')[2]
    if src_fname == dst_fname:
        raise DynamoException(error_codes.SAMEFILE, "Error: Trying to move file into itself.", src_path)
    dst_mount_point = kwargs['dst_mount_point']
    shutil.move("{0}/{1}/{2}".format(mount_point, src_dirpath, src_fname),
                "{0}/{1}/{2}".format(dst_mount_point, dst_dirpath, dst_fname))
    outgoing_data['rename_source'] = src_path
    outgoing_data['rename_dest'] = dst_path
    outgoing_data['uuid'] = incoming_data['uuid']
    return outgoing_data

def truncate(mount_point, incoming_data, **kwargs):
    outgoing_data = {}
    padding = random.choice(PADDING)
    offset = random.choice(OFFSETS_LIST) + padding
    fp = None
    try:
        fp = open("{0}{1}".format(mount_point, incoming_data['target']), 'r+')
        fcntl.lockf(fp.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
        fp.truncate(offset)
        fp.flush()
        os.fsync(fp.fileno())
        fcntl.lockf(fp.fileno(), fcntl.LOCK_UN)
        fp.close()
    except (IOError, OSError) as env_error:
        if fp:
            fcntl.lockf(fp.fileno(), fcntl.LOCK_UN)
            fp.close()
        raise env_error
    outgoing_data['size'] = offset
    outgoing_data['uuid'] = incoming_data['uuid']
    return outgoing_data
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Update HTPC-Manager from Github. Either through git command or tarball.

Updater and SourceUpdater written by styxit
https://github.com/styxit

Git updater written by mbw2001
https://github.com/mbw2001

Used as reference:
- https://github.com/mrkipling/maraschino
- https://github.com/midgetspy/Sick-Beard/
"""
import os
from threading import Thread
import urllib2
import subprocess
import re
from json import loads
import cherrypy
import htpc
import logging
import tarfile
import shutil

from htpc.root import do_restart

# configure git repo
gitUser = 'Hellowlol'
gitRepo = 'HTPC-Manager'

class Updater:
    """ Main class """
    def __init__(self):
        self.logger = logging.getLogger('htpc.updater')

        self.updateEngineName = 'Unknown'
        # Set update engine. Use git updater or update from source.
        self.updateEngine = self.getEngine()

    """ Determine the update method """
    def getEngine(self):
        self.logger.debug("Selecting Update engine.")
        gitDir = os.path.join(htpc.RUNDIR, '.git')
        validGitDir = os.path.isdir(gitDir)
        validGitCommand = GitUpdater().git_exec('branch')  # do simple command to test git functionality

        # If valid Git dir and git command succeeded, use Git updater
        if (validGitDir and validGitCommand):
            self.logger.info('Using GitUpdater engine')
            self.updateEngineName = 'Git'
            return GitUpdater()
        else:  # Otherwise update from Sourece
            self.logger.info('Using SourceUpdater engine')
            self.updateEngineName = 'Source'
            return SourceUpdater()

    @cherrypy.expose()
    @cherrypy.tools.json_out()
    def index(self, force=False):
        """ Update on POST. Check for new updates on GET. """
        if cherrypy.request.method.upper() == 'POST':
            Thread(target=self.updateEngine.update).start()
            return 1
        if cherrypy.request.method.upper() == 'POST' and force:
            self.check_update()
            Thread(target=self.updateEngine.update).start()
            return 1
        else:
            return self.check_update()

    @cherrypy.expose()
    @cherrypy.tools.json_out()
    def status(self):
        """ method to determine if HTPC Manager is currently updating """
        return self.updateEngine.UPDATING

    def check_update(self):
        """
        Check for updates

        Returns dict() with the following indexes:

        UpdateNeeded    True if an update is needed, False if an update is not needed OR not possible
        latestVersion   Commit hash of the most recent commit
        currentVersion  Commit hash for the version currently in use
        versionsBehind  How many versions is the current version behind the latest version
        """
        output = {'updateNeeded': True, 'latestVersion': 'Unknown', 'currentVersion': 'Unknown', 'versionsBehind': 'Unknown'}

        self.logger.info("Checking for updates from %s." % self.updateEngineName)

        # Get current and latest version
        current = self.updateEngine.current()
        latest = self.latest()

        if not latest:
            self.logger.error("Failed to determine the latest version for HTPC Manager.")
        else:
            output['latestVersion'] = latest

        if not current:
            self.logger.error("Failed to determine the current version for HTPC Manager.")
        else:
            output['currentVersion'] = current

        # If current or latest failed, updating is not possible
        if not current or not latest:
            self.logger.debug("Cancel update.")
            output['updateNeeded'] = False
            return output

        # If HTPC Manager is up to date, updating is not needed
        if current == latest:
            self.logger.info("HTPC-Manager is Up-To-Date.")
            output['versionsBehind'] = 0
            output['updateNeeded'] = False
        else:
            behind = self.behind_by(current, latest)
            output['versionsBehind'] = behind

        self.logger.info("Currently " + str(output['versionsBehind']) + " commits behind.")
        return output

    def latest(self):
        """ Get hash of latest commit on github """
        self.logger.debug('Getting latest version from github.')
        try:
            url = 'https://api.github.com/repos/%s/%s/commits/%s' % (gitUser, gitRepo, htpc.settings.get('branch', 'master'))
            result = loads(urllib2.urlopen(url).read())
            latest = result['sha'].strip()
            self.logger.debug('Latest version: ' + latest)
            self.updateEngine.latestHash = latest
            return latest
        except:
            return False

    def behind_by(self, current, latest):
        """ Check how many commits between current and latest """
        self.logger.debug('Checking how far behind latest')
        try:
            url = 'https://api.github.com/repos/%s/%s/compare/%s...%s' % (gitUser, gitRepo, current, latest)
            result = loads(urllib2.urlopen(url).read())
            behind = int(result['total_commits'])
            self.logger.debug('Behind: ' + str(behind))
            return behind
        except Exception, e:
            self.logger.error(str(e))
            self.logger.error('Could not determine how far behind')
            return 'Unknown'

    # Unlikely that a user has more the 100 branches, if they do use change this so it uses the link in the header
    @cherrypy.expose()
    @cherrypy.tools.json_out()
    def branches(self):
        """ Returns the all the branches to gitUser """
        d = {"branch": htpc.settings.get('branch', 'master'),
             "branches": []
            }
        try:
            url = "https://api.github.com/repos/%s/%s/branches?per_page=100" % (gitUser, gitRepo)
            branchlist = []
            branches = loads(urllib2.urlopen(url).read())
            for branch in branches:
                branchlist.append(branch["name"])
            d["branches"] = branchlist
            return d

        except Exception, e:
            self.logger.error(str(e))
            self.logger.error('Could not find any branches, setting default master')
            return [d]

""" Class to update HTPC Manager using git commands. """
class GitUpdater():
    """ Main class """
    def __init__(self):
        """ Set GitHub settings on load """
        self.UPDATING = 0

        self.git = htpc.settings.get('git_path', 'git')
        self.logger = logging.getLogger('htpc.updater')

    def current(self):
        """ Get hash of current Git commit """
        self.logger.debug('Getting current version.')
        output = self.git_exec('rev-parse HEAD')
        self.logger.debug('Current version: ' + output)

        if not output:
            self.logger.error('Got no response for current Git version.')
            return False

        if re.match('^[a-z0-9]+$', output):
            return output

    def update(self):
        """ Do update through git """
        self.logger.info("Attempting update through Git.")
        self.UPDATING = 1

        output = self.git_exec('pull origin %s' % htpc.settings.get('branch'))
        if not output:
            self.logger.error("Unable to update through git. Make sure that Git is located in your path and can be accessed by this application.")
        elif 'Aborting.' in output:
            self.logger.error("Update aborted.")
        else:
            # Restart HTPC Manager to make sure all new code is loaded
            self.logger.warning('Restarting HTPC Manager after update.')
            do_restart()

        self.UPDATING = 0

    def git_exec(self, args):
        """ Tool for running git program on system """
        try:
            proc = subprocess.Popen(self.git + " " + args, stdout=subprocess.PIPE,
                   stderr=subprocess.STDOUT, shell=True, cwd=htpc.RUNDIR)
            output, err = proc.communicate()
        except OSError, e:
            self.logger.warning(str(e))
            return ''

        if err:
            self.logger.warning(output + ' - ' + err)
            return ''
        elif any(s in output for s in ['not found', 'not recognized', 'fatal:']):
            self.logger.warning(output)
            return ''
        else:
            return output.strip()

""" Class to update HTPC Manager using Source code from Github. Requires a full download on every update."""
class SourceUpdater():
    """ Main class """
    def __init__(self):
        self.UPDATING = 0

        self.currentHash = False
        self.latestHash = False

        self.logger = logging.getLogger('htpc.updater')

        self.versionFile = os.path.join(htpc.RUNDIR, 'VERSION.txt')
        self.updateFile = os.path.join(htpc.DATADIR, 'htpc-manager-update.tar.gz')
        self.updateDir = os.path.join(htpc.DATADIR, 'update-source')

    """ Get hash of current runnig version """
    def current(self):
        self.logger.debug('Getting current version.')

        """ Check if version file exists """
        if not os.path.isfile(self.versionFile):
            self.logger.warning('Version file does not exists. Creating it now.')
            try:
                versionFileHandler = open(self.versionFile, 'w')
                versionFileHandler.close()
                return 'Unknown'
            except:
                # If version file can not be created updating is also not possible
                self.logger.error('Could not create version file.')
                return False

        """ Get version from version file """
        fp = open(self.versionFile, 'r')
        currentVersion = fp.read().strip(' \n\r')
        fp.close()

        self.logger.debug('Current version: ' + currentVersion)

        if not currentVersion:
            self.logger.error('No commit hash found in version file.')
            return True

        if re.match('^[a-z0-9]+$', currentVersion):
            self.currentHash = currentVersion
            return currentVersion

    """ Do update from source """
    def update(self):
        self.logger.info("Attempting update from source.")

        self.UPDATING = 1
        cherrypy.engine.exit()

        tarUrl = 'https://github.com/%s/%s/tarball/%s' % (gitUser, gitRepo, htpc.settings.get('branch'))

        # Download tar
        downloaded = self.__downloadTar(tarUrl, self.updateFile)
        if (downloaded is False):
            return False

        # Extract to temp folder
        extracted = self.__extractUpdate(self.updateFile, self.updateDir)
        if (extracted is False):
            return False

        # Overwite app source with source from extracted file
        overwritten = self.__updateSourcecode()
        if (overwritten is False):
            return False

        # Write new version to file
        self.__updateVersionFile(self.latestHash)

        # Restart HTPC Manager to make sure all new code is loaded
        self.logger.warning('Restarting HTPC Manager after update.')
        do_restart()

        # Cleanup after yourself
        self.__finishUpdate()

    """ Download source """
    def __downloadTar(self, url, destination):
        # Download tar
        self.logger.info('Downloading update from %s' % url)
        try:
            self.logger.debug('Downloading update file to %s' % destination)
            downloadedFile = urllib2.urlopen(url)
            f = open(destination, 'wb')
            f.write(downloadedFile.read())
            f.close()
            self.logger.info('Downloading update complete')
            return True
        except:
            self.logger.warning('Failed to download update file')
            self.__finishUpdate()
            return False

    """ Extract files from downloaded tar file """
    def __extractUpdate(self, filePath, destinationFolder):
        try:
            self.logger.debug('Extracting tar file: %s' % filePath)
            tarArchive = tarfile.open(filePath)
            tarArchive.extractall(destinationFolder)
            tarArchive.close()
            return True
        except:
            self.logger.error('Failed extracting update file.')
            self.__finishUpdate()
            return False

    """ Overwrite HTPC Manager sourcecode with (new) code from update path """
    def __updateSourcecode(self):
        # Determine the path where the updated should be located
        sourceUpdateFolder = os.path.join(self.updateDir, '%s-%s-%s' % (gitUser, gitRepo, self.latestHash[:7]))

        # Where to extract the update
        targetFolder = os.path.join(htpc.RUNDIR)

        self.logger.debug('Overwriting files.')
        try:
            # Loop files and folders and place them in the HTPC Manager path
            for src_dir, dirs, files in os.walk(sourceUpdateFolder):
                dst_dir = src_dir.replace(sourceUpdateFolder, targetFolder)
                if not os.path.exists(dst_dir):
                    os.mkdir(dst_dir)
                for file_ in files:
                    src_file = os.path.join(src_dir, file_)
                    dst_file = os.path.join(dst_dir, file_)
                    if os.path.exists(dst_file):
                        os.remove(dst_file)
                    shutil.move(src_file, dst_dir)
        except:
            self.logger.warning('Failed to overwrite old files')
            self.__finishUpdate()
            return False

        self.logger.info('updating files successfull')
        return True

    """
    Write the latest commit hash to th version file.

    Used when checking for update the next time.
    """
    def __updateVersionFile(self, newVersion):
        versionFileHandler = open(self.versionFile, 'wb')
        versionFileHandler.write(newVersion)
        versionFileHandler.close()

    """ Remove leftover files after the update """
    def __finishUpdate(self):
        self.UPDATING = 0

        if os.path.isfile(self.updateFile):
            self.logger.debug('Removing update archive')
            try:
                os.remove(self.updateFile)
            except:
                pass

        if os.path.isdir(self.updateDir):
            self.logger.debug('Removing update code folder')
            try:
                shutil.rmtree(self.updateDir)
            except:
                pass
<commit_msg>add git reset --hard after pull<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Update HTPC-Manager from Github. Either through git command or tarball.

Updater and SourceUpdater written by styxit
https://github.com/styxit

Git updater written by mbw2001
https://github.com/mbw2001

Used as reference:
- https://github.com/mrkipling/maraschino
- https://github.com/midgetspy/Sick-Beard/
"""
import os
from threading import Thread
import urllib2
import subprocess
import re
from json import loads
import cherrypy
import htpc
import logging
import tarfile
import shutil

from htpc.root import do_restart

# configure git repo
gitUser = 'Hellowlol'
gitRepo = 'HTPC-Manager'

class Updater:
    """ Main class """
    def __init__(self):
        self.logger = logging.getLogger('htpc.updater')

        self.updateEngineName = 'Unknown'
        # Set update engine. Use git updater or update from source.
        self.updateEngine = self.getEngine()

    """ Determine the update method """
    def getEngine(self):
        self.logger.debug("Selecting Update engine.")
        gitDir = os.path.join(htpc.RUNDIR, '.git')
        validGitDir = os.path.isdir(gitDir)
        validGitCommand = GitUpdater().git_exec('branch')  # do simple command to test git functionality

        # If valid Git dir and git command succeeded, use Git updater
        if (validGitDir and validGitCommand):
            self.logger.info('Using GitUpdater engine')
            self.updateEngineName = 'Git'
            return GitUpdater()
        else:  # Otherwise update from Sourece
            self.logger.info('Using SourceUpdater engine')
            self.updateEngineName = 'Source'
            return SourceUpdater()

    @cherrypy.expose()
    @cherrypy.tools.json_out()
    def index(self, force=False):
        """ Update on POST. Check for new updates on GET. """
        if cherrypy.request.method.upper() == 'POST':
            Thread(target=self.updateEngine.update).start()
            return 1
        if cherrypy.request.method.upper() == 'POST' and force:
            self.check_update()
            Thread(target=self.updateEngine.update).start()
            return 1
        else:
            return self.check_update()

    @cherrypy.expose()
    @cherrypy.tools.json_out()
    def status(self):
        """ method to determine if HTPC Manager is currently updating """
        return self.updateEngine.UPDATING

    def check_update(self):
        """
        Check for updates

        Returns dict() with the following indexes:

        UpdateNeeded    True if an update is needed, False if an update is not needed OR not possible
        latestVersion   Commit hash of the most recent commit
        currentVersion  Commit hash for the version currently in use
        versionsBehind  How many versions is the current version behind the latest version
        """
        output = {'updateNeeded': True, 'latestVersion': 'Unknown', 'currentVersion': 'Unknown', 'versionsBehind': 'Unknown'}

        self.logger.info("Checking for updates from %s." % self.updateEngineName)

        # Get current and latest version
        current = self.updateEngine.current()
        latest = self.latest()

        if not latest:
            self.logger.error("Failed to determine the latest version for HTPC Manager.")
        else:
            output['latestVersion'] = latest

        if not current:
            self.logger.error("Failed to determine the current version for HTPC Manager.")
        else:
            output['currentVersion'] = current

        # If current or latest failed, updating is not possible
        if not current or not latest:
            self.logger.debug("Cancel update.")
            output['updateNeeded'] = False
            return output

        # If HTPC Manager is up to date, updating is not needed
        if current == latest:
            self.logger.info("HTPC-Manager is Up-To-Date.")
            output['versionsBehind'] = 0
            output['updateNeeded'] = False
        else:
            behind = self.behind_by(current, latest)
            output['versionsBehind'] = behind

        self.logger.info("Currently " + str(output['versionsBehind']) + " commits behind.")
        return output

    def latest(self):
        """ Get hash of latest commit on github """
        self.logger.debug('Getting latest version from github.')
        try:
            url = 'https://api.github.com/repos/%s/%s/commits/%s' % (gitUser, gitRepo, htpc.settings.get('branch', 'master2'))
            result = loads(urllib2.urlopen(url).read())
            latest = result['sha'].strip()
            self.logger.debug('Latest version: ' + latest)
            self.updateEngine.latestHash = latest
            return latest
        except:
            return False

    def behind_by(self, current, latest):
        """ Check how many commits between current and latest """
        self.logger.debug('Checking how far behind latest')
        try:
            url = 'https://api.github.com/repos/%s/%s/compare/%s...%s' % (gitUser, gitRepo, current, latest)
            result = loads(urllib2.urlopen(url).read())
            behind = int(result['total_commits'])
            self.logger.debug('Behind: ' + str(behind))
            return behind
        except Exception, e:
            self.logger.error(str(e))
            self.logger.error('Could not determine how far behind')
            return 'Unknown'

    # Unlikely that a user has more the 100 branches, if they do use change this so it uses the link in the header
    @cherrypy.expose()
    @cherrypy.tools.json_out()
    def branches(self):
        """ Returns the all the branches to gitUser """
        d = {"branch": htpc.settings.get('branch', 'master'),
             "branches": []
            }
        try:
            url = "https://api.github.com/repos/%s/%s/branches?per_page=100" % (gitUser, gitRepo)
            branchlist = []
            branches = loads(urllib2.urlopen(url).read())
            for branch in branches:
                branchlist.append(branch["name"])
            d["branches"] = branchlist
            return d

        except Exception, e:
            self.logger.error(str(e))
            self.logger.error('Could not find any branches, setting default master')
            return [d]

""" Class to update HTPC Manager using git commands. """
class GitUpdater():
    """ Main class """
    def __init__(self):
        """ Set GitHub settings on load """
        self.UPDATING = 0

        self.git = htpc.settings.get('git_path', 'git')
        self.logger = logging.getLogger('htpc.updater')

    def current(self):
        """ Get hash of current Git commit """
        self.logger.debug('Getting current version.')
        output = self.git_exec('rev-parse HEAD')
        self.logger.debug('Current version: ' + output)

        if not output:
            self.logger.error('Got no response for current Git version.')
            return False

        if re.match('^[a-z0-9]+$', output):
            return output

    def update(self):
        """ Do update through git """
        self.logger.info("Attempting update through Git.")
        self.UPDATING = 1

        output = self.git_exec('pull origin %s' % htpc.settings.get('branch'))
        if not output:
            self.logger.error("Unable to update through git. Make sure that Git is located in your path and can be accessed by this application.")
        elif 'Aborting.' in output:
            self.logger.error("Update aborted.")
        else:
            # Restart HTPC Manager to make sure all new code is loaded
            self.logger.debug("Clean up after git")
            self.git_exec('reset --hard')
            self.logger.warning('Restarting HTPC Manager after update.')

            do_restart()

        self.UPDATING = 0

    def git_exec(self, args):
        """ Tool for running git program on system """
        try:
            proc = subprocess.Popen(self.git + " " + args, stdout=subprocess.PIPE,
                   stderr=subprocess.STDOUT, shell=True, cwd=htpc.RUNDIR)
            output, err = proc.communicate()
        except OSError, e:
            self.logger.warning(str(e))
            return ''

        if err:
            self.logger.warning(output + ' - ' + err)
            return ''
        elif any(s in output for s in ['not found', 'not recognized', 'fatal:']):
            self.logger.warning(output)
            return ''
        else:
            return output.strip()

""" Class to update HTPC Manager using Source code from Github. Requires a full download on every update."""
class SourceUpdater():
    """ Main class """
    def __init__(self):
        self.UPDATING = 0

        self.currentHash = False
        self.latestHash = False

        self.logger = logging.getLogger('htpc.updater')

        self.versionFile = os.path.join(htpc.RUNDIR, 'VERSION.txt')
        self.updateFile = os.path.join(htpc.DATADIR, 'htpc-manager-update.tar.gz')
        self.updateDir = os.path.join(htpc.DATADIR, 'update-source')

    """ Get hash of current runnig version """
    def current(self):
        self.logger.debug('Getting current version.')

        """ Check if version file exists """
        if not os.path.isfile(self.versionFile):
            self.logger.warning('Version file does not exists. Creating it now.')
            try:
                versionFileHandler = open(self.versionFile, 'w')
                versionFileHandler.close()
                return 'Unknown'
            except:
                # If version file can not be created updating is also not possible
                self.logger.error('Could not create version file.')
                return False

        """ Get version from version file """
        fp = open(self.versionFile, 'r')
        currentVersion = fp.read().strip(' \n\r')
        fp.close()

        self.logger.debug('Current version: ' + currentVersion)

        if not currentVersion:
            self.logger.error('No commit hash found in version file.')
            return True

        if re.match('^[a-z0-9]+$', currentVersion):
            self.currentHash = currentVersion
            return currentVersion

    """ Do update from source """
    def update(self):
        self.logger.info("Attempting update from source.")

        self.UPDATING = 1
        cherrypy.engine.exit()

        tarUrl = 'https://github.com/%s/%s/tarball/%s' % (gitUser, gitRepo, htpc.settings.get('branch'))

        # Download tar
        downloaded = self.__downloadTar(tarUrl, self.updateFile)
        if (downloaded is False):
            return False

        # Extract to temp folder
        extracted = self.__extractUpdate(self.updateFile, self.updateDir)
        if (extracted is False):
            return False

        # Overwite app source with source from extracted file
        overwritten = self.__updateSourcecode()
        if (overwritten is False):
            return False

        # Write new version to file
        self.__updateVersionFile(self.latestHash)

        # Restart HTPC Manager to make sure all new code is loaded
        self.logger.warning('Restarting HTPC Manager after update.')
        do_restart()

        # Cleanup after yourself
        self.__finishUpdate()

    """ Download source """
    def __downloadTar(self, url, destination):
        # Download tar
        self.logger.info('Downloading update from %s' % url)
        try:
            self.logger.debug('Downloading update file to %s' % destination)
            downloadedFile = urllib2.urlopen(url)
            f = open(destination, 'wb')
            f.write(downloadedFile.read())
            f.close()
            self.logger.info('Downloading update complete')
            return True
        except:
            self.logger.warning('Failed to download update file')
            self.__finishUpdate()
            return False

    """ Extract files from downloaded tar file """
    def __extractUpdate(self, filePath, destinationFolder):
        try:
            self.logger.debug('Extracting tar file: %s' % filePath)
            tarArchive = tarfile.open(filePath)
            tarArchive.extractall(destinationFolder)
            tarArchive.close()
            return True
        except:
            self.logger.error('Failed extracting update file.')
            self.__finishUpdate()
            return False

    """ Overwrite HTPC Manager sourcecode with (new) code from update path """
    def __updateSourcecode(self):
        # Determine the path where the updated should be located
        sourceUpdateFolder = os.path.join(self.updateDir, '%s-%s-%s' % (gitUser, gitRepo, self.latestHash[:7]))

        # Where to extract the update
        targetFolder = os.path.join(htpc.RUNDIR)

        self.logger.debug('Overwriting files.')
        try:
            # Loop files and folders and place them in the HTPC Manager path
            for src_dir, dirs, files in os.walk(sourceUpdateFolder):
                dst_dir = src_dir.replace(sourceUpdateFolder, targetFolder)
                if not os.path.exists(dst_dir):
                    os.mkdir(dst_dir)
                for file_ in files:
                    src_file = os.path.join(src_dir, file_)
                    dst_file = os.path.join(dst_dir, file_)
                    if os.path.exists(dst_file):
                        os.remove(dst_file)
                    shutil.move(src_file, dst_dir)
        except:
            self.logger.warning('Failed to overwrite old files')
            self.__finishUpdate()
            return False

        self.logger.info('updating files successfull')
        return True

    """
    Write the latest commit hash to th version file.

    Used when checking for update the next time.
    """
    def __updateVersionFile(self, newVersion):
        versionFileHandler = open(self.versionFile, 'wb')
        versionFileHandler.write(newVersion)
        versionFileHandler.close()

    """ Remove leftover files after the update """
    def __finishUpdate(self):
        self.UPDATING = 0

        if os.path.isfile(self.updateFile):
            self.logger.debug('Removing update archive')
            try:
                os.remove(self.updateFile)
            except:
                pass

        if os.path.isdir(self.updateDir):
            self.logger.debug('Removing update code folder')
            try:
                shutil.rmtree(self.updateDir)
            except:
                pass
<|endoftext|>


<commit_before>from .toolbox_EPICS import EPICSBooleanButton, EPICSLed, EPICSValueMeterWidget, EPICSPlotPV
from .toolbox_opencv import OpencvImRead, OpencvCrop, OpencvVideo, OpencvThreshold, OpencvSplit, OpencvCvtColor, OpencvAddWeighted, OpencvBitwiseNot, OpencvBitwiseAnd, OpencvBitwiseOr,\
                            OpencvBilateralFilter, OpencvBlurFilter, OpencvDilateFilter, OpencvErodeFilter, OpencvLaplacianFilter, OpencvCanny, OpencvFindContours
from .toolbox_scheduling import TimerWidget
from .toolbox_siemens import PLCSiemens, ButtonSetResetBit, BitStatusWidget, WordEditWidget
<commit_msg>Editor: fixed load external toolboxes.<commit_after>try:
    from .toolbox_EPICS import EPICSBooleanButton, EPICSLed, EPICSValueMeterWidget, EPICSPlotPV
except:
    print("In order to use EPICS widgets install pyepics")

try:
    from .toolbox_opencv import OpencvImRead, OpencvCrop, OpencvVideo, OpencvThreshold, OpencvSplit, OpencvCvtColor, OpencvAddWeighted, OpencvBitwiseNot, OpencvBitwiseAnd, OpencvBitwiseOr,\
                            OpencvBilateralFilter, OpencvBlurFilter, OpencvDilateFilter, OpencvErodeFilter, OpencvLaplacianFilter, OpencvCanny, OpencvFindContours
except:
    print("In order to use OpenCv widgets install python-opencv")

from .toolbox_scheduling import TimerWidget

try:
    from .toolbox_siemens import PLCSiemens, ButtonSetResetBit, BitStatusWidget, WordEditWidget
except:
    print("In order to use Siemens widgets install python-snap7")
<|endoftext|>


<commit_before>from datetime import date
from dateutil import parser

from .utils import DslBase, _make_dsl_class, ObjectBase, AttrDict

__all__ = ['construct_field', 'Object', 'Nested', 'Date']

def construct_field(name_or_field, **params):
    # {"type": "string", "index": "not_analyzed"}
    if isinstance(name_or_field, dict):
        if params:
            raise ValueError('construct_field() cannot accept parameters when passing in a dict.')
        params = name_or_field.copy()
        if 'type' not in params:
            # inner object can be implicitly defined
            if 'properties' in params:
                name = 'object'
            else:
                raise ValueError('construct_field() needs to have a "type" key.')
        else:
            name = params.pop('type')
        return Field.get_dsl_class(name)(**params)

    # String()
    if isinstance(name_or_field, Field):
        if params:
            raise ValueError('construct_field() cannot accept parameters when passing in a construct_field object.')
        return name_or_field

    # "string", index="not_analyzed"
    return Field.get_dsl_class(name_or_field)(**params)

class Field(DslBase):
    _type_name = 'field'
    _type_shortcut = staticmethod(construct_field)
    # all fields can be multifields
    _param_defs = {'fields': {'type': 'field', 'hash': True}}
    name = None

    def _to_python(self, data):
        return data

    def to_python(self, data):
        if isinstance(data, (list, tuple)):
            return [self._to_python(d) for d in data]
        return self._to_python(data)

    def to_dict(self):
        d = super(Field, self).to_dict()
        name, value = d.popitem()
        value['type'] = name
        return value

class InnerObjectWrapper(ObjectBase):
    def __init__(self, mapping, **kwargs):
        # mimic DocType behavior with _doc_type.mapping
        super(AttrDict, self).__setattr__('_doc_type', type('Meta', (), {'mapping': mapping}))
        super(InnerObjectWrapper, self).__init__(**kwargs)

class InnerObject(object):
    " Common functionality for nested and object fields. "
    _doc_class = InnerObjectWrapper
    _param_defs = {'properties': {'type': 'field', 'hash': True}}

    def property(self, name, *args, **kwargs):
        self.properties[name] = construct_field(*args, **kwargs)
        return self

    def empty(self):
        return self.to_python({})

    def _to_python(self, data):
        # don't wrap already wrapped data
        if isinstance(data, self._doc_class):
            return data

        return self._doc_class(self.properties, **data)

class Object(InnerObject, Field):
    name = 'object'

class Nested(InnerObject, Field):
    name = 'nested'

class Date(Field):
    name = 'date'

    def _to_python(self, data):
        if isinstance(data, date):
            return data

        try:
            # TODO: add format awareness
            return parser.parse(data)
        except (TypeError, ValueError):
            raise #XXX

FIELDS = (
    'string',
    'long',
)

# generate the query classes dynamicaly
for f in FIELDS:
    fclass = _make_dsl_class(Field, f)
    globals()[fclass.__name__] = fclass
    __all__.append(fclass.__name__)

<commit_msg>Adding other core types.<commit_after>from datetime import date
from dateutil import parser

from .utils import DslBase, _make_dsl_class, ObjectBase, AttrDict

__all__ = ['construct_field', 'Object', 'Nested', 'Date']

def construct_field(name_or_field, **params):
    # {"type": "string", "index": "not_analyzed"}
    if isinstance(name_or_field, dict):
        if params:
            raise ValueError('construct_field() cannot accept parameters when passing in a dict.')
        params = name_or_field.copy()
        if 'type' not in params:
            # inner object can be implicitly defined
            if 'properties' in params:
                name = 'object'
            else:
                raise ValueError('construct_field() needs to have a "type" key.')
        else:
            name = params.pop('type')
        return Field.get_dsl_class(name)(**params)

    # String()
    if isinstance(name_or_field, Field):
        if params:
            raise ValueError('construct_field() cannot accept parameters when passing in a construct_field object.')
        return name_or_field

    # "string", index="not_analyzed"
    return Field.get_dsl_class(name_or_field)(**params)

class Field(DslBase):
    _type_name = 'field'
    _type_shortcut = staticmethod(construct_field)
    # all fields can be multifields
    _param_defs = {'fields': {'type': 'field', 'hash': True}}
    name = None

    def _to_python(self, data):
        return data

    def to_python(self, data):
        if isinstance(data, (list, tuple)):
            return [self._to_python(d) for d in data]
        return self._to_python(data)

    def to_dict(self):
        d = super(Field, self).to_dict()
        name, value = d.popitem()
        value['type'] = name
        return value

class InnerObjectWrapper(ObjectBase):
    def __init__(self, mapping, **kwargs):
        # mimic DocType behavior with _doc_type.mapping
        super(AttrDict, self).__setattr__('_doc_type', type('Meta', (), {'mapping': mapping}))
        super(InnerObjectWrapper, self).__init__(**kwargs)

class InnerObject(object):
    " Common functionality for nested and object fields. "
    _doc_class = InnerObjectWrapper
    _param_defs = {'properties': {'type': 'field', 'hash': True}}

    def property(self, name, *args, **kwargs):
        self.properties[name] = construct_field(*args, **kwargs)
        return self

    def empty(self):
        return self.to_python({})

    def _to_python(self, data):
        # don't wrap already wrapped data
        if isinstance(data, self._doc_class):
            return data

        return self._doc_class(self.properties, **data)

class Object(InnerObject, Field):
    name = 'object'

class Nested(InnerObject, Field):
    name = 'nested'

class Date(Field):
    name = 'date'

    def _to_python(self, data):
        if isinstance(data, date):
            return data

        try:
            # TODO: add format awareness
            return parser.parse(data)
        except (TypeError, ValueError):
            raise #XXX

FIELDS = (
    'string',
    'long',
    'integer',
    'float',
    'double',
    'boolean',
)

# generate the query classes dynamicaly
for f in FIELDS:
    fclass = _make_dsl_class(Field, f)
    globals()[fclass.__name__] = fclass
    __all__.append(fclass.__name__)

<|endoftext|>


<commit_before>import appliance
import pandas

def concatenate_traces(traces, metadata=None, how="strict"):
    '''
    Given a list of appliance traces, returns a single concatenated
    trace. With how="strict" option, must be sampled at the same rate and
    consecutive, without overlapping datapoints.
    '''
    if not metadata:
        metadata = traces[0].metadata

    if how == "strict":
        # require ordered list of consecutive, similarly sampled traces with no
        # missing data.
        return ApplianceTrace(concat([t.series for t in traces],metadata))
    else:
        raise NotImplementedError

def aggregate_traces(traces, metadata, how="strict"):
    '''
    Given a list of temporally aligned traces, aggregate them into a single
    signal.
    '''
    if how == "strict":
        # require that traces are exactly aligned
        summed_series = traces[0].series
        for trace in traces[1:]:
            summed_series += trace.series
        return ApplianceTrace(summed_series,metadata)
    else:
        return NotImplementedError

def aggregate_instances(instances,how="strict"):
    '''
    Given a list of temporally aligned instances, aggregate them into a single
    signal.
    '''
    if how == "strict":
        traces = [instance.traces for instance in instances]
        traces = [list(t) for t in zip(*traces)] # transpose
        traces = [ aggregate_traces(t,{}) for t in traces]
        # TODO how to aggregate metadata?
        return ApplianceInstance(traces)
    else:
        return NotImplementedError

def order_traces(self,traces):
    '''
    Given a set of traces, orders them chronologically and catches
    overlapping traces.
    '''
    order = np.argsort([t.series[0] for t in traces])
    new_traces = [traces[i] for i in order]
    return new_traces

<commit_msg>remove self arg from order traces<commit_after>import appliance
import pandas

def concatenate_traces(traces, metadata=None, how="strict"):
    '''
    Given a list of appliance traces, returns a single concatenated
    trace. With how="strict" option, must be sampled at the same rate and
    consecutive, without overlapping datapoints.
    '''
    if not metadata:
        metadata = traces[0].metadata

    if how == "strict":
        # require ordered list of consecutive, similarly sampled traces with no
        # missing data.
        return ApplianceTrace(concat([t.series for t in traces],metadata))
    else:
        raise NotImplementedError

def aggregate_traces(traces, metadata, how="strict"):
    '''
    Given a list of temporally aligned traces, aggregate them into a single
    signal.
    '''
    if how == "strict":
        # require that traces are exactly aligned
        summed_series = traces[0].series
        for trace in traces[1:]:
            summed_series += trace.series
        return ApplianceTrace(summed_series,metadata)
    else:
        return NotImplementedError

def aggregate_instances(instances,how="strict"):
    '''
    Given a list of temporally aligned instances, aggregate them into a single
    signal.
    '''
    if how == "strict":
        traces = [instance.traces for instance in instances]
        traces = [list(t) for t in zip(*traces)] # transpose
        traces = [ aggregate_traces(t,{}) for t in traces]
        # TODO how to aggregate metadata?
        return ApplianceInstance(traces)
    else:
        return NotImplementedError

def order_traces(traces):
    '''
    Given a set of traces, orders them chronologically and catches
    overlapping traces.
    '''
    order = np.argsort([t.series[0] for t in traces])
    new_traces = [traces[i] for i in order]
    return new_traces

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

from openerp import api, fields, models

class SaleOrderLine(models.Model):
    _inherit = "sale.order.line"

    def _get_real_price_currency(self, product, pricelist_dict, qty, uom, pricelist_id):
        """Retrieve the price before applying the pricelist
            :param obj product: object of current product record
            :parem float qty: total quentity of product
            :param dict pricelist_dict: dict contain pricelist id and rule id
            :param obj uom: unit of measure of current order line
            :param integer pricelist_id: pricelist id of sale order"""
        PricelistItem = self.env['product.pricelist.item']
        field_name = 'lst_price'
        currency_id = None
        rule_id = pricelist_dict[pricelist_id][1] if pricelist_dict.get(pricelist_id) else False
        if rule_id:
            pricelist_item = PricelistItem.browse(rule_id)
            if pricelist_item.base == 'standard_price':
                field_name = 'standard_price'
            currency_id = pricelist_item.pricelist_id.currency_id

        product_currency = (product.company_id and product.company_id.currency_id) or self.env.user.company_id.currency_id
        if not currency_id:
            currency_id = product_currency
            cur_factor = 1.0
        else:
            if currency_id.id == product_currency.id:
                cur_factor = 1.0
            else:
                cur_factor = currency_id._get_conversion_rate(product_currency, currency_id)

        product_uom = self.env.context.get('uom') or product.uom_id.id
        if uom and uom.id != product_uom:
            # the unit price is in a different uom
            uom_factor = uom._compute_price(uom.id, 1.0, product.uom_id.id)
        else:
            uom_factor = 1.0

        return product[field_name] * uom_factor * cur_factor, currency_id.id

    @api.onchange('product_id', 'price_unit', 'product_uom', 'product_uom_qty', 'tax_id')
    def _onchange_discount(self):
        self.discount = 0.0
        if not (self.product_id and self.product_uom
                and self.order_id.partner_id and self.order_id.pricelist_id
                and self.order_id.pricelist_id.discount_policy == 'without_discount'
                and self.env.user.has_group('sale.group_discount_per_so_line')):
            return

        context_partner = dict(self.env.context, partner_id=self.order_id.partner_id.id)
        pricelist_context = dict(context_partner, uom=self.product_uom.id, date=self.order_id.date_order)

        list_price = self.order_id.pricelist_id.with_context(pricelist_context).price_rule_get(self.product_id.id, self.product_uom_qty or 1.0, self.order_id.partner_id)
        new_list_price, currency_id = self.with_context(context_partner)._get_real_price_currency(self.product_id, list_price, self.product_uom_qty, self.product_uom, self.order_id.pricelist_id.id)
        new_list_price = self.env['account.tax']._fix_tax_included_price(new_list_price, self.product_id.taxes_id, self.tax_id)
        list_price = list_price[self.order_id.pricelist_id.id][0]

        if list_price != 0 and new_list_price != 0:
            if self.product_id.company_id and self.order_id.pricelist_id.currency_id != self.product_id.company_id.currency_id:
                # new_list_price is in company's currency while price in pricelist currency
                ctx = dict(context_partner, date=self.order_id.date_order)
                new_list_price = self.env['res.currency'].browse(currency_id).with_context(ctx).compute(new_list_price, self.order_id.pricelist_id.currency_id)
            discount = (new_list_price - list_price) / new_list_price * 100
            if discount > 0:
                self.discount = discount

    @api.multi
    def _get_display_price(self, product):
        if self.order_id.pricelist_id.discount_policy == 'without_discount':
            return product.lst_price
        return super(SaleOrderLine, self)._get_display_price(product)
<commit_msg>[FIX] product_visible_discount: price currency<commit_after># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

from openerp import api, fields, models

class SaleOrderLine(models.Model):
    _inherit = "sale.order.line"

    def _get_real_price_currency(self, product, pricelist_dict, qty, uom, pricelist_id):
        """Retrieve the price before applying the pricelist
            :param obj product: object of current product record
            :parem float qty: total quentity of product
            :param dict pricelist_dict: dict contain pricelist id and rule id
            :param obj uom: unit of measure of current order line
            :param integer pricelist_id: pricelist id of sale order"""
        PricelistItem = self.env['product.pricelist.item']
        field_name = 'lst_price'
        currency_id = None
        rule_id = pricelist_dict[pricelist_id][1] if pricelist_dict.get(pricelist_id) else False
        if rule_id:
            pricelist_item = PricelistItem.browse(rule_id)
            if pricelist_item.base == 'standard_price':
                field_name = 'standard_price'
            currency_id = pricelist_item.pricelist_id.currency_id

        product_currency = (product.company_id and product.company_id.currency_id) or self.env.user.company_id.currency_id
        if not currency_id:
            currency_id = product_currency
            cur_factor = 1.0
        else:
            if currency_id.id == product_currency.id:
                cur_factor = 1.0
            else:
                cur_factor = currency_id._get_conversion_rate(product_currency, currency_id)

        product_uom = self.env.context.get('uom') or product.uom_id.id
        if uom and uom.id != product_uom:
            # the unit price is in a different uom
            uom_factor = uom._compute_price(uom.id, 1.0, product.uom_id.id)
        else:
            uom_factor = 1.0

        return product[field_name] * uom_factor * cur_factor, currency_id.id

    @api.onchange('product_id', 'price_unit', 'product_uom', 'product_uom_qty', 'tax_id')
    def _onchange_discount(self):
        self.discount = 0.0
        if not (self.product_id and self.product_uom
                and self.order_id.partner_id and self.order_id.pricelist_id
                and self.order_id.pricelist_id.discount_policy == 'without_discount'
                and self.env.user.has_group('sale.group_discount_per_so_line')):
            return

        context_partner = dict(self.env.context, partner_id=self.order_id.partner_id.id)
        pricelist_context = dict(context_partner, uom=self.product_uom.id, date=self.order_id.date_order)

        list_price = self.order_id.pricelist_id.with_context(pricelist_context).price_rule_get(self.product_id.id, self.product_uom_qty or 1.0, self.order_id.partner_id)
        new_list_price, currency_id = self.with_context(context_partner)._get_real_price_currency(self.product_id, list_price, self.product_uom_qty, self.product_uom, self.order_id.pricelist_id.id)
        new_list_price = self.env['account.tax']._fix_tax_included_price(new_list_price, self.product_id.taxes_id, self.tax_id)
        list_price = list_price[self.order_id.pricelist_id.id][0]

        if list_price != 0 and new_list_price != 0:
            if self.product_id.company_id and self.order_id.pricelist_id.currency_id != self.product_id.company_id.currency_id:
                # new_list_price is in company's currency while price in pricelist currency
                ctx = dict(context_partner, date=self.order_id.date_order)
                new_list_price = self.env['res.currency'].browse(currency_id).with_context(ctx).compute(new_list_price, self.order_id.pricelist_id.currency_id)
            discount = (new_list_price - list_price) / new_list_price * 100
            if discount > 0:
                self.discount = discount

    @api.multi
    def _get_display_price(self, product):
        if self.order_id.pricelist_id.discount_policy == 'without_discount':
            return self.order_id.company_id.currency_id.compute(
                product.lst_price, self.order_id.pricelist_id.currency_id)
        return super(SaleOrderLine, self)._get_display_price(product)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

from odoo import api, exceptions, models, _

class Issue(models.Model):
    _name = "project.issue"
    _inherit = ['project.issue']

    @api.multi
    def get_access_action(self):
        """ Instead of the classic form view, redirect to website for portal users
        that can read the issue. """
        self.ensure_one()
        if self.env.context.get('uid'):
            user = self.env['res.users'].browse(self.env.context['uid'])
        else:
            user = self.env.user
        if user.share:
            try:
                self.check_access_rule('read')
            except exceptions.AccessError:
                pass
            else:
                return {
                    'type': 'ir.actions.act_url',
                    'url': '/my/issues/%s' % self.id,
                    'target': 'self',
                    'res_id': self.id,
                }
        return super(Issue, self).get_access_action()

    @api.multi
    def _notification_recipients(self, message, groups):
        groups = super(Issue, self)._notification_recipients(message, groups)

        for group_name, group_method, group_data in groups:
            group_data['has_button_access'] = True

        return groups
<commit_msg>[FIX] website_project_issue: No buttons for portal, customers<commit_after># -*- coding: utf-8 -*-
# Part of Odoo. See LICENSE file for full copyright and licensing details.

from odoo import api, exceptions, models, _

class Issue(models.Model):
    _name = "project.issue"
    _inherit = ['project.issue']

    @api.multi
    def get_access_action(self):
        """ Instead of the classic form view, redirect to website for portal users
        that can read the issue. """
        self.ensure_one()
        if self.env.context.get('uid'):
            user = self.env['res.users'].browse(self.env.context['uid'])
        else:
            user = self.env.user
        if user.share:
            try:
                self.check_access_rule('read')
            except exceptions.AccessError:
                pass
            else:
                return {
                    'type': 'ir.actions.act_url',
                    'url': '/my/issues/%s' % self.id,
                    'target': 'self',
                    'res_id': self.id,
                }
        return super(Issue, self).get_access_action()

    @api.multi
    def _notification_recipients(self, message, groups):
        groups = super(Issue, self)._notification_recipients(message, groups)

        for group_name, group_method, group_data in groups:
            if group_name in ["customer", "portal"]:
                continue

            group_data['has_button_access'] = True

        return groups
<|endoftext|>