<commit_before>#!/usr/bin/env python3
#
# Copyright 2017 Jeff Bush
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""
Validate hardware JTAG debugging implementation. The Verilator model has a
stub module that simulates a JTAG host. This test communicates with it over
a socket, which allows sending and receiving data and instructions.
"""

import socket
import struct
import subprocess
import sys
import time
from threading import Thread

sys.path.insert(0, '..')
import test_harness

DEBUG = False
CONTROL_PORT = 8541
INSTRUCTION_LENGTH = 4
EXPECTED_IDCODE = 0x4d20dffb  # Derived from settings in hardware/core/config.sv

# JTAG instructions
INST_IDCODE = 0
INST_EXTEST = 1
INST_INTEST = 2
INST_CONTROL = 3
INST_INJECT_INST = 4
INST_TRANSFER_DATA = 5
INST_BYPASS = 15

# When passed, does not shift instruction register
INST_SAME = -1

# XXX does not test TRST signal

def mask_value(value, num_bits):
    return value & ((1 << num_bits) - 1)

class JTAGTestFixture(object):

    """
    Spawns the Verilator model and opens a socket to communicate with it
    Supports __enter__ and __exit__ methods so it can be used in the 'with'
    construct to automatically clean up after itself.
    """

    def __init__(self, hexfile):
        self.hexfile = hexfile
        self.process = None
        self.sock = None
        self.output = ''
        self.reader_thread = None
        self.last_response = 0

    def __enter__(self):
        verilator_args = [
            test_harness.BIN_DIR + 'verilator_model',
            '+bin=' + self.hexfile,
            '+jtag_port=' + str(CONTROL_PORT),
            self.hexfile
        ]

        self.process = subprocess.Popen(verilator_args, stdout=subprocess.PIPE,
                                        stderr=subprocess.STDOUT)

        # The process may take a little time to start listening for incoming
        # connections so retry a few times if it isn't ready yet.
        for _ in range(10):
            try:
                time.sleep(0.3)
                self.sock = socket.socket()
                self.sock.connect(('localhost', CONTROL_PORT))
                self.sock.settimeout(5)
                break
            except socket.error:
                pass
        else:
            self.process.kill()
            raise test_harness.TestException(
                'failed to connect to verilator model')

        self.reader_thread = Thread(target=self._read_output)
        self.reader_thread.daemon = True
        self.reader_thread.start()
        return self

    def __exit__(self, *unused):
        self.process.kill()
        if self.sock:
            self.sock.close()

    def jtag_transfer(self, instruction, data_length, data):
        """
        Shift an instruction and/or data to the target.
        If instruction is set to INST_SAME, this will not shift an instruction
        If data_length is zero, it will not shift any data. If both are set to
        not transfer, it will initiate a reset of the target.
        """

        if DEBUG:
            print('Sending JTAG command 0x{:x} data 0x{:x}'.format(
                instruction, data))

        if instruction == INST_SAME:
            instruction_length = 0
            instruction = 0
        else:
            instruction_length = INSTRUCTION_LENGTH

        self.sock.send(struct.pack('<BIBQ', instruction_length, instruction,
                                   data_length, data))

        response_data = self.sock.recv(12)
        if not response_data:
            raise test_harness.TestException(
                'error reading response:\n' + self.get_program_output())

        _, self.last_response = struct.unpack('<IQ', response_data)
        self.last_response = mask_value(self.last_response, data_length)
        if DEBUG:
            print('received JTAG response 0x{:x}'.format(self.last_response))

    def test_instruction_shift(self, value):
        """
        Shift a value through the instruction register, then capture
        the bits that are shifted out and check that they match.
        """

        # Send an instruction that is twice as long as the instruction register.
        # The first bits shifted in should come right back out in the high
        # bits of the result.
        self.sock.send(struct.pack('<BIBQ', INSTRUCTION_LENGTH * 2, value,
                                   0, 0))
        response_data = self.sock.recv(12)
        if not response_data:
            raise test_harness.TestException(
                'error reading response:\n' + self.get_program_output())

        instr_response, _ = struct.unpack('<IQ', response_data)
        if instr_response != value << INSTRUCTION_LENGTH:
            raise test_harness.TestException('invalid response: wanted {}, got {}'.format(
                value, instr_response))

    def get_program_output(self):
        """
        Return everything verilator printed to stdout before exiting.
        This won't read anything if the program was killed (which is the
        common case if the program didn't die with an assertion), but
        we usually call in the case that it has exited with an error.
        """
        # Give the reader thread time to finish reading responses.
        if self.reader_thread:
            self.reader_thread.join(0.5)

        return self.output

    def expect_data(self, expected_data):
        """
        Throw an exception if the bits shifted out of TDO during the last data
        transfer do not match the passed value
        """
        if self.last_response != expected_data:
            raise test_harness.TestException('unexpected JTAG data response. Wanted {} got {}:\n{}'
                                             .format(expected_data, self.last_response,
                                                     self.get_program_output()))

    def _read_output(self):
        """
        Read text that is printed by the verilator process to standard out.
        This needs to happen on a separate thread to avoid blocking the
        main thread. This seems to be the only way to do it portably in
        python.
        """
        while True:
            got = self.process.stdout.read(0x100)
            if not got:
                break

            if DEBUG:
                print(got)

            self.output += got.decode()

@test_harness.test
def jtag_idcode(_):
    """
    Validate response to IDCODE request
    """
    hexfile = test_harness.build_program(['test_program.S'])
    with JTAGTestFixture(hexfile) as fixture:
        # Ensure the default instruction after reset is IDCODE
        fixture.jtag_transfer(INST_SAME, 32, 0xffffffff)
        fixture.expect_data(EXPECTED_IDCODE)

        # Explicitly shift the IDCODE instruction to make sure it is
        # correct.
        fixture.jtag_transfer(INST_IDCODE, 32, 0xffffffff)
        fixture.expect_data(EXPECTED_IDCODE)

@test_harness.test
def jtag_reset(_):
    """
    Test transition to reset state
    """
    hexfile = test_harness.build_program(['test_program.S'])
    with JTAGTestFixture(hexfile) as fixture:
        # Load a different instruction
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0x3b643e9a)

        # Perform a reset (setting zero lengths)
        fixture.jtag_transfer(INST_SAME, 0, 0)

        # Perform data-only transfer. Ensure we get the idcode back.
        # If we hadn't performed a reset, we would get data value
        # that was shifted above.
        fixture.jtag_transfer(INST_SAME, 32, 0xffffffff)
        fixture.expect_data(EXPECTED_IDCODE)

@test_harness.test
def jtag_bypass(_):
    """
    Validate BYPASS instruction, which is a single bit data register
    We should get what we send, shifted by one bit.
    """
    hexfile = test_harness.build_program(['test_program.S'])
    with JTAGTestFixture(hexfile) as fixture:
        value = 0x267521cf
        fixture.jtag_transfer(INST_BYPASS, 32, value)
        fixture.expect_data(value << 1)

@test_harness.test
def test_instruction_shift(_):
    """
    Ensure instruction bits shifted into TDI come out TDO. This is necessary
    to properly chain JTAG devices together.
    """
    hexfile = test_harness.build_program(['test_program.S'])
    with JTAGTestFixture(hexfile) as fixture:
        fixture.test_instruction_shift(0xf)
        fixture.test_instruction_shift(0xa)
        fixture.test_instruction_shift(0x5)
        fixture.test_instruction_shift(0x3)
        fixture.test_instruction_shift(0xc)
        fixture.test_instruction_shift(0)

@test_harness.test
def jtag_data_transfer(_):
    """
    Validate bi-directional transfer. The TRANSFER_DATA instruction
    returns the old value of the control register while shifting a new
    one in, so we should see the previous value come out each time
    we write a new one.
    """
    hexfile = test_harness.build_program(['test_program.S'])
    with JTAGTestFixture(hexfile) as fixture:
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0x4be49e7c)
        fixture.jtag_transfer(INST_SAME, 32, 0xb282dc16)
        fixture.expect_data(0x4be49e7c)
        fixture.jtag_transfer(INST_SAME, 32, 0x7ee4838)
        fixture.expect_data(0xb282dc16)

@test_harness.test
def jtag_inject(_):
    """
    Test instruction injection, with multiple threads
    """
    hexfile = test_harness.build_program(['test_program.S'])
    with JTAGTestFixture(hexfile) as fixture:
        # Halt
        fixture.jtag_transfer(INST_CONTROL, 7, 0x1)

        # Load register values in thread 0
        # First value to transfer
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0x3b643e9a)
        fixture.jtag_transfer(INST_INJECT_INST, 32, 0xac0000b2)  # getcr s5, 18
        # Second value to transfer
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0xd1dc20a3)
        fixture.jtag_transfer(INST_INJECT_INST, 32, 0xac0000d2)  # getcr s6, 18

        # Load register values in thread 1
        fixture.jtag_transfer(INST_CONTROL, 7, 0x3)
        # First value to transfer
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0xa6532328)
        fixture.jtag_transfer(INST_INJECT_INST, 32, 0xac0000b2)  # getcr s5, 18
        # Second value to transfer
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0xf01839a0)
        fixture.jtag_transfer(INST_INJECT_INST, 32, 0xac0000d2)  # getcr s6, 18

        # Perform operation on thread 0
        fixture.jtag_transfer(INST_CONTROL, 7, 0x1)
        fixture.jtag_transfer(INST_INJECT_INST, 32,
                              0xc03300e5)  # xor s7, s5, s6
        fixture.jtag_transfer(INST_SAME, 32, 0x8c0000f2)  # setcr s7, 18
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0)
        fixture.expect_data(0xeab81e39)

        # Perform operation on thread 1
        fixture.jtag_transfer(INST_CONTROL, 7, 0x3)
        fixture.jtag_transfer(INST_INJECT_INST, 32,
                              0xc03300e5)  # xor s7, s5, s6
        fixture.jtag_transfer(INST_SAME, 32, 0x8c0000f2)  # setcr s7, 18
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0)
        fixture.expect_data(0x564b1a88)

@test_harness.test
def jtag_read_write_pc(_):
    """
    Use the call instruction to read the program counter. The injection
    logic is supposed to simulate each instruction having the PC of the
    interrupted thread. Then ensure performing a branch will properly
    update the PC of the selected thread. This then resumes the thread
    to ensure it operates properly.
    """
    hexfile = test_harness.build_program(['test_program.S'])
    with JTAGTestFixture(hexfile) as fixture:
        # Switch to thread 1, branch to new address
        fixture.jtag_transfer(INST_CONTROL, 7, 0x3)
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0x10e4)   # `jump_target`
        fixture.jtag_transfer(INST_INJECT_INST, 32, 0xac000012)  # getcr s0, 18
        fixture.jtag_transfer(INST_SAME, 32, 0xf0000000)  # b s0

        # Switch to thread 0, read PC to ensure it is read correctly
        # and that the branch didn't affect it.
        # (the return address will be the branch location + 4)
        fixture.jtag_transfer(INST_CONTROL, 7, 0x1)
        fixture.jtag_transfer(INST_INJECT_INST, 32, 0xf8000000)  # call 0
        fixture.jtag_transfer(INST_SAME, 32, 0x8c0003f2)  # setcr ra, 18
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0)
        fixture.expect_data(0x10d8)

        # Switch back to thread 1, read back PC to ensure it is in the new
        # location
        fixture.jtag_transfer(INST_CONTROL, 7, 0x3)
        fixture.jtag_transfer(INST_INJECT_INST, 32, 0xf8000000)  # call 0
        fixture.jtag_transfer(INST_SAME, 32, 0x8c0003f2)  # setcr ra, 18
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0)
        fixture.expect_data(0x10e8)

        # Resume the thread. It should now execute the instruction to
        # load a new value into s0
        fixture.jtag_transfer(INST_CONTROL, 7, 0x0)

        # Dummy transaction that ensures the processor has time to execute
        # the instructions
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0)

        # Halt again and verify register value.
        fixture.jtag_transfer(INST_CONTROL, 7, 0x3)
        fixture.jtag_transfer(INST_INJECT_INST, 32, 0x8c000012)  # setcr s0, 18
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0)
        fixture.expect_data(0x6bee68ca)

test_harness.execute_tests()
<commit_msg>issue #128: jtag_read_write_pc test failing intermittently<commit_after>#!/usr/bin/env python3
#
# Copyright 2017 Jeff Bush
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""
Validate hardware JTAG debugging implementation. The Verilator model has a
stub module that simulates a JTAG host. This test communicates with it over
a socket, which allows sending and receiving data and instructions.
"""

import socket
import struct
import subprocess
import sys
import time
from threading import Thread

sys.path.insert(0, '..')
import test_harness

DEBUG = False
CONTROL_PORT = 8541
INSTRUCTION_LENGTH = 4
EXPECTED_IDCODE = 0x4d20dffb  # Derived from settings in hardware/core/config.sv

# JTAG instructions
INST_IDCODE = 0
INST_EXTEST = 1
INST_INTEST = 2
INST_CONTROL = 3
INST_INJECT_INST = 4
INST_TRANSFER_DATA = 5
INST_BYPASS = 15

# When passed, does not shift instruction register
INST_SAME = -1

# XXX does not test TRST signal

def mask_value(value, num_bits):
    return value & ((1 << num_bits) - 1)

class JTAGTestFixture(object):

    """
    Spawns the Verilator model and opens a socket to communicate with it
    Supports __enter__ and __exit__ methods so it can be used in the 'with'
    construct to automatically clean up after itself.
    """

    def __init__(self, hexfile):
        self.hexfile = hexfile
        self.process = None
        self.sock = None
        self.output = ''
        self.reader_thread = None
        self.last_response = 0

    def __enter__(self):
        verilator_args = [
            test_harness.BIN_DIR + 'verilator_model',
            '+bin=' + self.hexfile,
            '+jtag_port=' + str(CONTROL_PORT),
            self.hexfile
        ]

        self.process = subprocess.Popen(verilator_args, stdout=subprocess.PIPE,
                                        stderr=subprocess.STDOUT)

        # The process may take a little time to start listening for incoming
        # connections so retry a few times if it isn't ready yet.
        for _ in range(10):
            try:
                time.sleep(0.3)
                self.sock = socket.socket()
                self.sock.connect(('localhost', CONTROL_PORT))
                self.sock.settimeout(5)
                break
            except socket.error:
                pass
        else:
            self.process.kill()
            raise test_harness.TestException(
                'failed to connect to verilator model')

        self.reader_thread = Thread(target=self._read_output)
        self.reader_thread.daemon = True
        self.reader_thread.start()
        return self

    def __exit__(self, *unused):
        self.process.kill()
        if self.sock:
            self.sock.close()

    def jtag_transfer(self, instruction, data_length, data):
        """
        Shift an instruction and/or data to the target.
        If instruction is set to INST_SAME, this will not shift an instruction
        If data_length is zero, it will not shift any data. If both are set to
        not transfer, it will initiate a reset of the target.
        """

        if DEBUG:
            print('Sending JTAG command 0x{:x} data 0x{:x}'.format(
                instruction, data))

        if instruction == INST_SAME:
            instruction_length = 0
            instruction = 0
        else:
            instruction_length = INSTRUCTION_LENGTH

        self.sock.send(struct.pack('<BIBQ', instruction_length, instruction,
                                   data_length, data))

        response_data = self.sock.recv(12)
        if not response_data:
            raise test_harness.TestException(
                'error reading response:\n' + self.get_program_output())

        _, self.last_response = struct.unpack('<IQ', response_data)
        self.last_response = mask_value(self.last_response, data_length)
        if DEBUG:
            print('received JTAG response 0x{:x}'.format(self.last_response))

    def test_instruction_shift(self, value):
        """
        Shift a value through the instruction register, then capture
        the bits that are shifted out and check that they match.
        """

        # Send an instruction that is twice as long as the instruction register.
        # The first bits shifted in should come right back out in the high
        # bits of the result.
        self.sock.send(struct.pack('<BIBQ', INSTRUCTION_LENGTH * 2, value,
                                   0, 0))
        response_data = self.sock.recv(12)
        if not response_data:
            raise test_harness.TestException(
                'error reading response:\n' + self.get_program_output())

        instr_response, _ = struct.unpack('<IQ', response_data)
        if instr_response != value << INSTRUCTION_LENGTH:
            raise test_harness.TestException('invalid response: wanted {}, got {}'.format(
                value, instr_response))

    def get_program_output(self):
        """
        Return everything verilator printed to stdout before exiting.
        This won't read anything if the program was killed (which is the
        common case if the program didn't die with an assertion), but
        we usually call in the case that it has exited with an error.
        """
        # Give the reader thread time to finish reading responses.
        if self.reader_thread:
            self.reader_thread.join(0.5)

        return self.output

    def expect_data(self, expected_data):
        """
        Throw an exception if the bits shifted out of TDO during the last data
        transfer do not match the passed value
        """
        if self.last_response != expected_data:
            raise test_harness.TestException('unexpected JTAG data response. Wanted {} got {}:\n{}'
                                             .format(expected_data, self.last_response,
                                                     self.get_program_output()))

    def _read_output(self):
        """
        Read text that is printed by the verilator process to standard out.
        This needs to happen on a separate thread to avoid blocking the
        main thread. This seems to be the only way to do it portably in
        python.
        """
        while True:
            got = self.process.stdout.read(0x100)
            if not got:
                break

            if DEBUG:
                print(got)

            self.output += got.decode()

@test_harness.test
def jtag_idcode(_):
    """
    Validate response to IDCODE request
    """
    hexfile = test_harness.build_program(['test_program.S'])
    with JTAGTestFixture(hexfile) as fixture:
        # Ensure the default instruction after reset is IDCODE
        fixture.jtag_transfer(INST_SAME, 32, 0xffffffff)
        fixture.expect_data(EXPECTED_IDCODE)

        # Explicitly shift the IDCODE instruction to make sure it is
        # correct.
        fixture.jtag_transfer(INST_IDCODE, 32, 0xffffffff)
        fixture.expect_data(EXPECTED_IDCODE)

@test_harness.test
def jtag_reset(_):
    """
    Test transition to reset state
    """
    hexfile = test_harness.build_program(['test_program.S'])
    with JTAGTestFixture(hexfile) as fixture:
        # Load a different instruction
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0x3b643e9a)

        # Perform a reset (setting zero lengths)
        fixture.jtag_transfer(INST_SAME, 0, 0)

        # Perform data-only transfer. Ensure we get the idcode back.
        # If we hadn't performed a reset, we would get data value
        # that was shifted above.
        fixture.jtag_transfer(INST_SAME, 32, 0xffffffff)
        fixture.expect_data(EXPECTED_IDCODE)

@test_harness.test
def jtag_bypass(_):
    """
    Validate BYPASS instruction, which is a single bit data register
    We should get what we send, shifted by one bit.
    """
    hexfile = test_harness.build_program(['test_program.S'])
    with JTAGTestFixture(hexfile) as fixture:
        value = 0x267521cf
        fixture.jtag_transfer(INST_BYPASS, 32, value)
        fixture.expect_data(value << 1)

@test_harness.test
def test_instruction_shift(_):
    """
    Ensure instruction bits shifted into TDI come out TDO. This is necessary
    to properly chain JTAG devices together.
    """
    hexfile = test_harness.build_program(['test_program.S'])
    with JTAGTestFixture(hexfile) as fixture:
        fixture.test_instruction_shift(0xf)
        fixture.test_instruction_shift(0xa)
        fixture.test_instruction_shift(0x5)
        fixture.test_instruction_shift(0x3)
        fixture.test_instruction_shift(0xc)
        fixture.test_instruction_shift(0)

@test_harness.test
def jtag_data_transfer(_):
    """
    Validate bi-directional transfer. The TRANSFER_DATA instruction
    returns the old value of the control register while shifting a new
    one in, so we should see the previous value come out each time
    we write a new one.
    """
    hexfile = test_harness.build_program(['test_program.S'])
    with JTAGTestFixture(hexfile) as fixture:
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0x4be49e7c)
        fixture.jtag_transfer(INST_SAME, 32, 0xb282dc16)
        fixture.expect_data(0x4be49e7c)
        fixture.jtag_transfer(INST_SAME, 32, 0x7ee4838)
        fixture.expect_data(0xb282dc16)

@test_harness.test
def jtag_inject(_):
    """
    Test instruction injection, with multiple threads
    """
    hexfile = test_harness.build_program(['test_program.S'])
    with JTAGTestFixture(hexfile) as fixture:
        # Halt
        fixture.jtag_transfer(INST_CONTROL, 7, 0x1)

        # Load register values in thread 0
        # First value to transfer
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0x3b643e9a)
        fixture.jtag_transfer(INST_INJECT_INST, 32, 0xac0000b2)  # getcr s5, 18
        # Second value to transfer
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0xd1dc20a3)
        fixture.jtag_transfer(INST_INJECT_INST, 32, 0xac0000d2)  # getcr s6, 18

        # Load register values in thread 1
        fixture.jtag_transfer(INST_CONTROL, 7, 0x3)
        # First value to transfer
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0xa6532328)
        fixture.jtag_transfer(INST_INJECT_INST, 32, 0xac0000b2)  # getcr s5, 18
        # Second value to transfer
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0xf01839a0)
        fixture.jtag_transfer(INST_INJECT_INST, 32, 0xac0000d2)  # getcr s6, 18

        # Perform operation on thread 0
        fixture.jtag_transfer(INST_CONTROL, 7, 0x1)
        fixture.jtag_transfer(INST_INJECT_INST, 32,
                              0xc03300e5)  # xor s7, s5, s6
        fixture.jtag_transfer(INST_SAME, 32, 0x8c0000f2)  # setcr s7, 18
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0)
        fixture.expect_data(0xeab81e39)

        # Perform operation on thread 1
        fixture.jtag_transfer(INST_CONTROL, 7, 0x3)
        fixture.jtag_transfer(INST_INJECT_INST, 32,
                              0xc03300e5)  # xor s7, s5, s6
        fixture.jtag_transfer(INST_SAME, 32, 0x8c0000f2)  # setcr s7, 18
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0)
        fixture.expect_data(0x564b1a88)

# XXX currently disabled because of issue #128
#@test_harness.test
def jtag_read_write_pc(_):
    """
    Use the call instruction to read the program counter. The injection
    logic is supposed to simulate each instruction having the PC of the
    interrupted thread. Then ensure performing a branch will properly
    update the PC of the selected thread. This then resumes the thread
    to ensure it operates properly.
    """
    hexfile = test_harness.build_program(['test_program.S'])
    with JTAGTestFixture(hexfile) as fixture:
        # Switch to thread 1, branch to new address
        fixture.jtag_transfer(INST_CONTROL, 7, 0x3)
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0x10e4)   # `jump_target`
        fixture.jtag_transfer(INST_INJECT_INST, 32, 0xac000012)  # getcr s0, 18
        fixture.jtag_transfer(INST_SAME, 32, 0xf0000000)  # b s0

        # Switch to thread 0, read PC to ensure it is read correctly
        # and that the branch didn't affect it.
        # (the return address will be the branch location + 4)
        fixture.jtag_transfer(INST_CONTROL, 7, 0x1)
        fixture.jtag_transfer(INST_INJECT_INST, 32, 0xf8000000)  # call 0
        fixture.jtag_transfer(INST_SAME, 32, 0x8c0003f2)  # setcr ra, 18
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0)
        fixture.expect_data(0x10d8)

        # Switch back to thread 1, read back PC to ensure it is in the new
        # location
        fixture.jtag_transfer(INST_CONTROL, 7, 0x3)
        fixture.jtag_transfer(INST_INJECT_INST, 32, 0xf8000000)  # call 0
        fixture.jtag_transfer(INST_SAME, 32, 0x8c0003f2)  # setcr ra, 18
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0)
        fixture.expect_data(0x10e8)

        # Resume the thread. It should now execute the instruction to
        # load a new value into s0
        fixture.jtag_transfer(INST_CONTROL, 7, 0x0)

        # Dummy transaction that ensures the processor has time to execute
        # the instructions
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0)

        # Halt again and verify register value.
        fixture.jtag_transfer(INST_CONTROL, 7, 0x3)
        fixture.jtag_transfer(INST_INJECT_INST, 32, 0x8c000012)  # setcr s0, 18
        fixture.jtag_transfer(INST_TRANSFER_DATA, 32, 0)
        fixture.expect_data(0x6bee68ca)

test_harness.execute_tests()
<|endoftext|>


<commit_before>def add_random(self, command):
    import random
    global selected_songs
    filelist = []
    for root, dirs, files in os.walk(MUSIC_PATH):
        for name in files:
            root = root.replace(MUSIC_PATH + os.sep, "")
            filelist.append(os.path.join(root, name))
    numsongs = int(self.confman.get_value("add_random", "NUMBER_OF_SONGS", 10))
    for i in range(1,numsongs):
        if len(selected_songs) == len(filelist):
            selected_songs = []
        filepath = ""
        while 1:
            filepath = filelist[random.randint(0, len(filelist)-1)]
            if not filepath.endswith(".m3u") and not filepath in selected_songs and not NICK + "_intros" + os.sep in filepath:
                break
        selected_songs.append(filepath)
        try:
            self.conman.mpc.add(filepath)
        except mpd.MPDError:
            pass

selected_songs = []
self.map_command("random", add_random)
self.map_help("random", ".random - adds 10 random tracks to the queue")
<commit_msg>Fix random module's help message<commit_after>def add_random(self, command):
    import random
    global selected_songs
    filelist = []
    for root, dirs, files in os.walk(MUSIC_PATH):
        for name in files:
            root = root.replace(MUSIC_PATH + os.sep, "")
            filelist.append(os.path.join(root, name))
    numsongs = int(self.confman.get_value("add_random", "NUMBER_OF_SONGS", 10))
    for i in range(1,numsongs):
        if len(selected_songs) == len(filelist):
            selected_songs = []
        filepath = ""
        while 1:
            filepath = filelist[random.randint(0, len(filelist)-1)]
            if not filepath.endswith(".m3u") and not filepath in selected_songs and not NICK + "_intros" + os.sep in filepath:
                break
        selected_songs.append(filepath)
        try:
            self.conman.mpc.add(filepath)
        except mpd.MPDError:
            pass

selected_songs = []
self.map_command("random", add_random)
self.map_help("random", ".random - adds %s random tracks to the queue" % self.confman.get_value("add_random", "NUMBER_OF_SONGS", 10))
<|endoftext|>


<commit_before>#! /usr/bin/env python

from __future__ import print_function
try: from . import *
except: from bacula_tools import *

class Director(DbDict):
    SETUP_KEYS = [ADDRESS,
                 FD_CONNECT_TIMEOUT, HEARTBEATINTERVAL,
                 PASSWORD, PIDDIRECTORY, QUERYFILE,
                 SCRIPTS_DIRECTORY, SD_CONNECT_TIMEOUT, SOURCEADDRESS, STATISTICS_RETENTION,
                 WORKINGDIRECTORY, DIRADDRESSES]
    INT_KEYS = [(DIRPORT, 9101), MAXIMUMCONCURRENTJOBS, MAXIMUMCONSOLECONNECTIONS,]
    NULL_KEYS = [MESSAGES_ID, ]
    table = DIRECTORS
    IDTAG = 1
    # {{{ parse_string(string, director_config, obj): Entry point for a recursive descent parser

    def parse_string(self, string, director_config, obj):
        '''Populate a new object from a string.

        Parsing is hard, so we're going to call out to the pyparsing
        library here.  I hope you installed it!
        FTR: this is hideous.
        '''
        from pyparsing import Suppress, Regex, quotedString, restOfLine, Keyword, nestedExpr, Group, OneOrMore, Word, Literal, alphanums, removeQuotes, replaceWith, nums, printables
        gr_eq = Literal('=')
        gr_stripped_string = quotedString.copy().setParseAction( removeQuotes )
        gr_opt_quoted_string = gr_stripped_string | restOfLine
        gr_number = Word(nums)
        gr_yn = Keyword('yes', caseless=True).setParseAction(replaceWith('1')) | Keyword('no', caseless=True).setParseAction(replaceWith('0'))

        def _handle_password(*x):
            a,b,c =  x[2]
            if type(obj) == bacula_tools.Client: klass = PasswordStore
            elif type(obj) == bacula_tools.Storage: klass = StoragePasswordStore
            else: print("WTH is a", type(obj), "being passed in?", director_config)
            a = klass(obj[ID], self[ID])
            a.password = c
            a.store()
            return

        def _handle_monitor(*x):
            a,b,c =  x[2]
            a = PasswordStore(obj[ID], self[ID])
            a.monitor = c
            a.store()
            return

        def _handle_ip(*x):
            a,b,c =  x[2]
            return '  %s = { %s }' % (a,c[0])

        def _handle_diraddr(*x):
            a,b,c =  x[2]
            self._set(DIRADDRESSES, '  %s' % '\n  '.join(c))
            return

        def np(words, fn = gr_opt_quoted_string, action=None):
            p = Keyword(words[0], caseless=True).setDebug(bacula_tools.DEBUG)
            for w in words[1:]:
                p = p | Keyword(w, caseless=True).setDebug(bacula_tools.DEBUG)
            p = p + gr_eq + fn
            p.setParseAction(action)
            return p

        gr_name = np((NAME,), action=lambda x: self._set_name(x[2]))
        gr_address = np((ADDRESS,), action=self._parse_setter(ADDRESS))
        gr_fd_conn = np(PList('fd connect timeout'), gr_number, self._parse_setter(FD_CONNECT_TIMEOUT, True))
        gr_heart = np(PList('heartbeat interval'), gr_number, self._parse_setter(HEARTBEATINTERVAL, True))
        gr_max_con = np(PList('maximum console connections'),
                        gr_number, self._parse_setter(MAXIMUMCONSOLECONNECTIONS, True))
        gr_max_jobs = np(PList('maximum concurrent jobs'), gr_number, action=self._parse_setter(MAXIMUMCONCURRENTJOBS, True))
        gr_pid = np(PList('pid directory'), action=self._parse_setter(PIDDIRECTORY))
        gr_query = np(PList('query file'), action=self._parse_setter(QUERYFILE))
        gr_scripts = np(PList('scripts directory'), action=self._parse_setter(SCRIPTS_DIRECTORY))
        gr_sd_conn = np(PList('sd connect timeout'), gr_number, self._parse_setter(SD_CONNECT_TIMEOUT, True))
        gr_source = np(PList('source address'), action=self._parse_setter(SOURCEADDRESS))
        gr_stats = np(PList('statistics retention'), action=self._parse_setter(STATISTICS_RETENTION))
        gr_messages = np((MESSAGES,), action=self._parse_setter(MESSAGES_ID, dereference=True))
        gr_work_dir = np(PList('working directory'), action=self._parse_setter(WORKINGDIRECTORY))
        gr_port = np(PList('dir port'), gr_number, self._parse_setter(DIRPORT, True))
        gr_monitor = np((MONITOR,), gr_yn, action=_handle_monitor)

        # This is a complicated one
        da_addr = np(('Addr','Port'), Word(printables), lambda x,y,z: ' '.join(z))
        da_ip = np(('IPv4','IPv6','IP'), nestedExpr('{','}', OneOrMore(da_addr).setParseAction(lambda x,y,z: ' ; '.join(z)))).setParseAction(_handle_ip)
        da_addresses = np(PList('dir addresses'), nestedExpr('{','}', OneOrMore(da_ip)), _handle_diraddr)

        # if this isn't a director, then we ignore the password
        if director_config: gr_pass = np((PASSWORD,), action=self._parse_setter(PASSWORD))
        else: gr_pass = np((PASSWORD,), action=_handle_password)

        gr_res = OneOrMore(gr_name | gr_address | gr_fd_conn | gr_heart | gr_max_con | gr_max_jobs | gr_pass | gr_pid | gr_query | gr_scripts | gr_sd_conn | gr_source | gr_stats | gr_verid | gr_messages | gr_work_dir | gr_port | gr_monitor | da_addresses)

        result = gr_res.parseString(string, parseAll=True)
        return 'Director: ' + self[NAME]

    # }}}
    # {{{ __str__():

    def __str__(self):
        self.output = ['Director {\n  Name = "%(name)s"' % self,'}']
        for key in self.NULL_KEYS: self._simple_phrase(key)
        # set the messages
        m = self._fk_reference(MESSAGES_ID)
        self.output.insert(-1, '  %s = "%s"' % (MESSAGES.capitalize(), m[NAME]))
        if self[DIRADDRESSES]:
            self.output.insert(-1, '  %s {' % DIRADDRESSES.capitalize())
            self.output.insert(-1,  self[DIRADDRESSES])
            self.output.insert(-1, '  }')
        return '\n'.join(self.output)

# }}}
    # {{{ fd(): return the string that describes the filedaemon configuration

    def fd(self):
        '''This is what we'll call to dump out the config for the file daemon'''
        self.output = ['Director {\n  Name = "%(name)s"' % self, '}']
        if getattr(self,CLIENT_ID, None):
            a = PasswordStore(self.client_id, self[ID])
            self.output.insert(-1,'  Password = "%s"' % a.password)
            if a.monitor: self.output.insert(-1,'  Monitor = "yes"' )
        return '\n'.join(self.output)

    # }}}
    # {{{ _cli_special_setup(): setup the weird phrases that go with filesets

    def _cli_special_setup(self):
        group = optparse.OptionGroup(self.parser,
                                     "Object Setters",
                                     "Various objects associated with a Job")
        group.add_option('--message-set')
        self.parser.add_option_group(group)
        return

    # }}}
    # {{{ _cli_special_do_parse(args): handle the weird phrases that go with filesets

    def _cli_special_do_parse(self, args):
        if args.message_set == None: return
        if args.message_set =='': return self._set(MESSAGES_ID, None)
        target = Messages().search(args.message_set)
        if target[ID]: self._set(MESSAGES_ID, target[ID])
        else: print('Unable to find a match for %s, continuing' % args.message_set)
        return

# }}}
    # {{{ _cli_special_print(): print out the weird phrases that go with filesets

    def _cli_special_print(self):
        if not self[MESSAGES_ID]: return
        fmt = '%' + str(self._maxlen) + 's: %s'
        print(fmt % ('Messages', self._fk_reference(MESSAGES_ID)[NAME]))
        return
    # }}}

    def _cli_special_clone(self): pass

def main():
    s = Director()
    s.cli()

if __name__ == "__main__": main()
<commit_msg>Missed removing gr_verid<commit_after>#! /usr/bin/env python

from __future__ import print_function
try: from . import *
except: from bacula_tools import *

class Director(DbDict):
    SETUP_KEYS = [ADDRESS,
                 FD_CONNECT_TIMEOUT, HEARTBEATINTERVAL,
                 PASSWORD, PIDDIRECTORY, QUERYFILE,
                 SCRIPTS_DIRECTORY, SD_CONNECT_TIMEOUT, SOURCEADDRESS, STATISTICS_RETENTION,
                 WORKINGDIRECTORY, DIRADDRESSES]
    INT_KEYS = [(DIRPORT, 9101), MAXIMUMCONCURRENTJOBS, MAXIMUMCONSOLECONNECTIONS,]
    NULL_KEYS = [MESSAGES_ID, ]
    table = DIRECTORS
    IDTAG = 1
    # {{{ parse_string(string, director_config, obj): Entry point for a recursive descent parser

    def parse_string(self, string, director_config, obj):
        '''Populate a new object from a string.

        Parsing is hard, so we're going to call out to the pyparsing
        library here.  I hope you installed it!
        FTR: this is hideous.
        '''
        from pyparsing import Suppress, Regex, quotedString, restOfLine, Keyword, nestedExpr, Group, OneOrMore, Word, Literal, alphanums, removeQuotes, replaceWith, nums, printables
        gr_eq = Literal('=')
        gr_stripped_string = quotedString.copy().setParseAction( removeQuotes )
        gr_opt_quoted_string = gr_stripped_string | restOfLine
        gr_number = Word(nums)
        gr_yn = Keyword('yes', caseless=True).setParseAction(replaceWith('1')) | Keyword('no', caseless=True).setParseAction(replaceWith('0'))

        def _handle_password(*x):
            a,b,c =  x[2]
            if type(obj) == bacula_tools.Client: klass = PasswordStore
            elif type(obj) == bacula_tools.Storage: klass = StoragePasswordStore
            else: print("WTH is a", type(obj), "being passed in?", director_config)
            a = klass(obj[ID], self[ID])
            a.password = c
            a.store()
            return

        def _handle_monitor(*x):
            a,b,c =  x[2]
            a = PasswordStore(obj[ID], self[ID])
            a.monitor = c
            a.store()
            return

        def _handle_ip(*x):
            a,b,c =  x[2]
            return '  %s = { %s }' % (a,c[0])

        def _handle_diraddr(*x):
            a,b,c =  x[2]
            self._set(DIRADDRESSES, '  %s' % '\n  '.join(c))
            return

        def np(words, fn = gr_opt_quoted_string, action=None):
            p = Keyword(words[0], caseless=True).setDebug(bacula_tools.DEBUG)
            for w in words[1:]:
                p = p | Keyword(w, caseless=True).setDebug(bacula_tools.DEBUG)
            p = p + gr_eq + fn
            p.setParseAction(action)
            return p

        gr_name = np((NAME,), action=lambda x: self._set_name(x[2]))
        gr_address = np((ADDRESS,), action=self._parse_setter(ADDRESS))
        gr_fd_conn = np(PList('fd connect timeout'), gr_number, self._parse_setter(FD_CONNECT_TIMEOUT, True))
        gr_heart = np(PList('heartbeat interval'), gr_number, self._parse_setter(HEARTBEATINTERVAL, True))
        gr_max_con = np(PList('maximum console connections'),
                        gr_number, self._parse_setter(MAXIMUMCONSOLECONNECTIONS, True))
        gr_max_jobs = np(PList('maximum concurrent jobs'), gr_number, action=self._parse_setter(MAXIMUMCONCURRENTJOBS, True))
        gr_pid = np(PList('pid directory'), action=self._parse_setter(PIDDIRECTORY))
        gr_query = np(PList('query file'), action=self._parse_setter(QUERYFILE))
        gr_scripts = np(PList('scripts directory'), action=self._parse_setter(SCRIPTS_DIRECTORY))
        gr_sd_conn = np(PList('sd connect timeout'), gr_number, self._parse_setter(SD_CONNECT_TIMEOUT, True))
        gr_source = np(PList('source address'), action=self._parse_setter(SOURCEADDRESS))
        gr_stats = np(PList('statistics retention'), action=self._parse_setter(STATISTICS_RETENTION))
        gr_messages = np((MESSAGES,), action=self._parse_setter(MESSAGES_ID, dereference=True))
        gr_work_dir = np(PList('working directory'), action=self._parse_setter(WORKINGDIRECTORY))
        gr_port = np(PList('dir port'), gr_number, self._parse_setter(DIRPORT, True))
        gr_monitor = np((MONITOR,), gr_yn, action=_handle_monitor)

        # This is a complicated one
        da_addr = np(('Addr','Port'), Word(printables), lambda x,y,z: ' '.join(z))
        da_ip = np(('IPv4','IPv6','IP'), nestedExpr('{','}', OneOrMore(da_addr).setParseAction(lambda x,y,z: ' ; '.join(z)))).setParseAction(_handle_ip)
        da_addresses = np(PList('dir addresses'), nestedExpr('{','}', OneOrMore(da_ip)), _handle_diraddr)

        # if this isn't a director, then we ignore the password
        if director_config: gr_pass = np((PASSWORD,), action=self._parse_setter(PASSWORD))
        else: gr_pass = np((PASSWORD,), action=_handle_password)

        gr_res = OneOrMore(gr_name | gr_address | gr_fd_conn | gr_heart | gr_max_con | gr_max_jobs | gr_pass | gr_pid | gr_query | gr_scripts | gr_sd_conn | gr_source | gr_stats | gr_messages | gr_work_dir | gr_port | gr_monitor | da_addresses)

        result = gr_res.parseString(string, parseAll=True)
        return 'Director: ' + self[NAME]

    # }}}
    # {{{ __str__():

    def __str__(self):
        self.output = ['Director {\n  Name = "%(name)s"' % self,'}']
        for key in self.NULL_KEYS: self._simple_phrase(key)
        # set the messages
        m = self._fk_reference(MESSAGES_ID)
        self.output.insert(-1, '  %s = "%s"' % (MESSAGES.capitalize(), m[NAME]))
        if self[DIRADDRESSES]:
            self.output.insert(-1, '  %s {' % DIRADDRESSES.capitalize())
            self.output.insert(-1,  self[DIRADDRESSES])
            self.output.insert(-1, '  }')
        return '\n'.join(self.output)

# }}}
    # {{{ fd(): return the string that describes the filedaemon configuration

    def fd(self):
        '''This is what we'll call to dump out the config for the file daemon'''
        self.output = ['Director {\n  Name = "%(name)s"' % self, '}']
        if getattr(self,CLIENT_ID, None):
            a = PasswordStore(self.client_id, self[ID])
            self.output.insert(-1,'  Password = "%s"' % a.password)
            if a.monitor: self.output.insert(-1,'  Monitor = "yes"' )
        return '\n'.join(self.output)

    # }}}
    # {{{ _cli_special_setup(): setup the weird phrases that go with filesets

    def _cli_special_setup(self):
        group = optparse.OptionGroup(self.parser,
                                     "Object Setters",
                                     "Various objects associated with a Job")
        group.add_option('--message-set')
        self.parser.add_option_group(group)
        return

    # }}}
    # {{{ _cli_special_do_parse(args): handle the weird phrases that go with filesets

    def _cli_special_do_parse(self, args):
        if args.message_set == None: return
        if args.message_set =='': return self._set(MESSAGES_ID, None)
        target = Messages().search(args.message_set)
        if target[ID]: self._set(MESSAGES_ID, target[ID])
        else: print('Unable to find a match for %s, continuing' % args.message_set)
        return

# }}}
    # {{{ _cli_special_print(): print out the weird phrases that go with filesets

    def _cli_special_print(self):
        if not self[MESSAGES_ID]: return
        fmt = '%' + str(self._maxlen) + 's: %s'
        print(fmt % ('Messages', self._fk_reference(MESSAGES_ID)[NAME]))
        return
    # }}}

    def _cli_special_clone(self): pass

def main():
    s = Director()
    s.cli()

if __name__ == "__main__": main()
<|endoftext|>


<commit_before>from test.test_support import verbose, findfile, TestFailed, TestSkipped

import errno
import fcntl
import ossaudiodev
import os
import sys
import select
import sunaudio
import time
import audioop

SND_FORMAT_MULAW_8 = 1

def read_sound_file(path):
    fp = open(path, 'r')
    size, enc, rate, nchannels, extra = sunaudio.gethdr(fp)
    data = fp.read()
    fp.close()

    if enc != SND_FORMAT_MULAW_8:
        print "Expect .au file with 8-bit mu-law samples"
        return

    # Convert the data to 16-bit signed.
    data = audioop.ulaw2lin(data, 2)
    return (data, rate, 16, nchannels)

def play_sound_file(data, rate, ssize, nchannels):
    try:
        a = ossaudiodev.open('w')
    except ossaudiodev.error, msg:
        if msg[0] in (errno.EACCES, errno.ENODEV, errno.EBUSY):
            raise TestSkipped, msg
        raise TestFailed, msg

    # set the data format
    if sys.byteorder == 'little':
        fmt = ossaudiodev.AFMT_S16_LE
    else:
        fmt = ossaudiodev.AFMT_S16_BE

    # at least check that these methods can be invoked
    a.bufsize()
    a.obufcount()
    a.obuffree()
    a.getptr()
    a.fileno()

    # set parameters based on .au file headers
    a.setparameters(rate, 16, nchannels, fmt)
    a.write(data)
    a.flush()
    a.close()

def test_errors():
    a = ossaudiodev.open("w")
    size = 8
    fmt = ossaudiodev.AFMT_U8
    rate = 8000
    nchannels = 1
    try:
        a.setparameters(-1, size, nchannels, fmt)
    except ValueError, msg:
        print msg
    try:
        a.setparameters(rate, -2, nchannels, fmt)
    except ValueError, msg:
        print msg
    try:
        a.setparameters(rate, size, 3, fmt)
    except ValueError, msg:
        print msg
    try:
        a.setparameters(rate, size, nchannels, 177)
    except ValueError, msg:
        print msg
    try:
        a.setparameters(rate, size, nchannels, ossaudiodev.AFMT_U16_LE)
    except ValueError, msg:
        print msg
    try:
        a.setparameters(rate, 16, nchannels, fmt)
    except ValueError, msg:
        print msg

def test():
    (data, rate, ssize, nchannels) = read_sound_file(findfile('audiotest.au'))
    play_sound_file(data, rate, ssize, nchannels)
    test_errors()

test()
<commit_msg>- be explicit: audio data files should be opened in binary mode - ossaudiodev.open() raises IOError, not ossaudiodev.error, for cases   which get mapped to TestSkipped<commit_after>from test.test_support import verbose, findfile, TestFailed, TestSkipped

import errno
import fcntl
import ossaudiodev
import os
import sys
import select
import sunaudio
import time
import audioop

SND_FORMAT_MULAW_8 = 1

def read_sound_file(path):
    fp = open(path, 'rb')
    size, enc, rate, nchannels, extra = sunaudio.gethdr(fp)
    data = fp.read()
    fp.close()

    if enc != SND_FORMAT_MULAW_8:
        print "Expect .au file with 8-bit mu-law samples"
        return

    # Convert the data to 16-bit signed.
    data = audioop.ulaw2lin(data, 2)
    return (data, rate, 16, nchannels)

def play_sound_file(data, rate, ssize, nchannels):
    try:
        a = ossaudiodev.open('w')
    except IOError, msg:
        if msg[0] in (errno.EACCES, errno.ENODEV, errno.EBUSY):
            raise TestSkipped, msg
        raise TestFailed, msg

    # set the data format
    if sys.byteorder == 'little':
        fmt = ossaudiodev.AFMT_S16_LE
    else:
        fmt = ossaudiodev.AFMT_S16_BE

    # at least check that these methods can be invoked
    a.bufsize()
    a.obufcount()
    a.obuffree()
    a.getptr()
    a.fileno()

    # set parameters based on .au file headers
    a.setparameters(rate, 16, nchannels, fmt)
    a.write(data)
    a.flush()
    a.close()

def test_errors():
    a = ossaudiodev.open("w")
    size = 8
    fmt = ossaudiodev.AFMT_U8
    rate = 8000
    nchannels = 1
    try:
        a.setparameters(-1, size, nchannels, fmt)
    except ValueError, msg:
        print msg
    try:
        a.setparameters(rate, -2, nchannels, fmt)
    except ValueError, msg:
        print msg
    try:
        a.setparameters(rate, size, 3, fmt)
    except ValueError, msg:
        print msg
    try:
        a.setparameters(rate, size, nchannels, 177)
    except ValueError, msg:
        print msg
    try:
        a.setparameters(rate, size, nchannels, ossaudiodev.AFMT_U16_LE)
    except ValueError, msg:
        print msg
    try:
        a.setparameters(rate, 16, nchannels, fmt)
    except ValueError, msg:
        print msg

def test():
    (data, rate, ssize, nchannels) = read_sound_file(findfile('audiotest.au'))
    play_sound_file(data, rate, ssize, nchannels)
    test_errors()

test()
<|endoftext|>


<commit_before>#! /usr/bin/env python

try:
    import matplotlib as mpl
    mpl.use('Agg')
except ImportError:
    pass

from distutils.dir_util import mkpath
from distutils.errors import DistutilsFileError
import logging
import numpy as np
import pdb
import sys
from emcee.utils import MPIPool
sys.path.insert(0, '..')
import chronostar.expectmax as em
import chronostar.groupfitter as gf
import chronostar.synthesiser as syn
import chronostar.datatool as dt

try:
    ass_name = sys.argv[1]
except IndexError:
    print(" ---------------- INCORRECT USAGE ---------------\n"
          "  Usage: nohup mpirun -np 19 python\n"
          "         perform_em_association_fit.py [ass_name]\n"
          " ------------------------------------------------")
    print("Using bpmg as default...")
    ass_name = 'bpmg_cand_w_gaia_dr2_astrometry_comb_binars'

# Setting key parameters for fit
try:
    NGROUPS = int(sys.argv[2])
except (IndexError, ValueError):
    NGROUPS = 1

try:
    rdir = "/data/mash/tcrun/em_fit/{}_{}/".format(ass_name.strip('/'),
                                                   NGROUPS)
    gdir = "/data/mash/tcrun/" # directory with master gaia data
    path_msg = "Storing data on mash data server"
    mkpath(rdir)
except (IOError, DistutilsFileError):
    path_msg = ("I'm guessing you're not Tim Crundall..."
                "or not on an RSAA server")
    rdir = "../results/em_fit/{}_{}/".format(ass_name.strip('/'),
                                             NGROUPS)
    gdir = "../data/" # directory with master gaia data
    path_msg = "Storing data on mash data server"
    # if rdir[-1] != '/':
    #     rdir += '/'
    mkpath(rdir)

gaia_xyzuvw_file = gdir + 'gaia_dr2_mean_xyzuvw.npy'
xyzuvw_file = '../data/' + ass_name + '_xyzuvw.fits'
best_fit_file = rdir + "best_fit.npy"
bg_hist_file = "bg_hists.npy"
bg_ln_ols_file = rdir + 'bg_ln_ols.npy'

logging.basicConfig(
    level=logging.INFO, filemode='w',
    filename=rdir + 'em.log',
)

# Initialize the MPI-based pool used for parallelization.
using_mpi = True
try:
    pool = MPIPool()
    logging.info("Successfully initialised mpi pool")
except:
    #print("MPI doesn't seem to be installed... maybe install it?")
    logging.info("MPI doesn't seem to be installed... maybe install it?")
    using_mpi = False
    pool=None

if using_mpi:
    if not pool.is_master():
        print("One thread is going to sleep")
        # Wait for instructions from the master process.
        pool.wait()
        sys.exit(0)
print("Only one thread is master")

#logging.info(path_msg)

print("Master should be working in the directory:\n{}".format(rdir))

star_pars = dt.loadXYZUVW(xyzuvw_file)

# --------------------------------------------------------------------------
# Calculate background overlap
# --------------------------------------------------------------------------
# GET BACKGROUND LOG OVERLAP DENSITIES:
logging.info("Acquiring background overlaps")
try:
    print("In try")
    logging.info("Calculating background overlaps")
    logging.info(" -- this step employs scipy's kernel density estimator")
    logging.info(" -- so could take a few minutes...")
    bg_ln_ols = np.load(bg_ln_ols_file)
    print("could load")
    assert len(bg_ln_ols) == len(star_pars['xyzuvw'])
    logging.info("Loaded bg_ln_ols from file")
except (IOError, AssertionError):
    print("in except")
    bg_ln_ols = dt.getKernelDensities(gaia_xyzuvw_file, star_pars['xyzuvw'], amp_scale=0.1)
    np.save(bg_ln_ols_file, bg_ln_ols)
print("continuing")
logging.info("DONE!")

# --------------------------------------------------------------------------
# Get grid-based Z membership
# --------------------------------------------------------------------------

# Grid
grid_x = [-10000, -40, -10, 10, 40, 10000]
grid_y = [-10000, -10, 10, 10000]
grid_z = [-10000, 0, 10000]
xyzuvw = star_pars['xyzuvw']

ncomps=(len(grid_x)-1)*(len(grid_y)-1)*(len(grid_z)-1)
print('ncomps: %d'%ncomps)

# Create init_z
#init_z=[]
for x1, x2 in zip(grid_x[:-1], grid_x[1:]):
    for y1, y2 in zip(grid_y[:-1], grid_y[1:]):
        for z1, z2 in zip(grid_z[:-1], grid_z[1:]):
            mask = (xyzuvw[:,0]>x1) & (xyzuvw[:,0]<=x2)
            mask = mask & (xyzuvw[:,1]>y1) & (xyzuvw[:,1]<=y2)
            mask = mask & (xyzuvw[:,2]>z1) & (xyzuvw[:,2]<=z2)

            if len(init_z)<1:
                init_z = mask.astype(int)
            else:
                init_z = np.vstack((init_z, mask.astype(int)))
init_z=init_z.T

# --------------------------------------------------------------------------
# Perform one EM fit
# --------------------------------------------------------------------------
logging.info("Using data file {}".format(xyzuvw_file))
logging.info("Everything loaded, about to fit with {} components"\
    .format(NGROUPS))
final_groups, final_med_errs, z = em.fitManyGroups(star_pars, ncomps, bg_ln_ols=bg_ln_ols,
                 rdir=rdir, pool=pool, init_z=init_z, ignore_dead_comps=True)

# --------------------------------------------------------------------------
# Repeat EM fit but killing components with too few members
# --------------------------------------------------------------------------
# e.g. maybe < 3 expected star count
# take results from entire past EM fit

#
# init_origin = False
# origins = None
# if NGROUPS == 1:
#     init_origin = True
#     nstars = star_means.shape[0]
#     bp_mean = np.mean(star_means, axis=0)
#     bp_cov = np.cov(star_means.T)
#     bp_dx = np.sqrt(np.min([bp_cov[0,0], bp_cov[1,1], bp_cov[2,2]]))
#     bp_dv = np.sqrt(np.min([bp_cov[3,3], bp_cov[4,4], bp_cov[5,5]]))
#     bp_age = 0.5
#     bp_pars = np.hstack((bp_mean, bp_dx, bp_dv, bp_age, nstars))
#     bp_group = syn.Group(bp_pars)
#     origins = [bp_group]
#
# #go through and compare overlap with groups with
# #background overlap
# #
# #bp_pars = np.array([
# #   2.98170398e+01,  4.43573995e+01,  2.29251498e+01, -9.65731744e-01,
# #   -3.42827894e+00, -3.99928052e-02 , 2.63084094e+00,  1.05302890e-01,
# #   1.59367119e+01, nstars
# #])
# #bp_group = syn.Group(bp_pars)
# #
# # --------------------------------------------------------------------------
# # Run fit
# # --------------------------------------------------------------------------
# logging.info("Using data file {}".format(xyzuvw_file))
# logging.info("Everything loaded, about to fit with {} components"\
#     .format(NGROUPS))
# em.fitManyGroups(star_pars, NGROUPS,
#                  rdir=rdir, pool=pool, offset=True, bg_hist_file=bg_hist_file,
#                  origins=origins, init_with_origin=init_origin, init_z=init_z
#                  )
if using_mpi:
    pool.close()
<commit_msg>init_z<commit_after>#! /usr/bin/env python

try:
    import matplotlib as mpl
    mpl.use('Agg')
except ImportError:
    pass

from distutils.dir_util import mkpath
from distutils.errors import DistutilsFileError
import logging
import numpy as np
import pdb
import sys
from emcee.utils import MPIPool
sys.path.insert(0, '..')
import chronostar.expectmax as em
import chronostar.groupfitter as gf
import chronostar.synthesiser as syn
import chronostar.datatool as dt

try:
    ass_name = sys.argv[1]
except IndexError:
    print(" ---------------- INCORRECT USAGE ---------------\n"
          "  Usage: nohup mpirun -np 19 python\n"
          "         perform_em_association_fit.py [ass_name]\n"
          " ------------------------------------------------")
    print("Using bpmg as default...")
    ass_name = 'bpmg_cand_w_gaia_dr2_astrometry_comb_binars'

# Setting key parameters for fit
try:
    NGROUPS = int(sys.argv[2])
except (IndexError, ValueError):
    NGROUPS = 1

try:
    rdir = "/data/mash/tcrun/em_fit/{}_{}/".format(ass_name.strip('/'),
                                                   NGROUPS)
    gdir = "/data/mash/tcrun/" # directory with master gaia data
    path_msg = "Storing data on mash data server"
    mkpath(rdir)
except (IOError, DistutilsFileError):
    path_msg = ("I'm guessing you're not Tim Crundall..."
                "or not on an RSAA server")
    rdir = "../results/em_fit/{}_{}/".format(ass_name.strip('/'),
                                             NGROUPS)
    gdir = "../data/" # directory with master gaia data
    path_msg = "Storing data on mash data server"
    # if rdir[-1] != '/':
    #     rdir += '/'
    mkpath(rdir)

gaia_xyzuvw_file = gdir + 'gaia_dr2_mean_xyzuvw.npy'
xyzuvw_file = '../data/' + ass_name + '_xyzuvw.fits'
best_fit_file = rdir + "best_fit.npy"
bg_hist_file = "bg_hists.npy"
bg_ln_ols_file = rdir + 'bg_ln_ols.npy'

logging.basicConfig(
    level=logging.INFO, filemode='w',
    filename=rdir + 'em.log',
)

# Initialize the MPI-based pool used for parallelization.
using_mpi = True
try:
    pool = MPIPool()
    logging.info("Successfully initialised mpi pool")
except:
    #print("MPI doesn't seem to be installed... maybe install it?")
    logging.info("MPI doesn't seem to be installed... maybe install it?")
    using_mpi = False
    pool=None

if using_mpi:
    if not pool.is_master():
        print("One thread is going to sleep")
        # Wait for instructions from the master process.
        pool.wait()
        sys.exit(0)
print("Only one thread is master")

#logging.info(path_msg)

print("Master should be working in the directory:\n{}".format(rdir))

star_pars = dt.loadXYZUVW(xyzuvw_file)

# --------------------------------------------------------------------------
# Calculate background overlap
# --------------------------------------------------------------------------
# GET BACKGROUND LOG OVERLAP DENSITIES:
logging.info("Acquiring background overlaps")
try:
    print("In try")
    logging.info("Calculating background overlaps")
    logging.info(" -- this step employs scipy's kernel density estimator")
    logging.info(" -- so could take a few minutes...")
    bg_ln_ols = np.load(bg_ln_ols_file)
    print("could load")
    assert len(bg_ln_ols) == len(star_pars['xyzuvw'])
    logging.info("Loaded bg_ln_ols from file")
except (IOError, AssertionError):
    print("in except")
    bg_ln_ols = dt.getKernelDensities(gaia_xyzuvw_file, star_pars['xyzuvw'], amp_scale=0.1)
    np.save(bg_ln_ols_file, bg_ln_ols)
print("continuing")
logging.info("DONE!")

# --------------------------------------------------------------------------
# Get grid-based Z membership
# --------------------------------------------------------------------------

# Grid
grid_x = [-10000, -40, -10, 10, 40, 10000]
grid_y = [-10000, -10, 10, 10000]
grid_z = [-10000, 0, 10000]
xyzuvw = star_pars['xyzuvw']

ncomps=(len(grid_x)-1)*(len(grid_y)-1)*(len(grid_z)-1)
print('ncomps: %d'%ncomps)

# Create init_z
init_z=[]
for x1, x2 in zip(grid_x[:-1], grid_x[1:]):
    for y1, y2 in zip(grid_y[:-1], grid_y[1:]):
        for z1, z2 in zip(grid_z[:-1], grid_z[1:]):
            mask = (xyzuvw[:,0]>x1) & (xyzuvw[:,0]<=x2)
            mask = mask & (xyzuvw[:,1]>y1) & (xyzuvw[:,1]<=y2)
            mask = mask & (xyzuvw[:,2]>z1) & (xyzuvw[:,2]<=z2)

            if len(init_z)<1:
                init_z = mask.astype(int)
            else:
                init_z = np.vstack((init_z, mask.astype(int)))
init_z=init_z.T

# --------------------------------------------------------------------------
# Perform one EM fit
# --------------------------------------------------------------------------
logging.info("Using data file {}".format(xyzuvw_file))
logging.info("Everything loaded, about to fit with {} components"\
    .format(NGROUPS))
final_groups, final_med_errs, z = em.fitManyGroups(star_pars, ncomps, bg_ln_ols=bg_ln_ols,
                 rdir=rdir, pool=pool, init_z=init_z, ignore_dead_comps=True)

# --------------------------------------------------------------------------
# Repeat EM fit but killing components with too few members
# --------------------------------------------------------------------------
# e.g. maybe < 3 expected star count
# take results from entire past EM fit

#
# init_origin = False
# origins = None
# if NGROUPS == 1:
#     init_origin = True
#     nstars = star_means.shape[0]
#     bp_mean = np.mean(star_means, axis=0)
#     bp_cov = np.cov(star_means.T)
#     bp_dx = np.sqrt(np.min([bp_cov[0,0], bp_cov[1,1], bp_cov[2,2]]))
#     bp_dv = np.sqrt(np.min([bp_cov[3,3], bp_cov[4,4], bp_cov[5,5]]))
#     bp_age = 0.5
#     bp_pars = np.hstack((bp_mean, bp_dx, bp_dv, bp_age, nstars))
#     bp_group = syn.Group(bp_pars)
#     origins = [bp_group]
#
# #go through and compare overlap with groups with
# #background overlap
# #
# #bp_pars = np.array([
# #   2.98170398e+01,  4.43573995e+01,  2.29251498e+01, -9.65731744e-01,
# #   -3.42827894e+00, -3.99928052e-02 , 2.63084094e+00,  1.05302890e-01,
# #   1.59367119e+01, nstars
# #])
# #bp_group = syn.Group(bp_pars)
# #
# # --------------------------------------------------------------------------
# # Run fit
# # --------------------------------------------------------------------------
# logging.info("Using data file {}".format(xyzuvw_file))
# logging.info("Everything loaded, about to fit with {} components"\
#     .format(NGROUPS))
# em.fitManyGroups(star_pars, NGROUPS,
#                  rdir=rdir, pool=pool, offset=True, bg_hist_file=bg_hist_file,
#                  origins=origins, init_with_origin=init_origin, init_z=init_z
#                  )
if using_mpi:
    pool.close()
<|endoftext|>


<commit_before>"""
TAS CAS EAS convert test functions
"""
import numpy as np
from numpy.testing import (assert_almost_equal)

from pyfme.utils.tascaseas import (tas2eas, eas2tas, cas2eas, eas2cas, tas2cas,
                                   cas2tas)
from pyfme.environment.isa import atm

rho_0 = 1.225  # density at sea level (kg/m3)
p_0 = 101325  # pressure at sea level (Pa)
a_0 = 340.293990543  # sound speed at sea level (m/s)
gamma = 1.4  # heat capacity ratio

def test_tas2eas():

    # sea level
    tas = 275
    eas_expected = 275

    eas = tas2eas(tas, rho_0)

    assert_almost_equal(eas, eas_expected)

    # Test at 11000m
    _, _, rho = atm(11000)
    tas = 275
    eas_expected = 149.88797172756003

    eas = tas2eas(tas, rho)

    assert_almost_equal(eas, eas_expected)

def test_eas2tas():

    # sea level
    eas = 149.88797172756003
    tas_expected = 149.88797172756003

    tas = eas2tas(eas, rho_0)

    assert_almost_equal(tas, tas_expected)

    # Test at 11000m
    _, _, rho = atm(11000)
    eas = 149.88797172756003
    tas_expected = 275

    tas = eas2tas(eas, rho)

    assert_almost_equal(tas, tas_expected)

def test_tas2cas():

    # sea level
    tas = 275
    cas_expected = 275

    cas = tas2cas(tas, p_0, rho_0)

    assert_almost_equal(cas, cas_expected)

    # Test at 11000m
    _, p, rho = atm(11000)
    tas = 275
    cas_expected = 162.03569680495048

    cas = tas2cas(tas, p, rho)

    assert_almost_equal(cas, cas_expected)

def test_cas2tas():

    # sea level
    cas = 275
    tas_expected = 275

    tas = cas2tas(cas, p_0, rho_0)

    assert_almost_equal(tas, tas_expected)

    # Test at 11000m
    _, p, rho = atm(11000)
    cas = 162.03569680495048
    tas_expected = 275

    tas = cas2tas(cas, p, rho)

    assert_almost_equal(tas, tas_expected)

def test_cas2eas():

    # sea level
    cas = 275
    eas_expected = 275

    eas = cas2eas(cas, p_0, rho_0)

    assert_almost_equal(eas, eas_expected)

    # Test at 11000m
    _, p, rho = atm(11000)
    cas = 162.03569680495048
    eas_expected = 149.88797172756003

    eas = cas2eas(cas, p, rho)

    assert_almost_equal(eas, eas_expected)

def test_eas2cas():

    # sea level
    eas = 275
    cas_expected = 275

    cas = eas2cas(eas, p_0, rho_0)

    assert_almost_equal(cas, cas_expected)

    # Test at 11000m
    _, p, rho = atm(11000)
    eas = 149.88797172756003
    cas_expected = 162.03569680495048

    cas = eas2cas(eas, p, rho)

    assert_almost_equal(cas, cas_expected)
<commit_msg>tests updated<commit_after>"""
TAS CAS EAS convert test functions
"""
import numpy as np
from numpy.testing import (assert_almost_equal)

from pyfme.utils.tascaseas import (tas2eas, eas2tas, cas2eas, eas2cas, tas2cas,
                                   cas2tas)
from pyfme.environment.isa import atm

rho_0 = 1.225  # density at sea level (kg/m3)
p_0 = 101325  # pressure at sea level (Pa)
a_0 = 340.293990543  # sound speed at sea level (m/s)
gamma = 1.4  # heat capacity ratio

def test_tas2eas():

    # sea level
    tas = 275
    eas_expected = 275

    eas = tas2eas(tas, rho_0)

    assert_almost_equal(eas, eas_expected)

    # Test at 11000m
    _, _, rho, _ = atm(11000)
    tas = 275
    eas_expected = 149.88797172756003

    eas = tas2eas(tas, rho)

    assert_almost_equal(eas, eas_expected)

def test_eas2tas():

    # sea level
    eas = 149.88797172756003
    tas_expected = 149.88797172756003

    tas = eas2tas(eas, rho_0)

    assert_almost_equal(tas, tas_expected)

    # Test at 11000m
    _, _, rho, _ = atm(11000)
    eas = 149.88797172756003
    tas_expected = 275

    tas = eas2tas(eas, rho)

    assert_almost_equal(tas, tas_expected)

def test_tas2cas():

    # sea level
    tas = 275
    cas_expected = 275

    cas = tas2cas(tas, p_0, rho_0)

    assert_almost_equal(cas, cas_expected)

    # Test at 11000m
    _, p, rho, _ = atm(11000)
    tas = 275
    cas_expected = 162.03569680495048

    cas = tas2cas(tas, p, rho)

    assert_almost_equal(cas, cas_expected)

def test_cas2tas():

    # sea level
    cas = 275
    tas_expected = 275

    tas = cas2tas(cas, p_0, rho_0)

    assert_almost_equal(tas, tas_expected)

    # Test at 11000m
    _, p, rho, _ = atm(11000)
    cas = 162.03569680495048
    tas_expected = 275

    tas = cas2tas(cas, p, rho)

    assert_almost_equal(tas, tas_expected)

def test_cas2eas():

    # sea level
    cas = 275
    eas_expected = 275

    eas = cas2eas(cas, p_0, rho_0)

    assert_almost_equal(eas, eas_expected)

    # Test at 11000m
    _, p, rho, _ = atm(11000)
    cas = 162.03569680495048
    eas_expected = 149.88797172756003

    eas = cas2eas(cas, p, rho)

    assert_almost_equal(eas, eas_expected)

def test_eas2cas():

    # sea level
    eas = 275
    cas_expected = 275

    cas = eas2cas(eas, p_0, rho_0)

    assert_almost_equal(cas, cas_expected)

    # Test at 11000m
    _, p, rho, _ = atm(11000)
    eas = 149.88797172756003
    cas_expected = 162.03569680495048

    cas = eas2cas(eas, p, rho)

    assert_almost_equal(cas, cas_expected)
<|endoftext|>


<commit_before># encoding: utf-8
import sys, os, time, threading, thread, logging, imp
from smisk.core import Application, URL

None2 = (None, None)
''':type: tuple'''

log = logging.getLogger(__name__)

class PerpetualTimer(threading._Timer):
  '''A subclass of threading._Timer whose run() method repeats.'''
  ident = None
  ''':type: int'''

  def __init__(self, frequency, callback, setup_callback=None, *args, **kwargs):
    threading._Timer.__init__(self, frequency, callback, *args, **kwargs)
    self.setup_callback = setup_callback

  def run(self):
    self.ident = thread.get_ident()
    if self.setup_callback is not None:
      self.setup_callback()
    while True:
      self.finished.wait(self.interval)
      if self.finished.isSet():
        return
      self.function(*self.args, **self.kwargs)

class Monitor(object):
  '''Periodically run a callback in its own thread.'''
  frequency = 60.0
  ''':type: float'''

  def __init__(self, callback, setup_callback=None, frequency=60.0):
    self.callback = callback
    self.setup_callback = setup_callback
    self.frequency = frequency
    self.thread = None

  def start(self):
    '''Start our callback in its own perpetual timer thread.'''
    if self.frequency > 0:
      threadname = "Monitor:%s" % self.__class__.__name__
      if self.thread is None:
        self.thread = PerpetualTimer(self.frequency, self.callback, self.setup_callback)
        self.thread.setName(threadname)
        self.thread.start()
        log.debug("Started thread %r", threadname)
      else:
        log.debug("Thread %r already started", threadname)
  start.priority = 70

  def stop(self):
    '''Stop our callback's perpetual timer thread.'''
    if self.thread is None:
      log.warn("No thread running for %s", self)
    else:
      # Note: For some reason threading._active dict freezes in some conditions
      # here, so we compare thread ids rather than comparing using threading.currentThread.
      if self.thread.ident != thread.get_ident():
        self.thread.cancel()
        self.thread.join()
        log.debug("Stopped thread %r", self.thread.getName())
      self.thread = None

  def restart(self):
    '''Stop the callback's perpetual timer thread and restart it.'''
    self.stop()
    self.start()

class Timer(object):
  '''A simple universal timer.'''
  def __init__(self, start=True):
    self.t0 = 0.0
    self.t1 = 0.0
    self.seconds = self.time
    if start:
      self.start()

  def start(self):
    self.t0 = time.time()

  def finish(self):
    self.t1 = time.time()
    return "%.0fs %.0fms %.0fus" % (self.seconds(), self.milli(), self.micro())

  def time(self):
    return self.t1 - self.t0

  def seconds(self): # alias for time
    return self.time()

  def milli(self):
    return (self.time() * 1000) % 1000

  def micro(self):
    return (self.time() * 1000000) % 1000

def parse_qvalue_header(s, accept_any_equals='*/*', partial_endswith='/*'):
  '''Parse a qvalue HTTP header'''
  vqs = []
  highqs = []
  partials = []
  accept_any = False

  if not partial_endswith:
    partial_endswith = None

  for part in s.split(','):
    part = part.strip(' ')
    p = part.find(';')
    if p != -1:
      # todo Find out what the undocumented, but revealed, level= tags in HTTP 1.1
      #      really mean and if they exists in reality. As they are not documented,
      #      we will not implement support for it. [RFC 2616, chapter 14.1 "Accept"]
      pp = part.find('q=', p)
      if pp != -1:
        q = int(float(part[pp+2:])*100.0)
        part = part[:p]
        vqs.append([part, q])
        if q == 100:
          highqs.append(part)
        continue
    # No qvalue; we use three classes: any (q=0), partial (q=50) and complete (q=100)
    qual = 100
    if part == accept_any_equals:
      qual = 0
      accept_any = True
    else:
      if partial_endswith is not None and part.endswith('/*'):
        partial = part[:-2]
        if not partial:
          continue
        qual = 50
        partials.append(partial) # remove last char '*'
      else:
        highqs.append(part)
    vqs.append([part, qual])
  # Order by qvalue
  vqs.sort(lambda a,b: b[1] - a[1])
  return vqs, highqs, partials, accept_any

def to_bool(o):
  if type(o) in (str, unicode, basestring):
    try:
      return int(o)
    except ValueError:
      o = o.lower()
      return o == 'true' or o == 'yes' or o == 'on' or o == 'enabled'
  else:
    return o

def wrap_exc_in_callable(exc):
  '''Wrap exc in a anonymous function, for later raising.

  :rtype: callable
  '''
  def a(*args, **kwargs):
    raise exc
  return a

def tokenize_path(path):
  '''Deconstruct a URI path into standardized tokens.

  :param path: A pathname
  :type  path: string
  :rtype: list'''
  tokens = []
  for tok in strip_filename_extension(path).split('/'):
    tok = URL.decode(tok)
    if len(tok):
      tokens.append(tok)
  return tokens

def strip_filename_extension(fn):
  '''Remove any file extension from filename.

  :rtype: string
  '''
  try:
    return fn[:fn.rindex('.')]
  except:
    return fn

def load_modules_in_dir(path, skip__init__=True):
  '''Import all modules in a directory.

  :returns: A list of modules imported
  :rtype:   list'''
  loaded = []
  for f in os.listdir(path):
    name = strip_filename_extension(f)
    if skip__init__ and name == '__init__':
      continue
    if f[0] != '.' and f[-3:] in ('.py', 'pyc') and f not in loaded:
      fp, pathname, desc = imp.find_module(name, [path])
      try:
        imp.load_module(name, fp, pathname, desc)
      finally:
        if fp:
          fp.close()
      loaded.append(f)
  return loaded

def normalize_url(url):
  ''':rtype: string'''
  if url.find('://') == -1:
    # url is actually a path
    path = url
    url = Application.current.request.url
    if len(path) == 0:
      path = '/'
    elif path[0] != '/':
      path = os.path.normpath(url.path) + '/' + os.path.normpath(path)
    else:
      path = os.path.normpath(path)
    url = url.to_s(port=url.port!=80, path=False) + path
  return url

def unique_sorted_modules_of_items(v):
  ''':rtype: list'''
  s = []
  for t in v:
    s.append(t.__module__)
  s = list_unique_wild(s)
  s.sort()
  return s

def list_python_filenames_in_dir(path, only_py=True):
  ''':rtype: list'''
  names = []
  for fn in os.listdir(path):
    if fn[-3:] == '.py':
      names.append(fn[:-3])
    elif not only_py:
      fn4 = fn[-4:]
      if fn4 == '.pyc' or fn4 == '.pyo':
        names.append(fn[:-4])
  if names:
    if not only_py:
      names = list_unique_wild(names)
    names.sort()
  return names

def find_modules_for_classtree(cls, exclude_root=True, unique=True):
  '''Returns a list of all modules in which cls and any subclasses are defined.

  :rtype: list
  '''
  if exclude_root:
    modules = []
  else:
    try:
      modules = [sys.modules[cls.__module__]]
    except KeyError:
      modules = [__import__(cls.__module__, globals(), locals())]
  for subcls in cls.__subclasses__():
    modules.extend(find_modules_for_classtree(subcls, False, False))
  if unique:
    modules = list_unique(modules)
  return modules

def inspect(o, as_string=True):
  '''Returns a dictionary or string of all members and their values in object `o`.

  :param o:         Object to inspect
  :type  o:         object
  :param as_string: Return a string formatted KEY=VALUE\\n...
  :type  as_string: bool
  :returns: string or dict
  :rtype: object
  '''
  items = {}
  for k in dir(o):
    items[k] = eval('o.'+k)
  if as_string:
    return "\n".join(['%s=%r' % kv for kv in items.items()])
  return items

def list_unique_wild(seq):
  '''
  :param seq:
  :type  seq: list
  :rtype: list'''
  # Not order preserving but faster than list_unique
  return list(set(seq))

def list_unique(seq):
  '''Return a list of the elements in s, but without duplicates.

  For example, unique([1,2,3,1,2,3]) is some permutation of [1,2,3],
  unique("abcabc") some permutation of ["a", "b", "c"], and
  unique(([1, 2], [2, 3], [1, 2])) some permutation of
  [[2, 3], [1, 2]].

  For best speed, all sequence elements should be hashable.  Then
  unique() will usually work in linear time.

  If not possible, the sequence elements should enjoy a total
  ordering, and if list(s).sort() doesn't raise TypeError it's
  assumed that they do enjoy a total ordering.  Then unique() will
  usually work in O(N*log2(N)) time.

  If that's not possible either, the sequence elements must support
  equality-testing.  Then unique() will usually work in quadratic
  time.

  :param seq:
  :type  seq: list
  :rtype: list
  '''
  n = len(seq)
  if n == 0:
    return []
  # Try using a dict first, as that's the fastest and will usually
  # work.  If it doesn't work, it will usually fail quickly, so it
  # usually doesn't cost much to *try* it.  It requires that all the
  # sequence elements be hashable, and support equality comparison.
  u = {}
  try:
    for x in seq:
      u[x] = 1
  except TypeError:
    del u  # move on to the next method
  else:
    return u.keys()
  # We can't hash all the elements.  Second fastest is to sort,
  # which brings the equal elements together; then duplicates are
  # easy to weed out in a single pass.
  # NOTE:  Python's list.sort() was designed to be efficient in the
  # presence of many duplicate elements.  This isn't true of all
  # sort functions in all languages or libraries, so this approach
  # is more effective in Python than it may be elsewhere.
  try:
    t = list(seq)
    t.sort()
  except TypeError:
    del t  # move on to the next method
  else:
    assert n > 0
    last = t[0]
    lasti = i = 1
    while i < n:
      if t[i] != last:
        t[lasti] = last = t[i]
        lasti += 1
      i += 1
    return t[:lasti]

  # Brute force is all that's left.
  u = []
  for x in seq:
    if x not in u:
      u.append(x)
  return u

def format_exc(exc=None, as_string=False):
  ''':rtype: string'''
  if exc is None:
    exc = sys.exc_info()
  if exc == (None, None, None):
    return ''
  import traceback
  if as_string:
    return ''.join(traceback.format_exception(*exc))
  else:
    return traceback.format_exception(*exc)
<commit_msg>Fixed a bug where the same module was imported twice<commit_after># encoding: utf-8
import sys, os, time, threading, thread, logging, imp
from smisk.core import Application, URL

None2 = (None, None)
''':type: tuple'''

log = logging.getLogger(__name__)

class PerpetualTimer(threading._Timer):
  '''A subclass of threading._Timer whose run() method repeats.'''
  ident = None
  ''':type: int'''

  def __init__(self, frequency, callback, setup_callback=None, *args, **kwargs):
    threading._Timer.__init__(self, frequency, callback, *args, **kwargs)
    self.setup_callback = setup_callback

  def run(self):
    self.ident = thread.get_ident()
    if self.setup_callback is not None:
      self.setup_callback()
    while True:
      self.finished.wait(self.interval)
      if self.finished.isSet():
        return
      self.function(*self.args, **self.kwargs)

class Monitor(object):
  '''Periodically run a callback in its own thread.'''
  frequency = 60.0
  ''':type: float'''

  def __init__(self, callback, setup_callback=None, frequency=60.0):
    self.callback = callback
    self.setup_callback = setup_callback
    self.frequency = frequency
    self.thread = None

  def start(self):
    '''Start our callback in its own perpetual timer thread.'''
    if self.frequency > 0:
      threadname = "Monitor:%s" % self.__class__.__name__
      if self.thread is None:
        self.thread = PerpetualTimer(self.frequency, self.callback, self.setup_callback)
        self.thread.setName(threadname)
        self.thread.start()
        log.debug("Started thread %r", threadname)
      else:
        log.debug("Thread %r already started", threadname)
  start.priority = 70

  def stop(self):
    '''Stop our callback's perpetual timer thread.'''
    if self.thread is None:
      log.warn("No thread running for %s", self)
    else:
      # Note: For some reason threading._active dict freezes in some conditions
      # here, so we compare thread ids rather than comparing using threading.currentThread.
      if self.thread.ident != thread.get_ident():
        self.thread.cancel()
        self.thread.join()
        log.debug("Stopped thread %r", self.thread.getName())
      self.thread = None

  def restart(self):
    '''Stop the callback's perpetual timer thread and restart it.'''
    self.stop()
    self.start()

class Timer(object):
  '''A simple universal timer.'''
  def __init__(self, start=True):
    self.t0 = 0.0
    self.t1 = 0.0
    self.seconds = self.time
    if start:
      self.start()

  def start(self):
    self.t0 = time.time()

  def finish(self):
    self.t1 = time.time()
    return "%.0fs %.0fms %.0fus" % (self.seconds(), self.milli(), self.micro())

  def time(self):
    return self.t1 - self.t0

  def seconds(self): # alias for time
    return self.time()

  def milli(self):
    return (self.time() * 1000) % 1000

  def micro(self):
    return (self.time() * 1000000) % 1000

def parse_qvalue_header(s, accept_any_equals='*/*', partial_endswith='/*'):
  '''Parse a qvalue HTTP header'''
  vqs = []
  highqs = []
  partials = []
  accept_any = False

  if not partial_endswith:
    partial_endswith = None

  for part in s.split(','):
    part = part.strip(' ')
    p = part.find(';')
    if p != -1:
      # todo Find out what the undocumented, but revealed, level= tags in HTTP 1.1
      #      really mean and if they exists in reality. As they are not documented,
      #      we will not implement support for it. [RFC 2616, chapter 14.1 "Accept"]
      pp = part.find('q=', p)
      if pp != -1:
        q = int(float(part[pp+2:])*100.0)
        part = part[:p]
        vqs.append([part, q])
        if q == 100:
          highqs.append(part)
        continue
    # No qvalue; we use three classes: any (q=0), partial (q=50) and complete (q=100)
    qual = 100
    if part == accept_any_equals:
      qual = 0
      accept_any = True
    else:
      if partial_endswith is not None and part.endswith('/*'):
        partial = part[:-2]
        if not partial:
          continue
        qual = 50
        partials.append(partial) # remove last char '*'
      else:
        highqs.append(part)
    vqs.append([part, qual])
  # Order by qvalue
  vqs.sort(lambda a,b: b[1] - a[1])
  return vqs, highqs, partials, accept_any

def to_bool(o):
  if type(o) in (str, unicode, basestring):
    try:
      return int(o)
    except ValueError:
      o = o.lower()
      return o == 'true' or o == 'yes' or o == 'on' or o == 'enabled'
  else:
    return o

def wrap_exc_in_callable(exc):
  '''Wrap exc in a anonymous function, for later raising.

  :rtype: callable
  '''
  def a(*args, **kwargs):
    raise exc
  return a

def tokenize_path(path):
  '''Deconstruct a URI path into standardized tokens.

  :param path: A pathname
  :type  path: string
  :rtype: list'''
  tokens = []
  for tok in strip_filename_extension(path).split('/'):
    tok = URL.decode(tok)
    if len(tok):
      tokens.append(tok)
  return tokens

def strip_filename_extension(fn):
  '''Remove any file extension from filename.

  :rtype: string
  '''
  try:
    return fn[:fn.rindex('.')]
  except:
    return fn

def load_modules_in_dir(path, skip__init__=True):
  '''Import all modules in a directory.

  :returns: A list of modules imported
  :rtype:   list'''
  loaded = []
  for f in os.listdir(path):
    name = strip_filename_extension(f)
    if skip__init__ and name == '__init__':
      continue
    if f[0] != '.' and f[-3:] in ('.py', 'pyc') and name not in loaded:
      fp, pathname, desc = imp.find_module(name, [path])
      try:
        imp.load_module(name, fp, pathname, desc)
      finally:
        if fp:
          fp.close()
      loaded.append(name)
  return loaded

def normalize_url(url):
  ''':rtype: string'''
  if url.find('://') == -1:
    # url is actually a path
    path = url
    url = Application.current.request.url
    if len(path) == 0:
      path = '/'
    elif path[0] != '/':
      path = os.path.normpath(url.path) + '/' + os.path.normpath(path)
    else:
      path = os.path.normpath(path)
    url = url.to_s(port=url.port!=80, path=False) + path
  return url

def unique_sorted_modules_of_items(v):
  ''':rtype: list'''
  s = []
  for t in v:
    s.append(t.__module__)
  s = list_unique_wild(s)
  s.sort()
  return s

def list_python_filenames_in_dir(path, only_py=True):
  ''':rtype: list'''
  names = []
  for fn in os.listdir(path):
    if fn[-3:] == '.py':
      names.append(fn[:-3])
    elif not only_py:
      fn4 = fn[-4:]
      if fn4 == '.pyc' or fn4 == '.pyo':
        names.append(fn[:-4])
  if names:
    if not only_py:
      names = list_unique_wild(names)
    names.sort()
  return names

def find_modules_for_classtree(cls, exclude_root=True, unique=True):
  '''Returns a list of all modules in which cls and any subclasses are defined.

  :rtype: list
  '''
  if exclude_root:
    modules = []
  else:
    try:
      modules = [sys.modules[cls.__module__]]
    except KeyError:
      modules = [__import__(cls.__module__, globals(), locals())]
  for subcls in cls.__subclasses__():
    modules.extend(find_modules_for_classtree(subcls, False, False))
  if unique:
    modules = list_unique(modules)
  return modules

def inspect(o, as_string=True):
  '''Returns a dictionary or string of all members and their values in object `o`.

  :param o:         Object to inspect
  :type  o:         object
  :param as_string: Return a string formatted KEY=VALUE\\n...
  :type  as_string: bool
  :returns: string or dict
  :rtype: object
  '''
  items = {}
  for k in dir(o):
    items[k] = eval('o.'+k)
  if as_string:
    return "\n".join(['%s=%r' % kv for kv in items.items()])
  return items

def list_unique_wild(seq):
  '''
  :param seq:
  :type  seq: list
  :rtype: list'''
  # Not order preserving but faster than list_unique
  return list(set(seq))

def list_unique(seq):
  '''Return a list of the elements in s, but without duplicates.

  For example, unique([1,2,3,1,2,3]) is some permutation of [1,2,3],
  unique("abcabc") some permutation of ["a", "b", "c"], and
  unique(([1, 2], [2, 3], [1, 2])) some permutation of
  [[2, 3], [1, 2]].

  For best speed, all sequence elements should be hashable.  Then
  unique() will usually work in linear time.

  If not possible, the sequence elements should enjoy a total
  ordering, and if list(s).sort() doesn't raise TypeError it's
  assumed that they do enjoy a total ordering.  Then unique() will
  usually work in O(N*log2(N)) time.

  If that's not possible either, the sequence elements must support
  equality-testing.  Then unique() will usually work in quadratic
  time.

  :param seq:
  :type  seq: list
  :rtype: list
  '''
  n = len(seq)
  if n == 0:
    return []
  # Try using a dict first, as that's the fastest and will usually
  # work.  If it doesn't work, it will usually fail quickly, so it
  # usually doesn't cost much to *try* it.  It requires that all the
  # sequence elements be hashable, and support equality comparison.
  u = {}
  try:
    for x in seq:
      u[x] = 1
  except TypeError:
    del u  # move on to the next method
  else:
    return u.keys()
  # We can't hash all the elements.  Second fastest is to sort,
  # which brings the equal elements together; then duplicates are
  # easy to weed out in a single pass.
  # NOTE:  Python's list.sort() was designed to be efficient in the
  # presence of many duplicate elements.  This isn't true of all
  # sort functions in all languages or libraries, so this approach
  # is more effective in Python than it may be elsewhere.
  try:
    t = list(seq)
    t.sort()
  except TypeError:
    del t  # move on to the next method
  else:
    assert n > 0
    last = t[0]
    lasti = i = 1
    while i < n:
      if t[i] != last:
        t[lasti] = last = t[i]
        lasti += 1
      i += 1
    return t[:lasti]

  # Brute force is all that's left.
  u = []
  for x in seq:
    if x not in u:
      u.append(x)
  return u

def format_exc(exc=None, as_string=False):
  ''':rtype: string'''
  if exc is None:
    exc = sys.exc_info()
  if exc == (None, None, None):
    return ''
  import traceback
  if as_string:
    return ''.join(traceback.format_exception(*exc))
  else:
    return traceback.format_exception(*exc)
<|endoftext|>