<commit_before>import itertools
import sys
import struct
from io import open, BytesIO, SEEK_CUR, SEEK_END  # noqa

PY2 = sys.version_info[0] == 2

# Kaitai Struct runtime streaming API version, defined as per PEP-0396
# standard. Used for two purposes:
#
# * .py files generated by ksc from .ksy check that they import proper
#   KS runtime library by this version number;
# * distribution utils (setup.py) use this when packaging for PyPI
#
__version__ = '0.9'

class KaitaiStruct(object):
    def __init__(self, stream):
        self._io = stream

    def __enter__(self):
        return self

    def __exit__(self, *args, **kwargs):
        self.close()

    def close(self):
        self._io.close()

    @classmethod
    def from_file(cls, filename):
        f = open(filename, 'rb')
        try:
            return cls(KaitaiStream(f))
        except Exception:
            # close file descriptor, then reraise the exception
            f.close()
            raise

    @classmethod
    def from_bytes(cls, buf):
        return cls(KaitaiStream(BytesIO(buf)))

    @classmethod
    def from_io(cls, io):
        return cls(KaitaiStream(io))

class KaitaiStream(object):
    def __init__(self, io):
        self._io = io
        self.align_to_byte()

    def __enter__(self):
        return self

    def __exit__(self, *args, **kwargs):
        self.close()

    def close(self):
        self._io.close()

    # ========================================================================
    # Stream positioning
    # ========================================================================

    def is_eof(self):
        if self.bits_left > 0:
            return False

        io = self._io
        t = io.read(1)
        if t == b'':
            return True
        else:
            io.seek(-1, SEEK_CUR)
            return False

    def seek(self, n):
        self._io.seek(n)

    def pos(self):
        return self._io.tell()

    def size(self):
        # Python has no internal File object API function to get
        # current file / StringIO size, thus we use the following
        # trick.
        io = self._io
        # Remember our current position
        cur_pos = io.tell()
        # Seek to the end of the File object
        io.seek(0, SEEK_END)
        # Remember position, which is equal to the full length
        full_size = io.tell()
        # Seek back to the current position
        io.seek(cur_pos)
        return full_size

    # ========================================================================
    # Integer numbers
    # ========================================================================

    packer_s1 = struct.Struct('b')
    packer_s2be = struct.Struct('>h')
    packer_s4be = struct.Struct('>i')
    packer_s8be = struct.Struct('>q')
    packer_s2le = struct.Struct('<h')
    packer_s4le = struct.Struct('<i')
    packer_s8le = struct.Struct('<q')

    packer_u1 = struct.Struct('B')
    packer_u2be = struct.Struct('>H')
    packer_u4be = struct.Struct('>I')
    packer_u8be = struct.Struct('>Q')
    packer_u2le = struct.Struct('<H')
    packer_u4le = struct.Struct('<I')
    packer_u8le = struct.Struct('<Q')

    # ------------------------------------------------------------------------
    # Signed
    # ------------------------------------------------------------------------

    def read_s1(self):
        return KaitaiStream.packer_s1.unpack(self.read_bytes(1))[0]

    # ........................................................................
    # Big-endian
    # ........................................................................

    def read_s2be(self):
        return KaitaiStream.packer_s2be.unpack(self.read_bytes(2))[0]

    def read_s4be(self):
        return KaitaiStream.packer_s4be.unpack(self.read_bytes(4))[0]

    def read_s8be(self):
        return KaitaiStream.packer_s8be.unpack(self.read_bytes(8))[0]

    # ........................................................................
    # Little-endian
    # ........................................................................

    def read_s2le(self):
        return KaitaiStream.packer_s2le.unpack(self.read_bytes(2))[0]

    def read_s4le(self):
        return KaitaiStream.packer_s4le.unpack(self.read_bytes(4))[0]

    def read_s8le(self):
        return KaitaiStream.packer_s8le.unpack(self.read_bytes(8))[0]

    # ------------------------------------------------------------------------
    # Unsigned
    # ------------------------------------------------------------------------

    def read_u1(self):
        return KaitaiStream.packer_u1.unpack(self.read_bytes(1))[0]

    # ........................................................................
    # Big-endian
    # ........................................................................

    def read_u2be(self):
        return KaitaiStream.packer_u2be.unpack(self.read_bytes(2))[0]

    def read_u4be(self):
        return KaitaiStream.packer_u4be.unpack(self.read_bytes(4))[0]

    def read_u8be(self):
        return KaitaiStream.packer_u8be.unpack(self.read_bytes(8))[0]

    # ........................................................................
    # Little-endian
    # ........................................................................

    def read_u2le(self):
        return KaitaiStream.packer_u2le.unpack(self.read_bytes(2))[0]

    def read_u4le(self):
        return KaitaiStream.packer_u4le.unpack(self.read_bytes(4))[0]

    def read_u8le(self):
        return KaitaiStream.packer_u8le.unpack(self.read_bytes(8))[0]

    # ========================================================================
    # Floating point numbers
    # ========================================================================

    packer_f4be = struct.Struct('>f')
    packer_f8be = struct.Struct('>d')
    packer_f4le = struct.Struct('<f')
    packer_f8le = struct.Struct('<d')

    # ........................................................................
    # Big-endian
    # ........................................................................

    def read_f4be(self):
        return KaitaiStream.packer_f4be.unpack(self.read_bytes(4))[0]

    def read_f8be(self):
        return KaitaiStream.packer_f8be.unpack(self.read_bytes(8))[0]

    # ........................................................................
    # Little-endian
    # ........................................................................

    def read_f4le(self):
        return KaitaiStream.packer_f4le.unpack(self.read_bytes(4))[0]

    def read_f8le(self):
        return KaitaiStream.packer_f8le.unpack(self.read_bytes(8))[0]

    # ========================================================================
    # Unaligned bit values
    # ========================================================================

    def align_to_byte(self):
        self.bits_left = 0
        self.bits = 0

    def read_bits_int_be(self, n):
        res = 0

        bits_needed = n - self.bits_left
        self.bits_left = -bits_needed % 8

        if bits_needed > 0:
            # 1 bit  => 1 byte
            # 8 bits => 1 byte
            # 9 bits => 2 bytes
            bytes_needed = ((bits_needed - 1) // 8) + 1 # `ceil(bits_needed / 8)`
            buf = self.read_bytes(bytes_needed)
            if PY2:
                buf = bytearray(buf)
            for byte in buf:
                res = res << 8 | byte

            new_bits = res
            res = res >> self.bits_left | self.bits << bits_needed
            self.bits = new_bits # will be masked at the end of the function
        else:
            res = self.bits >> -bits_needed # shift unneeded bits out

        mask = (1 << self.bits_left) - 1 # `bits_left` is in range 0..7
        self.bits &= mask

        return res

    # Unused since Kaitai Struct Compiler v0.9+ - compatibility with
    # older versions.
    def read_bits_int(self, n):
        return self.read_bits_int_be(n)

    def read_bits_int_le(self, n):
        res = 0
        bits_needed = n - self.bits_left

        if bits_needed > 0:
            # 1 bit  => 1 byte
            # 8 bits => 1 byte
            # 9 bits => 2 bytes
            bytes_needed = ((bits_needed - 1) // 8) + 1 # `ceil(bits_needed / 8)`
            buf = self.read_bytes(bytes_needed)
            if PY2:
                buf = bytearray(buf)
            for i, byte in enumerate(buf):
                res |= byte << (i * 8)

            new_bits = res >> bits_needed
            res = res << self.bits_left | self.bits
            self.bits = new_bits
        else:
            res = self.bits
            self.bits >>= n

        self.bits_left = -bits_needed % 8

        mask = (1 << n) - 1 # no problem with this in Python (arbitrary precision integers)
        res &= mask
        return res

    # ========================================================================
    # Byte arrays
    # ========================================================================

    def read_bytes(self, n):
        if n < 0:
            raise ValueError(
                "requested invalid %d amount of bytes" %
                (n,)
            )
        r = self._io.read(n)
        if len(r) < n:
            raise EOFError(
                "requested %d bytes, but got only %d bytes" %
                (n, len(r))
            )
        return r

    def read_bytes_full(self):
        return self._io.read()

    def read_bytes_term(self, term, include_term, consume_term, eos_error):
        r = b''
        while True:
            c = self._io.read(1)
            if c == b'':
                if eos_error:
                    raise Exception(
                        "end of stream reached, but no terminator %d found" %
                        (term,)
                    )
                else:
                    return r
            elif ord(c) == term:
                if include_term:
                    r += c
                if not consume_term:
                    self._io.seek(-1, SEEK_CUR)
                return r
            else:
                r += c

    def ensure_fixed_contents(self, expected):
        actual = self._io.read(len(expected))
        if actual != expected:
            raise Exception(
                "unexpected fixed contents: got %r, was waiting for %r" %
                (actual, expected)
            )
        return actual

    @staticmethod
    def bytes_strip_right(data, pad_byte):
        return data.rstrip(KaitaiStream.byte_from_int(pad_byte))

    @staticmethod
    def bytes_terminate(data, term, include_term):
        new_data, term_byte, _ = data.partition(KaitaiStream.byte_from_int(term))
        if include_term:
            new_data += term_byte
        return new_data

    # ========================================================================
    # Byte array processing
    # ========================================================================

    @staticmethod
    def process_xor_one(data, key):
        if PY2:
            return bytes(bytearray(v ^ key for v in bytearray(data)))
        else:
            return bytes(v ^ key for v in data)

    @staticmethod
    def process_xor_many(data, key):
        if PY2:
            return bytes(bytearray(a ^ b for a, b in zip(bytearray(data), itertools.cycle(bytearray(key)))))
        else:
            return bytes(a ^ b for a, b in zip(data, itertools.cycle(key)))

    @staticmethod
    def process_rotate_left(data, amount, group_size):
        if group_size != 1:
            raise Exception(
                "unable to rotate group of %d bytes yet" %
                (group_size,)
            )

        mask = group_size * 8 - 1
        anti_amount = -amount & mask

        r = bytearray(data)
        for i in range(len(r)):
            r[i] = (r[i] << amount) & 0xff | (r[i] >> anti_amount)
        return bytes(r)

    # ========================================================================
    # Misc
    # ========================================================================

    @staticmethod
    def int_from_byte(v):
        return ord(v) if PY2 else v

    @staticmethod
    def byte_from_int(i):
        return chr(i) if PY2 else bytes((i,))

    @staticmethod
    def byte_array_index(data, i):
        return KaitaiStream.int_from_byte(data[i])

    @staticmethod
    def byte_array_min(b):
        return KaitaiStream.int_from_byte(min(b))

    @staticmethod
    def byte_array_max(b):
        return KaitaiStream.int_from_byte(max(b))

    @staticmethod
    def resolve_enum(enum_obj, value):
        """Resolves value using enum: if the value is not found in the map,
        we'll just use literal value per se. Works around problem with Python
        enums throwing an exception when encountering unknown value.
        """
        try:
            return enum_obj(value)
        except ValueError:
            return value

class KaitaiStructError(Exception):
    """Common ancestor for all error originating from Kaitai Struct usage.
    Stores KSY source path, pointing to an element supposedly guilty of
    an error.
    """
    def __init__(self, msg, src_path):
        super(KaitaiStructError, self).__init__("%s: %s" % (src_path, msg))
        self.src_path = src_path

class UndecidedEndiannessError(KaitaiStructError):
    """Error that occurs when default endianness should be decided with
    switch, but nothing matches (although using endianness expression
    implies that there should be some positive result).
    """
    def __init__(self, src_path):
        super(KaitaiStructError, self).__init__("unable to decide on endianness for a type", src_path)

class ValidationFailedError(KaitaiStructError):
    """Common ancestor for all validation failures. Stores pointer to
    KaitaiStream IO object which was involved in an error.
    """
    def __init__(self, msg, io, src_path):
        super(ValidationFailedError, self).__init__("at pos %d: validation failed: %s" % (io.pos(), msg), src_path)
        self.io = io

class ValidationNotEqualError(ValidationFailedError):
    """Signals validation failure: we required "actual" value to be equal to
    "expected", but it turned out that it's not.
    """
    def __init__(self, expected, actual, io, src_path):
        super(ValidationNotEqualError, self).__init__("not equal, expected %s, but got %s" % (repr(expected), repr(actual)), io, src_path)
        self.expected = expected
        self.actual = actual

class ValidationLessThanError(ValidationFailedError):
    """Signals validation failure: we required "actual" value to be
    greater than or equal to "min", but it turned out that it's not.
    """
    def __init__(self, min, actual, io, src_path):
        super(ValidationLessThanError, self).__init__("not in range, min %s, but got %s" % (repr(min), repr(actual)), io, src_path)
        self.min = min
        self.actual = actual

class ValidationGreaterThanError(ValidationFailedError):
    """Signals validation failure: we required "actual" value to be
    less than or equal to "max", but it turned out that it's not.
    """
    def __init__(self, max, actual, io, src_path):
        super(ValidationGreaterThanError, self).__init__("not in range, max %s, but got %s" % (repr(max), repr(actual)), io, src_path)
        self.max = max
        self.actual = actual

class ValidationNotAnyOfError(ValidationFailedError):
    """Signals validation failure: we required "actual" value to be
    from the list, but it turned out that it's not.
    """
    def __init__(self, actual, io, src_path):
        super(ValidationNotAnyOfError, self).__init__("not any of the list, got %s" % (repr(actual)), io, src_path)
        self.actual = actual

class ValidationExprError(ValidationFailedError):
    """Signals validation failure: we required "actual" value to match
    the expression, but it turned out that it doesn't.
    """
    def __init__(self, actual, io, src_path):
        super(ValidationExprError, self).__init__("not matching the expression, got %s" % (repr(actual)), io, src_path)
        self.actual = actual
<commit_msg>Fix "pycodestyle" violations from previous commit 427ea191<commit_after>import itertools
import sys
import struct
from io import open, BytesIO, SEEK_CUR, SEEK_END  # noqa

PY2 = sys.version_info[0] == 2

# Kaitai Struct runtime streaming API version, defined as per PEP-0396
# standard. Used for two purposes:
#
# * .py files generated by ksc from .ksy check that they import proper
#   KS runtime library by this version number;
# * distribution utils (setup.py) use this when packaging for PyPI
#
__version__ = '0.9'

class KaitaiStruct(object):
    def __init__(self, stream):
        self._io = stream

    def __enter__(self):
        return self

    def __exit__(self, *args, **kwargs):
        self.close()

    def close(self):
        self._io.close()

    @classmethod
    def from_file(cls, filename):
        f = open(filename, 'rb')
        try:
            return cls(KaitaiStream(f))
        except Exception:
            # close file descriptor, then reraise the exception
            f.close()
            raise

    @classmethod
    def from_bytes(cls, buf):
        return cls(KaitaiStream(BytesIO(buf)))

    @classmethod
    def from_io(cls, io):
        return cls(KaitaiStream(io))

class KaitaiStream(object):
    def __init__(self, io):
        self._io = io
        self.align_to_byte()

    def __enter__(self):
        return self

    def __exit__(self, *args, **kwargs):
        self.close()

    def close(self):
        self._io.close()

    # ========================================================================
    # Stream positioning
    # ========================================================================

    def is_eof(self):
        if self.bits_left > 0:
            return False

        io = self._io
        t = io.read(1)
        if t == b'':
            return True
        else:
            io.seek(-1, SEEK_CUR)
            return False

    def seek(self, n):
        self._io.seek(n)

    def pos(self):
        return self._io.tell()

    def size(self):
        # Python has no internal File object API function to get
        # current file / StringIO size, thus we use the following
        # trick.
        io = self._io
        # Remember our current position
        cur_pos = io.tell()
        # Seek to the end of the File object
        io.seek(0, SEEK_END)
        # Remember position, which is equal to the full length
        full_size = io.tell()
        # Seek back to the current position
        io.seek(cur_pos)
        return full_size

    # ========================================================================
    # Integer numbers
    # ========================================================================

    packer_s1 = struct.Struct('b')
    packer_s2be = struct.Struct('>h')
    packer_s4be = struct.Struct('>i')
    packer_s8be = struct.Struct('>q')
    packer_s2le = struct.Struct('<h')
    packer_s4le = struct.Struct('<i')
    packer_s8le = struct.Struct('<q')

    packer_u1 = struct.Struct('B')
    packer_u2be = struct.Struct('>H')
    packer_u4be = struct.Struct('>I')
    packer_u8be = struct.Struct('>Q')
    packer_u2le = struct.Struct('<H')
    packer_u4le = struct.Struct('<I')
    packer_u8le = struct.Struct('<Q')

    # ------------------------------------------------------------------------
    # Signed
    # ------------------------------------------------------------------------

    def read_s1(self):
        return KaitaiStream.packer_s1.unpack(self.read_bytes(1))[0]

    # ........................................................................
    # Big-endian
    # ........................................................................

    def read_s2be(self):
        return KaitaiStream.packer_s2be.unpack(self.read_bytes(2))[0]

    def read_s4be(self):
        return KaitaiStream.packer_s4be.unpack(self.read_bytes(4))[0]

    def read_s8be(self):
        return KaitaiStream.packer_s8be.unpack(self.read_bytes(8))[0]

    # ........................................................................
    # Little-endian
    # ........................................................................

    def read_s2le(self):
        return KaitaiStream.packer_s2le.unpack(self.read_bytes(2))[0]

    def read_s4le(self):
        return KaitaiStream.packer_s4le.unpack(self.read_bytes(4))[0]

    def read_s8le(self):
        return KaitaiStream.packer_s8le.unpack(self.read_bytes(8))[0]

    # ------------------------------------------------------------------------
    # Unsigned
    # ------------------------------------------------------------------------

    def read_u1(self):
        return KaitaiStream.packer_u1.unpack(self.read_bytes(1))[0]

    # ........................................................................
    # Big-endian
    # ........................................................................

    def read_u2be(self):
        return KaitaiStream.packer_u2be.unpack(self.read_bytes(2))[0]

    def read_u4be(self):
        return KaitaiStream.packer_u4be.unpack(self.read_bytes(4))[0]

    def read_u8be(self):
        return KaitaiStream.packer_u8be.unpack(self.read_bytes(8))[0]

    # ........................................................................
    # Little-endian
    # ........................................................................

    def read_u2le(self):
        return KaitaiStream.packer_u2le.unpack(self.read_bytes(2))[0]

    def read_u4le(self):
        return KaitaiStream.packer_u4le.unpack(self.read_bytes(4))[0]

    def read_u8le(self):
        return KaitaiStream.packer_u8le.unpack(self.read_bytes(8))[0]

    # ========================================================================
    # Floating point numbers
    # ========================================================================

    packer_f4be = struct.Struct('>f')
    packer_f8be = struct.Struct('>d')
    packer_f4le = struct.Struct('<f')
    packer_f8le = struct.Struct('<d')

    # ........................................................................
    # Big-endian
    # ........................................................................

    def read_f4be(self):
        return KaitaiStream.packer_f4be.unpack(self.read_bytes(4))[0]

    def read_f8be(self):
        return KaitaiStream.packer_f8be.unpack(self.read_bytes(8))[0]

    # ........................................................................
    # Little-endian
    # ........................................................................

    def read_f4le(self):
        return KaitaiStream.packer_f4le.unpack(self.read_bytes(4))[0]

    def read_f8le(self):
        return KaitaiStream.packer_f8le.unpack(self.read_bytes(8))[0]

    # ========================================================================
    # Unaligned bit values
    # ========================================================================

    def align_to_byte(self):
        self.bits_left = 0
        self.bits = 0

    def read_bits_int_be(self, n):
        res = 0

        bits_needed = n - self.bits_left
        self.bits_left = -bits_needed % 8

        if bits_needed > 0:
            # 1 bit  => 1 byte
            # 8 bits => 1 byte
            # 9 bits => 2 bytes
            bytes_needed = ((bits_needed - 1) // 8) + 1  # `ceil(bits_needed / 8)`
            buf = self.read_bytes(bytes_needed)
            if PY2:
                buf = bytearray(buf)
            for byte in buf:
                res = res << 8 | byte

            new_bits = res
            res = res >> self.bits_left | self.bits << bits_needed
            self.bits = new_bits  # will be masked at the end of the function
        else:
            res = self.bits >> -bits_needed  # shift unneeded bits out

        mask = (1 << self.bits_left) - 1  # `bits_left` is in range 0..7
        self.bits &= mask

        return res

    # Unused since Kaitai Struct Compiler v0.9+ - compatibility with
    # older versions.
    def read_bits_int(self, n):
        return self.read_bits_int_be(n)

    def read_bits_int_le(self, n):
        res = 0
        bits_needed = n - self.bits_left

        if bits_needed > 0:
            # 1 bit  => 1 byte
            # 8 bits => 1 byte
            # 9 bits => 2 bytes
            bytes_needed = ((bits_needed - 1) // 8) + 1  # `ceil(bits_needed / 8)`
            buf = self.read_bytes(bytes_needed)
            if PY2:
                buf = bytearray(buf)
            for i, byte in enumerate(buf):
                res |= byte << (i * 8)

            new_bits = res >> bits_needed
            res = res << self.bits_left | self.bits
            self.bits = new_bits
        else:
            res = self.bits
            self.bits >>= n

        self.bits_left = -bits_needed % 8

        mask = (1 << n) - 1  # no problem with this in Python (arbitrary precision integers)
        res &= mask
        return res

    # ========================================================================
    # Byte arrays
    # ========================================================================

    def read_bytes(self, n):
        if n < 0:
            raise ValueError(
                "requested invalid %d amount of bytes" %
                (n,)
            )
        r = self._io.read(n)
        if len(r) < n:
            raise EOFError(
                "requested %d bytes, but got only %d bytes" %
                (n, len(r))
            )
        return r

    def read_bytes_full(self):
        return self._io.read()

    def read_bytes_term(self, term, include_term, consume_term, eos_error):
        r = b''
        while True:
            c = self._io.read(1)
            if c == b'':
                if eos_error:
                    raise Exception(
                        "end of stream reached, but no terminator %d found" %
                        (term,)
                    )
                else:
                    return r
            elif ord(c) == term:
                if include_term:
                    r += c
                if not consume_term:
                    self._io.seek(-1, SEEK_CUR)
                return r
            else:
                r += c

    def ensure_fixed_contents(self, expected):
        actual = self._io.read(len(expected))
        if actual != expected:
            raise Exception(
                "unexpected fixed contents: got %r, was waiting for %r" %
                (actual, expected)
            )
        return actual

    @staticmethod
    def bytes_strip_right(data, pad_byte):
        return data.rstrip(KaitaiStream.byte_from_int(pad_byte))

    @staticmethod
    def bytes_terminate(data, term, include_term):
        new_data, term_byte, _ = data.partition(KaitaiStream.byte_from_int(term))
        if include_term:
            new_data += term_byte
        return new_data

    # ========================================================================
    # Byte array processing
    # ========================================================================

    @staticmethod
    def process_xor_one(data, key):
        if PY2:
            return bytes(bytearray(v ^ key for v in bytearray(data)))
        else:
            return bytes(v ^ key for v in data)

    @staticmethod
    def process_xor_many(data, key):
        if PY2:
            return bytes(bytearray(a ^ b for a, b in zip(bytearray(data), itertools.cycle(bytearray(key)))))
        else:
            return bytes(a ^ b for a, b in zip(data, itertools.cycle(key)))

    @staticmethod
    def process_rotate_left(data, amount, group_size):
        if group_size != 1:
            raise Exception(
                "unable to rotate group of %d bytes yet" %
                (group_size,)
            )

        mask = group_size * 8 - 1
        anti_amount = -amount & mask

        r = bytearray(data)
        for i in range(len(r)):
            r[i] = (r[i] << amount) & 0xff | (r[i] >> anti_amount)
        return bytes(r)

    # ========================================================================
    # Misc
    # ========================================================================

    @staticmethod
    def int_from_byte(v):
        return ord(v) if PY2 else v

    @staticmethod
    def byte_from_int(i):
        return chr(i) if PY2 else bytes((i,))

    @staticmethod
    def byte_array_index(data, i):
        return KaitaiStream.int_from_byte(data[i])

    @staticmethod
    def byte_array_min(b):
        return KaitaiStream.int_from_byte(min(b))

    @staticmethod
    def byte_array_max(b):
        return KaitaiStream.int_from_byte(max(b))

    @staticmethod
    def resolve_enum(enum_obj, value):
        """Resolves value using enum: if the value is not found in the map,
        we'll just use literal value per se. Works around problem with Python
        enums throwing an exception when encountering unknown value.
        """
        try:
            return enum_obj(value)
        except ValueError:
            return value

class KaitaiStructError(Exception):
    """Common ancestor for all error originating from Kaitai Struct usage.
    Stores KSY source path, pointing to an element supposedly guilty of
    an error.
    """
    def __init__(self, msg, src_path):
        super(KaitaiStructError, self).__init__("%s: %s" % (src_path, msg))
        self.src_path = src_path

class UndecidedEndiannessError(KaitaiStructError):
    """Error that occurs when default endianness should be decided with
    switch, but nothing matches (although using endianness expression
    implies that there should be some positive result).
    """
    def __init__(self, src_path):
        super(KaitaiStructError, self).__init__("unable to decide on endianness for a type", src_path)

class ValidationFailedError(KaitaiStructError):
    """Common ancestor for all validation failures. Stores pointer to
    KaitaiStream IO object which was involved in an error.
    """
    def __init__(self, msg, io, src_path):
        super(ValidationFailedError, self).__init__("at pos %d: validation failed: %s" % (io.pos(), msg), src_path)
        self.io = io

class ValidationNotEqualError(ValidationFailedError):
    """Signals validation failure: we required "actual" value to be equal to
    "expected", but it turned out that it's not.
    """
    def __init__(self, expected, actual, io, src_path):
        super(ValidationNotEqualError, self).__init__("not equal, expected %s, but got %s" % (repr(expected), repr(actual)), io, src_path)
        self.expected = expected
        self.actual = actual

class ValidationLessThanError(ValidationFailedError):
    """Signals validation failure: we required "actual" value to be
    greater than or equal to "min", but it turned out that it's not.
    """
    def __init__(self, min, actual, io, src_path):
        super(ValidationLessThanError, self).__init__("not in range, min %s, but got %s" % (repr(min), repr(actual)), io, src_path)
        self.min = min
        self.actual = actual

class ValidationGreaterThanError(ValidationFailedError):
    """Signals validation failure: we required "actual" value to be
    less than or equal to "max", but it turned out that it's not.
    """
    def __init__(self, max, actual, io, src_path):
        super(ValidationGreaterThanError, self).__init__("not in range, max %s, but got %s" % (repr(max), repr(actual)), io, src_path)
        self.max = max
        self.actual = actual

class ValidationNotAnyOfError(ValidationFailedError):
    """Signals validation failure: we required "actual" value to be
    from the list, but it turned out that it's not.
    """
    def __init__(self, actual, io, src_path):
        super(ValidationNotAnyOfError, self).__init__("not any of the list, got %s" % (repr(actual)), io, src_path)
        self.actual = actual

class ValidationExprError(ValidationFailedError):
    """Signals validation failure: we required "actual" value to match
    the expression, but it turned out that it doesn't.
    """
    def __init__(self, actual, io, src_path):
        super(ValidationExprError, self).__init__("not matching the expression, got %s" % (repr(actual)), io, src_path)
        self.actual = actual
<|endoftext|>


<commit_before>""" This file organizes the different answers from each query result to load
"""

import os

import stackexchange
from pprint import pprint
from gso import search_google

SE_KEY = os.environ["SE_KEY"]

so = stackexchange.Site(
    stackexchange.StackOverflow,
    app_key=SE_KEY)

so_superuser = stackexchange.Site(
    stackexchange.SuperUser,
    app_key=SE_KEY)

so_unix = stackexchange.Site(
    stackexchange.UnixampLinux,
    app_key=SE_KEY)

so_tex = stackexchange.Site(
    stackexchange.TeXLaTeX,
    app_key=SE_KEY)

for site in [so, so_superuser, so_unix, so_tex]:
    site.impose_throttling = True
    site.throttle_stop = False

def load_up_questions(question, language='', answers=5):
    """ Load up stack overflow questions from a query

    Args:

        question: String question, what you would google for

        language: (optional) specific language inferred from filename

        answers: (optional, default 5) the number of answers to load

    Yields:

        lists of strings:
            0: URL to question
            1: Title of question
            2: Short description of question
    """
    query = search_google(question, language=language)
    results = query[u'items']
    for result in results:
        url = result[u'link']
        title = result[u'pagemap'][u'qapage'][0][u'title']
        description = result[u'pagemap'][u'qapage'][0][u'description']
        yield [url, title, description]

def load_up_answers(URL):
    """ Load answers from a stack overflow URL

    Args:

        URL: string of the url for the question (https:...)

    Returns:

        List of [score, body], where the body is html
    """

    split_url = URL.split('/')
    domain = split_url[2]
    site = {
        'unix.stackexchange.com': so_unix,
        'tex.stackexchange.com': so_tex,
        'superuser.com': so_superuser,
        'stackoverflow.com': so}[domain]
    question_id = split_url[4]
    answer_pointers = site.question(question_id).answers

    answer_pointers.sort(key=lambda x: x.score)
    answer_pointers = list(reversed(answer_pointers))

    answer_ids = [answer.id for answer in answer_pointers]
    # Only look at the first answer
    answer_ids = [answer_ids[0]]

    answer_objs = [
        site.answer(
            answer_id,
            body=True,
            score=True) for answer_id in answer_ids]

    answers = [
        [answer.score,
         answer.body] for answer in answer_objs]

    return answers

<commit_msg>Remove unused attrs<commit_after>""" This file organizes the different answers from each query result to load
"""

import os

import stackexchange
from pprint import pprint
from gso import search_google

SE_KEY = os.environ["SE_KEY"]

so = stackexchange.Site(
    stackexchange.StackOverflow,
    app_key=SE_KEY)

so_superuser = stackexchange.Site(
    stackexchange.SuperUser,
    app_key=SE_KEY)

so_unix = stackexchange.Site(
    stackexchange.UnixampLinux,
    app_key=SE_KEY)

so_tex = stackexchange.Site(
    stackexchange.TeXLaTeX,
    app_key=SE_KEY)

for site in [so, so_superuser, so_unix, so_tex]:
    site.impose_throttling = True
    site.throttle_stop = False

def load_up_questions(question, language='', answers=5):
    """ Load up stack overflow questions from a query

    Args:

        question: String question, what you would google for

        language: (optional) specific language inferred from filename

        answers: (optional, default 5) the number of answers to load

    Yields:

        lists of strings:
            0: URL to question
    """
    query = search_google(question, language=language)
    results = query[u'items']
    for result in results:
        url = result[u'link']
        yield [url]

def load_up_answers(URL):
    """ Load answers from a stack overflow URL

    Args:

        URL: string of the url for the question (https:...)

    Returns:

        List of [score, body], where the body is html
    """

    split_url = URL.split('/')
    domain = split_url[2]
    site = {
        'unix.stackexchange.com': so_unix,
        'tex.stackexchange.com': so_tex,
        'superuser.com': so_superuser,
        'stackoverflow.com': so}[domain]
    question_id = split_url[4]
    answer_pointers = site.question(question_id).answers

    answer_pointers.sort(key=lambda x: x.score)
    answer_pointers = list(reversed(answer_pointers))

    answer_ids = [answer.id for answer in answer_pointers]
    # Only look at the first answer
    answer_ids = [answer_ids[0]]

    answer_objs = [
        site.answer(
            answer_id,
            body=True,
            score=True) for answer_id in answer_ids]

    answers = [
        [answer.score,
         answer.body] for answer in answer_objs]

    return answers

<|endoftext|>


<commit_before>__author__ = 'Jon Hadfield'
import sys
from utils import is_ipv4, is_ipv6

class HostsEntry(object):
    """ An entry in a hosts file. """
    def __init__(self, entry_type=None, address=None, comment=None, names=None):
        if not entry_type or entry_type not in ('ipv4', 'ipv6', 'comment', 'blank'):
            raise Exception('entry_type invalid or not specified')

        if entry_type == 'comment' and not comment:
            raise Exception('entry_type comment supplied without value.')

        if entry_type in ('ipv4', 'ipv6'):
            if not all((address, names)):
                raise Exception('Address and Name(s) must be specified.')

        self.entry_type = entry_type
        self.address = address
        self.comment = comment
        self.names = names

    @staticmethod
    def get_entry_type(hosts_entry):
        if hosts_entry[0] == "#":
            return 'comment'
        if hosts_entry[0] == "\n":
            return 'blank'
        entry_chunks = hosts_entry.split()
        if is_ipv4(entry_chunks[0]):
            return "ipv4"
        if is_ipv6(entry_chunks[0]):
            return "ipv6"
        else:
            return False

class Hosts(object):
    """ A hosts file. """
    def __init__(self, path=None):
        """
        Returns a list of all entries in the hosts file.
        Each entry is represented in the form of a dict.
        """
        platform = sys.platform
        if path:
            self.hosts_path = path
        elif 'linux' in platform or 'darwin' == platform:
            self.hosts_path = '/etc/hosts'
        elif any((platform == 'win32', platform == 'windows')):
            self.hosts_path = r'c:\windows\system32\drivers\etc\hosts'
        else:
            exit("cannot determine platform")
        self.entries = []
        self.populate_entries()

    def write(self):
        with open(self.hosts_path, 'w') as hosts_file:
            for line in self.entries:
                if line.entry_type == 'comment':
                    hosts_file.write(line.comment)
                if line.entry_type == 'blank':
                    hosts_file.write("\n")
                if line.entry_type == 'ipv4':
                    hosts_file.write(
                        "{}\t{}\n".format(
                            line.address,
                            ' '.join(line.names),
                        )
                    )
                if line.entry_type == 'ipv6':
                    hosts_file.write(
                        "{}\t{}\n".format(
                            line.address,
                            ' '.join(line.names),
                        )
                    )

    def add(self, entry=None, force=False):
        """
        Adds an entry to a host file.
        :param entry: An instance of HostsEntry
        :param force: Remove conflicting, existing instances first
        :return: True if successfully added to hosts file
        """
        new_entry = entry
        if new_entry.entry_type == "comment":
            existing = self.count(new_entry).get('num_comment_matches')
            if existing:
                return False
        elif new_entry.entry_type == "ipv4" or new_entry.entry_type == "ipv6":
            existing = self.count(new_entry)
            existing_addresses = existing.get('num_address_matches')
            existing_names = existing.get('num_name_matches')
            if not force and (existing_addresses or existing_names):
                return False
            elif force:
                self.remove(passed_address=new_entry.address,
                            passed_names=new_entry.names)
        self.entries.append(new_entry)
        self.write()
        return True

    def count(self, entry=None):
        """
        Count the number of address, name or comment matches
        in the given HostsEntry instance or supplied values
        :param entry: An instance of HostsEntry
        :return: A dict listing the number of address, name and comment matches
        """
        if not entry:
            exit('valid entry not provided')

        num_address_matches = 0
        num_name_matches = 0
        num_comment_matches = 0
        for host in self.entries:
            existing_names = host.names
            existing_host_address = host.address
            existing_comment = host.comment
            if entry.entry_type == "ipv4" or entry.entry_type == "ipv6":
                if all((existing_names, entry.names)) and \
                        set(entry.names).intersection(existing_names):
                    print existing_names
                    num_name_matches += 1
                if existing_host_address and existing_host_address == entry.address:
                    print existing_host_address
                    num_address_matches += 1
            if entry.entry_type == "comment":
                if existing_comment == entry.comment:
                    print existing_comment
                    num_comment_matches += 1
        return {'address_matches': num_address_matches,
                'name_matches': num_name_matches,
                'comment_matches': num_comment_matches}

    def remove(self,
               entry=None,
               passed_address=None,
               passed_names=None,
               passed_comment=None):
        """
        Remove an entry from a hosts file
        :param entry: An instance of HostsEntry
        :param passed_address: The address of the host to remove
        :param passed_names: The name(s) of the host to remove
        :param passed_comment: The comment to remove
        :return:
        """
        removed = 0
        removal_list = []
        for existing_entry in self.entries:
            if entry:
                if existing_entry.names and entry.get("names"):
                    names_inter = set(existing_entry.names).intersection(entry.get("names"))
                    if any((existing_entry.address == entry.get('address'),
                           existing_entry.names == entry.get('names'),
                            names_inter)):
                        removal_list.append(existing_entry)
                        removed += 1
                if entry.comment and existing_entry.comment == entry.comment:
                    removal_list.append(existing_entry)
            else:
                if existing_entry.names and passed_names:
                    names_inter = set(existing_entry.names).intersection(passed_names)
                    if any((existing_entry.address == passed_address,
                            existing_entry.names == passed_names,
                            names_inter)):
                        removal_list.append(existing_entry)
                        removed += 1
                if passed_comment and existing_entry.comment == passed_comment:
                    removal_list.append(existing_entry)
        for entry_to_remove in removal_list:
            self.entries.remove(entry_to_remove)
        self.write()
        if removed > 0:
            return True
        return False

    def populate_entries(self):
        with open(self.hosts_path, 'r') as hosts_file:
            hosts_entries = [line for line in hosts_file]
            for hosts_entry in hosts_entries:
                entry_type = HostsEntry.get_entry_type(hosts_entry)
                if entry_type == "comment":
                    self.entries.append(HostsEntry(entry_type="comment", comment=hosts_entry))
                if entry_type == "blank":
                    self.entries.append(HostsEntry(entry_type="blank"))
                if entry_type == "ipv4" or entry_type == "ipv6":
                    chunked_entry = hosts_entry.split()
                    self.entries.append(HostsEntry(entry_type=entry_type,
                                                   address=chunked_entry[0],
                                                   names=chunked_entry[1:]))

<commit_msg>tidy up.<commit_after>__author__ = 'Jon Hadfield'
import sys
from utils import is_ipv4, is_ipv6

class HostsEntry(object):
    """ An entry in a hosts file. """
    def __init__(self, entry_type=None, address=None, comment=None, names=None):
        if not entry_type or entry_type not in ('ipv4', 'ipv6', 'comment', 'blank'):
            raise Exception('entry_type invalid or not specified')

        if entry_type == 'comment' and not comment:
            raise Exception('entry_type comment supplied without value.')

        if entry_type in ('ipv4', 'ipv6'):
            if not all((address, names)):
                raise Exception('Address and Name(s) must be specified.')

        self.entry_type = entry_type
        self.address = address
        self.comment = comment
        self.names = names

    @staticmethod
    def get_entry_type(hosts_entry):
        if hosts_entry[0] == "#":
            return 'comment'
        if hosts_entry[0] == "\n":
            return 'blank'
        entry_chunks = hosts_entry.split()
        if is_ipv4(entry_chunks[0]):
            return "ipv4"
        if is_ipv6(entry_chunks[0]):
            return "ipv6"
        else:
            return False

class Hosts(object):
    """ A hosts file. """
    def __init__(self, path=None):
        """
        Returns a list of all entries in the hosts file.
        Each entry is represented in the form of a dict.
        """
        platform = sys.platform
        if path:
            self.hosts_path = path
        elif 'linux' in platform or 'darwin' == platform:
            self.hosts_path = '/etc/hosts'
        elif any((platform == 'win32', platform == 'windows')):
            self.hosts_path = r'c:\windows\system32\drivers\etc\hosts'
        else:
            exit("cannot determine platform")
        self.entries = []
        self.populate_entries()

    def write(self):
        with open(self.hosts_path, 'w') as hosts_file:
            for line in self.entries:
                if line.entry_type == 'comment':
                    hosts_file.write(line.comment)
                if line.entry_type == 'blank':
                    hosts_file.write("\n")
                if line.entry_type == 'ipv4':
                    hosts_file.write(
                        "{}\t{}\n".format(
                            line.address,
                            ' '.join(line.names),
                        )
                    )
                if line.entry_type == 'ipv6':
                       hosts_file.write(
                           "{}\t{}\n".format(
                           line.address,
                            ' '.join(line.names),
                           )
                       )

    def add(self, entry=None, force=False):
        """
        Adds an entry to a host file.
        :param entry: An instance of HostsEntry
        :param force: Remove conflicting, existing instances first
        :return: True if successfully added to hosts file
        """
        new_entry = entry
        if new_entry.entry_type == "comment":
            existing = self.count(new_entry).get('num_comment_matches')
            if existing:
                return False
        elif new_entry.entry_type == "ipv4" or new_entry.entry_type == "ipv6":
            existing = self.count(new_entry)
            existing_addresses = existing.get('num_address_matches')
            existing_names = existing.get('num_name_matches')
            if not force and (existing_addresses or existing_names):
                return False
            elif force:
                self.remove(passed_address=new_entry.address,
                            passed_names=new_entry.names)
        self.entries.append(new_entry)
        self.write()
        return True

    def count(self, entry=None):
        """
        Count the number of address, name or comment matches
        in the given HostsEntry instance or supplied values
        :param entry: An instance of HostsEntry
        :return: A dict listing the number of address, name and comment matches
        """
        if not entry:
            exit('valid entry not provided')

        num_address_matches = 0
        num_name_matches = 0
        num_comment_matches = 0
        for host in self.entries:
            existing_names = host.names
            existing_host_address = host.address
            existing_comment = host.comment
            if entry.entry_type == "ipv4" or entry.entry_type == "ipv6":
                if all((existing_names, entry.names)) and \
                        set(entry.names).intersection(existing_names):
                    print existing_names
                    num_name_matches += 1
                if existing_host_address and existing_host_address == entry.address:
                    print existing_host_address
                    num_address_matches += 1
            if entry.entry_type == "comment":
                if existing_comment == entry.comment:
                    print existing_comment
                    num_comment_matches += 1
        return {'address_matches': num_address_matches,
                'name_matches': num_name_matches,
                'comment_matches': num_comment_matches}

    def remove(self,
               entry=None,
               passed_address=None,
               passed_names=None,
               passed_comment=None):
        """
        Remove an entry from a hosts file
        :param entry: An instance of HostsEntry
        :param passed_address: The address of the host to remove
        :param passed_names: The name(s) of the host to remove
        :param passed_comment: The comment to remove
        :return:
        """
        removed = 0
        removal_list = []
        for existing_entry in self.entries:
            if entry:
                if existing_entry.names and entry.get("names"):
                    names_inter = set(existing_entry.names).intersection(entry.get("names"))
                    if any((existing_entry.address == entry.get('address'),
                           existing_entry.names == entry.get('names'),
                            names_inter)):
                        removal_list.append(existing_entry)
                        removed += 1
                if entry.comment and existing_entry.comment == entry.comment:
                    removal_list.append(existing_entry)
            else:
                if existing_entry.names and passed_names:
                    names_inter = set(existing_entry.names).intersection(passed_names)
                    if any((existing_entry.address == passed_address,
                            existing_entry.names == passed_names,
                            names_inter)):
                        removal_list.append(existing_entry)
                        removed += 1
                if passed_comment and existing_entry.comment == passed_comment:
                    removal_list.append(existing_entry)
        for entry_to_remove in removal_list:
            self.entries.remove(entry_to_remove)
        self.write()
        if removed > 0:
            return True
        return False

    def populate_entries(self):
        with open(self.hosts_path, 'r') as hosts_file:
            hosts_entries = [line for line in hosts_file]
            for hosts_entry in hosts_entries:
                entry_type = HostsEntry.get_entry_type(hosts_entry)
                if entry_type == "comment":
                    self.entries.append(HostsEntry(entry_type="comment",
                                                   comment=hosts_entry))
                if entry_type == "blank":
                    self.entries.append(HostsEntry(entry_type="blank"))
                if entry_type == "ipv4" or entry_type == "ipv6":
                    chunked_entry = hosts_entry.split()
                    self.entries.append(HostsEntry(entry_type=entry_type,
                                                   address=chunked_entry[0],
                                                   names=chunked_entry[1:]))

<|endoftext|>


<commit_before># ===============================================================================
# Copyright 2016 Jake Ross
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ===============================================================================

# =============enthought library imports=======================

from traits.api import provides, HasTraits

# from pyface.timer.api import Timer
# =============standard library imports ========================
import random
# from threading import Lock
import inspect
import time
# =============local library imports  ==========================
# from traits.has_traits import provides
from i_core_device import ICoreDevice
# from pychron.core.helpers.timer import Timer
# from pychron.managers.data_managers.csv_data_manager import CSVDataManager
# from pychron.core.helpers.datetime_tools import generate_datetimestamp
from pychron.globals import globalv
from pychron.hardware.core.exceptions import TimeoutError, CRCError
from pychron.has_communicator import HasCommunicator
from pychron.hardware.core.communicators.scheduler import CommunicationScheduler
from pychron.consumer_mixin import ConsumerMixin

def crc_caller(func):
    def d(*args, **kw):
        try:
            return func(*args, **kw)
        except CRCError:
            stack = inspect.stack()
            print '{} called by {}'.format(func.func_name, stack[1][3])

    return d

@provides(ICoreDevice)
class BaseCoreDevice(HasCommunicator, ConsumerMixin):
    """
    """

    _auto_started = False
    _no_response_counter = 0
    _scheduler_name = None

    def send_email_notification(self, message):
        if self.application:
            tm = self.application.get_service('pychron.social.emailer.Emailer')
            if tm:
                tm.broadcast(message)
            else:
                self.warning('No emailer available')

    # ICoreDevice protocol
    def close(self):
        if self.communicator:
            self.communicator.close()

    def get(self, *args, **kw):
        return self.current_scan_value

    def set(self, v):
        pass

    def is_connected(self):
        if self.communicator:
            return not self.communicator.simulation

    def test_connection(self):
        if self.communicator:
            return self.communicator.test_connection()

    def set_simulation(self, tf):
        if self.communicator:
            self.communicator.simulation = tf

    def load(self, *args, **kw):
        """
            Load a configuration file.
            Get Communications info to make a new communicator
        """
        config = self.get_configuration()
        if config:

            if config.has_section('General'):
                name = self.config_get(config, 'General', 'name', optional=True)
                if name is not None:
                    self.name = name

            if config.has_section('Communications'):
                comtype = self.config_get(config, 'Communications', 'type')
                if not self._load_communicator(config, comtype):
                    return False

                self.set_attribute(config, '_scheduler_name', 'Communications', 'scheduler', optional=True)

            self._load_hook(config)

            # load additional child specific args
            r = self.load_additional_args(config)
            if r:
                self._loaded = True
            return r

    def open(self, *args, **kw):
        self.debug('open device')
        return HasCommunicator.open(self, **kw)

    def initialize(self, *args, **kw):
        a = super(BaseCoreDevice, self).initialize(*args, **kw)
        b = False
        if self.communicator is not None:
            b = self.communicator.initialize(*args, **kw)

        return a and b

    def load_additional_args(self, config):
        """
            remember to return a boolean in any subclass that overrides this method.
            if True bootstraping of this device will continue. otherwise device will not fully initialize
        """
        return True

    def blocking_poll(self, func, args=None, kwargs=None, period=1, timeout=None, script=None):
        """
            repeatedly ask func at 1/period rate
            if func returns true return True
            if timeout return False
        """
        if isinstance(func, str):
            func = getattr(self, func)
        if args is None:
            args = tuple()
        if kwargs is None:
            kwargs = dict()

        st = time.time()
        while 1:
            if script and script.is_canceled():
                return
            if func(*args, **kwargs):
                return True
            elif timeout:
                et = time.time() - st
                if et > timeout:
                    self.warning('blocking poll of "{}" timed out after {}s'.format(func.func_name, timeout))
                    raise TimeoutError(func.func_name, timeout)
            time.sleep(period)

    @crc_caller
    def ask(self, cmd, **kw):
        """
        """
        comm = self.communicator
        if comm is not None:
            if comm.scheduler:
                r = comm.scheduler.schedule(comm.ask, args=(cmd,),
                                            kwargs=kw)
            else:
                r = comm.ask(cmd, **kw)
            self._communicate_hook(cmd, r)
            return r
        else:
            self.info('no communicator for this device {}'.format(self.name))

    @crc_caller
    def write(self, *args, **kw):
        """
        """
        self.tell(*args, **kw)

    @crc_caller
    def tell(self, *args, **kw):
        """
        """
        if self.communicator is not None:
            cmd = ' '.join(map(str, args) + map(str, kw.iteritems()))
            self._communicate_hook(cmd, '-')
            return self.communicator.tell(*args, **kw)

    @crc_caller
    def read(self, *args, **kw):
        """
        """
        if self.communicator is not None:
            return self.communicator.read(*args, **kw)

    #        if self.simulation:
    #            return 'simulation'

    #            gdict = globals()
    #            if communicator_type in gdict:
    #                return gdict[communicator_type](name='_'.join((self.name, communicator_type)),
    #                                   id_query=self.id_query,
    #                                   id_response=self.id_response
    #                                )
    def post_initialize(self, *args, **kw):
        self.graph.set_y_title(self.graph_ytitle)
        self.setup_scan()
        self.setup_alarms()
        self.setup_scheduler()

        if self.auto_start:
            self.start_scan()

    def get_random_value(self, mi=0, ma=10):
        """
            convienent method for getting a random integer between min and max

            Defaults:
                min=0
                max=10

        """
        return random.randint(mi, ma) if globalv.communication_simulation else None

    def setup_scheduler(self, name=None):

        if self.application:
            if name is None:
                name = self._scheduler_name
            if name is not None:
                sc = self.application.get_service(CommunicationScheduler, 'name=="{}"'.format(name))
                if sc is None:
                    sc = CommunicationScheduler(name=name)
                    self.application.register_service(type(sc), sc)
                self.set_scheduler(sc)

    def set_scheduler(self, s):
        if self.communicator is not None:
            self.communicator.scheduler = s

    def repeat_command(self, cmd, ntries=2, check_val=None, check_type=None,
                       verbose=True, **kw):

        if isinstance(cmd, tuple):
            cmd = self._build_command(*cmd)
        else:
            cmd = self._build_command(cmd)

        resp = None
        for i in range(ntries + 1):
            resp = self._parse_response(self.ask(cmd, verbose=verbose))
            if verbose:
                m = 'repeat command {} response = {} len={} '.format(i + 1,
                                                                     resp,
                                                                     len(str(resp)) if resp is not None else None)
                self.debug(m)
            if check_val is not None:
                if self.simulation:
                    resp = check_val

                if resp == check_val:
                    break
                else:
                    continue

            if check_type is not None:
                if self.simulation:
                    resp = random.randint(1, 10)
                else:
                    try:
                        resp = check_type(resp)
                    except (ValueError, TypeError):
                        continue

            if resp is not None:
                break

        return resp

    # ===============================================================================
    # scanable interface
    # ===============================================================================
    def _scan_hook(self, v):
        for a in self.alarms:
            if a.test_condition(v):
                alarm_msg = a.get_message(v)
                self.warning(alarm_msg)
                manager = self.application.get_service('pychron.social.twitter_manager.TwitterManager')
                if manager is not None:
                    manager.post(alarm_msg)
                break

    def _parse_response(self, v):
        return v

    def _communicate_hook(self, cmd, r):
        self.last_command = cmd
        self.last_response = r if r else ''

    def _load_hook(self, config):
        pass
# ========================= EOF ============================================
<commit_msg>fixed visa<commit_after># ===============================================================================
# Copyright 2016 Jake Ross
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ===============================================================================

# =============enthought library imports=======================

from traits.api import provides, HasTraits

# from pyface.timer.api import Timer
# =============standard library imports ========================
import random
# from threading import Lock
import inspect
import time
# =============local library imports  ==========================
# from traits.has_traits import provides
from i_core_device import ICoreDevice
# from pychron.core.helpers.timer import Timer
# from pychron.managers.data_managers.csv_data_manager import CSVDataManager
# from pychron.core.helpers.datetime_tools import generate_datetimestamp
from pychron.globals import globalv
from pychron.hardware.core.exceptions import TimeoutError, CRCError
from pychron.has_communicator import HasCommunicator
from pychron.hardware.core.communicators.scheduler import CommunicationScheduler
from pychron.consumer_mixin import ConsumerMixin

def crc_caller(func):
    def d(*args, **kw):
        try:
            return func(*args, **kw)
        except CRCError:
            stack = inspect.stack()
            print '{} called by {}'.format(func.func_name, stack[1][3])

    return d

@provides(ICoreDevice)
class BaseCoreDevice(HasCommunicator, ConsumerMixin):
    """
    """

    _auto_started = False
    _no_response_counter = 0
    _scheduler_name = None

    def send_email_notification(self, message):
        if self.application:
            tm = self.application.get_service('pychron.social.emailer.Emailer')
            if tm:
                tm.broadcast(message)
            else:
                self.warning('No emailer available')

    # ICoreDevice protocol
    def close(self):
        if self.communicator:
            self.communicator.close()

    def get(self, *args, **kw):
        return self.current_scan_value

    def set(self, v):
        pass

    def is_connected(self):
        if self.communicator:
            return not self.communicator.simulation

    def test_connection(self):
        if self.communicator:
            return self.communicator.test_connection()

    def set_simulation(self, tf):
        if self.communicator:
            self.communicator.simulation = tf

    def load(self, *args, **kw):
        """
            Load a configuration file.
            Get Communications info to make a new communicator
        """
        config = self.get_configuration()
        if config:

            if config.has_section('General'):
                name = self.config_get(config, 'General', 'name', optional=True)
                if name is not None:
                    self.name = name

            if config.has_section('Communications'):
                comtype = self.config_get(config, 'Communications', 'type')
                if not self._load_communicator(config, comtype):
                    return False

                self.set_attribute(config, '_scheduler_name', 'Communications', 'scheduler', optional=True)

            self._load_hook(config)

            # load additional child specific args
            r = self.load_additional_args(config)
            if r:
                self._loaded = True
            return r

    def open(self, *args, **kw):
        self.debug('open device')
        return HasCommunicator.open(self, **kw)

    def initialize(self, *args, **kw):
        a = super(BaseCoreDevice, self).initialize(*args, **kw)
        b = False
        if self.communicator is not None:
            b = self.communicator.initialize(*args, **kw)

        return a and b

    def load_additional_args(self, config):
        """
            remember to return a boolean in any subclass that overrides this method.
            if True bootstraping of this device will continue. otherwise device will not fully initialize
        """
        return True

    def blocking_poll(self, func, args=None, kwargs=None, period=1, timeout=None, script=None):
        """
            repeatedly ask func at 1/period rate
            if func returns true return True
            if timeout return False
        """
        if isinstance(func, str):
            func = getattr(self, func)
        if args is None:
            args = tuple()
        if kwargs is None:
            kwargs = dict()

        st = time.time()
        while 1:
            if script and script.is_canceled():
                return
            if func(*args, **kwargs):
                return True
            elif timeout:
                et = time.time() - st
                if et > timeout:
                    self.warning('blocking poll of "{}" timed out after {}s'.format(func.func_name, timeout))
                    raise TimeoutError(func.func_name, timeout)
            time.sleep(period)

    @crc_caller
    def ask(self, cmd, **kw):
        """
        """
        comm = self.communicator
        if comm is not None:
            if comm.scheduler:
                r = comm.scheduler.schedule(comm.ask, args=(cmd,),
                                            kwargs=kw)
            else:
                r = comm.ask(cmd, **kw)
            self._communicate_hook(cmd, r)
            return r
        else:
            self.info('no communicator for this device {}'.format(self.name))

    @crc_caller
    def write(self, *args, **kw):
        """
        """
        self.tell(*args, **kw)

    @crc_caller
    def tell(self, *args, **kw):
        """
        """
        if self.communicator is not None:
            cmd = ' '.join(map(str, args) + map(str, kw.iteritems()))
            self._communicate_hook(cmd, '-')
            return self.communicator.tell(*args, **kw)

    @crc_caller
    def read(self, *args, **kw):
        """
        """
        if self.communicator is not None:
            return self.communicator.read(*args, **kw)

    #        if self.simulation:
    #            return 'simulation'

    #            gdict = globals()
    #            if communicator_type in gdict:
    #                return gdict[communicator_type](name='_'.join((self.name, communicator_type)),
    #                                   id_query=self.id_query,
    #                                   id_response=self.id_response
    #                                )
    def post_initialize(self, *args, **kw):
        self.graph.set_y_title(self.graph_ytitle)
        self.setup_scan()
        self.setup_alarms()
        self.setup_scheduler()

        if self.auto_start:
            self.start_scan()

    def get_random_value(self, mi=0, ma=10):
        """
            convienent method for getting a random integer between min and max

            Defaults:
                min=0
                max=10

        """
        return random.randint(mi, ma) if globalv.communication_simulation else None

    def setup_scheduler(self, name=None):

        if self.application:
            if name is None:
                name = self._scheduler_name
            if name is not None:
                sc = self.application.get_service(CommunicationScheduler, 'name=="{}"'.format(name))
                if sc is None:
                    sc = CommunicationScheduler(name=name)
                    self.application.register_service(type(sc), sc)
                self.set_scheduler(sc)

    def set_scheduler(self, s):
        if self.communicator is not None:
            self.communicator.scheduler = s

    def repeat_command(self, cmd, ntries=2, check_val=None, check_type=None,
                       verbose=True, **kw):

        if isinstance(cmd, tuple):
            cmd = self._build_command(*cmd)
        else:
            cmd = self._build_command(cmd)

        resp = None
        for i in range(ntries + 1):
            resp = self._parse_response(self.ask(cmd, verbose=verbose))
            if verbose:
                m = 'repeat command {} response = {} len={} '.format(i + 1,
                                                                     resp,
                                                                     len(str(resp)) if resp is not None else None)
                self.debug(m)
            if check_val is not None:
                if self.simulation:
                    resp = check_val

                if resp == check_val:
                    break
                else:
                    continue

            if check_type is not None:
                if self.simulation:
                    resp = random.randint(1, 10)
                else:
                    try:
                        resp = check_type(resp)
                    except (ValueError, TypeError):
                        continue

            if resp is not None:
                break

        return resp

    # ===============================================================================
    # scanable interface
    # ===============================================================================
    def _scan_hook(self, v):
        for a in self.alarms:
            if a.test_condition(v):
                alarm_msg = a.get_message(v)
                self.warning(alarm_msg)
                manager = self.application.get_service('pychron.social.twitter_manager.TwitterManager')
                if manager is not None:
                    manager.post(alarm_msg)
                break

    def _parse_response(self, v):
        return v

    def _communicate_hook(self, cmd, r):
        self.last_command = cmd
        self.last_response = str(r) if r else ''

    def _load_hook(self, config):
        pass
# ========================= EOF ============================================
<|endoftext|>


<commit_before>#!/usr/bin/env python

import os
import json
import socket
import logging
from random import randint
from bottle import route, request, response, run, error, default_app
from logentries import LogentriesHandler
from dicttoxml import dicttoxml
import pydenticon

def getIPAddress():
	try:
		if request.headers.get('Cf-Connecting-Ip') == None and request.headers.get('X-Forwarded-For') == None:
			raise TypeError
		elif request.headers.get('Cf-Connecting-Ip') != None:
			return request.headers.get('Cf-Connecting-Ip')
		else:
			return request.headers.get('X-Forwarded-For')
	except TypeError:
		return request.get('REMOTE_ADDR')

def getReverseHost():
	try:
		return socket.gethostbyaddr(getIPAddress())[0]
	except:
		return "Unable to resolve IP address to reverse hostname"

def getRequestHeaders():
	return request.headers.keys()

def addHeadersToAllRequests():
	response.set_header('X-ContactTheAuthor', 'themaster@ipinfo.in')
	return True

def buildContentResponse(content):
	# First, we're going to add our headers as needed
	addHeadersToAllRequests()
	# And define a default dictionary
	content = {
		"results": ""
	}
	return content

def isJSONResponse():
	if request.headers.get('Accept') == "application/json" \
	or request.path.endswith('.json'):
		response.content_type = 'application/json'
		return True

def isXMLResponse():
	if request.headers.get('Accept') == "application/xml" \
	or request.path.endswith('.xml'):
		response.content_type = 'application/xml'
		return True

@route('favicon.ico')
@error(404)
def error404(error):
	response.status = 404
	response.content_type = 'text/plain'
	return 'Nothing here, sorry'

@route('/version')
def getVersion():
	addHeadersToAllRequests()
	try:
		dirname, filename = os.path.split(os.path.abspath(__file__))
		f = open(os.getenv('VERSION_PATH', dirname + '/.git/refs/heads/master'), 'r')
		content = f.read()
		response.content_type = 'text/plain'
		return content
	except:
		return "Unable to open version file."

@route('/icon')
@route('/icon/<height:int>')
@route('/icon/<height:int>/<width:int>')
def getIcon(height=100,width=100):
	addHeadersToAllRequests()
	response.content_type = 'image/png'
	colors = []
	colors.append("rgb(" + str(randint(1, 255)) + "," + str(randint(1, 255)) + "," + str(randint(1, 255)) + ")")
	generator = pydenticon.Generator(8,8,foreground=colors)
	identicon = generator.generate(getIPAddress(), height, width)
	return identicon

@route('/headers')
@route('/headers.json')
@route('/headers.xml')
def headers():
	content = {
		"results": {}
	}
	for key in getRequestHeaders():
		content["results"][key] = request.headers.get(key)
	if isJSONResponse():
		return json.dumps(content)
	if isXMLResponse():
		return dicttoxml(content)
	else:
		content = ""
		for key in getRequestHeaders():
			content = content + "" + key + ": " + str(request.headers.get(key)) + "
"
		response.content_type = 'text/html'
		return content

@route('/reverse')
@route('/reverse.json')
@route('/reverse.xml')
def reverse():
	addHeadersToAllRequests()
	content = {
		"results": {
			'reverse': getReverseHost()
		}
	}
	if isJSONResponse():
		return json.dumps(content)
	elif isXMLResponse():
		return dicttoxml(content)
	else:
		response.content_type = 'text/plain'
		return getReverseHost()

@route('/')
@route('/ip')
@route('/ip.json')
@route('/ip.xml')
def ip():
	addHeadersToAllRequests()
	content = {
		"results": {
			'ip': getIPAddress()
		}
	}
	if isJSONResponse():
		return json.dumps(content)
	elif isXMLResponse():
		return dicttoxml(content)
	else:
		response.content_type = 'text/plain'
		return getIPAddress()

if __name__ == '__main__':

	application = default_app()

	serverHost = os.getenv('SERVER_HOST', 'localhost')
	serverPort = os.getenv('SERVER_PORT', '5000')

	# Now we're ready, so start the server
	# Instantiate the logger
	log = logging.getLogger('log')
	console = logging.StreamHandler()
	log.setLevel(logging.INFO)
	log.addHandler(console)

	if os.getenv('LOGENTRIES_TOKEN') == '':
		log.addHandler(LogentriesHandler(os.getenv('LOGENTRIES_TOKEN', '')))

	# Now we're ready, so start the server
	try:
		log.info("Successfully started application server on " + socket.gethostname())
		application.run(host=serverHost, port=serverPort)
	except:
		log.info("Failed to start application server on " + socket.gethostname())
<commit_msg>Converted indentation from tabs to spaces to appease pylint<commit_after>#!/usr/bin/env python

import os
import json
import socket
import logging
from random import randint
from bottle import route, request, response, run, error, default_app
from logentries import LogentriesHandler
from dicttoxml import dicttoxml
import pydenticon

def getIPAddress():
    try:
        if request.headers.get('Cf-Connecting-Ip') == None and request.headers.get('X-Forwarded-For') == None:
            raise TypeError
        elif request.headers.get('Cf-Connecting-Ip') != None:
            return request.headers.get('Cf-Connecting-Ip')
        else:
            return request.headers.get('X-Forwarded-For')
    except TypeError:
        return request.get('REMOTE_ADDR')

def getReverseHost():
    try:
        return socket.gethostbyaddr(getIPAddress())[0]
    except:
        return "Unable to resolve IP address to reverse hostname"

def getRequestHeaders():
    return request.headers.keys()

def addHeadersToAllRequests():
    response.set_header('X-ContactTheAuthor', 'themaster@ipinfo.in')
    return True

def buildContentResponse(content):
    # First, we're going to add our headers as needed
    addHeadersToAllRequests()
    # And define a default dictionary
    content = {
        "results": ""
    }
    return content

def isJSONResponse():
    if request.headers.get('Accept') == "application/json" \
    or request.path.endswith('.json'):
        response.content_type = 'application/json'
        return True

def isXMLResponse():
    if request.headers.get('Accept') == "application/xml" \
    or request.path.endswith('.xml'):
        response.content_type = 'application/xml'
        return True

@route('favicon.ico')
@error(404)
def error404(error):
    response.status = 404
    response.content_type = 'text/plain'
    return 'Nothing here, sorry'

@route('/version')
def getVersion():
    addHeadersToAllRequests()
    try:
        dirname, filename = os.path.split(os.path.abspath(__file__))
        f = open(os.getenv('VERSION_PATH', dirname + '/.git/refs/heads/master'), 'r')
        content = f.read()
        response.content_type = 'text/plain'
        return content
    except:
        return "Unable to open version file."

@route('/icon')
@route('/icon/<height:int>')
@route('/icon/<height:int>/<width:int>')
def getIcon(height=100,width=100):
    addHeadersToAllRequests()
    response.content_type = 'image/png'
    colors = []
    colors.append("rgb(" + str(randint(1, 255)) + "," + str(randint(1, 255)) + "," + str(randint(1, 255)) + ")")
    generator = pydenticon.Generator(8,8,foreground=colors)
    identicon = generator.generate(getIPAddress(), height, width)
    return identicon

@route('/headers')
@route('/headers.json')
@route('/headers.xml')
def headers():
    content = {
        "results": {}
    }
    for key in getRequestHeaders():
        content["results"][key] = request.headers.get(key)
    if isJSONResponse():
        return json.dumps(content)
    if isXMLResponse():
        return dicttoxml(content)
    else:
        content = ""
        for key in getRequestHeaders():
            content = content + "" + key + ": " + str(request.headers.get(key)) + "
"
        response.content_type = 'text/html'
        return content

@route('/reverse')
@route('/reverse.json')
@route('/reverse.xml')
def reverse():
    addHeadersToAllRequests()
    content = {
        "results": {
            'reverse': getReverseHost()
        }
    }
    if isJSONResponse():
        return json.dumps(content)
    elif isXMLResponse():
        return dicttoxml(content)
    else:
        response.content_type = 'text/plain'
        return getReverseHost()

@route('/')
@route('/ip')
@route('/ip.json')
@route('/ip.xml')
def ip():
    addHeadersToAllRequests()
    content = {
        "results": {
            'ip': getIPAddress()
        }
    }
    if isJSONResponse():
        return json.dumps(content)
    elif isXMLResponse():
        return dicttoxml(content)
    else:
        response.content_type = 'text/plain'
        return getIPAddress()

if __name__ == '__main__':

    application = default_app()

    serverHost = os.getenv('SERVER_HOST', 'localhost')
    serverPort = os.getenv('SERVER_PORT', '5000')

    # Now we're ready, so start the server
    # Instantiate the logger
    log = logging.getLogger('log')
    console = logging.StreamHandler()
    log.setLevel(logging.INFO)
    log.addHandler(console)

    if os.getenv('LOGENTRIES_TOKEN') == '':
        log.addHandler(LogentriesHandler(os.getenv('LOGENTRIES_TOKEN', '')))

    # Now we're ready, so start the server
    try:
        log.info("Successfully started application server on " + socket.gethostname())
        application.run(host=serverHost, port=serverPort)
    except:
        log.info("Failed to start application server on " + socket.gethostname())
<|endoftext|>


<commit_before># -*- coding: UTF-8 -*-
"""
Source processing routines
"""

from __future__ import absolute_import
from __future__ import unicode_literals
import warnings
from collections import OrderedDict
import numpy as np
import os
import pysynphot as S
import astropy.table as at
from . import io
from . import passband

def load_source(sourcenames):
    """
    Loads sources

    Parameters
    ----------
    sourcenames : array-like
        The source names. Passed to :py:func:`source_synphot.io.read_source`

    Returns
    -------
    sources : dict
        The dictionary of source spectra

    See Also
    --------
    :py:func:`source_synphot.io.read_source`
    """

    sources = OrderedDict()
    if np.isscalar(sourcenames):
        sourcenames = np.array(sourcenames, ndmin=1)
    else:
        sourcenames = np.array(sourcenames).flatten()
    nsource = len(sourcenames)

    for source in sourcenames:
        try:
            thissource  = io.read_source(source)
        except Exception as e:
            message = 'Source {} not loaded'.format(source)
            warnings.warn(message, RuntimeWarning)
            continue
        sources[source] = thissource

    return sources

def pre_process_source(source, sourcemag, sourcepb, sourcez, smooth=True):
    """
    Pre-process a source at some redshift ``sourcez`` back to the rest-frame
    and normalize it to have magnitude ``sourcemag`` in passband ``sourcepb``

    Parameters
    ----------
    sourcespec : str
            The source spectrum filename
    sourcemag : float
            The magnitude of the source spectrum in passband ``sourcepb``
    sourcepb : :py:class:`pysynphot.spectrum.ArraySpectralElement`
            The passband in which `source` has magnitude ``sourcemag``
    sourcez : float
            The redshift of `source`
    smooth : bool, optional
            Smooth the spectrum (default: True)

    Returns
    -------
    source : :py:class:`pysynphot.ArraySpectrum`
        The de-redshifted, normalized and optionally smoothed spectrum

    See Also
    --------
    :py:func:`astropy.table.Table.read`
    """
    inspec    = None
    inspecz   = np.nan
    inspecmag = np.nan
    inspecpb  = None

    source_table_file = os.path.join('sources', 'sourcetable.txt')
    source_table_file = io.get_pkgfile(source_table_file)
    source_table = at.Table.read(source_table_file, format='ascii')
    ind = (source_table['specname'] == sourcespec)
    nmatch = len(source_table['specname'][ind])
    if nmatch == 1:
        # load the file and the info
        inspec    = source_table['specname'][ind][0]
        inspecz   = source_table['redshift'][ind][0]
        inspecmag = source_table['g'][ind][0] # for now, just normalize the g-band mag
    elif nmatch == 0:
        message = 'Spectrum {} not listed in lookup table'.format(sourcespec)
        pass
    else:
        message = 'Spectrum {} not uniquely listed in lookup table'.format(sourcespec)
        pass

    if inspec is None:
        warnings.warn(message, RuntimeWarning)
        inspec    = sourcespec
        inspecz   = sourcez
        inspecmag = sourcemag
        inspecpb  = sourcepb

    if not os.path.exists(inspec):
        message = 'Spectrum {} could not be found'.format(inspec)
        raise valueError(message)

    try:
        spec = at.Table.read(inspec, names=('wave','flux','dflux'), format='ascii')
    except Exception as e:
        message = 'Could not read file {}'.format(sourcespec)
        raise ValueError(message)

    if hasattr(inspecpb,'wave') and hasattr(inspecpb, 'throughput'):
        pass
    else:
        pbs = passband.load_pbs([inspecpb], 0.)
        try:
            inspecpb = pbs[inspecpb]
        except KeyError as e:
            message = 'Could not load passband {}'.format(inspecpb)
            raise RuntimeError(message)

    try:
        inspecmag = float(inspecmag)
    except (TypeError, ValueError) as e:
        message = 'Source magnitude {} could not be interpreted as a float'.format(inspecmag)
        raise ValueError(message)

    try:
        inspecz = float(inspecz)
    except (TypeError, ValueError) as e:
        message = 'Source redshift  {} could not be interpreted as a float'.format(inspecz)
        raise ValueError(message)

    if inspecz < 0 :
        message = 'Source must have positive definite cosmological redshift'
        raise ValueError(message)

    inspec = S.ArraySpectrum(spec['wave'], spec['flux'], fluxunits='flam')
    zblue = 1./(1+inspecz) - 1.
    inspec_rest = inspec.redshift(zblue)
    # TODO renorm is basic and just calculates dmag = RNval - what the original spectrum's mag is
    # and renormalizes - there's some sanity checking for overlaps
    # we can do this without using it and relying on the .passband routines
    try:
        out = inspec_rest.renorm(sourcemag, 'ABmag', inspecpb[0])
    except Exception as e:
        message = 'Could not renormalize spectrum {}'.format(inspec)
        raise RuntimeError(message)
    return out

<commit_msg>fix typos in pre_process_spectrum<commit_after># -*- coding: UTF-8 -*-
"""
Source processing routines
"""

from __future__ import absolute_import
from __future__ import unicode_literals
import warnings
from collections import OrderedDict
import numpy as np
import os
import pysynphot as S
import astropy.table as at
from . import io
from . import passband

def load_source(sourcenames):
    """
    Loads sources

    Parameters
    ----------
    sourcenames : array-like
        The source names. Passed to :py:func:`source_synphot.io.read_source`

    Returns
    -------
    sources : dict
        The dictionary of source spectra

    See Also
    --------
    :py:func:`source_synphot.io.read_source`
    """

    sources = OrderedDict()
    if np.isscalar(sourcenames):
        sourcenames = np.array(sourcenames, ndmin=1)
    else:
        sourcenames = np.array(sourcenames).flatten()
    nsource = len(sourcenames)

    for source in sourcenames:
        try:
            thissource  = io.read_source(source)
        except Exception as e:
            message = 'Source {} not loaded'.format(source)
            warnings.warn(message, RuntimeWarning)
            continue
        sources[source] = thissource

    return sources

def pre_process_source(source, sourcemag, sourcepb, sourcez, smooth=True):
    """
    Pre-process a source at some redshift ``sourcez`` back to the rest-frame
    and normalize it to have magnitude ``sourcemag`` in passband ``sourcepb``

    Parameters
    ----------
    sourcespec : str
            The source spectrum filename
    sourcemag : float
            The magnitude of the source spectrum in passband ``sourcepb``
    sourcepb : :py:class:`pysynphot.spectrum.ArraySpectralElement`
            The passband in which `source` has magnitude ``sourcemag``
    sourcez : float
            The redshift of `source`
    smooth : bool, optional
            Smooth the spectrum (default: True)

    Returns
    -------
    source : :py:class:`pysynphot.ArraySpectrum`
        The de-redshifted, normalized and optionally smoothed spectrum

    See Also
    --------
    :py:func:`astropy.table.Table.read`
    """
    inspec    = None
    inspecz   = np.nan
    inspecmag = np.nan
    inspecpb  = None

    source_table_file = os.path.join('sources', 'sourcetable.txt')
    source_table_file = io.get_pkgfile(source_table_file)
    source_table = at.Table.read(source_table_file, format='ascii')
    ind = (source_table['specname'] == source)
    nmatch = len(source_table['specname'][ind])
    if nmatch == 1:
        # load the file and the info
        inspec    = source_table['specname'][ind][0]
        inspecz   = source_table['redshift'][ind][0]
        inspecmag = source_table['g'][ind][0] # for now, just normalize the g-band mag
    elif nmatch == 0:
        message = 'Spectrum {} not listed in lookup table'.format(source)
        pass
    else:
        message = 'Spectrum {} not uniquely listed in lookup table'.format(source)
        pass

    if inspec is None:
        warnings.warn(message, RuntimeWarning)
        inspec    = source
        inspecz   = sourcez
        inspecmag = sourcemag
        inspecpb  = sourcepb

    if not os.path.exists(inspec):
        message = 'Spectrum {} could not be found'.format(inspec)
        raise valueError(message)

    try:
        spec = at.Table.read(inspec, names=('wave','flux'), format='ascii')
    except Exception as e:
        message = 'Could not read file {}'.format(source)
        raise ValueError(message)

    if hasattr(inspecpb,'wave') and hasattr(inspecpb, 'throughput'):
        pass
    else:
        pbs = passband.load_pbs([inspecpb], 0.)
        try:
            inspecpb = pbs[inspecpb][0]
        except KeyError as e:
            message = 'Could not load passband {}'.format(inspecpb)
            raise RuntimeError(message)

    try:
        inspecmag = float(inspecmag)
    except (TypeError, ValueError) as e:
        message = 'Source magnitude {} could not be interpreted as a float'.format(inspecmag)
        raise ValueError(message)

    try:
        inspecz = float(inspecz)
    except (TypeError, ValueError) as e:
        message = 'Source redshift  {} could not be interpreted as a float'.format(inspecz)
        raise ValueError(message)

    if inspecz < 0 :
        message = 'Source must have positive definite cosmological redshift'
        raise ValueError(message)

    inspec = S.ArraySpectrum(spec['wave'], spec['flux'], fluxunits='flam')
    zblue = 1./(1+inspecz) - 1.
    inspec_rest = inspec.redshift(zblue)
    # TODO renorm is basic and just calculates dmag = RNval - what the original spectrum's mag is
    # and renormalizes - there's some sanity checking for overlaps
    # we can do this without using it and relying on the .passband routines
    try:
        out = inspec_rest.renorm(sourcemag, 'ABmag', inspecpb)
    except Exception as e:
        message = 'Could not renormalize spectrum {}'.format(inspec)
        raise RuntimeError(message)
    return out

<|endoftext|>