<commit_before>import itertools
import sys
import struct
from io import BytesIO, SEEK_CUR, SEEK_END  # noqa

PY2 = sys.version_info[0] == 2

# Kaitai Struct runtime streaming API version, defined as per PEP-0396
# standard. Used for two purposes:
#
# * .py files generated by ksc from .ksy check that they import proper
#   KS runtime library by this version number;
# * distribution utils (setup.py) use this when packaging for PyPI
#
__version__ = '0.9'

class KaitaiStruct(object):
    def __init__(self, stream):
        self._io = stream

    def __enter__(self):
        return self

    def __exit__(self, *args, **kwargs):
        self.close()

    def close(self):
        self._io.close()

    @classmethod
    def from_file(cls, filename):
        f = open(filename, 'rb')
        try:
            return cls(KaitaiStream(f))
        except Exception:
            # close file descriptor, then reraise the exception
            f.close()
            raise

    @classmethod
    def from_bytes(cls, buf):
        return cls(KaitaiStream(BytesIO(buf)))

    @classmethod
    def from_io(cls, io):
        return cls(KaitaiStream(io))

class KaitaiStream(object):
    def __init__(self, io):
        self._io = io
        self.align_to_byte()

    def __enter__(self):
        return self

    def __exit__(self, *args, **kwargs):
        self.close()

    def close(self):
        self._io.close()

    # ========================================================================
    # Stream positioning
    # ========================================================================

    def is_eof(self):
        if self.bits_left > 0:
            return False

        io = self._io
        t = io.read(1)
        if t == b'':
            return True
        else:
            io.seek(-1, SEEK_CUR)
            return False

    def seek(self, n):
        self._io.seek(n)

    def pos(self):
        return self._io.tell()

    def size(self):
        # Python has no internal File object API function to get
        # current file / StringIO size, thus we use the following
        # trick.
        io = self._io
        # Remember our current position
        cur_pos = io.tell()
        # Seek to the end of the File object
        io.seek(0, SEEK_END)
        # Remember position, which is equal to the full length
        full_size = io.tell()
        # Seek back to the current position
        io.seek(cur_pos)
        return full_size

    # ========================================================================
    # Integer numbers
    # ========================================================================

    packer_s1 = struct.Struct('b')
    packer_s2be = struct.Struct('>h')
    packer_s4be = struct.Struct('>i')
    packer_s8be = struct.Struct('>q')
    packer_s2le = struct.Struct('<h')
    packer_s4le = struct.Struct('<i')
    packer_s8le = struct.Struct('<q')

    packer_u1 = struct.Struct('B')
    packer_u2be = struct.Struct('>H')
    packer_u4be = struct.Struct('>I')
    packer_u8be = struct.Struct('>Q')
    packer_u2le = struct.Struct('<H')
    packer_u4le = struct.Struct('<I')
    packer_u8le = struct.Struct('<Q')

    # ------------------------------------------------------------------------
    # Signed
    # ------------------------------------------------------------------------

    def read_s1(self):
        return KaitaiStream.packer_s1.unpack(self.read_bytes(1))[0]

    # ........................................................................
    # Big-endian
    # ........................................................................

    def read_s2be(self):
        return KaitaiStream.packer_s2be.unpack(self.read_bytes(2))[0]

    def read_s4be(self):
        return KaitaiStream.packer_s4be.unpack(self.read_bytes(4))[0]

    def read_s8be(self):
        return KaitaiStream.packer_s8be.unpack(self.read_bytes(8))[0]

    # ........................................................................
    # Little-endian
    # ........................................................................

    def read_s2le(self):
        return KaitaiStream.packer_s2le.unpack(self.read_bytes(2))[0]

    def read_s4le(self):
        return KaitaiStream.packer_s4le.unpack(self.read_bytes(4))[0]

    def read_s8le(self):
        return KaitaiStream.packer_s8le.unpack(self.read_bytes(8))[0]

    # ------------------------------------------------------------------------
    # Unsigned
    # ------------------------------------------------------------------------

    def read_u1(self):
        return KaitaiStream.packer_u1.unpack(self.read_bytes(1))[0]

    # ........................................................................
    # Big-endian
    # ........................................................................

    def read_u2be(self):
        return KaitaiStream.packer_u2be.unpack(self.read_bytes(2))[0]

    def read_u4be(self):
        return KaitaiStream.packer_u4be.unpack(self.read_bytes(4))[0]

    def read_u8be(self):
        return KaitaiStream.packer_u8be.unpack(self.read_bytes(8))[0]

    # ........................................................................
    # Little-endian
    # ........................................................................

    def read_u2le(self):
        return KaitaiStream.packer_u2le.unpack(self.read_bytes(2))[0]

    def read_u4le(self):
        return KaitaiStream.packer_u4le.unpack(self.read_bytes(4))[0]

    def read_u8le(self):
        return KaitaiStream.packer_u8le.unpack(self.read_bytes(8))[0]

    # ========================================================================
    # Floating point numbers
    # ========================================================================

    packer_f4be = struct.Struct('>f')
    packer_f8be = struct.Struct('>d')
    packer_f4le = struct.Struct('<f')
    packer_f8le = struct.Struct('<d')

    # ........................................................................
    # Big-endian
    # ........................................................................

    def read_f4be(self):
        return KaitaiStream.packer_f4be.unpack(self.read_bytes(4))[0]

    def read_f8be(self):
        return KaitaiStream.packer_f8be.unpack(self.read_bytes(8))[0]

    # ........................................................................
    # Little-endian
    # ........................................................................

    def read_f4le(self):
        return KaitaiStream.packer_f4le.unpack(self.read_bytes(4))[0]

    def read_f8le(self):
        return KaitaiStream.packer_f8le.unpack(self.read_bytes(8))[0]

    # ========================================================================
    # Unaligned bit values
    # ========================================================================

    def align_to_byte(self):
        self.bits = 0
        self.bits_left = 0

    def read_bits_int(self, n):
        bits_needed = n - self.bits_left
        if bits_needed > 0:
            # 1 bit  => 1 byte
            # 8 bits => 1 byte
            # 9 bits => 2 bytes
            bytes_needed = ((bits_needed - 1) // 8) + 1
            buf = self.read_bytes(bytes_needed)
            for byte in buf:
                # Python 2 will get "byte" as one-character str, thus
                # we need to convert it to integer manually; Python 3
                # is fine as is.
                if isinstance(byte, str):
                    byte = ord(byte)
                self.bits <<= 8
                self.bits |= byte
                self.bits_left += 8

        # raw mask with required number of 1s, starting from lowest bit
        mask = (1 << n) - 1
        # shift mask to align with highest bits available in self.bits
        shift_bits = self.bits_left - n
        mask <<= shift_bits
        # derive reading result
        res = (self.bits & mask) >> shift_bits
        # clear top bits that we've just read => AND with 1s
        self.bits_left -= n
        mask = (1 << self.bits_left) - 1
        self.bits &= mask

        return res

    # ========================================================================
    # Byte arrays
    # ========================================================================

    def read_bytes(self, n):
        if n < 0:
            raise ValueError(
                "requested invalid %d amount of bytes" %
                (n,)
            )
        r = self._io.read(n)
        if len(r) < n:
            raise EOFError(
                "requested %d bytes, but got only %d bytes" %
                (n, len(r))
            )
        return r

    def read_bytes_full(self):
        return self._io.read()

    def read_bytes_term(self, term, include_term, consume_term, eos_error):
        r = b''
        while True:
            c = self._io.read(1)
            if c == b'':
                if eos_error:
                    raise Exception(
                        "end of stream reached, but no terminator %d found" %
                        (term,)
                    )
                else:
                    return r
            elif ord(c) == term:
                if include_term:
                    r += c
                if not consume_term:
                    self._io.seek(-1, SEEK_CUR)
                return r
            else:
                r += c

    def ensure_fixed_contents(self, expected):
        actual = self._io.read(len(expected))
        if actual != expected:
            raise Exception(
                "unexpected fixed contents: got %r, was waiting for %r" %
                (actual, expected)
            )
        return actual

    @staticmethod
    def bytes_strip_right(data, pad_byte):
        new_len = len(data)
        if PY2:
            # data[...] must yield an integer, to compare with integer pad_byte
            data = bytearray(data)

        while new_len > 0 and data[new_len - 1] == pad_byte:
            new_len -= 1

        return data[:new_len]

    @staticmethod
    def bytes_terminate(data, term, include_term):
        new_len = 0
        max_len = len(data)
        if PY2:
            # data[...] must yield an integer, to compare with integer term
            data = bytearray(data)

        while new_len < max_len and data[new_len] != term:
            new_len += 1

        if include_term and new_len < max_len:
            new_len += 1

        return data[:new_len]

    # ========================================================================
    # Byte array processing
    # ========================================================================

    @staticmethod
    def process_xor_one(data, key):
        if PY2:
            return bytes(bytearray(v ^ key for v in bytearray(data)))
        else:
            return bytes(v ^ key for v in data)

    @staticmethod
    def process_xor_many(data, key):
        if PY2:
            return bytes(bytearray(a ^ b for a, b in zip(bytearray(data), itertools.cycle(bytearray(key)))))
        else:
            return bytes(a ^ b for a, b in zip(data, itertools.cycle(key)))

    @staticmethod
    def process_rotate_left(data, amount, group_size):
        if group_size != 1:
            raise Exception(
                "unable to rotate group of %d bytes yet" %
                (group_size,)
            )

        mask = group_size * 8 - 1
        anti_amount = -amount & mask

        r = bytearray(data)
        for i in range(len(r)):
            r[i] = (r[i] << amount) & 0xff | (r[i] >> anti_amount)
        return bytes(r)

    # ========================================================================
    # Misc
    # ========================================================================

    @staticmethod
    def resolve_enum(enum_obj, value):
        """Resolves value using enum: if the value is not found in the map,
        we'll just use literal value per se. Works around problem with Python
        enums throwing an exception when encountering unknown value.
        """
        try:
            return enum_obj(value)
        except ValueError:
            return value
<commit_msg>Added system of exception<commit_after>import itertools
import sys
import struct
from io import BytesIO, SEEK_CUR, SEEK_END  # noqa

PY2 = sys.version_info[0] == 2

# Kaitai Struct runtime streaming API version, defined as per PEP-0396
# standard. Used for two purposes:
#
# * .py files generated by ksc from .ksy check that they import proper
#   KS runtime library by this version number;
# * distribution utils (setup.py) use this when packaging for PyPI
#
__version__ = '0.9'

class KaitaiStruct(object):
    def __init__(self, stream):
        self._io = stream

    def __enter__(self):
        return self

    def __exit__(self, *args, **kwargs):
        self.close()

    def close(self):
        self._io.close()

    @classmethod
    def from_file(cls, filename):
        f = open(filename, 'rb')
        try:
            return cls(KaitaiStream(f))
        except Exception:
            # close file descriptor, then reraise the exception
            f.close()
            raise

    @classmethod
    def from_bytes(cls, buf):
        return cls(KaitaiStream(BytesIO(buf)))

    @classmethod
    def from_io(cls, io):
        return cls(KaitaiStream(io))

class KaitaiStream(object):
    def __init__(self, io):
        self._io = io
        self.align_to_byte()

    def __enter__(self):
        return self

    def __exit__(self, *args, **kwargs):
        self.close()

    def close(self):
        self._io.close()

    # ========================================================================
    # Stream positioning
    # ========================================================================

    def is_eof(self):
        if self.bits_left > 0:
            return False

        io = self._io
        t = io.read(1)
        if t == b'':
            return True
        else:
            io.seek(-1, SEEK_CUR)
            return False

    def seek(self, n):
        self._io.seek(n)

    def pos(self):
        return self._io.tell()

    def size(self):
        # Python has no internal File object API function to get
        # current file / StringIO size, thus we use the following
        # trick.
        io = self._io
        # Remember our current position
        cur_pos = io.tell()
        # Seek to the end of the File object
        io.seek(0, SEEK_END)
        # Remember position, which is equal to the full length
        full_size = io.tell()
        # Seek back to the current position
        io.seek(cur_pos)
        return full_size

    # ========================================================================
    # Integer numbers
    # ========================================================================

    packer_s1 = struct.Struct('b')
    packer_s2be = struct.Struct('>h')
    packer_s4be = struct.Struct('>i')
    packer_s8be = struct.Struct('>q')
    packer_s2le = struct.Struct('<h')
    packer_s4le = struct.Struct('<i')
    packer_s8le = struct.Struct('<q')

    packer_u1 = struct.Struct('B')
    packer_u2be = struct.Struct('>H')
    packer_u4be = struct.Struct('>I')
    packer_u8be = struct.Struct('>Q')
    packer_u2le = struct.Struct('<H')
    packer_u4le = struct.Struct('<I')
    packer_u8le = struct.Struct('<Q')

    # ------------------------------------------------------------------------
    # Signed
    # ------------------------------------------------------------------------

    def read_s1(self):
        return KaitaiStream.packer_s1.unpack(self.read_bytes(1))[0]

    # ........................................................................
    # Big-endian
    # ........................................................................

    def read_s2be(self):
        return KaitaiStream.packer_s2be.unpack(self.read_bytes(2))[0]

    def read_s4be(self):
        return KaitaiStream.packer_s4be.unpack(self.read_bytes(4))[0]

    def read_s8be(self):
        return KaitaiStream.packer_s8be.unpack(self.read_bytes(8))[0]

    # ........................................................................
    # Little-endian
    # ........................................................................

    def read_s2le(self):
        return KaitaiStream.packer_s2le.unpack(self.read_bytes(2))[0]

    def read_s4le(self):
        return KaitaiStream.packer_s4le.unpack(self.read_bytes(4))[0]

    def read_s8le(self):
        return KaitaiStream.packer_s8le.unpack(self.read_bytes(8))[0]

    # ------------------------------------------------------------------------
    # Unsigned
    # ------------------------------------------------------------------------

    def read_u1(self):
        return KaitaiStream.packer_u1.unpack(self.read_bytes(1))[0]

    # ........................................................................
    # Big-endian
    # ........................................................................

    def read_u2be(self):
        return KaitaiStream.packer_u2be.unpack(self.read_bytes(2))[0]

    def read_u4be(self):
        return KaitaiStream.packer_u4be.unpack(self.read_bytes(4))[0]

    def read_u8be(self):
        return KaitaiStream.packer_u8be.unpack(self.read_bytes(8))[0]

    # ........................................................................
    # Little-endian
    # ........................................................................

    def read_u2le(self):
        return KaitaiStream.packer_u2le.unpack(self.read_bytes(2))[0]

    def read_u4le(self):
        return KaitaiStream.packer_u4le.unpack(self.read_bytes(4))[0]

    def read_u8le(self):
        return KaitaiStream.packer_u8le.unpack(self.read_bytes(8))[0]

    # ========================================================================
    # Floating point numbers
    # ========================================================================

    packer_f4be = struct.Struct('>f')
    packer_f8be = struct.Struct('>d')
    packer_f4le = struct.Struct('<f')
    packer_f8le = struct.Struct('<d')

    # ........................................................................
    # Big-endian
    # ........................................................................

    def read_f4be(self):
        return KaitaiStream.packer_f4be.unpack(self.read_bytes(4))[0]

    def read_f8be(self):
        return KaitaiStream.packer_f8be.unpack(self.read_bytes(8))[0]

    # ........................................................................
    # Little-endian
    # ........................................................................

    def read_f4le(self):
        return KaitaiStream.packer_f4le.unpack(self.read_bytes(4))[0]

    def read_f8le(self):
        return KaitaiStream.packer_f8le.unpack(self.read_bytes(8))[0]

    # ========================================================================
    # Unaligned bit values
    # ========================================================================

    def align_to_byte(self):
        self.bits = 0
        self.bits_left = 0

    def read_bits_int(self, n):
        bits_needed = n - self.bits_left
        if bits_needed > 0:
            # 1 bit  => 1 byte
            # 8 bits => 1 byte
            # 9 bits => 2 bytes
            bytes_needed = ((bits_needed - 1) // 8) + 1
            buf = self.read_bytes(bytes_needed)
            for byte in buf:
                # Python 2 will get "byte" as one-character str, thus
                # we need to convert it to integer manually; Python 3
                # is fine as is.
                if isinstance(byte, str):
                    byte = ord(byte)
                self.bits <<= 8
                self.bits |= byte
                self.bits_left += 8

        # raw mask with required number of 1s, starting from lowest bit
        mask = (1 << n) - 1
        # shift mask to align with highest bits available in self.bits
        shift_bits = self.bits_left - n
        mask <<= shift_bits
        # derive reading result
        res = (self.bits & mask) >> shift_bits
        # clear top bits that we've just read => AND with 1s
        self.bits_left -= n
        mask = (1 << self.bits_left) - 1
        self.bits &= mask

        return res

    # ========================================================================
    # Byte arrays
    # ========================================================================

    def read_bytes(self, n):
        if n < 0:
            raise ValueError(
                "requested invalid %d amount of bytes" %
                (n,)
            )
        r = self._io.read(n)
        if len(r) < n:
            raise EOFError(
                "requested %d bytes, but got only %d bytes" %
                (n, len(r))
            )
        return r

    def read_bytes_full(self):
        return self._io.read()

    def read_bytes_term(self, term, include_term, consume_term, eos_error):
        r = b''
        while True:
            c = self._io.read(1)
            if c == b'':
                if eos_error:
                    raise Exception(
                        "end of stream reached, but no terminator %d found" %
                        (term,)
                    )
                else:
                    return r
            elif ord(c) == term:
                if include_term:
                    r += c
                if not consume_term:
                    self._io.seek(-1, SEEK_CUR)
                return r
            else:
                r += c

    def ensure_fixed_contents(self, expected):
        actual = self._io.read(len(expected))
        if actual != expected:
            raise Exception(
                "unexpected fixed contents: got %r, was waiting for %r" %
                (actual, expected)
            )
        return actual

    @staticmethod
    def bytes_strip_right(data, pad_byte):
        new_len = len(data)
        if PY2:
            # data[...] must yield an integer, to compare with integer pad_byte
            data = bytearray(data)

        while new_len > 0 and data[new_len - 1] == pad_byte:
            new_len -= 1

        return data[:new_len]

    @staticmethod
    def bytes_terminate(data, term, include_term):
        new_len = 0
        max_len = len(data)
        if PY2:
            # data[...] must yield an integer, to compare with integer term
            data = bytearray(data)

        while new_len < max_len and data[new_len] != term:
            new_len += 1

        if include_term and new_len < max_len:
            new_len += 1

        return data[:new_len]

    # ========================================================================
    # Byte array processing
    # ========================================================================

    @staticmethod
    def process_xor_one(data, key):
        if PY2:
            return bytes(bytearray(v ^ key for v in bytearray(data)))
        else:
            return bytes(v ^ key for v in data)

    @staticmethod
    def process_xor_many(data, key):
        if PY2:
            return bytes(bytearray(a ^ b for a, b in zip(bytearray(data), itertools.cycle(bytearray(key)))))
        else:
            return bytes(a ^ b for a, b in zip(data, itertools.cycle(key)))

    @staticmethod
    def process_rotate_left(data, amount, group_size):
        if group_size != 1:
            raise Exception(
                "unable to rotate group of %d bytes yet" %
                (group_size,)
            )

        mask = group_size * 8 - 1
        anti_amount = -amount & mask

        r = bytearray(data)
        for i in range(len(r)):
            r[i] = (r[i] << amount) & 0xff | (r[i] >> anti_amount)
        return bytes(r)

    # ========================================================================
    # Misc
    # ========================================================================

    @staticmethod
    def resolve_enum(enum_obj, value):
        """Resolves value using enum: if the value is not found in the map,
        we'll just use literal value per se. Works around problem with Python
        enums throwing an exception when encountering unknown value.
        """
        try:
            return enum_obj(value)
        except ValueError:
            return value

class KaitaiStructError(BaseException):
    """Common ancestor for all error originating from Kaitai Struct usage.
    Stores KSY source path, pointing to an element supposedly guilty of
    an error.
    """
    def __init__(self, msg, src_path):
        super(KaitaiStructError, self).__init__("%s: %s" % (src_path, msg))
        self.src_path = src_path

class UndecidedEndiannessError(KaitaiStructError):
    """Error that occurs when default endianness should be decided with
    switch, but nothing matches (although using endianness expression
    implies that there should be some positive result).
    """
    def __init__(self, src_path):
        super(KaitaiStructError, self).__init__("unable to decide on endianness for a type", src_path)

class ValidationFailedError(KaitaiStructError):
    """Common ancestor for all validation failures. Stores pointer to
    KaitaiStream IO object which was involved in an error.
    """
    def __init__(self, msg, io, src_path):
        super(ValidationFailedError, self).__init__("at pos %d: validation failed: %s" % (io.pos(), msg), src_path)
        self.io = io

class ValidationNotEqualError(ValidationFailedError):
    """Signals validation failure: we required "actual" value to be equal to
    "expected", but it turned out that it's not.
    """
    def __init__(self, expected, actual, io, src_path):
        super(ValidationNotEqualError, self).__init__("not equal, expected %s, but got %s" % (repr(expected), repr(actual)), io, src_path)
        self.expected = expected
        self.actual = actual
<|endoftext|>


<commit_before>"""Custom message type for SQS messages."""
from __future__ import absolute_import

import pickle

import six

from kale import crypt
from kale import exceptions
from kale import settings
from kale import utils

_compressor = settings.COMPRESSOR
_decompressor = settings.DECOMPRESSOR
_task_size_limit = settings.SQS_TASK_SIZE_LIMIT
_get_current_timestamp = settings.TIMESTAMP_FUNC
_get_publisher_data = settings.PUBLISHER_STR_FUNC

class KaleMessage:
    """Kale message representing the data stored in an SQS queue."""

    # _task_mapper is a class cache mapping task paths to classes.
    # It will initially be populated with keys provided and will lazily
    # create other mappings.
    _task_mapper = None

    def __init__(self,
                 sqs_message_id=None,
                 sqs_receipt_handle=None,
                 sqs_queue_name=None,
                 task_class=None,
                 task_name=None,
                 task_id=None,
                 payload=None,
                 current_retry_num=None,
                 current_failure_num=None,
                 enqueued_time=None,
                 publisher_data=None,
                 instantiate_task=False,
                 delete_func=None
                 ):
        """Constructor.

        :param task_class: Class of task.
        :param task_name: Name of task. Required if task_class is not set.
        :param task_id: Id of task.
        :param payload: Payload holds the data that the task's run_task method will be called with.
        :param current_retry_num: Current task retry. This will be 0 from new tasks and will be
        incremented for each retry.
        :param current_failure_num: Current task failure. This will be 0 from new tasks and will be
        incremented for each failure.
        :param enqueued_time: Timestamp of when message was queued. If not provided then value set
        from setting's timestamp function.
        :param publisher_data: Str containing information about the publisher. If not provided the
        value from settings used.
        :param instantiate_task: Whether create instance of task_class. Default is false.
        :param delete_func: Delete function from the SQS message.

        """

        self._validate_task_payload(payload)
        retry_count = current_retry_num or 0
        failure_count = current_failure_num or 0

        self.id = sqs_message_id
        self.sqs_queue_name = sqs_queue_name
        self.sqs_receipt_handle = sqs_receipt_handle

        # This represents the path to the task. The consumer will have a
        # dictionary mapping these values to task classes.
        if task_class is not None:
            self.task_name = '.'.join([task_class.__module__, task_class.__name__])
        else:
            self.task_name = task_name

        self.task_id = task_id
        self.task_args = payload.get('args')
        self.task_kwargs = payload.get('kwargs')
        self.task_app_data = payload.get('app_data')
        self.task_retry_num = retry_count
        self.task_failure_num = failure_count
        self._enqueued_time = enqueued_time or _get_current_timestamp()
        self._publisher_data = publisher_data or _get_publisher_data()

        # Lazily instantiate the task mapper.
        if not self._task_mapper:
            self._task_mapper = {k: utils.class_import_from_path(v)
                                 for k, v in six.iteritems(settings.TASK_MAPPER)}

        # This will instantiate the task.
        if instantiate_task:
            self.task_inst = self._class_from_path(self.task_name)(self._get_message_body())

        self.delete_func = delete_func

    @staticmethod
    def _validate_task_payload(payload):
        """Validate that this is a valid task.

        :param payload: dictionary that will be submitted to the queue.
        :raises: AssertionError if payload is invalid.
        """

        assert 'args' in payload, 'args is required to be in the payload'
        assert 'kwargs' in payload, 'kwargs is required to be in the payload'

    def _get_message_body(self):
        message_body = {
            'id': self.task_id,
            'task': self.task_name,
            # Payload holds the data that the task's run_task method will be
            # called with.
            # Ex: mytask.ThisTask().run_task(
            # *payload['args'], **payload['kwargs'])
            'payload': {
                'args': self.task_args,
                'kwargs': self.task_kwargs,
                'app_data': self.task_app_data,
            },
            '_enqueued_time': self._enqueued_time,
            '_publisher': self._publisher_data,
            'retry_num': self.task_retry_num,
            'failure_num': self.task_failure_num,
        }
        return message_body

    def encode(self):
        """Custom encoding for Kale tasks.

        :return: string for encoded message.
        :rtype: str
        """

        compressed_msg = _compressor(
            pickle.dumps(self._get_message_body(), protocol=settings.PICKLE_PROTOCOL))
        compressed_msg = crypt.encrypt(compressed_msg)
        # Check compressed task size.
        if len(compressed_msg) >= _task_size_limit:
            raise exceptions.ChubbyTaskException(
                'Task %s is over the limit of %d bytes.' % (self.task_id,
                                                            _task_size_limit))

        return compressed_msg.decode("utf-8")

    @classmethod
    def decode_sqs(cls, sqs_message):
        """Custom decoding for Kale tasks from sqs messages

        :param boto3.resources.factory.sqs.Message sqs_message: message to decode.

        :return: a kale message
        :rtype: KaleMessage
        """

        message_body = crypt.decrypt(sqs_message.body)
        message_body = pickle.loads(_decompressor(message_body))
        # queue_url format is https://queue.amazonaws.com/<account id>/<queue name>
        sqs_queue_name = sqs_message.queue_url.rsplit('/', 1)[1]

        msg = cls(
            sqs_queue_name=sqs_queue_name,
            sqs_message_id=sqs_message.message_id,
            sqs_receipt_handle=sqs_message.receipt_handle,
            task_id=message_body.get('id'),
            task_name=message_body.get('task'),
            payload=message_body.get('payload'),
            enqueued_time=message_body.get('_enqueued_time'),
            publisher_data=message_body.get('_publisher'),
            current_retry_num=message_body.get('retry_num'),
            current_failure_num=message_body.get('failure_num'),
            instantiate_task=True,
            delete_func=sqs_message.delete
        )

        return msg

    @classmethod
    def decode_str(cls, message_str):
        """Custom decoding for Kale tasks from strings

        :param str message_str: message to decode.

        :return: a kale message
        :rtype: KaleMessage
        """

        message_body = crypt.decrypt(message_str)
        message_body = pickle.loads(_decompressor(message_body))

        msg = cls(
            task_id=message_body.get('id'),
            task_name=message_body.get('task'),
            payload=message_body.get('payload'),
            enqueued_time=message_body.get('_enqueued_time'),
            publisher_data=message_body.get('_publisher'),
            current_retry_num=message_body.get('retry_num'),
            current_failure_num = message_body.get('failure_num')
        )

        return msg

    def _class_from_path(self, task_path):
        """Return the task class given a task's path.

        :param task_path: string for a class, e.g., mytask.MyTask
        :return:
        """
        if task_path not in self._task_mapper:
            task_class = utils.class_import_from_path(task_path)
            self._task_mapper[task_path] = task_class
        return self._task_mapper[task_path]

    def delete(self):
        if self.delete_func is not None:
            self.delete_func()
<commit_msg>flake8 formatting<commit_after>"""Custom message type for SQS messages."""
from __future__ import absolute_import

import pickle

import six

from kale import crypt
from kale import exceptions
from kale import settings
from kale import utils

_compressor = settings.COMPRESSOR
_decompressor = settings.DECOMPRESSOR
_task_size_limit = settings.SQS_TASK_SIZE_LIMIT
_get_current_timestamp = settings.TIMESTAMP_FUNC
_get_publisher_data = settings.PUBLISHER_STR_FUNC

class KaleMessage:
    """Kale message representing the data stored in an SQS queue."""

    # _task_mapper is a class cache mapping task paths to classes.
    # It will initially be populated with keys provided and will lazily
    # create other mappings.
    _task_mapper = None

    def __init__(self,
                 sqs_message_id=None,
                 sqs_receipt_handle=None,
                 sqs_queue_name=None,
                 task_class=None,
                 task_name=None,
                 task_id=None,
                 payload=None,
                 current_retry_num=None,
                 current_failure_num=None,
                 enqueued_time=None,
                 publisher_data=None,
                 instantiate_task=False,
                 delete_func=None
                 ):
        """Constructor.

        :param task_class: Class of task.
        :param task_name: Name of task. Required if task_class is not set.
        :param task_id: Id of task.
        :param payload: Payload holds the data that the task's run_task method will be called with.
        :param current_retry_num: Current task retry. This will be 0 from new tasks and will be
        incremented for each retry.
        :param current_failure_num: Current task failure. This will be 0 from new tasks and will be
        incremented for each failure.
        :param enqueued_time: Timestamp of when message was queued. If not provided then value set
        from setting's timestamp function.
        :param publisher_data: Str containing information about the publisher. If not provided the
        value from settings used.
        :param instantiate_task: Whether create instance of task_class. Default is false.
        :param delete_func: Delete function from the SQS message.

        """

        self._validate_task_payload(payload)
        retry_count = current_retry_num or 0
        failure_count = current_failure_num or 0

        self.id = sqs_message_id
        self.sqs_queue_name = sqs_queue_name
        self.sqs_receipt_handle = sqs_receipt_handle

        # This represents the path to the task. The consumer will have a
        # dictionary mapping these values to task classes.
        if task_class is not None:
            self.task_name = '.'.join([task_class.__module__, task_class.__name__])
        else:
            self.task_name = task_name

        self.task_id = task_id
        self.task_args = payload.get('args')
        self.task_kwargs = payload.get('kwargs')
        self.task_app_data = payload.get('app_data')
        self.task_retry_num = retry_count
        self.task_failure_num = failure_count
        self._enqueued_time = enqueued_time or _get_current_timestamp()
        self._publisher_data = publisher_data or _get_publisher_data()

        # Lazily instantiate the task mapper.
        if not self._task_mapper:
            self._task_mapper = {k: utils.class_import_from_path(v)
                                 for k, v in six.iteritems(settings.TASK_MAPPER)}

        # This will instantiate the task.
        if instantiate_task:
            self.task_inst = self._class_from_path(self.task_name)(self._get_message_body())

        self.delete_func = delete_func

    @staticmethod
    def _validate_task_payload(payload):
        """Validate that this is a valid task.

        :param payload: dictionary that will be submitted to the queue.
        :raises: AssertionError if payload is invalid.
        """

        assert 'args' in payload, 'args is required to be in the payload'
        assert 'kwargs' in payload, 'kwargs is required to be in the payload'

    def _get_message_body(self):
        message_body = {
            'id': self.task_id,
            'task': self.task_name,
            # Payload holds the data that the task's run_task method will be
            # called with.
            # Ex: mytask.ThisTask().run_task(
            # *payload['args'], **payload['kwargs'])
            'payload': {
                'args': self.task_args,
                'kwargs': self.task_kwargs,
                'app_data': self.task_app_data,
            },
            '_enqueued_time': self._enqueued_time,
            '_publisher': self._publisher_data,
            'retry_num': self.task_retry_num,
            'failure_num': self.task_failure_num,
        }
        return message_body

    def encode(self):
        """Custom encoding for Kale tasks.

        :return: string for encoded message.
        :rtype: str
        """

        compressed_msg = _compressor(
            pickle.dumps(self._get_message_body(), protocol=settings.PICKLE_PROTOCOL))
        compressed_msg = crypt.encrypt(compressed_msg)
        # Check compressed task size.
        if len(compressed_msg) >= _task_size_limit:
            raise exceptions.ChubbyTaskException(
                'Task %s is over the limit of %d bytes.' % (self.task_id,
                                                            _task_size_limit))

        return compressed_msg.decode("utf-8")

    @classmethod
    def decode_sqs(cls, sqs_message):
        """Custom decoding for Kale tasks from sqs messages

        :param boto3.resources.factory.sqs.Message sqs_message: message to decode.

        :return: a kale message
        :rtype: KaleMessage
        """

        message_body = crypt.decrypt(sqs_message.body)
        message_body = pickle.loads(_decompressor(message_body))
        # queue_url format is https://queue.amazonaws.com/<account id>/<queue name>
        sqs_queue_name = sqs_message.queue_url.rsplit('/', 1)[1]

        msg = cls(
            sqs_queue_name=sqs_queue_name,
            sqs_message_id=sqs_message.message_id,
            sqs_receipt_handle=sqs_message.receipt_handle,
            task_id=message_body.get('id'),
            task_name=message_body.get('task'),
            payload=message_body.get('payload'),
            enqueued_time=message_body.get('_enqueued_time'),
            publisher_data=message_body.get('_publisher'),
            current_retry_num=message_body.get('retry_num'),
            current_failure_num=message_body.get('failure_num'),
            instantiate_task=True,
            delete_func=sqs_message.delete
        )

        return msg

    @classmethod
    def decode_str(cls, message_str):
        """Custom decoding for Kale tasks from strings

        :param str message_str: message to decode.

        :return: a kale message
        :rtype: KaleMessage
        """

        message_body = crypt.decrypt(message_str)
        message_body = pickle.loads(_decompressor(message_body))

        msg = cls(
            task_id=message_body.get('id'),
            task_name=message_body.get('task'),
            payload=message_body.get('payload'),
            enqueued_time=message_body.get('_enqueued_time'),
            publisher_data=message_body.get('_publisher'),
            current_retry_num=message_body.get('retry_num'),
            current_failure_num=message_body.get('failure_num')
        )

        return msg

    def _class_from_path(self, task_path):
        """Return the task class given a task's path.

        :param task_path: string for a class, e.g., mytask.MyTask
        :return:
        """
        if task_path not in self._task_mapper:
            task_class = utils.class_import_from_path(task_path)
            self._task_mapper[task_path] = task_class
        return self._task_mapper[task_path]

    def delete(self):
        if self.delete_func is not None:
            self.delete_func()
<|endoftext|>


<commit_before>import requests
from bs4 import BeautifulSoup
import sys
import orgs
import parse

reload(sys)
sys.setdefaultencoding('utf8')

def determine_bounds():
    ops = orgs.OPS()
    r = requests.get(ops.request_url)
    rtext = r.text
    soup = BeautifulSoup(rtext, "lxml")
    job_table = soup.find(ops.soup_find_list[0], ops.soup_find_list[1])
    job_table = job_table.parent.table
    nums = []
    rows = job_table.find_all("a", target="_self")
    for row in rows:
        url_text = row["href"].encode('utf-8')
        url_list = url_text.split("JobID=")
        job_id = int(url_list[1])
        nums.append(job_id)
    mi = max(nums) - 100
    ma = max(nums) + 100
    return mi, ma

def crawl(mi, ma):
    results = []
    for i in range(mi, ma):
        this_url = "https://www.gojobs.gov.on.ca/Preview.aspx?JobID=" + str(i)
        page = requests.get(this_url)
        ptext = page.text
        soup = BeautifulSoup(ptext, "lxml")
        if is_posting(soup):
            results.append(this_url)
            print "found OPS OT: {}".format(this_url)
    return results

def is_posting(soup):
    pared_soup = soup.find("table", id="JobAdTable_1")
    if not pared_soup:
        return False
    if pared_soup.find_next(text="Open Targeted "):
        return True
    return False

if __name__ == '__main__':
    mi = int(sys.argv[0])
    ma = int(sys.argv[1])
    data = parse.build_ot_list(mi, ma)
    parse.update_redis(data)
<commit_msg>debug<commit_after>import requests
from bs4 import BeautifulSoup
import sys
import orgs
import parse

reload(sys)
sys.setdefaultencoding('utf8')

def determine_bounds():
    ops = orgs.OPS()
    r = requests.get(ops.request_url)
    rtext = r.text
    soup = BeautifulSoup(rtext, "lxml")
    job_table = soup.find(ops.soup_find_list[0], ops.soup_find_list[1])
    job_table = job_table.parent.table
    nums = []
    rows = job_table.find_all("a", target="_self")
    for row in rows:
        url_text = row["href"].encode('utf-8')
        url_list = url_text.split("JobID=")
        job_id = int(url_list[1])
        nums.append(job_id)
    mi = max(nums) - 100
    ma = max(nums) + 100
    return mi, ma

def crawl(mi, ma):
    results = []
    for i in range(mi, ma):
        this_url = "https://www.gojobs.gov.on.ca/Preview.aspx?JobID=" + str(i)
        page = requests.get(this_url)
        ptext = page.text
        soup = BeautifulSoup(ptext, "lxml")
        if is_posting(soup):
            results.append(this_url)
            print "found OPS OT: {}".format(this_url)
    return results

def is_posting(soup):
    pared_soup = soup.find("table", id="JobAdTable_1")
    if not pared_soup:
        return False
    if pared_soup.find_next(text="Open Targeted "):
        return True
    return False

if __name__ == '__main__':
    mi = int(sys.argv[1])
    ma = int(sys.argv[2])
    print mi
    print ma
    # data = parse.build_ot_list(mi, ma)
    # parse.update_redis(data)
<|endoftext|>


<commit_before>import subprocess
import platform
import os
import sys

# Set these to None for compile/link debugging or subprocess.PIPE to silence
# compiler warnings and errors.
STDOUT = subprocess.PIPE
STDERR = subprocess.PIPE
# STDOUT = None
# STDERR = None

# This is the max length that I want a printed line to be.
MAX_LINE_LENGTH = 78

PY_MAJOR_VERSION = sys.version_info[0]

def line_wrap_paragraph(s):
    # Format s with terminal-friendly line wraps.
    done = False
    beginning = 0
    end = MAX_LINE_LENGTH - 1
    lines = [ ]
    while not done:
        if end >= len(s):
            done = True
            lines.append(s[beginning:])
        else:
            last_space = s[beginning:end].rfind(' ')

            lines.append(s[beginning:beginning + last_space])
            beginning += (last_space + 1)
            end = beginning + MAX_LINE_LENGTH - 1

    return lines

def print_bad_news(value_name, default):
    s = "Setup can't determine %s on your system, so it will default to %s which may not be correct." \
            % (value_name, default)
    plea = "Please report this message and your operating system info to the package maintainer listed in the README file."

    lines = line_wrap_paragraph(s) + [''] + line_wrap_paragraph(plea)

    border = '*' * MAX_LINE_LENGTH

    s = border + "\n* " + ('\n* '.join(lines)) + '\n' + border

    print (s)

def does_build_succeed(filename, linker_options = ""):
    # Utility function that returns True if the file compiles and links
    # successfully, False otherwise.
    # Two things to note here --
    #   - If there's a linker option like -lrt, it needs to come *after*
    #     the specification of the C file or linking will fail on Ubuntu 11.10
    #     (maybe because of the gcc version?)
    #   - Some versions of Linux place the sem_xxx() functions in libpthread.
    #     Rather than testing whether or not it's needed, I just specify it
    #     everywhere since it's harmless to specify it when it's not needed.
    cmd = "cc -Wall -o ./prober/foo ./prober/%s %s -lpthread" % (filename, linker_options)

    p = subprocess.Popen(cmd, shell=True, stdout=STDOUT, stderr=STDERR)

    # p.wait() returns the process' return code, so 0 implies that
    # the compile & link succeeded.
    return not bool(p.wait())

def compile_and_run(filename, linker_options = ""):
    # Utility function that returns the stdout output from running the
    # compiled source file; None if the compile fails.
    cmd = "cc -Wall -o ./prober/foo %s ./prober/%s" % (linker_options, filename)

    p = subprocess.Popen(cmd, shell=True, stdout=STDOUT, stderr=STDERR)

    if p.wait():
        # uh-oh, compile failed
        return None
    else:
        s = subprocess.Popen(["./prober/foo"],
                             stdout=subprocess.PIPE).communicate()[0]
        return s.strip().decode()

def get_sysctl_value(name):
    """Given a sysctl name (e.g. 'kern.mqueue.maxmsg'), returns sysctl's value
    for that variable or None if the sysctl call fails (unknown name, not
    a BSD-ish system, etc.)

    Only makes sense on systems that implement sysctl (BSD derivatives).
    """
    s = None
    try:
        s = subprocess.Popen(["sysctl", "-n", name],
                              stdout=subprocess.PIPE).communicate()[0]
        s = s.strip().decode()
    except:
        pass

    return s

def sniff_realtime_lib():
    rc = None
    filename = "sniff_realtime_lib.c"

    if does_build_succeed(filename):
        # Realtime libs not needed
        rc = False
    else:
        # cc failed when not linked to realtime libs; let's try again
        # with the realtime libs involved and see if things go better.
        if does_build_succeed(filename, "-lrt"):
            # Realtime libs are needed
            rc = True

    if rc == None:
        # Unable to determine whether or not I needed the realtime libs.
        # That's bad! Print a warning, set the return code to False
        # and hope for the best.
        rc = False
        print_bad_news("if it needs to link to the realtime libraries", "'no'")

    return rc

def sniff_sem_getvalue(linker_options):
    return does_build_succeed("sniff_sem_getvalue.c", linker_options)

def sniff_sem_timedwait(linker_options):
    return does_build_succeed("sniff_sem_timedwait.c", linker_options)

def sniff_sem_value_max():
    # default is to return None which means that it is #defined in a standard
    # header file and doesn't need to be added to my custom header file.
    sem_value_max = None

    if not does_build_succeed("sniff_sem_value_max.c"):
        # OpenSolaris 2008.05 doesn't #define SEM_VALUE_MAX. (This may
        # be true elsewhere too.) Ask sysconf() instead if it exists.
        # Note that sys.sysconf_names doesn't exist under Cygwin.
        if hasattr(os, "sysconf_names") and \
           ("SC_SEM_VALUE_MAX" in os.sysconf_names):
            sem_value_max = os.sysconf("SC_SEM_VALUE_MAX")
        else:
            # This value of last resort should be #defined everywhere. What
            # could possibly go wrong?
            sem_value_max = "_POSIX_SEM_VALUE_MAX"

    return sem_value_max

def sniff_page_size():
    DEFAULT_PAGE_SIZE = 4096

    # Linker options don't matter here because I'm not calling any
    # functions, just getting the value of a #define.
    page_size = compile_and_run("sniff_page_size.c")

    if page_size is None:
        page_size = DEFAULT_PAGE_SIZE
        print_bad_news("the value of PAGE_SIZE", page_size)

    return page_size

def sniff_mq_existence(linker_options):
    return does_build_succeed("sniff_mq_existence.c", linker_options)

def sniff_mq_prio_max():
    # MQ_PRIO_MAX is #defined in limits.h on all of the systems that I
    # checked that support message queues at all. (I checked 2 Linux boxes,
    # OpenSolaris and FreeBSD 8.0.)

    # 32 = minimum allowable max priority per POSIX; systems are permitted
    # to define a larger value.
    # ref: http://www.opengroup.org/onlinepubs/009695399/basedefs/limits.h.html
    DEFAULT_PRIORITY_MAX = 32

    max_priority = compile_and_run("sniff_mq_prio_max.c")

    if max_priority:
        try:
            max_priority = int(max_priority)
        except ValueError:
            max_priority = None

    if max_priority is None:
        # Looking for a #define didn't work; ask sysconf() instead.
        # Note that sys.sysconf_names doesn't exist under Cygwin.
        if hasattr(os, "sysconf_names") and \
           ("SC_MQ_PRIO_MAX" in os.sysconf_names):
            max_priority = os.sysconf("SC_MQ_PRIO_MAX")
        else:
            max_priority = DEFAULT_PRIORITY_MAX
            print_bad_news("the value of PRIORITY_MAX", max_priority)

    # Adjust for the fact that these are 0-based values; i.e. permitted
    # priorities range from 0 - (MQ_PRIO_MAX - 1). So why not just make
    # the #define one smaller? Because this one goes up to eleven...
    max_priority -= 1

    # priority is an unsigned int
    return str(max_priority).strip() + "U"

def sniff_mq_max_messages():
    # This value is not defined by POSIX.

    # On most systems I've tested, msg Qs are implemented via mmap-ed files
    # or a similar interface, so the only theoretical limits are imposed by the
    # file system. In practice, Linux and *BSD impose some fairly tight
    # limits.

    # On Linux it's available in a /proc file and often defaults to the wimpy
    # value of 10.

    # On FreeBSD (and other BSDs, I assume), it's available via sysctl as
    # kern.mqueue.maxmsg. On my FreeBSD 9.1 test system, it defaults to 100.

    # mqueue.h defines mq_attr.mq_maxmsg as a C long, so that's
    # a practical limit for this value.

    # ref: http://linux.die.net/man/7/mq_overview
    # ref: http://www.freebsd.org/cgi/man.cgi?query=mqueuefs&sektion=5&manpath=FreeBSD+7.0-RELEASE
    # http://fxr.watson.org/fxr/source/kern/uipc_mqueue.c?v=FREEBSD91#L195
    # ref: http://groups.google.com/group/comp.unix.solaris/browse_thread/thread/aa223fc7c91f8c38
    # ref: http://cygwin.com/cgi-bin/cvsweb.cgi/src/winsup/cygwin/posix_ipc.cc?cvsroot=src
    # ref: http://cygwin.com/cgi-bin/cvsweb.cgi/src/winsup/cygwin/include/mqueue.h?cvsroot=src
    mq_max_messages = None

    # Try to get the value from where Linux stores it.
    try:
        mq_max_messages = int(open("/proc/sys/fs/mqueue/msg_max").read())
    except:
        # Oh well.
        pass

    if not mq_max_messages:
        # Maybe we're on BSD.
        mq_max_messages = get_sysctl_value('kern.mqueue.maxmsg')
        if mq_max_messages:
            mq_max_messages = int(mq_max_messages)

    if not mq_max_messages:
        # Use an arbitrary default. The max possible is > 2 billion, but the
        # values used by Linux and FreeBSD suggest that a smaller default is
        # wiser.
        mq_max_messages = 1024

    return mq_max_messages

def sniff_mq_max_message_size_default():
    # The max message size is not defined by POSIX.

    # On most systems I've tested, msg Qs are implemented via mmap-ed files
    # or a similar interface, so the only theoretical limits are imposed by
    # the file system. In practice, Linux and *BSD impose some tighter limits.

    # On Linux, max message size is available in a /proc file and often
    # defaults to the value of 8192.

    # On FreeBSD (and other BSDs, I assume), it's available via sysctl as
    # kern.mqueue.maxmsgsize. On my FreeBSD 9.1 test system, it defaults to
    # 16384.

    # mqueue.h defines mq_attr.mq_msgsize as a C long, so that's
    # a practical limit for this value.

    # Further complicating things is the fact that the module has to allocate
    # a buffer the size of the queue's max message every time receive() is
    # called, so it would be a bad idea to set this default to the max.
    # I set it to 8192 -- not too small, not too big. I only set it smaller
    # if I'm on a system that tells me I must do so.
    DEFAULT = 8192
    mq_max_message_size_default = 0

    # Try to get the value from where Linux stores it.
    try:
        mq_max_message_size_default = \
                            int(open("/proc/sys/fs/mqueue/msgsize_max").read())
    except:
        # oh well
        pass

    if not mq_max_message_size_default:
        # Maybe we're on BSD.
        mq_max_message_size_default = get_sysctl_value('kern.mqueue.maxmsgsize')
        if mq_max_message_size_default:
            mq_max_message_size_default = int(mq_max_message_size_default)

    if not mq_max_message_size_default:
        mq_max_message_size_default = DEFAULT

    return mq_max_message_size_default

def probe():
    linker_options = ""
    d = { }

    f = open("VERSION")
    d["POSIX_IPC_VERSION"] = '"%s"' % f.read().strip()
    f.close()

    # Sniffing of the realtime libs has to go early in the list so as
    # to provide correct linker options to the rest of the tests.
    if "Darwin" in platform.uname():
        # I skip the test under Darwin/OS X for two reasons. First, I know
        # it isn't needed there. Second, I can't even compile the test for
        # the realtime lib because it references mq_unlink() which OS X
        # doesn't support. Unfortunately sniff_realtime_lib.c *must*
        # reference mq_unlink() or some other mq_xxx() function because
        # it is only the message queues that need the realtime libs under
        # FreeBSD.
        realtime_lib_is_needed = False
    else:
        # Some platforms (e.g. Linux & OpenSuse) require linking to librt
        realtime_lib_is_needed = sniff_realtime_lib()

    if realtime_lib_is_needed:
        d["REALTIME_LIB_IS_NEEDED"] = ""
        linker_options = " -lrt "

    d["PAGE_SIZE"] = sniff_page_size()

    if sniff_sem_getvalue(linker_options):
        d["SEM_GETVALUE_EXISTS"] = ""

    if ("SEM_GETVALUE_EXISTS" in d) and ("Darwin" in platform.uname()):
        # sem_getvalue() isn't available on OS X. The function exists but
        # always returns ENOSYS ("Function not implemented").
        del d["SEM_GETVALUE_EXISTS"]

    if sniff_sem_timedwait(linker_options):
        d["SEM_TIMEDWAIT_EXISTS"] = ""

    d["SEM_VALUE_MAX"] = sniff_sem_value_max()
    # A return of None means that I don't need to #define this myself.
    if d["SEM_VALUE_MAX"] is None:
        del d["SEM_VALUE_MAX"]

    if sniff_mq_existence(linker_options):
        d["MESSAGE_QUEUE_SUPPORT_EXISTS"] = ""

    d["QUEUE_MESSAGES_MAX_DEFAULT"] = sniff_mq_max_messages()
    d["QUEUE_MESSAGE_SIZE_MAX_DEFAULT"] = sniff_mq_max_message_size_default()
    d["QUEUE_PRIORITY_MAX"] = sniff_mq_prio_max()

    if PY_MAJOR_VERSION == 2:
        # I only need this for Python 2.x
        d["PY_INT_MAX"] = sys.maxint

    msg = """/*
This header file was generated when you ran setup. Once created, the setup
process won't overwrite it, so you can adjust the values by hand and
recompile if you need to.

On your platform, this file may contain only this comment -- that's OK!

To recreate this file, just delete it and re-run setup.py.
*/

"""
    filename = "probe_results.h"
    if not os.path.exists(filename):
        lines = ["#define %s\t\t%s" % (key, d[key]) for key in d if key != "PAGE_SIZE"]

        # PAGE_SIZE gets some special treatment. It's defined in header files
        # on some systems in which case I might get a redefinition error in
        # my header file, so I wrap it in #ifndef/#endif.

        lines.append("#ifndef PAGE_SIZE")
        lines.append("#define PAGE_SIZE\t\t%s" % d["PAGE_SIZE"])
        lines.append("#endif")

        # A trailing '\n' keeps compilers happy...
        open(filename, "w").write(msg + '\n'.join(lines) + '\n')

    return d

if __name__ == "__main__":
    print (probe())

<commit_msg>fixed a harmless but ugly bug under OSX where MQ_PRIO_MAX was written to probe_results.h as -2U. This didn't matter since OSX doesn't support message queues so the value of MQ_PRIO_MAX is irrelevant, but the mistake looks sloppy.<commit_after>import subprocess
import platform
import os
import sys

# Set these to None for compile/link debugging or subprocess.PIPE to silence
# compiler warnings and errors.
STDOUT = subprocess.PIPE
STDERR = subprocess.PIPE
# STDOUT = None
# STDERR = None

# This is the max length that I want a printed line to be.
MAX_LINE_LENGTH = 78

PY_MAJOR_VERSION = sys.version_info[0]

def line_wrap_paragraph(s):
    # Format s with terminal-friendly line wraps.
    done = False
    beginning = 0
    end = MAX_LINE_LENGTH - 1
    lines = [ ]
    while not done:
        if end >= len(s):
            done = True
            lines.append(s[beginning:])
        else:
            last_space = s[beginning:end].rfind(' ')

            lines.append(s[beginning:beginning + last_space])
            beginning += (last_space + 1)
            end = beginning + MAX_LINE_LENGTH - 1

    return lines

def print_bad_news(value_name, default):
    s = "Setup can't determine %s on your system, so it will default to %s which may not be correct." \
            % (value_name, default)
    plea = "Please report this message and your operating system info to the package maintainer listed in the README file."

    lines = line_wrap_paragraph(s) + [''] + line_wrap_paragraph(plea)

    border = '*' * MAX_LINE_LENGTH

    s = border + "\n* " + ('\n* '.join(lines)) + '\n' + border

    print (s)

def does_build_succeed(filename, linker_options = ""):
    # Utility function that returns True if the file compiles and links
    # successfully, False otherwise.
    # Two things to note here --
    #   - If there's a linker option like -lrt, it needs to come *after*
    #     the specification of the C file or linking will fail on Ubuntu 11.10
    #     (maybe because of the gcc version?)
    #   - Some versions of Linux place the sem_xxx() functions in libpthread.
    #     Rather than testing whether or not it's needed, I just specify it
    #     everywhere since it's harmless to specify it when it's not needed.
    cmd = "cc -Wall -o ./prober/foo ./prober/%s %s -lpthread" % (filename, linker_options)

    p = subprocess.Popen(cmd, shell=True, stdout=STDOUT, stderr=STDERR)

    # p.wait() returns the process' return code, so 0 implies that
    # the compile & link succeeded.
    return not bool(p.wait())

def compile_and_run(filename, linker_options = ""):
    # Utility function that returns the stdout output from running the
    # compiled source file; None if the compile fails.
    cmd = "cc -Wall -o ./prober/foo %s ./prober/%s" % (linker_options, filename)

    p = subprocess.Popen(cmd, shell=True, stdout=STDOUT, stderr=STDERR)

    if p.wait():
        # uh-oh, compile failed
        return None
    else:
        s = subprocess.Popen(["./prober/foo"],
                             stdout=subprocess.PIPE).communicate()[0]
        return s.strip().decode()

def get_sysctl_value(name):
    """Given a sysctl name (e.g. 'kern.mqueue.maxmsg'), returns sysctl's value
    for that variable or None if the sysctl call fails (unknown name, not
    a BSD-ish system, etc.)

    Only makes sense on systems that implement sysctl (BSD derivatives).
    """
    s = None
    try:
        s = subprocess.Popen(["sysctl", "-n", name],
                              stdout=subprocess.PIPE).communicate()[0]
        s = s.strip().decode()
    except:
        pass

    return s

def sniff_realtime_lib():
    rc = None
    filename = "sniff_realtime_lib.c"

    if does_build_succeed(filename):
        # Realtime libs not needed
        rc = False
    else:
        # cc failed when not linked to realtime libs; let's try again
        # with the realtime libs involved and see if things go better.
        if does_build_succeed(filename, "-lrt"):
            # Realtime libs are needed
            rc = True

    if rc == None:
        # Unable to determine whether or not I needed the realtime libs.
        # That's bad! Print a warning, set the return code to False
        # and hope for the best.
        rc = False
        print_bad_news("if it needs to link to the realtime libraries", "'no'")

    return rc

def sniff_sem_getvalue(linker_options):
    return does_build_succeed("sniff_sem_getvalue.c", linker_options)

def sniff_sem_timedwait(linker_options):
    return does_build_succeed("sniff_sem_timedwait.c", linker_options)

def sniff_sem_value_max():
    # default is to return None which means that it is #defined in a standard
    # header file and doesn't need to be added to my custom header file.
    sem_value_max = None

    if not does_build_succeed("sniff_sem_value_max.c"):
        # OpenSolaris 2008.05 doesn't #define SEM_VALUE_MAX. (This may
        # be true elsewhere too.) Ask sysconf() instead if it exists.
        # Note that sys.sysconf_names doesn't exist under Cygwin.
        if hasattr(os, "sysconf_names") and \
           ("SC_SEM_VALUE_MAX" in os.sysconf_names):
            sem_value_max = os.sysconf("SC_SEM_VALUE_MAX")
        else:
            # This value of last resort should be #defined everywhere. What
            # could possibly go wrong?
            sem_value_max = "_POSIX_SEM_VALUE_MAX"

    return sem_value_max

def sniff_page_size():
    DEFAULT_PAGE_SIZE = 4096

    # Linker options don't matter here because I'm not calling any
    # functions, just getting the value of a #define.
    page_size = compile_and_run("sniff_page_size.c")

    if page_size is None:
        page_size = DEFAULT_PAGE_SIZE
        print_bad_news("the value of PAGE_SIZE", page_size)

    return page_size

def sniff_mq_existence(linker_options):
    return does_build_succeed("sniff_mq_existence.c", linker_options)

def sniff_mq_prio_max():
    # MQ_PRIO_MAX is #defined in limits.h on all of the systems that I
    # checked that support message queues at all. (I checked 2 Linux boxes,
    # OpenSolaris and FreeBSD 8.0.)

    # 32 = minimum allowable max priority per POSIX; systems are permitted
    # to define a larger value.
    # ref: http://www.opengroup.org/onlinepubs/009695399/basedefs/limits.h.html
    DEFAULT_PRIORITY_MAX = 32

    max_priority = None
    # OS X up to and including 10.8 doesn't support POSIX messages queues and
    # doesn't define MQ_PRIO_MAX. Maybe this aggravation will cease in 10.9?
    if does_build_succeed("sniff_mq_prio_max.c"):
        max_priority = compile_and_run("sniff_mq_prio_max.c")

    if max_priority:
        try:
            max_priority = int(max_priority)
        except ValueError:
            max_priority = None

    if max_priority is None:
        # Looking for a #define didn't work; ask sysconf() instead.
        # Note that sys.sysconf_names doesn't exist under Cygwin.
        if hasattr(os, "sysconf_names") and \
           ("SC_MQ_PRIO_MAX" in os.sysconf_names):
            max_priority = os.sysconf("SC_MQ_PRIO_MAX")
        else:
            max_priority = DEFAULT_PRIORITY_MAX
            print_bad_news("the value of PRIORITY_MAX", max_priority)

    # Under OS X, os.sysconf("SC_MQ_PRIO_MAX") returns -1.
    if max_priority < 0:
        max_priority = DEFAULT_PRIORITY_MAX

    # Adjust for the fact that these are 0-based values; i.e. permitted
    # priorities range from 0 - (MQ_PRIO_MAX - 1). So why not just make
    # the #define one smaller? Because this one goes up to eleven...
    max_priority -= 1

    # priority is an unsigned int
    return str(max_priority).strip() + "U"

def sniff_mq_max_messages():
    # This value is not defined by POSIX.

    # On most systems I've tested, msg Qs are implemented via mmap-ed files
    # or a similar interface, so the only theoretical limits are imposed by the
    # file system. In practice, Linux and *BSD impose some fairly tight
    # limits.

    # On Linux it's available in a /proc file and often defaults to the wimpy
    # value of 10.

    # On FreeBSD (and other BSDs, I assume), it's available via sysctl as
    # kern.mqueue.maxmsg. On my FreeBSD 9.1 test system, it defaults to 100.

    # mqueue.h defines mq_attr.mq_maxmsg as a C long, so that's
    # a practical limit for this value.

    # ref: http://linux.die.net/man/7/mq_overview
    # ref: http://www.freebsd.org/cgi/man.cgi?query=mqueuefs&sektion=5&manpath=FreeBSD+7.0-RELEASE
    # http://fxr.watson.org/fxr/source/kern/uipc_mqueue.c?v=FREEBSD91#L195
    # ref: http://groups.google.com/group/comp.unix.solaris/browse_thread/thread/aa223fc7c91f8c38
    # ref: http://cygwin.com/cgi-bin/cvsweb.cgi/src/winsup/cygwin/posix_ipc.cc?cvsroot=src
    # ref: http://cygwin.com/cgi-bin/cvsweb.cgi/src/winsup/cygwin/include/mqueue.h?cvsroot=src
    mq_max_messages = None

    # Try to get the value from where Linux stores it.
    try:
        mq_max_messages = int(open("/proc/sys/fs/mqueue/msg_max").read())
    except:
        # Oh well.
        pass

    if not mq_max_messages:
        # Maybe we're on BSD.
        mq_max_messages = get_sysctl_value('kern.mqueue.maxmsg')
        if mq_max_messages:
            mq_max_messages = int(mq_max_messages)

    if not mq_max_messages:
        # Use an arbitrary default. The max possible is > 2 billion, but the
        # values used by Linux and FreeBSD suggest that a smaller default is
        # wiser.
        mq_max_messages = 1024

    return mq_max_messages

def sniff_mq_max_message_size_default():
    # The max message size is not defined by POSIX.

    # On most systems I've tested, msg Qs are implemented via mmap-ed files
    # or a similar interface, so the only theoretical limits are imposed by
    # the file system. In practice, Linux and *BSD impose some tighter limits.

    # On Linux, max message size is available in a /proc file and often
    # defaults to the value of 8192.

    # On FreeBSD (and other BSDs, I assume), it's available via sysctl as
    # kern.mqueue.maxmsgsize. On my FreeBSD 9.1 test system, it defaults to
    # 16384.

    # mqueue.h defines mq_attr.mq_msgsize as a C long, so that's
    # a practical limit for this value.

    # Further complicating things is the fact that the module has to allocate
    # a buffer the size of the queue's max message every time receive() is
    # called, so it would be a bad idea to set this default to the max.
    # I set it to 8192 -- not too small, not too big. I only set it smaller
    # if I'm on a system that tells me I must do so.
    DEFAULT = 8192
    mq_max_message_size_default = 0

    # Try to get the value from where Linux stores it.
    try:
        mq_max_message_size_default = \
                            int(open("/proc/sys/fs/mqueue/msgsize_max").read())
    except:
        # oh well
        pass

    if not mq_max_message_size_default:
        # Maybe we're on BSD.
        mq_max_message_size_default = get_sysctl_value('kern.mqueue.maxmsgsize')
        if mq_max_message_size_default:
            mq_max_message_size_default = int(mq_max_message_size_default)

    if not mq_max_message_size_default:
        mq_max_message_size_default = DEFAULT

    return mq_max_message_size_default

def probe():
    linker_options = ""
    d = { }

    f = open("VERSION")
    d["POSIX_IPC_VERSION"] = '"%s"' % f.read().strip()
    f.close()

    # Sniffing of the realtime libs has to go early in the list so as
    # to provide correct linker options to the rest of the tests.
    if "Darwin" in platform.uname():
        # I skip the test under Darwin/OS X for two reasons. First, I know
        # it isn't needed there. Second, I can't even compile the test for
        # the realtime lib because it references mq_unlink() which OS X
        # doesn't support. Unfortunately sniff_realtime_lib.c *must*
        # reference mq_unlink() or some other mq_xxx() function because
        # it is only the message queues that need the realtime libs under
        # FreeBSD.
        realtime_lib_is_needed = False
    else:
        # Some platforms (e.g. Linux & OpenSuse) require linking to librt
        realtime_lib_is_needed = sniff_realtime_lib()

    if realtime_lib_is_needed:
        d["REALTIME_LIB_IS_NEEDED"] = ""
        linker_options = " -lrt "

    d["PAGE_SIZE"] = sniff_page_size()

    if sniff_sem_getvalue(linker_options):
        d["SEM_GETVALUE_EXISTS"] = ""

    if ("SEM_GETVALUE_EXISTS" in d) and ("Darwin" in platform.uname()):
        # sem_getvalue() isn't available on OS X. The function exists but
        # always returns ENOSYS ("Function not implemented").
        del d["SEM_GETVALUE_EXISTS"]

    if sniff_sem_timedwait(linker_options):
        d["SEM_TIMEDWAIT_EXISTS"] = ""

    d["SEM_VALUE_MAX"] = sniff_sem_value_max()
    # A return of None means that I don't need to #define this myself.
    if d["SEM_VALUE_MAX"] is None:
        del d["SEM_VALUE_MAX"]

    if sniff_mq_existence(linker_options):
        d["MESSAGE_QUEUE_SUPPORT_EXISTS"] = ""

    d["QUEUE_MESSAGES_MAX_DEFAULT"] = sniff_mq_max_messages()
    d["QUEUE_MESSAGE_SIZE_MAX_DEFAULT"] = sniff_mq_max_message_size_default()
    d["QUEUE_PRIORITY_MAX"] = sniff_mq_prio_max()

    if PY_MAJOR_VERSION == 2:
        # I only need this for Python 2.x
        d["PY_INT_MAX"] = sys.maxint

    msg = """/*
This header file was generated when you ran setup. Once created, the setup
process won't overwrite it, so you can adjust the values by hand and
recompile if you need to.

On your platform, this file may contain only this comment -- that's OK!

To recreate this file, just delete it and re-run setup.py.
*/

"""
    filename = "probe_results.h"
    if not os.path.exists(filename):
        lines = ["#define %s\t\t%s" % (key, d[key]) for key in d if key != "PAGE_SIZE"]

        # PAGE_SIZE gets some special treatment. It's defined in header files
        # on some systems in which case I might get a redefinition error in
        # my header file, so I wrap it in #ifndef/#endif.

        lines.append("#ifndef PAGE_SIZE")
        lines.append("#define PAGE_SIZE\t\t%s" % d["PAGE_SIZE"])
        lines.append("#endif")

        # A trailing '\n' keeps compilers happy...
        open(filename, "w").write(msg + '\n'.join(lines) + '\n')

    return d

if __name__ == "__main__":
    print (probe())

<|endoftext|>