<commit_before>from collections import OrderedDict

import logging
log = logging.getLogger('payment_terminal')

from .fields import (
    BBSField, DelimitedField,
    ConstantField, EnumField,
    IntegerField, PriceField,
    TextField, FormattedTextField,
    DateTimeField,
)

class BBSMessageMeta(type):
    def __new__(mcls, cls, bases, d):
        fields = OrderedDict()

        # inherit fields from first base class with `_fields` attribute
        for base in bases:
            if hasattr(base, '_fields'):
                fields.update(base._fields)
                break

        # read fields from class body
        for name, field in d.items():
            if isinstance(field, BBSField):
                fields[name] = field

        d['_fields'] = fields
        return type.__new__(mcls, cls, bases, d)

    @classmethod
    def __prepare__(mcls, cls, bases):
        # the dictionary to use to store class attributes
        # need to return OrderedDict rather than default dict as attribute
        # order affects parsing
        return OrderedDict()

class BBSMessageBase(object):
    def __init__(self, **kwargs):
        for name, field in self._fields.items():
            if name in kwargs:
                value = kwargs[name]
            elif hasattr(field, 'default'):
                value = field.default
            else:
                raise TypeError('missing required argument: %r' % name)
            setattr(self, name, value)

    def pack(self):
        return b''.join(
            field.pack(getattr(self, name))
            for name, field in self._fields.items()
        )

    @classmethod
    def unpack_fields(cls, data):
        fields = OrderedDict()

        offset = 0
        for name, field in cls._fields.items():
            fields[name], size = field.unpack(data[offset:])
            offset += size

        return fields

    @classmethod
    def unpack(cls, data):
        return cls(**cls.unpack_fields(data))

    def __repr__(self):
        parts = [self.__class__.__name__]
        parts += (
            "%s=%r" % (name, getattr(self, name))
            for name in self._fields
        )

        return "<%s>" % " ".join(parts)

class BBSMessage(BBSMessageBase, metaclass=BBSMessageMeta):
    is_response = False

class DisplayTextMessage(BBSMessage):
    type = ConstantField(b'\x41')

    prompt_customer = EnumField({
        b'\x31': True,
        b'\x30': False,
    }, default=True)

    expects_input = EnumField({
        b'\x31': True,
        b'\x30': False,
    }, default=False)

    mode = ConstantField(b'\x30')

    text = TextField()

    def __init__(self, text, **kwargs):
        # allow `text` to be passed in as a positional argument
        super(DisplayTextMessage, self).__init__(text=text, **kwargs)

class PrintTextMessage(BBSMessage):
    type = ConstantField(b'\x42')

    sub_type = EnumField({b'\x20': 'formatted'})

    media = EnumField({
        b'\x20': 'print_on_receipt',
        b'\x21': 'print_on_journal',
        b'\x22': 'print_on_both',
    }, default='print_on_both')

    mode = EnumField({b'\x2a': 'normal_text'})

    commands = FormattedTextField()

class ResetTimerMessage(BBSMessage):
    type = ConstantField(b'\x43')

    seconds = IntegerField(3)

    def __init__(self, seconds, **kwargs):
        # allow `seconds` to be passed in as a positional argument
        super(ResetTimerMessage, self).__init__(seconds=seconds, **kwargs)

class LocalModeMessage(BBSMessage):
    type = ConstantField(b'\x44')

    result = EnumField({
        # indicates transaction OK
        b'\x20': 'success',
        # indicates transaction/operation rejected
        b'\x21': 'failure',
    })

    acc = EnumField({
        # indicates standard update of accumulator
        b'\x20': 'standard',
        # indicates transaction is finalised as Offline transaction
        b'\x22': 'offline',
        # indicates no update of accumulator
        b'\x30': 'none',
    })

    # 2 digit issuer number is indicating the card issuer. Used if the
    # transaction was accepted. As long as the data is available, the data
    # shall be sent regardless if transaction is rejected or accepted.
    issuer_id = IntegerField(2)

    # Variable field lenght, Max. 19 digit if present. The Primary Account
    # Number from the card holder. The PAN shall not be sent if some parts of
    # the card number is replaced with "*" in the printout. The PAN field is of
    # restricted use, due to security regulations
    pan = DelimitedField(TextField(19), optional=True, delimiter=b';')

    # 14 byte numeric data. Timestamp in format YYYYMMDDHHMMSS. The timestamp
    # shall be the same data as received from the Host to the terminal in the
    # response message
    timestamp = DelimitedField(DateTimeField(), delimiter=b';')

    # Cardholder Verification Method
    ver_method = DelimitedField(EnumField({
        # transaction is PIN based, also to be used if reversal transaction
        b'\x30': 'pin_based',
        # transaction is signature based
        b'\x31': 'signature_based',
        # no CVM. Only amount is verified by cardholder
        b'\x32': 'not_verified',
        # transaction is a Loyalty Transaction. Used for data capture
        # transactions. No accounts are debited or credited
        b'\x32': 'loyalty_transaction',
    }), delimiter=b';')

    # 3 byte, numeric data. The current session number received from the HOST.
    # The session number is uncertain in case that the transaction is an
    # Offline transaction.  This number is changed on reconciliation.
    session_num = DelimitedField(IntegerField(3), delimiter=b';')

    # 12 byte, Alphanumeric data (H20-H7F). The STAN_AUTH and the TIMESTAMP
    # will identify the transaction.
    #   * On-line: The STAN (System Trace Audit Number) is the 6 first bytes,
    #     and the Authorisation Code is the 6 last bytes.
    #   * Off-line: STAN=9xxxx9 where x is the message number for the actual
    #     transaction AUTH = <H20H20H20H20H20H20>
    stan_auth = DelimitedField(TextField(12), delimiter=b';')

    # 4 bytes numeric data (H30 .. H39). This is the customer number if the
    # transaction was Pre-Auth transaction. Must be used as reference in
    # Transfer Amount - Adjustment transaction.
    seq_no = DelimitedField(IntegerField(4), delimiter=b';')

    # 11 bytes numeric data (H30 .. H39). Normally not used. Only used in
    # Restaurant or Hotel environmet where TIP is added to the purchase amount
    # on the ITU. Used in the Purchase or Adjustment transaction.
    tip = DelimitedField(PriceField(), optional=True, delimiter=b';')

class KeyboardInputRequestMessage(BBSMessage):
    type = ConstantField(b'\x46')

    # Indicates if the entered chars should be echoed on the ECR display or not
    echo = EnumField({
        b'\x20': True,
        b'\x21': False,
    })

    # Minimum and maximum number of chars to enter (as a decimal ascii string)
    min_chars = TextField(2)
    max_chars = TextField(2)

class KeyboardInputMessage(BBSMessage):
    type = ConstantField(b'\x55')
    is_response = True

    text = TextField()

    # XXX how are you supposed to parse this
    delimiter = EnumField({
        b'0': 'enter',
        b'9': 'escape',
    })

    def __init__(self, text, **kwargs):
        # allow `text` to be passed in as a positional argument
        super(KeyboardInputMessage, self).__init__(text=text, **kwargs)

    @classmethod
    def unpack_fields(cls, data):
        # currently special cased because of fixed size `delimiter` field
        # following variable length `text` field.
        # TODO yuck yuck yuck
        fields = OrderedDict()

        fields['header'], size = cls.type.read(data)

        text_data = data[size:-cls.delimiter.size]
        fields['text'], size = cls.text.unpack(text_data)

        delimiter_data = data[-cls.delimiter.size:]
        fields['delimiter'], size = cls.delimiter.unpack(delimiter_data)

class SendDataMessageBase(BBSMessage):
    type = ConstantField(b'\x46')

    code = TextField(2)
    is_last_block = EnumField({
        b'\x32': True,
        b'\x31': False,
    })

    seq = TextField(4)  # ignored
    length = TextField(3)  # ignored

class SendReportsDataHeaderMessage(SendDataMessageBase):
    code = ConstantField(b'\x30\x31')

    site_number = TextField(6)
    session_number = TextField(3)
    timestamp = DateTimeField()

class SendReconciliationDataAmountsMessage(SendDataMessageBase):
    code = ConstantField(b'\x30\x32')

    issuer_id = TextField(2)
    num_transactions = IntegerField(4)

    # TODO

class SendDataMessage(SendDataMessageBase):
    code = EnumField({
        subfunction.code.value: subfunction
        for subfunction in [
            SendReportsDataHeaderMessage,
            SendReconciliationDataAmountsMessage,
            # TODO
        ]
    })

    @classmethod
    def unpack(cls, data):
        self = super(SendDataMessage, cls).unpack(data)

        return self.code.unpack(data)

class TransferAmountMessage(BBSMessage):
    type = ConstantField(b'\x51')

    timestamp = DateTimeField()  # not used
    id_no = TextField(6)  # not used
    # Normally set to "0000". If set in Pre-Auth, the number is a reference to
    # a previous Preauth. If set in Adjustment transaction, the field shall be
    # set to the corresponding number received in the Local Mode from the
    # Pre-Authorisation.
    seq_no = TextField(4)  # TODO
    # Operator identification. A fixed field with 4 characters. If not
    # implemented by the ECR vendor, the field should be filled with zeroes
    # (H30's).
    operator_id = TextField(4)
    # Not used, but tested by the ITU because of error prevention)
    mode = EnumField({
        b'\x30': None,
    })
    transfer_type = EnumField({
        b'\x30': 'eft_authorisation',
        b'\x31': 'return_of_goods',
        b'\x32': 'reversal',
        b'\x33': 'purchase_with_cashback',
        b'\x34': 'pre_authorisation',
        b'\x35': 'adjustment',
        b'\x36': 'balance_inquiry',
        b'\x37': 'complete_receipt',
        b'\x38': 'deposit',
        b'\x39': 'cash_withdrawal',
        b'\x3a': 'load_epurse_card',
        b'\x3b': 'merchandise_purchase',
        b'\x3c': 'merchandise_reversal',
        b'\x3d': 'merchandise_correction',
    })
    amount = PriceField(11)
    # Not used, but tested by the ITU because of error prevention)
    unused_type = EnumField({
        b'\x30': None,
    })
    # Only used if transfer_type == 'purchase_with_ashback' (H33), else it will
    # be filled with H20.
    cashback_amount = PriceField(11)
    is_top_up = EnumField({
        b'\x30': True,
        b'\x31': False,
    })
    art_amount = PriceField(11)

    data = DelimitedField(TextField(), delimiter=b';')

    # TODO ART#

class TransferCardDataMessage(BBSMessage):
    type = ConstantField(b'\x52')

    block = EnumField({b'\x30': None})
    track = EnumField({
        b'\x32': 'track_2',
        b'\x33': 'track_1',
        b'\x40': 'manual',
    })

    # TODO DATA and FS

class AdministrationMessage(BBSMessage):
    type = ConstantField(b'\x53')

    timestamp = DateTimeField()
    id_no = TextField(6)
    seq_no = TextField(4)
    opt = TextField(4)

    # TODO single character keyboard input
    adm_code = EnumField({
        b'\x30\x30': 'not_used',
        b'\x30\x39': 'not_used',
        # SEND from ECR should be mapped by ITU to perform RECONCILIATION
        # function.
        b'\x31\x30': 'send',
        # KLAR, validation key. Refer to the NOTE for details
        b'\x31\x31': 'ready',
        # AVBRYT, cancellation key. Refer to the NOTE for details.
        b'\x31\x32': 'cancel',
        # FEIL, correction key.
        b'\x31\x33': 'error',
        # ANNUL from ECR should be mapped by ITU to perform REVERSAL
        # transaction.
        b'\x31\x34': 'reverse',
        b'\x31\x35': 'balance_inquiry_transaction',
        b'\x31\x36': 'x_report',
        b'\x31\x37': 'z_report',
        b'\x31\x38': 'send_offline_transactions',
        b'\x31\x39': 'turnover_report',
        b'\x31\x3A': 'print_eot_transactions',
        b'\x31\x3B': 'not_used',
        b'\x31\x3C': 'not_used',
        b'\x31\x3D': 'not_used',
        b'\x31\x3E': 'not_used',
    })

    fs = ConstantField(b'\x1C')

class DeviceAttributeRequestMessage(BBSMessage):
    type = ConstantField(b'\x60')

class DeviceAttributeMessage(BBSMessage):
    type = ConstantField(b'\x61')

class StatusMessage(BBSMessage):
    type = ConstantField(b'\x62')
    is_response = True

class ResponseMessage(BBSMessage):
    type = ConstantField(b'\x5b')
    is_response = True

    code = EnumField({
        # OK. The Receiver has received and processed the data correctly
        b'\x30\x30': 'success',
        # Not OK. The receiver is not able to process the received data
        b'\x30\x33': 'failure',
        # Not OK. Shall be treated as if H3033 is received
        b'\x30\x34': 'failure',
        b'\x30\x35': 'failure',
        b'\x30\x36': 'failure',
        b'\x30\x37': 'failure',
        b'\x30\x38': 'failure',
        b'\x30\x39': 'failure',
        # ECR display busy, ITU may try again once
        b'\x31\x31': 'display_busy',
        # ECR printer busy, ITU may try again once
        b'\x31\x32': 'printer_busy',
        # ECR printer out of function.
        # If the ECR sends H3133, the ITU must interrupt the current
        # transaction, and wait for the next 'Bank-Mode' initiation from the
        # ECR
        b'\x31\x33': 'printer_broken'
    })

    endcode = ConstantField(b'\x5d')

_ITU_MESSAGE_TYPES = {
    DisplayTextMessage,
    PrintTextMessage,
    ResetTimerMessage,
    LocalModeMessage,
    KeyboardInputRequestMessage,
    SendDataMessage,
    DeviceAttributeRequestMessage,
    StatusMessage,
}

class ITUMessage(BBSMessage):
    type = EnumField({
        subtype.type.value: subtype
        for subtype in _ITU_MESSAGE_TYPES
    })

def unpack_itu_message(data):
    header = ITUMessage.unpack(data)
    return header.type.unpack(data)

_ECR_MESSAGE_TYPES = {
    KeyboardInputMessage,
    SendDataMessage,
    TransferAmountMessage,
    AdministrationMessage,
    DeviceAttributeMessage,
}

class ECRMessage(BBSMessage):
    type = EnumField({
        subtype.type.value: subtype
        for subtype in _ECR_MESSAGE_TYPES
    })

def unpack_ecr_message(data):
    header = ECRMessage.unpack(data)
    return header.type.unpack(data)
<commit_msg>change metaclass self argument from mcls to mcs to keep pylint happy<commit_after>from collections import OrderedDict

import logging
log = logging.getLogger('payment_terminal')

from .fields import (
    BBSField, DelimitedField,
    ConstantField, EnumField,
    IntegerField, PriceField,
    TextField, FormattedTextField,
    DateTimeField,
)

class BBSMessageMeta(type):
    def __new__(mcs, cls, bases, d):
        fields = OrderedDict()

        # inherit fields from first base class with `_fields` attribute
        for base in bases:
            if hasattr(base, '_fields'):
                fields.update(base._fields)
                break

        # read fields from class body
        for name, field in d.items():
            if isinstance(field, BBSField):
                fields[name] = field

        d['_fields'] = fields
        return type.__new__(mcs, cls, bases, d)

    @classmethod
    def __prepare__(mcs, cls, bases):
        # the dictionary to use to store class attributes
        # need to return OrderedDict rather than default dict as attribute
        # order affects parsing
        return OrderedDict()

class BBSMessageBase(object):
    def __init__(self, **kwargs):
        for name, field in self._fields.items():
            if name in kwargs:
                value = kwargs[name]
            elif hasattr(field, 'default'):
                value = field.default
            else:
                raise TypeError('missing required argument: %r' % name)
            setattr(self, name, value)

    def pack(self):
        return b''.join(
            field.pack(getattr(self, name))
            for name, field in self._fields.items()
        )

    @classmethod
    def unpack_fields(cls, data):
        fields = OrderedDict()

        offset = 0
        for name, field in cls._fields.items():
            fields[name], size = field.unpack(data[offset:])
            offset += size

        return fields

    @classmethod
    def unpack(cls, data):
        return cls(**cls.unpack_fields(data))

    def __repr__(self):
        parts = [self.__class__.__name__]
        parts += (
            "%s=%r" % (name, getattr(self, name))
            for name in self._fields
        )

        return "<%s>" % " ".join(parts)

class BBSMessage(BBSMessageBase, metaclass=BBSMessageMeta):
    is_response = False

class DisplayTextMessage(BBSMessage):
    type = ConstantField(b'\x41')

    prompt_customer = EnumField({
        b'\x31': True,
        b'\x30': False,
    }, default=True)

    expects_input = EnumField({
        b'\x31': True,
        b'\x30': False,
    }, default=False)

    mode = ConstantField(b'\x30')

    text = TextField()

    def __init__(self, text, **kwargs):
        # allow `text` to be passed in as a positional argument
        super(DisplayTextMessage, self).__init__(text=text, **kwargs)

class PrintTextMessage(BBSMessage):
    type = ConstantField(b'\x42')

    sub_type = EnumField({b'\x20': 'formatted'})

    media = EnumField({
        b'\x20': 'print_on_receipt',
        b'\x21': 'print_on_journal',
        b'\x22': 'print_on_both',
    }, default='print_on_both')

    mode = EnumField({b'\x2a': 'normal_text'})

    commands = FormattedTextField()

class ResetTimerMessage(BBSMessage):
    type = ConstantField(b'\x43')

    seconds = IntegerField(3)

    def __init__(self, seconds, **kwargs):
        # allow `seconds` to be passed in as a positional argument
        super(ResetTimerMessage, self).__init__(seconds=seconds, **kwargs)

class LocalModeMessage(BBSMessage):
    type = ConstantField(b'\x44')

    result = EnumField({
        # indicates transaction OK
        b'\x20': 'success',
        # indicates transaction/operation rejected
        b'\x21': 'failure',
    })

    acc = EnumField({
        # indicates standard update of accumulator
        b'\x20': 'standard',
        # indicates transaction is finalised as Offline transaction
        b'\x22': 'offline',
        # indicates no update of accumulator
        b'\x30': 'none',
    })

    # 2 digit issuer number is indicating the card issuer. Used if the
    # transaction was accepted. As long as the data is available, the data
    # shall be sent regardless if transaction is rejected or accepted.
    issuer_id = IntegerField(2)

    # Variable field lenght, Max. 19 digit if present. The Primary Account
    # Number from the card holder. The PAN shall not be sent if some parts of
    # the card number is replaced with "*" in the printout. The PAN field is of
    # restricted use, due to security regulations
    pan = DelimitedField(TextField(19), optional=True, delimiter=b';')

    # 14 byte numeric data. Timestamp in format YYYYMMDDHHMMSS. The timestamp
    # shall be the same data as received from the Host to the terminal in the
    # response message
    timestamp = DelimitedField(DateTimeField(), delimiter=b';')

    # Cardholder Verification Method
    ver_method = DelimitedField(EnumField({
        # transaction is PIN based, also to be used if reversal transaction
        b'\x30': 'pin_based',
        # transaction is signature based
        b'\x31': 'signature_based',
        # no CVM. Only amount is verified by cardholder
        b'\x32': 'not_verified',
        # transaction is a Loyalty Transaction. Used for data capture
        # transactions. No accounts are debited or credited
        b'\x32': 'loyalty_transaction',
    }), delimiter=b';')

    # 3 byte, numeric data. The current session number received from the HOST.
    # The session number is uncertain in case that the transaction is an
    # Offline transaction.  This number is changed on reconciliation.
    session_num = DelimitedField(IntegerField(3), delimiter=b';')

    # 12 byte, Alphanumeric data (H20-H7F). The STAN_AUTH and the TIMESTAMP
    # will identify the transaction.
    #   * On-line: The STAN (System Trace Audit Number) is the 6 first bytes,
    #     and the Authorisation Code is the 6 last bytes.
    #   * Off-line: STAN=9xxxx9 where x is the message number for the actual
    #     transaction AUTH = <H20H20H20H20H20H20>
    stan_auth = DelimitedField(TextField(12), delimiter=b';')

    # 4 bytes numeric data (H30 .. H39). This is the customer number if the
    # transaction was Pre-Auth transaction. Must be used as reference in
    # Transfer Amount - Adjustment transaction.
    seq_no = DelimitedField(IntegerField(4), delimiter=b';')

    # 11 bytes numeric data (H30 .. H39). Normally not used. Only used in
    # Restaurant or Hotel environmet where TIP is added to the purchase amount
    # on the ITU. Used in the Purchase or Adjustment transaction.
    tip = DelimitedField(PriceField(), optional=True, delimiter=b';')

class KeyboardInputRequestMessage(BBSMessage):
    type = ConstantField(b'\x46')

    # Indicates if the entered chars should be echoed on the ECR display or not
    echo = EnumField({
        b'\x20': True,
        b'\x21': False,
    })

    # Minimum and maximum number of chars to enter (as a decimal ascii string)
    min_chars = TextField(2)
    max_chars = TextField(2)

class KeyboardInputMessage(BBSMessage):
    type = ConstantField(b'\x55')
    is_response = True

    text = TextField()

    # XXX how are you supposed to parse this
    delimiter = EnumField({
        b'0': 'enter',
        b'9': 'escape',
    })

    def __init__(self, text, **kwargs):
        # allow `text` to be passed in as a positional argument
        super(KeyboardInputMessage, self).__init__(text=text, **kwargs)

    @classmethod
    def unpack_fields(cls, data):
        # currently special cased because of fixed size `delimiter` field
        # following variable length `text` field.
        # TODO yuck yuck yuck
        fields = OrderedDict()

        fields['header'], size = cls.type.read(data)

        text_data = data[size:-cls.delimiter.size]
        fields['text'], size = cls.text.unpack(text_data)

        delimiter_data = data[-cls.delimiter.size:]
        fields['delimiter'], size = cls.delimiter.unpack(delimiter_data)

class SendDataMessageBase(BBSMessage):
    type = ConstantField(b'\x46')

    code = TextField(2)
    is_last_block = EnumField({
        b'\x32': True,
        b'\x31': False,
    })

    seq = TextField(4)  # ignored
    length = TextField(3)  # ignored

class SendReportsDataHeaderMessage(SendDataMessageBase):
    code = ConstantField(b'\x30\x31')

    site_number = TextField(6)
    session_number = TextField(3)
    timestamp = DateTimeField()

class SendReconciliationDataAmountsMessage(SendDataMessageBase):
    code = ConstantField(b'\x30\x32')

    issuer_id = TextField(2)
    num_transactions = IntegerField(4)

    # TODO

class SendDataMessage(SendDataMessageBase):
    code = EnumField({
        subfunction.code.value: subfunction
        for subfunction in [
            SendReportsDataHeaderMessage,
            SendReconciliationDataAmountsMessage,
            # TODO
        ]
    })

    @classmethod
    def unpack(cls, data):
        self = super(SendDataMessage, cls).unpack(data)

        return self.code.unpack(data)

class TransferAmountMessage(BBSMessage):
    type = ConstantField(b'\x51')

    timestamp = DateTimeField()  # not used
    id_no = TextField(6)  # not used
    # Normally set to "0000". If set in Pre-Auth, the number is a reference to
    # a previous Preauth. If set in Adjustment transaction, the field shall be
    # set to the corresponding number received in the Local Mode from the
    # Pre-Authorisation.
    seq_no = TextField(4)  # TODO
    # Operator identification. A fixed field with 4 characters. If not
    # implemented by the ECR vendor, the field should be filled with zeroes
    # (H30's).
    operator_id = TextField(4)
    # Not used, but tested by the ITU because of error prevention)
    mode = EnumField({
        b'\x30': None,
    })
    transfer_type = EnumField({
        b'\x30': 'eft_authorisation',
        b'\x31': 'return_of_goods',
        b'\x32': 'reversal',
        b'\x33': 'purchase_with_cashback',
        b'\x34': 'pre_authorisation',
        b'\x35': 'adjustment',
        b'\x36': 'balance_inquiry',
        b'\x37': 'complete_receipt',
        b'\x38': 'deposit',
        b'\x39': 'cash_withdrawal',
        b'\x3a': 'load_epurse_card',
        b'\x3b': 'merchandise_purchase',
        b'\x3c': 'merchandise_reversal',
        b'\x3d': 'merchandise_correction',
    })
    amount = PriceField(11)
    # Not used, but tested by the ITU because of error prevention)
    unused_type = EnumField({
        b'\x30': None,
    })
    # Only used if transfer_type == 'purchase_with_ashback' (H33), else it will
    # be filled with H20.
    cashback_amount = PriceField(11)
    is_top_up = EnumField({
        b'\x30': True,
        b'\x31': False,
    })
    art_amount = PriceField(11)

    data = DelimitedField(TextField(), delimiter=b';')

    # TODO ART#

class TransferCardDataMessage(BBSMessage):
    type = ConstantField(b'\x52')

    block = EnumField({b'\x30': None})
    track = EnumField({
        b'\x32': 'track_2',
        b'\x33': 'track_1',
        b'\x40': 'manual',
    })

    # TODO DATA and FS

class AdministrationMessage(BBSMessage):
    type = ConstantField(b'\x53')

    timestamp = DateTimeField()
    id_no = TextField(6)
    seq_no = TextField(4)
    opt = TextField(4)

    # TODO single character keyboard input
    adm_code = EnumField({
        b'\x30\x30': 'not_used',
        b'\x30\x39': 'not_used',
        # SEND from ECR should be mapped by ITU to perform RECONCILIATION
        # function.
        b'\x31\x30': 'send',
        # KLAR, validation key. Refer to the NOTE for details
        b'\x31\x31': 'ready',
        # AVBRYT, cancellation key. Refer to the NOTE for details.
        b'\x31\x32': 'cancel',
        # FEIL, correction key.
        b'\x31\x33': 'error',
        # ANNUL from ECR should be mapped by ITU to perform REVERSAL
        # transaction.
        b'\x31\x34': 'reverse',
        b'\x31\x35': 'balance_inquiry_transaction',
        b'\x31\x36': 'x_report',
        b'\x31\x37': 'z_report',
        b'\x31\x38': 'send_offline_transactions',
        b'\x31\x39': 'turnover_report',
        b'\x31\x3A': 'print_eot_transactions',
        b'\x31\x3B': 'not_used',
        b'\x31\x3C': 'not_used',
        b'\x31\x3D': 'not_used',
        b'\x31\x3E': 'not_used',
    })

    fs = ConstantField(b'\x1C')

class DeviceAttributeRequestMessage(BBSMessage):
    type = ConstantField(b'\x60')

class DeviceAttributeMessage(BBSMessage):
    type = ConstantField(b'\x61')

class StatusMessage(BBSMessage):
    type = ConstantField(b'\x62')
    is_response = True

class ResponseMessage(BBSMessage):
    type = ConstantField(b'\x5b')
    is_response = True

    code = EnumField({
        # OK. The Receiver has received and processed the data correctly
        b'\x30\x30': 'success',
        # Not OK. The receiver is not able to process the received data
        b'\x30\x33': 'failure',
        # Not OK. Shall be treated as if H3033 is received
        b'\x30\x34': 'failure',
        b'\x30\x35': 'failure',
        b'\x30\x36': 'failure',
        b'\x30\x37': 'failure',
        b'\x30\x38': 'failure',
        b'\x30\x39': 'failure',
        # ECR display busy, ITU may try again once
        b'\x31\x31': 'display_busy',
        # ECR printer busy, ITU may try again once
        b'\x31\x32': 'printer_busy',
        # ECR printer out of function.
        # If the ECR sends H3133, the ITU must interrupt the current
        # transaction, and wait for the next 'Bank-Mode' initiation from the
        # ECR
        b'\x31\x33': 'printer_broken'
    })

    endcode = ConstantField(b'\x5d')

_ITU_MESSAGE_TYPES = {
    DisplayTextMessage,
    PrintTextMessage,
    ResetTimerMessage,
    LocalModeMessage,
    KeyboardInputRequestMessage,
    SendDataMessage,
    DeviceAttributeRequestMessage,
    StatusMessage,
}

class ITUMessage(BBSMessage):
    type = EnumField({
        subtype.type.value: subtype
        for subtype in _ITU_MESSAGE_TYPES
    })

def unpack_itu_message(data):
    header = ITUMessage.unpack(data)
    return header.type.unpack(data)

_ECR_MESSAGE_TYPES = {
    KeyboardInputMessage,
    SendDataMessage,
    TransferAmountMessage,
    AdministrationMessage,
    DeviceAttributeMessage,
}

class ECRMessage(BBSMessage):
    type = EnumField({
        subtype.type.value: subtype
        for subtype in _ECR_MESSAGE_TYPES
    })

def unpack_ecr_message(data):
    header = ECRMessage.unpack(data)
    return header.type.unpack(data)
<|endoftext|>


<commit_before># Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module containing UnixBench installation and cleanup functions."""

from perfkitbenchmarker import vm_util

UNIXBENCH_TAR = 'UnixBench5.1.3.tgz'
UNIXBENCH_URL = 'http://byte-unixbench.googlecode.com/files/' + UNIXBENCH_TAR
UNIXBENCH_DIR = '%s/UnixBench' % vm_util.VM_TMP_DIR

def _Install(vm):
  """Installs the UnixBench package on the VM."""
  vm.Install('build_tools')
  vm.Install('wget')
  vm.RemoteCommand('wget {0} -P {1}'.format(UNIXBENCH_URL, vm_util.VM_TMP_DIR))
  vm.RemoteCommand('cd {0} && tar xvzf {1}'.format(vm_util.VM_TMP_DIR,
                                                   UNIXBENCH_TAR))

def YumInstall(vm):
  """Installs the UnixBench package on the VM."""
  vm.InstallPackages('libX11-devel mesa-libGL-devel '
                     'perl-Time-HiRes libXext-devel')
  _Install(vm)

def AptInstall(vm):
  """Installs the UnixBench package on the VM."""
  vm.InstallPackages('libx11-dev libgl1-mesa-dev libxext-dev')
  _Install(vm)
<commit_msg>Update unixbench repository.<commit_after># Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Module containing UnixBench installation and cleanup functions."""

from perfkitbenchmarker import vm_util

UNIXBENCH_TAR = 'v5.1.3.tar.gz'
UNIXBENCH_URL = 'https://github.com/kdlucas/byte-unixbench/archive/' + UNIXBENCH_TAR
UNIXBENCH_DIR = '%s/byte-unixbench-5.1.3/UnixBench' % vm_util.VM_TMP_DIR

def _Install(vm):
  """Installs the UnixBench package on the VM."""
  vm.Install('build_tools')
  vm.Install('wget')
  vm.RemoteCommand('wget {0} -P {1}'.format(UNIXBENCH_URL, vm_util.VM_TMP_DIR))
  vm.RemoteCommand('cd {0} && tar xvzf {1}'.format(vm_util.VM_TMP_DIR,
                                                   UNIXBENCH_TAR))

def YumInstall(vm):
  """Installs the UnixBench package on the VM."""
  vm.InstallPackages('libX11-devel mesa-libGL-devel '
                     'perl-Time-HiRes libXext-devel')
  _Install(vm)

def AptInstall(vm):
  """Installs the UnixBench package on the VM."""
  vm.InstallPackages('libx11-dev libgl1-mesa-dev libxext-dev')
  _Install(vm)
<|endoftext|>


<commit_before># General imports.
import copy
import cPickle
import os
import Queue
import shutil
import sqlite3
import threading
import time
from collections import namedtuple as namedtuple

# Imports from this module.
from VectorClock import VectorClock
import Service

class GlobalStateError(Exception): pass
class ClockError(GlobalStateError): pass
class MessageError(GlobalStateError): pass
class OneToManyServiceError(GlobalStateError): pass

# TODO: support to detect crashed processes (send a message every peerWaitingTime time and require answer within peerWaitingTime, if not: process crashed)
# TODO: support to request lost messages (if missing message still not arrived after messageWaitingTime, send this request)
# TODO: add index on clock, if it makes a difference

class MessageProcessor(threading.Thread):
    """The message processor processes all incoming messages, and handles all
        election and host-related messages. It also handles checking of hosts/players
        leaving
    """

    KEEP_ALIVE_TYPE = 'KEEP-ALIVE'
    SERVER_MOVE_TYPE = 'SERVER-MESSAGE'

    def __init__(self, service, sessionUUID, senderUUID, peerWaitingTime=30, messageWaitingTime=2):
        # MulticastMessaging subclass.
        if not isinstance(service, Service.OneToManyService):
            raise OneToManyServiceError
        self.service = service

        # Identifiers.
        self.sessionUUID = sessionUUID
        self.senderUUID  = senderUUID

        #The last time the keep-alive message was sent (in seconds)
        #self.keepAliveSentTime = 0
        #The time to wait between keep-alive messages
        #self.keepAliveTreshold = 1
        #self.keepAliveLeftTime = 5 #the time to wait for a keep-alive message to arrive before we disconnect the player
        #self.keepAliveMessages = {} # Contains the last time a keep-alive message was received per player

        # Message storage.
        self.inbox       = Queue.Queue()
        self.outbox      = Queue.Queue()

        # Settings.
        self.peerWaitingTime    = int(peerWaitingTime)
        self.messageWaitingTime = int(messageWaitingTime)

        # Thread state variables.
        self.alive = True
        self.die = False
        self.lock = threading.Condition()

        # Other things.
        self._startedWaitingForMessages = None
        self.clock = VectorClock()
        self.clock.add(self.senderUUID)

        # Register this Global State's session UUID with the service as a
        # valid destination.
        self.service.registerDestination(self.sessionUUID)

        super(MessageProcessor, self).__init__()

    ##########################################################################
    # Message processing.                                                    #
    ##########################################################################

    def sendMessage(self, message, sendToSelf = True):
        """Enqueue a message to be sent."""
        print sendToSelf
        with self.lock:
            # print "\tGlobalState.sendMessage()", message
            envelope = self._wrapMessage(message)
            if sendToSelf:
                self.inbox.put((self.senderUUID, message))
            with self.service.lock:
                # print "\tGlobalState._sendMessage()", envelope
                self.service.sendMessage(self.sessionUUID, envelope)
                self.service.lock.notifyAll()

    def countReceivedMessages(self):
        with self.lock:
            return self.inbox.qsize()

    def receiveMessage(self):
        with self.lock:
            return self.inbox.get()

    def sendKeepAliveMessage(self):
        with self.service.lock:
            self.service.sendServiceMessage({'type' : self.KEEP_ALIVE_TYPE, 'originUUID':self.senderUUID})
            self.keepAliveSentTime = time.clock()

    def receiveKeepAliveMessage(self, message):
        with self.lock:
            self.keepAliveMessages[message['originUUID']] = time.clock()

    #checks if any players disconnected
    def checkKeepAlive(self):
        with self.lock:
            for key in self.keepAliveMessages.keys():
                if time.clock() - self.keepAliveMessages[key] > self.keepAliveLeftTime:
                    self.inbox.put((key, {'type':4}, None))
                    del self.keepAliveMessages[key]

    def _wrapMessage(self, message):
        """Wrap a message in an envelope to prepare it for sending."""

        envelope = {}
        # Increment the vector clock for our envelope.
        self.clock.increment(self.senderUUID)
        # Add the clock to the envelope.
        envelope['clock'] = copy.deepcopy(self.clock)
        # Add the sender UUUID and the original sender UUID to the envelope
        # (which is always ourselves).
        envelope['senderUUID'] = envelope['originUUID'] = self.senderUUID
        # Store the message in the envelope.
        envelope['message'] = message
        return envelope

    def processMessage(self, envelope):
        """Store a message in the global state and put it in the inbox."""

        # Merge the clocks and increment our own component.
        print envelope
        if envelope['message']['type'] == self.SERVER_MOVE_TYPE:
            if envelope['message']['target'] == self.senderUUID:
                with self.lock:
                    self.inbox.put((envelope['originUUID'], envelope['message']))

        else:
            # Move the message to the inbox queue, so it can be retrieved.
            with self.lock:
                self.inbox.put((envelope['originUUID'], envelope['message']))

    def erase(self):
        """Erase the global state."""
        # Remove this Global State's session UUID as a valid destination.
        self.service.removeDestination(self.sessionUUID)
        # Reset the database.
        self._dbCur.execute("DELETE FROM MessageHistory")
        self._dbCon.commit()
        self.clock = VectorClock()
        self.clock.add(self.senderUUID)

    def run(self):

        while self.alive:
            # Check if it's time to send liveness messages.
            # TODO

            # Retrieve incoming messages and store them in the waiting room.
            with self.lock:
                with self.service.lock:
                    if self.service.countReceivedMessages(self.sessionUUID) > 0:
                        envelope = self.service.receiveMessage(self.sessionUUID)
                        # Ignore our own messages.
                        if envelope['senderUUID'] != self.senderUUID:
                            self.processMessage(envelope)
                            # print "\tGlobalstate.run(): moved message to waiting room with clock", clock
                    if self.service.countReceivedServiceMessages() > 0:
                        envelope = self.service.receiveServiceMessage()
                        if envelope['type'] == self.KEEP_ALIVE_TYPE:
                            self.receiveKeepAliveMessage(envelope)
                        if ('originUUID' in envelope.keys()):
                            self.inbox.put((envelope['originUUID'], envelope))
                        else:
                            self.inbox.put((None, envelope))

            # 20 refreshes per second is plenty.
            time.sleep(0.05)

    def kill(self):
        # Let the thread know it should commit suicide. But first let it send
        # all remaining messages.
        # while self.outbox.qsize() > 0:
        #     if
        with self.lock:
            self.die = True

    def _commitSuicide(self):
        """Commit suicide when asked to. The lock must be acquired before
        calling this method.
        """

        # Stop us from running any further.
        self.alive = False
<commit_msg>ntp timestamps toegevoegd aan alle berichten<commit_after># General imports.
import copy
import cPickle
import os
import Queue
import shutil
import sqlite3
import threading
import time
from collections import namedtuple as namedtuple
import ntplib

# Imports from this module.
from VectorClock import VectorClock
import Service

class GlobalStateError(Exception): pass
class ClockError(GlobalStateError): pass
class MessageError(GlobalStateError): pass
class OneToManyServiceError(GlobalStateError): pass

# TODO: support to detect crashed processes (send a message every peerWaitingTime time and require answer within peerWaitingTime, if not: process crashed)
# TODO: support to request lost messages (if missing message still not arrived after messageWaitingTime, send this request)
# TODO: add index on clock, if it makes a difference

class MessageProcessor(threading.Thread):
    """The message processor processes all incoming messages, and handles all
        election and host-related messages. It also handles checking of hosts/players
        leaving
    """

    KEEP_ALIVE_TYPE = 'KEEP-ALIVE'
    SERVER_MOVE_TYPE = 'SERVER-MESSAGE'

    def __init__(self, service, sessionUUID, senderUUID, peerWaitingTime=30, messageWaitingTime=2):
        # MulticastMessaging subclass.
        if not isinstance(service, Service.OneToManyService):
            raise OneToManyServiceError
        self.service = service

        # Identifiers.
        self.sessionUUID = sessionUUID
        self.senderUUID  = senderUUID

        #The last time the keep-alive message was sent (in seconds)
        #self.keepAliveSentTime = 0
        #The time to wait between keep-alive messages
        #self.keepAliveTreshold = 1
        #self.keepAliveLeftTime = 5 #the time to wait for a keep-alive message to arrive before we disconnect the player
        #self.keepAliveMessages = {} # Contains the last time a keep-alive message was received per player

        # Message storage.
        self.inbox       = Queue.Queue()
        self.outbox      = Queue.Queue()

        # Settings.
        self.peerWaitingTime    = int(peerWaitingTime)
        self.messageWaitingTime = int(messageWaitingTime)

        # Thread state variables.
        self.alive = True
        self.die = False
        self.lock = threading.Condition()

        # Other things.
        self._startedWaitingForMessages = None
        self.NTPoffset = 0

        # Register this Global State's session UUID with the service as a
        # valid destination.
        self.service.registerDestination(self.sessionUUID)

        super(MessageProcessor, self).__init__()

    ##########################################################################
    # Message processing.                                                    #
    ##########################################################################

    def sendMessage(self, message, sendToSelf = True):
        """Enqueue a message to be sent."""
        print sendToSelf
        with self.lock:
            # print "\tGlobalState.sendMessage()", message
            envelope = self._wrapMessage(message)
            if sendToSelf:
                self.inbox.put((self.senderUUID, message))
            with self.service.lock:
                # print "\tGlobalState._sendMessage()", envelope
                self.service.sendMessage(self.sessionUUID, envelope)
                self.service.lock.notifyAll()

    def countReceivedMessages(self):
        with self.lock:
            return self.inbox.qsize()

    def receiveMessage(self):
        with self.lock:
            return self.inbox.get()

    def sendKeepAliveMessage(self):
        with self.service.lock:
            self.sendMessage({'type' : self.KEEP_ALIVE_TYPE, 'originUUID':self.senderUUID, 'timestamp' : time.time() + self.NTPoffset})
            self.keepAliveSentTime = time.time() + self.NTPoffset

    def receiveKeepAliveMessage(self, message):
        with self.lock:
            self.PlayerRTT[message['originUUID']] = message['timestamp']

    #checks if any players disconnected
    def checkKeepAlive(self):
        with self.lock:
            for key in self.keepAliveMessages.keys():
                if self.keepAliveMessages[key] > self.keepAliveLeftTime:
                    self.inbox.put(key, {'type':4})
                    del self.keepAliveMessages[key]

    def _wrapMessage(self, message):
        """Wrap a message in an envelope to prepare it for sending."""

        envelope = {}
        # Add the timestamp to the envelope.
        envelope['timestamp'] = time.time() + self.NTPoffset
        # Add the sender UUUID and the original sender UUID to the envelope
        # (which is always ourselves).
        envelope['senderUUID'] = envelope['originUUID'] = self.senderUUID
        # Store the message in the envelope.
        envelope['message'] = message
        return envelope

    def processMessage(self, envelope):
        """Process a message and put it in the inbox if its meant for us"""

        # Merge the clocks and increment our own component.
        if envelope['message']['type'] == self.SERVER_MOVE_TYPE:
            if envelope['message']['target'] == self.senderUUID:
                with self.lock:
                    self.inbox.put((envelope['originUUID'], envelope['message']))
        else:
            # Move the message to the inbox queue, so it can be retrieved.
            with self.lock:
                self.inbox.put((envelope['originUUID'], envelope['message']))

    def erase(self):
        """Erase the global state."""
        # Remove this Global State's session UUID as a valid destination.
        self.service.removeDestination(self.sessionUUID)
        # Reset the database.
        self._dbCur.execute("DELETE FROM MessageHistory")
        self._dbCon.commit()
        self.clock = VectorClock()
        self.clock.add(self.senderUUID)

    def getNTPoffset(self):
        # get the NTP offset of this computers clock
        try:
            client = ntplib.NTPClient()
            response = client.request('europe.pool.ntp.org', version=3)
            if response.leap != 3:
                self.NTPoffset = response.offset
        except:
            print 'Warning! NTP server could no be reached'
            pass

        # print 'NTPoffset is now: ' + str(self.NTPoffset)

    def run(self):
        self.getNTPoffset()

        while self.alive:
            # Check if it's time to send liveness messages.
            # TODO

            # Retrieve incoming messages and store them in the waiting room.
            with self.lock:
                with self.service.lock:
                    if self.service.countReceivedMessages(self.sessionUUID) > 0:
                        envelope = self.service.receiveMessage(self.sessionUUID)
                        # Ignore our own messages.
                        if envelope['senderUUID'] != self.senderUUID:
                            self.processMessage(envelope)
                            # print "\tGlobalstate.run(): moved message to waiting room with clock", clock
                    if self.service.countReceivedServiceMessages() > 0:
                        envelope = self.service.receiveServiceMessage()
                        if envelope['type'] == self.KEEP_ALIVE_TYPE:
                            self.receiveKeepAliveMessage(envelope)
                        if ('originUUID' in envelope.keys()):
                            self.inbox.put((envelope['originUUID'], envelope))
                        else:
                            self.inbox.put((None, envelope))

            # 20 refreshes per second is plenty.
            time.sleep(0.05)

    def kill(self):
        # Let the thread know it should commit suicide. But first let it send
        # all remaining messages.
        # while self.outbox.qsize() > 0:
        #     if
        with self.lock:
            self.die = True

    def _commitSuicide(self):
        """Commit suicide when asked to. The lock must be acquired before
        calling this method.
        """

        # Stop us from running any further.
        self.alive = False
<|endoftext|>


<commit_before>#! /usr/bin/env python3

import configparser
import os
import subprocess
import urllib.parse

URL = os.environ['PERU_MODULE_URL']
REV = os.environ['PERU_MODULE_REV'] or 'master'
REUP = os.environ['PERU_MODULE_REUP'] or 'master'

# Because peru gives each plugin a unique cache dir based on its cacheable
# fields (in this case, url) we could clone directly into cache_root. However,
# because the git plugin needs to handle subrepos as well, it still has to
# separate things out by repo url.
CACHE_ROOT = os.environ['PERU_PLUGIN_CACHE']

def git(*args, git_dir=None, capture_output=False):
    # Avoid forgetting this arg.
    assert git_dir is None or os.path.isdir(git_dir)

    command = ['git']
    if git_dir:
        command.append('--git-dir={0}'.format(git_dir))
    command.extend(args)

    stdout = subprocess.PIPE if capture_output else None
    # Always let stderr print to the caller.
    process = subprocess.Popen(
        command,
        stdin=subprocess.DEVNULL,
        stdout=stdout,
        universal_newlines=True)
    output, _ = process.communicate()
    if process.returncode != 0:
        raise RuntimeError(
            'Command exited with error code {0}:\n$ {1}\n{2}'.format(
                process.returncode,
                ' '.join(command),
                output))

    return output

def has_clone(url):
    return os.path.exists(repo_cache_path(url))

def clone_if_needed(url):
    repo_path = repo_cache_path(url)
    if not has_clone(url):
        git('clone', '--mirror', '--progress', url, repo_path)
    return repo_path

def repo_cache_path(url):
    escaped = urllib.parse.quote(url, safe='')
    return os.path.join(CACHE_ROOT, escaped)

def clone_and_maybe_print(url):
    if not has_clone(url):
        print('git clone ' + url)
    return clone_if_needed(url)

def git_fetch(url, repo_path):
    print('git fetch ' + url)
    git('fetch', '--prune', git_dir=repo_path)

def already_has_rev(repo, rev):
    try:
        # Make sure the rev exists.
        git('cat-file', '-e', rev, git_dir=repo)
        # Get the hash for the rev.
        output = git('rev-parse', rev, git_dir=repo, capture_output=True)
    except:
        return False

    # Only return True for revs that are absolute hashes.
    # We could consider treating tags the way, but...
    # 1) Tags actually can change.
    # 2) It's not clear at a glance if something is a branch or a tag.
    # Keep it simple.
    return output.strip() == rev

def checkout_tree(url, rev, dest):
    repo_path = clone_and_maybe_print(url)
    if not already_has_rev(repo_path, rev):
        git_fetch(url, repo_path)
    # If we just use `git checkout rev -- .` here, we get an error when rev is
    # an empty commit.
    git('--work-tree=' + dest, 'read-tree', rev, git_dir=repo_path)
    git('--work-tree=' + dest, 'checkout-index', '--all', git_dir=repo_path)
    checkout_subrepos(repo_path, rev, dest)

def checkout_subrepos(repo_path, rev, work_tree):
    gitmodules = os.path.join(work_tree, '.gitmodules')
    if not os.path.exists(gitmodules):
        return

    parser = configparser.ConfigParser()
    parser.read(gitmodules)
    for section in parser.sections():
        sub_relative_path = parser[section]['path']
        sub_full_path = os.path.join(work_tree, sub_relative_path)
        sub_url = parser[section]['url']
        ls_tree = git('ls-tree', '-r', rev, sub_relative_path,
                      git_dir=repo_path, capture_output=True)
        sub_rev = ls_tree.split()[2]
        checkout_tree(sub_url, sub_rev, sub_full_path)

def plugin_sync():
    checkout_tree(URL, REV, os.environ['PERU_SYNC_DEST'])

def plugin_reup():
    reup_output = os.environ['PERU_REUP_OUTPUT']
    repo_path = clone_if_needed(URL)
    git_fetch(URL, repo_path)
    output = git('rev-parse', REUP, git_dir=repo_path, capture_output=True)
    with open(reup_output, 'w') as out_file:
        print('rev:', output.strip(), file=out_file)

command = os.environ['PERU_PLUGIN_COMMAND']
if command == 'sync':
    plugin_sync()
elif command == 'reup':
    plugin_reup()
else:
    raise RuntimeError('Unknown command: ' + repr(command))
<commit_msg>hash repo URLs in the git plugin's cache paths<commit_after>#! /usr/bin/env python3

import configparser
import hashlib
import os
import subprocess

URL = os.environ['PERU_MODULE_URL']
REV = os.environ['PERU_MODULE_REV'] or 'master'
REUP = os.environ['PERU_MODULE_REUP'] or 'master'

# Because peru gives each plugin a unique cache dir based on its cacheable
# fields (in this case, url) we could clone directly into cache_root. However,
# because the git plugin needs to handle subrepos as well, it still has to
# separate things out by repo url.
CACHE_ROOT = os.environ['PERU_PLUGIN_CACHE']

def git(*args, git_dir=None, capture_output=False):
    # Avoid forgetting this arg.
    assert git_dir is None or os.path.isdir(git_dir)

    command = ['git']
    if git_dir:
        command.append('--git-dir={0}'.format(git_dir))
    command.extend(args)

    stdout = subprocess.PIPE if capture_output else None
    # Always let stderr print to the caller.
    process = subprocess.Popen(
        command,
        stdin=subprocess.DEVNULL,
        stdout=stdout,
        universal_newlines=True)
    output, _ = process.communicate()
    if process.returncode != 0:
        raise RuntimeError(
            'Command exited with error code {0}:\n$ {1}\n{2}'.format(
                process.returncode,
                ' '.join(command),
                output))

    return output

def has_clone(url):
    return os.path.exists(repo_cache_path(url))

def clone_if_needed(url):
    repo_path = repo_cache_path(url)
    if not has_clone(url):
        git('clone', '--mirror', '--progress', url, repo_path)
    return repo_path

def repo_cache_path(url):
    # If we just concatenate the escaped repo URL into the path, we start to
    # run up against the 260-character path limit on Windows.
    url_hash = hashlib.sha1(url.encode()).hexdigest()
    return os.path.join(CACHE_ROOT, url_hash)

def clone_and_maybe_print(url):
    if not has_clone(url):
        print('git clone ' + url)
    return clone_if_needed(url)

def git_fetch(url, repo_path):
    print('git fetch ' + url)
    git('fetch', '--prune', git_dir=repo_path)

def already_has_rev(repo, rev):
    try:
        # Make sure the rev exists.
        git('cat-file', '-e', rev, git_dir=repo)
        # Get the hash for the rev.
        output = git('rev-parse', rev, git_dir=repo, capture_output=True)
    except:
        return False

    # Only return True for revs that are absolute hashes.
    # We could consider treating tags the way, but...
    # 1) Tags actually can change.
    # 2) It's not clear at a glance if something is a branch or a tag.
    # Keep it simple.
    return output.strip() == rev

def checkout_tree(url, rev, dest):
    repo_path = clone_and_maybe_print(url)
    if not already_has_rev(repo_path, rev):
        git_fetch(url, repo_path)
    # If we just use `git checkout rev -- .` here, we get an error when rev is
    # an empty commit.
    git('--work-tree=' + dest, 'read-tree', rev, git_dir=repo_path)
    git('--work-tree=' + dest, 'checkout-index', '--all', git_dir=repo_path)
    checkout_subrepos(repo_path, rev, dest)

def checkout_subrepos(repo_path, rev, work_tree):
    gitmodules = os.path.join(work_tree, '.gitmodules')
    if not os.path.exists(gitmodules):
        return

    parser = configparser.ConfigParser()
    parser.read(gitmodules)
    for section in parser.sections():
        sub_relative_path = parser[section]['path']
        sub_full_path = os.path.join(work_tree, sub_relative_path)
        sub_url = parser[section]['url']
        ls_tree = git('ls-tree', '-r', rev, sub_relative_path,
                      git_dir=repo_path, capture_output=True)
        sub_rev = ls_tree.split()[2]
        checkout_tree(sub_url, sub_rev, sub_full_path)

def plugin_sync():
    checkout_tree(URL, REV, os.environ['PERU_SYNC_DEST'])

def plugin_reup():
    reup_output = os.environ['PERU_REUP_OUTPUT']
    repo_path = clone_if_needed(URL)
    git_fetch(URL, repo_path)
    output = git('rev-parse', REUP, git_dir=repo_path, capture_output=True)
    with open(reup_output, 'w') as out_file:
        print('rev:', output.strip(), file=out_file)

command = os.environ['PERU_PLUGIN_COMMAND']
if command == 'sync':
    plugin_sync()
elif command == 'reup':
    plugin_reup()
else:
    raise RuntimeError('Unknown command: ' + repr(command))
<|endoftext|>


<commit_before>from collections import namedtuple
from functools import singledispatch
import ctypes
import enum
import typing as py_typing

import numpy as np

from numba.core import types, utils, errors
from numba.np import numpy_support

# terminal color markup
_termcolor = errors.termcolor()

class Purpose(enum.Enum):
    # Value being typed is used as an argument
    argument = 1
    # Value being typed is used as a constant
    constant = 2

_TypeofContext = namedtuple("_TypeofContext", ("purpose",))

def typeof(val, purpose=Purpose.argument):
    """
    Get the Numba type of a Python value for the given purpose.
    """
    # Note the behaviour for Purpose.argument must match _typeof.c.
    c = _TypeofContext(purpose)
    ty = typeof_impl(val, c)
    if ty is None:
        msg = _termcolor.errmsg(
            f"Cannot determine Numba type of {type(val)}")
        raise ValueError(msg)
    return ty

@singledispatch
def typeof_impl(val, c):
    """
    Generic typeof() implementation.
    """
    tp = _typeof_buffer(val, c)
    if tp is not None:
        return tp

    # cffi is handled here as it does not expose a public base class
    # for exported functions or CompiledFFI instances.
    from numba.core.typing import cffi_utils
    if cffi_utils.SUPPORTED:
        if cffi_utils.is_cffi_func(val):
            return cffi_utils.make_function_type(val)
        if cffi_utils.is_ffi_instance(val):
            return types.ffi

    return getattr(val, "_numba_type_", None)

def _typeof_buffer(val, c):
    from numba.core.typing import bufproto
    try:
        m = memoryview(val)
    except TypeError:
        return
    # Object has the buffer protocol
    try:
        dtype = bufproto.decode_pep3118_format(m.format, m.itemsize)
    except ValueError:
        return
    type_class = bufproto.get_type_class(type(val))
    layout = bufproto.infer_layout(m)
    return type_class(dtype, m.ndim, layout=layout,
                      readonly=m.readonly)

@typeof_impl.register(ctypes._CFuncPtr)
def _typeof_ctypes_function(val, c):
    from .ctypes_utils import is_ctypes_funcptr, make_function_type
    if is_ctypes_funcptr(val):
        return make_function_type(val)

@typeof_impl.register(type)
def _typeof_type(val, c):
    """
    Type various specific Python types.
    """
    if issubclass(val, BaseException):
        return types.ExceptionClass(val)
    if issubclass(val, tuple) and hasattr(val, "_asdict"):
        return types.NamedTupleClass(val)

    if issubclass(val, np.generic):
        return types.NumberClass(numpy_support.from_dtype(val))

    from numba.typed import Dict
    if issubclass(val, Dict):
        return types.TypeRef(types.DictType)

    from numba.typed import List
    if issubclass(val, List):
        return types.TypeRef(types.ListType)

    if c.purpose == Purpose.argument:
        # We only return for argument contexts.
        # For situations like x = int(y), we want typeof(int) to return None,
        # so that Context.resolve_value_type calls Context._get_global_type.

        if val in (int, float, complex):
            return typeof_impl(val(0), c)

        if val is str:
            return typeof_impl(str("numba"), c)

@typeof_impl.register(py_typing.GenericMeta)
def _typeof_typing(val, c):
    if issubclass(val, py_typing.List):
        (element_py,) = val.__args__
        element_nb = typeof_impl(element_py, c)
        if element_nb is None:
            raise ValueError(
                f"Cannot type list element type {element_py}")
        return types.List(element_nb)

    if issubclass(val, py_typing.Dict):
        key_py, value_py = val.__args__
        key_nb = typeof_impl(key_py, c)
        value_nb = typeof_impl(value_py, c)
        if key_nb is None:
            raise ValueError(f"Cannot type dict key type {key_py}")
        if value_nb is None:
            raise ValueError(f"Cannot type dict value type {value_py}")
        return types.DictType(key_nb, value_nb)

    if issubclass(val, py_typing.Set):
        (element_py,) = val.__args__
        element_nb = typeof_impl(element_py, c)
        if element_nb is None:
            raise ValueError(
                f"Cannot type set element type {element_py}")
        return types.Set(element_nb)

    if issubclass(val, py_typing.Tuple):
        tys = tuple(typeof_impl(elem, c) for elem in val.__args__)
        if any(ty is None for ty in tys):
            return
        return types.BaseTuple.from_types(tys)

@typeof_impl.register(type(py_typing.Union))
def _typeof_optional(val, c):
    (arg_1_py, arg_2_py) = val.__args__
    if arg_2_py is not type(None): # noqa: E721
        raise ValueError(
            "Cannot type Union that is not an Optional "
            f"(second type {arg_2_py} is not NoneType")
    arg_1_nb = typeof_impl(arg_1_py, c)
    if arg_1_nb is None:
        raise ValueError(f"Cannot type optional inner type {arg_1_py}")
    return types.Optional(arg_1_nb)

@typeof_impl.register(bool)
def _typeof_bool(val, c):
    return types.boolean

@typeof_impl.register(float)
def _typeof_float(val, c):
    return types.float64

@typeof_impl.register(complex)
def _typeof_complex(val, c):
    return types.complex128

def _typeof_int(val, c):
    # As in _typeof.c
    nbits = utils.bit_length(val)
    if nbits < 32:
        typ = types.intp
    elif nbits < 64:
        typ = types.int64
    elif nbits == 64 and val >= 0:
        typ = types.uint64
    else:
        raise ValueError("Int value is too large: %s" % val)
    return typ

for cls in utils.INT_TYPES:
    typeof_impl.register(cls, _typeof_int)

@typeof_impl.register(np.generic)
def _typeof_numpy_scalar(val, c):
    try:
        return numpy_support.map_arrayscalar_type(val)
    except NotImplementedError:
        pass

@typeof_impl.register(str)
def _typeof_str(val, c):
    return types.string

@typeof_impl.register(type((lambda a: a).__code__))
def _typeof_code(val, c):
    return types.code_type

@typeof_impl.register(type(None))
def _typeof_none(val, c):
    return types.none

@typeof_impl.register(type(Ellipsis))
def _typeof_ellipsis(val, c):
    return types.ellipsis

@typeof_impl.register(tuple)
def _typeof_tuple(val, c):
    tys = [typeof_impl(v, c) for v in val]
    if any(ty is None for ty in tys):
        return
    return types.BaseTuple.from_types(tys, type(val))

@typeof_impl.register(list)
def _typeof_list(val, c):
    if len(val) == 0:
        raise ValueError("Cannot type empty list")
    ty = typeof_impl(val[0], c)
    if ty is None:
        raise ValueError(
            f"Cannot type list element type {type(val[0])}")
    return types.List(ty, reflected=True)

@typeof_impl.register(set)
def _typeof_set(val, c):
    if len(val) == 0:
        raise ValueError("Cannot type empty set")
    item = next(iter(val))
    ty = typeof_impl(item, c)
    if ty is None:
        raise ValueError(
            f"Cannot type set element type {type(item)}")
    return types.Set(ty, reflected=True)

@typeof_impl.register(slice)
def _typeof_slice(val, c):
    return types.slice2_type if val.step in (None, 1) else types.slice3_type

@typeof_impl.register(enum.Enum)
@typeof_impl.register(enum.IntEnum)
def _typeof_enum(val, c):
    clsty = typeof_impl(type(val), c)
    return clsty.member_type

@typeof_impl.register(enum.EnumMeta)
def _typeof_enum_class(val, c):
    cls = val
    members = list(cls.__members__.values())
    if len(members) == 0:
        raise ValueError("Cannot type enum with no members")
    dtypes = {typeof_impl(mem.value, c) for mem in members}
    if len(dtypes) > 1:
        raise ValueError("Cannot type heterogeneous enum: "
                         "got value types %s"
                         % ", ".join(sorted(str(ty) for ty in dtypes)))
    if issubclass(val, enum.IntEnum):
        typecls = types.IntEnumClass
    else:
        typecls = types.EnumClass
    return typecls(cls, dtypes.pop())

@typeof_impl.register(np.dtype)
def _typeof_dtype(val, c):
    tp = numpy_support.from_dtype(val)
    return types.DType(tp)

@typeof_impl.register(np.ndarray)
def _typeof_ndarray(val, c):
    try:
        dtype = numpy_support.from_dtype(val.dtype)
    except NotImplementedError:
        raise ValueError("Unsupported array dtype: %s" % (val.dtype,))
    layout = numpy_support.map_layout(val)
    readonly = not val.flags.writeable
    return types.Array(dtype, val.ndim, layout, readonly=readonly)

@typeof_impl.register(types.NumberClass)
def _typeof_number_class(val, c):
    return val

@typeof_impl.register(types.Literal)
def _typeof_literal(val, c):
    return val

@typeof_impl.register(types.TypeRef)
def _typeof_typeref(val, c):
    return val

@typeof_impl.register(types.Type)
def _typeof_nb_type(val, c):
    if isinstance(val, types.BaseFunction):
        return val
    elif isinstance(val, (types.Number, types.Boolean)):
        return types.NumberClass(val)
    else:
        return types.TypeRef(val)
<commit_msg>Fix for Python 3.7+<commit_after>from collections import namedtuple
from functools import singledispatch
import ctypes
import enum
import typing as py_typing

import numpy as np

from numba.core import types, utils, errors
from numba.np import numpy_support

# terminal color markup
_termcolor = errors.termcolor()

class Purpose(enum.Enum):
    # Value being typed is used as an argument
    argument = 1
    # Value being typed is used as a constant
    constant = 2

_TypeofContext = namedtuple("_TypeofContext", ("purpose",))

def typeof(val, purpose=Purpose.argument):
    """
    Get the Numba type of a Python value for the given purpose.
    """
    # Note the behaviour for Purpose.argument must match _typeof.c.
    c = _TypeofContext(purpose)
    ty = typeof_impl(val, c)
    if ty is None:
        msg = _termcolor.errmsg(
            f"Cannot determine Numba type of {type(val)}")
        raise ValueError(msg)
    return ty

@singledispatch
def typeof_impl(val, c):
    """
    Generic typeof() implementation.
    """
    tp = _typeof_buffer(val, c)
    if tp is not None:
        return tp

    # cffi is handled here as it does not expose a public base class
    # for exported functions or CompiledFFI instances.
    from numba.core.typing import cffi_utils
    if cffi_utils.SUPPORTED:
        if cffi_utils.is_cffi_func(val):
            return cffi_utils.make_function_type(val)
        if cffi_utils.is_ffi_instance(val):
            return types.ffi

    return getattr(val, "_numba_type_", None)

def _typeof_buffer(val, c):
    from numba.core.typing import bufproto
    try:
        m = memoryview(val)
    except TypeError:
        return
    # Object has the buffer protocol
    try:
        dtype = bufproto.decode_pep3118_format(m.format, m.itemsize)
    except ValueError:
        return
    type_class = bufproto.get_type_class(type(val))
    layout = bufproto.infer_layout(m)
    return type_class(dtype, m.ndim, layout=layout,
                      readonly=m.readonly)

@typeof_impl.register(ctypes._CFuncPtr)
def _typeof_ctypes_function(val, c):
    from .ctypes_utils import is_ctypes_funcptr, make_function_type
    if is_ctypes_funcptr(val):
        return make_function_type(val)

@typeof_impl.register(type)
def _typeof_type(val, c):
    """
    Type various specific Python types.
    """
    if issubclass(val, BaseException):
        return types.ExceptionClass(val)
    if issubclass(val, tuple) and hasattr(val, "_asdict"):
        return types.NamedTupleClass(val)

    if issubclass(val, np.generic):
        return types.NumberClass(numpy_support.from_dtype(val))

    from numba.typed import Dict
    if issubclass(val, Dict):
        return types.TypeRef(types.DictType)

    from numba.typed import List
    if issubclass(val, List):
        return types.TypeRef(types.ListType)

    if c.purpose == Purpose.argument:
        # We only return for argument contexts.
        # For situations like x = int(y), we want typeof(int) to return None,
        # so that Context.resolve_value_type calls Context._get_global_type.

        if val in (int, float, complex):
            return typeof_impl(val(0), c)

        if val is str:
            return typeof_impl(str("numba"), c)

# type(py_typing.List) is different in Python 3.6 vs. 3.7+.
@typeof_impl.register(type(py_typing.List))
def _typeof_typing(val, c):
    if issubclass(val, py_typing.List):
        (element_py,) = val.__args__
        element_nb = typeof_impl(element_py, c)
        if element_nb is None:
            raise ValueError(
                f"Cannot type list element type {element_py}")
        return types.List(element_nb)

    if issubclass(val, py_typing.Dict):
        key_py, value_py = val.__args__
        key_nb = typeof_impl(key_py, c)
        value_nb = typeof_impl(value_py, c)
        if key_nb is None:
            raise ValueError(f"Cannot type dict key type {key_py}")
        if value_nb is None:
            raise ValueError(f"Cannot type dict value type {value_py}")
        return types.DictType(key_nb, value_nb)

    if issubclass(val, py_typing.Set):
        (element_py,) = val.__args__
        element_nb = typeof_impl(element_py, c)
        if element_nb is None:
            raise ValueError(
                f"Cannot type set element type {element_py}")
        return types.Set(element_nb)

    if issubclass(val, py_typing.Tuple):
        tys = tuple(typeof_impl(elem, c) for elem in val.__args__)
        if any(ty is None for ty in tys):
            return
        return types.BaseTuple.from_types(tys)

@typeof_impl.register(type(py_typing.Optional))
def _typeof_optional(val, c):
    (arg_1_py, arg_2_py) = val.__args__
    if arg_2_py is not type(None): # noqa: E721
        raise ValueError(
            "Cannot type Union that is not an Optional "
            f"(second type {arg_2_py} is not NoneType")
    arg_1_nb = typeof_impl(arg_1_py, c)
    if arg_1_nb is None:
        raise ValueError(f"Cannot type optional inner type {arg_1_py}")
    return types.Optional(arg_1_nb)

@typeof_impl.register(bool)
def _typeof_bool(val, c):
    return types.boolean

@typeof_impl.register(float)
def _typeof_float(val, c):
    return types.float64

@typeof_impl.register(complex)
def _typeof_complex(val, c):
    return types.complex128

def _typeof_int(val, c):
    # As in _typeof.c
    nbits = utils.bit_length(val)
    if nbits < 32:
        typ = types.intp
    elif nbits < 64:
        typ = types.int64
    elif nbits == 64 and val >= 0:
        typ = types.uint64
    else:
        raise ValueError("Int value is too large: %s" % val)
    return typ

for cls in utils.INT_TYPES:
    typeof_impl.register(cls, _typeof_int)

@typeof_impl.register(np.generic)
def _typeof_numpy_scalar(val, c):
    try:
        return numpy_support.map_arrayscalar_type(val)
    except NotImplementedError:
        pass

@typeof_impl.register(str)
def _typeof_str(val, c):
    return types.string

@typeof_impl.register(type((lambda a: a).__code__))
def _typeof_code(val, c):
    return types.code_type

@typeof_impl.register(type(None))
def _typeof_none(val, c):
    return types.none

@typeof_impl.register(type(Ellipsis))
def _typeof_ellipsis(val, c):
    return types.ellipsis

@typeof_impl.register(tuple)
def _typeof_tuple(val, c):
    tys = [typeof_impl(v, c) for v in val]
    if any(ty is None for ty in tys):
        return
    return types.BaseTuple.from_types(tys, type(val))

@typeof_impl.register(list)
def _typeof_list(val, c):
    if len(val) == 0:
        raise ValueError("Cannot type empty list")
    ty = typeof_impl(val[0], c)
    if ty is None:
        raise ValueError(
            f"Cannot type list element type {type(val[0])}")
    return types.List(ty, reflected=True)

@typeof_impl.register(set)
def _typeof_set(val, c):
    if len(val) == 0:
        raise ValueError("Cannot type empty set")
    item = next(iter(val))
    ty = typeof_impl(item, c)
    if ty is None:
        raise ValueError(
            f"Cannot type set element type {type(item)}")
    return types.Set(ty, reflected=True)

@typeof_impl.register(slice)
def _typeof_slice(val, c):
    return types.slice2_type if val.step in (None, 1) else types.slice3_type

@typeof_impl.register(enum.Enum)
@typeof_impl.register(enum.IntEnum)
def _typeof_enum(val, c):
    clsty = typeof_impl(type(val), c)
    return clsty.member_type

@typeof_impl.register(enum.EnumMeta)
def _typeof_enum_class(val, c):
    cls = val
    members = list(cls.__members__.values())
    if len(members) == 0:
        raise ValueError("Cannot type enum with no members")
    dtypes = {typeof_impl(mem.value, c) for mem in members}
    if len(dtypes) > 1:
        raise ValueError("Cannot type heterogeneous enum: "
                         "got value types %s"
                         % ", ".join(sorted(str(ty) for ty in dtypes)))
    if issubclass(val, enum.IntEnum):
        typecls = types.IntEnumClass
    else:
        typecls = types.EnumClass
    return typecls(cls, dtypes.pop())

@typeof_impl.register(np.dtype)
def _typeof_dtype(val, c):
    tp = numpy_support.from_dtype(val)
    return types.DType(tp)

@typeof_impl.register(np.ndarray)
def _typeof_ndarray(val, c):
    try:
        dtype = numpy_support.from_dtype(val.dtype)
    except NotImplementedError:
        raise ValueError("Unsupported array dtype: %s" % (val.dtype,))
    layout = numpy_support.map_layout(val)
    readonly = not val.flags.writeable
    return types.Array(dtype, val.ndim, layout, readonly=readonly)

@typeof_impl.register(types.NumberClass)
def _typeof_number_class(val, c):
    return val

@typeof_impl.register(types.Literal)
def _typeof_literal(val, c):
    return val

@typeof_impl.register(types.TypeRef)
def _typeof_typeref(val, c):
    return val

@typeof_impl.register(types.Type)
def _typeof_nb_type(val, c):
    if isinstance(val, types.BaseFunction):
        return val
    elif isinstance(val, (types.Number, types.Boolean)):
        return types.NumberClass(val)
    else:
        return types.TypeRef(val)
<|endoftext|>


<commit_before>
# missing Numarray defined names (in from numarray import *)
##__all__ = ['ArrayType', 'CLIP', 'ClassicUnpickler', 'Complex32_fromtype',
##           'Complex64_fromtype', 'ComplexArray', 'EarlyEOFError', 'Error',
##           'FileSeekWarning', 'MAX_ALIGN', 'MAX_INT_SIZE', 'MAX_LINE_WIDTH',
##           'MathDomainError', 'NDArray', 'NewArray', 'NewAxis', 'NumArray',
##           'NumError', 'NumOverflowError', 'PRECISION', 'Py2NumType',
##           'PyINT_TYPES', 'PyLevel2Type', 'PyNUMERIC_TYPES', 'PyREAL_TYPES',
##           'RAISE', 'SLOPPY', 'STRICT', 'SUPPRESS_SMALL', 'SizeMismatchError',
##           'SizeMismatchWarning', 'SuitableBuffer', 'USING_BLAS',
##           'UnderflowError', 'UsesOpPriority', 'WARN', 'WRAP', 'all',
##           'allclose', 'alltrue', 'and_', 'any', 'arange', 'argmax',
##           'argmin', 'argsort', 'around', 'array2list', 'array_equal',
##           'array_equiv', 'array_repr', 'array_str', 'arrayprint',
##           'arrayrange', 'average', 'choose', 'clip',
##           'codegenerator', 'compress', 'concatenate', 'conjugate',
##           'copy', 'copy_reg', 'diagonal', 'divide_remainder',
##           'dotblas', 'e', 'explicit_type', 'flush_caches', 'fromfile',
##           'fromfunction', 'fromlist', 'fromstring', 'generic',
##           'genericCoercions', 'genericPromotionExclusions', 'genericTypeRank',
##           'getShape', 'getTypeObject', 'handleError', 'identity', 'indices',
##           'info', 'innerproduct', 'inputarray', 'isBigEndian',
##           'kroneckerproduct', 'lexsort', 'libnumarray', 'libnumeric',
##           'load', 'make_ufuncs', 'math', 'memory',
##           'numarrayall', 'numarraycore', 'numerictypes', 'numinclude',
##           'operator', 'os', 'outerproduct', 'pi', 'put', 'putmask',
##           'pythonTypeMap', 'pythonTypeRank', 'rank', 'repeat',
##           'reshape', 'resize', 'round', 'safethread', 'save', 'scalarTypeMap',
##           'scalarTypes', 'searchsorted', 'session', 'shape', 'sign', 'size',
##           'sometrue', 'sort', 'swapaxes', 'sys', 'take', 'tcode',
##           'tensormultiply', 'tname', 'trace', 'transpose', 'typeDict',
##           'typecode', 'typecodes', 'typeconv', 'types', 'ufunc',
##           'ufuncFactory', 'value', ]

__all__ = ['asarray', 'ones', 'zeros', 'array', 'where']
__all__ += ['vdot', 'dot', 'matrixmultiply', 'ravel', 'indices',
            'arange', 'concatenate']

from numpy import dot as matrixmultiply, dot, vdot, ravel

def array(sequence=None, typecode=None, copy=1, savespace=0,
          type=None, shape=None, dtype=None):
    dtype = type2dtype(typecode, type, dtype)
    if sequence is None:
        if shape is None:
            return None
        if dtype is None:
            dtype = 'l'
        return N.empty(shape, dtype)
    arr = N.array(sequence, dtype, copy=copy)
    if shape is not None:
        arr.shape = shape
    return arr

def asarray(seq, type=None, typecode=None, dtype=None):
    if seq is None:
        return None
    dtype = type2dtype(typecode, type, dtype)
    return N.array(seq, dtype, copy=0)

def ones(shape, type=None, typecode=None, dtype=None):
    dtype = type2dtype(typecode, type, dtype)
    return N.ones(shape, dtype)

def zeros(shape, type=None, typecode=None, dtype=None):
    dtype = type2dtype(typecode, type, dtype)
    return N.zeros(shape, dtype)

def where(condition, x=None, y=None, out=None):
    if x is None and y is None:
        arr = N.where(condition)
    else:
        arr = N.where(condition, x, y)
    if out is not None:
        out[...] = arr
        return out
    return arr

def indices(shape, type=None):
    return N.indices(shape, type)

def arange(a1, a2=None, stride=1, type=None, shape=None,
           typecode=None, dtype=None):
    dtype = type2dtype(typecode, type, dtype)
    return N.arange(a1, a2, stride, dtype)
<commit_msg>Fix import problem<commit_after>
# missing Numarray defined names (in from numarray import *)
##__all__ = ['ArrayType', 'CLIP', 'ClassicUnpickler', 'Complex32_fromtype',
##           'Complex64_fromtype', 'ComplexArray', 'EarlyEOFError', 'Error',
##           'FileSeekWarning', 'MAX_ALIGN', 'MAX_INT_SIZE', 'MAX_LINE_WIDTH',
##           'MathDomainError', 'NDArray', 'NewArray', 'NewAxis', 'NumArray',
##           'NumError', 'NumOverflowError', 'PRECISION', 'Py2NumType',
##           'PyINT_TYPES', 'PyLevel2Type', 'PyNUMERIC_TYPES', 'PyREAL_TYPES',
##           'RAISE', 'SLOPPY', 'STRICT', 'SUPPRESS_SMALL', 'SizeMismatchError',
##           'SizeMismatchWarning', 'SuitableBuffer', 'USING_BLAS',
##           'UnderflowError', 'UsesOpPriority', 'WARN', 'WRAP', 'all',
##           'allclose', 'alltrue', 'and_', 'any', 'arange', 'argmax',
##           'argmin', 'argsort', 'around', 'array2list', 'array_equal',
##           'array_equiv', 'array_repr', 'array_str', 'arrayprint',
##           'arrayrange', 'average', 'choose', 'clip',
##           'codegenerator', 'compress', 'concatenate', 'conjugate',
##           'copy', 'copy_reg', 'diagonal', 'divide_remainder',
##           'dotblas', 'e', 'explicit_type', 'flush_caches', 'fromfile',
##           'fromfunction', 'fromlist', 'fromstring', 'generic',
##           'genericCoercions', 'genericPromotionExclusions', 'genericTypeRank',
##           'getShape', 'getTypeObject', 'handleError', 'identity', 'indices',
##           'info', 'innerproduct', 'inputarray', 'isBigEndian',
##           'kroneckerproduct', 'lexsort', 'libnumarray', 'libnumeric',
##           'load', 'make_ufuncs', 'math', 'memory',
##           'numarrayall', 'numarraycore', 'numerictypes', 'numinclude',
##           'operator', 'os', 'outerproduct', 'pi', 'put', 'putmask',
##           'pythonTypeMap', 'pythonTypeRank', 'rank', 'repeat',
##           'reshape', 'resize', 'round', 'safethread', 'save', 'scalarTypeMap',
##           'scalarTypes', 'searchsorted', 'session', 'shape', 'sign', 'size',
##           'sometrue', 'sort', 'swapaxes', 'sys', 'take', 'tcode',
##           'tensormultiply', 'tname', 'trace', 'transpose', 'typeDict',
##           'typecode', 'typecodes', 'typeconv', 'types', 'ufunc',
##           'ufuncFactory', 'value', ]

__all__ = ['asarray', 'ones', 'zeros', 'array', 'where']
__all__ += ['vdot', 'dot', 'matrixmultiply', 'ravel', 'indices',
            'arange', 'concatenate']

from numpy import dot as matrixmultiply, dot, vdot, ravel, concatenate

def array(sequence=None, typecode=None, copy=1, savespace=0,
          type=None, shape=None, dtype=None):
    dtype = type2dtype(typecode, type, dtype)
    if sequence is None:
        if shape is None:
            return None
        if dtype is None:
            dtype = 'l'
        return N.empty(shape, dtype)
    arr = N.array(sequence, dtype, copy=copy)
    if shape is not None:
        arr.shape = shape
    return arr

def asarray(seq, type=None, typecode=None, dtype=None):
    if seq is None:
        return None
    dtype = type2dtype(typecode, type, dtype)
    return N.array(seq, dtype, copy=0)

def ones(shape, type=None, typecode=None, dtype=None):
    dtype = type2dtype(typecode, type, dtype)
    return N.ones(shape, dtype)

def zeros(shape, type=None, typecode=None, dtype=None):
    dtype = type2dtype(typecode, type, dtype)
    return N.zeros(shape, dtype)

def where(condition, x=None, y=None, out=None):
    if x is None and y is None:
        arr = N.where(condition)
    else:
        arr = N.where(condition, x, y)
    if out is not None:
        out[...] = arr
        return out
    return arr

def indices(shape, type=None):
    return N.indices(shape, type)

def arange(a1, a2=None, stride=1, type=None, shape=None,
           typecode=None, dtype=None):
    dtype = type2dtype(typecode, type, dtype)
    return N.arange(a1, a2, stride, dtype)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# Copyright (c) 2012, AT&T Labs, Yun Mao <yunmao@gmail.com>
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""pylint error checking."""

import cStringIO as StringIO
import json
import re
import sys

from pylint import lint
from pylint.reporters import text

# Note(maoy): E1103 is error code related to partial type inference
ignore_codes = ["E1103"]
# Note(maoy): the error message is the pattern of E0202. It should be ignored
# for nova.tests modules
ignore_messages = ["An attribute affected in nova.tests"]
# Note(maoy): we ignore all errors in openstack.common because it should be
# checked elsewhere. We also ignore nova.tests for now due to high false
# positive rate.
ignore_modules = ["nova/openstack/common/", "nova/tests/"]

KNOWN_PYLINT_EXCEPTIONS_FILE = "tools/pylint_exceptions"

class LintOutput(object):

    _cached_filename = None
    _cached_content = None

    def __init__(self, filename, lineno, line_content, code, message,
                 lintoutput):
        self.filename = filename
        self.lineno = lineno
        self.line_content = line_content
        self.code = code
        self.message = message
        self.lintoutput = lintoutput

    @classmethod
    def from_line(cls, line):
        m = re.search(r"(\S+):(\d+): \[(\S+)(, \S+)?] (.*)", line)
        matched = m.groups()
        filename, lineno, code, message = (matched[0], int(matched[1]),
                                           matched[2], matched[-1])
        if cls._cached_filename != filename:
            with open(filename) as f:
                cls._cached_content = list(f.readlines())
                cls._cached_filename = filename
        line_content = cls._cached_content[lineno - 1].rstrip()
        return cls(filename, lineno, line_content, code, message,
                   line.rstrip())

    @classmethod
    def from_msg_to_dict(cls, msg):
        """From the output of pylint msg, to a dict, where each key
        is a unique error identifier, value is a list of LintOutput
        """
        result = {}
        for line in msg.splitlines():
            obj = cls.from_line(line)
            if obj.is_ignored():
                continue
            key = obj.key()
            if key not in result:
                result[key] = []
            result[key].append(obj)
        return result

    def is_ignored(self):
        if self.code in ignore_codes:
            return True
        if any(self.filename.startswith(name) for name in ignore_modules):
            return True
        if any(msg in self.message for msg in ignore_messages):
            return True
        return False

    def key(self):
        if self.code in ["E1101", "E1103"]:
            # These two types of errors are like Foo class has no member bar.
            # We discard the source code so that the error will be ignored
            # next time another Foo.bar is encountered.
            return self.message, ""
        return self.message, self.line_content.strip()

    def json(self):
        return json.dumps(self.__dict__)

    def review_str(self):
        return ("File %(filename)s\nLine %(lineno)d:%(line_content)s\n"
                "%(code)s: %(message)s" % self.__dict__)

class ErrorKeys(object):

    @classmethod
    def print_json(cls, errors, output=sys.stdout):
        print >>output, "# automatically generated by tools/lintstack.py"
        for i in sorted(errors.keys()):
            print >>output, json.dumps(i)

    @classmethod
    def from_file(cls, filename):
        keys = set()
        for line in open(filename):
            if line and line[0] != "#":
                d = json.loads(line)
                keys.add(tuple(d))
        return keys

def run_pylint():
    buff = StringIO.StringIO()
    reporter = text.ParseableTextReporter(output=buff)
    args = ["--include-ids=y", "-E", "nova"]
    lint.Run(args, reporter=reporter, exit=False)
    val = buff.getvalue()
    buff.close()
    return val

def generate_error_keys(msg=None):
    print "Generating", KNOWN_PYLINT_EXCEPTIONS_FILE
    if msg is None:
        msg = run_pylint()
    errors = LintOutput.from_msg_to_dict(msg)
    with open(KNOWN_PYLINT_EXCEPTIONS_FILE, "w") as f:
        ErrorKeys.print_json(errors, output=f)

def validate(newmsg=None):
    print "Loading", KNOWN_PYLINT_EXCEPTIONS_FILE
    known = ErrorKeys.from_file(KNOWN_PYLINT_EXCEPTIONS_FILE)
    if newmsg is None:
        print "Running pylint. Be patient..."
        newmsg = run_pylint()
    errors = LintOutput.from_msg_to_dict(newmsg)

    print "Unique errors reported by pylint: was %d, now %d." \
           % (len(known), len(errors))
    passed = True
    for err_key, err_list in errors.items():
        for err in err_list:
            if err_key not in known:
                print err.lintoutput
                print
                passed = False
    if passed:
        print "Congrats! pylint check passed."
        redundant = known - set(errors.keys())
        if redundant:
            print "Extra credit: some known pylint exceptions disappeared."
            for i in sorted(redundant):
                print json.dumps(i)
            print "Consider regenerating the exception file if you will."
    else:
        print ("Please fix the errors above. If you believe they are false"
              " positives, run 'tools/lintstack.py generate' to overwrite.")
        sys.exit(1)

def usage():
    print """Usage: tools/lintstack.py [generate|validate]
    To generate pylint_exceptions file: tools/lintstack.py generate
    To validate the current commit: tools/lintstack.py
    """

def main():
    option = "validate"
    if len(sys.argv) > 1:
        option = sys.argv[1]
    if option == "generate":
        generate_error_keys()
    elif option == "validate":
        validate()
    else:
        usage()

if __name__ == "__main__":
    main()
<commit_msg>Remove white space between print ()<commit_after>#!/usr/bin/env python

# Copyright (c) 2012, AT&T Labs, Yun Mao <yunmao@gmail.com>
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

"""pylint error checking."""

import cStringIO as StringIO
import json
import re
import sys

from pylint import lint
from pylint.reporters import text

# Note(maoy): E1103 is error code related to partial type inference
ignore_codes = ["E1103"]
# Note(maoy): the error message is the pattern of E0202. It should be ignored
# for nova.tests modules
ignore_messages = ["An attribute affected in nova.tests"]
# Note(maoy): we ignore all errors in openstack.common because it should be
# checked elsewhere. We also ignore nova.tests for now due to high false
# positive rate.
ignore_modules = ["nova/openstack/common/", "nova/tests/"]

KNOWN_PYLINT_EXCEPTIONS_FILE = "tools/pylint_exceptions"

class LintOutput(object):

    _cached_filename = None
    _cached_content = None

    def __init__(self, filename, lineno, line_content, code, message,
                 lintoutput):
        self.filename = filename
        self.lineno = lineno
        self.line_content = line_content
        self.code = code
        self.message = message
        self.lintoutput = lintoutput

    @classmethod
    def from_line(cls, line):
        m = re.search(r"(\S+):(\d+): \[(\S+)(, \S+)?] (.*)", line)
        matched = m.groups()
        filename, lineno, code, message = (matched[0], int(matched[1]),
                                           matched[2], matched[-1])
        if cls._cached_filename != filename:
            with open(filename) as f:
                cls._cached_content = list(f.readlines())
                cls._cached_filename = filename
        line_content = cls._cached_content[lineno - 1].rstrip()
        return cls(filename, lineno, line_content, code, message,
                   line.rstrip())

    @classmethod
    def from_msg_to_dict(cls, msg):
        """From the output of pylint msg, to a dict, where each key
        is a unique error identifier, value is a list of LintOutput
        """
        result = {}
        for line in msg.splitlines():
            obj = cls.from_line(line)
            if obj.is_ignored():
                continue
            key = obj.key()
            if key not in result:
                result[key] = []
            result[key].append(obj)
        return result

    def is_ignored(self):
        if self.code in ignore_codes:
            return True
        if any(self.filename.startswith(name) for name in ignore_modules):
            return True
        if any(msg in self.message for msg in ignore_messages):
            return True
        return False

    def key(self):
        if self.code in ["E1101", "E1103"]:
            # These two types of errors are like Foo class has no member bar.
            # We discard the source code so that the error will be ignored
            # next time another Foo.bar is encountered.
            return self.message, ""
        return self.message, self.line_content.strip()

    def json(self):
        return json.dumps(self.__dict__)

    def review_str(self):
        return ("File %(filename)s\nLine %(lineno)d:%(line_content)s\n"
                "%(code)s: %(message)s" % self.__dict__)

class ErrorKeys(object):

    @classmethod
    def print_json(cls, errors, output=sys.stdout):
        print >>output, "# automatically generated by tools/lintstack.py"
        for i in sorted(errors.keys()):
            print >>output, json.dumps(i)

    @classmethod
    def from_file(cls, filename):
        keys = set()
        for line in open(filename):
            if line and line[0] != "#":
                d = json.loads(line)
                keys.add(tuple(d))
        return keys

def run_pylint():
    buff = StringIO.StringIO()
    reporter = text.ParseableTextReporter(output=buff)
    args = ["--include-ids=y", "-E", "nova"]
    lint.Run(args, reporter=reporter, exit=False)
    val = buff.getvalue()
    buff.close()
    return val

def generate_error_keys(msg=None):
    print "Generating", KNOWN_PYLINT_EXCEPTIONS_FILE
    if msg is None:
        msg = run_pylint()
    errors = LintOutput.from_msg_to_dict(msg)
    with open(KNOWN_PYLINT_EXCEPTIONS_FILE, "w") as f:
        ErrorKeys.print_json(errors, output=f)

def validate(newmsg=None):
    print "Loading", KNOWN_PYLINT_EXCEPTIONS_FILE
    known = ErrorKeys.from_file(KNOWN_PYLINT_EXCEPTIONS_FILE)
    if newmsg is None:
        print "Running pylint. Be patient..."
        newmsg = run_pylint()
    errors = LintOutput.from_msg_to_dict(newmsg)

    print "Unique errors reported by pylint: was %d, now %d." \
           % (len(known), len(errors))
    passed = True
    for err_key, err_list in errors.items():
        for err in err_list:
            if err_key not in known:
                print err.lintoutput
                print
                passed = False
    if passed:
        print "Congrats! pylint check passed."
        redundant = known - set(errors.keys())
        if redundant:
            print "Extra credit: some known pylint exceptions disappeared."
            for i in sorted(redundant):
                print json.dumps(i)
            print "Consider regenerating the exception file if you will."
    else:
        print("Please fix the errors above. If you believe they are false"
              " positives, run 'tools/lintstack.py generate' to overwrite.")
        sys.exit(1)

def usage():
    print """Usage: tools/lintstack.py [generate|validate]
    To generate pylint_exceptions file: tools/lintstack.py generate
    To validate the current commit: tools/lintstack.py
    """

def main():
    option = "validate"
    if len(sys.argv) > 1:
        option = sys.argv[1]
    if option == "generate":
        generate_error_keys()
    elif option == "validate":
        validate()
    else:
        usage()

if __name__ == "__main__":
    main()
<|endoftext|>