<commit_before>"""Knowledge about cryptographic mechanisms implemented in Mbed TLS.

This module is entirely based on the PSA API.
"""

# Copyright The Mbed TLS Contributors
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import re
from typing import List, Optional, Tuple

class KeyType:
    """Knowledge about a PSA key type."""

    def __init__(self, name: str, params: Optional[List[str]] = None):
        """Analyze a key type.

        The key type must be specified in PSA syntax. In its simplest form,
        this is a string 'PSA_KEY_TYPE_xxx' which is the name of a PSA key
        type macro. For key types that take arguments, the arguments can
        be passed either through the optional argument `params` or by
        passing an expression of the form 'PSA_KEY_TYPE_xxx(param1, param2)'
        as the a string.
        """
        self.name = name.strip()
        if params is None:
            if '(' in self.name:
                m = re.match(r'(\w+)\s*\((.*)\)\Z', self.name)
                assert m is not None
                self.name = m.group(1)
                params = ','.split(m.group(2))
        if params is None:
            self.params = params
        else:
            self.params = [param.strip() for param in params]
        self.expression = self.name
        if self.params is not None:
            self.expression += '(' + ', '.join(self.params) + ')'
        self.private_type = re.sub(r'_PUBLIC_KEY\Z', r'_KEY_PAIR', self.name)

    ECC_KEY_SIZES = {
        'PSA_ECC_FAMILY_SECP_K1': (192, 224, 256),
        'PSA_ECC_FAMILY_SECP_R1': (192, 225, 256, 384, 521),
        'PSA_ECC_FAMILY_SECP_R2': (160,),
        'PSA_ECC_FAMILY_SECT_K1': (163, 233, 239, 283, 409, 571),
        'PSA_ECC_FAMILY_SECT_R1': (163, 233, 283, 409, 571),
        'PSA_ECC_FAMILY_SECT_R2': (163,),
        'PSA_ECC_FAMILY_BRAINPOOL_P_R1': (160, 192, 224, 256, 320, 384, 512),
        'PSA_ECC_FAMILY_MONTGOMERY': (255, 448),
    }
    KEY_TYPE_SIZES = {
        'PSA_KEY_TYPE_AES': (128, 192, 256), # exhaustive
        'PSA_KEY_TYPE_ARC4': (8, 128, 2048), # extremes + sensible
        'PSA_KEY_TYPE_ARIA': (128, 192, 256), # exhaustive
        'PSA_KEY_TYPE_CAMELLIA': (128, 192, 256), # exhaustive
        'PSA_KEY_TYPE_CHACHA20': (256,), # exhaustive
        'PSA_KEY_TYPE_DERIVE': (120, 128), # sample
        'PSA_KEY_TYPE_DES': (64, 128, 192), # exhaustive
        'PSA_KEY_TYPE_HMAC': (128, 160, 224, 256, 384, 512), # standard size for each supported hash
        'PSA_KEY_TYPE_RAW_DATA': (8, 40, 128), # sample
        'PSA_KEY_TYPE_RSA_KEY_PAIR': (1024, 1536), # small sample
    }
    def sizes_to_test(self) -> Tuple[int, ...]:
        """Return a tuple of key sizes to test.

        For key types that only allow a single size, or only a small set of
        sizes, these are all the possible sizes. For key types that allow a
        wide range of sizes, these are a representative sample of sizes,
        excluding large sizes for which a typical resource-constrained platform
        may run out of memory.
        """
        if self.private_type == 'PSA_KEY_TYPE_ECC_KEY_PAIR':
            assert self.params is not None
            return self.ECC_KEY_SIZES[self.params[0]]
        return self.KEY_TYPE_SIZES[self.private_type]
<commit_msg>Create sample key material for symmetric keys<commit_after>"""Knowledge about cryptographic mechanisms implemented in Mbed TLS.

This module is entirely based on the PSA API.
"""

# Copyright The Mbed TLS Contributors
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import re
from typing import List, Optional, Tuple

class KeyType:
    """Knowledge about a PSA key type."""

    def __init__(self, name: str, params: Optional[List[str]] = None):
        """Analyze a key type.

        The key type must be specified in PSA syntax. In its simplest form,
        this is a string 'PSA_KEY_TYPE_xxx' which is the name of a PSA key
        type macro. For key types that take arguments, the arguments can
        be passed either through the optional argument `params` or by
        passing an expression of the form 'PSA_KEY_TYPE_xxx(param1, param2)'
        as the a string.
        """
        self.name = name.strip()
        if params is None:
            if '(' in self.name:
                m = re.match(r'(\w+)\s*\((.*)\)\Z', self.name)
                assert m is not None
                self.name = m.group(1)
                params = ','.split(m.group(2))
        if params is None:
            self.params = params
        else:
            self.params = [param.strip() for param in params]
        self.expression = self.name
        if self.params is not None:
            self.expression += '(' + ', '.join(self.params) + ')'
        self.private_type = re.sub(r'_PUBLIC_KEY\Z', r'_KEY_PAIR', self.name)

    ECC_KEY_SIZES = {
        'PSA_ECC_FAMILY_SECP_K1': (192, 224, 256),
        'PSA_ECC_FAMILY_SECP_R1': (192, 225, 256, 384, 521),
        'PSA_ECC_FAMILY_SECP_R2': (160,),
        'PSA_ECC_FAMILY_SECT_K1': (163, 233, 239, 283, 409, 571),
        'PSA_ECC_FAMILY_SECT_R1': (163, 233, 283, 409, 571),
        'PSA_ECC_FAMILY_SECT_R2': (163,),
        'PSA_ECC_FAMILY_BRAINPOOL_P_R1': (160, 192, 224, 256, 320, 384, 512),
        'PSA_ECC_FAMILY_MONTGOMERY': (255, 448),
    }
    KEY_TYPE_SIZES = {
        'PSA_KEY_TYPE_AES': (128, 192, 256), # exhaustive
        'PSA_KEY_TYPE_ARC4': (8, 128, 2048), # extremes + sensible
        'PSA_KEY_TYPE_ARIA': (128, 192, 256), # exhaustive
        'PSA_KEY_TYPE_CAMELLIA': (128, 192, 256), # exhaustive
        'PSA_KEY_TYPE_CHACHA20': (256,), # exhaustive
        'PSA_KEY_TYPE_DERIVE': (120, 128), # sample
        'PSA_KEY_TYPE_DES': (64, 128, 192), # exhaustive
        'PSA_KEY_TYPE_HMAC': (128, 160, 224, 256, 384, 512), # standard size for each supported hash
        'PSA_KEY_TYPE_RAW_DATA': (8, 40, 128), # sample
        'PSA_KEY_TYPE_RSA_KEY_PAIR': (1024, 1536), # small sample
    }
    def sizes_to_test(self) -> Tuple[int, ...]:
        """Return a tuple of key sizes to test.

        For key types that only allow a single size, or only a small set of
        sizes, these are all the possible sizes. For key types that allow a
        wide range of sizes, these are a representative sample of sizes,
        excluding large sizes for which a typical resource-constrained platform
        may run out of memory.
        """
        if self.private_type == 'PSA_KEY_TYPE_ECC_KEY_PAIR':
            assert self.params is not None
            return self.ECC_KEY_SIZES[self.params[0]]
        return self.KEY_TYPE_SIZES[self.private_type]

    # "48657265006973206b6579a064617461"
    DATA_BLOCK = b'Here\000is key\240data'
    def key_material(self, bits: int) -> bytes:
        """Return a byte string containing suitable key material with the given bit length.

        Use the PSA export representation. The resulting byte string is one that
        can be obtained with the following code:
        ```
        psa_set_key_type(&attributes, `self.expression`);
        psa_set_key_bits(&attributes, `bits`);
        psa_set_key_usage_flags(&attributes, PSA_KEY_USAGE_EXPORT);
        psa_generate_key(&attributes, &id);
        psa_export_key(id, `material`, ...);
        ```
        """
        if bits % 8 != 0:
            raise ValueError('Non-integer number of bytes: {} bits'.format(bits))
        length = bits // 8
        if self.name == 'PSA_KEY_TYPE_DES':
            # "644573206b457901644573206b457902644573206b457904"
            des3 = b'dEs kEy\001dEs kEy\002dEs kEy\004'
            return des3[:length]
        # TODO: ECC, RSA
        return b''.join([self.DATA_BLOCK] * (length // len(self.DATA_BLOCK)) +
                        [self.DATA_BLOCK[:length % len(self.DATA_BLOCK)]])
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from __future__ import division

import numpy as np
from nose.tools import assert_raises

from .. import flow

def test_qn():
    """flow vectors q_n"""

    # This is _almost_ unnecessary.  It would be tough to screw up q_n.

    # q_n(0) = 1
    q = flow.qn(2, 0)
    assert q == 1., \
        'Incorrect single-particle q_n ({} != 1).'.format(q)

    # q_3(isotropic phi) = -1
    q = flow.qn(3, (0, np.pi/3, -np.pi/3))
    assert q == -1., \
        'Incorrect isotropic q_3 ({} != -1).'.format(q)

def test_flow_cumulant():
    """flow correlations and cumulants"""

    Nev = 10
    M = 100

    # Fabricate q_2 for perfectly fluctuating "flow" (no actual flow).
    qn = {2: M**.5*np.ones(Nev)}
    vnk = flow.FlowCumulant(M*np.ones(Nev), qn)

    corr22 = vnk.correlation(2, 2)
    assert corr22 == 0., \
        'Purely statistical correlation must vanish ({} != 0).'.format(corr22)

    c22 = vnk.cumulant(2, 2)
    assert c22 == 0., \
        'Purely statistical cumulant must vanish ({} != 0).'.format(c22)

    v22 = vnk.flow(2, 2)
    assert v22 == 0., \
        'Purely statistical flow must vanish ({} != 0).'.format(v22)

    assert_raises(ValueError, vnk.correlation, 2, 4)
<commit_msg>Significantly improve test coverage and rigor.<commit_after># -*- coding: utf-8 -*-

from __future__ import division

import itertools
import warnings

import numpy as np
from nose.tools import assert_raises, nottest

from .. import flow

def test_qn():
    """flow vectors q_n"""

    # This is _almost_ unnecessary.  It would be tough to screw up q_n.

    # q_n(0) = 1
    q = flow.qn(2, 0)
    assert q == 1., \
        'Incorrect single-particle q_n.\n{} != 1'.format(q)

    # q_3(isotropic phi) = -1
    q = flow.qn(3, (0, np.pi/3, -np.pi/3))
    assert q == -1., \
        'Incorrect isotropic q_3.\n{} != -1'.format(q)

@nottest
def assert_close(a, b, msg='', tol=1e-15):
    assert abs(a - b) < tol, \
        '{}\n{} != {}'.format(msg, a, b)

def test_flow_cumulant():
    """flow correlations and cumulants"""

    # Compare n-particle correlations to explicitly calculating all
    # n-permutations.
    M = 8
    phi = 2*np.pi*np.random.rand(8)

    corr2_true = np.exp(2j*np.array([
        (p1-p2) for p1, p2 in itertools.permutations(phi, 2)
    ])).mean()
    corr4_true = np.exp(2j*np.array([
        (p1+p2-p3-p4) for p1, p2, p3, p4 in itertools.permutations(phi, 4)
    ])).mean()

    qn = {n: np.exp(1j*n*phi).sum() for n in (2, 4)}
    vnk = flow.FlowCumulant(M, qn)
    corr2, corr4 = (vnk.correlation(2, n) for n in (2, 4))

    assert_close(corr2, corr2_true, 'Incorrect 2-particle correlation.')
    assert_close(corr4, corr4_true, 'Incorrect 4-particle correlation.')

    # Now verify cumulants.
    c2_true = corr2_true
    c4_true = corr4_true - 2*corr2_true*corr2_true

    c2, c4 = (vnk.cumulant(2, n) for n in (2, 4))

    assert_close(c2, c2_true, 'Incorrect 2-particle cumulant.')
    assert_close(c4, c4_true, 'Incorrect 4-particle cumulant.')

    # Fabricate q_n so that v2 is positive, v3 vanishes, and v4 is imaginary.
    Nev = 10
    M = 100
    qfluct = M**.5*np.ones(Nev)
    qn = {2: 1.1*qfluct, 3: qfluct, 4: 0.9*qfluct}
    vnk = flow.FlowCumulant(M*np.ones(Nev), qn)

    v22 = vnk.flow(2, 2)
    assert v22 > 0, \
        'This v_2{2} must be positive.\n{}'.format(v22)

    v32 = vnk.flow(3, 2)
    assert v32 == 0, \
        'This v_3{2} must vanish.\n{}'.format(v32)

    with warnings.catch_warnings(record=True) as w:
        v42_nan = vnk.flow(4, 2)
        assert len(w) == 1 and issubclass(w[0].category, RuntimeWarning), \
            'There should be RuntimeWarning here.'
    assert np.isnan(v42_nan), \
        'This v_4{2} must be NaN.\n{}'.format(v42_nan)

    v42_zero = vnk.flow(4, 2, imaginary='zero')
    assert v42_zero == 0, \
        'This v_4{2} must be zero.\n{}'.format(v42_zero)

    v42_negative = vnk.flow(4, 2, imaginary='negative')
    assert v42_negative < 0, \
        'This v_4{2} must be negative.\n{}'.format(v42_negative)

    # Test bad arguments.

    # invalid k
    assert_raises(ValueError, vnk.correlation, 2, 3)

    # missing qn
    assert_raises(ValueError, vnk.cumulant, 2, 6)
<|endoftext|>


<commit_before>import struct
import logging
import array

logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)-8s] (%(threadName)-10s) %(message)s")

cmd_modes = {
    0: 'positive edge',
    1: 'negative edge',
    2: 'Manchester Code IEEE 802.3',
    3: 'Manchester Code G.E. Thomas'
}

trigger_modes = {
    0: 'external trigger',
    1: 'TLU no handshake',
    2: 'TLU simple handshake',
    3: 'TLU trigger data handshake'
}

class ReadoutUtils(object):
    def __init__(self, device):
        self.device = device
        self.rx_base_address = dict([(idx, addr) for idx, addr in enumerate(range(0x8600, 0x8200, -0x0100))])

    def configure_rx_fsm(self, channels=None, invert_rx_data=False, **kwargs):
        '''Setting up RX FSM.

        Parameters
        ----------
        channels : list, tuple
            List of readout channels to which the setting is applied.
        invert_rx_data : bool
            Enables inverting RX data. This can be used to compensate swapped n and p LVDS signal lines.
        '''
        if invert_rx_data:
            logging.info('Inverting of RX data enabled')
        reg = 0
        if invert_rx_data:
            reg |= 0x02
        else:
            reg &= ~0x02
        if channels == None:
            channels = self.rx_base_address.iterkeys()
        filter(lambda i: self.device.WriteExternal(address=self.rx_base_address[i] + 2, data=[reg]), channels)  # overwriting selected register

    def configure_command_fsm(self, enable_ext_trigger=False, cmd_mode=0, diable_clock=False, disable_command_trigger=False, **kwargs):
        '''Setting up command FSM to receive external triggers.

        Parameters
        ----------
        enable_ext_trigger : bool
            Enabling external trigger and TLU trigger.
        cmd_mode : bool
            Changing CMD output mode. From 0 to 3.
            0: positive edge (default)
            1: negative edge
            2: Manchester Code IEEE 802.3 (for capacitively coupled Din)
            3: Manchester Code G.E. Thomas
        diable_clock : bool
            Disabling FE clock.
        disable_command_trigger : bool
            Disabling command trigger. Command trigger sends pulse to LEMO TX1 when sending command to FE. Sending pulses over LEMO TX1 only when enable_ext_trigger is set to false.
        '''
        if cmd_mode != 0:
            logging.info('Command mode: %s' % cmd_modes[cmd_mode])
        logging.info('External trigger %s' % ('enabled' if enable_ext_trigger else 'disabled'))
#         array = self.device.ReadExternal(address=0 + 2, size=1)  # get stored register value
#         reg = struct.unpack('B', array)[0]
        reg = 0
        if enable_ext_trigger:
            reg |= 0x01
        else:
            reg &= ~0x01
        reg = ((cmd_mode & 0x03) << 1) | (reg & 0xf9)
        if diable_clock:
            reg |= 0x08
        else:
            reg &= ~0x08
        if disable_command_trigger:
            reg |= 0x10
        else:
            reg &= ~0x10
        self.device.WriteExternal(address=0 + 2, data=[reg])  # overwriting register

    def configure_trigger_fsm(self, trigger_mode=0, trigger_data_msb_first=False, disable_veto=False, trigger_data_delay=0, trigger_clock_cycles=16, enable_reset=False, invert_lemo_trigger_input=False, write_timestamp=False, trigger_low_timeout=0, reset_trigger_counter=False, **kwargs):
        '''Setting up external trigger mode and TLU trigger FSM.

        Parameters
        ----------
        trigger_mode : string
            TLU handshake mode. External trigger has to be enabled in command FSM. From 0 to 3.
            0: External trigger (LEMO RX0 only, TLU port disabled (TLU port/RJ45)).
            1: TLU no handshake (automatic detection of TLU connection (TLU port/RJ45)).
            2: TLU simple handshake (automatic detection of TLU connection (TLU port/RJ45)).
            3: TLU trigger data handshake (automatic detection of TLU connection (TLU port/RJ45)).
        trigger_data_msb_first : bool
            Setting endianness of TLU trigger data.
        disable_veto : bool
            Disabling TLU veto support.
        trigger_data_delay : int
            Addition wait cycles before latching TLU trigger data. From 0 to 15.
        trigger_clock_cycles : int
            Number of clock cycles sent to TLU to clock out TLU trigger data (only in TLU trigger data handshake mode). The number of clock cycles is usually (bit length of TLU trigger data + 1). From 0 to 31 (0 = 32 clock cycles).
        enable_reset : bool
            Enable resetting of internal trigger counter when TLU asserts reset signal.
        invert_lemo_trigger_input : bool
            Enable inverting of LEMO RX0 trigger input.
        write_timestamp : bool
            Writing time stamp to SRAM every trigger (not available in TLU trigger data handshake mode).
        trigger_low_timeout : int
            Enabling timeout for waiting for de-asserting TLU trigger signal. From 0 to 255.
        reset_trigger_counter : bool
            Reset trigger counter to zero.
        '''
        logging.info('Trigger mode: %s' % trigger_modes[trigger_mode])
#         array = self.device.ReadExternal(address = 0x8200+1, size = 3)  # get stored register value
#         reg = struct.unpack(4*'B', array)
        reg_1 = (trigger_mode & 0x03)
        if trigger_data_msb_first:
            reg_1 |= 0x04
        else:
            reg_1 &= ~0x04
        if disable_veto:
            reg_1 |= 0x08
        else:
            reg_1 &= ~0x08
        reg_1 = ((trigger_data_delay & 0x0f) << 4) | (reg_1 & 0x0f)
        reg_2 = (trigger_clock_cycles & 0x1F)  # 0 = 32 clock cycles
        if enable_reset:
            reg_2 |= 0x20
        else:
            reg_2 &= ~0x20
        if invert_lemo_trigger_input:
            reg_2 |= 0x40
        else:
            reg_2 &= ~0x40
        if write_timestamp:
            reg_2 |= 0x80
        else:
            reg_2 &= ~0x80
        reg_3 = trigger_low_timeout
        if reset_trigger_counter:
            self.set_trigger_number(value=0)
        self.device.WriteExternal(address=0x8200 + 1, data=[reg_1, reg_2, reg_3])  # overwriting registers
#         if not force_use_rj45:
#             array = self.device.ReadExternal(address=0x8200 + 2, size=1)  # get stored register value
#             reg = struct.unpack('B', array)
#             if reg[0] & 0x80:
#                 logging.info('TLU detected on RJ45 port')
#         else:
#             logging.info('Using RJ45 port for triggering')

    def configure_tdc_fsm(self, enable_tdc=False, enable_tdc_arming=False, **kwargs):
        '''Setting up TDC (time-to-digital converter) FSM.

        Parameters
        ----------
        enable_tdc : bool
            Enables TDC. TDC will measure signal at RX0 (LEMO trigger input).
        reject_small_tot : bool
            If true rejecting signals shorter than 25ns (40MHz).
        enable_tdc_arming : bool
            Enables arming of TDC. TDC will only measure a signal when triggered (command is sent out).
        '''
#         array = self.device.ReadExternal(address=0x8700 + 1, size=1)  # get stored register value
#         reg = struct.unpack('B', array)[0]
        reg = 0
        if enable_tdc:
            reg |= 0x01
        else:
            reg &= ~0x01
        if enable_tdc_arming:
            reg |= 0x02
        else:
            reg &= ~0x02
        self.device.WriteExternal(address=0x8700 + 1, data=[reg])

    def get_tlu_trigger_number(self):
        '''Reading most recent TLU trigger data/number.
        '''
        trigger_number_array = self.device.ReadExternal(address=0x8200 + 4, size=4)
        return struct.unpack('I', trigger_number_array)[0]

    def get_trigger_number(self):
        '''Reading internal trigger counter.
        '''
        trigger_number_array = self.device.ReadExternal(address=0x8200 + 8, size=4)
        return struct.unpack('I', trigger_number_array)[0]

    def set_trigger_number(self, value=0):
        '''Reading internal trigger counter.
        '''
        trigger_number = array.array('B', struct.pack('I', value))
        self.device.WriteExternal(address=0x8200 + 8, data=trigger_number)
        read_value = self.get_trigger_number()
        if read_value != value:
            logging.warning('Trigger counter is not %d (read %d)' % (value, read_value))
<commit_msg>MAINT: commend variable type fix<commit_after>import struct
import logging
import array

logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)-8s] (%(threadName)-10s) %(message)s")

cmd_modes = {
    0: 'positive edge',
    1: 'negative edge',
    2: 'Manchester Code IEEE 802.3',
    3: 'Manchester Code G.E. Thomas'
}

trigger_modes = {
    0: 'external trigger',
    1: 'TLU no handshake',
    2: 'TLU simple handshake',
    3: 'TLU trigger data handshake'
}

class ReadoutUtils(object):
    def __init__(self, device):
        self.device = device
        self.rx_base_address = dict([(idx, addr) for idx, addr in enumerate(range(0x8600, 0x8200, -0x0100))])

    def configure_rx_fsm(self, channels=None, invert_rx_data=False, **kwargs):
        '''Setting up RX FSM.

        Parameters
        ----------
        channels : list, tuple
            List of readout channels to which the setting is applied.
        invert_rx_data : bool
            Enables inverting RX data. This can be used to compensate swapped n and p LVDS signal lines.
        '''
        if invert_rx_data:
            logging.info('Inverting of RX data enabled')
        reg = 0
        if invert_rx_data:
            reg |= 0x02
        else:
            reg &= ~0x02
        if channels == None:
            channels = self.rx_base_address.iterkeys()
        filter(lambda i: self.device.WriteExternal(address=self.rx_base_address[i] + 2, data=[reg]), channels)  # overwriting selected register

    def configure_command_fsm(self, enable_ext_trigger=False, cmd_mode=0, diable_clock=False, disable_command_trigger=False, **kwargs):
        '''Setting up command FSM to receive external triggers.

        Parameters
        ----------
        enable_ext_trigger : bool
            Enabling external trigger and TLU trigger.
        cmd_mode : bool
            Changing CMD output mode. From 0 to 3.
            0: positive edge (default)
            1: negative edge
            2: Manchester Code IEEE 802.3 (for capacitively coupled Din)
            3: Manchester Code G.E. Thomas
        diable_clock : bool
            Disabling FE clock.
        disable_command_trigger : bool
            Disabling command trigger. Command trigger sends pulse to LEMO TX1 when sending command to FE. Sending pulses over LEMO TX1 only when enable_ext_trigger is set to false.
        '''
        if cmd_mode != 0:
            logging.info('Command mode: %s' % cmd_modes[cmd_mode])
        logging.info('External trigger %s' % ('enabled' if enable_ext_trigger else 'disabled'))
#         array = self.device.ReadExternal(address=0 + 2, size=1)  # get stored register value
#         reg = struct.unpack('B', array)[0]
        reg = 0
        if enable_ext_trigger:
            reg |= 0x01
        else:
            reg &= ~0x01
        reg = ((cmd_mode & 0x03) << 1) | (reg & 0xf9)
        if diable_clock:
            reg |= 0x08
        else:
            reg &= ~0x08
        if disable_command_trigger:
            reg |= 0x10
        else:
            reg &= ~0x10
        self.device.WriteExternal(address=0 + 2, data=[reg])  # overwriting register

    def configure_trigger_fsm(self, trigger_mode=0, trigger_data_msb_first=False, disable_veto=False, trigger_data_delay=0, trigger_clock_cycles=16, enable_reset=False, invert_lemo_trigger_input=False, write_timestamp=False, trigger_low_timeout=0, reset_trigger_counter=False, **kwargs):
        '''Setting up external trigger mode and TLU trigger FSM.

        Parameters
        ----------
        trigger_mode : int
            TLU handshake mode. External trigger has to be enabled in command FSM. From 0 to 3.
            0: External trigger (LEMO RX0 only, TLU port disabled (TLU port/RJ45)).
            1: TLU no handshake (automatic detection of TLU connection (TLU port/RJ45)).
            2: TLU simple handshake (automatic detection of TLU connection (TLU port/RJ45)).
            3: TLU trigger data handshake (automatic detection of TLU connection (TLU port/RJ45)).
        trigger_data_msb_first : bool
            Setting endianness of TLU trigger data.
        disable_veto : bool
            Disabling TLU veto support.
        trigger_data_delay : int
            Addition wait cycles before latching TLU trigger data. From 0 to 15.
        trigger_clock_cycles : int
            Number of clock cycles sent to TLU to clock out TLU trigger data (only in TLU trigger data handshake mode). The number of clock cycles is usually (bit length of TLU trigger data + 1). From 0 to 31 (0 = 32 clock cycles).
        enable_reset : bool
            Enable resetting of internal trigger counter when TLU asserts reset signal.
        invert_lemo_trigger_input : bool
            Enable inverting of LEMO RX0 trigger input.
        write_timestamp : bool
            Writing time stamp to SRAM every trigger (not available in TLU trigger data handshake mode).
        trigger_low_timeout : int
            Enabling timeout for waiting for de-asserting TLU trigger signal. From 0 to 255.
        reset_trigger_counter : bool
            Reset trigger counter to zero.
        '''
        logging.info('Trigger mode: %s' % trigger_modes[trigger_mode])
#         array = self.device.ReadExternal(address = 0x8200+1, size = 3)  # get stored register value
#         reg = struct.unpack(4*'B', array)
        reg_1 = (trigger_mode & 0x03)
        if trigger_data_msb_first:
            reg_1 |= 0x04
        else:
            reg_1 &= ~0x04
        if disable_veto:
            reg_1 |= 0x08
        else:
            reg_1 &= ~0x08
        reg_1 = ((trigger_data_delay & 0x0f) << 4) | (reg_1 & 0x0f)
        reg_2 = (trigger_clock_cycles & 0x1F)  # 0 = 32 clock cycles
        if enable_reset:
            reg_2 |= 0x20
        else:
            reg_2 &= ~0x20
        if invert_lemo_trigger_input:
            reg_2 |= 0x40
        else:
            reg_2 &= ~0x40
        if write_timestamp:
            reg_2 |= 0x80
        else:
            reg_2 &= ~0x80
        reg_3 = trigger_low_timeout
        if reset_trigger_counter:
            self.set_trigger_number(value=0)
        self.device.WriteExternal(address=0x8200 + 1, data=[reg_1, reg_2, reg_3])  # overwriting registers
#         if not force_use_rj45:
#             array = self.device.ReadExternal(address=0x8200 + 2, size=1)  # get stored register value
#             reg = struct.unpack('B', array)
#             if reg[0] & 0x80:
#                 logging.info('TLU detected on RJ45 port')
#         else:
#             logging.info('Using RJ45 port for triggering')

    def configure_tdc_fsm(self, enable_tdc=False, enable_tdc_arming=False, **kwargs):
        '''Setting up TDC (time-to-digital converter) FSM.

        Parameters
        ----------
        enable_tdc : bool
            Enables TDC. TDC will measure signal at RX0 (LEMO trigger input).
        reject_small_tot : bool
            If true rejecting signals shorter than 25ns (40MHz).
        enable_tdc_arming : bool
            Enables arming of TDC. TDC will only measure a signal when triggered (command is sent out).
        '''
#         array = self.device.ReadExternal(address=0x8700 + 1, size=1)  # get stored register value
#         reg = struct.unpack('B', array)[0]
        reg = 0
        if enable_tdc:
            reg |= 0x01
        else:
            reg &= ~0x01
        if enable_tdc_arming:
            reg |= 0x02
        else:
            reg &= ~0x02
        self.device.WriteExternal(address=0x8700 + 1, data=[reg])

    def get_tlu_trigger_number(self):
        '''Reading most recent TLU trigger data/number.
        '''
        trigger_number_array = self.device.ReadExternal(address=0x8200 + 4, size=4)
        return struct.unpack('I', trigger_number_array)[0]

    def get_trigger_number(self):
        '''Reading internal trigger counter.
        '''
        trigger_number_array = self.device.ReadExternal(address=0x8200 + 8, size=4)
        return struct.unpack('I', trigger_number_array)[0]

    def set_trigger_number(self, value=0):
        '''Reading internal trigger counter.
        '''
        trigger_number = array.array('B', struct.pack('I', value))
        self.device.WriteExternal(address=0x8200 + 8, data=trigger_number)
        read_value = self.get_trigger_number()
        if read_value != value:
            logging.warning('Trigger counter is not %d (read %d)' % (value, read_value))
<|endoftext|>


<commit_before>#! /usr/bin/env python3

from mll_calc.mll_calc import *

import numpy as np
import pandas as pd

def ll_calc(x):
    # where x = y_sim - y_mes
    ll = -0.5 * (x**2 + np.log(2 * np.pi))
    return ll

# dummy test (don't need to test single line scipy func, I think)
def test_like_calc():
    y_sim = pd.Series([1, 1, 1, 1])
    y_mes = pd.Series([1, 1, 1, 1])
    std = pd.Series([1, 1, 1, 1])
    exp = (1 / np.sqrt(2 * np.pi)) ** len(y_sim)
    obs = like_calc(y_sim, y_mes, std)
    assert obs == exp

def test_ratios():
    ratio_list = ['A/B', 'B/A']
    labels = ['label']
    XY = pd.DataFrame({'A' : [1., 2., 3., np.nan],
                       'B' : [1., 1., 0., 0],
                       'label' : [1, 1, 1, 1]})
    exp = pd.DataFrame({'A/B' : [1., 2., 0., 0.],
                        'B/A' : [1., 0.5, 0, 0],
                        'label' : [1, 1, 1, 1]})
    obs = ratios(XY, ratio_list, labels)
    assert obs.equals(exp)

def test_get_pred():
    XY = pd.DataFrame({'feature' : [1, 3],
                       'label' : ['Y', 'Z']},
                       index = [1, 2])
    test_sample = pd.Series([0], index=['feature'])
    unc = 1
    lbls = ['label']
    ll_name = 'LogLikelihood_' + str(unc)
    exp_1 = pd.DataFrame({'pred_label' : ['Y'],
                          ll_name : [ll_calc(1)]},
                          index = [1])
    exp_2 = ['pred_label', ll_name]
    obs_1, obs_2 = get_pred(XY, test_sample, unc, lbls)
    assert obs_1.equals(exp_1)
    assert obs_2 == exp_2

def test_mll_testset_XY():
    XY = pd.DataFrame({'feature' : [1., 2., 4.],
                       'label' : ['X', 'Y', 'Z']},
                       index = [0, 1, 2])
    test = XY.copy()
    unc = 1
    lbls = ['label']
    ll_name = 'LogLikelihood_' + str(unc)
    exp_1 = pd.DataFrame({'sim_idx' : [0, 1, 2],
                          'label' : ['X', 'Y', 'Z'],
                          'pred_idx' : [1, 0, 1],
                          'pred_label' : ['Y', 'X', 'Y'],
                          ll_name : [ll_calc(1), ll_calc(1), ll_calc(2)]},
                          index = [0, 1, 2])
    exp_2 = ['pred_label', ll_name]
    obs_1, obs_2 = mll_testset(XY, test, unc, lbls)
    assert obs_1.equals(exp_1)
    assert obs_2 == exp_2

#def test_mll_testset_ext():
#
#    XY = pd.DataFrame({'A' : [1., 2., 3., np.nan],
#                       'B' : [1., 1., 0., 0],
#                       'label' : [1, 1, 1, 1]})
#    test = pd.DataFrame({'A' : [1., 2., 3., np.nan],
#                         'B' : [1., 1., 0., 0],
#                         'label' : [1, 1, 1, 1]})
#    unc = 1
#    lbls = ['label']
#    exp_1 = pd.DataFrame({'A' : [1., 2., 3., np.nan],
#                          'B' : [1., 1., 0., 0],
#                          'label' : [1, 1, 1, 1]})
#    exp_2 = ['Pred_label']
#    obs_1, obs_2 = mll_testset(XY, test, unc, lbls)
#    assert obs_1.equals(exp_1)
#    assert obs_2 == exp_2

# def test_mll_testset_drop_replace():
# need to test that the db is in fact stating the same (not slowly getting deleted)
<commit_msg>added coverage notes, and trying to figure out why get_pred test runs but numbers don't calculate as expected with same data in mll_testset<commit_after>#! /usr/bin/env python3

from mll_calc.mll_calc import *

import numpy as np
import pandas as pd

def ll_calc(x):
    # where x = y_sim - y_mes
    ll = -0.5 * (x**2 + np.log(2 * np.pi))
    return ll

# dummy test (don't need to test single line scipy func, I think)
def test_like_calc():
    y_sim = pd.Series([1, 1, 1, 1])
    y_mes = pd.Series([1, 1, 1, 1])
    std = pd.Series([1, 1, 1, 1])
    exp = (1 / np.sqrt(2 * np.pi)) ** len(y_sim)
    obs = like_calc(y_sim, y_mes, std)
    assert obs == exp

def test_ratios():
    ratio_list = ['A/B', 'B/A']
    labels = ['label']
    XY = pd.DataFrame({'A' : [1., 2., 3., np.nan],
                       'B' : [1., 1., 0., 0],
                       'label' : [1, 1, 1, 1]})
    exp = pd.DataFrame({'A/B' : [1., 2., 0., 0.],
                        'B/A' : [1., 0.5, 0, 0],
                        'label' : [1, 1, 1, 1]})
    obs = ratios(XY, ratio_list, labels)
    assert obs.equals(exp)

def test_get_pred():
    XY = pd.DataFrame({'feature' : [1, 3],
                       'label' : ['Y', 'Z']},
                       index = [1, 2])
    test_sample = pd.Series([0], index=['feature'])
    unc = 1
    lbls = ['label']
    ll_name = 'LogLikelihood_' + str(unc)
    exp_1 = pd.DataFrame({'pred_label' : ['Y'],
                          ll_name : [ll_calc(1)]},
                          index = [1])
    exp_2 = ['pred_label', ll_name]
    obs_1, obs_2 = get_pred(XY, test_sample, unc, lbls)
    assert obs_1.equals(exp_1)
    assert obs_2 == exp_2

def test_mll_testset_XY():
    XY = pd.DataFrame({'feature' : [0, 1, 3],
                       'label' : ['X', 'Y', 'Z']},
                       index = [0, 1, 2])
    test = XY.copy()
    unc = 1
    lbls = ['label']
    ll_name = 'LogLikelihood_' + str(unc)
    exp_1 = pd.DataFrame({'sim_idx' : [0, 1, 2],
                          'label' : ['X', 'Y', 'Z'],
                          'pred_idx' : [1, 0, 1],
                          'pred_label' : ['Y', 'X', 'Y'],
                          ll_name : [ll_calc(1), ll_calc(1), ll_calc(2)]},
                          index = [0, 1, 2])
    exp_2 = ['pred_label', ll_name]
    obs_1, obs_2 = mll_testset(XY, test, unc, lbls)
    assert obs_1.equals(exp_1)
    assert obs_2 == exp_2

#def test_mll_testset_ext():
#
#    XY = pd.DataFrame({'A' : [1., 2., 3., np.nan],
#                       'B' : [1., 1., 0., 0],
#                       'label' : [1, 1, 1, 1]})
#    test = pd.DataFrame({'A' : [1., 2., 3., np.nan],
#                         'B' : [1., 1., 0., 0],
#                         'label' : [1, 1, 1, 1]})
#    unc = 1
#    lbls = ['label']
#    exp_1 = pd.DataFrame({'A' : [1., 2., 3., np.nan],
#                          'B' : [1., 1., 0., 0],
#                          'label' : [1, 1, 1, 1]})
#    exp_2 = ['Pred_label']
#    obs_1, obs_2 = mll_testset(XY, test, unc, lbls)
#    assert obs_1.equals(exp_1)
#    assert obs_2 == exp_2

# def test_mll_testset_drop_replace():
# need to test that the db is in fact stating the same (not slowly getting deleted)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function, division, absolute_import
import warnings
import numpy as np
from numba.decorators import jit
from numba.targets.registry import target_registry
from numba.targets.descriptors import TargetDescriptor
from numba.targets.options import TargetOptions
from numba import utils, compiler, types, sigutils
from . import _internal
from .sigparse import parse_signature
from .wrappers import build_ufunc_wrapper, build_gufunc_wrapper
from numba.targets import registry

class UFuncTargetOptions(TargetOptions):
    OPTIONS = {
        "nopython" : bool,
        "forceobj" : bool,
    }

class UFuncTarget(registry.CPUTarget):
    options = UFuncTargetOptions

class UFuncDispatcher(object):
    targetdescr = UFuncTarget()

    def __init__(self, py_func, locals={}, targetoptions={}):
        self.py_func = py_func
        self.overloads = utils.UniqueDict()
        self.targetoptions = targetoptions
        self.locals = locals

    def compile(self, sig, locals={}, **targetoptions):
        locs = self.locals.copy()
        locs.update(locals)

        topt = self.targetoptions.copy()
        topt.update(targetoptions)

        flags = compiler.Flags()
        self.targetdescr.options.parse_as_flags(flags, topt)
        flags.set("no_compile")
        # Disable loop lifting
        # The feature requires a real python function
        flags.unset("enable_looplift")

        typingctx = self.targetdescr.typing_context
        targetctx = self.targetdescr.target_context

        args, return_type = sigutils.normalize_signature(sig)

        cres = compiler.compile_extra(typingctx, targetctx, self.py_func,
                                      args=args, return_type=return_type,
                                      flags=flags, locals=locals)

        self.overloads[cres.signature] = cres
        return cres

target_registry['npyufunc'] = UFuncDispatcher

class UFuncBuilder(object):
    def __init__(self, py_func, targetoptions={}):
        self.py_func = py_func
        self.nb_func = jit(target='npyufunc', **targetoptions)(py_func)

    def add(self, sig=None, argtypes=None, restype=None):
        # Handle argtypes
        if argtypes is not None:
            warnings.warn("Keyword argument argtypes is deprecated",
                          DeprecationWarning)
            assert sig is None
            if restype is None:
                sig = tuple(argtypes)
            else:
                sig = restype(*argtypes)
        # Do compilation
        # Return CompileResult to test
        return self.nb_func.compile(sig)

    def build_ufunc(self):
        dtypelist = []
        ptrlist = []
        if not self.nb_func:
            raise TypeError("No definition")
        for sig, cres in self.nb_func.overloads.items():
            dtypenums, ptr = self.build(cres)
            dtypelist.append(dtypenums)
            ptrlist.append(utils.longint(ptr))
        datlist = [None] * len(ptrlist)

        inct = len(cres.signature.args)
        outct = 1

        # Becareful that fromfunc does not provide full error checking yet.
        # If typenum is out-of-bound, we have nasty memory corruptions.
        # For instance, -1 for typenum will cause segfault.
        # If elements of type-list (2nd arg) is tuple instead,
        # there will also memory corruption. (Seems like code rewrite.)
        ufunc = _internal.fromfunc(ptrlist, dtypelist, inct, outct, datlist)

        return ufunc

    def build(self, cres):
        # Buider wrapper for ufunc entry point
        ctx = cres.target_context
        signature = cres.signature
        wrapper = build_ufunc_wrapper(ctx, cres.llvm_func, signature)
        ctx.finalize(wrapper, cres.fndesc)
        ctx.engine.add_module(wrapper.module)
        ptr = ctx.engine.get_pointer_to_function(wrapper)
        # Get dtypes
        dtypenums = [np.dtype(a.name).num for a in signature.args]
        dtypenums.append(np.dtype(signature.return_type.name).num)
        return dtypenums, ptr

class GUFuncBuilder(object):
    # TODO handle scalar
    def __init__(self, py_func, signature, targetoptions={}):
        self.py_func = py_func
        self.nb_func = jit(target='npyufunc')(py_func)
        self.signature = signature
        self.sin, self.sout = parse_signature(signature)
        self.targetoptions = targetoptions
        self._envs = []

    def add(self, sig=None, argtypes=None, restype=None):
        # Handle argtypes
        if argtypes is not None:
            warnings.warn("Keyword argument argtypes is deprecated",
                          DeprecationWarning)
            assert sig is None
            if restype is None:
                sig = tuple(argtypes)
            else:
                sig = restype(*argtypes)
        # Actual work begins
        cres = self.nb_func.compile(sig, **self.targetoptions)

        if not cres.objectmode and cres.signature.return_type != types.void:
            raise TypeError("gufunc kernel must have void return type")

        return cres

    def build_ufunc(self):
        dtypelist = []
        ptrlist = []
        if not self.nb_func:
            raise TypeError("No definition")

        for sig, cres in self.nb_func.overloads.items():
            dtypenums, ptr, env = self.build(cres)
            self._envs.append(env)   # keep env object alive
            dtypelist.append(dtypenums)
            ptrlist.append(utils.longint(ptr))
        datlist = [None] * len(ptrlist)

        inct = len(self.sin)
        outct = len(self.sout)

        ufunc = _internal.fromfuncsig(ptrlist, dtypelist, inct, outct, datlist,
                                      self.signature)

        return ufunc

    def build(self, cres):
        """
        Returns (dtype numbers, function ptr, EnviornmentObject)
        """
        # Buider wrapper for ufunc entry point
        ctx = cres.target_context
        signature = cres.signature
        wrapper, env = build_gufunc_wrapper(ctx, cres.llvm_func, signature,
                                            self.sin, self.sout,
                                            fndesc=cres.fndesc)

        ctx.finalize(wrapper, cres.fndesc)
        ctx.engine.add_module(wrapper.module)
        ptr = ctx.engine.get_pointer_to_function(wrapper)
        # Get dtypes
        dtypenums = []
        for a in signature.args:
            if isinstance(a, types.Array):
                ty = a.dtype
            else:
                ty = a
            dtypenums.append(np.dtype(ty.name).num)
        return dtypenums, ptr, env

<commit_msg>Fix env keepalive<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function, division, absolute_import
import warnings
import numpy as np
from numba.decorators import jit
from numba.targets.registry import target_registry
from numba.targets.descriptors import TargetDescriptor
from numba.targets.options import TargetOptions
from numba import utils, compiler, types, sigutils
from . import _internal
from .sigparse import parse_signature
from .wrappers import build_ufunc_wrapper, build_gufunc_wrapper
from numba.targets import registry

class UFuncTargetOptions(TargetOptions):
    OPTIONS = {
        "nopython" : bool,
        "forceobj" : bool,
    }

class UFuncTarget(registry.CPUTarget):
    options = UFuncTargetOptions

class UFuncDispatcher(object):
    targetdescr = UFuncTarget()

    def __init__(self, py_func, locals={}, targetoptions={}):
        self.py_func = py_func
        self.overloads = utils.UniqueDict()
        self.targetoptions = targetoptions
        self.locals = locals

    def compile(self, sig, locals={}, **targetoptions):
        locs = self.locals.copy()
        locs.update(locals)

        topt = self.targetoptions.copy()
        topt.update(targetoptions)

        flags = compiler.Flags()
        self.targetdescr.options.parse_as_flags(flags, topt)
        flags.set("no_compile")
        # Disable loop lifting
        # The feature requires a real python function
        flags.unset("enable_looplift")

        typingctx = self.targetdescr.typing_context
        targetctx = self.targetdescr.target_context

        args, return_type = sigutils.normalize_signature(sig)

        cres = compiler.compile_extra(typingctx, targetctx, self.py_func,
                                      args=args, return_type=return_type,
                                      flags=flags, locals=locals)

        self.overloads[cres.signature] = cres
        return cres

target_registry['npyufunc'] = UFuncDispatcher

class UFuncBuilder(object):
    def __init__(self, py_func, targetoptions={}):
        self.py_func = py_func
        self.nb_func = jit(target='npyufunc', **targetoptions)(py_func)

    def add(self, sig=None, argtypes=None, restype=None):
        # Handle argtypes
        if argtypes is not None:
            warnings.warn("Keyword argument argtypes is deprecated",
                          DeprecationWarning)
            assert sig is None
            if restype is None:
                sig = tuple(argtypes)
            else:
                sig = restype(*argtypes)
        # Do compilation
        # Return CompileResult to test
        return self.nb_func.compile(sig)

    def build_ufunc(self):
        dtypelist = []
        ptrlist = []
        if not self.nb_func:
            raise TypeError("No definition")
        for sig, cres in self.nb_func.overloads.items():
            dtypenums, ptr = self.build(cres)
            dtypelist.append(dtypenums)
            ptrlist.append(utils.longint(ptr))
        datlist = [None] * len(ptrlist)

        inct = len(cres.signature.args)
        outct = 1

        # Becareful that fromfunc does not provide full error checking yet.
        # If typenum is out-of-bound, we have nasty memory corruptions.
        # For instance, -1 for typenum will cause segfault.
        # If elements of type-list (2nd arg) is tuple instead,
        # there will also memory corruption. (Seems like code rewrite.)
        ufunc = _internal.fromfunc(ptrlist, dtypelist, inct, outct, datlist)

        return ufunc

    def build(self, cres):
        # Buider wrapper for ufunc entry point
        ctx = cres.target_context
        signature = cres.signature
        wrapper = build_ufunc_wrapper(ctx, cres.llvm_func, signature)
        ctx.finalize(wrapper, cres.fndesc)
        ctx.engine.add_module(wrapper.module)
        ptr = ctx.engine.get_pointer_to_function(wrapper)
        # Get dtypes
        dtypenums = [np.dtype(a.name).num for a in signature.args]
        dtypenums.append(np.dtype(signature.return_type.name).num)
        return dtypenums, ptr

class GUFuncBuilder(object):
    # TODO handle scalar
    def __init__(self, py_func, signature, targetoptions={}):
        self.py_func = py_func
        self.nb_func = jit(target='npyufunc')(py_func)
        self.signature = signature
        self.sin, self.sout = parse_signature(signature)
        self.targetoptions = targetoptions

    def add(self, sig=None, argtypes=None, restype=None):
        # Handle argtypes
        if argtypes is not None:
            warnings.warn("Keyword argument argtypes is deprecated",
                          DeprecationWarning)
            assert sig is None
            if restype is None:
                sig = tuple(argtypes)
            else:
                sig = restype(*argtypes)
        # Actual work begins
        cres = self.nb_func.compile(sig, **self.targetoptions)

        if not cres.objectmode and cres.signature.return_type != types.void:
            raise TypeError("gufunc kernel must have void return type")

        return cres

    def build_ufunc(self):
        dtypelist = []
        ptrlist = []
        keepalive = []

        if not self.nb_func:
            raise TypeError("No definition")

        for sig, cres in self.nb_func.overloads.items():
            dtypenums, ptr, env = self.build(cres)
            keepalive.append(env)   # keep env object alive
            dtypelist.append(dtypenums)
            ptrlist.append(utils.longint(ptr))
        datlist = [None] * len(ptrlist)

        inct = len(self.sin)
        outct = len(self.sout)

        # Pass envs to fromfuncsig to bind to the lifetime of the ufunc object
        ufunc = _internal.fromfuncsig(ptrlist, dtypelist, inct, outct, datlist,
                                      self.signature, keepalive)
        return ufunc

    def build(self, cres):
        """
        Returns (dtype numbers, function ptr, EnviornmentObject)
        """
        # Buider wrapper for ufunc entry point
        ctx = cres.target_context
        signature = cres.signature
        wrapper, env = build_gufunc_wrapper(ctx, cres.llvm_func, signature,
                                            self.sin, self.sout,
                                            fndesc=cres.fndesc)

        ctx.finalize(wrapper, cres.fndesc)
        ctx.engine.add_module(wrapper.module)
        ptr = ctx.engine.get_pointer_to_function(wrapper)
        # Get dtypes
        dtypenums = []
        for a in signature.args:
            if isinstance(a, types.Array):
                ty = a.dtype
            else:
                ty = a
            dtypenums.append(np.dtype(ty.name).num)
        return dtypenums, ptr, env

<|endoftext|>


<commit_before>from __future__ import print_function
import itertools
import numba.unittest_support as unittest
from numba.compiler import compile_isolated, Flags
from numba import types, jit
import numpy as np
import sys

def identity(x):
    return x

def addition(x, y):
    return x + y

def equality(x, y):
    return x == y

def foobar(x, y, z):
    return x

class TestConversion(unittest.TestCase):
    """
    Testing Python to Native conversion
    """
    def test_complex_identity(self):
        pyfunc = identity
        cres = compile_isolated(pyfunc, [types.complex64],
                                return_type=types.complex64)

        xs = [1.0j, (1+1j), (-1-1j), (1+0j)]
        for x in xs:
            self.assertEqual(cres.entry_point(x=x), x)
        for x in np.complex64(xs):
            self.assertEqual(cres.entry_point(x=x), x)

        cres = compile_isolated(pyfunc, [types.complex128],
                                return_type=types.complex128)

        xs = [1.0j, (1+1j), (-1-1j), (1+0j)]
        for x in xs:
            self.assertEqual(cres.entry_point(x=x), x)
        for x in np.complex128(xs):
            self.assertEqual(cres.entry_point(x=x), x)

    def test_complex_addition(self):
        pyfunc = addition
        cres = compile_isolated(pyfunc, [types.complex64, types.complex64],
                                return_type=types.complex64)

        xs = [1.0j, (1+1j), (-1-1j), (1+0j)]
        for x in xs:
            y = x
            self.assertEqual(cres.entry_point(x, y), x + y)
        for x in np.complex64(xs):
            y = x
            self.assertEqual(cres.entry_point(x, y), x + y)

        cres = compile_isolated(pyfunc, [types.complex128, types.complex128],
                                return_type=types.complex128)

        xs = [1.0j, (1+1j), (-1-1j), (1+0j)]
        for x in xs:
            y = x
            self.assertEqual(cres.entry_point(x, y), x + y)
        for x in np.complex128(xs):
            y = x
            self.assertEqual(cres.entry_point(x, y), x + y)

    def test_boolean_as_int(self):
        pyfunc = equality
        cres = compile_isolated(pyfunc, [types.boolean, types.intp])
        cfunc = cres.entry_point

        xs = True, False
        ys = -1, 0, 1

        for xs, ys in itertools.product(xs, ys):
            self.assertEqual(pyfunc(xs, ys), cfunc(xs, ys))

    def test_boolean_as_float(self):
        pyfunc = equality
        cres = compile_isolated(pyfunc, [types.boolean, types.float64])
        cfunc = cres.entry_point

        xs = True, False
        ys = -1, 0, 1

        for xs, ys in itertools.product(xs, ys):
            self.assertEqual(pyfunc(xs, ys), cfunc(xs, ys))

    def test_boolean_eq_boolean(self):
        pyfunc = equality
        cres = compile_isolated(pyfunc, [types.boolean, types.boolean])
        cfunc = cres.entry_point

        xs = True, False
        ys = True, False

        for xs, ys in itertools.product(xs, ys):
            self.assertEqual(pyfunc(xs, ys), cfunc(xs, ys))

    # test when a function parameters are jitted as unsigned types
    # when the function is called with negative parameters the Python error
    # that it generates is correctly handled -- a Python error is returned to the user
    # For more info, see the comment in Include/longobject.h for _PyArray_AsByteArray
    # which PyLong_AsUnsignedLongLong calls
    def test_negative_to_unsigned(self):
        def f(x):
            return x
        # TypeError is for 2.6
        if sys.version_info >= (2, 7):
            with self.assertRaises(OverflowError):
                jit('uintp(uintp)', nopython=True)(f)(-5)
        else:
            with self.assertRaises(TypeError):
                jit('uintp(uintp)', nopython=True)(f)(-5)

    # test that the switch logic in callwraper.py:build_wrapper() works for more than one argument
    # and when it occurs in different positions
    def test_multiple_args_negative_to_unsigned(self):
        pyfunc = foobar
        cres = compile_isolated(pyfunc, [types.uint64, types.uint64, types.uint64],
                                return_type=types.uint64)
        cfunc = cres.entry_point
        test_fail_args = ((-1, 0, 1), (0, -1, 1), (0, 1, -1))
        # TypeError is for 2.6
        if sys.version_info >= (2, 7):
            with self.assertRaises(OverflowError):
                for a, b, c in test_fail_args:
                    cfunc(a, b, c)
        else:
            with self.assertRaises(TypeError):
                for a, b, c in test_fail_args:
                    cfunc(a, b, c)

if __name__ == '__main__':
    unittest.main()
<commit_msg>refactored test for numpy records to test records and uint types as function parameters<commit_after>from __future__ import print_function
import itertools
import numba.unittest_support as unittest
from numba.compiler import compile_isolated, Flags
from numba import types, jit, numpy_support

import numpy as np
import sys

def identity(x):
    return x

def addition(x, y):
    return x + y

def equality(x, y):
    return x == y

def foobar(x, y, z):
    return x

class TestConversion(unittest.TestCase):
    """
    Testing Python to Native conversion
    """
    def test_complex_identity(self):
        pyfunc = identity
        cres = compile_isolated(pyfunc, [types.complex64],
                                return_type=types.complex64)

        xs = [1.0j, (1+1j), (-1-1j), (1+0j)]
        for x in xs:
            self.assertEqual(cres.entry_point(x=x), x)
        for x in np.complex64(xs):
            self.assertEqual(cres.entry_point(x=x), x)

        cres = compile_isolated(pyfunc, [types.complex128],
                                return_type=types.complex128)

        xs = [1.0j, (1+1j), (-1-1j), (1+0j)]
        for x in xs:
            self.assertEqual(cres.entry_point(x=x), x)
        for x in np.complex128(xs):
            self.assertEqual(cres.entry_point(x=x), x)

    def test_complex_addition(self):
        pyfunc = addition
        cres = compile_isolated(pyfunc, [types.complex64, types.complex64],
                                return_type=types.complex64)

        xs = [1.0j, (1+1j), (-1-1j), (1+0j)]
        for x in xs:
            y = x
            self.assertEqual(cres.entry_point(x, y), x + y)
        for x in np.complex64(xs):
            y = x
            self.assertEqual(cres.entry_point(x, y), x + y)

        cres = compile_isolated(pyfunc, [types.complex128, types.complex128],
                                return_type=types.complex128)

        xs = [1.0j, (1+1j), (-1-1j), (1+0j)]
        for x in xs:
            y = x
            self.assertEqual(cres.entry_point(x, y), x + y)
        for x in np.complex128(xs):
            y = x
            self.assertEqual(cres.entry_point(x, y), x + y)

    def test_boolean_as_int(self):
        pyfunc = equality
        cres = compile_isolated(pyfunc, [types.boolean, types.intp])
        cfunc = cres.entry_point

        xs = True, False
        ys = -1, 0, 1

        for xs, ys in itertools.product(xs, ys):
            self.assertEqual(pyfunc(xs, ys), cfunc(xs, ys))

    def test_boolean_as_float(self):
        pyfunc = equality
        cres = compile_isolated(pyfunc, [types.boolean, types.float64])
        cfunc = cres.entry_point

        xs = True, False
        ys = -1, 0, 1

        for xs, ys in itertools.product(xs, ys):
            self.assertEqual(pyfunc(xs, ys), cfunc(xs, ys))

    def test_boolean_eq_boolean(self):
        pyfunc = equality
        cres = compile_isolated(pyfunc, [types.boolean, types.boolean])
        cfunc = cres.entry_point

        xs = True, False
        ys = True, False

        for xs, ys in itertools.product(xs, ys):
            self.assertEqual(pyfunc(xs, ys), cfunc(xs, ys))

    # test when a function parameters are jitted as unsigned types
    # when the function is called with negative parameters the Python error
    # that it generates is correctly handled -- a Python error is returned to the user
    # For more info, see the comment in Include/longobject.h for _PyArray_AsByteArray
    # which PyLong_AsUnsignedLongLong calls
    def test_negative_to_unsigned(self):
        def f(x):
            return x
        # TypeError is for 2.6
        if sys.version_info >= (2, 7):
            with self.assertRaises(OverflowError):
                jit('uintp(uintp)', nopython=True)(f)(-5)
        else:
            with self.assertRaises(TypeError):
                jit('uintp(uintp)', nopython=True)(f)(-5)

    # test the switch logic in callwraper.py:build_wrapper() works for more than one argument
    # and where the error occurs
    def test_multiple_args_negative_to_unsigned(self):
        pyfunc = foobar
        cres = compile_isolated(pyfunc, [types.uint64, types.uint64, types.uint64],
                                return_type=types.uint64)
        cfunc = cres.entry_point
        test_fail_args = ((-1, 0, 1), (0, -1, 1), (0, 1, -1))
        # TypeError is for 2.6
        if sys.version_info >= (2, 7):
            with self.assertRaises(OverflowError):
                for a, b, c in test_fail_args:
                    cfunc(a, b, c)
        else:
            with self.assertRaises(TypeError):
                for a, b, c in test_fail_args:
                    cfunc(a, b, c)

    # test with records as function parameters
    def test_multiple_args_records(self):
        pyfunc = foobar

        mystruct_dt = np.dtype([('p', np.float64),
                           ('row', np.float64),
                           ('col', np.float64)])
        mystruct = numpy_support.from_dtype(mystruct_dt)

        cres = compile_isolated(pyfunc, [mystruct[:], types.uint64, types.uint64],
                return_type=mystruct[:])
        cfunc = cres.entry_point

        st1 = np.recarray(3, dtype=mystruct_dt)
        st2 = np.recarray(3, dtype=mystruct_dt)

        st1.p = np.arange(st1.size) + 1
        st1.row = np.arange(st1.size) + 1
        st1.col = np.arange(st1.size) + 1

        st2.p = np.arange(st2.size) + 1
        st2.row = np.arange(st2.size) + 1
        st2.col = np.arange(st2.size) + 1

        test_fail_args = ((st1, -1, st2), (st1, st2, -1))

        # TypeError is for 2.6
        if sys.version_info >= (2, 7):
            with self.assertRaises(OverflowError):
                for a, b, c in test_fail_args:
                    cfunc(a, b, c)
        else:
            with self.assertRaises(TypeError):
                for a, b, c in test_fail_args:
                    cfunc(a, b, c)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""This module provides an abstract base class, containing common methods and
attributes that all types of workers and foreman instances may use."""
from __future__ import (absolute_import, print_function, unicode_literals)

from abc import ABCMeta, abstractmethod

from contracts import contract

try:
    from botocore.exceptions import ClientError
except ImportError:
    ClientError = None
    exit('boto3 is required but not installed.')

from taran.helpers.aws import get_account_id
from taran.helpers.aws.clients import get_swf_client
from taran.helpers.aws.swf import get_activity_history
from taran.utils.host import get_hostname
import logging
import inspect
import time
import os
import json
try:
    from collections import Counter
except ImportError:
    from backport_collections import Counter

class Taran(object):
    """The base class for all processors that interact with SWF (Simple WorkFlow).

    Attributes:
        processor (unicode): The type of processor that is processing decisions or tasks.
        hostname (unicode): The name of the host the processor is running on.
        configuration (module): The workflow configuration.
        domain_name: (unicode): The domain the workflow exists in.
        swf_client: (SWF): An instance of the SWF client.
        activity_task (unicode): Name of the activity task.
    """

    __metaclass__ = ABCMeta

    @contract(configuration='*')
    @abstractmethod
    def __init__(self, configuration=None):
        """Initialise state that applies to all processors.

        Args:
            configuration (module): The workflow configuration.
        """
        self.aws_region = configuration.AWS_REGION if hasattr(configuration, 'AWS_REGION') else None
        self.swf_client = get_swf_client(region=self.aws_region)
        self.configuration = configuration
        if hasattr(configuration, 'AWS_ACCOUNT_ID'):
            if configuration.AWS_ACCOUNT_ID != get_account_id():
                exit('Your configuration prevents this running outside of AWS account: {0}'.format(
                    configuration.AWS_ACCOUNT_ID))
        self.processor = None
        self.hostname = get_hostname()
        self.workflow_history = None
        self.task_list = None
        self.identity = None
        self.workflow_id = None
        self.run_id = None
        self.activity_type_name = None
        self.domain_name = configuration.DOMAIN_NAME if hasattr(configuration, 'DOMAIN_NAME') else None
        self.workflow_name = configuration.WORKFLOW_NAME if hasattr(configuration, 'WORKFLOW_NAME') else '-'
        self.workflow_version = configuration.WORKFLOW_VERSION if hasattr(configuration, 'WORKFLOW_VERSION') else '-'
        self.activity_task = None
        self.activity_list = configuration.ACTIVITY_LIST if hasattr(configuration, 'ACTIVITY_LIST') else None
        self.decision_task = None
        self.task_token = None
        self.log_level = configuration.LOG_LEVEL if hasattr(configuration, 'LOG_LEVEL') else logging.INFO
        self.logger = self.get_logger()

    @staticmethod
    @contract(workflow_history='dict[>0]', scheduled_ids='list|None', activity_type='unicode|None')
    def get_activity_history(workflow_history=None, scheduled_ids=None, activity_type=None):
        """Get a subset of the workflow history that relates to a specific activity type and (optionally) event ids.

        Args:
            workflow_history (Dict): The entire history of the workflow.
            scheduled_ids (Optional[List]): A list of scheduled event ids to reduce the event search by.
            activity_type (unicode): The type of activity to retrieve events for.

        Returns:
            task (dict): Workflow events matching the activity type and (oprtionally) event ids.
        """
        return get_activity_history(workflow_history=workflow_history, scheduled_ids=scheduled_ids,
                                    activity_type=activity_type)

    @contract(reason='unicode', domain_name='unicode|None', details='unicode',
              child_policy='unicode|None')
    def terminate_workflow(self, reason=None, domain_name=None,
                           details=None, child_policy='TERMINATE'):
        """End the workflow as the result of completion or failure.

        Args:
            reason (unicode): The type of activity to retrieve events for.
            domain_name (unicode): The name of domain the workflow to terminate is a member of
            details (unicode): Details of the termination.
            child_policy (Optional[unicode]): How to treat child workflows. Terminate by default.
        """
        try:
            if not domain_name:
                domain_name = self.domain_name
            self.swf_client.terminate_workflow_execution(
                domain=domain_name,
                workflowId=self.workflow_id,
                runId=self.run_id,
                reason=reason,
                details=details,
                childPolicy=child_policy
            )
            self.msg(message='Termination details: {0}'.format(details))
            return True
        except ClientError as ce:
            if 'Unknown execution' in ce.response['Error']['Code']:
                self.msg(message='Workflow already terminated')
        except Exception:
            raise

    def get_workflow_input(self):
        """Get the input specified when the workflow execution was started"""
        for event in self.workflow_history.get('events'):
            if event.get('eventType') == 'WorkflowExecutionStarted':
                return json.loads(event['workflowExecutionStartedEventAttributes']['input'])

    def get_activity_status(self, activity=None):
        """Retrieve the activity history and a count of each status recorded"""
        activity_history = self.get_activity_history(workflow_history=self.workflow_history,
                                                     activity_type=activity)

        activity_list = list()
        if activity_history:
            for activity_event in activity_history:
                if activity_event.get('status') in (
                        'failed', 'timed_out', 'cancelled', 'cancel_requested', 'scheduled', 'started'):
                    activity_list.append({'status': activity_event.get('status')})
                elif activity_event.get('status') == 'completed':
                    activity_list.append(
                        {'status': activity_event.get('status'), 'result': activity_event.get('result')})
        status_counts = Counter(token['status'] for token in activity_list)
        return dict(events=activity_list, counts=status_counts)

    @contract(message='unicode|None', level='unicode|None')
    def msg(self, message='-', level='info'):
        """Accept specific attributes to help produce an output message.

         Args:
             message (unicode): The message to output.
             level (unicode): The log level to output.
         """
        workflow_id = self.workflow_id if self.workflow_id else '-'
        run_id = self.run_id if self.run_id else '-'
        output = '\"{0}\" \"{1}\" \"{2}\" {3} {4} {5}'.format(self.domain_name, self.workflow_name,
                                                              self.workflow_version, workflow_id,
                                                              run_id, message)
        if level == 'debug':
            self.logger.debug(msg=output)
        elif level == 'info':
            self.logger.info(msg=output)
        elif level == 'warning':
            self.logger.warning(msg=output)
        elif level == 'error':
            self.logger.error(msg=output)
        elif level == 'critical':
            self.logger.critical(msg=output)

    def get_logger(self):
        """Return a logger we can use to write messages to files."""
        root_module_name = inspect.stack()[-1][1].split('/')[-1].split('.')[0]
        logger = logging.getLogger(root_module_name)
        if not len(logger.handlers):
            logger.setLevel(self.log_level)
            ch = logging.StreamHandler()
            ch.setLevel(self.log_level)
            log_dir = self.configuration.LOG_DIR if hasattr(self.configuration,
                                                            'LOG_DIR') else os.getcwd() + os.path.sep
            fh = logging.FileHandler('{0}{1}.log'.format(log_dir, root_module_name))
            fh.setLevel(logging.INFO)
            formatter = logging.Formatter(fmt='%(asctime)s %(levelname)s %(message)s')
            formatter.converter = time.gmtime
            ch.setFormatter(formatter)
            fh.setFormatter(formatter)
            # Suppress verbose boto messages
            logging.getLogger("botocore.vendored.requests.packages.urllib3.connectionpool").setLevel(logging.WARN)
            logger.addHandler(ch)
            logger.addHandler(fh)
            logger.propagate = False
        return logger
<commit_msg>handle different log level scenario.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""This module provides an abstract base class, containing common methods and
attributes that all types of workers and foreman instances may use."""
from __future__ import (absolute_import, print_function, unicode_literals)

from abc import ABCMeta, abstractmethod

from contracts import contract

try:
    from botocore.exceptions import ClientError
except ImportError:
    ClientError = None
    exit('boto3 is required but not installed.')

from taran.helpers.aws import get_account_id
from taran.helpers.aws.clients import get_swf_client
from taran.helpers.aws.swf import get_activity_history
from taran.utils.host import get_hostname
import logging
import inspect
import time
import os
import json

try:
    from collections import Counter
except ImportError:
    from backport_collections import Counter

class Taran(object):
    """The base class for all processors that interact with SWF (Simple WorkFlow).

    Attributes:
        processor (unicode): The type of processor that is processing decisions or tasks.
        hostname (unicode): The name of the host the processor is running on.
        configuration (module): The workflow configuration.
        domain_name: (unicode): The domain the workflow exists in.
        swf_client: (SWF): An instance of the SWF client.
        activity_task (unicode): Name of the activity task.
    """

    __metaclass__ = ABCMeta

    @contract(configuration='*')
    @abstractmethod
    def __init__(self, configuration=None):
        """Initialise state that applies to all processors.

        Args:
            configuration (module): The workflow configuration.
        """
        self.aws_region = configuration.AWS_REGION if hasattr(configuration, 'AWS_REGION') else None
        self.swf_client = get_swf_client(region=self.aws_region)
        self.configuration = configuration
        if hasattr(configuration, 'AWS_ACCOUNT_ID'):
            if configuration.AWS_ACCOUNT_ID != get_account_id():
                exit('Your configuration prevents this running outside of AWS account: {0}'.format(
                    configuration.AWS_ACCOUNT_ID))
        self.processor = None
        self.hostname = get_hostname()
        self.workflow_history = None
        self.task_list = None
        self.identity = None
        self.workflow_id = None
        self.run_id = None
        self.activity_type_name = None
        self.domain_name = configuration.DOMAIN_NAME if hasattr(configuration, 'DOMAIN_NAME') else None
        self.workflow_name = configuration.WORKFLOW_NAME if hasattr(configuration, 'WORKFLOW_NAME') else '-'
        self.workflow_version = configuration.WORKFLOW_VERSION if hasattr(configuration, 'WORKFLOW_VERSION') else '-'
        self.activity_task = None
        self.activity_list = configuration.ACTIVITY_LIST if hasattr(configuration, 'ACTIVITY_LIST') else None
        self.decision_task = None
        self.task_token = None
        self.log_level = self.get_log_level()
        self.logger = self.get_logger()

    def get_log_level(self):
        """Return a log level depending on specified LOG_LEVEL in configuration or default to INFO"""
        levels = {'NOTSET': 0, 'DEBUG': 10, 'INFO': 20, 'WARNING': 30, 'ERROR': 40, 'CRITICAL': 50}
        config_specified = self.configuration.LOG_LEVEL if hasattr(self.configuration, 'LOG_LEVEL') else None
        if config_specified:
            if isinstance(config_specified, int) and config_specified in levels.values():
                return config_specified
            elif levels[config_specified]:
                return levels.get(config_specified)
            else:
                output = 'Invalid log level defined. Defaulting to INFO.'
        else:
            output = 'Log level not defined. Defaulting to INFO.'
        print('WARNING: {0}'.format(output))
        return 10

    @staticmethod
    @contract(workflow_history='dict[>0]', scheduled_ids='list|None', activity_type='unicode|None')
    def get_activity_history(workflow_history=None, scheduled_ids=None, activity_type=None):
        """Get a subset of the workflow history that relates to a specific activity type and (optionally) event ids.

        Args:
            workflow_history (Dict): The entire history of the workflow.
            scheduled_ids (Optional[List]): A list of scheduled event ids to reduce the event search by.
            activity_type (unicode): The type of activity to retrieve events for.

        Returns:
            task (dict): Workflow events matching the activity type and (oprtionally) event ids.
        """
        return get_activity_history(workflow_history=workflow_history, scheduled_ids=scheduled_ids,
                                    activity_type=activity_type)

    @contract(reason='unicode', domain_name='unicode|None', details='unicode',
              child_policy='unicode|None')
    def terminate_workflow(self, reason=None, domain_name=None,
                           details=None, child_policy='TERMINATE'):
        """End the workflow as the result of completion or failure.

        Args:
            reason (unicode): The type of activity to retrieve events for.
            domain_name (unicode): The name of domain the workflow to terminate is a member of
            details (unicode): Details of the termination.
            child_policy (Optional[unicode]): How to treat child workflows. Terminate by default.
        """
        try:
            if not domain_name:
                domain_name = self.domain_name
            self.swf_client.terminate_workflow_execution(
                domain=domain_name,
                workflowId=self.workflow_id,
                runId=self.run_id,
                reason=reason,
                details=details,
                childPolicy=child_policy
            )
            self.msg(message='Termination details: {0}'.format(details))
            return True
        except ClientError as ce:
            if 'Unknown execution' in ce.response['Error']['Code']:
                self.msg(message='Workflow already terminated')
        except Exception:
            raise

    def get_workflow_input(self):
        """Get the input specified when the workflow execution was started"""
        for event in self.workflow_history.get('events'):
            if event.get('eventType') == 'WorkflowExecutionStarted':
                return json.loads(event['workflowExecutionStartedEventAttributes']['input'])

    def get_activity_status(self, activity=None):
        """Retrieve the activity history and a count of each status recorded"""
        activity_history = self.get_activity_history(workflow_history=self.workflow_history,
                                                     activity_type=activity)

        activity_list = list()
        if activity_history:
            for activity_event in activity_history:
                if activity_event.get('status') in (
                        'failed', 'timed_out', 'cancelled', 'cancel_requested', 'scheduled', 'started'):
                    activity_list.append({'status': activity_event.get('status')})
                elif activity_event.get('status') == 'completed':
                    activity_list.append(
                        {'status': activity_event.get('status'), 'result': activity_event.get('result')})
        status_counts = Counter(token['status'] for token in activity_list)
        return dict(events=activity_list, counts=status_counts)

    @contract(message='unicode|None', level='unicode|None')
    def msg(self, message='-', level='info'):
        """Accept specific attributes to help produce an output message.

         Args:
             message (unicode): The message to output.
             level (unicode): The log level to output.
         """
        workflow_id = self.workflow_id if self.workflow_id else '-'
        run_id = self.run_id if self.run_id else '-'
        output = '\"{0}\" \"{1}\" \"{2}\" {3} {4} {5}'.format(self.domain_name, self.workflow_name,
                                                              self.workflow_version, workflow_id,
                                                              run_id, message)
        if level == 'debug':
            self.logger.debug(msg=output)
        elif level == 'info':
            self.logger.info(msg=output)
        elif level == 'warning':
            self.logger.warning(msg=output)
        elif level == 'error':
            self.logger.error(msg=output)
        elif level == 'critical':
            self.logger.critical(msg=output)

    def get_logger(self):
        """Return a logger we can use to write messages to files."""
        root_module_name = inspect.stack()[-1][1].split('/')[-1].split('.')[0]
        logger = logging.getLogger(root_module_name)
        if not len(logger.handlers):
            logger.setLevel(self.log_level)
            ch = logging.StreamHandler()
            ch.setLevel(self.log_level)
            log_dir = self.configuration.LOG_DIR if hasattr(self.configuration,
                                                            'LOG_DIR') else os.getcwd() + os.path.sep
            fh = logging.FileHandler('{0}{1}.log'.format(log_dir, root_module_name))
            fh.setLevel(logging.INFO)
            formatter = logging.Formatter(fmt='%(asctime)s %(levelname)s %(message)s')
            formatter.converter = time.gmtime
            ch.setFormatter(formatter)
            fh.setFormatter(formatter)
            # Suppress verbose boto messages
            logging.getLogger("botocore.vendored.requests.packages.urllib3.connectionpool").setLevel(logging.WARN)
            logger.addHandler(ch)
            logger.addHandler(fh)
            logger.propagate = False
        return logger
<|endoftext|>


<commit_before># coding: utf-8

from __future__ import unicode_literals
import re
from .Lemmatizer import Lemmatizer
from .WordTokenizer import WordTokenizer
from .utils import maketrans

compile_patterns = lambda patterns: [(re.compile(pattern), repl) for pattern, repl in patterns]

class Normalizer(object):
	def __init__(self, remove_extra_spaces=True, persian_style=True, persian_numbers=True, remove_diacritics=True, affix_spacing=True, token_based=False, punctuation_spacing=True):
		self._punctuation_spacing = punctuation_spacing
		self._affix_spacing = affix_spacing
		self._token_based = token_based

		translation_src, translation_dst = ' ىكي“”', ' یکی""'
		if persian_numbers:
			translation_src += '0123456789%'
			translation_dst += '۰۱۲۳۴۵۶۷۸۹٪'
		self.translations = maketrans(translation_src, translation_dst)

		if self._token_based:
			lemmatizer = Lemmatizer()
			self.words = lemmatizer.words
			self.verbs = lemmatizer.verbs
			self.tokenizer = WordTokenizer(join_verb_parts=False)
			self.suffixes = {'ی', 'ای', 'ها', 'های', 'تر', 'تری', 'ترین', 'گر', 'گری', 'ام', 'ات', 'اش'}

		self.character_refinement_patterns = []

		if remove_extra_spaces:
			self.character_refinement_patterns.extend([
				(r' +', ' '),  # remove extra spaces
				(r'\n\n+', '\n\n'),  # remove extra newlines
				(r'[ـ\r]', ''),  # remove keshide, carriage returns
			])

		if persian_style:
			self.character_refinement_patterns.extend([
				('"([^\n"]+)"', r'«\1»'),  # replace quotation with gyoome
				('([\d+])\.([\d+])', r'\1٫\2'),  # replace dot with momayez
				(r' ?\.\.\.', ' …'),  # replace 3 dots
			])

		if remove_diacritics:
			self.character_refinement_patterns.append(
				('[\u064B\u064C\u064D\u064E\u064F\u0650\u0651\u0652]', ''),  # remove FATHATAN, DAMMATAN, KASRATAN, FATHA, DAMMA, KASRA, SHADDA, SUKUN
			)

		self.character_refinement_patterns = compile_patterns(self.character_refinement_patterns)

		punc_after, punc_before = r'\.:!،؛؟»\]\)\}', r'«\[\(\{'
		if punctuation_spacing:
			self.punctuation_spacing_patterns = compile_patterns([
				('" ([^\n"]+) "', r'"\1"'),  # remove space before and after quotation
				(' (['+ punc_after +'])', r'\1'),  # remove space before
				('(['+ punc_before +']) ', r'\1'),  # remove space after
				('(['+ punc_after[:3] +'])([^ '+ punc_after +'\d۰۱۲۳۴۵۶۷۸۹])', r'\1 \2'),  # put space after . and :
				('(['+ punc_after[3:] +'])([^ '+ punc_after +'])', r'\1 \2'),  # put space after
				('([^ '+ punc_before +'])(['+ punc_before +'])', r'\1 \2'),  # put space before
			])

		if affix_spacing:
			self.affix_spacing_patterns = compile_patterns([
				(r'([^ ]ه) ی ', r'\1ی '),  # fix ی space
				(r'(^| )(ن?می) ', r'\1\2'),  # put zwnj after می, نمی
				(r'(?<=[^\n\d '+ punc_after + punc_before +']{2}) (تر(ین?)?|گری?|های?)(?=[ \n'+ punc_after + punc_before +']|$)', r'\1'),  # put zwnj before تر, تری, ترین, گر, گری, ها, های
				(r'([^ ]ه) (ا(م|یم|ش|ند|ی|ید|ت))(?=[ \n'+ punc_after +']|$)', r'\1\2'),  # join ام, ایم, اش, اند, ای, اید, ات
			])

	def normalize(self, text):
		text = self.character_refinement(text)
		if self._affix_spacing:
			text = self.affix_spacing(text)

		if self._token_based:
			tokens = self.tokenizer.tokenize(text.translate(self.translations))
			text = ' '.join(self.token_spacing(tokens))

		if self._punctuation_spacing:
			text = self.punctuation_spacing(text)

		return text

	def character_refinement(self, text):
		"""
		>>> normalizer = Normalizer()
		>>> normalizer.character_refinement('اصلاح كاف و ياي عربي')
		'اصلاح کاف و یای عربی'

		>>> normalizer.character_refinement('عراق سال 2012 قراردادی به ارزش "4.2 میلیارد دلار" برای خرید تجهیزات نظامی با روسیه امضا  کرد.')
		'عراق سال ۲۰۱۲ قراردادی به ارزش «۴٫۲ میلیارد دلار» برای خرید تجهیزات نظامی با روسیه امضا کرد.'

		>>> normalizer.character_refinement('رمــــان')
		'رمان'

		>>> normalizer.character_refinement('بُشقابِ مَن را بِگیر')
		'بشقاب من را بگیر'
		"""

		text = text.translate(self.translations)
		for pattern, repl in self.character_refinement_patterns:
			text = pattern.sub(repl, text)
		return text

	def punctuation_spacing(self, text):
		"""
		>>> normalizer = Normalizer()
		>>> normalizer.punctuation_spacing('اصلاح ( پرانتزها ) در متن .')
		'اصلاح (پرانتزها) در متن.'

		>>> normalizer.punctuation_spacing('نسخه 0.5 در ساعت 22:00 تهران،1396')
		'نسخه 0.5 در ساعت 22:00 تهران، 1396'

		>>> normalizer.punctuation_spacing('اتریش ٧.٩ میلیون.')
		'اتریش ٧.٩ میلیون.'

		"""

		for pattern, repl in self.punctuation_spacing_patterns:
			text = pattern.sub(repl, text)
		return text

	def affix_spacing(self, text):
		"""
		>>> normalizer = Normalizer()
		>>> normalizer.affix_spacing('خانه ی پدری')
		'خانه‌ی پدری'

		>>> normalizer.affix_spacing('فاصله میان پیشوند ها و پسوند ها را اصلاح می کند.')
		'فاصله میان پیشوند‌ها و پسوند‌ها را اصلاح می‌کند.'

		>>> normalizer.affix_spacing('می روم')
		'می‌روم'

		>>> normalizer.affix_spacing('حرفه ای')
		'حرفه‌ای'

		>>> normalizer.affix_spacing('محبوب ترین ها')
		'محبوب‌ترین‌ها'
		"""

		for pattern, repl in self.affix_spacing_patterns:
			text = pattern.sub(repl, text)
		return text

	def token_spacing(self, tokens):
		"""
		>>> normalizer = Normalizer(token_based=True)
		>>> normalizer.token_spacing(['کتاب', 'ها'])
		['کتاب‌ها']

		>>> normalizer.token_spacing(['او', 'می', 'رود'])
		['او', 'می‌رود']

		>>> normalizer.token_spacing(['ماه', 'می', 'سال', 'جدید'])
		['ماه', 'می', 'سال', 'جدید']

		>>> normalizer.token_spacing(['اخلال', 'گر'])
		['اخلال‌گر']

		>>> normalizer.token_spacing(['پرداخت', 'شده', 'است'])
		['پرداخت', 'شده', 'است']

		>>> normalizer.token_spacing(['زمین', 'لرزه', 'ای'])
		['زمین‌لرزه‌ای']
		"""

		result = []
		for t, token in enumerate(tokens):
			joined = False

			if result:
				token_pair = result[-1]+''+token
				if token_pair in self.verbs or token_pair in self.words and self.words[token_pair][0] > 0:
					joined = True

					if t < len(tokens)-1 and token+'_'+tokens[t+1] in self.verbs:
						joined = False

				elif token in self.suffixes and result[-1] in self.words:
					joined = True

			if joined:
				result.pop()
				result.append(token_pair)
			else:
				result.append(token)

		return result
<commit_msg>fix odd test case character<commit_after># coding: utf-8

from __future__ import unicode_literals
import re
from .Lemmatizer import Lemmatizer
from .WordTokenizer import WordTokenizer
from .utils import maketrans

compile_patterns = lambda patterns: [(re.compile(pattern), repl) for pattern, repl in patterns]

class Normalizer(object):
	def __init__(self, remove_extra_spaces=True, persian_style=True, persian_numbers=True, remove_diacritics=True, affix_spacing=True, token_based=False, punctuation_spacing=True):
		self._punctuation_spacing = punctuation_spacing
		self._affix_spacing = affix_spacing
		self._token_based = token_based

		translation_src, translation_dst = ' ىكي“”', ' یکی""'
		if persian_numbers:
			translation_src += '0123456789%'
			translation_dst += '۰۱۲۳۴۵۶۷۸۹٪'
		self.translations = maketrans(translation_src, translation_dst)

		if self._token_based:
			lemmatizer = Lemmatizer()
			self.words = lemmatizer.words
			self.verbs = lemmatizer.verbs
			self.tokenizer = WordTokenizer(join_verb_parts=False)
			self.suffixes = {'ی', 'ای', 'ها', 'های', 'تر', 'تری', 'ترین', 'گر', 'گری', 'ام', 'ات', 'اش'}

		self.character_refinement_patterns = []

		if remove_extra_spaces:
			self.character_refinement_patterns.extend([
				(r' +', ' '),  # remove extra spaces
				(r'\n\n+', '\n\n'),  # remove extra newlines
				(r'[ـ\r]', ''),  # remove keshide, carriage returns
			])

		if persian_style:
			self.character_refinement_patterns.extend([
				('"([^\n"]+)"', r'«\1»'),  # replace quotation with gyoome
				('([\d+])\.([\d+])', r'\1٫\2'),  # replace dot with momayez
				(r' ?\.\.\.', ' …'),  # replace 3 dots
			])

		if remove_diacritics:
			self.character_refinement_patterns.append(
				('[\u064B\u064C\u064D\u064E\u064F\u0650\u0651\u0652]', ''),  # remove FATHATAN, DAMMATAN, KASRATAN, FATHA, DAMMA, KASRA, SHADDA, SUKUN
			)

		self.character_refinement_patterns = compile_patterns(self.character_refinement_patterns)

		punc_after, punc_before = r'\.:!،؛؟»\]\)\}', r'«\[\(\{'
		if punctuation_spacing:
			self.punctuation_spacing_patterns = compile_patterns([
				('" ([^\n"]+) "', r'"\1"'),  # remove space before and after quotation
				(' (['+ punc_after +'])', r'\1'),  # remove space before
				('(['+ punc_before +']) ', r'\1'),  # remove space after
				('(['+ punc_after[:3] +'])([^ '+ punc_after +'\d۰۱۲۳۴۵۶۷۸۹])', r'\1 \2'),  # put space after . and :
				('(['+ punc_after[3:] +'])([^ '+ punc_after +'])', r'\1 \2'),  # put space after
				('([^ '+ punc_before +'])(['+ punc_before +'])', r'\1 \2'),  # put space before
			])

		if affix_spacing:
			self.affix_spacing_patterns = compile_patterns([
				(r'([^ ]ه) ی ', r'\1ی '),  # fix ی space
				(r'(^| )(ن?می) ', r'\1\2'),  # put zwnj after می, نمی
				(r'(?<=[^\n\d '+ punc_after + punc_before +']{2}) (تر(ین?)?|گری?|های?)(?=[ \n'+ punc_after + punc_before +']|$)', r'\1'),  # put zwnj before تر, تری, ترین, گر, گری, ها, های
				(r'([^ ]ه) (ا(م|یم|ش|ند|ی|ید|ت))(?=[ \n'+ punc_after +']|$)', r'\1\2'),  # join ام, ایم, اش, اند, ای, اید, ات
			])

	def normalize(self, text):
		text = self.character_refinement(text)
		if self._affix_spacing:
			text = self.affix_spacing(text)

		if self._token_based:
			tokens = self.tokenizer.tokenize(text.translate(self.translations))
			text = ' '.join(self.token_spacing(tokens))

		if self._punctuation_spacing:
			text = self.punctuation_spacing(text)

		return text

	def character_refinement(self, text):
		"""
		>>> normalizer = Normalizer()
		>>> normalizer.character_refinement('اصلاح كاف و ياي عربي')
		'اصلاح کاف و یای عربی'

		>>> normalizer.character_refinement('عراق سال 2012 قراردادی به ارزش "4.2 میلیارد دلار" برای خرید تجهیزات نظامی با روسیه امضا  کرد.')
		'عراق سال ۲۰۱۲ قراردادی به ارزش «۴٫۲ میلیارد دلار» برای خرید تجهیزات نظامی با روسیه امضا کرد.'

		>>> normalizer.character_refinement('رمــــان')
		'رمان'

		>>> normalizer.character_refinement('بُشقابِ مَن را بِگیر')
		'بشقاب من را بگیر'
		"""

		text = text.translate(self.translations)
		for pattern, repl in self.character_refinement_patterns:
			text = pattern.sub(repl, text)
		return text

	def punctuation_spacing(self, text):
		"""
		>>> normalizer = Normalizer()
		>>> normalizer.punctuation_spacing('اصلاح ( پرانتزها ) در متن .')
		'اصلاح (پرانتزها) در متن.'

		>>> normalizer.punctuation_spacing('نسخه 0.5 در ساعت 22:00 تهران،1396')
		'نسخه 0.5 در ساعت 22:00 تهران، 1396'

		>>> normalizer.punctuation_spacing('اتریش ۷.۹ میلیون.')
		'اتریش ۷.۹ میلیون.'

		"""

		for pattern, repl in self.punctuation_spacing_patterns:
			text = pattern.sub(repl, text)
		return text

	def affix_spacing(self, text):
		"""
		>>> normalizer = Normalizer()
		>>> normalizer.affix_spacing('خانه ی پدری')
		'خانه‌ی پدری'

		>>> normalizer.affix_spacing('فاصله میان پیشوند ها و پسوند ها را اصلاح می کند.')
		'فاصله میان پیشوند‌ها و پسوند‌ها را اصلاح می‌کند.'

		>>> normalizer.affix_spacing('می روم')
		'می‌روم'

		>>> normalizer.affix_spacing('حرفه ای')
		'حرفه‌ای'

		>>> normalizer.affix_spacing('محبوب ترین ها')
		'محبوب‌ترین‌ها'
		"""

		for pattern, repl in self.affix_spacing_patterns:
			text = pattern.sub(repl, text)
		return text

	def token_spacing(self, tokens):
		"""
		>>> normalizer = Normalizer(token_based=True)
		>>> normalizer.token_spacing(['کتاب', 'ها'])
		['کتاب‌ها']

		>>> normalizer.token_spacing(['او', 'می', 'رود'])
		['او', 'می‌رود']

		>>> normalizer.token_spacing(['ماه', 'می', 'سال', 'جدید'])
		['ماه', 'می', 'سال', 'جدید']

		>>> normalizer.token_spacing(['اخلال', 'گر'])
		['اخلال‌گر']

		>>> normalizer.token_spacing(['پرداخت', 'شده', 'است'])
		['پرداخت', 'شده', 'است']

		>>> normalizer.token_spacing(['زمین', 'لرزه', 'ای'])
		['زمین‌لرزه‌ای']
		"""

		result = []
		for t, token in enumerate(tokens):
			joined = False

			if result:
				token_pair = result[-1]+''+token
				if token_pair in self.verbs or token_pair in self.words and self.words[token_pair][0] > 0:
					joined = True

					if t < len(tokens)-1 and token+'_'+tokens[t+1] in self.verbs:
						joined = False

				elif token in self.suffixes and result[-1] in self.words:
					joined = True

			if joined:
				result.pop()
				result.append(token_pair)
			else:
				result.append(token)

		return result
<|endoftext|>