<commit_before>from array import array
from copy import deepcopy
import constants
from pyipmi.errors import CompletionCodeError, EncodingError, DecodingError
from pyipmi.utils import push_unsigned_int, pop_unsigned_int

class BaseField:
    def __init__(self, name, length, default=None):
        self.name = name
        self.length = length
        self.default = default

    def decode(self, obj, data):
        raise NotImplementedError()

    def encode(self, obj, data):
        raise NotImplementedError()

class ByteArray(BaseField):
    def __init__(self, name, length, default=None):
        BaseField.__init__(self, name, length)
        if default is not None:
            self.default = array('B', default)
        else:
            self.default = array('B')

    def encode(self, obj, data):
        a = getattr(obj, self.name)
        if len(a) != self.length:
            raise EncodingError('Array must be exaclty %d bytes long '
                    '(but is %d long)' % (self.length, len(a)))
        for i in xrange(self.length):
            push_unsigned_int(data, a[i], 1)

    def decode(self, obj, data):
        a = getattr(obj, self.name)
        bytes = []
        for i in xrange(self.length):
            bytes.append(pop_unsigned_int(data, 1))
        setattr(obj, self.name, array('B', bytes))

class UnsignedInt(BaseField):
    def encode(self, obj, data):
        value = getattr(obj, self.name)
        push_unsigned_int(data, value, self.length)

    def decode(self, obj, data):
        value = pop_unsigned_int(data, self.length)
        setattr(obj, self.name, value)

class String(BaseField):
    def encode(self, obj, data):
        value = getattr(obj, self.name)
        data.fromstring(value)

    def decode(self, obj, data):
        setattr(obj, self.name, data[0:self.length])
        del data[0:self.length]

class CompletionCode(UnsignedInt):
    def __init__(self, name='completion_code'):
        UnsignedInt.__init__(self, name, 1, None)

    def decode(self, obj, data):
        UnsignedInt.decode(self, obj, data)
        cc = getattr(obj, self.name)
        if cc != 0x00:
            raise CompletionCodeError(cc)

class UnsignedIntMask(UnsignedInt):
    def __init__(self, name, length, mask, default=None):
        UnsignedInt.__init__(self, name, length, default)

class Timestamp(UnsignedInt):
    def __init__(self, name):
        UnsignedInt.__init__(self, name, 4, None)

class Conditional:
    def __init__(self, cond_fn, field):
        self._condition_fn = cond_fn
        self._field = field

    def __getattr__(self, name):
        return getattr(self._field, name)

    def encode(self, obj, data):
        if self._condition_fn(obj):
            self._field.encode(obj, data)

    def decode(self, obj, data):
        if self._condition_fn(obj):
            self._field.decode(obj, data)

class Bitfield(BaseField):
    class Bit:
        def __init__(self, name, width=1, default=None):
            self.name = name
            self._width = width
            self.default = default

    class ReservedBit(Bit):
        counter = 0
        def __init__(self, width, default=0):
            Bitfield.Bit.__init__(self, 'reserved_bit_%d' %
                    Bitfield.reserved_bit_counter, width, default)
            Bitfield.reserved_bit_counter += 1

    class BitWrapper:
        def __str__(self):
            s = '['
            for attr in dir(self):
                if attr.startswith('_'):
                    continue
                s += '%s=%s ' % (attr, getattr(self, attr))
            s += ']'
            return s

    reserved_bit_counter = 0

    def __init__(self, name, length, *bits):
        BaseField.__init__(self, name, length)
        self._bits = bits
        self._set_offsets()
        self._wrapper = self.BitWrapper()
        self.default = self._wrapper
        self._create_fields()

    def _create_fields(self):
        for bit in self._bits:
            if hasattr(self._wrapper, bit.name):
                raise DescriptionError('Bit with name "%s" already added' %
                        bit.name)
            setattr(self._wrapper, bit.name, bit.default)

    def _set_offsets(self):
        offset = 0
        for b in self._bits:
            b._offset = offset
            offset += b._width
        if offset != 8 * self.length:
            raise DescriptionError('Bit description does not match bitfield '
                    'length')

    def encode(self, obj, data):
        value = 0
        for bit in self._bits:
            wrapper = getattr(obj, self.name)
            bit_value = getattr(wrapper, bit.name)
            if bit_value is None:
                bit_value = bit.default
            if bit_value == None:
                raise EncodingError('Bitfield "%s" not set.' % bit.name)

            value |= (bit_value & (2**bit._width - 1)) << bit._offset
        for i in xrange(self.length):
            data.append(chr((value >> (8*i)) & 0xff))

    def decode(self, obj, data):
        value = 0
        for i in xrange(self.length):
            try:
                value |= ord(data.pop(0)) << (8*i)
            except IndexError:
                raise DecodingError('Data too short for message')
        for bit in self._bits:
            tmp = (value >> bit._offset) & (2**bit._width - 1)
            wrapper = getattr(obj, self.name)
            setattr(wrapper, bit.name, tmp)

class Message:
    _REQ_DESC = ()
    _RSP_DESC = ()

    class _Req:
        def __init__(self, parent):
            self._parent = parent
        def encode(self):
            return self._parent._encode_req()
        def decode(self, data):
            self._parent._decode_req(data)
        def __str__(self):
            s = '%sReq(' % self._parent.__class__.__name__
            if not hasattr(self._parent, '_REQ_DESC'):
                raise NotImplementedError('You have to overwrite this method')
            for field in self._parent._REQ_DESC:
                s += '%s=%s ' % (field.name, getattr(self, field.name))
            s += ")"
            return s

    class _Rsp:
        def __init__(self, parent):
            self._parent = parent
        def encode(self):
            return self._parent._encode_rsp()
        def decode(self, data):
            self._parent._decode_rsp(data)
        def encode_cmd(self):
            return self._parent._cmd_encode_rsp()
        def decode_cmd(self, data):
            self._parent._decode_cmd_rsp(data)
        def __str__(self):
            s = '%sRsp(' % self._parent.__class__.__name__
            if not hasattr(self._parent, '_RSP_DESC'):
                raise NotImplementedError('You have to overwrite this method')
            for field in self._parent._RSP_DESC:
                s += '%s=%s ' % (field.name, getattr(self, field.name))
            s += ")"
            return s

    def __init__(self):
        self.req = self._Req(self)
        self.rsp = self._Rsp(self)
        self._create_fields()

    def _create_fields(self):
        if hasattr(self, '_REQ_DESC'):
            for field in self._REQ_DESC:
                if hasattr(self.req, field.name):
                    raise DescriptionError('Field "%s" already added',
                            field.name)
                # make a deep copy, default may be a mutable object
                setattr(self.req, field.name, deepcopy(field.default))
        if hasattr(self, '_RSP_DESC'):
            for field in self._RSP_DESC:
                if hasattr(self.rsp, field.name):
                    raise DescriptionError('Field "%s" already added',
                            field.name)
                # make a deep copy, default may be a mutable object
                setattr(self.rsp, field.name, deepcopy(field.default))

    def _csum(self, data):
        csum = 0
        for b in data:
            csum += b
        return -csum % 256

    def _encode_req(self):
        # messages can extend this class and provide their own encoding
        # and decoding functions
        if not hasattr(self, '_REQ_DESC'):
            raise NotImplementedError('You have to overwrite this method')

        data = array('c')
        for field in self._REQ_DESC:
            if getattr(self.req, field.name) == None:
                raise EncodingError('Field "%s" not set.' % field.name)
            field.encode(self.req, data)
        return data.tostring()

    def _decode_req(self, data):
        if not hasattr(self, '_REQ_DESC'):
            raise NotImplementedError('You have to overwrite this method')

        data = array('c', data)
        for field in self._REQ_DESC:
            field.decode(self.req, data)
        if len(data) > 0:
            raise DecodingError('Data has extra bytes')

    def _encode_rsp(self):
        if not hasattr(self, '_RSP_DESC'):
            raise NotImplementedError('You have to overwrite this method')

        data = array('c')
        for field in self._RSP_DESC:
            if getattr(self.rsp, field.name) == None:
                raise EncodingError('Field "%s" not set.' % field.name)
            field.encode(self.rsp, data)
        return data.tostring()

    def _decode_rsp(self, data):
        if not hasattr(self, '_RSP_DESC'):
            raise NotImplementedError('You have to overwrite this method')

        data = array('c', data)
        for field in self._RSP_DESC:
            try:
                field.decode(self.rsp, data)
            except CompletionCodeError:
                # stop decoding on completion code != 0
                break
        if len(data) > 0:
            raise DecodingError('Data has extra bytes')
<commit_msg>msgs: allow extra bytes with failed responses<commit_after>from array import array
from copy import deepcopy
import constants
from pyipmi.errors import CompletionCodeError, EncodingError, DecodingError
from pyipmi.utils import push_unsigned_int, pop_unsigned_int

class BaseField:
    def __init__(self, name, length, default=None):
        self.name = name
        self.length = length
        self.default = default

    def decode(self, obj, data):
        raise NotImplementedError()

    def encode(self, obj, data):
        raise NotImplementedError()

class ByteArray(BaseField):
    def __init__(self, name, length, default=None):
        BaseField.__init__(self, name, length)
        if default is not None:
            self.default = array('B', default)
        else:
            self.default = array('B')

    def encode(self, obj, data):
        a = getattr(obj, self.name)
        if len(a) != self.length:
            raise EncodingError('Array must be exaclty %d bytes long '
                    '(but is %d long)' % (self.length, len(a)))
        for i in xrange(self.length):
            push_unsigned_int(data, a[i], 1)

    def decode(self, obj, data):
        a = getattr(obj, self.name)
        bytes = []
        for i in xrange(self.length):
            bytes.append(pop_unsigned_int(data, 1))
        setattr(obj, self.name, array('B', bytes))

class UnsignedInt(BaseField):
    def encode(self, obj, data):
        value = getattr(obj, self.name)
        push_unsigned_int(data, value, self.length)

    def decode(self, obj, data):
        value = pop_unsigned_int(data, self.length)
        setattr(obj, self.name, value)

class String(BaseField):
    def encode(self, obj, data):
        value = getattr(obj, self.name)
        data.fromstring(value)

    def decode(self, obj, data):
        setattr(obj, self.name, data[0:self.length])
        del data[0:self.length]

class CompletionCode(UnsignedInt):
    def __init__(self, name='completion_code'):
        UnsignedInt.__init__(self, name, 1, None)

    def decode(self, obj, data):
        UnsignedInt.decode(self, obj, data)
        cc = getattr(obj, self.name)
        if cc != 0x00:
            raise CompletionCodeError(cc)

class UnsignedIntMask(UnsignedInt):
    def __init__(self, name, length, mask, default=None):
        UnsignedInt.__init__(self, name, length, default)

class Timestamp(UnsignedInt):
    def __init__(self, name):
        UnsignedInt.__init__(self, name, 4, None)

class Conditional:
    def __init__(self, cond_fn, field):
        self._condition_fn = cond_fn
        self._field = field

    def __getattr__(self, name):
        return getattr(self._field, name)

    def encode(self, obj, data):
        if self._condition_fn(obj):
            self._field.encode(obj, data)

    def decode(self, obj, data):
        if self._condition_fn(obj):
            self._field.decode(obj, data)

class Bitfield(BaseField):
    class Bit:
        def __init__(self, name, width=1, default=None):
            self.name = name
            self._width = width
            self.default = default

    class ReservedBit(Bit):
        counter = 0
        def __init__(self, width, default=0):
            Bitfield.Bit.__init__(self, 'reserved_bit_%d' %
                    Bitfield.reserved_bit_counter, width, default)
            Bitfield.reserved_bit_counter += 1

    class BitWrapper:
        def __str__(self):
            s = '['
            for attr in dir(self):
                if attr.startswith('_'):
                    continue
                s += '%s=%s ' % (attr, getattr(self, attr))
            s += ']'
            return s

    reserved_bit_counter = 0

    def __init__(self, name, length, *bits):
        BaseField.__init__(self, name, length)
        self._bits = bits
        self._set_offsets()
        self._wrapper = self.BitWrapper()
        self.default = self._wrapper
        self._create_fields()

    def _create_fields(self):
        for bit in self._bits:
            if hasattr(self._wrapper, bit.name):
                raise DescriptionError('Bit with name "%s" already added' %
                        bit.name)
            setattr(self._wrapper, bit.name, bit.default)

    def _set_offsets(self):
        offset = 0
        for b in self._bits:
            b._offset = offset
            offset += b._width
        if offset != 8 * self.length:
            raise DescriptionError('Bit description does not match bitfield '
                    'length')

    def encode(self, obj, data):
        value = 0
        for bit in self._bits:
            wrapper = getattr(obj, self.name)
            bit_value = getattr(wrapper, bit.name)
            if bit_value is None:
                bit_value = bit.default
            if bit_value == None:
                raise EncodingError('Bitfield "%s" not set.' % bit.name)

            value |= (bit_value & (2**bit._width - 1)) << bit._offset
        for i in xrange(self.length):
            data.append(chr((value >> (8*i)) & 0xff))

    def decode(self, obj, data):
        value = 0
        for i in xrange(self.length):
            try:
                value |= ord(data.pop(0)) << (8*i)
            except IndexError:
                raise DecodingError('Data too short for message')
        for bit in self._bits:
            tmp = (value >> bit._offset) & (2**bit._width - 1)
            wrapper = getattr(obj, self.name)
            setattr(wrapper, bit.name, tmp)

class Message:
    _REQ_DESC = ()
    _RSP_DESC = ()

    class _Req:
        def __init__(self, parent):
            self._parent = parent
        def encode(self):
            return self._parent._encode_req()
        def decode(self, data):
            self._parent._decode_req(data)
        def __str__(self):
            s = '%sReq(' % self._parent.__class__.__name__
            if not hasattr(self._parent, '_REQ_DESC'):
                raise NotImplementedError('You have to overwrite this method')
            for field in self._parent._REQ_DESC:
                s += '%s=%s ' % (field.name, getattr(self, field.name))
            s += ")"
            return s

    class _Rsp:
        def __init__(self, parent):
            self._parent = parent
        def encode(self):
            return self._parent._encode_rsp()
        def decode(self, data):
            self._parent._decode_rsp(data)
        def encode_cmd(self):
            return self._parent._cmd_encode_rsp()
        def decode_cmd(self, data):
            self._parent._decode_cmd_rsp(data)
        def __str__(self):
            s = '%sRsp(' % self._parent.__class__.__name__
            if not hasattr(self._parent, '_RSP_DESC'):
                raise NotImplementedError('You have to overwrite this method')
            for field in self._parent._RSP_DESC:
                s += '%s=%s ' % (field.name, getattr(self, field.name))
            s += ")"
            return s

    def __init__(self):
        self.req = self._Req(self)
        self.rsp = self._Rsp(self)
        self._create_fields()

    def _create_fields(self):
        if hasattr(self, '_REQ_DESC'):
            for field in self._REQ_DESC:
                if hasattr(self.req, field.name):
                    raise DescriptionError('Field "%s" already added',
                            field.name)
                # make a deep copy, default may be a mutable object
                setattr(self.req, field.name, deepcopy(field.default))
        if hasattr(self, '_RSP_DESC'):
            for field in self._RSP_DESC:
                if hasattr(self.rsp, field.name):
                    raise DescriptionError('Field "%s" already added',
                            field.name)
                # make a deep copy, default may be a mutable object
                setattr(self.rsp, field.name, deepcopy(field.default))

    def _csum(self, data):
        csum = 0
        for b in data:
            csum += b
        return -csum % 256

    def _encode_req(self):
        # messages can extend this class and provide their own encoding
        # and decoding functions
        if not hasattr(self, '_REQ_DESC'):
            raise NotImplementedError('You have to overwrite this method')

        data = array('c')
        for field in self._REQ_DESC:
            if getattr(self.req, field.name) == None:
                raise EncodingError('Field "%s" not set.' % field.name)
            field.encode(self.req, data)
        return data.tostring()

    def _decode_req(self, data):
        if not hasattr(self, '_REQ_DESC'):
            raise NotImplementedError('You have to overwrite this method')

        data = array('c', data)
        for field in self._REQ_DESC:
            field.decode(self.req, data)
        if len(data) > 0:
            raise DecodingError('Data has extra bytes')

    def _encode_rsp(self):
        if not hasattr(self, '_RSP_DESC'):
            raise NotImplementedError('You have to overwrite this method')

        data = array('c')
        for field in self._RSP_DESC:
            if getattr(self.rsp, field.name) == None:
                raise EncodingError('Field "%s" not set.' % field.name)
            field.encode(self.rsp, data)
        return data.tostring()

    def _decode_rsp(self, data):
        if not hasattr(self, '_RSP_DESC'):
            raise NotImplementedError('You have to overwrite this method')

        data = array('c', data)
        cc = 0
        for field in self._RSP_DESC:
            try:
                field.decode(self.rsp, data)
            except CompletionCodeError, e:
                # stop decoding on completion code != 0
                cc = e.cc
                break
        if cc == 0 and len(data) > 0:
            raise DecodingError('Data has extra bytes')
<|endoftext|>


<commit_before>""" Implementação do algoritmo de busca binária com recursão """

def busca_binaria(valor, vetor, esquerda, direita):
    """
    Implementação de um algoritmo de busca binária com recursão.

    Argumentos:
    valor: Any. Valor a ser buscado na lista
    vetor: list. lista ordenada na qual o valor será buscado
    esquerda: Any. Valor inicial da metade buscada
    direita: Any. Valor final da metade buscada

    Retorna o índice do valor em "vetor" ou -1 caso não exista nela.
    """
    meio = int((esquerda + direita) / 2)

    if esquerda <= direita:
        if valor > vetor[meio]:
            esquerda = meio + 1
            return busca_binaria(valor, vetor, esquerda, direita)
        elif valor < vetor[meio]:
            direita = meio - 1
            return busca_binaria(valor, vetor, esquerda, direita)
        return meio
    return -1

lista = [0, 1, 3, 5, 6, 7, 8, 9, 10, 11, 12]
print(busca_binaria(12, lista, 0, len(lista)))
<commit_msg>fix bug in binary search python<commit_after>""" Implementação do algoritmo de busca binária com recursão """

def busca_binaria(valor, vetor, esquerda, direita):
    """
    Implementação de um algoritmo de busca binária com recursão.

    Argumentos:
    valor: Any. Valor a ser buscado na lista
    vetor: list. lista ordenada na qual o valor será buscado
    esquerda: Any. Valor inicial da metade buscada
    direita: Any. Valor final da metade buscada

    Retorna o índice do valor em "vetor" ou -1 caso não exista nela.
    """
    meio = int((esquerda + direita) / 2)

    if esquerda <= direita:
        if valor > vetor[meio]:
            esquerda = meio + 1
            return busca_binaria(valor, vetor, esquerda, direita)
        elif valor < vetor[meio]:
            direita = meio - 1
            return busca_binaria(valor, vetor, esquerda, direita)
        return meio
    return -1

lista = [0, 1, 3, 5, 6, 7, 8, 9, 10, 11, 12]
print(busca_binaria(12, lista, 0, len(lista) - 1))
<|endoftext|>


<commit_before># Copyright 2016 by Raytheon BBN Technologies Corp.  All Rights Reserved

"""
Analyze the body of a with-qfor statement to see whether it can be
converted into a "repeat" statement, and make the transformation if
appropriate.
"""

from pyqgl2.ast_util import ast2str, expr2ast

def with_qiter_eq(stmnt1, stmnt2):
    """
    Given two "with-qiter" statements, determine whether they are
    identical and therefore the effect of the two statements
    can be implemented by repeating one of the statements twice.

    Returns True if the statements are identical, False otherwise.

    Note that the statements are assumed to be consecutive,
    NOT separated by a third statement.  If the statements are
    separated, then the comparison is still legitimate, but
    does not imply that the two statements can be replaced with
    iteration.

    Also note that the comparison is assumed to be done
    AFTER the evaluator has finished transforming the statements,
    including rebinding any variables and/or variable names.
    At this point, only a small subset of valid Python statements
    and expressions may appear in the given statements.  We use
    this assumption liberally, and therefore this method may
    need to change if/when the evaluator changes.
    """

    try:
        stmnt_txt1 = ast2str(stmnt1).strip()
    except BaseException as exc:
        print('with_qiter_eq ast2str failed stmnt1')
        return False

    try:
        stmnt_txt2 = ast2str(stmnt2).strip()
    except BaseException as exc:
        print('with_qiter_eq ast2str failed stmnt2')
        return False

    # Just to be doubly certain, try converting the text back
    # to AST again, and then check those results.

    try:
        stmnt1_ast_again = expr2ast(stmnt1_txt1)
        stmnt1_txt_again = ast2str(stmnt1_ast_again).strip()
    except BaseException as exc:
        print('with_qiter_eq expr2ast failed stmnt1')
        return False

    try:
        stmnt2_ast_again = expr2ast(stmnt2_txt1)
        stmnt2_txt_again = ast2str(stmnt2_ast_again).strip()
    except BaseException as exc:
        print('with_qiter_eq expr2ast failed stmnt2')
        return False

    if stmnt_txt1 != stmnt_txt2:
        return False
    elif stmnt1_txt_again != stmnt2_txt_again:
        return False
    else:
        return True

def find_dup_qiters(stmnts, compare=None):
    """
    Given a list of statements, return a tuple (base, iter_cnt, iter_len)
    where base is the offset where the iterations begin, iter_cnt is
    the number of repetitions discovered, and iter_len is the number
    of statements in each iteration.  Note that if iter_cnt is 1, then
    no iterations were discovered at all.

    A function to "compare" two statements for equality may be
    passed in as a parameter.  This simplifies testing.

    For example, the sequence [ A, B, C, A, B, C, A ] would result in
    (0, 2, 2), meaning that starting at a base offset of zero,
    there are two iterations of length three.

    This function only attempts to find one iteration, starting at
    the beginning of the stmnts list.  If may be necessary to call
    this function repeatedly, in order to find all the possible
    iterations.  For example, [ A, A, A, B, B, B ] would find the
    three As, and return (0, 3, 1), and then would have to be called
    again on the remainder of the list to find the three Bs.

    The current heuristics for discovering repetitions are incomplete.
    We look for the simple patterns that we expect to be the common case.
    The heurstics may be updated if our expectations turn out to be
    incorrect.

    The current heuristic is that it's better to try to maximize iter_cnt
    and minimize iter_len, in order to compress the code as much as
    possible.  Therefore we look for short subsequences first, starting
    at the beginning of the stmnts, and then if that fails, look for
    longer subsequences.  This greedy approach can miss longer sequences.
    For example, [ A, A, B, A, A, B, A, A, B ] can be expressed as
    three iterations of [ A, A, B ], but the current heuristic will
    instead group this as three iterations of two As separated by Bs.

    The goodness of a grouping is defined to be (iter_cnt - 1) * iter_len,
    which is simplistic (because it pretends that all of the statements
    have the same length).

    TODO: make sure this comment is up-to-date as the heuristics evolve.
    """

    if compare is None:
        compare = with_qiter_eq

    n_stmnts = len(stmnts)

    # this is the longest possible iter length: we can't divide
    # the stmnts into pieces longer than this
    #
    max_len = int(n_stmnts / 2)

    # in order to prevent an O(n^2) term in the simple implementation,
    # we cap max_len to a small, magic number.  We expect this to suffice
    # in most cases, but this assertion is untested
    #
    # FIXME: this should be a parameter in some sense
    #
    if max_len > 5:
        max_len = 5

    # the best thing we've found so far, which is failure to find
    # anything.
    #
    best = (0, 1, n_stmnts)
    best_score = (best[1] - 1) * best[2]

    for base in range(0, n_stmnts - 2):
        for iter_len in range(1, max_len + 1):

            start1 = base
            start2 = base + iter_len
            end2 = start2 + iter_len

            print('- start1 %d start2 %d' % (start1, start2))

            print('base %d iter_len %d end2 %d' % (base, iter_len, end2))

            while end2 <= n_stmnts:
                cand = (start1, int((end2 - start1) / iter_len), iter_len)
                cand_score = (cand[1] - 1) * cand[2]

                print('    cand base %d cnt %d len %d' %
                        (cand[0], cand[1], cand[2]))

                success = True
                for t in range(iter_len):
                    if not compare(
                            stmnts[start1 + t], stmnts[start2 + t]):
                        print('  failed base %d cnt %d len %d' %
                                (best[0], best[1], best[2]))
                        success = False
                        break

                if success:
                    print('matched! %d %d ' % (best_score, cand_score))
                    if best_score < cand_score:
                        print('     old base %d cnt %d len %d' %
                                (best[0], best[1], best[2]))
                        print('     new base %d cnt %d len %d' %
                                (cand[0], cand[1], cand[2]))
                        best = cand
                        best_score = cand_score
                    else:
                        print('  not a new high score')
                    start2 += iter_len
                    end2 = start2 + iter_len
                else:
                    print('not a match')
                    break

    return best

if __name__ == '__main__':

    def simple_compare(a, b):
        return a == b

    def test_find_dup(arr):
        base, iter_cnt, iter_len = find_dup_qiters(arr, simple_compare)

        print('ARR = %s, base %d cnt %d len %d' %
                (str(arr), base, iter_cnt, iter_len))

        return base, iter_cnt, iter_len

    def main():
        test_find_dup([1, 1, 1, 1])
        test_find_dup([2, 1, 1, 1])
        test_find_dup([2, 1, 1, 2])
        test_find_dup([1, 2, 1, 2])

    main()
<commit_msg>take the leftmost match, not the longest<commit_after># Copyright 2016 by Raytheon BBN Technologies Corp.  All Rights Reserved

"""
Analyze the body of a with-qfor statement to see whether it can be
converted into a "repeat" statement, and make the transformation if
appropriate.
"""

from pyqgl2.ast_util import ast2str, expr2ast

def with_qiter_eq(stmnt1, stmnt2):
    """
    Given two "with-qiter" statements, determine whether they are
    identical and therefore the effect of the two statements
    can be implemented by repeating one of the statements twice.

    Returns True if the statements are identical, False otherwise.

    Note that the statements are assumed to be consecutive,
    NOT separated by a third statement.  If the statements are
    separated, then the comparison is still legitimate, but
    does not imply that the two statements can be replaced with
    iteration.

    Also note that the comparison is assumed to be done
    AFTER the evaluator has finished transforming the statements,
    including rebinding any variables and/or variable names.
    At this point, only a small subset of valid Python statements
    and expressions may appear in the given statements.  We use
    this assumption liberally, and therefore this method may
    need to change if/when the evaluator changes.
    """

    try:
        stmnt_txt1 = ast2str(stmnt1).strip()
    except BaseException as exc:
        print('with_qiter_eq ast2str failed stmnt1')
        return False

    try:
        stmnt_txt2 = ast2str(stmnt2).strip()
    except BaseException as exc:
        print('with_qiter_eq ast2str failed stmnt2')
        return False

    # Just to be doubly certain, try converting the text back
    # to AST again, and then check those results.

    try:
        stmnt1_ast_again = expr2ast(stmnt1_txt1)
        stmnt1_txt_again = ast2str(stmnt1_ast_again).strip()
    except BaseException as exc:
        print('with_qiter_eq expr2ast failed stmnt1')
        return False

    try:
        stmnt2_ast_again = expr2ast(stmnt2_txt1)
        stmnt2_txt_again = ast2str(stmnt2_ast_again).strip()
    except BaseException as exc:
        print('with_qiter_eq expr2ast failed stmnt2')
        return False

    if stmnt_txt1 != stmnt_txt2:
        return False
    elif stmnt1_txt_again != stmnt2_txt_again:
        return False
    else:
        return True

def find_left_dup_qiters(stmnts, compare=None):
    """
    Given a list of statements, return a tuple (base, iter_cnt, iter_len)
    where base is the offset where the iterations begin, iter_cnt is
    the number of repetitions discovered, and iter_len is the number
    of statements in each iteration.  Note that if iter_cnt is 1, then
    no iterations were discovered at all.

    A function to "compare" two statements for equality may be
    passed in as a parameter.  This simplifies testing.

    For example, the sequence [ A, B, C, A, B, C, A ] would result in
    (0, 2, 2), meaning that starting at a base offset of zero,
    there are two iterations of length three.

    This function only attempts to find one iteration, starting at
    the beginning of the stmnts list.  If may be necessary to call
    this function repeatedly, in order to find all the possible
    iterations.  For example, [ A, A, A, B, B, B ] would find the
    three As, and return (0, 3, 1), and then would have to be called
    again on the remainder of the list to find the three Bs.

    The current heuristics for discovering repetitions are incomplete.
    We look for the simple patterns that we expect to be the common case.
    The heurstics may be updated if our expectations turn out to be
    incorrect.

    The current heuristic is that it's better to try to maximize iter_cnt
    and minimize iter_len, in order to compress the code as much as
    possible.  Therefore we look for short subsequences first, starting
    at the beginning of the stmnts, and then if that fails, look for
    longer subsequences.  This greedy approach can miss longer sequences.
    For example, [ A, A, B, A, A, B, A, A, B ] can be expressed as
    three iterations of [ A, A, B ], but the current heuristic will
    instead group this as three iterations of two As separated by Bs.

    The goodness of a grouping is defined to be (iter_cnt - 1) * iter_len,
    which is simplistic (because it pretends that all of the statements
    have the same length).

    TODO: make sure this comment is up-to-date as the heuristics evolve.
    """

    if compare is None:
        compare = with_qiter_eq

    n_stmnts = len(stmnts)

    # this is the longest possible iter length: we can't divide
    # the stmnts into pieces longer than this
    #
    max_len = int(n_stmnts / 2)

    # in order to prevent an O(n^2) term in the simple implementation,
    # we cap max_len to a small, magic number.  We expect this to suffice
    # in most cases, but this assertion is untested
    #
    # FIXME: this should be a parameter in some sense
    #
    if max_len > 5:
        max_len = 5

    # the best thing we've found so far, which is failure to find
    # anything.
    #
    best = (0, 1, n_stmnts)
    best_score = (best[1] - 1) * best[2]

    for base in range(0, n_stmnts - 1):
        for iter_len in range(1, max_len + 1):

            start1 = base
            start2 = base + iter_len
            end2 = start2 + iter_len

            print('- start1 %d start2 %d' % (start1, start2))

            print('base %d iter_len %d end2 %d' % (base, iter_len, end2))

            while end2 <= n_stmnts:
                cand = (start1, int((end2 - start1) / iter_len), iter_len)
                cand_score = (cand[1] - 1) * cand[2]

                print('    cand base %d cnt %d len %d' %
                        (cand[0], cand[1], cand[2]))

                success = True
                for t in range(iter_len):
                    if not compare(
                            stmnts[start1 + t], stmnts[start2 + t]):
                        print('  failed base %d cnt %d len %d' %
                                (best[0], best[1], best[2]))
                        success = False
                        break

                if success:
                    print('matched! %d %d ' % (best_score, cand_score))
                    if best_score < cand_score:
                        print('     old base %d cnt %d len %d' %
                                (best[0], best[1], best[2]))
                        print('     new base %d cnt %d len %d' %
                                (cand[0], cand[1], cand[2]))
                        best = cand
                        best_score = cand_score
                    else:
                        print('  not a new high score')
                    start2 += iter_len
                    end2 = start2 + iter_len
                else:
                    print('not a match')
                    break

        # We're not looking for the best score globally: we're looking
        # for the leftmost possible non-degenerate score.  For example,
        # if the input is [x, x, y, y, y, y], we'll take the [x, x]
        # segment as a good optimization even though the [y, y, y, y]
        # is better.  We'll get the scores to the right in later passes.

        if best_score > 0:
            break

    return best

if __name__ == '__main__':

    def simple_compare(a, b):
        return a == b

    def test_find_left_dup(arr):
        base, iter_cnt, iter_len = find_left_dup_qiters(arr, simple_compare)

        print('ARR = %s, base %d cnt %d len %d' %
                (str(arr), base, iter_cnt, iter_len))

        return base, iter_cnt, iter_len

    def main():
        assert (0, 4, 1) == test_find_left_dup([1, 1, 1, 1])
        assert (1, 3, 1) == test_find_left_dup([2, 1, 1, 1])
        assert (1, 2, 1) == test_find_left_dup([2, 1, 1, 2])
        assert (0, 2, 2) == test_find_left_dup([1, 2, 1, 2])
        assert (0, 4, 3) == test_find_left_dup(
                [1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2, 3, 4])
        assert (0, 4, 1) == test_find_left_dup(
                [1, 1, 1, 1, 2, 2, 2, 3, 3, 4, 5])
        assert (2, 2, 2) == test_find_left_dup(
                [5, 4, 3, 2, 3, 2, 2, 3, 3, 4, 5])
        assert (2, 3, 2) == test_find_left_dup(
                [5, 4, 3, 2, 3, 2, 3, 2, 3, 4, 5])

        assert (0, 1, 9) == test_find_left_dup([1, 2, 3, 4, 5, 4, 3, 2, 1])
        assert (8, 2, 1) == test_find_left_dup([1, 2, 3, 4, 5, 4, 3, 2, 1, 1])

        # test that we take the leftmost match, not the longest.

        assert (0, 2, 1) == test_find_left_dup(
                [1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 4])
        assert (8, 2, 1) == test_find_left_dup(
                [1, 2, 3, 4, 5, 4, 3, 2, 1, 1, 2, 2, 2])

    main()
<|endoftext|>


<commit_before>#!/usr/bin/python

"""Anonymize DICOM files. Write in-place or into another directory."""

from __future__ import absolute_import, division, print_function
import argparse
import os

import dicom

# ftp://medical.nema.org/medical/dicom/final/sup55_ft.pdf
# Digital Imaging and Communications in Medicine (DICOM)
# Supplement 55: Attribute Level Confidentiality (including De-identification)
# Table X.1-1
# Basic Application Level Confidentiality Profile Attributes
CONFIDENTIAL_TAGS = {
    (0x0008, 0x0014): "Instance Creator UID",
    (0x0008, 0x0018): "SOP Instance UID",
    (0x0008, 0x0050): "Accession Number",
    (0x0008, 0x0080): "Institution Name",
    (0x0008, 0x0081): "Institution Address",
    (0x0008, 0x0090): "Referring Physician's Name",
    (0x0008, 0x0092): "Referring Physician's Address",
    (0x0008, 0x0094): "Referring Physician's Telephone Numbers",
    (0x0008, 0x1010): "Station Name",
    (0x0008, 0x1030): "Study Description",
    (0x0008, 0x103E): "Series Description",
    (0x0008, 0x1040): "Institutional Department Name",
    (0x0008, 0x1048): "Physician(s) of Record",
    (0x0008, 0x1050): "Performing Physicians' Name",
    (0x0008, 0x1060): "Name of Physician(s) Reading Study",
    (0x0008, 0x1070): "Operators' Name",
    (0x0008, 0x1080): "Admitting Diagnoses Description",
    (0x0008, 0x1155): "Referenced SOP Instance UID",
    (0x0008, 0x2111): "Derivation Description",
    (0x0010, 0x0010): "Patient's Name",
    (0x0010, 0x0020): "Patient ID",
    (0x0010, 0x0030): "Patient's Birth Date",
    (0x0010, 0x0032): "Patient's Birth Time",
    (0x0010, 0x0040): "Patient's Sex",
    (0x0010, 0x1000): "Other Patient Ids",
    (0x0010, 0x1001): "Other Patient Names",
    (0x0010, 0x1010): "Patient's Age",
    (0x0010, 0x1020): "Patient's Size",
    (0x0010, 0x1030): "Patient's Weight",
    (0x0010, 0x1090): "Medical Record Locator",
    (0x0010, 0x2160): "Ethnic Group",
    (0x0010, 0x2180): "Occupation",
    (0x0010, 0x21B0): "Additional Patient's History",
    (0x0010, 0x4000): "Patient Comments",
    (0x0018, 0x1000): "Device Serial Number",
    (0x0018, 0x1030): "Protocol Name",
    (0x0020, 0x000D): "Study Instance UID",
    (0x0020, 0x000E): "Series Instance UID",
    (0x0020, 0x0010): "Study ID",
    (0x0020, 0x0052): "Frame of Reference UID",
    (0x0020, 0x0200): "Synchronization Frame of Reference UID",
    (0x0020, 0x4000): "Image Comments",
    (0x0040, 0x0275): "Request Attributes Sequence",
    (0x0040, 0xA124): "UID",
    (0x0040, 0xA730): "Content Sequence",
    (0x0088, 0x0140): "Storage Media File-set UID",
    (0x3006, 0x0024): "Referenced Frame of Reference UID",
    (0x3006, 0x00C2): "Related Frame of Reference UID",
}

def parse_args():
    """Parse command-line arguments."""
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument('-v', '--verbose', action='count',
                   help='increase verbosity')
    p.add_argument('-n', '--dry-run', action='store_true',
                   help='do not write any changes')
    p.add_argument('-i', '--input', metavar='FILENAME', nargs='+',
                   help='input DICOM files')
    p.add_argument('-o', '--output', metavar='PATHNAME',
                   help='output directory')
    return p.parse_args()

def delete_confidential(dataset, elem):
    """Delete confidential elements from dataset."""
    if elem.tag in CONFIDENTIAL_TAGS:
        deleted.append((elem.tag, elem.name))
        del dataset[elem.tag]

args = parse_args()
for infile in args.input:
    if args.output:
        outfile = os.path.join(args.output, os.path.basename(infile))
    else:
        outfile = infile
    deleted = []
    f = dicom.read_file(infile)
    f.walk(delete_confidential, recursive=True)
    if deleted:
        if args.verbose > 1:
            for elem in deleted:
                print('{i}: {t} {n}'.format(i=infile, t=elem[0], n=elem[1]))
        if args.verbose:
            d = dict(i=infile, o=outfile, n=len(deleted))
            print('{i}: Deleted {n} elements, writing to {o}.'.format(**d))
        if not args.dry_run:
            f.save_as(outfile, WriteLikeOriginal=True)
<commit_msg>call py3<commit_after>#!/usr/bin/python3

"""Anonymize DICOM files. Write in-place or into another directory."""

from __future__ import absolute_import, division, print_function
import argparse
import os

import dicom

# ftp://medical.nema.org/medical/dicom/final/sup55_ft.pdf
# Digital Imaging and Communications in Medicine (DICOM)
# Supplement 55: Attribute Level Confidentiality (including De-identification)
# Table X.1-1
# Basic Application Level Confidentiality Profile Attributes
CONFIDENTIAL_TAGS = {
    (0x0008, 0x0014): "Instance Creator UID",
    (0x0008, 0x0018): "SOP Instance UID",
    (0x0008, 0x0050): "Accession Number",
    (0x0008, 0x0080): "Institution Name",
    (0x0008, 0x0081): "Institution Address",
    (0x0008, 0x0090): "Referring Physician's Name",
    (0x0008, 0x0092): "Referring Physician's Address",
    (0x0008, 0x0094): "Referring Physician's Telephone Numbers",
    (0x0008, 0x1010): "Station Name",
    (0x0008, 0x1030): "Study Description",
    (0x0008, 0x103E): "Series Description",
    (0x0008, 0x1040): "Institutional Department Name",
    (0x0008, 0x1048): "Physician(s) of Record",
    (0x0008, 0x1050): "Performing Physicians' Name",
    (0x0008, 0x1060): "Name of Physician(s) Reading Study",
    (0x0008, 0x1070): "Operators' Name",
    (0x0008, 0x1080): "Admitting Diagnoses Description",
    (0x0008, 0x1155): "Referenced SOP Instance UID",
    (0x0008, 0x2111): "Derivation Description",
    (0x0010, 0x0010): "Patient's Name",
    (0x0010, 0x0020): "Patient ID",
    (0x0010, 0x0030): "Patient's Birth Date",
    (0x0010, 0x0032): "Patient's Birth Time",
    (0x0010, 0x0040): "Patient's Sex",
    (0x0010, 0x1000): "Other Patient Ids",
    (0x0010, 0x1001): "Other Patient Names",
    (0x0010, 0x1010): "Patient's Age",
    (0x0010, 0x1020): "Patient's Size",
    (0x0010, 0x1030): "Patient's Weight",
    (0x0010, 0x1090): "Medical Record Locator",
    (0x0010, 0x2160): "Ethnic Group",
    (0x0010, 0x2180): "Occupation",
    (0x0010, 0x21B0): "Additional Patient's History",
    (0x0010, 0x4000): "Patient Comments",
    (0x0018, 0x1000): "Device Serial Number",
    (0x0018, 0x1030): "Protocol Name",
    (0x0020, 0x000D): "Study Instance UID",
    (0x0020, 0x000E): "Series Instance UID",
    (0x0020, 0x0010): "Study ID",
    (0x0020, 0x0052): "Frame of Reference UID",
    (0x0020, 0x0200): "Synchronization Frame of Reference UID",
    (0x0020, 0x4000): "Image Comments",
    (0x0040, 0x0275): "Request Attributes Sequence",
    (0x0040, 0xA124): "UID",
    (0x0040, 0xA730): "Content Sequence",
    (0x0088, 0x0140): "Storage Media File-set UID",
    (0x3006, 0x0024): "Referenced Frame of Reference UID",
    (0x3006, 0x00C2): "Related Frame of Reference UID",
}

def parse_args():
    """Parse command-line arguments."""
    p = argparse.ArgumentParser(description=__doc__)
    p.add_argument('-v', '--verbose', action='count',
                   help='increase verbosity')
    p.add_argument('-n', '--dry-run', action='store_true',
                   help='do not write any changes')
    p.add_argument('-i', '--input', metavar='FILENAME', nargs='+',
                   help='input DICOM files')
    p.add_argument('-o', '--output', metavar='PATHNAME',
                   help='output directory')
    return p.parse_args()

def delete_confidential(dataset, elem):
    """Delete confidential elements from dataset."""
    if elem.tag in CONFIDENTIAL_TAGS:
        deleted.append((elem.tag, elem.name))
        del dataset[elem.tag]

args = parse_args()
for infile in args.input:
    if args.output:
        outfile = os.path.join(args.output, os.path.basename(infile))
    else:
        outfile = infile
    deleted = []
    f = dicom.read_file(infile)
    f.walk(delete_confidential, recursive=True)
    if deleted:
        if args.verbose > 1:
            for elem in deleted:
                print('{i}: {t} {n}'.format(i=infile, t=elem[0], n=elem[1]))
        if args.verbose:
            d = dict(i=infile, o=outfile, n=len(deleted))
            print('{i}: Deleted {n} elements, writing to {o}.'.format(**d))
        if not args.dry_run:
            f.save_as(outfile, WriteLikeOriginal=True)
<|endoftext|>


<commit_before>"""Exceptions are defined here"""

class NotFoundError(Exception):
    """Exception for Not Found error"""
    status_code = 404

    def to_dict(self):
        """Convert exception to dictionary representation"""
        return {'error': str(self), 'status': self.status_code}
<commit_msg>add `UnauthorizedError` exception<commit_after>"""Exceptions are defined here"""

class NotFoundError(Exception):
    """Exception for HTTP Not Found error"""
    status_code = 404

    def to_dict(self):
        """Convert exception to dictionary representation"""
        return {'error': str(self), 'status': self.status_code}

class UnauthorizedError(Exception):
    """Exception for HTTP Unauthorized error"""
    status_code = 401

    def to_dict(self):
        """Convert exception to dictionary representation"""
        return {'error': str(self), 'status': self.status_code}
<|endoftext|>


<commit_before>'''
Replace ast.Name nodes with strings wherever possible:
    foo + bar -> 'foo' + 'bar'
    sorted([sjhsdfjsdfsdf, abc, 'de'], key=len) -> sorted(['de', 'abc', 'sjhsdfjsdfsdf'], key=len)

Ignore:
    foo[0], foo[::], foo()
    arguments TODO: fix scope escaping
    locals/globals unless they're direct children of root expression
'''

import ast
from recognition.actions import library

class NameToStringTransformer(ast.NodeTransformer):

    def __init__(self, root, namespace, arguments):
        super().__init__()
        self.namespace = namespace
        self.arguments = arguments
        self.nodes_to_replace = self.to_replace(root)

    def to_replace(self, root):
        nodes_to_replace = set()
        names_to_keep = set()
        for node in ast.walk(root):
            for child in ast.iter_child_nodes(node):
                if isinstance(child, ast.arg):
                    names_to_keep.add(child.arg)
                if not isinstance(child, ast.Name):
                    continue
                if child.id in self.arguments:
                    names_to_keep.add(child.id)
                    continue
                is_call_or_slice = isinstance(node, (ast.Subscript, ast.Attribute)) or (isinstance(node, ast.Call) and node.func is child)
                if not (is_call_or_slice or
                    (child.id in self.namespace and node is not root)):
                    nodes_to_replace.add(child)
        return set(n for n in nodes_to_replace if n.id not in names_to_keep)

    def visit_Name(self, node):
        if node in self.nodes_to_replace:
            return ast.Str(s=node.id)
        return self.generic_visit(node)

class SetLiteralTransformer(ast.NodeTransformer):

    def visit_Set(self, node):
        func = ast.Name(id='keys', ctx=ast.Load())
        return ast.Call(func=func, args=node.elts, keywords=[])

class LambdaArgTransformer(ast.NodeTransformer):

    def visit_Call(self, node):
        if self.is_deferred_func(node):
            newargs = []
            for arg in node.args:
                largs = ast.arguments(args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[])
                newarg = ast.Lambda(args=largs, body=arg)
                newargs.append(newarg)
            node.args = newargs
        return self.generic_visit(node)

    def is_deferred_func(self, node):
        path = node_path(node.func)
        # TODO: needs cleanup
        if (path in library.lambda_arguments or
            path == ('context', '_meta', 'call_or_type') and isinstance(node.func, ast.Call)):
            return True
        return False

class VariableArgumentTransformer(ast.NodeTransformer):

    def __init__(self, root):
        self.parent_map = self.build_parent_map(root)

    def visit_Call(self, node):
        if self.is_variable_call(node):
            path = self.get_containing_function_path(node)
            if path:
                func = path[0]
                # only type results for certain arguments. ex: 'foo' in repeat('foo', 10), but not 10
                if node_path(func) != ('repeat',) or func.args[-1] is path[1]:
                    node.keywords.append(ast.keyword(arg='perform_results', value=ast.NameConstant(value=False)))
        return self.generic_visit(node)

    def build_parent_map(self, root):
        parent_map = {root: None}
        for node in ast.walk(root):
            for child in ast.iter_child_nodes(node):
                parent_map[child] = node
        return parent_map

    def is_variable_call(self, node):
        return isinstance(node, ast.Call) and node_path(node.func) == ('context', 'var')

    def get_containing_function_path(self, node):
        path = [node]
        while node:
            parent = self.parent_map[node]
            path.append(parent)
            if isinstance(parent, ast.Call) and not self.is_variable_call(parent):
                return path[::-1]
            node = parent

def node_path(node):
    if isinstance(node, ast.Call):
        return node_path(node.func)
    if isinstance(node, ast.Name):
        return node.id,
    if isinstance(node, ast.Lambda):
        return ()
    return node_path(node.value) + (node.attr,)

def transform_expression(expr_text, namespace=None, arguments=None):
    arguments = arguments or []
    namespace = get_builtins() if namespace is None else namespace
    expr = ast.parse(expr_text, mode='eval')
    new_expr = NameToStringTransformer(expr, namespace, arguments).visit(expr)
    new_expr = SetLiteralTransformer().visit(new_expr)
    new_expr = LambdaArgTransformer().visit(new_expr)
    new_expr = VariableArgumentTransformer(new_expr).visit(new_expr)
    return compile(ast.fix_missing_locations(new_expr), filename=f'<{expr_text}>', mode='eval')

def get_builtins():
    return __builtins__ if isinstance(__builtins__, dict) else dir(__builtins__)<commit_msg>remove namespace from asttransform<commit_after>'''
Replace ast.Name nodes with strings wherever possible:
    foo + bar -> 'foo' + 'bar'
    sorted([sjhsdfjsdfsdf, abc, 'de'], key=len) -> sorted(['de', 'abc', 'sjhsdfjsdfsdf'], key=len)

Ignore:
    foo[0], foo[::], foo()
    arguments TODO: fix scope escaping
    locals/globals unless they're direct children of root expression
'''

import ast
from recognition.actions import library

class NameToStringTransformer(ast.NodeTransformer):

    def __init__(self, root, arguments):
        super().__init__()
        self.arguments = arguments
        self.nodes_to_replace = self.to_replace(root)

    def to_replace(self, root):
        nodes_to_replace = set()
        names_to_keep = set()
        for node in ast.walk(root):
            for child in ast.iter_child_nodes(node):
                if isinstance(child, ast.arg):
                    names_to_keep.add(child.arg)
                if not isinstance(child, ast.Name):
                    continue
                if child.id in self.arguments:
                    names_to_keep.add(child.id)
                    continue
                is_call_or_slice = isinstance(node, (ast.Subscript, ast.Attribute)) or (isinstance(node, ast.Call) and node.func is child)
                if not is_call_or_slice:
                    nodes_to_replace.add(child)
        return set(n for n in nodes_to_replace if n.id not in names_to_keep)

    def visit_Name(self, node):
        if node in self.nodes_to_replace:
            return ast.Str(s=node.id)
        return self.generic_visit(node)

class SetLiteralTransformer(ast.NodeTransformer):

    def visit_Set(self, node):
        func = ast.Name(id='keys', ctx=ast.Load())
        return ast.Call(func=func, args=node.elts, keywords=[])

class LambdaArgTransformer(ast.NodeTransformer):

    def visit_Call(self, node):
        if self.is_deferred_func(node):
            newargs = []
            for arg in node.args:
                largs = ast.arguments(args=[], vararg=None, kwonlyargs=[], kw_defaults=[], kwarg=None, defaults=[])
                newarg = ast.Lambda(args=largs, body=arg)
                newargs.append(newarg)
            node.args = newargs
        return self.generic_visit(node)

    def is_deferred_func(self, node):
        path = node_path(node.func)
        # TODO: needs cleanup
        if (path in library.lambda_arguments or
            path == ('context', '_meta', 'call_or_type') and isinstance(node.func, ast.Call)):
            return True
        return False

class VariableArgumentTransformer(ast.NodeTransformer):

    def __init__(self, root):
        self.parent_map = self.build_parent_map(root)

    def visit_Call(self, node):
        if self.is_variable_call(node):
            path = self.get_containing_function_path(node)
            if path:
                func = path[0]
                # only type results for certain arguments. ex: 'foo' in repeat('foo', 10), but not 10
                if node_path(func) != ('repeat',) or func.args[-1] is path[1]:
                    node.keywords.append(ast.keyword(arg='perform_results', value=ast.NameConstant(value=False)))
        return self.generic_visit(node)

    def build_parent_map(self, root):
        parent_map = {root: None}
        for node in ast.walk(root):
            for child in ast.iter_child_nodes(node):
                parent_map[child] = node
        return parent_map

    def is_variable_call(self, node):
        return isinstance(node, ast.Call) and node_path(node.func) == ('context', 'var')

    def get_containing_function_path(self, node):
        path = [node]
        while node:
            parent = self.parent_map[node]
            path.append(parent)
            if isinstance(parent, ast.Call) and not self.is_variable_call(parent):
                return path[::-1]
            node = parent

def node_path(node):
    if isinstance(node, ast.Call):
        return node_path(node.func)
    if isinstance(node, ast.Name):
        return node.id,
    if isinstance(node, ast.Lambda):
        return ()
    return node_path(node.value) + (node.attr,)

def transform_expression(expr_text, namespace=None, arguments=None):
    arguments = arguments or []
    expr = ast.parse(expr_text, mode='eval')
    new_expr = NameToStringTransformer(expr, arguments).visit(expr)
    new_expr = SetLiteralTransformer().visit(new_expr)
    new_expr = LambdaArgTransformer().visit(new_expr)
    new_expr = VariableArgumentTransformer(new_expr).visit(new_expr)
    return compile(ast.fix_missing_locations(new_expr), filename=f'<{expr_text}>', mode='eval')

def get_builtins():
    return __builtins__ if isinstance(__builtins__, dict) else dir(__builtins__)<|endoftext|>


<commit_before>from __future__ import with_statement
import os, subprocess, re, sys, glob
from memoize import memoize

__all__ = ["include_imports"]

file_mtimes = {}
file_contents = {}
lib_imports_fast = {}
lib_imports_slow = {}

IMPORT_REG = re.compile('^R[0-9]+:[0-9]+ ([^ ]+) <> <> lib$', re.MULTILINE)
IMPORT_LINE_REG = re.compile(r'^\s*(?:Require\s+Import|Require\s+Export|Require|Load\s+Verbose|Load)\s+(.*?)\.(?:\s|$)', re.MULTILINE | re.DOTALL)

def DEFAULT_LOG(text):
    print(text)

def filename_of_lib(lib, topname='__TOP__', ext='.v'):
    if lib[:len(topname + '.')] == topname + '.':
        lib = lib[len(topname + '.'):]
        lib = lib.replace('.', os.sep)
    else:
        # is this the right thing to do?
        lib = lib.replace('.', os.sep)
    return os.path.relpath(os.path.normpath(lib + ext), '.')

def lib_of_filename(filename, topname='__TOP__', exts=('.v', '.glob')):
    filename = os.path.relpath(filename, '.')
    for ext in exts:
        if filename[-len(ext):] == ext:
            filename = filename[:-len(ext)]
            break
    if '.' in filename:
        print("WARNING: There is a dot (.) in filename %s; the library conversion probably won't work." % filename)
    return topname + '.' + filename.replace(os.sep, '.')

def get_file(filename, verbose=True, log=DEFAULT_LOG):
    if filename[-2:] != '.v': filename += '.v'
    if filename not in file_contents.keys() or file_mtimes[filename] < os.stat(filename).st_mtime:
        if verbose: log('getting %s' % filename)
        try:
            with open(filename, 'r', encoding='UTF-8') as f:
                file_contents[filename] = f.read()
        except TypeError:
            with open(filename, 'r') as f:
                file_contents[filename] = f.read()
        file_mtimes[filename] = os.stat(filename).st_mtime
    return file_contents[filename]

def get_all_v_files(directory, exclude=tuple()):
    all_files = []
    exclude = [os.path.normpath(i) for i in exclude]
    for dirpath, dirnames, filenames in os.walk(directory):
        all_files += [os.path.relpath(name, '.') for name in glob.glob(os.path.join(dirpath, '*.v'))
                      if os.path.normpath(name) not in exclude]
    return all_files

@memoize
def get_makefile_contents(coqc, topname, v_files, verbose, log):
    cmds = ['coq_makefile', 'COQC', '=', coqc, '-R', '.', topname] + list(v_files)
    if verbose:
        log(' '.join(cmds))
    p_make_makefile = subprocess.Popen(cmds,
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE)
    return p_make_makefile.communicate()

def make_globs(libnames, verbose=True, topname='__TOP__', log=DEFAULT_LOG, coqc='coqc'):
    extant_libnames = [i for i in libnames
                       if os.path.exists(filename_of_lib(i, topname=topname, ext='.v'))]
    if len(extant_libnames) == 0: return
    filenames_v = [filename_of_lib(i, topname=topname, ext='.v') for i in extant_libnames]
    filenames_glob = [filename_of_lib(i, topname=topname, ext='.glob') for i in extant_libnames]
    if all(os.path.exists(glob_name) for glob_name in filenames_glob):
        return
    extra_filenames_v = get_all_v_files('.', filenames_v)
    (stdout, stderr) = get_makefile_contents(coqc, topname, tuple(sorted(filenames_v + extra_filenames_v)), verbose, log)
    if verbose:
        log(' '.join(['make', '-k', '-f', '-'] + filenames_glob))
    p_make = subprocess.Popen(['make', '-k', '-f', '-'] + filenames_glob, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    (stdout_make, stderr_make) = p_make.communicate(stdout)

def get_imports(lib, verbose=True, fast=False, log=DEFAULT_LOG, topname='__TOP__', coqc='coqc'):
    glob_name = filename_of_lib(lib, topname=topname, ext='.glob')
    v_name = filename_of_lib(lib, topname=topname, ext='.v')
    if not fast:
        if lib not in lib_imports_slow.keys():
            if not os.path.exists(glob_name):
                make_globs([lib], verbose=verbose, topname=topname, log=log, coqc=coqc)
            if os.path.exists(glob_name): # making succeeded
                with open(glob_name, 'r') as f:
                    contents = f.read()
                lines = contents.split('\n')
                lib_imports_slow[lib] = tuple(sorted(set(IMPORT_REG.findall(contents))))
                return lib_imports_slow[lib]
    # making globs failed, or we want the fast way, fall back to regexp
    if lib not in lib_imports_fast.keys():
        contents = get_file(v_name, verbose=verbose, log=log)
        imports_string = re.sub('\\s+', ' ', ' '.join(IMPORT_LINE_REG.findall(contents))).strip()
        lib_imports_fast[lib] = tuple(sorted(set(i for i in imports_string.split(' ') if i != '')))
    return lib_imports_fast[lib]

def norm_libname(lib, topname='__TOP__'):
    if lib[:len(topname + '.')] != topname + '.' and os.path.exists(filename_of_lib(lib, topname=topname)):
        return topname + '.' + lib
    return lib

def merge_imports(imports, topname='__TOP__'):
    rtn = []
    for import_list in imports:
        for i in import_list:
            if norm_libname(i, topname=topname) not in rtn:
                rtn.append(norm_libname(i, topname=topname))
    return rtn

def recursively_get_imports(lib, verbose=True, fast=False, log=DEFAULT_LOG, topname='__TOP__', coqc='coqc'):
    glob_name = filename_of_lib(lib, topname=topname, ext='.glob')
    v_name = filename_of_lib(lib, topname=topname, ext='.v')
    if os.path.exists(v_name):
        imports = get_imports(lib, verbose=verbose, fast=fast, topname=topname, coqc=coqc)
        if not fast: make_globs(imports, verbose=verbose, topname=topname, log=log, coqc=coqc)
        imports_list = [recursively_get_imports(i, verbose=verbose, fast=fast, log=log, topname=topname, coqc=coqc)
                        for i in imports]
        return merge_imports(imports_list + [[lib]], topname=topname)
    return [lib]

def contents_without_imports(lib, verbose=True, log=DEFAULT_LOG, topname='__TOP__'):
    v_file = filename_of_lib(lib, topname=topname, ext='.v')
    contents = get_file(v_file, verbose=verbose, log=log)
    if '(*' in ' '.join(IMPORT_LINE_REG.findall(contents)):
        print('Warning: There are comments in your Require/Import/Export lines in %s.' % filename)
    return IMPORT_LINE_REG.sub('', contents)

def escape_lib(lib):
    return lib.replace('.', '_DOT_')

def group_by_first_component(lib_libname_pairs):
    rtn = dict((lib.split('.')[0], []) for lib, libname in lib_libname_pairs)
    for lib, libname in lib_libname_pairs:
        split_lib = lib.split('.')
        rtn[split_lib[0]].append(('.'.join(split_lib[1:]), libname))
    return rtn

def recreate_path_structure(lib_libname_pairs, uid='', topname='__TOP__'):
    '''Takes a list of (<dotted name, escaped name>) pairs, and turns it into a nested module structure'''
    rtn = ''
    lib_libname_pairs = [(lib[len(topname + '.'):]
                          if lib[:len(topname + '.')] == topname + '.'
                          else lib,
                          libname)
                         for lib, libname in lib_libname_pairs]
    if topname != '': rtn += 'Module Import %s%s.\n' % (uid, topname)
    top_libnames = [libname for lib, libname in lib_libname_pairs if lib == '']
    lib_libname_pairs = [(lib, libname) for lib, libname in lib_libname_pairs
                         if lib != '']
    for top_libname in top_libnames:
        rtn += 'Include %s.\n' % top_libname
    for cur_lib, lib_list in group_by_first_component(lib_libname_pairs).items():
        rtn += 'Module %s.\n' % cur_lib
        rtn += recreate_path_structure(lib_list, topname='')
        if cur_lib in [other_lib for other_lib, other_libname in lib_list] and cur_lib not in top_libnames:
            rtn += 'Include %s.\n' % cur_lib
        rtn += 'End %s.\n' % cur_lib
    if topname != '': rtn += 'End %s%s.\n' % (uid, topname)
    return rtn

def contents_as_module_without_require(lib, other_imports, verbose=True, log=DEFAULT_LOG, topname='__TOP__'):
    v_name = filename_of_lib(lib, topname=topname, ext='.v')
    contents = get_file(v_name, verbose=verbose, log=log)
    contents = re.sub(r'^\s*Require\s+((?:Import|Export)\s)',
                      r'\1',
                      contents,
                      flags=re.MULTILINE)
    contents = re.sub(r'^\s*Require\s+((?!Import\s+|Export\s+)(?:[^\.]|\.(?!\s|$))+\.(?:\s|$))',
                      r'',
                      contents,
                      flags=re.MULTILINE)
    print(contents)
    module_name = escape_lib(lib)
    existing_imports = recreate_path_structure([(i, escape_lib(i)) for i in other_imports],
                                               uid=module_name,
                                               topname=topname)
    contents = 'Module %s.\n%s\n%s\nEnd %s.\n' % (module_name, existing_imports, contents, module_name)
    return contents

def include_imports(filename, as_modules=True, verbose=True, fast=False, log=DEFAULT_LOG, topname='__TOP__', coqc='coqc'):
    """Return the contents of filename, with any top-level imports inlined.

    If as_modules == True, then the imports will be wrapped in modules.

    This method requires access to the coqdep program if fast == False.
    Without it, it will fall back to manual parsing of the imports,
    which may change behavior.

    >>> import tempfile, os
    >>> f = tempfile.NamedTemporaryFile(dir='.', suffix='.v', delete=False)
    >>> g = tempfile.NamedTemporaryFile(dir='.', suffix='.v', delete=False)
    >>> g_name = os.path.relpath(g.name, '.')[:-2]
    >>> f.write("  Require  Import %s Coq.Init.Logic Arith.\n  Require  Export \tPArith\t Coq.Init.Logic.\n  Require Bool.\n Import Bool. (* asdf *)\n Require Import QArith\n  ZArith\n  Setoid.\nRequire Import %s.\n Require\n  Import\n%s\n\n\t.\t(*foo*)\n\nInductive t := a | b.\n\n(*asdf*)" % (g_name, g_name, g_name))
    >>> g.write(r"Require Export Ascii String.\n\nInductive q := c | d.")
    >>> f.close()
    >>> g.close()
    >>> print(include_imports(f.name, as_modules=False, verbose=False))
    Require Import Coq.Arith.Arith Coq.Bool.Bool Coq.Init.Logic Coq.PArith.PArith Coq.QArith.QArith Coq.Setoids.Setoid Coq.ZArith.ZArith Coq.Strings.Ascii Coq.Strings.String.

    Inductive q := c | d.
    (* asdf *)

    Inductive t := a | b.

    (*asdf*)

    >>> print(include_imports(f.name, as_modules=False, fast=True, verbose=False))
    Require Import Arith Bool Coq.Init.Logic PArith QArith Setoid ZArith Ascii String.

    Inductive q := c | d.
    (* asdf *)

    Inductive t := a | b.

    (*asdf*)
    >>> exts = ('.v', '.v.d', '.glob', '.vo', '.o', '.cmi', '.cmxs', '.native', '.cmx')
    >>> names = [f.name[:-2] + ext for ext in exts] + [g.name[:-2] + ext for ext in exts]
    >>> names = [i for i in names if os.path.exists(i)]
    >>> for name in names: os.remove(name)
    """
    if filename[-2:] != '.v': filename += '.v'
    lib = lib_of_filename(filename, topname=topname)
    all_imports = recursively_get_imports(lib, verbose=verbose, fast=fast, log=log, topname=topname, coqc=coqc)
    remaining_imports = []
    rtn = ''
    imports_done = []
    for import_name in all_imports:
        try:
            if as_modules:
                rtn += contents_as_module_without_require(import_name, imports_done, verbose=verbose, log=log, topname=topname) + '\n'
            else:
                rtn += contents_without_imports(import_name, verbose=verbose, log=log, topname=topname) + '\n'
            imports_done.append(import_name)
        except IOError:
            remaining_imports.append(import_name)
    if len(remaining_imports) > 0:
        if verbose:
            log('remaining imports:')
            log(remaining_imports)
        if as_modules:
            pattern = 'Require %s.\n%s'
        else:
            pattern = 'Require Import %s.\n%s'
        rtn = pattern % (' '.join(remaining_imports), rtn)
    return rtn

if __name__ == "__main__":
    # if we're working in python 3.3, we can test this file
    try:
        import doctest
        success = True
    except ImportError:
        print('This is not the main file to use.\nOnly run it if you have doctest (python 3.3+) and are testing things.')
        success = False
    if success:
        doctest.testmod()
<commit_msg>Fix for python 2.6<commit_after>from __future__ import with_statement
import os, subprocess, re, sys, glob
from memoize import memoize

__all__ = ["include_imports"]

file_mtimes = {}
file_contents = {}
lib_imports_fast = {}
lib_imports_slow = {}

IMPORT_REG = re.compile('^R[0-9]+:[0-9]+ ([^ ]+) <> <> lib$', re.MULTILINE)
IMPORT_LINE_REG = re.compile(r'^\s*(?:Require\s+Import|Require\s+Export|Require|Load\s+Verbose|Load)\s+(.*?)\.(?:\s|$)', re.MULTILINE | re.DOTALL)

def DEFAULT_LOG(text):
    print(text)

def filename_of_lib(lib, topname='__TOP__', ext='.v'):
    if lib[:len(topname + '.')] == topname + '.':
        lib = lib[len(topname + '.'):]
        lib = lib.replace('.', os.sep)
    else:
        # is this the right thing to do?
        lib = lib.replace('.', os.sep)
    return os.path.relpath(os.path.normpath(lib + ext), '.')

def lib_of_filename(filename, topname='__TOP__', exts=('.v', '.glob')):
    filename = os.path.relpath(filename, '.')
    for ext in exts:
        if filename[-len(ext):] == ext:
            filename = filename[:-len(ext)]
            break
    if '.' in filename:
        print("WARNING: There is a dot (.) in filename %s; the library conversion probably won't work." % filename)
    return topname + '.' + filename.replace(os.sep, '.')

def get_file(filename, verbose=True, log=DEFAULT_LOG):
    if filename[-2:] != '.v': filename += '.v'
    if filename not in file_contents.keys() or file_mtimes[filename] < os.stat(filename).st_mtime:
        if verbose: log('getting %s' % filename)
        try:
            with open(filename, 'r', encoding='UTF-8') as f:
                file_contents[filename] = f.read()
        except TypeError:
            with open(filename, 'r') as f:
                file_contents[filename] = f.read()
        file_mtimes[filename] = os.stat(filename).st_mtime
    return file_contents[filename]

def get_all_v_files(directory, exclude=tuple()):
    all_files = []
    exclude = [os.path.normpath(i) for i in exclude]
    for dirpath, dirnames, filenames in os.walk(directory):
        all_files += [os.path.relpath(name, '.') for name in glob.glob(os.path.join(dirpath, '*.v'))
                      if os.path.normpath(name) not in exclude]
    return all_files

@memoize
def get_makefile_contents(coqc, topname, v_files, verbose, log):
    cmds = ['coq_makefile', 'COQC', '=', coqc, '-R', '.', topname] + list(v_files)
    if verbose:
        log(' '.join(cmds))
    p_make_makefile = subprocess.Popen(cmds,
                                       stdout=subprocess.PIPE,
                                       stderr=subprocess.PIPE)
    return p_make_makefile.communicate()

def make_globs(libnames, verbose=True, topname='__TOP__', log=DEFAULT_LOG, coqc='coqc'):
    extant_libnames = [i for i in libnames
                       if os.path.exists(filename_of_lib(i, topname=topname, ext='.v'))]
    if len(extant_libnames) == 0: return
    filenames_v = [filename_of_lib(i, topname=topname, ext='.v') for i in extant_libnames]
    filenames_glob = [filename_of_lib(i, topname=topname, ext='.glob') for i in extant_libnames]
    if all(os.path.exists(glob_name) for glob_name in filenames_glob):
        return
    extra_filenames_v = get_all_v_files('.', filenames_v)
    (stdout, stderr) = get_makefile_contents(coqc, topname, tuple(sorted(filenames_v + extra_filenames_v)), verbose, log)
    if verbose:
        log(' '.join(['make', '-k', '-f', '-'] + filenames_glob))
    p_make = subprocess.Popen(['make', '-k', '-f', '-'] + filenames_glob, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    (stdout_make, stderr_make) = p_make.communicate(stdout)

def get_imports(lib, verbose=True, fast=False, log=DEFAULT_LOG, topname='__TOP__', coqc='coqc'):
    glob_name = filename_of_lib(lib, topname=topname, ext='.glob')
    v_name = filename_of_lib(lib, topname=topname, ext='.v')
    if not fast:
        if lib not in lib_imports_slow.keys():
            if not os.path.exists(glob_name):
                make_globs([lib], verbose=verbose, topname=topname, log=log, coqc=coqc)
            if os.path.exists(glob_name): # making succeeded
                with open(glob_name, 'r') as f:
                    contents = f.read()
                lines = contents.split('\n')
                lib_imports_slow[lib] = tuple(sorted(set(IMPORT_REG.findall(contents))))
                return lib_imports_slow[lib]
    # making globs failed, or we want the fast way, fall back to regexp
    if lib not in lib_imports_fast.keys():
        contents = get_file(v_name, verbose=verbose, log=log)
        imports_string = re.sub('\\s+', ' ', ' '.join(IMPORT_LINE_REG.findall(contents))).strip()
        lib_imports_fast[lib] = tuple(sorted(set(i for i in imports_string.split(' ') if i != '')))
    return lib_imports_fast[lib]

def norm_libname(lib, topname='__TOP__'):
    if lib[:len(topname + '.')] != topname + '.' and os.path.exists(filename_of_lib(lib, topname=topname)):
        return topname + '.' + lib
    return lib

def merge_imports(imports, topname='__TOP__'):
    rtn = []
    for import_list in imports:
        for i in import_list:
            if norm_libname(i, topname=topname) not in rtn:
                rtn.append(norm_libname(i, topname=topname))
    return rtn

def recursively_get_imports(lib, verbose=True, fast=False, log=DEFAULT_LOG, topname='__TOP__', coqc='coqc'):
    glob_name = filename_of_lib(lib, topname=topname, ext='.glob')
    v_name = filename_of_lib(lib, topname=topname, ext='.v')
    if os.path.exists(v_name):
        imports = get_imports(lib, verbose=verbose, fast=fast, topname=topname, coqc=coqc)
        if not fast: make_globs(imports, verbose=verbose, topname=topname, log=log, coqc=coqc)
        imports_list = [recursively_get_imports(i, verbose=verbose, fast=fast, log=log, topname=topname, coqc=coqc)
                        for i in imports]
        return merge_imports(imports_list + [[lib]], topname=topname)
    return [lib]

def contents_without_imports(lib, verbose=True, log=DEFAULT_LOG, topname='__TOP__'):
    v_file = filename_of_lib(lib, topname=topname, ext='.v')
    contents = get_file(v_file, verbose=verbose, log=log)
    if '(*' in ' '.join(IMPORT_LINE_REG.findall(contents)):
        print('Warning: There are comments in your Require/Import/Export lines in %s.' % filename)
    return IMPORT_LINE_REG.sub('', contents)

def escape_lib(lib):
    return lib.replace('.', '_DOT_')

def group_by_first_component(lib_libname_pairs):
    rtn = dict((lib.split('.')[0], []) for lib, libname in lib_libname_pairs)
    for lib, libname in lib_libname_pairs:
        split_lib = lib.split('.')
        rtn[split_lib[0]].append(('.'.join(split_lib[1:]), libname))
    return rtn

def recreate_path_structure(lib_libname_pairs, uid='', topname='__TOP__'):
    '''Takes a list of (<dotted name, escaped name>) pairs, and turns it into a nested module structure'''
    rtn = ''
    lib_libname_pairs = [(lib[len(topname + '.'):]
                          if lib[:len(topname + '.')] == topname + '.'
                          else lib,
                          libname)
                         for lib, libname in lib_libname_pairs]
    if topname != '': rtn += 'Module Import %s%s.\n' % (uid, topname)
    top_libnames = [libname for lib, libname in lib_libname_pairs if lib == '']
    lib_libname_pairs = [(lib, libname) for lib, libname in lib_libname_pairs
                         if lib != '']
    for top_libname in top_libnames:
        rtn += 'Include %s.\n' % top_libname
    for cur_lib, lib_list in group_by_first_component(lib_libname_pairs).items():
        rtn += 'Module %s.\n' % cur_lib
        rtn += recreate_path_structure(lib_list, topname='')
        if cur_lib in [other_lib for other_lib, other_libname in lib_list] and cur_lib not in top_libnames:
            rtn += 'Include %s.\n' % cur_lib
        rtn += 'End %s.\n' % cur_lib
    if topname != '': rtn += 'End %s%s.\n' % (uid, topname)
    return rtn

def contents_as_module_without_require(lib, other_imports, verbose=True, log=DEFAULT_LOG, topname='__TOP__'):
    v_name = filename_of_lib(lib, topname=topname, ext='.v')
    contents = get_file(v_name, verbose=verbose, log=log)
    reg1 = re.compile(r'^\s*Require\s+((?:Import|Export)\s)', flags=re.MULTILINE)
    contents = reg1.sub(r'\1', contents)
    reg2 = re.compile(r'^\s*Require\s+((?!Import\s+|Export\s+)(?:[^\.]|\.(?!\s|$))+\.(?:\s|$))', flags=re.MULTILINE)
    contents = reg2.sub(r'', contents)
    print(contents)
    module_name = escape_lib(lib)
    existing_imports = recreate_path_structure([(i, escape_lib(i)) for i in other_imports],
                                               uid=module_name,
                                               topname=topname)
    contents = 'Module %s.\n%s\n%s\nEnd %s.\n' % (module_name, existing_imports, contents, module_name)
    return contents

def include_imports(filename, as_modules=True, verbose=True, fast=False, log=DEFAULT_LOG, topname='__TOP__', coqc='coqc'):
    """Return the contents of filename, with any top-level imports inlined.

    If as_modules == True, then the imports will be wrapped in modules.

    This method requires access to the coqdep program if fast == False.
    Without it, it will fall back to manual parsing of the imports,
    which may change behavior.

    >>> import tempfile, os
    >>> f = tempfile.NamedTemporaryFile(dir='.', suffix='.v', delete=False)
    >>> g = tempfile.NamedTemporaryFile(dir='.', suffix='.v', delete=False)
    >>> g_name = os.path.relpath(g.name, '.')[:-2]
    >>> f.write("  Require  Import %s Coq.Init.Logic Arith.\n  Require  Export \tPArith\t Coq.Init.Logic.\n  Require Bool.\n Import Bool. (* asdf *)\n Require Import QArith\n  ZArith\n  Setoid.\nRequire Import %s.\n Require\n  Import\n%s\n\n\t.\t(*foo*)\n\nInductive t := a | b.\n\n(*asdf*)" % (g_name, g_name, g_name))
    >>> g.write(r"Require Export Ascii String.\n\nInductive q := c | d.")
    >>> f.close()
    >>> g.close()
    >>> print(include_imports(f.name, as_modules=False, verbose=False))
    Require Import Coq.Arith.Arith Coq.Bool.Bool Coq.Init.Logic Coq.PArith.PArith Coq.QArith.QArith Coq.Setoids.Setoid Coq.ZArith.ZArith Coq.Strings.Ascii Coq.Strings.String.

    Inductive q := c | d.
    (* asdf *)

    Inductive t := a | b.

    (*asdf*)

    >>> print(include_imports(f.name, as_modules=False, fast=True, verbose=False))
    Require Import Arith Bool Coq.Init.Logic PArith QArith Setoid ZArith Ascii String.

    Inductive q := c | d.
    (* asdf *)

    Inductive t := a | b.

    (*asdf*)
    >>> exts = ('.v', '.v.d', '.glob', '.vo', '.o', '.cmi', '.cmxs', '.native', '.cmx')
    >>> names = [f.name[:-2] + ext for ext in exts] + [g.name[:-2] + ext for ext in exts]
    >>> names = [i for i in names if os.path.exists(i)]
    >>> for name in names: os.remove(name)
    """
    if filename[-2:] != '.v': filename += '.v'
    lib = lib_of_filename(filename, topname=topname)
    all_imports = recursively_get_imports(lib, verbose=verbose, fast=fast, log=log, topname=topname, coqc=coqc)
    remaining_imports = []
    rtn = ''
    imports_done = []
    for import_name in all_imports:
        try:
            if as_modules:
                rtn += contents_as_module_without_require(import_name, imports_done, verbose=verbose, log=log, topname=topname) + '\n'
            else:
                rtn += contents_without_imports(import_name, verbose=verbose, log=log, topname=topname) + '\n'
            imports_done.append(import_name)
        except IOError:
            remaining_imports.append(import_name)
    if len(remaining_imports) > 0:
        if verbose:
            log('remaining imports:')
            log(remaining_imports)
        if as_modules:
            pattern = 'Require %s.\n%s'
        else:
            pattern = 'Require Import %s.\n%s'
        rtn = pattern % (' '.join(remaining_imports), rtn)
    return rtn

if __name__ == "__main__":
    # if we're working in python 3.3, we can test this file
    try:
        import doctest
        success = True
    except ImportError:
        print('This is not the main file to use.\nOnly run it if you have doctest (python 3.3+) and are testing things.')
        success = False
    if success:
        doctest.testmod()
<|endoftext|>


<commit_before>"""Heuristic search for generating action labels
"""

__author__ = "Liyan Chen"
__copyright__ = "Copyright (c) 2017 Malmactor"
__license__ = "MIT"

import copy
import itertools
import numpy as np
import pqdict

def l1_distance(a, b):
    return np.sum(np.abs(a - np.array(b)))

def l2_distance(a, b):
    if not isinstance(a, np.ndarray) and not isinstance(b, np.ndarray):
        a = np.array(a)
    return np.linalg.norm(a-b)

def heuristic(pos, target):
    return l1_distance(pos, target)

def encode_state(simulation):
    return tuple(simulation.mario.state.flatten())

def decode_state(encoding, simulation):
    dynamics = np.array(encoding).reshape((3, 3))
    simulation.mario.state = dynamics
    return simulation

def get_state_pos(encoding):
    if isinstance(encoding, tuple):
        return np.array(encoding).reshape((3, 3))[:, 0]
    else:
        return encoding.mario.state[:, 0]

def in_bound(state, bound):
    minx, miny, maxx, maxy = bound
    x, y = state[0:2]

    return x >= minx and y >= miny and x <= maxx and y <= maxy

def a_star(layout, simulation, init_pos, end_pos, actions, interval=5, config=None):
    """
    A* search for path-finding
    :param layout: 2D numpy array layout
    :param simulation: Physics simulation object
    :param init_pos: 3D vector of the initial position
    :param end_pos: 3D vector of the target position for search
    :param actions: Possible actions
    :param interval: Frame interval between two actions
    :return: Action sequence list
    """
    epsilon = 0.001 if config is None or "epsilon" not in config else config["epsilon"]
    empty_action = "remains" if config is None or "empty_action" not in config else config["empty_action"]

    bound = -1, -1, layout.shape[0], layout.shape[1]

    simulation.mario.state[0:3, 0] = init_pos
    init_state = encode_state(simulation)
    end_state = None

    state_pre = {init_state: None}
    state_preaction_map = {}
    cost = {init_state: 0}
    reversed_action_path = []

    frontier_queue = pqdict.minpq({init_state: heuristic(get_state_pos(init_state), end_pos)})

    expansion = 0
    greatest_x = 0

    while frontier_queue and not end_state:

        frontier = frontier_queue.pop()
        expansion += 1
        if frontier[0] > greatest_x:
            greatest_x = frontier[0]
            simulation = decode_state(frontier, simulation)
            config["render"].render(simulation.mario)
            print np.array(frontier).reshape((3, 3))

        # Expand frontier
        for act in actions:
            simulation = decode_state(frontier, simulation)
            simulation.advance_frame(act)

            next_cost = l2_distance(get_state_pos(frontier), get_state_pos(simulation)) + cost[frontier] + 1

            # Downsample actions
            for i in range(interval - 1):
                mid_state = encode_state(simulation)
                simulation.advance_frame(empty_action)
                next_cost += l2_distance(get_state_pos(mid_state), get_state_pos(simulation))

            next_state = encode_state(simulation)

            if in_bound(next_state, bound) and (next_state not in cost or next_cost < cost[next_state]):

                cost[next_state] = next_cost
                state_pre[next_state] = frontier
                state_preaction_map[next_state] = act

                h = heuristic(get_state_pos(next_state), end_pos) + next_cost
                frontier_queue[next_state] = h

                # Reach the end and exit
                if l1_distance(get_state_pos(next_state), end_pos) <= 0.5:
                    end_state = next_state
                    break

    # No solution found
    if not end_state:
        return []

    # Generate action sequences
    node = end_state
    while node in state_preaction_map:
        reversed_action_path.extend(itertools.repeat(empty_action, interval - 1))

        reversed_action_path.append(state_preaction_map[node])

        node = state_pre[node]

    return list(reversed(reversed_action_path))
<commit_msg>pruning search<commit_after>"""Heuristic search for generating action labels
"""

__author__ = "Liyan Chen"
__copyright__ = "Copyright (c) 2017 Malmactor"
__license__ = "MIT"

import copy
import itertools
import numpy as np
import pqdict

def l1_distance(a, b):
    return np.sum(np.abs(a - np.array(b)))

def l2_distance(a, b):
    if not isinstance(a, np.ndarray) and not isinstance(b, np.ndarray):
        a = np.array(a)
    return np.linalg.norm(a-b)

def heuristic(pos, target):
    return l1_distance(pos, target)

def encode_state(simulation):
    return tuple(simulation.mario.state.flatten())

def decode_state(encoding, simulation):
    dynamics = np.array(encoding).reshape((3, 3))
    simulation.mario.state = dynamics
    return simulation

def get_state_pos(encoding):
    if isinstance(encoding, tuple):
        return np.array(encoding).reshape((3, 3))[:, 0]
    else:
        return encoding.mario.state[:, 0]

def in_bound(state, bound):
    minx, miny, maxx, maxy = bound
    x, y = state[0:2]

    return x >= minx and y >= miny and x <= maxx and y <= maxy

def a_star(layout, simulation, init_pos, end_pos, actions, interval=5, config=None):
    """
    A* search for path-finding
    :param layout: 2D numpy array layout
    :param simulation: Physics simulation object
    :param init_pos: 3D vector of the initial position
    :param end_pos: 3D vector of the target position for search
    :param actions: Possible actions
    :param interval: Frame interval between two actions
    :return: Action sequence list
    """
    epsilon = 0.001 if config is None or "epsilon" not in config else config["epsilon"]
    empty_action = "remains" if config is None or "empty_action" not in config else config["empty_action"]

    bound = -1, -1, layout.shape[0], layout.shape[1]

    simulation.mario.state[0:3, 0] = init_pos
    init_state = encode_state(simulation)
    end_state = None

    state_pre = {init_state: None}
    state_preaction_map = {}
    cost = {init_state: 0}
    reversed_action_path = []

    frontier_queue = pqdict.minpq({init_state: heuristic(get_state_pos(init_state), end_pos)})

    expansion = 0
    greatest_x = 0
    max_q = l2_distance(init_pos, end_pos) * 10
    pruned_num = 0

    while frontier_queue and not end_state:

        frontier = frontier_queue.pop()
        expansion += 1

        # Expand frontier
        for act in actions:
            simulation = decode_state(frontier, simulation)
            simulation.advance_frame(act)

            next_cost = l2_distance(get_state_pos(frontier), get_state_pos(simulation)) + cost[frontier] + 1

            # Downsample actions
            for i in range(interval - 1):
                mid_state = encode_state(simulation)
                simulation.advance_frame(empty_action)
                next_cost += l2_distance(get_state_pos(mid_state), get_state_pos(simulation))

            next_state = encode_state(simulation)

            if in_bound(next_state, bound) and (next_state not in cost or next_cost < cost[next_state]):

                cost[next_state] = next_cost
                state_pre[next_state] = frontier
                state_preaction_map[next_state] = act

                h = heuristic(get_state_pos(next_state), end_pos) + next_cost
                if h < max_q:
                    frontier_queue[next_state] = h
                else:
                    pruned_num += 1
                    print pruned_num

                # Reach the end and exit
                if l1_distance(get_state_pos(next_state), end_pos) <= 0.5:
                    end_state = next_state
                    break

    # No solution found
    if not end_state:
        return []

    # Generate action sequences
    node = end_state
    while node in state_preaction_map:
        reversed_action_path.extend(itertools.repeat(empty_action, interval - 1))

        reversed_action_path.append(state_preaction_map[node])

        node = state_pre[node]

    return list(reversed(reversed_action_path))
<|endoftext|>


<commit_before>#!/usr/bin/env python

import igraph as ig
import tagspace as ts
import itertools

class ClusterTopics:
    def __init__(self, topics):
        self.topics = topics
        self.graph = None

    def make_complete_graph(self):
        self.graph = ig.Graph.Full(len(self.topics))
        self.graph.vs["name"] = self.topics
        self.graph.es["weight"] = 1.0
        return self.graph

    def assign_edge_weights(self):
        vertices = self.graph.vs["name"]
        vertex_pairs = list(itertools.combinations(vertices, 2))

        for pair in vertex_pairs:
            self.graph[pair[0], pair[1]] = topic_sim(pair[0], pair[1])

        return self.graph

    def cluster(self):
        return self.graph.community_fastgreedy()

<commit_msg>TopicRelation class<commit_after>#!/usr/bin/env python

import igraph as ig
import tagspace as ts
import itertools

class ClusterTopics:
    def __init__(self, topics):
        self.topics = topics
        self.graph = None

    def make_complete_graph(self):
        self.graph = ig.Graph.Full(len(self.topics))
        self.graph.vs["name"] = self.topics
        self.graph.es["weight"] = 1.0
        return self.graph

    def assign_edge_weights(self):
        vertices = self.graph.vs["name"]
        vertex_pairs = list(itertools.combinations(vertices, 2))

        for pair in vertex_pairs:
            self.graph[pair[0], pair[1]] = topic_sim(pair[0], pair[1])

        return self.graph

    def cluster(self):
        return self.graph.community_fastgreedy()

class TopicRelation:
    def __init__(self, articles):
        self.articles = articles
        self.article_paths = self.build_paths()

    def find_intersection(self, results_x, results_y):
        return set(results_x).intersection(results_y)

    def build_paths(self):
        article_paths = {}
        for article in self.articles:
            article_paths[article] = ts.IterateArticles(article).traverse()
        return article_paths
<|endoftext|>


<commit_before>import RPi.GPIO as GPIO
import time
import sys
from enum import Enum

class Direction(Enum):
    forward = 1
    backward = 1

class Stepper:
    def __init__(self, power_a, ground_a, power_b, ground_b):
        self.coil_A_1_pin = power_a
        self.coil_A_2_pin = ground_a
        self.coil_B_1_pin = power_b
        self.coil_B_2_pin = ground_b

    def forward(delay, steps):
        for i in range(0, steps):
            setStep(1, 0, 1, 0)
            time.sleep(delay)
            setStep(0, 1, 1, 0)
            time.sleep(delay)
            setStep(0, 1, 0, 1)
            time.sleep(delay)
            setStep(1, 0, 0, 1)
            time.sleep(delay)

    def backwards(delay, steps):
        for i in range(0, steps):
            setStep(1, 0, 0, 1)
            time.sleep(delay)
            setStep(0, 1, 0, 1)
            time.sleep(delay)
            setStep(0, 1, 1, 0)
            time.sleep(delay)
            setStep(1, 0, 1, 0)
            time.sleep(delay)

    def setStep(w1, w2, w3, w4):
        GPIO.output(coil_A_1_pin, w1)
        GPIO.output(coil_A_2_pin, w2)
        GPIO.output(coil_B_1_pin, w3)
        GPIO.output(coil_B_2_pin, w4)

    def run(delay, steps, direction)
        if direction == Direction.Forward
            forward(int(delay) / 1000.0, int(steps))
        else
            backwards(int(delay) / 1000.0, int(steps))

<commit_msg>fixed syntax errors<commit_after>import RPi.GPIO as GPIO
import time
import sys
from enum import Enum

class Direction(Enum):
    forward = 1
    backward = 1

class Stepper:
    def __init__(self, power_a, ground_a, power_b, ground_b):
        self.coil_A_1_pin = power_a
        self.coil_A_2_pin = ground_a
        self.coil_B_1_pin = power_b
        self.coil_B_2_pin = ground_b

    def forward(delay, steps):
        for i in range(0, steps):
            setStep(1, 0, 1, 0)
            time.sleep(delay)
            setStep(0, 1, 1, 0)
            time.sleep(delay)
            setStep(0, 1, 0, 1)
            time.sleep(delay)
            setStep(1, 0, 0, 1)
            time.sleep(delay)

    def backwards(delay, steps):
        for i in range(0, steps):
            setStep(1, 0, 0, 1)
            time.sleep(delay)
            setStep(0, 1, 0, 1)
            time.sleep(delay)
            setStep(0, 1, 1, 0)
            time.sleep(delay)
            setStep(1, 0, 1, 0)
            time.sleep(delay)

    def setStep(w1, w2, w3, w4):
        GPIO.output(coil_A_1_pin, w1)
        GPIO.output(coil_A_2_pin, w2)
        GPIO.output(coil_B_1_pin, w3)
        GPIO.output(coil_B_2_pin, w4)

    def run(delay, steps, direction):
        if direction == Direction.Forward:
            forward(int(delay) / 1000.0, int(steps))
        else:
            backwards(int(delay) / 1000.0, int(steps))

<|endoftext|>