<commit_before>from gevent.event import AsyncResult
import gevent
import time
from eth_protocol import TransientBlock
from ethereum.slogging import get_logger
import traceback

log = get_logger('eth.sync')
log_st = get_logger('eth.sync.task')

class SyncTask(object):

    """
    synchronizes a the chain starting from a given blockhash
    blockchain hash is fetched from a single peer (which led to the unknown blockhash)
    blocks are fetched from the best peers

    with missing block:
        fetch hashes
            until known block
    for hashes
        fetch blocks
            for each block
                chainservice.add_blocks() # blocks if queue is full
    """
    max_blocks_per_request = 256
    initial_blockhashes_per_request = 16
    max_blockhashes_per_request = 2048
    blocks_request_timeout = 8.  # 256 * ~2KB = 512KB
    blockhashes_request_timeout = 8.  # 32 * 2048 = 65KB

    def __init__(self, synchronizer, proto, blockhash, chain_difficulty=0, originator_only=False):
        self.synchronizer = synchronizer
        self.chain = synchronizer.chain
        self.chainservice = synchronizer.chainservice
        self.originating_proto = proto
        self.originator_only = originator_only
        self.blockhash = blockhash
        self.chain_difficulty = chain_difficulty
        self.requests = dict()  # proto: Event
        gevent.spawn(self.run)

    def run(self):
        log_st.info('spawning new syntask')
        try:
            self.fetch_hashchain()
        except Exception:
            print(traceback.format_exc())
            self.exit(success=False)

    def exit(self, success=False):
        if not success:
            log_st.warn('syncing failed')
        else:
            log_st.debug('successfully synced')
        self.synchronizer.synctask_exited(success)

    @property
    def protocols(self):
        if self.originator_only:
            return [self.originating_proto]
        return self.synchronizer.protocols

    def fetch_hashchain(self):
        log_st.debug('fetching hashchain')
        blockhashes_chain = [self.blockhash]  # youngest to oldest

        blockhash = self.blockhash
        assert blockhash not in self.chain

        # get block hashes until we found a known one
        max_blockhashes_per_request = self.initial_blockhashes_per_request
        while blockhash not in self.chain:
            # proto with highest_difficulty should be the proto we got the newblock from
            blockhashes_batch = []

            # try with protos
            protocols = self.protocols
            if not protocols:
                log_st.warn('no protocols available')
                return self.exit(success=False)

            for proto in protocols:
                log.debug('syncing with', proto=proto)
                if proto.is_stopped:
                    continue

                # request
                assert proto not in self.requests
                deferred = AsyncResult()
                self.requests[proto] = deferred
                proto.send_getblockhashes(blockhash, max_blockhashes_per_request)
                try:
                    blockhashes_batch = deferred.get(block=True,
                                                     timeout=self.blockhashes_request_timeout)
                except gevent.Timeout:
                    log_st.warn('syncing hashchain timed out')
                    continue
                finally:
                    # is also executed 'on the way out' when any other clause of the try statement
                    # is left via a break, continue or return statement.
                    del self.requests[proto]

                if not blockhashes_batch:
                    log_st.warn('empty getblockhashes result')
                    continue
                if not all(isinstance(bh, bytes) for bh in blockhashes_batch):
                    log_st.warn('got wrong data type', expected='bytes',
                                received=type(blockhashes_batch[0]))
                    continue
                break

            if not blockhashes_batch:
                log_st.warn('syncing failed with all peers', num_protos=len(protocols))
                return self.exit(success=False)

            for blockhash in blockhashes_batch:  # youngest to oldest
                assert isinstance(blockhash, bytes)
                if blockhash not in self.chain:
                    blockhashes_chain.append(blockhash)
                else:
                    log_st.debug('found known blockhash', blockhash=blockhash.encode('hex'),
                                 is_genesis=bool(blockhash == self.chain.genesis.hash))
                    break
            log_st.debug('downloaded ' + str(len(blockhashes_chain)) + ' block hashes')
            max_blockhashes_per_request = self.max_blockhashes_per_request

        self.fetch_blocks(blockhashes_chain)

    def fetch_blocks(self, blockhashes_chain):
        # fetch blocks (no parallelism here)
        log_st.debug('fetching blocks', num=len(blockhashes_chain))
        assert blockhashes_chain
        blockhashes_chain.reverse()  # oldest to youngest
        num_blocks = len(blockhashes_chain)
        num_fetched = 0

        while blockhashes_chain:
            blockhashes_batch = blockhashes_chain[:self.max_blocks_per_request]
            t_blocks = []

            # try with protos
            protocols = self.protocols
            if not protocols:
                log_st.warn('no protocols available')
                return self.exit(success=False)

            for proto in protocols:
                if proto.is_stopped:
                    continue
                assert proto not in self.requests
                # request
                log_st.debug('requesting blocks', num=len(blockhashes_batch))
                deferred = AsyncResult()
                self.requests[proto] = deferred
                proto.send_getblocks(*blockhashes_batch)
                try:
                    t_blocks = deferred.get(block=True, timeout=self.blocks_request_timeout)
                except gevent.Timeout:
                    log_st.warn('getblocks timed out, trying next proto')
                    continue
                finally:
                    del self.requests[proto]
                if not t_blocks:
                    log_st.warn('empty getblocks reply, trying next proto')
                    continue
                elif not isinstance(t_blocks[0], TransientBlock):
                    log_st.warn('received unexpected data', data=repr(t_blocks))
                    t_blocks = []
                    continue
                # we have results
                if not [b.header.hash for b in t_blocks] == blockhashes_batch[:len(t_blocks)]:
                    log_st.warn('received wrong blocks, should ban peer')
                    t_blocks = []
                    continue
                break

            # add received t_blocks
            num_fetched += len(t_blocks)
            log_st.debug('received blocks', num=len(t_blocks), num_fetched=num_fetched,
                         total=num_blocks, missing=num_blocks - num_fetched)

            if not t_blocks:
                log_st.warn('failed to fetch blocks', missing=len(blockhashes_chain))
                return self.exit(success=False)

            ts = time.time()
            log_st.debug('adding blocks', qsize=self.chainservice.block_queue.qsize())
            for t_block in t_blocks:
                b = blockhashes_chain.pop(0)
                assert t_block.header.hash == b
                assert t_block.header.hash not in blockhashes_chain
                self.chainservice.add_block(t_block, proto)  # this blocks if the queue is full
            log_st.debug('adding blocks done', took=time.time() - ts)

        # done
        last_block = t_block
        assert not len(blockhashes_chain)
        assert last_block.header.hash == self.blockhash
        log_st.debug('syncing finished')
        # at this point blocks are not in the chain yet, but in the add_block queue
        if self.chain_difficulty >= self.chain.head.chain_difficulty():
            self.chainservice.broadcast_newblock(last_block, self.chain_difficulty, origin=proto)

        self.exit(success=True)

    def receive_blocks(self, proto, t_blocks):
        log.debug('blocks received', proto=proto, num=len(t_blocks))
        if proto not in self.requests:
            log.debug('unexpected blocks')
            return
        self.requests[proto].set(t_blocks)

    def receive_blockhashes(self, proto, blockhashes):
        log.debug('blockhashes received', proto=proto, num=len(blockhashes))
        if proto not in self.requests:
            log.debug('unexpected blockhashes')
            return
        self.requests[proto].set(blockhashes)

class Synchronizer(object):

    """
    handles the synchronization of blocks

    there is only one synctask active at a time
    in order to deal with the worst case of initially syncing the wrong chain,
        a checkpoint blockhash can be specified and synced via force_sync

    received blocks are given to chainservice.add_block
    which has a fixed size queue, the synchronization blocks if the queue is full

    on_status:
        if peer.head.chain_difficulty > chain.head.chain_difficulty
            fetch peer.head and handle as newblock
    on_newblock:
        if block.parent:
            add
        else:
            sync
    on_blocks/on_blockhashes:
        if synctask:
            handle to requester
        elif unknown and has parent:
            add to chain
        else:
            drop
    """

    def __init__(self, chainservice, force_sync=None):
        """
        @param: force_sync None or tuple(blockhash, chain_difficulty)
                helper for long initial syncs to get on the right chain
                used with first status_received
        """
        self.chainservice = chainservice
        self.force_sync = force_sync
        self.chain = chainservice.chain
        self._protocols = dict()  # proto: chain_difficulty
        self.synctask = None

    def synctask_exited(self, success=False):
        # note: synctask broadcasts best block
        if success:
            self.force_sync = None
        self.synctask = None

    @property
    def protocols(self):
        "return protocols which are not stopped sorted by highest chain_difficulty"
        # filter and cleanup
        self._protocols = dict((p, cd) for p, cd in self._protocols.items() if not p.is_stopped)
        return sorted(self._protocols.keys(), key=lambda p: self._protocols[p], reverse=True)

    def receive_newblock(self, proto, t_block, chain_difficulty):
        "called if there's a newblock announced on the network"
        log.debug('newblock', proto=proto, block=t_block, chain_difficulty=chain_difficulty,
                  client=proto.peer.remote_client_version)

        if t_block.header.hash in self.chain:
            assert chain_difficulty == self.chain.get(t_block.header.hash).chain_difficulty()

        # memorize proto with difficulty
        self._protocols[proto] = chain_difficulty

        if self.chainservice.knows_block(block_hash=t_block.header.hash):
            log.debug('known block')
            return

        # check pow
        if not t_block.header.check_pow():
            log.warn('check pow failed, should ban!')
            return

        expected_difficulty = self.chain.head.chain_difficulty() + t_block.header.difficulty
        if chain_difficulty >= self.chain.head.chain_difficulty():
            # broadcast duplicates filtering is done in eth_service
            log.debug('sufficient difficulty, broadcasting',
                      client=proto.peer.remote_client_version)
            self.chainservice.broadcast_newblock(t_block, chain_difficulty, origin=proto)
        else:
            # any criteria for which blocks/chains not to add?
            max_age = 5
            age = self.chain.head.number - t_block.header.number
            log.debug('low difficulty', client=proto.peer.remote_client_version,
                      chain_difficulty=chain_difficulty, expected_difficulty=expected_difficulty,
                      block_age=age)
            if age > max_age:
                log.debug('newblock is too old, not adding', block_age=age, max_age=max_age)
                return

        # unknown and pow check and highest difficulty

        # check if we have parent
        if self.chainservice.knows_block(block_hash=t_block.header.prevhash):
            log.debug('adding block')
            self.chainservice.add_block(t_block, proto)
        else:
            log.debug('missing parent')
            if not self.synctask:
                self.synctask = SyncTask(self, proto, t_block.header.hash, chain_difficulty)
            else:
                log.debug('existing task, discarding')

    def receive_status(self, proto, blockhash, chain_difficulty):
        "called if a new peer is connected"
        log.debug('status received', proto=proto, chain_difficulty=chain_difficulty)

        # memorize proto with difficulty
        self._protocols[proto] = chain_difficulty

        if self.chainservice.knows_block(blockhash) or self.synctask:
            log.debug('existing task or known hash, discarding')
            return

        if self.force_sync:
            blockhash, chain_difficulty = self.force_sync
            log.debug('starting forced syctask', blockhash=blockhash.encode('hex'))
            self.synctask = SyncTask(self, proto, blockhash, chain_difficulty)

        elif chain_difficulty > self.chain.head.chain_difficulty():
            log.debug('sufficient difficulty')
            self.synctask = SyncTask(self, proto, blockhash, chain_difficulty)

    def receive_newblockhashes(self, proto, newblockhashes):
        """
        no way to check if this really an interesting block at this point.
        might lead to an amplification attack, need to track this proto and judge usefullness
        """
        log.debug('received newblockhashes', num=len(newblockhashes), proto=proto)
        # log.debug('DISABLED')
        # return
        newblockhashes = [h for h in newblockhashes if not self.chainservice.knows_block(h)]
        if (proto not in self.protocols) or (not newblockhashes) or self.synctask:
            log.debug('discarding', known=bool(not newblockhashes), synctask=bool(self.synctask))
            return
        if len(newblockhashes) != 1:
            log.warn('supporting only one newblockhash', num=len(newblockhashes))
        blockhash = newblockhashes[0]
        log.debug('starting synctask for newblockhashes', blockhash=blockhash.encode('hex'))
        self.synctask = SyncTask(self, proto, blockhash, 0, originator_only=True)

    def receive_blocks(self, proto, t_blocks):
        log.debug('blocks received', proto=proto, num=len(t_blocks))
        if self.synctask:
            self.synctask.receive_blocks(proto, t_blocks)
        else:
            log.warn('no synctask, not expecting blocks')

    def receive_blockhashes(self, proto, blockhashes):
        log.debug('blockhashes received', proto=proto, num=len(blockhashes))
        if self.synctask:
            self.synctask.receive_blockhashes(proto, blockhashes)
        else:
            log.warn('no synctask, not expecting blockhashes')
<commit_msg>adjust sync request sizes<commit_after>from gevent.event import AsyncResult
import gevent
import time
from eth_protocol import TransientBlock
from ethereum.slogging import get_logger
import traceback

log = get_logger('eth.sync')
log_st = get_logger('eth.sync.task')

class SyncTask(object):

    """
    synchronizes a the chain starting from a given blockhash
    blockchain hash is fetched from a single peer (which led to the unknown blockhash)
    blocks are fetched from the best peers

    with missing block:
        fetch hashes
            until known block
    for hashes
        fetch blocks
            for each block
                chainservice.add_blocks() # blocks if queue is full
    """
    max_blocks_per_request = 32
    initial_blockhashes_per_request = 16
    max_blockhashes_per_request = 96
    blocks_request_timeout = 32.
    blockhashes_request_timeout = 32.

    def __init__(self, synchronizer, proto, blockhash, chain_difficulty=0, originator_only=False):
        self.synchronizer = synchronizer
        self.chain = synchronizer.chain
        self.chainservice = synchronizer.chainservice
        self.originating_proto = proto
        self.originator_only = originator_only
        self.blockhash = blockhash
        self.chain_difficulty = chain_difficulty
        self.requests = dict()  # proto: Event
        gevent.spawn(self.run)

    def run(self):
        log_st.info('spawning new syntask')
        try:
            self.fetch_hashchain()
        except Exception:
            print(traceback.format_exc())
            self.exit(success=False)

    def exit(self, success=False):
        if not success:
            log_st.warn('syncing failed')
        else:
            log_st.debug('successfully synced')
        self.synchronizer.synctask_exited(success)

    @property
    def protocols(self):
        if self.originator_only:
            return [self.originating_proto]
        return self.synchronizer.protocols

    def fetch_hashchain(self):
        log_st.debug('fetching hashchain')
        blockhashes_chain = [self.blockhash]  # youngest to oldest

        blockhash = self.blockhash
        assert blockhash not in self.chain

        # get block hashes until we found a known one
        max_blockhashes_per_request = self.initial_blockhashes_per_request
        while blockhash not in self.chain:
            # proto with highest_difficulty should be the proto we got the newblock from
            blockhashes_batch = []

            # try with protos
            protocols = self.protocols
            if not protocols:
                log_st.warn('no protocols available')
                return self.exit(success=False)

            for proto in protocols:
                log.debug('syncing with', proto=proto)
                if proto.is_stopped:
                    continue

                # request
                assert proto not in self.requests
                deferred = AsyncResult()
                self.requests[proto] = deferred
                proto.send_getblockhashes(blockhash, max_blockhashes_per_request)
                try:
                    blockhashes_batch = deferred.get(block=True,
                                                     timeout=self.blockhashes_request_timeout)
                except gevent.Timeout:
                    log_st.warn('syncing hashchain timed out')
                    continue
                finally:
                    # is also executed 'on the way out' when any other clause of the try statement
                    # is left via a break, continue or return statement.
                    del self.requests[proto]

                if not blockhashes_batch:
                    log_st.warn('empty getblockhashes result')
                    continue
                if not all(isinstance(bh, bytes) for bh in blockhashes_batch):
                    log_st.warn('got wrong data type', expected='bytes',
                                received=type(blockhashes_batch[0]))
                    continue
                break

            if not blockhashes_batch:
                log_st.warn('syncing failed with all peers', num_protos=len(protocols))
                return self.exit(success=False)

            for blockhash in blockhashes_batch:  # youngest to oldest
                assert isinstance(blockhash, bytes)
                if blockhash not in self.chain:
                    blockhashes_chain.append(blockhash)
                else:
                    log_st.debug('found known blockhash', blockhash=blockhash.encode('hex'),
                                 is_genesis=bool(blockhash == self.chain.genesis.hash))
                    break
            log_st.debug('downloaded ' + str(len(blockhashes_chain)) + ' block hashes')
            max_blockhashes_per_request = self.max_blockhashes_per_request

        self.fetch_blocks(blockhashes_chain)

    def fetch_blocks(self, blockhashes_chain):
        # fetch blocks (no parallelism here)
        log_st.debug('fetching blocks', num=len(blockhashes_chain))
        assert blockhashes_chain
        blockhashes_chain.reverse()  # oldest to youngest
        num_blocks = len(blockhashes_chain)
        num_fetched = 0

        while blockhashes_chain:
            blockhashes_batch = blockhashes_chain[:self.max_blocks_per_request]
            t_blocks = []

            # try with protos
            protocols = self.protocols
            if not protocols:
                log_st.warn('no protocols available')
                return self.exit(success=False)

            for proto in protocols:
                if proto.is_stopped:
                    continue
                assert proto not in self.requests
                # request
                log_st.debug('requesting blocks', num=len(blockhashes_batch))
                deferred = AsyncResult()
                self.requests[proto] = deferred
                proto.send_getblocks(*blockhashes_batch)
                try:
                    t_blocks = deferred.get(block=True, timeout=self.blocks_request_timeout)
                except gevent.Timeout:
                    log_st.warn('getblocks timed out, trying next proto')
                    continue
                finally:
                    del self.requests[proto]
                if not t_blocks:
                    log_st.warn('empty getblocks reply, trying next proto')
                    continue
                elif not isinstance(t_blocks[0], TransientBlock):
                    log_st.warn('received unexpected data', data=repr(t_blocks))
                    t_blocks = []
                    continue
                # we have results
                if not [b.header.hash for b in t_blocks] == blockhashes_batch[:len(t_blocks)]:
                    log_st.warn('received wrong blocks, should ban peer')
                    t_blocks = []
                    continue
                break

            # add received t_blocks
            num_fetched += len(t_blocks)
            log_st.debug('received blocks', num=len(t_blocks), num_fetched=num_fetched,
                         total=num_blocks, missing=num_blocks - num_fetched)

            if not t_blocks:
                log_st.warn('failed to fetch blocks', missing=len(blockhashes_chain))
                return self.exit(success=False)

            ts = time.time()
            log_st.debug('adding blocks', qsize=self.chainservice.block_queue.qsize())
            for t_block in t_blocks:
                b = blockhashes_chain.pop(0)
                assert t_block.header.hash == b
                assert t_block.header.hash not in blockhashes_chain
                self.chainservice.add_block(t_block, proto)  # this blocks if the queue is full
            log_st.debug('adding blocks done', took=time.time() - ts)

        # done
        last_block = t_block
        assert not len(blockhashes_chain)
        assert last_block.header.hash == self.blockhash
        log_st.debug('syncing finished')
        # at this point blocks are not in the chain yet, but in the add_block queue
        if self.chain_difficulty >= self.chain.head.chain_difficulty():
            self.chainservice.broadcast_newblock(last_block, self.chain_difficulty, origin=proto)

        self.exit(success=True)

    def receive_blocks(self, proto, t_blocks):
        log.debug('blocks received', proto=proto, num=len(t_blocks))
        if proto not in self.requests:
            log.debug('unexpected blocks')
            return
        self.requests[proto].set(t_blocks)

    def receive_blockhashes(self, proto, blockhashes):
        log.debug('blockhashes received', proto=proto, num=len(blockhashes))
        if proto not in self.requests:
            log.debug('unexpected blockhashes')
            return
        self.requests[proto].set(blockhashes)

class Synchronizer(object):

    """
    handles the synchronization of blocks

    there is only one synctask active at a time
    in order to deal with the worst case of initially syncing the wrong chain,
        a checkpoint blockhash can be specified and synced via force_sync

    received blocks are given to chainservice.add_block
    which has a fixed size queue, the synchronization blocks if the queue is full

    on_status:
        if peer.head.chain_difficulty > chain.head.chain_difficulty
            fetch peer.head and handle as newblock
    on_newblock:
        if block.parent:
            add
        else:
            sync
    on_blocks/on_blockhashes:
        if synctask:
            handle to requester
        elif unknown and has parent:
            add to chain
        else:
            drop
    """

    def __init__(self, chainservice, force_sync=None):
        """
        @param: force_sync None or tuple(blockhash, chain_difficulty)
                helper for long initial syncs to get on the right chain
                used with first status_received
        """
        self.chainservice = chainservice
        self.force_sync = force_sync
        self.chain = chainservice.chain
        self._protocols = dict()  # proto: chain_difficulty
        self.synctask = None

    def synctask_exited(self, success=False):
        # note: synctask broadcasts best block
        if success:
            self.force_sync = None
        self.synctask = None

    @property
    def protocols(self):
        "return protocols which are not stopped sorted by highest chain_difficulty"
        # filter and cleanup
        self._protocols = dict((p, cd) for p, cd in self._protocols.items() if not p.is_stopped)
        return sorted(self._protocols.keys(), key=lambda p: self._protocols[p], reverse=True)

    def receive_newblock(self, proto, t_block, chain_difficulty):
        "called if there's a newblock announced on the network"
        log.debug('newblock', proto=proto, block=t_block, chain_difficulty=chain_difficulty,
                  client=proto.peer.remote_client_version)

        if t_block.header.hash in self.chain:
            assert chain_difficulty == self.chain.get(t_block.header.hash).chain_difficulty()

        # memorize proto with difficulty
        self._protocols[proto] = chain_difficulty

        if self.chainservice.knows_block(block_hash=t_block.header.hash):
            log.debug('known block')
            return

        # check pow
        if not t_block.header.check_pow():
            log.warn('check pow failed, should ban!')
            return

        expected_difficulty = self.chain.head.chain_difficulty() + t_block.header.difficulty
        if chain_difficulty >= self.chain.head.chain_difficulty():
            # broadcast duplicates filtering is done in eth_service
            log.debug('sufficient difficulty, broadcasting',
                      client=proto.peer.remote_client_version)
            self.chainservice.broadcast_newblock(t_block, chain_difficulty, origin=proto)
        else:
            # any criteria for which blocks/chains not to add?
            max_age = 5
            age = self.chain.head.number - t_block.header.number
            log.debug('low difficulty', client=proto.peer.remote_client_version,
                      chain_difficulty=chain_difficulty, expected_difficulty=expected_difficulty,
                      block_age=age)
            if age > max_age:
                log.debug('newblock is too old, not adding', block_age=age, max_age=max_age)
                return

        # unknown and pow check and highest difficulty

        # check if we have parent
        if self.chainservice.knows_block(block_hash=t_block.header.prevhash):
            log.debug('adding block')
            self.chainservice.add_block(t_block, proto)
        else:
            log.debug('missing parent')
            if not self.synctask:
                self.synctask = SyncTask(self, proto, t_block.header.hash, chain_difficulty)
            else:
                log.debug('existing task, discarding')

    def receive_status(self, proto, blockhash, chain_difficulty):
        "called if a new peer is connected"
        log.debug('status received', proto=proto, chain_difficulty=chain_difficulty)

        # memorize proto with difficulty
        self._protocols[proto] = chain_difficulty

        if self.chainservice.knows_block(blockhash) or self.synctask:
            log.debug('existing task or known hash, discarding')
            return

        if self.force_sync:
            blockhash, chain_difficulty = self.force_sync
            log.debug('starting forced syctask', blockhash=blockhash.encode('hex'))
            self.synctask = SyncTask(self, proto, blockhash, chain_difficulty)

        elif chain_difficulty > self.chain.head.chain_difficulty():
            log.debug('sufficient difficulty')
            self.synctask = SyncTask(self, proto, blockhash, chain_difficulty)

    def receive_newblockhashes(self, proto, newblockhashes):
        """
        no way to check if this really an interesting block at this point.
        might lead to an amplification attack, need to track this proto and judge usefullness
        """
        log.debug('received newblockhashes', num=len(newblockhashes), proto=proto)
        # log.debug('DISABLED')
        # return
        newblockhashes = [h for h in newblockhashes if not self.chainservice.knows_block(h)]
        if (proto not in self.protocols) or (not newblockhashes) or self.synctask:
            log.debug('discarding', known=bool(not newblockhashes), synctask=bool(self.synctask))
            return
        if len(newblockhashes) != 1:
            log.warn('supporting only one newblockhash', num=len(newblockhashes))
        blockhash = newblockhashes[0]
        log.debug('starting synctask for newblockhashes', blockhash=blockhash.encode('hex'))
        self.synctask = SyncTask(self, proto, blockhash, 0, originator_only=True)

    def receive_blocks(self, proto, t_blocks):
        log.debug('blocks received', proto=proto, num=len(t_blocks))
        if self.synctask:
            self.synctask.receive_blocks(proto, t_blocks)
        else:
            log.warn('no synctask, not expecting blocks')

    def receive_blockhashes(self, proto, blockhashes):
        log.debug('blockhashes received', proto=proto, num=len(blockhashes))
        if self.synctask:
            self.synctask.receive_blockhashes(proto, blockhashes)
        else:
            log.warn('no synctask, not expecting blockhashes')
<|endoftext|>


<commit_before>#!/usr/bin/env python2.3
#
# (C) 2004 British Broadcasting Corporation and Kamaelia Contributors(1)
#     All Rights Reserved.
#
# You may only modify and redistribute this under the terms of any of the
# following licenses(2): Mozilla Public License, V1.1, GNU General
# Public License, V2.0, GNU Lesser General Public License, V2.1
#
# (1) Kamaelia Contributors are listed in the AUTHORS file and at
#     http://kamaelia.sourceforge.net/AUTHORS - please extend this file,
#     not this notice.
# (2) Reproduced in the COPYING file, and at:
#     http://kamaelia.sourceforge.net/COPYING
# Under section 3.5 of the MPL, we are using this text since we deem the MPL
# notice inappropriate for this file. As per MPL/GPL/LGPL removal of this
# notice is prohibited.
#
# Please contact us via: kamaelia-list-owner@lists.sourceforge.net
# to discuss alternative licensing.
# -------------------------------------------------------------------------
"""Kamaelia Concurrency Component Framework.

SCHEDULER

You provide the scheduler with microthreads of microprocesses,
and it runs them, clean and simple. This is usually handled by the component
system via returning a newComponent value. An alternative is to call the
component's activate method.

It also has a slow motion mode designed to help with debugging & testing.

"""
import time
import gc as _gc

from util import removeAll
from idGen import strId, numId
from debug import debug
from Microprocess import microprocess
from Axon import AxonObject as _AxonObject
from Ipc import *

def _sort(somelist):
   a=[ x for x in somelist]
   a.sort()
   return a

def activateMicroprocesses(result):
     "activatedMicroprocesses = self.activateMicroprocesses(result)"
     # The class sent us a newComponent message, which
     # can contain several components.
     activatedMicroprocesses = 0
     for C in result.components():   # For each one
        t = C.activate()      # Activate it - adds itself to the runset
        if t._activityCreator():
           activatedMicroprocesses +=1
     return activatedMicroprocesses

class scheduler(microprocess):
   """Scheduler - runs microthreads of control."""
   run = None
   def __init__(self):
      """Creates a scheduler object. If scheduler.run has not been set, sets it.
      Class initialisation ensures that this object/class attribute is initialised - client
      modules always have access to a standalone scheduler.
      Internal attributes:
         * time = time when this object was last active.
         * threads = list of threads to run.
      Whilst there can be more than one scheduler active in the general case you will NOT
      want to create a custom scheduler.
      """
      super(scheduler, self).__init__()
      if not scheduler.run:         # If no scheduler already allocated...
         scheduler.run = self       # Make this scheduler the singleton scheduler.

      self.time = time.time()
      self.threads = list()     # Don't use [] to avoid Python wierdness
      self.newthreads = list()

   def _addThread(self, mprocess):
      """A Microprocess adds itself to the runqueue using this method, using
      the mannerism scheduler.run._addThread(). Generally components will *not*
      use this method to activate themselves, see the component class to see
      how you do that.
      """
      self.newthreads.append(mprocess)

   def handleMicroprocessShutdownKnockon(self, knockon):
     if isinstance(knockon, shutdownMicroprocess):
        for i in xrange(len(self.threads)):
           if self.threads[i] in knockon.microprocesses():
              self.threads[i] = None

     if isinstance(knockon, reactivate):
        self._addThread(knockon.original)

   def main(self,slowmo=0):
      """This is the meat of the scheduler  - this actively loops round the threads
      that it has available to run, and runs them. The only control over the scheduler
      at present is a means to slow it down - ie run in slow motion.
      The way this is runn is as follows:
          scheduler.run.runThreads(slowmo=/delay/)
      where delay is in seconds. If the delay is 0, the the system runs all the
      threads as fast as it can. If the delay is non zero - eg 0.5, then the system
      runs all the threads for one "cycle", waits until the delay has passed, and then
      times again. Note : the delay is between the start points of cycles, and not
      between the start and end points of cycles. The delay is NOT 100% accurate
      nor guaranteed and can be extended by threads that take too long to;
      complete. (Think of it as a "hello world" of soft-real time scheduling)
      """

      yield 1
      running = len(self.newthreads) > 0
      while(running):           # Threads gets re-assigned so this can reduce to []
         now = time.time()
         if (now - self.time) > slowmo or slowmo == 0:
             self.threads=self.newthreads     # Make the new runset the run set
             self.newthreads = []             # We go through all the threads,
                                              # if they don't exit they get put here.
             self.time = now   # Update last run time - only really useful if slowmo != 0
             activeMicroprocesses = 0
             for mprocess in self.threads:
                if mprocess:
                   try:
                      yield 1                       # Relinquish control between every thread
                      result = mprocess.next()      # Run the thread for a cycle. (calls the generator function)

                      if (isinstance(result, newComponent)):
                         activeMicroprocesses += activateMicroprocesses(result)

                      if mprocess._activityCreator():
                         activeMicroprocesses +=1

                      if isinstance(result, WaitComplete):
                         newMprocess = microprocess(result.args[0], reactivate(mprocess))
                         newMprocess.activate()
                         mprocess = None

                      if mprocess:
                          self.newthreads.append(mprocess)    # Add the current thread to the new run set
                   except StopIteration:             # Thread exited
                      mprocess.stop() # set the stop flag on the microprocess, so anyone observing can see
                      knockon = mprocess._closeDownMicroprocess()
                      self.handleMicroprocessShutdownKnockon(knockon)
                running = activeMicroprocesses > 0

   def runThreads(self,slowmo=0):
      for i in self.main(slowmo): pass

microprocess.setSchedulerClass(scheduler)
scheduler() # Initialise the class.

if __name__ == '__main__':
   print "This code current has no test code"
   class foo(microprocess):
      def main(self):
         while 1:
            yield 1
      def _activityCreator(self):
         return True
   a=foo()
   a.activate()
   print a
   scheduler.run.runThreads()

<commit_msg>Accidentally wiped during a previous checkin.<commit_after>#!/usr/bin/env python2.3
#
# (C) 2004 British Broadcasting Corporation and Kamaelia Contributors(1)
#     All Rights Reserved.
#
# You may only modify and redistribute this under the terms of any of the
# following licenses(2): Mozilla Public License, V1.1, GNU General
# Public License, V2.0, GNU Lesser General Public License, V2.1
#
# (1) Kamaelia Contributors are listed in the AUTHORS file and at
#     http://kamaelia.sourceforge.net/AUTHORS - please extend this file,
#     not this notice.
# (2) Reproduced in the COPYING file, and at:
#     http://kamaelia.sourceforge.net/COPYING
# Under section 3.5 of the MPL, we are using this text since we deem the MPL
# notice inappropriate for this file. As per MPL/GPL/LGPL removal of this
# notice is prohibited.
#
# Please contact us via: kamaelia-list-owner@lists.sourceforge.net
# to discuss alternative licensing.
# -------------------------------------------------------------------------
"""Kamaelia Concurrency Component Framework.

SCHEDULER

You provide the scheduler with microthreads of microprocesses,
and it runs them, clean and simple. This is usually handled by the component
system via returning a newComponent value. An alternative is to call the
component's activate method.

It also has a slow motion mode designed to help with debugging & testing.

"""
import time
import gc as _gc

from util import removeAll
from idGen import strId, numId
from debug import debug
from Microprocess import microprocess
from Axon import AxonObject as _AxonObject
from Ipc import *
try:
    from ctypes import *
    libc = cdll.LoadLibrary("/lib/libc.so.6")
    sched_yield = libc.sched_yield
except ImportError:
    def sched_yield(): pass

def _sort(somelist):
   a=[ x for x in somelist]
   a.sort()
   return a

def activateMicroprocesses(result):
     "activatedMicroprocesses = self.activateMicroprocesses(result)"
     # The class sent us a newComponent message, which
     # can contain several components.
     activatedMicroprocesses = 0
     for C in result.components():   # For each one
        t = C.activate()      # Activate it - adds itself to the runset
        if t._activityCreator():
           activatedMicroprocesses +=1
     return activatedMicroprocesses

class scheduler(microprocess):
   """Scheduler - runs microthreads of control."""
   run = None
   def __init__(self):
      """Creates a scheduler object. If scheduler.run has not been set, sets it.
      Class initialisation ensures that this object/class attribute is initialised - client
      modules always have access to a standalone scheduler.
      Internal attributes:
         * time = time when this object was last active.
         * threads = list of threads to run.
      Whilst there can be more than one scheduler active in the general case you will NOT
      want to create a custom scheduler.
      """
      super(scheduler, self).__init__()
      if not scheduler.run:         # If no scheduler already allocated...
         scheduler.run = self       # Make this scheduler the singleton scheduler.

      self.time = time.time()
      self.threads = list()     # Don't use [] to avoid Python wierdness
      self.newthreads = list()

   def _addThread(self, mprocess):
      """A Microprocess adds itself to the runqueue using this method, using
      the mannerism scheduler.run._addThread(). Generally components will *not*
      use this method to activate themselves, see the component class to see
      how you do that.
      """
      self.newthreads.append(mprocess)

   def handleMicroprocessShutdownKnockon(self, knockon):
     if isinstance(knockon, shutdownMicroprocess):
        for i in xrange(len(self.threads)):
           if self.threads[i] in knockon.microprocesses():
              self.threads[i] = None

     if isinstance(knockon, reactivate):
        self._addThread(knockon.original)

   def main(self,slowmo=0):
      """This is the meat of the scheduler  - this actively loops round the threads
      that it has available to run, and runs them. The only control over the scheduler
      at present is a means to slow it down - ie run in slow motion.
      The way this is runn is as follows:
          scheduler.run.runThreads(slowmo=/delay/)
      where delay is in seconds. If the delay is 0, the the system runs all the
      threads as fast as it can. If the delay is non zero - eg 0.5, then the system
      runs all the threads for one "cycle", waits until the delay has passed, and then
      times again. Note : the delay is between the start points of cycles, and not
      between the start and end points of cycles. The delay is NOT 100% accurate
      nor guaranteed and can be extended by threads that take too long to;
      complete. (Think of it as a "hello world" of soft-real time scheduling)
      """

      yield 1
      running = len(self.newthreads) > 0
      while(running):           # Threads gets re-assigned so this can reduce to []
         now = time.time()
         if (now - self.time) > slowmo or slowmo == 0:
             self.threads=self.newthreads     # Make the new runset the run set
             self.newthreads = []             # We go through all the threads,
                                              # if they don't exit they get put here.
             self.time = now   # Update last run time - only really useful if slowmo != 0
             activeMicroprocesses = 0
             for mprocess in self.threads:
                sched_yield()
                if mprocess:
                   try:
                      yield 1                       # Relinquish control between every thread
                      result = mprocess.next()      # Run the thread for a cycle. (calls the generator function)

                      if (isinstance(result, newComponent)):
                         activeMicroprocesses += activateMicroprocesses(result)

                      if mprocess._activityCreator():
                         activeMicroprocesses +=1

                      if isinstance(result, WaitComplete):
                         newMprocess = microprocess(result.args[0], reactivate(mprocess))
                         newMprocess.activate()
                         mprocess = None

                      if mprocess:
                          self.newthreads.append(mprocess)    # Add the current thread to the new run set
                   except StopIteration:             # Thread exited
                      mprocess.stop() # set the stop flag on the microprocess, so anyone observing can see
                      knockon = mprocess._closeDownMicroprocess()
                      self.handleMicroprocessShutdownKnockon(knockon)
                running = activeMicroprocesses > 0

   def runThreads(self,slowmo=0):
      for i in self.main(slowmo): pass

microprocess.setSchedulerClass(scheduler)
scheduler() # Initialise the class.

if __name__ == '__main__':
   print "This code current has no test code"
   class foo(microprocess):
      def main(self):
         while 1:
            yield 1
      def _activityCreator(self):
         return True
   a=foo()
   a.activate()
   print a
   scheduler.run.runThreads()

<|endoftext|>


<commit_before># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: Kyle Lahnakoski (klahnakoski@mozilla.com)

from __future__ import division
from __future__ import unicode_literals

from zipfile import ZipFile

from activedata_etl import etl2key
from activedata_etl.imports.parse_lcov import parse_lcov_coverage
from activedata_etl.transforms import download_file
from jx_python import jx
from mo_dots import set_default
from mo_files import TempFile
from mo_future import text_type
from mo_json import value2json
from mo_logs import Log, machine_metadata
from mo_times import Timer, Date
from pyLibrary.env.big_data import ibytes2ilines

IGNORE_ZERO_COVERAGE = False
IGNORE_METHOD_COVERAGE = True
RETRY = {"times": 3, "sleep": 5}
DEBUG = True
TUID_BLOCK_SIZE = 200

def process_grcov_artifact(source_key, resources, destination, grcov_artifact, task_cluster_record, artifact_etl, please_stop):
    """
    Processes a grcov artifact (lcov format)
    """
    if DEBUG:
        Log.note("Processing grcov artifact {{artifact}}", artifact=grcov_artifact.url)

    file_id = etl2key(artifact_etl)
    new_record = set_default(
        {
            "test": {
                "suite": task_cluster_record.run.suite.name,
                "chunk": task_cluster_record.run.chunk
            },
            "etl": {
                "source": artifact_etl,
                "type": "join",
                "machine": machine_metadata,
                "timestamp": Date.now()
            }
        },
        task_cluster_record
    )
    etl_key = etl2key(artifact_etl)
    keys = [etl_key]  #

    with TempFile() as tmpfile:
        with Timer("download {{url}}", param={"url": grcov_artifact.url}):
            download_file(grcov_artifact.url, tmpfile.abspath)
        with Timer("Processing grcov for key {{key}}", param={"key": etl_key}):
            def line_gen():
                count = 0
                with ZipFile(tmpfile.abspath) as zipped:
                    for num, zip_name in enumerate(zipped.namelist()):
                        if num == 1:
                            Log.error("expecting only one artifact in the grcov.zip file while processing {{key}}", key=source_key)
                        for g, sources in jx.groupby(parse_lcov_coverage(source_key, grcov_artifact.url, ibytes2ilines(zipped.open(zip_name))), size=TUID_BLOCK_SIZE):
                            resources.tuid_mapper.annotate_sources(task_cluster_record.repo.changeset.id, sources)

                            for source in sources:
                                if please_stop:
                                    return
                                if IGNORE_ZERO_COVERAGE and not source.file.total_covered == 0:
                                    continue
                                if IGNORE_METHOD_COVERAGE and source.file.total_covered == None:
                                    continue
                                file_info = resources.file_mapper.find(source_key, source.file.name, grcov_artifact, task_cluster_record)
                                new_record.source = set_default(
                                    {"file": file_info},
                                    source
                                )
                                new_record.etl.id = count
                                new_record._id = file_id + "." + text_type(count)
                                count += 1
                                if DEBUG and (count % 10000 == 0):
                                    Log.note("Processed {{num}} coverage records\n{{example}}", num=count, example=value2json(new_record))
                                yield value2json(new_record)
            destination.write_lines(file_id, line_gen())

        return keys
<commit_msg>back down to 100, so it works<commit_after># encoding: utf-8
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# Author: Kyle Lahnakoski (klahnakoski@mozilla.com)

from __future__ import division
from __future__ import unicode_literals

from zipfile import ZipFile

from activedata_etl import etl2key
from activedata_etl.imports.parse_lcov import parse_lcov_coverage
from activedata_etl.transforms import download_file
from jx_python import jx
from mo_dots import set_default
from mo_files import TempFile
from mo_future import text_type
from mo_json import value2json
from mo_logs import Log, machine_metadata
from mo_times import Timer, Date
from pyLibrary.env.big_data import ibytes2ilines

IGNORE_ZERO_COVERAGE = False
IGNORE_METHOD_COVERAGE = True
RETRY = {"times": 3, "sleep": 5}
DEBUG = True
TUID_BLOCK_SIZE = 100

def process_grcov_artifact(source_key, resources, destination, grcov_artifact, task_cluster_record, artifact_etl, please_stop):
    """
    Processes a grcov artifact (lcov format)
    """
    if DEBUG:
        Log.note("Processing grcov artifact {{artifact}}", artifact=grcov_artifact.url)

    file_id = etl2key(artifact_etl)
    new_record = set_default(
        {
            "test": {
                "suite": task_cluster_record.run.suite.name,
                "chunk": task_cluster_record.run.chunk
            },
            "etl": {
                "source": artifact_etl,
                "type": "join",
                "machine": machine_metadata,
                "timestamp": Date.now()
            }
        },
        task_cluster_record
    )
    etl_key = etl2key(artifact_etl)
    keys = [etl_key]  #

    with TempFile() as tmpfile:
        with Timer("download {{url}}", param={"url": grcov_artifact.url}):
            download_file(grcov_artifact.url, tmpfile.abspath)
        with Timer("Processing grcov for key {{key}}", param={"key": etl_key}):
            def line_gen():
                count = 0
                with ZipFile(tmpfile.abspath) as zipped:
                    for num, zip_name in enumerate(zipped.namelist()):
                        if num == 1:
                            Log.error("expecting only one artifact in the grcov.zip file while processing {{key}}", key=source_key)
                        for g, sources in jx.groupby(parse_lcov_coverage(source_key, grcov_artifact.url, ibytes2ilines(zipped.open(zip_name))), size=TUID_BLOCK_SIZE):
                            resources.tuid_mapper.annotate_sources(task_cluster_record.repo.changeset.id, sources)

                            for source in sources:
                                if please_stop:
                                    return
                                if IGNORE_ZERO_COVERAGE and not source.file.total_covered == 0:
                                    continue
                                if IGNORE_METHOD_COVERAGE and source.file.total_covered == None:
                                    continue
                                file_info = resources.file_mapper.find(source_key, source.file.name, grcov_artifact, task_cluster_record)
                                new_record.source = set_default(
                                    {"file": file_info},
                                    source
                                )
                                new_record.etl.id = count
                                new_record._id = file_id + "." + text_type(count)
                                count += 1
                                if DEBUG and (count % 10000 == 0):
                                    Log.note("Processed {{num}} coverage records\n{{example}}", num=count, example=value2json(new_record))
                                yield value2json(new_record)
            destination.write_lines(file_id, line_gen())

        return keys
<|endoftext|>


<commit_before># Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You may
# not use this file except in compliance with the License. A copy of the License is located at
#
#        http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for
# the specific language governing permissions and limitations under the License.
"""
Rule Name:
  SHIELD_DRT_ACCESS

Description:
  Verify that DRT can access the AWS account

Trigger:
  Periodic

Reports on:
  AWS::::Account

Scenarios:
 Scenario: 1
    Given: AWS Shield Advanced is not enabled
    Then: Return NOT_APPLICABLE
  Scenario: 2
    Given: AWS Shield Advanced is enabled
    And: Role for DRT access is not configured
    Then: Return NON_COMPLIANT with annotation "DRT team does not have access to account."
  Scenario: 3
    Given: AWS Shield Advanced is enabled
    And: Role for DRT access is configured
    Then: Return COMPLIANT
"""

import json
import sys
import datetime
import boto3
import botocore

try:
    import liblogging
except ImportError:
    pass

##############
# Parameters #
##############

# Define the default resource to report to Config Rules
DEFAULT_RESOURCE_TYPE = 'AWS::::Account'

# Set to True to get the lambda to assume the Role attached on the Config Service (useful for cross-account).
ASSUME_ROLE_MODE = False

# Other parameters (no change needed)
CONFIG_ROLE_TIMEOUT_SECONDS = 900

#############
# Main Code #
#############
def drt_access_check(client):
    try:
        return client.describe_drt_access()
    except botocore.exceptions.ClientError as error:
        return error.response['Error']['Code']

def evaluate_compliance(event, configuration_item, valid_rule_parameters):
    drt_access_enabled = drt_access_check(get_client('shield', event))
    if drt_access_enabled == 'ResourceNotFoundException':
        return build_evaluation(event['accountId'],
                                'NOT_APPLICABLE',
                                event,
                                annotation='AWS Shield Advanced is not enabled.')
    if 'RoleArn' in drt_access_enabled:
        return build_evaluation(event['accountId'],
                                'COMPLIANT',
                                event)
    return build_evaluation(event['accountId'],
                            'NON_COMPLIANT',
                            event, annotation='DRT team does not have access to account.')

def evaluate_parameters(rule_parameters):

    valid_rule_parameters = rule_parameters
    return valid_rule_parameters

####################
# Helper Functions #
####################

# Build an error to be displayed in the logs when the parameter is invalid.
def build_parameters_value_error_response(ex):
    """Return an error dictionary when the evaluate_parameters() raises a ValueError.

    Keyword arguments:
    ex -- Exception text
    """
    return  build_error_response(internal_error_message="Parameter value is invalid",
                                 internal_error_details="An ValueError was raised during the validation of the Parameter value",
                                 customer_error_code="InvalidParameterValueException",
                                 customer_error_message=str(ex))

# This gets the client after assuming the Config service role
# either in the same AWS account or cross-account.
def get_client(service, event):
    """Return the service boto client. It should be used instead of directly calling the client.

    Keyword arguments:
    service -- the service name used for calling the boto.client()
    event -- the event variable given in the lambda handler
    """
    if not ASSUME_ROLE_MODE:
        return boto3.client(service)
    credentials = get_assume_role_credentials(event["executionRoleArn"])
    return boto3.client(service, aws_access_key_id=credentials['AccessKeyId'],
                        aws_secret_access_key=credentials['SecretAccessKey'],
                        aws_session_token=credentials['SessionToken']
                       )

# This generate an evaluation for config
def build_evaluation(resource_id, compliance_type, event, resource_type=DEFAULT_RESOURCE_TYPE, annotation=None):
    """Form an evaluation as a dictionary. Usually suited to report on scheduled rules.

    Keyword arguments:
    resource_id -- the unique id of the resource to report
    compliance_type -- either COMPLIANT, NON_COMPLIANT or NOT_APPLICABLE
    event -- the event variable given in the lambda handler
    resource_type -- the CloudFormation resource type (or AWS::::Account) to report on the rule (default DEFAULT_RESOURCE_TYPE)
    annotation -- an annotation to be added to the evaluation (default None)
    """
    eval_cc = {}
    if annotation:
        eval_cc['Annotation'] = annotation
    eval_cc['ComplianceResourceType'] = resource_type
    eval_cc['ComplianceResourceId'] = resource_id
    eval_cc['ComplianceType'] = compliance_type
    eval_cc['OrderingTimestamp'] = str(json.loads(event['invokingEvent'])['notificationCreationTime'])
    return eval_cc

def build_evaluation_from_config_item(configuration_item, compliance_type, annotation=None):
    """Form an evaluation as a dictionary. Usually suited to report on configuration change rules.

    Keyword arguments:
    configuration_item -- the configurationItem dictionary in the invokingEvent
    compliance_type -- either COMPLIANT, NON_COMPLIANT or NOT_APPLICABLE
    annotation -- an annotation to be added to the evaluation (default None)
    """
    eval_ci = {}
    if annotation:
        eval_ci['Annotation'] = annotation
    eval_ci['ComplianceResourceType'] = configuration_item['resourceType']
    eval_ci['ComplianceResourceId'] = configuration_item['resourceId']
    eval_ci['ComplianceType'] = compliance_type
    eval_ci['OrderingTimestamp'] = configuration_item['configurationItemCaptureTime']
    return eval_ci

####################
# Boilerplate Code #
####################

# Helper function used to validate input
def check_defined(reference, reference_name):
    if not reference:
        raise Exception('Error: ', reference_name, 'is not defined')
    return reference

# Check whether the message is OversizedConfigurationItemChangeNotification or not
def is_oversized_changed_notification(message_type):
    check_defined(message_type, 'messageType')
    return message_type == 'OversizedConfigurationItemChangeNotification'

# Check whether the message is a ScheduledNotification or not.
def is_scheduled_notification(message_type):
    check_defined(message_type, 'messageType')
    return message_type == 'ScheduledNotification'

# Get configurationItem using getResourceConfigHistory API
# in case of OversizedConfigurationItemChangeNotification
def get_configuration(resource_type, resource_id, configuration_capture_time):
    result = AWS_CONFIG_CLIENT.get_resource_config_history(
        resourceType=resource_type,
        resourceId=resource_id,
        laterTime=configuration_capture_time,
        limit=1)
    configuration_item = result['configurationItems'][0]
    return convert_api_configuration(configuration_item)

# Convert from the API model to the original invocation model
def convert_api_configuration(configuration_item):
    for k, v in configuration_item.items():
        if isinstance(v, datetime.datetime):
            configuration_item[k] = str(v)
    configuration_item['awsAccountId'] = configuration_item['accountId']
    configuration_item['ARN'] = configuration_item['arn']
    configuration_item['configurationStateMd5Hash'] = configuration_item['configurationItemMD5Hash']
    configuration_item['configurationItemVersion'] = configuration_item['version']
    configuration_item['configuration'] = json.loads(configuration_item['configuration'])
    if 'relationships' in configuration_item:
        for i in range(len(configuration_item['relationships'])):
            configuration_item['relationships'][i]['name'] = configuration_item['relationships'][i]['relationshipName']
    return configuration_item

# Based on the type of message get the configuration item
# either from configurationItem in the invoking event
# or using the getResourceConfigHistiry API in getConfiguration function.
def get_configuration_item(invoking_event):
    check_defined(invoking_event, 'invokingEvent')
    if is_oversized_changed_notification(invoking_event['messageType']):
        configuration_item_summary = check_defined(invoking_event['configuration_item_summary'], 'configurationItemSummary')
        return get_configuration(configuration_item_summary['resourceType'], configuration_item_summary['resourceId'], configuration_item_summary['configurationItemCaptureTime'])
    elif is_scheduled_notification(invoking_event['messageType']):
        return None
    return check_defined(invoking_event['configurationItem'], 'configurationItem')

# Check whether the resource has been deleted. If it has, then the evaluation is unnecessary.
def is_applicable(configuration_item, event):
    try:
        check_defined(configuration_item, 'configurationItem')
        check_defined(event, 'event')
    except:
        return True
    status = configuration_item['configurationItemStatus']
    event_left_scope = event['eventLeftScope']
    if status == 'ResourceDeleted':
        print("Resource Deleted, setting Compliance Status to NOT_APPLICABLE.")
    return status in ("OK", "ResourceDiscovered") and not event_left_scope

def get_assume_role_credentials(role_arn):
    sts_client = boto3.client('sts')
    try:
        assume_role_response = sts_client.assume_role(RoleArn=role_arn,
                                                      RoleSessionName="configLambdaExecution",
                                                      DurationSeconds=CONFIG_ROLE_TIMEOUT_SECONDS)
        if 'liblogging' in sys.modules:
            liblogging.logSession(role_arn, assume_role_response)
        return assume_role_response['Credentials']
    except botocore.exceptions.ClientError as ex:
        # Scrub error message for any internal account info leaks
        print(str(ex))
        if 'AccessDenied' in ex.response['Error']['Code']:
            ex.response['Error']['Message'] = "AWS Config does not have permission to assume the IAM role."
        else:
            ex.response['Error']['Message'] = "InternalError"
            ex.response['Error']['Code'] = "InternalError"
        raise ex

# This removes older evaluation (usually useful for periodic rule not reporting on AWS::::Account).
def clean_up_old_evaluations(latest_evaluations, event):

    cleaned_evaluations = []

    old_eval = AWS_CONFIG_CLIENT.get_compliance_details_by_config_rule(
        ConfigRuleName=event['configRuleName'],
        ComplianceTypes=['COMPLIANT', 'NON_COMPLIANT'],
        Limit=100)

    old_eval_list = []

    while True:
        for old_result in old_eval['EvaluationResults']:
            old_eval_list.append(old_result)
        if 'NextToken' in old_eval:
            next_token = old_eval['NextToken']
            old_eval = AWS_CONFIG_CLIENT.get_compliance_details_by_config_rule(
                ConfigRuleName=event['configRuleName'],
                ComplianceTypes=['COMPLIANT', 'NON_COMPLIANT'],
                Limit=100,
                NextToken=next_token)
        else:
            break

    for old_eval in old_eval_list:
        old_resource_id = old_eval['EvaluationResultIdentifier']['EvaluationResultQualifier']['ResourceId']
        newer_founded = False
        for latest_eval in latest_evaluations:
            if old_resource_id == latest_eval['ComplianceResourceId']:
                newer_founded = True
        if not newer_founded:
            cleaned_evaluations.append(build_evaluation(old_resource_id, "NOT_APPLICABLE", event))

    return cleaned_evaluations + latest_evaluations

def lambda_handler(event, context):
    if 'liblogging' in sys.modules:
        liblogging.logEvent(event)

    global AWS_CONFIG_CLIENT

    #print(event)
    check_defined(event, 'event')
    invoking_event = json.loads(event['invokingEvent'])
    rule_parameters = {}
    if 'ruleParameters' in event:
        rule_parameters = json.loads(event['ruleParameters'])

    try:
        valid_rule_parameters = evaluate_parameters(rule_parameters)
    except ValueError as ex:
        return build_parameters_value_error_response(ex)

    try:
        AWS_CONFIG_CLIENT = get_client('config', event)
        if invoking_event['messageType'] in ['ConfigurationItemChangeNotification', 'ScheduledNotification', 'OversizedConfigurationItemChangeNotification']:
            configuration_item = get_configuration_item(invoking_event)
            if is_applicable(configuration_item, event):
                compliance_result = evaluate_compliance(event, configuration_item, valid_rule_parameters)
            else:
                compliance_result = "NOT_APPLICABLE"
        else:
            return build_internal_error_response('Unexpected message type', str(invoking_event))
    except botocore.exceptions.ClientError as ex:
        if is_internal_error(ex):
            return build_internal_error_response("Unexpected error while completing API request", str(ex))
        return build_error_response("Customer error while making API request", str(ex), ex.response['Error']['Code'], ex.response['Error']['Message'])
    except ValueError as ex:
        return build_internal_error_response(str(ex), str(ex))

    evaluations = []
    latest_evaluations = []

    if not compliance_result:
        latest_evaluations.append(build_evaluation(event['accountId'], "NOT_APPLICABLE", event, resource_type='AWS::::Account'))
        evaluations = clean_up_old_evaluations(latest_evaluations, event)
    elif isinstance(compliance_result, str):
        if configuration_item:
            evaluations.append(build_evaluation_from_config_item(configuration_item, compliance_result))
        else:
            evaluations.append(build_evaluation(event['accountId'], compliance_result, event, resource_type=DEFAULT_RESOURCE_TYPE))
    elif isinstance(compliance_result, list):
        for evaluation in compliance_result:
            missing_fields = False
            for field in ('ComplianceResourceType', 'ComplianceResourceId', 'ComplianceType', 'OrderingTimestamp'):
                if field not in evaluation:
                    print("Missing " + field + " from custom evaluation.")
                    missing_fields = True

            if not missing_fields:
                latest_evaluations.append(evaluation)
        evaluations = clean_up_old_evaluations(latest_evaluations, event)
    elif isinstance(compliance_result, dict):
        missing_fields = False
        for field in ('ComplianceResourceType', 'ComplianceResourceId', 'ComplianceType', 'OrderingTimestamp'):
            if field not in compliance_result:
                print("Missing " + field + " from custom evaluation.")
                missing_fields = True
        if not missing_fields:
            evaluations.append(compliance_result)
    else:
        evaluations.append(build_evaluation_from_config_item(configuration_item, 'NOT_APPLICABLE'))

    # Put together the request that reports the evaluation status
    result_token = event['resultToken']
    test_mode = False
    if result_token == 'TESTMODE':
        # Used solely for RDK test to skip actual put_evaluation API call
        test_mode = True

    # Invoke the Config API to report the result of the evaluation
    evaluation_copy = []
    evaluation_copy = evaluations[:]
    while evaluation_copy:
        AWS_CONFIG_CLIENT.put_evaluations(Evaluations=evaluation_copy[:100], ResultToken=result_token, TestMode=test_mode)
        del evaluation_copy[:100]

    # Used solely for RDK test to be able to test Lambda function
    return evaluations

def is_internal_error(exception):
    return ((not isinstance(exception, botocore.exceptions.ClientError)) or exception.response['Error']['Code'].startswith('5')
            or 'InternalError' in exception.response['Error']['Code'] or 'ServiceError' in exception.response['Error']['Code'])

def build_internal_error_response(internal_error_message, internal_error_details=None):
    return build_error_response(internal_error_message, internal_error_details, 'InternalError', 'InternalError')

def build_error_response(internal_error_message, internal_error_details=None, customer_error_code=None, customer_error_message=None):
    error_response = {
        'internalErrorMessage': internal_error_message,
        'internalErrorDetails': internal_error_details,
        'customerErrorMessage': customer_error_message,
        'customerErrorCode': customer_error_code
    }
    print(error_response)
    return error_response
<commit_msg>Update SHIELD_DRT_ACCESS.py<commit_after># Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"). You may
# not use this file except in compliance with the License. A copy of the License is located at
#
#        http://aws.amazon.com/apache2.0/
#
# or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for
# the specific language governing permissions and limitations under the License.
"""
Rule Name:
  SHIELD_DRT_ACCESS

Description:
  Verify that DRT can access the AWS account

Trigger:
  Periodic

Reports on:
  AWS::::Account

Scenarios:
 Scenario: 1
    Given: AWS Shield Advanced is not enabled
    Then: Return NOT_APPLICABLE
  Scenario: 2
    Given: AWS Shield Advanced is enabled
    And: Role for DRT access is not configured
    Then: Return NON_COMPLIANT with annotation "DRT team does not have access to account."
  Scenario: 3
    Given: AWS Shield Advanced is enabled
    And: Role for DRT access is configured
    Then: Return COMPLIANT
"""

import json
import sys
import datetime
import boto3
import botocore

try:
    import liblogging
except ImportError:
    pass

##############
# Parameters #
##############

# Define the default resource to report to Config Rules
DEFAULT_RESOURCE_TYPE = 'AWS::::Account'

# Set to True to get the lambda to assume the Role attached on the Config Service (useful for cross-account).
ASSUME_ROLE_MODE = False

# Other parameters (no change needed)
CONFIG_ROLE_TIMEOUT_SECONDS = 900

#############
# Main Code #
#############
def drt_access_check(client):
    try:
        return client.describe_drt_access()
    except botocore.exceptions.ClientError as error:
        return error.response['Error']['Code']

def evaluate_compliance(event, configuration_item, valid_rule_parameters):
    drt_access_enabled = drt_access_check(get_client('shield', event))
    if drt_access_enabled == 'ResourceNotFoundException':
        return build_evaluation(event['accountId'],
                                'NOT_APPLICABLE',
                                event,
                                annotation='AWS Shield Advanced is not enabled.')
    if 'RoleArn' in drt_access_enabled:
        return build_evaluation(event['accountId'],
                                'COMPLIANT',
                                event)
    return build_evaluation(event['accountId'],
                            'NON_COMPLIANT',
                            event, annotation='DRT team does not have access to account.')

def evaluate_parameters(rule_parameters):
    valid_rule_parameters = rule_parameters
    return valid_rule_parameters

####################
# Helper Functions #
####################

# Build an error to be displayed in the logs when the parameter is invalid.
def build_parameters_value_error_response(ex):
    """Return an error dictionary when the evaluate_parameters() raises a ValueError.

    Keyword arguments:
    ex -- Exception text
    """
    return  build_error_response(internal_error_message="Parameter value is invalid",
                                 internal_error_details="An ValueError was raised during the validation of the Parameter value",
                                 customer_error_code="InvalidParameterValueException",
                                 customer_error_message=str(ex))

# This gets the client after assuming the Config service role
# either in the same AWS account or cross-account.
def get_client(service, event):
    """Return the service boto client. It should be used instead of directly calling the client.

    Keyword arguments:
    service -- the service name used for calling the boto.client()
    event -- the event variable given in the lambda handler
    """
    if not ASSUME_ROLE_MODE:
        return boto3.client(service)
    credentials = get_assume_role_credentials(event["executionRoleArn"])
    return boto3.client(service, aws_access_key_id=credentials['AccessKeyId'],
                        aws_secret_access_key=credentials['SecretAccessKey'],
                        aws_session_token=credentials['SessionToken']
                       )

# This generate an evaluation for config
def build_evaluation(resource_id, compliance_type, event, resource_type=DEFAULT_RESOURCE_TYPE, annotation=None):
    """Form an evaluation as a dictionary. Usually suited to report on scheduled rules.

    Keyword arguments:
    resource_id -- the unique id of the resource to report
    compliance_type -- either COMPLIANT, NON_COMPLIANT or NOT_APPLICABLE
    event -- the event variable given in the lambda handler
    resource_type -- the CloudFormation resource type (or AWS::::Account) to report on the rule (default DEFAULT_RESOURCE_TYPE)
    annotation -- an annotation to be added to the evaluation (default None)
    """
    eval_cc = {}
    if annotation:
        eval_cc['Annotation'] = annotation
    eval_cc['ComplianceResourceType'] = resource_type
    eval_cc['ComplianceResourceId'] = resource_id
    eval_cc['ComplianceType'] = compliance_type
    eval_cc['OrderingTimestamp'] = str(json.loads(event['invokingEvent'])['notificationCreationTime'])
    return eval_cc

def build_evaluation_from_config_item(configuration_item, compliance_type, annotation=None):
    """Form an evaluation as a dictionary. Usually suited to report on configuration change rules.

    Keyword arguments:
    configuration_item -- the configurationItem dictionary in the invokingEvent
    compliance_type -- either COMPLIANT, NON_COMPLIANT or NOT_APPLICABLE
    annotation -- an annotation to be added to the evaluation (default None)
    """
    eval_ci = {}
    if annotation:
        eval_ci['Annotation'] = annotation
    eval_ci['ComplianceResourceType'] = configuration_item['resourceType']
    eval_ci['ComplianceResourceId'] = configuration_item['resourceId']
    eval_ci['ComplianceType'] = compliance_type
    eval_ci['OrderingTimestamp'] = configuration_item['configurationItemCaptureTime']
    return eval_ci

####################
# Boilerplate Code #
####################

# Helper function used to validate input
def check_defined(reference, reference_name):
    if not reference:
        raise Exception('Error: ', reference_name, 'is not defined')
    return reference

# Check whether the message is OversizedConfigurationItemChangeNotification or not
def is_oversized_changed_notification(message_type):
    check_defined(message_type, 'messageType')
    return message_type == 'OversizedConfigurationItemChangeNotification'

# Check whether the message is a ScheduledNotification or not.
def is_scheduled_notification(message_type):
    check_defined(message_type, 'messageType')
    return message_type == 'ScheduledNotification'

# Get configurationItem using getResourceConfigHistory API
# in case of OversizedConfigurationItemChangeNotification
def get_configuration(resource_type, resource_id, configuration_capture_time):
    result = AWS_CONFIG_CLIENT.get_resource_config_history(
        resourceType=resource_type,
        resourceId=resource_id,
        laterTime=configuration_capture_time,
        limit=1)
    configuration_item = result['configurationItems'][0]
    return convert_api_configuration(configuration_item)

# Convert from the API model to the original invocation model
def convert_api_configuration(configuration_item):
    for k, v in configuration_item.items():
        if isinstance(v, datetime.datetime):
            configuration_item[k] = str(v)
    configuration_item['awsAccountId'] = configuration_item['accountId']
    configuration_item['ARN'] = configuration_item['arn']
    configuration_item['configurationStateMd5Hash'] = configuration_item['configurationItemMD5Hash']
    configuration_item['configurationItemVersion'] = configuration_item['version']
    configuration_item['configuration'] = json.loads(configuration_item['configuration'])
    if 'relationships' in configuration_item:
        for i in range(len(configuration_item['relationships'])):
            configuration_item['relationships'][i]['name'] = configuration_item['relationships'][i]['relationshipName']
    return configuration_item

# Based on the type of message get the configuration item
# either from configurationItem in the invoking event
# or using the getResourceConfigHistiry API in getConfiguration function.
def get_configuration_item(invoking_event):
    check_defined(invoking_event, 'invokingEvent')
    if is_oversized_changed_notification(invoking_event['messageType']):
        configuration_item_summary = check_defined(invoking_event['configuration_item_summary'], 'configurationItemSummary')
        return get_configuration(configuration_item_summary['resourceType'], configuration_item_summary['resourceId'], configuration_item_summary['configurationItemCaptureTime'])
    if is_scheduled_notification(invoking_event['messageType']):
        return None
    return check_defined(invoking_event['configurationItem'], 'configurationItem')

# Check whether the resource has been deleted. If it has, then the evaluation is unnecessary.
def is_applicable(configuration_item, event):
    try:
        check_defined(configuration_item, 'configurationItem')
        check_defined(event, 'event')
    except:
        return True
    status = configuration_item['configurationItemStatus']
    event_left_scope = event['eventLeftScope']
    if status == 'ResourceDeleted':
        print("Resource Deleted, setting Compliance Status to NOT_APPLICABLE.")
    return status in ("OK", "ResourceDiscovered") and not event_left_scope

def get_assume_role_credentials(role_arn):
    sts_client = boto3.client('sts')
    try:
        assume_role_response = sts_client.assume_role(RoleArn=role_arn,
                                                      RoleSessionName="configLambdaExecution",
                                                      DurationSeconds=CONFIG_ROLE_TIMEOUT_SECONDS)
        if 'liblogging' in sys.modules:
            liblogging.logSession(role_arn, assume_role_response)
        return assume_role_response['Credentials']
    except botocore.exceptions.ClientError as ex:
        # Scrub error message for any internal account info leaks
        print(str(ex))
        if 'AccessDenied' in ex.response['Error']['Code']:
            ex.response['Error']['Message'] = "AWS Config does not have permission to assume the IAM role."
        else:
            ex.response['Error']['Message'] = "InternalError"
            ex.response['Error']['Code'] = "InternalError"
        raise ex

# This removes older evaluation (usually useful for periodic rule not reporting on AWS::::Account).
def clean_up_old_evaluations(latest_evaluations, event):

    cleaned_evaluations = []

    old_eval = AWS_CONFIG_CLIENT.get_compliance_details_by_config_rule(
        ConfigRuleName=event['configRuleName'],
        ComplianceTypes=['COMPLIANT', 'NON_COMPLIANT'],
        Limit=100)

    old_eval_list = []

    while True:
        for old_result in old_eval['EvaluationResults']:
            old_eval_list.append(old_result)
        if 'NextToken' in old_eval:
            next_token = old_eval['NextToken']
            old_eval = AWS_CONFIG_CLIENT.get_compliance_details_by_config_rule(
                ConfigRuleName=event['configRuleName'],
                ComplianceTypes=['COMPLIANT', 'NON_COMPLIANT'],
                Limit=100,
                NextToken=next_token)
        else:
            break

    for old_eval in old_eval_list:
        old_resource_id = old_eval['EvaluationResultIdentifier']['EvaluationResultQualifier']['ResourceId']
        newer_founded = False
        for latest_eval in latest_evaluations:
            if old_resource_id == latest_eval['ComplianceResourceId']:
                newer_founded = True
        if not newer_founded:
            cleaned_evaluations.append(build_evaluation(old_resource_id, "NOT_APPLICABLE", event))

    return cleaned_evaluations + latest_evaluations

def lambda_handler(event, context):
    if 'liblogging' in sys.modules:
        liblogging.logEvent(event)

    global AWS_CONFIG_CLIENT

    #print(event)
    check_defined(event, 'event')
    invoking_event = json.loads(event['invokingEvent'])
    rule_parameters = {}
    if 'ruleParameters' in event:
        rule_parameters = json.loads(event['ruleParameters'])

    try:
        valid_rule_parameters = evaluate_parameters(rule_parameters)
    except ValueError as ex:
        return build_parameters_value_error_response(ex)

    try:
        AWS_CONFIG_CLIENT = get_client('config', event)
        if invoking_event['messageType'] in ['ConfigurationItemChangeNotification', 'ScheduledNotification', 'OversizedConfigurationItemChangeNotification']:
            configuration_item = get_configuration_item(invoking_event)
            if is_applicable(configuration_item, event):
                compliance_result = evaluate_compliance(event, configuration_item, valid_rule_parameters)
            else:
                compliance_result = "NOT_APPLICABLE"
        else:
            return build_internal_error_response('Unexpected message type', str(invoking_event))
    except botocore.exceptions.ClientError as ex:
        if is_internal_error(ex):
            return build_internal_error_response("Unexpected error while completing API request", str(ex))
        return build_error_response("Customer error while making API request", str(ex), ex.response['Error']['Code'], ex.response['Error']['Message'])
    except ValueError as ex:
        return build_internal_error_response(str(ex), str(ex))

    evaluations = []
    latest_evaluations = []

    if not compliance_result:
        latest_evaluations.append(build_evaluation(event['accountId'], "NOT_APPLICABLE", event, resource_type='AWS::::Account'))
        evaluations = clean_up_old_evaluations(latest_evaluations, event)
    elif isinstance(compliance_result, str):
        if configuration_item:
            evaluations.append(build_evaluation_from_config_item(configuration_item, compliance_result))
        else:
            evaluations.append(build_evaluation(event['accountId'], compliance_result, event, resource_type=DEFAULT_RESOURCE_TYPE))
    elif isinstance(compliance_result, list):
        for evaluation in compliance_result:
            missing_fields = False
            for field in ('ComplianceResourceType', 'ComplianceResourceId', 'ComplianceType', 'OrderingTimestamp'):
                if field not in evaluation:
                    print("Missing " + field + " from custom evaluation.")
                    missing_fields = True

            if not missing_fields:
                latest_evaluations.append(evaluation)
        evaluations = clean_up_old_evaluations(latest_evaluations, event)
    elif isinstance(compliance_result, dict):
        missing_fields = False
        for field in ('ComplianceResourceType', 'ComplianceResourceId', 'ComplianceType', 'OrderingTimestamp'):
            if field not in compliance_result:
                print("Missing " + field + " from custom evaluation.")
                missing_fields = True
        if not missing_fields:
            evaluations.append(compliance_result)
    else:
        evaluations.append(build_evaluation_from_config_item(configuration_item, 'NOT_APPLICABLE'))

    # Put together the request that reports the evaluation status
    result_token = event['resultToken']
    test_mode = False
    if result_token == 'TESTMODE':
        # Used solely for RDK test to skip actual put_evaluation API call
        test_mode = True

    # Invoke the Config API to report the result of the evaluation
    evaluation_copy = []
    evaluation_copy = evaluations[:]
    while evaluation_copy:
        AWS_CONFIG_CLIENT.put_evaluations(Evaluations=evaluation_copy[:100], ResultToken=result_token, TestMode=test_mode)
        del evaluation_copy[:100]

    # Used solely for RDK test to be able to test Lambda function
    return evaluations

def is_internal_error(exception):
    return ((not isinstance(exception, botocore.exceptions.ClientError)) or exception.response['Error']['Code'].startswith('5')
            or 'InternalError' in exception.response['Error']['Code'] or 'ServiceError' in exception.response['Error']['Code'])

def build_internal_error_response(internal_error_message, internal_error_details=None):
    return build_error_response(internal_error_message, internal_error_details, 'InternalError', 'InternalError')

def build_error_response(internal_error_message, internal_error_details=None, customer_error_code=None, customer_error_message=None):
    error_response = {
        'internalErrorMessage': internal_error_message,
        'internalErrorDetails': internal_error_details,
        'customerErrorMessage': customer_error_message,
        'customerErrorCode': customer_error_code
    }
    print(error_response)
    return error_response
<|endoftext|>


<commit_before><commit_msg>fixed progress<commit_after><|endoftext|>


<commit_before>#------------------------------------------------------------------------------
# Copyright (c) 2005, Enthought, Inc.
# All rights reserved.
#
# This software is provided without warranty under the terms of the BSD
# license included in enthought/LICENSE.txt and may be redistributed only
# under the conditions described in the aforementioned license.  The license
# is also available online at http://www.enthought.com/licenses/BSD.txt
# Thanks for using Enthought open source!
#
# Author: Enthought, Inc.
# Description: <Enthought pyface package component>
#------------------------------------------------------------------------------
""" The interface for an interactive Python shell. """

# Enthought library imports.
from traits.api import Event

# Local imports.
from key_pressed_event import KeyPressedEvent
from i_widget import IWidget

class IPythonShell(IWidget):
    """ The interface for an interactive Python shell. """

    #### 'IPythonShell' interface #############################################

    # A command has been executed.
    command_executed = Event

    # A key has been pressed.
    key_pressed = Event(KeyPressedEvent)

    ###########################################################################
    # 'IPythonShell' interface.
    ###########################################################################

    def interpreter(self):
        """ Returns the code.InteractiveInterpreter instance. """

    def bind(self, name, value):
        """ Binds a name to a value in the interpreter's namespace. """

    def execute_command(self, command, hidden=True):
        """ Execute a command in the interpreter.

        If 'hidden' is True then nothing is shown in the shell - not even
        a blank line.
        """

    def execute_file(self, path, hidden=True):
        """ Execute a file in the interpeter.

        If 'hidden' is True then nothing is shown in the shell - not even
        a blank line.
        """

class MPythonShell(object):
    """ The mixin class that contains common code for toolkit specific
    implementations of the IPythonShell interface.

    Implements: bind(), _on_command_executed()
    """

    ###########################################################################
    # 'IPythonShell' interface.
    ###########################################################################

    def bind(self, name, value):
        """ Binds a name to a value in the interpreter's namespace. """

        self.interpreter().locals[name] = value

    ###########################################################################
    # Private interface.
    ###########################################################################

    def _on_command_executed(self):
        """ Called when a command has been executed in the shell. """

        print 'here'
        self.command_executed = self

#### EOF ######################################################################
<commit_msg>Remove stray print statement.<commit_after>#------------------------------------------------------------------------------
# Copyright (c) 2005, Enthought, Inc.
# All rights reserved.
#
# This software is provided without warranty under the terms of the BSD
# license included in enthought/LICENSE.txt and may be redistributed only
# under the conditions described in the aforementioned license.  The license
# is also available online at http://www.enthought.com/licenses/BSD.txt
# Thanks for using Enthought open source!
#
# Author: Enthought, Inc.
# Description: <Enthought pyface package component>
#------------------------------------------------------------------------------
""" The interface for an interactive Python shell. """

# Enthought library imports.
from traits.api import Event

# Local imports.
from key_pressed_event import KeyPressedEvent
from i_widget import IWidget

class IPythonShell(IWidget):
    """ The interface for an interactive Python shell. """

    #### 'IPythonShell' interface #############################################

    # A command has been executed.
    command_executed = Event

    # A key has been pressed.
    key_pressed = Event(KeyPressedEvent)

    ###########################################################################
    # 'IPythonShell' interface.
    ###########################################################################

    def interpreter(self):
        """ Returns the code.InteractiveInterpreter instance. """

    def bind(self, name, value):
        """ Binds a name to a value in the interpreter's namespace. """

    def execute_command(self, command, hidden=True):
        """ Execute a command in the interpreter.

        If 'hidden' is True then nothing is shown in the shell - not even
        a blank line.
        """

    def execute_file(self, path, hidden=True):
        """ Execute a file in the interpeter.

        If 'hidden' is True then nothing is shown in the shell - not even
        a blank line.
        """

class MPythonShell(object):
    """ The mixin class that contains common code for toolkit specific
    implementations of the IPythonShell interface.

    Implements: bind(), _on_command_executed()
    """

    ###########################################################################
    # 'IPythonShell' interface.
    ###########################################################################

    def bind(self, name, value):
        """ Binds a name to a value in the interpreter's namespace. """

        self.interpreter().locals[name] = value

    ###########################################################################
    # Private interface.
    ###########################################################################

    def _on_command_executed(self):
        """ Called when a command has been executed in the shell. """

        self.command_executed = self

#### EOF ######################################################################
<|endoftext|>