<commit_before>from mpi4py import MPI
import mpiunittest as unittest

class GReqCtx(object):

    source = 3
    tag    = 7
    completed = False

    cancel_called = False
    free_called = False

    def query(self, status):
        status.Set_source(self.source)
        status.Set_tag(self.tag)
    def free(self):
        self.free_called = True
    def cancel(self, completed):
        self.cancel_called = True
        if completed is not self.completed:
            raise MPI.Exception(MPI.ERR_PENDING)

@unittest.skipMPI('MPI(<2.0)')
class TestGrequest(unittest.TestCase):

    def testAll(self):
        ctx = GReqCtx()
        greq = MPI.Grequest.Start(ctx.query, ctx.free, ctx.cancel)
        self.assertFalse(greq.Test())
        self.assertFalse(ctx.free_called)
        greq.Cancel()
        self.assertTrue(ctx.cancel_called)
        ctx.cancel_called = False
        greq.Complete()
        ctx.completed = True
        greq.Cancel()
        self.assertTrue(ctx.cancel_called)
        status = MPI.Status()
        self.assertTrue(greq.Test(status))
        self.assertEqual(status.Get_source(), ctx.source)
        self.assertEqual(status.Get_tag(), ctx.tag)
        self.assertEqual(status.Get_error(), MPI.SUCCESS)
        greq.Wait()
        self.assertTrue(ctx.free_called)

    def testAll1(self):
        ctx = GReqCtx()
        greq = MPI.Grequest.Start(ctx.query, None, None)
        self.assertFalse(greq.Test())
        greq.Cancel()
        greq.Complete()
        status = MPI.Status()
        self.assertTrue(greq.Test(status))
        self.assertEqual(status.Get_source(), ctx.source)
        self.assertEqual(status.Get_tag(), ctx.tag)
        self.assertEqual(status.Get_error(), MPI.SUCCESS)
        self.assertFalse(status.Is_cancelled())
        greq.Wait()

    def testAll2(self):
        greq = MPI.Grequest.Start(None, None, None)
        self.assertFalse(greq.Test())
        greq.Cancel()
        greq.Complete()
        status = MPI.Status()
        self.assertTrue(greq.Test(status))
        self.assertEqual(status.Get_source(), MPI.ANY_SOURCE)
        self.assertEqual(status.Get_tag(), MPI.ANY_TAG)
        self.assertEqual(status.Get_error(), MPI.SUCCESS)
        self.assertFalse(status.Is_cancelled())
        greq.Wait()

if __name__ == '__main__':
    unittest.main()
<commit_msg>test: Disable tests for generalized requests with Open MPI 4.1.0<commit_after>from mpi4py import MPI
import mpiunittest as unittest

class GReqCtx(object):

    source = 3
    tag    = 7
    completed = False

    cancel_called = False
    free_called = False

    def query(self, status):
        status.Set_source(self.source)
        status.Set_tag(self.tag)
    def free(self):
        self.free_called = True
    def cancel(self, completed):
        self.cancel_called = True
        if completed is not self.completed:
            raise MPI.Exception(MPI.ERR_PENDING)

@unittest.skipMPI('MPI(<2.0)')
@unittest.skipMPI('openmpi(==4.1.0)')
class TestGrequest(unittest.TestCase):

    def testAll(self):
        ctx = GReqCtx()
        greq = MPI.Grequest.Start(ctx.query, ctx.free, ctx.cancel)
        self.assertFalse(greq.Test())
        self.assertFalse(ctx.free_called)
        greq.Cancel()
        self.assertTrue(ctx.cancel_called)
        ctx.cancel_called = False
        greq.Complete()
        ctx.completed = True
        greq.Cancel()
        self.assertTrue(ctx.cancel_called)
        status = MPI.Status()
        self.assertTrue(greq.Test(status))
        self.assertEqual(status.Get_source(), ctx.source)
        self.assertEqual(status.Get_tag(), ctx.tag)
        self.assertEqual(status.Get_error(), MPI.SUCCESS)
        greq.Wait()
        self.assertTrue(ctx.free_called)

    def testAll1(self):
        ctx = GReqCtx()
        greq = MPI.Grequest.Start(ctx.query, None, None)
        self.assertFalse(greq.Test())
        greq.Cancel()
        greq.Complete()
        status = MPI.Status()
        self.assertTrue(greq.Test(status))
        self.assertEqual(status.Get_source(), ctx.source)
        self.assertEqual(status.Get_tag(), ctx.tag)
        self.assertEqual(status.Get_error(), MPI.SUCCESS)
        self.assertFalse(status.Is_cancelled())
        greq.Wait()

    def testAll2(self):
        greq = MPI.Grequest.Start(None, None, None)
        self.assertFalse(greq.Test())
        greq.Cancel()
        greq.Complete()
        status = MPI.Status()
        self.assertTrue(greq.Test(status))
        self.assertEqual(status.Get_source(), MPI.ANY_SOURCE)
        self.assertEqual(status.Get_tag(), MPI.ANY_TAG)
        self.assertEqual(status.Get_error(), MPI.SUCCESS)
        self.assertFalse(status.Is_cancelled())
        greq.Wait()

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>import unittest

from myhdl import *
from myhdl_lib.mux import ls_demux
import myhdl_lib.simulation as sim

import random

class TestLsDemux(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls.simulators = ["myhdl", "icarus"]

    def testLsDemux5(self):
        ''' LS_DEMUX: 5 outputs '''
        DMAX = 100
        NUM_INPUTS = 5

        def ls_demux_top(sel,
                       di_0, di_1, di_2,
                       do_00, do_01, do_02,
                       do_10, do_11, do_12,
                       do_20, do_21, do_22,
                       do_30, do_31, do_32,
                       do_40, do_41, do_42):
            ''' Needed when ls_demux is co-simulated as top level'''
            ls_di_0 = Signal(bool(0))
            ls_di_1 = Signal(intbv(0,min=0,max=DMAX))
            ls_di_2 = Signal(intbv(0,min=-DMAX,max=DMAX))

            lsls_do_00 = Signal(bool(0))
            lsls_do_01 = Signal(intbv(0,min=0,max=DMAX))
            lsls_do_02 = Signal(intbv(0,min=-DMAX,max=DMAX))

            lsls_do_10 = Signal(bool(0))
            lsls_do_11 = Signal(intbv(0,min=0,max=DMAX))
            lsls_do_12 = Signal(intbv(0,min=-DMAX,max=DMAX))

            lsls_do_20 = Signal(bool(0))
            lsls_do_21 = Signal(intbv(0,min=0,max=DMAX))
            lsls_do_22 = Signal(intbv(0,min=-DMAX,max=DMAX))

            lsls_do_30 = Signal(bool(0))
            lsls_do_31 = Signal(intbv(0,min=0,max=DMAX))
            lsls_do_32 = Signal(intbv(0,min=-DMAX,max=DMAX))

            lsls_do_40 = Signal(bool(0))
            lsls_do_41 = Signal(intbv(0,min=0,max=DMAX))
            lsls_do_42 = Signal(intbv(0,min=-DMAX,max=DMAX))

            @always_comb
            def _assign():
                ls_di_0.next = di_0
                ls_di_1.next = di_1
                ls_di_2.next = di_2

                do_00.next = lsls_do_00
                do_01.next = lsls_do_01
                do_02.next = lsls_do_02
                do_10.next = lsls_do_10
                do_11.next = lsls_do_11
                do_12.next = lsls_do_12
                do_20.next = lsls_do_20
                do_21.next = lsls_do_21
                do_22.next = lsls_do_22
                do_30.next = lsls_do_30
                do_31.next = lsls_do_31
                do_32.next = lsls_do_32
                do_40.next = lsls_do_40
                do_41.next = lsls_do_41
                do_42.next = lsls_do_42

            lsls_do = [[lsls_do_00, lsls_do_01, lsls_do_02],
                       [lsls_do_10, lsls_do_11, lsls_do_12],
                       [lsls_do_20, lsls_do_21, lsls_do_22],
                       [lsls_do_30, lsls_do_31, lsls_do_32],
                       [lsls_do_40, lsls_do_41, lsls_do_42]]

            ls_di = [ls_di_0, ls_di_1, ls_di_2]

            _ls_demux = ls_demux(sel=sel, ls_di=ls_di, lsls_do=lsls_do)
            return instances()

        sel = Signal(intbv(0,min=0,max=NUM_INPUTS))

        ls_di = [Signal(bool(0)),
                 Signal(intbv(0,min=0,max=DMAX)),
                 Signal(intbv(0,min=-DMAX,max=DMAX))]

        lsls_do = [[Signal(bool(0)),
                    Signal(intbv(0,min=0,max=DMAX)),
                    Signal(intbv(0,min=-DMAX,max=DMAX))] for _ in range(NUM_INPUTS)]

        argl = {"sel":sel,
                "di_0":ls_di[0],"di_1":ls_di[1],"di_2":ls_di[2],
                "do_00":lsls_do[0][0], "do_01":lsls_do[0][1], "do_02":lsls_do[0][2],
                "do_10":lsls_do[1][0], "do_11":lsls_do[1][1], "do_12":lsls_do[1][2],
                "do_20":lsls_do[2][0], "do_21":lsls_do[2][1], "do_22":lsls_do[2][2],
                "do_30":lsls_do[3][0], "do_31":lsls_do[3][1], "do_32":lsls_do[3][2],
                "do_40":lsls_do[4][0], "do_41":lsls_do[4][1], "do_42":lsls_do[4][2]}

        def stim():
            @instance
            def _inst():
                sel.next = 0
                for i in range(3):
                    ls_di[i].next = 0
                yield delay(10)

                for _ in range(2):
                    d = [[random.randint(0,1),
                          random.randint(0,DMAX-1),
                          random.randint(-DMAX,DMAX-1)] for _ in range(NUM_INPUTS)]

                    for s in range(NUM_INPUTS):
                        for j in range(3):
                            ls_di[j].next = d[s][j]
                        sel.next = s
                        yield delay(10)

                        for o in range(NUM_INPUTS):
                            if s == o:
                                assert d[s][0] == lsls_do[o][0], "Mux output {} (sel={}): expected {}, detected {}".format(o, sel, d[s], [int(x) for x in lsls_do[o]])
                                assert d[s][1] == lsls_do[o][1], "Mux output {} (sel={}): expected {}, detected {}".format(o, sel, d[s], [int(x) for x in lsls_do[o]])
                                assert d[s][2] == lsls_do[o][2], "Mux output {} (sel={}): expected {}, detected {}".format(o, sel, d[s], [int(x) for x in lsls_do[o]])
                            else:
                                assert 0 == lsls_do[o][0], "Mux output {} (sel={}): expected {}, detected {}".format(o, sel, 3*[0], [int(x) for x in lsls_do[o]])
                                assert 0 == lsls_do[o][1], "Mux output {} (sel={}): expected {}, detected {}".format(o, sel, 3*[0], [int(x) for x in lsls_do[o]])
                                assert 0 == lsls_do[o][2], "Mux output {} (sel={}): expected {}, detected {}".format(o, sel, 3*[0], [int(x) for x in lsls_do[o]])

                yield delay(10)
                raise StopSimulation
            return _inst

        getDut = sim.DUTer()

        for s in self.simulators:
            getDut.selectSimulator(s)

            dut=getDut(ls_demux_top, **argl)
            stm = stim()
            Simulation( dut, stm).run()
            del dut, stm

if __name__ == "__main__":
    #import sys;sys.argv = ['', 'Test.testName']
    unittest.main()<commit_msg>Renamed an internal constant<commit_after>import unittest

from myhdl import *
from myhdl_lib.mux import ls_demux
import myhdl_lib.simulation as sim

import random

class TestLsDemux(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls.simulators = ["myhdl", "icarus"]

    def testLsDemux5(self):
        ''' LS_DEMUX: 5 outputs '''
        DMAX = 100
        NUM_OUTPUTS = 5

        def ls_demux_top(sel,
                       di_0, di_1, di_2,
                       do_00, do_01, do_02,
                       do_10, do_11, do_12,
                       do_20, do_21, do_22,
                       do_30, do_31, do_32,
                       do_40, do_41, do_42):
            ''' Needed when ls_demux is co-simulated as top level'''
            ls_di_0 = Signal(bool(0))
            ls_di_1 = Signal(intbv(0,min=0,max=DMAX))
            ls_di_2 = Signal(intbv(0,min=-DMAX,max=DMAX))

            lsls_do_00 = Signal(bool(0))
            lsls_do_01 = Signal(intbv(0,min=0,max=DMAX))
            lsls_do_02 = Signal(intbv(0,min=-DMAX,max=DMAX))

            lsls_do_10 = Signal(bool(0))
            lsls_do_11 = Signal(intbv(0,min=0,max=DMAX))
            lsls_do_12 = Signal(intbv(0,min=-DMAX,max=DMAX))

            lsls_do_20 = Signal(bool(0))
            lsls_do_21 = Signal(intbv(0,min=0,max=DMAX))
            lsls_do_22 = Signal(intbv(0,min=-DMAX,max=DMAX))

            lsls_do_30 = Signal(bool(0))
            lsls_do_31 = Signal(intbv(0,min=0,max=DMAX))
            lsls_do_32 = Signal(intbv(0,min=-DMAX,max=DMAX))

            lsls_do_40 = Signal(bool(0))
            lsls_do_41 = Signal(intbv(0,min=0,max=DMAX))
            lsls_do_42 = Signal(intbv(0,min=-DMAX,max=DMAX))

            @always_comb
            def _assign():
                ls_di_0.next = di_0
                ls_di_1.next = di_1
                ls_di_2.next = di_2

                do_00.next = lsls_do_00
                do_01.next = lsls_do_01
                do_02.next = lsls_do_02
                do_10.next = lsls_do_10
                do_11.next = lsls_do_11
                do_12.next = lsls_do_12
                do_20.next = lsls_do_20
                do_21.next = lsls_do_21
                do_22.next = lsls_do_22
                do_30.next = lsls_do_30
                do_31.next = lsls_do_31
                do_32.next = lsls_do_32
                do_40.next = lsls_do_40
                do_41.next = lsls_do_41
                do_42.next = lsls_do_42

            lsls_do = [[lsls_do_00, lsls_do_01, lsls_do_02],
                       [lsls_do_10, lsls_do_11, lsls_do_12],
                       [lsls_do_20, lsls_do_21, lsls_do_22],
                       [lsls_do_30, lsls_do_31, lsls_do_32],
                       [lsls_do_40, lsls_do_41, lsls_do_42]]

            ls_di = [ls_di_0, ls_di_1, ls_di_2]

            _ls_demux = ls_demux(sel=sel, ls_di=ls_di, lsls_do=lsls_do)
            return instances()

        sel = Signal(intbv(0,min=0,max=NUM_OUTPUTS))

        ls_di = [Signal(bool(0)),
                 Signal(intbv(0,min=0,max=DMAX)),
                 Signal(intbv(0,min=-DMAX,max=DMAX))]

        lsls_do = [[Signal(bool(0)),
                    Signal(intbv(0,min=0,max=DMAX)),
                    Signal(intbv(0,min=-DMAX,max=DMAX))] for _ in range(NUM_OUTPUTS)]

        argl = {"sel":sel,
                "di_0":ls_di[0],"di_1":ls_di[1],"di_2":ls_di[2],
                "do_00":lsls_do[0][0], "do_01":lsls_do[0][1], "do_02":lsls_do[0][2],
                "do_10":lsls_do[1][0], "do_11":lsls_do[1][1], "do_12":lsls_do[1][2],
                "do_20":lsls_do[2][0], "do_21":lsls_do[2][1], "do_22":lsls_do[2][2],
                "do_30":lsls_do[3][0], "do_31":lsls_do[3][1], "do_32":lsls_do[3][2],
                "do_40":lsls_do[4][0], "do_41":lsls_do[4][1], "do_42":lsls_do[4][2]}

        def stim():
            @instance
            def _inst():
                sel.next = 0
                for i in range(3):
                    ls_di[i].next = 0
                yield delay(10)

                for _ in range(2):
                    d = [[random.randint(0,1),
                          random.randint(0,DMAX-1),
                          random.randint(-DMAX,DMAX-1)] for _ in range(NUM_OUTPUTS)]

                    for s in range(NUM_OUTPUTS):
                        for j in range(3):
                            ls_di[j].next = d[s][j]
                        sel.next = s
                        yield delay(10)

                        for o in range(NUM_OUTPUTS):
                            if s == o:
                                assert d[s][0] == lsls_do[o][0], "Mux output {} (sel={}): expected {}, detected {}".format(o, sel, d[s], [int(x) for x in lsls_do[o]])
                                assert d[s][1] == lsls_do[o][1], "Mux output {} (sel={}): expected {}, detected {}".format(o, sel, d[s], [int(x) for x in lsls_do[o]])
                                assert d[s][2] == lsls_do[o][2], "Mux output {} (sel={}): expected {}, detected {}".format(o, sel, d[s], [int(x) for x in lsls_do[o]])
                            else:
                                assert 0 == lsls_do[o][0], "Mux output {} (sel={}): expected {}, detected {}".format(o, sel, 3*[0], [int(x) for x in lsls_do[o]])
                                assert 0 == lsls_do[o][1], "Mux output {} (sel={}): expected {}, detected {}".format(o, sel, 3*[0], [int(x) for x in lsls_do[o]])
                                assert 0 == lsls_do[o][2], "Mux output {} (sel={}): expected {}, detected {}".format(o, sel, 3*[0], [int(x) for x in lsls_do[o]])

                yield delay(10)
                raise StopSimulation
            return _inst

        getDut = sim.DUTer()

        for s in self.simulators:
            getDut.selectSimulator(s)

            dut=getDut(ls_demux_top, **argl)
            stm = stim()
            Simulation( dut, stm).run()
            del dut, stm

if __name__ == "__main__":
    #import sys;sys.argv = ['', 'Test.testName']
    unittest.main()<|endoftext|>


<commit_before>import pytest
import random
import itertools
from pypeerassets import Kutil
from pypeerassets.protocol import (CardTransfer, Deck, IssueMode,
                                   validate_card_issue_modes)

def test_deck_object():
    '''test creation of deck objects'''

    deck = Deck(
        name="decky",
        number_of_decimals=2,
        issue_mode=IssueMode.MULTI.value,
        network="ppc",
        production=True,
        version=1,
        asset_specific_data="Just testing.",
    )

    assert deck.__dict__ == {'id': None,
                             'asset_specific_data': 'Just testing.',
                             'issue_mode': IssueMode.MULTI.value,
                             'issue_time': None,
                             'confirms': None,
                             'issuer': '',
                             'name': 'decky',
                             'network': 'ppc',
                             'number_of_decimals': 2,
                             'production': True,
                             'testnet': False,
                             'version': 1
                            }

    assert deck.metainfo_to_protobuf == b'\x08\x01\x12\x05decky\x18\x02 \x04*\rJust testing.'

    assert deck.metainfo_to_dict == {'issue_mode': IssueMode.MULTI.value,
                                     'name': 'decky',
                                     'number_of_decimals': 2,
                                     'version': 1,
                                     'asset_specific_data': 'Just testing.',
                                    }

def test_card_transfer_object():

    deck = Deck(
        name="decky",
        number_of_decimals=2,
        issue_mode=IssueMode.MULTI.value,
        network="ppc",
        production=True,
        version=1,
        asset_specific_data="Just testing.",
    )

    card_transfer = CardTransfer(
        deck=deck,
        receiver=["PDZ9MPBPPjtT6qdJm98PhLVY9gNtFUoSLT"],
        amount=[1],
        version=1,
    )

    assert card_transfer.metainfo_to_protobuf == b'\x08\x01\x12\x01\x01\x18\x02'

    assert card_transfer.__dict__ == {'amount': [1],
                                      'asset_specific_data': None,
                                      'blockhash': 0,
                                      'blocknum': 0,
                                      'blockseq': 0,
                                      'cardseq': 0,
                                      'confirms': 0,
                                      'deck_id': None,
                                      'number_of_decimals': 2,
                                      'receiver': ['PDZ9MPBPPjtT6qdJm98PhLVY9gNtFUoSLT'],
                                      'sender': None,
                                      'timestamp': 0,
                                      'txid': None,
                                      'type': 'CardTransfer',
                                      'version': 1,
                                      'deck_p2th': None
                                      }

@pytest.mark.parametrize("combo", [IssueMode.ONCE, IssueMode.MULTI, IssueMode.MONO])
def test_issue_mode_combos(combo):

    base_issue_mode = IssueMode.CUSTOM

    if combo == IssueMode.ONCE:
        assert base_issue_mode.value + combo.value == 3

    if combo == IssueMode.MULTI:
        assert base_issue_mode.value + combo.value == 5

    if combo == IssueMode.MONO:
        assert base_issue_mode.value + combo.value == 9

def few_random_cards(deck: Deck, n: int, card_type: str='random',
                     amount: int=None) -> list:
    '''returns <n> randomly generated cards'''

    types = ['CardIssue', 'CardBurn', 'CardTransfer']

    cards = [CardTransfer(
        deck=deck,
        receiver=[Kutil(network='tppc').address],
        amount=[random.randint(1, 100)],
        ) for i in range(n)]

    if card_type == 'transfer':
        for i in cards:
            i.__setattr__('type', 'CardTransfer')

    if card_type == 'random':
        for i in cards:
            i.__setattr__('type', random.choice(types))

    if card_type == 'issue':
        for i in cards:
            i.__setattr__('type', 'CardIssue')

    if card_type == 'burn':
        for i in cards:
            i.__setattr__('type', 'CardBurn')

    if amount:  # if there is strict requirement for amount to be <int>
        for i in cards:
            i.amount = [amount]

    return cards

def test_validate_multi_card_issue_mode():
    '''test card filtering against MULTI deck'''

    deck = Deck(
        name="decky",
        number_of_decimals=2,
        issue_mode=IssueMode.MULTI.value,
        network="tppc",
        production=True,
        version=1,
        )

    cards = few_random_cards(deck, 4, 'issue')

    assert len(validate_card_issue_modes(deck.issue_mode, cards)) == 4

def test_validate_once_card_issue_mode():
    '''test card filtering against ONCE deck'''

    deck = Deck(
        name="decky",
        number_of_decimals=2,
        issue_mode=IssueMode.ONCE.value,
        network="tppc",
        production=True,
        version=1,
        )

    cards = few_random_cards(deck, 8, 'issue')

    assert len(validate_card_issue_modes(deck.issue_mode, cards)) == 1

def test_validate_none_card_issue_mode():
    '''test card filtering against None deck'''

    deck = Deck(
        name="decky",
        number_of_decimals=0,
        issue_mode=IssueMode.NONE.value,
        network="tppc",
        production=True,
        version=1,
        )

    cards = few_random_cards(deck, 8, 'issue')

    assert len(validate_card_issue_modes(deck.issue_mode, cards)) == 0

def test_validate_unflushable_card_issue_mode():
    '''test card filtering against None deck'''

    deck = Deck(
        name="decky",
        number_of_decimals=0,
        issue_mode=IssueMode.UNFLUSHABLE.value,
        network="tppc",
        production=True,
        version=1,
        )

    cards_issues = few_random_cards(deck, 8, 'issue')
    random_cards = few_random_cards(deck, 16, 'transfer')

    assert len(validate_card_issue_modes(deck.issue_mode, cards_issues + random_cards)) == 8

def test_validate_mono_card_issue_mode():
    '''test card filtering against MONO deck'''

    deck = Deck(
        name="decky",
        number_of_decimals=0,
        issue_mode=IssueMode.MONO.value,
        network="tppc",
        production=True,
        version=1,
        )

    issues = few_random_cards(deck, 10, 'issue', 1)
    other = few_random_cards(deck, 9, 'transfer', 1)
    invalid = few_random_cards(deck, 9, 'transfer', 2)

    assert len(validate_card_issue_modes(deck.issue_mode, issues + other + invalid)) == 19

def test_validate_singlet_card_issue_mode():
    '''test card filtering against SINGLET deck'''

    deck = Deck(
        name="decky",
        number_of_decimals=0,
        issue_mode=IssueMode.SINGLET.value,
        network="tppc",
        production=True,
        version=1,
        )

    first_issue = few_random_cards(deck, 10, 'issue', 1)  # first 10, with amount=1
    second_issue = few_random_cards(deck, 5, 'issue')  # with random amounts

    assert len(validate_card_issue_modes(deck.issue_mode, first_issue + second_issue)) == 1

def test_validate_subscription_card_issue_mode():
    '''test card filtering against SUBSCRIPTION deck'''

    deck = Deck(
        name="decky",
        number_of_decimals=0,
        issue_mode=IssueMode.SUBSCRIPTION.value,
        network="tppc",
        production=True,
        version=1,
        )

    issues = few_random_cards(deck, 10, 'issue')
    other = few_random_cards(deck, 9, 'transfer', 1)

    assert len(validate_card_issue_modes(deck.issue_mode, issues + other)) == 10

def test_validate_3combo_card_issue_mode():
    '''combo ONCE [2] and CUSTOM [1]'''

    deck = Deck(
        name="decky",
        number_of_decimals=0,
        issue_mode=3,
        network="tppc",
        production=True,
        version=1,
        )

    issues = few_random_cards(deck, 8, 'issue')
    other = few_random_cards(deck, 20, 'transfer')

    assert len(validate_card_issue_modes(deck.issue_mode, issues + other)) == 21

def test_validate_10combo_card_issue_mode():
    '''combo ONCE [2] and MONO [8]'''

    deck = Deck(
        name="decky",
        number_of_decimals=0,
        issue_mode=10,
        network="tppc",
        production=True,
        version=1,
        )

    issues = few_random_cards(deck, 3, 'issue', 1)
    mono_transfers = few_random_cards(deck, 2, 'transfer', 1)
    other = few_random_cards(deck, 20, 'transfer')

    assert len(validate_card_issue_modes(
               deck.issue_mode, issues + other + mono_transfers)) == 3

def test_validate_6combo_card_issue_mode():
    '''combo ONCE [2] and MULTI [4]'''

    deck = Deck(
        name="decky",
        number_of_decimals=1,
        issue_mode=6,
        network="tppc",
        production=True,
        version=1,
        )

    issues = few_random_cards(deck, 3, 'issue',)
    other = few_random_cards(deck, 2, 'transfer')

    assert len(validate_card_issue_modes(
               deck.issue_mode, issues + other)) == 3

def test_validate_28combo_card_issue_mode():
    '''combo MULTI [4], MONO [8] and UNFLUSHABLE [10]'''

    mode = IssueMode.MULTI.value | IssueMode.MONO.value | IssueMode.UNFLUSHABLE.value  # 28

    deck = Deck(
        name="decky",
        number_of_decimals=0,
        issue_mode=mode,
        network="tppc",
        production=True,
        version=1,
        )

    issues = few_random_cards(deck, 10, 'issue', 1)
    bad_issues = few_random_cards(deck, 2, 'issue')
    other = few_random_cards(deck, 2, 'transfer', 1)

    assert len(validate_card_issue_modes(
               deck.issue_mode, issues + bad_issues + other)) == 10

def test_validate_13combo_card_issue_mode():
    '''combo CUSTOM [1], MULTI [4] and MONO [8]'''

    mode = IssueMode.CUSTOM.value | IssueMode.MULTI.value | IssueMode.MONO.value  # 13

    deck = Deck(
        name="decky",
        number_of_decimals=0,
        issue_mode=mode,
        network="tppc",
        production=True,
        version=1,
        )

    issues = few_random_cards(deck, 10, 'issue', 1)
    bad_issues = few_random_cards(deck, 2, 'issue')
    other = few_random_cards(deck, 2, 'transfer', 1)

    assert len(validate_card_issue_modes(
               deck.issue_mode, issues + bad_issues + other)) == 12

@pytest.mark.parametrize("combo", list(
                         itertools.combinations(
                             [0, 1, 2, 4, 8, 16, 52, 10], 2))
                         )
def test_validate_wild_two_way_combos(combo):

    deck = Deck(
        name="decky",
        number_of_decimals=0,
        issue_mode=combo[0] + combo[1],
        network="tppc",
        production=True,
        version=1,
        )

    issues = few_random_cards(deck, 5, 'issue')
    other = few_random_cards(deck, 15, 'transfer')

    assert isinstance(validate_card_issue_modes(
                      deck.issue_mode, issues + other), list)

@pytest.mark.parametrize("combo", list(
                         itertools.combinations(
                             [0, 1, 2, 4, 8, 16, 52, 10], 3))
                         )
def test_validate_wild_three_way_combos(combo):

    deck = Deck(
        name="decky",
        number_of_decimals=0,
        issue_mode=combo[0] + combo[1] + combo[2],
        network="tppc",
        production=True,
        version=1,
        )

    issues = few_random_cards(deck, 5, 'issue')
    other = few_random_cards(deck, 15, 'transfer')

    assert isinstance(validate_card_issue_modes(
                      deck.issue_mode, issues + other), list)
<commit_msg>implement test for the DeckState<commit_after>import pytest
import random
import itertools
from pypeerassets import Kutil
from pypeerassets.protocol import (CardTransfer, Deck, IssueMode,
                                   validate_card_issue_modes, DeckState)

def test_deck_object():
    '''test creation of deck objects'''

    deck = Deck(
        name="decky",
        number_of_decimals=2,
        issue_mode=IssueMode.MULTI.value,
        network="ppc",
        production=True,
        version=1,
        asset_specific_data="Just testing.",
    )

    assert deck.__dict__ == {'id': None,
                             'asset_specific_data': 'Just testing.',
                             'issue_mode': IssueMode.MULTI.value,
                             'issue_time': None,
                             'confirms': None,
                             'issuer': '',
                             'name': 'decky',
                             'network': 'ppc',
                             'number_of_decimals': 2,
                             'production': True,
                             'testnet': False,
                             'version': 1
                            }

    assert deck.metainfo_to_protobuf == b'\x08\x01\x12\x05decky\x18\x02 \x04*\rJust testing.'

    assert deck.metainfo_to_dict == {'issue_mode': IssueMode.MULTI.value,
                                     'name': 'decky',
                                     'number_of_decimals': 2,
                                     'version': 1,
                                     'asset_specific_data': 'Just testing.',
                                    }

def test_card_transfer_object():

    deck = Deck(
        name="decky",
        number_of_decimals=2,
        issue_mode=IssueMode.MULTI.value,
        network="ppc",
        production=True,
        version=1,
        asset_specific_data="Just testing.",
    )

    card_transfer = CardTransfer(
        deck=deck,
        receiver=["PDZ9MPBPPjtT6qdJm98PhLVY9gNtFUoSLT"],
        amount=[1],
        version=1,
    )

    assert card_transfer.metainfo_to_protobuf == b'\x08\x01\x12\x01\x01\x18\x02'

    assert card_transfer.__dict__ == {'amount': [1],
                                      'asset_specific_data': None,
                                      'blockhash': 0,
                                      'blocknum': 0,
                                      'blockseq': 0,
                                      'cardseq': 0,
                                      'confirms': 0,
                                      'deck_id': None,
                                      'number_of_decimals': 2,
                                      'receiver': ['PDZ9MPBPPjtT6qdJm98PhLVY9gNtFUoSLT'],
                                      'sender': None,
                                      'timestamp': 0,
                                      'txid': None,
                                      'type': 'CardTransfer',
                                      'version': 1,
                                      'deck_p2th': None
                                      }

@pytest.mark.parametrize("combo", [IssueMode.ONCE, IssueMode.MULTI, IssueMode.MONO])
def test_issue_mode_combos(combo):

    base_issue_mode = IssueMode.CUSTOM

    if combo == IssueMode.ONCE:
        assert base_issue_mode.value + combo.value == 3

    if combo == IssueMode.MULTI:
        assert base_issue_mode.value + combo.value == 5

    if combo == IssueMode.MONO:
        assert base_issue_mode.value + combo.value == 9

def few_random_cards(deck: Deck, n: int, card_type: str='random',
                     amount: int=None) -> list:
    '''returns <n> randomly generated cards'''

    types = ['CardIssue', 'CardBurn', 'CardTransfer']

    cards = [CardTransfer(
        deck=deck,
        receiver=[Kutil(network='tppc').address],
        amount=[random.randint(1, 100)],
        ) for i in range(n)]

    if card_type == 'transfer':
        for i in cards:
            i.__setattr__('type', 'CardTransfer')

    if card_type == 'random':
        for i in cards:
            i.__setattr__('type', random.choice(types))

    if card_type == 'issue':
        for i in cards:
            i.__setattr__('type', 'CardIssue')

    if card_type == 'burn':
        for i in cards:
            i.__setattr__('type', 'CardBurn')

    if amount:  # if there is strict requirement for amount to be <int>
        for i in cards:
            i.amount = [amount]

    return cards

def test_validate_multi_card_issue_mode():
    '''test card filtering against MULTI deck'''

    deck = Deck(
        name="decky",
        number_of_decimals=2,
        issue_mode=IssueMode.MULTI.value,
        network="tppc",
        production=True,
        version=1,
        )

    cards = few_random_cards(deck, 4, 'issue')

    assert len(validate_card_issue_modes(deck.issue_mode, cards)) == 4

def test_validate_once_card_issue_mode():
    '''test card filtering against ONCE deck'''

    deck = Deck(
        name="decky",
        number_of_decimals=2,
        issue_mode=IssueMode.ONCE.value,
        network="tppc",
        production=True,
        version=1,
        )

    cards = few_random_cards(deck, 8, 'issue')

    assert len(validate_card_issue_modes(deck.issue_mode, cards)) == 1

def test_validate_none_card_issue_mode():
    '''test card filtering against None deck'''

    deck = Deck(
        name="decky",
        number_of_decimals=0,
        issue_mode=IssueMode.NONE.value,
        network="tppc",
        production=True,
        version=1,
        )

    cards = few_random_cards(deck, 8, 'issue')

    assert len(validate_card_issue_modes(deck.issue_mode, cards)) == 0

def test_validate_unflushable_card_issue_mode():
    '''test card filtering against None deck'''

    deck = Deck(
        name="decky",
        number_of_decimals=0,
        issue_mode=IssueMode.UNFLUSHABLE.value,
        network="tppc",
        production=True,
        version=1,
        )

    cards_issues = few_random_cards(deck, 8, 'issue')
    random_cards = few_random_cards(deck, 16, 'transfer')

    assert len(validate_card_issue_modes(deck.issue_mode, cards_issues + random_cards)) == 8

def test_validate_mono_card_issue_mode():
    '''test card filtering against MONO deck'''

    deck = Deck(
        name="decky",
        number_of_decimals=0,
        issue_mode=IssueMode.MONO.value,
        network="tppc",
        production=True,
        version=1,
        )

    issues = few_random_cards(deck, 10, 'issue', 1)
    other = few_random_cards(deck, 9, 'transfer', 1)
    invalid = few_random_cards(deck, 9, 'transfer', 2)

    assert len(validate_card_issue_modes(deck.issue_mode, issues + other + invalid)) == 19

def test_validate_singlet_card_issue_mode():
    '''test card filtering against SINGLET deck'''

    deck = Deck(
        name="decky",
        number_of_decimals=0,
        issue_mode=IssueMode.SINGLET.value,
        network="tppc",
        production=True,
        version=1,
        )

    first_issue = few_random_cards(deck, 10, 'issue', 1)  # first 10, with amount=1
    second_issue = few_random_cards(deck, 5, 'issue')  # with random amounts

    assert len(validate_card_issue_modes(deck.issue_mode, first_issue + second_issue)) == 1

def test_validate_subscription_card_issue_mode():
    '''test card filtering against SUBSCRIPTION deck'''

    deck = Deck(
        name="decky",
        number_of_decimals=0,
        issue_mode=IssueMode.SUBSCRIPTION.value,
        network="tppc",
        production=True,
        version=1,
        )

    issues = few_random_cards(deck, 10, 'issue')
    other = few_random_cards(deck, 9, 'transfer', 1)

    assert len(validate_card_issue_modes(deck.issue_mode, issues + other)) == 10

def test_validate_3combo_card_issue_mode():
    '''combo ONCE [2] and CUSTOM [1]'''

    deck = Deck(
        name="decky",
        number_of_decimals=0,
        issue_mode=3,
        network="tppc",
        production=True,
        version=1,
        )

    issues = few_random_cards(deck, 8, 'issue')
    other = few_random_cards(deck, 20, 'transfer')

    assert len(validate_card_issue_modes(deck.issue_mode, issues + other)) == 21

def test_validate_10combo_card_issue_mode():
    '''combo ONCE [2] and MONO [8]'''

    deck = Deck(
        name="decky",
        number_of_decimals=0,
        issue_mode=10,
        network="tppc",
        production=True,
        version=1,
        )

    issues = few_random_cards(deck, 3, 'issue', 1)
    mono_transfers = few_random_cards(deck, 2, 'transfer', 1)
    other = few_random_cards(deck, 20, 'transfer')

    assert len(validate_card_issue_modes(
               deck.issue_mode, issues + other + mono_transfers)) == 3

def test_validate_6combo_card_issue_mode():
    '''combo ONCE [2] and MULTI [4]'''

    deck = Deck(
        name="decky",
        number_of_decimals=1,
        issue_mode=6,
        network="tppc",
        production=True,
        version=1,
        )

    issues = few_random_cards(deck, 3, 'issue',)
    other = few_random_cards(deck, 2, 'transfer')

    assert len(validate_card_issue_modes(
               deck.issue_mode, issues + other)) == 3

def test_validate_28combo_card_issue_mode():
    '''combo MULTI [4], MONO [8] and UNFLUSHABLE [10]'''

    mode = IssueMode.MULTI.value | IssueMode.MONO.value | IssueMode.UNFLUSHABLE.value  # 28

    deck = Deck(
        name="decky",
        number_of_decimals=0,
        issue_mode=mode,
        network="tppc",
        production=True,
        version=1,
        )

    issues = few_random_cards(deck, 10, 'issue', 1)
    bad_issues = few_random_cards(deck, 2, 'issue')
    other = few_random_cards(deck, 2, 'transfer', 1)

    assert len(validate_card_issue_modes(
               deck.issue_mode, issues + bad_issues + other)) == 10

def test_validate_13combo_card_issue_mode():
    '''combo CUSTOM [1], MULTI [4] and MONO [8]'''

    mode = IssueMode.CUSTOM.value | IssueMode.MULTI.value | IssueMode.MONO.value  # 13

    deck = Deck(
        name="decky",
        number_of_decimals=0,
        issue_mode=mode,
        network="tppc",
        production=True,
        version=1,
        )

    issues = few_random_cards(deck, 10, 'issue', 1)
    bad_issues = few_random_cards(deck, 2, 'issue')
    other = few_random_cards(deck, 2, 'transfer', 1)

    assert len(validate_card_issue_modes(
               deck.issue_mode, issues + bad_issues + other)) == 12

@pytest.mark.parametrize("combo", list(
                         itertools.combinations(
                             [0, 1, 2, 4, 8, 16, 52, 10], 2))
                         )
def test_validate_wild_two_way_combos(combo):

    deck = Deck(
        name="decky",
        number_of_decimals=0,
        issue_mode=combo[0] + combo[1],
        network="tppc",
        production=True,
        version=1,
        )

    issues = few_random_cards(deck, 5, 'issue')
    other = few_random_cards(deck, 15, 'transfer')

    assert isinstance(validate_card_issue_modes(
                      deck.issue_mode, issues + other), list)

@pytest.mark.parametrize("combo", list(
                         itertools.combinations(
                             [0, 1, 2, 4, 8, 16, 52, 10], 3))
                         )
def test_validate_wild_three_way_combos(combo):

    deck = Deck(
        name="decky",
        number_of_decimals=0,
        issue_mode=combo[0] + combo[1] + combo[2],
        network="tppc",
        production=True,
        version=1,
        )

    issues = few_random_cards(deck, 5, 'issue')
    other = few_random_cards(deck, 15, 'transfer')

    assert isinstance(validate_card_issue_modes(
                      deck.issue_mode, issues + other), list)

def test_deck_state():
    '''test DeckState calculations'''

    deck = Deck(
        name="my_test_deck",
        number_of_decimals=0,
        issue_mode=12,  # MULTI | MONO
        network="tppc",
        production=True,
        version=1,
        issuer='msnHPXDWuJhRBPVNQnwXdKvEMQHLr9z1P5'
        )

    receiver_roster = ['mzsMJgqVABFhrEGrqKH7qURhmxESx4K8Ti',
                       'mmsiUudS9W5xLoWeA44JmKa28cioFg7Yzx'
                       'muMpqVjUDq5voY9WnxvFb9sFvZm8wwKihu',
                       'mxrr8ALSs9fmHEszs5y1w5tRsDv9r7M2bK']
    amounts = [10, 20, 30, 40]

    card_issues = [CardTransfer(deck=deck,
                                receiver=[r],
                                amount=[a],
                                sender=deck.issuer,
                                blockseq=0,
                                txid='fe8f88c2a3a700a664f9547cb9c48466f900553d0a6bdb504ad52340ef00c9a0',
                                cardseq=amounts.index(a)
                                ) for r, a in zip(receiver_roster, amounts)]

    transfers = []  # list of card transfers

    # first member of the roster sends it's 10 cards to third member of the roster
    transfers.append(CardTransfer(deck=deck,
                                  sender=receiver_roster[0],
                                  receiver=[receiver_roster[2]],
                                  amount=[amounts[0]],
                                  blockseq=1,
                                  cardseq=0,
                                  txid='08c886a43ce9f95a5673bc95374259b0f9eca9de1e5fb9bb7aa7826834820133',
                                  type='CardTransfer'
                                  ))

    # second member of the roster burns it's 20 cards, he calls it a scam too
    transfers.append(CardTransfer(deck=deck,
                                  sender=receiver_roster[1],
                                  receiver=[deck.issuer],
                                  amount=[amounts[1]],
                                  blockseq=1,
                                  cardseq=0,
                                  txid='b27161ba476d29c2255d097aaa4e236752b9891a46d1fdb88f5225ee677b976e',
                                  type='CardBurn'
                                  ))

    # third member of the roster sends out it's cards to r[0] and r[4]
    transfers.append(CardTransfer(deck=deck,
                                  sender=receiver_roster[2],
                                  receiver=[deck.issuer],
                                  amount=[10],
                                  blockseq=1,
                                  cardseq=0,
                                  txid='ebe36158ca3f364910f8a1c0f9b1b2696bed4522f84551bdb42ffd57360ce232',
                                  type='CardTransfer'
                                  ))

    transfers.append(CardTransfer(deck=deck,
                                  sender=receiver_roster[2],
                                  receiver=[deck.issuer],
                                  amount=[20],
                                  blockseq=1,
                                  cardseq=1,
                                  txid='ebe36158ca3f364910f8a1c0f9b1b2696bed4522f84551bdb42ffd57360ce232',
                                  type='CardTransfer'
                                  ))

    state = DeckState(card_issues + transfers)

    assert state.balances['mzsMJgqVABFhrEGrqKH7qURhmxESx4K8Ti'] == 10
    assert state.balances['mt6U36PBurMsgjNVkPefYoku7uypLMRrbx'] == 0
    assert state.balances['mmsiUudS9W5xLoWeA44JmKa28cioFg7Yzx'] == 10
    assert state.balances['muobvRLAAMgEWBDNGLz1bGJ1Kj4yZUqCGj'] == 60
<|endoftext|>


<commit_before>########################################################################
<commit_msg>Turns out other parts of the test code need this file. I did test it, but it picked up the system installation, not the test one. Serve me right for not running nosetests from the highest level<commit_after>########################################################################
# test/xslt/__init__.py
import os
import gc
import sys
import uuid
import difflib

from amara import Error
from amara.lib import inputsource, iri, treecompare
from amara.xpath import datatypes, util
from amara.xslt import XsltError, transform
from amara.xslt.processor import processor

# Defined to ignore this module in tracebacks
__unittest = True

#from amara.xslt.functions import generate_id_function as xslt_generate_id
#g_idmap = {}
#class generate_id_function(xslt_generate_id):
#    """
#    Replacement for XSLT's generate-id(). Generates IDs that are
#    unique, but not random, for comparisons in the test suites.
#    """
#    def evaluate(self, context):
#        result = xslt_generate_id.evaluate(self, context)
#        if result:
#            result = g_idmap.setdefault(result, len(g_idmap) + 1)
#            result = datatypes.string(u'id%d' % result)
#        return result

class _mapping_resolver(iri.default_resolver):
    def __init__(self, uris):
        iri.default_resolver.__init__(self)
        self._uris = uris

    def normalize(self, uriref, baseuri):
        if uriref in self._uris:
            return uriref
        return iri.default_resolver.normalize(self, uriref, baseuri)

    def resolve(self, uri, baseuri=None):
        if uri in self._uris:
            return cStringIO.StringIO(self._uris[uri])
        return iri.default_resolver.resolve(self, uri, baseuri)

def get_mapping_factory():
    return

class testsource(object):
    """
    Encapsulates an inputsource given as a string or URI, so that it can be
    referenced in various ways. Used by XsltTest().
    """
    __slots__ = ('source', 'uri', 'validate', 'xinclude', 'external')
    def __init__(self, source, uri=None, validate=False, xinclude=True,
                 external=False):
        self.source = source
        self.uri = uri
        self.validate = validate
        self.xinclude = xinclude
        self.external = external

class filesource(testsource):
    def __init__(self, path, validate=False, xinclude=True, external=False):
        # Same logic that exists in _4xslt.py
        # The processor only deals with URIs, not file paths
        if not os.path.isabs(path):
            # it is relative to the calling module
            module = sys._getframe(1).f_globals['__name__']
            module = sys.modules[module]
            moduledir = os.path.dirname(os.path.abspath(module.__file__))
            path = os.path.join(moduledir, path)
            assert os.path.exists(path)
        testsource.__init__(self, path, None, validate, xinclude)

class stringsource(testsource):
    def __init__(self, source, uri=None, validate=False, xinclude=True,
                 external=False):
        if not uri:
            # it is relative to the calling module
            module = sys._getframe(1).f_globals['__name__']
            module = sys.modules[module]
            moduledir = os.path.dirname(os.path.abspath(module.__file__))
            path = str(uuid.uuid4())
            uri = iri.os_path_to_uri(os.path.join(moduledir, path))
        testsource.__init__(self, source, uri, validate, xinclude)

from amara.test import test_case, TestError

class xslt_test(test_case):

    source = None
    transform = None
    expected = None
    parameters = None
    force_method = None

    class __metaclass__(type):
        def __init__(cls, name, bases, namespace):
            transform = cls.transform
            if transform and not isinstance(transform, testsource):
                cls.transform = tuple(transform)

    def _format_error(self, error_class, error_code):
        if not issubclass(error_class, Error):
            return error_class.__name__
        for name, value in error_class.__dict__.iteritems():
            if value == error_code:
                error_code = error_class.__name__ + '.' + name
                break
        return '%s(%s)' % (error_class.__name__, error_code)

    def setUp(self):
        self.source = inputsource(self.source.source, self.source.uri)
        if isinstance(self.transform, testsource):
            T = self.transform
            self.transform = [inputsource(T.source, T.uri)]
        elif self.transform:
              self.transform = [ inputsource(T.source, T.uri)
                                 for T in self.transform ]
        else:
            self.transform = ()
        return

    def _assert_result(self, result):
        method = (None, self.force_method) if self.force_method else result.parameters.method
        omit_decl = result.parameters.omit_xml_declaration
        expected, compared = self.expected, result
        #print expected, compared
        if method == (None, 'xml') and not omit_decl:
            diff = treecompare.xml_diff(expected, compared)
        elif method == (None, 'html'):
            diff = treecompare.html_diff(expected, compared)
        else:
            expected = expected.splitlines()
            compared = compared.splitlines()
            diff = difflib.unified_diff(expected, compared,
                                        'expected', 'compared',
                                        n=2, lineterm='')
        diff = '\n'.join(diff)
        self.assertFalse(diff, msg=(None, diff))
        gc.collect()

    def test_processor(self):
        P = processor()
        for transform in self.transform:
            P.append_transform(transform)
        parameters = self.parameters
        if parameters:
            parameters = util.parameterize(parameters)
        result = P.run(self.source, parameters=parameters)
        self._assert_result(result)

    #def test_transform(self):
    #    result = transform(self.source, self.transform, params=self.parameters)
    #    self._assert_result(result)

class xslt_error(xslt_test):
    error_class = XsltError
    error_code = None

    class __metaclass__(xslt_test.__metaclass__):
        def __init__(cls, name, bases, namespace):
            xslt_test.__metaclass__.__init__(cls, name, bases, namespace)
            if (issubclass(cls.error_class, Error)
                and cls.error_code is None
                and 'error_code' not in namespace
                and '__unittest__' not in namespace):
                raise ValueError("class '%s' must define 'error_code'" %
                                 name)

    def test_processor(self):
        try:
            xslt_test.test_processor(self)
        except self.error_class, error:
            expected = self._format_error(self.error_class, self.error_code)
            compared = self._format_error(self.error_class,
                                          getattr(error, 'code', None))
            self.assertEquals(expected, compared)
        else:
            expected = self._format_error(self.error_class, self.error_code)
            self.fail('%s not raised' % expected)

if __name__ == '__main__':
    from amara.test import test_main, test_case, TestError
    import glob
    module_dir = os.path.dirname(os.path.abspath(__file__))
    pattern = os.path.join(module_dir, 'test_*.py')
    test_modules = [ os.path.basename(fn)[:-3] for fn in glob.glob(pattern) ]
    # Re-arrange the modules /slightly/
    for name in ('test_basics', 'test_exslt', 'test_borrowed'):
        try:
            test_modules.remove(name)
        except ValueError:
            pass
        else:
            test_modules.append(name)
    test_modules.remove('test_borrowed')
    test_main(*test_modules)
<|endoftext|>


<commit_before>"""
Tests speed at which IO occurs, reading and writing values to the
Office program

Times:

"""

from leadmacro import Office, Translation

from time import time
from random import shuffle

try:
    import xlwings as xw
except ImportError:
    xw = None

OVERALL_TIME_TO_BEAT = None

SOURCE_COLUMN_NAME_KEY = 'source_column_name'
TARGET_COLUMN_NAME_KEY = 'target_column_name'
WHITESPACE_CHK_KEY = 'check_for_whitespace'
DUPLICATE_CHK_KEY = 'check_for_duplicates'

DFT_SRC = 'st_src_1'
DFT_TGT = 'st_tgt_1'

def test_translation_speed_is_improved(src_sheet=None, tgt_sheet=None):
    """
    Test speed at which a test translation is applied.
    :return: None
    """
    if src_sheet is None:
        src_sheet = DFT_SRC
    if tgt_sheet is None:
        tgt_sheet = DFT_TGT
    model = Office.get_model()
    print('getting sheets')
    sheet_finding_start_time = time()
    src_sheet = model[src_sheet]
    tgt_sheet = model[tgt_sheet]
    # enable caching, as should be the case in regular use
    src_sheet.exclusive_editor = True
    tgt_sheet.exclusive_editor = True
    print('finished getting sheet instances (%ss)' %
          (time() - sheet_finding_start_time))
    assert isinstance(src_sheet, Office.XW.Sheet), src_sheet
    assert isinstance(tgt_sheet, Office.XW.Sheet), tgt_sheet
    print('getting src col names')
    src_col_start_time = time()
    src_col_names = [name for name in src_sheet.columns.names]
    print('finished getting src col names (%ss)'
          % (time() - src_col_start_time ))
    print('getting tgt col names')
    tgt_col_start_time = time()
    tgt_col_names = [name for name in tgt_sheet.columns.names]
    print('finished getting src col names (%ss)'
          % (time() - tgt_col_start_time))
    shuffle(src_col_names)
    # make translation dicts
    translation_dicts = []
    i = 0
    n = len(src_col_names)
    assert len(src_col_names) == len(tgt_col_names)
    print('building translation')
    for src_col_name, tgt_col_name in zip(src_col_names, tgt_col_names):
        # in first 1/5 of column transforms, check for duplicates
        if i <= n / 5:
            duplicate_chk = True
        else:
            duplicate_chk = False
        translation_dicts.append({
            SOURCE_COLUMN_NAME_KEY: src_col_name,
            TARGET_COLUMN_NAME_KEY: tgt_col_name,
            DUPLICATE_CHK_KEY: duplicate_chk,
            WHITESPACE_CHK_KEY: True
        })
    print('done building translation dictionaries')
    print('starting clock')
    start_time = time()
    print('building Translation')
    translation_build_t1 = time()
    translation = Translation(
        None,
        source_sheet=src_sheet,
        target_sheet=tgt_sheet,
        column_translations=translation_dicts,
    )
    print('finished building translation (%ss)'
          % (time() - translation_build_t1))
    print('committing translation')
    commit_t1 = time()
    translation.commit()
    print('finished commit. (%ss)' % (time() - commit_t1))
    elapsed_time = time() - start_time
    print('finished')
    print(
        '\nRUN TIME     : %s\n TIME TO BEAT : %s\n' %
        (elapsed_time, OVERALL_TIME_TO_BEAT)
    )

if __name__ == '__main__':

    if xw is None:
        assert False, "XW was unable to be imported"

    test_translation_speed_is_improved()
<commit_msg>xw_speed_test now uses cProfile to save results to file<commit_after>"""
Tests speed at which IO occurs, reading and writing values to the
Office program

Times:

"""

from leadmacro import Office, Translation

from time import time
from random import shuffle

from cProfile import run

import settings

import os

try:
    import xlwings as xw
except ImportError:
    xw = None

OVERALL_TIME_TO_BEAT = None

SOURCE_COLUMN_NAME_KEY = 'source_column_name'
TARGET_COLUMN_NAME_KEY = 'target_column_name'
WHITESPACE_CHK_KEY = 'check_for_whitespace'
DUPLICATE_CHK_KEY = 'check_for_duplicates'

DFT_SRC = 'st_src_1'
DFT_TGT = 'st_tgt_1'

def test_translation_speed_is_improved(src_sheet=None, tgt_sheet=None):
    """
    Test speed at which a test translation is applied.
    :return: None
    """
    if src_sheet is None:
        src_sheet = DFT_SRC
    if tgt_sheet is None:
        tgt_sheet = DFT_TGT
    model = Office.get_model()
    print('getting sheets')
    sheet_finding_start_time = time()
    src_sheet = model[src_sheet]
    tgt_sheet = model[tgt_sheet]
    # enable caching, as should be the case in regular use
    src_sheet.exclusive_editor = True
    tgt_sheet.exclusive_editor = True
    print('finished getting sheet instances (%ss)' %
          (time() - sheet_finding_start_time))
    assert isinstance(src_sheet, Office.XW.Sheet), src_sheet
    assert isinstance(tgt_sheet, Office.XW.Sheet), tgt_sheet
    print('getting src col names')
    src_col_start_time = time()
    src_col_names = [name for name in src_sheet.columns.names]
    print('finished getting src col names (%ss)'
          % (time() - src_col_start_time ))
    print('getting tgt col names')
    tgt_col_start_time = time()
    tgt_col_names = [name for name in tgt_sheet.columns.names]
    print('finished getting src col names (%ss)'
          % (time() - tgt_col_start_time))
    shuffle(src_col_names)
    # make translation dicts
    translation_dicts = []
    i = 0
    n = len(src_col_names)
    assert len(src_col_names) == len(tgt_col_names)
    print('building translation')
    for src_col_name, tgt_col_name in zip(src_col_names, tgt_col_names):
        # in first 1/5 of column transforms, check for duplicates
        if i <= n / 5:
            duplicate_chk = True
        else:
            duplicate_chk = False
        translation_dicts.append({
            SOURCE_COLUMN_NAME_KEY: src_col_name,
            TARGET_COLUMN_NAME_KEY: tgt_col_name,
            DUPLICATE_CHK_KEY: duplicate_chk,
            WHITESPACE_CHK_KEY: True
        })
    print('done building translation dictionaries')
    print('starting clock')
    start_time = time()
    print('building Translation')
    translation_build_t1 = time()
    translation = Translation(
        None,
        source_sheet=src_sheet,
        target_sheet=tgt_sheet,
        column_translations=translation_dicts,
    )
    print('finished building translation (%ss)'
          % (time() - translation_build_t1))
    print('committing translation')
    commit_t1 = time()
    translation.commit()
    print('finished commit. (%ss)' % (time() - commit_t1))
    elapsed_time = time() - start_time
    print('finished')
    print(
        '\nRUN TIME     : %s\n TIME TO BEAT : %s\n' %
        (elapsed_time, OVERALL_TIME_TO_BEAT)
    )

if __name__ == '__main__':

    if xw is None:
        assert False, "XW was unable to be imported"

    out = os.path.join(settings.PROJECT_ROOT, 'out_test', 'test_cgf')
    run('test_translation_speed_is_improved()', out)
<|endoftext|>


<commit_before>import pytest
import render_little
import models
from sqlalchemy import create_engine
from datetime import datetime
import mock
import unittest
import os
import factory
import factory.alchemy

@pytest.fixture()
def setup_dirs(monkeypatch):
    from zipfile import ZipFile
    from shutil import rmtree
    monkeypatch.setattr(render_little,
                        'PATH_DOWNLOAD',
                        str(TestProcess.test_tmp_download)
                        )

    if os.path.exists(TestProcess.test_tmp_download):
        rmtree(TestProcess.test_tmp_download)
    if not os.path.exists(TestProcess.test_input_path):
        os.makedirs(TestProcess.test_input_path)
        try:
            with ZipFile('test_tiffs_Archive.zip', 'r') as zip_file:
                zip_file.extractall(TestProcess.test_input_path)
        except IOError:
            print("Archive does not exist - downloading files")
            bands, input_path, scene_id = render_little.download_and_set(
                TestProcess.fake_job_message)

@pytest.fixture(scope='session', autouse=True)
def connection(request):
    engine = create_engine('postgresql://postgres@/test_bar')
    models.Base.metadata.create_all(engine)
    connection = engine.connect()
    models.DBSession.registry.clear()
    models.DBSession.configure(bind=connection)
    models.Base.metadata.bind = engine
    request.addfinalizer(models.Base.metadata.drop_all)
    return connection

@pytest.fixture(autouse=True)
def db_session(request, connection):
    from transaction import abort
    trans = connection.begin()
    request.addfinalizer(trans.rollback)
    request.addfinalizer(abort)

    from models import DBSession
    return DBSession

@pytest.mark.usefixtures("connection", "db_session")
class JobFactory(factory.alchemy.SQLAlchemyModelFactory):
    class Meta():
        model = models.UserJob_Model

        sqlalchemy_session = db_session

    jobstatus = 0
    starttime = datetime.utcnow()
    lastmodified = datetime.utcnow()
    band1 = u'4'
    band2 = u'3'
    band3 = u'2'
    entityid = u'LC80470272015005LGN00'
    email = u'test@test.com'
    jobid = factory.Sequence(lambda n: n)

@pytest.fixture(scope='class')
def fake_job1(db_session):
    model_instance = models.UserJob_Model(
        jobstatus=0,
        starttime=datetime.utcnow(),
        lastmodified=datetime.utcnow()
    )
    db_session.add(model_instance)
    db_session.flush()

# --test db functionality tests
def test_db_lookup(db_session):
    model_instance = models.UserJob_Model(jobstatus=0,
                                          starttime=datetime.utcnow(),
                                          lastmodified=datetime.utcnow())
    db_session.add(model_instance)
    db_session.flush()

    assert 1 == db_session.query(models.UserJob_Model).count()

def test_db_is_rolled_back(db_session):
    assert 0 == db_session.query(models.UserJob_Model).count()

# --module function tests
def test_cleanup_downloads():
    test_dir = os.getcwd() + '/testdir'

    if not os.path.exists(test_dir):
        os.makedirs(test_dir)

    f = open(test_dir + '/test.txt', 'a')
    f.write('this is a test')
    f.close()
    # cleanup_downloads returns True if works
    assert render_little.cleanup_downloads(test_dir)

def test_write_activity(monkeypatch, tmpdir):
    tmp_activity_log = tmpdir.mkdir('log').join('tmp_act_log.txt')
    monkeypatch.setattr(render_little,
                        'PATH_ACTIVITY_LOG',
                        str(tmp_activity_log)
                        )
    render_little.write_activity('test message')
    assert 'test message' in tmp_activity_log.read()

def test_write_error(monkeypatch, tmpdir):
    tmp_error_log = tmpdir.mkdir('log').join('tmp_error_log.txt')
    monkeypatch.setattr(render_little,
                        'PATH_ERROR_LOG',
                        str(tmp_error_log)
                        )
    render_little.write_error('test message')
    assert 'test message' in tmp_error_log.read()

# --jobs queue
class TestQueue(unittest.TestCase):

    def test(self):
        pass

# --process tests
@pytest.mark.usefixtures("setup_dirs")
class TestImageFiles(unittest.TestCase):
    """These tests require real files"""

    @mock.patch('worker.render_little.Downloader')
    def test_download_returns_correct_values(self, Downloader):
        bands, input_path, scene_id = (render_little.download_and_set(
            TestProcess.fake_job_message))
        self.assertEqual(input_path,
                         os.getcwd() + '/test_download/LC80470272015005LGN00')
        self.assertEqual(bands, [u'4', u'3', u'2'])
        self.assertEqual(scene_id, 'LC80470272015005LGN00')

@pytest.mark.usefixtures("connection", "db_session", "fake_job1")
class TestProcess(unittest.TestCase):

    fake_job_message = {u'job_id': u'1',
                        u'band_2': u'3',
                        u'band_3': u'2',
                        u'band_1': u'4',
                        u'scene_id': u'LC80470272015005LGN00',
                        u'email': u'test@test.com'}

    # bad_job_message is missing band_1
    bad_job_message = {u'job_id': u'1',
                       u'band_2': u'3',
                       u'band_3': u'2',
                       u'scene_id': u'LC80470272015005LGN00',
                       u'email': u'test@test.com'}

    test_tmp_download = os.getcwd() + '/test_download'
    test_input_path = os.getcwd() + '/test_download/LC80470272015005LGN00'
    test_bands = [u'4', u'3', u'2']
    bad_test_bands = [u'4', u'3']
    test_scene_id = 'LC80470272015005LGN00'
    test_file_location = (os.getcwd() +
        '/download/LC80470272015005LGN00/LC80470272015005LGN00_bands_432.TIF')
    test_file_name = 'LC80470272015005LGN00_bands_432'
    test_file_name_zip = 'LC80470272015005LGN00_bands_432.zip'
    test_file_png = 'pre_LC80470272015005LGN00_bands_432.png'
    test_file_tif = 'pre_LC80470272015005LGN00_bands_432.TIF'

    @mock.patch('worker.render_little.Downloader')
    def test_download_errors_correctly(self, Downloader):
        with pytest.raises(Exception):
            bands, input_path, scene_id = (render_little.download_and_set(
                self.bad_job_message))

    def test_resize_bands_creates_files(self):
        """If test files don't exist, make them exist

        The files are either downloaded from a fileserver, or unzipped
        from an archive file if it exists.
        """
        delete_me, rename_me = (
            render_little.resize_bands(self.test_bands, self.test_input_path,
                                       self.test_scene_id)
        )
        expected_delete_me = (
            [self.test_input_path + '/LC80470272015005LGN00_B4.TIF',
             self.test_input_path + '/LC80470272015005LGN00_B3.TIF',
             self.test_input_path + '/LC80470272015005LGN00_B2.TIF']
        )
        self.assertEqual(delete_me, expected_delete_me)

    def test_resize_bands_fails_with_message(self):
        with pytest.raises(Exception) as e:
            delete_me, rename_me = (
                render_little.resize_bands(self.bad_test_bands,
                                           '',
                                           self.test_scene_id)
            )
        print(e.value)
        assert 'gdal_translate did not downsize images' in str(e.value)

    @mock.patch('worker.render_little.os')
    def test_remove_and_rename(self, mock_os):
        render_little.remove_and_rename(['filelist1'], ['filelist2'])
        mock_os.remove.assert_called_with('filelist1')
        mock_os.rename.assert_called_with('filelist2', 'filelist1')

    @mock.patch('worker.render_little.Process')
    def test_merge_images(self, Process):
        render_little.merge_images(self.test_input_path, self.test_bands)
        render_little.Process.assert_called_with(
            self.test_input_path,
            dst_path=render_little.PATH_DOWNLOAD,
            verbose=False,
            bands=self.test_bands
        )

    @mock.patch('worker.render_little.Process')
    def test_merge_images_fails_with_exception(self, Process):
        render_little.Process.side_effect = Exception()
        with pytest.raises(Exception) as e:
            render_little.merge_images('', self.bad_test_bands)
        assert 'Processing/landsat-util failed' in str(e.value)

    def test_name_files(self):
        file_location, file_name, file_tif = (
            render_little.name_files(self.test_bands,
                                     self.test_input_path,
                                     self.test_scene_id))
        assert file_location == (
            self.test_input_path + '/LC80470272015005LGN00_bands_432.png')
        assert file_name == 'LC80470272015005LGN00_bands_432'
        assert file_tif == (
            self.test_input_path + '/LC80470272015005LGN00_bands_432.TIF')

    @mock.patch('worker.render_little.subprocess')
    def test_tif_to_png(self, mock_subp):
        file_png = render_little.tif_to_png(self.test_file_location,
                                            self.test_file_name,
                                            self.test_file_tif)
        assert file_png == self.test_file_png
        mock_subp.call.assert_called_with(['convert',
                                           self.test_file_tif,
                                           self.test_file_location])

    @mock.patch('worker.render_little.Key')
    @mock.patch('worker.render_little.boto')
    def test_upload_to_s3(self, boto, Key):
        self.assertIsNone(render_little.upload_to_s3(self.test_file_location,
                                                     self.test_file_png,
                                                     self.fake_job_message
                                                     ))

    @mock.patch('worker.render_little.Key')
    @mock.patch('worker.render_little.boto')
    def test_upload_to_s3_fails_with_exception(self, boto, Key):
        # missing job argument to cause exception
        render_little.boto.connect_s3.side_effect = Exception()
        with pytest.raises(Exception) as e:
            render_little.upload_to_s3(None,
                                       '',
                                       self.bad_job_message,
                                       )
        assert 'S3 Upload failed' in str(e.value)

    @mock.patch('worker.render_little.rmtree')
    def test_delete_files(self, mock_rmtree):
        render_little.delete_files(self.test_input_path)
        mock_rmtree.assert_called_with(self.test_input_path)
        # check error checking:
        render_little.rmtree.side_effect = Exception(OSError)
        with pytest.raises(Exception):
            render_little.delete_files('files')

@mock.patch('worker.render_little.Key')
@mock.patch('worker.render_little.boto')
def test_whole_process_run(Key, boto, setup_dirs):

    result = render_little.process(TestProcess.fake_job_message)
    # render_little.process returns True if it works:
    assert result
<commit_msg>refactor write functions into fixtures<commit_after>import pytest
import render_little
import models
from sqlalchemy import create_engine
from datetime import datetime
import mock
import unittest
import os
import factory
import factory.alchemy

@pytest.fixture()
def setup_dirs(monkeypatch):
    from zipfile import ZipFile
    from shutil import rmtree
    monkeypatch.setattr(render_little,
                        'PATH_DOWNLOAD',
                        str(TestProcess.test_tmp_download)
                        )

    if os.path.exists(TestProcess.test_tmp_download):
        rmtree(TestProcess.test_tmp_download)
    if not os.path.exists(TestProcess.test_input_path):
        os.makedirs(TestProcess.test_input_path)
        try:
            with ZipFile('test_tiffs_Archive.zip', 'r') as zip_file:
                zip_file.extractall(TestProcess.test_input_path)
        except IOError:
            print("Archive does not exist - downloading files")
            bands, input_path, scene_id = render_little.download_and_set(
                TestProcess.fake_job_message)

@pytest.fixture(scope='session', autouse=True)
def connection(request):
    engine = create_engine('postgresql://postgres@/test_bar')
    models.Base.metadata.create_all(engine)
    connection = engine.connect()
    models.DBSession.registry.clear()
    models.DBSession.configure(bind=connection)
    models.Base.metadata.bind = engine
    request.addfinalizer(models.Base.metadata.drop_all)
    return connection

@pytest.fixture(autouse=True)
def db_session(request, connection):
    from transaction import abort
    trans = connection.begin()
    request.addfinalizer(trans.rollback)
    request.addfinalizer(abort)

    from models import DBSession
    return DBSession

@pytest.mark.usefixtures("connection", "db_session")
class JobFactory(factory.alchemy.SQLAlchemyModelFactory):
    class Meta():
        model = models.UserJob_Model

        sqlalchemy_session = db_session

    jobstatus = 0
    starttime = datetime.utcnow()
    lastmodified = datetime.utcnow()
    band1 = u'4'
    band2 = u'3'
    band3 = u'2'
    entityid = u'LC80470272015005LGN00'
    email = u'test@test.com'
    jobid = factory.Sequence(lambda n: n)

@pytest.fixture(scope='class')
def fake_job1(db_session):
    model_instance = models.UserJob_Model(
        jobstatus=0,
        starttime=datetime.utcnow(),
        lastmodified=datetime.utcnow()
    )
    db_session.add(model_instance)
    db_session.flush()

@pytest.fixture()
def write_activity_fix(monkeypatch, tmpdir):
    tmp_activity_log = tmpdir.mkdir('log').join('tmp_act_log.txt')
    monkeypatch.setattr(render_little,
                        'PATH_ACTIVITY_LOG',
                        str(tmp_activity_log)
                        )
    return tmp_activity_log

@pytest.fixture()
def write_error_fix(monkeypatch, tmpdir):
    tmp_error_log = tmpdir.mkdir('log').join('tmp_error_log.txt')
    monkeypatch.setattr(render_little,
                        'PATH_ERROR_LOG',
                        str(tmp_error_log)
                        )
    return tmp_error_log

# --test db functionality tests
def test_db_lookup(db_session):
    model_instance = models.UserJob_Model(jobstatus=0,
                                          starttime=datetime.utcnow(),
                                          lastmodified=datetime.utcnow())
    db_session.add(model_instance)
    db_session.flush()

    assert 1 == db_session.query(models.UserJob_Model).count()

def test_db_is_rolled_back(db_session):
    assert 0 == db_session.query(models.UserJob_Model).count()

# --module function tests
def test_cleanup_downloads():
    test_dir = os.getcwd() + '/testdir'

    if not os.path.exists(test_dir):
        os.makedirs(test_dir)

    f = open(test_dir + '/test.txt', 'a')
    f.write('this is a test')
    f.close()
    # cleanup_downloads returns True if works
    assert render_little.cleanup_downloads(test_dir)

def test_write_activity(write_activity_fix):
    render_little.write_activity('test message')
    assert 'test message' in write_activity_fix.read()

def test_write_error(write_error_fix):
    render_little.write_error('test message')
    assert 'test message' in write_error_fix.read()

# --jobs queue
@pytest.mark.usefixtures("connection", "db_session", "fake_job1")
class TestQueue(unittest.TestCase):

    pass

# --process tests
@pytest.mark.usefixtures("setup_dirs")
class TestImageFiles(unittest.TestCase):
    """These tests require real files"""

    @mock.patch('worker.render_little.Downloader')
    def test_download_returns_correct_values(self, Downloader):
        bands, input_path, scene_id = (render_little.download_and_set(
            TestProcess.fake_job_message))
        self.assertEqual(input_path,
                         os.getcwd() + '/test_download/LC80470272015005LGN00')
        self.assertEqual(bands, [u'4', u'3', u'2'])
        self.assertEqual(scene_id, 'LC80470272015005LGN00')

@pytest.mark.usefixtures("connection", "db_session", "fake_job1")
class TestProcess(unittest.TestCase):

    fake_job_message = {u'job_id': u'1',
                        u'band_2': u'3',
                        u'band_3': u'2',
                        u'band_1': u'4',
                        u'scene_id': u'LC80470272015005LGN00',
                        u'email': u'test@test.com'}

    # bad_job_message is missing band_1
    bad_job_message = {u'job_id': u'1',
                       u'band_2': u'3',
                       u'band_3': u'2',
                       u'scene_id': u'LC80470272015005LGN00',
                       u'email': u'test@test.com'}

    test_tmp_download = os.getcwd() + '/test_download'
    test_input_path = os.getcwd() + '/test_download/LC80470272015005LGN00'
    test_bands = [u'4', u'3', u'2']
    bad_test_bands = [u'4', u'3']
    test_scene_id = 'LC80470272015005LGN00'
    test_file_location = (os.getcwd() +
        '/download/LC80470272015005LGN00/LC80470272015005LGN00_bands_432.TIF')
    test_file_name = 'LC80470272015005LGN00_bands_432'
    test_file_name_zip = 'LC80470272015005LGN00_bands_432.zip'
    test_file_png = 'pre_LC80470272015005LGN00_bands_432.png'
    test_file_tif = 'pre_LC80470272015005LGN00_bands_432.TIF'

    @mock.patch('worker.render_little.Downloader')
    def test_download_errors_correctly(self, Downloader):
        with pytest.raises(Exception):
            bands, input_path, scene_id = (render_little.download_and_set(
                self.bad_job_message))

    def test_resize_bands_creates_files(self):
        """If test files don't exist, make them exist

        The files are either downloaded from a fileserver, or unzipped
        from an archive file if it exists.
        """
        delete_me, rename_me = (
            render_little.resize_bands(self.test_bands, self.test_input_path,
                                       self.test_scene_id)
        )
        expected_delete_me = (
            [self.test_input_path + '/LC80470272015005LGN00_B4.TIF',
             self.test_input_path + '/LC80470272015005LGN00_B3.TIF',
             self.test_input_path + '/LC80470272015005LGN00_B2.TIF']
        )
        self.assertEqual(delete_me, expected_delete_me)

    def test_resize_bands_fails_with_message(self):
        with pytest.raises(Exception) as e:
            delete_me, rename_me = (
                render_little.resize_bands(self.bad_test_bands,
                                           '',
                                           self.test_scene_id)
            )
        print(e.value)
        assert 'gdal_translate did not downsize images' in str(e.value)

    @mock.patch('worker.render_little.os')
    def test_remove_and_rename(self, mock_os):
        render_little.remove_and_rename(['filelist1'], ['filelist2'])
        mock_os.remove.assert_called_with('filelist1')
        mock_os.rename.assert_called_with('filelist2', 'filelist1')

    @mock.patch('worker.render_little.Process')
    def test_merge_images(self, Process):
        render_little.merge_images(self.test_input_path, self.test_bands)
        render_little.Process.assert_called_with(
            self.test_input_path,
            dst_path=render_little.PATH_DOWNLOAD,
            verbose=False,
            bands=self.test_bands
        )

    @mock.patch('worker.render_little.Process')
    def test_merge_images_fails_with_exception(self, Process):
        render_little.Process.side_effect = Exception()
        with pytest.raises(Exception) as e:
            render_little.merge_images('', self.bad_test_bands)
        assert 'Processing/landsat-util failed' in str(e.value)

    def test_name_files(self):
        file_location, file_name, file_tif = (
            render_little.name_files(self.test_bands,
                                     self.test_input_path,
                                     self.test_scene_id))
        assert file_location == (
            self.test_input_path + '/LC80470272015005LGN00_bands_432.png')
        assert file_name == 'LC80470272015005LGN00_bands_432'
        assert file_tif == (
            self.test_input_path + '/LC80470272015005LGN00_bands_432.TIF')

    @mock.patch('worker.render_little.subprocess')
    def test_tif_to_png(self, mock_subp):
        file_png = render_little.tif_to_png(self.test_file_location,
                                            self.test_file_name,
                                            self.test_file_tif)
        assert file_png == self.test_file_png
        mock_subp.call.assert_called_with(['convert',
                                           self.test_file_tif,
                                           self.test_file_location])

    @mock.patch('worker.render_little.Key')
    @mock.patch('worker.render_little.boto')
    def test_upload_to_s3(self, boto, Key):
        self.assertIsNone(render_little.upload_to_s3(self.test_file_location,
                                                     self.test_file_png,
                                                     self.fake_job_message
                                                     ))

    @mock.patch('worker.render_little.Key')
    @mock.patch('worker.render_little.boto')
    def test_upload_to_s3_fails_with_exception(self, boto, Key):
        # missing job argument to cause exception
        render_little.boto.connect_s3.side_effect = Exception()
        with pytest.raises(Exception) as e:
            render_little.upload_to_s3(None,
                                       '',
                                       self.bad_job_message,
                                       )
        assert 'S3 Upload failed' in str(e.value)

    @mock.patch('worker.render_little.rmtree')
    def test_delete_files(self, mock_rmtree):
        render_little.delete_files(self.test_input_path)
        mock_rmtree.assert_called_with(self.test_input_path)
        # check error checking:
        render_little.rmtree.side_effect = Exception(OSError)
        with pytest.raises(Exception):
            render_little.delete_files('files')

@mock.patch('worker.render_little.Key')
@mock.patch('worker.render_little.boto')
def test_whole_process_run(Key, boto, setup_dirs):

    result = render_little.process(TestProcess.fake_job_message)
    # render_little.process returns True if it works:
    assert result
<|endoftext|>


<commit_before>import sys
sys.path.append('./landsat-util/landsat')
import pytest
import render_little
import models
from sqlalchemy import create_engine
from datetime import datetime
import mock
import unittest
import os
import factory
import factory.alchemy

@pytest.fixture(scope='session', autouse=True)
def connection(request):
    engine = create_engine('postgresql://postgres@/test_bar')
    models.Base.metadata.create_all(engine)
    connection = engine.connect()
    models.DBSession.registry.clear()
    models.DBSession.configure(bind=connection)
    models.Base.metadata.bind = engine
    request.addfinalizer(models.Base.metadata.drop_all)
    return connection

@pytest.fixture(autouse=True)
def db_session(request, connection):
    from transaction import abort
    trans = connection.begin()
    request.addfinalizer(trans.rollback)
    request.addfinalizer(abort)

    from models import DBSession
    return DBSession

@pytest.mark.usefixtures("connection", "db_session")
class JobFactory(factory.alchemy.SQLAlchemyModelFactory):
    class Meta():
        model = models.UserJob_Model

        sqlalchemy_session = db_session

    jobstatus = 0
    starttime = datetime.utcnow()
    lastmodified = datetime.utcnow()
    band1 = u'4'
    band2 = u'3'
    band3 = u'2'
    entityid = u'LC80470272015005LGN00'
    email = u'test@test.com'
    jobid = factory.Sequence(lambda n: n)

@pytest.fixture(scope='class')
def fake_job1(db_session):
    model_instance = models.UserJob_Model(
        jobstatus=0,
        starttime=datetime.utcnow(),
        lastmodified=datetime.utcnow()
    )
    db_session.add(model_instance)
    db_session.flush()

# --- test db functionality tests

def test_db_lookup(db_session):
    model_instance = models.UserJob_Model(jobstatus=0,
                                          starttime=datetime.utcnow(),
                                          lastmodified=datetime.utcnow())
    db_session.add(model_instance)
    db_session.flush()

    assert 1 == db_session.query(models.UserJob_Model).count()

def test_db_is_rolled_back(db_session):
    assert 0 == db_session.query(models.UserJob_Model).count()

# --- process tests

@pytest.mark.usefixtures("connection", "db_session", "fake_job1")
class TestProcess(unittest.TestCase):

    fake_job_message = {u'job_id': u'1',
                        u'band_2': u'3',
                        u'band_3': u'2',
                        u'band_1': u'4',
                        u'scene_id': u'LC80470272015005LGN00',
                        u'email': u'test@test.com'}

    # bad_job_message is missing band_1
    bad_job_message = {u'job_id': u'1',
                       u'band_2': u'3',
                       u'band_3': u'2',
                       u'scene_id': u'LC80470272015005LGN00',
                       u'email': u'test@test.com'}

    test_input_path = os.getcwd() + '/test_download/LC80470272015005LGN00'
    test_bands = [u'4', u'3', u'2']
    bad_test_bands = [u'4', u'3']
    test_scene_id = 'LC80470272015005LGN00'
    test_band_output = '432'
    test_file_location = (os.getcwd() +
        '/download/LC80470272015005LGN00/LC80470272015005LGN00_bands_432.TIF')
    test_file_name = 'LC80470272015005LGN00_bands_432'
    test_file_name_zip = 'LC80470272015005LGN00_bands_432.zip'
    test_file_png = 'pre_LC80470272015005LGN00_bands_432.png'
    test_file_tif = 'pre_LC80470272015005LGN00_bands_432.TIF'

    @mock.patch('worker.render_little.Downloader')
    def test_download_returns_correct_values(self, Downloader):
        bands, input_path, scene_id = (render_little.download_and_set(
            self.fake_job_message))
        self.assertEqual(input_path,
                         os.getcwd() + '/download/LC80470272015005LGN00')
        self.assertEqual(bands, [u'4', u'3', u'2'])
        self.assertEqual(scene_id, 'LC80470272015005LGN00')

    @mock.patch('worker.render_little.Downloader')
    def test_download_errors_correctly(self, Downloader):
        with pytest.raises(Exception):
            bands, input_path, scene_id = (render_little.download_and_set(
                self.bad_job_message))

    def test_resize_bands_creates_files(self):
        if not os.path.exists(self.test_input_path):
            os.makedirs(self.test_input_path)
        delete_me, rename_me = (
            render_little.resize_bands(self.test_bands, self.test_input_path,
                                       self.test_scene_id)
            )
        expected_delete_me = (
            [self.test_input_path + '/LC80470272015005LGN00_B4.TIF',
             self.test_input_path + '/LC80470272015005LGN00_B3.TIF',
             self.test_input_path + '/LC80470272015005LGN00_B2.TIF']
            )
        self.assertEqual(delete_me, expected_delete_me)

    def test_resize_bands_fails_with_message(self):
        with pytest.raises(Exception) as e:
            delete_me, rename_me = (
                render_little.resize_bands(self.bad_test_bands,
                                           '',
                                           self.test_scene_id)
            )
        print(e.value)
        assert 'gdal_translate did not downsize images' in str(e.value)

    @mock.patch('worker.render_little.Key')
    @mock.patch('worker.render_little.boto')
    def test_upload_to_s3(self, boto, Key):
        self.assertIsNone(render_little.upload_to_s3(self.test_file_location,
                                                     self.test_file_png,
                                                     self.fake_job_message
                                                     ))

    @mock.patch('worker.render_little.Key')
    @mock.patch('worker.render_little.boto')
    def test_upload_to_s3_fails_with_exception(self, boto, Key):
        # missing job argument to cause exception
        with self.assertRaises(Exception):
            render_little.upload_to_s3(self.test_file_location,
                                       self.test_file_name_zip,
                                       self.test_input_path,
                                       None
                                       )

    @mock.patch('worker.render_little.Process')
    def test_merge_images(self, Process):
        render_little.merge_images(self.test_input_path, self.test_bands)
        render_little.Process.assert_called_with(
            self.test_input_path,
            dst_path=render_little.PATH_DOWNLOAD,
            verbose=False,
            bands=self.test_bands
        )

    @mock.patch('worker.render_little.Process')
    def test_merge_images_fails_with_exception(self, Process):
        render_little.Process.side_effect = Exception()
        with pytest.raises(Exception) as e:
            render_little.merge_images('', self.bad_test_bands)
        assert 'Processing/landsat-util failed' in str(e.value)

    @mock.patch('worker.render_little.os')
    def test_remove_and_rename(self, mock_os):
        render_little.remove_and_rename(['filelist1'], ['filelist2'])
        mock_os.remove.assert_called_with('filelist1')
        mock_os.rename.assert_called_with('filelist2', 'filelist1')

    def test_name_files(self):
        file_location, file_name, file_tif = (
            render_little.name_files(self.test_bands,
                                     self.test_input_path,
                                     self.test_scene_id))
        assert file_location == (
            self.test_input_path + '/LC80470272015005LGN00_bands_432.png')
        assert file_name == 'LC80470272015005LGN00_bands_432'
        assert file_tif == (
            self.test_input_path + '/LC80470272015005LGN00_bands_432.TIF')

    @mock.patch('worker.render_little.subprocess')
    def test_tif_to_png(self, mock_subp):
        file_png = render_little.tif_to_png(self.test_file_location,
                                            self.test_file_name,
                                            self.test_file_tif)
        assert file_png == self.test_file_png
        mock_subp.call.assert_called_with(['convert',
                                           self.test_file_tif,
                                           self.test_file_location])

def test_cleanup_downloads():
    test_dir = os.getcwd() + '/testdir'

    if not os.path.exists(test_dir):
        os.makedirs(test_dir)

    f = open(test_dir + '/test.txt', 'a')
    f.write('this is a test')
    f.close()

    assert render_little.cleanup_downloads(test_dir) == True
<commit_msg>fix test_upload_to_s3_fails_with_exception<commit_after>import sys
sys.path.append('./landsat-util/landsat')
import pytest
import render_little
import models
from sqlalchemy import create_engine
from datetime import datetime
import mock
import unittest
import os
import factory
import factory.alchemy

@pytest.fixture(scope='session', autouse=True)
def connection(request):
    engine = create_engine('postgresql://postgres@/test_bar')
    models.Base.metadata.create_all(engine)
    connection = engine.connect()
    models.DBSession.registry.clear()
    models.DBSession.configure(bind=connection)
    models.Base.metadata.bind = engine
    request.addfinalizer(models.Base.metadata.drop_all)
    return connection

@pytest.fixture(autouse=True)
def db_session(request, connection):
    from transaction import abort
    trans = connection.begin()
    request.addfinalizer(trans.rollback)
    request.addfinalizer(abort)

    from models import DBSession
    return DBSession

@pytest.mark.usefixtures("connection", "db_session")
class JobFactory(factory.alchemy.SQLAlchemyModelFactory):
    class Meta():
        model = models.UserJob_Model

        sqlalchemy_session = db_session

    jobstatus = 0
    starttime = datetime.utcnow()
    lastmodified = datetime.utcnow()
    band1 = u'4'
    band2 = u'3'
    band3 = u'2'
    entityid = u'LC80470272015005LGN00'
    email = u'test@test.com'
    jobid = factory.Sequence(lambda n: n)

@pytest.fixture(scope='class')
def fake_job1(db_session):
    model_instance = models.UserJob_Model(
        jobstatus=0,
        starttime=datetime.utcnow(),
        lastmodified=datetime.utcnow()
    )
    db_session.add(model_instance)
    db_session.flush()

# --- test db functionality tests

def test_db_lookup(db_session):
    model_instance = models.UserJob_Model(jobstatus=0,
                                          starttime=datetime.utcnow(),
                                          lastmodified=datetime.utcnow())
    db_session.add(model_instance)
    db_session.flush()

    assert 1 == db_session.query(models.UserJob_Model).count()

def test_db_is_rolled_back(db_session):
    assert 0 == db_session.query(models.UserJob_Model).count()

# --- process tests

@pytest.mark.usefixtures("connection", "db_session", "fake_job1")
class TestProcess(unittest.TestCase):

    fake_job_message = {u'job_id': u'1',
                        u'band_2': u'3',
                        u'band_3': u'2',
                        u'band_1': u'4',
                        u'scene_id': u'LC80470272015005LGN00',
                        u'email': u'test@test.com'}

    # bad_job_message is missing band_1
    bad_job_message = {u'job_id': u'1',
                       u'band_2': u'3',
                       u'band_3': u'2',
                       u'scene_id': u'LC80470272015005LGN00',
                       u'email': u'test@test.com'}

    test_input_path = os.getcwd() + '/test_download/LC80470272015005LGN00'
    test_bands = [u'4', u'3', u'2']
    bad_test_bands = [u'4', u'3']
    test_scene_id = 'LC80470272015005LGN00'
    test_band_output = '432'
    test_file_location = (os.getcwd() +
        '/download/LC80470272015005LGN00/LC80470272015005LGN00_bands_432.TIF')
    test_file_name = 'LC80470272015005LGN00_bands_432'
    test_file_name_zip = 'LC80470272015005LGN00_bands_432.zip'
    test_file_png = 'pre_LC80470272015005LGN00_bands_432.png'
    test_file_tif = 'pre_LC80470272015005LGN00_bands_432.TIF'

    @mock.patch('worker.render_little.Downloader')
    def test_download_returns_correct_values(self, Downloader):
        bands, input_path, scene_id = (render_little.download_and_set(
            self.fake_job_message))
        self.assertEqual(input_path,
                         os.getcwd() + '/download/LC80470272015005LGN00')
        self.assertEqual(bands, [u'4', u'3', u'2'])
        self.assertEqual(scene_id, 'LC80470272015005LGN00')

    @mock.patch('worker.render_little.Downloader')
    def test_download_errors_correctly(self, Downloader):
        with pytest.raises(Exception):
            bands, input_path, scene_id = (render_little.download_and_set(
                self.bad_job_message))

    def test_resize_bands_creates_files(self):
        if not os.path.exists(self.test_input_path):
            os.makedirs(self.test_input_path)
        delete_me, rename_me = (
            render_little.resize_bands(self.test_bands, self.test_input_path,
                                       self.test_scene_id)
            )
        expected_delete_me = (
            [self.test_input_path + '/LC80470272015005LGN00_B4.TIF',
             self.test_input_path + '/LC80470272015005LGN00_B3.TIF',
             self.test_input_path + '/LC80470272015005LGN00_B2.TIF']
            )
        self.assertEqual(delete_me, expected_delete_me)

    def test_resize_bands_fails_with_message(self):
        with pytest.raises(Exception) as e:
            delete_me, rename_me = (
                render_little.resize_bands(self.bad_test_bands,
                                           '',
                                           self.test_scene_id)
            )
        print(e.value)
        assert 'gdal_translate did not downsize images' in str(e.value)

    @mock.patch('worker.render_little.Key')
    @mock.patch('worker.render_little.boto')
    def test_upload_to_s3(self, boto, Key):
        self.assertIsNone(render_little.upload_to_s3(self.test_file_location,
                                                     self.test_file_png,
                                                     self.fake_job_message
                                                     ))

    @mock.patch('worker.render_little.Key')
    @mock.patch('worker.render_little.boto')
    def test_upload_to_s3_fails_with_exception(self, boto, Key):
        # missing job argument to cause exception
        render_little.boto.connect_s3.side_effect = Exception()
        with pytest.raises(Exception) as e:
            render_little.upload_to_s3(None,
                                       '',
                                       self.bad_job_message,
                                       )
        assert 'S3 Upload failed' in str(e.value)

    @mock.patch('worker.render_little.Process')
    def test_merge_images(self, Process):
        render_little.merge_images(self.test_input_path, self.test_bands)
        render_little.Process.assert_called_with(
            self.test_input_path,
            dst_path=render_little.PATH_DOWNLOAD,
            verbose=False,
            bands=self.test_bands
        )

    @mock.patch('worker.render_little.Process')
    def test_merge_images_fails_with_exception(self, Process):
        render_little.Process.side_effect = Exception()
        with pytest.raises(Exception) as e:
            render_little.merge_images('', self.bad_test_bands)
        assert 'Processing/landsat-util failed' in str(e.value)

    @mock.patch('worker.render_little.os')
    def test_remove_and_rename(self, mock_os):
        render_little.remove_and_rename(['filelist1'], ['filelist2'])
        mock_os.remove.assert_called_with('filelist1')
        mock_os.rename.assert_called_with('filelist2', 'filelist1')

    def test_name_files(self):
        file_location, file_name, file_tif = (
            render_little.name_files(self.test_bands,
                                     self.test_input_path,
                                     self.test_scene_id))
        assert file_location == (
            self.test_input_path + '/LC80470272015005LGN00_bands_432.png')
        assert file_name == 'LC80470272015005LGN00_bands_432'
        assert file_tif == (
            self.test_input_path + '/LC80470272015005LGN00_bands_432.TIF')

    @mock.patch('worker.render_little.subprocess')
    def test_tif_to_png(self, mock_subp):
        file_png = render_little.tif_to_png(self.test_file_location,
                                            self.test_file_name,
                                            self.test_file_tif)
        assert file_png == self.test_file_png
        mock_subp.call.assert_called_with(['convert',
                                           self.test_file_tif,
                                           self.test_file_location])

def test_cleanup_downloads():
    test_dir = os.getcwd() + '/testdir'

    if not os.path.exists(test_dir):
        os.makedirs(test_dir)

    f = open(test_dir + '/test.txt', 'a')
    f.write('this is a test')
    f.close()

    assert render_little.cleanup_downloads(test_dir) == True
<|endoftext|>