<commit_before># -*- coding: utf-8 -*-

#   Copyright (c) 2010-2015, MIT Probabilistic Computing Project
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import StringIO
import contextlib
import pytest
import sqlite3
import tempfile

import bayeslite
import bayeslite.guess

from bayeslite import bql_quote_name

import test_core

@contextlib.contextmanager
def bayesdb_csv_stream(csv):
    with test_core.bayesdb() as bdb:
        yield (bdb, StringIO.StringIO(csv))

@contextlib.contextmanager
def bayesdb_csv_file(csv):
    with test_core.bayesdb() as bdb:
        with tempfile.NamedTemporaryFile(prefix='bayeslite') as f:
            with open(f.name, 'w') as out:
                out.write(csv)
            yield (bdb, f.name)

def test_csv_import_empty():
    with bayesdb_csv_stream('') as (bdb, f):
        with pytest.raises(IOError):
            bayeslite.bayesdb_read_csv(bdb, 'empty', f, header=True,
                create=True)

def test_csv_import_nocols():
    with bayesdb_csv_stream('\n') as (bdb, f):
        # CSV import rejects no columns.
        with pytest.raises(IOError):
            bayeslite.bayesdb_read_csv(bdb, 'nocols', f, header=True,
                create=True)

def test_csv_import_onecol_key():
    with bayesdb_csv_stream('foo\n0\none\n2\n') as (bdb, f):
        # foo will be a key column, hence no columns to model.
        bayeslite.bayesdb_read_csv(bdb, 'onecol_key', f, header=True,
            create=True)
        with pytest.raises(ValueError):
            bayeslite.guess.bayesdb_guess_generator(bdb, 'onecol_key_cc',
                'onecol_key', 'crosscat')

def test_csv_import_onecol():
    with bayesdb_csv_stream('foo\n0\none\n2\n0\n') as (bdb, f):
        bayeslite.bayesdb_read_csv(bdb, 'onecol', f, header=True, create=True)

def test_csv_import_toofewcols():
    with bayesdb_csv_stream('foo,bar\n0,1\n0\n') as (bdb, f):
        with pytest.raises(IOError):
            bayeslite.bayesdb_read_csv(bdb, 'bad', f, header=True, create=True)

def test_csv_import_toomanycols():
    with bayesdb_csv_stream('foo,bar\n0,1\n0,1,2\n') as (bdb, f):
        with pytest.raises(IOError):
            bayeslite.bayesdb_read_csv(bdb, 'bad', f, header=True, create=True)

def test_csv_import_dupcols():
    with bayesdb_csv_stream('foo,foo\n0,1\n') as (bdb, f):
        with pytest.raises(IOError):
            bayeslite.bayesdb_read_csv(bdb, 'bad', f, header=True, create=True)
    with bayesdb_csv_stream('foo,FOO\n0,1\n') as (bdb, f):
        with pytest.raises(IOError):
            bayeslite.bayesdb_read_csv(bdb, 'bad', f, header=True, create=True)

# Where did I get these data?  The gender balance needs work, as does
# representation of nonbinaries.
csv_data = '''age, gender, salary, height, division, rank
34, M, 74000, 65, sales, 3
41, M, 65600, 72, marketing, 4
25, M, 52000, 69, accounting, 5
23, F, 81000, 67, data science, 3
36, F, 96000, 70, management, 2
30, M, 70000, 73, sales, 4
30, F, 81000, 73, engineering, 3
'''

def test_csv_import():
    with bayesdb_csv_stream(csv_data) as (bdb, f):
        bayeslite.bayesdb_read_csv(bdb, 'employees', f, header=True,
            create=True)

def test_csv_import_schema():
    with bayesdb_csv_stream(csv_data) as (bdb, f):
        bdb.sql_execute('''
            CREATE TABLE employees(
                age INTEGER,
                gender TEXT,
                salary REAL,
                height INTEGER,
                division TEXT,
                rank INTEGER
            )
        ''')
        bayeslite.bayesdb_read_csv(bdb, 'employees', f, header=True,
            create=False)
        bdb.execute('select height from employees').fetchall()
        # XXX Currently this test fails because we compile the query
        # into `SELECT "idontexist" FROM "employees"', and for
        # compatibility with MySQL idiocy or something, SQLite treats
        # double-quotes as single-quotes if the alternative would be
        # an error.
        with pytest.raises(sqlite3.OperationalError):
            bdb.execute('select idontexist from employees')
            raise sqlite3.OperationalError('BQL compiler is broken;'
                ' a.k.a. sqlite3 is stupid.')
        bdb.execute('''
            CREATE GENERATOR employees_cc FOR employees USING crosscat(
                age NUMERICAL,
                gender CATEGORICAL,
                salary CYCLIC,
                division CATEGORICAL,
                rank CATEGORICAL
            )
        ''')
        bdb.execute('estimate height from employees_cc').fetchall()
        with pytest.raises(bayeslite.BQLError):
            bdb.execute('estimate predict height with confidence 0.9'
                ' from employees_cc')

def test_csv_import_schema_case():
    with bayesdb_csv_stream(csv_data) as (bdb, f):
        bdb.sql_execute('''
            CREATE TABLE emPloyEES(
                AGE INTeger,
                geNder Text,
                saLAry REal,
                heighT inteGER,
                DIVision TEXt,
                rank INTEGER
            )
        ''')
        bayeslite.bayesdb_read_csv(bdb, 'employees', f, header=True,
            create=False)
        bayeslite.guess.bayesdb_guess_generator(bdb, 'employees_cc',
            'employees', 'crosscat')

def test_csv_import_badschema0():
    with bayesdb_csv_stream(csv_data) as (bdb, f):
        bdb.sql_execute('''
            CREATE TABLE emPloyEES(
                AGE INTeger,
                geNder Text,
                -- saLAry REal,
                heighT inteGER,
                DIVision TEXt,
                rank INTEGER
            )
        ''')
        with pytest.raises(IOError):
            bayeslite.bayesdb_read_csv(bdb, 'employees', f, header=True,
                create=False)

def test_csv_import_badschema1():
    with bayesdb_csv_stream(csv_data) as (bdb, f):
        bdb.sql_execute('''
            CREATE TABLE employees(
                age INTEGER,
                zorblaxianism TEXT,
                salary INTEGER,
                height INTEGER NOT NULL PRIMARY KEY,
                division TEXT,
                rank CATEGORICAL
            )
        ''')
        with pytest.raises(IOError):
            bayeslite.bayesdb_read_csv(bdb, 'employees', f, header=True,
                create=False)

csv_data_missing = '''a,b,c
1,2,3
10,,30
100,200,nan
4,5,6
'''

def test_csv_missing():
    with bayesdb_csv_stream(csv_data_missing) as (bdb, f):
        # XXX Test the automatic column type guessing too.
        bdb.sql_execute('CREATE TABLE t(a REAL, b REAL, c REAL)')
        bayeslite.bayesdb_read_csv(bdb, 't', f, header=True, create=False)
        def clean(column_name):
            qcn = bql_quote_name(column_name)
            sql = "UPDATE t SET %s = NULL WHERE %s = '' OR %s LIKE 'NaN'" % \
                (qcn, qcn, qcn)
            bdb.sql_execute(sql)
        clean('a')
        clean('b')
        clean('c')
        assert bdb.execute('select * from t').fetchall() == [
            (1.0, 2.0, 3.0),
            (10.0, None, 30.0),
            (100.0, 200.0, None),
            (4.0, 5.0, 6.0),
        ]
<commit_msg>Convert tests/test_csv.py from sqlite3 to apsw.<commit_after># -*- coding: utf-8 -*-

#   Copyright (c) 2010-2015, MIT Probabilistic Computing Project
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import StringIO
import apsw
import contextlib
import pytest
import tempfile

import bayeslite
import bayeslite.guess

from bayeslite import bql_quote_name

import test_core

@contextlib.contextmanager
def bayesdb_csv_stream(csv):
    with test_core.bayesdb() as bdb:
        yield (bdb, StringIO.StringIO(csv))

@contextlib.contextmanager
def bayesdb_csv_file(csv):
    with test_core.bayesdb() as bdb:
        with tempfile.NamedTemporaryFile(prefix='bayeslite') as f:
            with open(f.name, 'w') as out:
                out.write(csv)
            yield (bdb, f.name)

def test_csv_import_empty():
    with bayesdb_csv_stream('') as (bdb, f):
        with pytest.raises(IOError):
            bayeslite.bayesdb_read_csv(bdb, 'empty', f, header=True,
                create=True)

def test_csv_import_nocols():
    with bayesdb_csv_stream('\n') as (bdb, f):
        # CSV import rejects no columns.
        with pytest.raises(IOError):
            bayeslite.bayesdb_read_csv(bdb, 'nocols', f, header=True,
                create=True)

def test_csv_import_onecol_key():
    with bayesdb_csv_stream('foo\n0\none\n2\n') as (bdb, f):
        # foo will be a key column, hence no columns to model.
        bayeslite.bayesdb_read_csv(bdb, 'onecol_key', f, header=True,
            create=True)
        with pytest.raises(ValueError):
            bayeslite.guess.bayesdb_guess_generator(bdb, 'onecol_key_cc',
                'onecol_key', 'crosscat')

def test_csv_import_onecol():
    with bayesdb_csv_stream('foo\n0\none\n2\n0\n') as (bdb, f):
        bayeslite.bayesdb_read_csv(bdb, 'onecol', f, header=True, create=True)

def test_csv_import_toofewcols():
    with bayesdb_csv_stream('foo,bar\n0,1\n0\n') as (bdb, f):
        with pytest.raises(IOError):
            bayeslite.bayesdb_read_csv(bdb, 'bad', f, header=True, create=True)

def test_csv_import_toomanycols():
    with bayesdb_csv_stream('foo,bar\n0,1\n0,1,2\n') as (bdb, f):
        with pytest.raises(IOError):
            bayeslite.bayesdb_read_csv(bdb, 'bad', f, header=True, create=True)

def test_csv_import_dupcols():
    with bayesdb_csv_stream('foo,foo\n0,1\n') as (bdb, f):
        with pytest.raises(IOError):
            bayeslite.bayesdb_read_csv(bdb, 'bad', f, header=True, create=True)
    with bayesdb_csv_stream('foo,FOO\n0,1\n') as (bdb, f):
        with pytest.raises(IOError):
            bayeslite.bayesdb_read_csv(bdb, 'bad', f, header=True, create=True)

# Where did I get these data?  The gender balance needs work, as does
# representation of nonbinaries.
csv_data = '''age, gender, salary, height, division, rank
34, M, 74000, 65, sales, 3
41, M, 65600, 72, marketing, 4
25, M, 52000, 69, accounting, 5
23, F, 81000, 67, data science, 3
36, F, 96000, 70, management, 2
30, M, 70000, 73, sales, 4
30, F, 81000, 73, engineering, 3
'''

def test_csv_import():
    with bayesdb_csv_stream(csv_data) as (bdb, f):
        bayeslite.bayesdb_read_csv(bdb, 'employees', f, header=True,
            create=True)

def test_csv_import_schema():
    with bayesdb_csv_stream(csv_data) as (bdb, f):
        bdb.sql_execute('''
            CREATE TABLE employees(
                age INTEGER,
                gender TEXT,
                salary REAL,
                height INTEGER,
                division TEXT,
                rank INTEGER
            )
        ''')
        bayeslite.bayesdb_read_csv(bdb, 'employees', f, header=True,
            create=False)
        bdb.execute('select height from employees').fetchall()
        # XXX Currently this test fails because we compile the query
        # into `SELECT "idontexist" FROM "employees"', and for
        # compatibility with MySQL idiocy or something, SQLite treats
        # double-quotes as single-quotes if the alternative would be
        # an error.
        with pytest.raises(apsw.SQLError):
            bdb.execute('select idontexist from employees')
            raise apsw.SQLError('BQL compiler is broken;'
                ' a.k.a. sqlite3 is stupid.')
        bdb.execute('''
            CREATE GENERATOR employees_cc FOR employees USING crosscat(
                age NUMERICAL,
                gender CATEGORICAL,
                salary CYCLIC,
                division CATEGORICAL,
                rank CATEGORICAL
            )
        ''')
        bdb.execute('estimate height from employees_cc').fetchall()
        with pytest.raises(bayeslite.BQLError):
            bdb.execute('estimate predict height with confidence 0.9'
                ' from employees_cc')

def test_csv_import_schema_case():
    with bayesdb_csv_stream(csv_data) as (bdb, f):
        bdb.sql_execute('''
            CREATE TABLE emPloyEES(
                AGE INTeger,
                geNder Text,
                saLAry REal,
                heighT inteGER,
                DIVision TEXt,
                rank INTEGER
            )
        ''')
        bayeslite.bayesdb_read_csv(bdb, 'employees', f, header=True,
            create=False)
        bayeslite.guess.bayesdb_guess_generator(bdb, 'employees_cc',
            'employees', 'crosscat')

def test_csv_import_badschema0():
    with bayesdb_csv_stream(csv_data) as (bdb, f):
        bdb.sql_execute('''
            CREATE TABLE emPloyEES(
                AGE INTeger,
                geNder Text,
                -- saLAry REal,
                heighT inteGER,
                DIVision TEXt,
                rank INTEGER
            )
        ''')
        with pytest.raises(IOError):
            bayeslite.bayesdb_read_csv(bdb, 'employees', f, header=True,
                create=False)

def test_csv_import_badschema1():
    with bayesdb_csv_stream(csv_data) as (bdb, f):
        bdb.sql_execute('''
            CREATE TABLE employees(
                age INTEGER,
                zorblaxianism TEXT,
                salary INTEGER,
                height INTEGER NOT NULL PRIMARY KEY,
                division TEXT,
                rank CATEGORICAL
            )
        ''')
        with pytest.raises(IOError):
            bayeslite.bayesdb_read_csv(bdb, 'employees', f, header=True,
                create=False)

csv_data_missing = '''a,b,c
1,2,3
10,,30
100,200,nan
4,5,6
'''

def test_csv_missing():
    with bayesdb_csv_stream(csv_data_missing) as (bdb, f):
        # XXX Test the automatic column type guessing too.
        bdb.sql_execute('CREATE TABLE t(a REAL, b REAL, c REAL)')
        bayeslite.bayesdb_read_csv(bdb, 't', f, header=True, create=False)
        def clean(column_name):
            qcn = bql_quote_name(column_name)
            sql = "UPDATE t SET %s = NULL WHERE %s = '' OR %s LIKE 'NaN'" % \
                (qcn, qcn, qcn)
            bdb.sql_execute(sql)
        clean('a')
        clean('b')
        clean('c')
        assert bdb.execute('select * from t').fetchall() == [
            (1.0, 2.0, 3.0),
            (10.0, None, 30.0),
            (100.0, 200.0, None),
            (4.0, 5.0, 6.0),
        ]
<|endoftext|>


<commit_before>'''
'''

from pyOCD.board.board import Board
from pyOCD.interface import INTERFACE, usb_backend
from pyOCD.target.cortex_m import CORE_TYPE_NAME
from pyOCD.target.target import TARGET_RUNNING, TARGET_HALTED

HID_VID = 0xA600
HID_PID = 0xE2C0

FREQ = 100000

ARMv6M = 0xC
ARMv7M = 0xF

ARCH_NAME = {
    ARMv6M : "ARMv6M",
    ARMv7M : "ARMv7M",
}

MAP_BOOL_Y_N = {
    True  : "Yes",
    False : "No",
}

REG_NAMES = [
  'r0', 'r1',  'r2',  'r3',  'r4', 'r5', 'r6', 'r7',
  'r8', 'r9', 'r10', 'r11', 'r12', 'sp', 'lr', 'pc', 'xpsr',
]

TARGET_STATE = {
    TARGET_RUNNING : "Running",
    TARGET_HALTED  : "Halted",
}

PART_NAME = {
    0x00050080 : "LPC1115",
    0x00017C40 : "LPC11U37",
    0x1A020525 : "LPC1317",
    0x08020543 : "LPC1347",
    0x481D3F47 : "LPC4088",
}

GETIDS_CODE = [
    0x4685480B, 0x1F2C4D0B, 0x60202036, 0xF80BF000,
    0x60202037, 0xF0003508, 0x203AF806, 0x35086020,
    0xF801F000, 0x4620BE00, 0x4A034629, 0x00004710,
    0x10000400, 0x10000100, 0x1FFF1FF1,
]

SIMPLE_CODE = [
 0x21002005, 0x38011809, 0xBE00D1FC,
]

LOAD_ADDR   = 0x10000000
START_ADDR  = 0x10000001
RESULT_ADDR = 0x10000100
RESULT_SIZE = 9

def get_lipicano():
    lipicans = INTERFACE[usb_backend].getAllConnectedInterface(HID_VID, HID_PID)
    print("")
    for lip in lipicans:
        print("Lipicano found: %s, %s\n" % (lip.vendor_name, lip.product_name))
    return lipicans[0]

def print_target_info(target):
    print("Target info:")
    print("IDCODE  : 0x%08X" % (target.idcode))
    print("Arch    : %s" % (ARCH_NAME[target.arch]))
    print("Core    : %s" % (CORE_TYPE_NAME[target.core_type]))
    print("FPU     : %s" % (MAP_BOOL_Y_N[target.has_fpu]))
    print("Hw-BPt  : %s" % (len(target.breakpoints)))
    print("Hw-WPt  : %s" % (len(target.watchpoints)))
    print("State   : %s" % (TARGET_STATE[target.getState()]))

def print_target_ids(target):
    reg_list = []
    data_list = []
    target.halt()
    target.writeBlockMemoryAligned32(LOAD_ADDR, GETIDS_CODE)
    reg_list.append('pc')
    data_list.append(START_ADDR)
    target.writeCoreRegistersRaw(reg_list, data_list)
    target.resume()
    while(target.getState() == TARGET_RUNNING):
        pass
    result = target.readBlockMemoryAligned32(RESULT_ADDR, RESULT_SIZE)
    print("PartID  : 0x%08X" % (result[1]))
    print("BootVer : %d.%d" % ((result[3] >> 8) & 0xFF, result[3] & 0xFF))
    print("PartSN  : 0x%08X-%08X-%08X-%08X" % (result[8], result[7], result[6], result[5]))
    name = "Unknown"
    if (result[1] in PART_NAME):
        name = PART_NAME[result[1]];
    print("Name    : %s" % (name))

def print_regs(target):
    reg_vals = target.readCoreRegistersRaw(REG_NAMES)
    print("Regs:")
    line = ""
    cnt = 1
    for i, reg in enumerate(REG_NAMES):
        line += "%4s = 0x%08X  " % (reg, reg_vals[i])
        cnt += 1
        if (cnt == 5):
            print(line)
            line = ""
            cnt = 1
    if (len(line) > 0):
        print(line)
    return reg_vals

def step_simple_code(target):
    reg_list = []
    data_list = []
    target.halt()
    target.writeBlockMemoryAligned32(LOAD_ADDR, SIMPLE_CODE)
    reg_list.append('pc')
    data_list.append(START_ADDR)
    target.writeCoreRegistersRaw(reg_list, data_list)
    print("\nStep simple_code:")
    stepit = True
    while stepit :
        print("")
        reg_vals = print_regs(target)
        instr_code = target.read16(reg_vals[15])
        print("Next instr: %04X" % (instr_code))
        if (instr_code == 0xBE00):
            print("Breakpoint reached")
            stepit = False
        else:
            c = raw_input("Step? [y/n]")
            if c.upper() == 'N':
                print("User break")
                stepit = False
            else:
                target.step()

if __name__ == '__main__':
    lipicano = get_lipicano()
    board = Board(target = "cortex_m", flash = "cortex_m", interface = lipicano, transport = "cmsis_dap", frequency = FREQ)
    board.init()
    target = board.target
    print_target_info(target)
    print_target_ids(target)
    # print_regs(target)
    step_simple_code(target)

<commit_msg>Comment out step_simple_code()<commit_after>'''
'''

from pyOCD.board.board import Board
from pyOCD.interface import INTERFACE, usb_backend
from pyOCD.target.cortex_m import CORE_TYPE_NAME
from pyOCD.target.target import TARGET_RUNNING, TARGET_HALTED

HID_VID = 0xA600
HID_PID = 0xE2C0

FREQ = 100000

ARMv6M = 0xC
ARMv7M = 0xF

ARCH_NAME = {
    ARMv6M : "ARMv6M",
    ARMv7M : "ARMv7M",
}

MAP_BOOL_Y_N = {
    True  : "Yes",
    False : "No",
}

REG_NAMES = [
  'r0', 'r1',  'r2',  'r3',  'r4', 'r5', 'r6', 'r7',
  'r8', 'r9', 'r10', 'r11', 'r12', 'sp', 'lr', 'pc', 'xpsr',
]

TARGET_STATE = {
    TARGET_RUNNING : "Running",
    TARGET_HALTED  : "Halted",
}

PART_NAME = {
    0x00050080 : "LPC1115",
    0x00017C40 : "LPC11U37",
    0x1A020525 : "LPC1317",
    0x08020543 : "LPC1347",
    0x481D3F47 : "LPC4088",
}

GETIDS_CODE = [
    0x4685480B, 0x1F2C4D0B, 0x60202036, 0xF80BF000,
    0x60202037, 0xF0003508, 0x203AF806, 0x35086020,
    0xF801F000, 0x4620BE00, 0x4A034629, 0x00004710,
    0x10000400, 0x10000100, 0x1FFF1FF1,
]

SIMPLE_CODE = [
 0x21002005, 0x38011809, 0xBE00D1FC,
]

LOAD_ADDR   = 0x10000000
START_ADDR  = 0x10000001
RESULT_ADDR = 0x10000100
RESULT_SIZE = 9

def get_lipicano():
    lipicans = INTERFACE[usb_backend].getAllConnectedInterface(HID_VID, HID_PID)
    print("")
    for lip in lipicans:
        print("Lipicano found: %s, %s\n" % (lip.vendor_name, lip.product_name))
    return lipicans[0]

def print_target_info(target):
    print("Target info:")
    print("IDCODE  : 0x%08X" % (target.idcode))
    print("Arch    : %s" % (ARCH_NAME[target.arch]))
    print("Core    : %s" % (CORE_TYPE_NAME[target.core_type]))
    print("FPU     : %s" % (MAP_BOOL_Y_N[target.has_fpu]))
    print("Hw-BPt  : %s" % (len(target.breakpoints)))
    print("Hw-WPt  : %s" % (len(target.watchpoints)))
    print("State   : %s" % (TARGET_STATE[target.getState()]))

def print_target_ids(target):
    reg_list = []
    data_list = []
    target.halt()
    target.writeBlockMemoryAligned32(LOAD_ADDR, GETIDS_CODE)
    reg_list.append('pc')
    data_list.append(START_ADDR)
    target.writeCoreRegistersRaw(reg_list, data_list)
    target.resume()
    while(target.getState() == TARGET_RUNNING):
        pass
    result = target.readBlockMemoryAligned32(RESULT_ADDR, RESULT_SIZE)
    print("PartID  : 0x%08X" % (result[1]))
    print("BootVer : %d.%d" % ((result[3] >> 8) & 0xFF, result[3] & 0xFF))
    print("PartSN  : 0x%08X-%08X-%08X-%08X" % (result[8], result[7], result[6], result[5]))
    name = "Unknown"
    if (result[1] in PART_NAME):
        name = PART_NAME[result[1]];
    print("Name    : %s" % (name))

def print_regs(target):
    reg_vals = target.readCoreRegistersRaw(REG_NAMES)
    print("Regs:")
    line = ""
    cnt = 1
    for i, reg in enumerate(REG_NAMES):
        line += "%4s = 0x%08X  " % (reg, reg_vals[i])
        cnt += 1
        if (cnt == 5):
            print(line)
            line = ""
            cnt = 1
    if (len(line) > 0):
        print(line)
    return reg_vals

def step_simple_code(target):
    reg_list = []
    data_list = []
    target.halt()
    target.writeBlockMemoryAligned32(LOAD_ADDR, SIMPLE_CODE)
    reg_list.append('pc')
    data_list.append(START_ADDR)
    target.writeCoreRegistersRaw(reg_list, data_list)
    print("\nStep simple_code:")
    stepit = True
    while stepit :
        print("")
        reg_vals = print_regs(target)
        instr_code = target.read16(reg_vals[15])
        print("Next instr: %04X" % (instr_code))
        if (instr_code == 0xBE00):
            print("Breakpoint reached")
            stepit = False
        else:
            c = raw_input("Step? [y/n]")
            if c.upper() == 'N':
                print("User break")
                stepit = False
            else:
                target.step()

if __name__ == '__main__':
    lipicano = get_lipicano()
    board = Board(target = "cortex_m", flash = "cortex_m", interface = lipicano, transport = "cmsis_dap", frequency = FREQ)
    board.init()
    target = board.target
    print_target_info(target)
    print_target_ids(target)
    # print_regs(target)
    # step_simple_code(target)

<|endoftext|>


<commit_before>import unittest
import pytest
from lists import (
    get_rounds,
    concatenate_rounds,
    list_contains_round,
    card_average,
    approx_average_is_average,
    average_even_is_average_odd,
    maybe_double_last,
)

class CardGamesTest(unittest.TestCase):

    @pytest.mark.task(taskno=1)
    def test_get_rounds(self):

        input_vars = [0, 1, 10, 27, 99, 666]

        results = [[0, 1, 2], [1, 2, 3],
                   [10, 11, 12], [27, 28, 29],
                   [99, 100, 101], [666, 667, 668]]

        for variant, (number, rounds) in enumerate(zip(input_vars, results), start=1):
            error_message = f'Expected rounds {rounds} given the current round {number}.'
            with self.subTest(f'variation #{variant}', input=number, output=rounds):
                self.assertEqual(rounds, get_rounds(number), msg=error_message)

    @pytest.mark.task(taskno=2)
    def test_concatenate_rounds(self):

        input_vars = [([], []), ([0, 1], []), ([], [1, 2]),
                      ([1], [2]), ([27, 28, 29], [35, 36]),
                      ([1, 2, 3], [4, 5, 6])]

        results = [[], [0, 1], [1, 2], [1, 2],
                   [27, 28, 29, 35, 36],
                   [1, 2, 3, 4, 5, 6]]

        for variant, ((rounds_1, rounds_2), rounds) in enumerate(zip(input_vars, results), start=1):
            error_message = f'Expected {rounds} as the concatenation of {rounds_1} and {rounds_2}.'
            with self.subTest(f'variation #{variant}', input=(rounds_1, rounds_2), output=rounds):
                self.assertEqual(rounds, concatenate_rounds(rounds_1, rounds_2), msg=error_message)

    @pytest.mark.task(taskno=3)
    def test_list_contains_round(self):

        input_vars = [([], 1), ([1, 2, 3], 0), ([27, 28, 29, 35, 36], 30),
                      ([1], 1), ([1, 2, 3], 1), ([27, 28, 29, 35, 36], 29)]

        results = [False, False, False, True, True, True]

        for variant, ((rounds, round_number), contains) in enumerate(zip(input_vars, results), start=1):
            error_message = f'Round {round_number} {"is" if contains else "is not"} in {rounds}.'
            with self.subTest(f'variation #{variant}', input=(rounds, round_number), output=contains):
                self.assertEqual(contains, list_contains_round(rounds, round_number), msg=error_message)

    @pytest.mark.task(taskno=4)
    def test_card_average(self):

        input_vars = [[1], [5, 6, 7], [1, 2, 3, 4], [1, 10, 100]]

        results = [1.0, 6.0, 2.5, 37.0]

        for variant, (hand, average) in enumerate(zip(input_vars, results), start=1):
            error_message = f'Expected {average} as the average of {hand}.'
            with self.subTest(f'variation #{variant}', input=hand, output=average):
                self.assertEqual(average, card_average(hand), msg=error_message)

    @pytest.mark.task(taskno=5)
    def test_approx_average_is_average(self):

        input_vars = [[0, 1, 5], [3, 6, 9, 12, 150], [1, 2, 3, 5, 9],
                      [2, 3, 4, 7, 8], [1, 2, 3], [2, 3, 4],
                      [2, 3, 4, 8, 8], [1, 2, 4, 5, 8]]

        results = [False, False, False, False, True, True, True, True]

        for variant, (hand, same) in enumerate(zip(input_vars, results), start=1):
            error_message = f'Hand {hand} {"does" if same else "does not"} yield the same approximate average.'
            with self.subTest(f'variation #{variant}', input=hand, output=same):
                self.assertEqual(same, approx_average_is_average(hand), msg=error_message)

    @pytest.mark.task(taskno=6)
    def test_average_even_is_average_odd(self):

        input_vars = [[5, 6, 8], [1, 2, 3, 4], [1, 2, 3], [5, 6, 7], ]

        results = [False, False, True, True]

        for variant, (hand, same) in enumerate(zip(input_vars, results), start=1):
            error_message = f'Hand {hand} {"does" if same else "does not"} yield the same odd-even average.'
            with self.subTest(f'variation #{variant}', input=hand, output=same):
                self.assertEqual(same, average_even_is_average_odd(hand), msg=error_message)

    @pytest.mark.task(taskno=7)
    def test_maybe_double_last(self):

        input_vars = [[1, 2, 11], [5, 9, 11], [5, 9, 10], [1, 2, 3]]

        results = [[1, 2, 22], [5, 9, 22], [5, 9, 10], [1, 2, 3]]

        for variant, (hand, doubled_hand) in enumerate(zip(input_vars, results), start=1):
            error_message = f'Expected {doubled_hand} as the maybe-doubled version of {hand}.'
            with self.subTest(f'variation #{variant}', input=hand, output=doubled_hand):
                self.assertEqual(doubled_hand, maybe_double_last(hand), msg=error_message)
<commit_msg>[Card Games] Additional test for Task 6.<commit_after>import unittest
import pytest
from lists import (
    get_rounds,
    concatenate_rounds,
    list_contains_round,
    card_average,
    approx_average_is_average,
    average_even_is_average_odd,
    maybe_double_last,
)

class CardGamesTest(unittest.TestCase):

    @pytest.mark.task(taskno=1)
    def test_get_rounds(self):

        input_vars = [0, 1, 10, 27, 99, 666]

        results = [[0, 1, 2], [1, 2, 3],
                   [10, 11, 12], [27, 28, 29],
                   [99, 100, 101], [666, 667, 668]]

        for variant, (number, rounds) in enumerate(zip(input_vars, results), start=1):
            error_message = f'Expected rounds {rounds} given the current round {number}.'
            with self.subTest(f'variation #{variant}', input=number, output=rounds):
                self.assertEqual(rounds, get_rounds(number), msg=error_message)

    @pytest.mark.task(taskno=2)
    def test_concatenate_rounds(self):

        input_vars = [([], []), ([0, 1], []), ([], [1, 2]),
                      ([1], [2]), ([27, 28, 29], [35, 36]),
                      ([1, 2, 3], [4, 5, 6])]

        results = [[], [0, 1], [1, 2], [1, 2],
                   [27, 28, 29, 35, 36],
                   [1, 2, 3, 4, 5, 6]]

        for variant, ((rounds_1, rounds_2), rounds) in enumerate(zip(input_vars, results), start=1):
            error_message = f'Expected {rounds} as the concatenation of {rounds_1} and {rounds_2}.'
            with self.subTest(f'variation #{variant}', input=(rounds_1, rounds_2), output=rounds):
                self.assertEqual(rounds, concatenate_rounds(rounds_1, rounds_2), msg=error_message)

    @pytest.mark.task(taskno=3)
    def test_list_contains_round(self):

        input_vars = [([], 1), ([1, 2, 3], 0), ([27, 28, 29, 35, 36], 30),
                      ([1], 1), ([1, 2, 3], 1), ([27, 28, 29, 35, 36], 29)]

        results = [False, False, False, True, True, True]

        for variant, ((rounds, round_number), contains) in enumerate(zip(input_vars, results), start=1):
            error_message = f'Round {round_number} {"is" if contains else "is not"} in {rounds}.'
            with self.subTest(f'variation #{variant}', input=(rounds, round_number), output=contains):
                self.assertEqual(contains, list_contains_round(rounds, round_number), msg=error_message)

    @pytest.mark.task(taskno=4)
    def test_card_average(self):

        input_vars = [[1], [5, 6, 7], [1, 2, 3, 4], [1, 10, 100]]

        results = [1.0, 6.0, 2.5, 37.0]

        for variant, (hand, average) in enumerate(zip(input_vars, results), start=1):
            error_message = f'Expected {average} as the average of {hand}.'
            with self.subTest(f'variation #{variant}', input=hand, output=average):
                self.assertEqual(average, card_average(hand), msg=error_message)

    @pytest.mark.task(taskno=5)
    def test_approx_average_is_average(self):

        input_vars = [[0, 1, 5], [3, 6, 9, 12, 150], [1, 2, 3, 5, 9],
                      [2, 3, 4, 7, 8], [1, 2, 3], [2, 3, 4],
                      [2, 3, 4, 8, 8], [1, 2, 4, 5, 8]]

        results = [False, False, False, False, True, True, True, True]

        for variant, (hand, same) in enumerate(zip(input_vars, results), start=1):
            error_message = f'Hand {hand} {"does" if same else "does not"} yield the same approximate average.'
            with self.subTest(f'variation #{variant}', input=hand, output=same):
                self.assertEqual(same, approx_average_is_average(hand), msg=error_message)

    @pytest.mark.task(taskno=6)
    def test_average_even_is_average_odd(self):

        input_vars = [[5, 6, 8], [1, 2, 3, 4], [1, 2, 3], [5, 6, 7], [1, 3, 5, 7, 9]]

        results = [False, False, True, True, True]

        for variant, (hand, same) in enumerate(zip(input_vars, results), start=1):
            error_message = f'Hand {hand} {"does" if same else "does not"} yield the same odd-even average.'
            with self.subTest(f'variation #{variant}', input=hand, output=same):
                self.assertEqual(same, average_even_is_average_odd(hand), msg=error_message)

    @pytest.mark.task(taskno=7)
    def test_maybe_double_last(self):

        input_vars = [[1, 2, 11], [5, 9, 11], [5, 9, 10], [1, 2, 3]]

        results = [[1, 2, 22], [5, 9, 22], [5, 9, 10], [1, 2, 3]]

        for variant, (hand, doubled_hand) in enumerate(zip(input_vars, results), start=1):
            error_message = f'Expected {doubled_hand} as the maybe-doubled version of {hand}.'
            with self.subTest(f'variation #{variant}', input=hand, output=doubled_hand):
                self.assertEqual(doubled_hand, maybe_double_last(hand), msg=error_message)
<|endoftext|>


<commit_before>from __future__ import absolute_import, unicode_literals

import unittest

from mopidy import config, ext

class ExtensionTest(unittest.TestCase):

    def setUp(self):  # noqa: N802
        self.ext = ext.Extension()

    def test_dist_name_is_none(self):
        self.assertIsNone(self.ext.dist_name)

    def test_ext_name_is_none(self):
        self.assertIsNone(self.ext.ext_name)

    def test_version_is_none(self):
        self.assertIsNone(self.ext.version)

    def test_get_default_config_raises_not_implemented(self):
        with self.assertRaises(NotImplementedError):
            self.ext.get_default_config()

    def test_get_config_schema_returns_extension_schema(self):
        schema = self.ext.get_config_schema()
        self.assertIsInstance(schema['enabled'], config.Boolean)

    def test_validate_environment_does_nothing_by_default(self):
        self.assertIsNone(self.ext.validate_environment())

    def test_setup_raises_not_implemented(self):
        with self.assertRaises(NotImplementedError):
            self.ext.setup(None)
<commit_msg>tests: Convert ext test to pytests<commit_after>from __future__ import absolute_import, unicode_literals

import pytest

from mopidy import config, ext

@pytest.fixture
def extension():
    return ext.Extension()

def test_dist_name_is_none(extension):
    assert extension.dist_name is None

def test_ext_name_is_none(extension):
    assert extension.ext_name is None

def test_version_is_none(extension):
    assert extension.version is None

def test_get_default_config_raises_not_implemented(extension):
    with pytest.raises(NotImplementedError):
        extension.get_default_config()

def test_get_config_schema_returns_extension_schema(extension):
    schema = extension.get_config_schema()
    assert isinstance(schema['enabled'], config.Boolean)

def test_validate_environment_does_nothing_by_default(extension):
    assert extension.validate_environment() is None

def test_setup_raises_not_implemented(extension):
    with pytest.raises(NotImplementedError):
        extension.setup(None)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
##
##  test_inflow.py
##  RAPIDpy
##
##  Created by Alan D. Snow.
##  Copyright © 2016 Alan D Snow. All rights reserved.
##

from nose.tools import ok_
import os

#local import
from RAPIDpy.gis.workflow import CreateAllStaticECMWFRAPIDFiles
from RAPIDpy.helper_functions import (compare_csv_decimal_files,
                                      remove_files)
#GLOBAL VARIABLES
MAIN_TESTS_FOLDER = os.path.dirname(os.path.abspath(__file__))
COMPARE_DATA_PATH = os.path.join(MAIN_TESTS_FOLDER, 'compare', 'gis')
INPUT_DATA_PATH = os.path.join(MAIN_TESTS_FOLDER, 'data', 'gis')
OUTPUT_DATA_PATH = os.path.join(MAIN_TESTS_FOLDER, 'output')

#------------------------------------------------------------------------------
# MAIN TEST SCRIPTS
#------------------------------------------------------------------------------
def test_run_era_interim_inflow():
    """
    Checks generating inflow file from ERA Interim LSM
    """
    print "TEST 1: TEST GENERATE INFLOW FILE FROM ERA INTERIM DATA"
    CreateAllStaticECMWFRAPIDFiles(in_drainage_line=os.path.join(INPUT_DATA_PATH, 'flowline.shp'),
                                   river_id="COMID",
                                   length_id="LENGTHKM",
                                   slope_id="Slope",
                                   next_down_river_id="NextDownID",
                                   in_catchment=os.path.join(INPUT_DATA_PATH, 'catchment.shp'),
                                   catchment_river_id="FEATUREID",
                                   rapid_output_folder=OUTPUT_DATA_PATH,
                                   kfac_length_units="km",
                                   )

    #CHECK OUTPUT
    #comid_lat_lon_z
    generated_comid_lat_lon_z_file = os.path.join(OUTPUT_DATA_PATH,
                                                  "comid_lat_lon_z.csv")
    generated_comid_lat_lon_z_file_solution = os.path.join(COMPARE_DATA_PATH,
                                                           "comid_lat_lon_z.csv")
    ok_(compare_csv_decimal_files(generated_comid_lat_lon_z_file,
                                  generated_comid_lat_lon_z_file_solution))

    #rapid_connect
    generated_rapid_connect_file = os.path.join(OUTPUT_DATA_PATH,
                                                  "rapid_connect.csv")
    generated_rapid_connect_file_solution = os.path.join(COMPARE_DATA_PATH,
                                                           "rapid_connect.csv")
    ok_(compare_csv_decimal_files(generated_rapid_connect_file,
                                  generated_rapid_connect_file_solution))

    #riv_bas_id
    generated_riv_bas_id_file = os.path.join(OUTPUT_DATA_PATH,
                                             "riv_bas_id.csv")
    generated_riv_bas_id_file_solution = os.path.join(COMPARE_DATA_PATH,
                                                      "riv_bas_id.csv")
    ok_(compare_csv_decimal_files(generated_riv_bas_id_file,
                                  generated_riv_bas_id_file_solution))

    #kfac
    generated_kfac_file = os.path.join(OUTPUT_DATA_PATH,
                                       "kfac.csv")
    generated_kfac_file_solution = os.path.join(COMPARE_DATA_PATH,
                                                "kfac.csv")
    ok_(compare_csv_decimal_files(generated_kfac_file,
                                  generated_kfac_file_solution))

    #k
    generated_k_file = os.path.join(OUTPUT_DATA_PATH,
                                    "k.csv")
    generated_k_file_solution = os.path.join(COMPARE_DATA_PATH,
                                             "k.csv")
    ok_(compare_csv_decimal_files(generated_k_file,
                                  generated_k_file_solution))

    #x
    generated_x_file = os.path.join(OUTPUT_DATA_PATH,
                                    "x.csv")
    generated_x_file_solution = os.path.join(COMPARE_DATA_PATH,
                                             "x.csv")
    ok_(compare_csv_decimal_files(generated_x_file,
                                  generated_x_file_solution))

    #weight_ecmwf_t1279
    generated_weight_ecmwf_t1279_file = os.path.join(OUTPUT_DATA_PATH,
                                                     "weight_ecmwf_t1279.csv")
    generated_weight_ecmwf_t1279_file_solution = os.path.join(COMPARE_DATA_PATH,
                                                              "weight_ecmwf_t1279.csv")
    ok_(compare_csv_decimal_files(generated_weight_ecmwf_t1279_file,
                                  generated_weight_ecmwf_t1279_file_solution))

    #weight_ecmwf_tco369
    generated_weight_ecmwf_tco369_file = os.path.join(OUTPUT_DATA_PATH,
                                                     "weight_ecmwf_tco369.csv")
    generated_weight_ecmwf_tco369_file_solution = os.path.join(COMPARE_DATA_PATH,
                                                              "weight_ecmwf_tco369.csv")
    ok_(compare_csv_decimal_files(generated_weight_ecmwf_tco369_file,
                                  generated_weight_ecmwf_tco369_file_solution))

    #weight_era_t511
    generated_weight_era_t511_file = os.path.join(OUTPUT_DATA_PATH,
                                                     "weight_era_t511.csv")
    generated_weight_era_t511_file_solution = os.path.join(COMPARE_DATA_PATH,
                                                              "weight_era_t511.csv")
    ok_(compare_csv_decimal_files(generated_weight_era_t511_file,
                                  generated_weight_era_t511_file_solution))

    remove_files(generated_comid_lat_lon_z_file,
                 generated_rapid_connect_file,
                 generated_riv_bas_id_file,
                 generated_kfac_file,
                 generated_k_file,
                 generated_x_file,
                 generated_weight_ecmwf_t1279_file,
                 generated_weight_ecmwf_tco369_file,
                 generated_weight_era_t511_file)

if __name__ == '__main__':
    import nose
    nose.main()<commit_msg>fixed name<commit_after># -*- coding: utf-8 -*-
##
##  test_inflow.py
##  RAPIDpy
##
##  Created by Alan D. Snow.
##  Copyright © 2016 Alan D Snow. All rights reserved.
##

from nose.tools import ok_
import os

#local import
from RAPIDpy.gis.workflow import CreateAllStaticECMWFRAPIDFiles
from RAPIDpy.helper_functions import (compare_csv_decimal_files,
                                      remove_files)
#GLOBAL VARIABLES
MAIN_TESTS_FOLDER = os.path.dirname(os.path.abspath(__file__))
COMPARE_DATA_PATH = os.path.join(MAIN_TESTS_FOLDER, 'compare', 'gis')
INPUT_DATA_PATH = os.path.join(MAIN_TESTS_FOLDER, 'data', 'gis')
OUTPUT_DATA_PATH = os.path.join(MAIN_TESTS_FOLDER, 'output')

#------------------------------------------------------------------------------
# MAIN TEST SCRIPTS
#------------------------------------------------------------------------------
def test_run_era_interim_inflow():
    """
    Checks generating inflow file from ERA Interim LSM
    """
    print "TEST 1: TEST GENERATE INFLOW FILE FROM ERA INTERIM DATA"
    CreateAllStaticECMWFRAPIDFiles(in_drainage_line=os.path.join(INPUT_DATA_PATH, 'flowline.shp'),
                                   river_id="COMID",
                                   length_id="LENGTHKM",
                                   slope_id="Slope",
                                   next_down_river_id="NextDownID",
                                   in_catchment=os.path.join(INPUT_DATA_PATH, 'catchment.shp'),
                                   catchment_river_id="FEATUREID",
                                   rapid_output_folder=OUTPUT_DATA_PATH,
                                   kfac_length_units="km",
                                   )

    #CHECK OUTPUT
    #comid_lat_lon_z
    generated_comid_lat_lon_z_file = os.path.join(OUTPUT_DATA_PATH,
                                                  "comid_lat_lon_z.csv")
    generated_comid_lat_lon_z_file_solution = os.path.join(COMPARE_DATA_PATH,
                                                           "comid_lat_lon_z.csv")
    ok_(compare_csv_decimal_files(generated_comid_lat_lon_z_file,
                                  generated_comid_lat_lon_z_file_solution))

    #rapid_connect
    generated_rapid_connect_file = os.path.join(OUTPUT_DATA_PATH,
                                                  "rapid_connect.csv")
    generated_rapid_connect_file_solution = os.path.join(COMPARE_DATA_PATH,
                                                           "rapid_connect.csv")
    ok_(compare_csv_decimal_files(generated_rapid_connect_file,
                                  generated_rapid_connect_file_solution))

    #riv_bas_id
    generated_riv_bas_id_file = os.path.join(OUTPUT_DATA_PATH,
                                             "riv_bas_id.csv")
    generated_riv_bas_id_file_solution = os.path.join(COMPARE_DATA_PATH,
                                                      "riv_bas_id.csv")
    ok_(compare_csv_decimal_files(generated_riv_bas_id_file,
                                  generated_riv_bas_id_file_solution))

    #kfac
    generated_kfac_file = os.path.join(OUTPUT_DATA_PATH,
                                       "kfac.csv")
    generated_kfac_file_solution = os.path.join(COMPARE_DATA_PATH,
                                                "kfac.csv")
    ok_(compare_csv_decimal_files(generated_kfac_file,
                                  generated_kfac_file_solution))

    #k
    generated_k_file = os.path.join(OUTPUT_DATA_PATH,
                                    "k.csv")
    generated_k_file_solution = os.path.join(COMPARE_DATA_PATH,
                                             "k.csv")
    ok_(compare_csv_decimal_files(generated_k_file,
                                  generated_k_file_solution))

    #x
    generated_x_file = os.path.join(OUTPUT_DATA_PATH,
                                    "x.csv")
    generated_x_file_solution = os.path.join(COMPARE_DATA_PATH,
                                             "x.csv")
    ok_(compare_csv_decimal_files(generated_x_file,
                                  generated_x_file_solution))

    #weight_ecmwf_t1279
    generated_weight_ecmwf_t1279_file = os.path.join(OUTPUT_DATA_PATH,
                                                     "weight_ecmwf_t1279.csv")
    generated_weight_ecmwf_t1279_file_solution = os.path.join(COMPARE_DATA_PATH,
                                                              "weight_ecmwf_t1279.csv")
    ok_(compare_csv_decimal_files(generated_weight_ecmwf_t1279_file,
                                  generated_weight_ecmwf_t1279_file_solution))

    #weight_ecmwf_tco369
    generated_weight_ecmwf_tco639_file = os.path.join(OUTPUT_DATA_PATH,
                                                     "weight_ecmwf_tco639.csv")
    generated_weight_ecmwf_tco639_file_solution = os.path.join(COMPARE_DATA_PATH,
                                                              "weight_ecmwf_tco639.csv")
    ok_(compare_csv_decimal_files(generated_weight_ecmwf_tco639_file,
                                  generated_weight_ecmwf_tco639_file_solution))

    #weight_era_t511
    generated_weight_era_t511_file = os.path.join(OUTPUT_DATA_PATH,
                                                     "weight_era_t511.csv")
    generated_weight_era_t511_file_solution = os.path.join(COMPARE_DATA_PATH,
                                                              "weight_era_t511.csv")
    ok_(compare_csv_decimal_files(generated_weight_era_t511_file,
                                  generated_weight_era_t511_file_solution))

    remove_files(generated_comid_lat_lon_z_file,
                 generated_rapid_connect_file,
                 generated_riv_bas_id_file,
                 generated_kfac_file,
                 generated_k_file,
                 generated_x_file,
                 generated_weight_ecmwf_t1279_file,
                 generated_weight_ecmwf_tco639_file,
                 generated_weight_era_t511_file)

if __name__ == '__main__':
    import nose
    nose.main()<|endoftext|>


<commit_before>import datetime
import traceback
from exporters.export_managers.bypass import RequisitesNotMet
from exporters.logger.base_logger import ExportManagerLogger
from exporters.notifications.notifiers_list import NotifiersList
from exporters.module_loader import ModuleLoader
from exporters.exporter_config import ExporterConfig
from exporters.notifications.receiver_groups import CLIENTS, TEAM
from exporters.writers.base_writer import ItemsLimitReached

class BaseExporter(object):

    def __init__(self, configuration):
        self.config = ExporterConfig(configuration)
        self.logger = ExportManagerLogger(self.config.log_options)
        self.module_loader = ModuleLoader()
        self.reader = self.module_loader.load_reader(self.config.reader_options)
        self.filter_before = self.module_loader.load_filter(self.config.filter_before_options)
        self.filter_after = self.module_loader.load_filter(self.config.filter_after_options)
        self.transform = self.module_loader.load_transform(self.config.transform_options)
        self.writer = self.module_loader.load_writer(self.config.writer_options)
        self.persistence = self.module_loader.load_persistence(self.config.persistence_options)
        self.export_formatter = self.module_loader.load_formatter(self.config.formatter_options)
        self.grouper = self.module_loader.load_grouper(self.config.grouper_options)
        self.notifiers = NotifiersList(self.config.notifiers)
        self.logger.debug('{} has been initiated'.format(self.__class__.__name__))
        job_info = {
            'configuration': configuration,
            'items_count': 0,
            'start_time': datetime.datetime.now(),
            'script_name': 'basic_export_manager'
        }
        self.stats_manager = self.module_loader.load_stats_manager(self.config.stats_options)
        self.stats_manager.stats = job_info

    def _run_pipeline_iteration(self):
        self.logger.debug('Getting new batch')
        next_batch = self.reader.get_next_batch()
        last_position = self.reader.get_last_position()
        next_batch = self.filter_before.filter_batch(next_batch)
        next_batch = self.transform.transform_batch(next_batch)
        next_batch = self.filter_after.filter_batch(next_batch)
        next_batch = self.grouper.group_batch(next_batch)
        next_batch = self.export_formatter.format(next_batch)
        self.writer.write_batch(batch=next_batch)
        self.persistence.commit_position(last_position)

    def _init_export_job(self):
        self.notifiers.notify_start_dump(receivers=[CLIENTS, TEAM], info=self.stats_manager.stats)
        last_position = self.persistence.get_last_position()
        self.reader.set_last_position(last_position)

    def _clean_export_job(self):
        self.persistence.delete_instance()
        self.writer.close_writer()

    def _finish_export_job(self):
        self.stats_manager.stats['items_count'] = self.writer.items_count
        self.stats_manager.stats['end_time'] = datetime.datetime.now()
        self.stats_manager.stats['elapsed_time'] = self.stats_manager.stats['end_time'] - self.stats_manager.stats['start_time']

    @property
    def bypass_cases(self):
        return []

    def bypass_exporter(self, bypass_script):
        self.logger.info('Executing bypass {}.'.format(bypass_script.__class__.__name__))
        self.notifiers.notify_start_dump(receivers=[CLIENTS, TEAM], info=self.stats_manager.stats)
        bypass_script.bypass()
        self.logger.info('Finished executing bypass {}.'.format(bypass_script.__class__.__name__))
        self.notifiers.notify_complete_dump(receivers=[CLIENTS, TEAM], info=self.stats_manager.stats)

    def bypass(self):
        for bypass_script in self.bypass_cases:
            try:
                bypass_script.meets_conditions()
                self.bypass_exporter(bypass_script)
                return True
            except RequisitesNotMet:
                self.logger.debug('{} bypass skipped.'.format(bypass_script.__class__.__name__))
        return False

    def _handle_export_exception(self, exception):
        self.logger.error(traceback.format_exc(exception))
        self.logger.error(str(exception))
        self.notifiers.notify_failed_job(str(exception), str(traceback.format_exc(exception)), receivers=[TEAM], info=self.stats_manager.stats)

    def _run_pipeline(self):
        while not self.reader.is_finished():
            try:
                self._run_pipeline_iteration()
            except ItemsLimitReached as e:
                self.logger.info('{!r}'.format(e))
                break
            else:
                self.stats_manager.stats['items_count'] = self.writer.items_count
                self.stats_manager.populate()

    def export(self):
        if not self.bypass():
            try:
                self._init_export_job()
                self._run_pipeline()
                self._finish_export_job()
                self.notifiers.notify_complete_dump(receivers=[CLIENTS, TEAM], info=self.stats_manager.stats)
            except Exception as e:
                self._handle_export_exception(e)
                raise e
            finally:
                self._clean_export_job()
        self.logger.info(str(self.stats_manager.stats))
<commit_msg>Populate stats when finishing.<commit_after>import datetime
import traceback
from exporters.export_managers.bypass import RequisitesNotMet
from exporters.logger.base_logger import ExportManagerLogger
from exporters.notifications.notifiers_list import NotifiersList
from exporters.module_loader import ModuleLoader
from exporters.exporter_config import ExporterConfig
from exporters.notifications.receiver_groups import CLIENTS, TEAM
from exporters.writers.base_writer import ItemsLimitReached

class BaseExporter(object):

    def __init__(self, configuration):
        self.config = ExporterConfig(configuration)
        self.logger = ExportManagerLogger(self.config.log_options)
        self.module_loader = ModuleLoader()
        self.reader = self.module_loader.load_reader(self.config.reader_options)
        self.filter_before = self.module_loader.load_filter(self.config.filter_before_options)
        self.filter_after = self.module_loader.load_filter(self.config.filter_after_options)
        self.transform = self.module_loader.load_transform(self.config.transform_options)
        self.writer = self.module_loader.load_writer(self.config.writer_options)
        self.persistence = self.module_loader.load_persistence(self.config.persistence_options)
        self.export_formatter = self.module_loader.load_formatter(self.config.formatter_options)
        self.grouper = self.module_loader.load_grouper(self.config.grouper_options)
        self.notifiers = NotifiersList(self.config.notifiers)
        self.logger.debug('{} has been initiated'.format(self.__class__.__name__))
        job_info = {
            'configuration': configuration,
            'items_count': 0,
            'start_time': datetime.datetime.now(),
            'script_name': 'basic_export_manager'
        }
        self.stats_manager = self.module_loader.load_stats_manager(self.config.stats_options)
        self.stats_manager.stats = job_info

    def _run_pipeline_iteration(self):
        self.logger.debug('Getting new batch')
        next_batch = self.reader.get_next_batch()
        last_position = self.reader.get_last_position()
        next_batch = self.filter_before.filter_batch(next_batch)
        next_batch = self.transform.transform_batch(next_batch)
        next_batch = self.filter_after.filter_batch(next_batch)
        next_batch = self.grouper.group_batch(next_batch)
        next_batch = self.export_formatter.format(next_batch)
        self.writer.write_batch(batch=next_batch)
        self.persistence.commit_position(last_position)

    def _init_export_job(self):
        self.notifiers.notify_start_dump(receivers=[CLIENTS, TEAM], info=self.stats_manager.stats)
        last_position = self.persistence.get_last_position()
        self.reader.set_last_position(last_position)

    def _clean_export_job(self):
        self.persistence.delete_instance()
        self.writer.close_writer()

    def _finish_export_job(self):
        self.stats_manager.stats['items_count'] = self.writer.items_count
        self.stats_manager.stats['end_time'] = datetime.datetime.now()
        self.stats_manager.stats['elapsed_time'] = self.stats_manager.stats['end_time'] - self.stats_manager.stats['start_time']

    @property
    def bypass_cases(self):
        return []

    def bypass_exporter(self, bypass_script):
        self.logger.info('Executing bypass {}.'.format(bypass_script.__class__.__name__))
        self.notifiers.notify_start_dump(receivers=[CLIENTS, TEAM], info=self.stats_manager.stats)
        bypass_script.bypass()
        self.logger.info('Finished executing bypass {}.'.format(bypass_script.__class__.__name__))
        self.notifiers.notify_complete_dump(receivers=[CLIENTS, TEAM], info=self.stats_manager.stats)

    def bypass(self):
        for bypass_script in self.bypass_cases:
            try:
                bypass_script.meets_conditions()
                self.bypass_exporter(bypass_script)
                return True
            except RequisitesNotMet:
                self.logger.debug('{} bypass skipped.'.format(bypass_script.__class__.__name__))
        return False

    def _handle_export_exception(self, exception):
        self.logger.error(traceback.format_exc(exception))
        self.logger.error(str(exception))
        self.notifiers.notify_failed_job(str(exception), str(traceback.format_exc(exception)), receivers=[TEAM], info=self.stats_manager.stats)

    def _run_pipeline(self):
        while not self.reader.is_finished():
            try:
                self._run_pipeline_iteration()
            except ItemsLimitReached as e:
                self.logger.info('{!r}'.format(e))
                break
            else:
                self.stats_manager.stats['items_count'] = self.writer.items_count
                self.stats_manager.populate()

    def export(self):
        if not self.bypass():
            try:
                self._init_export_job()
                self._run_pipeline()
                self._finish_export_job()
                self.stats_manager.populate()
                self.notifiers.notify_complete_dump(receivers=[CLIENTS, TEAM], info=self.stats_manager.stats)
            except Exception as e:
                self._handle_export_exception(e)
                raise e
            finally:
                self._clean_export_job()
        self.logger.info(str(self.stats_manager.stats))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os
import shutil
from sh import git
from mock import patch, PropertyMock
from django.test import TestCase
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext_lazy as _
from waliki.models import Page
from waliki.git import Git
from waliki.settings import WALIKI_DATA_DIR, WALIKI_COMMITTER_EMAIL, WALIKI_COMMITTER_NAME
from .factories import PageFactory

class TestGit(TestCase):

    def setUp(self):
        self.page = PageFactory()
        self.edit_url = reverse('waliki_edit', args=(self.page.slug,))

    def test_init_git_create_repo(self):
        git_dir = os.path.join(WALIKI_DATA_DIR, '.git')
        shutil.rmtree(git_dir)
        Git()
        self.assertTrue(os.path.isdir(git_dir))
        self.assertEqual(git.config('user.name').stdout.decode('utf8')[:-1], WALIKI_COMMITTER_NAME)
        self.assertEqual(git.config('user.email').stdout.decode('utf8')[:-1], WALIKI_COMMITTER_EMAIL)

    def test_commit_existent_page_with_no_previous_commits(self):
        response = self.client.get(self.edit_url)
        data = response.context[0]['form'].initial
        data["raw"] = "test content"
        data["message"] = "testing :)"
        response = self.client.post(self.edit_url, data)
        self.assertEqual(self.page.raw, "test content")
        self.assertEqual(Git().version(self.page, 'HEAD'), "test content")
        self.assertIn("testing :)", git.log('-s', '--format=%s', self.page.path))

    def test_commit_existent_page_with_previous_commits(self):
        self.page.raw = "lala"
        Git().commit(self.page, message="previous commit")
        assert "previous commit" in git.log('-s', '--format=%s', self.page.path)
        response = self.client.get(self.edit_url)
        data = response.context[0]['form'].initial
        data["raw"] = "test content"
        response = self.client.post(self.edit_url, data)
        self.assertEqual(self.page.raw, "test content")
        self.assertEqual(Git().version(self.page, 'HEAD'), "test content")

    def test_commit_new_page(self):
        assert not Page.objects.filter(slug='test').exists()
        url = reverse('waliki_edit', args=('test',))
        response = self.client.post(url)
        # it exists now
        self.assertTrue(Page.objects.filter(slug='test').exists())
        data = response.context[0]['form'].initial
        data["raw"] = "hey there\n"
        data["title"] = "Test Page"
        data["message"] = ""
        response = self.client.post(url, data)
        page = Page.objects.get(slug='test')
        self.assertEqual(page.title, "Test Page")
        self.assertEqual(page.raw, "hey there\n")
        self.assertEqual(Git().version(page, 'HEAD'), "hey there\n")

    def test_concurrent_edition_with_no_conflict(self):
        self.page.raw = "\n- item1\n"
        Git().commit(self.page, message="original")

        response1 = self.client.get(self.edit_url)
        response2 = self.client.get(self.edit_url)

        data1 = response1.context[0]['form'].initial
        data1["raw"] = self.page.raw + '\n- item2'
        data1["message"] = "add item2"

        data2 = response2.context[0]['form'].initial
        data2["raw"] = '- item0\n' + self.page.raw
        data1["message"] = "add item0"
        self.client.post(self.edit_url, data1)
        with patch('waliki.views.messages') as messages:
            self.client.post(self.edit_url, data2)
        # there is a message
        self.assertTrue(messages.warning.called)
        self.assertIn('There were changes', messages.warning.call_args[0][1])

        # newer versions of git don't leave a blank line at the end
        self.assertRegexpMatches("- item0\n\n- item1\n\n- item2\n?$", self.page.raw)

    def test_concurrent_edition_with_conflict(self):
        self.page.raw = "- item1"
        Git().commit(self.page, message="original")
        response1 = self.client.get(self.edit_url)
        response2 = self.client.get(self.edit_url)

        data1 = response1.context[0]['form'].initial
        data1["raw"] = '- item2'
        data1["message"] = "add item2"

        data2 = response2.context[0]['form'].initial
        data2["raw"] = '- item0'
        data1["message"] = "add item0"
        r = self.client.post(self.edit_url, data1)
        self.assertRedirects(r, reverse('waliki_detail', args=(self.page.slug,)))
        r = self.client.post(self.edit_url, data2)
        self.assertRedirects(r, self.edit_url)

        self.assertEqual('', git.status('--porcelain', self.page.path).stdout.decode('utf8'))
        self.assertIn('Merged with conflict', git.log("--no-color", "--pretty=format:%s", "-n 1", self.page.path).stdout.decode('utf8'))
        self.assertRegexpMatches(self.page.raw,'<<<<<<< HEAD\n- item2\n=======\n- item0\n>>>>>>> [0-9a-f]{7}\n')

        # can edit in conflict
        response = self.client.get(self.edit_url)
        data1 = response.context[0]['form'].initial
        data1["raw"] = '- item0\n- item2'
        data1["message"] = "fixing :)"
        response = self.client.post(self.edit_url, data1)
        self.assertRedirects(response, reverse('waliki_detail', args=(self.page.slug,)))
        self.assertEqual(self.page.raw, '- item0\n- item2')

    def test_concurrent_edition_no_existent_page(self):
        assert not Page.objects.filter(slug='test2').exists()
        url = reverse('waliki_edit', args=('test2',))
        response1 = self.client.post(url)
        response2 = self.client.post(url)
        page = Page.objects.get(slug='test2')

        data1 = response1.context[0]['form'].initial
        data1["raw"] = '- item2\n'
        data1["message"] = "add item2"
        data1["title"] = "a title"

        data2 = response2.context[0]['form'].initial
        data2["raw"] = '- item0\n'
        data2["message"] = "add item0"
        data2["title"] = "another title"

        self.client.post(url, data1)
        with patch('waliki.views.messages') as messages:
            response = self.client.post(url, data2)
        # there is a warning
        self.assertTrue(messages.warning.called)
        self.assertIsInstance(messages.warning.call_args[0][1], Page.EditionConflict)

        # redirect
        self.assertRedirects(response, url)

        # file committed with conflict
        self.assertEqual('', git.status('--porcelain', page.path).stdout.decode('utf8'))
        self.assertIn('Merged with conflict', git.log("--pretty=format:%s", "-n 1", page.path).stdout.decode('utf8'))

        self.assertRegexpMatches(page.raw, r"""<<<<<<< HEAD\n- item2
=======\n- item0\n>>>>>>> [0-9a-f]{7}\n""")
        page = Page.objects.get(slug='test2')       # refresh
        self.assertEqual(page.title, "another title")

        # can edit in conflict
        response = self.client.get(url)

        data1 = response.context[0]['form'].initial
        data1["raw"] = '- item0\n- item2\n'
        data1["message"] = "fixing :)"
        response = self.client.post(url, data1)
        self.assertRedirects(response, reverse('waliki_detail', args=(page.slug,)))
        self.assertEqual(page.raw, '- item0\n- item2\n')

    def test_unicode_content(self):
        response = self.client.get(self.edit_url)
        data = response.context[0]['form'].initial
        data['raw'] = u'Ω'
        data['message'] = 'testing :)'
        response = self.client.post(self.edit_url, data)
        self.assertRedirects(response, reverse('waliki_detail', args=(self.page.slug,)))
        self.assertEqual(Git().version(self.page, 'HEAD'), u'Ω')

    def test_commit_page_with_no_changes(self):
        self.page.raw = 'lala'
        Git().commit(self.page, message='previous commit')

        response = self.client.get(self.edit_url)
        data = response.context[0]['form'].initial
        data['title'] = 'new title'
        data['raw'] = self.page.raw
        data['message'] = 'testing :)'
        response = self.client.post(self.edit_url, data)
        self.assertRedirects(response, self.page.get_absolute_url())
        page = Page.objects.get(id=self.page.id)       # refresh
        self.assertEqual(page.raw, 'lala')      # same body
        self.assertEqual(page.title, 'new title')      # different title
        response = self.client.get(reverse('waliki_history', args=(self.page.slug,)))
        history = response.context[0].get('history')
        self.assertEqual(len(history), 1)
        self.assertEqual(history[0]['message'], u'previous commit')

    def test_history_log(self):
        self.page.raw = 'line\n' * 10
        Git().commit(self.page, message=u'"10 lines ñoñas"')  # include quotes and non ascii

        self.page.raw = 'another line2\n' * 2
        Git().commit(self.page, message=u'changed 20%')  # include quotes and non ascii

        response = self.client.get(reverse('waliki_history', args=(self.page.slug,)))
        self.assertEqual(response.status_code, 200)

        history = response.context[0].get('history')

        self.assertEqual(history[0]['message'], u'changed 20%')
        self.assertEqual(history[0]['deletion'], 10)
        self.assertEqual(history[0]['insertion'], 2)
        self.assertTrue(str(history[0]['insertion_relative']).startswith('16.6'))
        self.assertEqual(history[1]['message'], u'"10 lines ñoñas"')

    def test_diff_with_space_and_tab(self):
        self.page.raw = 'line with space\n'
        Git().commit(self.page, message=u'one')
        old = Git().last_version(self.page)

        self.page.raw = '\tline with tab\n'
        Git().commit(self.page, message=u'two')
        new = Git().last_version(self.page)

        response = self.client.get(reverse('waliki_diff', args=(self.page.slug, old, new,)))
        old_content = response.context[0]['old_content']
        new_content = response.context[0]['new_content']

        self.assertEqual(old_content, u'line\u00a0with\u00a0space\n')
        self.assertEqual(new_content, u'\u00a0\u00a0\u00a0\u00a0line\u00a0with\u00a0tab\n')

class TestWhatChanged(TestCase):

    def setUp(self):
        self.page = PageFactory()
        self.edit_url = reverse('waliki_edit', args=(self.page.slug,))

    def test_low_level_whatchanged(self):
        self.page.raw = 'line\n'
        Git().commit(self.page, message=u'"//"')
        another_page = PageFactory(path='another-page.rst')
        another_page.raw = "hello!"
        self.page.raw = "hello 2!"
        git.add('.')
        git.commit('-am', 'commit all')
        wc = Git().whatchanged()
        import ipdb; ipdb.set_trace()

    def test_whatchanged(self):
        self.page.raw = 'line\n'
        Git().commit(self.page, message=u'"//"')
        another_page = PageFactory(path='another-page.rst')
        another_page.raw = "hello!"
        Git().commit(another_page, message=u'hello history')
        response = self.client.get(reverse('waliki_whatchanged'))
        changes = response.context[0]['changes']

        self.assertEqual(len(changes), 2)
        self.assertEqual(changes[0]['page'], another_page)
        self.assertEqual(changes[1]['page'], self.page)
        self.assertEqual(changes[0]['message'], 'hello history')
        self.assertEqual(changes[1]['message'], '"//"')

    def test_whatchanged_pagination(self):
        self.page.raw = 'line\n'
        Git().commit(self.page, message=u'one')
        self.page.raw += 'line 2\n'
        Git().commit(self.page, message=u'two')
        self.page.raw += 'line 3\n'
        Git().commit(self.page, message=u'three')
        with patch('waliki.git.views.settings') as s_mock:
            type(s_mock).WALIKI_PAGINATE_BY = PropertyMock(return_value=2)
            response1 = self.client.get(reverse('waliki_whatchanged'))
            response2 = self.client.get(reverse('waliki_whatchanged', args=('2',)))

        # first page has no previous page
        self.assertIsNone(response1.context[0]['prev'])
        self.assertEqual(response1.context[0]['next'], 2)
        self.assertIsNone(response2.context[0]['next'])
        self.assertEqual(response2.context[0]['prev'], 1)

        changes1 = response1.context[0]['changes']
        changes2 = response2.context[0]['changes']
        self.assertEqual(len(changes1), 2)
        self.assertEqual(len(changes2), 1)
        self.assertEqual(changes1[0]['message'], 'three')
        self.assertEqual(changes1[1]['message'], 'two')
        self.assertEqual(changes2[0]['message'], 'one')

    def test_whatchanged_multiples_files_in_one_commit(self):
        git_dir = os.path.join(WALIKI_DATA_DIR, '.git')
        shutil.rmtree(git_dir)
        Git()

        self.page.raw = 'line\n'
        another_page = PageFactory(path='another-page.rst')
        another_page.raw = "hello!"
        git.add('.')
        git.commit('-am', 'commit all')
        response = self.client.get(reverse('waliki_whatchanged'))
        changes = response.context[0]['changes']
        self.assertEqual(len(changes), 2)
        self.assertEqual(changes[0]['page'], another_page)
        self.assertEqual(changes[1]['page'], self.page)
        self.assertEqual(changes[0]['message'], 'commit all')
        self.assertEqual(changes[1]['message'], 'commit all')

class TestGitMove(TestCase):

    def setUp(self):
        self.page = PageFactory()

    def test_legacy_history(self):
        original_slug = self.page.slug
        self.page.raw = "\n- item1\n"
        Git().commit(self.page, message="original")
        self.page.raw = "\n- otra\n"
        Git().commit(self.page, message="second version")
        self.client.post(reverse('waliki_move', args=(self.page.slug,)), {'slug': 'new-slug'})

        response = self.client.get(reverse('waliki_history', args=('new-slug',)))
        history = response.context[0].get('history')
        self.assertEqual(history[0]['message'], u'Page moved from %s' % original_slug)
        self.assertEqual(history[1]['message'], u'second version')
        self.assertEqual(history[2]['message'], u'original')
<commit_msg>couple of test for whatchanged<commit_after># -*- coding: utf-8 -*-
import os
import shutil
from sh import git
from mock import patch, PropertyMock
from django.test import TestCase
from django.core.urlresolvers import reverse
from django.utils.translation import ugettext_lazy as _
from waliki.models import Page
from waliki.git import Git
from waliki.settings import WALIKI_DATA_DIR, WALIKI_COMMITTER_EMAIL, WALIKI_COMMITTER_NAME
from .factories import PageFactory

class TestGit(TestCase):

    def setUp(self):
        self.page = PageFactory()
        self.edit_url = reverse('waliki_edit', args=(self.page.slug,))

    def test_init_git_create_repo(self):
        git_dir = os.path.join(WALIKI_DATA_DIR, '.git')
        shutil.rmtree(git_dir)
        Git()
        self.assertTrue(os.path.isdir(git_dir))
        self.assertEqual(git.config('user.name').stdout.decode('utf8')[:-1], WALIKI_COMMITTER_NAME)
        self.assertEqual(git.config('user.email').stdout.decode('utf8')[:-1], WALIKI_COMMITTER_EMAIL)

    def test_commit_existent_page_with_no_previous_commits(self):
        response = self.client.get(self.edit_url)
        data = response.context[0]['form'].initial
        data["raw"] = "test content"
        data["message"] = "testing :)"
        response = self.client.post(self.edit_url, data)
        self.assertEqual(self.page.raw, "test content")
        self.assertEqual(Git().version(self.page, 'HEAD'), "test content")
        self.assertIn("testing :)", git.log('-s', '--format=%s', self.page.path))

    def test_commit_existent_page_with_previous_commits(self):
        self.page.raw = "lala"
        Git().commit(self.page, message="previous commit")
        assert "previous commit" in git.log('-s', '--format=%s', self.page.path)
        response = self.client.get(self.edit_url)
        data = response.context[0]['form'].initial
        data["raw"] = "test content"
        response = self.client.post(self.edit_url, data)
        self.assertEqual(self.page.raw, "test content")
        self.assertEqual(Git().version(self.page, 'HEAD'), "test content")

    def test_commit_new_page(self):
        assert not Page.objects.filter(slug='test').exists()
        url = reverse('waliki_edit', args=('test',))
        response = self.client.post(url)
        # it exists now
        self.assertTrue(Page.objects.filter(slug='test').exists())
        data = response.context[0]['form'].initial
        data["raw"] = "hey there\n"
        data["title"] = "Test Page"
        data["message"] = ""
        response = self.client.post(url, data)
        page = Page.objects.get(slug='test')
        self.assertEqual(page.title, "Test Page")
        self.assertEqual(page.raw, "hey there\n")
        self.assertEqual(Git().version(page, 'HEAD'), "hey there\n")

    def test_concurrent_edition_with_no_conflict(self):
        self.page.raw = "\n- item1\n"
        Git().commit(self.page, message="original")

        response1 = self.client.get(self.edit_url)
        response2 = self.client.get(self.edit_url)

        data1 = response1.context[0]['form'].initial
        data1["raw"] = self.page.raw + '\n- item2'
        data1["message"] = "add item2"

        data2 = response2.context[0]['form'].initial
        data2["raw"] = '- item0\n' + self.page.raw
        data1["message"] = "add item0"
        self.client.post(self.edit_url, data1)
        with patch('waliki.views.messages') as messages:
            self.client.post(self.edit_url, data2)
        # there is a message
        self.assertTrue(messages.warning.called)
        self.assertIn('There were changes', messages.warning.call_args[0][1])

        # newer versions of git don't leave a blank line at the end
        self.assertRegexpMatches("- item0\n\n- item1\n\n- item2\n?$", self.page.raw)

    def test_concurrent_edition_with_conflict(self):
        self.page.raw = "- item1"
        Git().commit(self.page, message="original")
        response1 = self.client.get(self.edit_url)
        response2 = self.client.get(self.edit_url)

        data1 = response1.context[0]['form'].initial
        data1["raw"] = '- item2'
        data1["message"] = "add item2"

        data2 = response2.context[0]['form'].initial
        data2["raw"] = '- item0'
        data1["message"] = "add item0"
        r = self.client.post(self.edit_url, data1)
        self.assertRedirects(r, reverse('waliki_detail', args=(self.page.slug,)))
        r = self.client.post(self.edit_url, data2)
        self.assertRedirects(r, self.edit_url)

        self.assertEqual('', git.status('--porcelain', self.page.path).stdout.decode('utf8'))
        self.assertIn('Merged with conflict', git.log("--no-color", "--pretty=format:%s", "-n 1", self.page.path).stdout.decode('utf8'))
        self.assertRegexpMatches(self.page.raw,'<<<<<<< HEAD\n- item2\n=======\n- item0\n>>>>>>> [0-9a-f]{7}\n')

        # can edit in conflict
        response = self.client.get(self.edit_url)
        data1 = response.context[0]['form'].initial
        data1["raw"] = '- item0\n- item2'
        data1["message"] = "fixing :)"
        response = self.client.post(self.edit_url, data1)
        self.assertRedirects(response, reverse('waliki_detail', args=(self.page.slug,)))
        self.assertEqual(self.page.raw, '- item0\n- item2')

    def test_concurrent_edition_no_existent_page(self):
        assert not Page.objects.filter(slug='test2').exists()
        url = reverse('waliki_edit', args=('test2',))
        response1 = self.client.post(url)
        response2 = self.client.post(url)
        page = Page.objects.get(slug='test2')

        data1 = response1.context[0]['form'].initial
        data1["raw"] = '- item2\n'
        data1["message"] = "add item2"
        data1["title"] = "a title"

        data2 = response2.context[0]['form'].initial
        data2["raw"] = '- item0\n'
        data2["message"] = "add item0"
        data2["title"] = "another title"

        self.client.post(url, data1)
        with patch('waliki.views.messages') as messages:
            response = self.client.post(url, data2)
        # there is a warning
        self.assertTrue(messages.warning.called)
        self.assertIsInstance(messages.warning.call_args[0][1], Page.EditionConflict)

        # redirect
        self.assertRedirects(response, url)

        # file committed with conflict
        self.assertEqual('', git.status('--porcelain', page.path).stdout.decode('utf8'))
        self.assertIn('Merged with conflict', git.log("--pretty=format:%s", "-n 1", page.path).stdout.decode('utf8'))

        self.assertRegexpMatches(page.raw, r"""<<<<<<< HEAD\n- item2
=======\n- item0\n>>>>>>> [0-9a-f]{7}\n""")
        page = Page.objects.get(slug='test2')       # refresh
        self.assertEqual(page.title, "another title")

        # can edit in conflict
        response = self.client.get(url)

        data1 = response.context[0]['form'].initial
        data1["raw"] = '- item0\n- item2\n'
        data1["message"] = "fixing :)"
        response = self.client.post(url, data1)
        self.assertRedirects(response, reverse('waliki_detail', args=(page.slug,)))
        self.assertEqual(page.raw, '- item0\n- item2\n')

    def test_unicode_content(self):
        response = self.client.get(self.edit_url)
        data = response.context[0]['form'].initial
        data['raw'] = u'Ω'
        data['message'] = 'testing :)'
        response = self.client.post(self.edit_url, data)
        self.assertRedirects(response, reverse('waliki_detail', args=(self.page.slug,)))
        self.assertEqual(Git().version(self.page, 'HEAD'), u'Ω')

    def test_commit_page_with_no_changes(self):
        self.page.raw = 'lala'
        Git().commit(self.page, message='previous commit')

        response = self.client.get(self.edit_url)
        data = response.context[0]['form'].initial
        data['title'] = 'new title'
        data['raw'] = self.page.raw
        data['message'] = 'testing :)'
        response = self.client.post(self.edit_url, data)
        self.assertRedirects(response, self.page.get_absolute_url())
        page = Page.objects.get(id=self.page.id)       # refresh
        self.assertEqual(page.raw, 'lala')      # same body
        self.assertEqual(page.title, 'new title')      # different title
        response = self.client.get(reverse('waliki_history', args=(self.page.slug,)))
        history = response.context[0].get('history')
        self.assertEqual(len(history), 1)
        self.assertEqual(history[0]['message'], u'previous commit')

    def test_history_log(self):
        self.page.raw = 'line\n' * 10
        Git().commit(self.page, message=u'"10 lines ñoñas"')  # include quotes and non ascii

        self.page.raw = 'another line2\n' * 2
        Git().commit(self.page, message=u'changed 20%')  # include quotes and non ascii

        response = self.client.get(reverse('waliki_history', args=(self.page.slug,)))
        self.assertEqual(response.status_code, 200)

        history = response.context[0].get('history')

        self.assertEqual(history[0]['message'], u'changed 20%')
        self.assertEqual(history[0]['deletion'], 10)
        self.assertEqual(history[0]['insertion'], 2)
        self.assertTrue(str(history[0]['insertion_relative']).startswith('16.6'))
        self.assertEqual(history[1]['message'], u'"10 lines ñoñas"')

    def test_diff_with_space_and_tab(self):
        self.page.raw = 'line with space\n'
        Git().commit(self.page, message=u'one')
        old = Git().last_version(self.page)

        self.page.raw = '\tline with tab\n'
        Git().commit(self.page, message=u'two')
        new = Git().last_version(self.page)

        response = self.client.get(reverse('waliki_diff', args=(self.page.slug, old, new,)))
        old_content = response.context[0]['old_content']
        new_content = response.context[0]['new_content']

        self.assertEqual(old_content, u'line\u00a0with\u00a0space\n')
        self.assertEqual(new_content, u'\u00a0\u00a0\u00a0\u00a0line\u00a0with\u00a0tab\n')

class TestWhatChanged(TestCase):

    def setUp(self):
        self.page = PageFactory()
        self.edit_url = reverse('waliki_edit', args=(self.page.slug,))

    def test_low_level_whatchanged(self):
        self.page.raw = 'line\n'
        Git().commit(self.page, message=u'"//"')
        another_page = PageFactory(path='another-page.rst')
        another_page.raw = "hello!"
        self.page.raw = "hello 2!"
        git.add(another_page.path)
        git.add(self.page.path)
        git.commit('-m', 'commit all')
        wc = Git().whatchanged()
        self.assertEqual(wc[0][3], 'commit all')
        self.assertEqual(wc[0][5], [another_page.path, self.page.path])

    def test_whatchanged(self):
        self.page.raw = 'line\n'
        Git().commit(self.page, message=u'"//"')
        another_page = PageFactory(path='random_page.rst')
        another_page.raw = "hello!"
        Git().commit(another_page, message=u'hello history')
        response = self.client.get(reverse('waliki_whatchanged'))
        changes = response.context[0]['changes']

        self.assertEqual(len(changes), 2)
        self.assertEqual(changes[0]['page'], another_page)
        self.assertEqual(changes[1]['page'], self.page)
        self.assertEqual(changes[0]['message'], 'hello history')
        self.assertEqual(changes[1]['message'], '"//"')

    def test_whatchanged_pagination(self):
        self.page.raw = 'line\n'
        Git().commit(self.page, message=u'one')
        self.page.raw += 'line 2\n'
        Git().commit(self.page, message=u'two')
        self.page.raw += 'line 3\n'
        Git().commit(self.page, message=u'three')
        with patch('waliki.git.views.settings') as s_mock:
            type(s_mock).WALIKI_PAGINATE_BY = PropertyMock(return_value=2)
            response1 = self.client.get(reverse('waliki_whatchanged'))
            response2 = self.client.get(reverse('waliki_whatchanged', args=('2',)))

        # first page has no previous page
        self.assertIsNone(response1.context[0]['prev'])
        self.assertEqual(response1.context[0]['next'], 2)
        self.assertIsNone(response2.context[0]['next'])
        self.assertEqual(response2.context[0]['prev'], 1)

        changes1 = response1.context[0]['changes']
        changes2 = response2.context[0]['changes']
        self.assertEqual(len(changes1), 2)
        self.assertEqual(len(changes2), 1)
        self.assertEqual(changes1[0]['message'], 'three')
        self.assertEqual(changes1[1]['message'], 'two')
        self.assertEqual(changes2[0]['message'], 'one')

    def test_whatchanged_multiples_files_in_one_commit(self):
        git_dir = os.path.join(WALIKI_DATA_DIR, '.git')
        shutil.rmtree(git_dir)
        Git()

        self.page.raw = 'line\n'
        another_page = PageFactory(path='another-page.rst')
        another_page.raw = "hello!"
        git.add('.')
        git.commit('-am', 'commit all')
        response = self.client.get(reverse('waliki_whatchanged'))
        changes = response.context[0]['changes']
        self.assertEqual(len(changes), 2)
        self.assertEqual(changes[0]['page'], another_page)
        self.assertEqual(changes[1]['page'], self.page)
        self.assertEqual(changes[0]['message'], 'commit all')
        self.assertEqual(changes[1]['message'], 'commit all')

class TestGitMove(TestCase):

    def setUp(self):
        self.page = PageFactory()

    def test_legacy_history(self):
        original_slug = self.page.slug
        self.page.raw = "\n- item1\n"
        Git().commit(self.page, message="original")
        self.page.raw = "\n- otra\n"
        Git().commit(self.page, message="second version")
        self.client.post(reverse('waliki_move', args=(self.page.slug,)), {'slug': 'new-slug'})

        response = self.client.get(reverse('waliki_history', args=('new-slug',)))
        history = response.context[0].get('history')
        self.assertEqual(history[0]['message'], u'Page moved from %s' % original_slug)
        self.assertEqual(history[1]['message'], u'second version')
        self.assertEqual(history[2]['message'], u'original')
<|endoftext|>


<commit_before># $Id: test_gtk.py,v 1.8 2001/08/21 10:08:22 kjetilja Exp $

## System modules
import sys, threading

## Gtk modules
from gtk import *

## PycURL module
import pycurl

def progress(download_t, download_d, upload_t, upload_d):
    global round, pbar
    threads_enter()
    if download_t == 0:
        pbar.set_activity_mode(1)
        round = round + 0.1
        if round >= 1.0:  round = 0.0
    else:
        pbar.set_activity_mode(0)
        round = float(download_d) / float(download_t)
    pbar.update(round)
    threads_leave()
    return 0 # Anything else indicates an error

def close_app(*args):
    global t, win
    win.destroy()
    mainquit()
    return TRUE

class Test(threading.Thread):

    def __init__(self, url, target_file):
        threading.Thread.__init__(self)
        self.target_file = target_file
        self.curl = pycurl.init()
        self.curl.setopt(pycurl.URL, url)
        self.curl.setopt(pycurl.FILE, target_file)
        self.curl.setopt(pycurl.FOLLOWLOCATION, 1)
        self.curl.setopt(pycurl.NOPROGRESS, 0)
        self.curl.setopt(pycurl.PROGRESSFUNCTION, progress)
        self.curl.setopt(pycurl.MAXREDIRS, 5)

    def run(self):
        self.curl.perform()
        self.curl.cleanup()
        self.target_file.close()

# Check command line args
if len(sys.argv) < 3:
    print "Usage: %s <URL> <filename>" % sys.argv[0]
    raise SystemExit

# Launch a window with a statusbar
win = GtkDialog()
win.set_title("PycURL progress")
win.show()
vbox = GtkVBox(spacing=5)
vbox.set_border_width(10)
win.vbox.pack_start(vbox)
vbox.show()
label = GtkLabel("Downloading %s" % sys.argv[1])
label.set_alignment(0, 0.5)
vbox.pack_start(label, expand=FALSE)
label.show()
pbar = GtkProgressBar()
pbar.set_usize(200, 20)
vbox.pack_start(pbar)
pbar.show()
win.connect("destroy", close_app)
win.connect("delete_event", close_app)

# Start thread for fetching url
f = open(sys.argv[2], 'w')
round = 0.0
t = Test(sys.argv[1], f)
t.start()

# Start GTK mainloop
threads_enter()
mainloop()
threads_leave()
<commit_msg>- removed some unecessary code<commit_after># $Id: test_gtk.py,v 1.9 2001/08/21 10:10:25 kjetilja Exp $

## System modules
import sys, threading
from gtk import *

## PycURL module
import pycurl

def progress(download_t, download_d, upload_t, upload_d):
    global round, pbar
    threads_enter()
    if download_t == 0:
        pbar.set_activity_mode(1)
        round = round + 0.1
        if round >= 1.0:  round = 0.0
    else:
        pbar.set_activity_mode(0)
        round = float(download_d) / float(download_t)
    pbar.update(round)
    threads_leave()
    return 0 # Anything else indicates an error

def close_app(*args):
    global t, win
    win.destroy()
    mainquit()
    return TRUE

class Test(threading.Thread):

    def __init__(self, url, target_file):
        threading.Thread.__init__(self)
        self.target_file = target_file
        self.curl = pycurl.init()
        self.curl.setopt(pycurl.URL, url)
        self.curl.setopt(pycurl.FILE, target_file)
        self.curl.setopt(pycurl.FOLLOWLOCATION, 1)
        self.curl.setopt(pycurl.NOPROGRESS, 0)
        self.curl.setopt(pycurl.PROGRESSFUNCTION, progress)
        self.curl.setopt(pycurl.MAXREDIRS, 5)

    def run(self):
        self.curl.perform()
        self.curl.cleanup()
        self.target_file.close()

# Check command line args
if len(sys.argv) < 3:
    print "Usage: %s <URL> <filename>" % sys.argv[0]
    raise SystemExit

# Launch a window with a statusbar
win = GtkDialog()
win.set_title("PycURL progress")
win.show()
vbox = GtkVBox(spacing=5)
vbox.set_border_width(10)
win.vbox.pack_start(vbox)
vbox.show()
label = GtkLabel("Downloading %s" % sys.argv[1])
label.set_alignment(0, 0.5)
vbox.pack_start(label, expand=FALSE)
label.show()
pbar = GtkProgressBar()
pbar.set_usize(200, 20)
vbox.pack_start(pbar)
pbar.show()
win.connect("destroy", close_app)
win.connect("delete_event", close_app)

# Start thread for fetching url
round = 0.0
t = Test(sys.argv[1], open(sys.argv[2], 'w'))
t.start()

# Start GTK mainloop
threads_enter()
mainloop()
threads_leave()
<|endoftext|>