<commit_before>import os
import contextlib
import unittest
import warnings
import flask

import flask_dotenv as dotenv

@contextlib.contextmanager
def capture():
    import sys
    import cStringIO
    oldout, olderr = sys.stdout, sys.stderr
    try:
        out = [cStringIO.StringIO(), cStringIO.StringIO()]
        sys.stdout, sys.stderr = out
        yield out
    finally:
        sys.stdout, sys.stderr = oldout, olderr
        out[0] = out[0].getvalue()
        out[1] = out[1].getvalue()

class DotEnvTestCase(unittest.TestCase):
    def setUp(self):
        self.app = flask.Flask(__name__)
        self.env = dotenv.DotEnv()

    def tearDown(self):
        config_keys = [
            'FOO',
            'SECRET_KEY',
            'DEVELOPMENT_DATABASE_URL',
            'TEST_DATABASE_URL',
            'DATABASE_URL',
            'BAR'
        ]
        for key in config_keys:
            if key in self.app.config:
                del self.app.config[key]

    def test_warning_if_env_file_is_missing(self):
        with warnings.catch_warnings(record=True) as w:
            self.env.init_app(self.app, "/does/not/exist/.env")
            self.assertEqual(
                "can't read /does/not/exist/.env - it doesn't exist",
                str(w[0].message)
            )
        self.assertFalse('FOO' in self.app.config)

    def test_read_default_env_file(self):
        self.env.init_app(self.app)
        self.assertTrue('FOO' in self.app.config)

    def test_read_specified_env_file(self):
        root_dir = os.path.dirname(os.path.abspath(__file__))
        self.env.init_app(self.app, os.path.join(root_dir, '.env.min'))
        self.assertTrue('BAR' in self.app.config)

    def test_loaded_value_dose_not_contain_double_quote(self):
        self.env.init_app(self.app)
        self.assertEqual(
            'postgresql://user:password@localhost/development',
            self.app.config['DEVELOPMENT_DATABASE_URL'])

    def test_loaded_value_dose_not_contain_single_quote(self):
        self.env.init_app(self.app)
        self.assertEqual(
            'postgresql://user:password@localhost/test',
            self.app.config['TEST_DATABASE_URL'])

    def test_loaded_value_can_contain_equal_signs(self):
        self.env.init_app(self.app)
        self.assertEqual(
            'postgresql://user:password@localhost/production?sslmode=require',
            self.app.config['DATABASE_URL'])

    def test_overwrite_an_existing_config_var(self):
        # flask has secret_key in default
        self.assertEqual(None, self.app.config['SECRET_KEY'])
        self.env.init_app(self.app)
        self.assertEqual(':)', self.app.config['SECRET_KEY'])

    def test_alias_sets_it_as_same_value(self):
        self.env.init_app(self.app)
        self.env.alias(maps={
            'TEST_DATABASE_URL': 'SQLALCHEMY_DATABASE_URL'
        })
        self.assertEqual(
            'postgresql://user:password@localhost/test',
            self.app.config['SQLALCHEMY_DATABASE_URL']
        )

    def test_init_app_assigns_app(self):
        self.env.init_app(self.app)
        self.assertEqual(self.app, self.env.app)

    def test_init_app_assigns_default_verbose_mode(self):
        self.env.init_app(self.app)
        self.assertFalse(self.env.verbose_mode)

    def test_import_vars_raises_in_env_file_does_not_exist(self):
        try:
            FileNotFoundError
        except NameError:
            # python 2
            FileNotFoundError = IOError
        with self.assertRaises(FileNotFoundError) as e:
            self.env._DotEnv__import_vars('/does/not/exist/.env')
        self.assertEqual(e.exception.strerror, 'No such file or directory')

    def test_import_vars_will_output_logs_in_vobose_mode(self):
        with capture() as out:
            self.env.app = self.app
            self.env.verbose_mode = True
            root_dir = os.path.dirname(os.path.abspath(__file__))
            self.env._DotEnv__import_vars(os.path.join(root_dir, '.env.min'))
        # flask has secret_key in default
        self.assertIn(
            ' * Setting an entirely new config var: BAR\n'
            ' * Overwriting an existing config var: SECRET_KEY\n',
            out
        )

    def test_alias_will_output_log_in_vobose_mode(self):
        with capture() as out:
            self.env.init_app(self.app)
            self.env.verbose_mode = True
            self.env.alias(maps={
                'TEST_DATABASE_URL': 'SQLALCHEMY_DATABASE_URL'
            })
        self.assertIn(
            ' * Mapping a specified var as a alias:'
            ' SQLALCHEMY_DATABASE_URL => TEST_DATABASE_URL\n',
            out
        )

if __name__ == '__main__':
    unittest.main()
<commit_msg>Fix StringIO consistency problem<commit_after>import os
import contextlib
import unittest
import warnings
import flask

import flask_dotenv as dotenv

@contextlib.contextmanager
def capture():
    import sys
    try:
        # python 2
        from cStringIO import StringIO
    except ImportError:
        from io import StringIO
    oldout, olderr = sys.stdout, sys.stderr
    try:
        out = [StringIO(), StringIO()]
        sys.stdout, sys.stderr = out
        yield out
    finally:
        sys.stdout, sys.stderr = oldout, olderr
        out[0] = out[0].getvalue()
        out[1] = out[1].getvalue()

class DotEnvTestCase(unittest.TestCase):
    def setUp(self):
        self.app = flask.Flask(__name__)
        self.env = dotenv.DotEnv()

    def tearDown(self):
        config_keys = [
            'FOO',
            'SECRET_KEY',
            'DEVELOPMENT_DATABASE_URL',
            'TEST_DATABASE_URL',
            'DATABASE_URL',
            'BAR'
        ]
        for key in config_keys:
            if key in self.app.config:
                del self.app.config[key]

    def test_warning_if_env_file_is_missing(self):
        with warnings.catch_warnings(record=True) as w:
            self.env.init_app(self.app, "/does/not/exist/.env")
            self.assertEqual(
                "can't read /does/not/exist/.env - it doesn't exist",
                str(w[0].message)
            )
        self.assertFalse('FOO' in self.app.config)

    def test_read_default_env_file(self):
        self.env.init_app(self.app)
        self.assertTrue('FOO' in self.app.config)

    def test_read_specified_env_file(self):
        root_dir = os.path.dirname(os.path.abspath(__file__))
        self.env.init_app(self.app, os.path.join(root_dir, '.env.min'))
        self.assertTrue('BAR' in self.app.config)

    def test_loaded_value_dose_not_contain_double_quote(self):
        self.env.init_app(self.app)
        self.assertEqual(
            'postgresql://user:password@localhost/development',
            self.app.config['DEVELOPMENT_DATABASE_URL'])

    def test_loaded_value_dose_not_contain_single_quote(self):
        self.env.init_app(self.app)
        self.assertEqual(
            'postgresql://user:password@localhost/test',
            self.app.config['TEST_DATABASE_URL'])

    def test_loaded_value_can_contain_equal_signs(self):
        self.env.init_app(self.app)
        self.assertEqual(
            'postgresql://user:password@localhost/production?sslmode=require',
            self.app.config['DATABASE_URL'])

    def test_overwrite_an_existing_config_var(self):
        # flask has secret_key in default
        self.assertEqual(None, self.app.config['SECRET_KEY'])
        self.env.init_app(self.app)
        self.assertEqual(':)', self.app.config['SECRET_KEY'])

    def test_alias_sets_it_as_same_value(self):
        self.env.init_app(self.app)
        self.env.alias(maps={
            'TEST_DATABASE_URL': 'SQLALCHEMY_DATABASE_URL'
        })
        self.assertEqual(
            'postgresql://user:password@localhost/test',
            self.app.config['SQLALCHEMY_DATABASE_URL']
        )

    def test_init_app_assigns_app(self):
        self.env.init_app(self.app)
        self.assertEqual(self.app, self.env.app)

    def test_init_app_assigns_default_verbose_mode(self):
        self.env.init_app(self.app)
        self.assertFalse(self.env.verbose_mode)

    def test_import_vars_raises_in_env_file_does_not_exist(self):
        try:
            FileNotFoundError
        except NameError:
            # python 2
            FileNotFoundError = IOError
        with self.assertRaises(FileNotFoundError) as e:
            self.env._DotEnv__import_vars('/does/not/exist/.env')
        self.assertEqual(e.exception.strerror, 'No such file or directory')

    def test_import_vars_will_output_logs_in_vobose_mode(self):
        with capture() as out:
            self.env.app = self.app
            self.env.verbose_mode = True
            root_dir = os.path.dirname(os.path.abspath(__file__))
            self.env._DotEnv__import_vars(os.path.join(root_dir, '.env.min'))
        # flask has secret_key in default
        self.assertIn(
            ' * Setting an entirely new config var: BAR\n'
            ' * Overwriting an existing config var: SECRET_KEY\n',
            out
        )

    def test_alias_will_output_log_in_vobose_mode(self):
        with capture() as out:
            self.env.init_app(self.app)
            self.env.verbose_mode = True
            self.env.alias(maps={
                'TEST_DATABASE_URL': 'SQLALCHEMY_DATABASE_URL'
            })
        self.assertIn(
            ' * Mapping a specified var as a alias:'
            ' SQLALCHEMY_DATABASE_URL => TEST_DATABASE_URL\n',
            out
        )

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from __future__ import absolute_import, division, print_function

from libtbx.test_utils.pytest import discover

tst_list = discover()

# To write tests for dials:

# 1. Test file should be named test_*.py
# 2. Test methods should be named test_*()
# 3. Nothing else needed. Rest happens by magic.

# To run dials tests:

# run 'pytest' inside dials directory

# For more information see:
#   https://github.com/dials/dials/wiki/pytest
<commit_msg>disable libtbx testing for DIALS<commit_after>from __future__ import absolute_import, division, print_function

exit(
    """
To run dials tests please run 'pytest' inside the dials directory

For more information see:
  https://github.com/dials/dials/wiki/pytest
"""
)
<|endoftext|>


<commit_before>#!/usr/bin/env python3
#Will execute the unit tests manually and provide a better report than tox
import subprocess
import os
from pprint import pprint
import argparse
PYS = ['3.3', '3.4']
SOLRS = ['5.2.1', '5.5.3', '6.1.0', '6.2.1']
TESTS = ['test_client', 'test_indexq', 'test_reindexer', 'test_resp',
         'test_collections', 'test_zk']

parser = argparse.ArgumentParser()
parser.add_argument('-test', type=str, nargs='*', help='Which test.')
parser.add_argument('-solr', type=str, nargs='*', help='Which Solr.')
parser.add_argument('-py', type=str, nargs='*', help='PY')
args = parser.parse_args()

if args.test:
    TESTS = args.test
if args.solr:
    SOLRS = args.solr
if args.py:
    PYS = args.py

print("Starting Testing")
REPORT = {}
for ver in PYS:
    res = subprocess.call(["python3", "/usr/local/bin/tox", "-e", "py{}".format(ver.replace('.',''))])
    python = '.tox/py{}/bin/python{}'.format(ver.replace('.', ''), ver)
    subprocess.call([".tox/py{}/bin/pip".format(ver.replace('.', '')), "install", "-r", "requirements.txt"])
    REPORT[ver] = {}
    for solr in SOLRS:
        REPORT[ver][solr] = {}
        os.environ['SOLR_TEST_URL'] = 'http://localhost:9{}/solr/'.format(solr.replace('.',''))
        for test in TESTS:
            print("Running: {}- {}- {}".format(ver, solr, test))
            REPORT[ver][solr][test] = subprocess.call([python, "-m", "unittest", "test.{}".format(test)])
pprint(REPORT)
<commit_msg>pep8 run_tests.py<commit_after>#!/usr/bin/env python3
# Will execute the unit tests manually and provide a better report than tox
import subprocess
import os
from pprint import pprint
import argparse

PYS = ['3.3', '3.4']
SOLRS = ['5.2.1', '5.5.3', '6.1.0', '6.2.1']
TESTS = ['test_client', 'test_indexq', 'test_reindexer', 'test_resp',
         'test_collections', 'test_zk']

parser = argparse.ArgumentParser()
parser.add_argument('-test', type=str, nargs='*', help='Which test.')
parser.add_argument('-solr', type=str, nargs='*', help='Which Solr.')
parser.add_argument('-py', type=str, nargs='*', help='PY')
args = parser.parse_args()

if args.test:
    TESTS = args.test
if args.solr:
    SOLRS = args.solr
if args.py:
    PYS = args.py

print("Starting Testing")
REPORT = {}
for ver in PYS:
    res = subprocess.call(["python3", "/usr/local/bin/tox", "-e", "py{}".format(ver.replace('.', ''))])
    python = '.tox/py{}/bin/python{}'.format(ver.replace('.', ''), ver)
    subprocess.call([".tox/py{}/bin/pip".format(ver.replace('.', '')), "install", "-r", "requirements.txt"])
    REPORT[ver] = {}
    for solr in SOLRS:
        REPORT[ver][solr] = {}
        os.environ['SOLR_TEST_URL'] = 'http://localhost:9{}/solr/'.format(solr.replace('.', ''))
        for test in TESTS:
            print("Running: {}- {}- {}".format(ver, solr, test))
            REPORT[ver][solr][test] = subprocess.call([python, "-m", "unittest", "test.{}".format(test)])
pprint(REPORT)
<|endoftext|>


<commit_before>from devicehive.transports.base_transport import BaseTransport
import websocket
import threading
import time
import uuid

class WebsocketTransport(BaseTransport):
    """Websocket transport class."""

    def __init__(self, data_format_class, data_format_options, handler_class,
                 handler_options):
        BaseTransport.__init__(self, 'websocket', data_format_class,
                               data_format_options, handler_class,
                               handler_options)
        self._connect_thread = None
        self._websocket = websocket.WebSocket()
        self._pong_received = False
        self._obj_queue = []
        self._data_opcode = self._get_data_opcode()

    def _connect(self, url, options):
        timeout = options.get('timeout', None)
        pong_timeout = options.get('pong_timeout', None)
        self._websocket.connect(url, **options)
        self._websocket.settimeout(timeout)
        self._connected = True
        self._call_handler_method('handle_connected')
        send_ping_thread = None
        if pong_timeout:
            send_ping_thread = threading.Thread(target=self._send_ping,
                                                args=(pong_timeout,))
            send_ping_thread.daemon = True
            send_ping_thread.start()
        self._receive_data()
        self._close()
        if pong_timeout:
            send_ping_thread.join()

    def _send_ping(self, pong_timeout):
        while self._connected:
            self._websocket.ping()
            self._pong_received = False
            time.sleep(pong_timeout)
            if not self._pong_received:
                self._connected = False
                return

    def _receive_data(self):
        while self._connected:
            if len(self._obj_queue):
                obj = self._obj_queue.pop(0)
                self._call_handler_method('handle_event', obj)
                continue
            opcode, frame = self._websocket.recv_data_frame(True)
            if opcode == websocket.ABNF.OPCODE_TEXT:
                obj = self._decode_data(frame.data.decode('utf-8'))
                self._call_handler_method('handle_event', obj)
                continue
            if opcode == websocket.ABNF.OPCODE_BINARY:
                obj = self._decode_data(frame.data)
                self._call_handler_method('handle_event', obj)
                continue
            if opcode == websocket.ABNF.OPCODE_PONG:
                self._pong_received = True
                continue
            if opcode == websocket.ABNF.OPCODE_CLOSE:
                return

    def _close(self):
        self._pong_received = False
        self._obj_queue = []
        self._websocket.close()
        self._call_handler_method('handle_closed')

    def _get_data_opcode(self):
        if self._data_type == 'text':
            return websocket.ABNF.OPCODE_TEXT
        return websocket.ABNF.OPCODE_BINARY

    def connect(self, url, **options):
        self._assert_not_connected()
        self._connect_thread = threading.Thread(target=self._connect,
                                                args=(url, options))
        self._connect_thread.daemon = True
        self._connect_thread.start()

    def send_request(self, action, obj, **params):
        self._assert_connected()
        obj_id = str(uuid.uuid1())
        obj['requestId'] = obj_id
        obj['action'] = action
        self._websocket.send(self._encode_obj(obj), opcode=self._data_opcode)
        return obj_id

    def request(self, action, obj, **params):
        timeout = params.get('timeout', 30)
        obj_id = self.send_request(action, obj, **params)
        send_time = time.time()
        while time.time() - timeout < send_time:
            obj = self._decode_data(self._websocket.recv())
            if obj.get('requestId') == obj_id:
                return obj
            self._obj_queue.append(obj)
        raise WebsocketTransportException('Object receive timeout occurred')

    def close(self):
        self._assert_connected()
        self._connected = False

    def join(self, timeout=None):
        self._connect_thread.join(timeout)

class WebsocketTransportException(websocket.WebSocketException):
    """Websocket transport exception."""
    pass
<commit_msg>Remove send ping thread join<commit_after>from devicehive.transports.base_transport import BaseTransport
import websocket
import threading
import time
import uuid

class WebsocketTransport(BaseTransport):
    """Websocket transport class."""

    def __init__(self, data_format_class, data_format_options, handler_class,
                 handler_options):
        BaseTransport.__init__(self, 'websocket', data_format_class,
                               data_format_options, handler_class,
                               handler_options)
        self._connect_thread = None
        self._websocket = websocket.WebSocket()
        self._pong_received = False
        self._obj_queue = []
        self._data_opcode = self._get_data_opcode()

    def _connect(self, url, options):
        timeout = options.get('timeout', None)
        pong_timeout = options.get('pong_timeout', None)
        self._websocket.connect(url, **options)
        self._websocket.settimeout(timeout)
        self._connected = True
        self._call_handler_method('handle_connected')
        if pong_timeout:
            send_ping_thread = threading.Thread(target=self._send_ping,
                                                args=(pong_timeout,))
            send_ping_thread.daemon = True
            send_ping_thread.start()
        self._receive_data()
        self._close()

    def _send_ping(self, pong_timeout):
        while self._connected:
            self._websocket.ping()
            self._pong_received = False
            time.sleep(pong_timeout)
            if not self._pong_received:
                self._connected = False
                return

    def _receive_data(self):
        while self._connected:
            if len(self._obj_queue):
                obj = self._obj_queue.pop(0)
                self._call_handler_method('handle_event', obj)
                continue
            opcode, frame = self._websocket.recv_data_frame(True)
            if opcode == websocket.ABNF.OPCODE_TEXT:
                obj = self._decode_data(frame.data.decode('utf-8'))
                self._call_handler_method('handle_event', obj)
                continue
            if opcode == websocket.ABNF.OPCODE_BINARY:
                obj = self._decode_data(frame.data)
                self._call_handler_method('handle_event', obj)
                continue
            if opcode == websocket.ABNF.OPCODE_PONG:
                self._pong_received = True
                continue
            if opcode == websocket.ABNF.OPCODE_CLOSE:
                return

    def _close(self):
        self._pong_received = False
        self._obj_queue = []
        self._websocket.close()
        self._call_handler_method('handle_closed')

    def _get_data_opcode(self):
        if self._data_type == 'text':
            return websocket.ABNF.OPCODE_TEXT
        return websocket.ABNF.OPCODE_BINARY

    def connect(self, url, **options):
        self._assert_not_connected()
        self._connect_thread = threading.Thread(target=self._connect,
                                                args=(url, options))
        self._connect_thread.daemon = True
        self._connect_thread.start()

    def send_request(self, action, obj, **params):
        self._assert_connected()
        obj_id = str(uuid.uuid1())
        obj['requestId'] = obj_id
        obj['action'] = action
        self._websocket.send(self._encode_obj(obj), opcode=self._data_opcode)
        return obj_id

    def request(self, action, obj, **params):
        timeout = params.get('timeout', 30)
        obj_id = self.send_request(action, obj, **params)
        send_time = time.time()
        while time.time() - timeout < send_time:
            obj = self._decode_data(self._websocket.recv())
            if obj.get('requestId') == obj_id:
                return obj
            self._obj_queue.append(obj)
        raise WebsocketTransportException('Object receive timeout occurred')

    def close(self):
        self._assert_connected()
        self._connected = False

    def join(self, timeout=None):
        self._connect_thread.join(timeout)

class WebsocketTransportException(websocket.WebSocketException):
    """Websocket transport exception."""
    pass
<|endoftext|>


<commit_before>#
# run all the regression tests
#
# [or, you can request specific ones by giving their names on the command line]
#

import sys
import compile
import os
import subprocess

def run_test (cmd, *args):
    cmd = PJ ('tests', cmd)
    p = subprocess.Popen ([cmd] + list(args), stdout=subprocess.PIPE)
    out = p.stdout.read()
    return out

def test_t17():
    lines = run_test ('t17').split ('\n')
    # we can't say anything about the address returned by malloc
    # but we should expect to read this number
    assert (lines[1] == '3141')
    # and the sizeof (pxll_int) is random too, since even on the same
    #  platform we might compile 32 or 64 bit, so just ignore it.

def test_t_dump_image():
    # generate the output
    run_test ('t_dump_image')
    # load the image and run it
    exp0 = run_test ('t_dump_image','-l')
    exp1 = open ('tests/t_dump_image.exp').read()
    assert (exp0 == exp1)

def test_t21():
    out = run_test ('t21')
    exp = open ('gc.c').read()
    # make sure the first part matches the contents of gc.c
    assert (out[:len(exp)] == exp)
    # the chars are too hard to tests for, and unlikely to be wrong.
    # should really make a separate char test.

def test_t22():
    out = run_test ('t22')
    lines = out.split ('\n')
    assert (lines[0].count ('<closure pc=') == 5)
    r6 = [ str(x) for x in range (6) ]
    assert (lines[1:] == (r6 + r6 + ['#u', '']))

def test_t_lex():
    out = run_test ('t_lex')
    assert (out.split ('\n')[-3] == '{u0 newline "\\0x0a"}')

PJ = os.path.join

if len(sys.argv) > 1:
    # run only these specific tests
    files = [x + '.scm' for x in sys.argv[1:]]
else:
    files = os.listdir ('tests')

# When looking for things that are broken, I prefer to work with the smallest
#  test that reproduces a problem.  Thus, run the tests in source-size order...
files = [ (os.stat(PJ ('tests', x)).st_size, x) for x in files ]
files.sort()

# tests that need special handling
special = [x[5:] for x in dir() if x.startswith ('test_')]

for size, file in files:
    if file.endswith ('.scm'):
        base, ext = os.path.splitext (file)
        path = os.path.join ('tests', file)
        print 'compiling', path
        fail = file.startswith ('f')
        try:
            compile.compile_file (open (path, 'rb'), path)
        except:
            if not fail:
                raise
        else:
            if fail:
                raise ValueError ("oops - expected compilation to fail")
            compile.cc (path, optimize=False)
            if base not in special:
                out = run_test (base)
                exp_path = PJ ('tests', base + '.exp')
                if os.path.isfile (exp_path):
                    exp = open (exp_path).read()
                    if out != exp:
                        raise ValueError ("oops - output didn't match on test '%s'" % (base,))
            else:
                # tests that require special handling for whatever reason.
                eval ('test_%s()' % (base,))
<commit_msg>sometimes the lexer gets swapped out<commit_after>#
# run all the regression tests
#
# [or, you can request specific ones by giving their names on the command line]
#

import sys
import compile
import os
import subprocess

def run_test (cmd, *args):
    cmd = PJ ('tests', cmd)
    p = subprocess.Popen ([cmd] + list(args), stdout=subprocess.PIPE)
    out = p.stdout.read()
    return out

def test_t17():
    lines = run_test ('t17').split ('\n')
    # we can't say anything about the address returned by malloc
    # but we should expect to read this number
    assert (lines[1] == '3141')
    # and the sizeof (pxll_int) is random too, since even on the same
    #  platform we might compile 32 or 64 bit, so just ignore it.

def test_t_dump_image():
    # generate the output
    run_test ('t_dump_image')
    # load the image and run it
    exp0 = run_test ('t_dump_image','-l')
    exp1 = open ('tests/t_dump_image.exp').read()
    assert (exp0 == exp1)

def test_t21():
    out = run_test ('t21')
    exp = open ('gc.c').read()
    # make sure the first part matches the contents of gc.c
    assert (out[:len(exp)] == exp)
    # the chars are too hard to tests for, and unlikely to be wrong.
    # should really make a separate char test.

def test_t22():
    out = run_test ('t22')
    lines = out.split ('\n')
    assert (lines[0].count ('<closure pc=') == 5)
    r6 = [ str(x) for x in range (6) ]
    assert (lines[1:] == (r6 + r6 + ['#u', '']))

def test_t_lex():
    out = run_test ('t_lex')
    assert (out.split ('\n')[-3].lower() == '{u0 newline "\\0x0a"}')

PJ = os.path.join

if len(sys.argv) > 1:
    # run only these specific tests
    files = [x + '.scm' for x in sys.argv[1:]]
else:
    files = os.listdir ('tests')

# When looking for things that are broken, I prefer to work with the smallest
#  test that reproduces a problem.  Thus, run the tests in source-size order...
files = [ (os.stat(PJ ('tests', x)).st_size, x) for x in files ]
files.sort()

# tests that need special handling
special = [x[5:] for x in dir() if x.startswith ('test_')]

for size, file in files:
    if file.endswith ('.scm'):
        base, ext = os.path.splitext (file)
        path = os.path.join ('tests', file)
        print 'compiling', path
        fail = file.startswith ('f')
        try:
            compile.compile_file (open (path, 'rb'), path)
        except:
            if not fail:
                raise
        else:
            if fail:
                raise ValueError ("oops - expected compilation to fail")
            compile.cc (path, optimize=False)
            if base not in special:
                out = run_test (base)
                exp_path = PJ ('tests', base + '.exp')
                if os.path.isfile (exp_path):
                    exp = open (exp_path).read()
                    if out != exp:
                        raise ValueError ("oops - output didn't match on test '%s'" % (base,))
            else:
                # tests that require special handling for whatever reason.
                eval ('test_%s()' % (base,))
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Driver script for testing TuLiP.  Try calling it with "-h" flag.
"""

import imp
import sys
import os.path
from os import walk
import argparse

import nose

class ArgParser(argparse.ArgumentParser):
    def error(self, message):
        sys.stderr.write('error: %s\n' % message)
        self.print_help()
        sys.exit(1)

testfiles_help = (
"""
space-separated list of test file names,
where the suffix "_test.py" is added to
each given name.  E.g.,

    run_tests.py gr1cint

causes the gr1cint_test.py file to be used
and no others.

If no arguments are given,
then default is to run all tests.

If TESTFILES... each have a prefix of "-",
then all tests *except* those listed will be run.
Use the pseudoargument -- in this case. E.g.,

    run_tests.py --fast -- -abstract -synth

Besides what is below, OPTIONS... are passed on to nose."""
)

if __name__ == "__main__":
    parser = ArgParser(formatter_class=argparse.RawTextHelpFormatter)

    parser.add_argument('testfiles', nargs='*', default=None,
                        help=testfiles_help)

    parser.add_argument('--fast', action='store_true',
                        help='exclude tests that are marked as slow')
    parser.add_argument('--cover', action='store_true',
                        help='generate a coverage report')
    parser.add_argument('--outofsource', action='store_true',
                        help='import tulip from outside the current directory')
    parser.add_argument(
        '-w', '--where', dest='dir', default='tests',
        help='search for tests in directory WHERE\n'
             '(this is exactly the "-w" or "--where" option of nose)')

    args, unknown_args = parser.parse_known_args()

    basenames = args.testfiles
    skip_slow = args.fast
    measure_coverage = args.cover
    require_nonlocaldir_tulip = args.outofsource
    tests_dir = args.dir

    if require_nonlocaldir_tulip:
        # Scrub local directory from search path for modules
        import os
        try:
            while True:
                sys.path.remove('')
        except ValueError:
            pass
        try:
            while True:
                sys.path.remove(os.path.abspath(os.curdir))
        except ValueError:
            pass
    try:
        modtuple = imp.find_module("tulip", sys.path)
        imp.load_module("tulip", *modtuple)
    except ImportError:
        if require_nonlocaldir_tulip:
            raise ImportError('tulip package not found, '
                              'besides in the local directory')
        else:
            raise()

    argv = ["nosetests"]
    if skip_slow:
        argv.append("--attr=!slow")

    if measure_coverage:
        argv.extend(["--with-coverage", "--cover-html", "--cover-package=tulip"])

    available = []
    for dirpath, dirnames, filenames in walk(tests_dir):
        available.extend(filenames)
        break

    testfiles = []
    excludefiles = []
    for basename in basenames:
        match = [f for f in available
                 if f.startswith(basename) and f.endswith('.py')]

        if len(match) > 1:
            raise Exception('ambiguous base name: %s, matches: %s' %
                            (basename, match))
        elif match[0].endswith('_test.py'):
            testfiles.append(match[0])
            continue

        if os.path.exists(os.path.join(tests_dir, basename + '_test.py')):
            testfiles.append(basename + '_test.py')
        elif basename[0] == '-':
            if os.path.exists(os.path.join(tests_dir, basename[1:] + "_test.py")):
                excludefiles.append(basename[1:] + "_test.py")
            else:
                argv.append(basename)
        else:
            argv.append(basename)

    if testfiles and excludefiles:
        print("You can specify files to exclude or include, but not both.")
        print("Try calling it with \"-h\" flag.")
        exit(1)

    if excludefiles:
        argv.append("--exclude=" + "|".join(excludefiles))

    argv.extend(testfiles)
    argv += ["--where=" + tests_dir]
    argv.extend(unknown_args)

    print('calling nose')
    nose.main(argv=argv + ["--verbosity=3", "--exe"])
<commit_msg>TRIV: Delete trailing whitespace<commit_after>#!/usr/bin/env python
"""
Driver script for testing TuLiP.  Try calling it with "-h" flag.
"""

import imp
import sys
import os.path
from os import walk
import argparse

import nose

class ArgParser(argparse.ArgumentParser):
    def error(self, message):
        sys.stderr.write('error: %s\n' % message)
        self.print_help()
        sys.exit(1)

testfiles_help = (
"""
space-separated list of test file names,
where the suffix "_test.py" is added to
each given name.  E.g.,

    run_tests.py gr1cint

causes the gr1cint_test.py file to be used
and no others.

If no arguments are given,
then default is to run all tests.

If TESTFILES... each have a prefix of "-",
then all tests *except* those listed will be run.
Use the pseudoargument -- in this case. E.g.,

    run_tests.py --fast -- -abstract -synth

Besides what is below, OPTIONS... are passed on to nose."""
)

if __name__ == "__main__":
    parser = ArgParser(formatter_class=argparse.RawTextHelpFormatter)

    parser.add_argument('testfiles', nargs='*', default=None,
                        help=testfiles_help)

    parser.add_argument('--fast', action='store_true',
                        help='exclude tests that are marked as slow')
    parser.add_argument('--cover', action='store_true',
                        help='generate a coverage report')
    parser.add_argument('--outofsource', action='store_true',
                        help='import tulip from outside the current directory')
    parser.add_argument(
        '-w', '--where', dest='dir', default='tests',
        help='search for tests in directory WHERE\n'
             '(this is exactly the "-w" or "--where" option of nose)')

    args, unknown_args = parser.parse_known_args()

    basenames = args.testfiles
    skip_slow = args.fast
    measure_coverage = args.cover
    require_nonlocaldir_tulip = args.outofsource
    tests_dir = args.dir

    if require_nonlocaldir_tulip:
        # Scrub local directory from search path for modules
        import os
        try:
            while True:
                sys.path.remove('')
        except ValueError:
            pass
        try:
            while True:
                sys.path.remove(os.path.abspath(os.curdir))
        except ValueError:
            pass
    try:
        modtuple = imp.find_module("tulip", sys.path)
        imp.load_module("tulip", *modtuple)
    except ImportError:
        if require_nonlocaldir_tulip:
            raise ImportError('tulip package not found, '
                              'besides in the local directory')
        else:
            raise()

    argv = ["nosetests"]
    if skip_slow:
        argv.append("--attr=!slow")

    if measure_coverage:
        argv.extend(["--with-coverage", "--cover-html", "--cover-package=tulip"])

    available = []
    for dirpath, dirnames, filenames in walk(tests_dir):
        available.extend(filenames)
        break

    testfiles = []
    excludefiles = []
    for basename in basenames:
        match = [f for f in available
                 if f.startswith(basename) and f.endswith('.py')]

        if len(match) > 1:
            raise Exception('ambiguous base name: %s, matches: %s' %
                            (basename, match))
        elif match[0].endswith('_test.py'):
            testfiles.append(match[0])
            continue

        if os.path.exists(os.path.join(tests_dir, basename + '_test.py')):
            testfiles.append(basename + '_test.py')
        elif basename[0] == '-':
            if os.path.exists(os.path.join(tests_dir, basename[1:] + "_test.py")):
                excludefiles.append(basename[1:] + "_test.py")
            else:
                argv.append(basename)
        else:
            argv.append(basename)

    if testfiles and excludefiles:
        print("You can specify files to exclude or include, but not both.")
        print("Try calling it with \"-h\" flag.")
        exit(1)

    if excludefiles:
        argv.append("--exclude=" + "|".join(excludefiles))

    argv.extend(testfiles)
    argv += ["--where=" + tests_dir]
    argv.extend(unknown_args)

    print('calling nose')
    nose.main(argv=argv + ["--verbosity=3", "--exe"])
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
Notes on writing new test files:
--------------------------------

A test file can contain anything, but usually it is similar to a regular input
file (defining a test problem), with a mandatory Test class. This class holds
all the test_* functions, as well as the from_conf(), which serves to
initialize the test (conf is in fact the test file itself, options are
command-line options).

All variables defined in a test file are collected in 'conf' variable passed to
a Test.__init__(). For example, 'input_name' in test_input_*.py files is
accessible as 'conf.input_name'. This is usefull if the test class is defined
outside the test file, as the classes in tests_basic.py are.

The test_* functions are collected automatically by run_tests.py, with one
exception: if a certain order of their evaluation is required, a class
attribute 'test' of the Test class with a list of the test function names
should be defined (example: test_meshio.py)."""

import sys
import time
import os
import os.path as op
from optparse import OptionParser

import sfepy
from sfepy.base.conf import ProblemConf, get_standard_keywords

class OutputFilter(object):

    def __init__(self, allowed_lines):
        self.allowed_lines = allowed_lines
        self.msg_type1 = ['...', '!!!', '+++', '---']
        self.msg_type2 = ['<<<', '>>>']
        self.start()

    def start(self):
        self.stdout = sys.stdout
        sys.stdout = self

    def write(self, msg):
        if self.stdout is not None:
            msg_type = msg[:3]
            if msg_type in self.allowed_lines:
                if msg_type in self.msg_type1:
                    msg = ''.join((msg[:3], '  ', msg[3:]))
                elif msg_type in self.msg_type2:
                    msg = msg[4:]
                self.stdout.write(msg)
                self.stdout.write('\n')

    def stop(self):
        sys.stdout = self.stdout
        self.stdout = None

def run_test(conf_name, options):
    try:
        os.makedirs(options.out_dir)
    except OSError, e:
        if e.errno != 17: # [Errno 17] File exists
            raise

    if options.filter_none or options.debug:
        of = None
    elif options.filter_less:
        of = OutputFilter(['<<<', '>>>', '...', '!!!', '+++', '---'])
    elif options.filter_more:
        of = OutputFilter(['+++', '---'])
    else:
        of = OutputFilter(['<<<', '+++', '---'])

    print '<<< %s' % conf_name

    _required, other = get_standard_keywords()
    required = ['Test']

    num = 1
    test_time = 0.0
    try:
        conf = ProblemConf.from_file(conf_name, required, _required + other)
        test = conf.funmod.Test.from_conf(conf, options)
        num = test.get_number()
        ok = True
        print '>>> test instance prepared (%d test(s))' % num
    except KeyboardInterrupt:
        print '>>> interrupted'
        sys.exit(0)
    except:
        print '--- test instance creation failed'
        if options.debug:
            raise
        ok, n_fail, n_total = False, num, num

    if ok:
        try:
            tt = time.clock()
            ok, n_fail, n_total = test.run(options.debug)
            test_time = time.clock() - tt
        except KeyboardInterrupt:
            print '>>> interrupted'
            sys.exit(0)
        except Exception, e:
            print '>>> %s' % e.__class__
            if options.debug:
                raise
            ok, n_fail, n_total = False, num, num

    if ok:
        print '>>> all passed in %.2f s' % test_time
    else:
        print '!!! %s test failed' % n_fail

    if of is not None:
        of.stop()

    return n_fail, n_total, test_time

def wrap_run_tests(options):
    def run_tests(stats, dir_name, filenames):
        filenames = [filename for filename in sorted(filenames)
                     if (len(filename) > 8) and
                     filename[:5] == 'test_' and filename[-3:] == '.py']

        print '<<< directory: %s, test files: %d' % (dir_name, len(filenames))

        for filename in filenames:
            conf_name = op.join(dir_name, filename)

            n_fail, n_total, test_time = run_test(conf_name, options)

            stats[0] += 1
            stats[1] += n_fail
            stats[2] += n_total
            stats[3] += test_time

    return run_tests

def get_dir(default):
    if sfepy.in_source_tree:
        out = default
    else:
        out = op.normpath(op.join(sfepy.data_dir, default))
    return out

usage = """%prog [options] [test_filename[ test_filename ...]]"""

help = {
    'dir' : 'directory with tests [default: %default]',
    'out_dir' : 'directory for storing test results and temporary files'
    ' [default: %default]',
    'debug' : 'raise silenced exceptions to see what was wrong',
    'filter-none' : 'do not filter any messages',
    'filter-less' : 'filter output (suppress all except test messages)',
    'filter-more' : 'filter output (suppress all except test result messages)',
    'print-doc' : 'print the docstring of this file (howto write new tests)',
}

def main():
    parser = OptionParser(usage=usage, version="%prog " + sfepy.__version__)
    parser.add_option("", "--print-doc",
                      action="store_true", dest="print_doc",
                      default=False, help=help['print-doc'])
    parser.add_option("-d", "--dir", metavar='directory',
                      action="store", dest="test_dir",
                      default=get_dir('tests'),
                      help=help['dir'])
    parser.add_option("-o", "--output", metavar='directory',
                      action="store", dest="out_dir",
                      default=get_dir('output-tests'),
                      help=help['out_dir'])
    parser.add_option("", "--debug",
                      action="store_true", dest="debug",
                      default=False, help=help['debug'])
    parser.add_option("", "--filter-none",
                      action="store_true", dest="filter_none",
                      default=False, help=help['filter-none'])
    parser.add_option("", "--filter-less",
                      action="store_true", dest="filter_less",
                      default=False, help=help['filter-less'])
    parser.add_option("", "--filter-more",
                      action="store_true", dest="filter_more",
                      default=False, help=help['filter-more'])
    options, args = parser.parse_args()

    if options.print_doc:
        print __doc__
        return

    run_tests = wrap_run_tests(options)
    stats = [0, 0, 0, 0.0]

    if len(args) >= 1:
        for test_filename in args:
            dirname, filename = op.split(test_filename)
            run_tests(stats, dirname, [filename])

    else:
        op.walk(options.test_dir, run_tests, stats)

    print '%d test file(s) executed in %.2f s, %d failure(s) of %d test(s)'\
          % (stats[0], stats[3], stats[1], stats[2])

if __name__ == '__main__':
    main()
<commit_msg>new flush() method in OutputFilter class - needed for multiprocessing<commit_after>#!/usr/bin/env python
"""
Notes on writing new test files:
--------------------------------

A test file can contain anything, but usually it is similar to a regular input
file (defining a test problem), with a mandatory Test class. This class holds
all the test_* functions, as well as the from_conf(), which serves to
initialize the test (conf is in fact the test file itself, options are
command-line options).

All variables defined in a test file are collected in 'conf' variable passed to
a Test.__init__(). For example, 'input_name' in test_input_*.py files is
accessible as 'conf.input_name'. This is usefull if the test class is defined
outside the test file, as the classes in tests_basic.py are.

The test_* functions are collected automatically by run_tests.py, with one
exception: if a certain order of their evaluation is required, a class
attribute 'test' of the Test class with a list of the test function names
should be defined (example: test_meshio.py)."""

import sys
import time
import os
import os.path as op
from optparse import OptionParser

import sfepy
from sfepy.base.conf import ProblemConf, get_standard_keywords

class OutputFilter(object):

    def __init__(self, allowed_lines):
        self.allowed_lines = allowed_lines
        self.msg_type1 = ['...', '!!!', '+++', '---']
        self.msg_type2 = ['<<<', '>>>']
        self.start()

    def start(self):
        self.stdout = sys.stdout
        sys.stdout = self

    def write(self, msg):
        if self.stdout is not None:
            msg_type = msg[:3]
            if msg_type in self.allowed_lines:
                if msg_type in self.msg_type1:
                    msg = ''.join((msg[:3], '  ', msg[3:]))
                elif msg_type in self.msg_type2:
                    msg = msg[4:]
                self.stdout.write(msg)
                self.stdout.write('\n')

    def flush(self):
        self.stdout.flush()

    def stop(self):
        sys.stdout = self.stdout
        self.stdout = None

def run_test(conf_name, options):
    try:
        os.makedirs(options.out_dir)
    except OSError, e:
        if e.errno != 17: # [Errno 17] File exists
            raise

    if options.filter_none or options.debug:
        of = None
    elif options.filter_less:
        of = OutputFilter(['<<<', '>>>', '...', '!!!', '+++', '---'])
    elif options.filter_more:
        of = OutputFilter(['+++', '---'])
    else:
        of = OutputFilter(['<<<', '+++', '---'])

    print '<<< %s' % conf_name

    _required, other = get_standard_keywords()
    required = ['Test']

    num = 1
    test_time = 0.0
    try:
        conf = ProblemConf.from_file(conf_name, required, _required + other)
        test = conf.funmod.Test.from_conf(conf, options)
        num = test.get_number()
        ok = True
        print '>>> test instance prepared (%d test(s))' % num
    except KeyboardInterrupt:
        print '>>> interrupted'
        sys.exit(0)
    except:
        print '--- test instance creation failed'
        if options.debug:
            raise
        ok, n_fail, n_total = False, num, num

    if ok:
        try:
            tt = time.clock()
            ok, n_fail, n_total = test.run(options.debug)
            test_time = time.clock() - tt
        except KeyboardInterrupt:
            print '>>> interrupted'
            sys.exit(0)
        except Exception, e:
            print '>>> %s' % e.__class__
            if options.debug:
                raise
            ok, n_fail, n_total = False, num, num

    if ok:
        print '>>> all passed in %.2f s' % test_time
    else:
        print '!!! %s test failed' % n_fail

    if of is not None:
        of.stop()

    return n_fail, n_total, test_time

def wrap_run_tests(options):
    def run_tests(stats, dir_name, filenames):
        filenames = [filename for filename in sorted(filenames)
                     if (len(filename) > 8) and
                     filename[:5] == 'test_' and filename[-3:] == '.py']

        print '<<< directory: %s, test files: %d' % (dir_name, len(filenames))

        for filename in filenames:
            conf_name = op.join(dir_name, filename)

            n_fail, n_total, test_time = run_test(conf_name, options)

            stats[0] += 1
            stats[1] += n_fail
            stats[2] += n_total
            stats[3] += test_time

    return run_tests

def get_dir(default):
    if sfepy.in_source_tree:
        out = default
    else:
        out = op.normpath(op.join(sfepy.data_dir, default))
    return out

usage = """%prog [options] [test_filename[ test_filename ...]]"""

help = {
    'dir' : 'directory with tests [default: %default]',
    'out_dir' : 'directory for storing test results and temporary files'
    ' [default: %default]',
    'debug' : 'raise silenced exceptions to see what was wrong',
    'filter-none' : 'do not filter any messages',
    'filter-less' : 'filter output (suppress all except test messages)',
    'filter-more' : 'filter output (suppress all except test result messages)',
    'print-doc' : 'print the docstring of this file (howto write new tests)',
}

def main():
    parser = OptionParser(usage=usage, version="%prog " + sfepy.__version__)
    parser.add_option("", "--print-doc",
                      action="store_true", dest="print_doc",
                      default=False, help=help['print-doc'])
    parser.add_option("-d", "--dir", metavar='directory',
                      action="store", dest="test_dir",
                      default=get_dir('tests'),
                      help=help['dir'])
    parser.add_option("-o", "--output", metavar='directory',
                      action="store", dest="out_dir",
                      default=get_dir('output-tests'),
                      help=help['out_dir'])
    parser.add_option("", "--debug",
                      action="store_true", dest="debug",
                      default=False, help=help['debug'])
    parser.add_option("", "--filter-none",
                      action="store_true", dest="filter_none",
                      default=False, help=help['filter-none'])
    parser.add_option("", "--filter-less",
                      action="store_true", dest="filter_less",
                      default=False, help=help['filter-less'])
    parser.add_option("", "--filter-more",
                      action="store_true", dest="filter_more",
                      default=False, help=help['filter-more'])
    options, args = parser.parse_args()

    if options.print_doc:
        print __doc__
        return

    run_tests = wrap_run_tests(options)
    stats = [0, 0, 0, 0.0]

    if len(args) >= 1:
        for test_filename in args:
            dirname, filename = op.split(test_filename)
            run_tests(stats, dirname, [filename])

    else:
        op.walk(options.test_dir, run_tests, stats)

    print '%d test file(s) executed in %.2f s, %d failure(s) of %d test(s)'\
          % (stats[0], stats[3], stats[1], stats[2])

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#! /usr/bin/env python2
#
# This file is part of khmer, http://github.com/ged-lab/khmer/, and is
# Copyright (C) Michigan State University, 2009-2013. It is licensed under
# the three-clause BSD license; see doc/LICENSE.txt.
# Contact: khmer-project@idyll.org
#
import sys
import screed

total_bp = 0
total_seqs = 0

output = []
for filename in sys.argv[1:]:
    bp = 0
    seqs = 0
    for record in screed.open(filename):
        if seqs % 100000 == 0:
            print >>sys.stderr, '...', filename, seqs
        bp += len(record.sequence)
        seqs += 1

    if seqs == 0:
        print >>sys.stderr, 'No sequences found in %s' % filename
    else:
        avg_len = bp / float(seqs)
        s = '%d bp / %d seqs; %.1f average length -- %s' % (bp,
                                                            seqs,
                                                            avg_len,
                                                            filename)
        print >>sys.stderr, '... found', s
        output.append(s)

        total_bp += bp
        total_seqs += seqs

if total_seqs == 0:
    print 'No sequences found in %d files' % len(sys.argv[1:])
else:
    print '---------------'
    print "\n".join(output)
    print '---------------'
    print '%d bp / %d seqs; %.1f average length -- total' %
        (total_bp, total_seqs, total_bp / float(total_seqs))
<commit_msg>Syntax error<commit_after>#! /usr/bin/env python2
#
# This file is part of khmer, http://github.com/ged-lab/khmer/, and is
# Copyright (C) Michigan State University, 2009-2013. It is licensed under
# the three-clause BSD license; see doc/LICENSE.txt.
# Contact: khmer-project@idyll.org
#
import sys
import screed

total_bp = 0
total_seqs = 0

output = []
for filename in sys.argv[1:]:
    bp = 0
    seqs = 0
    for record in screed.open(filename):
        if seqs % 100000 == 0:
            print >>sys.stderr, '...', filename, seqs
        bp += len(record.sequence)
        seqs += 1

    if seqs == 0:
        print >>sys.stderr, 'No sequences found in %s' % filename
    else:
        avg_len = bp / float(seqs)
        s = '%d bp / %d seqs; %.1f average length -- %s' % (bp,
                                                            seqs,
                                                            avg_len,
                                                            filename)
        print >>sys.stderr, '... found', s
        output.append(s)

        total_bp += bp
        total_seqs += seqs

if total_seqs == 0:
    print 'No sequences found in %d files' % len(sys.argv[1:])
else:
    print '---------------'
    print "\n".join(output)
    print '---------------'
    print '%d bp / %d seqs; %.1f average length -- total'.format(
        total_bp, total_seqs, total_bp / float(total_seqs))
<|endoftext|>


<commit_before>import os
import unittest

from JSOV import generator

class Run_Tests(unittest.TestCase):
	INPUT_JSON = "tests/input/sample.json"
	INPUT_TEMPLATE = "tests/input/template.jsov"
	OUTPUT_HTML = "tests/output/output.html"
	OUTPUT_CSS = "tests/output/style.css"
	GENERATED_DIR = "tests/generated/"
	GENERATED_HTML = "tests/generated/output.html"
	GENERATED_CSS = "tests/generated/style.css"

	def test_input_json_exists(self):
		self.assertTrue(os.path.exists(self.INPUT_JSON))

	def test_input_jsov_exists(self):
		self.assertTrue(os.path.exists(self.INPUT_TEMPLATE))

	def test_output_html_exists(self):
		self.assertTrue(os.path.exists(self.OUTPUT_HTML))

	def test_output_css_exists(self):
		self.assertTrue(os.path.exists(self.OUTPUT_CSS))

	def test_generated_dir_exists(self):
		self.assertTrue(os.path.exists(self.GENERATED_DIR))

if __name__ == "__main__":
	unittest.main()
<commit_msg>Added create_instance() to run_tests.py<commit_after>import os
import unittest

from JSOV import generator

class Run_Tests(unittest.TestCase):
	INPUT_JSON = "tests/input/sample.json"
	INPUT_TEMPLATE = "tests/input/template.jsov"
	OUTPUT_HTML = "tests/output/output.html"
	OUTPUT_CSS = "tests/output/style.css"
	GENERATED_DIR = "tests/generated/"
	GENERATED_HTML = "tests/generated/output.html"
	GENERATED_CSS = "tests/generated/style.css"

	def create_instance(self):
		self.visualizer = generator.Generator(self.INPUT_JSON, self.INPUT_TEMPLATE)

	def test_input_json_exists(self):
		self.assertTrue(os.path.exists(self.INPUT_JSON))

	def test_input_jsov_exists(self):
		self.assertTrue(os.path.exists(self.INPUT_TEMPLATE))

	def test_output_html_exists(self):
		self.assertTrue(os.path.exists(self.OUTPUT_HTML))

	def test_output_css_exists(self):
		self.assertTrue(os.path.exists(self.OUTPUT_CSS))

	def test_generated_dir_exists(self):
		self.assertTrue(os.path.exists(self.GENERATED_DIR))

if __name__ == "__main__":
	unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# hily 2018-05-06 M.Kolodin 2018-05-08 0.11
# Highly Likely Project

# ver. 0.11. it must scan folders in hily.ini and print result (same files) into file hily.out.

# 1st ver. will work with sorted list of tiples
# 2nd ver. will use databases
#import sqlite3

import os.path

# ------------------------------------ config
# file with config
CONFIG = "hily.ini"

# folders list
flist = []

# outfile with pretty printed result, with default value
outfile = "./outfile.txt"

# ------------------------------------ start

def good_config ():
    """read configuration and cry if none or wrong."""

    global flist, outfile

    print ("Looking for configuration file.")

    if os.path.isfile (CONFIG):
        with open (CONFIG) as config:
            for line in config:
                line = line.strip ()
                if len (line):
                    print (line)
                    cmd, data = line.split (maxsplit=1)
                    if cmd == "in":
                        extdata = os.path.abspath (data)
                        flist += [extdata]
                    elif cmd == "out":
                        outfile = os.path.abspath (data)
        print (f"\nFolders list is {flist},\nresult will be written to {outfile}.")

        if not flist:
            print ("Folders list is empty. No work is possible. Quitting.")
            return False

    else:
        print (f"Config file {CONFIG} does not exist. Quitting.")
        return False

    return True

# ------------------------------------ init_db

def init_db ():
    """create database if it does not exists,
    connect to it and use
    """
    pass

# ------------------------------------ scan_folders

def scan_folders ():
    """make main job: scan multiple folders recursively"""
    pass

# ------------------------------------ print_result

def print_result ():
    """print result from renewed database in pleasant way"""
    print ("OK.")

# ------------------------------------ main

def main ():
    """starter"""
    print ("This is HiLy project: Highly Likely folders scanner.\n")
    if good_config ():
        init_db ()
        scan_folders ()
        print_result ()

# ------------------------------------ init

if __name__ == "__main__":
    main ()

# ------------------------------------ the end.

# NOTES:

# note: PEP8 change: function names are space separated from parameters lists even when they are empty; it is a test to see if it works better than traditional spacing.
<commit_msg>org flist folders<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# hily 2018-05-06 M.Kolodin 2018-05-09 0.12
# Highly Likely Project

# ver. 0. it must scan folders in hily.ini and print result (same files) into file hily.out.

# 1st ver. will work with sorted list of tiples
# 2nd ver. will use databases
#import sqlite3

import os.path

# ------------------------------------ config
# file with config
CONFIG = "hily.ini"

# folders list
flist = []

# outfile with pretty printed result, with default value
outfile = "./outfile.txt"

# ------------------------------------ start

def good_config ():
    """read configuration and cry if none or wrong."""

    global flist, outfile

    print ("Looking for configuration file.")

    if os.path.isfile (CONFIG):
        with open (CONFIG) as config:
            for line in config:
                line = line.strip ()
                if len (line):
                    print (line)
                    cmd, data = line.split (maxsplit=1)
                    if cmd == "in":
                        extdata = os.path.abspath (data)
                        flist += [extdata]
                    elif cmd == "out":
                        outfile = os.path.abspath (data)
        print (f"\nFolders list is {flist},\nresult will be written to {outfile}.")

        if not flist:
            print ("Folders list is empty. No work is possible. Quitting.")
            return False

    else:
        print (f"Config file {CONFIG} does not exist. Quitting.")
        return False

    return True

# ------------------------------------ init_db

def init_db ():
    """create database if it does not exists,
    connect to it and use
    """
    pass

# ------------------------------------ scan_folders

def scan_folders ():
    """make main job: scan multiple folders recursively"""
    for folder in flist:
        print (f"scanning {folder}")

# ------------------------------------ print_result

def print_result ():
    """print result from renewed database in pleasant way"""
    print ("OK.")

# ------------------------------------ main

def main ():
    """starter"""
    print ("This is HiLy project: Highly Likely folders scanner.\n")
    if good_config ():
#        init_db ()
        scan_folders ()
        print_result ()

# ------------------------------------ init

if __name__ == "__main__":
    main ()

# ------------------------------------ the end.

# NOTES:

# note: PEP8 change: function names are space separated from parameters lists even when they are empty; it is a test to see if it works better than traditional spacing.
<|endoftext|>


<commit_before>"""*sandman2*'s main module."""

# Third-party imports
from flask import Flask, current_app, jsonify
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.sql import sqltypes

# Application imports
from sandman2.exception import (
    BadRequestException,
    ForbiddenException,
    NotFoundException,
    NotAcceptableException,
    NotImplementedException,
    ConflictException,
    ServerErrorException,
    ServiceUnavailableException,
    )
from sandman2.service import Service
from sandman2.model import db, Model
from sandman2.admin import CustomAdminView
from flask.ext.admin import Admin

__version__ = '0.0.6'

# Augment sandman2's Model class with the Automap and Flask-SQLAlchemy model
# classes
Model = declarative_base(cls=(Model, db.Model))
AutomapModel = automap_base(Model)

def get_app(
        database_uri,
        exclude_tables=None,
        user_models=None,
        reflect_all=True,
        read_only=False):
    """Return an application instance connected to the database described in
    *database_uri*.

    :param str database_uri: The URI connection string for the database
    :param list exclude_tables: A list of tables to exclude from the API
                                service
    :param list user_models: A list of user-defined models to include in the
                             API service
    :param bool reflect_all: Include all database tables in the API service
    :param bool read_only: Only allow HTTP GET commands for all endpoints
    """
    app = Flask('sandman2')
    app.config['SQLALCHEMY_DATABASE_URI'] = database_uri
    app.config['SANDMAN2_READ_ONLY'] = read_only
    db.init_app(app)
    admin = Admin(app, base_template='layout.html', template_mode='bootstrap3')
    _register_error_handlers(app)
    if user_models:
        with app.app_context():
            _register_user_models(user_models, admin)
    elif reflect_all:
        with app.app_context():
            _reflect_all(exclude_tables, admin, read_only)
    return app

def _register_error_handlers(app):
    """Register error-handlers for the application.

    :param app: The application instance
    """
    @app.errorhandler(BadRequestException)
    @app.errorhandler(ForbiddenException)
    @app.errorhandler(NotAcceptableException)
    @app.errorhandler(NotFoundException)
    @app.errorhandler(ConflictException)
    @app.errorhandler(ServerErrorException)
    @app.errorhandler(NotImplementedException)
    @app.errorhandler(ServiceUnavailableException)
    def handle_application_error(error):  # pylint:disable=unused-variable
        """Handler used to send JSON error messages rather than default HTML
        ones."""
        response = jsonify(error.to_dict())
        response.status_code = error.code
        return response

def register_service(cls, primary_key_type):
    """Register an API service endpoint.

    :param cls: The class to register
    :param str primary_key_type: The type (as a string) of the primary_key
                                 field
    """
    view_func = cls.as_view(cls.__name__.lower())  # pylint: disable=no-member
    methods = set(cls.__model__.__methods__)  # pylint: disable=no-member

    if 'GET' in methods:  # pylint: disable=no-member
        current_app.add_url_rule(
            cls.__model__.__url__, defaults={'resource_id': None},
            view_func=view_func,
            methods=['GET'])
        current_app.add_url_rule(
            '{resource}/meta'.format(resource=cls.__model__.__url__),
            view_func=view_func,
            methods=['GET'])
    if 'POST' in methods:  # pylint: disable=no-member
        current_app.add_url_rule(
            cls.__model__.__url__, view_func=view_func, methods=['POST', ])
    current_app.add_url_rule(
        '{resource}/<{pk_type}:{pk}>'.format(
            resource=cls.__model__.__url__,
            pk='resource_id', pk_type=primary_key_type),
        view_func=view_func,
        methods=methods - {'POST'})

def _reflect_all(exclude_tables=None, admin=None, read_only=False):
    """Register all tables in the given database as services.

    :param list exclude_tables: A list of tables to exclude from the API
                                service
    """
    AutomapModel.prepare(  # pylint:disable=maybe-no-member
        db.engine, reflect=True)
    for cls in AutomapModel.classes:
        if exclude_tables and cls.__table__.name in exclude_tables:
            continue
        if read_only:
            cls.__methods__ = {'GET'}
        register_model(cls, admin)

def register_model(cls, admin=None):
    """Register *cls* to be included in the API service

    :param cls: Class deriving from :class:`sandman2.models.Model`
    """
    cls.__url__ = '/{}'.format(cls.__name__.lower())
    service_class = type(
        cls.__name__ + 'Service',
        (Service,),
        {
            '__model__': cls,
            '__version__': __version__,
        })

    # inspect primary key
    cols = list(cls().__table__.primary_key.columns)

    # composite keys not supported (yet)
    if len(cols) != 1:
        primary_key_type = None
    else:
        col_type = cols[0].type
        # types defined at http://flask.pocoo.org/docs/0.10/api/#url-route-registrations
        if isinstance(col_type, sqltypes.String):
            primary_key_type = 'string'
        elif isinstance(col_type, sqltypes.Integer):
            primary_key_type = 'int'
        elif isinstance(col_type, sqltypes.Numeric):
            primary_key_type = 'float'
        else:
            # unsupported primary key type
            primary_key_type = 'string'

    # registration
    register_service(service_class, primary_key_type)
    if admin is not None:
        admin.add_view(CustomAdminView(cls, db.session))

def _register_user_models(user_models, admin=None):
    """Register any user-defined models with the API Service.

    :param list user_models: A list of user-defined models to include in the
                             API service
    """
    if any([True for cls in user_models if issubclass(cls, AutomapModel)]):
        AutomapModel.prepare(  # pylint:disable=maybe-no-member
                               db.engine, reflect=True)

    for user_model in user_models:
        if not issubclass(user_model, AutomapModel):
            model_type = type(user_model.__name__, (user_model, Model), {})
            register_model(model_type, admin)
        else:
            register_model(user_model, admin)
<commit_msg>default composites to string for the time being<commit_after>"""*sandman2*'s main module."""

# Third-party imports
from flask import Flask, current_app, jsonify
from sqlalchemy.ext.automap import automap_base
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.sql import sqltypes

# Application imports
from sandman2.exception import (
    BadRequestException,
    ForbiddenException,
    NotFoundException,
    NotAcceptableException,
    NotImplementedException,
    ConflictException,
    ServerErrorException,
    ServiceUnavailableException,
    )
from sandman2.service import Service
from sandman2.model import db, Model
from sandman2.admin import CustomAdminView
from flask.ext.admin import Admin

__version__ = '0.0.6'

# Augment sandman2's Model class with the Automap and Flask-SQLAlchemy model
# classes
Model = declarative_base(cls=(Model, db.Model))
AutomapModel = automap_base(Model)

def get_app(
        database_uri,
        exclude_tables=None,
        user_models=None,
        reflect_all=True,
        read_only=False):
    """Return an application instance connected to the database described in
    *database_uri*.

    :param str database_uri: The URI connection string for the database
    :param list exclude_tables: A list of tables to exclude from the API
                                service
    :param list user_models: A list of user-defined models to include in the
                             API service
    :param bool reflect_all: Include all database tables in the API service
    :param bool read_only: Only allow HTTP GET commands for all endpoints
    """
    app = Flask('sandman2')
    app.config['SQLALCHEMY_DATABASE_URI'] = database_uri
    app.config['SANDMAN2_READ_ONLY'] = read_only
    db.init_app(app)
    admin = Admin(app, base_template='layout.html', template_mode='bootstrap3')
    _register_error_handlers(app)
    if user_models:
        with app.app_context():
            _register_user_models(user_models, admin)
    elif reflect_all:
        with app.app_context():
            _reflect_all(exclude_tables, admin, read_only)
    return app

def _register_error_handlers(app):
    """Register error-handlers for the application.

    :param app: The application instance
    """
    @app.errorhandler(BadRequestException)
    @app.errorhandler(ForbiddenException)
    @app.errorhandler(NotAcceptableException)
    @app.errorhandler(NotFoundException)
    @app.errorhandler(ConflictException)
    @app.errorhandler(ServerErrorException)
    @app.errorhandler(NotImplementedException)
    @app.errorhandler(ServiceUnavailableException)
    def handle_application_error(error):  # pylint:disable=unused-variable
        """Handler used to send JSON error messages rather than default HTML
        ones."""
        response = jsonify(error.to_dict())
        response.status_code = error.code
        return response

def register_service(cls, primary_key_type):
    """Register an API service endpoint.

    :param cls: The class to register
    :param str primary_key_type: The type (as a string) of the primary_key
                                 field
    """
    view_func = cls.as_view(cls.__name__.lower())  # pylint: disable=no-member
    methods = set(cls.__model__.__methods__)  # pylint: disable=no-member

    if 'GET' in methods:  # pylint: disable=no-member
        current_app.add_url_rule(
            cls.__model__.__url__, defaults={'resource_id': None},
            view_func=view_func,
            methods=['GET'])
        current_app.add_url_rule(
            '{resource}/meta'.format(resource=cls.__model__.__url__),
            view_func=view_func,
            methods=['GET'])
    if 'POST' in methods:  # pylint: disable=no-member
        current_app.add_url_rule(
            cls.__model__.__url__, view_func=view_func, methods=['POST', ])
    current_app.add_url_rule(
        '{resource}/<{pk_type}:{pk}>'.format(
            resource=cls.__model__.__url__,
            pk='resource_id', pk_type=primary_key_type),
        view_func=view_func,
        methods=methods - {'POST'})

def _reflect_all(exclude_tables=None, admin=None, read_only=False):
    """Register all tables in the given database as services.

    :param list exclude_tables: A list of tables to exclude from the API
                                service
    """
    AutomapModel.prepare(  # pylint:disable=maybe-no-member
        db.engine, reflect=True)
    for cls in AutomapModel.classes:
        if exclude_tables and cls.__table__.name in exclude_tables:
            continue
        if read_only:
            cls.__methods__ = {'GET'}
        register_model(cls, admin)

def register_model(cls, admin=None):
    """Register *cls* to be included in the API service

    :param cls: Class deriving from :class:`sandman2.models.Model`
    """
    cls.__url__ = '/{}'.format(cls.__name__.lower())
    service_class = type(
        cls.__name__ + 'Service',
        (Service,),
        {
            '__model__': cls,
            '__version__': __version__,
        })

    # inspect primary key
    cols = list(cls().__table__.primary_key.columns)

    # composite keys not supported (yet)
    if len(cols) != 1:
        primary_key_type = 'string'
    else:
        col_type = cols[0].type
        # types defined at http://flask.pocoo.org/docs/0.10/api/#url-route-registrations
        if isinstance(col_type, sqltypes.String):
            primary_key_type = 'string'
        elif isinstance(col_type, sqltypes.Integer):
            primary_key_type = 'int'
        elif isinstance(col_type, sqltypes.Numeric):
            primary_key_type = 'float'
        else:
            # unsupported primary key type
            primary_key_type = 'string'

    # registration
    register_service(service_class, primary_key_type)
    if admin is not None:
        admin.add_view(CustomAdminView(cls, db.session))

def _register_user_models(user_models, admin=None):
    """Register any user-defined models with the API Service.

    :param list user_models: A list of user-defined models to include in the
                             API service
    """
    if any([True for cls in user_models if issubclass(cls, AutomapModel)]):
        AutomapModel.prepare(  # pylint:disable=maybe-no-member
                               db.engine, reflect=True)

    for user_model in user_models:
        if not issubclass(user_model, AutomapModel):
            model_type = type(user_model.__name__, (user_model, Model), {})
            register_model(model_type, admin)
        else:
            register_model(user_model, admin)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- cpy-indent-level: 4; indent-tabs-mode: nil -*-
# ex: set expandtab softtabstop=4 shiftwidth=4:
#
# Copyright (C) 2012,2013,2014  Contributor
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Copy AQDB to a different backend.
#
# The source backend is taken from $AQDCONF as usual, the destination backend
# is taken from command line.
#
# Warning: making a copy will cause the sequences in the source database to
# jump - there's no easy way around that.
#
# Examples:
#    ./aqdb_migrate sqlite:////path/aquilon.db
#    ./aqdb_migrate postgresql://<username>@/

from __future__ import print_function

import os
import signal
import sys

# -- begin path_setup --
BINDIR = os.path.dirname(os.path.realpath(sys.argv[0]))
LIBDIR = os.path.join(BINDIR, "..", "lib")

if LIBDIR not in sys.path:
    sys.path.append(LIBDIR)
# -- end path_setup --

from aquilon.aqdb import depends  # pylint: disable=W0611

import argparse

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, configure_mappers
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.sql import text

from aquilon.aqdb.db_factory import DbFactory
from aquilon.aqdb.model import Base

signalled = 0

def dummy_mapper(table):
    Base = declarative_base()

    class DummyMapper(Base):
        __table__ = table

    return DummyMapper

def signal_handler(signum, frame):
    global signalled
    signalled = 1

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='copy AQDB between backends')
    parser.add_argument('-v', '--verbose',
                        action='store_true',
                        dest='verbose',
                        help='show queries (metadata bind.echo = True)')
    parser.add_argument('dsn', metavar='DSN',
                        help='DSN of the target backend (driver://user[:password]@host[:port]/database)')
    opts = parser.parse_args()

    # Use aquilon.aqdb for connecting to the source backend
    db = DbFactory()
    if opts.verbose:
        db.engine.echo = True
    src_session = sessionmaker(bind=db.engine)()

    if db.engine.dialect.name == 'oracle' or \
       db.engine.dialect.name == 'postgresql':
        src_session.execute(text('SET TRANSACTION ISOLATION LEVEL SERIALIZABLE'))

    dest_engine = create_engine(opts.dsn, convert_unicode=True, echo=opts.verbose)
    dest_session = sessionmaker(bind=dest_engine)()

    if db.engine.dialect.supports_sequences and \
       dest_engine.dialect.supports_sequences:
        # The only operation on sequences that all DBs support is getting the
        # next value. Unfortunately that means we have to alter the state of the
        # source database here.
        for seq in Base.metadata._sequences.values():
            nextid = src_session.execute(seq)
            # Make sure the sequence is re-created with the right start index
            seq.drop(dest_engine, checkfirst=True)
            seq.start = nextid

    # Avoid auto-populating tables as that would interfere with the copying
    Base.populate_table_on_create = False

    # Need to call this explicitely to make the __extra_table_args__ hack work
    configure_mappers()

    Base.metadata.create_all(dest_engine, checkfirst=True)

    if dest_engine.dialect.name == 'postgresql':
        dest_session.execute(text('SET CONSTRAINTS ALL DEFERRED'))
    elif dest_engine.dialect.name == 'oracle':
        dest_session.execute(text('ALTER SESSION SET CONSTRAINTS = DEFERRED'))
    elif dest_engine.dialect.name == 'sqlite':
        # SQLite does not allow changing the deferred state at run-time, but
        # it does allow disabling foreign keys entirely
        dest_session.execute(text('PRAGMA foreign_keys = 0;'))

    signal.signal(signal.SIGALRM, signal_handler)

    # Oracle does not like EINTR
    signal.siginterrupt(signal.SIGALRM, False)

    for table in Base.metadata.sorted_tables:
        total = src_session.execute(table.count()).scalar()
        print('Processing %s (%d rows)' % (table, total), end=' ')
        sys.stdout.flush()
        cnt = 0

        NewRecord = dummy_mapper(table)
        signal.setitimer(signal.ITIMER_REAL, 5, 5)
        for record in src_session.execute(table.select()):
            cnt = cnt + 1
            if signalled:
                print("... %d" % cnt, end=' ')
                sys.stdout.flush()
                signalled = 0

            data = {col.key: getattr(record, col.key) for col in table.columns}

            # insert() is faster, but using .merge() is restartable
            # dest_session.merge(NewRecord(**data))
            dest_session.execute(table.insert().values(**data))

        signal.setitimer(signal.ITIMER_REAL, 0)
        dest_session.flush()
        print()

    dest_session.commit()
<commit_msg>Improve aqdb_migrate<commit_after>#!/usr/bin/env python
# -*- cpy-indent-level: 4; indent-tabs-mode: nil -*-
# ex: set expandtab softtabstop=4 shiftwidth=4:
#
# Copyright (C) 2012,2013,2014  Contributor
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Copy AQDB to a different backend.
#
# The source backend is taken from $AQDCONF as usual, the destination backend
# is taken from command line.
#
# Warning: making a copy will cause the sequences in the source database to
# jump - there's no easy way around that.
#
# Examples:
#    ./aqdb_migrate sqlite:////path/aquilon.db
#    ./aqdb_migrate postgresql://<username>@/

from __future__ import print_function

import os
import signal
import sys

# -- begin path_setup --
BINDIR = os.path.dirname(os.path.realpath(sys.argv[0]))
LIBDIR = os.path.join(BINDIR, "..", "lib")

if LIBDIR not in sys.path:
    sys.path.append(LIBDIR)
# -- end path_setup --

from aquilon.aqdb import depends  # pylint: disable=W0611

import argparse

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, configure_mappers
from sqlalchemy.sql import text

from aquilon.aqdb.db_factory import DbFactory
from aquilon.aqdb.model import Base
from aquilon.utils import chunk

signalled = 0

def signal_handler(signum, frame):  # pylint: disable=W0613
    global signalled  # pylint: disable=W0603
    signalled = 1

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='copy AQDB between backends')
    parser.add_argument('-v', '--verbose',
                        action='store_true',
                        dest='verbose',
                        help='show queries (metadata bind.echo = True)')
    parser.add_argument('dsn', metavar='DSN',
                        help='DSN of the target backend (driver://user[:password]@host[:port]/database)')
    opts = parser.parse_args()

    # Use aquilon.aqdb for connecting to the source backend
    db = DbFactory()
    if opts.verbose:
        db.engine.echo = True
    src_session = sessionmaker(bind=db.engine)()

    if db.engine.dialect.name == 'oracle' or \
       db.engine.dialect.name == 'postgresql':
        src_session.execute(text('SET TRANSACTION ISOLATION LEVEL SERIALIZABLE'))

    dest_engine = create_engine(opts.dsn, convert_unicode=True, echo=opts.verbose)
    dest_session = sessionmaker(bind=dest_engine)()

    if db.engine.dialect.supports_sequences and \
       dest_engine.dialect.supports_sequences:
        # The only operation on sequences that all DBs support is getting the
        # next value. Unfortunately that means we have to alter the state of the
        # source database here.
        for seq in Base.metadata._sequences.values():
            nextid = src_session.execute(seq)
            # Make sure the sequence is re-created with the right start index
            seq.drop(dest_engine, checkfirst=True)
            seq.start = nextid

    # Avoid auto-populating tables as that would interfere with the copying
    Base.populate_table_on_create = False

    # Need to call this explicitely to make the __extra_table_args__ hack work
    configure_mappers()

    Base.metadata.create_all(dest_engine, checkfirst=True)

    if dest_engine.dialect.name == 'postgresql':
        dest_session.execute(text('SET CONSTRAINTS ALL DEFERRED'))
    elif dest_engine.dialect.name == 'oracle':
        dest_session.execute(text('ALTER SESSION SET CONSTRAINTS = DEFERRED'))
    elif dest_engine.dialect.name == 'sqlite':
        # SQLite does not allow changing the deferred state at run-time, but
        # it does allow disabling foreign keys entirely
        dest_session.execute(text('PRAGMA foreign_keys = 0;'))

    signal.signal(signal.SIGALRM, signal_handler)

    # Oracle does not like EINTR
    signal.siginterrupt(signal.SIGALRM, False)

    for table in Base.metadata.sorted_tables:
        total = src_session.execute(table.count()).scalar()
        print('Processing %s (%d rows)' % (table, total), end=' ')
        sys.stdout.flush()
        cnt = 0

        signal.setitimer(signal.ITIMER_REAL, 5, 5)
        for rows in chunk(src_session.execute(table.select()), 1000):
            cnt = cnt + len(rows)
            if signalled:
                print("... %d" % cnt, end=' ')
                sys.stdout.flush()
                signalled = 0

            data = [{col.key: getattr(record, col.key)
                     for col in table.columns}
                    for row in rows]

            dest_session.execute(table.insert().values(data))

        signal.setitimer(signal.ITIMER_REAL, 0)
        dest_session.flush()
        print()

    dest_session.commit()
<|endoftext|>