<commit_before>"""
BenchExec is a framework for reliable benchmarking.
This file is part of BenchExec.

Copyright (C) 2007-2015  Dirk Beyer
All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

# prepare for Python 3
from __future__ import absolute_import, division, print_function, unicode_literals

import logging
import os
import sys
import tempfile
import threading
import time
import unittest
sys.dont_write_bytecode = True # prevent creation of .pyc files

from benchexec.runexecutor import RunExecutor
from benchexec.runexecutor import _reduce_file_size_if_necessary

class TestRunExecutor(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls.longMessage = True
        cls.maxDiff = None
        logging.disable(logging.CRITICAL)
        if not hasattr(cls, 'assertRegex'):
            cls.assertRegex = cls.assertRegexpMatches

    def setUp(self):
        self.runexecutor = RunExecutor()

    def execute_run(self, *args, **kwargs):
        (output_fd, output_filename) = tempfile.mkstemp('.log', 'output_', text=True)
        try:
            result = self.runexecutor.execute_run(args, output_filename, **kwargs)
            output_lines = os.read(output_fd, 4096).decode().splitlines()
            return (result, output_lines)
        finally:
            os.close(output_fd)
            os.remove(output_filename)

    def test_command_output(self):
        if not os.path.exists('/bin/echo'):
            self.skipTest('missing /bin/echo')
        (_, output) = self.execute_run('/bin/echo', 'TEST_TOKEN')
        self.assertEqual(output[0], '/bin/echo TEST_TOKEN', 'run output misses executed command')
        self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
        for line in output[1:-1]:
            self.assertRegex(line, '^-*$', 'unexpected text in run output')

    def test_command_result(self):
        if not os.path.exists('/bin/echo'):
            self.skipTest('missing /bin/echo')
        (result, _) = self.execute_run('/bin/echo', 'TEST_TOKEN')
        self.assertEqual(result['exitcode'], 0, 'exit code of /bin/echo is not zero')
        self.assertAlmostEqual(result['walltime'], 0.2, delta=0.2, msg='walltime of /bin/echo not as expected')
        self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of /bin/echo not as expected')
        for key in result.keys():
            self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode'}, 'unexpected result value ' + key)

    def test_cputime_hardlimit(self):
        if not os.path.exists('/bin/sh'):
            self.skipTest('missing /bin/sh')
        (result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
                                            hardtimelimit=1)
        self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
        if 'terminationreason' in result:
            # not produced currently if killed by ulimit
            self.assertEqual(result['terminationreason'], 'cputime', 'termination reason is not "cputime"')
        self.assertAlmostEqual(result['walltime'], 1.4, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
        self.assertAlmostEqual(result['cputime'], 1.4, delta=0.5, msg='cputime is not approximately the time after which the process should have been killed')
        for key in result.keys():
            self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'terminationreason'}, 'unexpected result value ' + key)

        for line in output[1:]:
            self.assertRegex(line, '^-*$', 'unexpected text in run output')

    def test_cputime_softlimit(self):
        if not os.path.exists('/bin/sh'):
            self.skipTest('missing /bin/sh')
        try:
            (result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
                                                softtimelimit=1)
        except SystemExit as e:
            self.assertEqual(str(e), 'Soft time limit cannot be specified without cpuacct cgroup.')
            self.skipTest(e)

        self.assertEqual(result['exitcode'], 15, 'exit code of killed process is not 15')
        self.assertEqual(result['terminationreason'], 'cputime-soft', 'termination reason is not "cputime-soft"')
        self.assertAlmostEqual(result['walltime'], 4, delta=3, msg='walltime is not approximately the time after which the process should have been killed')
        self.assertAlmostEqual(result['cputime'], 4, delta=3, msg='cputime is not approximately the time after which the process should have been killed')
        for key in result.keys():
            self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'terminationreason'}, 'unexpected result value ' + key)

        for line in output[1:]:
            self.assertRegex(line, '^-*$', 'unexpected text in run output')

    def test_walltime_limit(self):
        if not os.path.exists('/bin/sleep'):
            self.skipTest('missing /bin/sleep')
        try:
            (result, output) = self.execute_run('/bin/sleep', '10', walltimelimit=1)
        except SystemExit as e:
            self.assertEqual(str(e), 'Wall time limit is not implemented for systems without cpuacct cgroup.')
            self.skipTest(e)

        self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
        self.assertEqual(result['terminationreason'], 'walltime', 'termination reason is not "walltime"')
        self.assertAlmostEqual(result['walltime'], 4, delta=3, msg='walltime is not approximately the time after which the process should have been killed')
        self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of /bin/sleep is not approximately zero')
        for key in result.keys():
            self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'terminationreason'}, 'unexpected result value ' + key)

        self.assertEqual(output[0], '/bin/sleep 10', 'run output misses executed command')
        for line in output[1:]:
            self.assertRegex(line, '^-*$', 'unexpected text in run output')

    def test_cputime_walltime_limit(self):
        if not os.path.exists('/bin/sh'):
            self.skipTest('missing /bin/sh')
        (result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
                                            hardtimelimit=1, walltimelimit=5)

        self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
        if 'terminationreason' in result:
            # not produced currently if killed by ulimit
            self.assertEqual(result['terminationreason'], 'cputime', 'termination reason is not "cputime"')
        self.assertAlmostEqual(result['walltime'], 1.4, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
        self.assertAlmostEqual(result['cputime'], 1.4, delta=0.5, msg='cputime is not approximately the time after which the process should have been killed')
        for key in result.keys():
            self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'terminationreason'}, 'unexpected result value ' + key)

        for line in output[1:]:
            self.assertRegex(line, '^-*$', 'unexpected text in run output')

    def test_all_timelimits(self):
        if not os.path.exists('/bin/sh'):
            self.skipTest('missing /bin/sh')
        try:
            (result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
                                                softtimelimit=1, hardtimelimit=2, walltimelimit=5)
        except SystemExit as e:
            self.assertEqual(str(e), 'Soft time limit cannot be specified without cpuacct cgroup.')
            self.skipTest(e)

        self.assertEqual(result['exitcode'], 15, 'exit code of killed process is not 15')
        self.assertEqual(result['terminationreason'], 'cputime-soft', 'termination reason is not "cputime-soft"')
        self.assertAlmostEqual(result['walltime'], 1.4, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
        self.assertAlmostEqual(result['cputime'], 1.4, delta=0.5, msg='cputime is not approximately the time after which the process should have been killed')
        for key in result.keys():
            self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'terminationreason'}, 'unexpected result value ' + key)

        for line in output[1:]:
            self.assertRegex(line, '^-*$', 'unexpected text in run output')

    def test_input_is_redirected_from_devnull(self):
        if not os.path.exists('/bin/cat'):
            self.skipTest('missing /bin/cat')
        try:
            (result, output) = self.execute_run('/bin/cat', walltimelimit=1)
        except SystemExit as e:
            self.assertEqual(str(e), 'Wall time limit is not implemented for systems without cpuacct cgroup.')
            self.skipTest(e)

        self.assertEqual(result['exitcode'], 0, 'exit code of process is not 0')
        self.assertAlmostEqual(result['walltime'], 0.2, delta=0.2, msg='walltime of "/bin/cat < /dev/null" is not approximately zero')
        self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of "/bin/cat < /dev/null" is not approximately zero')
        for key in result.keys():
            self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode'}, 'unexpected result value ' + key)

        self.assertEqual(output[0], '/bin/cat', 'run output misses executed command')
        for line in output[1:]:
            self.assertRegex(line, '^-*$', 'unexpected text in run output')

    def test_stop_run(self):
        if not os.path.exists('/bin/sleep'):
            self.skipTest('missing /bin/sleep')
        thread = _StopRunThread(1, self.runexecutor)
        thread.start()
        (result, output) = self.execute_run('/bin/sleep', '10')
        thread.join()

        self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
        self.assertEqual(result['terminationreason'], 'killed', 'termination reason is not "killed"')
        self.assertAlmostEqual(result['walltime'], 1, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
        self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of /bin/sleep is not approximately zero')
        for key in result.keys():
            self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'terminationreason'}, 'unexpected result value ' + key)

        self.assertEqual(output[0], '/bin/sleep 10', 'run output misses executed command')
        for line in output[1:]:
            self.assertRegex(line, '^-*$', 'unexpected text in run output')

    def test_reduce_file_size_empty_file(self):
        with tempfile.NamedTemporaryFile() as tmp:
            _reduce_file_size_if_necessary(tmp.name, 0)
            self.assertEqual(os.path.getsize(tmp.name), 0)

    def test_reduce_file_size_empty_file2(self):
        with tempfile.NamedTemporaryFile() as tmp:
            _reduce_file_size_if_necessary(tmp.name, 500)
            self.assertEqual(os.path.getsize(tmp.name), 0)

    def test_reduce_file_size_long_line_not_truncated(self):
        with tempfile.NamedTemporaryFile(mode='wt') as tmp:
            content = 'Long line ' * 500
            tmp.write(content)
            tmp.flush()
            _reduce_file_size_if_necessary(tmp.name, 500)
            with open(tmp.name, 'rt') as tmp2:
                self.assertMultiLineEqual(tmp2.read(), content)

    REDUCE_WARNING_MSG = "WARNING: YOUR LOGFILE WAS TOO LONG, SOME LINES IN THE MIDDLE WERE REMOVED."
    REDUCE_OVERHEAD = 100

    def test_reduce_file_size(self):
        with tempfile.NamedTemporaryFile(mode='wt') as tmp:
            line = 'Some text\n'
            tmp.write(line * 500)
            tmp.flush()
            limit = 500
            _reduce_file_size_if_necessary(tmp.name, limit)
            self.assertLessEqual(os.path.getsize(tmp.name), limit + self.REDUCE_OVERHEAD)
            with open(tmp.name, 'rt') as tmp2:
                new_content = tmp2.read()
        self.assertIn(self.REDUCE_WARNING_MSG, new_content)
        self.assertTrue(new_content.startswith(line))
        self.assertTrue(new_content.endswith(line))

    def test_reduce_file_size_limit_zero(self):
        with tempfile.NamedTemporaryFile(mode='wt') as tmp:
            line = 'Some text\n'
            tmp.write(line * 500)
            tmp.flush()
            _reduce_file_size_if_necessary(tmp.name, 0)
            self.assertLessEqual(os.path.getsize(tmp.name), self.REDUCE_OVERHEAD)
            with open(tmp.name, 'rt') as tmp2:
                new_content = tmp2.read()
        self.assertIn(self.REDUCE_WARNING_MSG, new_content)
        self.assertTrue(new_content.startswith(line))

class _StopRunThread(threading.Thread):
    def __init__(self, delay, runexecutor):
        super(_StopRunThread, self).__init__()
        self.daemon = True
        self.delay = delay
        self.runexecutor = runexecutor

    def run(self):
        time.sleep(self.delay)
        self.runexecutor.stop()
<commit_msg>Add integration test for runexec.<commit_after>"""
BenchExec is a framework for reliable benchmarking.
This file is part of BenchExec.

Copyright (C) 2007-2015  Dirk Beyer
All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

# prepare for Python 3
from __future__ import absolute_import, division, print_function, unicode_literals

import logging
import os
import subprocess
import sys
import tempfile
import threading
import time
import unittest
sys.dont_write_bytecode = True # prevent creation of .pyc files

try:
    from subprocess import DEVNULL
except ImportError:
    DEVNULL = open(os.devnull, 'wb')

from benchexec.runexecutor import RunExecutor
from benchexec.runexecutor import _reduce_file_size_if_necessary

here = os.path.dirname(__file__)
base_dir = os.path.join(here, '..')
bin_dir = os.path.join(base_dir, 'bin')
runexec = os.path.join(bin_dir, 'runexec')
python = 'python2' if sys.version_info[0] == 2 else 'python3'

class TestRunExecutor(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls.longMessage = True
        cls.maxDiff = None
        logging.disable(logging.CRITICAL)
        if not hasattr(cls, 'assertRegex'):
            cls.assertRegex = cls.assertRegexpMatches

    def setUp(self):
        self.runexecutor = RunExecutor()

    def execute_run(self, *args, **kwargs):
        (output_fd, output_filename) = tempfile.mkstemp('.log', 'output_', text=True)
        try:
            result = self.runexecutor.execute_run(args, output_filename, **kwargs)
            output_lines = os.read(output_fd, 4096).decode().splitlines()
            return (result, output_lines)
        finally:
            os.close(output_fd)
            os.remove(output_filename)

    def execute_run_extern(self, *args, **kwargs):
        (output_fd, output_filename) = tempfile.mkstemp('.log', 'output_', text=True)
        try:
            runexec_output = subprocess.check_output(
                    args=[python, runexec] + list(args) + ['--output', output_filename],
                    stderr=DEVNULL,
                    **kwargs
                    ).decode()
            output_lines = os.read(output_fd, 4096).decode().splitlines()
        except subprocess.CalledProcessError as e:
            print(e.output.decode())
            raise e
        finally:
            os.close(output_fd)
            os.remove(output_filename)

        result={key.strip(): value.strip() for (key, _, value) in (line.partition('=') for line in runexec_output.splitlines())}
        return (result, output_lines)

    def test_command_output(self):
        if not os.path.exists('/bin/echo'):
            self.skipTest('missing /bin/echo')
        (_, output) = self.execute_run('/bin/echo', 'TEST_TOKEN')
        self.assertEqual(output[0], '/bin/echo TEST_TOKEN', 'run output misses executed command')
        self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
        for line in output[1:-1]:
            self.assertRegex(line, '^-*$', 'unexpected text in run output')

    def test_command_result(self):
        if not os.path.exists('/bin/echo'):
            self.skipTest('missing /bin/echo')
        (result, _) = self.execute_run('/bin/echo', 'TEST_TOKEN')
        self.assertEqual(result['exitcode'], 0, 'exit code of /bin/echo is not zero')
        self.assertAlmostEqual(result['walltime'], 0.2, delta=0.2, msg='walltime of /bin/echo not as expected')
        self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of /bin/echo not as expected')
        for key in result.keys():
            self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode'}, 'unexpected result value ' + key)

    def test_cputime_hardlimit(self):
        if not os.path.exists('/bin/sh'):
            self.skipTest('missing /bin/sh')
        (result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
                                            hardtimelimit=1)
        self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
        if 'terminationreason' in result:
            # not produced currently if killed by ulimit
            self.assertEqual(result['terminationreason'], 'cputime', 'termination reason is not "cputime"')
        self.assertAlmostEqual(result['walltime'], 1.4, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
        self.assertAlmostEqual(result['cputime'], 1.4, delta=0.5, msg='cputime is not approximately the time after which the process should have been killed')
        for key in result.keys():
            self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'terminationreason'}, 'unexpected result value ' + key)

        for line in output[1:]:
            self.assertRegex(line, '^-*$', 'unexpected text in run output')

    def test_cputime_softlimit(self):
        if not os.path.exists('/bin/sh'):
            self.skipTest('missing /bin/sh')
        try:
            (result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
                                                softtimelimit=1)
        except SystemExit as e:
            self.assertEqual(str(e), 'Soft time limit cannot be specified without cpuacct cgroup.')
            self.skipTest(e)

        self.assertEqual(result['exitcode'], 15, 'exit code of killed process is not 15')
        self.assertEqual(result['terminationreason'], 'cputime-soft', 'termination reason is not "cputime-soft"')
        self.assertAlmostEqual(result['walltime'], 4, delta=3, msg='walltime is not approximately the time after which the process should have been killed')
        self.assertAlmostEqual(result['cputime'], 4, delta=3, msg='cputime is not approximately the time after which the process should have been killed')
        for key in result.keys():
            self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'terminationreason'}, 'unexpected result value ' + key)

        for line in output[1:]:
            self.assertRegex(line, '^-*$', 'unexpected text in run output')

    def test_walltime_limit(self):
        if not os.path.exists('/bin/sleep'):
            self.skipTest('missing /bin/sleep')
        try:
            (result, output) = self.execute_run('/bin/sleep', '10', walltimelimit=1)
        except SystemExit as e:
            self.assertEqual(str(e), 'Wall time limit is not implemented for systems without cpuacct cgroup.')
            self.skipTest(e)

        self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
        self.assertEqual(result['terminationreason'], 'walltime', 'termination reason is not "walltime"')
        self.assertAlmostEqual(result['walltime'], 4, delta=3, msg='walltime is not approximately the time after which the process should have been killed')
        self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of /bin/sleep is not approximately zero')
        for key in result.keys():
            self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'terminationreason'}, 'unexpected result value ' + key)

        self.assertEqual(output[0], '/bin/sleep 10', 'run output misses executed command')
        for line in output[1:]:
            self.assertRegex(line, '^-*$', 'unexpected text in run output')

    def test_cputime_walltime_limit(self):
        if not os.path.exists('/bin/sh'):
            self.skipTest('missing /bin/sh')
        (result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
                                            hardtimelimit=1, walltimelimit=5)

        self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
        if 'terminationreason' in result:
            # not produced currently if killed by ulimit
            self.assertEqual(result['terminationreason'], 'cputime', 'termination reason is not "cputime"')
        self.assertAlmostEqual(result['walltime'], 1.4, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
        self.assertAlmostEqual(result['cputime'], 1.4, delta=0.5, msg='cputime is not approximately the time after which the process should have been killed')
        for key in result.keys():
            self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'terminationreason'}, 'unexpected result value ' + key)

        for line in output[1:]:
            self.assertRegex(line, '^-*$', 'unexpected text in run output')

    def test_all_timelimits(self):
        if not os.path.exists('/bin/sh'):
            self.skipTest('missing /bin/sh')
        try:
            (result, output) = self.execute_run('/bin/sh', '-c', 'i=0; while [ $i -lt 10000000 ]; do i=$(($i+1)); done; echo $i',
                                                softtimelimit=1, hardtimelimit=2, walltimelimit=5)
        except SystemExit as e:
            self.assertEqual(str(e), 'Soft time limit cannot be specified without cpuacct cgroup.')
            self.skipTest(e)

        self.assertEqual(result['exitcode'], 15, 'exit code of killed process is not 15')
        self.assertEqual(result['terminationreason'], 'cputime-soft', 'termination reason is not "cputime-soft"')
        self.assertAlmostEqual(result['walltime'], 1.4, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
        self.assertAlmostEqual(result['cputime'], 1.4, delta=0.5, msg='cputime is not approximately the time after which the process should have been killed')
        for key in result.keys():
            self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'terminationreason'}, 'unexpected result value ' + key)

        for line in output[1:]:
            self.assertRegex(line, '^-*$', 'unexpected text in run output')

    def test_input_is_redirected_from_devnull(self):
        if not os.path.exists('/bin/cat'):
            self.skipTest('missing /bin/cat')
        try:
            (result, output) = self.execute_run('/bin/cat', walltimelimit=1)
        except SystemExit as e:
            self.assertEqual(str(e), 'Wall time limit is not implemented for systems without cpuacct cgroup.')
            self.skipTest(e)

        self.assertEqual(result['exitcode'], 0, 'exit code of process is not 0')
        self.assertAlmostEqual(result['walltime'], 0.2, delta=0.2, msg='walltime of "/bin/cat < /dev/null" is not approximately zero')
        self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of "/bin/cat < /dev/null" is not approximately zero')
        for key in result.keys():
            self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode'}, 'unexpected result value ' + key)

        self.assertEqual(output[0], '/bin/cat', 'run output misses executed command')
        for line in output[1:]:
            self.assertRegex(line, '^-*$', 'unexpected text in run output')

    def test_stop_run(self):
        if not os.path.exists('/bin/sleep'):
            self.skipTest('missing /bin/sleep')
        thread = _StopRunThread(1, self.runexecutor)
        thread.start()
        (result, output) = self.execute_run('/bin/sleep', '10')
        thread.join()

        self.assertEqual(result['exitcode'], 9, 'exit code of killed process is not 9')
        self.assertEqual(result['terminationreason'], 'killed', 'termination reason is not "killed"')
        self.assertAlmostEqual(result['walltime'], 1, delta=0.5, msg='walltime is not approximately the time after which the process should have been killed')
        self.assertAlmostEqual(result['cputime'], 0.2, delta=0.2, msg='cputime of /bin/sleep is not approximately zero')
        for key in result.keys():
            self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'terminationreason'}, 'unexpected result value ' + key)

        self.assertEqual(output[0], '/bin/sleep 10', 'run output misses executed command')
        for line in output[1:]:
            self.assertRegex(line, '^-*$', 'unexpected text in run output')

    def test_reduce_file_size_empty_file(self):
        with tempfile.NamedTemporaryFile() as tmp:
            _reduce_file_size_if_necessary(tmp.name, 0)
            self.assertEqual(os.path.getsize(tmp.name), 0)

    def test_reduce_file_size_empty_file2(self):
        with tempfile.NamedTemporaryFile() as tmp:
            _reduce_file_size_if_necessary(tmp.name, 500)
            self.assertEqual(os.path.getsize(tmp.name), 0)

    def test_reduce_file_size_long_line_not_truncated(self):
        with tempfile.NamedTemporaryFile(mode='wt') as tmp:
            content = 'Long line ' * 500
            tmp.write(content)
            tmp.flush()
            _reduce_file_size_if_necessary(tmp.name, 500)
            with open(tmp.name, 'rt') as tmp2:
                self.assertMultiLineEqual(tmp2.read(), content)

    REDUCE_WARNING_MSG = "WARNING: YOUR LOGFILE WAS TOO LONG, SOME LINES IN THE MIDDLE WERE REMOVED."
    REDUCE_OVERHEAD = 100

    def test_reduce_file_size(self):
        with tempfile.NamedTemporaryFile(mode='wt') as tmp:
            line = 'Some text\n'
            tmp.write(line * 500)
            tmp.flush()
            limit = 500
            _reduce_file_size_if_necessary(tmp.name, limit)
            self.assertLessEqual(os.path.getsize(tmp.name), limit + self.REDUCE_OVERHEAD)
            with open(tmp.name, 'rt') as tmp2:
                new_content = tmp2.read()
        self.assertIn(self.REDUCE_WARNING_MSG, new_content)
        self.assertTrue(new_content.startswith(line))
        self.assertTrue(new_content.endswith(line))

    def test_reduce_file_size_limit_zero(self):
        with tempfile.NamedTemporaryFile(mode='wt') as tmp:
            line = 'Some text\n'
            tmp.write(line * 500)
            tmp.flush()
            _reduce_file_size_if_necessary(tmp.name, 0)
            self.assertLessEqual(os.path.getsize(tmp.name), self.REDUCE_OVERHEAD)
            with open(tmp.name, 'rt') as tmp2:
                new_content = tmp2.read()
        self.assertIn(self.REDUCE_WARNING_MSG, new_content)
        self.assertTrue(new_content.startswith(line))

    def test_integration(self):
        if not os.path.exists('/bin/echo'):
            self.skipTest('missing /bin/echo')
        (result, output) = self.execute_run_extern('/bin/echo', 'TEST_TOKEN')
        self.assertEqual(int(result['exitcode']), 0, 'exit code of /bin/echo is not zero')
        for key in result.keys():
            self.assertIn(key, {'cputime', 'walltime', 'memory', 'exitcode', 'returnvalue'}, 'unexpected result value ' + key)

        self.assertEqual(output[0], '/bin/echo TEST_TOKEN', 'run output misses executed command')
        self.assertEqual(output[-1], 'TEST_TOKEN', 'run output misses command output')
        for line in output[1:-1]:
            self.assertRegex(line, '^-*$', 'unexpected text in run output')

class _StopRunThread(threading.Thread):
    def __init__(self, delay, runexecutor):
        super(_StopRunThread, self).__init__()
        self.daemon = True
        self.delay = delay
        self.runexecutor = runexecutor

    def run(self):
        time.sleep(self.delay)
        self.runexecutor.stop()
<|endoftext|>


<commit_before>
#
# TODO: Other useful operations
#

class ItemList(list):
    '''
        A list that allows adding items to it via the += operator.

        DSL Implementation Object Usage:

            class Foo(object):
                def __init__(self):
                    self.items = ItemList()

        DSL usage:

            with Foo:
                items += 'item1'
                items += 'item2

        Result:

            foo.items = ['item1', 'item2']
    '''
    def __iadd__(self, item):
        self.append(item)
        return self

class list_of(list):
    '''
        When seen in a DSL Implementation Object, this allows the 'with'
        statement to occur multiple times, and each time a new object will
        be appended to the list.

        @warning Don't use it like this, or you will get odd errors:

            class Foo:
                bar = dsltool.list_of(Bar)

        Instead, prefer this:

            class Foo:
                def __init__(self):
                    self.bar = dsltool.list_of(Bar)
    '''

    def __init__(self, cls):
        list.__init__(self)
        self._original_cls = cls
<commit_msg>Update doc<commit_after>
#
# TODO: Other useful operations
#

class ItemList(list):
    '''
        A list that allows adding items to it via the += operator.

        DSL Implementation Object Usage:

            class Foo(object):
                def __init__(self):
                    self.items = ItemList()

        DSL usage:

            with Foo:
                items += 'item1'
                items += 'item2

        Result:

            foo.items == ['item1', 'item2']
    '''
    def __iadd__(self, item):
        self.append(item)
        return self

class list_of(list):
    '''
        When seen in a DSL Implementation Object, this allows the 'with'
        statement to occur multiple times, and each time a new object will
        be appended to the list.

        @warning Don't use it like this, or you will get odd errors:

            class Foo:
                bar = dsltool.list_of(Bar)

        Instead, prefer this:

            class Foo:
                def __init__(self):
                    self.bar = dsltool.list_of(Bar)
    '''

    def __init__(self, cls):
        list.__init__(self)
        self._original_cls = cls
<|endoftext|>


<commit_before>from browserid.errors import TrustError
from pyramid.config import ConfigurationError
from pyramid.httpexceptions import (
    HTTPForbidden,
    HTTPFound,
)
from pyramid.security import (
    NO_PERMISSION_REQUIRED,
    authenticated_userid,
    remember,
    forget,
)
from pyramid.settings import (
    asbool,
    aslist,
)
from pyramid.view import (
    view_config,
)
from .contentbase import make_subrequest
from .storage import (
    DBSession,
    Key,
)
from .validation import ValidationFailure

AUDIENCES_MESSAGE = """\
Missing persona.audiences settings. This is needed for security reasons. \
See https://developer.mozilla.org/en-US/docs/Mozilla/Persona/Security_Considerations \
for details."""

def includeme(config):
    config.scan(__name__)
    settings = config.registry.settings

    if not 'persona.audiences' in settings:
        raise ConfigurationError(AUDIENCES_MESSAGE)
    # Construct a browserid Verifier using the configured audience.
    # This will pre-compile some regexes to reduce per-request overhead.
    verifier_factory = config.maybe_dotted(settings.get('persona.verifier',
                                                        'browserid.RemoteVerifier'))
    audiences = aslist(settings['persona.audiences'])
    config.registry['persona.verifier'] = verifier_factory(audiences)

# in a perfect world these would inherit from Classes shared by api module
def verify_assertion(request):
    """Verifies the assertion in the given request.

    Returns the email of the user if everything is valid, otherwise raises
    a ValidationFailure
    """
    verifier = request.registry['persona.verifier']
    try:
        assertion = request.json['assertion']
    except KeyError:
        msg = 'Missing assertion.'
        raise ValidationFailure('body', ['assertion'], msg)
    try:
        data = verifier.verify(assertion)
    except (ValueError, TrustError) as e:
        msg = 'Invalid assertion: %s (%s)' % (e, type(e).__name__)
        raise ValidationFailure('body', ['assertion'], msg)
    else:
        request.validated = data

# Unfortunately, X-Requested-With is not sufficient.
# http://lists.webappsec.org/pipermail/websecurity_lists.webappsec.org/2011-February/007533.html
# Checking the CSRF token in middleware is easier
@view_config(name='login', physical_path='/', request_method='POST',
             subpath_segments=0, permission=NO_PERMISSION_REQUIRED,
             validators=[verify_assertion])
def login(request):
    """View to check the persona assertion and remember the user"""
    data = request.validated
    email = data['email'].lower()
    session = DBSession()
    model = session.query(Key).get(('user:email', email))
    if model is None:
        raise HTTPForbidden()
    login = 'mailto:' + email
    request.response.headerlist.extend(remember(request, login))
    return data

@view_config(name='logout', physical_path='/',
             subpath_segments=0, permission=NO_PERMISSION_REQUIRED)
def logout(request):
    """View to forget the user"""
    request.response.headers = forget(request)
    if asbool(request.params.get('redirect', True)):
        raise HTTPFound(location=request.resource_path(request.root))
    return {'status': 'okay'}

@view_config(name='session', physical_path='/', request_method='GET',
             subpath_segments=0, permission=NO_PERMISSION_REQUIRED)
def session(context, request):
    """ Give the user a CSRF token
    """
    token = request.session.get_csrf_token()
    login = authenticated_userid(request)
    result = {'csrf_token': token, 'persona': None, 'user_properties': {}}
    if login is None:
        return result

    namespace, userid = login.split(':', 1)
    if namespace != 'mailto':
        raise HTTPForbidden()

    result['persona'] = userid
    subreq = make_subrequest(request, '/current-user')
    subreq.override_renderer = 'null_renderer'
    result['user_properties'] = request.invoke_subrequest(subreq)
    return result
<commit_msg>Don't remove existing headers, they are needed.<commit_after>from browserid.errors import TrustError
from pyramid.config import ConfigurationError
from pyramid.httpexceptions import (
    HTTPForbidden,
    HTTPFound,
)
from pyramid.security import (
    NO_PERMISSION_REQUIRED,
    authenticated_userid,
    remember,
    forget,
)
from pyramid.settings import (
    asbool,
    aslist,
)
from pyramid.view import (
    view_config,
)
from .contentbase import make_subrequest
from .storage import (
    DBSession,
    Key,
)
from .validation import ValidationFailure

AUDIENCES_MESSAGE = """\
Missing persona.audiences settings. This is needed for security reasons. \
See https://developer.mozilla.org/en-US/docs/Mozilla/Persona/Security_Considerations \
for details."""

def includeme(config):
    config.scan(__name__)
    settings = config.registry.settings

    if not 'persona.audiences' in settings:
        raise ConfigurationError(AUDIENCES_MESSAGE)
    # Construct a browserid Verifier using the configured audience.
    # This will pre-compile some regexes to reduce per-request overhead.
    verifier_factory = config.maybe_dotted(settings.get('persona.verifier',
                                                        'browserid.RemoteVerifier'))
    audiences = aslist(settings['persona.audiences'])
    config.registry['persona.verifier'] = verifier_factory(audiences)

# in a perfect world these would inherit from Classes shared by api module
def verify_assertion(request):
    """Verifies the assertion in the given request.

    Returns the email of the user if everything is valid, otherwise raises
    a ValidationFailure
    """
    verifier = request.registry['persona.verifier']
    try:
        assertion = request.json['assertion']
    except KeyError:
        msg = 'Missing assertion.'
        raise ValidationFailure('body', ['assertion'], msg)
    try:
        data = verifier.verify(assertion)
    except (ValueError, TrustError) as e:
        msg = 'Invalid assertion: %s (%s)' % (e, type(e).__name__)
        raise ValidationFailure('body', ['assertion'], msg)
    else:
        request.validated = data

# Unfortunately, X-Requested-With is not sufficient.
# http://lists.webappsec.org/pipermail/websecurity_lists.webappsec.org/2011-February/007533.html
# Checking the CSRF token in middleware is easier
@view_config(name='login', physical_path='/', request_method='POST',
             subpath_segments=0, permission=NO_PERMISSION_REQUIRED,
             validators=[verify_assertion])
def login(request):
    """View to check the persona assertion and remember the user"""
    data = request.validated
    email = data['email'].lower()
    session = DBSession()
    model = session.query(Key).get(('user:email', email))
    if model is None:
        raise HTTPForbidden()
    login = 'mailto:' + email
    request.response.headerlist.extend(remember(request, login))
    return data

@view_config(name='logout', physical_path='/',
             subpath_segments=0, permission=NO_PERMISSION_REQUIRED)
def logout(request):
    """View to forget the user"""
    request.response.headerlist.extend(forget(request))
    if asbool(request.params.get('redirect', True)):
        raise HTTPFound(location=request.resource_path(request.root))
    return {'status': 'okay'}

@view_config(name='session', physical_path='/', request_method='GET',
             subpath_segments=0, permission=NO_PERMISSION_REQUIRED)
def session(context, request):
    """ Give the user a CSRF token
    """
    token = request.session.get_csrf_token()
    login = authenticated_userid(request)
    result = {'csrf_token': token, 'persona': None, 'user_properties': {}}
    if login is None:
        return result

    namespace, userid = login.split(':', 1)
    if namespace != 'mailto':
        raise HTTPForbidden()

    result['persona'] = userid
    subreq = make_subrequest(request, '/current-user')
    subreq.override_renderer = 'null_renderer'
    result['user_properties'] = request.invoke_subrequest(subreq)
    return result
<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys
import json

def format_(credentials):
    key_map = {"SecretAccessKey": "AWS_SECRET_ACCESS_KEY",
               "SessionToken": "AWS_SESSION_TOKEN",
               "AccessKeyId": "AWS_ACCESS_KEY_ID"}

    for key, value in key_map.items():
        print "export {0}={1}".format(value, credentials[key])
    print "export AWS_SECURITY_TOKEN={0}".format(credentials['SessionToken'])

if __name__ == "__main__":
    input_str = sys.stdin.read()
    format_(json.loads(input_str)['Credentials'])
<commit_msg>remove obsolete stuff<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python3
from __future__ import print_function

VERSION = '0.4.8'

import os
import sys
import shutil
import fileinput
import functools
import tarfile
import gzip
import subprocess
import multiprocessing
import argparse
import string
import random
import time
from contextlib import closing

try:
    from urllib.request import urlopen, urlretrieve
except ImportError:
    from urllib2 import urlopen
    from urllib import urlretrieve

try:
    width = shutil.get_terminal_size((40, 0)).columns
except AttributeError:
    width = 40

def get_args(raw_args):
    parser = argparse.ArgumentParser(
        description='Simplified kernel rolling tool'
    )
    parser.add_argument(
        '--version',
        action='version',
        version=VERSION
    )
    parser.add_argument(
        '-v', '--verbose',
        action='store_true',
        help='Increase verbosity of output'
    )
    parser.add_argument(
        '-k', '--kernel',
        dest='new_version',
        default=get_latest_kernel_version(),
        help='Kernel version to build',
    )
    parser.add_argument(
        '-n', '--new-revision',
        dest='new_revision',
        type=str,
        default=None,
        help='Kernel revision to create',
    )
    parser.add_argument(
        '-c', '--config',
        dest='config_version',
        default=get_current_kernel_version(),
        help='Config version to work from',
    )
    parser.add_argument(
        '-r', '--config-revision',
        dest='config_revision',
        type=str,
        default='current',
        help='Config revision to work from',
    )
    parser.add_argument(
        '-s', '--skip-install',
        dest='skip_install',
        action='store_true',
        help='Do not install kernel to /boot'
    )
    parser.add_argument(
        '-p', '--patch',
        dest='patch',
        action='store_true',
        help='Open a shell before configuration to allow patching'
    )
    parser.add_argument(
        '-b', '--build-dir',
        dest='build_dir',
        type=str,
        default='/tmp',
        help='directory for downloading, extracting, and building the kernel'
    )
    parser.add_argument(
        '-d', '--config-dir',
        dest='config_dir',
        type=str,
        default=None,
        help='directory for kernel configs'
    )

    return parser.parse_args(raw_args)

def get_latest_kernel_version(kind='stable'):
    kernel_url = 'https://www.kernel.org/finger_banner'
    search_string = 'The latest {0}'.format(kind)
    with closing(urlopen(kernel_url)) as handle:
        for raw_line in handle.readlines():
            line = raw_line.decode('utf-8').rstrip('\n')
            if search_string in line:
                return str(line.rstrip(' (EOL)').rsplit(' ', 1)[1])
    raise LookupError('Could not find the latest {0} kernel'.format(kind))

def get_current_kernel_version():
    return os.uname()[2].split('_', 1)[0]

def get_current_kernel_revision():
    current_kernel = os.uname()[2].rsplit('_', 1)
    if len(current_kernel) < 2:
        return '0'
    current_revision = current_kernel[1]
    if all(x in set(string.digits) for x in current_revision):
        return current_revision
    else:
        return '0'

def devnull():
    return open(os.devnull, 'w')

def progress_bar(current, goal):
    marker_width = width - 7
    percent = round(current / goal, 2)
    mark_count = int(round(marker_width * percent))
    text_bar = '{0:3}% [{1}{2}]'.format(
        int(percent * 100),
        '*' * mark_count,
        ' ' * (marker_width - mark_count),
    )
    try:
        print('\r' + text_bar, end='')
    except:
        sys.stdout.flush()

def download_progress(current_block, block_size, total_size):
    if current_block % 5 == 0:
        current_size = min(current_block * block_size, total_size)
        progress_bar(current_size, total_size)

def extract_progress(extracted_count, total_count):
    if extracted_count % 50 == 0:
        progress_bar(extracted_count, total_count)

def require_attr(attribute):
    def decorator(method):
        @functools.wraps(method)
        def wrapper(self, *args, **kwargs):
            if getattr(self, attribute, None) is None:
                raise LookupError(
                    'Required attribute is unset: {0}'.format(attribute)
                )
            method(self, *args, **kwargs)
        return wrapper
    return decorator

class TarFileWithProgress(tarfile.TarFile):
    def __init__(self, *args, **kwargs):
        self.callback = kwargs.pop('callback', None)
        super().__init__(*args, **kwargs)
        if self.callback is not None:
            self._total_count = len(self.getmembers())
            self._extracted_count = 0

    def extract(self, *args, **kwargs):
        if self.callback is not None:
            self.callback(self._extracted_count, self._total_count)
            self._extracted_count = self._extracted_count + 1
        super().extract(*args, **kwargs)

class Kernel(object):
    def __init__(self, build_dir=None, config_dir=None, verbose=True):
        self.version = None
        self.revision = None
        self.config_version = None
        self.config_revision = None
        self.verbose = verbose

        if build_dir is None:
            self.build_dir = os.path.dirname(sys.argv[0])
        else:
            self.build_dir = os.path.expanduser(build_dir.rstrip('/'))
        self.build_dir = os.path.abspath(self.build_dir)

        if config_dir is None:
            self.config_dir = self.build_dir + '/configs'
        else:
            self.config_dir = os.path.expanduser(config_dir.rstrip('/'))
        self.config_dir = os.path.abspath(self.config_dir)
        if not os.path.isdir(self.config_dir):
            os.makedirs(self.config_dir, 0o755)

        for subdir in ['/sources', '/archives']:
            if not os.path.isdir(self.build_dir + subdir):
                os.makedirs(self.build_dir + subdir, 0o755)

        raw_configs = [
            x.split('_')
            for x in os.listdir(self.config_dir)
            if '_' in x
        ]
        self.existing_configs = {
            x[0]: [] for x in raw_configs
        }
        for config in raw_configs:
            self.existing_configs[config[0]].append((config[1]))

    def log(self, message):
        if self.verbose:
            print(message)

    @require_attr('version')
    def download(self):
        if self.version[0] == '2':
            major = self.version[0:3]
        else:
            major = '3.x'
        if 'rc' in self.version:
            testing = 'testing/'
        else:
            testing = ''

        destination = '{0}/archives/linux-{1}.tar.gz'.format(
            self.build_dir,
            self.version
        )
        base_url = 'http://www.kernel.org/pub/linux/kernel'
        source = '{0}/v{1}/{2}linux-{3}.tar.gz'.format(
            base_url,
            major,
            testing,
            self.version
        )

        if os.path.isfile(destination):
            self.log('Kernel already downloaded: {0}'.format(self.version))
            return
        self.log('Downloading kernel: {0}'.format(self.version))
        if self.verbose:
            hook = download_progress
        else:
            hook = None
        try:
            urlretrieve(
                source,
                filename=destination,
                reporthook=hook
            )
        except:
            os.remove(destination)
            raise

    @require_attr('version')
    def extract(self):
        destination = '{0}/sources/'.format(self.build_dir)
        source = '{0}/archives/linux-{1}.tar.gz'.format(
            self.build_dir,
            self.version
        )

        if os.path.isdir('{0}linux-{1}'.format(destination, self.version)):
            self.log('Kernel already extracted')
            return
        if not os.path.isfile(source):
            raise EnvironmentError('Archived kernel does not exist')
        self.log('Extracting kernel')
        if self.verbose:
            callback = extract_progress
        else:
            callback = None
        try:
            print("Trying to open {0}".format(source))
            archive = TarFileWithProgress.open(source, callback=callback)
            archive.extractall(destination)
        except:
            shutil.rmtree(
                '{0}linux-{1}'.format(destination, self.version),
                ignore_errors=True
            )
            raise

    @require_attr('version')
    @require_attr('revision')
    @require_attr('config_version')
    @require_attr('config_revision')
    def configure(self, merge_method='oldconfig'):
        os.chdir('{0}/sources/linux-{1}'.format(self.build_dir, self.version))
        self.log('Cleaning your kernel tree')
        try:
            subprocess.call(['make', 'mrproper'], stdout=devnull())
        except:
            raise EnvironmentError('Failed to clean your kernel tree')
        if self.config_revision == 'current':
            self.log('Inserting config from current system kernel')
            with gzip.open('/proc/config.gz') as old_config:
                with open('.config', 'wb') as new_config:
                    new_config.write(old_config.read())
        else:
            self.log('Copying saved config: {0}_{1}'.format(
                self.config_version,
                self.config_revision,
            ))
            shutil.copy(
                '{0}/{1}_{2}'.format(
                    self.config_dir,
                    self.config_version,
                    self.config_revision
                ),
                '.config'
            )
        done = False
        for line in fileinput.input('.config', inplace=True):
            if not done and line.find('CONFIG_LOCALVERSION') == 0:
                print('CONFIG_LOCALVERSION="_{0}"'.format(self.revision))
                done = True
            else:
                print(line.rstrip())
        if self.config_version != self.version:
            self.log('Merging your kernel config via "{0}"'.format(
                merge_method)
            )
            subprocess.call(['make', merge_method])

    @require_attr('version')
    @require_attr('revision')
    def modify(self):
        os.chdir('{0}/sources/linux-{1}'.format(self.build_dir, self.version))
        self.log('Running menuconfig')
        subprocess.call(['make', 'menuconfig'])
        self.log('Saving configuration: {0}_{1}'.format(
            self.version,
            self.revision
        ))
        shutil.copy(
            '.config',
            '{0}/{1}_{2}'.format(
                self.config_dir,
                self.version,
                self.revision,
            ),
        )

    @require_attr('version')
    def make(self, jobs=None, background=True):
        os.chdir('{0}/sources/linux-{1}'.format(self.build_dir, self.version))
        if background:
            stdout = devnull()
        else:
            stdout = None
        if jobs is None:
            jobs = str(multiprocessing.cpu_count())
        self.log('Making the kernel')
        make_process = subprocess.Popen(
            ['make', '-j' + jobs],
            stdout=stdout,
            stderr=stdout,
        )
        while make_process.poll() is None:
            if self.verbose:
                progress_bar(random.randint(1, 99), 100)
            time.sleep(1)
        if make_process.returncode != 0:
            print('Failed to make kernel')
            raise SystemExit

    @require_attr('version')
    @require_attr('revision')
    def install(self):
        os.chdir('{0}/sources/linux-{1}'.format(self.build_dir, self.version))
        self.log('Installing the kernel image')
        if not os.path.isdir('/boot'):
            os.makedirs('/boot', 0o755, True)
        shutil.copy(
            'arch/x86/boot/bzImage',
            '/boot/vmlinuz-{0}_{1}'.format(self.version, self.revision)
        )
        try:
            with open('/etc/fstab') as handle:
                device = [
                    x.split()[0]
                    for x in handle.readlines()
                    if 'ext' in x
                ][0]
        except IndexError:
            device = '/dev/xvda'
        try:
            int(device[-1])
        except ValueError:
            hd = '(hd0)'
        else:
            hd = '(hd0,0)'
        grub_config = '''
title {0}_{1}
root {3}
kernel /boot/vmlinuz-{0}_{1} root={2} ro\n'''.format(
            self.version, self.revision, device, hd)
        if not os.path.isdir('/boot/grub'):
            self.log('Making /boot/grub')
            os.makedirs('/boot/grub', 0o755, True)
        if not os.path.isfile('/boot/grub/menu.lst'):
            self.log('Creating initial menu.lst')
            with open('/boot/grub/menu.lst', 'w') as handle:
                handle.write('timeout 25\ndefault 0\n\n#START\n')
        self.log('Inserting new kernel into menu.lst')
        done = False
        for line in fileinput.input('/boot/grub/menu.lst', inplace=True):
            print(line.rstrip())
            if not done and line == '#START\n':
                print(grub_config)
                done = True
        if not done:
            raise EnvironmentError('Failed to update /boot/grub/menu.lst')

    def where(self):
        print('{0}/sources/linux-{1}/arch/x86/boot/bzImage'.format(
            self.build_dir, self.version
        ))

    def cleanup(self):
        self.log('Cleaning old archives and sources')
        for archive in os.listdir(self.build_dir + '/archives'):
            os.remove(self.build_dir + '/archives/' + archive)
        for source in os.listdir(self.build_dir + '/sources'):
            shutil.rmtree(self.build_dir + '/sources/' + source)

def easy_roll(raw_args):
    args = get_args(raw_args)
    kernel = Kernel(
        build_dir=args.build_dir,
        config_dir=args.config_dir,
        verbose=args.verbose
    )

    kernel.version = args.new_version
    kernel.config_version = args.config_version
    kernel.config_revision = args.config_revision
    if args.new_revision == 'next':
        if args.new_version in kernel.existing_configs:
            kernel.revision = str(
                int(max(kernel.existing_configs[args.new_version])) + 1
            )
        else:
            kernel.revision = '1'
        modify = True
    elif args.new_revision is None:
        if args.new_version == args.config_version:
            if args.config_revision == 'current':
                kernel.revision = get_current_kernel_revision()
            else:
                kernel.revision = args.config_revision
        else:
            kernel.revision = '0'
        modify = False
    else:
        kernel.revision = args.new_revision
        modify = True

    kernel.download()
    kernel.extract()
    if args.patch:
        print('Dropping into a bash shell for patching; `exit` to continue')
        system('bash')
    kernel.configure()
    if modify:
        kernel.modify()
    kernel.make()
    if args.skip_install:
        kernel.where()
    else:
        kernel.install()

if __name__ == '__main__':
    easy_roll(sys.argv[1:])
<commit_msg>fix verbose fail<commit_after>#!/usr/bin/env python3
from __future__ import print_function

VERSION = '0.4.9'

import os
import sys
import shutil
import fileinput
import functools
import tarfile
import gzip
import subprocess
import multiprocessing
import argparse
import string
import random
import time
from contextlib import closing

try:
    from urllib.request import urlopen, urlretrieve
except ImportError:
    from urllib2 import urlopen
    from urllib import urlretrieve

try:
    width = shutil.get_terminal_size((40, 0)).columns
except AttributeError:
    width = 40

def get_args(raw_args):
    parser = argparse.ArgumentParser(
        description='Simplified kernel rolling tool'
    )
    parser.add_argument(
        '--version',
        action='version',
        version=VERSION
    )
    parser.add_argument(
        '-v', '--verbose',
        action='store_true',
        help='Increase verbosity of output'
    )
    parser.add_argument(
        '-k', '--kernel',
        dest='new_version',
        default=get_latest_kernel_version(),
        help='Kernel version to build',
    )
    parser.add_argument(
        '-n', '--new-revision',
        dest='new_revision',
        type=str,
        default=None,
        help='Kernel revision to create',
    )
    parser.add_argument(
        '-c', '--config',
        dest='config_version',
        default=get_current_kernel_version(),
        help='Config version to work from',
    )
    parser.add_argument(
        '-r', '--config-revision',
        dest='config_revision',
        type=str,
        default='current',
        help='Config revision to work from',
    )
    parser.add_argument(
        '-s', '--skip-install',
        dest='skip_install',
        action='store_true',
        help='Do not install kernel to /boot'
    )
    parser.add_argument(
        '-p', '--patch',
        dest='patch',
        action='store_true',
        help='Open a shell before configuration to allow patching'
    )
    parser.add_argument(
        '-b', '--build-dir',
        dest='build_dir',
        type=str,
        default='/tmp',
        help='directory for downloading, extracting, and building the kernel'
    )
    parser.add_argument(
        '-d', '--config-dir',
        dest='config_dir',
        type=str,
        default=None,
        help='directory for kernel configs'
    )

    return parser.parse_args(raw_args)

def get_latest_kernel_version(kind='stable'):
    kernel_url = 'https://www.kernel.org/finger_banner'
    search_string = 'The latest {0}'.format(kind)
    with closing(urlopen(kernel_url)) as handle:
        for raw_line in handle.readlines():
            line = raw_line.decode('utf-8').rstrip('\n')
            if search_string in line:
                return str(line.rstrip(' (EOL)').rsplit(' ', 1)[1])
    raise LookupError('Could not find the latest {0} kernel'.format(kind))

def get_current_kernel_version():
    return os.uname()[2].split('_', 1)[0]

def get_current_kernel_revision():
    current_kernel = os.uname()[2].rsplit('_', 1)
    if len(current_kernel) < 2:
        return '0'
    current_revision = current_kernel[1]
    if all(x in set(string.digits) for x in current_revision):
        return current_revision
    else:
        return '0'

def devnull():
    return open(os.devnull, 'w')

def progress_bar(current, goal):
    marker_width = width - 7
    percent = round(current / goal, 2)
    mark_count = int(round(marker_width * percent))
    text_bar = '{0:3}% [{1}{2}]'.format(
        int(percent * 100),
        '*' * mark_count,
        ' ' * (marker_width - mark_count),
    )
    try:
        print('\r' + text_bar, end='')
    except:
        sys.stdout.flush()

def download_progress(current_block, block_size, total_size):
    if current_block % 5 == 0:
        current_size = min(current_block * block_size, total_size)
        progress_bar(current_size, total_size)

def extract_progress(extracted_count, total_count):
    if extracted_count % 50 == 0:
        progress_bar(extracted_count, total_count)

def require_attr(attribute):
    def decorator(method):
        @functools.wraps(method)
        def wrapper(self, *args, **kwargs):
            if getattr(self, attribute, None) is None:
                raise LookupError(
                    'Required attribute is unset: {0}'.format(attribute)
                )
            method(self, *args, **kwargs)
        return wrapper
    return decorator

class TarFileWithProgress(tarfile.TarFile):
    def __init__(self, *args, **kwargs):
        self.callback = kwargs.pop('callback', None)
        super().__init__(*args, **kwargs)
        if self.callback is not None:
            self._total_count = len(self.getmembers())
            self._extracted_count = 0

    def extract(self, *args, **kwargs):
        if self.callback is not None:
            self.callback(self._extracted_count, self._total_count)
            self._extracted_count = self._extracted_count + 1
        super().extract(*args, **kwargs)

class Kernel(object):
    def __init__(self, build_dir=None, config_dir=None, verbose=True):
        self.version = None
        self.revision = None
        self.config_version = None
        self.config_revision = None
        self.verbose = verbose

        if build_dir is None:
            self.build_dir = os.path.dirname(sys.argv[0])
        else:
            self.build_dir = os.path.expanduser(build_dir.rstrip('/'))
        self.build_dir = os.path.abspath(self.build_dir)

        if config_dir is None:
            self.config_dir = self.build_dir + '/configs'
        else:
            self.config_dir = os.path.expanduser(config_dir.rstrip('/'))
        self.config_dir = os.path.abspath(self.config_dir)
        if not os.path.isdir(self.config_dir):
            os.makedirs(self.config_dir, 0o755)

        for subdir in ['/sources', '/archives']:
            if not os.path.isdir(self.build_dir + subdir):
                os.makedirs(self.build_dir + subdir, 0o755)

        raw_configs = [
            x.split('_')
            for x in os.listdir(self.config_dir)
            if '_' in x
        ]
        self.existing_configs = {
            x[0]: [] for x in raw_configs
        }
        for config in raw_configs:
            self.existing_configs[config[0]].append((config[1]))

    def log(self, message):
        if self.verbose:
            print(message)

    @require_attr('version')
    def download(self):
        if self.version[0] == '2':
            major = self.version[0:3]
        else:
            major = '3.x'
        if 'rc' in self.version:
            testing = 'testing/'
        else:
            testing = ''

        destination = '{0}/archives/linux-{1}.tar.gz'.format(
            self.build_dir,
            self.version
        )
        base_url = 'http://www.kernel.org/pub/linux/kernel'
        source = '{0}/v{1}/{2}linux-{3}.tar.gz'.format(
            base_url,
            major,
            testing,
            self.version
        )

        if os.path.isfile(destination):
            self.log('Kernel already downloaded: {0}'.format(self.version))
            return
        self.log('Downloading kernel: {0}'.format(self.version))
        if self.verbose:
            hook = download_progress
        else:
            hook = None
        try:
            urlretrieve(
                source,
                filename=destination,
                reporthook=hook
            )
        except:
            os.remove(destination)
            raise

    @require_attr('version')
    def extract(self):
        destination = '{0}/sources/'.format(self.build_dir)
        source = '{0}/archives/linux-{1}.tar.gz'.format(
            self.build_dir,
            self.version
        )

        if os.path.isdir('{0}linux-{1}'.format(destination, self.version)):
            self.log('Kernel already extracted')
            return
        if not os.path.isfile(source):
            raise EnvironmentError('Archived kernel does not exist')
        self.log('Extracting kernel')
        if self.verbose:
            callback = extract_progress
        else:
            callback = None
        try:
            self.log("Trying to open {0}".format(source))
            archive = TarFileWithProgress.open(source, callback=callback)
            archive.extractall(destination)
        except:
            shutil.rmtree(
                '{0}linux-{1}'.format(destination, self.version),
                ignore_errors=True
            )
            raise

    @require_attr('version')
    @require_attr('revision')
    @require_attr('config_version')
    @require_attr('config_revision')
    def configure(self, merge_method='oldconfig'):
        os.chdir('{0}/sources/linux-{1}'.format(self.build_dir, self.version))
        self.log('Cleaning your kernel tree')
        try:
            subprocess.call(['make', 'mrproper'], stdout=devnull())
        except:
            raise EnvironmentError('Failed to clean your kernel tree')
        if self.config_revision == 'current':
            self.log('Inserting config from current system kernel')
            with gzip.open('/proc/config.gz') as old_config:
                with open('.config', 'wb') as new_config:
                    new_config.write(old_config.read())
        else:
            self.log('Copying saved config: {0}_{1}'.format(
                self.config_version,
                self.config_revision,
            ))
            shutil.copy(
                '{0}/{1}_{2}'.format(
                    self.config_dir,
                    self.config_version,
                    self.config_revision
                ),
                '.config'
            )
        done = False
        for line in fileinput.input('.config', inplace=True):
            if not done and line.find('CONFIG_LOCALVERSION') == 0:
                print('CONFIG_LOCALVERSION="_{0}"'.format(self.revision))
                done = True
            else:
                print(line.rstrip())
        if self.config_version != self.version:
            self.log('Merging your kernel config via "{0}"'.format(
                merge_method)
            )
            subprocess.call(['make', merge_method])

    @require_attr('version')
    @require_attr('revision')
    def modify(self):
        os.chdir('{0}/sources/linux-{1}'.format(self.build_dir, self.version))
        self.log('Running menuconfig')
        subprocess.call(['make', 'menuconfig'])
        self.log('Saving configuration: {0}_{1}'.format(
            self.version,
            self.revision
        ))
        shutil.copy(
            '.config',
            '{0}/{1}_{2}'.format(
                self.config_dir,
                self.version,
                self.revision,
            ),
        )

    @require_attr('version')
    def make(self, jobs=None, background=True):
        os.chdir('{0}/sources/linux-{1}'.format(self.build_dir, self.version))
        if background:
            stdout = devnull()
        else:
            stdout = None
        if jobs is None:
            jobs = str(multiprocessing.cpu_count())
        self.log('Making the kernel')
        make_process = subprocess.Popen(
            ['make', '-j' + jobs],
            stdout=stdout,
            stderr=stdout,
        )
        while make_process.poll() is None:
            if self.verbose:
                progress_bar(random.randint(1, 99), 100)
            time.sleep(1)
        if make_process.returncode != 0:
            print('Failed to make kernel')
            raise SystemExit

    @require_attr('version')
    @require_attr('revision')
    def install(self):
        os.chdir('{0}/sources/linux-{1}'.format(self.build_dir, self.version))
        self.log('Installing the kernel image')
        if not os.path.isdir('/boot'):
            os.makedirs('/boot', 0o755, True)
        shutil.copy(
            'arch/x86/boot/bzImage',
            '/boot/vmlinuz-{0}_{1}'.format(self.version, self.revision)
        )
        try:
            with open('/etc/fstab') as handle:
                device = [
                    x.split()[0]
                    for x in handle.readlines()
                    if 'ext' in x
                ][0]
        except IndexError:
            device = '/dev/xvda'
        try:
            int(device[-1])
        except ValueError:
            hd = '(hd0)'
        else:
            hd = '(hd0,0)'
        grub_config = '''
title {0}_{1}
root {3}
kernel /boot/vmlinuz-{0}_{1} root={2} ro\n'''.format(
            self.version, self.revision, device, hd)
        if not os.path.isdir('/boot/grub'):
            self.log('Making /boot/grub')
            os.makedirs('/boot/grub', 0o755, True)
        if not os.path.isfile('/boot/grub/menu.lst'):
            self.log('Creating initial menu.lst')
            with open('/boot/grub/menu.lst', 'w') as handle:
                handle.write('timeout 25\ndefault 0\n\n#START\n')
        self.log('Inserting new kernel into menu.lst')
        done = False
        for line in fileinput.input('/boot/grub/menu.lst', inplace=True):
            print(line.rstrip())
            if not done and line == '#START\n':
                print(grub_config)
                done = True
        if not done:
            raise EnvironmentError('Failed to update /boot/grub/menu.lst')

    def where(self):
        print('{0}/sources/linux-{1}/arch/x86/boot/bzImage'.format(
            self.build_dir, self.version
        ))

    def cleanup(self):
        self.log('Cleaning old archives and sources')
        for archive in os.listdir(self.build_dir + '/archives'):
            os.remove(self.build_dir + '/archives/' + archive)
        for source in os.listdir(self.build_dir + '/sources'):
            shutil.rmtree(self.build_dir + '/sources/' + source)

def easy_roll(raw_args):
    args = get_args(raw_args)
    kernel = Kernel(
        build_dir=args.build_dir,
        config_dir=args.config_dir,
        verbose=args.verbose
    )

    kernel.version = args.new_version
    kernel.config_version = args.config_version
    kernel.config_revision = args.config_revision
    if args.new_revision == 'next':
        if args.new_version in kernel.existing_configs:
            kernel.revision = str(
                int(max(kernel.existing_configs[args.new_version])) + 1
            )
        else:
            kernel.revision = '1'
        modify = True
    elif args.new_revision is None:
        if args.new_version == args.config_version:
            if args.config_revision == 'current':
                kernel.revision = get_current_kernel_revision()
            else:
                kernel.revision = args.config_revision
        else:
            kernel.revision = '0'
        modify = False
    else:
        kernel.revision = args.new_revision
        modify = True

    kernel.download()
    kernel.extract()
    if args.patch:
        print('Dropping into a bash shell for patching; `exit` to continue')
        system('bash')
    kernel.configure()
    if modify:
        kernel.modify()
    kernel.make()
    if args.skip_install:
        kernel.where()
    else:
        kernel.install()

if __name__ == '__main__':
    easy_roll(sys.argv[1:])
<|endoftext|>


<commit_before>import os

PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__))
DEBUG = True
ROOT_URLCONF = 'proxly.urls'
INSTALLED_APPS = ( 'bettercache', )

CACHES = {
        'default': {
            'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
        },
}

BETTERCACHE_LOCAL_POSTCHECK = 120
DATABASES = {
    'default': {
                'ENGINE': 'sqlite3',
                'NAME': ':memory:',
    }
}
ADMINS = ( )
TIME_ZONE = 'America/Chicago'
LANGUAGE_CODE = 'en-us'
SITE_ID = 1
USE_I18N = False
USE_L10N = False
TEMPLATE_LOADERS = (
            ('django.template.loaders.cached.Loader', (
                'django.template.loaders.filesystem.Loader',
            )),
)
TEMPLATE_DIRS = ( os.path.join(PROJECT_ROOT, 'templates'),)
MIDDLEWARE_CLASSES = ( )
<commit_msg>typo for proxley in test settings<commit_after>import os

PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__))
DEBUG = True
ROOT_URLCONF = 'proxley.urls'
INSTALLED_APPS = ( 'bettercache', )

CACHES = {
        'default': {
            'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
        },
}

BETTERCACHE_LOCAL_POSTCHECK = 120
DATABASES = {
    'default': {
                'ENGINE': 'sqlite3',
                'NAME': ':memory:',
    }
}
ADMINS = ( )
TIME_ZONE = 'America/Chicago'
LANGUAGE_CODE = 'en-us'
SITE_ID = 1
USE_I18N = False
USE_L10N = False
TEMPLATE_LOADERS = (
            ('django.template.loaders.cached.Loader', (
                'django.template.loaders.filesystem.Loader',
            )),
)
TEMPLATE_DIRS = ( os.path.join(PROJECT_ROOT, 'templates'),)
MIDDLEWARE_CLASSES = ( )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals
import os
import subprocess
import locale
import const as c

def java_version():
    """Call java and return version information.

    :return unicode: Java version string
    """
    result = subprocess.check_output(
        [c.JAVA, '-version'], stderr=subprocess.STDOUT
    )
    first_line = result.splitlines()[0]
    return first_line.decode()

def epubcheck_help():
    """Return epubcheck.jar commandline help text.

    :return unicode: helptext from epubcheck.jar
    """

    tc = locale.getdefaultlocale()[1]

    with open(os.devnull, "w") as devnull:
        p = subprocess.Popen(
            [c.JAVA, '-jar', c.EPUBCHECK, '-h'],
            stdout=subprocess.PIPE,
            stderr=devnull,
        )
        result = p.communicate()[0]

    return result.decode(tc)

def epubcheck_version():
    """Call epubcheck -h and return helptext.

    :return unicode: Epubcheck verstion string
    """
    return epubcheck_help().splitlines()[0]

if __name__ == "__main__":
    print(epubcheck_help())
<commit_msg>fix ImportError: No module named 'const' 2<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals
import os
import subprocess
import locale
from epubcheck import const as c

def java_version():
    """Call java and return version information.

    :return unicode: Java version string
    """
    result = subprocess.check_output(
        [c.JAVA, '-version'], stderr=subprocess.STDOUT
    )
    first_line = result.splitlines()[0]
    return first_line.decode()

def epubcheck_help():
    """Return epubcheck.jar commandline help text.

    :return unicode: helptext from epubcheck.jar
    """

    tc = locale.getdefaultlocale()[1]

    with open(os.devnull, "w") as devnull:
        p = subprocess.Popen(
            [c.JAVA, '-jar', c.EPUBCHECK, '-h'],
            stdout=subprocess.PIPE,
            stderr=devnull,
        )
        result = p.communicate()[0]

    return result.decode(tc)

def epubcheck_version():
    """Call epubcheck -h and return helptext.

    :return unicode: Epubcheck verstion string
    """
    return epubcheck_help().splitlines()[0]

if __name__ == "__main__":
    print(epubcheck_help())
<|endoftext|>


<commit_before>import unittest

import pytest

import cupy
from cupy import testing

import cupyx.scipy.signal

try:
    import scipy.signal  # NOQA
except ImportError:
    pass

@testing.parameterize(*testing.product({
    'size1': [(10,), (5, 10), (10, 3), (3, 4, 10)],
    'size2': [3, 4, 5, 10],
    'mode': ['full', 'same', 'valid'],
}))
@testing.gpu
@testing.with_requires('scipy')
class TestConvolveCorrelate(unittest.TestCase):
    def _filter(self, func, dtype, xp, scp):
        in1 = testing.shaped_random(self.size1, xp, dtype)
        in2 = testing.shaped_random((self.size2,)*in1.ndim, xp, dtype)
        return getattr(scp.signal, func)(in1, in2, self.mode, method='direct')

    # TODO: support complex
    # Note: float16 is tested separately
    @testing.for_all_dtypes(no_float16=True, no_complex=True)
    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
                                 accept_error=ValueError)
    def test_convolve(self, xp, scp, dtype):
        return self._filter('convolve', dtype, xp, scp)

    # TODO: support complex
    # Note: float16 is tested separately
    @testing.for_all_dtypes(no_float16=True, no_complex=True)
    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
                                 accept_error=ValueError)
    def test_correlate(self, xp, scp, dtype):
        return self._filter('correlate', dtype, xp, scp)

    # float16 has significantly worse error tolerances
    @testing.numpy_cupy_allclose(atol=1e-3, rtol=1e-3, scipy_name='scp',
                                 accept_error=ValueError)
    def test_convolve_float16(self, xp, scp, dtype=cupy.float16):
        return self._filter('convolve', dtype, xp, scp)

    # float16 has significantly worse error tolerances
    @testing.numpy_cupy_allclose(atol=1e-3, rtol=1e-3, scipy_name='scp',
                                 accept_error=ValueError)
    def test_correlate_float16(self, xp, scp, dtype=cupy.float16):
        return self._filter('correlate', dtype, xp, scp)

@testing.parameterize(*testing.product({
    'size1': [(10,), (5, 10), (10, 3), (3, 4, 10)],
    'size2': [3, 4, 5, 10],
    'mode': ['full', 'same', 'valid'],
}))
@testing.gpu
@testing.with_requires('scipy')
class TestFFTConvolve(unittest.TestCase):
    def _filter(self, func, dtype, xp, scp, **kwargs):
        in1 = testing.shaped_random(self.size1, xp, dtype)
        in2 = testing.shaped_random((self.size2,)*in1.ndim, xp, dtype)
        return getattr(scp.signal, func)(in1, in2, self.mode, **kwargs)

    # Note: float16 is tested separately
    @testing.for_all_dtypes(no_float16=True)
    @testing.numpy_cupy_allclose(atol=1e-4, rtol=5e-4, scipy_name='scp',
                                 accept_error=ValueError)
    def test_fftconvolve(self, xp, scp, dtype):
        return self._filter('fftconvolve', dtype, xp, scp)

    # float16 has significantly worse error tolerances
    @testing.numpy_cupy_allclose(atol=1e-3, rtol=1e-3, scipy_name='scp',
                                 accept_error=ValueError)
    def test_fftconvolve_float16(self, xp, scp, dtype=cupy.float16):
        return self._filter('fftconvolve', dtype, xp, scp)

    # Note: float16 is tested separately
    @testing.for_all_dtypes(no_float16=True, no_bool=True)
    @testing.numpy_cupy_allclose(atol=1e-4, rtol=5e-4, scipy_name='scp',
                                 accept_error=ValueError)
    def test_convolve_fft(self, xp, scp, dtype):
        return self._filter('convolve', dtype, xp, scp, method='fft')

    # float16 has significantly worse error tolerances
    @testing.numpy_cupy_allclose(atol=1e-3, rtol=1e-3, scipy_name='scp',
                                 accept_error=ValueError)
    def test_convolve_fft_float16(self, xp, scp, dtype=cupy.float16):
        return self._filter('convolve', dtype, xp, scp, method='fft')

    # Note: float16 is tested separately
    @testing.for_all_dtypes(no_float16=True, no_bool=True)
    @testing.numpy_cupy_allclose(atol=1e-4, rtol=5e-4, scipy_name='scp',
                                 accept_error=ValueError)
    def test_correlate_fft(self, xp, scp, dtype):
        return self._filter('correlate', dtype, xp, scp, method='fft')

    # float16 has significantly worse error tolerances
    @testing.numpy_cupy_allclose(atol=1e-3, rtol=1e-3, scipy_name='scp',
                                 accept_error=ValueError)
    def test_correlate_fft_float16(self, xp, scp, dtype=cupy.float16):
        return self._filter('correlate', dtype, xp, scp, method='fft')

@testing.parameterize(*(testing.product({
    'size1': [(5, 10), (10, 7)],
    'size2': [(3, 2), (3, 3), (2, 2), (10, 10), (11, 11)],
    'mode': ['full', 'same', 'valid'],
    'boundary': ['fill'],
    'fillvalue': [0, 1, -1],
}) + testing.product({
    'size1': [(5, 10), (10, 7)],
    'size2': [(3, 2), (3, 3), (2, 2), (10, 10), (11, 11)],
    'mode': ['full', 'same', 'valid'],
    'boundary': ['wrap', 'symm'],
    'fillvalue': [0],
})))
@testing.gpu
@testing.with_requires('scipy')
class TestConvolveCorrelate2D(unittest.TestCase):
    def _filter(self, func, dtype, xp, scp):
        if self.mode == 'full' and self.boundary != 'constant':
            # See https://github.com/scipy/scipy/issues/12685
            raise unittest.SkipTest('broken in scipy')
        in1 = testing.shaped_random(self.size1, xp, dtype)
        in2 = testing.shaped_random(self.size2, xp, dtype)
        return getattr(scp.signal, func)(in1, in2, self.mode, self.boundary,
                                         self.fillvalue)

    # TODO: support complex
    # Note: float16 is tested separately
    @testing.for_all_dtypes(no_float16=True, no_complex=True)
    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
                                 accept_error=ValueError)
    def test_convolve2d(self, xp, scp, dtype):
        return self._filter('convolve2d', dtype, xp, scp)

    # TODO: support complex
    # Note: float16 is tested separately
    @testing.for_all_dtypes(no_float16=True, no_complex=True)
    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
                                 accept_error=ValueError)
    def test_correlate2d(self, xp, scp, dtype):
        return self._filter('correlate2d', dtype, xp, scp)

    # float16 has significantly worse error tolerances
    @testing.numpy_cupy_allclose(atol=1e-3, rtol=1e-3, scipy_name='scp',
                                 accept_error=ValueError)
    def test_convolve2d_float16(self, xp, scp, dtype=cupy.float16):
        return self._filter('convolve2d', dtype, xp, scp)

    # float16 has significantly worse error tolerances
    @testing.numpy_cupy_allclose(atol=1e-3, rtol=1e-3, scipy_name='scp',
                                 accept_error=ValueError)
    def test_correlate2d_float16(self, xp, scp, dtype=cupy.float16):
        return self._filter('correlate2d', dtype, xp, scp)

@testing.gpu
@testing.parameterize(*testing.product({
    'mode': ['valid', 'same', 'full']
}))
class TestChooseConvMethod(unittest.TestCase):

    @testing.for_dtypes('efdFD')
    def test_choose_conv_method1(self, dtype):
        a = testing.shaped_arange((10000,), cupy, dtype)
        b = testing.shaped_arange((5000,), cupy, dtype)
        assert cupyx.scipy.signal.choose_conv_method(
            a, b, mode=self.mode) == 'fft'

    @testing.for_dtypes('efdFD')
    def test_choose_conv_method2(self, dtype):
        a = testing.shaped_arange((5000,), cupy, dtype)
        b = testing.shaped_arange((10000,), cupy, dtype)
        assert cupyx.scipy.signal.choose_conv_method(
            a, b, mode=self.mode) == 'fft'

    @testing.for_int_dtypes()
    def test_choose_conv_method_int(self, dtype):
        a = testing.shaped_arange((10,), cupy, dtype)
        b = testing.shaped_arange((5,), cupy, dtype)
        assert cupyx.scipy.signal.choose_conv_method(
            a, b, mode=self.mode) == 'direct'

    @testing.for_all_dtypes()
    def test_choose_conv_method_ndim(self, dtype):
        a = testing.shaped_arange((3, 4, 5), cupy, dtype)
        b = testing.shaped_arange((1, 2), cupy, dtype)
        with pytest.raises(NotImplementedError):
            cupyx.scipy.signal.choose_conv_method(a, b, mode=self.mode)

    @testing.for_all_dtypes()
    def test_choose_conv_method_zero_dim(self, dtype):
        a = testing.shaped_arange((), cupy, dtype)
        b = testing.shaped_arange((5,), cupy, dtype)
        with pytest.raises(NotImplementedError):
            cupyx.scipy.signal.choose_conv_method(a, b, mode=self.mode)

@testing.parameterize(*testing.product({
    'im': [(10,), (5, 10), (10, 3), (3, 4, 10)],
    'mysize': [3, 4, (3, 4, 5)],
    'noise': [False, True],
}))
@testing.gpu
@testing.with_requires('scipy')
class TestWiener(unittest.TestCase):
    # TODO: support complex
    # Note: float16 is tested separately
    @testing.for_all_dtypes(no_float16=True, no_complex=True)
    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
    def test_wiener(self, xp, scp, dtype):
        im = testing.shaped_random(self.im, xp, dtype)
        mysize = self.mysize
        if isinstance(mysize, tuple):
            mysize = mysize[:im.ndim]
        noise = (testing.shaped_random(self.im, xp, dtype)
                 if self.noise else None)
        return scp.signal.wiener(im, mysize, noise)

    # float16 has significantly worse error tolerances
    @testing.numpy_cupy_allclose(atol=1e-3, rtol=1e-3, scipy_name='scp')
    def test_wiener_float16(self, xp, scp, dtype=cupy.float16):
        im = testing.shaped_random(self.im, xp, dtype)
        mysize = self.mysize
        if isinstance(mysize, tuple):
            mysize = mysize[:im.ndim]
        noise = (testing.shaped_random(self.im, xp, dtype)
                 if self.noise else None)
        return scp.signal.wiener(im, mysize, noise)

@testing.parameterize(*testing.product({
    'a': [(10,), (5, 10), (10, 3), (3, 4, 10)],
    'domain': [3, 4, (3, 3, 5)],
    'rank': [0, 1, 2],
}))
@testing.gpu
@testing.with_requires('scipy')
class TestOrderFilter(unittest.TestCase):
    @testing.for_all_dtypes()
    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
                                 accept_error=ValueError)  # for even kernels
    def test_order_filter(self, xp, scp, dtype):
        a = testing.shaped_random(self.a, xp, dtype)
        d = self.domain
        d = d[:a.ndim] if isinstance(d, tuple) else (d,)*a.ndim
        domain = testing.shaped_random(d, xp) > 0.25
        rank = min(self.rank, domain.sum())
        return scp.signal.order_filter(a, domain, rank)

@testing.parameterize(*testing.product({
    'volume': [(10,), (5, 10), (10, 5), (5, 6, 10)],
    'kernel_size': [3, 4, (3, 3, 5)],
}))
@testing.gpu
@testing.with_requires('scipy')
class TestMedFilt(unittest.TestCase):
    @testing.for_all_dtypes()
    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
                                 accept_error=ValueError)  # for even kernels
    def test_medfilt(self, xp, scp, dtype):
        volume = testing.shaped_random(self.volume, xp, dtype)
        kernel_size = self.kernel_size
        if isinstance(kernel_size, tuple):
            kernel_size = kernel_size[:volume.ndim]
        return scp.signal.medfilt(volume, kernel_size)

@testing.parameterize(*testing.product({
    'input': [(5, 10), (10, 5)],
    'kernel_size': [3, 4, (3, 5)],
}))
@testing.gpu
@testing.with_requires('scipy')
class TestMedFilt2d(unittest.TestCase):
    @testing.for_all_dtypes()
    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
                                 accept_error=ValueError)  # for even kernels
    def test_medfilt2d(self, xp, scp, dtype):
        input = testing.shaped_random(self.input, xp, dtype)
        kernel_size = self.kernel_size
        return scp.signal.medfilt2d(input, kernel_size)
<commit_msg>Decrease sensitivity of tests due to float32 inaccuracies.<commit_after>import unittest

import pytest

import cupy
from cupy import testing

import cupyx.scipy.signal

try:
    import scipy.signal  # NOQA
except ImportError:
    pass

@testing.parameterize(*testing.product({
    'size1': [(10,), (5, 10), (10, 3), (3, 4, 10)],
    'size2': [3, 4, 5, 10],
    'mode': ['full', 'same', 'valid'],
}))
@testing.gpu
@testing.with_requires('scipy')
class TestConvolveCorrelate(unittest.TestCase):
    def _filter(self, func, dtype, xp, scp):
        in1 = testing.shaped_random(self.size1, xp, dtype)
        in2 = testing.shaped_random((self.size2,)*in1.ndim, xp, dtype)
        return getattr(scp.signal, func)(in1, in2, self.mode, method='direct')

    # TODO: support complex
    # Note: float16 is tested separately
    @testing.for_all_dtypes(no_float16=True, no_complex=True)
    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
                                 accept_error=ValueError)
    def test_convolve(self, xp, scp, dtype):
        return self._filter('convolve', dtype, xp, scp)

    # TODO: support complex
    # Note: float16 is tested separately
    @testing.for_all_dtypes(no_float16=True, no_complex=True)
    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
                                 accept_error=ValueError)
    def test_correlate(self, xp, scp, dtype):
        return self._filter('correlate', dtype, xp, scp)

    # float16 has significantly worse error tolerances
    @testing.numpy_cupy_allclose(atol=1e-3, rtol=1e-3, scipy_name='scp',
                                 accept_error=ValueError)
    def test_convolve_float16(self, xp, scp, dtype=cupy.float16):
        return self._filter('convolve', dtype, xp, scp)

    # float16 has significantly worse error tolerances
    @testing.numpy_cupy_allclose(atol=1e-3, rtol=1e-3, scipy_name='scp',
                                 accept_error=ValueError)
    def test_correlate_float16(self, xp, scp, dtype=cupy.float16):
        return self._filter('correlate', dtype, xp, scp)

@testing.parameterize(*testing.product({
    'size1': [(10,), (5, 10), (10, 3), (3, 4, 10)],
    'size2': [3, 4, 5, 10],
    'mode': ['full', 'same', 'valid'],
}))
@testing.gpu
@testing.with_requires('scipy')
class TestFFTConvolve(unittest.TestCase):
    def _filter(self, func, dtype, xp, scp, **kwargs):
        in1 = testing.shaped_random(self.size1, xp, dtype)
        in2 = testing.shaped_random((self.size2,)*in1.ndim, xp, dtype)
        return getattr(scp.signal, func)(in1, in2, self.mode, **kwargs)

    # Note: float16 is tested separately
    @testing.for_all_dtypes(no_float16=True)
    @testing.numpy_cupy_allclose(atol=1e-4, rtol=1e-4, scipy_name='scp',
                                 accept_error=ValueError)
    def test_fftconvolve(self, xp, scp, dtype):
        return self._filter('fftconvolve', dtype, xp, scp)

    # float16 has significantly worse error tolerances
    @testing.numpy_cupy_allclose(atol=1e-3, rtol=1e-3, scipy_name='scp',
                                 accept_error=ValueError)
    def test_fftconvolve_float16(self, xp, scp, dtype=cupy.float16):
        return self._filter('fftconvolve', dtype, xp, scp)

    # Note: float16 is tested separately
    @testing.for_all_dtypes(no_float16=True, no_bool=True)
    @testing.numpy_cupy_allclose(atol=1e-4, rtol=1e-4, scipy_name='scp',
                                 accept_error=ValueError)
    def test_convolve_fft(self, xp, scp, dtype):
        return self._filter('convolve', dtype, xp, scp, method='fft')

    # float16 has significantly worse error tolerances
    @testing.numpy_cupy_allclose(atol=1e-3, rtol=1e-3, scipy_name='scp',
                                 accept_error=ValueError)
    def test_convolve_fft_float16(self, xp, scp, dtype=cupy.float16):
        return self._filter('convolve', dtype, xp, scp, method='fft')

    # Note: float16 is tested separately
    @testing.for_all_dtypes(no_float16=True, no_bool=True)
    @testing.numpy_cupy_allclose(atol=1e-4, rtol=1e-4, scipy_name='scp',
                                 accept_error=ValueError)
    def test_correlate_fft(self, xp, scp, dtype):
        return self._filter('correlate', dtype, xp, scp, method='fft')

    # float16 has significantly worse error tolerances
    @testing.numpy_cupy_allclose(atol=1e-3, rtol=1e-3, scipy_name='scp',
                                 accept_error=ValueError)
    def test_correlate_fft_float16(self, xp, scp, dtype=cupy.float16):
        return self._filter('correlate', dtype, xp, scp, method='fft')

@testing.parameterize(*(testing.product({
    'size1': [(5, 10), (10, 7)],
    'size2': [(3, 2), (3, 3), (2, 2), (10, 10), (11, 11)],
    'mode': ['full', 'same', 'valid'],
    'boundary': ['fill'],
    'fillvalue': [0, 1, -1],
}) + testing.product({
    'size1': [(5, 10), (10, 7)],
    'size2': [(3, 2), (3, 3), (2, 2), (10, 10), (11, 11)],
    'mode': ['full', 'same', 'valid'],
    'boundary': ['wrap', 'symm'],
    'fillvalue': [0],
})))
@testing.gpu
@testing.with_requires('scipy')
class TestConvolveCorrelate2D(unittest.TestCase):
    def _filter(self, func, dtype, xp, scp):
        if self.mode == 'full' and self.boundary != 'constant':
            # See https://github.com/scipy/scipy/issues/12685
            raise unittest.SkipTest('broken in scipy')
        in1 = testing.shaped_random(self.size1, xp, dtype)
        in2 = testing.shaped_random(self.size2, xp, dtype)
        return getattr(scp.signal, func)(in1, in2, self.mode, self.boundary,
                                         self.fillvalue)

    # TODO: support complex
    # Note: float16 is tested separately
    @testing.for_all_dtypes(no_float16=True, no_complex=True)
    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
                                 accept_error=ValueError)
    def test_convolve2d(self, xp, scp, dtype):
        return self._filter('convolve2d', dtype, xp, scp)

    # TODO: support complex
    # Note: float16 is tested separately
    @testing.for_all_dtypes(no_float16=True, no_complex=True)
    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
                                 accept_error=ValueError)
    def test_correlate2d(self, xp, scp, dtype):
        return self._filter('correlate2d', dtype, xp, scp)

    # float16 has significantly worse error tolerances
    @testing.numpy_cupy_allclose(atol=1e-3, rtol=1e-3, scipy_name='scp',
                                 accept_error=ValueError)
    def test_convolve2d_float16(self, xp, scp, dtype=cupy.float16):
        return self._filter('convolve2d', dtype, xp, scp)

    # float16 has significantly worse error tolerances
    @testing.numpy_cupy_allclose(atol=1e-3, rtol=1e-3, scipy_name='scp',
                                 accept_error=ValueError)
    def test_correlate2d_float16(self, xp, scp, dtype=cupy.float16):
        return self._filter('correlate2d', dtype, xp, scp)

@testing.gpu
@testing.parameterize(*testing.product({
    'mode': ['valid', 'same', 'full']
}))
class TestChooseConvMethod(unittest.TestCase):

    @testing.for_dtypes('efdFD')
    def test_choose_conv_method1(self, dtype):
        a = testing.shaped_arange((10000,), cupy, dtype)
        b = testing.shaped_arange((5000,), cupy, dtype)
        assert cupyx.scipy.signal.choose_conv_method(
            a, b, mode=self.mode) == 'fft'

    @testing.for_dtypes('efdFD')
    def test_choose_conv_method2(self, dtype):
        a = testing.shaped_arange((5000,), cupy, dtype)
        b = testing.shaped_arange((10000,), cupy, dtype)
        assert cupyx.scipy.signal.choose_conv_method(
            a, b, mode=self.mode) == 'fft'

    @testing.for_int_dtypes()
    def test_choose_conv_method_int(self, dtype):
        a = testing.shaped_arange((10,), cupy, dtype)
        b = testing.shaped_arange((5,), cupy, dtype)
        assert cupyx.scipy.signal.choose_conv_method(
            a, b, mode=self.mode) == 'direct'

    @testing.for_all_dtypes()
    def test_choose_conv_method_ndim(self, dtype):
        a = testing.shaped_arange((3, 4, 5), cupy, dtype)
        b = testing.shaped_arange((1, 2), cupy, dtype)
        with pytest.raises(NotImplementedError):
            cupyx.scipy.signal.choose_conv_method(a, b, mode=self.mode)

    @testing.for_all_dtypes()
    def test_choose_conv_method_zero_dim(self, dtype):
        a = testing.shaped_arange((), cupy, dtype)
        b = testing.shaped_arange((5,), cupy, dtype)
        with pytest.raises(NotImplementedError):
            cupyx.scipy.signal.choose_conv_method(a, b, mode=self.mode)

@testing.parameterize(*testing.product({
    'im': [(10,), (5, 10), (10, 3), (3, 4, 10)],
    'mysize': [3, 4, (3, 4, 5)],
    'noise': [False, True],
}))
@testing.gpu
@testing.with_requires('scipy')
class TestWiener(unittest.TestCase):
    # TODO: support complex
    # Note: float16 is tested separately
    @testing.for_all_dtypes(no_float16=True, no_complex=True)
    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp')
    def test_wiener(self, xp, scp, dtype):
        im = testing.shaped_random(self.im, xp, dtype)
        mysize = self.mysize
        if isinstance(mysize, tuple):
            mysize = mysize[:im.ndim]
        noise = (testing.shaped_random(self.im, xp, dtype)
                 if self.noise else None)
        return scp.signal.wiener(im, mysize, noise)

    # float16 has significantly worse error tolerances
    @testing.numpy_cupy_allclose(atol=1e-3, rtol=1e-3, scipy_name='scp')
    def test_wiener_float16(self, xp, scp, dtype=cupy.float16):
        im = testing.shaped_random(self.im, xp, dtype)
        mysize = self.mysize
        if isinstance(mysize, tuple):
            mysize = mysize[:im.ndim]
        noise = (testing.shaped_random(self.im, xp, dtype)
                 if self.noise else None)
        return scp.signal.wiener(im, mysize, noise)

@testing.parameterize(*testing.product({
    'a': [(10,), (5, 10), (10, 3), (3, 4, 10)],
    'domain': [3, 4, (3, 3, 5)],
    'rank': [0, 1, 2],
}))
@testing.gpu
@testing.with_requires('scipy')
class TestOrderFilter(unittest.TestCase):
    @testing.for_all_dtypes()
    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
                                 accept_error=ValueError)  # for even kernels
    def test_order_filter(self, xp, scp, dtype):
        a = testing.shaped_random(self.a, xp, dtype)
        d = self.domain
        d = d[:a.ndim] if isinstance(d, tuple) else (d,)*a.ndim
        domain = testing.shaped_random(d, xp) > 0.25
        rank = min(self.rank, domain.sum())
        return scp.signal.order_filter(a, domain, rank)

@testing.parameterize(*testing.product({
    'volume': [(10,), (5, 10), (10, 5), (5, 6, 10)],
    'kernel_size': [3, 4, (3, 3, 5)],
}))
@testing.gpu
@testing.with_requires('scipy')
class TestMedFilt(unittest.TestCase):
    @testing.for_all_dtypes()
    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
                                 accept_error=ValueError)  # for even kernels
    def test_medfilt(self, xp, scp, dtype):
        volume = testing.shaped_random(self.volume, xp, dtype)
        kernel_size = self.kernel_size
        if isinstance(kernel_size, tuple):
            kernel_size = kernel_size[:volume.ndim]
        return scp.signal.medfilt(volume, kernel_size)

@testing.parameterize(*testing.product({
    'input': [(5, 10), (10, 5)],
    'kernel_size': [3, 4, (3, 5)],
}))
@testing.gpu
@testing.with_requires('scipy')
class TestMedFilt2d(unittest.TestCase):
    @testing.for_all_dtypes()
    @testing.numpy_cupy_allclose(atol=1e-5, rtol=1e-5, scipy_name='scp',
                                 accept_error=ValueError)  # for even kernels
    def test_medfilt2d(self, xp, scp, dtype):
        input = testing.shaped_random(self.input, xp, dtype)
        kernel_size = self.kernel_size
        return scp.signal.medfilt2d(input, kernel_size)
<|endoftext|>