<commit_before># -*- coding: utf-8 -*-

# Import python libs
from __future__ import absolute_import, print_function, unicode_literals
import io
import os
import sys
import threading
import time
import signal
import multiprocessing
import functools

# Import Salt Testing libs
from tests.support.unit import TestCase, skipIf
from tests.support.mock import (
    patch,
    NO_MOCK,
    NO_MOCK_REASON
)

# Import salt libs
import salt.utils.platform
import salt.utils.process

# Import 3rd-party libs
from salt.ext import six
from salt.ext.six.moves import range  # pylint: disable=import-error,redefined-builtin
import psutil

def die(func):
    '''
    Add proc title
    '''
    @functools.wraps(func)
    def wrapper(self):
        # Strip off the "test_" from the function name
        name = func.__name__[5:]

        def _die():
            salt.utils.process.appendproctitle('test_{0}'.format(name))
        setattr(self, 'die_' + name, _die)

    return wrapper

def incr(func):
    '''
    Increment counter
    '''
    @functools.wraps(func)
    def wrapper(self):
        # Strip off the "test_" from the function name
        name = func.__name__[5:]

        def _incr(counter, num):
            salt.utils.process.appendproctitle('test_{0}'.format(name))
            for _ in range(0, num):
                counter.value += 1
        setattr(self, 'incr_' + name, _incr)

    return wrapper

def spin(func):
    '''
    Spin indefinitely
    '''
    @functools.wraps(func)
    def wrapper(self):
        # Strip off the "test_" from the function name
        name = func.__name__[5:]

        def _spin():
            salt.utils.process.appendproctitle('test_{0}'.format(name))
            while True:
                time.sleep(1)
        setattr(self, 'spin_' + name, _spin)

    return wrapper

class TestProcessManager(TestCase):

    @spin
    def test_basic(self):
        '''
        Make sure that the process is alive 2s later
        '''
        process_manager = salt.utils.process.ProcessManager()
        process_manager.add_process(self.spin_basic)
        initial_pid = next(six.iterkeys(process_manager._process_map))
        time.sleep(2)
        process_manager.check_children()
        try:
            assert initial_pid == next(six.iterkeys(process_manager._process_map))
        finally:
            process_manager.stop_restarting()
            process_manager.kill_children()
            time.sleep(0.5)
            # Are there child processes still running?
            if process_manager._process_map.keys():
                process_manager.send_signal_to_processes(signal.SIGKILL)
                process_manager.stop_restarting()
                process_manager.kill_children()

    @spin
    def test_kill(self):
        process_manager = salt.utils.process.ProcessManager()
        process_manager.add_process(self.spin_kill)
        initial_pid = next(six.iterkeys(process_manager._process_map))
        # kill the child
        if salt.utils.platform.is_windows():
            os.kill(initial_pid, signal.SIGTERM)
        else:
            os.kill(initial_pid, signal.SIGKILL)
        # give the OS time to give the signal...
        time.sleep(0.1)
        process_manager.check_children()
        try:
            assert initial_pid != next(six.iterkeys(process_manager._process_map))
        finally:
            process_manager.stop_restarting()
            process_manager.kill_children()
            time.sleep(0.5)
            # Are there child processes still running?
            if process_manager._process_map.keys():
                process_manager.send_signal_to_processes(signal.SIGKILL)
                process_manager.stop_restarting()
                process_manager.kill_children()

    @die
    def test_restarting(self):
        '''
        Make sure that the process is alive 2s later
        '''
        process_manager = salt.utils.process.ProcessManager()
        process_manager.add_process(self.die_restarting)
        initial_pid = next(six.iterkeys(process_manager._process_map))
        time.sleep(2)
        process_manager.check_children()
        try:
            assert initial_pid != next(six.iterkeys(process_manager._process_map))
        finally:
            process_manager.stop_restarting()
            process_manager.kill_children()
            time.sleep(0.5)
            # Are there child processes still running?
            if process_manager._process_map.keys():
                process_manager.send_signal_to_processes(signal.SIGKILL)
                process_manager.stop_restarting()
                process_manager.kill_children()

    @skipIf(sys.version_info < (2, 7), 'Needs > Py 2.7 due to bug in stdlib')
    @incr
    def test_counter(self):
        counter = multiprocessing.Value('i', 0)
        process_manager = salt.utils.process.ProcessManager()
        process_manager.add_process(self.incr_counter, args=(counter, 2))
        time.sleep(1)
        process_manager.check_children()
        time.sleep(1)
        # we should have had 2 processes go at it
        try:
            assert counter.value == 4
        finally:
            process_manager.stop_restarting()
            process_manager.kill_children()
            time.sleep(0.5)
            # Are there child processes still running?
            if process_manager._process_map.keys():
                process_manager.send_signal_to_processes(signal.SIGKILL)
                process_manager.stop_restarting()
                process_manager.kill_children()

class TestThreadPool(TestCase):

    def test_basic(self):
        '''
        Make sure the threadpool can do things
        '''
        def incr_counter(counter):
            counter.value += 1
        counter = multiprocessing.Value('i', 0)

        pool = salt.utils.process.ThreadPool()
        sent = pool.fire_async(incr_counter, args=(counter,))
        self.assertTrue(sent)
        time.sleep(1)  # Sleep to let the threads do things
        self.assertEqual(counter.value, 1)
        self.assertEqual(pool._job_queue.qsize(), 0)

    def test_full_queue(self):
        '''
        Make sure that a full threadpool acts as we expect
        '''
        def incr_counter(counter):
            counter.value += 1
        counter = multiprocessing.Value('i', 0)

        # Create a pool with no workers and 1 queue size
        pool = salt.utils.process.ThreadPool(0, 1)
        # make sure we can put the one item in
        sent = pool.fire_async(incr_counter, args=(counter,))
        self.assertTrue(sent)
        # make sure we can't put more in
        sent = pool.fire_async(incr_counter, args=(counter,))
        self.assertFalse(sent)
        time.sleep(1)  # Sleep to let the threads do things
        # make sure no one updated the counter
        self.assertEqual(counter.value, 0)
        # make sure the queue is still full
        self.assertEqual(pool._job_queue.qsize(), 1)

class TestProcess(TestCase):

    @skipIf(NO_MOCK, NO_MOCK_REASON)
    def test_daemonize_if(self):
        # pylint: disable=assignment-from-none
        with patch('sys.argv', ['salt-call']):
            ret = salt.utils.process.daemonize_if({})
            self.assertEqual(None, ret)

        ret = salt.utils.process.daemonize_if({'multiprocessing': False})
        self.assertEqual(None, ret)

        with patch('sys.platform', 'win'):
            ret = salt.utils.process.daemonize_if({})
            self.assertEqual(None, ret)

        with patch('salt.utils.process.daemonize'), \
                patch('sys.platform', 'linux2'):
            salt.utils.process.daemonize_if({})
            self.assertTrue(salt.utils.process.daemonize.called)
        # pylint: enable=assignment-from-none

class TestSignalHandlingMultiprocessingProcess(TestCase):

    @classmethod
    def Process(cls, pid):
        raise psutil.NoSuchProcess(pid)

    @classmethod
    def target(cls):
        os.kill(os.getpid(), signal.SIGTERM)

    @classmethod
    def children(cls, *args, **kwargs):
        raise psutil.NoSuchProcess(1)

    @skipIf(NO_MOCK, NO_MOCK_REASON)
    def test_process_does_not_exist(self):
        try:
            with patch('psutil.Process', self.Process):
                proc = salt.utils.process.SignalHandlingMultiprocessingProcess(target=self.target)
                proc.start()
        except psutil.NoSuchProcess:
            assert False, "psutil.NoSuchProcess raised"

    @skipIf(NO_MOCK, NO_MOCK_REASON)
    def test_process_children_do_not_exist(self):
        try:
            with patch('psutil.Process.children', self.children):
                proc = salt.utils.process.SignalHandlingMultiprocessingProcess(target=self.target)
                proc.start()
        except psutil.NoSuchProcess:
            assert False, "psutil.NoSuchProcess raised"

    @staticmethod
    def run_forever_sub_target(evt):
        'Used by run_forever_target to create a sub-process'
        while not evt.is_set():
            time.sleep(1)

    @staticmethod
    def run_forever_target(sub_target, evt):
        'A target that will run forever or until an event is set'
        p = multiprocessing.Process(target=sub_target, args=(evt,))
        p.start()
        p.join()

    @staticmethod
    def kill_target_sub_proc():
        pid = os.fork()
        if pid == 0:
            return
        pid = os.fork()
        if pid == 0:
            return
        time.sleep(.1)
        try:
            os.kill(os.getpid(), signal.SIGINT)
        except KeyboardInterrupt:
            pass

    @skipIf(sys.platform.startswith('win'), 'No os.fork on Windows')
    def test_signal_processing_regression_test(self):
        evt = multiprocessing.Event()
        sh_proc = salt.utils.process.SignalHandlingMultiprocessingProcess(
            target=self.run_forever_target,
            args=(self.run_forever_sub_target, evt)
        )
        sh_proc.start()
        proc = multiprocessing.Process(target=self.kill_target_sub_proc)
        proc.start()
        proc.join()
        # When the bug exists, the kill_target_sub_proc signal will kill both
        # processes. sh_proc will be alive if the bug is fixed
        try:
            assert sh_proc.is_alive()
        finally:
            evt.set()
            sh_proc.join()

    @staticmethod
    def no_op_target():
        pass

    @skipIf(NO_MOCK, NO_MOCK_REASON)
    def test_signal_processing_test_after_fork_called(self):
        'Validate MultiprocessingProcess and sub classes call after fork methods'
        evt = multiprocessing.Event()
        sig_to_mock = 'salt.utils.process.SignalHandlingMultiprocessingProcess._setup_signals'
        log_to_mock = 'salt.utils.process.MultiprocessingProcess._setup_process_logging'
        with patch(sig_to_mock) as ma, patch(log_to_mock) as mb:
            self.sh_proc = salt.utils.process.SignalHandlingMultiprocessingProcess(target=self.no_op_target)
            self.sh_proc.run()
        ma.assert_called()
        mb.assert_called()

    @skipIf(NO_MOCK, NO_MOCK_REASON)
    def test_signal_processing_test_final_methods_called(self):
        'Validate MultiprocessingProcess and sub classes call finalize methods'
        evt = multiprocessing.Event()
        teardown_to_mock = 'salt.log.setup.shutdown_multiprocessing_logging'
        log_to_mock = 'salt.utils.process.MultiprocessingProcess._setup_process_logging'
        sig_to_mock = 'salt.utils.process.SignalHandlingMultiprocessingProcess._setup_signals'
        # Mock _setup_signals so we do not register one for this process.
        with patch(sig_to_mock):
            with patch(teardown_to_mock) as ma, patch(log_to_mock) as mb:
                self.sh_proc = salt.utils.process.SignalHandlingMultiprocessingProcess(target=self.no_op_target)
                self.sh_proc.run()
        ma.assert_called()
        mb.assert_called()

    @staticmethod
    def pid_setting_target(sub_target, val, evt):
        val.value = os.getpid()
        p = multiprocessing.Process(target=sub_target, args=(evt,))
        p.start()
        p.join()

    @skipIf(sys.platform.startswith('win'), 'Required signals not supported on windows')
    def test_signal_processing_handle_signals_called(self):
        'Validate SignalHandlingMultiprocessingProcess handles signals'
        # Gloobal event to stop all processes we're creating
        evt = multiprocessing.Event()

        # Create a process to test signal handler
        val = multiprocessing.Value('i', 0)
        proc = salt.utils.process.SignalHandlingMultiprocessingProcess(
            target=self.pid_setting_target,
            args=(self.run_forever_sub_target, val, evt),
        )
        proc.start()

        # Create a second process that should not respond to SIGINT or SIGTERM
        proc2 = multiprocessing.Process(
            target=self.run_forever_target,
            args=(self.run_forever_sub_target, evt),
        )
        proc2.start()

        # Wait for the sub process to set it's pid
        while not val.value:
            time.sleep(.3)

        assert not proc.signal_handled()

        # Send a signal that should get handled by the subprocess
        os.kill(val.value, signal.SIGTERM)

        # wait up to 10 seconds for signal handler:
        start = time.time()
        while time.time() - start < 10:
            if proc.signal_handled():
                break
            time.sleep(.3)

        try:
            # Allow some time for the signal handler to do it's thing
            assert proc.signal_handled()
            # Reap the signaled process
            proc.join(1)
            assert proc2.is_alive()
        finally:
            evt.set()
            proc2.join(30)
            proc.join(30)

class TestDup2(TestCase):

    def test_dup2_no_fileno(self):
        'The dup2 method does not fail on streams without fileno support'
        f1 = io.StringIO("some initial text data")
        f2 = io.StringIO("some initial other text data")
        with self.assertRaises(io.UnsupportedOperation):
            f1.fileno()
        with patch('os.dup2') as dup_mock:
            try:
                salt.utils.process.dup2(f1, f2)
            except io.UnsupportedOperation:
                assert False, 'io.UnsupportedOperation was raised'
        assert not dup_mock.called

class TestProcessList(TestCase):

    def null_target(self):
        pass

    def event_target(self, event):
        while True:
            if event.wait(5):
                break

    def wait_for_proc(self, proc, timeout=10):
        start = time.time()
        while proc.is_alive():
            if time.time() - start > timeout:
                raise Exception("Process did not finishe before timeout")
            time.sleep(.3)

    def test_process_list_process(self):
        plist = salt.utils.process.SubprocessList()
        proc = multiprocessing.Process(target=self.null_target)
        proc.start()
        plist.add(proc)
        assert proc in plist.processes
        self.wait_for_proc(proc)
        assert not proc.is_alive()
        plist.cleanup()
        assert proc not in plist.processes

    def test_process_list_thread(self):
        plist = salt.utils.process.SubprocessList()
        thread = threading.Thread(target=self.null_target)
        thread.start()
        plist.add(thread)
        assert thread in plist.processes
        self.wait_for_proc(thread)
        assert not thread.is_alive()
        plist.cleanup()
        assert thread not in plist.processes

    def test_process_list_cleanup(self):
        plist = salt.utils.process.SubprocessList()
        event = multiprocessing.Event()
        proc = multiprocessing.Process(target=self.event_target, args=[event])
        proc.start()
        plist.add(proc)
        assert proc in plist.processes
        plist.cleanup()
        event.set()
        assert proc in plist.processes
        self.wait_for_proc(proc)
        assert not proc.is_alive()
        plist.cleanup()
        assert proc not in plist.processes
<commit_msg>Fix process list tests on windows py3<commit_after># -*- coding: utf-8 -*-

# Import python libs
from __future__ import absolute_import, print_function, unicode_literals
import io
import os
import sys
import threading
import time
import signal
import multiprocessing
import functools

# Import Salt Testing libs
from tests.support.unit import TestCase, skipIf
from tests.support.mock import (
    patch,
    NO_MOCK,
    NO_MOCK_REASON
)

# Import salt libs
import salt.utils.platform
import salt.utils.process

# Import 3rd-party libs
from salt.ext import six
from salt.ext.six.moves import range  # pylint: disable=import-error,redefined-builtin
import psutil

def die(func):
    '''
    Add proc title
    '''
    @functools.wraps(func)
    def wrapper(self):
        # Strip off the "test_" from the function name
        name = func.__name__[5:]

        def _die():
            salt.utils.process.appendproctitle('test_{0}'.format(name))
        setattr(self, 'die_' + name, _die)

    return wrapper

def incr(func):
    '''
    Increment counter
    '''
    @functools.wraps(func)
    def wrapper(self):
        # Strip off the "test_" from the function name
        name = func.__name__[5:]

        def _incr(counter, num):
            salt.utils.process.appendproctitle('test_{0}'.format(name))
            for _ in range(0, num):
                counter.value += 1
        setattr(self, 'incr_' + name, _incr)

    return wrapper

def spin(func):
    '''
    Spin indefinitely
    '''
    @functools.wraps(func)
    def wrapper(self):
        # Strip off the "test_" from the function name
        name = func.__name__[5:]

        def _spin():
            salt.utils.process.appendproctitle('test_{0}'.format(name))
            while True:
                time.sleep(1)
        setattr(self, 'spin_' + name, _spin)

    return wrapper

class TestProcessManager(TestCase):

    @spin
    def test_basic(self):
        '''
        Make sure that the process is alive 2s later
        '''
        process_manager = salt.utils.process.ProcessManager()
        process_manager.add_process(self.spin_basic)
        initial_pid = next(six.iterkeys(process_manager._process_map))
        time.sleep(2)
        process_manager.check_children()
        try:
            assert initial_pid == next(six.iterkeys(process_manager._process_map))
        finally:
            process_manager.stop_restarting()
            process_manager.kill_children()
            time.sleep(0.5)
            # Are there child processes still running?
            if process_manager._process_map.keys():
                process_manager.send_signal_to_processes(signal.SIGKILL)
                process_manager.stop_restarting()
                process_manager.kill_children()

    @spin
    def test_kill(self):
        process_manager = salt.utils.process.ProcessManager()
        process_manager.add_process(self.spin_kill)
        initial_pid = next(six.iterkeys(process_manager._process_map))
        # kill the child
        if salt.utils.platform.is_windows():
            os.kill(initial_pid, signal.SIGTERM)
        else:
            os.kill(initial_pid, signal.SIGKILL)
        # give the OS time to give the signal...
        time.sleep(0.1)
        process_manager.check_children()
        try:
            assert initial_pid != next(six.iterkeys(process_manager._process_map))
        finally:
            process_manager.stop_restarting()
            process_manager.kill_children()
            time.sleep(0.5)
            # Are there child processes still running?
            if process_manager._process_map.keys():
                process_manager.send_signal_to_processes(signal.SIGKILL)
                process_manager.stop_restarting()
                process_manager.kill_children()

    @die
    def test_restarting(self):
        '''
        Make sure that the process is alive 2s later
        '''
        process_manager = salt.utils.process.ProcessManager()
        process_manager.add_process(self.die_restarting)
        initial_pid = next(six.iterkeys(process_manager._process_map))
        time.sleep(2)
        process_manager.check_children()
        try:
            assert initial_pid != next(six.iterkeys(process_manager._process_map))
        finally:
            process_manager.stop_restarting()
            process_manager.kill_children()
            time.sleep(0.5)
            # Are there child processes still running?
            if process_manager._process_map.keys():
                process_manager.send_signal_to_processes(signal.SIGKILL)
                process_manager.stop_restarting()
                process_manager.kill_children()

    @skipIf(sys.version_info < (2, 7), 'Needs > Py 2.7 due to bug in stdlib')
    @incr
    def test_counter(self):
        counter = multiprocessing.Value('i', 0)
        process_manager = salt.utils.process.ProcessManager()
        process_manager.add_process(self.incr_counter, args=(counter, 2))
        time.sleep(1)
        process_manager.check_children()
        time.sleep(1)
        # we should have had 2 processes go at it
        try:
            assert counter.value == 4
        finally:
            process_manager.stop_restarting()
            process_manager.kill_children()
            time.sleep(0.5)
            # Are there child processes still running?
            if process_manager._process_map.keys():
                process_manager.send_signal_to_processes(signal.SIGKILL)
                process_manager.stop_restarting()
                process_manager.kill_children()

class TestThreadPool(TestCase):

    def test_basic(self):
        '''
        Make sure the threadpool can do things
        '''
        def incr_counter(counter):
            counter.value += 1
        counter = multiprocessing.Value('i', 0)

        pool = salt.utils.process.ThreadPool()
        sent = pool.fire_async(incr_counter, args=(counter,))
        self.assertTrue(sent)
        time.sleep(1)  # Sleep to let the threads do things
        self.assertEqual(counter.value, 1)
        self.assertEqual(pool._job_queue.qsize(), 0)

    def test_full_queue(self):
        '''
        Make sure that a full threadpool acts as we expect
        '''
        def incr_counter(counter):
            counter.value += 1
        counter = multiprocessing.Value('i', 0)

        # Create a pool with no workers and 1 queue size
        pool = salt.utils.process.ThreadPool(0, 1)
        # make sure we can put the one item in
        sent = pool.fire_async(incr_counter, args=(counter,))
        self.assertTrue(sent)
        # make sure we can't put more in
        sent = pool.fire_async(incr_counter, args=(counter,))
        self.assertFalse(sent)
        time.sleep(1)  # Sleep to let the threads do things
        # make sure no one updated the counter
        self.assertEqual(counter.value, 0)
        # make sure the queue is still full
        self.assertEqual(pool._job_queue.qsize(), 1)

class TestProcess(TestCase):

    @skipIf(NO_MOCK, NO_MOCK_REASON)
    def test_daemonize_if(self):
        # pylint: disable=assignment-from-none
        with patch('sys.argv', ['salt-call']):
            ret = salt.utils.process.daemonize_if({})
            self.assertEqual(None, ret)

        ret = salt.utils.process.daemonize_if({'multiprocessing': False})
        self.assertEqual(None, ret)

        with patch('sys.platform', 'win'):
            ret = salt.utils.process.daemonize_if({})
            self.assertEqual(None, ret)

        with patch('salt.utils.process.daemonize'), \
                patch('sys.platform', 'linux2'):
            salt.utils.process.daemonize_if({})
            self.assertTrue(salt.utils.process.daemonize.called)
        # pylint: enable=assignment-from-none

class TestSignalHandlingMultiprocessingProcess(TestCase):

    @classmethod
    def Process(cls, pid):
        raise psutil.NoSuchProcess(pid)

    @classmethod
    def target(cls):
        os.kill(os.getpid(), signal.SIGTERM)

    @classmethod
    def children(cls, *args, **kwargs):
        raise psutil.NoSuchProcess(1)

    @skipIf(NO_MOCK, NO_MOCK_REASON)
    def test_process_does_not_exist(self):
        try:
            with patch('psutil.Process', self.Process):
                proc = salt.utils.process.SignalHandlingMultiprocessingProcess(target=self.target)
                proc.start()
        except psutil.NoSuchProcess:
            assert False, "psutil.NoSuchProcess raised"

    @skipIf(NO_MOCK, NO_MOCK_REASON)
    def test_process_children_do_not_exist(self):
        try:
            with patch('psutil.Process.children', self.children):
                proc = salt.utils.process.SignalHandlingMultiprocessingProcess(target=self.target)
                proc.start()
        except psutil.NoSuchProcess:
            assert False, "psutil.NoSuchProcess raised"

    @staticmethod
    def run_forever_sub_target(evt):
        'Used by run_forever_target to create a sub-process'
        while not evt.is_set():
            time.sleep(1)

    @staticmethod
    def run_forever_target(sub_target, evt):
        'A target that will run forever or until an event is set'
        p = multiprocessing.Process(target=sub_target, args=(evt,))
        p.start()
        p.join()

    @staticmethod
    def kill_target_sub_proc():
        pid = os.fork()
        if pid == 0:
            return
        pid = os.fork()
        if pid == 0:
            return
        time.sleep(.1)
        try:
            os.kill(os.getpid(), signal.SIGINT)
        except KeyboardInterrupt:
            pass

    @skipIf(sys.platform.startswith('win'), 'No os.fork on Windows')
    def test_signal_processing_regression_test(self):
        evt = multiprocessing.Event()
        sh_proc = salt.utils.process.SignalHandlingMultiprocessingProcess(
            target=self.run_forever_target,
            args=(self.run_forever_sub_target, evt)
        )
        sh_proc.start()
        proc = multiprocessing.Process(target=self.kill_target_sub_proc)
        proc.start()
        proc.join()
        # When the bug exists, the kill_target_sub_proc signal will kill both
        # processes. sh_proc will be alive if the bug is fixed
        try:
            assert sh_proc.is_alive()
        finally:
            evt.set()
            sh_proc.join()

    @staticmethod
    def no_op_target():
        pass

    @skipIf(NO_MOCK, NO_MOCK_REASON)
    def test_signal_processing_test_after_fork_called(self):
        'Validate MultiprocessingProcess and sub classes call after fork methods'
        evt = multiprocessing.Event()
        sig_to_mock = 'salt.utils.process.SignalHandlingMultiprocessingProcess._setup_signals'
        log_to_mock = 'salt.utils.process.MultiprocessingProcess._setup_process_logging'
        with patch(sig_to_mock) as ma, patch(log_to_mock) as mb:
            self.sh_proc = salt.utils.process.SignalHandlingMultiprocessingProcess(target=self.no_op_target)
            self.sh_proc.run()
        ma.assert_called()
        mb.assert_called()

    @skipIf(NO_MOCK, NO_MOCK_REASON)
    def test_signal_processing_test_final_methods_called(self):
        'Validate MultiprocessingProcess and sub classes call finalize methods'
        evt = multiprocessing.Event()
        teardown_to_mock = 'salt.log.setup.shutdown_multiprocessing_logging'
        log_to_mock = 'salt.utils.process.MultiprocessingProcess._setup_process_logging'
        sig_to_mock = 'salt.utils.process.SignalHandlingMultiprocessingProcess._setup_signals'
        # Mock _setup_signals so we do not register one for this process.
        with patch(sig_to_mock):
            with patch(teardown_to_mock) as ma, patch(log_to_mock) as mb:
                self.sh_proc = salt.utils.process.SignalHandlingMultiprocessingProcess(target=self.no_op_target)
                self.sh_proc.run()
        ma.assert_called()
        mb.assert_called()

    @staticmethod
    def pid_setting_target(sub_target, val, evt):
        val.value = os.getpid()
        p = multiprocessing.Process(target=sub_target, args=(evt,))
        p.start()
        p.join()

    @skipIf(sys.platform.startswith('win'), 'Required signals not supported on windows')
    def test_signal_processing_handle_signals_called(self):
        'Validate SignalHandlingMultiprocessingProcess handles signals'
        # Gloobal event to stop all processes we're creating
        evt = multiprocessing.Event()

        # Create a process to test signal handler
        val = multiprocessing.Value('i', 0)
        proc = salt.utils.process.SignalHandlingMultiprocessingProcess(
            target=self.pid_setting_target,
            args=(self.run_forever_sub_target, val, evt),
        )
        proc.start()

        # Create a second process that should not respond to SIGINT or SIGTERM
        proc2 = multiprocessing.Process(
            target=self.run_forever_target,
            args=(self.run_forever_sub_target, evt),
        )
        proc2.start()

        # Wait for the sub process to set it's pid
        while not val.value:
            time.sleep(.3)

        assert not proc.signal_handled()

        # Send a signal that should get handled by the subprocess
        os.kill(val.value, signal.SIGTERM)

        # wait up to 10 seconds for signal handler:
        start = time.time()
        while time.time() - start < 10:
            if proc.signal_handled():
                break
            time.sleep(.3)

        try:
            # Allow some time for the signal handler to do it's thing
            assert proc.signal_handled()
            # Reap the signaled process
            proc.join(1)
            assert proc2.is_alive()
        finally:
            evt.set()
            proc2.join(30)
            proc.join(30)

class TestDup2(TestCase):

    def test_dup2_no_fileno(self):
        'The dup2 method does not fail on streams without fileno support'
        f1 = io.StringIO("some initial text data")
        f2 = io.StringIO("some initial other text data")
        with self.assertRaises(io.UnsupportedOperation):
            f1.fileno()
        with patch('os.dup2') as dup_mock:
            try:
                salt.utils.process.dup2(f1, f2)
            except io.UnsupportedOperation:
                assert False, 'io.UnsupportedOperation was raised'
        assert not dup_mock.called

class TestProcessList(TestCase):

    @staticmethod
    def null_target():
        pass

    @staticmethod
    def event_target(event):
        while True:
            if event.wait(5):
                break

    @staticmethod
    def wait_for_proc(proc, timeout=10):
        start = time.time()
        while proc.is_alive():
            if time.time() - start > timeout:
                raise Exception("Process did not finishe before timeout")
            time.sleep(.3)

    def test_process_list_process(self):
        plist = salt.utils.process.SubprocessList()
        proc = multiprocessing.Process(target=self.null_target)
        proc.start()
        plist.add(proc)
        assert proc in plist.processes
        self.wait_for_proc(proc)
        assert not proc.is_alive()
        plist.cleanup()
        assert proc not in plist.processes

    def test_process_list_thread(self):
        plist = salt.utils.process.SubprocessList()
        thread = threading.Thread(target=self.null_target)
        thread.start()
        plist.add(thread)
        assert thread in plist.processes
        self.wait_for_proc(thread)
        assert not thread.is_alive()
        plist.cleanup()
        assert thread not in plist.processes

    def test_process_list_cleanup(self):
        plist = salt.utils.process.SubprocessList()
        event = multiprocessing.Event()
        proc = multiprocessing.Process(target=self.event_target, args=[event])
        proc.start()
        plist.add(proc)
        assert proc in plist.processes
        plist.cleanup()
        event.set()
        assert proc in plist.processes
        self.wait_for_proc(proc)
        assert not proc.is_alive()
        plist.cleanup()
        assert proc not in plist.processes
<|endoftext|>


<commit_before>#!/usr/bin/env python3
"""Python binding of 8x8R Click wrapper of LetMeCreate library.

You must initialise the SPI bus and select the right one before using any of
these functions.
"""

import ctypes

_LIB = ctypes.CDLL('libletmecreate_click.so')

def enable():
    """Enable the Led Matrix.

    Switch off all LED's and set the intensity to its maximum.

    Note: An exception is thrown if it fails to enable the chip.
    """
    ret = _LIB.led_matrix_click_enable()
    if ret < 0:
        raise Exception("led matrix click enable failed")

def set_intensity(intensity):
    """Sets the intensity of the led matrix.

    intensity: must be in range 0-15.

    Note: An exception is thrown if it fails to set the intensity.
    """
    ret = _LIB.led_matrix_click_set_intensity(intensity)
    if ret < 0:
        raise Exception("led matrix click set intensity failed")

def set_column(column_index, data):
    """Switch on/off LED's of one column.

    column_index: must be in range 0-7.

    data: A 8-bit integer. data is interpreted as a bit string. LED's, whose
    corresponding bits are 1, are switched on, otherwise they are switched off.

    Note: An exception is thrown if it fails to switch on/off LED's of a column.
    """
    ret = _LIB.led_matrix_click_set_column(column_index, data)
    if ret < 0:
        raise Exception("led matrix click set column failed")

def display_number(number):
    """Displays a decimal number on the LED matrix.

    number: must be in range 0-99

    Note: An exception is thrown if it fails to display the number.
    """
    ret = _LIB.led_matrix_click_display_number(number)
    if ret < 0:
        raise Exception("led matrix click display number failed")

def set_columns(columns):
    """Switch on/off LED's of the LED matrix.

    columns: A list of bytes of length 8. Each element is an integer
    interpreted as a bit string. LED's, whose corresponding bits are set to 1,
    are switched on, otherwise they are switched off.

    Note: An exception is thrown if it fails to switch on/off LED's.
    """
    data = (ctypes.c_uint8 * len(columns))(*columns)
    ret = _LIB.led_matrix_click_set(data)
    if ret < 0:
        raise Exception("led matrix click set columns failed")

def disable():
    """Disables the LED matrix.

    All LED's are switched off.

    Note: An exception is thrown if it fails to disable the LED matrix.
    """
    ret = _LIB.led_matrix_click_disable()
    if ret < 0:
        raise Exception("led matrix click disable failed")
<commit_msg>led_matrix: fix pep8 error<commit_after>#!/usr/bin/env python3
"""Python binding of 8x8R Click wrapper of LetMeCreate library.

You must initialise the SPI bus and select the right one before using any of
these functions.
"""

import ctypes

_LIB = ctypes.CDLL('libletmecreate_click.so')

def enable():
    """Enable the Led Matrix.

    Switch off all LED's and set the intensity to its maximum.

    Note: An exception is thrown if it fails to enable the chip.
    """
    ret = _LIB.led_matrix_click_enable()
    if ret < 0:
        raise Exception("led matrix click enable failed")

def set_intensity(intensity):
    """Sets the intensity of the led matrix.

    intensity: must be in range 0-15.

    Note: An exception is thrown if it fails to set the intensity.
    """
    ret = _LIB.led_matrix_click_set_intensity(intensity)
    if ret < 0:
        raise Exception("led matrix click set intensity failed")

def set_column(column_index, data):
    """Switch on/off LED's of one column.

    column_index: must be in range 0-7.

    data: A 8-bit integer. data is interpreted as a bit string. LED's, whose
    corresponding bits are 1, are switched on, otherwise they are switched off.

    Note: An exception is thrown if it fails to switch on/off LED's of a
    column.
    """
    ret = _LIB.led_matrix_click_set_column(column_index, data)
    if ret < 0:
        raise Exception("led matrix click set column failed")

def display_number(number):
    """Displays a decimal number on the LED matrix.

    number: must be in range 0-99

    Note: An exception is thrown if it fails to display the number.
    """
    ret = _LIB.led_matrix_click_display_number(number)
    if ret < 0:
        raise Exception("led matrix click display number failed")

def set_columns(columns):
    """Switch on/off LED's of the LED matrix.

    columns: A list of bytes of length 8. Each element is an integer
    interpreted as a bit string. LED's, whose corresponding bits are set to 1,
    are switched on, otherwise they are switched off.

    Note: An exception is thrown if it fails to switch on/off LED's.
    """
    data = (ctypes.c_uint8 * len(columns))(*columns)
    ret = _LIB.led_matrix_click_set(data)
    if ret < 0:
        raise Exception("led matrix click set columns failed")

def disable():
    """Disables the LED matrix.

    All LED's are switched off.

    Note: An exception is thrown if it fails to disable the LED matrix.
    """
    ret = _LIB.led_matrix_click_disable()
    if ret < 0:
        raise Exception("led matrix click disable failed")
<|endoftext|>


<commit_before>SEG_NON_POSITIVE_MESSAGE = "The segment size must be a positive integer"
OVERLAP_LARGE_MESSAGE = "The segment size must be greater than " \
    "the overlap of words between chunks"
PROP_NEGATIVE_MESSAGE = "The proportional size of the last segment must be " \
    "zero or greater"
OVERLAP_NEGATIVE_MESSAGE = "The overlap between segments must be zero or " \
                           "greater"
NON_POSITIVE_SEGMENT_MESSAGE = "the segment size should be positive"
NEG_OVERLAP_LAST_PROP_MESSAGE = "the overlap or last segment proportion" \
                                "should not be negative"
LARGER_SEG_SIZE_MESSAGE = "the segment size should be larger " \
    "than the overlap size"
NON_NEGATIVE_INDEX_MESSAGE = "the index should be larger than or equal to zero"
MATRIX_DIMENSION_UNEQUAL_MESSAGE = "the dimension of the matrix " \
    "should be equal"
EMPTY_LIST_MESSAGE = "the list should not be empty"
WINDOW_SIZE_LARGE_MESSAGE = "The window size must be less than or equal to" \
                            " the length of the given document"
INVALID_CUTTING_TYPE_MESSAGE = "the cutting type should be letters, lines, " \
    "number, words or milestone"
EMPTY_MILESTONE_MESSAGE = "the milestone should not be empty"
EMPTY_NP_ARRAY_MESSAGE = "The input numpy array should not be empty, since" \
    "the input file should not be empty."
NOT_ONE_REPLACEMENT_COLON_MESSAGE = "Invalid number of colons: "
REPLACEMENT_RIGHT_OPERAND_MESSAGE = "Too many values on right side of " \
                                    "replacement string: "
REPLACEMENT_NO_LEFTHAND_MESSAGE = "Missing value on the left side of " \
                                  "replacement string: "
EMPTY_STRING_MESSAGE = "the string should not be empty"
WINDOW_NON_POSITIVE_MESSAGE = "The window size must be a positive integer"

# ============= Base Model Errors ===========================
NO_DATA_SENT_ERROR_MESSAGE = 'Front end did not send data to backend'
INVALID_DATA_KEY_MESSAGE_FORMAT = \
    'no data with name {data_key} sended to backend'

# ============= Topword Model Error =========================
NOT_ENOUGH_CLASSES_MESSAGE = "Only one class given, cannot do Z-test by " \
                             "class, at least 2 classes needed"
<commit_msg>Add period<commit_after>SEG_NON_POSITIVE_MESSAGE = "The segment size must be a positive integer"
OVERLAP_LARGE_MESSAGE = "The segment size must be greater than " \
    "the overlap of words between chunks"
PROP_NEGATIVE_MESSAGE = "The proportional size of the last segment must be " \
    "zero or greater"
OVERLAP_NEGATIVE_MESSAGE = "The overlap between segments must be zero or " \
                           "greater"
NON_POSITIVE_SEGMENT_MESSAGE = "the segment size should be positive"
NEG_OVERLAP_LAST_PROP_MESSAGE = "the overlap or last segment proportion" \
                                "should not be negative"
LARGER_SEG_SIZE_MESSAGE = "the segment size should be larger " \
    "than the overlap size"
NON_NEGATIVE_INDEX_MESSAGE = "the index should be larger than or equal to zero"
MATRIX_DIMENSION_UNEQUAL_MESSAGE = "the dimension of the matrix " \
    "should be equal"
EMPTY_LIST_MESSAGE = "the list should not be empty"
WINDOW_SIZE_LARGE_MESSAGE = "The window size must be less than or equal to" \
                            " the length of the given document"
INVALID_CUTTING_TYPE_MESSAGE = "the cutting type should be letters, lines, " \
    "number, words or milestone"
EMPTY_MILESTONE_MESSAGE = "the milestone should not be empty"
EMPTY_NP_ARRAY_MESSAGE = "The input numpy array should not be empty, since" \
    "the input file should not be empty."
NOT_ONE_REPLACEMENT_COLON_MESSAGE = "Invalid number of colons: "
REPLACEMENT_RIGHT_OPERAND_MESSAGE = "Too many values on right side of " \
                                    "replacement string: "
REPLACEMENT_NO_LEFTHAND_MESSAGE = "Missing value on the left side of " \
                                  "replacement string: "
EMPTY_STRING_MESSAGE = "the string should not be empty"
WINDOW_NON_POSITIVE_MESSAGE = "The window size must be a positive integer"

# ============= Base Model Errors ===========================
NO_DATA_SENT_ERROR_MESSAGE = 'Front end did not send data to backend'
INVALID_DATA_KEY_MESSAGE_FORMAT = \
    'no data with name {data_key} sended to backend'

# ============= Topword Model Error =========================
NOT_ENOUGH_CLASSES_MESSAGE = "Only one class given, cannot do Z-test by " \
                             "class, at least 2 classes needed."
<|endoftext|>


<commit_before>import dask.array as da
import numpy as np
import dask
from dask.array.ghost import *
from dask.core import get

def eq(a, b):
    if isinstance(a, Array):
        a = a.compute(get=dask.get)
    if isinstance(b, Array):
        b = b.compute(get=dask.get)
    c = a == b
    if isinstance(c, np.ndarray):
        c = c.all()
    return c

def test_fractional_slice():
    assert fractional_slice(('x', 4.9), {0: 2}) == \
            (getitem, ('x', 5), (slice(0, 2),))

    assert fractional_slice(('x', 3, 5.1), {0: 2, 1: 3}) == \
            (getitem, ('x', 3, 5), (slice(None, None, None), slice(-3, None)))

    assert fractional_slice(('x', 2.9, 5.1), {0: 2, 1: 3}) == \
            (getitem, ('x', 3, 5), (slice(0, 2), slice(-3, None)))

def test_ghost_internal():
    x = np.arange(64).reshape((8, 8))
    d = da.from_array(x, chunks=(4, 4))

    g = ghost_internal(d, {0: 2, 1: 1})
    result = g.compute(get=get)
    assert g.chunks == ((6, 6), (5, 5))

    expected = np.array([
        [ 0,  1,  2,  3,  4,    3,  4,  5,  6,  7],
        [ 8,  9, 10, 11, 12,   11, 12, 13, 14, 15],
        [16, 17, 18, 19, 20,   19, 20, 21, 22, 23],
        [24, 25, 26, 27, 28,   27, 28, 29, 30, 31],
        [32, 33, 34, 35, 36,   35, 36, 37, 38, 39],
        [40, 41, 42, 43, 44,   43, 44, 45, 46, 47],

        [16, 17, 18, 19, 20,   19, 20, 21, 22, 23],
        [24, 25, 26, 27, 28,   27, 28, 29, 30, 31],
        [32, 33, 34, 35, 36,   35, 36, 37, 38, 39],
        [40, 41, 42, 43, 44,   43, 44, 45, 46, 47],
        [48, 49, 50, 51, 52,   51, 52, 53, 54, 55],
        [56, 57, 58, 59, 60,   59, 60, 61, 62, 63]])

    assert eq(result, expected)

def test_trim_internal():
    d = da.ones((40, 60), chunks=(10, 10))
    e = trim_internal(d, axes={0: 1, 1: 2})

    assert e.chunks == ((8, 8, 8, 8), (6, 6, 6, 6, 6, 6))

def test_periodic():
    x = np.arange(64).reshape((8, 8))
    d = da.from_array(x, chunks=(4, 4))

    e = periodic(d, axis=0, depth=2)
    assert e.shape[0] == d.shape[0] + 4
    assert e.shape[1] == d.shape[1]

    assert eq(e[1, :], d[-1, :])
    assert eq(e[0, :], d[-2, :])

def test_reflect():
    x = np.arange(10)
    d = da.from_array(x, chunks=(5, 5))

    e = reflect(d, axis=0, depth=2)
    expected = np.array([1, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9, 8])
    assert eq(e, expected)

    e = reflect(d, axis=0, depth=1)
    expected = np.array([0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9])
    assert eq(e, expected)

def test_nearest():
    x = np.arange(10)
    d = da.from_array(x, chunks=(5, 5))

    e = nearest(d, axis=0, depth=2)
    expected = np.array([0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9, 9])
    assert eq(e, expected)

    e = nearest(d, axis=0, depth=1)
    expected = np.array([0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9])
    assert eq(e, expected)

def test_constant():
    x = np.arange(64).reshape((8, 8))
    d = da.from_array(x, chunks=(4, 4))

    e = constant(d, axis=0, depth=2, value=10)
    assert e.shape[0] == d.shape[0] + 4
    assert e.shape[1] == d.shape[1]

    assert eq(e[1, :], 10)
    assert eq(e[-1, :], 10)

def test_boundaries():
    x = np.arange(64).reshape((8, 8))
    d = da.from_array(x, chunks=(4, 4))

    e = boundaries(d, {0: 2, 1: 1}, {0: 0, 1: 'periodic'})

    expected = np.array(
        [[ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [ 7, 0, 1, 2, 3, 4, 5, 6, 7, 0],
         [15, 8, 9,10,11,12,13,14,15, 8],
         [23,16,17,18,19,20,21,22,23,16],
         [31,24,25,26,27,28,29,30,31,24],
         [39,32,33,34,35,36,37,38,39,32],
         [47,40,41,42,43,44,45,46,47,40],
         [55,48,49,50,51,52,53,54,55,48],
         [63,56,57,58,59,60,61,62,63,56],
         [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])
    assert eq(e, expected)

def test_ghost():
    x = np.arange(64).reshape((8, 8))
    d = da.from_array(x, chunks=(4, 4))
    g = ghost(d, depth={0: 2, 1: 1}, boundary={0: 100, 1: 'reflect'})
    assert g.chunks == ((8, 8), (6, 6))
    expected = np.array(
      [[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
       [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
       [  0,   0,   1,   2,   3,   4,   3,   4,   5,   6,   7,   7],
       [  8,   8,   9,  10,  11,  12,  11,  12,  13,  14,  15,  15],
       [ 16,  16,  17,  18,  19,  20,  19,  20,  21,  22,  23,  23],
       [ 24,  24,  25,  26,  27,  28,  27,  28,  29,  30,  31,  31],
       [ 32,  32,  33,  34,  35,  36,  35,  36,  37,  38,  39,  39],
       [ 40,  40,  41,  42,  43,  44,  43,  44,  45,  46,  47,  47],
       [ 16,  16,  17,  18,  19,  20,  19,  20,  21,  22,  23,  23],
       [ 24,  24,  25,  26,  27,  28,  27,  28,  29,  30,  31,  31],
       [ 32,  32,  33,  34,  35,  36,  35,  36,  37,  38,  39,  39],
       [ 40,  40,  41,  42,  43,  44,  43,  44,  45,  46,  47,  47],
       [ 48,  48,  49,  50,  51,  52,  51,  52,  53,  54,  55,  55],
       [ 56,  56,  57,  58,  59,  60,  59,  60,  61,  62,  63,  63],
       [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
       [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100]])
    assert eq(g, expected)

    g = ghost(d, depth={0: 2, 1: 1}, boundary={0: 100})
    assert g.chunks == ((8, 8), (5, 5))
<commit_msg>Remove wildcard import<commit_after>import dask.array as da
import numpy as np
import dask
from dask.array.ghost import (Array, fractional_slice, getitem, trim_internal,
                              ghost_internal, nearest, constant, boundaries,
                              reflect, periodic, ghost)
from dask.core import get

def eq(a, b):
    if isinstance(a, Array):
        a = a.compute(get=dask.get)
    if isinstance(b, Array):
        b = b.compute(get=dask.get)
    c = a == b
    if isinstance(c, np.ndarray):
        c = c.all()
    return c

def test_fractional_slice():
    assert fractional_slice(('x', 4.9), {0: 2}) == \
            (getitem, ('x', 5), (slice(0, 2),))

    assert fractional_slice(('x', 3, 5.1), {0: 2, 1: 3}) == \
            (getitem, ('x', 3, 5), (slice(None, None, None), slice(-3, None)))

    assert fractional_slice(('x', 2.9, 5.1), {0: 2, 1: 3}) == \
            (getitem, ('x', 3, 5), (slice(0, 2), slice(-3, None)))

def test_ghost_internal():
    x = np.arange(64).reshape((8, 8))
    d = da.from_array(x, chunks=(4, 4))

    g = ghost_internal(d, {0: 2, 1: 1})
    result = g.compute(get=get)
    assert g.chunks == ((6, 6), (5, 5))

    expected = np.array([
        [ 0,  1,  2,  3,  4,    3,  4,  5,  6,  7],
        [ 8,  9, 10, 11, 12,   11, 12, 13, 14, 15],
        [16, 17, 18, 19, 20,   19, 20, 21, 22, 23],
        [24, 25, 26, 27, 28,   27, 28, 29, 30, 31],
        [32, 33, 34, 35, 36,   35, 36, 37, 38, 39],
        [40, 41, 42, 43, 44,   43, 44, 45, 46, 47],

        [16, 17, 18, 19, 20,   19, 20, 21, 22, 23],
        [24, 25, 26, 27, 28,   27, 28, 29, 30, 31],
        [32, 33, 34, 35, 36,   35, 36, 37, 38, 39],
        [40, 41, 42, 43, 44,   43, 44, 45, 46, 47],
        [48, 49, 50, 51, 52,   51, 52, 53, 54, 55],
        [56, 57, 58, 59, 60,   59, 60, 61, 62, 63]])

    assert eq(result, expected)

def test_trim_internal():
    d = da.ones((40, 60), chunks=(10, 10))
    e = trim_internal(d, axes={0: 1, 1: 2})

    assert e.chunks == ((8, 8, 8, 8), (6, 6, 6, 6, 6, 6))

def test_periodic():
    x = np.arange(64).reshape((8, 8))
    d = da.from_array(x, chunks=(4, 4))

    e = periodic(d, axis=0, depth=2)
    assert e.shape[0] == d.shape[0] + 4
    assert e.shape[1] == d.shape[1]

    assert eq(e[1, :], d[-1, :])
    assert eq(e[0, :], d[-2, :])

def test_reflect():
    x = np.arange(10)
    d = da.from_array(x, chunks=(5, 5))

    e = reflect(d, axis=0, depth=2)
    expected = np.array([1, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9, 8])
    assert eq(e, expected)

    e = reflect(d, axis=0, depth=1)
    expected = np.array([0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9])
    assert eq(e, expected)

def test_nearest():
    x = np.arange(10)
    d = da.from_array(x, chunks=(5, 5))

    e = nearest(d, axis=0, depth=2)
    expected = np.array([0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9, 9])
    assert eq(e, expected)

    e = nearest(d, axis=0, depth=1)
    expected = np.array([0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9])
    assert eq(e, expected)

def test_constant():
    x = np.arange(64).reshape((8, 8))
    d = da.from_array(x, chunks=(4, 4))

    e = constant(d, axis=0, depth=2, value=10)
    assert e.shape[0] == d.shape[0] + 4
    assert e.shape[1] == d.shape[1]

    assert eq(e[1, :], 10)
    assert eq(e[-1, :], 10)

def test_boundaries():
    x = np.arange(64).reshape((8, 8))
    d = da.from_array(x, chunks=(4, 4))

    e = boundaries(d, {0: 2, 1: 1}, {0: 0, 1: 'periodic'})

    expected = np.array(
        [[ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [ 7, 0, 1, 2, 3, 4, 5, 6, 7, 0],
         [15, 8, 9,10,11,12,13,14,15, 8],
         [23,16,17,18,19,20,21,22,23,16],
         [31,24,25,26,27,28,29,30,31,24],
         [39,32,33,34,35,36,37,38,39,32],
         [47,40,41,42,43,44,45,46,47,40],
         [55,48,49,50,51,52,53,54,55,48],
         [63,56,57,58,59,60,61,62,63,56],
         [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])
    assert eq(e, expected)

def test_ghost():
    x = np.arange(64).reshape((8, 8))
    d = da.from_array(x, chunks=(4, 4))
    g = ghost(d, depth={0: 2, 1: 1}, boundary={0: 100, 1: 'reflect'})
    assert g.chunks == ((8, 8), (6, 6))
    expected = np.array(
      [[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
       [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
       [  0,   0,   1,   2,   3,   4,   3,   4,   5,   6,   7,   7],
       [  8,   8,   9,  10,  11,  12,  11,  12,  13,  14,  15,  15],
       [ 16,  16,  17,  18,  19,  20,  19,  20,  21,  22,  23,  23],
       [ 24,  24,  25,  26,  27,  28,  27,  28,  29,  30,  31,  31],
       [ 32,  32,  33,  34,  35,  36,  35,  36,  37,  38,  39,  39],
       [ 40,  40,  41,  42,  43,  44,  43,  44,  45,  46,  47,  47],
       [ 16,  16,  17,  18,  19,  20,  19,  20,  21,  22,  23,  23],
       [ 24,  24,  25,  26,  27,  28,  27,  28,  29,  30,  31,  31],
       [ 32,  32,  33,  34,  35,  36,  35,  36,  37,  38,  39,  39],
       [ 40,  40,  41,  42,  43,  44,  43,  44,  45,  46,  47,  47],
       [ 48,  48,  49,  50,  51,  52,  51,  52,  53,  54,  55,  55],
       [ 56,  56,  57,  58,  59,  60,  59,  60,  61,  62,  63,  63],
       [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
       [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100]])
    assert eq(g, expected)

    g = ghost(d, depth={0: 2, 1: 1}, boundary={0: 100})
    assert g.chunks == ((8, 8), (5, 5))
<|endoftext|>


<commit_before>import pytest
import numpy as np
from numpy.testing import assert_array_almost_equal, assert_array_equal

import dask
import dask.array as da
from dask.array.ghost import (Array, fractional_slice, getitem, trim_internal,
                              ghost_internal, nearest, constant, boundaries,
                              reflect, periodic, ghost)
from dask.core import get

def eq(a, b):
    if isinstance(a, Array):
        a = a.compute(get=dask.get)
    if isinstance(b, Array):
        b = b.compute(get=dask.get)
    c = a == b
    if isinstance(c, np.ndarray):
        c = c.all()
    return c

def test_fractional_slice():
    assert fractional_slice(('x', 4.9), {0: 2}) == \
            (getitem, ('x', 5), (slice(0, 2),))

    assert fractional_slice(('x', 3, 5.1), {0: 2, 1: 3}) == \
            (getitem, ('x', 3, 5), (slice(None, None, None), slice(-3, None)))

    assert fractional_slice(('x', 2.9, 5.1), {0: 2, 1: 3}) == \
            (getitem, ('x', 3, 5), (slice(0, 2), slice(-3, None)))

def test_ghost_internal():
    x = np.arange(64).reshape((8, 8))
    d = da.from_array(x, chunks=(4, 4))

    g = ghost_internal(d, {0: 2, 1: 1})
    result = g.compute(get=get)
    assert g.chunks == ((6, 6), (5, 5))

    expected = np.array([
        [ 0,  1,  2,  3,  4,    3,  4,  5,  6,  7],
        [ 8,  9, 10, 11, 12,   11, 12, 13, 14, 15],
        [16, 17, 18, 19, 20,   19, 20, 21, 22, 23],
        [24, 25, 26, 27, 28,   27, 28, 29, 30, 31],
        [32, 33, 34, 35, 36,   35, 36, 37, 38, 39],
        [40, 41, 42, 43, 44,   43, 44, 45, 46, 47],

        [16, 17, 18, 19, 20,   19, 20, 21, 22, 23],
        [24, 25, 26, 27, 28,   27, 28, 29, 30, 31],
        [32, 33, 34, 35, 36,   35, 36, 37, 38, 39],
        [40, 41, 42, 43, 44,   43, 44, 45, 46, 47],
        [48, 49, 50, 51, 52,   51, 52, 53, 54, 55],
        [56, 57, 58, 59, 60,   59, 60, 61, 62, 63]])

    assert eq(result, expected)

def test_trim_internal():
    d = da.ones((40, 60), chunks=(10, 10))
    e = trim_internal(d, axes={0: 1, 1: 2})

    assert e.chunks == ((8, 8, 8, 8), (6, 6, 6, 6, 6, 6))

def test_periodic():
    x = np.arange(64).reshape((8, 8))
    d = da.from_array(x, chunks=(4, 4))

    e = periodic(d, axis=0, depth=2)
    assert e.shape[0] == d.shape[0] + 4
    assert e.shape[1] == d.shape[1]

    assert eq(e[1, :], d[-1, :])
    assert eq(e[0, :], d[-2, :])

def test_reflect():
    x = np.arange(10)
    d = da.from_array(x, chunks=(5, 5))

    e = reflect(d, axis=0, depth=2)
    expected = np.array([1, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9, 8])
    assert eq(e, expected)

    e = reflect(d, axis=0, depth=1)
    expected = np.array([0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9])
    assert eq(e, expected)

def test_nearest():
    x = np.arange(10)
    d = da.from_array(x, chunks=(5, 5))

    e = nearest(d, axis=0, depth=2)
    expected = np.array([0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9, 9])
    assert eq(e, expected)

    e = nearest(d, axis=0, depth=1)
    expected = np.array([0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9])
    assert eq(e, expected)

def test_constant():
    x = np.arange(64).reshape((8, 8))
    d = da.from_array(x, chunks=(4, 4))

    e = constant(d, axis=0, depth=2, value=10)
    assert e.shape[0] == d.shape[0] + 4
    assert e.shape[1] == d.shape[1]

    assert eq(e[1, :], 10)
    assert eq(e[-1, :], 10)

def test_boundaries():
    x = np.arange(64).reshape((8, 8))
    d = da.from_array(x, chunks=(4, 4))

    e = boundaries(d, {0: 2, 1: 1}, {0: 0, 1: 'periodic'})

    expected = np.array(
        [[ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [ 7, 0, 1, 2, 3, 4, 5, 6, 7, 0],
         [15, 8, 9,10,11,12,13,14,15, 8],
         [23,16,17,18,19,20,21,22,23,16],
         [31,24,25,26,27,28,29,30,31,24],
         [39,32,33,34,35,36,37,38,39,32],
         [47,40,41,42,43,44,45,46,47,40],
         [55,48,49,50,51,52,53,54,55,48],
         [63,56,57,58,59,60,61,62,63,56],
         [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])
    assert eq(e, expected)

def test_ghost():
    x = np.arange(64).reshape((8, 8))
    d = da.from_array(x, chunks=(4, 4))
    g = ghost(d, depth={0: 2, 1: 1}, boundary={0: 100, 1: 'reflect'})
    assert g.chunks == ((8, 8), (6, 6))
    expected = np.array(
      [[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
       [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
       [  0,   0,   1,   2,   3,   4,   3,   4,   5,   6,   7,   7],
       [  8,   8,   9,  10,  11,  12,  11,  12,  13,  14,  15,  15],
       [ 16,  16,  17,  18,  19,  20,  19,  20,  21,  22,  23,  23],
       [ 24,  24,  25,  26,  27,  28,  27,  28,  29,  30,  31,  31],
       [ 32,  32,  33,  34,  35,  36,  35,  36,  37,  38,  39,  39],
       [ 40,  40,  41,  42,  43,  44,  43,  44,  45,  46,  47,  47],
       [ 16,  16,  17,  18,  19,  20,  19,  20,  21,  22,  23,  23],
       [ 24,  24,  25,  26,  27,  28,  27,  28,  29,  30,  31,  31],
       [ 32,  32,  33,  34,  35,  36,  35,  36,  37,  38,  39,  39],
       [ 40,  40,  41,  42,  43,  44,  43,  44,  45,  46,  47,  47],
       [ 48,  48,  49,  50,  51,  52,  51,  52,  53,  54,  55,  55],
       [ 56,  56,  57,  58,  59,  60,  59,  60,  61,  62,  63,  63],
       [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
       [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100]])
    assert eq(g, expected)

    g = ghost(d, depth={0: 2, 1: 1}, boundary={0: 100})
    assert g.chunks == ((8, 8), (5, 5))

def test_map_overlap():
    x = da.arange(10, chunks=5)

    y = x.map_overlap(lambda x: x + len(x), depth=2)
    assert eq(y, np.arange(10) + 5 + 2 + 2)

def test_nearest_ghost():
    a = np.arange(144).reshape(12, 12).astype(float)

    darr = da.from_array(a, chunks=(6, 6))
    garr = ghost(darr, depth={0: 5, 1: 5},
                 boundary={0: 'nearest', 1: 'nearest'})
    tarr = trim_internal(garr, {0: 5, 1: 5})
    assert_array_almost_equal(np.array(tarr), a)

def test_0_depth():
    expected = np.arange(100).reshape(10, 10)
    darr = da.from_array(expected, chunks=(5, 2))

    depth = {0: 0, 1: 0}

    reflected = ghost(darr, depth=depth, boundary='reflect')
    nearest = ghost(darr, depth=depth, boundary='nearest')
    periodic = ghost(darr, depth=depth, boundary='periodic')
    constant = ghost(darr, depth=depth, boundary=42)

    result = trim_internal(reflected, depth)
    assert_array_equal(result, expected)

    result = trim_internal(nearest, depth)
    assert_array_equal(result, expected)

    result = trim_internal(periodic, depth)
    assert_array_equal(result, expected)

    result = trim_internal(constant, depth)
    assert_array_equal(result, expected)

def test_depth_equals_boundary_length():
    expected = np.arange(100).reshape(10, 10)
    darr = da.from_array(expected, chunks=(5, 5))

    depth = {0: 5, 1: 5}

    reflected = ghost(darr, depth=depth, boundary='reflect')
    nearest = ghost(darr, depth=depth, boundary='nearest')
    periodic = ghost(darr, depth=depth, boundary='periodic')
    constant = ghost(darr, depth=depth, boundary=42)

    result = trim_internal(reflected, depth)
    assert_array_equal(result, expected)

    result = trim_internal(nearest, depth)
    assert_array_equal(result, expected)

    result = trim_internal(periodic, depth)
    assert_array_equal(result, expected)

    result = trim_internal(constant, depth)
    assert_array_equal(result, expected)

@pytest.mark.xfail
def test_depth_greater_than_boundary_length():
    expected = np.arange(100).reshape(10, 10)
    darr = da.from_array(expected, chunks=(5, 5))

    depth = {0: 8, 1: 7}

    reflected = ghost(darr, depth=depth, boundary='reflect')
    nearest = ghost(darr, depth=depth, boundary='nearest')
    periodic = ghost(darr, depth=depth, boundary='periodic')
    constant = ghost(darr, depth=depth, boundary=42)

    result = trim_internal(reflected, depth)
    assert_array_equal(result, expected)

    result = trim_internal(nearest, depth)
    assert_array_equal(result, expected)

    result = trim_internal(periodic, depth)
    assert_array_equal(result, expected)

    result = trim_internal(constant, depth)
    assert_array_equal(result, expected)
<commit_msg>Don't use `np.array()` in assert_array_equal<commit_after>import pytest
import numpy as np
from numpy.testing import assert_array_almost_equal, assert_array_equal

import dask
import dask.array as da
from dask.array.ghost import (Array, fractional_slice, getitem, trim_internal,
                              ghost_internal, nearest, constant, boundaries,
                              reflect, periodic, ghost)
from dask.core import get

def eq(a, b):
    if isinstance(a, Array):
        a = a.compute(get=dask.get)
    if isinstance(b, Array):
        b = b.compute(get=dask.get)
    c = a == b
    if isinstance(c, np.ndarray):
        c = c.all()
    return c

def test_fractional_slice():
    assert fractional_slice(('x', 4.9), {0: 2}) == \
            (getitem, ('x', 5), (slice(0, 2),))

    assert fractional_slice(('x', 3, 5.1), {0: 2, 1: 3}) == \
            (getitem, ('x', 3, 5), (slice(None, None, None), slice(-3, None)))

    assert fractional_slice(('x', 2.9, 5.1), {0: 2, 1: 3}) == \
            (getitem, ('x', 3, 5), (slice(0, 2), slice(-3, None)))

def test_ghost_internal():
    x = np.arange(64).reshape((8, 8))
    d = da.from_array(x, chunks=(4, 4))

    g = ghost_internal(d, {0: 2, 1: 1})
    result = g.compute(get=get)
    assert g.chunks == ((6, 6), (5, 5))

    expected = np.array([
        [ 0,  1,  2,  3,  4,    3,  4,  5,  6,  7],
        [ 8,  9, 10, 11, 12,   11, 12, 13, 14, 15],
        [16, 17, 18, 19, 20,   19, 20, 21, 22, 23],
        [24, 25, 26, 27, 28,   27, 28, 29, 30, 31],
        [32, 33, 34, 35, 36,   35, 36, 37, 38, 39],
        [40, 41, 42, 43, 44,   43, 44, 45, 46, 47],

        [16, 17, 18, 19, 20,   19, 20, 21, 22, 23],
        [24, 25, 26, 27, 28,   27, 28, 29, 30, 31],
        [32, 33, 34, 35, 36,   35, 36, 37, 38, 39],
        [40, 41, 42, 43, 44,   43, 44, 45, 46, 47],
        [48, 49, 50, 51, 52,   51, 52, 53, 54, 55],
        [56, 57, 58, 59, 60,   59, 60, 61, 62, 63]])

    assert eq(result, expected)

def test_trim_internal():
    d = da.ones((40, 60), chunks=(10, 10))
    e = trim_internal(d, axes={0: 1, 1: 2})

    assert e.chunks == ((8, 8, 8, 8), (6, 6, 6, 6, 6, 6))

def test_periodic():
    x = np.arange(64).reshape((8, 8))
    d = da.from_array(x, chunks=(4, 4))

    e = periodic(d, axis=0, depth=2)
    assert e.shape[0] == d.shape[0] + 4
    assert e.shape[1] == d.shape[1]

    assert eq(e[1, :], d[-1, :])
    assert eq(e[0, :], d[-2, :])

def test_reflect():
    x = np.arange(10)
    d = da.from_array(x, chunks=(5, 5))

    e = reflect(d, axis=0, depth=2)
    expected = np.array([1, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9, 8])
    assert eq(e, expected)

    e = reflect(d, axis=0, depth=1)
    expected = np.array([0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9])
    assert eq(e, expected)

def test_nearest():
    x = np.arange(10)
    d = da.from_array(x, chunks=(5, 5))

    e = nearest(d, axis=0, depth=2)
    expected = np.array([0, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9, 9])
    assert eq(e, expected)

    e = nearest(d, axis=0, depth=1)
    expected = np.array([0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 9])
    assert eq(e, expected)

def test_constant():
    x = np.arange(64).reshape((8, 8))
    d = da.from_array(x, chunks=(4, 4))

    e = constant(d, axis=0, depth=2, value=10)
    assert e.shape[0] == d.shape[0] + 4
    assert e.shape[1] == d.shape[1]

    assert eq(e[1, :], 10)
    assert eq(e[-1, :], 10)

def test_boundaries():
    x = np.arange(64).reshape((8, 8))
    d = da.from_array(x, chunks=(4, 4))

    e = boundaries(d, {0: 2, 1: 1}, {0: 0, 1: 'periodic'})

    expected = np.array(
        [[ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [ 7, 0, 1, 2, 3, 4, 5, 6, 7, 0],
         [15, 8, 9,10,11,12,13,14,15, 8],
         [23,16,17,18,19,20,21,22,23,16],
         [31,24,25,26,27,28,29,30,31,24],
         [39,32,33,34,35,36,37,38,39,32],
         [47,40,41,42,43,44,45,46,47,40],
         [55,48,49,50,51,52,53,54,55,48],
         [63,56,57,58,59,60,61,62,63,56],
         [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
         [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]])
    assert eq(e, expected)

def test_ghost():
    x = np.arange(64).reshape((8, 8))
    d = da.from_array(x, chunks=(4, 4))
    g = ghost(d, depth={0: 2, 1: 1}, boundary={0: 100, 1: 'reflect'})
    assert g.chunks == ((8, 8), (6, 6))
    expected = np.array(
      [[100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
       [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
       [  0,   0,   1,   2,   3,   4,   3,   4,   5,   6,   7,   7],
       [  8,   8,   9,  10,  11,  12,  11,  12,  13,  14,  15,  15],
       [ 16,  16,  17,  18,  19,  20,  19,  20,  21,  22,  23,  23],
       [ 24,  24,  25,  26,  27,  28,  27,  28,  29,  30,  31,  31],
       [ 32,  32,  33,  34,  35,  36,  35,  36,  37,  38,  39,  39],
       [ 40,  40,  41,  42,  43,  44,  43,  44,  45,  46,  47,  47],
       [ 16,  16,  17,  18,  19,  20,  19,  20,  21,  22,  23,  23],
       [ 24,  24,  25,  26,  27,  28,  27,  28,  29,  30,  31,  31],
       [ 32,  32,  33,  34,  35,  36,  35,  36,  37,  38,  39,  39],
       [ 40,  40,  41,  42,  43,  44,  43,  44,  45,  46,  47,  47],
       [ 48,  48,  49,  50,  51,  52,  51,  52,  53,  54,  55,  55],
       [ 56,  56,  57,  58,  59,  60,  59,  60,  61,  62,  63,  63],
       [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100],
       [100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 100]])
    assert eq(g, expected)

    g = ghost(d, depth={0: 2, 1: 1}, boundary={0: 100})
    assert g.chunks == ((8, 8), (5, 5))

def test_map_overlap():
    x = da.arange(10, chunks=5)

    y = x.map_overlap(lambda x: x + len(x), depth=2)
    assert eq(y, np.arange(10) + 5 + 2 + 2)

def test_nearest_ghost():
    a = np.arange(144).reshape(12, 12).astype(float)

    darr = da.from_array(a, chunks=(6, 6))
    garr = ghost(darr, depth={0: 5, 1: 5},
                 boundary={0: 'nearest', 1: 'nearest'})
    tarr = trim_internal(garr, {0: 5, 1: 5})
    assert_array_almost_equal(tarr, a)

def test_0_depth():
    expected = np.arange(100).reshape(10, 10)
    darr = da.from_array(expected, chunks=(5, 2))

    depth = {0: 0, 1: 0}

    reflected = ghost(darr, depth=depth, boundary='reflect')
    nearest = ghost(darr, depth=depth, boundary='nearest')
    periodic = ghost(darr, depth=depth, boundary='periodic')
    constant = ghost(darr, depth=depth, boundary=42)

    result = trim_internal(reflected, depth)
    assert_array_equal(result, expected)

    result = trim_internal(nearest, depth)
    assert_array_equal(result, expected)

    result = trim_internal(periodic, depth)
    assert_array_equal(result, expected)

    result = trim_internal(constant, depth)
    assert_array_equal(result, expected)

def test_depth_equals_boundary_length():
    expected = np.arange(100).reshape(10, 10)
    darr = da.from_array(expected, chunks=(5, 5))

    depth = {0: 5, 1: 5}

    reflected = ghost(darr, depth=depth, boundary='reflect')
    nearest = ghost(darr, depth=depth, boundary='nearest')
    periodic = ghost(darr, depth=depth, boundary='periodic')
    constant = ghost(darr, depth=depth, boundary=42)

    result = trim_internal(reflected, depth)
    assert_array_equal(result, expected)

    result = trim_internal(nearest, depth)
    assert_array_equal(result, expected)

    result = trim_internal(periodic, depth)
    assert_array_equal(result, expected)

    result = trim_internal(constant, depth)
    assert_array_equal(result, expected)

@pytest.mark.xfail
def test_depth_greater_than_boundary_length():
    expected = np.arange(100).reshape(10, 10)
    darr = da.from_array(expected, chunks=(5, 5))

    depth = {0: 8, 1: 7}

    reflected = ghost(darr, depth=depth, boundary='reflect')
    nearest = ghost(darr, depth=depth, boundary='nearest')
    periodic = ghost(darr, depth=depth, boundary='periodic')
    constant = ghost(darr, depth=depth, boundary=42)

    result = trim_internal(reflected, depth)
    assert_array_equal(result, expected)

    result = trim_internal(nearest, depth)
    assert_array_equal(result, expected)

    result = trim_internal(periodic, depth)
    assert_array_equal(result, expected)

    result = trim_internal(constant, depth)
    assert_array_equal(result, expected)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import (absolute_import, print_function, unicode_literals)
import datetime
from acli.output.cloudwatch import (output_ec2_cpu, output_ec2_net,
                                    output_asg_cpu, output_ec2_vols)
from acli.services.ec2 import ec2_get_instance_vols
from acli.connections import get_client

def ec2_net(aws_config=None, instance_id=None, intervals=None, period=None,
            start=None, end=datetime.datetime.utcnow()):
    """
    @type aws_config: Config
    @type instance_id: unicode
    @type intervals: int
    @type period: int
    @type start: datetime
    @type end: datetime
    """
    if not intervals:
        intervals = 60
    if not period:
        period = 7200
    cloudwatch_client = get_client(client_type='cloudwatch', config=aws_config)
    if not start:
        start = end - datetime.timedelta(seconds=period)
    net_in = cloudwatch_client.get_metric_statistics(
            Namespace='AWS/EC2',
            MetricName='NetworkIn',
            Dimensions=[{'Name': 'InstanceId', 'Value': instance_id}],
            StartTime=start,
            EndTime=datetime.datetime.utcnow(),
            Period=intervals,
            Statistics=['Average'],
            Unit='Bytes'
        )
    net_out = cloudwatch_client.get_metric_statistics(
            Namespace='AWS/EC2',
            MetricName='NetworkOut',
            Dimensions=[{'Name': 'InstanceId', 'Value': instance_id}],
            StartTime=start,
            EndTime=datetime.datetime.utcnow(),
            Period=intervals,
            Statistics=['Average'],
            Unit='Bytes'
        )

    net_in_datapoints, net_out_datapoints = net_in.get('Datapoints'), net_out.get('Datapoints')
    if not all((net_in_datapoints, net_out_datapoints)):
        exit("Metrics unavailable.")
    sorted_net_in_datapoints = sorted(net_in_datapoints, key=lambda v: v.get('Timestamp'))
    sorted_net_out_datapoints = sorted(net_out_datapoints, key=lambda v: v.get('Timestamp'))
    in_dates = [x1.get('Timestamp') for x1 in sorted_net_in_datapoints]
    in_values = [x2.get('Average') for x2 in sorted_net_in_datapoints]
    out_dates = [x3.get('Timestamp') for x3 in sorted_net_out_datapoints]
    out_values = [x4.get('Average') for x4 in sorted_net_out_datapoints]
    output_ec2_net(in_dates=in_dates, in_values=in_values,
                   out_dates=out_dates, out_values=out_values,
                   instance_id=instance_id)
    exit(0)

def ec2_cpu(aws_config=None, instance_id=None, intervals=None, period=None,
            start=None, end=datetime.datetime.utcnow()):
    """
    @type aws_config: Config
    @type instance_id: unicode
    @type intervals: int
    @type period: int
    @type start: datetime
    @type end: datetime
    """
    if not intervals:
        intervals = 60
    if not period:
        period = 7200
    cloudwatch_client = get_client(client_type='cloudwatch', config=aws_config)
    if not start:
        start = end - datetime.timedelta(seconds=period)
    out = cloudwatch_client.get_metric_statistics(
            Namespace='AWS/EC2',
            MetricName='CPUUtilization',
            Dimensions=[
                {
                 'Name': 'InstanceId',
                 'Value': instance_id
                }
            ],
            StartTime=start,
            EndTime=datetime.datetime.utcnow(),
            Period=intervals,
            Statistics=[
                'Average',
            ],
            Unit='Percent'
        )
    datapoints = out.get('Datapoints')
    sorted_datapoints = sorted(datapoints, key=lambda v: v.get('Timestamp'))
    dates = list()
    values = list()
    for datapoint in sorted_datapoints:
        dates.append(datapoint.get('Timestamp'))
        values.append(datapoint.get('Average'))
    output_ec2_cpu(dates=dates, values=values, instance_id=instance_id)
    exit(0)

def asg_cpu(aws_config=None, asg_name=None, intervals=None, period=None,
            start=None, end=datetime.datetime.utcnow(), output_type=None):
    """
    @type aws_config: Config
    @type asg_name: unicode
    @type intervals: int
    @type period: int
    @type start: datetime
    @type end: datetime
    @type output_type: unicode
    """
    if not output_type or output_type == 'graph':
        if not intervals:
            intervals = 60
        if not period:
            period = 7200
        cloudwatch_client = get_client(client_type='cloudwatch', config=aws_config)
        if not start:
            start = end - datetime.timedelta(seconds=period)
        out = cloudwatch_client.get_metric_statistics(
                Namespace='AWS/EC2',
                MetricName='CPUUtilization',
                Dimensions=[
                    {
                     'Name': 'AutoScalingGroupName',
                     'Value': asg_name
                    }
                ],
                StartTime=start,
                EndTime=datetime.datetime.utcnow(),
                Period=intervals,
                Statistics=[
                    'Average',
                ],
                Unit='Percent'
            )
        datapoints = out.get('Datapoints')
        sorted_datapoints = sorted(datapoints, key=lambda v: v.get('Timestamp'))
        dates = [y1.get('Timestamp') for y1 in sorted_datapoints]
        values = [y2.get('Average') for y2 in sorted_datapoints]

        output_asg_cpu(dates=dates, values=values, asg_name=asg_name)
        exit(0)
    elif output_type == 'table':
        print("table")
        exit(0)

def ec2_vol(aws_config=None, instance_id=None, intervals=None, period=None,
            start=None, end=datetime.datetime.utcnow()):
    """
    @type aws_config: Config
    @type instance_id: unicode
    @type intervals: int
    @type period: int
    @type start: datetime
    @type end: datetime
    """
    ebs_vols = ec2_get_instance_vols(aws_config=aws_config, instance_id=instance_id)
    if not intervals:
        intervals = 60
    if not period:
        period = 7200

    cloudwatch_client = get_client(client_type='cloudwatch', config=aws_config)
    if not start:
        start = end - datetime.timedelta(seconds=period)
    vol_datapoints = list()
    for ebs_vol in ebs_vols:
        read_ops = cloudwatch_client.get_metric_statistics(
                Namespace='AWS/EBS',
                MetricName='VolumeReadBytes',
                Dimensions=[{'Name': 'VolumeId', 'Value': ebs_vol['Ebs']['VolumeId']}],
                StartTime=start,
                EndTime=datetime.datetime.utcnow(),
                Period=intervals,
                Statistics=['Average'],
                Unit='Bytes'
        )
        write_ops = cloudwatch_client.get_metric_statistics(
                Namespace='AWS/EBS',
                MetricName='VolumeWriteBytes',
                Dimensions=[{'Name': 'VolumeId', 'Value': ebs_vol['Ebs']['VolumeId']}],
                StartTime=start,
                EndTime=datetime.datetime.utcnow(),
                Period=intervals,
                Statistics=['Average'],
                Unit='Bytes'
            )
        sorted_read_datapoints = sorted(read_ops.get('Datapoints'), key=lambda v: v.get('Timestamp'))
        sorted_write_datapoints = sorted(write_ops.get('Datapoints'), key=lambda v: v.get('Timestamp'))
        read_dates = [x1.get('Timestamp') for x1 in sorted_read_datapoints]
        read_values = [x2.get('Average') for x2 in sorted_read_datapoints]
        write_dates = [y1.get('Timestamp') for y1 in sorted_write_datapoints]
        write_values = [y2.get('Average') for y2 in sorted_write_datapoints]
        vol_datapoints.append({'device_name': ebs_vol['DeviceName'],
                               'read_dates': read_dates, 'read_values': read_values,
                               'write_dates': write_dates, 'write_values': write_values})
    output_ec2_vols(vols_datapoints=vol_datapoints, instance_id=instance_id)
<commit_msg>refactor.<commit_after># -*- coding: utf-8 -*-
from __future__ import (absolute_import, print_function, unicode_literals)
import datetime
from acli.output.cloudwatch import (output_ec2_cpu, output_ec2_net,
                                    output_asg_cpu, output_ec2_vols)
from acli.services.ec2 import ec2_get_instance_vols
from acli.connections import get_client

def ec2_net(aws_config=None, instance_id=None, intervals=None, period=None,
            start=None, end=datetime.datetime.utcnow()):
    """
    @type aws_config: Config
    @type instance_id: unicode
    @type intervals: int
    @type period: int
    @type start: datetime
    @type end: datetime
    """
    if not intervals:
        intervals = 60
    if not period:
        period = 7200
    cloudwatch_client = get_client(client_type='cloudwatch', config=aws_config)
    if not start:
        start = end - datetime.timedelta(seconds=period)
    net_in = cloudwatch_client.get_metric_statistics(
            Namespace='AWS/EC2',
            MetricName='NetworkIn',
            Dimensions=[{'Name': 'InstanceId', 'Value': instance_id}],
            StartTime=start,
            EndTime=datetime.datetime.utcnow(),
            Period=intervals,
            Statistics=['Average'],
            Unit='Bytes'
        )
    net_out = cloudwatch_client.get_metric_statistics(
            Namespace='AWS/EC2',
            MetricName='NetworkOut',
            Dimensions=[{'Name': 'InstanceId', 'Value': instance_id}],
            StartTime=start,
            EndTime=datetime.datetime.utcnow(),
            Period=intervals,
            Statistics=['Average'],
            Unit='Bytes'
        )

    net_in_datapoints, net_out_datapoints = net_in.get('Datapoints'), net_out.get('Datapoints')
    if not all((net_in_datapoints, net_out_datapoints)):
        exit("Metrics unavailable.")
    sorted_net_in_datapoints = sorted(net_in_datapoints, key=lambda v: v.get('Timestamp'))
    sorted_net_out_datapoints = sorted(net_out_datapoints, key=lambda v: v.get('Timestamp'))
    in_dates = [x1.get('Timestamp') for x1 in sorted_net_in_datapoints]
    in_values = [x2.get('Average') for x2 in sorted_net_in_datapoints]
    out_dates = [x3.get('Timestamp') for x3 in sorted_net_out_datapoints]
    out_values = [x4.get('Average') for x4 in sorted_net_out_datapoints]
    output_ec2_net(in_dates=in_dates, in_values=in_values,
                   out_dates=out_dates, out_values=out_values,
                   instance_id=instance_id)
    exit(0)

def ec2_cpu(aws_config=None, instance_id=None, intervals=None, period=None,
            start=None, end=datetime.datetime.utcnow()):
    """
    @type aws_config: Config
    @type instance_id: unicode
    @type intervals: int
    @type period: int
    @type start: datetime
    @type end: datetime
    """
    if not intervals:
        intervals = 60
    if not period:
        period = 7200
    cloudwatch_client = get_client(client_type='cloudwatch', config=aws_config)
    if not start:
        start = end - datetime.timedelta(seconds=period)
    out = cloudwatch_client.get_metric_statistics(
            Namespace='AWS/EC2',
            MetricName='CPUUtilization',
            Dimensions=[
                {
                 'Name': 'InstanceId',
                 'Value': instance_id
                }
            ],
            StartTime=start,
            EndTime=datetime.datetime.utcnow(),
            Period=intervals,
            Statistics=[
                'Average',
            ],
            Unit='Percent'
        )
    datapoints = out.get('Datapoints')
    sorted_datapoints = sorted(datapoints, key=lambda v: v.get('Timestamp'))
    dates = list()
    values = list()
    for datapoint in sorted_datapoints:
        dates.append(datapoint.get('Timestamp'))
        values.append(datapoint.get('Average'))
    output_ec2_cpu(dates=dates, values=values, instance_id=instance_id)
    exit(0)

def asg_cpu(aws_config=None, asg_name=None, start=None, period=None, intervals=None,
            output_type=None):
    """
    @type aws_config: Config
    @type asg_name: unicode
    @type intervals: int
    @type period: int
    @type start: datetime
    @type output_type: unicode
    """
    end = datetime.datetime.utcnow()
    if not output_type or output_type == 'graph':
        if not intervals:
            intervals = 60
        if not period:
            period = 7200
        cloudwatch_client = get_client(client_type='cloudwatch', config=aws_config)
        if not start:
            start = end - datetime.timedelta(seconds=period)
        out = cloudwatch_client.get_metric_statistics(
                Namespace='AWS/EC2',
                MetricName='CPUUtilization',
                Dimensions=[
                    {
                     'Name': 'AutoScalingGroupName',
                     'Value': asg_name
                    }
                ],
                StartTime=start,
                EndTime=datetime.datetime.utcnow(),
                Period=intervals,
                Statistics=[
                    'Average',
                ],
                Unit='Percent'
            )
        datapoints = out.get('Datapoints')
        sorted_datapoints = sorted(datapoints, key=lambda v: v.get('Timestamp'))
        dates = [y1.get('Timestamp') for y1 in sorted_datapoints]
        values = [y2.get('Average') for y2 in sorted_datapoints]

        output_asg_cpu(dates=dates, values=values, asg_name=asg_name)
        exit(0)
    elif output_type == 'table':
        print("table")
        exit(0)

def ec2_vol(aws_config=None, instance_id=None, intervals=None, period=None,
            start=None, end=datetime.datetime.utcnow()):
    """
    @type aws_config: Config
    @type instance_id: unicode
    @type intervals: int
    @type period: int
    @type start: datetime
    @type end: datetime
    """
    ebs_vols = ec2_get_instance_vols(aws_config=aws_config, instance_id=instance_id)
    if not intervals:
        intervals = 60
    if not period:
        period = 7200

    cloudwatch_client = get_client(client_type='cloudwatch', config=aws_config)
    if not start:
        start = end - datetime.timedelta(seconds=period)
    vol_datapoints = list()
    for ebs_vol in ebs_vols:
        read_ops = cloudwatch_client.get_metric_statistics(
                Namespace='AWS/EBS',
                MetricName='VolumeReadBytes',
                Dimensions=[{'Name': 'VolumeId', 'Value': ebs_vol['Ebs']['VolumeId']}],
                StartTime=start,
                EndTime=datetime.datetime.utcnow(),
                Period=intervals,
                Statistics=['Average'],
                Unit='Bytes'
        )
        write_ops = cloudwatch_client.get_metric_statistics(
                Namespace='AWS/EBS',
                MetricName='VolumeWriteBytes',
                Dimensions=[{'Name': 'VolumeId', 'Value': ebs_vol['Ebs']['VolumeId']}],
                StartTime=start,
                EndTime=datetime.datetime.utcnow(),
                Period=intervals,
                Statistics=['Average'],
                Unit='Bytes'
            )
        sorted_read_datapoints = sorted(read_ops.get('Datapoints'), key=lambda v: v.get('Timestamp'))
        sorted_write_datapoints = sorted(write_ops.get('Datapoints'), key=lambda v: v.get('Timestamp'))
        read_dates = [x1.get('Timestamp') for x1 in sorted_read_datapoints]
        read_values = [x2.get('Average') for x2 in sorted_read_datapoints]
        write_dates = [y1.get('Timestamp') for y1 in sorted_write_datapoints]
        write_values = [y2.get('Average') for y2 in sorted_write_datapoints]
        vol_datapoints.append({'device_name': ebs_vol['DeviceName'],
                               'read_dates': read_dates, 'read_values': read_values,
                               'write_dates': write_dates, 'write_values': write_values})
    output_ec2_vols(vols_datapoints=vol_datapoints, instance_id=instance_id)
<|endoftext|>