<commit_before>import os
import shutil
import socket
import subprocess
import sys
import tempfile
from errno import EBADF

from pywincffi.core import dist
from pywincffi.dev.testutil import TestCase
from pywincffi.exceptions import InputError, WindowsAPIError
from pywincffi.kernel32 import (
    GetStdHandle, CloseHandle, OpenProcess, WaitForSingleObject,
    handle_from_file, GetHandleInformation, SetHandleInformation)

try:
    WindowsError
except NameError:  # pragma: no cover
    WindowsError = OSError  # pylint: disable=redefined-builtin

class TestGetStdHandle(TestCase):
    """
    Tests for :func:`pywincffi.kernel32.GetStdHandle`
    """
    def test_stdin_handle(self):
        _, library = dist.load()
        self.assertEqual(
            GetStdHandle(library.STD_INPUT_HANDLE),
            library.GetStdHandle(library.STD_INPUT_HANDLE)
        )

    def test_stdout_handle(self):
        _, library = dist.load()
        self.assertEqual(
            GetStdHandle(library.STD_OUTPUT_HANDLE),
            library.GetStdHandle(library.STD_OUTPUT_HANDLE)
        )

    def test_stderr_handle(self):
        _, library = dist.load()
        self.assertEqual(
            GetStdHandle(library.STD_ERROR_HANDLE),
            library.GetStdHandle(library.STD_ERROR_HANDLE)
        )

class TestGetHandleFromFile(TestCase):
    """
    Tests for :func:`pywincffi.kernel32.handle_from_file`
    """
    def test_fails_if_not_a_file(self):
        with self.assertRaises(InputError):
            handle_from_file(0)

    def test_fails_if_file_is_not_open(self):
        fd, _ = tempfile.mkstemp()
        test_file = os.fdopen(fd, "r")
        test_file.close()

        with self.assertRaises(InputError):
            handle_from_file(test_file)

    def test_opens_correct_file_handle(self):
        fd, path = tempfile.mkstemp()
        os.close(fd)

        test_file = open(path, "w")
        handle = handle_from_file(test_file)

        CloseHandle(handle)

        # If CloseHandle() was passed the same handle
        # that test_file is trying to write to the file
        # and/or flushing it should fail.
        try:
            test_file.write("foo")
            test_file.flush()
        except (OSError, IOError, WindowsError) as error:
            # EBADF == Bad file descriptor (because CloseHandle closed it)
            self.assertEqual(error.errno, EBADF)
        else:
            self.fail("Expected os.close(%r) to fail" % fd)

class TestWaitForSingleObject(TestCase):
    """
    Tests for :func:`pywincffi.kernel32.WaitForSingleObject`
    """
    def test_wait_failed(self):
        # This should cause WAIT_FAILED to be returned by the underlying
        # WaitForSingleObject because we didn't request the SYNCHRONIZE
        # permission.
        process = self.create_python_process("import time; time.sleep(3)")
        _, library = dist.load()

        hProcess = OpenProcess(
            library.PROCESS_QUERY_INFORMATION, False, process.pid)
        self.addCleanup(CloseHandle, hProcess)

        with self.assertRaises(WindowsAPIError) as exec_:
            WaitForSingleObject(hProcess, 3)
            self.assertEqual(exec_.code, library.WAIT_FAILED)

    def test_wait_on_running_process(self):
        process = self.create_python_process("import time; time.sleep(1)")
        _, library = dist.load()

        hProcess = OpenProcess(
            library.PROCESS_QUERY_INFORMATION | library.SYNCHRONIZE,
            False, process.pid)
        self.addCleanup(CloseHandle, hProcess)
        self.assertEqual(
            WaitForSingleObject(hProcess, 0), library.WAIT_TIMEOUT)

    def test_process_dies_before_timeout(self):
        process = self.create_python_process("import time; time.sleep(1)")
        _, library = dist.load()

        hProcess = OpenProcess(
            library.PROCESS_QUERY_INFORMATION | library.SYNCHRONIZE,
            False, process.pid)
        self.addCleanup(CloseHandle, hProcess)
        self.assertEqual(
            WaitForSingleObject(hProcess, library.INFINITE),
            library.WAIT_OBJECT_0)

class TestGetHandleInformation(TestCase):
    """
    Tests for :func:`pywincffi.kernel32.GetHandleInformation`
    """
    def test_get_handle_info_stdin(self):
        _, library = dist.load()
        stdin_handle = GetStdHandle(library.STD_INPUT_HANDLE)
        handle_flags = GetHandleInformation(stdin_handle)
        inherit = handle_flags & library.HANDLE_FLAG_INHERIT
        expected = (0, library.HANDLE_FLAG_INHERIT)
        self.assertIn(inherit, expected)

    def _expected_inheritance(self):
        # Python >= 3.4 creates non-inheritable handles (PEP 0446)
        _, library = dist.load()
        inherit = library.HANDLE_FLAG_INHERIT
        return inherit if sys.version_info[:2] < (3, 4) else 0

    def test_get_handle_info_file(self):
        _, library = dist.load()
        # can't use mkstemp: not inheritable on Python < 3.4
        tempdir = tempfile.mkdtemp()
        self.addCleanup(os.rmdir, tempdir)
        filename = os.path.join(tempdir, "test_file")
        with open(filename, "w") as test_file:
            self.addCleanup(os.unlink, filename)
            test_file.write("data")
            file_handle = handle_from_file(test_file)
            handle_flags = GetHandleInformation(file_handle)
            inherit = handle_flags & library.HANDLE_FLAG_INHERIT
        expected = self._expected_inheritance()
        self.assertEqual(inherit, expected)

    def test_get_handle_info_socket(self):
        ffi, library = dist.load()
        sock = socket.socket()
        self.addCleanup(sock.close)
        sock_handle = ffi.cast("void *", sock.fileno())
        handle_flags = GetHandleInformation(sock_handle)
        inherit = handle_flags & library.HANDLE_FLAG_INHERIT
        expected = self._expected_inheritance()
        self.assertEqual(inherit, expected)

class TestSetHandleInformation(TestCase):
    """
    Tests for :func:`pywincffi.kernel32.SetHandleInformation`
    """
    def _set_handle_info_file(self, inherit, check=False):
        _, library = dist.load()
        tempdir = tempfile.mkdtemp()
        self.addCleanup(os.rmdir, tempdir)
        filename = os.path.join(tempdir, "test_file")
        with open(filename, "w") as test_file:
            self.addCleanup(os.unlink, filename)
            test_file.write("data")
            file_handle = handle_from_file(test_file)
            SetHandleInformation(
                file_handle,
                library.HANDLE_FLAG_INHERIT,
                inherit
            )
            if check:
                result = GetHandleInformation(file_handle)
                self.assertEqual(inherit, result)

    def test_set_handle_info_file_noinherit(self):
        self._set_handle_info_file(0)

    def test_set_handle_info_file_inherit(self):
        self._set_handle_info_file(1)

    def test_set_get_handle_info_file_noinherit(self):
        self._set_handle_info_file(0, check=True)

    def test_set_get_handle_info_file_inherit(self):
        self._set_handle_info_file(1, check=True)

    def _set_handle_info_socket(self, inherit, check=False):
        ffi, library = dist.load()
        sock = socket.socket()
        self.addCleanup(sock.close)
        sock_handle = ffi.cast("void *", sock.fileno())
        SetHandleInformation(
            sock_handle,
            library.HANDLE_FLAG_INHERIT,
            inherit
        )
        if check:
            result = GetHandleInformation(sock_handle)
            self.assertEqual(inherit, result)

    def test_set_handle_info_socket_noinherit(self):
        self._set_handle_info_socket(0)

    def test_set_handle_info_socket_inherit(self):
        self._set_handle_info_socket(1)

    def test_set_get_handle_info_socket_noinherit(self):
        self._set_handle_info_socket(0, check=True)

    def test_set_get_handle_info_socket_inherit(self):
        self._set_handle_info_socket(1, check=True)

class TestSetHandleInformationForks(TestCase):
    """
    Integration tests for :func:`pywincffi.kernel32.SetHandleInformation`
    """
    def test_file_rename_after_fork(self):
        _, library = dist.load()
        tempdir = tempfile.mkdtemp()
        self.addCleanup(shutil.rmtree, tempdir, ignore_errors=True)
        filename = os.path.join(tempdir, "original_name")
        with open(filename, "w") as test_file:
            test_file.write("data")
            file_handle = handle_from_file(test_file)
            # prevent file_handle inheritance
            SetHandleInformation(file_handle, library.HANDLE_FLAG_INHERIT, 0)
            # spawn child while test_file is open
            p = subprocess.Popen(
                args=[sys.executable],
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE
            )
        newfilename = os.path.join(tempdir, "new_name")
        # works as long as file is closed and not inherited by child
        os.rename(filename, newfilename)
        # stop the child process
        p.stdin.close()
        p.wait()

    def test_socket_rebind_after_fork(self):
        ffi, library = dist.load()
        bind_addr = (('127.0.0.1', 0))
        sock = socket.socket()
        try:
            sock.bind(bind_addr)
            bind_addr = sock.getsockname()
            sock_handle = ffi.cast("void *", sock.fileno())
            SetHandleInformation(sock_handle, library.HANDLE_FLAG_INHERIT, 0)
            # spawn child while sock is bound
            p = subprocess.Popen(
                args=[sys.executable],
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE
            )
        finally:
            sock.close()
        sock = socket.socket()
        self.addCleanup(sock.close)
        sock.bind(bind_addr)
        # stop the child process
        p.stdin.close()
        p.wait()
<commit_msg>Ensure cleanup if test_file_rename_after_fork fails<commit_after>import os
import shutil
import socket
import subprocess
import sys
import tempfile
from errno import EBADF

from pywincffi.core import dist
from pywincffi.dev.testutil import TestCase
from pywincffi.exceptions import InputError, WindowsAPIError
from pywincffi.kernel32 import (
    GetStdHandle, CloseHandle, OpenProcess, WaitForSingleObject,
    handle_from_file, GetHandleInformation, SetHandleInformation)

try:
    WindowsError
except NameError:  # pragma: no cover
    WindowsError = OSError  # pylint: disable=redefined-builtin

class TestGetStdHandle(TestCase):
    """
    Tests for :func:`pywincffi.kernel32.GetStdHandle`
    """
    def test_stdin_handle(self):
        _, library = dist.load()
        self.assertEqual(
            GetStdHandle(library.STD_INPUT_HANDLE),
            library.GetStdHandle(library.STD_INPUT_HANDLE)
        )

    def test_stdout_handle(self):
        _, library = dist.load()
        self.assertEqual(
            GetStdHandle(library.STD_OUTPUT_HANDLE),
            library.GetStdHandle(library.STD_OUTPUT_HANDLE)
        )

    def test_stderr_handle(self):
        _, library = dist.load()
        self.assertEqual(
            GetStdHandle(library.STD_ERROR_HANDLE),
            library.GetStdHandle(library.STD_ERROR_HANDLE)
        )

class TestGetHandleFromFile(TestCase):
    """
    Tests for :func:`pywincffi.kernel32.handle_from_file`
    """
    def test_fails_if_not_a_file(self):
        with self.assertRaises(InputError):
            handle_from_file(0)

    def test_fails_if_file_is_not_open(self):
        fd, _ = tempfile.mkstemp()
        test_file = os.fdopen(fd, "r")
        test_file.close()

        with self.assertRaises(InputError):
            handle_from_file(test_file)

    def test_opens_correct_file_handle(self):
        fd, path = tempfile.mkstemp()
        os.close(fd)

        test_file = open(path, "w")
        handle = handle_from_file(test_file)

        CloseHandle(handle)

        # If CloseHandle() was passed the same handle
        # that test_file is trying to write to the file
        # and/or flushing it should fail.
        try:
            test_file.write("foo")
            test_file.flush()
        except (OSError, IOError, WindowsError) as error:
            # EBADF == Bad file descriptor (because CloseHandle closed it)
            self.assertEqual(error.errno, EBADF)
        else:
            self.fail("Expected os.close(%r) to fail" % fd)

class TestWaitForSingleObject(TestCase):
    """
    Tests for :func:`pywincffi.kernel32.WaitForSingleObject`
    """
    def test_wait_failed(self):
        # This should cause WAIT_FAILED to be returned by the underlying
        # WaitForSingleObject because we didn't request the SYNCHRONIZE
        # permission.
        process = self.create_python_process("import time; time.sleep(3)")
        _, library = dist.load()

        hProcess = OpenProcess(
            library.PROCESS_QUERY_INFORMATION, False, process.pid)
        self.addCleanup(CloseHandle, hProcess)

        with self.assertRaises(WindowsAPIError) as exec_:
            WaitForSingleObject(hProcess, 3)
            self.assertEqual(exec_.code, library.WAIT_FAILED)

    def test_wait_on_running_process(self):
        process = self.create_python_process("import time; time.sleep(1)")
        _, library = dist.load()

        hProcess = OpenProcess(
            library.PROCESS_QUERY_INFORMATION | library.SYNCHRONIZE,
            False, process.pid)
        self.addCleanup(CloseHandle, hProcess)
        self.assertEqual(
            WaitForSingleObject(hProcess, 0), library.WAIT_TIMEOUT)

    def test_process_dies_before_timeout(self):
        process = self.create_python_process("import time; time.sleep(1)")
        _, library = dist.load()

        hProcess = OpenProcess(
            library.PROCESS_QUERY_INFORMATION | library.SYNCHRONIZE,
            False, process.pid)
        self.addCleanup(CloseHandle, hProcess)
        self.assertEqual(
            WaitForSingleObject(hProcess, library.INFINITE),
            library.WAIT_OBJECT_0)

class TestGetHandleInformation(TestCase):
    """
    Tests for :func:`pywincffi.kernel32.GetHandleInformation`
    """
    def test_get_handle_info_stdin(self):
        _, library = dist.load()
        stdin_handle = GetStdHandle(library.STD_INPUT_HANDLE)
        handle_flags = GetHandleInformation(stdin_handle)
        inherit = handle_flags & library.HANDLE_FLAG_INHERIT
        expected = (0, library.HANDLE_FLAG_INHERIT)
        self.assertIn(inherit, expected)

    def _expected_inheritance(self):
        # Python >= 3.4 creates non-inheritable handles (PEP 0446)
        _, library = dist.load()
        inherit = library.HANDLE_FLAG_INHERIT
        return inherit if sys.version_info[:2] < (3, 4) else 0

    def test_get_handle_info_file(self):
        _, library = dist.load()
        # can't use mkstemp: not inheritable on Python < 3.4
        tempdir = tempfile.mkdtemp()
        self.addCleanup(os.rmdir, tempdir)
        filename = os.path.join(tempdir, "test_file")
        with open(filename, "w") as test_file:
            self.addCleanup(os.unlink, filename)
            test_file.write("data")
            file_handle = handle_from_file(test_file)
            handle_flags = GetHandleInformation(file_handle)
            inherit = handle_flags & library.HANDLE_FLAG_INHERIT
        expected = self._expected_inheritance()
        self.assertEqual(inherit, expected)

    def test_get_handle_info_socket(self):
        ffi, library = dist.load()
        sock = socket.socket()
        self.addCleanup(sock.close)
        sock_handle = ffi.cast("void *", sock.fileno())
        handle_flags = GetHandleInformation(sock_handle)
        inherit = handle_flags & library.HANDLE_FLAG_INHERIT
        expected = self._expected_inheritance()
        self.assertEqual(inherit, expected)

class TestSetHandleInformation(TestCase):
    """
    Tests for :func:`pywincffi.kernel32.SetHandleInformation`
    """
    def _set_handle_info_file(self, inherit, check=False):
        _, library = dist.load()
        tempdir = tempfile.mkdtemp()
        self.addCleanup(os.rmdir, tempdir)
        filename = os.path.join(tempdir, "test_file")
        with open(filename, "w") as test_file:
            self.addCleanup(os.unlink, filename)
            test_file.write("data")
            file_handle = handle_from_file(test_file)
            SetHandleInformation(
                file_handle,
                library.HANDLE_FLAG_INHERIT,
                inherit
            )
            if check:
                result = GetHandleInformation(file_handle)
                self.assertEqual(inherit, result)

    def test_set_handle_info_file_noinherit(self):
        self._set_handle_info_file(0)

    def test_set_handle_info_file_inherit(self):
        self._set_handle_info_file(1)

    def test_set_get_handle_info_file_noinherit(self):
        self._set_handle_info_file(0, check=True)

    def test_set_get_handle_info_file_inherit(self):
        self._set_handle_info_file(1, check=True)

    def _set_handle_info_socket(self, inherit, check=False):
        ffi, library = dist.load()
        sock = socket.socket()
        self.addCleanup(sock.close)
        sock_handle = ffi.cast("void *", sock.fileno())
        SetHandleInformation(
            sock_handle,
            library.HANDLE_FLAG_INHERIT,
            inherit
        )
        if check:
            result = GetHandleInformation(sock_handle)
            self.assertEqual(inherit, result)

    def test_set_handle_info_socket_noinherit(self):
        self._set_handle_info_socket(0)

    def test_set_handle_info_socket_inherit(self):
        self._set_handle_info_socket(1)

    def test_set_get_handle_info_socket_noinherit(self):
        self._set_handle_info_socket(0, check=True)

    def test_set_get_handle_info_socket_inherit(self):
        self._set_handle_info_socket(1, check=True)

class TestSetHandleInformationForks(TestCase):
    """
    Integration tests for :func:`pywincffi.kernel32.SetHandleInformation`
    """
    def test_file_rename_after_fork(self):
        _, library = dist.load()
        tempdir = tempfile.mkdtemp()
        self.addCleanup(shutil.rmtree, tempdir, ignore_errors=True)
        filename = os.path.join(tempdir, "original_name")
        with open(filename, "w") as test_file:
            test_file.write("data")
            file_handle = handle_from_file(test_file)
            # prevent file_handle inheritance
            SetHandleInformation(file_handle, library.HANDLE_FLAG_INHERIT, 0)
            # spawn child while test_file is open
            p = subprocess.Popen(
                args=[sys.executable],
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE
            )
        try:
            newfilename = os.path.join(tempdir, "new_name")
            # works as long as file is closed and not inherited by child
            os.rename(filename, newfilename)
        finally:
            # stop the child process
            p.stdin.close()
            p.wait()

    def test_socket_rebind_after_fork(self):
        ffi, library = dist.load()
        bind_addr = (('127.0.0.1', 0))
        sock = socket.socket()
        try:
            sock.bind(bind_addr)
            bind_addr = sock.getsockname()
            sock_handle = ffi.cast("void *", sock.fileno())
            SetHandleInformation(sock_handle, library.HANDLE_FLAG_INHERIT, 0)
            # spawn child while sock is bound
            p = subprocess.Popen(
                args=[sys.executable],
                stdin=subprocess.PIPE,
                stdout=subprocess.PIPE,
                stderr=subprocess.PIPE
            )
        finally:
            sock.close()
        sock = socket.socket()
        self.addCleanup(sock.close)
        sock.bind(bind_addr)
        # stop the child process
        p.stdin.close()
        p.wait()
<|endoftext|>


<commit_before>"""
Filename:     calc_heat_budget_change.py
Author:       Damien Irving, irving.damien@gmail.com
Description:  Calculate the heat budget change between two time periods

"""

# Import general Python modules

import sys, os, pdb
import collections
import argparse
import numpy
import pandas
import iris
import iris.coord_categorisation

# Import my modules

cwd = os.getcwd()
repo_dir = '/'
for directory in cwd.split('/')[1:]:
    repo_dir = os.path.join(repo_dir, directory)
    if directory == 'ocean-analysis':
        break

modules_dir = os.path.join(repo_dir, 'modules')
sys.path.append(modules_dir)
try:
    import general_io as gio
    import convenient_universal as uconv
    import spatial_weights
    import timeseries
except ImportError:
    raise ImportError('Must run this script from anywhere within the ocean-analysis git repo')

# Define functions

history = []

def save_history(cube, field, filename):
    """Save the history attribute when reading the data.
    (This is required because the history attribute differs between input files
      and is therefore deleted upon equilising attributes)
    """

    history.append(cube.attributes['history'])

def get_control_time_constraint(control_cube, hist_cube, time_bounds):
    """Define the time constraints for the control data."""

    iris.coord_categorisation.add_year(control_cube, 'time')
    iris.coord_categorisation.add_year(hist_cube, 'time')

    branch_time = hist_cube.attributes['branch_time']

    index = 0
    for bounds in control_cube.coord('time').bounds:
        lower, upper = bounds
        if lower <= branch_time < upper:
            break
        else:
            index = index + 1

    branch_year = control_cube.coord('year').points[index]
    hist_start_year = hist_cube.coord('year').points[0]
    start_gap = int(time_bounds[0].split('-')[0]) - hist_start_year
    end_gap = int(time_bounds[1].split('-')[0]) - hist_start_year

    control_start_year = branch_year + start_gap
    control_end_year = branch_year + end_gap

    control_start_date = str(control_start_year).zfill(4)+'-01-01'
    control_end_date = str(control_end_year).zfill(4)+'-01-01'

    time_constraint = get_time_constraint([control_start_date, control_end_date])

    return time_constraint

def get_attributes(cube):
    """Get the model, experiment and rip information."""

    model = cube.attributes['model_id']
    experiment = cube.attributes['experiment_id']

    realization = cube.attributes['realization']
    initialization = cube.attributes['initialization_method']
    physics = cube.attributes['physics_version']

    rip = 'r%si%sp%s' %(realization, initialization, physics)

    return model, experiment, rip

def generate_results(data_dict, cube_list, time_constraint, time_bounds):
    """Generate results."""

    model, experiment, rip = get_attributes(cube_list[0])
    period = '%s-%s' %(time_bounds[0].split('-')[0], time_bounds[0].split('-')[1])

    data_dict['model'].append(model)
    data_dict['experiment'].append(experiment)
    data_dict['rip'].append(rip)
    data_dict['period'].append(period)

    for cube in cube_list:
        temporal_subset = cube.extract(time_constraint)
        clim = temporal_subset.collapsed('time', iris.analysis.MEAN)
        data_dict[clim.var_name] = clim.data

    return data_dict

def main(inargs):
    """Run the program."""

    hist_cube_list = iris.load(inargs.historical_file)
    control_cube_list = iris.load(inargs.control_file)

    hist_start_constraint = gio.get_time_constraint(inargs.start_time)
    hist_end_constraint = gio.get_time_constraint(inargs.end_time)
    start_year = inargs.start_time[0].split('-')[0]
    control_start_constraint = get_control_time_constraint(control_cube_list[0], hist_cube_list[0], inargs.start_time)
    control_end_constraint = get_control_time_constraint(control_cube_list[0], hist_cube_list[0], inargs.end_time)

    column_headers = ['model', 'experiment', 'rip', 'period',
                      'hfds-globe-sum', 'hfds-nh-sum', 'hfds-sh-sum', 'hfds-nhext-sum', 'hfds-tropics-sum', 'hfds-shext-sum',
                      'ohc-globe-sum', 'ohc-nh-sum', 'ohc-sh-sum', 'ohc-nhext-sum', 'ohc-tropics-sum', 'ohc-shext-sum']

    data_dict = collections.OrderedDict()
    for column in column_headers:
        data_dict[column] = []
    data_dict = generate_results(data_dict, hist_cube_list, hist_start_constaint, inargs.start_time)
    data_dict = generate_results(data_dict, hist_cube_list, hist_end_constaint, inargs.end_time)
    data_dict = generate_results(data_dict, control_cube_list, control_start_constaint, inargs.start_time)
    data_dict = generate_results(data_dict, control_cube_list, control_end_constaint, inargs.end_time)

    data_df = pandas.DataFrame.from_dict(data_dict)
    data_df.to_csv(inargs.outfile)

    metadata_dict = {inargs.historical_file: hist_cube_list[0].attributes['history'],
                     inargs.control_file: control_cube_list[0].attributes['history']}
    gio.write_metadata(inargs.outfile, file_info=metadata_dict)

if __name__ == '__main__':

    extra_info ="""

author:
    Damien Irving, irving.damien@gmail.com

"""

    description = 'Calculate the heat budget change between two time periods'
    parser = argparse.ArgumentParser(description=description,
                                     epilog=extra_info,
                                     argument_default=argparse.SUPPRESS,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument("historical_file", type=str,
                        help="historical experiment heat budget file from calc_heat_budget_timeseries.py")
    parser.add_argument("control_file", type=str,
                        help="control experiment heat budget file from calc_heat_budget_timeseries.py")
    parser.add_argument("outfile", type=str, help="Output .csv file")

    parser.add_argument("--start_time", type=str, nargs=2, metavar=('START_DATE', 'END_DATE'),
                        default=('1861-01-01', '1880-12-31'), help="Start time period")
    parser.add_argument("--end_time", type=str, nargs=2, metavar=('START_DATE', 'END_DATE'),
                        default=('1986-01-01', '2005-12-31'), help="End time period")

    args = parser.parse_args()
    main(args)
<commit_msg>Fixed typos<commit_after>"""
Filename:     calc_heat_budget_change.py
Author:       Damien Irving, irving.damien@gmail.com
Description:  Calculate the heat budget change between two time periods

"""

# Import general Python modules

import sys, os, pdb
import collections
import argparse
import numpy
import pandas
import iris
import iris.coord_categorisation

# Import my modules

cwd = os.getcwd()
repo_dir = '/'
for directory in cwd.split('/')[1:]:
    repo_dir = os.path.join(repo_dir, directory)
    if directory == 'ocean-analysis':
        break

modules_dir = os.path.join(repo_dir, 'modules')
sys.path.append(modules_dir)
try:
    import general_io as gio
    import convenient_universal as uconv
    import spatial_weights
    import timeseries
except ImportError:
    raise ImportError('Must run this script from anywhere within the ocean-analysis git repo')

# Define functions

history = []

def save_history(cube, field, filename):
    """Save the history attribute when reading the data.
    (This is required because the history attribute differs between input files
      and is therefore deleted upon equilising attributes)
    """

    history.append(cube.attributes['history'])

def get_control_time_constraint(control_cube, hist_cube, time_bounds):
    """Define the time constraints for the control data."""

    iris.coord_categorisation.add_year(control_cube, 'time')
    iris.coord_categorisation.add_year(hist_cube, 'time')

    branch_time = hist_cube.attributes['branch_time']

    index = 0
    for bounds in control_cube.coord('time').bounds:
        lower, upper = bounds
        if lower <= branch_time < upper:
            break
        else:
            index = index + 1

    branch_year = control_cube.coord('year').points[index]
    hist_start_year = hist_cube.coord('year').points[0]
    start_gap = int(time_bounds[0].split('-')[0]) - hist_start_year
    end_gap = int(time_bounds[1].split('-')[0]) - hist_start_year

    control_start_year = branch_year + start_gap
    control_end_year = branch_year + end_gap

    control_start_date = str(control_start_year).zfill(4)+'-01-01'
    control_end_date = str(control_end_year).zfill(4)+'-01-01'

    time_constraint = get_time_constraint([control_start_date, control_end_date])

    return time_constraint

def get_attributes(cube):
    """Get the model, experiment and rip information."""

    model = cube.attributes['model_id']
    experiment = cube.attributes['experiment_id']

    realization = cube.attributes['realization']
    initialization = cube.attributes['initialization_method']
    physics = cube.attributes['physics_version']

    rip = 'r%si%sp%s' %(realization, initialization, physics)

    return model, experiment, rip

def generate_results(data_dict, cube_list, time_constraint, time_bounds):
    """Generate results."""

    model, experiment, rip = get_attributes(cube_list[0])
    period = '%s-%s' %(time_bounds[0].split('-')[0], time_bounds[1].split('-')[0])

    data_dict['model'].append(model)
    data_dict['experiment'].append(experiment)
    data_dict['rip'].append(rip)
    data_dict['period'].append(period)

    for cube in cube_list:
        temporal_subset = cube.extract(time_constraint)
        clim = temporal_subset.collapsed('time', iris.analysis.MEAN)
        data_dict[clim.var_name] = float(clim.data)

    return data_dict

def main(inargs):
    """Run the program."""

    hist_cube_list = iris.load(inargs.historical_file)
    control_cube_list = iris.load(inargs.control_file)

    hist_start_constraint = gio.get_time_constraint(inargs.start_time)
    hist_end_constraint = gio.get_time_constraint(inargs.end_time)
    start_year = inargs.start_time[0].split('-')[0]
    control_start_constraint = get_control_time_constraint(control_cube_list[0], hist_cube_list[0], inargs.start_time)
    control_end_constraint = get_control_time_constraint(control_cube_list[0], hist_cube_list[0], inargs.end_time)

    column_headers = ['model', 'experiment', 'rip', 'period',
                      'hfds-globe-sum', 'hfds-nh-sum', 'hfds-sh-sum', 'hfds-nhext-sum', 'hfds-tropics-sum', 'hfds-shext-sum',
                      'ohc-globe-sum', 'ohc-nh-sum', 'ohc-sh-sum', 'ohc-nhext-sum', 'ohc-tropics-sum', 'ohc-shext-sum']

    data_dict = collections.OrderedDict()
    for column in column_headers:
        data_dict[column] = []
    data_dict = generate_results(data_dict, hist_cube_list, hist_start_constraint, inargs.start_time)
    data_dict = generate_results(data_dict, hist_cube_list, hist_end_constraint, inargs.end_time)

    data_dict = generate_results(data_dict, control_cube_list, control_start_constraint, inargs.start_time)
    data_dict = generate_results(data_dict, control_cube_list, control_end_constraint, inargs.end_time)

    data_df = pandas.DataFrame.from_dict(data_dict)
    data_df.to_csv(inargs.outfile)

    metadata_dict = {inargs.historical_file: hist_cube_list[0].attributes['history'],
                     inargs.control_file: control_cube_list[0].attributes['history']}
    gio.write_metadata(inargs.outfile, file_info=metadata_dict)

if __name__ == '__main__':

    extra_info ="""

author:
    Damien Irving, irving.damien@gmail.com

"""

    description = 'Calculate the heat budget change between two time periods'
    parser = argparse.ArgumentParser(description=description,
                                     epilog=extra_info,
                                     argument_default=argparse.SUPPRESS,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument("historical_file", type=str,
                        help="historical experiment heat budget file from calc_heat_budget_timeseries.py")
    parser.add_argument("control_file", type=str,
                        help="control experiment heat budget file from calc_heat_budget_timeseries.py")
    parser.add_argument("outfile", type=str, help="Output .csv file")

    parser.add_argument("--start_time", type=str, nargs=2, metavar=('START_DATE', 'END_DATE'),
                        default=('1861-01-01', '1880-12-31'), help="Start time period")
    parser.add_argument("--end_time", type=str, nargs=2, metavar=('START_DATE', 'END_DATE'),
                        default=('1986-01-01', '2005-12-31'), help="End time period")

    args = parser.parse_args()
    main(args)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# Copyright (c) 2012 Cloudera, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Verfier for common impalad metrics

import pytest

# List of metrics that should be equal to zero when there are no outstanding queries.
METRIC_LIST = ["impala-server.backends.client-cache.clients-in-use",
               "impala-server.hash-table.total-bytes",
               "impala-server.io-mgr.num-open-files",
               "impala-server.mem-pool.total-bytes",
               "impala-server.num-files-open-for-insert",
               # Disable checking of num-missing-volume-id due to IMPALA-467
               # "impala-server.scan-ranges.num-missing-volume-id",
               ]

class MetricVerifier(object):
  """Reuseable class that can verify common metrics"""
  def __init__(self, impalad_service):
    """Initialize module given an ImpalaService object"""
    self.impalad_service = impalad_service

  def verify_metrics_are_zero(self, timeout=60):
    """Test that all the metric in METRIC_LIST are 0"""
    for metric in METRIC_LIST:
      self.wait_for_metric(metric, 0, timeout)

  def verify_no_open_files(self, timeout=60):
    """Tests there are no open files"""
    self.wait_for_metric("impala-server.num-files-open-for-insert", 0, timeout)
    self.wait_for_metric("impala-server.io-mgr.num-open-files", 0, timeout)

  def verify_num_unused_buffers(self, timeout=60):
    """Test that all buffers are unused"""
    buffers =\
        self.impalad_service.get_metric_value("impala-server.io-mgr.num-buffers")
    self.wait_for_metric("impala-server.io-mgr.num-unused-buffers", buffers,
        timeout)

  def wait_for_metric(self, metric_name, expected_value, timeout=60):
    self.impalad_service.wait_for_metric_value(metric_name, expected_value, timeout)
<commit_msg>Disable metric verification for mem-pool.total-bytes.<commit_after>#!/usr/bin/env python
# Copyright (c) 2012 Cloudera, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Verfier for common impalad metrics

import pytest

# List of metrics that should be equal to zero when there are no outstanding queries.
METRIC_LIST = ["impala-server.backends.client-cache.clients-in-use",
               "impala-server.hash-table.total-bytes",
               "impala-server.io-mgr.num-open-files",
               # Disable checking of mem-pool.total-bytes DUE TO IMPALA-1057
               #"impala-server.mem-pool.total-bytes",
               "impala-server.num-files-open-for-insert",
               # Disable checking of num-missing-volume-id due to IMPALA-467
               # "impala-server.scan-ranges.num-missing-volume-id",
               ]

class MetricVerifier(object):
  """Reuseable class that can verify common metrics"""
  def __init__(self, impalad_service):
    """Initialize module given an ImpalaService object"""
    self.impalad_service = impalad_service

  def verify_metrics_are_zero(self, timeout=60):
    """Test that all the metric in METRIC_LIST are 0"""
    for metric in METRIC_LIST:
      self.wait_for_metric(metric, 0, timeout)

  def verify_no_open_files(self, timeout=60):
    """Tests there are no open files"""
    self.wait_for_metric("impala-server.num-files-open-for-insert", 0, timeout)
    self.wait_for_metric("impala-server.io-mgr.num-open-files", 0, timeout)

  def verify_num_unused_buffers(self, timeout=60):
    """Test that all buffers are unused"""
    buffers =\
        self.impalad_service.get_metric_value("impala-server.io-mgr.num-buffers")
    self.wait_for_metric("impala-server.io-mgr.num-unused-buffers", buffers,
        timeout)

  def wait_for_metric(self, metric_name, expected_value, timeout=60):
    self.impalad_service.wait_for_metric_value(metric_name, expected_value, timeout)
<|endoftext|>


<commit_before>import contextlib
import logging

from cStringIO import StringIO

from teuthology import misc
from teuthology.job_status import set_status
from teuthology.orchestra import run

log = logging.getLogger(__name__)

@contextlib.contextmanager
def syslog(ctx, config):
    """
    start syslog / stop syslog on exit.
    """
    if ctx.archive is None:
        # disable this whole feature if we're not going to archive the data
        # anyway
        yield
        return

    log.info('Starting syslog monitoring...')

    archive_dir = misc.get_archive_dir(ctx)
    log_dir = '{adir}/syslog'.format(adir=archive_dir)
    run.wait(
        ctx.cluster.run(
            args=['mkdir', '-p', '-m0755', '--', log_dir],
            wait=False,
        )
    )

    CONF = '/etc/rsyslog.d/80-cephtest.conf'
    kern_log = '{log_dir}/kern.log'.format(log_dir=log_dir)
    misc_log = '{log_dir}/misc.log'.format(log_dir=log_dir)
    conf_lines = [
        'kern.* -{kern_log};RSYSLOG_FileFormat'.format(kern_log=kern_log),
        '*.*;kern.none -{misc_log};RSYSLOG_FileFormat'.format(
            misc_log=misc_log),
    ]
    conf_fp = StringIO('\n'.join(conf_lines))
    try:
        for rem in ctx.cluster.remotes.keys():
            log_context = 'system_u:object_r:var_log_t:s0'
            for log_path in (kern_log, misc_log):
                rem.run(args=['install', '-m', '666', '/dev/null', log_path])
                rem.chcon(log_path, log_context)
            misc.sudo_write_file(
                remote=rem,
                path=CONF,
                data=conf_fp,
            )
            conf_fp.seek(0)
        run.wait(
            ctx.cluster.run(
                args=[
                    'sudo',
                    'service',
                    # a mere reload (SIGHUP) doesn't seem to make
                    # rsyslog open the files
                    'rsyslog',
                    'restart',
                ],
                wait=False,
            ),
        )

        yield
    finally:
        log.info('Shutting down syslog monitoring...')

        run.wait(
            ctx.cluster.run(
                args=[
                    'sudo',
                    'rm',
                    '-f',
                    '--',
                    CONF,
                    run.Raw('&&'),
                    'sudo',
                    'service',
                    'rsyslog',
                    'restart',
                ],
                wait=False,
            ),
        )
        # race condition: nothing actually says rsyslog had time to
        # flush the file fully. oh well.

        log.info('Checking logs for errors...')
        for rem in ctx.cluster.remotes.keys():
            log.debug('Checking %s', rem.name)
            r = rem.run(
                args=[
                    'egrep', '--binary-files=text',
                    '\\bBUG\\b|\\bINFO\\b|\\bDEADLOCK\\b',
                    run.Raw('{adir}/syslog/*.log'.format(adir=archive_dir)),
                    run.Raw('|'),
                    'grep', '-v', 'task .* blocked for more than .* seconds',
                    run.Raw('|'),
                    'grep', '-v', 'lockdep is turned off',
                    run.Raw('|'),
                    'grep', '-v', 'trying to register non-static key',
                    run.Raw('|'),
                    'grep', '-v', 'DEBUG: fsize',  # xfs_fsr
                    run.Raw('|'),
                    'grep', '-v', 'CRON',  # ignore cron noise
                    run.Raw('|'),
                    'grep', '-v', 'BUG: bad unlock balance detected',  # #6097
                    run.Raw('|'),
                    'grep', '-v', 'inconsistent lock state',  # FIXME see #2523
                    run.Raw('|'),
                    'grep', '-v', '*** DEADLOCK ***',  # part of lockdep output
                    run.Raw('|'),
                    'grep', '-v',
                    # FIXME see #2590 and #147
                    'INFO: possible irq lock inversion dependency detected',
                    run.Raw('|'),
                    'grep', '-v',
                    'INFO: NMI handler (perf_event_nmi_handler) took too long to run',  # noqa
                    run.Raw('|'),
                    'grep', '-v', 'INFO: recovery required on readonly',
                    run.Raw('|'),
                    'grep', '-v', 'ceph-create-keys: INFO',
                    run.Raw('|'),
                    'grep', '-v', 'INFO:ceph-create-keys',
                    run.Raw('|'),
                    'grep', '-v', 'Loaded datasource DataSourceOpenStack',
                    run.Raw('|'),
                    'grep', '-v', 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined',  # noqa
                    run.Raw('|'),
                    'egrep', '-v', '\\bsalt-master\\b|\\bsalt-minion\\b|\\bsalt-api\\b',
                    run.Raw('|'),
                    'grep', '-v', 'ceph-crash',
                    run.Raw('|'),
                    'egrep', '-v', '\\btcmu-runner\\b.*\\bINFO\\b',
                    run.Raw('|'),
                    'head', '-n', '1',
                ],
                stdout=StringIO(),
            )
            stdout = r.stdout.getvalue()
            if stdout != '':
                log.error('Error in syslog on %s: %s', rem.name, stdout)
                set_status(ctx.summary, 'fail')
                if 'failure_reason' not in ctx.summary:
                    ctx.summary['failure_reason'] = \
                        "'{error}' in syslog".format(error=stdout)

        log.info('Compressing syslogs...')
        run.wait(
            ctx.cluster.run(
                args=[
                    'find',
                    '{adir}/syslog'.format(adir=archive_dir),
                    '-name',
                    '*.log',
                    '-print0',
                    run.Raw('|'),
                    'sudo',
                    'xargs',
                    '-0',
                    '--no-run-if-empty',
                    '--',
                    'gzip',
                    '--',
                ],
                wait=False,
            ),
        )
<commit_msg>task/internal/syslog: use remote.sh for py3 compat<commit_after>import contextlib
import logging

from io import BytesIO

from teuthology import misc
from teuthology.job_status import set_status
from teuthology.orchestra import run

log = logging.getLogger(__name__)

@contextlib.contextmanager
def syslog(ctx, config):
    """
    start syslog / stop syslog on exit.
    """
    if ctx.archive is None:
        # disable this whole feature if we're not going to archive the data
        # anyway
        yield
        return

    log.info('Starting syslog monitoring...')

    archive_dir = misc.get_archive_dir(ctx)
    log_dir = '{adir}/syslog'.format(adir=archive_dir)
    run.wait(
        ctx.cluster.run(
            args=['mkdir', '-p', '-m0755', '--', log_dir],
            wait=False,
        )
    )

    CONF = '/etc/rsyslog.d/80-cephtest.conf'
    kern_log = '{log_dir}/kern.log'.format(log_dir=log_dir)
    misc_log = '{log_dir}/misc.log'.format(log_dir=log_dir)
    conf_lines = [
        'kern.* -{kern_log};RSYSLOG_FileFormat'.format(kern_log=kern_log),
        '*.*;kern.none -{misc_log};RSYSLOG_FileFormat'.format(
            misc_log=misc_log),
    ]
    conf_fp = BytesIO('\n'.join(conf_lines).encode())
    try:
        for rem in ctx.cluster.remotes.keys():
            log_context = 'system_u:object_r:var_log_t:s0'
            for log_path in (kern_log, misc_log):
                rem.run(args=['install', '-m', '666', '/dev/null', log_path])
                rem.chcon(log_path, log_context)
            misc.sudo_write_file(
                remote=rem,
                path=CONF,
                data=conf_fp,
            )
            conf_fp.seek(0)
        run.wait(
            ctx.cluster.run(
                args=[
                    'sudo',
                    'service',
                    # a mere reload (SIGHUP) doesn't seem to make
                    # rsyslog open the files
                    'rsyslog',
                    'restart',
                ],
                wait=False,
            ),
        )

        yield
    finally:
        log.info('Shutting down syslog monitoring...')

        run.wait(
            ctx.cluster.run(
                args=[
                    'sudo',
                    'rm',
                    '-f',
                    '--',
                    CONF,
                    run.Raw('&&'),
                    'sudo',
                    'service',
                    'rsyslog',
                    'restart',
                ],
                wait=False,
            ),
        )
        # race condition: nothing actually says rsyslog had time to
        # flush the file fully. oh well.

        log.info('Checking logs for errors...')
        for rem in ctx.cluster.remotes.keys():
            log.debug('Checking %s', rem.name)
            stdout = rem.sh(
                [
                    'egrep', '--binary-files=text',
                    '\\bBUG\\b|\\bINFO\\b|\\bDEADLOCK\\b',
                    run.Raw('{adir}/syslog/*.log'.format(adir=archive_dir)),
                    run.Raw('|'),
                    'grep', '-v', 'task .* blocked for more than .* seconds',
                    run.Raw('|'),
                    'grep', '-v', 'lockdep is turned off',
                    run.Raw('|'),
                    'grep', '-v', 'trying to register non-static key',
                    run.Raw('|'),
                    'grep', '-v', 'DEBUG: fsize',  # xfs_fsr
                    run.Raw('|'),
                    'grep', '-v', 'CRON',  # ignore cron noise
                    run.Raw('|'),
                    'grep', '-v', 'BUG: bad unlock balance detected',  # #6097
                    run.Raw('|'),
                    'grep', '-v', 'inconsistent lock state',  # FIXME see #2523
                    run.Raw('|'),
                    'grep', '-v', '*** DEADLOCK ***',  # part of lockdep output
                    run.Raw('|'),
                    'grep', '-v',
                    # FIXME see #2590 and #147
                    'INFO: possible irq lock inversion dependency detected',
                    run.Raw('|'),
                    'grep', '-v',
                    'INFO: NMI handler (perf_event_nmi_handler) took too long to run',  # noqa
                    run.Raw('|'),
                    'grep', '-v', 'INFO: recovery required on readonly',
                    run.Raw('|'),
                    'grep', '-v', 'ceph-create-keys: INFO',
                    run.Raw('|'),
                    'grep', '-v', 'INFO:ceph-create-keys',
                    run.Raw('|'),
                    'grep', '-v', 'Loaded datasource DataSourceOpenStack',
                    run.Raw('|'),
                    'grep', '-v', 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined',  # noqa
                    run.Raw('|'),
                    'egrep', '-v', '\\bsalt-master\\b|\\bsalt-minion\\b|\\bsalt-api\\b',
                    run.Raw('|'),
                    'grep', '-v', 'ceph-crash',
                    run.Raw('|'),
                    'egrep', '-v', '\\btcmu-runner\\b.*\\bINFO\\b',
                    run.Raw('|'),
                    'head', '-n', '1',
                ],
            )
            if stdout != '':
                log.error('Error in syslog on %s: %s', rem.name, stdout)
                set_status(ctx.summary, 'fail')
                if 'failure_reason' not in ctx.summary:
                    ctx.summary['failure_reason'] = \
                        "'{error}' in syslog".format(error=stdout)

        log.info('Compressing syslogs...')
        run.wait(
            ctx.cluster.run(
                args=[
                    'find',
                    '{adir}/syslog'.format(adir=archive_dir),
                    '-name',
                    '*.log',
                    '-print0',
                    run.Raw('|'),
                    'sudo',
                    'xargs',
                    '-0',
                    '--no-run-if-empty',
                    '--',
                    'gzip',
                    '--',
                ],
                wait=False,
            ),
        )
<|endoftext|>


<commit_before># Generated by Django 2.0.13 on 2021-04-18 23:10

import ddcz.models.magic
from django.db import migrations

class Migration(migrations.Migration):

    dependencies = [
        ('ddcz', '0061_market'),
    ]

    operations = [
        migrations.AlterField(
            model_name='market',
            name='sekce',
            field=ddcz.models.magic.MisencodedCharField(choices=[('nabizim', 'Nabízím'), ('shanim', 'Sháním'), ('vymenim', 'Vyměním'), ('daruji', 'Daruji')], max_length=20),
        ),
    ]
<commit_msg>Fix: Wild migration<commit_after><|endoftext|>


<commit_before>import os
#import timeit
#TODO add a shit ton more threads
#just counts files were indexing
def counting():
	root=os.path.expanduser('~')#change this before development
	#print "start"
	i=1
	#start = timeit.default_timer()
	for dir_name, sub_dirs, files in os.walk(root):
		#print i
		i = i+1#change to i+=1 later
		print "I counted to file " + str(i)
	return i #returns the count
	#stop = timeit.default_timer()<commit_msg>Logging/Catching any error<commit_after>import os
import logging

#Log Here
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

#import timeit
#TODO add a shit ton more threads
#just counts files were indexing
def counting():
	root=os.path.expanduser('~')#change this before development
	#print "start"
	i=1
	#start = timeit.default_timer()
	for dir_name, sub_dirs, files in os.walk(root):
		#print i
		try:
			i = i+1#change to i+=1 later
			logger.info("Counted to file " + str(i))
		except BaseException:
			logger.debug("An Error Occured, Continuing")
			continue
	return i #returns the count
	#stop = timeit.default_timer()<|endoftext|>


<commit_before>import sys
import logging
from thespian.actors import *
from thespian.system.utilis import thesplog, AssocList
from thespian.system.systemCommon import systemCommonBase
from thespian.system.messages.status import Thespian_SystemStatus
from thespian.system.messages.admin import *
from thespian.system.messages.logcontrol import SetLogging
from thespian.system.transport import TransmitIntent
from thespian.system.sourceLoader import SourceHashFinder

class AdminCore(systemCommonBase):

    def __init__(self, transport, address, capabilities, logdefs, concurrency_context):
        thesplog('++++ Starting Admin from %s', sys.modules['thespian'].__file__, level=logging.DEBUG)
        super(AdminCore, self).__init__(address, transport)
        self.init_replicator(transport, concurrency_context)
        self.capabilities = capabilities
        self.logdefs      = logdefs
        self._pendingChildren = {}  # key = childLocalAddr instance #, value = PendingActorEnvelope
        # Things that help us look like an Actor, even though we're not
        self._sourceHash  = None
        thesplog('++++ Admin started @ %s / gen %s', self.transport.myAddress, str(ThespianGeneration), level=logging.INFO, primary=True)
        self._nannying = AssocList()  # child actorAddress -> parent Address
        self._deadLetterHandler = None
        self._sources = {}  # Index is sourcehash, value is requestor
                            # ActorAddress or zipsrc (when validated)
        self._sourceAuthority = None

    def _activate(self):
        """Called when the full ActorSystem initialization is completed.  This
           should then activate any functionality that needed to wait
           for completion of initialization.
        """
        pass

    def run(self):
        try:
            while True:
                r = self.transport.run(self.handleIncoming, None)
                if isinstance(r, Thespian__UpdateWork):
                    self._send_intent(
                        TransmitIntent(self.myAddress, r))  # tickle the transmit queues
                    continue
                # Expects that on completion of self.transport.run
                # that the Actor is done processing and that it has
                # been shutdown gracefully.
                break
        except Exception as ex:
            import traceback
            thesplog('ActorAdmin uncaught exception: %s', traceback.format_exc(),
                     level=logging.ERROR, exc_info=True)
        thesplog('Admin time to die', level=logging.DEBUG)

    def handleIncoming(self, envelope):
        self._sCBStats.inc('Admin Message Received.Total')
        handled, result = self._handleReplicatorMessages(envelope)
        if handled:
            return result
        if isinstance(envelope.message, (ActorSystemMessage, logging.LogRecord)):
            thesplog('Admin of %s', envelope.identify(), level=logging.DEBUG)
            return getattr(self,
                           'h_' + envelope.message.__class__.__name__, self.unrecognized)(envelope)
        # else discard random non-admin messages
        self._sCBStats.inc('Admin Message Received.Ignored')
        thesplog('ADMIN DISCARD %s', envelope.identify(), level=logging.WARNING)
        return True

    def unrecognized(self, envelope):
        self._sCBStats.inc('Admin Message Received.Discarded')
        thesplog("Admin got incoming %s from %s; discarded because I don't know how to handle it!",
                 envelope.message, envelope.sender, level=logging.WARNING, primary=True)
        return True

    def isShuttingDown(self): return hasattr(self, '_exiting')

    def h_QueryExists(self, envelope):
        self._sCBStats.inc('Admin Message Received.Type.QueryExists')
        self._send_intent(
            TransmitIntent(envelope.sender,
                           QueryAck(
                               self.capabilities.get('Thespian ActorSystem Name',
                                                     'misc Actor System'),
                               self.capabilities.get('Thespian ActorSystem Version',
                                                     'unknown Version'),
                               self.isShuttingDown())))
        return True

    def getStatus(self):
        resp = Thespian_SystemStatus(self.myAddress,
                                     capabilities = self.capabilities,
                                     inShutdown = self.isShuttingDown())
        resp.setDeadLetterHandler(self._deadLetterHandler)
        self._updateStatusResponse(resp)
        resp.setLoadedSources(list(self._sources.keys()))
        resp.sourceAuthority = self._sourceAuthority
        return resp

    def h_Thespian_StatusReq(self, envelope):
        self._sCBStats.inc('Admin Message Received.Type.StatusReq')
        self._send_intent(TransmitIntent(envelope.sender, self.getStatus()))
        return True

    def thesplogStatus(self):
        "Write status to thesplog"
        from io import StringIO
        sd = StringIO()
        from thespian.system.messages.status import formatStatus
        try:
            sd.write('')
            SSD = lambda v: v
        except TypeError:
            class SSD(object):
                def __init__(self, sd): self.sd = sd
                def write(self, str_arg): self.sd.write(str_arg.decode('utf-8'))
        formatStatus(self.getStatus(), tofd=SSD(sd))
        thesplog('STATUS: %s', sd.getvalue())

    def h_SetLogging(self, envelope):
        return self.setLoggingControls(envelope)

    def h_SystemShutdown(self, envelope):
        self._exiting = envelope.sender
        thesplog('---- shutdown initiated by %s', envelope.sender, level=logging.DEBUG)

        # Send failure notices and clear out any pending children.  If
        # any pending child ready notifications are received after
        # this, they will automatically be sent an ActorExitRequest.
        for each in self._pendingChildren:
            pendingReq = self._pendingChildren[each]
            self._send_intent(
                TransmitIntent(
                    pendingReq.sender,
                    PendingActorResponse(
                        pendingReq.message.forActor,
                        pendingReq.message.instanceNum,
                        pendingReq.message.globalName,
                        errorCode = PendingActorResponse.ERROR_ActorSystem_Shutting_Down)))
        self._pendingChildren = []

        if not self.childAddresses:  # no children?
            self._sayGoodbye()
            self.transport.abort_run(drain=True)
            return True

        # Now shutdown any direct children
        self._killLocalActors()

        # Once children confirm their exits the callback will shutdown the Admin.
        return True

    def _killLocalActors(self):
        for each in self.childAddresses:
            self._send_intent(
                TransmitIntent(each, ActorExitRequest(recursive=True),
                               onError=lambda r,m,a=each:
                               self._handleChildExited(a)))

    def _sayGoodbye(self):
        self._cleanupAdmin()
        self._send_intent(TransmitIntent(self._exiting, SystemShutdownCompleted()))
        thesplog('---- shutdown completed', level=logging.INFO)

    def h_ChildActorExited(self, envelope):
        return self._handleChildExited(envelope.message.childAddress)

    def _handleChildExited(self, childAddress):
        parentAddr = self._nannying.find(childAddress)
        if parentAddr:
            self._nannying.rmv(childAddress)
            # Let original requesting Actor (that *thinks* it's the
            # parent) know about this child exit as well.
            self._send_intent(TransmitIntent(parentAddr,
                                             ChildActorExited(childAddress)))
        return super(AdminCore, self)._handleChildExited(childAddress)

    def h_PendingActor(self, envelope):
        """Admin is creating an Actor.  This covers one of the following cases:
            1. Creating for external requester via ActorSystem.
                  envelope.message.forActor will be None
               The Admin is the "parent" for all externally-created Actors.
            2. Creating for another Actor when direct creation fails.  Usually means
               that the current ActorSyste cannot meet the new Actor's requirements
               and the Admin should find another convention member that can create
               the new Actor.
            3. GlobalName Actor.  The Admin is the "parent" for all GlobalName Actors.
        """
        self._sCBStats.inc('Admin Message Received.Type.Pending Actor Request')

        if self.isShuttingDown():
            self._sendPendingActorResponse(envelope, None,
                                           errorCode = PendingActorResponse.ERROR_ActorSystem_Shutting_Down)
            return True

        sourceHash = envelope.message.sourceHash
        if sourceHash and sourceHash not in self._sources:
            self._sendPendingActorResponse(envelope, None,
                                           errorCode = PendingActorResponse.ERROR_Invalid_SourceHash)
            return True

        # Note, both Admin and remote requester will have a local
        # child address for the child (with a different instanceNumber
        # each).
        childAddr = self._addrManager.createLocalAddress()
        childInstance = childAddr.addressDetails.addressInstanceNum

        try:
            self._startChildActor(childAddr, envelope.message.actorClassName,
                                  parentAddr = self.myAddress, # Admin is surrogate parent
                                  notifyAddr = self.myAddress,
                                  childRequirements = envelope.message.targetActorReq,
                                  sourceHash = sourceHash,
                                  sourceToLoad = (self._sources[sourceHash]
                                                  if sourceHash else None))
        except NoCompatibleSystemForActor:
            self._sendPendingActorResponse(envelope, None,
                                           errorCode = PendingActorResponse.ERROR_No_Compatible_ActorSystem)
            self._retryPendingChildOperations(childInstance, None)
            return True

        self._pendingChildren[childInstance] = envelope
        # transport will contrive to call _pendingActorReady when the
        # child is initialized and connected to this parent.
        return True

    def _sendPendingActorResponse(self, requestEnvelope, actualAddress,
                                  errorCode = None, errorStr = None):
        # actualAddress is None for failure
        if actualAddress is None and errorCode is None:
            raise ValueError('Must specify either actualAddress or errorCode')
        self._send_intent(
            TransmitIntent(requestEnvelope.message.forActor or requestEnvelope.sender,
                           PendingActorResponse(requestEnvelope.message.forActor,
                                                requestEnvelope.message.instanceNum,
                                                requestEnvelope.message.globalName,
                                                errorCode = errorCode,
                                                errorStr = errorStr,
                                                actualAddress = actualAddress)))

    def _pendingActorReady(self, childInstance, actualAddress):
        if childInstance not in self._pendingChildren:
            thesplog('Pending actor is ready at %s for %s but latter is unknown'
                     '; sending child a shutdown',
                     actualAddress, childInstance, level=logging.WARNING)
            self._send_intent(
                TransmitIntent(actualAddress, ActorExitRequest(recursive=True)))
            return

        requestEnvelope = self._pendingChildren[childInstance]
        del self._pendingChildren[childInstance]
        if requestEnvelope.message.globalName or \
           not requestEnvelope.message.forActor:
            # The Admin is the responsible Parent for these children
            self._registerChild(actualAddress)
        else:
            # Anything the Admin was requested to create is a adoptive
            # child and should be killed when the Admin exits.
            self._registerChild(actualAddress)

        if requestEnvelope.message.forActor:
            # Proxy-parenting; remember the real parent
            self._nannying.add(actualAddress, requestEnvelope.message.forActor)
        self._addrManager.associateUseableAddress(self.myAddress, childInstance, actualAddress)
        # n.b. childInstance is for this Admin, but caller's childInstance is in original request
        self._sendPendingActorResponse(requestEnvelope, actualAddress)
        self._retryPendingChildOperations(childInstance, actualAddress)

    def h_HandleDeadLetters(self, envelope):  # handlerAddr, enableHandler
        self._sCBStats.inc('Admin Message Received.Type.Set Dead Letter Handler')
        if envelope.message.enableHandler:
            self._deadLetterHandler = envelope.message.handlerAddr
        else:
            if self._deadLetterHandler == envelope.message.handlerAddr:
                self._deadLetterHandler = None
        return True

    def h_DeadEnvelope(self, envelope):
        self._sCBStats.inc('Admin Message Received.Type.Dead Letter')
        if self._deadLetterHandler:
            self._send_intent(
                TransmitIntent(self._deadLetterHandler,
                               envelope.message))
        return True

    def h_RegisterSourceAuthority(self, envelope):
        self._sourceAuthority = envelope.message.authorityAddress

    def h_ValidateSource(self, envelope):
        sourceHash = envelope.message.sourceHash
        if not envelope.message.sourceData:
            self.unloadActorSource(sourceHash)
            logging.getLogger('Thespian').info('Source hash %s unloaded', sourceHash)
            return
        if sourceHash in self._sources:
            logging.getLogger('Thespian').info('Source hash %s already loaded', sourceHash)
            return
        if self._sourceAuthority:
            self._send_intent(TransmitIntent(self._sourceAuthority, envelope.message))
            return
        # Any attempt to load sources is ignored if there is no active
        # Source Authority.  This is a security measure to protect the
        # un-protected.
        logging.getLogger('Thespian').warning(
            'No source authority to validate source hash %s',
            sourceHash)

    def h_ValidatedSource(self, envelope):
        self._loadValidatedActorSource(envelope.message.sourceHash,
                                       envelope.message.sourceZip)
        thesplog('Source hash %s validated by Source Authority; now available.',
                 envelope.message.sourceHash)

    def _loadValidatedActorSource(self, sourceHash, sourceZip):
        # Validate the source file; this doesn't actually utilize the
        # sourceZip, but it ensures that the sourceZip isn't garbage
        # before registering it as active source.
        try:
            f = SourceHashFinder(sourceHash, lambda v: v, sourceZip)
            namelist = f.getZipNames()
            logging.getLogger('Thespian').info(
                'Validated source hash %s, %s modules (%s)',
                sourceHash, len(namelist),
                ', '.join(namelist if len(namelist) < 10 else
                          namelist[:9] + ['...']))
        except Exception as ex:
            logging.getLogger('Thespian').error('Validated source (hash %s) is corrupted: %s',
                                                sourceHash, ex)
            return

        # Store this registered source
        self._sources[sourceHash] = sourceZip

    def unloadActorSource(self, sourceHash):
        if sourceHash in self._sources:
            del self._sources[sourceHash]
        for pnum, metapath in enumerate(sys.meta_path):
            if getattr(metapath, 'srcHash', None) == sourceHash:
                rmmods = [M for M in sys.modules
                          if M and M.startswith(metapath.hashRoot())]
                for each in rmmods:
                    del sys.modules[each]
                del sys.meta_path[pnum]
                break

    def h_CapabilityUpdate(self, envelope):
        if self._updSystemCapabilities(envelope.message.capabilityName,
                                       envelope.message.capabilityValue):
            self._capUpdateLocalActors()

    def _updSystemCapabilities(self, cName, cVal):
        updateLocals = False
        if cVal is not None:
            updateLocals = cName not in self.capabilities or \
                           self.capabilities[cName] != cVal
            self.capabilities[cName] = cVal
        else:
            if cName in self.capabilities:
                updateLocals = True
                del self.capabilities[cName]
        return updateLocals

    def _capUpdateLocalActors(self):
        newCaps = NewCapabilities(self.capabilities, self.myAddress)
        for each in self.childAddresses:
            self._send_intent(TransmitIntent(each, newCaps))

<commit_msg>Misc unused import and variable and formatting cleanup in adminCore<commit_after>import sys
import logging
from thespian.actors import *
from thespian.system.utilis import thesplog, AssocList
from thespian.system.systemCommon import systemCommonBase
from thespian.system.messages.status import Thespian_SystemStatus
from thespian.system.messages.admin import *
from thespian.system.transport import TransmitIntent
from thespian.system.sourceLoader import SourceHashFinder

class AdminCore(systemCommonBase):

    def __init__(self, transport, address, capabilities,
                 logdefs,
                 concurrency_context):
        thesplog('++++ Starting Admin from %s',
                 sys.modules['thespian'].__file__,
                 level=logging.DEBUG)
        super(AdminCore, self).__init__(address, transport)
        self.init_replicator(transport, concurrency_context)
        self.capabilities = capabilities
        self.logdefs = logdefs
        self._pendingChildren = {}  # Use: childLocalAddr instance # : PendingActorEnvelope
        # Things that help us look like an Actor, even though we're not
        self._sourceHash = None
        thesplog('++++ Admin started @ %s / gen %s',
                 self.transport.myAddress, str(ThespianGeneration),
                 level=logging.INFO,
                 primary=True)
        self._nannying = AssocList()  # child actorAddress -> parent Address
        self._deadLetterHandler = None
        self._sources = {}  # Index is sourcehash, value is requestor
                            # ActorAddress or zipsrc (when validated)
        self._sourceAuthority = None

    def _activate(self):
        """Called when the full ActorSystem initialization is completed.  This
           should then activate any functionality that needed to wait
           for completion of initialization.
        """
        pass

    def run(self):
        try:
            while True:
                r = self.transport.run(self.handleIncoming, None)
                if isinstance(r, Thespian__UpdateWork):
                    # tickle the transmit queues
                    self._send_intent(
                        TransmitIntent(self.myAddress, r))
                    continue
                # Expects that on completion of self.transport.run
                # that the Actor is done processing and that it has
                # been shutdown gracefully.
                break
        except Exception:
            import traceback
            thesplog('ActorAdmin uncaught exception: %s',
                     traceback.format_exc(),
                     level=logging.ERROR, exc_info=True)
        thesplog('Admin time to die', level=logging.DEBUG)

    def handleIncoming(self, envelope):
        self._sCBStats.inc('Admin Message Received.Total')
        handled, result = self._handleReplicatorMessages(envelope)
        if handled:
            return result
        if isinstance(envelope.message, (ActorSystemMessage,
                                         logging.LogRecord)):
            thesplog('Admin of %s', envelope.identify(), level=logging.DEBUG)
            return getattr(self,
                           'h_' + envelope.message.__class__.__name__,
                           self.unrecognized)(envelope)
        # else discard random non-admin messages
        self._sCBStats.inc('Admin Message Received.Ignored')
        thesplog('ADMIN DISCARD %s', envelope.identify(),
                 level=logging.WARNING)
        return True

    def unrecognized(self, envelope):
        self._sCBStats.inc('Admin Message Received.Discarded')
        thesplog("Admin got incoming %s from %s;"
                 " discarded because I don't know how to handle it!",
                 envelope.message, envelope.sender,
                 level=logging.WARNING, primary=True)
        return True

    def isShuttingDown(self): return hasattr(self, '_exiting')

    def h_QueryExists(self, envelope):
        self._sCBStats.inc('Admin Message Received.Type.QueryExists')
        self._send_intent(
            TransmitIntent(
                envelope.sender,
                QueryAck(
                    self.capabilities.get('Thespian ActorSystem Name',
                                          'misc Actor System'),
                    self.capabilities.get('Thespian ActorSystem Version',
                                          'unknown Version'),
                    self.isShuttingDown())))
        return True

    def getStatus(self):
        resp = Thespian_SystemStatus(self.myAddress,
                                     capabilities=self.capabilities,
                                     inShutdown=self.isShuttingDown())
        resp.setDeadLetterHandler(self._deadLetterHandler)
        self._updateStatusResponse(resp)
        resp.setLoadedSources(list(self._sources.keys()))
        resp.sourceAuthority = self._sourceAuthority
        return resp

    def h_Thespian_StatusReq(self, envelope):
        self._sCBStats.inc('Admin Message Received.Type.StatusReq')
        self._send_intent(TransmitIntent(envelope.sender, self.getStatus()))
        return True

    def thesplogStatus(self):
        "Write status to thesplog"
        from io import StringIO
        sd = StringIO()
        from thespian.system.messages.status import formatStatus
        try:
            sd.write('')
            SSD = lambda v: v
        except TypeError:
            class SSD(object):
                def __init__(self, sd): self.sd = sd
                def write(self, str_arg): self.sd.write(str_arg.decode('utf-8'))
        formatStatus(self.getStatus(), tofd=SSD(sd))
        thesplog('STATUS: %s', sd.getvalue())

    def h_SetLogging(self, envelope):
        return self.setLoggingControls(envelope)

    def h_SystemShutdown(self, envelope):
        self._exiting = envelope.sender
        thesplog('---- shutdown initiated by %s', envelope.sender,
                 level=logging.DEBUG)

        # Send failure notices and clear out any pending children.  If
        # any pending child ready notifications are received after
        # this, they will automatically be sent an ActorExitRequest.
        for each in self._pendingChildren:
            pendingReq = self._pendingChildren[each]
            self._send_intent(
                TransmitIntent(
                    pendingReq.sender,
                    PendingActorResponse(
                        pendingReq.message.forActor,
                        pendingReq.message.instanceNum,
                        pendingReq.message.globalName,
                        errorCode=PendingActorResponse.ERROR_ActorSystem_Shutting_Down)))
        self._pendingChildren = []

        if not self.childAddresses:  # no children?
            self._sayGoodbye()
            self.transport.abort_run(drain=True)
            return True

        # Now shutdown any direct children
        self._killLocalActors()

        # Callback will shutdown the Admin Once the children confirm
        # their exits.
        return True

    def _killLocalActors(self):
        for each in self.childAddresses:
            self._send_intent(
                TransmitIntent(each, ActorExitRequest(recursive=True),
                               onError=lambda r, m, a=each:
                               self._handleChildExited(a)))

    def _sayGoodbye(self):
        self._cleanupAdmin()
        self._send_intent(TransmitIntent(self._exiting,
                                         SystemShutdownCompleted()))
        thesplog('---- shutdown completed', level=logging.INFO)

    def h_ChildActorExited(self, envelope):
        return self._handleChildExited(envelope.message.childAddress)

    def _handleChildExited(self, childAddress):
        parentAddr = self._nannying.find(childAddress)
        if parentAddr:
            self._nannying.rmv(childAddress)
            # Let original requesting Actor (that *thinks* it's the
            # parent) know about this child exit as well.
            self._send_intent(TransmitIntent(parentAddr,
                                             ChildActorExited(childAddress)))
        return super(AdminCore, self)._handleChildExited(childAddress)

    def h_PendingActor(self, envelope):
        """Admin is creating an Actor.  This covers one of the following cases:

            1. Creating for external requester via ActorSystem.
                  envelope.message.forActor will be None
               The Admin is the "parent" for all externally-created Actors.

            2. Creating for another Actor when direct creation fails.
               Usually means that the current ActorSyste cannot meet
               the new Actor's requirements and the Admin should find
               another convention member that can create the new
               Actor.

            3. GlobalName Actor.  The Admin is the "parent" for all
               GlobalName Actors.

        """
        self._sCBStats.inc('Admin Message Received.Type.Pending Actor Request')

        if self.isShuttingDown():
            self._sendPendingActorResponse(
                envelope, None,
                errorCode=PendingActorResponse.ERROR_ActorSystem_Shutting_Down)
            return True

        sourceHash = envelope.message.sourceHash
        if sourceHash and sourceHash not in self._sources:
            self._sendPendingActorResponse(
                envelope, None,
                errorCode=PendingActorResponse.ERROR_Invalid_SourceHash)
            return True

        # Note, both Admin and remote requester will have a local
        # child address for the child (with a different instanceNumber
        # each).
        childAddr = self._addrManager.createLocalAddress()
        childInstance = childAddr.addressDetails.addressInstanceNum

        try:
            self._startChildActor(
                childAddr, envelope.message.actorClassName,
                parentAddr=self.myAddress,  # Admin is surrogate parent
                notifyAddr=self.myAddress,
                childRequirements=envelope.message.targetActorReq,
                sourceHash=sourceHash,
                sourceToLoad=(self._sources[sourceHash]
                              if sourceHash else None))
        except NoCompatibleSystemForActor:
            self._sendPendingActorResponse(
                envelope, None,
                errorCode=PendingActorResponse.ERROR_No_Compatible_ActorSystem)
            self._retryPendingChildOperations(childInstance, None)
            return True

        self._pendingChildren[childInstance] = envelope
        # transport will contrive to call _pendingActorReady when the
        # child is initialized and connected to this parent.
        return True

    def _sendPendingActorResponse(self, requestEnvelope, actualAddress,
                                  errorCode=None, errorStr=None):
        # actualAddress is None for failure
        if actualAddress is None and errorCode is None:
            raise ValueError('Must specify either actualAddress or errorCode')
        self._send_intent(
            TransmitIntent(
                requestEnvelope.message.forActor or requestEnvelope.sender,
                PendingActorResponse(requestEnvelope.message.forActor,
                                     requestEnvelope.message.instanceNum,
                                     requestEnvelope.message.globalName,
                                     errorCode=errorCode,
                                     errorStr=errorStr,
                                     actualAddress=actualAddress)))

    def _pendingActorReady(self, childInstance, actualAddress):
        if childInstance not in self._pendingChildren:
            thesplog('Pending actor is ready at %s for UNKNOWN %s'
                     '; sending child a shutdown',
                     actualAddress, childInstance, level=logging.WARNING)
            self._send_intent(
                TransmitIntent(actualAddress, ActorExitRequest(recursive=True)))
            return

        requestEnvelope = self._pendingChildren[childInstance]
        del self._pendingChildren[childInstance]
        if requestEnvelope.message.globalName or \
           not requestEnvelope.message.forActor:
            # The Admin is the responsible Parent for these children
            self._registerChild(actualAddress)
        else:
            # Anything the Admin was requested to create is a adoptive
            # child and should be killed when the Admin exits.
            self._registerChild(actualAddress)

        if requestEnvelope.message.forActor:
            # Proxy-parenting; remember the real parent
            self._nannying.add(actualAddress, requestEnvelope.message.forActor)
        self._addrManager.associateUseableAddress(self.myAddress,
                                                  childInstance,
                                                  actualAddress)
        # n.b. childInstance is for this Admin, but caller's
        # childInstance is in original request
        self._sendPendingActorResponse(requestEnvelope, actualAddress)
        self._retryPendingChildOperations(childInstance, actualAddress)

    def h_HandleDeadLetters(self, envelope):  # handlerAddr, enableHandler
        self._sCBStats.inc('Admin Message Received.Type.Set Dead Letter Handler')
        if envelope.message.enableHandler:
            self._deadLetterHandler = envelope.message.handlerAddr
        else:
            if self._deadLetterHandler == envelope.message.handlerAddr:
                self._deadLetterHandler = None
        return True

    def h_DeadEnvelope(self, envelope):
        self._sCBStats.inc('Admin Message Received.Type.Dead Letter')
        if self._deadLetterHandler:
            self._send_intent(
                TransmitIntent(self._deadLetterHandler,
                               envelope.message))
        return True

    def h_RegisterSourceAuthority(self, envelope):
        self._sourceAuthority = envelope.message.authorityAddress

    def h_ValidateSource(self, envelope):
        sourceHash = envelope.message.sourceHash
        if not envelope.message.sourceData:
            self.unloadActorSource(sourceHash)
            logging.getLogger('Thespian')\
                   .info('Source hash %s unloaded', sourceHash)
            return
        if sourceHash in self._sources:
            logging.getLogger('Thespian')\
                   .info('Source hash %s already loaded', sourceHash)
            return
        if self._sourceAuthority:
            self._send_intent(TransmitIntent(self._sourceAuthority,
                                             envelope.message))
            return
        # Any attempt to load sources is ignored if there is no active
        # Source Authority.  This is a security measure to protect the
        # un-protected.
        logging.getLogger('Thespian').warning(
            'No source authority to validate source hash %s',
            sourceHash)

    def h_ValidatedSource(self, envelope):
        self._loadValidatedActorSource(envelope.message.sourceHash,
                                       envelope.message.sourceZip)
        thesplog('Source hash %s validated by Source Authority; now available.',
                 envelope.message.sourceHash)

    def _loadValidatedActorSource(self, sourceHash, sourceZip):
        # Validate the source file; this doesn't actually utilize the
        # sourceZip, but it ensures that the sourceZip isn't garbage
        # before registering it as active source.
        try:
            f = SourceHashFinder(sourceHash, lambda v: v, sourceZip)
            namelist = f.getZipNames()
            logging.getLogger('Thespian').info(
                'Validated source hash %s, %s modules (%s)',
                sourceHash, len(namelist),
                ', '.join(namelist if len(namelist) < 10 else
                          namelist[:9] + ['...']))
        except Exception as ex:
            logging.getLogger('Thespian')\
                   .error('Validated source (hash %s) is corrupted: %s',
                          sourceHash, ex)
            return

        # Store this registered source
        self._sources[sourceHash] = sourceZip

    def unloadActorSource(self, sourceHash):
        if sourceHash in self._sources:
            del self._sources[sourceHash]
        for pnum, metapath in enumerate(sys.meta_path):
            if getattr(metapath, 'srcHash', None) == sourceHash:
                rmmods = [M for M in sys.modules
                          if M and M.startswith(metapath.hashRoot())]
                for each in rmmods:
                    del sys.modules[each]
                del sys.meta_path[pnum]
                break

    def h_CapabilityUpdate(self, envelope):
        if self._updSystemCapabilities(envelope.message.capabilityName,
                                       envelope.message.capabilityValue):
            self._capUpdateLocalActors()

    def _updSystemCapabilities(self, cName, cVal):
        updateLocals = False
        if cVal is not None:
            updateLocals = cName not in self.capabilities or \
                           self.capabilities[cName] != cVal
            self.capabilities[cName] = cVal
        else:
            if cName in self.capabilities:
                updateLocals = True
                del self.capabilities[cName]
        return updateLocals

    def _capUpdateLocalActors(self):
        newCaps = NewCapabilities(self.capabilities, self.myAddress)
        for each in self.childAddresses:
            self._send_intent(TransmitIntent(each, newCaps))
<|endoftext|>


<commit_before>class Display():

    def __init__(self):
        self.start = "Welcome"
        self.draw = "Draw"
        self.computer = "Computer Wins"
        self.human = "You Win"
        self.next_move = "What is your next move?"
        self.bad_move = "That is not a legal move."

    def show(self, text):
        print text

    def render_board(self, raw_board):
        stringified_board = self.stringify_board(raw_board)
        rendered_board = self.construct_board(stringified_board)
        print rendered_board
        return rendered_board

    def stringify_board(self, board):
        for i in range(0, len(board)):
            if board[i] == 0:
                board[i] = "   "
            elif board[i] == 1:
                board[i] = " X "
            elif board[i] == 10:
                board[i] = " O "
        return board

    def construct_board(self, board):
        rack = self.construct_rack(board)
        rows = self.construct_rows(board)
        constructed_board = '\n' + rack.join(rows) + '\n'
        return constructed_board

    def construct_rack(self, board):
        board_size = self.get_board_size(board)
        corner = '+'
        shelves = []
        for i in range(0, board_size):
            shelves.append('---')
        rack = '\n' + corner.join(shelves) + '\n'
        return rack

    def construct_rows(self, board):
        board_size = self.get_board_size(board)
        wall = '|'
        rows = []
        for i in range(0, len(board), board_size):
            rows.append(wall.join(self.construct_a_row(board, i)))
            #working_row = rows[i]
            #rows[i] = wall.join(working_row)
        return rows

    def construct_a_row(self, board, rows_counter):
        board_size = self.get_board_size(board)
        a_row = []
        for i in range(rows_counter, rows_counter + board_size):
            a_row.append(board[i])
        return a_row

    def get_board_size(self, board):
        from math import sqrt
        return int(sqrt(len(board)))
<commit_msg>refactor method names for layer of abstraction<commit_after>class Display():

    def __init__(self):
        self.start = "Welcome"
        self.draw = "Draw"
        self.computer = "Computer Wins"
        self.human = "You Win"
        self.next_move = "What is your next move?"
        self.bad_move = "That is not a legal move."

    def show(self, text):
        print text

    def render_board(self, raw_board):
        stringified_board = self.stringify_board(raw_board)
        rendered_board = self.construct_board(stringified_board)
        print rendered_board
        return rendered_board

    def stringify_board(self, board):
        for i in range(0, len(board)):
            if board[i] == 0:
                board[i] = "   "
            elif board[i] == 1:
                board[i] = " X "
            elif board[i] == 10:
                board[i] = " O "
        return board

    def construct_board(self, board):
        rack = self.assemble_rack(board)
        rows = self.assemble_rows(board)
        constructed_board = '\n' + rack.join(rows) + '\n'
        return constructed_board

    def assemble_rack(self, board):
        board_size = self.get_board_size(board)
        corner = '+'
        shelves = []
        for i in range(0, board_size):
            shelves.append('---')
        rack = '\n' + corner.join(shelves) + '\n'
        return rack

    def assemble_rows(self, board):
        board_size = self.get_board_size(board)
        wall = '|'
        rows = []
        for i in range(0, len(board), board_size):
            rows.append(wall.join(self.build_proto_row(board, i)))
        return rows

    def build_proto_row(self, board, rows_counter):
        board_size = self.get_board_size(board)
        a_row = []
        for i in range(rows_counter, rows_counter + board_size):
            a_row.append(board[i])
        return a_row

    def get_board_size(self, board):
        from math import sqrt
        return int(sqrt(len(board)))
<|endoftext|>


<commit_before>from thinglang.lexer.symbols import LexicalGroupEnd
from thinglang.lexer.symbols.base import LexicalIdentifier
from thinglang.lexer.symbols.functions import LexicalDeclarationConstructor
from thinglang.parser.tokens import DefinitionPairToken, BaseToken
from thinglang.parser.tokens.functions import ArgumentList

class ThingDefinition(DefinitionPairToken):

    def __contains__(self, item):
        return any(child.value == item for child in self.children)

    def __getitem__(self, item):
        return [child for child in self.children if child.value == item][0]

    def describe(self):
        return self.name

class MethodDefinition(BaseToken):
    def __init__(self, slice):
        super(MethodDefinition, self).__init__(slice)

        if isinstance(slice[0], LexicalDeclarationConstructor):
            self.name = LexicalIdentifier.constructor()
            argument_list = slice[1]
        else:
            self.name = slice[1]
            argument_list = slice[2]

        if isinstance(argument_list, ArgumentList):
            self.arguments = argument_list
        else:
            self.arguments = ArgumentList()

    def describe(self):
        return '{}, args={}'.format(self.name, self.arguments)

class MemberDefinition(BaseToken):
    def __init__(self, slice):
        super(MemberDefinition, self).__init__(slice)

        _, self.type, self.name = slice

    def describe(self):
        return 'has {} {}'.format(self.type, self.name)
<commit_msg>Fix value/name ambiguity in ThingDefinition<commit_after>from thinglang.lexer.symbols import LexicalGroupEnd
from thinglang.lexer.symbols.base import LexicalIdentifier
from thinglang.lexer.symbols.functions import LexicalDeclarationConstructor
from thinglang.parser.tokens import DefinitionPairToken, BaseToken
from thinglang.parser.tokens.functions import ArgumentList

class ThingDefinition(DefinitionPairToken):

    def __contains__(self, item):
        return any(child.name == item for child in self.children)

    def __getitem__(self, item):
        return [child for child in self.children if child.name == item][0]

    def describe(self):
        return self.name

class MethodDefinition(BaseToken):
    def __init__(self, slice):
        super(MethodDefinition, self).__init__(slice)

        if isinstance(slice[0], LexicalDeclarationConstructor):
            self.name = LexicalIdentifier.constructor()
            argument_list = slice[1]
        else:
            self.name = slice[1]
            argument_list = slice[2]

        if isinstance(argument_list, ArgumentList):
            self.arguments = argument_list
        else:
            self.arguments = ArgumentList()

    def describe(self):
        return '{}, args={}'.format(self.name, self.arguments)

class MemberDefinition(BaseToken):
    def __init__(self, slice):
        super(MemberDefinition, self).__init__(slice)

        _, self.type, self.name = slice

    def describe(self):
        return 'has {} {}'.format(self.type, self.name)
<|endoftext|>


<commit_before>def if_mkl_open_source_only(if_true, if_false = []):
    """Shorthand for select()'ing on whether we're building with
    MKL-DNN v0.x open source library only, without depending on MKL binary form.

    Returns a select statement which evaluates to if_true if we're building
    with MKL-DNN v0.x open source library only. Otherwise, the select statement
    evaluates to if_false.

    """
    return select({
        str(Label("//third_party/mkl_dnn:build_with_mkl_dnn_only")): if_true,
        "//conditions:default": if_false,
    })

def if_mkl_v1_open_source_only(if_true, if_false = []):
    """Shorthand for select()'ing on whether we're building with MKL-DNN v1.x
    (and v0.x) open source library only, without depending on MKL binary form.

    Returns a select statement which evaluates to if_true if we're building
    with MKL-DNN v1.x (and v0.x) open source library only. Otherwise, the
    select statement evaluates to if_false.

    """
    return select({
        str(Label("//third_party/mkl_dnn:build_with_mkl_dnn_v1_only")): if_true,
        "//conditions:default": if_false,
    })
<commit_msg>Minor change.<commit_after>def if_mkl_open_source_only(if_true, if_false = []):
    """Shorthand for select()'ing on whether we're building with
    MKL-DNN v0.x open source library only, without depending on MKL binary form.

    Returns a select statement which evaluates to if_true if we're building
    with MKL-DNN v0.x open source library only. Otherwise, the select statement
    evaluates to if_false.

    """
    return select({
        str(Label("//third_party/mkl_dnn:build_with_mkl_dnn_only")): if_true,
        "//conditions:default": if_false,
    })

def if_mkl_v1_open_source_only(if_true, if_false = []):
    """Shorthand for select()'ing on whether we're building with
    MKL-DNN v1.x open source library only, without depending on MKL binary form.

    Returns a select statement which evaluates to if_true if we're building
    with MKL-DNN v1.x open source library only. Otherwise, the
    select statement evaluates to if_false.

    """
    return select({
        str(Label("//third_party/mkl_dnn:build_with_mkl_dnn_v1_only")): if_true,
        "//conditions:default": if_false,
    })
<|endoftext|>


<commit_before>import unittest
from deepchem.feat import MATFeaturizer
import numpy as np

class TestMATFeaturizer(unittest.TestCase):
  """
  Test MATFeaturizer.
  """

  def setUp(self):
    """
    Set up tests.
    """
    from rdkit import Chem
    smiles = 'CCC'
    self.mol = Chem.MolFromSmiles(smiles)

  def test_mat_featurizer(self):
    """
    Test featurizer.py
    """
    featurizer = MATFeaturizer()
    out = featurizer.featurize(self.mol)
    assert (type(out) == np.ndarray)
    assert (out.shape == (1, 2, 31))
    correct_array = np.array([[[
        0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.,
        0., 0., 1., 0., 0., 1., 0., 0., 0., 0., 1., 0., 1.
    ], [
        0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.,
        0., 0., 1., 0., 0., 1., 0., 0., 0., 1., 0., 1., 0.
    ]]])
    assert (np.array_equal(out, correct_array))
<commit_msg>Removed blank line<commit_after>import unittest
from deepchem.feat import MATFeaturizer
import numpy as np

class TestMATFeaturizer(unittest.TestCase):
  """
  Test MATFeaturizer.
  """

  def setUp(self):
    """
    Set up tests.
    """
    from rdkit import Chem
    smiles = 'CCC'
    self.mol = Chem.MolFromSmiles(smiles)

  def test_mat_featurizer(self):
    """
    Test featurizer.py
    """
    featurizer = MATFeaturizer()
    out = featurizer.featurize(self.mol)
    assert (type(out) == np.ndarray)
    assert (out.shape == (1, 2, 31))
    correct_array = np.array([[[
        0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.,
        0., 0., 1., 0., 0., 1., 0., 0., 0., 0., 1., 0., 1.
    ], [
        0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0.,
        0., 0., 1., 0., 0., 1., 0., 0., 0., 1., 0., 1., 0.
    ]]])
    assert (np.array_equal(out, correct_array))
<|endoftext|>