<commit_before>import re
from collections import namedtuple

ASCII_BYTE = " !\"#\$%&\'\(\)\*\+,-\./0123456789:;<=>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\]\^_`abcdefghijklmnopqrstuvwxyz\{\|\}\\\~\t"
ASCII_RE_4 = re.compile("([%s]{%d,})" % (ASCII_BYTE, 4))
UNICODE_RE_4 = re.compile(b"((?:[%s]\x00){%d,})" % (ASCII_BYTE, 4))

String = namedtuple("String", ["s", "offset"])

def extract_ascii_strings(buf, n=4):
    '''
    Extract ASCII strings from the given binary data.

    :param buf: A bytestring.
    :type buf: str
    :param n: The minimum length of strings to extract.
    :type n: int
    :rtype: Sequence[String]
    '''
    r = None
    if n == 4:
        r = ASCII_RE_4
    else:
        reg = "([%s]{%d,})" % (ASCII_BYTE, n)
        r = re.compile(reg)
    for match in r.finditer(buf):
        yield String(match.group().decode("ascii"), match.start())

def extract_unicode_strings(buf, n=4):
    '''
    Extract naive UTF-16 strings from the given binary data.

    :param buf: A bytestring.
    :type buf: str
    :param n: The minimum length of strings to extract.
    :type n: int
    :rtype: Sequence[String]
    '''
    if n == 4:
        r = UNICODE_RE_4
    else:
        reg = b"((?:[%s]\x00){%d,})" % (ASCII_BYTE, n)
        r = re.compile(reg)
    for match in r.finditer(buf):
        try:
            yield String(match.group().decode("utf-16"), match.start())
        except UnicodeDecodeError:
            pass

def main():
    import sys

    with open(sys.argv[1], 'rb') as f:
        b = f.read()

    for s in extract_ascii_strings(b):
        print('0x{:x}: {:s}'.format(s.offset, s.s))

    for s in extract_unicode_strings(b):
        print('0x{:x}: {:s}'.format(s.offset, s.s))

if __name__ == '__main__':
    main()
<commit_msg>skip extraction on bufs of one repeated character<commit_after>import re
from collections import namedtuple

ASCII_BYTE = " !\"#\$%&\'\(\)\*\+,-\./0123456789:;<=>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\]\^_`abcdefghijklmnopqrstuvwxyz\{\|\}\\\~\t"
ASCII_RE_4 = re.compile("([%s]{%d,})" % (ASCII_BYTE, 4))
UNICODE_RE_4 = re.compile(b"((?:[%s]\x00){%d,})" % (ASCII_BYTE, 4))
REPEATS = ["A", "\x00", "\xfe", "\xff"]

String = namedtuple("String", ["s", "offset"])

def extract_ascii_strings(buf, n=4):
    '''
    Extract ASCII strings from the given binary data.

    :param buf: A bytestring.
    :type buf: str
    :param n: The minimum length of strings to extract.
    :type n: int
    :rtype: Sequence[String]
    '''
    if (buf[0] in REPEATS) and (buf.count(buf[0]) == len(buf)):
        return

    r = None
    if n == 4:
        r = ASCII_RE_4
    else:
        reg = "([%s]{%d,})" % (ASCII_BYTE, n)
        r = re.compile(reg)
    for match in r.finditer(buf):
        yield String(match.group().decode("ascii"), match.start())

def extract_unicode_strings(buf, n=4):
    '''
    Extract naive UTF-16 strings from the given binary data.

    :param buf: A bytestring.
    :type buf: str
    :param n: The minimum length of strings to extract.
    :type n: int
    :rtype: Sequence[String]
    '''
    if (buf[0] in REPEATS) and (buf.count(buf[0]) == len(buf)):
        return

    if n == 4:
        r = UNICODE_RE_4
    else:
        reg = b"((?:[%s]\x00){%d,})" % (ASCII_BYTE, n)
        r = re.compile(reg)
    for match in r.finditer(buf):
        try:
            yield String(match.group().decode("utf-16"), match.start())
        except UnicodeDecodeError:
            pass

def main():
    import sys

    with open(sys.argv[1], 'rb') as f:
        b = f.read()

    for s in extract_ascii_strings(b):
        print('0x{:x}: {:s}'.format(s.offset, s.s))

    for s in extract_unicode_strings(b):
        print('0x{:x}: {:s}'.format(s.offset, s.s))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>"""Spherical harmonic vector wind analysis."""
# Copyright (c) 2012-2013 Andrew Dawson
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
from __future__ import absolute_import

from . import standard
from . import tools

# List to define the behaviour of imports of the form:
#     from windspharm import *
__all__ = []

# Package version number.
__version__ = '1.2.0'

try:
    from . import cdms
    __all__.append('cdms')
    metadata = cdms
except ImportError:
    pass

try:
    from . import iris
    __all__.append('iris')
except ImportError:
    pass
<commit_msg>Reset version number for maintenance branch.<commit_after>"""Spherical harmonic vector wind analysis."""
# Copyright (c) 2012-2013 Andrew Dawson
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
from __future__ import absolute_import

from . import standard
from . import tools

# List to define the behaviour of imports of the form:
#     from windspharm import *
__all__ = []

# Package version number.
__version__ = '1.2.x'

try:
    from . import cdms
    __all__.append('cdms')
    metadata = cdms
except ImportError:
    pass

try:
    from . import iris
    __all__.append('iris')
except ImportError:
    pass
<|endoftext|>


<commit_before>"""LNT based measurements."""
import logging
from glob import glob
from os import path

from plumbum import FG, local

from benchbuild.project import Project
from benchbuild.settings import CFG
from benchbuild.utils.cmd import cat, mkdir, rm, virtualenv
from benchbuild.utils.compiler import lt_clang, lt_clang_cxx
from benchbuild.utils.downloader import CopyNoFail, Git
from benchbuild.utils.wrapping import wrap_dynamic

LOG = logging.getLogger(__name__)

class LNTGroup(Project):
    """LNT ProjectGroup for running the lnt test suite."""

    DOMAIN = 'lnt'
    GROUP = 'lnt'
    VERSION = '9.0.1.13'

    NAME_FILTERS = [
        r'(?P<name>.+)\.simple',
        r'(?P<name>.+)-(dbl|flt)',
    ]

    SUBDIR = None

    src_dir = "lnt"
    src_uri = "http://llvm.org/git/lnt"
    test_suite_dir = "test-suite"
    test_suite_uri = "http://llvm.org/git/test-suite"

    # Will be set by configure.
    lnt = None
    sandbox_dir = None
    clang = None
    clang_cxx = None
    binary = None

    def download(self):
        Git(self.src_uri, self.src_dir)
        Git(self.test_suite_uri, self.test_suite_dir)

        virtualenv("local", "--python=python2", )
        pip = local[path.join("local", "bin", "pip")]
        with local.cwd(self.src_dir):
            pip("install", "-e", ".")

    def configure(self):
        sandbox_dir = path.join(self.builddir, "run")
        if path.exists(sandbox_dir):
            rm("-rf", sandbox_dir)

        mkdir(sandbox_dir)

        self.lnt = local[path.join("local", "bin", "lnt")]
        self.sandbox_dir = path.join(self.builddir, "run")
        self.clang = lt_clang(self.cflags, self.ldflags, self.compiler_extension)
        self.clang_cxx = lt_clang_cxx(self.cflags, self.ldflags,
                                      self.compiler_extension)
    @staticmethod
    def after_run_tests(sandbox_dir):
        logfiles = glob(path.join(sandbox_dir, "./*/test.log"))
        for log in logfiles:
            LOG.info("Dumping contents of: %s", log)
            (cat[log] & FG) # pylint: disable=pointless-statement

    def build(self):
        self.lnt("runtest", "test-suite", "-v", "-j1",
                 "--sandbox", self.sandbox_dir,
                 "--benchmarking-only",
                 "--only-compile",
                 "--cc", str(self.clang),
                 "--cxx", str(self.clang_cxx),
                 "--test-suite", path.join(self.builddir,
                                             self.test_suite_dir),
                 "--only-test=" + self.SUBDIR)

    def run_tests(self, experiment, runner):
        binary = wrap_dynamic(self, "lnt_runner", experiment,
                              name_filters=LNTGroup.NAME_FILTERS)

        runner(self.lnt["runtest", "nt", "-v", "-j1",
                        "--sandbox", self.sandbox_dir,
                        "--benchmarking-only",
                        "--cc", str(self.clang),
                        "--cxx", str(self.clang_cxx),
                        "--test-suite", path.join(self.builddir,
                                                  self.test_suite_dir),
                        "--test-style", "simple",
                        "--test-externals", self.builddir,
                        "--make-param=RUNUNDER=" + str(binary),
                        "--only-test=" + self.SUBDIR])

        LNTGroup.after_run_tests(self.sandbox_dir)

class SingleSourceBenchmarks(LNTGroup):
    NAME = 'SingleSourceBenchmarks'
    DOMAIN = 'LNT (SSB)'
    SUBDIR = path.join("SingleSource", "Benchmarks")

class MultiSourceBenchmarks(LNTGroup):
    NAME = 'MultiSourceBenchmarks'
    DOMAIN = 'LNT (MSB)'
    SUBDIR = path.join("MultiSource", "Benchmarks")

class MultiSourceApplications(LNTGroup):
    NAME = 'MultiSourceApplications'
    DOMAIN = 'LNT (MSA)'
    SUBDIR = path.join("MultiSource", "Applications")

class SPEC2006(LNTGroup):
    NAME = 'SPEC2006'
    DOMAIN = 'LNT (Ext)'
    SUBDIR = path.join("External", "SPEC")

    def download(self):
        if CopyNoFail('speccpu2006'):
            super(SPEC2006, self).download()
        else:
            print('======================================================')
            print(('SPECCPU2006 not found in %s. This project will fail.',
                   CFG['tmp_dir']))
            print('======================================================')

class Povray(LNTGroup):
    NAME = 'Povray'
    DOMAIN = 'LNT (Ext)'
    SUBDIR = path.join("External", "Povray")

    povray_url = "https://github.com/POV-Ray/povray"
    povray_src_dir = "Povray"

    def download(self):
        super(Povray, self).download()
        Git(self.povray_url, self.povray_src_dir)
<commit_msg>lnt: enable detect_project<commit_after>"""LNT based measurements."""
import logging
from glob import glob
from os import path

from plumbum import FG, local

from benchbuild.project import Project
from benchbuild.settings import CFG
from benchbuild.utils.cmd import cat, mkdir, rm, virtualenv
from benchbuild.utils.compiler import lt_clang, lt_clang_cxx
from benchbuild.utils.downloader import CopyNoFail, Git
from benchbuild.utils.wrapping import wrap_dynamic

LOG = logging.getLogger(__name__)

class LNTGroup(Project):
    """LNT ProjectGroup for running the lnt test suite."""

    DOMAIN = 'lnt'
    GROUP = 'lnt'
    VERSION = '9.0.1.13'

    NAME_FILTERS = [
        r'(?P<name>.+)\.simple',
        r'(?P<name>.+)-(dbl|flt)',
    ]

    SUBDIR = None

    src_dir = "lnt"
    src_uri = "http://llvm.org/git/lnt"
    test_suite_dir = "test-suite"
    test_suite_uri = "http://llvm.org/git/test-suite"

    # Will be set by configure.
    lnt = None
    sandbox_dir = None
    clang = None
    clang_cxx = None
    binary = None

    def download(self):
        Git(self.src_uri, self.src_dir)
        Git(self.test_suite_uri, self.test_suite_dir)

        virtualenv("local", "--python=python2", )
        pip = local[path.join("local", "bin", "pip")]
        with local.cwd(self.src_dir):
            pip("install", "-e", ".")

    def configure(self):
        sandbox_dir = path.join(self.builddir, "run")
        if path.exists(sandbox_dir):
            rm("-rf", sandbox_dir)

        mkdir(sandbox_dir)

        self.lnt = local[path.join("local", "bin", "lnt")]
        self.sandbox_dir = path.join(self.builddir, "run")
        self.clang = lt_clang(self.cflags, self.ldflags,
                              self.compiler_extension,
                              detect_project=True)
        self.clang_cxx = lt_clang_cxx(self.cflags, self.ldflags,
                                      self.compiler_extension)
    @staticmethod
    def after_run_tests(sandbox_dir):
        logfiles = glob(path.join(sandbox_dir, "./*/test.log"))
        for log in logfiles:
            LOG.info("Dumping contents of: %s", log)
            (cat[log] & FG) # pylint: disable=pointless-statement

    def build(self):
        self.lnt("runtest", "test-suite", "-v", "-j1",
                 "--sandbox", self.sandbox_dir,
                 "--benchmarking-only",
                 "--only-compile",
                 "--cc", str(self.clang),
                 "--cxx", str(self.clang_cxx),
                 "--test-suite", path.join(self.builddir,
                                             self.test_suite_dir),
                 "--only-test=" + self.SUBDIR)

    def run_tests(self, experiment, runner):
        binary = wrap_dynamic(self, "lnt_runner", experiment,
                              name_filters=LNTGroup.NAME_FILTERS)

        runner(self.lnt["runtest", "nt", "-v", "-j1",
                        "--sandbox", self.sandbox_dir,
                        "--benchmarking-only",
                        "--cc", str(self.clang),
                        "--cxx", str(self.clang_cxx),
                        "--test-suite", path.join(self.builddir,
                                                  self.test_suite_dir),
                        "--test-style", "simple",
                        "--test-externals", self.builddir,
                        "--make-param=RUNUNDER=" + str(binary),
                        "--only-test=" + self.SUBDIR])

        LNTGroup.after_run_tests(self.sandbox_dir)

class SingleSourceBenchmarks(LNTGroup):
    NAME = 'SingleSourceBenchmarks'
    DOMAIN = 'LNT (SSB)'
    SUBDIR = path.join("SingleSource", "Benchmarks")

class MultiSourceBenchmarks(LNTGroup):
    NAME = 'MultiSourceBenchmarks'
    DOMAIN = 'LNT (MSB)'
    SUBDIR = path.join("MultiSource", "Benchmarks")

class MultiSourceApplications(LNTGroup):
    NAME = 'MultiSourceApplications'
    DOMAIN = 'LNT (MSA)'
    SUBDIR = path.join("MultiSource", "Applications")

class SPEC2006(LNTGroup):
    NAME = 'SPEC2006'
    DOMAIN = 'LNT (Ext)'
    SUBDIR = path.join("External", "SPEC")

    def download(self):
        if CopyNoFail('speccpu2006'):
            super(SPEC2006, self).download()
        else:
            print('======================================================')
            print(('SPECCPU2006 not found in %s. This project will fail.',
                   CFG['tmp_dir']))
            print('======================================================')

class Povray(LNTGroup):
    NAME = 'Povray'
    DOMAIN = 'LNT (Ext)'
    SUBDIR = path.join("External", "Povray")

    povray_url = "https://github.com/POV-Ray/povray"
    povray_src_dir = "Povray"

    def download(self):
        super(Povray, self).download()
        Git(self.povray_url, self.povray_src_dir)
<|endoftext|>


<commit_before>import time
from mpf.tests.MpfTestCase import MpfTestCase

class TestAssets(MpfTestCase):
    def getMachinePath(self):
        return 'tests/machine_files/asset_manager'

    def getConfigFile(self):
        return 'test_asset_loading.yaml'

    def test_machine_wide_asset_loading(self):

        # test that the shows asset class gets built correctly
        self.assertTrue(self.machine, 'shows')
        self.assertTrue(self.machine.asset_manager._asset_classes)
        self.assertEqual(self.machine.asset_manager._asset_classes[0]
                         ['path_string'], 'shows')

        # tests that assets are registered as expected with various conditions

        # /shows folder
        self.assertIn('show1', self.machine.shows)
        self.assertIn('show2', self.machine.shows)
        self.assertIn('show3', self.machine.shows)

        # test subfolders listed in assets:shows machine-wide config folders
        self.assertIn('show4', self.machine.shows)  # /shows/preload
        self.assertIn('show5', self.machine.shows)  # /shows/on_demand

        # test shows from subfolder not listed in assets:shows
        self.assertIn('show11', self.machine.shows)  # /shows/custom1

        # test shows from the shows: section that have names configured to be
        # different from their file names
        self.assertIn('show_12_new_name', self.machine.shows)  # show12.png
        # custom1/show13.png
        self.assertIn('show_13_new_name', self.machine.shows)

        # test that the shows that were renamed were not also loaded based on
        # their original names
        self.assertNotIn('show12', self.machine.shows)
        self.assertNotIn('show13', self.machine.shows)

        # test that config dicts are merged and/or overwritten properly

        # test custom k/v pair from default config based on the folder the
        # asset was in
        self.assertEqual(self.machine.shows['show4'].config['test_key'],
                         'test_value')

        # test custom k/v pair from asset entry in the shows: section
        self.assertEqual(self.machine.shows['show3'].config['test_key'],
                         'test_value_override3')

        # same as above, but test that it also works when the asset name is
        # different from the file name
        self.assertEqual(
                self.machine.shows['show_12_new_name'].config['test_key'],
                'test_value_override12')

        # Test that mode assets were loaded properly
        self.assertIn('show6', self.machine.shows)
        self.assertIn('show7', self.machine.shows)
        self.assertIn('show8', self.machine.shows)
        self.assertIn('show9', self.machine.shows)
        self.assertIn('show10', self.machine.shows)

        # Make sure all the assets are loaded. Wait if not
        while (self.machine.asset_manager.num_assets_to_load <
                   self.machine.asset_manager.num_assets_loaded):
            self.advance_time_and_run()

        # Need to wait a bit since the loading was a separate thread
        self.advance_time_and_run(.1)

        # Make sure the ones that should have loaded on startup actually loaded
        self.assertTrue(self.machine.shows['show1'].loaded)
        self.assertFalse(self.machine.shows['show1'].loading)
        self.assertFalse(self.machine.shows['show1'].unloading)

        self.assertTrue(self.machine.shows['show2'].loaded)
        self.assertFalse(self.machine.shows['show2'].loading)
        self.assertFalse(self.machine.shows['show2'].unloading)

        self.assertTrue(self.machine.shows['show3'].loaded)
        self.assertFalse(self.machine.shows['show3'].loading)
        self.assertFalse(self.machine.shows['show3'].unloading)

        self.assertTrue(self.machine.shows['show8'].loaded)
        self.assertFalse(self.machine.shows['show8'].loading)
        self.assertFalse(self.machine.shows['show8'].unloading)

        self.assertTrue(self.machine.shows['show2'].loaded)
        self.assertFalse(self.machine.shows['show2'].loading)
        self.assertFalse(self.machine.shows['show2'].unloading)

        self.assertTrue(self.machine.shows['show4'].loaded)
        self.assertFalse(self.machine.shows['show4'].loading)
        self.assertFalse(self.machine.shows['show4'].unloading)

        self.assertTrue(self.machine.shows['show7'].loaded)
        self.assertFalse(self.machine.shows['show7'].loading)
        self.assertFalse(self.machine.shows['show7'].unloading)

        self.assertTrue(self.machine.shows['show11'].loaded)
        self.assertFalse(self.machine.shows['show11'].loading)
        self.assertFalse(self.machine.shows['show11'].unloading)

        self.assertTrue(self.machine.shows['show_12_new_name'].loaded)
        self.assertFalse(self.machine.shows['show_12_new_name'].loading)
        self.assertFalse(self.machine.shows['show_12_new_name'].unloading)

        self.assertTrue(self.machine.shows['show_13_new_name'].loaded)
        self.assertFalse(self.machine.shows['show_13_new_name'].loading)
        self.assertFalse(self.machine.shows['show_13_new_name'].unloading)

        # Make sure the ones that should not have loaded on startup didn't load
        self.assertFalse(self.machine.shows['show5'].loaded)
        self.assertFalse(self.machine.shows['show5'].loading)
        self.assertFalse(self.machine.shows['show5'].unloading)

        self.assertFalse(self.machine.shows['show9'].loaded)
        self.assertFalse(self.machine.shows['show9'].loading)
        self.assertFalse(self.machine.shows['show9'].unloading)

        self.assertFalse(self.machine.shows['show10'].loaded)
        self.assertFalse(self.machine.shows['show10'].loading)
        self.assertFalse(self.machine.shows['show10'].unloading)

        # Start the mode and make sure those assets load
        self.machine.modes['mode1'].start()
        self.advance_time_and_run()

        # Give it a second to load. This file is tiny, so it shouldn't take
        # this long

        start_time = time.time()
        while (not self.machine.shows['show9'].loaded and
                time.time() < start_time + 5):
            self.assertTrue(self.machine.shows['show9'].loading)
            self.advance_time_and_run(.1)

        self.assertTrue(self.machine.shows['show9'].loaded)
        self.assertFalse(self.machine.shows['show9'].loading)
        self.assertFalse(self.machine.shows['show9'].unloading)

        # test mode stop which should unload those assets
        self.machine.modes['mode1'].stop()
        self.advance_time_and_run(.1)

        start_time = time.time()
        while (self.machine.shows['show9'].loaded and
                time.time() < start_time + 5):
            self.assertTrue(self.machine.shows['show9'].unloading)
            self.advance_time_and_run(.1)

        self.assertFalse(self.machine.shows['show9'].loaded)
        self.assertFalse(self.machine.shows['show9'].loading)
        self.assertFalse(self.machine.shows['show9'].unloading)

    def test_random_asset_group(self):
        # three assets, no weights

        # make sure the asset group was created
        self.assertIn('group1', self.machine.shows)

        # make sure the randomness is working. To test this, we request the
        # asset 10,000 times and then count the results and assume that each
        # should be 3,333 +- 500 just to make sure the test never fails/
        res = list()
        for x in range(10000):
            res.append(self.machine.shows['group1'].show)

        self.assertAlmostEqual(3333, res.count(self.machine.shows['show1']),
                               delta=500)
        self.assertAlmostEqual(3333, res.count(self.machine.shows['show2']),
                               delta=500)
        self.assertAlmostEqual(3333, res.count(self.machine.shows['show3']),
                               delta=500)

    def test_random_asset_group_with_weighting(self):
        # three assets, third one has a weight of 2

        # make sure the asset group was created
        self.assertIn('group2', self.machine.shows)

        # make sure the randomness is working. To test this, we request the
        # asset 10,000 times and then count the results and assume that each
        # should be 3,333 +- 500 just to make sure the test never fails/
        res = list()
        for x in range(10000):
            res.append(self.machine.shows['group2'].show)

        self.assertAlmostEqual(2500, res.count(self.machine.shows['show1']),
                               delta=500)
        self.assertAlmostEqual(2500, res.count(self.machine.shows['show2']),
                               delta=500)
        self.assertAlmostEqual(5000, res.count(self.machine.shows['show3']),
                               delta=500)

    def test_sequence_asset_group(self):
        # three assets, no weights

        self.assertIn('group3', self.machine.shows)

        # Should always return in order, 1, 2, 3, 1, 2, 3...
        self.assertIs(self.machine.shows['group3'].show, self.machine.shows['show1'])
        self.assertIs(self.machine.shows['group3'].show, self.machine.shows['show2'])
        self.assertIs(self.machine.shows['group3'].show, self.machine.shows['show3'])
        self.assertIs(self.machine.shows['group3'].show, self.machine.shows['show1'])
        self.assertIs(self.machine.shows['group3'].show, self.machine.shows['show2'])
        self.assertIs(self.machine.shows['group3'].show, self.machine.shows['show3'])
        self.assertIs(self.machine.shows['group3'].show, self.machine.shows['show1'])
        self.assertIs(self.machine.shows['group3'].show, self.machine.shows['show2'])
        self.assertIs(self.machine.shows['group3'].show, self.machine.shows['show3'])

    def test_sequence_asset_group_with_count(self):
        # three assets, no weights

        self.assertIn('group4', self.machine.shows)

        # Should always return in order, 1, 1, 1, 1, 2, 2, 3, 1, 1, 1, 1 ...
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show1'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show1'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show1'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show1'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show2'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show2'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show3'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show1'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show1'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show1'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show1'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show2'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show2'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show3'])

    def test_random_force_next(self):
        # random, except it ensures the same one does not show up twice in a
        # row

        self.assertIn('group5', self.machine.shows)

        # do it 10,000 times just to be sure. :)
        last = self.machine.shows['group5'].show
        res = list()

        for x in range(10000):
            show = self.machine.shows['group5'].show
            self.assertIsNot(last, show)
            last = show

            res.append(show)

        # Also check that the weights were right

        # BTW these weights are non-intuitive since the last asset is not
        # considered for the next round. e.g. show1 = 1, show2 = 5,
        # show3 = 1, so you'd think they would be 1400, 7200, 1400, but in
        # reality, 50% of the time, asset2 is not in contention, so really
        # asset2 has a 6-to-1 (84%) chance of being selected 66% of the time,
        # but a 0% chance of being selected 33% of the time, etc. So trust that
        # these numbers are right. :)
        self.assertAlmostEqual(2733, res.count(self.machine.shows['show1']),
                               delta=500)
        self.assertAlmostEqual(4533, res.count(self.machine.shows['show2']),
                               delta=500)
        self.assertAlmostEqual(2733, res.count(self.machine.shows['show3']),
                               delta=500)

    def test_random_force_all(self):
        # random, except it ensures the same one does not show up twice before
        # they're all shown

        self.assertIn('group6', self.machine.shows)

        for x in range(1000):
            this_set = set()
            this_set.add(self.machine.shows['group6'].show)
            this_set.add(self.machine.shows['group6'].show)
            this_set.add(self.machine.shows['group6'].show)

            self.assertEqual(len(this_set), 3)
<commit_msg>make this one large method to speed up test<commit_after>import time
from mpf.tests.MpfTestCase import MpfTestCase

class TestAssets(MpfTestCase):
    def getMachinePath(self):
        return 'tests/machine_files/asset_manager'

    def getConfigFile(self):
        return 'test_asset_loading.yaml'

    def test_assert_loading(self):
        self.expected_duration = 1.0
        self._test_machine_wide_asset_loading()
        self._test_random_asset_group()
        self._test_random_asset_group_with_weighting()
        self._test_random_force_all()
        self._test_random_force_next()
        self._test_sequence_asset_group()
        self._test_sequence_asset_group_with_count()

    def _test_machine_wide_asset_loading(self):

        # test that the shows asset class gets built correctly
        self.assertTrue(self.machine, 'shows')
        self.assertTrue(self.machine.asset_manager._asset_classes)
        self.assertEqual(self.machine.asset_manager._asset_classes[0]
                         ['path_string'], 'shows')

        # tests that assets are registered as expected with various conditions

        # /shows folder
        self.assertIn('show1', self.machine.shows)
        self.assertIn('show2', self.machine.shows)
        self.assertIn('show3', self.machine.shows)

        # test subfolders listed in assets:shows machine-wide config folders
        self.assertIn('show4', self.machine.shows)  # /shows/preload
        self.assertIn('show5', self.machine.shows)  # /shows/on_demand

        # test shows from subfolder not listed in assets:shows
        self.assertIn('show11', self.machine.shows)  # /shows/custom1

        # test shows from the shows: section that have names configured to be
        # different from their file names
        self.assertIn('show_12_new_name', self.machine.shows)  # show12.png
        # custom1/show13.png
        self.assertIn('show_13_new_name', self.machine.shows)

        # test that the shows that were renamed were not also loaded based on
        # their original names
        self.assertNotIn('show12', self.machine.shows)
        self.assertNotIn('show13', self.machine.shows)

        # test that config dicts are merged and/or overwritten properly

        # test custom k/v pair from default config based on the folder the
        # asset was in
        self.assertEqual(self.machine.shows['show4'].config['test_key'],
                         'test_value')

        # test custom k/v pair from asset entry in the shows: section
        self.assertEqual(self.machine.shows['show3'].config['test_key'],
                         'test_value_override3')

        # same as above, but test that it also works when the asset name is
        # different from the file name
        self.assertEqual(
                self.machine.shows['show_12_new_name'].config['test_key'],
                'test_value_override12')

        # Test that mode assets were loaded properly
        self.assertIn('show6', self.machine.shows)
        self.assertIn('show7', self.machine.shows)
        self.assertIn('show8', self.machine.shows)
        self.assertIn('show9', self.machine.shows)
        self.assertIn('show10', self.machine.shows)

        # Make sure all the assets are loaded. Wait if not
        while (self.machine.asset_manager.num_assets_to_load <
               self.machine.asset_manager.num_assets_loaded):
            self.advance_time_and_run()

        # Need to wait a bit since the loading was a separate thread
        self.advance_time_and_run(.1)

        # Make sure the ones that should have loaded on startup actually loaded
        self.assertTrue(self.machine.shows['show1'].loaded)
        self.assertFalse(self.machine.shows['show1'].loading)
        self.assertFalse(self.machine.shows['show1'].unloading)

        self.assertTrue(self.machine.shows['show2'].loaded)
        self.assertFalse(self.machine.shows['show2'].loading)
        self.assertFalse(self.machine.shows['show2'].unloading)

        self.assertTrue(self.machine.shows['show3'].loaded)
        self.assertFalse(self.machine.shows['show3'].loading)
        self.assertFalse(self.machine.shows['show3'].unloading)

        self.assertTrue(self.machine.shows['show8'].loaded)
        self.assertFalse(self.machine.shows['show8'].loading)
        self.assertFalse(self.machine.shows['show8'].unloading)

        self.assertTrue(self.machine.shows['show2'].loaded)
        self.assertFalse(self.machine.shows['show2'].loading)
        self.assertFalse(self.machine.shows['show2'].unloading)

        self.assertTrue(self.machine.shows['show4'].loaded)
        self.assertFalse(self.machine.shows['show4'].loading)
        self.assertFalse(self.machine.shows['show4'].unloading)

        self.assertTrue(self.machine.shows['show7'].loaded)
        self.assertFalse(self.machine.shows['show7'].loading)
        self.assertFalse(self.machine.shows['show7'].unloading)

        self.assertTrue(self.machine.shows['show11'].loaded)
        self.assertFalse(self.machine.shows['show11'].loading)
        self.assertFalse(self.machine.shows['show11'].unloading)

        self.assertTrue(self.machine.shows['show_12_new_name'].loaded)
        self.assertFalse(self.machine.shows['show_12_new_name'].loading)
        self.assertFalse(self.machine.shows['show_12_new_name'].unloading)

        self.assertTrue(self.machine.shows['show_13_new_name'].loaded)
        self.assertFalse(self.machine.shows['show_13_new_name'].loading)
        self.assertFalse(self.machine.shows['show_13_new_name'].unloading)

        # Make sure the ones that should not have loaded on startup didn't load
        self.assertFalse(self.machine.shows['show5'].loaded)
        self.assertFalse(self.machine.shows['show5'].loading)
        self.assertFalse(self.machine.shows['show5'].unloading)

        self.assertFalse(self.machine.shows['show9'].loaded)
        self.assertFalse(self.machine.shows['show9'].loading)
        self.assertFalse(self.machine.shows['show9'].unloading)

        self.assertFalse(self.machine.shows['show10'].loaded)
        self.assertFalse(self.machine.shows['show10'].loading)
        self.assertFalse(self.machine.shows['show10'].unloading)

        # Start the mode and make sure those assets load
        self.machine.modes['mode1'].start()
        self.advance_time_and_run()

        # Give it a second to load. This file is tiny, so it shouldn't take
        # this long

        start_time = time.time()
        while (not self.machine.shows['show9'].loaded and
                time.time() < start_time + 5):
            self.assertTrue(self.machine.shows['show9'].loading)
            self.advance_time_and_run(.1)

        self.assertTrue(self.machine.shows['show9'].loaded)
        self.assertFalse(self.machine.shows['show9'].loading)
        self.assertFalse(self.machine.shows['show9'].unloading)

        # test mode stop which should unload those assets
        self.machine.modes['mode1'].stop()
        self.advance_time_and_run(.1)

        start_time = time.time()
        while (self.machine.shows['show9'].loaded and
                time.time() < start_time + 5):
            self.assertTrue(self.machine.shows['show9'].unloading)
            self.advance_time_and_run(.1)

        self.assertFalse(self.machine.shows['show9'].loaded)
        self.assertFalse(self.machine.shows['show9'].loading)
        self.assertFalse(self.machine.shows['show9'].unloading)

    def _test_random_asset_group(self):
        # three assets, no weights

        # make sure the asset group was created
        self.assertIn('group1', self.machine.shows)

        # make sure the randomness is working. To test this, we request the
        # asset 10,000 times and then count the results and assume that each
        # should be 3,333 +- 500 just to make sure the test never fails/
        res = list()
        for x in range(10000):
            res.append(self.machine.shows['group1'].show)

        self.assertAlmostEqual(3333, res.count(self.machine.shows['show1']),
                               delta=500)
        self.assertAlmostEqual(3333, res.count(self.machine.shows['show2']),
                               delta=500)
        self.assertAlmostEqual(3333, res.count(self.machine.shows['show3']),
                               delta=500)

    def _test_random_asset_group_with_weighting(self):
        # three assets, third one has a weight of 2

        # make sure the asset group was created
        self.assertIn('group2', self.machine.shows)

        # make sure the randomness is working. To test this, we request the
        # asset 10,000 times and then count the results and assume that each
        # should be 3,333 +- 500 just to make sure the test never fails/
        res = list()
        for x in range(10000):
            res.append(self.machine.shows['group2'].show)

        self.assertAlmostEqual(2500, res.count(self.machine.shows['show1']),
                               delta=500)
        self.assertAlmostEqual(2500, res.count(self.machine.shows['show2']),
                               delta=500)
        self.assertAlmostEqual(5000, res.count(self.machine.shows['show3']),
                               delta=500)

    def _test_sequence_asset_group(self):
        # three assets, no weights

        self.assertIn('group3', self.machine.shows)

        # Should always return in order, 1, 2, 3, 1, 2, 3...
        self.assertIs(self.machine.shows['group3'].show, self.machine.shows['show1'])
        self.assertIs(self.machine.shows['group3'].show, self.machine.shows['show2'])
        self.assertIs(self.machine.shows['group3'].show, self.machine.shows['show3'])
        self.assertIs(self.machine.shows['group3'].show, self.machine.shows['show1'])
        self.assertIs(self.machine.shows['group3'].show, self.machine.shows['show2'])
        self.assertIs(self.machine.shows['group3'].show, self.machine.shows['show3'])
        self.assertIs(self.machine.shows['group3'].show, self.machine.shows['show1'])
        self.assertIs(self.machine.shows['group3'].show, self.machine.shows['show2'])
        self.assertIs(self.machine.shows['group3'].show, self.machine.shows['show3'])

    def _test_sequence_asset_group_with_count(self):
        # three assets, no weights

        self.assertIn('group4', self.machine.shows)

        # Should always return in order, 1, 1, 1, 1, 2, 2, 3, 1, 1, 1, 1 ...
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show1'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show1'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show1'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show1'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show2'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show2'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show3'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show1'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show1'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show1'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show1'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show2'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show2'])
        self.assertIs(self.machine.shows['group4'].show, self.machine.shows['show3'])

    def _test_random_force_next(self):
        # random, except it ensures the same one does not show up twice in a
        # row

        self.assertIn('group5', self.machine.shows)

        # do it 10,000 times just to be sure. :)
        last = self.machine.shows['group5'].show
        res = list()

        for x in range(10000):
            show = self.machine.shows['group5'].show
            self.assertIsNot(last, show)
            last = show

            res.append(show)

        # Also check that the weights were right

        # BTW these weights are non-intuitive since the last asset is not
        # considered for the next round. e.g. show1 = 1, show2 = 5,
        # show3 = 1, so you'd think they would be 1400, 7200, 1400, but in
        # reality, 50% of the time, asset2 is not in contention, so really
        # asset2 has a 6-to-1 (84%) chance of being selected 66% of the time,
        # but a 0% chance of being selected 33% of the time, etc. So trust that
        # these numbers are right. :)
        self.assertAlmostEqual(2733, res.count(self.machine.shows['show1']),
                               delta=500)
        self.assertAlmostEqual(4533, res.count(self.machine.shows['show2']),
                               delta=500)
        self.assertAlmostEqual(2733, res.count(self.machine.shows['show3']),
                               delta=500)

    def _test_random_force_all(self):
        # random, except it ensures the same one does not show up twice before
        # they're all shown

        self.assertIn('group6', self.machine.shows)

        for x in range(1000):
            this_set = set()
            this_set.add(self.machine.shows['group6'].show)
            this_set.add(self.machine.shows['group6'].show)
            this_set.add(self.machine.shows['group6'].show)

            self.assertEqual(len(this_set), 3)
<|endoftext|>


<commit_before>"""
Filename:     mom_to_cmip.py
Author:       Damien Irving, irving.damien@gmail.com
Description:  convert MOM data files to be like ACCESS-CM2 CMIP6 files

"""

# Import general Python modules

import sys, os, pdb
import argparse
import numpy as np
import xarray as xr
import iris
import cf_units
import cmdline_provenance as cmdprov

# Import my modules

cwd = os.getcwd()
repo_dir = '/'
for directory in cwd.split('/')[1:]:
    repo_dir = os.path.join(repo_dir, directory)
    if directory == 'ocean-analysis':
        break

modules_dir = os.path.join(repo_dir, 'modules')
sys.path.append(modules_dir)

try:
    import general_io as gio
except ImportError:
    raise ImportError('Must run this script from anywhere within the ocean-analysis git repo')

# Define functions

def check_coords(dset, ref_cube, mom_var):
    """Check that the reference coordinates match input file."""

    assert dset[mom_var].shape[1:] == ref_cube.shape[1:], \
        "MOM and reference data do not have same spatial coordinates"

#    in_coords = dset[mom_var].coords._names
#    in_lat = 'geolat_t' if 'geolat_t' in in_coords else 'yt_ocean'
#    in_lon = 'geolon_t' if 'geolon_t' in in_coords else 'xt_ocean'

#    assert np.allclose(ref_cube.coord('latitude').points, dset[in_lat].values)
#    mom_lons = np.where(dset[in_lon].values < 0.0, dset[in_lon].values + 360, dset[in_lon].values)
#    assert np.allclose(ref_cube.coord('longitude').points, mom_lons)

def xr_to_cube(dset, ref_cube, inargs):
    """Create iris cube from xarray dataset"""

    input_units = dset[inargs.mom_var].units
    if input_units[0] == 'W':
        new_units = 'W m-2'
    elif 'kg' in input_units[0:3]:
        new_units = 'kg m-2 s-1'
    elif input_units == 'deg_C':
        new_units = 'degC'
    else:
        raise AttributeError(f'input units not supported: {input_units}')

    mom_ndim = dset[inargs.mom_var].ndim
    check_coords(dset, ref_cube, inargs.mom_var)

    time_coord = iris.coords.DimCoord(dset['time'].values,
                                      standard_name='time',
                                      long_name='time',
                                      var_name='time',
                                      units=cf_units.Unit(dset['time'].units,
                                                          calendar=dset['time'].calendar.lower()))
    dim_coord_list = [(time_coord, 0)]
    for index, coord in enumerate(ref_cube.dim_coords[1:]):
        dim_coord_list.append((coord, index + 1))

    new_data = dset[inargs.mom_var].to_masked_array()
    if inargs.ref_names:
        long_name = ref_cube.long_name
        var_name = ref_cube.var_name
        standard_name = ref_cube.standard_name
    else:
        long_name = dset[inargs.mom_var].long_name
        var_name = dset[inargs.mom_var].name
        if 'standard_name' in dset[inargs.mom_var].attrs:
            standard_name = dset[inargs.mom_var].standard_name
        else:
            standard_name = None

    new_cube = iris.cube.Cube(new_data,
                              long_name=long_name,
                              var_name=var_name,
                              units=new_units,
                              attributes=ref_cube.attributes,
                              dim_coords_and_dims=dim_coord_list,
                              aux_coords_and_dims=[(ref_cube.aux_coords[0], (mom_ndim - 2, mom_ndim - 1)),
                                                   (ref_cube.aux_coords[1], (mom_ndim - 2, mom_ndim - 1))])
    if standard_name in iris.std_names.STD_NAMES:
        new_cube.standard_name = standard_name

    return new_cube

def main(inargs):
    """Run the program."""

    ref_cube = iris.load_cube(inargs.ref_file, inargs.ref_var)
    if inargs.single:
        cube_list = iris.cube.CubeList([])
        for mom_file in inargs.mom_files:
            print(mom_file)
            dset = xr.open_dataset(mom_file, decode_times=False)
            cube = xr_to_cube(dset, ref_cube, inargs)
            cube_list.append(cube)
        outcube = gio.combine_cubes(cube_list)
    else:
        dset = xr.open_mfdataset(inargs.mom_files, decode_times=False, combine='by_coords')
        outcube = xr_to_cube(dset, ref_cube, inargs)

    outcube.attributes['history'] = cmdprov.new_log(git_repo=repo_dir)
    iris.save(outcube, inargs.outfile)

if __name__ == '__main__':

    extra_info ="""

author:
    Damien Irving, irving.damien@gmail.com

"""

    description = 'convert MOM data files to be like ACCESS-CM2 CMIP6 files'
    parser = argparse.ArgumentParser(description=description,
                                     epilog=extra_info,
                                     argument_default=argparse.SUPPRESS,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument("mom_files", type=str, nargs='*', help="Input MOM files")
    parser.add_argument("mom_var", type=str, help="variable name")
    parser.add_argument("ref_file", type=str, help="reference file")
    parser.add_argument("ref_var", type=str, help="reference variable")
    parser.add_argument("outfile", type=str, help="output file")

    parser.add_argument("--single", action="store_true", default=False,
                        help="Process each mom_file separately before merging")
    parser.add_argument("--ref_names", action="store_true", default=False,
                        help="Use the reference file names (var, long and standard) for output file")

    args = parser.parse_args()
    main(args)
<commit_msg>Salinity unit handling<commit_after>"""
Filename:     mom_to_cmip.py
Author:       Damien Irving, irving.damien@gmail.com
Description:  convert MOM data files to be like ACCESS-CM2 CMIP6 files

"""

# Import general Python modules

import sys, os, pdb
import argparse
import numpy as np
import xarray as xr
import iris
import cf_units
import cmdline_provenance as cmdprov

# Import my modules

cwd = os.getcwd()
repo_dir = '/'
for directory in cwd.split('/')[1:]:
    repo_dir = os.path.join(repo_dir, directory)
    if directory == 'ocean-analysis':
        break

modules_dir = os.path.join(repo_dir, 'modules')
sys.path.append(modules_dir)

try:
    import general_io as gio
except ImportError:
    raise ImportError('Must run this script from anywhere within the ocean-analysis git repo')

# Define functions

def check_coords(dset, ref_cube, mom_var):
    """Check that the reference coordinates match input file."""

    assert dset[mom_var].shape[1:] == ref_cube.shape[1:], \
        "MOM and reference data do not have same spatial coordinates"

#    in_coords = dset[mom_var].coords._names
#    in_lat = 'geolat_t' if 'geolat_t' in in_coords else 'yt_ocean'
#    in_lon = 'geolon_t' if 'geolon_t' in in_coords else 'xt_ocean'

#    assert np.allclose(ref_cube.coord('latitude').points, dset[in_lat].values)
#    mom_lons = np.where(dset[in_lon].values < 0.0, dset[in_lon].values + 360, dset[in_lon].values)
#    assert np.allclose(ref_cube.coord('longitude').points, mom_lons)

def xr_to_cube(dset, ref_cube, inargs):
    """Create iris cube from xarray dataset"""

    input_units = dset[inargs.mom_var].units
    if input_units[0] == 'W':
        new_units = 'W m-2'
    elif 'kg' in input_units[0:3]:
        new_units = 'kg m-2 s-1'
    elif input_units == 'deg_C':
        new_units = 'degC'
    elif input_units == 'psu':
        new_units = 'g/kg'
    else:
        raise AttributeError(f'input units not supported: {input_units}')

    mom_ndim = dset[inargs.mom_var].ndim
    check_coords(dset, ref_cube, inargs.mom_var)

    time_coord = iris.coords.DimCoord(dset['time'].values,
                                      standard_name='time',
                                      long_name='time',
                                      var_name='time',
                                      units=cf_units.Unit(dset['time'].units,
                                                          calendar=dset['time'].calendar.lower()))
    dim_coord_list = [(time_coord, 0)]
    for index, coord in enumerate(ref_cube.dim_coords[1:]):
        dim_coord_list.append((coord, index + 1))

    new_data = dset[inargs.mom_var].to_masked_array()
    if inargs.ref_names:
        long_name = ref_cube.long_name
        var_name = ref_cube.var_name
        standard_name = ref_cube.standard_name
    else:
        long_name = dset[inargs.mom_var].long_name
        var_name = dset[inargs.mom_var].name
        if 'standard_name' in dset[inargs.mom_var].attrs:
            standard_name = dset[inargs.mom_var].standard_name
        else:
            standard_name = None

    new_cube = iris.cube.Cube(new_data,
                              long_name=long_name,
                              var_name=var_name,
                              units=new_units,
                              attributes=ref_cube.attributes,
                              dim_coords_and_dims=dim_coord_list,
                              aux_coords_and_dims=[(ref_cube.aux_coords[0], (mom_ndim - 2, mom_ndim - 1)),
                                                   (ref_cube.aux_coords[1], (mom_ndim - 2, mom_ndim - 1))])
    if standard_name in iris.std_names.STD_NAMES:
        new_cube.standard_name = standard_name

    return new_cube

def main(inargs):
    """Run the program."""

    ref_cube = iris.load_cube(inargs.ref_file, inargs.ref_var)
    if inargs.single:
        cube_list = iris.cube.CubeList([])
        for mom_file in inargs.mom_files:
            print(mom_file)
            dset = xr.open_dataset(mom_file, decode_times=False)
            cube = xr_to_cube(dset, ref_cube, inargs)
            cube_list.append(cube)
        outcube = gio.combine_cubes(cube_list)
    else:
        dset = xr.open_mfdataset(inargs.mom_files, decode_times=False, combine='by_coords')
        outcube = xr_to_cube(dset, ref_cube, inargs)

    outcube.attributes['history'] = cmdprov.new_log(git_repo=repo_dir)
    iris.save(outcube, inargs.outfile)

if __name__ == '__main__':

    extra_info ="""

author:
    Damien Irving, irving.damien@gmail.com

"""

    description = 'convert MOM data files to be like ACCESS-CM2 CMIP6 files'
    parser = argparse.ArgumentParser(description=description,
                                     epilog=extra_info,
                                     argument_default=argparse.SUPPRESS,
                                     formatter_class=argparse.RawDescriptionHelpFormatter)

    parser.add_argument("mom_files", type=str, nargs='*', help="Input MOM files")
    parser.add_argument("mom_var", type=str, help="variable name")
    parser.add_argument("ref_file", type=str, help="reference file")
    parser.add_argument("ref_var", type=str, help="reference variable")
    parser.add_argument("outfile", type=str, help="output file")

    parser.add_argument("--single", action="store_true", default=False,
                        help="Process each mom_file separately before merging")
    parser.add_argument("--ref_names", action="store_true", default=False,
                        help="Use the reference file names (var, long and standard) for output file")

    args = parser.parse_args()
    main(args)
<|endoftext|>


<commit_before>#!/bin/python
from __future__ import with_statement
import re
import os
import os.path
import sys
import threading
import calendar
import datetime
import email
import Queue
import mimetypes
import logging
from boto.s3.connection import S3Connection
from assetman.tools import make_output_path, make_absolute_static_path, make_relative_static_path, get_static_pattern, get_shard_from_list

class S3UploadThread(threading.Thread):
    """Thread that knows how to read asset file names from a queue and upload
    them to S3. Any exceptions encountered will be added to a shared errors
    list.

    Each asset will be uploaded twice:  Once with a special "/cdn/" prefix for
    assets to be served by a proxy on our own domain, and again without the
    prefix for assets to be served via CloudFront.  For the second upload,
    static URLs inside each asset will have to be rewritten *again* to point
    at CloudFront instead of our local CDN proxy.
    """

    def __init__(self, queue, errors, manifest, settings):
        threading.Thread.__init__(self)
        cx = S3Connection(settings.get('aws_access_key'), settings.get('aws_secret_key'))
        self.bucket = cx.get_bucket(settings.get('s3_assets_bucket'))
        self.queue = queue
        self.errors = errors
        self.manifest = manifest
        self.settings = settings

    def run(self):
        while True:
            file_name, file_path = self.queue.get()
            try:
                self.start_upload_file(file_name, file_path)
            except Exception, e:
                logging.error('Error uploading %s: %s', file_name, e)
                self.errors.append((sys.exc_info(), self))
            finally:
                self.queue.task_done()

    def start_upload_file(self, file_name, file_path):
        """Starts the procecss of uploading a file to S3. Each file will be
        uploaded twice (once for CDN and once for our local CDN proxy).
        """
        assert isinstance(file_name, (str, unicode))
        assert isinstance(file_path, (str, unicode))
        assert os.path.isfile(file_path)

        content_type, content_encoding = mimetypes.guess_type(file_name)
        if not content_type:
            ext = os.path.splitext(file_name)
            content_type = {
                '.woff': 'application/font-woff',
                '.ttf': 'font/ttf',
                '.otf': 'font/opentype',
                '.eot': 'application/vnd.ms-fontobject'
            }.get(ext, 'application/octet-stream')
        headers = {
            'Content-Type': content_type,
            'Expires': self.get_expires(),
            'Cache-Control': self.get_cache_control(),
            'x-amz-acl': 'public-read',
        }

        with open(file_path, 'rb') as f:
            file_data = f.read()
            # First we will upload the asset for serving via CloudFront CDN,
            # so its S3 key will not have a prefix.
            key = self.bucket.new_key(file_name)
            self.upload_file(key, file_data, headers, for_cdn=True)

            # Next we will upload the same file with a prefixed key, to be
            # served by our "local CDN proxy".
            key_prefix = self.settings.get('local_cdn_url_prefix').lstrip('/').rstrip('/')
            key = self.bucket.new_key(key_prefix + '/' + file_name)
            self.upload_file(key, file_data, headers, for_cdn=False)

    def upload_file(self, key, file_data, headers, for_cdn):
        """Uploads the given file_data to the given S3 key. If the file is a
        compiled asset (ie, JS or CSS file), any static URL references it
        contains will be rewritten before upload.

        If use_cdn is True, static URL references will be updated to point to
        our CloudFront CDN domains. Otherwise, they will be updated to point
        to our local CDN proxy.
        """
        if not key.exists() or self.settings.get('force_s3_upload'):
            # Do we need to do URL replacement?
            if re.search(r'\.(css|js)$', key.name):
                if for_cdn:
                    logging.info('Rewriting URLs => CDN: %s', key.name)
                    replacement_prefix = self.settings.get('cdn_url_prefix')
                else:
                    logging.info('Rewriting URLs => local proxy: %s', key.name)
                    replacement_prefix = self.settings.get('local_cdn_url_prefix')
                file_data = sub_static_version(
                    file_data,
                    self.manifest,
                    replacement_prefix,
                    self.settings['static_dir'],
                    self.settings.get('static_url_prefix'))
            key.set_contents_from_string(file_data, headers, replace=False)
            logging.info('Uploaded %s', key.name)
            logging.debug('Headers: %r', headers)
        else:
            logging.info('Skipping %s; already exists', key.name)

    def get_expires(self):
        # Get a properly formatted date and time, via Tornado's set_header()
        dt = datetime.datetime.utcnow() + datetime.timedelta(days=365*10)
        t = calendar.timegm(dt.utctimetuple())
        return email.utils.formatdate(t, localtime=False, usegmt=True)

    def get_cache_control(self):
        return 'public, max-age=%s' % (86400 * 365 * 10)

def upload_assets_to_s3(manifest, settings, skip_s3_upload=False):
    """Uploads any assets that are in the given manifest and in our compiled
    output dir but missing from our static assets bucket to that bucket on S3.
    """

    # We will gather a set of (file_name, file_path) tuples to be uploaded
    to_upload = set()

    # We know we want to upload each asset block (these correspond to the
    # assetman.include_* blocks in each template)
    for depspec in manifest.blocks.itervalues():
        file_name = depspec['versioned_path']
        file_path = make_output_path(settings['compiled_asset_root'], file_name)
        assert os.path.isfile(file_path), 'Missing compiled asset %s' % file_path
        to_upload.add((file_name, file_path))

    # And we know that we'll want to upload any statically-referenced assets
    # (from assetman.static_url calls or referenced in any compiled assets),
    # but we'll need to filter out other entries in the complete 'assets'
    # block of the manifest.
    should_skip = re.compile(r'\.(scss|less|css|js|html)$', re.I).search
    for rel_path, depspec in manifest.assets.iteritems():
        if should_skip(rel_path):
            continue
        file_path = make_absolute_static_path(settings['static_dir'], rel_path)
        assert os.path.isfile(file_path), 'Missing static asset %s' % file_path
        file_name = depspec['versioned_path']
        to_upload.add((file_name, file_path))

    logging.info('Found %d assets to upload to S3', len(to_upload))
    if skip_s3_upload:
        logging.info('Skipping asset upload to S3 %s', to_upload)
        return to_upload

    # Upload assets to S3 using 5 threads
    queue = Queue.Queue()
    errors = []
    for i in xrange(5):
        uploader = S3UploadThread(queue, errors, manifest, settings)
        uploader.setDaemon(True)
        uploader.start()
    map(queue.put, to_upload)
    queue.join()
    if errors:
        raise Exception(errors)
    return to_upload

def sub_static_version(src, manifest, replacement_prefix, static_dir, static_url_prefix):
    """Adjusts any static URLs in the given source to point to a different
    location.

    Static URLs are determined based on the the 'static_url_prefix' setting.
    They will be updated to point to the given replacement_prefix, which can
    be a string or a list of strings (in which case the actual replacement
    prefix will be chosen by sharding each asset's base name).
    """
    def replacer(match):
        prefix, rel_path = match.groups()
        path = make_relative_static_path(static_dir, rel_path)
        if path in manifest.assets:
            versioned_path = manifest.assets[path]['versioned_path']
            if isinstance(replacement_prefix, (list, tuple)):
                prefix = get_shard_from_list(replacement_prefix, versioned_path)
            else:
                prefix = replacement_prefix
            return prefix.rstrip('/') + '/' + versioned_path.lstrip('/')
        logging.warn('Missing path %s in manifest, using %s', path, match.group(0))
        return match.group(0)
    pattern = get_static_pattern(static_url_prefix)
    return re.sub(pattern, replacer, src)
<commit_msg>better logging on URL replacements<commit_after>#!/bin/python
from __future__ import with_statement
import re
import os
import os.path
import sys
import threading
import calendar
import datetime
import email
import Queue
import mimetypes
import logging
from boto.s3.connection import S3Connection
from assetman.tools import make_output_path, make_absolute_static_path, make_relative_static_path, get_static_pattern, get_shard_from_list

class S3UploadThread(threading.Thread):
    """Thread that knows how to read asset file names from a queue and upload
    them to S3. Any exceptions encountered will be added to a shared errors
    list.

    Each asset will be uploaded twice:  Once with a special "/cdn/" prefix for
    assets to be served by a proxy on our own domain, and again without the
    prefix for assets to be served via CloudFront.  For the second upload,
    static URLs inside each asset will have to be rewritten *again* to point
    at CloudFront instead of our local CDN proxy.
    """

    def __init__(self, queue, errors, manifest, settings):
        threading.Thread.__init__(self)
        cx = S3Connection(settings.get('aws_access_key'), settings.get('aws_secret_key'))
        self.bucket = cx.get_bucket(settings.get('s3_assets_bucket'))
        self.queue = queue
        self.errors = errors
        self.manifest = manifest
        self.settings = settings

    def run(self):
        while True:
            file_name, file_path = self.queue.get()
            try:
                self.start_upload_file(file_name, file_path)
            except Exception, e:
                logging.error('Error uploading %s: %s', file_name, e)
                self.errors.append((sys.exc_info(), self))
            finally:
                self.queue.task_done()

    def start_upload_file(self, file_name, file_path):
        """Starts the procecss of uploading a file to S3. Each file will be
        uploaded twice (once for CDN and once for our local CDN proxy).
        """
        assert isinstance(file_name, (str, unicode))
        assert isinstance(file_path, (str, unicode))
        assert os.path.isfile(file_path)

        content_type, content_encoding = mimetypes.guess_type(file_name)
        if not content_type:
            ext = os.path.splitext(file_name)
            content_type = {
                '.woff': 'application/font-woff',
                '.ttf': 'font/ttf',
                '.otf': 'font/opentype',
                '.eot': 'application/vnd.ms-fontobject'
            }.get(ext, 'application/octet-stream')
        headers = {
            'Content-Type': content_type,
            'Expires': self.get_expires(),
            'Cache-Control': self.get_cache_control(),
            'x-amz-acl': 'public-read',
        }

        with open(file_path, 'rb') as f:
            file_data = f.read()
            # First we will upload the asset for serving via CloudFront CDN,
            # so its S3 key will not have a prefix.
            key = self.bucket.new_key(file_name)
            self.upload_file(key, file_data, headers, for_cdn=True)

            # Next we will upload the same file with a prefixed key, to be
            # served by our "local CDN proxy".
            key_prefix = self.settings.get('local_cdn_url_prefix').lstrip('/').rstrip('/')
            key = self.bucket.new_key(key_prefix + '/' + file_name)
            self.upload_file(key, file_data, headers, for_cdn=False)

    def upload_file(self, key, file_data, headers, for_cdn):
        """Uploads the given file_data to the given S3 key. If the file is a
        compiled asset (ie, JS or CSS file), any static URL references it
        contains will be rewritten before upload.

        If use_cdn is True, static URL references will be updated to point to
        our CloudFront CDN domains. Otherwise, they will be updated to point
        to our local CDN proxy.
        """
        if not key.exists() or self.settings.get('force_s3_upload'):
            # Do we need to do URL replacement?
            if re.search(r'\.(css|js)$', key.name):
                if for_cdn:
                    logging.info('Rewriting URLs => CDN in %s', key.name)
                    replacement_prefix = self.settings.get('cdn_url_prefix')
                else:
                    logging.info('Rewriting URLs => local proxy in %s', key.name)
                    replacement_prefix = self.settings.get('local_cdn_url_prefix')
                file_data = sub_static_version(
                    file_data,
                    self.manifest,
                    replacement_prefix,
                    self.settings['static_dir'],
                    self.settings.get('static_url_prefix'))
            key.set_contents_from_string(file_data, headers, replace=self.settings.get('force_s3_upload', False))
            logging.info('Uploaded %s', key.name)
            logging.debug('Headers: %r', headers)
        else:
            logging.info('Skipping upload of %s; already exists (use force_s3_upload to override)', key.name)

    def get_expires(self):
        # Get a properly formatted date and time, via Tornado's set_header()
        dt = datetime.datetime.utcnow() + datetime.timedelta(days=365*10)
        t = calendar.timegm(dt.utctimetuple())
        return email.utils.formatdate(t, localtime=False, usegmt=True)

    def get_cache_control(self):
        return 'public, max-age=%s' % (86400 * 365 * 10)

def upload_assets_to_s3(manifest, settings, skip_s3_upload=False):
    """Uploads any assets that are in the given manifest and in our compiled
    output dir but missing from our static assets bucket to that bucket on S3.
    """

    # We will gather a set of (file_name, file_path) tuples to be uploaded
    to_upload = set()

    # We know we want to upload each asset block (these correspond to the
    # assetman.include_* blocks in each template)
    for depspec in manifest.blocks.itervalues():
        file_name = depspec['versioned_path']
        file_path = make_output_path(settings['compiled_asset_root'], file_name)
        assert os.path.isfile(file_path), 'Missing compiled asset %s' % file_path
        to_upload.add((file_name, file_path))

    # And we know that we'll want to upload any statically-referenced assets
    # (from assetman.static_url calls or referenced in any compiled assets),
    # but we'll need to filter out other entries in the complete 'assets'
    # block of the manifest.
    should_skip = re.compile(r'\.(scss|less|css|js|html)$', re.I).search
    for rel_path, depspec in manifest.assets.iteritems():
        if should_skip(rel_path):
            continue
        file_path = make_absolute_static_path(settings['static_dir'], rel_path)
        assert os.path.isfile(file_path), 'Missing static asset %s' % file_path
        file_name = depspec['versioned_path']
        to_upload.add((file_name, file_path))

    logging.info('Found %d assets to upload to S3', len(to_upload))
    if skip_s3_upload:
        logging.info('Skipping asset upload to S3 %s', to_upload)
        return to_upload

    # Upload assets to S3 using 5 threads
    queue = Queue.Queue()
    errors = []
    for i in xrange(5):
        uploader = S3UploadThread(queue, errors, manifest, settings)
        uploader.setDaemon(True)
        uploader.start()
    map(queue.put, to_upload)
    queue.join()
    if errors:
        raise Exception(errors)
    return to_upload

def sub_static_version(src, manifest, replacement_prefix, static_dir, static_url_prefix):
    """Adjusts any static URLs in the given source to point to a different
    location.

    Static URLs are determined based on the the 'static_url_prefix' setting.
    They will be updated to point to the given replacement_prefix, which can
    be a string or a list of strings (in which case the actual replacement
    prefix will be chosen by sharding each asset's base name).
    """
    def replacer(match):
        prefix, rel_path = match.groups()
        path = make_relative_static_path(static_dir, rel_path)
        if path in manifest.assets:
            versioned_path = manifest.assets[path]['versioned_path']
            if isinstance(replacement_prefix, (list, tuple)):
                prefix = get_shard_from_list(replacement_prefix, versioned_path)
            else:
                prefix = replacement_prefix
            replacement_link = prefix.rstrip('/') + '/' + versioned_path.lstrip('/')
            logging.info('replacing %s -> %s', path, replacement_link)
            return replacement_link
        logging.warn('Missing path %s in manifest, using %s', path, match.group(0))
        return match.group(0)
    pattern = get_static_pattern(static_url_prefix)
    return re.sub(pattern, replacer, src)
<|endoftext|>


<commit_before>from __future__ import print_function
import time
import threading
from sys import stderr

import pcapy

class Capturer(threading.Thread):
    def __init__(self, device, snaplen, promisc=False,
                 to_ms=0, filename="capture_data", cap_filter=None):
        threading.Thread.__init__(self)
        self.setDaemon(True)
        self.device = device
        self.snaplen = snaplen
        self.promisc = promisc
        self.to_ms = to_ms
        self.filename = filename
        self.cap_filter = cap_filter
        self.stop = False
        self.dumper = None

    def run(self):
        self.start_capture()

    def start_capture(self):
        self.stop = False
        pc = pcapy.open_live(self.device, self.snaplen, self.promisc, self.to_ms)
        if self.cap_filter is not None:
            pc.setfilter(self.cap_filter)
        self.dumper = pc.dump_open(self.filename)
        while not self.stop:
            try:
                header, data = pc.next()
                self._handle_packet(header, data)
            except pcapy.PcapError as e:
                print("[Warning] An error occurs when capturing a packet: {}".format(e.message),
                      file=stderr)
        print("Capture stopped.")

    def stop_capture(self):
        self.stop = True

    def _handle_packet(self, header, data):
        print("A new packet captured - timestamp: {}; capture length: {}; total length: {}".format(
            header.getts(), header.getcaplen(), header.getlen()))
        self.dumper.dump(header, data)

if __name__ == '__main__':
    cap = Capturer('en0', snaplen=65535, to_ms=3000)
    cap.start()
    time.sleep(5)
    cap.stop_capture()
    cap.join()
<commit_msg>change default filename of captured data<commit_after>from __future__ import print_function
import time
import threading
from sys import stderr

import pcapy

class Capturer(threading.Thread):
    def __init__(self, device, snaplen, promisc=False,
                 to_ms=0, filename="capture_data.pcap", cap_filter=None):
        threading.Thread.__init__(self)
        self.setDaemon(True)
        self.device = device
        self.snaplen = snaplen
        self.promisc = promisc
        self.to_ms = to_ms
        self.filename = filename
        self.cap_filter = cap_filter
        self.stop = False
        self.dumper = None

    def run(self):
        self.start_capture()

    def start_capture(self):
        self.stop = False
        pc = pcapy.open_live(self.device, self.snaplen, self.promisc, self.to_ms)
        if self.cap_filter is not None:
            pc.setfilter(self.cap_filter)
        self.dumper = pc.dump_open(self.filename)
        while not self.stop:
            try:
                header, data = pc.next()
                self._handle_packet(header, data)
            except pcapy.PcapError as e:
                print("[Warning] An error occurs when capturing a packet: {}".format(e.message),
                      file=stderr)
        print("Capture stopped.")

    def stop_capture(self):
        self.stop = True

    def _handle_packet(self, header, data):
        print("A new packet captured - timestamp: {}; capture length: {}; total length: {}".format(
            header.getts(), header.getcaplen(), header.getlen()))
        self.dumper.dump(header, data)

if __name__ == '__main__':
    cap = Capturer('en0', snaplen=65535, to_ms=3000)
    cap.start()
    time.sleep(5)
    cap.stop_capture()
    cap.join()
<|endoftext|>


<commit_before>"""
=====================
The Component Manager
=====================

The ``vivarium`` component manager system is responsible for maintaining a
reference to all of the managers and components in a simulation, as well as
initiating the ``setup`` stage of their lifecycle. This module provides the
default implementation and interface.

TODO: note about letting components add to and ask for components

The :class:`ComponentManager` is the first plugin loaded by the :class:`SimulationContext`,
and managers and components are given to it by the context. It is called on to setup
everything it holds when the context itself is setup.

"""
import inspect
from typing import Union, List, Tuple, Any

from vivarium import VivariumError, ConfigTree

class ComponentConfigError(VivariumError):
    """Error while interpreting configuration file or initializing components"""
    pass

class ComponentManager:
    """Maintains references to all components and managers in a ``vivarium``
    simulation and is responsible for applying their default configuration and
    initiating their ``setup`` life-cycle stage.

    """

    def __init__(self):
        self._managers = []
        self._components = []

    def add_managers(self, managers: Union[List, Tuple, Any]):
        """Registers new managers with the component manager. Managers are
        configured and setup before components.

        Parameters
        ----------
        managers:
          Instantiated managers to register.
        """
        self._add_components(self._managers, managers)

    def add_components(self, components: Union[List, Tuple, Any]):
        """Register new components with the component manager. Components are
        configured and setup after managers.

        Parameters
        ----------
        components:
          Instantiated components to register.
        """
        self._add_components(self._components, components)

    def _add_components(self, component_list: Union[List, Tuple], components: Union[List, Tuple, Any]):
        for component in components:
            if isinstance(component, list) or isinstance(component, tuple):
                self._add_components(component_list, component)
            else:
                component_list.append(component)

    def get_components(self, component_type: Any) -> List:
        """Gets a list of components currently held by the component manager
        that are instances of a specific type. Does not include managers.

        Parameters
        ----------
        component_type
            A component type.
        Returns
        -------
            A list of components.
        """
        return [c for c in self._components if isinstance(c, component_type)]

    def setup_components(self, builder, configuration: ConfigTree):
        """Apply default configurations to the global configuration and run
        component-level configuration defaults to the global
        configuration and run setup methods. Runs first on managers, then on
        components.

        This can result in new components due to side effects of a component's
        setup function.

        Parameters
        ----------
        builder:
            Interface to several simulation tools.
        configuration:
            Simulation configuration parameters.
        """
        self._managers = setup_components(builder, self._managers, configuration)
        self._components = setup_components(builder, self._components, configuration)

class ComponentInterface:
    """The builder interface for the component manager system. This class
    defines the methods other components can access from the builder. It
    provides methods for querying and adding components to the simulation.

    """

    def __init__(self, component_manager: ComponentManager):
        self._component_manager = component_manager

    def add_components(self, components: Union[List, Tuple, Any]):
        """Register new components with the component manager. Components are
        configured  and setup after managers.

        Parameters
        ----------
        components:
          Instantiated components to register.

        """
        self._component_manager.add_components(components)

    def get_components(self, component_type: Any) -> List:
        """Get all components that are an instance of ``component_type``
        currently held by the component manager.

        Parameters
        ----------
        component_type
            A component type to get, compared against internal components
            using isinstance().
        Returns
        -------
            A list of components.

        """
        return self._component_manager.get_components(component_type)

def setup_components(builder, component_list: Union[List, Tuple], configuration: ConfigTree) -> List:
    """Configures and sets up a list of components or managers.

    This function first loops over ``component_list`` and applies configuration
    defaults if present, modifying the ``configuration`` object in the process.
    Then, the ``component_list`` is looped over until empty, with components
    being popped and setup. Because of the side effect described below, new
    components can be added to the list. This necessitates a while loop and a
    check against unconfigured components.

    Importantly, this function is called by the :class:`ComponentManager` and it
    is passed a reference to one of the lists that manager holds. This list is
    also accessible through the :class:`ComponentInterface` on the builder, so a
    component's setup method can mutate the list.

    Parameters
    ----------
    builder
        Interface to several simulation tools.
    component_list
        A list of vivarium components.
    configuration
        A vivarium configuration object.

    Returns
    -------
        A list of components which have been configured and setup.
    """
    configured = []
    for c in component_list:  # apply top-level configurations first
        if hasattr(c, "configuration_defaults") and c not in configured:
            apply_component_default_configuration(configuration, c)
            configured.append(c)

    setup = []
    while component_list:  # mutated at runtime by calls to setup
        c = component_list.pop(0)
        if c is None:
            raise ComponentConfigError('None in component list. This likely '
                                       'indicates a bug in a factory function')

        if hasattr(c, "configuration_defaults") and c not in configured:
            apply_component_default_configuration(configuration, c)
            configured.append(c)

        if c not in setup:
            if hasattr(c, "setup"):
                c.setup(builder)
            setup.append(c)

    return setup

def apply_component_default_configuration(configuration: ConfigTree, component: Any):
    """Checks if a default configuration attached to a component is duplicated
    in the simulation configuration, which will raise. If it is not, the
    configuration is applied.

    Parameters
    ----------
    configuration
        A vivarium configuration object.
    component
        A vivarium component.

    """

    # This reapplies configuration from some components but it is idempotent so there's no effect.
    if component.__module__ == '__main__':
        # This is defined directly in a script or notebook so there's no file to attribute it to
        source = '__main__'
    else:
        source = inspect.getfile(component.__class__)
    check_duplicated_default_configuration(component.configuration_defaults, configuration, source)
    configuration.update(component.configuration_defaults, layer='component_configs', source=source)

def check_duplicated_default_configuration(component: Any, config: ConfigTree, source: str):
    """Checks that the keys present in a component's default configuration are
    not already present in the configtree. Source is just for making a cogent
    error message.

    Parameters
    ----------
    component
        A vivarium component.
    config
        A vivarium configuration object.
    source
        The file containing the code that describes the component.

    Raises
    -------
    ComponentConfigError
        A component's default configuration is already present in the config tree.

    """
    overlapped = set(component.keys()).intersection(config.keys())
    if not overlapped:
        pass

    while overlapped:
        key = overlapped.pop()

        try:
            sub_config = config.get_from_layer(key, layer='component_configs')
            sub_component = component[key]

            if isinstance(sub_component, dict) and isinstance(sub_config, ConfigTree):
                check_duplicated_default_configuration(sub_component, sub_config, source)
            elif isinstance(sub_component, dict) or isinstance(sub_config, ConfigTree):
                raise ComponentConfigError(f'These two sources have different structure of configurations for {component}.'
                                           f' Check {source} and {sub_config}')
            else:
                raise ComponentConfigError(f'Check these two {source} and {config._children[key].get_value_with_source()}'
                                           f'Both try to set the default configurations for {component}/{key}')

        except KeyError:
            pass
<commit_msg>more docstring edits<commit_after>"""
============================
The Component Manager System
============================
The ``vivarium`` component manager system is responsible for maintaining a
reference to all of the managers and components in a simulation, providing an
interface for adding additional components or managers, and applying default
configurations and initiating the ``setup`` stage of the lifecycle. This module
provides the default implementation and interface.

The :class:`ComponentManager` is the first plugin loaded by the
:class:`SimulationContext`[vivarium.framework.engine.SimulationContext],
and managers and components are given to it by the context. It is called on to
setup everything it holds when the context itself is setup.

"""
import inspect
from typing import Union, List, Tuple, Any

from vivarium import VivariumError, ConfigTree

class ComponentConfigError(VivariumError):
    """Error while interpreting configuration file or initializing components"""
    pass

class ComponentManager:
    """Maintains references to all components and managers in a ``vivarium``
    simulation, applies their default configuration and initiates their
    ``setup`` life-cycle stage.

    The component manager maintains a separate list of managers and components
    and provides methods for adding to these lists and getting members that
    correspond to a specific type.  It also initiates the ``setup`` lifecycle
    phase for all components and managers it controls. This is done first for
    managers and then components, and involves applying default configurations
    and calling the object's ``setup`` method.

    """

    def __init__(self):
        self._managers = []
        self._components = []

    def add_managers(self, managers: Union[List, Tuple, Any]):
        """Registers new managers with the component manager. Managers are
        configured and setup before components.

        Parameters
        ----------
        managers:
          Instantiated managers to register.
        """
        self._add_components(self._managers, managers)

    def add_components(self, components: Union[List, Tuple, Any]):
        """Register new components with the component manager. Components are
        configured and setup after managers.

        Parameters
        ----------
        components:
          Instantiated components to register.
        """
        self._add_components(self._components, components)

    def _add_components(self, component_list: Union[List, Tuple], components: Union[List, Tuple, Any]):
        for component in components:
            if isinstance(component, list) or isinstance(component, tuple):
                self._add_components(component_list, component)
            else:
                component_list.append(component)

    def get_components(self, component_type: Any) -> List:
        """Gets a list of components currently held by the component manager
        that are instances of a specific type. Does not include managers.

        Parameters
        ----------
        component_type
            A component type.
        Returns
        -------
            A list of components.
        """
        return [c for c in self._components if isinstance(c, component_type)]

    def setup_components(self, builder, configuration: ConfigTree):
        """Separately configure and set up the managers and components held by
        the component manager, in that order.

        The setup process involves applying default configurations and then
        calling the manager or component's setup method. This can result in new
        components as a side effect of setup because components themselves have
        access to this interface through the builder in their setup method.

        Parameters
        ----------
        builder:
            Interface to several simulation tools.
        configuration:
            Simulation configuration parameters.
        """
        self._managers = setup_components(builder, self._managers, configuration)
        self._components = setup_components(builder, self._components, configuration)

class ComponentInterface:
    """The builder interface for the component manager system. This class
    defines component manager methods a ``vivarium`` component can access from
    the builder. It provides methods for querying and adding components to the
    :class:`ComponentManager`.

    """

    def __init__(self, component_manager: ComponentManager):
        self._component_manager = component_manager

    def add_components(self, components: Union[List, Tuple, Any]):
        """Register new components with the component manager. Components are
        configured  and setup after managers.

        Parameters
        ----------
        components:
          Instantiated components to register.

        """
        self._component_manager.add_components(components)

    def get_components(self, component_type: Any) -> List:
        """Get all components that are an instance of ``component_type``
        currently held by the component manager.

        Parameters
        ----------
        component_type
            A component type to get, compared against internal components
            using isinstance().
        Returns
        -------
            A list of components.

        """
        return self._component_manager.get_components(component_type)

def setup_components(builder, component_list: Union[List, Tuple], configuration: ConfigTree) -> List:
    """Configure and set up a list of components or managers.

    This function first loops over ``component_list`` and applies configuration
    defaults if present, modifying the ``configuration`` object in the process.
    Then, the ``component_list`` is looped over until empty, with components
    being popped and setup. Because of the side effect described below, new
    components can be added to the list. This necessitates a while loop and a
    check against unconfigured components.

    Importantly, this function is called by the :class:`ComponentManager` and it
    is passed a reference to one of the lists that manager holds. This list is
    also accessible through the :class:`ComponentInterface` on the builder, so a
    component's setup method can mutate the list.

    Parameters
    ----------
    builder
        Interface to several simulation tools.
    component_list
        A list of vivarium components.
    configuration
        A vivarium configuration object.

    Returns
    -------
        A list of components which have been configured and setup.
    """
    configured = []
    for c in component_list:  # apply top-level configurations first
        if hasattr(c, "configuration_defaults") and c not in configured:
            apply_component_default_configuration(configuration, c)
            configured.append(c)

    setup = []
    while component_list:  # mutated at runtime by calls to setup
        c = component_list.pop(0)
        if c is None:
            raise ComponentConfigError('None in component list. This likely '
                                       'indicates a bug in a factory function')

        if hasattr(c, "configuration_defaults") and c not in configured:
            apply_component_default_configuration(configuration, c)
            configured.append(c)

        if c not in setup:
            if hasattr(c, "setup"):
                c.setup(builder)
            setup.append(c)

    return setup

def apply_component_default_configuration(configuration: ConfigTree, component: Any):
    """Check if a default configuration attached to a component is duplicated
    in the simulation configuration, which will raise. If it is not, apply the
    configuration.

    Parameters
    ----------
    configuration
        A vivarium configuration object.
    component
        A vivarium component.

    """

    # This reapplies configuration from some components but it is idempotent so there's no effect.
    if component.__module__ == '__main__':
        # This is defined directly in a script or notebook so there's no file to attribute it to
        source = '__main__'
    else:
        source = inspect.getfile(component.__class__)
    check_duplicated_default_configuration(component.configuration_defaults, configuration, source)
    configuration.update(component.configuration_defaults, layer='component_configs', source=source)

def check_duplicated_default_configuration(component: Any, config: ConfigTree, source: str):
    """Check that the keys present in a component's default configuration
    ``component`` are not already present in the global configtree ``config``.

    Parameters
    ----------
    component
        A vivarium component.
    config
        A vivarium configuration object.
    source
        The file containing the code that describes the component. Only used
        to generate a cogent error message.

    Raises
    -------
    ComponentConfigError
        A component's default configuration is already present in the config tree.

    """
    overlapped = set(component.keys()).intersection(config.keys())
    if not overlapped:
        pass

    while overlapped:
        key = overlapped.pop()

        try:
            sub_config = config.get_from_layer(key, layer='component_configs')
            sub_component = component[key]

            if isinstance(sub_component, dict) and isinstance(sub_config, ConfigTree):
                check_duplicated_default_configuration(sub_component, sub_config, source)
            elif isinstance(sub_component, dict) or isinstance(sub_config, ConfigTree):
                raise ComponentConfigError(f'These two sources have different structure of configurations for {component}.'
                                           f' Check {source} and {sub_config}')
            else:
                raise ComponentConfigError(f'Check these two {source} and {config._children[key].get_value_with_source()}'
                                           f'Both try to set the default configurations for {component}/{key}')

        except KeyError:
            pass
<|endoftext|>


<commit_before>"""Contains classes which are used to process config files for the media
controller.

"""

import logging
from mpf.core.config_processor import ConfigProcessor as ConfigProcessorBase

# from mpfmc.uix.display import Display is imported deeper in this file
# we need it there because if we do a standard import, the Clock is created
# before we have a chance to read the config and set the maxfps

class ConfigProcessor(ConfigProcessorBase):
    def __init__(self, machine):
        self.mc = machine
        self.machine = machine
        self.system_config = self.mc.machine_config['mpf-mc']
        self.log = logging.getLogger('ConfigProcessor')
        self.machine_sections = None
        self.mode_sections = None

        self.machine_sections = dict()
        self.mode_sections = dict()

        # process mode-based and machine-wide configs
        self.register_load_methods()

        self.mc.events.add_handler('init_phase_1', self._init)

        # todo need to clean this up
        try:
            self.process_displays(config=self.mc.machine_config['displays'])
        except KeyError:
            pass

        if not self.mc.displays:
            from mpfmc.uix.display import Display
            Display.create_default_display(self.mc)

    def _init(self):
        self.process_config_file(section_dict=self.machine_sections,
                                 config=self.mc.machine_config)

    def process_displays(self, config):
        # config is localized to 'displays' section
        for display, settings in config.items():
            self.mc.displays[display] = self.create_display(display, settings)

    def create_display(self, name, config):
        # config is localized display settings
        from mpfmc.uix.display import Display
        return Display(self.mc, name,
            **self.machine.config_validator.validate_config('displays',
                                                            config))
<commit_msg>move code from MPF to MPF-MC which is only needed in MC<commit_after>"""Contains classes which are used to process config files for the media
controller.

"""

import logging
from mpf.core.config_processor import ConfigProcessor as ConfigProcessorBase

# from mpfmc.uix.display import Display is imported deeper in this file
# we need it there because if we do a standard import, the Clock is created
# before we have a chance to read the config and set the maxfps

class ConfigProcessor(ConfigProcessorBase):

    config_spec = None

    def __init__(self, machine):
        self.mc = machine
        self.machine = machine
        self.system_config = self.mc.machine_config['mpf-mc']
        self.log = logging.getLogger('ConfigProcessor')
        self.machine_sections = None
        self.mode_sections = None

        self.machine_sections = dict()
        self.mode_sections = dict()

        # process mode-based and machine-wide configs
        self.register_load_methods()

        self.mc.events.add_handler('init_phase_1', self._init)

        # todo need to clean this up
        try:
            self.process_displays(config=self.mc.machine_config['displays'])
        except KeyError:
            pass

        if not self.mc.displays:
            from mpfmc.uix.display import Display
            Display.create_default_display(self.mc)

    def _init(self):
        self.process_config_file(section_dict=self.machine_sections,
                                 config=self.mc.machine_config)

    def register_load_methods(self):
        """Register load method for modes."""
        for section in self.mode_sections:
            self.machine.mode_controller.register_load_method(
                load_method=self.process_mode_config,
                config_section_name=section, section=section)

    def process_config_file(self, section_dict, config):
        """Called to process a config file (can be a mode or machine config)."""
        for section in section_dict:
            if section in section_dict and section in config:
                self.process_localized_config_section(config=config[section],
                                                      section=section)

    def process_localized_config_section(self, config, section):
        """Process a single key within a config file.

        Args:
            config: The subsection of a config dict to process
            section: The name of the section, either 'scripts' or 'shows'.

        """
        self.machine_sections[section](config)

    def process_mode_config(self, config, mode, mode_path, section, **kwargs):
        """Process a mode config."""
        del mode
        del mode_path
        del kwargs
        self.process_localized_config_section(config, section)

    def process_displays(self, config):
        # config is localized to 'displays' section
        for display, settings in config.items():
            self.mc.displays[display] = self.create_display(display, settings)

    def create_display(self, name, config):
        # config is localized display settings
        from mpfmc.uix.display import Display
        return Display(self.mc, name,
            **self.machine.config_validator.validate_config('displays',
                                                            config))
<|endoftext|>