<commit_before>import pytest
import yaml

from cekit.descriptor.base import _merge_descriptors, _merge_lists
from cekit.descriptor import Descriptor, Image, Module, Overrides, Run
from cekit.errors import CekitError

class TestDescriptor(Descriptor):
    def __init__(self, descriptor):
        self.schemas = [yaml.safe_load("""type: any""")]
        super(TestDescriptor, self).__init__(descriptor)

        for key, val in descriptor.items():
            if isinstance(val, dict):
                self._descriptor[key] = TestDescriptor(val)

def test_merging_description_image():
    desc1 = Image({'name': 'foo', 'version': 1}, None)

    desc2 = Module({'name': 'mod1',
                    'description': 'mod_desc'}, None, None)

    merged = _merge_descriptors(desc1, desc2)
    assert 'description' not in merged

def test_merging_description_modules():
    desc1 = Module({'name': 'foo'}, None, None)

    desc2 = Module({'name': 'mod1',
                    'description': 'mod_desc'}, None, None)

    merged = _merge_descriptors(desc1, desc2)
    assert 'description' not in merged

def test_merging_description_override():
    desc1 = Image({'name': 'foo', 'version': 1}, None)

    desc2 = Overrides({'name': 'mod1',
                       'description': 'mod_desc'}, None)

    merged = _merge_descriptors(desc2, desc1)
    assert 'description' in merged

def test_merging_plain_descriptors():
    desc1 = TestDescriptor({'name': 'foo',
                            'a': 1,
                            'b': 2})

    desc2 = TestDescriptor({'name': 'foo',
                            'b': 5,
                            'c': 3})

    expected = TestDescriptor({'name': 'foo',
                               'a': 1,
                               'b': 2,
                               'c': 3})
    assert expected == _merge_descriptors(desc1, desc2)
    assert expected.items() == _merge_descriptors(desc1, desc2).items()

def test_merging_emdedded_descriptors():
    desc1 = TestDescriptor({'name': 'a',
                            'a': 1,
                            'b': {'name': 'b',
                                  'b1': 10,
                                  'b2': 20}})
    desc2 = TestDescriptor({'b': {'name': 'b',
                                  'b2': 50,
                                  'b3': 30},
                            'c': {'name': 'c'}})

    expected = TestDescriptor({'name': 'a',
                               'a': 1,
                               'b': {'name': 'b',
                                     'b1': 10,
                                     'b2': 20,
                                     'b3': 30},
                               'c': {'name': 'c'}})

    assert expected == _merge_descriptors(desc1, desc2)

def test_merging_plain_lists():
    list1 = [2, 3, 4, 5]
    list2 = [1, 2, 3]
    expected = [1, 2, 3, 4, 5]
    assert _merge_lists(list1, list2) == expected

def test_merging_plain_list_of_list():
    list1 = [1, 2, 3]
    list2 = [3, 4, []]
    with pytest.raises(CekitError):
        _merge_lists(list1, list2)

def test_merging_list_of_descriptors():
    desc1 = [TestDescriptor({'name': 1,
                             'a': 1,
                             'b': 2})]

    desc2 = [TestDescriptor({'name': 2,
                             'a': 123}),
             TestDescriptor({'name': 1,
                             'b': 3,
                             'c': 3})]

    expected = [TestDescriptor({'name': 2,
                                'a': 123}),
                TestDescriptor({'name': 1,
                                'a': 1,
                                'b': 2,
                                'c': 3})]

    assert expected == _merge_lists(desc1, desc2)

def test_merge_run_cmd():
    run1 = Run({'user': 'foo', 'cmd': ['a', 'b', 'c'], 'entrypoint': ['a', 'b']})
    run2 = Run({'user': 'foo', 'cmd': ['1', '2', '3'], 'entrypoint': ['1', '2']})

    run1.merge(run2)
    assert run1['cmd'] == ['a', 'b', 'c']
    assert run1['entrypoint'] == ['a', 'b']
<commit_msg>renamed run1/run2 in tests for better readability<commit_after>import pytest
import yaml

from cekit.descriptor.base import _merge_descriptors, _merge_lists
from cekit.descriptor import Descriptor, Image, Module, Overrides, Run
from cekit.errors import CekitError

class TestDescriptor(Descriptor):
    def __init__(self, descriptor):
        self.schemas = [yaml.safe_load("""type: any""")]
        super(TestDescriptor, self).__init__(descriptor)

        for key, val in descriptor.items():
            if isinstance(val, dict):
                self._descriptor[key] = TestDescriptor(val)

def test_merging_description_image():
    desc1 = Image({'name': 'foo', 'version': 1}, None)

    desc2 = Module({'name': 'mod1',
                    'description': 'mod_desc'}, None, None)

    merged = _merge_descriptors(desc1, desc2)
    assert 'description' not in merged

def test_merging_description_modules():
    desc1 = Module({'name': 'foo'}, None, None)

    desc2 = Module({'name': 'mod1',
                    'description': 'mod_desc'}, None, None)

    merged = _merge_descriptors(desc1, desc2)
    assert 'description' not in merged

def test_merging_description_override():
    desc1 = Image({'name': 'foo', 'version': 1}, None)

    desc2 = Overrides({'name': 'mod1',
                       'description': 'mod_desc'}, None)

    merged = _merge_descriptors(desc2, desc1)
    assert 'description' in merged

def test_merging_plain_descriptors():
    desc1 = TestDescriptor({'name': 'foo',
                            'a': 1,
                            'b': 2})

    desc2 = TestDescriptor({'name': 'foo',
                            'b': 5,
                            'c': 3})

    expected = TestDescriptor({'name': 'foo',
                               'a': 1,
                               'b': 2,
                               'c': 3})
    assert expected == _merge_descriptors(desc1, desc2)
    assert expected.items() == _merge_descriptors(desc1, desc2).items()

def test_merging_emdedded_descriptors():
    desc1 = TestDescriptor({'name': 'a',
                            'a': 1,
                            'b': {'name': 'b',
                                  'b1': 10,
                                  'b2': 20}})
    desc2 = TestDescriptor({'b': {'name': 'b',
                                  'b2': 50,
                                  'b3': 30},
                            'c': {'name': 'c'}})

    expected = TestDescriptor({'name': 'a',
                               'a': 1,
                               'b': {'name': 'b',
                                     'b1': 10,
                                     'b2': 20,
                                     'b3': 30},
                               'c': {'name': 'c'}})

    assert expected == _merge_descriptors(desc1, desc2)

def test_merging_plain_lists():
    list1 = [2, 3, 4, 5]
    list2 = [1, 2, 3]
    expected = [1, 2, 3, 4, 5]
    assert _merge_lists(list1, list2) == expected

def test_merging_plain_list_of_list():
    list1 = [1, 2, 3]
    list2 = [3, 4, []]
    with pytest.raises(CekitError):
        _merge_lists(list1, list2)

def test_merging_list_of_descriptors():
    desc1 = [TestDescriptor({'name': 1,
                             'a': 1,
                             'b': 2})]

    desc2 = [TestDescriptor({'name': 2,
                             'a': 123}),
             TestDescriptor({'name': 1,
                             'b': 3,
                             'c': 3})]

    expected = [TestDescriptor({'name': 2,
                                'a': 123}),
                TestDescriptor({'name': 1,
                                'a': 1,
                                'b': 2,
                                'c': 3})]

    assert expected == _merge_lists(desc1, desc2)

def test_merge_run_cmd():
    override = Run({'user': 'foo', 'cmd': ['a', 'b', 'c'], 'entrypoint': ['a', 'b']})
    image = Run({'user': 'foo', 'cmd': ['1', '2', '3'], 'entrypoint': ['1', '2']})

    override.merge(image)
    assert override['cmd'] == ['a', 'b', 'c']
    assert override['entrypoint'] == ['a', 'b']
<|endoftext|>


<commit_before>
from unittest import TestCase
import pytest

from security_id import validation

from security_id.validation import NullError

xfail = pytest.mark.xfail

# empty exception tests
@pytest.mark.parametrize('obj',[
    pytest.mark.xfail(raises=NullError)((validation.Sedol())),
    pytest.mark.xfail(raises=NullError)((validation.Isin())),
    pytest.mark.xfail(raises=NullError)((validation.Cusip()))])
def test_none(obj):
    obj.validate(None)

@pytest.mark.parametrize('obj',[
    pytest.mark.xfail(raises=NullError)((validation.Sedol())),
    pytest.mark.xfail(raises=NullError)((validation.Isin())),
    pytest.mark.xfail(raises=NullError)((validation.Cusip()))])
def test_empty_string(obj):
    obj.validate("")

# SEDOL tests
@pytest.mark.parametrize('input',['0263494','2046251','0798059'])
def test_numeric_sedol(input):
    validation.Sedol().validate(input) == True

@pytest.mark.parametrize('input',['BCV7KT2',])
def test_char_sedol(input):
    validation.Sedol().validate(input) == True

@pytest.mark.parametrize('input',[
                            xfail('0263494567',raises=validation.LengthError),
                            xfail('BCV7KTM',raises=validation.CheckDigitError),
                            xfail('BCV7KT5',raises=validation.CheckSumError),
                            xfail(None,raises=validation.NullError)])
def test_sedol_failures(input):
    validation.Sedol().validate(input) == True

# ISIN tests
@pytest.mark.parametrize('input',['US0378331005','ES0109067019','GB0002374006','HK0941009539'])
def test_numeric_isin(input):
    validation.Isin().validate(input) == True

@pytest.mark.parametrize('input',['US30231G1022','CNE1000007Q1','US66987V1098'])
def test_char_isin(input):
    validation.Isin().validate(input) == True

@pytest.mark.parametrize('input',[
                            xfail('US03783310055',raises=validation.LengthError),
                            xfail('US037833100G',raises=validation.CheckDigitError),
                            xfail('XA0109067019',raises=validation.CountryCodeError),
                            xfail('US0378331009',raises=validation.CheckSumError)])
def test_isin_failures(input):
    validation.Isin().validate(input) == True

# CUSIP tests
@pytest.mark.parametrize('input',['037833100'])
def test_numeric_cusip(input):
    validation.Cusip().validate(input) == True

@pytest.mark.parametrize('input',['30303M102'])
def test_char_cusip(input):
    validation.Cusip().validate(input) == True

@pytest.mark.parametrize('input',[
                            xfail('30303M1024',raises=validation.LengthError),
                            xfail('30303M10#',raises=validation.CheckDigitError),
                            xfail('30303M102',raises=validation.CheckSumError)])
def test_cusip_failures(input):
    validation.Cusip().validate(input) == True
<commit_msg>improve coverage.<commit_after>
import pytest

from security_id.validation import Cusip, Isin, Sedol, val_check_digit, luhn_modn_checksum
from security_id.exceptions import CheckDigitError, CheckSumError, CountryCodeError, LengthError, NullError

xfail = pytest.mark.xfail

# class SecurityId shared methods

# empty exception tests
@pytest.mark.parametrize('obj',[
    pytest.mark.xfail(raises=NullError)((Sedol())),
    pytest.mark.xfail(raises=NullError)((Isin())),
    pytest.mark.xfail(raises=NullError)((Cusip()))])
def test_none(obj):
    obj.validate(None)

# empty string
@pytest.mark.parametrize('obj',[
    pytest.mark.xfail(raises=NullError)((Sedol())),
    pytest.mark.xfail(raises=NullError)((Isin())),
    pytest.mark.xfail(raises=NullError)((Cusip()))])
def test_empty_string(obj):
    obj.validate("")

# is_valid
@pytest.mark.parametrize('input,expected',
                         [('30303M102',True),
                          ('30303M101',False)])
def test_is_valid(input,expected):
    assert Cusip().is_valid(input) == expected

# appending checksum
@pytest.mark.parametrize('obj,input,expected',
                         [(Cusip(),'03783310','037833100'),
                          (Cusip(),'3783310','37833100')])
def test_append_checksum(obj,input,expected):
    assert obj.append_checksum(input) == expected

# length error
@pytest.mark.parametrize('obj,input',[
    pytest.mark.xfail(raises=LengthError)((Sedol(),'0263494567')),
    pytest.mark.xfail(raises=LengthError)((Isin(),'US03783310055')),
    pytest.mark.xfail(raises=LengthError)((Cusip(),'30303M1024'))])
def test_length_error(obj,input):
    obj.validate(input)

# checksum digit error
@pytest.mark.parametrize('obj,input',[
    pytest.mark.xfail(raises=CheckDigitError)((Sedol(),'BCV7KTM')),
    pytest.mark.xfail(raises=CheckDigitError)((Isin(),'US037833100G')),
    pytest.mark.xfail(raises=CheckDigitError)((Cusip(),'30303M10#'))])
def test_length_error(obj,input):
    obj.validate(input)

# SEDOL tests
@pytest.mark.parametrize('input',['0263494','2046251','0798059'])
def test_numeric_sedol(input):
    assert Sedol().validate(input) == True

@pytest.mark.parametrize('input',['BCV7KT2'])
def test_char_sedol(input):
    assert Sedol().validate(input) == True

@pytest.mark.parametrize('input',[
                            xfail('BCV7KT5',raises=CheckSumError)])
def test_sedol_failures(input):
    assert Sedol().validate(input) == True

# ISIN tests
@pytest.mark.parametrize('input',['US0378331005','ES0109067019','GB0002374006','HK0941009539'])
def test_numeric_isin(input):
    assert Isin().validate(input) == True

@pytest.mark.parametrize('input',['US30231G1022','CNE1000007Q1','US66987V1098'])
def test_char_isin(input):
    assert Isin().validate(input) == True

@pytest.mark.parametrize('input',[
                            xfail('XA0109067019',raises=CountryCodeError),
                            xfail('US0378331009',raises=CheckSumError)])
def test_isin_failures(input):
    assert Isin().validate(input) == True

# CUSIP tests
@pytest.mark.parametrize('input',['037833100'])
def test_numeric_cusip(input):
    assert Cusip().validate(input) == True

@pytest.mark.parametrize('input',['30303M102'])
def test_char_cusip(input):
    assert Cusip().validate(input) == True

@pytest.mark.parametrize('input',[
                            xfail('30303M102',raises=CheckSumError)])
def test_cusip_failures(input):
    assert Cusip().validate(input) == True

@pytest.mark.parametrize('input,expected',[('30303M102',True),('30303M101',False)])
def test_is_valid(input,expected):
    assert Cusip().is_valid(input) == expected

@pytest.mark.parametrize('input,expected',[('0',0),('A',8)])
def test_luhn_modn_checksum(input,expected):
    assert luhn_modn_checksum(input) == expected

def test_val_check_digit():
    assert val_check_digit('abc1') == 1

@xfail(raises=CheckDigitError)
def test_val_check_digit():
    assert val_check_digit('abc') == 1<|endoftext|>


<commit_before>import nose.tools
from nose.plugins.attrib import attr

from tests.common import WpsTestClient, TESTDATA, assert_response_success

@attr('online')
def test_wps_subset_countries():
    wps = WpsTestClient()
    datainputs = "[resource={0};region=FRA]".format(TESTDATA['eurocordex_nc_1'])
    resp = wps.get(service='wps', request='execute', version='1.0.0', identifier='subset_countries',
                   datainputs=datainputs)
    assert_response_success(resp)

<commit_msg>using local testdata<commit_after>import nose.tools
from nose.plugins.attrib import attr

from tests.common import WpsTestClient, TESTDATA, assert_response_success

def test_wps_subset_countries():
    wps = WpsTestClient()
    datainputs = "[resource={0};region=FRA]".format(TESTDATA['cmip5_tasmax_nc'])
    resp = wps.get(service='wps', request='execute', version='1.0.0', identifier='subset_countries',
                   datainputs=datainputs)
    assert_response_success(resp)

<|endoftext|>


<commit_before>"""
The sole purpose of this module is to alter the sys.path upon
import in such a way to get pygr from the source directory rather than
@CTB finish comment?

See the README.txt file for details on how to change the behavior.

NOTE in place builds are required:

python setup.py build_ext -i
"""

import sys, os, distutils.util, platform

import testoptions

def path_join(*args):
    "Joins and normalizes paths"
    return os.path.abspath(os.path.join(*args))

def stop(msg):
    "A fatal unrecoverable error"
    sys.stderr.write(msg + '\n')
    sys.exit()

# get the current directory of the current module
curr_dir = os.path.dirname(__file__)

# this is the extra path that needs be added
base_dir = path_join(curr_dir, '..', '..')

# get the pygr source directory
pygr_source_dir = path_join(base_dir, 'pygr')

# build specific directories
os_info = distutils.util.get_platform()
version = ".".join([str(x) for x in platform.python_version_tuple()[:2]])
lib_dir  = 'lib.%s-%s' % (os_info, version,)
temp_dir = 'temp.%s-%s' % (os_info, version,)
pygr_build_dir = path_join(base_dir, 'build', lib_dir)
pygr_temp_dir  = path_join(base_dir, 'build', temp_dir)

# we'll do a top level option parsing in this module as well
parser = testoptions.option_parser()

# parse the arguments
options, args = parser.parse_args()

# this makes it less clunky
use_pathfix = not options.no_pathfix

if use_pathfix:
    # alter the import path
    if options.builddir:
        sys.path = [ pygr_build_dir  ] + sys.path
        required_prefix = pygr_build_dir
    else:
        sys.path = [ base_dir  ] + sys.path
        required_prefix = pygr_source_dir

###

# also, start coverage

def start_coverage():
    import figleaf
    from figleaf import annotate_html

    # Fix for figleaf misbehaving. It is adding a logger at root level
    # and that will add a handler to all subloggers (ours as well)
    # needs to be fixed in figleaf
    import logging
    root = logging.getLogger()

    # remove all root handlers
    for hand in root.handlers:
        root.removeHandler(hand)

    figleaf.start()

if options.coverage:
    start_coverage()

###

try:
    # import the main pygr module
    import pygr
except ImportError, exc:
    stop ("unable to import: %s" %  exc)

try:
    # import an extension module
    from pygr import cnestedlist
except ImportError, exc:
    stop("""
    Unable to import extension modules: '%s'

    Either build in place with: python setup.py build_ext -i

    Or pass the -b flag to runtest.py (see runtest.py -h for more details)
    """ % exc)

if use_pathfix:

    for mod in [ pygr, cnestedlist ]:
        # test that the imported python modules have the required prefix
        if not mod.__file__.startswith(required_prefix):
            stop ("module %s imported from invalid path: %s" % \
                  (mod.__name__, mod.__file__))
<commit_msg>better error message, more details printed about the import paths, both on error and on success<commit_after>"""
The sole purpose of this module is to alter the sys.path upon
import in such a way to get pygr from the source directory rather than
@CTB finish comment?

See the README.txt file for details on how to change the behavior.

NOTE in place builds are required:

python setup.py build_ext -i
"""

import sys, os, distutils.util, platform

import testoptions

def path_join(*args):
    "Joins and normalizes paths"
    return os.path.abspath(os.path.join(*args))

# we cannot use the main logger, because the import paths
# may not be set up yet
def info(msg):
    "Prints a message"
    sys.stderr.write(msg + '\n')

def stop(msg):
    "A fatal unrecoverable error"
    info(msg)
    sys.exit()

# get the current directory of the current module
curr_dir = os.path.dirname(__file__)

# this is the extra path that needs be added
base_dir = path_join(curr_dir, '..', '..')

# get the pygr source directory
pygr_source_dir = path_join(base_dir, 'pygr')

# build specific directories
os_info = distutils.util.get_platform()
version = ".".join([str(x) for x in platform.python_version_tuple()[:2]])
lib_dir  = 'lib.%s-%s' % (os_info, version,)
temp_dir = 'temp.%s-%s' % (os_info, version,)
pygr_build_dir = path_join(base_dir, 'build', lib_dir)
pygr_temp_dir  = path_join(base_dir, 'build', temp_dir)

# we'll do a top level option parsing in this module as well
parser = testoptions.option_parser()

# parse the arguments
options, args = parser.parse_args()

# this makes it less clunky
use_pathfix = not options.no_pathfix

# stores the error message about the import path
path_errmsg = None

if use_pathfix:
    # alter the import path
    if options.builddir:
        path_errmsg = "Importing pygr from platform build path %s" %  pygr_build_dir
        sys.path = [ pygr_build_dir  ] + sys.path
        required_prefix = pygr_build_dir
    else:
        path_errmsg = "Importing pygr from source directory %s" % base_dir
        sys.path = [ base_dir  ] + sys.path
        required_prefix = pygr_source_dir
else:
    path_errmsg = "Importing pygr from default path"

###

# also, start coverage

def start_coverage():
    import figleaf
    from figleaf import annotate_html

    # Fix for figleaf misbehaving. It is adding a logger at root level
    # and that will add a handler to all subloggers (ours as well)
    # needs to be fixed in figleaf
    import logging
    root = logging.getLogger()

    # remove all root handlers
    for hand in root.handlers:
        root.removeHandler(hand)

    figleaf.start()

if options.coverage:
    start_coverage()

###

try:
    # try to import the main pygr module
    import pygr
    from pygr import logger

    # we have a logger now
    logger.info("importing pygr from %s" %  pygr.__file__)

    # try to import an extension module
    from pygr import cnestedlist

except ImportError, exc:
    stop("""
    %s

    Error: '%s'

    Possible solutions:

        1. build the extension modules in place with:
                     python setup.py build_ext -i

        2. add the -b flag to runtest.py
                    (see runtest.py -h for more details)

        3. install a binary version of pygr into the system path

    """ % (path_errmsg, exc))

if use_pathfix:

    for mod in [ pygr, cnestedlist ]:
        # test that the imported python modules have the required prefix
        if not mod.__file__.startswith(required_prefix):
            stop ("module %s imported from invalid path: %s" % \
                  (mod.__name__, mod.__file__))
<|endoftext|>


<commit_before>import os.path
import pytest

from pip.download import PipSession
from pip.index import HTMLPage
from pip.index import PackageFinder, Link

def test_sort_locations_file_expand_dir(data):
    """
    Test that a file:// dir gets listdir run with expand_dir
    """
    finder = PackageFinder([data.find_links], [], session=PipSession())
    files, urls = finder._sort_locations([data.find_links], expand_dir=True)
    assert files and not urls, (
        "files and not urls should have been found at find-links url: %s" %
        data.find_links
    )

def test_sort_locations_file_not_find_link(data):
    """
    Test that a file:// url dir that's not a find-link, doesn't get a listdir
    run
    """
    finder = PackageFinder([], [], session=PipSession())
    files, urls = finder._sort_locations([data.index_url("empty_with_pkg")])
    assert urls and not files, "urls, but not files should have been found"

def test_sort_locations_non_existing_path():
    """
    Test that a non-existing path is ignored.
    """
    finder = PackageFinder([], [], session=PipSession())
    files, urls = finder._sort_locations(
        [os.path.join('this', 'doesnt', 'exist')])
    assert not urls and not files, "nothing should have been found"

class TestLink(object):

    def test_splitext(self):
        assert ('wheel', '.whl') == Link('http://yo/wheel.whl').splitext()

    @pytest.mark.parametrize(
        ("url", "expected"),
        [
            ("http://yo/wheel.whl", "wheel.whl"),
            ("http://yo/wheel", "wheel"),
            (
                "http://yo/myproject-1.0%2Bfoobar.0-py2.py3-none-any.whl",
                "myproject-1.0+foobar.0-py2.py3-none-any.whl",
            ),
        ],
    )
    def test_filename(self, url, expected):
        assert Link(url).filename == expected

    def test_no_ext(self):
        assert '' == Link('http://yo/wheel').ext

    def test_ext(self):
        assert '.whl' == Link('http://yo/wheel.whl').ext

    def test_ext_fragment(self):
        assert '.whl' == Link('http://yo/wheel.whl#frag').ext

    def test_ext_query(self):
        assert '.whl' == Link('http://yo/wheel.whl?a=b').ext

    def test_is_wheel(self):
        assert Link('http://yo/wheel.whl').is_wheel

    def test_is_wheel_false(self):
        assert not Link('http://yo/not_a_wheel').is_wheel

@pytest.mark.parametrize(
    ("html", "url", "expected"),
    [
        ("", "https://example.com/", "https://example.com/"),
        (
            ""
            "<base href=\"https://foo.example.com/\">"
            "",
            "https://example.com/",
            "https://foo.example.com/",
        ),
        (
            ""
            "<base><base href=\"https://foo.example.com/\">"
            "",
            "https://example.com/",
            "https://foo.example.com/",
        ),
    ],
)
def test_base_url(html, url, expected):
    assert HTMLPage(html, url).base_url == expected

class MockLogger(object):
    def __init__(self):
        self.called = False

    def warning(self, *args, **kwargs):
        self.called = True

@pytest.mark.parametrize(
    ("location", "trusted", "expected"),
    [
        ("http://pypi.python.org/something", [], True),
        ("https://pypi.python.org/something", [], False),
        ("git+http://pypi.python.org/something", [], True),
        ("git+https://pypi.python.org/something", [], False),
        ("git+ssh://git@pypi.python.org/something", [], False),
        ("http://localhost", [], False),
        ("http://127.0.0.1", [], False),
        ("http://example.com/something/", [], True),
        ("http://example.com/something/", ["example.com"], False),
    ],
)
def test_secure_origin(location, trusted, expected):
    finder = PackageFinder([], [], session=[], trusted_hosts=trusted)
    logger = MockLogger()
    finder._validate_secure_origin(logger, location)
    assert logger.called == expected
<commit_msg>add test for link fragments<commit_after>import os.path
import pytest

from pip.download import PipSession
from pip.index import HTMLPage
from pip.index import PackageFinder, Link

def test_sort_locations_file_expand_dir(data):
    """
    Test that a file:// dir gets listdir run with expand_dir
    """
    finder = PackageFinder([data.find_links], [], session=PipSession())
    files, urls = finder._sort_locations([data.find_links], expand_dir=True)
    assert files and not urls, (
        "files and not urls should have been found at find-links url: %s" %
        data.find_links
    )

def test_sort_locations_file_not_find_link(data):
    """
    Test that a file:// url dir that's not a find-link, doesn't get a listdir
    run
    """
    finder = PackageFinder([], [], session=PipSession())
    files, urls = finder._sort_locations([data.index_url("empty_with_pkg")])
    assert urls and not files, "urls, but not files should have been found"

def test_sort_locations_non_existing_path():
    """
    Test that a non-existing path is ignored.
    """
    finder = PackageFinder([], [], session=PipSession())
    files, urls = finder._sort_locations(
        [os.path.join('this', 'doesnt', 'exist')])
    assert not urls and not files, "nothing should have been found"

class TestLink(object):

    def test_splitext(self):
        assert ('wheel', '.whl') == Link('http://yo/wheel.whl').splitext()

    @pytest.mark.parametrize(
        ("url", "expected"),
        [
            ("http://yo/wheel.whl", "wheel.whl"),
            ("http://yo/wheel", "wheel"),
            (
                "http://yo/myproject-1.0%2Bfoobar.0-py2.py3-none-any.whl",
                "myproject-1.0+foobar.0-py2.py3-none-any.whl",
            ),
        ],
    )
    def test_filename(self, url, expected):
        assert Link(url).filename == expected

    def test_no_ext(self):
        assert '' == Link('http://yo/wheel').ext

    def test_ext(self):
        assert '.whl' == Link('http://yo/wheel.whl').ext

    def test_ext_fragment(self):
        assert '.whl' == Link('http://yo/wheel.whl#frag').ext

    def test_ext_query(self):
        assert '.whl' == Link('http://yo/wheel.whl?a=b').ext

    def test_is_wheel(self):
        assert Link('http://yo/wheel.whl').is_wheel

    def test_is_wheel_false(self):
        assert not Link('http://yo/not_a_wheel').is_wheel

    def test_fragments(self):
        url = 'git+https://example.com/package#egg=eggname'
        assert 'eggname' == Link(url).egg_fragment
        assert None is Link(url).subdirectory_fragment
        url = 'git+https://example.com/package#egg=eggname&subdirectory=subdir'
        assert 'eggname' == Link(url).egg_fragment
        assert 'subdir' == Link(url).subdirectory_fragment
        url = 'git+https://example.com/package#subdirectory=subdir&egg=eggname'
        assert 'eggname' == Link(url).egg_fragment
        assert 'subdir' == Link(url).subdirectory_fragment

@pytest.mark.parametrize(
    ("html", "url", "expected"),
    [
        ("", "https://example.com/", "https://example.com/"),
        (
            ""
            "<base href=\"https://foo.example.com/\">"
            "",
            "https://example.com/",
            "https://foo.example.com/",
        ),
        (
            ""
            "<base><base href=\"https://foo.example.com/\">"
            "",
            "https://example.com/",
            "https://foo.example.com/",
        ),
    ],
)
def test_base_url(html, url, expected):
    assert HTMLPage(html, url).base_url == expected

class MockLogger(object):
    def __init__(self):
        self.called = False

    def warning(self, *args, **kwargs):
        self.called = True

@pytest.mark.parametrize(
    ("location", "trusted", "expected"),
    [
        ("http://pypi.python.org/something", [], True),
        ("https://pypi.python.org/something", [], False),
        ("git+http://pypi.python.org/something", [], True),
        ("git+https://pypi.python.org/something", [], False),
        ("git+ssh://git@pypi.python.org/something", [], False),
        ("http://localhost", [], False),
        ("http://127.0.0.1", [], False),
        ("http://example.com/something/", [], True),
        ("http://example.com/something/", ["example.com"], False),
    ],
)
def test_secure_origin(location, trusted, expected):
    finder = PackageFinder([], [], session=[], trusted_hosts=trusted)
    logger = MockLogger()
    finder._validate_secure_origin(logger, location)
    assert logger.called == expected
<|endoftext|>


<commit_before>"""
Copyright (c) 2020 Red Hat, Inc
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the LICENSE file for details.
"""

from __future__ import absolute_import

from flexmock import flexmock
from osbs.utils.yaml import read_yaml, read_yaml_from_file_path

import json
import os
import pkg_resources
import pytest
import yaml

def test_read_yaml_file_ioerrors(tmpdir):
    config_path = os.path.join(str(tmpdir), 'nosuchfile.yaml')
    with pytest.raises(IOError):
        read_yaml_from_file_path(config_path, 'schemas/nosuchfile.json')

@pytest.mark.parametrize('from_file', [True, False])
@pytest.mark.parametrize('config', [
    ("""\
      compose:
          modules:
          - mod_name:mod_stream:mod_version
    """),
])
def test_read_yaml_file_or_yaml(tmpdir, from_file, config):
    expected = yaml.safe_load(config)

    if from_file:
        config_path = os.path.join(str(tmpdir), 'config.yaml')
        with open(config_path, 'w') as fp:
            fp.write(config)
        output = read_yaml_from_file_path(config_path, 'schemas/container.json')
    else:
        output = read_yaml(config, 'schemas/container.json')

    assert output == expected

def test_read_yaml_bad_package(caplog):
    with pytest.raises(ImportError):
        read_yaml("", 'schemas/container.json', package='bad_package')
    assert 'Unable to find package bad_package' in caplog.text

def test_read_yaml_file_bad_extract(tmpdir, caplog):
    class FakeProvider(object):
        def get_resource_stream(self, pkg, rsc):
            raise IOError

    # pkg_resources.resource_stream() cannot be mocked directly
    # Instead mock the module-level function it calls.
    (flexmock(pkg_resources)
        .should_receive('get_provider')
        .and_return(FakeProvider()))

    config_path = os.path.join(str(tmpdir), 'config.yaml')
    with open(config_path, 'w'):
        pass

    with pytest.raises(IOError):
        read_yaml_from_file_path(config_path, 'schemas/container.json')
    assert "unable to extract JSON schema, cannot validate" in caplog.text

def test_read_yaml_file_bad_decode(tmpdir, caplog):
    (flexmock(json)
        .should_receive('load')
        .and_raise(ValueError))

    config_path = os.path.join(str(tmpdir), 'config.yaml')
    with open(config_path, 'w'):
        pass

    with pytest.raises(ValueError):
        read_yaml_from_file_path(config_path, 'schemas/container.json')
    assert "unable to decode JSON schema, cannot validate" in caplog.text
<commit_msg>Add testing of validation errors<commit_after>"""
Copyright (c) 2020 Red Hat, Inc
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the LICENSE file for details.
"""

from __future__ import absolute_import

from flexmock import flexmock
from osbs.utils.yaml import read_yaml, read_yaml_from_file_path
from osbs.exceptions import OsbsValidationException
from osbs.constants import PY3

import json
import os
import pkg_resources
import pytest
import yaml

def test_read_yaml_file_ioerrors(tmpdir):
    config_path = os.path.join(str(tmpdir), 'nosuchfile.yaml')
    with pytest.raises(IOError):
        read_yaml_from_file_path(config_path, 'schemas/nosuchfile.json')

@pytest.mark.parametrize('from_file', [True, False])
@pytest.mark.parametrize('config', [
    ("""\
      compose:
          modules:
          - mod_name:mod_stream:mod_version
    """),
])
def test_read_yaml_file_or_yaml(tmpdir, from_file, config):
    expected = yaml.safe_load(config)

    if from_file:
        config_path = os.path.join(str(tmpdir), 'config.yaml')
        with open(config_path, 'w') as fp:
            fp.write(config)
        output = read_yaml_from_file_path(config_path, 'schemas/container.json')
    else:
        output = read_yaml(config, 'schemas/container.json')

    assert output == expected

def test_read_yaml_bad_package(caplog):
    with pytest.raises(ImportError):
        read_yaml("", 'schemas/container.json', package='bad_package')
    assert 'Unable to find package bad_package' in caplog.text

def test_read_yaml_file_bad_extract(tmpdir, caplog):
    class FakeProvider(object):
        def get_resource_stream(self, pkg, rsc):
            raise IOError

    # pkg_resources.resource_stream() cannot be mocked directly
    # Instead mock the module-level function it calls.
    (flexmock(pkg_resources)
        .should_receive('get_provider')
        .and_return(FakeProvider()))

    config_path = os.path.join(str(tmpdir), 'config.yaml')
    with open(config_path, 'w'):
        pass

    with pytest.raises(IOError):
        read_yaml_from_file_path(config_path, 'schemas/container.json')
    assert "unable to extract JSON schema, cannot validate" in caplog.text

def test_read_yaml_file_bad_decode(tmpdir, caplog):
    (flexmock(json)
        .should_receive('load')
        .and_raise(ValueError))

    config_path = os.path.join(str(tmpdir), 'config.yaml')
    with open(config_path, 'w'):
        pass

    with pytest.raises(ValueError):
        read_yaml_from_file_path(config_path, 'schemas/container.json')
    assert "unable to decode JSON schema, cannot validate" in caplog.text

@pytest.mark.parametrize(('config', 'expected'), [
    ("""\
        operator_manifests:
            enable_digest_pinning: true
            repo_replacements: [] """,
     ("at top level: validating 'anyOf' has failed "
      "({}'manifests_dir' is a required property)"
      ).format('' if PY3 else 'u')),
    ("""\
        compose:
            packages: []
            pulp_repos: true
        mage_build_method: "imagebuilder" """,
     ("at top level: validating 'anyOf' has failed "
      "(Additional properties are not allowed ('mage_build_method' was unexpected))")),
])
def test_read_yaml_validation_error(config, expected, caplog):
    with pytest.raises(OsbsValidationException) as exc_info:
        read_yaml(config, 'schemas/container.json')

    assert "schema validation error" in caplog.text
    assert expected == str(exc_info.value)
<|endoftext|>


<commit_before>#pylint: disable=W0104,W0108
import pyop.operators as operators
from pyop import adjointTest

import random

import numpy as np

from numpy import linspace, reshape, array, zeros, ravel
import scipy.signal as signal

num_tests = 25

#################
#  Convolution  #
#################

def test2DConvolution():

    A = reshape(linspace(1,9,9), (3,3), order = 'C')
    kernel = array([[-1, 1]])
    kernel_square = array([[0, 0, 0], [0, -1, 1], [0, 0, 0]])

    C = operators.convolve(kernel, (3, 3), order = 'F')
    adjointTest(C)

    np.testing.assert_allclose(
        reshape(C._forward(ravel(A, order = 'F')), (3,3), order = 'F'),
        signal.convolve(A, kernel, 'same'))

    np.testing.assert_allclose(
        reshape(C._forward(ravel(A, order = 'F')), (3,3), order = 'F'),
        signal.convolve(A, kernel_square, 'same'))

    np.testing.assert_allclose(
        reshape(C._adjoint(ravel(A, order = 'F')), (3,3), order = 'F'),
        signal.convolve(A, np.flipud(np.fliplr(kernel_square)), 'same'))

def test2DConvolutionSquare():

    A = reshape(linspace(1,9,9), (3,3), order = 'C')
    kernel = array([[0, -1, 0],
                    [-1, 0, 1],
                    [0,  1, 0]])

    C = operators.convolve(kernel, (3, 3), order = 'F')
    adjointTest(C)

    np.testing.assert_allclose(
        reshape(C._forward(ravel(A, order = 'F')), (3,3), order = 'F'),
        signal.convolve(A, kernel, 'same'))

    np.testing.assert_allclose(
        reshape(C._adjoint(ravel(A, order = 'F')), (3,3), order = 'F'),
        signal.convolve(A, np.flipud(np.fliplr(kernel)), 'same'))

def test3DConvolution():
    kernel = array([[[-1, 1], [1, 0]], [[1, 0], [0, 0]]])
    kernel_cube = array([
        zeros((3,3)),
        [[0, 0, 0], [0, -1, 1], [0, 1, 0]],
        [[0, 0, 0], [0,  1, 0], [0, 0, 0]]])

    A = reshape(linspace(1,9,9), (3,3), order = 'C')
    B = array((A, A, A))

    C = operators.convolve(kernel, (3, 3, 3), order = 'F')
    adjointTest(C)

    np.testing.assert_allclose(
        reshape(C._forward(ravel(B, order = 'F')), (3, 3, 3), order = 'F'),
        signal.convolve(B, kernel, 'same'))

    np.testing.assert_allclose(
        reshape(C._forward(ravel(B, order = 'F')), (3, 3, 3), order = 'F'),
        signal.convolve(B, kernel_cube, 'same'))

    np.testing.assert_allclose(
        reshape(C._adjoint(ravel(B, order = 'F')), (3, 3, 3), order = 'F'),
        signal.convolve(B,
            np.flipud(np.fliplr(kernel_cube[:,:,::-1])), 'same'))

def testConvolutionRandom():
    kernel_max_size = 5
    image_max_size = 5
    dimensions_max = 4

    for _ in range(num_tests):
        d = random.randint(1, dimensions_max + 1)

        kernel = np.random.rand(*tuple(
            random.randint(1, kernel_max_size + 1) for _ in range(d)))

        image = np.random.rand(*tuple(
            random.randint(1, image_max_size + 1) for _ in range(d)))

        order = ('C', 'F', 'A')[random.randint(0, 2)]

        C = operators.convolve(kernel, image.shape, order)
        adjointTest(C)

        np.testing.assert_allclose(
            reshape(C._forward(ravel(image, order)), image.shape, order),
            signal.convolve(image, kernel, 'same'))
<commit_msg>Slight test cleanup for testConvolutionRandom<commit_after>#pylint: disable=W0104,W0108
import pyop.operators as operators
from pyop import adjointTest

import random

import numpy as np

from numpy import linspace, reshape, array, zeros, ravel
import scipy.signal as signal

num_tests = 25

#################
#  Convolution  #
#################

def test2DConvolution():

    A = reshape(linspace(1,9,9), (3,3), order = 'C')
    kernel = array([[-1, 1]])
    kernel_square = array([[0, 0, 0], [0, -1, 1], [0, 0, 0]])

    C = operators.convolve(kernel, (3, 3), order = 'F')
    adjointTest(C)

    np.testing.assert_allclose(
        reshape(C._forward(ravel(A, order = 'F')), (3,3), order = 'F'),
        signal.convolve(A, kernel, 'same'))

    np.testing.assert_allclose(
        reshape(C._forward(ravel(A, order = 'F')), (3,3), order = 'F'),
        signal.convolve(A, kernel_square, 'same'))

    np.testing.assert_allclose(
        reshape(C._adjoint(ravel(A, order = 'F')), (3,3), order = 'F'),
        signal.convolve(A, np.flipud(np.fliplr(kernel_square)), 'same'))

def test2DConvolutionSquare():

    A = reshape(linspace(1,9,9), (3,3), order = 'C')
    kernel = array([[0, -1, 0],
                    [-1, 0, 1],
                    [0,  1, 0]])

    C = operators.convolve(kernel, (3, 3), order = 'F')
    adjointTest(C)

    np.testing.assert_allclose(
        reshape(C._forward(ravel(A, order = 'F')), (3,3), order = 'F'),
        signal.convolve(A, kernel, 'same'))

    np.testing.assert_allclose(
        reshape(C._adjoint(ravel(A, order = 'F')), (3,3), order = 'F'),
        signal.convolve(A, np.flipud(np.fliplr(kernel)), 'same'))

def test3DConvolution():
    kernel = array([[[-1, 1], [1, 0]], [[1, 0], [0, 0]]])
    kernel_cube = array([
        zeros((3,3)),
        [[0, 0, 0], [0, -1, 1], [0, 1, 0]],
        [[0, 0, 0], [0,  1, 0], [0, 0, 0]]])

    A = reshape(linspace(1,9,9), (3,3), order = 'C')
    B = array((A, A, A))

    C = operators.convolve(kernel, (3, 3, 3), order = 'F')
    adjointTest(C)

    np.testing.assert_allclose(
        reshape(C._forward(ravel(B, order = 'F')), (3, 3, 3), order = 'F'),
        signal.convolve(B, kernel, 'same'))

    np.testing.assert_allclose(
        reshape(C._forward(ravel(B, order = 'F')), (3, 3, 3), order = 'F'),
        signal.convolve(B, kernel_cube, 'same'))

    np.testing.assert_allclose(
        reshape(C._adjoint(ravel(B, order = 'F')), (3, 3, 3), order = 'F'),
        signal.convolve(B,
            np.flipud(np.fliplr(kernel_cube[:,:,::-1])), 'same'))

def testConvolutionRandom():
    kernel_max_size = 5
    image_max_size = 5
    dimensions_max = 4

    for _ in range(num_tests):
        d = random.randint(1, dimensions_max + 1)

        kernel = np.random.rand(*tuple(
            random.randint(1, kernel_max_size + 1) for _ in range(d)))

        image = np.random.rand(*tuple(
            random.randint(1, image_max_size + 1) for _ in range(d)))

        order = random.choice(('C', 'F', 'A'))

        C = operators.convolve(kernel, image.shape, order)
        adjointTest(C)

        np.testing.assert_allclose(
            reshape(C._forward(ravel(image, order)), image.shape, order),
            signal.convolve(image, kernel, 'same'))
<|endoftext|>


<commit_before>#!/bin/env python2

import sys
sys.path.append('../')

## What is clearer, a class structure

import serial
import re
import lewas.models
import lewas.parsers
import lewas.datastores

weather_station_metrics = { 'Pa': ( 'air', 'pressure' ),
                            'Racc': ( 'rain', 'accumulation' ),
                            'T': ('air', 'temperature'),
                            'Ws': ('air', 'velocity'),
                            'Vs': ('battery', 'voltage'),
                            'Dm': ('wind', 'direction'),
                            'Ua': ('air', 'humidity')
}

unit_conversion = { 'H': 'hPa',
                    'P': '%RH'
                }

def weather_helper(astring):
    """A helper should take a string representing a single measurement and
    return a tuple or object representing that value. In this case we
    return a Measurement object, the parser will then return a list of
    Measurement objects
    """

    (key, value) = astring.split("=")
    if key in weather_station_metrics:
        key = weather_station_metrics[key]
    m = None
    try:
        (value,unit) = re.search(r'([0-9]+(?:\.[0-9]+)?)([a-zA-Z#/]+)',value).groups()
        if unit in unit_conversion:
            unit = unit_conversion[unit]
        m = lewas.models.Measurement(value, key, unit, "weather station", "stroubles1")
    except AttributeError, e:
        print("Error parsing: {}".format(value))
    return m

class WeatherStation(lewas.models.Instrument):
    ## should we define sensors in an array
    sensors = { 'wind.DirectionMin': { 'metric': 'wind_speed', 'unit': 'm/s' },
                'wind.DirectionAvg': { 'metric': 'wind_speed', 'unit': 'm/s' },
                'wind.DirectionMax': { 'metric': 'wind_speed', 'unit': 'm/s' }
            }

    ## let's just use the output of the sensor to determine metrics: one parser to rule them all.
    parsers = { r'^0R[0-5],(.*)': lewas.parsers.split_parser(delim=',',helper=weather_helper) }

    ## still figuring out what makes sense here
    DirectionMin = lewas.models.Sensor(metric='wind_speed', unit='m/s')
    WindGust = lewas.models.Sensor(metric='wind_gust', unit='m/s')

    #parsers = { r'/^0R1 (.*)$/': lewas.parsers.Split([ "wind.DirectionMin", "wind.DirectionAvg", "wind.DirectMax", "wind.speed", "wind.gust", "wind.lull"]),
    #            r'/^0R2 (.*)$/': lewas.parsers.Split([ "pth.Temp", "pth.Humidity", "pth.Pressure" ]),
    #            r'/^0R3 (.*)$/': lewas.parsers.Split([ "rain.rainAcc", "rain.rainDur", "rain.rainIn", "rain.hailAcc", "rain.hailDur", "rain.hailIn", "rain.peakIn", "rain.hailPeakIn" ])
    #}

    def start(self):
        pass

    ## Custom methods.  Anything beyond what is handled by the automatic API
    ## Diagnostic methods

    def check(self, metric):
        codes = { 'wind': '0WU',
          'PTH': '0TU',
          'rain': '0RU',
          'self': '0SU'
        }

        self.cmd_output(codes[metric]+'\n\r')

    def reset(self, code):
        codes = { 'rain': '0XZRU',
                  'intensity': '0XZRI'
              }
        output = self.cmd_output(codes[code]+'\n\r')

if __name__ == "__main__":
    interactive = False

    if len(sys.argv) == 1:
        datastream = open("weather_data.txt", "r")
    else:
        datastream = serial.Serial("/dev/tty{}".format(sys.argv[1]), 19200) #argv[1] e.g. USB0

    ws = WeatherStation(datastream)
    print(ws)
    ws.run(lewas.datastores.leapi())

    #if not interactive:
    #    ws.run(lewas.datastores.ActiveRecord(WeatherStation))
    #else:
    #    pass
        ## run checks, prompt user to reset, check a particular value,
        ## etc.

<commit_msg>add more metrics, put units in with metrics<commit_after>#!/bin/env python2

import sys
sys.path.append('../')

## What is clearer, a class structure

import serial
import re
import lewas.models
import lewas.parsers
import lewas.datastores

weather_station_metrics = { 'Pa': ( 'air', 'pressure', 'hPa' ),
                            'Rc': ( 'rain', 'accumulation', 'mm' ),
                            'Rd': ( 'rain', 'duration', 's' ),
                            'Ri': ( 'rain', 'intensity', 'mm/h' ),
                            'Ta': ('air', 'temperature', 'C'),
                            'Hc': ( 'hail', 'accumulation', 'hits/cm2' ),
                            'Hd': ( 'hail', 'duration', 's' ),
                            'Hi': ( 'hail', 'intensity', 'hits/cm2h'),
                            'Sm': ('wind', 'speed', 'm/s'),
                            'Vs': ('battery', 'voltage', 'V'),
                            'Dm': ('wind', 'direction', 'D'),
                            'Ua': ('air', 'humidity', '%RH')
}

unit_conversion = { 'H': 'hPa',
                    'P': '%RH',
                    'M': 'mm',

                }

def weather_helper(astring):
    """A helper should take a string representing a single measurement and
    return a tuple or object representing that value. In this case we
    return a Measurement object, the parser will then return a list of
    Measurement objects
    """

    (key, value) = astring.split("=")
    units = None
    if key in weather_station_metrics:
        key = weather_station_metrics[key]
        units = key[2]
    m = None
    try:
        (value,unit) = re.search(r'([0-9]+(?:\.[0-9]+)?)([a-zA-Z#/]+)',value).groups()
        m = lewas.models.Measurement(value, key, units, "weather station", "stroubles1")
    except AttributeError, e:
        print("Error parsing: {}".format(value))
    return m

class WeatherStation(lewas.models.Instrument):
    ## should we define sensors in an array
    sensors = { 'wind.DirectionMin': { 'metric': 'wind_speed', 'unit': 'm/s' },
                'wind.DirectionAvg': { 'metric': 'wind_speed', 'unit': 'm/s' },
                'wind.DirectionMax': { 'metric': 'wind_speed', 'unit': 'm/s' }
            }

    ## let's just use the output of the sensor to determine metrics: one parser to rule them all.
    parsers = { r'^0R[0-5],(.*)': lewas.parsers.split_parser(delim=',',helper=weather_helper) }

    ## still figuring out what makes sense here
    DirectionMin = lewas.models.Sensor(metric='wind_speed', unit='m/s')
    WindGust = lewas.models.Sensor(metric='wind_gust', unit='m/s')

    #parsers = { r'/^0R1 (.*)$/': lewas.parsers.Split([ "wind.DirectionMin", "wind.DirectionAvg", "wind.DirectMax", "wind.speed", "wind.gust", "wind.lull"]),
    #            r'/^0R2 (.*)$/': lewas.parsers.Split([ "pth.Temp", "pth.Humidity", "pth.Pressure" ]),
    #            r'/^0R3 (.*)$/': lewas.parsers.Split([ "rain.rainAcc", "rain.rainDur", "rain.rainIn", "rain.hailAcc", "rain.hailDur", "rain.hailIn", "rain.peakIn", "rain.hailPeakIn" ])
    #}

    def start(self):
        pass

    ## Custom methods.  Anything beyond what is handled by the automatic API
    ## Diagnostic methods

    def check(self, metric):
        codes = { 'wind': '0WU',
          'PTH': '0TU',
          'rain': '0RU',
          'self': '0SU'
        }

        self.cmd_output(codes[metric]+'\n\r')

    def reset(self, code):
        codes = { 'rain': '0XZRU',
                  'intensity': '0XZRI'
              }
        output = self.cmd_output(codes[code]+'\n\r')

if __name__ == "__main__":
    interactive = False

    if len(sys.argv) == 1:
        datastream = open("weather_data.txt", "r")
    else:
        datastream = serial.Serial("/dev/tty{}".format(sys.argv[1]), 19200) #argv[1] e.g. USB0

    ws = WeatherStation(datastream)
    print(ws)
    ws.run(lewas.datastores.leapi())

    #if not interactive:
    #    ws.run(lewas.datastores.ActiveRecord(WeatherStation))
    #else:
    #    pass
        ## run checks, prompt user to reset, check a particular value,
        ## etc.

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
stream_power_smooth_threshold.py: Defines the StreamPowerSmoothThresholdEroder,
which is a version of the FastscapeEroder (derived from it).

StreamPowerSmoothThresholdEroder uses a mathematically smooth threshold
formulation, rather than one with a singularity. The erosion rate is defined as
follows:

$\omega = K A^m S$

$E = \omega - \omega_c \left[ 1 - \exp ( -\omega / \omega_c ) \right]$

Created on Sat Nov 26 08:36:49 2016

@author: gtucker
"""

if __name__ == '__main__':
    from landlab.components import FastscapeEroder
else:
    from .fastscape_stream_power import FastscapeEroder
import numpy as np
from scipy.optimize import newton

UNDEFINED_INDEX = -1

def new_elev(x, a, b, c, d, e):
    """Equation for elevation of a node at timestep t+1.

    Parameters
    ----------
    x : float
        Value of new elevation
    a : float
        Parameter = K A^m dt / L (nondimensional)
    b : float
        Elevation of downstream node, z_j
    c : float
        Parameter = omega_c * dt (dimension of L, because omega_c [=] L/T)
    d : float
        Parameter = K A^m / (L * wc) [=] L^{-1} (so d * z [=] [-])
    e : float
        z(t) + a z_j + (wc * dt)
    """
    return x * (1.0 + a) + c * np.exp(-d * (x - b)) - e

def new_elev_prime(x, a, b, c, d, e=0.0):
    """Equation for elevation of a node at timestep t+1.

    Parameters
    ----------
    x : float
        Value of new elevation
    a : float
        Parameter = K A^m dt / L
    b : float
        Elevation of downstream node, z_j
    c : float
        Parameter = omega_c * dt (dimension of L, because omega_c [=] L/T)
    d : float
        Parameter = K A^m / (L * wc) [=] L^{-1} (so d * z [=] [-])
    e : n/a
        Placeholder; not used
    """
    return (1.0 + a) - c * d * np.exp(-d * (x - b))

def new_elev_prime2(x, a, b, c, d, e=0.0):
    """Equation for elevation of a node at timestep t+1.

    Parameters
    ----------
    x : float
        Value of new elevation
    a : float
        Placeholder; not used
    b : float
        Elevation of downstream node, z_j
    c : float
        Parameter = omega_c * dt (dimension of L, because omega_c [=] L/T)
    d : float
        Parameter = K A^m / (L * wc) [=] L^{-1} (so d * z [=] [-])
    e : n/a
        Placeholder; not used
    """
    return c * d * d * np.exp(-d * (x - b))

class StreamPowerSmoothThresholdEroder(FastscapeEroder):
    """Stream erosion component with smooth threshold function.

    Parameters
    ----------
    grid : ModelGrid
        A grid.
    K_sp : float, array, or field name
        K in the stream power equation (units vary with other parameters).
    m_sp : float, optional
        m in the stream power equation (power on drainage area).
    n_sp : float, optional, ~ 0.5<n_sp<4.
        n in the stream power equation (power on slope). NOTE: NOT PRESENTLY
        HONORED BY StreamPowerSmoothThresholdEroder (TODO)
    threshold_sp : float (TODO: array, or field name)
        The threshold stream power.
    rainfall_intensity : float; optional
        NOT PRESENTLY HONORED (TODO)

    Examples
    --------
    >>> from landlab import RasterModelGrid
    >>> rg = RasterModelGrid((3, 4), 1.0)
    >>> rg.set_closed_boundaries_at_grid_edges(False, True, True, True)
    >>> z = rg.add_zeros('node', 'topographic__elevation')
    >>> z[5] = 2.0
    >>> z[6] = 1.0
    >>> from landlab.components import FlowRouter
    >>> fr = FlowRouter(rg, method='D4')
    >>> fr.run_one_step()
    >>> from landlab.components import StreamPowerSmoothThresholdEroder
    >>> sp = StreamPowerSmoothThresholdEroder(rg, K_sp=1.0)
    >>> sp.thresholds
    1.0
    >>> sp.run_one_step(dt=1.0)
    >>> import numpy as np
    >>> np.round(z[5:7], 3)
    array([ 1.646,  0.667])
    >>> z[5] = 2.0
    >>> z[6] = 1.0
    >>> import numpy as np
    >>> q = np.zeros(rg.number_of_nodes) + 0.25
    >>> q[6] = 100.0
    >>> sp = StreamPowerSmoothThresholdEroder(rg, K_sp=1.0, use_Q=q)
    >>> sp.run_one_step(dt=1.0)
    >>> np.round(z[5:7], 3)
    array([ 1.754,  0.164])
    """

    def __init__(self, grid, K_sp=None, m_sp=0.5, n_sp=1., threshold_sp=1.,
                 rainfall_intensity=1., use_Q=None, **kwargs):
        """Initialize StreamPowerSmoothThresholdEroder."""

        # Call base-class init
        super(StreamPowerSmoothThresholdEroder,
              self).__init__(grid, K_sp=K_sp, m_sp=m_sp, n_sp=n_sp,
                             threshold_sp=threshold_sp,
                             rainfall_intensity=rainfall_intensity, **kwargs)

        # Handle "use_Q" option (ideally should be done by base class, but
        # FastscapeEroder, unlike StreamPowerEroder, lacks this option)
        if use_Q is not None:
            if type(use_Q) is str:  # if str, assume it's a field name
                self.area_or_discharge = self._grid.at_node[use_Q]
            elif type(use_Q) is np.ndarray:  # if array, use it
                self.area_or_discharge = use_Q
            else:
                print('Warning: use_Q must be field name or array')
                self.area_or_discharge = self._grid.at_node['drainage_area']
        else:
            self.area_or_discharge = self._grid.at_node['drainage_area']

        # Arrays with parameters for use in implicit solver
        self.gamma = grid.empty(at='node')
        self.delta = grid.empty(at='node')

    def run_one_step(self, dt, flooded_nodes=None, runoff_rate=None, **kwds):
        """Run one forward iteration of duration dt.

        Parameters
        ----------
        dt : float
            Time step size
        flooded_nodes : ndarray of int (optional)
            Indices of nodes in lakes/depressions
        runoff_rate : (not used yet)
            (to be added later)

        Examples
        --------
        >>> from landlab import RasterModelGrid
        >>> rg = RasterModelGrid((3, 3), 1.0)
        >>> rg.set_closed_boundaries_at_grid_edges(False, True, True, True)
        >>> z = rg.add_zeros('node', 'topographic__elevation')
        >>> z[4] = 1.0
        >>> from landlab.components import FlowRouter
        >>> fr = FlowRouter(rg, method='D4')
        >>> fr.run_one_step()
        >>> from landlab.components import StreamPowerSmoothThresholdEroder
        >>> sp = StreamPowerSmoothThresholdEroder(rg, K_sp=1.0)
        >>> sp.run_one_step(dt=1.0)
        >>> sp.alpha
        array([ 0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.])
        >>> sp.gamma
        array([ 0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.])
        >>> sp.delta
        array([ 0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.])
        """

        # Set up needed arrays
        #
        # Get shorthand for elevation field ("z"), and for up-to-downstream
        # ordered array of node IDs ("upstream_order_IDs")
        upstream_order_IDs = self._grid['node']['flow__upstream_node_order']
        z = self._grid['node']['topographic__elevation']
        flow_receivers = self._grid['node']['flow__receiver_node']

        # Get an array of flow-link length for each node that has a defined
        # receiver (i.e., that drains to another node).
        defined_flow_receivers = np.not_equal(self._grid['node'][
            'flow__link_to_receiver_node'], UNDEFINED_INDEX)
        if flooded_nodes is not None:
            defined_flow_receivers[flooded_nodes] = False
        flow_link_lengths = self._grid._length_of_link_with_diagonals[
            self._grid['node']['flow__link_to_receiver_node'][
                defined_flow_receivers]]

        # (Later on, add functionality for a runoff rate, or discharge, or
        # variable K)

        # Set up alpha, beta, delta arrays
        #
        #   First, compute drainage area or discharge raised to the power m.
        np.power(self.area_or_discharge, self.m,
                 out=self.A_to_the_m)

        #   Alpha
        self.alpha[defined_flow_receivers==False] = 0.0
        self.alpha[defined_flow_receivers] = (self.K * dt
            * self.A_to_the_m[defined_flow_receivers] / flow_link_lengths)

        #   Gamma
        self.gamma[defined_flow_receivers==False] = 0.0
        self.gamma[defined_flow_receivers] = dt * self.thresholds

        #   Delta
        self.delta[defined_flow_receivers] = ((self.K
            * self.A_to_the_m[defined_flow_receivers] )
            / (self.thresholds * flow_link_lengths))

        # Iterate over nodes from downstream to upstream, using scipy's
        # 'newton' function to find new elevation at each node in turn.
        for node in upstream_order_IDs:
            if defined_flow_receivers[node]:
                epsilon = (self.alpha[node] * z[flow_receivers[node]]
                           + self.gamma[node] + z[node])
                z[node] = newton(new_elev, z[node],
                                 fprime=new_elev_prime,
                                 args=(self.alpha[node],
                                       z[flow_receivers[node]],
                                       self.gamma[node],
                                       self.delta[node],
                                       epsilon))  #,
                                 #fprime2=new_elev_prime2)

        # TODO: handle case self.thresholds = 0

#if __name__ == '__main__':
#    from landlab import RasterModelGrid
#    rg = RasterModelGrid((3, 4), 1.0)
#    z = rg.add_zeros('node', 'topographic__elevation')
#    rg.set_closed_boundaries_at_grid_edges(False, True, True, True)
#    z[5] = 2.0
#    z[6] = 1.0
#    from landlab.components import FlowRouter
#    fr = FlowRouter(rg, method='D4')
#    fr.run_one_step()
#    sp = StreamPowerSmoothThresholdEroder(rg, K_sp=1.0)
#    sp.run_one_step(dt=1.0)
#    print(z)<commit_msg>oops, one more: handle var thresh<commit_after># -*- coding: utf-8 -*-
"""
stream_power_smooth_threshold.py: Defines the StreamPowerSmoothThresholdEroder,
which is a version of the FastscapeEroder (derived from it).

StreamPowerSmoothThresholdEroder uses a mathematically smooth threshold
formulation, rather than one with a singularity. The erosion rate is defined as
follows:

$\omega = K A^m S$

$E = \omega - \omega_c \left[ 1 - \exp ( -\omega / \omega_c ) \right]$

Created on Sat Nov 26 08:36:49 2016

@author: gtucker
"""

if __name__ == '__main__':
    from landlab.components import FastscapeEroder
else:
    from .fastscape_stream_power import FastscapeEroder
import numpy as np
from scipy.optimize import newton

UNDEFINED_INDEX = -1

def new_elev(x, a, b, c, d, e):
    """Equation for elevation of a node at timestep t+1.

    Parameters
    ----------
    x : float
        Value of new elevation
    a : float
        Parameter = K A^m dt / L (nondimensional)
    b : float
        Elevation of downstream node, z_j
    c : float
        Parameter = omega_c * dt (dimension of L, because omega_c [=] L/T)
    d : float
        Parameter = K A^m / (L * wc) [=] L^{-1} (so d * z [=] [-])
    e : float
        z(t) + a z_j + (wc * dt)
    """
    return x * (1.0 + a) + c * np.exp(-d * (x - b)) - e

def new_elev_prime(x, a, b, c, d, e=0.0):
    """Equation for elevation of a node at timestep t+1.

    Parameters
    ----------
    x : float
        Value of new elevation
    a : float
        Parameter = K A^m dt / L
    b : float
        Elevation of downstream node, z_j
    c : float
        Parameter = omega_c * dt (dimension of L, because omega_c [=] L/T)
    d : float
        Parameter = K A^m / (L * wc) [=] L^{-1} (so d * z [=] [-])
    e : n/a
        Placeholder; not used
    """
    return (1.0 + a) - c * d * np.exp(-d * (x - b))

def new_elev_prime2(x, a, b, c, d, e=0.0):
    """Equation for elevation of a node at timestep t+1.

    Parameters
    ----------
    x : float
        Value of new elevation
    a : float
        Placeholder; not used
    b : float
        Elevation of downstream node, z_j
    c : float
        Parameter = omega_c * dt (dimension of L, because omega_c [=] L/T)
    d : float
        Parameter = K A^m / (L * wc) [=] L^{-1} (so d * z [=] [-])
    e : n/a
        Placeholder; not used
    """
    return c * d * d * np.exp(-d * (x - b))

class StreamPowerSmoothThresholdEroder(FastscapeEroder):
    """Stream erosion component with smooth threshold function.

    Parameters
    ----------
    grid : ModelGrid
        A grid.
    K_sp : float, array, or field name
        K in the stream power equation (units vary with other parameters).
    m_sp : float, optional
        m in the stream power equation (power on drainage area).
    n_sp : float, optional, ~ 0.5<n_sp<4.
        n in the stream power equation (power on slope). NOTE: NOT PRESENTLY
        HONORED BY StreamPowerSmoothThresholdEroder (TODO)
    threshold_sp : float (TODO: array, or field name)
        The threshold stream power.
    rainfall_intensity : float; optional
        NOT PRESENTLY HONORED (TODO)

    Examples
    --------
    >>> from landlab import RasterModelGrid
    >>> rg = RasterModelGrid((3, 4), 1.0)
    >>> rg.set_closed_boundaries_at_grid_edges(False, True, True, True)
    >>> z = rg.add_zeros('node', 'topographic__elevation')
    >>> z[5] = 2.0
    >>> z[6] = 1.0
    >>> from landlab.components import FlowRouter
    >>> fr = FlowRouter(rg, method='D4')
    >>> fr.run_one_step()
    >>> from landlab.components import StreamPowerSmoothThresholdEroder
    >>> sp = StreamPowerSmoothThresholdEroder(rg, K_sp=1.0)
    >>> sp.thresholds
    1.0
    >>> sp.run_one_step(dt=1.0)
    >>> import numpy as np
    >>> np.round(z[5:7], 3)
    array([ 1.646,  0.667])
    >>> z[5] = 2.0
    >>> z[6] = 1.0
    >>> import numpy as np
    >>> q = np.zeros(rg.number_of_nodes) + 0.25
    >>> q[6] = 100.0
    >>> sp = StreamPowerSmoothThresholdEroder(rg, K_sp=1.0, use_Q=q)
    >>> sp.run_one_step(dt=1.0)
    >>> np.round(z[5:7], 3)
    array([ 1.754,  0.164])
    """

    def __init__(self, grid, K_sp=None, m_sp=0.5, n_sp=1., threshold_sp=1.,
                 rainfall_intensity=1., use_Q=None, **kwargs):
        """Initialize StreamPowerSmoothThresholdEroder."""

        # Call base-class init
        super(StreamPowerSmoothThresholdEroder,
              self).__init__(grid, K_sp=K_sp, m_sp=m_sp, n_sp=n_sp,
                             threshold_sp=threshold_sp,
                             rainfall_intensity=rainfall_intensity, **kwargs)

        # Handle "use_Q" option (ideally should be done by base class, but
        # FastscapeEroder, unlike StreamPowerEroder, lacks this option)
        if use_Q is not None:
            if type(use_Q) is str:  # if str, assume it's a field name
                self.area_or_discharge = self._grid.at_node[use_Q]
            elif type(use_Q) is np.ndarray:  # if array, use it
                self.area_or_discharge = use_Q
            else:
                print('Warning: use_Q must be field name or array')
                self.area_or_discharge = self._grid.at_node['drainage_area']
        else:
            self.area_or_discharge = self._grid.at_node['drainage_area']

        # Arrays with parameters for use in implicit solver
        self.gamma = grid.empty(at='node')
        self.delta = grid.empty(at='node')

    def run_one_step(self, dt, flooded_nodes=None, runoff_rate=None, **kwds):
        """Run one forward iteration of duration dt.

        Parameters
        ----------
        dt : float
            Time step size
        flooded_nodes : ndarray of int (optional)
            Indices of nodes in lakes/depressions
        runoff_rate : (not used yet)
            (to be added later)

        Examples
        --------
        >>> from landlab import RasterModelGrid
        >>> rg = RasterModelGrid((3, 3), 1.0)
        >>> rg.set_closed_boundaries_at_grid_edges(False, True, True, True)
        >>> z = rg.add_zeros('node', 'topographic__elevation')
        >>> z[4] = 1.0
        >>> from landlab.components import FlowRouter
        >>> fr = FlowRouter(rg, method='D4')
        >>> fr.run_one_step()
        >>> from landlab.components import StreamPowerSmoothThresholdEroder
        >>> sp = StreamPowerSmoothThresholdEroder(rg, K_sp=1.0)
        >>> sp.run_one_step(dt=1.0)
        >>> sp.alpha
        array([ 0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.])
        >>> sp.gamma
        array([ 0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.])
        >>> sp.delta
        array([ 0.,  0.,  0.,  0.,  1.,  0.,  0.,  0.,  0.])
        """

        # Set up needed arrays
        #
        # Get shorthand for elevation field ("z"), and for up-to-downstream
        # ordered array of node IDs ("upstream_order_IDs")
        upstream_order_IDs = self._grid['node']['flow__upstream_node_order']
        z = self._grid['node']['topographic__elevation']
        flow_receivers = self._grid['node']['flow__receiver_node']

        # Get an array of flow-link length for each node that has a defined
        # receiver (i.e., that drains to another node).
        defined_flow_receivers = np.not_equal(self._grid['node'][
            'flow__link_to_receiver_node'], UNDEFINED_INDEX)
        if flooded_nodes is not None:
            defined_flow_receivers[flooded_nodes] = False
        flow_link_lengths = self._grid._length_of_link_with_diagonals[
            self._grid['node']['flow__link_to_receiver_node'][
                defined_flow_receivers]]

        # (Later on, add functionality for a runoff rate, or discharge, or
        # variable K)

        # Handle possibility of spatially varying threshold
        if type(self.thresholds) is np.ndarray:
            thresh = self.thresholds[defined_flow_receivers]
        else:
            thresh = self.thresholds

        # Set up alpha, beta, delta arrays
        #
        #   First, compute drainage area or discharge raised to the power m.
        np.power(self.area_or_discharge, self.m,
                 out=self.A_to_the_m)

        #   Alpha
        self.alpha[defined_flow_receivers==False] = 0.0
        self.alpha[defined_flow_receivers] = (self.K * dt
            * self.A_to_the_m[defined_flow_receivers] / flow_link_lengths)

        #   Gamma
        self.gamma[defined_flow_receivers==False] = 0.0
        self.gamma[defined_flow_receivers] = dt * thresh

        #   Delta
        self.delta[defined_flow_receivers] = ((self.K
            * self.A_to_the_m[defined_flow_receivers] )
            / (thresh * flow_link_lengths))

        # Iterate over nodes from downstream to upstream, using scipy's
        # 'newton' function to find new elevation at each node in turn.
        for node in upstream_order_IDs:
            if defined_flow_receivers[node]:
                epsilon = (self.alpha[node] * z[flow_receivers[node]]
                           + self.gamma[node] + z[node])
                z[node] = newton(new_elev, z[node],
                                 fprime=new_elev_prime,
                                 args=(self.alpha[node],
                                       z[flow_receivers[node]],
                                       self.gamma[node],
                                       self.delta[node],
                                       epsilon))  #,
                                 #fprime2=new_elev_prime2)

        # TODO: handle case self.thresholds = 0

#if __name__ == '__main__':
#    from landlab import RasterModelGrid
#    rg = RasterModelGrid((3, 4), 1.0)
#    z = rg.add_zeros('node', 'topographic__elevation')
#    rg.set_closed_boundaries_at_grid_edges(False, True, True, True)
#    z[5] = 2.0
#    z[6] = 1.0
#    from landlab.components import FlowRouter
#    fr = FlowRouter(rg, method='D4')
#    fr.run_one_step()
#    sp = StreamPowerSmoothThresholdEroder(rg, K_sp=1.0)
#    sp.run_one_step(dt=1.0)
#    print(z)<|endoftext|>


<commit_before>'''
SPDX-License-Identifier: Apache-2.0
Copyright 2017 Massachusetts Institute of Technology.
'''

import ast
import codecs
import hashlib
import struct
import os

from keylime import config
from keylime import keylime_logging

logger = keylime_logging.init_logging('ima')

#         m = ima_measure_re.match(measure_line)
#         measure  = m.group('file_hash')
#         filename = m.group('file_path')

START_HASH = (codecs.decode('0000000000000000000000000000000000000000', 'hex'))
FF_HASH = (codecs.decode('ffffffffffffffffffffffffffffffffffffffff', 'hex'))

# struct event {
#     struct {
#         u_int32_t pcr;
#         u_int8_t digest[SHA_DIGEST_LENGTH];
#         u_int32_t name_len;
#     } header;
#     char name[TCG_EVENT_NAME_LEN_MAX + 1];
#     struct ima_template_desc *template_desc; /* template descriptor */
#     u_int32_t template_data_len;
#     u_int8_t *template_data;    /* template related data */
# };

def read_unpack(fd, fmt):
    return struct.unpack(fmt, fd.read(struct.calcsize(fmt)))

TCG_EVENT_NAME_LEN_MAX = 255
SHA_DIGEST_LEN = 20

defined_templates = {
    'ima': 'd|n',
    'ima-ng': 'd-ng|n-ng',
    'ima-sig': 'd-ng|n-ng|sig;',
}

def ima_eventname_parse():
    pass

def ima_eventdigest_ng_parse():
    pass

def ima_eventname_ng_parse():
    pass

def ima_eventsig_parse():
    pass

supported_fields = {
    "n": ima_eventname_parse,
    "d-ng": ima_eventdigest_ng_parse,
    "n-ng": ima_eventname_ng_parse,
    "sig": ima_eventsig_parse,
}

def read_measurement_list_bin(path):
    raise Exception("not implementated fully yet")
    # pylint: disable=W0101
    f = open(path, 'rb')

    while True:
        template = {}
        (template['pcr'], template['digest'],
         template['name_len']) = read_unpack(f, "<I20sI")

        if template['name_len'] > TCG_EVENT_NAME_LEN_MAX:
            raise Exception("Error event name too long: %d" %
                            template['name_len'])

        name = read_unpack(f, "<%ds" % template['name_len'])[0]

        is_ima_template = name == 'ima'
        if not is_ima_template:
            template['data_len'] = read_unpack(f, "<I")[0]
            print("reading ima len %d" % template['data_len'])
        else:
            template['data_len'] = SHA_DIGEST_LEN + TCG_EVENT_NAME_LEN_MAX + 1

        template['data'] = read_unpack(f, "<%ds" % template['data_len'])[0]

        if is_ima_template:
            field_len = read_unpack(f, "<I")[0]
            extra_data = read_unpack(f, "<%ds" % field_len)[0]
            template['data'] += extra_data

        print("record %s" % template)

        if template['name'] not in list(defined_templates.keys()):
            template['name'] = 'ima'
        template['desc-fmt'] = defined_templates[template['name']]

        # tokens = template['desc-fmt'].split('|')

def _extract_from_ima(tokens, template_hash):
    """ Extract filedata_hash & path form 'ima' entry; return 1 in case error occurred """
    filedata_hash = codecs.decode(tokens[3], "hex")
    path = str(tokens[4])

    # this is some IMA weirdness
    if template_hash == START_HASH:
        return filedata_hash, path, 0

    # verify template hash. yep this is terrible
    # name needs to be null padded out to MAX len. +1 is for the null terminator of the string itself
    fmt = "<%ds%ds%ds" % (len(filedata_hash), len(path), TCG_EVENT_NAME_LEN_MAX - len(path) + 1)
    tohash = struct.pack(fmt,
                         filedata_hash, path.encode("utf-8"),
                         bytearray(TCG_EVENT_NAME_LEN_MAX - len(path) + 1))
    expected_template_hash = hashlib.sha1(tohash).digest()

    if expected_template_hash != template_hash:
        error = 1
        logger.warning("template hash for file %s does not match %s != %s" %
                       (path,
                        codecs.encode(expected_template_hash, 'hex').decode('utf-8'),
                        codecs.encode(template_hash, 'hex').decode('utf-8')))
    else:
        error = 0

    return filedata_hash, path, error

def _extract_from_ima_ng(tokens, template_hash):
    """ Extract filedata_hash & path form 'ima-ng' entry; return 1 in case error occurred """
    filedata = tokens[3]
    ftokens = filedata.split(":")
    filedata_hash = codecs.decode(ftokens[1], 'hex')
    path = str(tokens[4])

    # this is some IMA weirdness
    if template_hash == START_HASH:
        return filedata_hash, path, 0

    filedata_algo = str(ftokens[0])
    # verify template hash. yep this is terrible
    fmt = "<I%dsBB%dsI%dsB" % (len(filedata_algo), len(filedata_hash), len(path))
    # +2 for the : and the null terminator, and +1 on path for null terminator
    tohash = struct.pack(fmt,
                         len(filedata_hash) + len(filedata_algo) + 2,
                         filedata_algo.encode('utf-8'), ord(':'), ord('\0'),
                         filedata_hash,
                         len(path) + 1,
                         path.encode("utf-8"),
                         ord('\0'))
    expected_template_hash = hashlib.sha1(tohash).digest()

    if expected_template_hash != template_hash:
        error = 1
        logger.warning("template hash for file %s does not match %s != %s" %
                       (path,
                        codecs.encode(expected_template_hash, 'hex').decode('utf-8'),
                        codecs.encode(template_hash, 'hex').decode('utf-8')))
    else:
        error = 0

    return filedata_hash, path, error

def process_measurement_list(lines, lists=None, m2w=None, pcrval=None):
    errs = [0, 0, 0, 0]
    runninghash = START_HASH
    found_pcr = (pcrval is None)

    if lists is not None:
        lists = ast.literal_eval(lists)
        allowlist = lists['allowlist']
        exclude_list = lists['exclude']
    else:
        allowlist = None
        exclude_list = None

    is_valid, compiled_regex, err_msg = config.valid_exclude_list(exclude_list)
    if not is_valid:
        # This should not happen as the exclude list has already been validated
        # by the verifier before acceping it. This is a safety net just in case.
        err_msg += " Exclude list will be ignored."
        logger.error(err_msg)

    for line in lines:
        line = line.strip()
        if line == '':
            continue

        tokens = line.split(None, 4)
        if len(tokens) != 5:
            logger.error("invalid measurement list file line: -%s-" % (line))
            return None

        # print tokens
        # pcr = tokens[0]
        template_hash = codecs.decode(tokens[1], 'hex')
        mode = tokens[2]

        if mode == "ima-ng":
            filedata_hash, path, error = _extract_from_ima_ng(tokens,
                                                              template_hash)
        elif mode == 'ima':
            filedata_hash, path, error = _extract_from_ima(tokens,
                                                           template_hash)
        else:
            raise Exception("unsupported ima template mode: %s" % mode)

        errs[0] += error
        if template_hash == START_HASH:
            template_hash = FF_HASH

        # update hash
        runninghash = hashlib.sha1(runninghash + template_hash).digest()

        if not found_pcr:
            found_pcr = \
                (codecs.encode(runninghash, 'hex').decode('utf-8') == pcrval)

        # write out the new hash
        if m2w is not None:
            m2w.write("%s %s\n" %
                      (codecs.encode(filedata_hash, 'hex').decode('utf-8'),
                       path))

        if allowlist is not None:

            # just skip if it is a weird overwritten path
            if template_hash == FF_HASH:
                # print "excluding ffhash %s"%path
                continue

            # determine if path matches any exclusion list items
            if compiled_regex is not None and compiled_regex.match(path):
                logger.debug("IMA: ignoring excluded path %s" % path)
                continue

            accept_list = allowlist.get(path, None)
            if accept_list is None:
                logger.warning("File not found in allowlist: %s" % (path))
                errs[1] += 1
                continue

            # print('codecs.encode', codecs.encode(filedata_hash, 'hex').decode('utf-8'))
            # print('accept_list:', accept_list)
            if codecs.encode(filedata_hash, 'hex').decode('utf-8') not in accept_list:
                logger.warning("Hashes for file %s don't match %s not in %s" %
                               (path,
                                codecs.encode(filedata_hash, 'hex').decode('utf-8'),
                                accept_list))
                errs[2] += 1
                continue

        errs[3] += 1

    # check PCR value has been found
    if not found_pcr:
        logger.error("IMA measurement list does not match TPM PCR %s" % pcrval)
        return None

    # clobber the retval if there were IMA file errors
    if sum(errs[:3]) > 0:
        logger.error(
            "IMA ERRORS: template-hash %d fnf %d hash %d good %d" % tuple(errs))
        return None

    return codecs.encode(runninghash, 'hex').decode('utf-8')

def process_allowlists(al_data, excl_data):
    # Pull in default config values if not specified
    if al_data is None:
        al_data = read_allowlist()
    if excl_data is None:
        excl_data = read_excllist()

    allowlist = {}
    for line in al_data:
        line = line.strip()
        tokens = line.split(None, 1)
        if len(tokens) != 2:
            continue
        fhash = tokens[0]
        path = str(tokens[1])
        tmp = allowlist.get(path, [])
        tmp.append(fhash)
        allowlist[path] = tmp

    if allowlist.get('boot_aggregate', None) is None:
        logger.warning(
            "No boot_aggregate value found in allowlist, adding an empty one")
        allowlist['boot_aggregate'] = [
            '0000000000000000000000000000000000000000']

    for excl in excl_data:
        if excl.startswith("#"):
            excl_data.remove(excl)
        # don't allow empty lines in exclude list, it will match everything
        if excl == "":
            excl_data.remove(excl)

    return{'allowlist': allowlist, 'exclude': excl_data}

def read_allowlist(al_path=None):
    if al_path is None:
        al_path = config.get('tenant', 'ima_allowlist')
        if config.STUB_IMA:
            al_path = '../scripts/ima/allowlist.txt'

    # Purposefully die if path doesn't exist
    with open(al_path, 'r') as f:
        alist = f.read()
    alist = alist.splitlines()

    logger.debug("Loaded allowlist from %s" % (al_path))

    return alist

def read_excllist(exclude_path=None):
    if exclude_path is None:
        exclude_path = config.get('tenant', 'ima_excludelist')
        if config.STUB_IMA:
            exclude_path = '../scripts/ima/exclude.txt'

    excl_list = []
    if os.path.exists(exclude_path):
        with open(exclude_path, 'r') as f:
            excl_list = f.read()
        excl_list = excl_list.splitlines()

        logger.debug("Loaded exclusion list from %s: %s" %
                     (exclude_path, excl_list))

    return excl_list

def main():
    # read_measurement_list_bin("/sys/kernel/security/ima/binary_runtime_measurements")

    allowlist_path = 'allowlist.txt'
    print("reading allowlist from %s" % allowlist_path)

    exclude_path = 'exclude.txt'
    # exclude_path = '../scripts/ima/exclude.txt'
    print("reading exclude list from %s" % exclude_path)

    al_data = read_allowlist(allowlist_path)
    excl_data = read_excllist(exclude_path)
    lists = process_allowlists(al_data, excl_data)

    measure_path = config.IMA_ML
    # measure_path='../scripts/ima/ascii_runtime_measurements_ima'
    # measure_path = '../scripts/gerardo/ascii_runtime_measurements'
    print("reading measurement list from %s" % measure_path)
    f = open(measure_path, 'r')
    lines = f.readlines()

    m2a = open('measure2allow.txt', "w")
    digest = process_measurement_list(lines, lists, m2a)
    print("final digest is %s" % digest)
    f.close()
    m2a.close()

    print("using m2a")

    al_data = read_allowlist('measure2allow.txt')
    excl_data = read_excllist(exclude_path)
    lists2 = process_allowlists(al_data, excl_data)
    process_measurement_list(lines, lists2)

    print("done")

if __name__ == "__main__":
    try:
        main()
    except Exception as e:
        logger.exception(e)
<commit_msg>ima: Add support for ima-sig template<commit_after>'''
SPDX-License-Identifier: Apache-2.0
Copyright 2017 Massachusetts Institute of Technology.
'''

import ast
import codecs
import hashlib
import struct
import os

from keylime import config
from keylime import keylime_logging

logger = keylime_logging.init_logging('ima')

#         m = ima_measure_re.match(measure_line)
#         measure  = m.group('file_hash')
#         filename = m.group('file_path')

START_HASH = (codecs.decode('0000000000000000000000000000000000000000', 'hex'))
FF_HASH = (codecs.decode('ffffffffffffffffffffffffffffffffffffffff', 'hex'))

# struct event {
#     struct {
#         u_int32_t pcr;
#         u_int8_t digest[SHA_DIGEST_LENGTH];
#         u_int32_t name_len;
#     } header;
#     char name[TCG_EVENT_NAME_LEN_MAX + 1];
#     struct ima_template_desc *template_desc; /* template descriptor */
#     u_int32_t template_data_len;
#     u_int8_t *template_data;    /* template related data */
# };

def read_unpack(fd, fmt):
    return struct.unpack(fmt, fd.read(struct.calcsize(fmt)))

TCG_EVENT_NAME_LEN_MAX = 255
SHA_DIGEST_LEN = 20

defined_templates = {
    'ima': 'd|n',
    'ima-ng': 'd-ng|n-ng',
    'ima-sig': 'd-ng|n-ng|sig;',
}

def ima_eventname_parse():
    pass

def ima_eventdigest_ng_parse():
    pass

def ima_eventname_ng_parse():
    pass

def ima_eventsig_parse():
    pass

supported_fields = {
    "n": ima_eventname_parse,
    "d-ng": ima_eventdigest_ng_parse,
    "n-ng": ima_eventname_ng_parse,
    "sig": ima_eventsig_parse,
}

def read_measurement_list_bin(path):
    raise Exception("not implementated fully yet")
    # pylint: disable=W0101
    f = open(path, 'rb')

    while True:
        template = {}
        (template['pcr'], template['digest'],
         template['name_len']) = read_unpack(f, "<I20sI")

        if template['name_len'] > TCG_EVENT_NAME_LEN_MAX:
            raise Exception("Error event name too long: %d" %
                            template['name_len'])

        name = read_unpack(f, "<%ds" % template['name_len'])[0]

        is_ima_template = name == 'ima'
        if not is_ima_template:
            template['data_len'] = read_unpack(f, "<I")[0]
            print("reading ima len %d" % template['data_len'])
        else:
            template['data_len'] = SHA_DIGEST_LEN + TCG_EVENT_NAME_LEN_MAX + 1

        template['data'] = read_unpack(f, "<%ds" % template['data_len'])[0]

        if is_ima_template:
            field_len = read_unpack(f, "<I")[0]
            extra_data = read_unpack(f, "<%ds" % field_len)[0]
            template['data'] += extra_data

        print("record %s" % template)

        if template['name'] not in list(defined_templates.keys()):
            template['name'] = 'ima'
        template['desc-fmt'] = defined_templates[template['name']]

        # tokens = template['desc-fmt'].split('|')

def _extract_from_ima(tokens, template_hash):
    """ Extract filedata_hash & path form 'ima' entry; return 1 in case error occurred """
    filedata_hash = codecs.decode(tokens[3], "hex")
    path = str(tokens[4])

    # this is some IMA weirdness
    if template_hash == START_HASH:
        return filedata_hash, path, 0

    # verify template hash. yep this is terrible
    # name needs to be null padded out to MAX len. +1 is for the null terminator of the string itself
    fmt = "<%ds%ds%ds" % (len(filedata_hash), len(path), TCG_EVENT_NAME_LEN_MAX - len(path) + 1)
    tohash = struct.pack(fmt,
                         filedata_hash, path.encode("utf-8"),
                         bytearray(TCG_EVENT_NAME_LEN_MAX - len(path) + 1))
    expected_template_hash = hashlib.sha1(tohash).digest()

    if expected_template_hash != template_hash:
        error = 1
        logger.warning("template hash for file %s does not match %s != %s" %
                       (path,
                        codecs.encode(expected_template_hash, 'hex').decode('utf-8'),
                        codecs.encode(template_hash, 'hex').decode('utf-8')))
    else:
        error = 0

    return filedata_hash, path, error

def _extract_from_ima_ng(tokens, template_hash):
    """ Extract filedata_hash & path form 'ima-ng' entry; return 1 in case error occurred """
    filedata = tokens[3]
    ftokens = filedata.split(":")
    filedata_hash = codecs.decode(ftokens[1], 'hex')
    path = str(tokens[4])

    # this is some IMA weirdness
    if template_hash == START_HASH:
        return filedata_hash, path, 0

    filedata_algo = str(ftokens[0])
    # verify template hash. yep this is terrible
    fmt = "<I%dsBB%dsI%dsB" % (len(filedata_algo), len(filedata_hash), len(path))
    # +2 for the : and the null terminator, and +1 on path for null terminator
    tohash = struct.pack(fmt,
                         len(filedata_hash) + len(filedata_algo) + 2,
                         filedata_algo.encode('utf-8'), ord(':'), ord('\0'),
                         filedata_hash,
                         len(path) + 1,
                         path.encode("utf-8"),
                         ord('\0'))
    expected_template_hash = hashlib.sha1(tohash).digest()

    if expected_template_hash != template_hash:
        error = 1
        logger.warning("template hash for file %s does not match %s != %s" %
                       (path,
                        codecs.encode(expected_template_hash, 'hex').decode('utf-8'),
                        codecs.encode(template_hash, 'hex').decode('utf-8')))
    else:
        error = 0

    return filedata_hash, path, error

def _extract_from_ima_sig(tokens, template_hash):
    """ Extract filedata_hash & path form 'ima-sig' entry; return 1 in case error occurred """
    filedata = tokens[3]
    ftokens = filedata.split(":")
    filedata_hash = codecs.decode(ftokens[1], 'hex')
    path = str(tokens[4])

    # this is some IMA weirdness
    if template_hash == START_HASH:
        return filedata_hash, path, None, None, 0

    filedata_algo = str(ftokens[0])
    # verify template hash. yep this is terrible
    fmt = "<I%dsBB%dsI%dsB" % (len(filedata_algo), len(filedata_hash), len(path))
    # +2 for the : and the null terminator, and +1 on path for null terminator
    tohash = struct.pack(fmt,
                         len(filedata_hash) + len(filedata_algo) + 2,
                         filedata_algo.encode('utf-8'), ord(':'), ord('\0'),
                         filedata_hash,
                         len(path) + 1,
                         path.encode("utf-8"),
                         ord('\0'))
    signature = b''
    if len(tokens) == 6:
        signature = codecs.decode(tokens[5], 'hex')
    tohash += struct.pack("<I%ds" % len(signature), len(signature), signature)
    expected_template_hash = hashlib.sha1(tohash).digest()

    if expected_template_hash != template_hash:
        error = 1
        logger.warning("template hash for file %s does not match %s != %s" %
                       (path,
                        codecs.encode(expected_template_hash, 'hex').decode('utf-8'),
                        codecs.encode(template_hash, 'hex').decode('utf-8')))
    else:
        error = 0

    return filedata_hash, path, signature, filedata_algo, error

def process_measurement_list(lines, lists=None, m2w=None, pcrval=None):
    errs = [0, 0, 0, 0]
    runninghash = START_HASH
    found_pcr = (pcrval is None)

    if lists is not None:
        lists = ast.literal_eval(lists)
        allowlist = lists['allowlist']
        exclude_list = lists['exclude']
    else:
        allowlist = None
        exclude_list = None

    is_valid, compiled_regex, err_msg = config.valid_exclude_list(exclude_list)
    if not is_valid:
        # This should not happen as the exclude list has already been validated
        # by the verifier before acceping it. This is a safety net just in case.
        err_msg += " Exclude list will be ignored."
        logger.error(err_msg)

    for line in lines:
        line = line.strip()
        if line == '':
            continue

        tokens = line.split(None, 5)
        if len(tokens) < 5 or len(tokens) > 6:
            logger.error("invalid measurement list file line: -%s-" % (line))
            return None

        # print tokens
        # pcr = tokens[0]
        template_hash = codecs.decode(tokens[1], 'hex')
        mode = tokens[2]

        if mode == "ima-ng":
            filedata_hash, path, error = _extract_from_ima_ng(tokens,
                                                              template_hash)
        elif mode == "ima-sig":
            filedata_hash, path, _, _, error = \
                _extract_from_ima_sig(tokens, template_hash)
        elif mode == 'ima':
            filedata_hash, path, error = _extract_from_ima(tokens,
                                                           template_hash)
        else:
            raise Exception("unsupported ima template mode: %s" % mode)

        errs[0] += error
        if template_hash == START_HASH:
            template_hash = FF_HASH

        # update hash
        runninghash = hashlib.sha1(runninghash + template_hash).digest()

        if not found_pcr:
            found_pcr = \
                (codecs.encode(runninghash, 'hex').decode('utf-8') == pcrval)

        # write out the new hash
        if m2w is not None:
            m2w.write("%s %s\n" %
                      (codecs.encode(filedata_hash, 'hex').decode('utf-8'),
                       path))

        if allowlist is not None:

            # just skip if it is a weird overwritten path
            if template_hash == FF_HASH:
                # print "excluding ffhash %s"%path
                continue

            # determine if path matches any exclusion list items
            if compiled_regex is not None and compiled_regex.match(path):
                logger.debug("IMA: ignoring excluded path %s" % path)
                continue

            accept_list = allowlist.get(path, None)
            if accept_list is None:
                logger.warning("File not found in allowlist: %s" % (path))
                errs[1] += 1
                continue

            # print('codecs.encode', codecs.encode(filedata_hash, 'hex').decode('utf-8'))
            # print('accept_list:', accept_list)
            if codecs.encode(filedata_hash, 'hex').decode('utf-8') not in accept_list:
                logger.warning("Hashes for file %s don't match %s not in %s" %
                               (path,
                                codecs.encode(filedata_hash, 'hex').decode('utf-8'),
                                accept_list))
                errs[2] += 1
                continue

        errs[3] += 1

    # check PCR value has been found
    if not found_pcr:
        logger.error("IMA measurement list does not match TPM PCR %s" % pcrval)
        return None

    # clobber the retval if there were IMA file errors
    if sum(errs[:3]) > 0:
        logger.error(
            "IMA ERRORS: template-hash %d fnf %d hash %d good %d" % tuple(errs))
        return None

    return codecs.encode(runninghash, 'hex').decode('utf-8')

def process_allowlists(al_data, excl_data):
    # Pull in default config values if not specified
    if al_data is None:
        al_data = read_allowlist()
    if excl_data is None:
        excl_data = read_excllist()

    allowlist = {}
    for line in al_data:
        line = line.strip()
        tokens = line.split(None, 1)
        if len(tokens) != 2:
            continue
        fhash = tokens[0]
        path = str(tokens[1])
        tmp = allowlist.get(path, [])
        tmp.append(fhash)
        allowlist[path] = tmp

    if allowlist.get('boot_aggregate', None) is None:
        logger.warning(
            "No boot_aggregate value found in allowlist, adding an empty one")
        allowlist['boot_aggregate'] = [
            '0000000000000000000000000000000000000000']

    for excl in excl_data:
        if excl.startswith("#"):
            excl_data.remove(excl)
        # don't allow empty lines in exclude list, it will match everything
        if excl == "":
            excl_data.remove(excl)

    return{'allowlist': allowlist, 'exclude': excl_data}

def read_allowlist(al_path=None):
    if al_path is None:
        al_path = config.get('tenant', 'ima_allowlist')
        if config.STUB_IMA:
            al_path = '../scripts/ima/allowlist.txt'

    # Purposefully die if path doesn't exist
    with open(al_path, 'r') as f:
        alist = f.read()
    alist = alist.splitlines()

    logger.debug("Loaded allowlist from %s" % (al_path))

    return alist

def read_excllist(exclude_path=None):
    if exclude_path is None:
        exclude_path = config.get('tenant', 'ima_excludelist')
        if config.STUB_IMA:
            exclude_path = '../scripts/ima/exclude.txt'

    excl_list = []
    if os.path.exists(exclude_path):
        with open(exclude_path, 'r') as f:
            excl_list = f.read()
        excl_list = excl_list.splitlines()

        logger.debug("Loaded exclusion list from %s: %s" %
                     (exclude_path, excl_list))

    return excl_list

def main():
    # read_measurement_list_bin("/sys/kernel/security/ima/binary_runtime_measurements")

    allowlist_path = 'allowlist.txt'
    print("reading allowlist from %s" % allowlist_path)

    exclude_path = 'exclude.txt'
    # exclude_path = '../scripts/ima/exclude.txt'
    print("reading exclude list from %s" % exclude_path)

    al_data = read_allowlist(allowlist_path)
    excl_data = read_excllist(exclude_path)
    lists = process_allowlists(al_data, excl_data)

    measure_path = config.IMA_ML
    # measure_path='../scripts/ima/ascii_runtime_measurements_ima'
    # measure_path = '../scripts/gerardo/ascii_runtime_measurements'
    print("reading measurement list from %s" % measure_path)
    f = open(measure_path, 'r')
    lines = f.readlines()

    m2a = open('measure2allow.txt', "w")
    digest = process_measurement_list(lines, lists, m2a)
    print("final digest is %s" % digest)
    f.close()
    m2a.close()

    print("using m2a")

    al_data = read_allowlist('measure2allow.txt')
    excl_data = read_excllist(exclude_path)
    lists2 = process_allowlists(al_data, excl_data)
    process_measurement_list(lines, lists2)

    print("done")

if __name__ == "__main__":
    try:
        main()
    except Exception as e:
        logger.exception(e)
<|endoftext|>