<commit_before>    def __del__(self):
        try:
           for cb,opaque in self.domainEventCallbacks.items():
               del self.domainEventCallbacks[cb]
           self.domainEventCallbacks = None
           libvirtmod.virConnectDomainEventDeregister(self._o, self)
        except AttributeError:
           pass

        if self._o != None:
            libvirtmod.virConnectClose(self._o)
        self._o = None

    def domainEventDeregister(self, cb):
        """Removes a Domain Event Callback. De-registering for a
           domain callback will disable delivery of this event type """
        try:
            del self.domainEventCallbacks[cb]
            if len(self.domainEventCallbacks) == 0:
                ret = libvirtmod.virConnectDomainEventDeregister(self._o, self)
                if ret == -1: raise libvirtError ('virConnectDomainEventDeregister() failed', conn=self)
        except AttributeError:
            pass

    def domainEventRegister(self, cb, opaque):
        """Adds a Domain Event Callback. Registering for a domain
           callback will enable delivery of the events """
        try:
            self.domainEventCallbacks[cb] = opaque
        except AttributeError:
            self.domainEventCallbacks = {cb:opaque}
            ret = libvirtmod.virConnectDomainEventRegister(self._o, self)
            if ret == -1: raise libvirtError ('virConnectDomainEventRegister() failed', conn=self)

    def dispatchDomainEventCallbacks(self, dom, event, detail):
        """Dispatches events to python user domain event callbacks
        """
        try:
            for cb,opaque in self.domainEventCallbacks.items():
                cb(self,dom,event,detail,opaque)
            return 0
        except AttributeError:
            pass

    def dispatchDomainEventLifecycleCallback(self, dom, event, detail, cbData):
        """Dispatches events to python user domain lifecycle event callbacks
        """
        cb = cbData["cb"]
        opaque = cbData["opaque"]

        cb(self, virDomain(self, _obj=dom), event, detail, opaque)
        return 0

    def dispatchDomainEventGenericCallback(self, dom, cbData):
        """Dispatches events to python user domain generic event callbacks
        """
        try:
            cb = cbData["cb"]
            opaque = cbData["opaque"]

            cb(self, virDomain(self, _obj=dom), opaque)
            return 0
        except AttributeError:
            pass

    def dispatchDomainEventRTCChangeCallback(self, dom, offset, cbData):
        """Dispatches events to python user domain RTC change event callbacks
        """
        try:
            cb = cbData["cb"]
            opaque = cbData["opaque"]

            cb(self, virDomain(self, _obj=dom), offset ,opaque)
            return 0
        except AttributeError:
            pass

    def dispatchDomainEventWatchdogCallback(self, dom, action, cbData):
        """Dispatches events to python user domain watchdog event callbacks
        """
        try:
            cb = cbData["cb"]
            opaque = cbData["opaque"]

            cb(self, virDomain(self, _obj=dom), action, opaque)
            return 0
        except AttributeError:
            pass

    def dispatchDomainEventIOErrorCallback(self, dom, srcPath, devAlias, action, cbData):
        """Dispatches events to python user domain IO error event callbacks
        """
        try:
            cb = cbData["cb"]
            opaque = cbData["opaque"]

            cb(self, virDomain(self, _obj=dom), srcPath, devAlias, opaque)
            return 0
        except AttributeError:
            pass

    def dispatchDomainEventIOErrorReasonCallback(self, dom, srcPath, devAlias, action, reason, cbData):
        """Dispatches events to python user domain IO error event callbacks
        """
        try:
            cb = cbData["cb"]
            opaque = cbData["opaque"]

            cb(self, virDomain(self, _obj=dom), srcPath, devAlias, action, reason, opaque)
            return 0
        except AttributeError:
            pass

    def dispatchDomainEventGraphicsCallback(self, dom, phase, localAddr, remoteAddr, authScheme, subject, cbData):
        """Dispatches events to python user domain graphics event callbacks
        """
        try:
            cb = cbData["cb"]
            opaque = cbData["opaque"]

            cb(self, virDomain(self, _obj=dom), phase, localAddr, remoteAddr, authScheme, subject, opaque)
            return 0
        except AttributeError:
            pass

    def domainEventDeregisterAny(self, callbackID):
        """Removes a Domain Event Callback. De-registering for a
           domain callback will disable delivery of this event type """
        try:
            ret = libvirtmod.virConnectDomainEventDeregisterAny(self._o, callbackID)
            if ret == -1: raise libvirtError ('virConnectDomainEventDeregisterAny() failed', conn=self)
            del self.domainEventCallbackID[callbackID]
        except AttributeError:
            pass

    def domainEventRegisterAny(self, dom, eventID, cb, opaque):
        """Adds a Domain Event Callback. Registering for a domain
           callback will enable delivery of the events """
        if not hasattr(self, 'domainEventCallbackID'):
            self.domainEventCallbackID = {}
        cbData = { "cb": cb, "conn": self, "opaque": opaque }
        if dom is None:
            ret = libvirtmod.virConnectDomainEventRegisterAny(self._o, None, eventID, cbData)
        else:
            ret = libvirtmod.virConnectDomainEventRegisterAny(self._o, dom._o, eventID, cbData)
        if ret == -1:
            raise libvirtError ('virConnectDomainEventRegisterAny() failed', conn=self)
        self.domainEventCallbackID[ret] = opaque
<commit_msg>Add missing parameter in python Disk IO error callback<commit_after>    def __del__(self):
        try:
           for cb,opaque in self.domainEventCallbacks.items():
               del self.domainEventCallbacks[cb]
           self.domainEventCallbacks = None
           libvirtmod.virConnectDomainEventDeregister(self._o, self)
        except AttributeError:
           pass

        if self._o != None:
            libvirtmod.virConnectClose(self._o)
        self._o = None

    def domainEventDeregister(self, cb):
        """Removes a Domain Event Callback. De-registering for a
           domain callback will disable delivery of this event type """
        try:
            del self.domainEventCallbacks[cb]
            if len(self.domainEventCallbacks) == 0:
                ret = libvirtmod.virConnectDomainEventDeregister(self._o, self)
                if ret == -1: raise libvirtError ('virConnectDomainEventDeregister() failed', conn=self)
        except AttributeError:
            pass

    def domainEventRegister(self, cb, opaque):
        """Adds a Domain Event Callback. Registering for a domain
           callback will enable delivery of the events """
        try:
            self.domainEventCallbacks[cb] = opaque
        except AttributeError:
            self.domainEventCallbacks = {cb:opaque}
            ret = libvirtmod.virConnectDomainEventRegister(self._o, self)
            if ret == -1: raise libvirtError ('virConnectDomainEventRegister() failed', conn=self)

    def dispatchDomainEventCallbacks(self, dom, event, detail):
        """Dispatches events to python user domain event callbacks
        """
        try:
            for cb,opaque in self.domainEventCallbacks.items():
                cb(self,dom,event,detail,opaque)
            return 0
        except AttributeError:
            pass

    def dispatchDomainEventLifecycleCallback(self, dom, event, detail, cbData):
        """Dispatches events to python user domain lifecycle event callbacks
        """
        cb = cbData["cb"]
        opaque = cbData["opaque"]

        cb(self, virDomain(self, _obj=dom), event, detail, opaque)
        return 0

    def dispatchDomainEventGenericCallback(self, dom, cbData):
        """Dispatches events to python user domain generic event callbacks
        """
        try:
            cb = cbData["cb"]
            opaque = cbData["opaque"]

            cb(self, virDomain(self, _obj=dom), opaque)
            return 0
        except AttributeError:
            pass

    def dispatchDomainEventRTCChangeCallback(self, dom, offset, cbData):
        """Dispatches events to python user domain RTC change event callbacks
        """
        try:
            cb = cbData["cb"]
            opaque = cbData["opaque"]

            cb(self, virDomain(self, _obj=dom), offset ,opaque)
            return 0
        except AttributeError:
            pass

    def dispatchDomainEventWatchdogCallback(self, dom, action, cbData):
        """Dispatches events to python user domain watchdog event callbacks
        """
        try:
            cb = cbData["cb"]
            opaque = cbData["opaque"]

            cb(self, virDomain(self, _obj=dom), action, opaque)
            return 0
        except AttributeError:
            pass

    def dispatchDomainEventIOErrorCallback(self, dom, srcPath, devAlias, action, cbData):
        """Dispatches events to python user domain IO error event callbacks
        """
        try:
            cb = cbData["cb"]
            opaque = cbData["opaque"]

            cb(self, virDomain(self, _obj=dom), srcPath, devAlias, action, opaque)
            return 0
        except AttributeError:
            pass

    def dispatchDomainEventIOErrorReasonCallback(self, dom, srcPath, devAlias, action, reason, cbData):
        """Dispatches events to python user domain IO error event callbacks
        """
        try:
            cb = cbData["cb"]
            opaque = cbData["opaque"]

            cb(self, virDomain(self, _obj=dom), srcPath, devAlias, action, reason, opaque)
            return 0
        except AttributeError:
            pass

    def dispatchDomainEventGraphicsCallback(self, dom, phase, localAddr, remoteAddr, authScheme, subject, cbData):
        """Dispatches events to python user domain graphics event callbacks
        """
        try:
            cb = cbData["cb"]
            opaque = cbData["opaque"]

            cb(self, virDomain(self, _obj=dom), phase, localAddr, remoteAddr, authScheme, subject, opaque)
            return 0
        except AttributeError:
            pass

    def domainEventDeregisterAny(self, callbackID):
        """Removes a Domain Event Callback. De-registering for a
           domain callback will disable delivery of this event type """
        try:
            ret = libvirtmod.virConnectDomainEventDeregisterAny(self._o, callbackID)
            if ret == -1: raise libvirtError ('virConnectDomainEventDeregisterAny() failed', conn=self)
            del self.domainEventCallbackID[callbackID]
        except AttributeError:
            pass

    def domainEventRegisterAny(self, dom, eventID, cb, opaque):
        """Adds a Domain Event Callback. Registering for a domain
           callback will enable delivery of the events """
        if not hasattr(self, 'domainEventCallbackID'):
            self.domainEventCallbackID = {}
        cbData = { "cb": cb, "conn": self, "opaque": opaque }
        if dom is None:
            ret = libvirtmod.virConnectDomainEventRegisterAny(self._o, None, eventID, cbData)
        else:
            ret = libvirtmod.virConnectDomainEventRegisterAny(self._o, dom._o, eventID, cbData)
        if ret == -1:
            raise libvirtError ('virConnectDomainEventRegisterAny() failed', conn=self)
        self.domainEventCallbackID[ret] = opaque
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import sys
import os
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..', 'python-hosts')))
from hostman import output_message, import_from_file, import_from_url, add, backup_hosts, strip_entry_value, remove
import pytest

def test_output_message_with_failed():
    with pytest.raises(SystemExit) as cm:
        output_message({'result': 'failed', 'message': 'test failed'})
    assert cm.value.code == 1

def test_output_message_with_success():
    with pytest.raises(SystemExit) as cm:
        output_message({'result': 'success', 'message': 'test success'})
    assert cm.value.code == 0

def test_add_add_duplicate_ipv4_host_with_force(tmpdir):
    """
    Test that an ipv4 type host is replaced with force set
    """
    hosts_file = tmpdir.mkdir("etc").join("hosts")
    hosts_file.write("3.4.5.6\tlocalhost\n")
    new_entry = '3.4.5.6 bob jane.com'
    output = add(entry_line=new_entry, hosts_path=hosts_file.strpath, force_add=True)
    assert output.get('result') == 'success'
    assert output.get('message').startswith('Entry added. Matching entries replaced.')

def test_import_hosts_with_invalid_hosts_path(tmpdir):
    """
    Test the import of a file where the hosts path is invalid
    """
    hosts_file = tmpdir.mkdir("etc").join("hosts")
    hosts_file.write("3.4.5.6\tlocalhost\n")
    import_file = tmpdir.mkdir("tmp").join("in")
    import_file.write("8.8.8.8\tgoogledns\n")
    result = import_from_file(hosts_path='invalid',
                              file_path=import_file.strpath)
    assert result.get('message').startswith('Cannot read hosts file:')

def test_import_hosts_with_invalid_import_file_path(tmpdir):
    """
    Test the import of a file where the import file path is invalid
    """
    hosts_file = tmpdir.mkdir("etc").join("hosts")
    hosts_file.write("3.4.5.6\tlocalhost\n")
    import_file = tmpdir.mkdir("tmp").join("in")
    import_file.write("8.8.8.8\tgoogledns\n")
    result = import_from_file(hosts_path=hosts_file.strpath,
                              file_path='invalid')
    assert result.get('message').startswith('Cannot read import file:')

def test_add_duplicate_ipv4_host_without_force(tmpdir):
    """
    Test that nothing is added when attempt to replace a similar
    entry without using force option
    """
    hosts_file = tmpdir.mkdir("etc").join("hosts")
    hosts_file.write("9.9.9.9\tgeorge\n")
    new_entry = '9.9.9.9 george bungle.com'
    output = add(entry_line=new_entry, hosts_path=hosts_file.strpath, force_add=False)
    assert output.get('result') == 'failed'
    assert output.get('message').startswith('New entry matches')

def test_add_new_ipv4_host(tmpdir):
    """
    Test successful addition of an ipv4 host
    """
    hosts_file = tmpdir.mkdir("etc").join("hosts")
    hosts_file.write("127.0.0.1\tlocalhost\n")
    new_entry = '3.4.5.6 bob jane.com'
    output = add(entry_line=new_entry, hosts_path=hosts_file.strpath, force_add=False)
    assert output.get('result') == 'success'

def test_backup_hosts_file(tmpdir):
    """
    Test hosts file backup
    """
    hosts_file = tmpdir.mkdir("etc").join("hosts")
    hosts_file.write("127.0.0.1\tlocalhost\n")
    test_extension = 'test'
    output = backup_hosts(source=hosts_file.strpath, extension=test_extension)
    assert 'Backup written to:' in output.get('message')

def test_backup_hosts_file_fails_with_invalid_source(tmpdir):
    """
    Test backup fails with an invalid source
    """
    hosts_file = tmpdir.mkdir("etc").join("hosts")
    hosts_file.write("127.0.0.1\tlocalhost\n")
    output = backup_hosts(source="invalid")
    assert 'Cannot create backup file:' in output.get('message')

# def test_failure_with_invalid_host_entry():
#    with pytest.raises(SystemExit) as cm:
#        hostman.add(entry_line='256.255.255.255 badaddr.com')
#    assert cm.value.code == 1

# def test_output_message_with_failure():
#    with pytest.raises(SystemExit) as cm:
#       hostman.output_message({'result': 'failed', 'message': 'test failure'})
#    assert cm.value.code == 1

# def test_output_message_with_failure():
#    with pytest.raises(SystemExit) as cm:
#        hostman.output_message({'result': 'failed', 'message': 'test failure'})
#    assert cm.value.code == 1

# def test_output_message_with_success():
#    with pytest.raises(SystemExit) as cm:
#        hostman.output_message({'result': 'success', 'message': 'test success'})
#    assert cm.value.code == 0

def test_output_message_with_continue():
    assert output_message({'result': 'continue', 'message': 'test continue'})

def test_import_hosts_from_file(tmpdir):
    """
    Test the import of a file
    """
    hosts_file = tmpdir.mkdir("etc").join("hosts")
    hosts_file.write("127.0.0.1\tlocalhost\n")
    import_file = tmpdir.mkdir("tmp").join("in")
    import_file.write("8.8.8.8\tgoogledns\n")
    assert import_from_file(hosts_path=hosts_file.strpath,
                            file_path=import_file.strpath)

def test_import_hosts_from_url(tmpdir):
    """
    Test the import of a url
    """
    hosts_file = tmpdir.mkdir("etc").join("hosts")
    hosts_file.write("127.0.0.1\tlocalhost\n")
    import_url = "https://dl.dropboxusercontent.com/u/167103/hosts"
    assert import_from_url(hosts_path=hosts_file.strpath, url=import_url)

def test_removal_of_entry(tmpdir):
    hosts_file = tmpdir.mkdir("etc").join("hosts")
    hosts_file.write("15.15.15.15\texample.com\n16.16.16.16\ttest.com\n")
    assert remove(address_to_remove='15.15.15.15', remove_from_path=hosts_file.strpath)

def test_stripping():
    string_with_spaces = " test "
    list_string_with_spaces = [" example.com ", "example "]
    assert strip_entry_value(list_string_with_spaces) == 'example.com example'
    assert strip_entry_value(string_with_spaces) == 'test'
<commit_msg>add test for invalid host entry.<commit_after># -*- coding: utf-8 -*-
import sys
import os
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '../..', 'python-hosts')))
from hostman import output_message, import_from_file, import_from_url, add, backup_hosts, strip_entry_value, remove
import pytest

def test_output_message_with_failed():
    with pytest.raises(SystemExit) as cm:
        output_message({'result': 'failed', 'message': 'test failed'})
    assert cm.value.code == 1

def test_output_message_with_success():
    with pytest.raises(SystemExit) as cm:
        output_message({'result': 'success', 'message': 'test success'})
    assert cm.value.code == 0

def test_add_add_duplicate_ipv4_host_with_force(tmpdir):
    """
    Test that an ipv4 type host is replaced with force set
    """
    hosts_file = tmpdir.mkdir("etc").join("hosts")
    hosts_file.write("3.4.5.6\tlocalhost\n")
    new_entry = '3.4.5.6 bob jane.com'
    output = add(entry_line=new_entry, hosts_path=hosts_file.strpath, force_add=True)
    assert output.get('result') == 'success'
    assert output.get('message').startswith('Entry added. Matching entries replaced.')

def test_import_hosts_with_invalid_hosts_path(tmpdir):
    """
    Test the import of a file where the hosts path is invalid
    """
    hosts_file = tmpdir.mkdir("etc").join("hosts")
    hosts_file.write("3.4.5.6\tlocalhost\n")
    import_file = tmpdir.mkdir("tmp").join("in")
    import_file.write("8.8.8.8\tgoogledns\n")
    result = import_from_file(hosts_path='invalid',
                              file_path=import_file.strpath)
    assert result.get('message').startswith('Cannot read hosts file:')

def test_import_hosts_with_invalid_import_file_path(tmpdir):
    """
    Test the import of a file where the import file path is invalid
    """
    hosts_file = tmpdir.mkdir("etc").join("hosts")
    hosts_file.write("3.4.5.6\tlocalhost\n")
    import_file = tmpdir.mkdir("tmp").join("in")
    import_file.write("8.8.8.8\tgoogledns\n")
    result = import_from_file(hosts_path=hosts_file.strpath,
                              file_path='invalid')
    assert result.get('message').startswith('Cannot read import file:')

def test_add_duplicate_ipv4_host_without_force(tmpdir):
    """
    Test that nothing is added when attempt to replace a similar
    entry without using force option
    """
    hosts_file = tmpdir.mkdir("etc").join("hosts")
    hosts_file.write("9.9.9.9\tgeorge\n")
    new_entry = '9.9.9.9 george bungle.com'
    output = add(entry_line=new_entry, hosts_path=hosts_file.strpath, force_add=False)
    assert output.get('result') == 'failed'
    assert output.get('message').startswith('New entry matches')

def test_add_new_ipv4_host(tmpdir):
    """
    Test successful addition of an ipv4 host
    """
    hosts_file = tmpdir.mkdir("etc").join("hosts")
    hosts_file.write("127.0.0.1\tlocalhost\n")
    new_entry = '3.4.5.6 bob jane.com'
    output = add(entry_line=new_entry, hosts_path=hosts_file.strpath, force_add=False)
    assert output.get('result') == 'success'

def test_backup_hosts_file(tmpdir):
    """
    Test hosts file backup
    """
    hosts_file = tmpdir.mkdir("etc").join("hosts")
    hosts_file.write("127.0.0.1\tlocalhost\n")
    test_extension = 'test'
    output = backup_hosts(source=hosts_file.strpath, extension=test_extension)
    assert 'Backup written to:' in output.get('message')

def test_backup_hosts_file_fails_with_invalid_source(tmpdir):
    """
    Test backup fails with an invalid source
    """
    hosts_file = tmpdir.mkdir("etc").join("hosts")
    hosts_file.write("127.0.0.1\tlocalhost\n")
    output = backup_hosts(source="invalid")
    assert 'Cannot create backup file:' in output.get('message')

def test_failure_with_invalid_host_entry():
    with pytest.raises(SystemExit) as cm:
        add(entry_line='256.255.255.255 badaddr.com')
    assert cm.value.code == 1

# def test_output_message_with_failure():
#    with pytest.raises(SystemExit) as cm:
#       hostman.output_message({'result': 'failed', 'message': 'test failure'})
#    assert cm.value.code == 1

# def test_output_message_with_failure():
#    with pytest.raises(SystemExit) as cm:
#        hostman.output_message({'result': 'failed', 'message': 'test failure'})
#    assert cm.value.code == 1

# def test_output_message_with_success():
#    with pytest.raises(SystemExit) as cm:
#        hostman.output_message({'result': 'success', 'message': 'test success'})
#    assert cm.value.code == 0

def test_output_message_with_continue():
    assert output_message({'result': 'continue', 'message': 'test continue'})

def test_import_hosts_from_file(tmpdir):
    """
    Test the import of a file
    """
    hosts_file = tmpdir.mkdir("etc").join("hosts")
    hosts_file.write("127.0.0.1\tlocalhost\n")
    import_file = tmpdir.mkdir("tmp").join("in")
    import_file.write("8.8.8.8\tgoogledns\n")
    assert import_from_file(hosts_path=hosts_file.strpath,
                            file_path=import_file.strpath)

def test_import_hosts_from_url(tmpdir):
    """
    Test the import of a url
    """
    hosts_file = tmpdir.mkdir("etc").join("hosts")
    hosts_file.write("127.0.0.1\tlocalhost\n")
    import_url = "https://dl.dropboxusercontent.com/u/167103/hosts"
    assert import_from_url(hosts_path=hosts_file.strpath, url=import_url)

def test_removal_of_entry(tmpdir):
    hosts_file = tmpdir.mkdir("etc").join("hosts")
    hosts_file.write("15.15.15.15\texample.com\n16.16.16.16\ttest.com\n")
    assert remove(address_to_remove='15.15.15.15', remove_from_path=hosts_file.strpath)

def test_stripping():
    string_with_spaces = " test "
    list_string_with_spaces = [" example.com ", "example "]
    assert strip_entry_value(list_string_with_spaces) == 'example.com example'
    assert strip_entry_value(string_with_spaces) == 'test'
<|endoftext|>


<commit_before>"""
Range of time stuff
"""

import bisect
import operator
import collections

VERSION = (0, 0, 2)

__all__ = ('INFINITY', 'NEGATIVE_INFINITY',
           'RangeSet')

class _Indeterminate(object):
    pass

class _Infinity(_Indeterminate):
    def __cmp__(self, other):
        return 0 if other is self else 1
    def __str__(self):
        return 'inf'

class _NegativeInfinity(_Indeterminate):
    def __cmp__(self, other):
        return 0 if other is self else -1
    def __str__(self):
        return '-inf'

INFINITY = _Infinity()
NEGATIVE_INFINITY = _NegativeInfinity()

_parent = collections.namedtuple('RangeSet_', ['ends'])

class RangeSet(_parent):
    def __new__(cls, start, end):
        if end is _RAW_ENDS:
            ends = start
        else:
            if start > end:
                start, end = end, start
            ends = ((start, _START), (end, _END))
        return _parent.__new__(cls, ends)

    def __merged_ends(self, *others):
        data = (self,) + others
        sorted_ends = list(reduce(operator.add,
                                  (RangeSet.__promote(x).ends for x in data)))
        sorted_ends.sort()
        return sorted_ends

    @classmethod
    def __promote(cls, value):
        if isinstance(value, RangeSet):
            return value
        else:
            return RangeSet(value[0], value[1])

    @classmethod
    def __iterate_state(cls, ends):
        state = 0
        for _, end in ends:
            if end == _START:
                state += 1
            else:
                state -= 1
            yield _, end, state

    def __or__(self, *other):
        sorted_ends = self.__merged_ends(*other)
        new_ends = []
        for _, end, state in RangeSet.__iterate_state(sorted_ends):
            if state > 1 and end == _START:
                continue
            elif state > 0 and end == _END:
                continue
            new_ends.append((_, end))
        return RangeSet(tuple(new_ends), _RAW_ENDS)

    union = __or__

    def __and__(self, *other):
        sorted_ends = self.__merged_ends(*other)
        new_ends = []
        for _, end, state in RangeSet.__iterate_state(sorted_ends):
            if state == 2 and end == _START:
                new_ends.append((_, end))
            elif state == 1 and end == _END:
                new_ends.append((_, end))
        return RangeSet(tuple(new_ends), _RAW_ENDS)

    intersect = __and__

    def __ror__(self, other):
        return self.__or__(other)

    def __rand__(self, other):
        return self.__rand__(other)

    def __xor__(self, *other):
        sorted_ends = self.__merged_ends(*other)
        new_ends = []
        old_val = None
        for _, end, state in RangeSet.__iterate_state(sorted_ends):
            if state == 2 and end == _START:
                new_ends.append((_, _NEGATE[end]))
            elif state == 1 and end == _END:
                new_ends.append((_, _NEGATE[end]))
            elif state == 1 and end == _START:
                new_ends.append((_, end))
            elif state == 0 and end == _END:
                new_ends.append((_, end))
        return RangeSet(tuple(new_ends), _RAW_ENDS)

    symmetric_difference = __xor__

    def __contains__(self, test):
        last_val, last_end = None, None
        for _, end, state in RangeSet.__iterate_state(self.ends):
            if last_val is not None and _ > test:
                return last_end == _START
            elif _ > test:
                return False
            last_val, last_end = _, end
        return False

    def issuperset(self, test):
        if isinstance(test, RangeSet):
            rangeset = test
        else:
            rangeset = RangeSet.__promote(test)
        difference = rangeset - ~self
        return difference == rangeset

    __ge__ = issuperset

    def __gt__(self, other):
        return self != other and self >= other

    def issubset(self, other):
        return RangeSet.__promote(other).issuperset(self)

    __le__ = issubset

    def __lt__(self, other):
        return self != other and self <= other

    def isdisjoint(self, other):
        return not bool(self & other)

    def __nonzero__(self):
        return bool(self.ends)

    def __invert__(self):
        if not self.ends:
            new_ends = ((NEGATIVE_INFINITY, _START),
                        (INFINITY, _END))
            return RangeSet(new_ends, _RAW_ENDS)
        new_ends = list(self.ends)
        head, tail = [], []
        if new_ends[0][0] == NEGATIVE_INFINITY:
            new_ends.pop(0)
        else:
            head = [(NEGATIVE_INFINITY, _START)]
        if new_ends[-1][0] == INFINITY:
            new_ends.pop(-1)
        else:
            tail = [(INFINITY, _END)]
        for i, value in enumerate(new_ends):
            new_ends[i] = (value[0], _NEGATE[value[1]])
        return RangeSet(tuple(head + new_ends + tail), _RAW_ENDS)

    invert = __invert__

    def __sub__(self, other):
        return self & ~other

    def difference(self, other):
        return self.__sub__(other)

    def __rsub__(self, other):
        return RangeSet.__promote(other) - self

    def __len__(self):
        return self.measure()

    def measure(self):
        if not self.ends:
            return 0
        if isinstance(self.ends[0][0], _Indeterminate) or isinstance(self.ends[-1][0], _Indeterminate):
            raise ValueError("Cannot compute range with unlimited bounds.")
        return reduce(operator.add, (self.ends[i + 1][0] - self.ends[i][0] for i in range(0, len(self.ends), 2)))

    def range(self):
        if not self.ends:
            return 0
        if isinstance(self.ends[0][0], _Indeterminate) or isinstance(self.ends[-1][0], _Indeterminate):
            raise ValueError("Cannot compute range with unlimited bounds.")
        return self.ends[-1][0] - self.ends[0][0]

    def __str__(self):
        pieces = ["{} -- {}".format(self.ends[i][0], self.ends[i + 1][0])
                                    for i in range(0, len(self.ends), 2)]
        return "<RangeSet {}>".format(", ".join(pieces))

    __repr__ = __str__

    def __eq__(self, other):
        if self is other:
            return True
        elif not isinstance(other, RangeSet):
            try:
                other = RangeSet.__promote(other)
            except TypeError:
                return False
        return self.ends == other.ends

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return hash(self.ends)

    @classmethod
    def mutual_overlaps(cls, *ranges):
        return cls.__promote(ranges[0]).intersect(*ranges[1:])

    @classmethod
    def mutual_union(cls, *ranges):
        return cls.__promote(ranges[0]).union(*ranges[1:])

_START = -1
_END = 1

_NEGATE = {_START: _END, _END: _START}

_RAW_ENDS = object()
<commit_msg>Added more documentation.<commit_after>"""
Range of time stuff
"""

import bisect
import operator
import collections

VERSION = (0, 0, 2)

__all__ = ('INFINITY', 'NEGATIVE_INFINITY',
           'RangeSet')

class _Indeterminate(object):
    pass

class _Infinity(_Indeterminate):
    def __cmp__(self, other):
        return 0 if other is self else 1
    def __str__(self):
        return 'inf'

class _NegativeInfinity(_Indeterminate):
    def __cmp__(self, other):
        return 0 if other is self else -1
    def __str__(self):
        return '-inf'

INFINITY = _Infinity()
NEGATIVE_INFINITY = _NegativeInfinity()

_parent = collections.namedtuple('RangeSet_', ['ends'])

class RangeSet(_parent):
    def __new__(cls, start, end):
        if end is _RAW_ENDS:
            ends = start
        else:
            if start > end:
                start, end = end, start
            ends = ((start, _START), (end, _END))
        return _parent.__new__(cls, ends)

    def __merged_ends(self, *others):
        data = (self,) + others
        sorted_ends = list(reduce(operator.add,
                                  (RangeSet.__promote(x).ends for x in data)))
        sorted_ends.sort()
        return sorted_ends

    @classmethod
    def __promote(cls, value):
        if isinstance(value, RangeSet):
            return value
        else:
            return RangeSet(value[0], value[1])

    @classmethod
    def __iterate_state(cls, ends):
        state = 0
        for _, end in ends:
            if end == _START:
                state += 1
            else:
                state -= 1
            yield _, end, state

    def __or__(self, *other):
        sorted_ends = self.__merged_ends(*other)
        new_ends = []
        for _, end, state in RangeSet.__iterate_state(sorted_ends):
            if state > 1 and end == _START:
                continue
            elif state > 0 and end == _END:
                continue
            new_ends.append((_, end))
        return RangeSet(tuple(new_ends), _RAW_ENDS)

    union = __or__

    def __and__(self, *other):
        sorted_ends = self.__merged_ends(*other)
        new_ends = []
        for _, end, state in RangeSet.__iterate_state(sorted_ends):
            if state == 2 and end == _START:
                new_ends.append((_, end))
            elif state == 1 and end == _END:
                new_ends.append((_, end))
        return RangeSet(tuple(new_ends), _RAW_ENDS)

    intersect = __and__

    def __ror__(self, other):
        return self.__or__(other)

    def __rand__(self, other):
        return self.__rand__(other)

    def __xor__(self, *other):
        sorted_ends = self.__merged_ends(*other)
        new_ends = []
        old_val = None
        for _, end, state in RangeSet.__iterate_state(sorted_ends):
            if state == 2 and end == _START:
                new_ends.append((_, _NEGATE[end]))
            elif state == 1 and end == _END:
                new_ends.append((_, _NEGATE[end]))
            elif state == 1 and end == _START:
                new_ends.append((_, end))
            elif state == 0 and end == _END:
                new_ends.append((_, end))
        return RangeSet(tuple(new_ends), _RAW_ENDS)

    symmetric_difference = __xor__

    def __contains__(self, test):
        last_val, last_end = None, None
        for _, end, state in RangeSet.__iterate_state(self.ends):
            if last_val is not None and _ > test:
                return last_end == _START
            elif _ > test:
                return False
            last_val, last_end = _, end
        return False

    def issuperset(self, test):
        if isinstance(test, RangeSet):
            rangeset = test
        else:
            rangeset = RangeSet.__promote(test)
        difference = rangeset - ~self
        return difference == rangeset

    __ge__ = issuperset

    def __gt__(self, other):
        return self != other and self >= other

    def issubset(self, other):
        return RangeSet.__promote(other).issuperset(self)

    __le__ = issubset

    def __lt__(self, other):
        return self != other and self <= other

    def isdisjoint(self, other):
        return not bool(self & other)

    def __nonzero__(self):
        return bool(self.ends)

    def __invert__(self):
        if not self.ends:
            new_ends = ((NEGATIVE_INFINITY, _START),
                        (INFINITY, _END))
            return RangeSet(new_ends, _RAW_ENDS)
        new_ends = list(self.ends)
        head, tail = [], []
        if new_ends[0][0] == NEGATIVE_INFINITY:
            new_ends.pop(0)
        else:
            head = [(NEGATIVE_INFINITY, _START)]
        if new_ends[-1][0] == INFINITY:
            new_ends.pop(-1)
        else:
            tail = [(INFINITY, _END)]
        for i, value in enumerate(new_ends):
            new_ends[i] = (value[0], _NEGATE[value[1]])
        return RangeSet(tuple(head + new_ends + tail), _RAW_ENDS)

    invert = __invert__

    def __sub__(self, other):
        return self & ~other

    def difference(self, other):
        return self.__sub__(other)

    def __rsub__(self, other):
        return RangeSet.__promote(other) - self

    def __len__(self):
        return self.measure()

    def measure(self):
        if not self.ends:
            return 0
        if isinstance(self.ends[0][0], _Indeterminate) or isinstance(self.ends[-1][0], _Indeterminate):
            raise ValueError("Cannot compute range with unlimited bounds.")
        return reduce(operator.add, (self.ends[i + 1][0] - self.ends[i][0] for i in range(0, len(self.ends), 2)))

    def range(self):
        if not self.ends:
            return 0
        if isinstance(self.ends[0][0], _Indeterminate) or isinstance(self.ends[-1][0], _Indeterminate):
            raise ValueError("Cannot compute range with unlimited bounds.")
        return self.ends[-1][0] - self.ends[0][0]

    def __str__(self):
        pieces = ["{} -- {}".format(self.ends[i][0], self.ends[i + 1][0])
                                    for i in range(0, len(self.ends), 2)]
        return "<RangeSet {}>".format(", ".join(pieces))

    __repr__ = __str__

    def __eq__(self, other):
        if self is other:
            return True
        elif not isinstance(other, RangeSet):
            try:
                other = RangeSet.__promote(other)
            except TypeError:
                return False
        return self.ends == other.ends

    def __ne__(self, other):
        return not self.__eq__(other)

    def __hash__(self):
        return hash(self.ends)

    @classmethod
    def mutual_overlaps(cls, *ranges):
        return cls.__promote(ranges[0]).intersect(*ranges[1:])

    @classmethod
    def mutual_union(cls, *ranges):
        return cls.__promote(ranges[0]).union(*ranges[1:])

    @property
    def min(self):
        return self.ends[0][0]

    @property
    def max(self):
        return self.ends[-1][0]

_START = -1
_END = 1

_NEGATE = {_START: _END, _END: _START}

_RAW_ENDS = object()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
#
# test_map.py
#
# Created by Brett Andrews on 2 Jul 2017.

import numpy as np
import astropy
import matplotlib
import pytest

from marvin.tools.maps import Maps
from marvin.tools.map import Map

def _get_maps_kwargs(galaxy, data_origin):

    if data_origin == 'file':
        maps_kwargs = dict(filename=galaxy.mapspath)
    else:
        maps_kwargs = dict(plateifu=galaxy.plateifu, release=galaxy.release,
                           bintype=galaxy.bintype, template_kin=galaxy.template,
                           mode='local' if data_origin == 'db' else 'remote')

    return maps_kwargs

@pytest.fixture(scope='function', params=[('emline_gflux', 'ha_6564'),
                                          ('stellar_vel', None)])
def map_(request, galaxy, data_origin):

    # TODO Remove
    files_to_download = ['manga-7443-12701-MAPS-SPX-GAU-MILESHC.fits.gz',
                         'manga-7443-12701-MAPS-ALL-GAU-MILESHC.fits.gz',
                         'manga-7443-12701-MAPS-NRE-GAU-MILESHC.fits.gz',
                         'manga-7443-12701-MAPS-VOR10-GAU-MILESHC.fits.gz',
                         'manga-7443-12701-LOGCUBE_MAPS-NONE-003.fits.gz',
                         'manga-7443-12701-LOGCUBE_MAPS-NONE-013.fits.gz',
                         'manga-7443-12701-LOGCUBE_MAPS-NONE-023.fits.gz',
                         'manga-8485-1901-LOGCUBE_MAPS-NONE-023.fits.gz',
                         'manga-7443-12701-LOGCUBE_MAPS-RADIAL-007.fits.gz',
                         'manga-8485-1901-LOGCUBE_MAPS-RADIAL-007.fits.gz',
                         'manga-7443-12701-LOGCUBE_MAPS-RADIAL-017.fits.gz',
                         'manga-8485-1901-LOGCUBE_MAPS-RADIAL-017.fits.gz',
                         'manga-7443-12701-LOGCUBE_MAPS-RADIAL-027.fits.gz',
                         'manga-8485-1901-LOGCUBE_MAPS-RADIAL-027.fits.gz',
                         'manga-7443-12701-LOGCUBE_MAPS-STON-001.fits.gz',
                         'manga-8485-1901-LOGCUBE_MAPS-STON-001.fits.gz',
                         'manga-7443-12701-LOGCUBE_MAPS-STON-011.fits.gz',
                         'manga-8485-1901-LOGCUBE_MAPS-STON-011.fits.gz',
                         'manga-7443-12701-LOGCUBE_MAPS-STON-021.fits.gz',
                         'manga-8485-1901-LOGCUBE_MAPS-STON-021.fits.gz']
    if galaxy.mapspath.split('/')[-1] in files_to_download:
        pytest.skip('Remove this skip once I download the files.')

    maps = Maps(**_get_maps_kwargs(galaxy, data_origin))
    map_ = maps.getMap(property_name=request.param[0], channel=request.param[1])
    return map_

class TestMap(object):

    def test_map(self, map_, galaxy):

        assert map_.release == galaxy.release

        assert tuple(map_.shape) == tuple(galaxy.shape)
        assert map_.value.shape == tuple(galaxy.shape)
        assert map_.ivar.shape == tuple(galaxy.shape)
        assert map_.mask.shape == tuple(galaxy.shape)

        assert (map_.masked.data == map_.value).all()
        assert (map_.masked.mask == map_.mask.astype(bool)).all()

        assert pytest.approx(map_.snr, np.abs(map_.value * np.sqrt(map_.ivar)))

        assert map_.header['BUNIT'] == map_.unit

        assert isinstance(map_.header, astropy.io.fits.header.Header)

    def test_plot(self, map_):

        fig, ax = map_.plot()
        assert isinstance(fig, matplotlib.figure.Figure)
        assert isinstance(ax, matplotlib.axes._subplots.Subplot)

    def test_save_and_restore(self, temp_scratch, map_):

        fout = temp_scratch.join('test_map.mpf')
        map_.save(str(fout))
        assert fout.check() is True

        map_restored = Map.restore(str(fout), delete=False)
        assert tuple(map_.shape) == tuple(map_restored.shape)

# extend the number of property name + channel combinations

<commit_msg>adds OIII vel and stellar sigma to maps tests<commit_after>#!/usr/bin/env python
# encoding: utf-8
#
# test_map.py
#
# Created by Brett Andrews on 2 Jul 2017.

import numpy as np
import astropy
import matplotlib
import pytest

from marvin.tools.maps import Maps
from marvin.tools.map import Map

def _get_maps_kwargs(galaxy, data_origin):

    if data_origin == 'file':
        maps_kwargs = dict(filename=galaxy.mapspath)
    else:
        maps_kwargs = dict(plateifu=galaxy.plateifu, release=galaxy.release,
                           bintype=galaxy.bintype, template_kin=galaxy.template,
                           mode='local' if data_origin == 'db' else 'remote')

    return maps_kwargs

@pytest.fixture(scope='function', params=[('emline_gflux', 'ha_6564'),
                                          ('emline_gvel', 'oiii_5008'),
                                          ('stellar_vel', None),
                                          ('stellar_sigma', None)])
def map_(request, galaxy, data_origin):

    # TODO Remove
    files_to_download = ['manga-7443-12701-MAPS-SPX-GAU-MILESHC.fits.gz',
                         'manga-7443-12701-MAPS-ALL-GAU-MILESHC.fits.gz',
                         'manga-7443-12701-MAPS-NRE-GAU-MILESHC.fits.gz',
                         'manga-7443-12701-MAPS-VOR10-GAU-MILESHC.fits.gz',
                         'manga-7443-12701-LOGCUBE_MAPS-NONE-003.fits.gz',
                         'manga-7443-12701-LOGCUBE_MAPS-NONE-013.fits.gz',
                         'manga-7443-12701-LOGCUBE_MAPS-NONE-023.fits.gz',
                         'manga-8485-1901-LOGCUBE_MAPS-NONE-023.fits.gz',
                         'manga-7443-12701-LOGCUBE_MAPS-RADIAL-007.fits.gz',
                         'manga-8485-1901-LOGCUBE_MAPS-RADIAL-007.fits.gz',
                         'manga-7443-12701-LOGCUBE_MAPS-RADIAL-017.fits.gz',
                         'manga-8485-1901-LOGCUBE_MAPS-RADIAL-017.fits.gz',
                         'manga-7443-12701-LOGCUBE_MAPS-RADIAL-027.fits.gz',
                         'manga-8485-1901-LOGCUBE_MAPS-RADIAL-027.fits.gz',
                         'manga-7443-12701-LOGCUBE_MAPS-STON-001.fits.gz',
                         'manga-8485-1901-LOGCUBE_MAPS-STON-001.fits.gz',
                         'manga-7443-12701-LOGCUBE_MAPS-STON-011.fits.gz',
                         'manga-8485-1901-LOGCUBE_MAPS-STON-011.fits.gz',
                         'manga-7443-12701-LOGCUBE_MAPS-STON-021.fits.gz',
                         'manga-8485-1901-LOGCUBE_MAPS-STON-021.fits.gz']
    if galaxy.mapspath.split('/')[-1] in files_to_download:
        pytest.skip('Remove this skip once I download the files.')

    maps = Maps(**_get_maps_kwargs(galaxy, data_origin))
    map_ = maps.getMap(property_name=request.param[0], channel=request.param[1])
    return map_

class TestMap(object):

    def test_map(self, map_, galaxy):

        assert map_.release == galaxy.release

        assert tuple(map_.shape) == tuple(galaxy.shape)
        assert map_.value.shape == tuple(galaxy.shape)
        assert map_.ivar.shape == tuple(galaxy.shape)
        assert map_.mask.shape == tuple(galaxy.shape)

        assert (map_.masked.data == map_.value).all()
        assert (map_.masked.mask == map_.mask.astype(bool)).all()

        assert pytest.approx(map_.snr, np.abs(map_.value * np.sqrt(map_.ivar)))

        assert map_.header['BUNIT'] == map_.unit

        assert isinstance(map_.header, astropy.io.fits.header.Header)

    def test_plot(self, map_):

        fig, ax = map_.plot()
        assert isinstance(fig, matplotlib.figure.Figure)
        assert isinstance(ax, matplotlib.axes._subplots.Subplot)

    def test_save_and_restore(self, temp_scratch, map_):

        fout = temp_scratch.join('test_map.mpf')
        map_.save(str(fout))
        assert fout.check() is True

        map_restored = Map.restore(str(fout), delete=False)
        assert tuple(map_.shape) == tuple(map_restored.shape)

# extend the number of property name + channel combinations

<|endoftext|>


<commit_before>from itertools import zip_longest

import pytest

from umongo import Document, EmbeddedDocument, fields
from umongo.indexes import (
    explicit_key, parse_index,
    IndexModel, ASCENDING, DESCENDING, TEXT, HASHED)

from .common import BaseTest

def assert_indexes(indexes1, indexes2):
    if hasattr(indexes1, '__iter__'):
        for e1, e2 in zip_longest(indexes1, indexes2):
            assert e1, "missing index %s" % e2.document
            assert e2, "too much indexes: %s" % e1.document
            assert e1.document == e2.document
    else:
        assert indexes1.document == indexes2.document

class TestIndexes(BaseTest):

    def test_parse_index(self):
        for value, expected in (
            ('my_index', IndexModel([('my_index', ASCENDING)])),
            ('+my_index', IndexModel([('my_index', ASCENDING)])),
            ('-my_index', IndexModel([('my_index', DESCENDING)])),
            ('$my_index', IndexModel([('my_index', TEXT)])),
            ('#my_index', IndexModel([('my_index', HASHED)])),
            # Compound indexes
            (('index1', '-index2'), IndexModel([('index1', ASCENDING), ('index2', DESCENDING)])),
            # No changes if not needed
            (IndexModel([('my_index', ASCENDING)]), IndexModel([('my_index', ASCENDING)])),
            # Custom index
            (
                {
                    'name': 'my-custom-index',
                    'key': ['+index1', '-index2'],
                    'sparse': True,
                    'unique': True,
                    'expireAfterSeconds': 42
                },
                IndexModel([('index1', ASCENDING), ('index2', DESCENDING)],
                           name='my-custom-index', sparse=True,
                           unique=True, expireAfterSeconds=42)
            ),
        ):
            assert_indexes(parse_index(value), expected)

    def test_explicit_key(self):
        for value, expected in (
            ('my_index', ('my_index', ASCENDING)),
            ('+my_index', ('my_index', ASCENDING)),
            ('-my_index', ('my_index', DESCENDING)),
            ('$my_index', ('my_index', TEXT)),
            ('#my_index', ('my_index', HASHED)),
            # No changes if not needed
            (('my_index', ASCENDING), ('my_index', ASCENDING)),
        ):
            assert explicit_key(value) == expected

    def test_inheritance(self):

        @self.instance.register
        class Parent(Document):
            last_name = fields.StrField()

            class Meta:
                allow_inheritance = True
                indexes = ['last_name']

        @self.instance.register
        class Child(Parent):
            first_name = fields.StrField()

            class Meta:
                indexes = ['-first_name']

        assert_indexes(Parent.opts.indexes, [IndexModel([('last_name', ASCENDING)])])
        assert_indexes(Child.opts.indexes,
            [
                IndexModel([('last_name', ASCENDING)]),
                IndexModel([('first_name', DESCENDING), ('_cls', ASCENDING)]),
                IndexModel([('_cls', ASCENDING)])
            ])

    def test_bad_index(self):
        for bad in [1, None, object()]:
            with pytest.raises(TypeError) as exc:
                parse_index(1)
            assert exc.value.args[0] == (
                'Index type must be <str>, <list>, <dict> or <pymongo.IndexModel>')

    def test_nested_indexes(self):

        @self.instance.register
        class NestedDoc(EmbeddedDocument):
            simple = fields.StrField()
            listed = fields.ListField(fields.StrField())

        @self.instance.register
        class Doc(Document):
            nested = fields.EmbeddedField(NestedDoc)
            listed = fields.ListField(fields.EmbeddedField(NestedDoc))

            class Meta:
                indexes = ['nested', 'nested.simple', 'listed', 'listed.simple', 'listed.listed']

        assert_indexes(Doc.opts.indexes,
            [
                IndexModel([('nested', ASCENDING)]),
                IndexModel([('nested.simple', ASCENDING)]),
                IndexModel([('listed', ASCENDING)]),
                IndexModel([('listed.simple', ASCENDING)]),
                IndexModel([('listed.listed', ASCENDING)]),
            ])
<commit_msg>Add test_unique_indexes<commit_after>from itertools import zip_longest

import pytest

from umongo import Document, EmbeddedDocument, fields
from umongo.indexes import (
    explicit_key, parse_index,
    IndexModel, ASCENDING, DESCENDING, TEXT, HASHED)

from .common import BaseTest

def assert_indexes(indexes1, indexes2):
    if hasattr(indexes1, '__iter__'):
        for e1, e2 in zip_longest(indexes1, indexes2):
            assert e1, "missing index %s" % e2.document
            assert e2, "too much indexes: %s" % e1.document
            assert e1.document == e2.document
    else:
        assert indexes1.document == indexes2.document

class TestIndexes(BaseTest):

    def test_parse_index(self):
        for value, expected in (
            ('my_index', IndexModel([('my_index', ASCENDING)])),
            ('+my_index', IndexModel([('my_index', ASCENDING)])),
            ('-my_index', IndexModel([('my_index', DESCENDING)])),
            ('$my_index', IndexModel([('my_index', TEXT)])),
            ('#my_index', IndexModel([('my_index', HASHED)])),
            # Compound indexes
            (('index1', '-index2'), IndexModel([('index1', ASCENDING), ('index2', DESCENDING)])),
            # No changes if not needed
            (IndexModel([('my_index', ASCENDING)]), IndexModel([('my_index', ASCENDING)])),
            # Custom index
            (
                {
                    'name': 'my-custom-index',
                    'key': ['+index1', '-index2'],
                    'sparse': True,
                    'unique': True,
                    'expireAfterSeconds': 42
                },
                IndexModel([('index1', ASCENDING), ('index2', DESCENDING)],
                           name='my-custom-index', sparse=True,
                           unique=True, expireAfterSeconds=42)
            ),
        ):
            assert_indexes(parse_index(value), expected)

    def test_explicit_key(self):
        for value, expected in (
            ('my_index', ('my_index', ASCENDING)),
            ('+my_index', ('my_index', ASCENDING)),
            ('-my_index', ('my_index', DESCENDING)),
            ('$my_index', ('my_index', TEXT)),
            ('#my_index', ('my_index', HASHED)),
            # No changes if not needed
            (('my_index', ASCENDING), ('my_index', ASCENDING)),
        ):
            assert explicit_key(value) == expected

    def test_inheritance(self):

        @self.instance.register
        class Parent(Document):
            last_name = fields.StrField()

            class Meta:
                allow_inheritance = True
                indexes = ['last_name']

        @self.instance.register
        class Child(Parent):
            first_name = fields.StrField()

            class Meta:
                indexes = ['-first_name']

        assert_indexes(Parent.opts.indexes, [IndexModel([('last_name', ASCENDING)])])
        assert_indexes(Child.opts.indexes,
            [
                IndexModel([('last_name', ASCENDING)]),
                IndexModel([('first_name', DESCENDING), ('_cls', ASCENDING)]),
                IndexModel([('_cls', ASCENDING)])
            ])

    def test_bad_index(self):
        for bad in [1, None, object()]:
            with pytest.raises(TypeError) as exc:
                parse_index(1)
            assert exc.value.args[0] == (
                'Index type must be <str>, <list>, <dict> or <pymongo.IndexModel>')

    def test_nested_indexes(self):

        @self.instance.register
        class NestedDoc(EmbeddedDocument):
            simple = fields.StrField()
            listed = fields.ListField(fields.StrField())

        @self.instance.register
        class Doc(Document):
            nested = fields.EmbeddedField(NestedDoc)
            listed = fields.ListField(fields.EmbeddedField(NestedDoc))

            class Meta:
                indexes = ['nested', 'nested.simple', 'listed', 'listed.simple', 'listed.listed']

        assert_indexes(Doc.opts.indexes,
            [
                IndexModel([('nested', ASCENDING)]),
                IndexModel([('nested.simple', ASCENDING)]),
                IndexModel([('listed', ASCENDING)]),
                IndexModel([('listed.simple', ASCENDING)]),
                IndexModel([('listed.listed', ASCENDING)]),
            ])

    @pytest.mark.parametrize("unique_field", ("nested", "list"))
    def test_unique_indexes(self, unique_field):

        @self.instance.register
        class NestedDoc(EmbeddedDocument):
            simple = fields.StrField(unique=True)

        u_field, index = {
            "nested": (
                fields.EmbeddedField(NestedDoc),
                IndexModel([('field.simple', ASCENDING)], unique=True, sparse=True),
            ),
            "list": (
                fields.ListField(fields.EmbeddedField(NestedDoc)),
                IndexModel([('field.simple', ASCENDING)], unique=True, sparse=True),
            ),
        }[unique_field]

        @self.instance.register
        class Doc(Document):
            field = u_field

        assert_indexes(Doc.opts.indexes, [index])
<|endoftext|>


<commit_before>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Jun  9 08:03:07 2017

@author: Ben Marshall
"""

### Imports ###
import os
import sys
import shutil
import argparse
from glob import glob
import contextlib

### Class definitions ###
class Error(Exception):
    """Base class for exceptions in this module."""
    pass

class ConfigError(Error):
    """Exception raised for options not defined in config.

    Attributes:
        message -- explanation of the error
    """

    def __init__(self, message):
        self.message = message

### Support Functions ###
def try_delete(item):
    try:
        shutil.rmtree(item)
    except OSError:
        try:
            os.remove(item)
        except OSError:
            return 1
        else:
            return 0
    else:
        return 0

def get_vars_from_file(filename):
    import imp
    f = open(filename)
    config = imp.load_source('config', '', f)
    f.close()
    return config

def parse_config_vars(config_loaded, config, errors):
    config_loaded_dict = dict((name, getattr(config_loaded, name)) for name in dir(config_loaded) if not name.startswith('__'))
    config_loaded_set = set(config_loaded_dict)
    config_set = set(config)
    options_defined = config_loaded_set.intersection(config_set)
    for name in config:
        if str(name) in options_defined:
            config[name] = config_loaded_dict[name]
        try:
            if not config[name]:
                raise ConfigError("Error: " + name + " is not defined in config file. No default exists, please define a value in the config file.")
        except ConfigError as err:
            errors.append(err)
            continue

def just_loop_on(input):
  if isinstance(input, str):
    yield input
  else:
    try:
      for item in input:
        yield item
    except TypeError:
      yield input

# Set up default config dictionary
config = {
    "project_name" : "proj_" + os.path.relpath(".",".."),
    "top_level_function_name" : "",
    "src_dir_name" : "src",
    "tb_dir_name" : "tb",
    "src_files" : "",
    "tb_files" : "",
    "part_name" : "",
    "clock_period" : "",
    "language" : "vhdl",
}

# Parse command line arguments
parser = argparse.ArgumentParser(description="Helper tool for using Vivado HLS through the command line. If no arguments are specified then a default run is executed which includes C simulation, C synthesis, Cosimulation and export for both Vivado IP Catalog and System Generator. If any of the run options are specified then only those specified are performed.")
parser.add_argument("-clean", help="Remove all Vivado HLS generated files", action="store_true")
parser.add_argument("-keep", help="keep all previous solution and generate a new one", action="store_true")
parser.add_argument("-csim", help="perform C simulation stage", action="store_true")
parser.add_argument("-syn", help="perform C synthesis stage", action="store_true")
cosim_group = parser.add_mutually_exclusive_group()
cosim_group.add_argument("-cosim", help="perform cosimulation", action="store_true")
cosim_group.add_argument("-cosim_debug", help="perform cosimulation with debug logging", action="store_true")
export_ip_group = parser.add_mutually_exclusive_group()
export_ip_group.add_argument("-export_ip", help="perform export for Vivado IP Catalog", action="store_true")
export_ip_group.add_argument("-evaluate_ip", help="perform export for Vivado IP Catalog with build to place and route", action="store_true")
export_dsp_group = parser.add_mutually_exclusive_group()
export_dsp_group.add_argument("-export_dsp", help="perform export for System Generator", action="store_true")
export_dsp_group.add_argument("-evaluate_dsp", help="perform export for System Generator with build to place and route", action="store_true")
args = parser.parse_args()

# Check for clean argument
if args.clean:
    if len(sys.argv) > 2:
        print("Warning: The 'Clean' option is exclusive. All other arguments will be ignored.")
    if try_delete(config["project_name"]) + try_delete("run_hls.tcl") + try_delete("vivado_hls.log") == 3:
        print("Warning: Nothing to remove!")
    else:
        print("Cleaned up generated files.")
    sys.exit()

# Load project specifics from local config file and add to config dict
config_loaded = get_vars_from_file('hls_config.py')
errors = []
parse_config_vars(config_loaded, config, errors)
if len(errors) != 0:
    for err in errors:
        print(err)
    print("Config Errors, exiting...")
    sys.exit()

# Write out TCL file
file = open("run_hls.tcl","w")
file.write("open_project " + config["project_name"] + "\n")
file.write("set_top " + config["top_level_function_name"] + "\n")
for src_file in config["src_files"]:
    file.write("add_files " + config["src_dir_name"] + "/" + src_file + "\n")
for tb_file in config["tb_files"]:
    file.write("add_files -tb " + config["tb_dir_name"] + "/" + tb_file + "\n")
if args.keep:
    paths = glob(project_name + "/solution*/")
    solution_num = len(paths) + 1
    if solution_num == 1:
        file.write("open_solution -reset \"solution1\"" + "\n")
    else:
        file.write("open_solution -reset \"solution" + str(solution_num) + "\"" + "\n")
else:
    file.write("open_solution -reset \"solution1\"" + "\n")
file.write("set_part \{" + config["part_name"] + "\}" + "\n")
file.write("create_clock -period " + config["clock_period"] + " -name default" + "\n")

if not(args.csim or args.syn or args.cosim or args.cosim_debug or args.export_ip or args.export_dsp or args.evaluate_ip or args.evaluate_dsp):
    file.write("csim_design -clean" + "\n")
    file.write("csynth_design" + "\n")
    file.write("cosim_design -O -rtl " + config["language"] + "\n")
    file.write("export_design -format ip_catalog" + "\n")
    file.write("export_design -format sysgen" + "\n")
    file.write("exit" + "\n")
else:
    if args.csim:
        file.write("csim_design -clean" + "\n")
    if args.syn:
        file.write("csynth_design" + "\n")
    if args.cosim:
        for language in just_loop_on(config["language"]):
            file.write("cosim_design -O -rtl " + language + "\n")
    if args.cosim_debug:
        for language in just_loop_on(config["language"]):
            file.write("cosim_design -rtl " + language + "-trace_level all" + "\n")
    if args.export_dsp:
        file.write("export_design -format ip_catalog")
    if args.export_ip:
        file.write("export_design -format sysgen")
    if args.evaluate_dsp:
        for language in just_loop_on(config["language"]):
            file.write("export_design -format ip_catalog -evaluate " + language)
    if args.evaluate_ip:
        for language in just_loop_on(config["language"]):
            file.write("export_design -format sysgen -evaluate " + language)
file.write("exit")
file.close()

# Call the Vivado HLS process
os.system("vivado_hls -f run_hls.tcl")
<commit_msg>Added main function and fixed a few lingering bugs in less used HLS process statements.<commit_after>#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Jun  9 08:03:07 2017

@author: Ben Marshall
"""

### Imports ###
import os
import sys
import shutil
import argparse
from glob import glob
import contextlib

### Class definitions ###
class Error(Exception):
    """Base class for exceptions in this module."""
    pass

class ConfigError(Error):
    """Exception raised for options not defined in config.

    Attributes:
        message -- explanation of the error
    """

    def __init__(self, message):
        self.message = message

### Support Functions ###
def try_delete(item):
    try:
        shutil.rmtree(item)
    except OSError:
        try:
            os.remove(item)
        except OSError:
            return 1
        else:
            return 0
    else:
        return 0

def get_vars_from_file(filename):
    import imp
    f = open(filename)
    config = imp.load_source('config', '', f)
    f.close()
    return config

def parse_config_vars(config_loaded, config, errors):
    config_loaded_dict = dict((name, getattr(config_loaded, name)) for name in dir(config_loaded) if not name.startswith('__'))
    config_loaded_set = set(config_loaded_dict)
    config_set = set(config)
    options_defined = config_loaded_set.intersection(config_set)
    for name in config:
        if str(name) in options_defined:
            config[name] = config_loaded_dict[name]
        try:
            if not config[name]:
                raise ConfigError("Error: " + name + " is not defined in config file. No default exists, please define a value in the config file.")
        except ConfigError as err:
            errors.append(err)
            continue

def just_loop_on(input):
  if isinstance(input, str):
    yield input
  else:
    try:
      for item in input:
        yield item
    except TypeError:
      yield input

def main():
    # Set up default config dictionary
    config = {
        "project_name" : "proj_" + os.path.relpath(".",".."),
        "top_level_function_name" : "",
        "src_dir_name" : "src",
        "tb_dir_name" : "tb",
        "src_files" : "",
        "tb_files" : "",
        "part_name" : "",
        "clock_period" : "",
        "language" : "vhdl",
    }

    # Parse command line arguments
    parser = argparse.ArgumentParser(description="Helper tool for using Vivado HLS through the command line. If no arguments are specified then a default run is executed which includes C simulation, C synthesis, Cosimulation and export for both Vivado IP Catalog and System Generator. If any of the run options are specified then only those specified are performed.")
    parser.add_argument("-clean", help="Remove all Vivado HLS generated files", action="store_true")
    parser.add_argument("-keep", help="keep all previous solution and generate a new one", action="store_true")
    parser.add_argument("-csim", help="perform C simulation stage", action="store_true")
    parser.add_argument("-syn", help="perform C synthesis stage", action="store_true")
    cosim_group = parser.add_mutually_exclusive_group()
    cosim_group.add_argument("-cosim", help="perform cosimulation", action="store_true")
    cosim_group.add_argument("-cosim_debug", help="perform cosimulation with debug logging", action="store_true")
    export_ip_group = parser.add_mutually_exclusive_group()
    export_ip_group.add_argument("-export_ip", help="perform export for Vivado IP Catalog", action="store_true")
    export_ip_group.add_argument("-evaluate_ip", help="perform export for Vivado IP Catalog with build to place and route", action="store_true")
    export_dsp_group = parser.add_mutually_exclusive_group()
    export_dsp_group.add_argument("-export_dsp", help="perform export for System Generator", action="store_true")
    export_dsp_group.add_argument("-evaluate_dsp", help="perform export for System Generator with build to place and route", action="store_true")
    args = parser.parse_args()

    # Check for clean argument
    if args.clean:
        if len(sys.argv) > 2:
            print("Warning: The 'Clean' option is exclusive. All other arguments will be ignored.")
        if try_delete(config["project_name"]) + try_delete("run_hls.tcl") + try_delete("vivado_hls.log") == 3:
            print("Warning: Nothing to remove!")
        else:
            print("Cleaned up generated files.")
        sys.exit()

    # Load project specifics from local config file and add to config dict
    config_loaded = get_vars_from_file('hls_config.py')
    errors = []
    parse_config_vars(config_loaded, config, errors)
    if len(errors) != 0:
        for err in errors:
            print(err)
        print("Config Errors, exiting...")
        sys.exit()

    # Write out TCL file
    file = open("run_hls.tcl","w")
    file.write("open_project " + config["project_name"] + "\n")
    file.write("set_top " + config["top_level_function_name"] + "\n")
    for src_file in config["src_files"]:
        file.write("add_files " + config["src_dir_name"] + "/" + src_file + "\n")
    for tb_file in config["tb_files"]:
        file.write("add_files -tb " + config["tb_dir_name"] + "/" + tb_file + "\n")
    if args.keep:
        paths = glob(config["project_name"] + "/solution*/")
        solution_num = len(paths) + 1
        if solution_num == 1:
            file.write("open_solution -reset \"solution1\"" + "\n")
        else:
            file.write("open_solution -reset \"solution" + str(solution_num) + "\"" + "\n")
    else:
        file.write("open_solution \"solution1\"" + "\n")
    file.write("set_part \{" + config["part_name"] + "\}" + "\n")
    file.write("create_clock -period " + config["clock_period"] + " -name default" + "\n")

    if not(args.csim or args.syn or args.cosim or args.cosim_debug or args.export_ip or args.export_dsp or args.evaluate_ip or args.evaluate_dsp):
        file.write("csim_design -clean" + "\n")
        file.write("csynth_design" + "\n")
        file.write("cosim_design -O -rtl " + config["language"] + "\n")
        file.write("export_design -format ip_catalog" + "\n")
        file.write("export_design -format sysgen" + "\n")
        file.write("exit" + "\n")
    else:
        if args.csim:
            file.write("csim_design -clean" + "\n")
        if args.syn:
            file.write("csynth_design" + "\n")
        if args.cosim:
            for language in just_loop_on(config["language"]):
                file.write("cosim_design -O -rtl " + language + "\n")
        if args.cosim_debug:
            for language in just_loop_on(config["language"]):
                file.write("cosim_design -rtl " + language + " -trace_level all" + "\n")
        if args.export_dsp:
            file.write("export_design -format ip_catalog" + "\n")
        if args.export_ip:
            file.write("export_design -format sysgen" + "\n")
        if args.evaluate_dsp:
            for language in just_loop_on(config["language"]):
                file.write("export_design -format ip_catalog -evaluate " + language + "\n")
        if args.evaluate_ip:
            for language in just_loop_on(config["language"]):
                file.write("export_design -format sysgen -evaluate " + language + "\n")
    file.write("exit")
    file.close()

    # Call the Vivado HLS process
    os.system("vivado_hls -f run_hls.tcl")

if __name__ == "__main__": main()
<|endoftext|>


<commit_before>from unittest import TestCase
import unittest
from equadratures.poly import Poly
from equadratures.sampling_methods.induced import Induced
from equadratures.parameter import Parameter
from equadratures.basis import Basis
import numpy as np
import time
<<<<<<< HEAD

class TestSamplingGeneration(TestCase):
=======

class TestSamplingGeneration(TestCase):

>>>>>>> fix induced sampling interface with poly and quadrature class
    def test_sampling(self):
        d = 3
        order = 3
        param = Parameter(distribution='uniform', order=order, lower=-1.0, upper=1.0)
        myparameters = [param for _ in range(d)]
<<<<<<< HEAD
        mybasis2 = Basis('total-order')
        mypoly2 = Poly(myparameters, mybasis2, method='least-squares', sampling_args={'mesh':'induced', 'subsampling-algorithm':'qr', 'sampling-ratio':1.0})
        assert mypoly2._quadrature_points.shape == (mypoly2.basis.cardinality, d)
=======
        # mybasis = Basis('total-order')
        # mypoly1 = Poly(myparameters, mybasis, method='least-squares', sampling_args={'mesh':'monte-carlo', 'subsampling-algorithm':'qr', 'sampling-ratio':1.0} )
        # assert mypoly1._quadrature_points.shape == (order+1, d)
        mybasis2 = Basis('total-order')
        mypoly2 = Poly(myparameters, mybasis2, method='least-squares', sampling_args={'mesh':'induced', 'subsampling-algorithm':'qr', 'sampling-ratio':1.0})
        print(mypoly2._quadrature_points.shape)
        assert mypoly2._quadrature_points.shape == (mypoly2.basis.cardinality, d)

>>>>>>> fix induced sampling interface with poly and quadrature class
    def test_induced_jacobi_evaluation(self):
        dimension = 3
        parameters = [Parameter(1, "Uniform", upper=1, lower=-1)]*dimension
        basis = Basis("total-order")
<<<<<<< HEAD
        induced_sampling = Induced(parameters, basis)
=======

        initialise_time = time.time()
        induced_sampling = Induced(parameters, basis)
        print(f"time taken to initialise class: {time.time()-initialise_time}")

>>>>>>> fix induced sampling interface with poly and quadrature class
        parameter = parameters[0]
        parameter.order = 3
        cdf_value = induced_sampling.induced_jacobi_evaluation(0, 0, 0, parameter)
        np.testing.assert_allclose(cdf_value, 0.5, atol=0.00001)
        cdf_value = induced_sampling.induced_jacobi_evaluation(0, 0, 1, parameter)
        assert cdf_value == 1
        cdf_value = induced_sampling.induced_jacobi_evaluation(0, 0, -1, parameter)
        assert cdf_value == 0
        cdf_value = induced_sampling.induced_jacobi_evaluation(0, 0, 0.6, parameter)
        np.testing.assert_allclose(cdf_value, 0.7462, atol=0.00005)
        cdf_value = induced_sampling.induced_jacobi_evaluation(0, 0, 0.999, parameter)
        np.testing.assert_allclose(cdf_value, 0.99652, atol=0.000005)
<<<<<<< HEAD
=======

    # def test_induced_sampling(self):
    #     """
    #     An integration test for the whole routine
    #     """
    #     dimension = 3
    #     parameters = [Parameter(3, "Uniform", upper=1, lower=-1)]*dimension
    #     basis = Basis("total-order", [3]*dimension)

    #     induced_sampling = Induced(parameters, basis)

    #     quadrature_points = induced_sampling.get_points()
    #     assert quadrature_points.shape == (63, 3)

>>>>>>> fix induced sampling interface with poly and quadrature class
if __name__ == '__main__':
    unittest.main()<commit_msg>remove test_induced print statement<commit_after>from unittest import TestCase
import unittest
from equadratures.poly import Poly
from equadratures.sampling_methods.induced import Induced
from equadratures.parameter import Parameter
from equadratures.basis import Basis

import numpy as np
# import time

class TestSamplingGeneration(TestCase):

    def test_sampling(self):
        d = 3
        order = 3
        param = Parameter(distribution='uniform', order=order, lower=-1.0, upper=1.0)
        myparameters = [param for _ in range(d)]
        mybasis2 = Basis('total-order')
        mypoly2 = Poly(myparameters, mybasis2, method='least-squares', sampling_args={'mesh':'induced', 'subsampling-algorithm':'qr', 'sampling-ratio':1.0})
        assert mypoly2._quadrature_points.shape == (mypoly2.basis.cardinality, d)

    def test_induced_jacobi_evaluation(self):
        dimension = 3
        parameters = [Parameter(1, "Uniform", upper=1, lower=-1)]*dimension
        basis = Basis("total-order")
        induced_sampling = Induced(parameters, basis)

        parameter = parameters[0]
        parameter.order = 3
        cdf_value = induced_sampling.induced_jacobi_evaluation(0, 0, 0, parameter)
        np.testing.assert_allclose(cdf_value, 0.5, atol=0.00001)
        cdf_value = induced_sampling.induced_jacobi_evaluation(0, 0, 1, parameter)
        assert cdf_value == 1
        cdf_value = induced_sampling.induced_jacobi_evaluation(0, 0, -1, parameter)
        assert cdf_value == 0
        cdf_value = induced_sampling.induced_jacobi_evaluation(0, 0, 0.6, parameter)
        np.testing.assert_allclose(cdf_value, 0.7462, atol=0.00005)
        cdf_value = induced_sampling.induced_jacobi_evaluation(0, 0, 0.999, parameter)
        np.testing.assert_allclose(cdf_value, 0.99652, atol=0.000005)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from pathlib import Path
import numpy as np
import struct
import time
import socket
try:
    import h5py
except ImportError:
    h5py = None

NPKT = 10000

def ipv4bcast_tx(port: int, N: int=NPKT, twait: float=0.1, verbose: bool=True):
    with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
        s.bind(('', 0))
        s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)

        for i in range(N):
            payload = i.to_bytes(14, 'big')
            s.sendto(payload, ('<broadcast>', port))
            if verbose:
                print('broadcast', payload, '\r', end="")

            time.sleep(twait)

def ipv4bcast_rx(port: int, N: int=NPKT, timeout: float=1.0):
    with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
        s.settimeout(timeout)
        s.bind(('', port))

        for i in range(N):
            print('package', i)
            m = s.recv(14)
            print(m)
# %%

def udpunicast(host: str, port: int, h5fn: Path=None,
               Nupdate: int=1000, bufsize: int=8192, Nelbyte: int=4, N: int=NPKT):
    """
    maxshape parameters:
    limit ~ 100000*BUFSIZE to overfill hardrive with extremely fast demo
    normally you would take other more sensible measures, and have sane datarates.
    None means unlimited size (entire hard drive)
    """

    S = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
    # Do NOT connect or bind
    try:
        first = True

        if h5py is not None and h5fn:
            h5 = h5py.File(h5fn, 'w')
            h5d = h5.create_dataset('/data',
                                    dtype=np.float32,
                                    chunks=True,
                                    shape=(Nupdate*bufsize/Nelbyte,),
                                    maxshape=(100000*bufsize/Nelbyte,))
        else:
            h5 = h5d = None

# %% main loop
        for i in range(N):
            tic = time.time()
# %% host-> device
            # host (other program) is programmed to send payload for any input--including simply '\n'
            S.sendto(b'\n', (host, port, 0, 0))
# %% device -> host
            Nel = struct.unpack('<1L', S.recv(4))[0]  # int len
            Nbyte_dg = Nel*Nelbyte

            A = S.recv(Nbyte_dg)

            if len(A) != Nbyte_dg:
                continue
# %% parse result
            dat = struct.unpack(f'<{Nel:d}f', A)
            if first:  # to avoid having to restart C code each time
                first = False
                last = dat[0]-1
                rtoc = time.time()-tic

            rtoc = np.mean((time.time()-tic, rtoc))
# %% optional write to disk
            if h5 is not None:
                h5d[i*Nel:(i+1)*Nel] = dat
# %% progress report / writing
            if not i % Nupdate:
                print(rtoc, i)
                h5 = updatemsg(i, h5, h5d, Nupdate, bufsize, Nelbyte)
# %% data validation due to a priori pattern

# for single precision float, large integer jumps are experienced at large values by IEEE754 definition
            # assert_allclose(diff(dat),1,err_msg=str(dat))

            np.testing.assert_allclose(last, dat[0]-1,
                                       err_msg=f'{last} be mindful of min/max values of your datatype')

            last = dat[-1]
    except KeyboardInterrupt:
        pass

    S.close()
    if h5 is not None:
        h5.close()

def updatemsg(i: int, h5, h5d,
              Nupdate: int, bufsize: int, Nelbyte: int) -> h5py.Dataset:
    if h5 is not None:
        h5.flush()  # NOTE: every N loops to improve performance
        try:
            h5d.resize((i+Nupdate)*bufsize/Nelbyte, axis=0)
        except ValueError as e:
            print('stopping HDF5 writing for this fast demo. in real life you can set maxshape=(None,) for no limit.', e)
            h5.close()
            h5 = None

    return h5

# def getselfaddr():
    # return socket.getaddrinfo(None, PORT, socket.AF_INET6, socket.SOCK_DGRAM,socket.IPPROTO_IP)[0]
<commit_msg>soften prereq<commit_after>from pathlib import Path
import numpy as np
import struct
import time
import socket
try:
    import h5py
except ImportError:
    h5py = None

NPKT = 10000

def ipv4bcast_tx(port: int, N: int=NPKT, twait: float=0.1, verbose: bool=True):
    with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
        s.bind(('', 0))
        s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)

        for i in range(N):
            payload = i.to_bytes(14, 'big')
            s.sendto(payload, ('<broadcast>', port))
            if verbose:
                print('broadcast', payload, '\r', end="")

            time.sleep(twait)

def ipv4bcast_rx(port: int, N: int=NPKT, timeout: float=1.0):
    with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
        s.settimeout(timeout)
        s.bind(('', port))

        for i in range(N):
            print('package', i)
            m = s.recv(14)
            print(m)
# %%

def udpunicast(host: str, port: int, h5fn: Path=None,
               Nupdate: int=1000, bufsize: int=8192, Nelbyte: int=4, N: int=NPKT):
    """
    maxshape parameters:
    limit ~ 100000*BUFSIZE to overfill hardrive with extremely fast demo
    normally you would take other more sensible measures, and have sane datarates.
    None means unlimited size (entire hard drive)
    """

    S = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
    # Do NOT connect or bind
    try:
        first = True

        if h5py is not None and h5fn:
            h5 = h5py.File(h5fn, 'w')
            h5d = h5.create_dataset('/data',
                                    dtype=np.float32,
                                    chunks=True,
                                    shape=(Nupdate*bufsize/Nelbyte,),
                                    maxshape=(100000*bufsize/Nelbyte,))
        else:
            h5 = h5d = None

# %% main loop
        for i in range(N):
            tic = time.time()
# %% host-> device
            # host (other program) is programmed to send payload for any input--including simply '\n'
            S.sendto(b'\n', (host, port, 0, 0))
# %% device -> host
            Nel = struct.unpack('<1L', S.recv(4))[0]  # int len
            Nbyte_dg = Nel*Nelbyte

            A = S.recv(Nbyte_dg)

            if len(A) != Nbyte_dg:
                continue
# %% parse result
            dat = struct.unpack(f'<{Nel:d}f', A)
            if first:  # to avoid having to restart C code each time
                first = False
                last = dat[0]-1
                rtoc = time.time()-tic

            rtoc = np.mean((time.time()-tic, rtoc))
# %% optional write to disk
            if h5 is not None:
                h5d[i*Nel:(i+1)*Nel] = dat
# %% progress report / writing
            if not i % Nupdate:
                print(rtoc, i)
                h5 = updatemsg(i, h5, h5d, Nupdate, bufsize, Nelbyte)
# %% data validation due to a priori pattern

# for single precision float, large integer jumps are experienced at large values by IEEE754 definition
            # assert_allclose(diff(dat),1,err_msg=str(dat))

            np.testing.assert_allclose(last, dat[0]-1,
                                       err_msg=f'{last} be mindful of min/max values of your datatype')

            last = dat[-1]
    except KeyboardInterrupt:
        pass

    S.close()
    if h5 is not None:
        h5.close()

def updatemsg(i: int, h5, h5d,
              Nupdate: int, bufsize: int, Nelbyte: int):
    if h5 is not None:
        h5.flush()  # NOTE: every N loops to improve performance
        try:
            h5d.resize((i+Nupdate)*bufsize/Nelbyte, axis=0)
        except ValueError as e:
            print('stopping HDF5 writing for this fast demo. in real life you can set maxshape=(None,) for no limit.', e)
            h5.close()
            h5 = None

    return h5

# def getselfaddr():
    # return socket.getaddrinfo(None, PORT, socket.AF_INET6, socket.SOCK_DGRAM,socket.IPPROTO_IP)[0]
<|endoftext|>


<commit_before>import pytest
import testinfra

@pytest.mark.parametrize('name', [
    ('~/rpmbuild/RPMS/x86_64/apache-storm-1.0.2-1.el6.x86_64.rpm'),
])
def test_install_rpm(Command, Package, name):
    '''
    It should install the apache-storm RPM without errors
    '''
    # ~/rpmbuild/RPMS/x86_64/apache-storm-1.0.2-1.el6.x86_64.rpm
    yum_install = Command('yum install -y {0}'.format(name))

    pkg = Package('apache-storm')
    assert pkg.is_installed

def test_opt_apache_storm_exists(File):
    f = File('/opt/apache-storm')
    assert f.exists
    assert f.is_directory
    assert f.user == 'storm'
    assert f.group == 'storm'
    assert f.mode == 0755

def test_etc_sysconfig_strom_exists(File):
    f = File('/etc/sysconfig/storm')
    assert f.is_file
    assert f.exists
    assert f.user == 'root'
    assert f.group == 'root'
    assert f.mode  == 0644

@pytest.mark.parametrize('name', [
    'storm-nimbus',
    'storm-ui',
    'storm-supervisor',
    'storm-logviewer',
    'storm-drpc',
])
def test_etc_initd_scripts_exist(File, name):
    f = File('/etc/init.d/{0}'.format(name))
    assert f.is_file
    assert f.exists
    assert f.mode == 0755
    assert f.user == 'root'
    assert f.group == 'root'

def test_var_log_apache_storm_exists(File):
    f = File('/var/log/apache-storm')
    assert f.is_symlink
    assert f.linked_to == '/opt/apache-storm/logs'

def test_var_run_storm_exists(File):
    f = File('/var/run/storm')
    assert f.is_directory
    assert f.user == 'storm'
    assert f.group == 'storm'
    assert f.mode == 0755

@pytest.mark.parametrize('name', [
    'flight.bash',
    'storm',
    'storm.cmd',
    'storm-config.cmd',
    'storm.py',
])
def test_opt_apache_storm_bin_scripts_exists(File, name):
    f = File('/opt/apache-storm/bin/{0}'.format(name))
    assert f.is_file
    assert f.exists
    assert f.mode == 0755
    assert f.user == 'storm'
    assert f.group == 'storm'
<commit_msg>using local yum repo for testing package install<commit_after>import pytest
import testinfra

@pytest.mark.parametrize('name', [
    ('apache-storm-1.0.2-1.el6.x86_64.rpm'),
])
def test_install_rpm(Command, Package, name):
    '''
    It should install the apache-storm RPM without errors
    '''
    # ~/rpmbuild/RPMS/x86_64/apache-storm-1.0.2-1.el6.x86_64.rpm
    yum_install = Command('yum install -y {0}'.format(name))

    pkg = Package('apache-storm')
    assert pkg.is_installed

def test_opt_apache_storm_exists(File):
    f = File('/opt/apache-storm')
    assert f.exists
    assert f.is_directory
    assert f.user == 'storm'
    assert f.group == 'storm'
    assert f.mode == 0755

def test_etc_sysconfig_strom_exists(File):
    f = File('/etc/sysconfig/storm')
    assert f.is_file
    assert f.exists
    assert f.user == 'root'
    assert f.group == 'root'
    assert f.mode  == 0644

@pytest.mark.parametrize('name', [
    'storm-nimbus',
    'storm-ui',
    'storm-supervisor',
    'storm-logviewer',
    'storm-drpc',
])
def test_etc_initd_scripts_exist(File, name):
    f = File('/etc/init.d/{0}'.format(name))
    assert f.is_file
    assert f.exists
    assert f.mode == 0755
    assert f.user == 'root'
    assert f.group == 'root'

def test_var_log_apache_storm_exists(File):
    f = File('/var/log/apache-storm')
    assert f.is_symlink
    assert f.linked_to == '/opt/apache-storm/logs'

def test_var_run_storm_exists(File):
    f = File('/var/run/storm')
    assert f.is_directory
    assert f.user == 'storm'
    assert f.group == 'storm'
    assert f.mode == 0755

@pytest.mark.parametrize('name', [
    'flight.bash',
    'storm',
    'storm.cmd',
    'storm-config.cmd',
    'storm.py',
])
def test_opt_apache_storm_bin_scripts_exists(File, name):
    f = File('/opt/apache-storm/bin/{0}'.format(name))
    assert f.is_file
    assert f.exists
    assert f.mode == 0755
    assert f.user == 'storm'
    assert f.group == 'storm'
<|endoftext|>


<commit_before>class TestBoundaryValue:
    def test_WeakNormal(self):
        pass

    def test_WeakRobust(self):
        pass

    def test_StrongNormal(self):
        pass

    def test_StrongRobust(self):
        pass

<commit_msg>Failed boundary test for ipCheck function<commit_after>from PyCat import pycat

class TestBoundaryValue:
    def test_WeakNormal(self):
        assert pycat.ipCheck("1.128.128.128") == True
        assert pycat.ipCheck("223.128.128.128") == True

        assert pycat.ipCheck("128.1.128.128") == True
        assert pycat.ipCheck("128.254.128.128") == True

        assert pycat.ipCheck("128.128.1.128") == True
        assert pycat.ipCheck("128.128.254.128") == True

        assert pycat.ipCheck("128.128.128.1") == True
        assert pycat.ipCheck("128.128.128.254") == True

    def test_WeakRobust(self):
        assert pycat.ipCheck("0.128.128.128") == False
        assert pycat.ipCheck("224.128.128.128") == False

        assert pycat.ipCheck("128.0.128.128") == False
        assert pycat.ipCheck("128.255.128.128") == False

        assert pycat.ipCheck("128.128.0.128") == False
        assert pycat.ipCheck("128.128.255.128") == False

        assert pycat.ipCheck("128.128.128.0") == False
        assert pycat.ipCheck("128.128.128.255") == False

    def test_StrongNormal(self):
        assert pycat.ipCheck("1.1.128.128") == True
        assert pycat.ipCheck("1.254.128.128") == True
        assert pycat.ipCheck("223.1.128.128") == True
        assert pycat.ipCheck("223.254.128.128") == True

        assert pycat.ipCheck("1.128.1.128") == True
        assert pycat.ipCheck("1.128.254.128") == True
        assert pycat.ipCheck("223.128.1.128") == True
        assert pycat.ipCheck("223.128.254.128") == True

        assert pycat.ipCheck("1.128.128.1") == True
        assert pycat.ipCheck("1.128.128.254") == True
        assert pycat.ipCheck("223.128.128.1") == True
        assert pycat.ipCheck("223.128.128.254") == True

        assert pycat.ipCheck("128.1.1.128") == True
        assert pycat.ipCheck("128.1.254.128") == True
        assert pycat.ipCheck("128.254.1.128") == True
        assert pycat.ipCheck("128.254.254.128") == True

        assert pycat.ipCheck("128.1.128.1") == True
        assert pycat.ipCheck("128.1.128.254") == True
        assert pycat.ipCheck("128.254.128.1") == True
        assert pycat.ipCheck("128.254.128.254") == True

        assert pycat.ipCheck("128.128.1.1") == True
        assert pycat.ipCheck("128.128.1.254") == True
        assert pycat.ipCheck("128.128.254.1") == True
        assert pycat.ipCheck("128.128.254.254") == True

        assert pycat.ipCheck("1.1.1.128") == True
        assert pycat.ipCheck("223.1.1.128") == True
        assert pycat.ipCheck("1.254.1.128") == True
        assert pycat.ipCheck("1.1.254.128") == True
        assert pycat.ipCheck("223.254.1.128") == True
        assert pycat.ipCheck("223.1.254.128") == True
        assert pycat.ipCheck("1.254.254.128") == True
        assert pycat.ipCheck("223.254.254.128") == True

        assert pycat.ipCheck("1.1.128.1") == True
        assert pycat.ipCheck("223.1.128.1") == True
        assert pycat.ipCheck("1.254.128.1") == True
        assert pycat.ipCheck("1.1.128.254") == True
        assert pycat.ipCheck("223.254.128.1") == True
        assert pycat.ipCheck("223.1.128.254") == True
        assert pycat.ipCheck("1.254.128.254") == True
        assert pycat.ipCheck("223.254.128.254") == True

        assert pycat.ipCheck("1.128.1.1") == True
        assert pycat.ipCheck("223.128.1.1") == True
        assert pycat.ipCheck("1.128.254.1") == True
        assert pycat.ipCheck("1.128.1.254") == True
        assert pycat.ipCheck("223.128.254.1") == True
        assert pycat.ipCheck("223.128.1.254") == True
        assert pycat.ipCheck("1.128.254.254") == True
        assert pycat.ipCheck("223.128.254.254") == True

        assert pycat.ipCheck("128.1.1.1") == True
        assert pycat.ipCheck("128.254.1.1") == True
        assert pycat.ipCheck("128.1.254.1") == True
        assert pycat.ipCheck("128.1.1.254") == True
        assert pycat.ipCheck("128.254.254.1") == True
        assert pycat.ipCheck("128.254.1.254") == True
        assert pycat.ipCheck("128.1.254.254") == True
        assert pycat.ipCheck("128.254.254.254") == True

        assert pycat.ipCheck("1.1.1.1") == True
        assert pycat.ipCheck("223.1.1.1") == True
        assert pycat.ipCheck("1.254.1.1") == True
        assert pycat.ipCheck("1.1.254.1") == True
        assert pycat.ipCheck("1.1.1.254") == True
        assert pycat.ipCheck("223.254.1.1") == True
        assert pycat.ipCheck("223.1.254.1") == True
        assert pycat.ipCheck("223.1.1.254") == True
        assert pycat.ipCheck("1.254.254.1") == True
        assert pycat.ipCheck("1.254.1.254") == True
        assert pycat.ipCheck("1.1.254.254") == True
        assert pycat.ipCheck("223.254.254.1") == True
        assert pycat.ipCheck("223.254.1.254") == True
        assert pycat.ipCheck("223.1.254.254") == True
        assert pycat.ipCheck("1.254.254.254") == True
        assert pycat.ipCheck("223.254.254.254") == True

    def test_StrongRobust(self):
        assert pycat.ipCheck("0.0.128.128") == False
        assert pycat.ipCheck("0.255.128.128") == False
        assert pycat.ipCheck("224.0.128.128") == False
        assert pycat.ipCheck("224.255.128.128") == False

        assert pycat.ipCheck("0.128.0.128") == False
        assert pycat.ipCheck("0.128.255.128") == False
        assert pycat.ipCheck("224.128.0.128") == False
        assert pycat.ipCheck("224.128.255.128") == False

        assert pycat.ipCheck("0.128.128.0") == False
        assert pycat.ipCheck("0.128.128.255") == False
        assert pycat.ipCheck("224.128.128.0") == False
        assert pycat.ipCheck("224.128.128.255") == False

        assert pycat.ipCheck("128.0.0.128") == False
        assert pycat.ipCheck("128.0.255.128") == False
        assert pycat.ipCheck("128.255.0.128") == False
        assert pycat.ipCheck("128.255.255.128") == False

        assert pycat.ipCheck("128.0.128.0") == False
        assert pycat.ipCheck("128.0.128.255") == False
        assert pycat.ipCheck("128.255.128.0") == False
        assert pycat.ipCheck("128.255.128.255") == False

        assert pycat.ipCheck("128.128.0.0") == False
        assert pycat.ipCheck("128.128.0.255") == False
        assert pycat.ipCheck("128.128.255.0") == False
        assert pycat.ipCheck("128.128.255.255") == False

        assert pycat.ipCheck("0.0.0.128") == False
        assert pycat.ipCheck("224.0.0.128") == False
        assert pycat.ipCheck("0.255.0.128") == False
        assert pycat.ipCheck("0.0.255.128") == False
        assert pycat.ipCheck("224.255.0.128") == False
        assert pycat.ipCheck("224.0.255.128") == False
        assert pycat.ipCheck("0.255.255.128") == False
        assert pycat.ipCheck("224.255.255.128") == False

        assert pycat.ipCheck("0.0.128.0") == False
        assert pycat.ipCheck("224.0.128.0") == False
        assert pycat.ipCheck("0.255.128.0") == False
        assert pycat.ipCheck("0.0.128.255") == False
        assert pycat.ipCheck("224.255.128.0") == False
        assert pycat.ipCheck("224.0.128.255") == False
        assert pycat.ipCheck("0.255.128.255") == False
        assert pycat.ipCheck("224.255.128.255") == False

        assert pycat.ipCheck("0.128.0.0") == False
        assert pycat.ipCheck("224.128.0.0") == False
        assert pycat.ipCheck("0.128.255.0") == False
        assert pycat.ipCheck("0.128.0.255") == False
        assert pycat.ipCheck("224.128.255.0") == False
        assert pycat.ipCheck("224.128.0.255") == False
        assert pycat.ipCheck("0.128.255.255") == False
        assert pycat.ipCheck("224.128.255.255") == False

        assert pycat.ipCheck("128.0.0.0") == False
        assert pycat.ipCheck("128.255.0.0") == False
        assert pycat.ipCheck("128.0.255.0") == False
        assert pycat.ipCheck("128.0.0.255") == False
        assert pycat.ipCheck("128.255.255.0") == False
        assert pycat.ipCheck("128.255.0.255") == False
        assert pycat.ipCheck("128.0.255.255") == False
        assert pycat.ipCheck("128.255.255.255") == False

        assert pycat.ipCheck("0.0.0.0") == False
        assert pycat.ipCheck("224.0.0.0") == False
        assert pycat.ipCheck("0.255.0.0") == False
        assert pycat.ipCheck("0.0.255.0") == False
        assert pycat.ipCheck("0.0.0.255") == False
        assert pycat.ipCheck("224.255.0.0") == False
        assert pycat.ipCheck("224.0.255.0") == False
        assert pycat.ipCheck("224.0.0.255") == False
        assert pycat.ipCheck("0.255.255.0") == False
        assert pycat.ipCheck("0.255.0.255") == False
        assert pycat.ipCheck("0.0.255.255") == False
        assert pycat.ipCheck("224.255.255.0") == False
        assert pycat.ipCheck("224.255.0.255") == False
        assert pycat.ipCheck("224.0.255.255") == False
        assert pycat.ipCheck("0.255.255.255") == False
        assert pycat.ipCheck("224.255.255.255") == False
<|endoftext|>