<commit_before>from __future__ import print_function
from __future__ import with_statement

import pytest
import portalocker
from portalocker import utils

def test_exceptions(tmpfile):
    # Open the file 2 times
    a = open(tmpfile, 'a')
    b = open(tmpfile, 'a')

    # Lock exclusive non-blocking
    lock_flags = portalocker.LOCK_EX | portalocker.LOCK_NB

    # First lock file a
    portalocker.lock(a, lock_flags)

    # Now see if we can lock file b
    with pytest.raises(portalocker.LockException):
        portalocker.lock(b, lock_flags)

    # Cleanup
    a.close()
    b.close()

def test_utils_base():
    class Test(utils.LockBase):
        pass

def test_with_timeout(tmpfile):
    # Open the file 2 times
    with pytest.raises(portalocker.AlreadyLocked):
        with portalocker.Lock(tmpfile, timeout=0.1) as fh:
            print('writing some stuff to my cache...', file=fh)
            with portalocker.Lock(tmpfile, timeout=0.1, mode='wb',
                                  fail_when_locked=True):
                pass
            print('writing more stuff to my cache...', file=fh)

def test_without_timeout(tmpfile):
    # Open the file 2 times
    with pytest.raises(portalocker.LockException):
        with portalocker.Lock(tmpfile, timeout=None) as fh:
            print('writing some stuff to my cache...', file=fh)
            with portalocker.Lock(tmpfile, timeout=None, mode='w'):
                pass
            print('writing more stuff to my cache...', file=fh)

def test_without_fail(tmpfile):
    # Open the file 2 times
    with pytest.raises(portalocker.LockException):
        with portalocker.Lock(tmpfile, timeout=0.1) as fh:
            print('writing some stuff to my cache...', file=fh)
            lock = portalocker.Lock(tmpfile, timeout=0.1)
            lock.acquire(check_interval=0.05, fail_when_locked=False)

def test_simple(tmpfile):
    with open(tmpfile, 'w') as fh:
        fh.write('spam and eggs')

    fh = open(tmpfile, 'r+')
    portalocker.lock(fh, portalocker.LOCK_EX)

    fh.seek(13)
    fh.write('foo')

    # Make sure we didn't overwrite the original text
    fh.seek(0)
    assert fh.read(13) == 'spam and eggs'

    portalocker.unlock(fh)
    fh.close()

def test_truncate(tmpfile):
    with open(tmpfile, 'w') as fh:
        fh.write('spam and eggs')

    with portalocker.Lock(tmpfile, mode='a+') as fh:
        # Make sure we didn't overwrite the original text
        fh.seek(0)
        assert fh.read(13) == 'spam and eggs'

    with portalocker.Lock(tmpfile, mode='w+') as fh:
        # Make sure we truncated the file
        assert fh.read() == ''

def test_class(tmpfile):
    lock = portalocker.Lock(tmpfile)
    lock2 = portalocker.Lock(tmpfile, fail_when_locked=False, timeout=0.01)

    with lock:
        lock.acquire()

        with pytest.raises(portalocker.LockException):
            with lock2:
                pass

    with lock2:
        pass

def test_acquire_release(tmpfile):
    lock = portalocker.Lock(tmpfile)
    lock2 = portalocker.Lock(tmpfile, fail_when_locked=False)

    lock.acquire()  # acquire lock when nobody is using it
    with pytest.raises(portalocker.LockException):
        # another party should not be able to acquire the lock
        lock2.acquire(timeout=0.01)

        # re-acquire a held lock is a no-op
        lock.acquire()

    lock.release()  # release the lock
    lock.release()  # second release does nothing

def test_rlock_acquire_release_count(tmpfile):
    lock = portalocker.RLock(tmpfile)
    # Twice acquire
    h = lock.acquire()
    assert not h.closed
    lock.acquire()
    assert not h.closed

    # Two release
    lock.release()
    assert not h.closed
    lock.release()
    assert h.closed

def test_rlock_acquire_release(tmpfile):
    lock = portalocker.RLock(tmpfile)
    lock2 = portalocker.RLock(tmpfile, fail_when_locked=False)

    lock.acquire()  # acquire lock when nobody is using it
    with pytest.raises(portalocker.LockException):
        # another party should not be able to acquire the lock
        lock2.acquire(timeout=0.01)

    # Now acquire again
    lock.acquire()

    lock.release()  # release the lock
    lock.release()  # second release does nothing

def test_release_unacquired(tmpfile):
    with pytest.raises(portalocker.LockException):
        portalocker.RLock(tmpfile).release()

def test_exlusive(tmpfile):
    with open(tmpfile, 'w') as fh:
        fh.write('spam and eggs')

    fh = open(tmpfile, 'r')
    portalocker.lock(fh, portalocker.LOCK_EX | portalocker.LOCK_NB)

    # Make sure we can't read the locked file
    with pytest.raises(portalocker.LockException):
        with open(tmpfile, 'r') as fh2:
            portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB)
            fh2.read()

    # Make sure we can't write the locked file
    with pytest.raises(portalocker.LockException):
        with open(tmpfile, 'w+') as fh2:
            portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB)
            fh2.write('surprise and fear')

    # Make sure we can explicitly unlock the file
    portalocker.unlock(fh)
    fh.close()

def test_shared(tmpfile):
    with open(tmpfile, 'w') as fh:
        fh.write('spam and eggs')

    f = open(tmpfile, 'r')
    portalocker.lock(f, portalocker.LOCK_SH | portalocker.LOCK_NB)

    # Make sure we can read the locked file
    with open(tmpfile, 'r') as fh2:
        portalocker.lock(fh2, portalocker.LOCK_SH | portalocker.LOCK_NB)
        assert fh2.read() == 'spam and eggs'

    # Make sure we can't write the locked file
    with pytest.raises(portalocker.LockException):
        with open(tmpfile, 'w+') as fh2:
            portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB)
            fh2.write('surprise and fear')

    # Make sure we can explicitly unlock the file
    portalocker.unlock(f)
    f.close()

def test_blocking_timeout(tmpfile):
    flags = portalocker.LockFlags.SHARED

    with pytest.warns(UserWarning):
        with portalocker.Lock(tmpfile, timeout=5, flags=flags):
            pass

    lock = portalocker.Lock(tmpfile, flags=flags)
    with pytest.warns(UserWarning):
        lock.acquire(timeout=5)
<commit_msg>Added extra shared locks tests<commit_after>from __future__ import print_function
from __future__ import with_statement

import multiprocessing
import time

import pytest
import portalocker
from portalocker import utils

def test_exceptions(tmpfile):
    # Open the file 2 times
    a = open(tmpfile, 'a')
    b = open(tmpfile, 'a')

    # Lock exclusive non-blocking
    lock_flags = portalocker.LOCK_EX | portalocker.LOCK_NB

    # First lock file a
    portalocker.lock(a, lock_flags)

    # Now see if we can lock file b
    with pytest.raises(portalocker.LockException):
        portalocker.lock(b, lock_flags)

    # Cleanup
    a.close()
    b.close()

def test_utils_base():
    class Test(utils.LockBase):
        pass

def test_with_timeout(tmpfile):
    # Open the file 2 times
    with pytest.raises(portalocker.AlreadyLocked):
        with portalocker.Lock(tmpfile, timeout=0.1) as fh:
            print('writing some stuff to my cache...', file=fh)
            with portalocker.Lock(
                tmpfile, timeout=0.1, mode='wb',
                fail_when_locked=True
            ):
                pass
            print('writing more stuff to my cache...', file=fh)

def test_without_timeout(tmpfile):
    # Open the file 2 times
    with pytest.raises(portalocker.LockException):
        with portalocker.Lock(tmpfile, timeout=None) as fh:
            print('writing some stuff to my cache...', file=fh)
            with portalocker.Lock(tmpfile, timeout=None, mode='w'):
                pass
            print('writing more stuff to my cache...', file=fh)

def test_without_fail(tmpfile):
    # Open the file 2 times
    with pytest.raises(portalocker.LockException):
        with portalocker.Lock(tmpfile, timeout=0.1) as fh:
            print('writing some stuff to my cache...', file=fh)
            lock = portalocker.Lock(tmpfile, timeout=0.1)
            lock.acquire(check_interval=0.05, fail_when_locked=False)

def test_simple(tmpfile):
    with open(tmpfile, 'w') as fh:
        fh.write('spam and eggs')

    fh = open(tmpfile, 'r+')
    portalocker.lock(fh, portalocker.LOCK_EX)

    fh.seek(13)
    fh.write('foo')

    # Make sure we didn't overwrite the original text
    fh.seek(0)
    assert fh.read(13) == 'spam and eggs'

    portalocker.unlock(fh)
    fh.close()

def test_truncate(tmpfile):
    with open(tmpfile, 'w') as fh:
        fh.write('spam and eggs')

    with portalocker.Lock(tmpfile, mode='a+') as fh:
        # Make sure we didn't overwrite the original text
        fh.seek(0)
        assert fh.read(13) == 'spam and eggs'

    with portalocker.Lock(tmpfile, mode='w+') as fh:
        # Make sure we truncated the file
        assert fh.read() == ''

def test_class(tmpfile):
    lock = portalocker.Lock(tmpfile)
    lock2 = portalocker.Lock(tmpfile, fail_when_locked=False, timeout=0.01)

    with lock:
        lock.acquire()

        with pytest.raises(portalocker.LockException):
            with lock2:
                pass

    with lock2:
        pass

def test_acquire_release(tmpfile):
    lock = portalocker.Lock(tmpfile)
    lock2 = portalocker.Lock(tmpfile, fail_when_locked=False)

    lock.acquire()  # acquire lock when nobody is using it
    with pytest.raises(portalocker.LockException):
        # another party should not be able to acquire the lock
        lock2.acquire(timeout=0.01)

        # re-acquire a held lock is a no-op
        lock.acquire()

    lock.release()  # release the lock
    lock.release()  # second release does nothing

def test_rlock_acquire_release_count(tmpfile):
    lock = portalocker.RLock(tmpfile)
    # Twice acquire
    h = lock.acquire()
    assert not h.closed
    lock.acquire()
    assert not h.closed

    # Two release
    lock.release()
    assert not h.closed
    lock.release()
    assert h.closed

def test_rlock_acquire_release(tmpfile):
    lock = portalocker.RLock(tmpfile)
    lock2 = portalocker.RLock(tmpfile, fail_when_locked=False)

    lock.acquire()  # acquire lock when nobody is using it
    with pytest.raises(portalocker.LockException):
        # another party should not be able to acquire the lock
        lock2.acquire(timeout=0.01)

    # Now acquire again
    lock.acquire()

    lock.release()  # release the lock
    lock.release()  # second release does nothing

def test_release_unacquired(tmpfile):
    with pytest.raises(portalocker.LockException):
        portalocker.RLock(tmpfile).release()

def test_exlusive(tmpfile):
    with open(tmpfile, 'w') as fh:
        fh.write('spam and eggs')

    fh = open(tmpfile, 'r')
    portalocker.lock(fh, portalocker.LOCK_EX | portalocker.LOCK_NB)

    # Make sure we can't read the locked file
    with pytest.raises(portalocker.LockException):
        with open(tmpfile, 'r') as fh2:
            portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB)
            fh2.read()

    # Make sure we can't write the locked file
    with pytest.raises(portalocker.LockException):
        with open(tmpfile, 'w+') as fh2:
            portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB)
            fh2.write('surprise and fear')

    # Make sure we can explicitly unlock the file
    portalocker.unlock(fh)
    fh.close()

def test_shared(tmpfile):
    with open(tmpfile, 'w') as fh:
        fh.write('spam and eggs')

    f = open(tmpfile, 'r')
    portalocker.lock(f, portalocker.LOCK_SH | portalocker.LOCK_NB)

    # Make sure we can read the locked file
    with open(tmpfile, 'r') as fh2:
        portalocker.lock(fh2, portalocker.LOCK_SH | portalocker.LOCK_NB)
        assert fh2.read() == 'spam and eggs'

    # Make sure we can't write the locked file
    with pytest.raises(portalocker.LockException):
        with open(tmpfile, 'w+') as fh2:
            portalocker.lock(fh2, portalocker.LOCK_EX | portalocker.LOCK_NB)
            fh2.write('surprise and fear')

    # Make sure we can explicitly unlock the file
    portalocker.unlock(f)
    f.close()

def test_blocking_timeout(tmpfile):
    flags = portalocker.LockFlags.SHARED

    with pytest.warns(UserWarning):
        with portalocker.Lock(tmpfile, timeout=5, flags=flags):
            pass

    lock = portalocker.Lock(tmpfile, flags=flags)
    with pytest.warns(UserWarning):
        lock.acquire(timeout=5)

def shared_lock(filename, **kwargs):
    with portalocker.Lock(
        filename,
        timeout=0.1,
        fail_when_locked=False,
        flags=portalocker.LockFlags.SHARED | portalocker.LockFlags.NON_BLOCKING,
    ):
        time.sleep(0.2)
        return True

def shared_lock_fail(filename, **kwargs):
    with portalocker.Lock(
        filename,
        timeout=0.1,
        fail_when_locked=True,
        flags=portalocker.LockFlags.SHARED | portalocker.LockFlags.NON_BLOCKING,
    ):
        time.sleep(0.2)
        return True

def test_shared_processes(tmpfile):
    # Force spawning the process so we don't accidently inherit the lock
    # I'm not a 100% certain this will work correctly unfortunately... there
    # is some potential for breaking other tests
    multiprocessing.set_start_method('spawn')

    with multiprocessing.Pool(processes=2) as pool:
        for result in pool.imap_unordered(shared_lock, 3 * [tmpfile]):
            assert result is True

    with multiprocessing.Pool(processes=2) as pool:
        for result in pool.imap_unordered(shared_lock_fail, 3 * [tmpfile]):
            assert result is True
<|endoftext|>


<commit_before>import os
import logging
import functools

def find_readers():
    """Auto discover readers in installed apps.

    Each readers.py should have the READERS list with reader classes
    """

    from django.conf import settings
    from django.utils.importlib import import_module
    from django.utils.module_loading import module_has_submodule

    readers = []
    for app in settings.INSTALLED_APPS:
        mod = import_module(app)
        # Attempt to import the app's admin module.
        try:
            walk_mod = import_module('%s.readers' % app)
            if hasattr(walk_mod, 'READERS'):
                readers.extend(walk_mod.READERS)
            else:
                logging.warn(
                    'readers found in app %s, but has no READERS attrib',
                    app)
        except:
            # Decide whether to bubble up this error. If the app just
            # doesn't have an readers module, we can ignore the error
            # attempting to import it, otherwise we want it to bubble up.
            if module_has_submodule(mod, 'readers'):
                raise

    return readers

def find_files(root, extensions):
    """Generator finding files from a root dir with specific extensions

    :param root: Root directory for the search
    :param extensions: List of extensions to search (e.g: ['.rst', '.txt'])
    """

    for root, dirs, files in os.walk(root):
        for f in files:
            if os.path.splitext(f)[1] in extensions:
                yield os.path.join(root, f)

def i18n_permalink(*args):
    """Return the correct URL in case of not the default language.
    relaces djangos models.permalink.

    Assumes the name of the i18n version of the url pattern is prefix with
    'i18n_' (This is done in the statirator's default urls.py).

    It looks for a `language` field for the instance. If the name is different,
    pass it as the first param to the decorator. e.g::

        @i18n_permalink('lang_code')
        def get_absolute_url(self):
            return (...)

    """
    def outer(f):
        @functools.wraps(f)
        def wrapper(obj, *args, **kwargs):

            bits = f(obj, *args, **kwargs)
            name = bits[0]

            lang = getattr(obj, language_field)
            return i18n_reverse(lang, name, None, *bits[1:3])

        return wrapper

    if len(args) == 1 and callable(args[0]):
        # No arguments, this is the decorator
        # Set default values for the arguments
        language_field = 'language'
        return outer(args[0])
    else:
        language_field = args[0]
        return outer

def i18n_reverse(language, viewname, *args, **kwargs):
    """Django's reverse with a pinch of i18n.

    Assumes the name of the i18n version of the url pattern is prefix with
    'i18n_' (This is done in the statirator's default urls.py).

    """
    from django.core.urlresolvers import reverse
    from django.conf import settings
    from django.utils.translation import activate, get_language

    cur_lang = get_language()

    # activate the obj's lang for correct i18n_patterns reverse
    activate(language)
    if language != settings.LANGUAGE_CODE:
        viewname = 'i18n_' + viewname

    res = reverse(viewname, *args, **kwargs)
    activate(cur_lang)

    return res
<commit_msg>RequestFactory, partial template rendering<commit_after>import os
import logging
import functools

def find_readers():
    """Auto discover readers in installed apps.

    Each readers.py should have the READERS list with reader classes
    """

    from django.conf import settings
    from django.utils.importlib import import_module
    from django.utils.module_loading import module_has_submodule

    readers = []
    for app in settings.INSTALLED_APPS:
        mod = import_module(app)
        # Attempt to import the app's admin module.
        try:
            walk_mod = import_module('%s.readers' % app)
            if hasattr(walk_mod, 'READERS'):
                readers.extend(walk_mod.READERS)
            else:
                logging.warn(
                    'readers found in app %s, but has no READERS attrib',
                    app)
        except:
            # Decide whether to bubble up this error. If the app just
            # doesn't have an readers module, we can ignore the error
            # attempting to import it, otherwise we want it to bubble up.
            if module_has_submodule(mod, 'readers'):
                raise

    return readers

def find_files(root, extensions):
    """Generator finding files from a root dir with specific extensions

    :param root: Root directory for the search
    :param extensions: List of extensions to search (e.g: ['.rst', '.txt'])
    """

    for root, dirs, files in os.walk(root):
        for f in files:
            if os.path.splitext(f)[1] in extensions:
                yield os.path.join(root, f)

def i18n_permalink(*args):
    """Return the correct URL in case of not the default language.
    relaces djangos models.permalink.

    Assumes the name of the i18n version of the url pattern is prefix with
    'i18n_' (This is done in the statirator's default urls.py).

    It looks for a `language` field for the instance. If the name is different,
    pass it as the first param to the decorator. e.g::

        @i18n_permalink('lang_code')
        def get_absolute_url(self):
            return (...)

    """
    def outer(f):
        @functools.wraps(f)
        def wrapper(obj, *args, **kwargs):

            bits = f(obj, *args, **kwargs)
            name = bits[0]

            lang = getattr(obj, language_field)
            return i18n_reverse(lang, name, None, *bits[1:3])

        return wrapper

    if len(args) == 1 and callable(args[0]):
        # No arguments, this is the decorator
        # Set default values for the arguments
        language_field = 'language'
        return outer(args[0])
    else:
        language_field = args[0]
        return outer

def i18n_reverse(language, viewname, *args, **kwargs):
    """Django's reverse with a pinch of i18n.

    Assumes the name of the i18n version of the url pattern is prefix with
    'i18n_' (This is done in the statirator's default urls.py).

    """
    from django.core.urlresolvers import reverse
    from django.conf import settings
    from django.utils.translation import activate, get_language

    cur_lang = get_language()

    # activate the obj's lang for correct i18n_patterns reverse
    activate(language)
    if language != settings.LANGUAGE_CODE:
        viewname = 'i18n_' + viewname

    res = reverse(viewname, *args, **kwargs)
    activate(cur_lang)

    return res

# The following allow rendering specific template blocks (for data extraction)
# Taken from: http://djangosnippets.org/snippets/942/

def get_template(template):
    from django.template import loader

    if isinstance(template, (tuple, list)):
        return loader.select_template(template)
    return loader.get_template(template)

class BlockNotFound(Exception):
    pass

def render_template_block(template, block, context):
    """
    Renders a single block from a template. This template should have previously been rendered.
    """
    return render_template_block_nodelist(template.nodelist, block, context)

def render_template_block_nodelist(nodelist, block, context):
    from django.template.loader_tags import BlockNode, ExtendsNode

    for node in nodelist:
        if isinstance(node, BlockNode) and node.name == block:
            return node.render(context)
        for key in ('nodelist', 'nodelist_true', 'nodelist_false'):
            if hasattr(node, key):
                try:
                    return render_template_block_nodelist(getattr(node, key), block, context)
                except:
                    pass
    for node in nodelist:
        if isinstance(node, ExtendsNode):
            try:
                return render_template_block(node.get_parent(context), block, context)
            except BlockNotFound:
                pass
    raise BlockNotFound

def render_block_to_string(template_or_name, block, dictionary=None,
                           context_instance=None):
    """
    Loads the given template_name and renders the given block with the given dictionary as
    context. Returns a string.
    """
    from django.template import Context, Template

    dictionary = dictionary or {}
    if isinstance(template_or_name, Template):
        t = template_or_name
    else:
        t = get_template(template_or_name)

    if context_instance:
        context_instance.update(dictionary)
    else:
        context_instance = Context(dictionary)
    t.render(context_instance)
    return render_template_block(t, block, context_instance)

from django.test import Client
from django.core.handlers.wsgi import WSGIRequest

class RequestFactory(Client):
    """
    Class that lets you create mock Request objects for use in testing.

    Usage:

    rf = RequestFactory()
    get_request = rf.get('/hello/')
    post_request = rf.post('/submit/', {'foo': 'bar'})

    This class re-uses the django.test.client.Client interface, docs here:
    http://www.djangoproject.com/documentation/testing/#the-test-client

    Once you have a request object you can pass it to any view function,
    just as if that view had been hooked up using a URLconf.

    """
    def request(self, **request):
        """
        Similar to parent class, but returns the request object as soon as it
        has created it.
        """
        environ = {
            'HTTP_COOKIE': self.cookies,
            'PATH_INFO': '/',
            'QUERY_STRING': '',
            'REQUEST_METHOD': 'GET',
            'SCRIPT_NAME': '',
            'SERVER_NAME': 'testserver',
            'SERVER_PORT': 80,
            'SERVER_PROTOCOL': 'HTTP/1.1',
        }
        environ.update(self.defaults)
        environ.update(request)
        return WSGIRequest(environ)
<|endoftext|>


<commit_before>__version__ = "0.0.29"
<commit_msg>version 0.0.30<commit_after>__version__ = "0.0.30"
<|endoftext|>


<commit_before>import logging

import numpy as np
import scipy as sp
import pandas as pd
import h5py

import inspect
import time

from .instruments.instrument import Instrument

class Quantity(object):
    """Physical quantity to be measured."""
    def __init__(self, name, unit=None):
        super(Quantity, self).__init__()
        self.name   = name
        self.unit   = unit
        self.method = None
        self._value = None
        self.delay_before = 0
        self.delay_after = 0

    @property
    def value(self):
        return self._value

    @value.setter
    def value(self, value):
        self._value = value

    def assign_method(self, method):
        logging.debug("Setting method of Quantity %s to %s" % (self.name, str(method)) )
        self.method = method

    def measure(self):
        logging.debug("%s Being asked to measure" % self.name)
        if self.delay_before is not None:
            time.sleep(self.delay_before)

        try:
            self._value = self.method()
        except:
            self._value = None
            print("Unable to measure %s." % self.name)

        if self.delay_after is not None:
            time.sleep(self.delay_after)

    def __str__(self):
        result = ""
        result += "%s" % str(self._value)
        if self.unit:
            result += " %s" % self.unit
        return result

    def __repr__(self):
        result = "<Quantity(name='%s'" % self.name
        result += ",value=%s" % repr(self._value)
        if self.unit:
            result += ",unit='%s'" % self.unit
        return result + ")>"

class Parameter(object):
    """ Encapsulates the information for an experiment parameter"""

    def __init__(self, name, unit=None, default=None, abstract=False):
        self.name     = name
        self._value   = default
        self.unit     = unit
        self.default  = default
        self.method   = None
        self.abstract = abstract # Is this something we can actually push?

        # Hooks to be called before or after updating a sweep parameter
        self.pre_push_hooks = []
        self.post_push_hooks = []

    def add_pre_push_hook(self, hook):
        self.pre_push_hooks.append(hook)

    def add_post_push_hook(self, hook):
        self.post_push_hooks.append(hook)

    @property
    def value(self):
        return self._value

    @value.setter
    def value(self, value):
        self._value = value

    def __str__(self):
        result = ""
        result += "%s" % str(self.value)
        if self.unit:
            result += " %s" % self.unit
        return result

    def __repr__(self):
        result = "<Parameter(name='%s'" % self.name
        result += ",value=%s" % repr(self.value)
        if self.unit:
            result += ",unit='%s'" % self.unit
        return result + ")>"

    def assign_method(self, method):
        logging.debug("Setting method of Parameter %s to %s" % (self.name, str(method)) )
        self.method = method

    def push(self):
        if not self.abstract:
            self.method(self._value)

class FloatParameter(Parameter):

    @property
    def value(self):
        return self._value

    @value.setter
    def value(self, value):
        try:
            self._value = float(value)
        except ValueError:
            raise ValueError("FloatParameter given non-float value of "
                             "type '%s'" % type(value))

    def __repr__(self):
        result = super(FloatParameter, self).__repr__()
        return result.replace("<Parameter", "<FloatParameter", 1)

class IntParameter(Parameter):

    @property
    def value(self):
        return self._value

    @value.setter
    def value(self, value):
        try:
            self._value = int(value)
        except ValueError:
            raise ValueError("IntParameter given non-int value of "
                             "type '%s'" % type(value))

    def __repr__(self):
        result = super(IntParameter, self).__repr__()
        return result.replace("<Parameter", "<IntParameter", 1)

class Procedure(object):
    """The measurement loop to be run for each set of sweep parameters."""
    def __init__(self):
        super(Procedure, self).__init__()
        self._gather_parameters()
        self._gather_quantities()
        self._gather_instruments()

    def _gather_parameters(self):
        """ Collects all the Parameter objects for this procedure and stores\
        them in a dictionary.
        """
        self._parameters  = {}
        for item in dir(self):
            parameter = getattr(self, item)
            if isinstance(parameter, Parameter):
                self._parameters[item] = parameter

    def _gather_quantities(self):
        """ Collects all the Quantity objects for this procedure and stores\
        them in a dictionary.
        """
        self._quantities  = {}
        for item in dir(self):
            quantity = getattr(self, item)
            if isinstance(quantity, Quantity):
                self._quantities[item] = quantity

    def _gather_instruments(self):
        """ Collects all the Quantity objects for this procedure and stores\
        them in a dictionary.
        """
        self._instruments = {}
        for item in dir(self):
            inst = getattr(self, item)
            if isinstance(inst, Instrument):
                self._instruments[item] = inst

    def init_instruments(self):
        """Gets run before a sweep starts"""
        pass

    def shutdown_instruments(self):
        """Gets run after a sweep ends, or when the program is terminated."""
        pass

    def run(self):
        """The actual measurement that gets run for each set of values in a sweep."""
        pass
<commit_msg>Moved procedure introspection into a metaclass. Instruments, parameters, and quantities are now accumulated before the class is instantiated.<commit_after>import logging

import numpy as np
import scipy as sp
import pandas as pd
import h5py

import inspect
import time

from .instruments.instrument import Instrument

class Quantity(object):
    """Physical quantity to be measured."""
    def __init__(self, name, unit=None):
        super(Quantity, self).__init__()
        self.name   = name
        self.unit   = unit
        self.method = None
        self._value = None
        self.delay_before = 0
        self.delay_after = 0

    @property
    def value(self):
        return self._value

    @value.setter
    def value(self, value):
        self._value = value

    def assign_method(self, method):
        logging.debug("Setting method of Quantity %s to %s" % (self.name, str(method)) )
        self.method = method

    def measure(self):
        logging.debug("%s Being asked to measure" % self.name)
        if self.delay_before is not None:
            time.sleep(self.delay_before)

        try:
            self._value = self.method()
        except:
            self._value = None
            print("Unable to measure %s." % self.name)

        if self.delay_after is not None:
            time.sleep(self.delay_after)

    def __str__(self):
        result = ""
        result += "%s" % str(self._value)
        if self.unit:
            result += " %s" % self.unit
        return result

    def __repr__(self):
        result = "<Quantity(name='%s'" % self.name
        result += ",value=%s" % repr(self._value)
        if self.unit:
            result += ",unit='%s'" % self.unit
        return result + ")>"

class Parameter(object):
    """ Encapsulates the information for an experiment parameter"""

    def __init__(self, name, unit=None, default=None, abstract=False):
        self.name     = name
        self._value   = default
        self.unit     = unit
        self.default  = default
        self.method   = None
        self.abstract = abstract # Is this something we can actually push?

        # Hooks to be called before or after updating a sweep parameter
        self.pre_push_hooks = []
        self.post_push_hooks = []

    def add_pre_push_hook(self, hook):
        self.pre_push_hooks.append(hook)

    def add_post_push_hook(self, hook):
        self.post_push_hooks.append(hook)

    @property
    def value(self):
        return self._value

    @value.setter
    def value(self, value):
        self._value = value

    def __str__(self):
        result = ""
        result += "%s" % str(self.value)
        if self.unit:
            result += " %s" % self.unit
        return result

    def __repr__(self):
        result = "<Parameter(name='%s'" % self.name
        result += ",value=%s" % repr(self.value)
        if self.unit:
            result += ",unit='%s'" % self.unit
        return result + ")>"

    def assign_method(self, method):
        logging.debug("Setting method of Parameter %s to %s" % (self.name, str(method)) )
        self.method = method

    def push(self):
        if not self.abstract:
            self.method(self._value)

class FloatParameter(Parameter):

    @property
    def value(self):
        return self._value

    @value.setter
    def value(self, value):
        try:
            self._value = float(value)
        except ValueError:
            raise ValueError("FloatParameter given non-float value of "
                             "type '%s'" % type(value))

    def __repr__(self):
        result = super(FloatParameter, self).__repr__()
        return result.replace("<Parameter", "<FloatParameter", 1)

class IntParameter(Parameter):

    @property
    def value(self):
        return self._value

    @value.setter
    def value(self, value):
        try:
            self._value = int(value)
        except ValueError:
            raise ValueError("IntParameter given non-int value of "
                             "type '%s'" % type(value))

    def __repr__(self):
        result = super(IntParameter, self).__repr__()
        return result.replace("<Parameter", "<IntParameter", 1)

class MetaProcedure(type):
    """Meta class to bake the instrument objects into a class description
    """
    logging.basicConfig(format='%(levelname)s:\t%(message)s', level=logging.DEBUG)

    def __init__(self, name, bases, dct):
        type.__init__(self, name, bases, dct)
        logging.debug("Adding controls to %s", name)
        self._parameters  = {}
        self._quantities  = {}
        self._instruments  = {}

        for k,v in dct.items():
            if isinstance(v, Instrument):
                logging.debug("Found '%s' instrument", k)
                self._instruments[k] = v
            elif isinstance(v, Parameter):
                logging.debug("Found '%s' parameter", k)
                self._parameters[k] = v
            elif isinstance(v, Quantity):
                logging.debug("Found '%s' quantity", k)
                self._quantities[k] = v

class Procedure(metaclass=MetaProcedure):
    """The measurement loop to be run for each set of sweep parameters."""
    def __init__(self):
        super(Procedure, self).__init__()

    def init_instruments(self):
        """Gets run before a sweep starts"""
        pass

    def shutdown_instruments(self):
        """Gets run after a sweep ends, or when the program is terminated."""
        pass

    def run(self):
        """The actual measurement that gets run for each set of values in a sweep."""
        pass
<|endoftext|>


<commit_before>"""
Base tools for handling various kinds of data structures, attaching metadata to
results, and doing data cleaning
"""
from statsmodels.compat.python import reduce, iteritems, lmap, zip, range
import numpy as np
from pandas import DataFrame, Series, TimeSeries, isnull
from statsmodels.tools.decorators import (resettable_cache, cache_readonly,
                                          cache_writable)
import statsmodels.tools.data as data_util
from statsmodels.tools.sm_exceptions import MissingDataError

def _asarray_2dcolumns(x):
    if np.asarray(x).ndim > 1 and np.asarray(x).squeeze().ndim == 1:
        return

def _asarray_2d_null_rows(x):
    """
    Makes sure input is an array and is 2d. Makes sure output is 2d. True
    indicates a null in the rows of 2d x.
    """
    #Have to have the asarrays because isnull doesn't account for array-like
    #input
    x = np.asarray(x)
    if x.ndim == 1:
        x = x[:, None]
    return np.any(isnull(x), axis=1)[:, None]

def _nan_rows(*arrs):
    """
    Returns a boolean array which is True where any of the rows in any
    of the _2d_ arrays in arrs are NaNs. Inputs can be any mixture of Series,
    DataFrames or array-like.
    """
    if len(arrs) == 1:
        arrs += ([[False]],)

    def _nan_row_maybe_two_inputs(x, y):
        # check for dtype bc dataframe has dtypes
        x_is_boolean_array = hasattr(x, 'dtype') and x.dtype == bool and x
        return np.logical_or(_asarray_2d_null_rows(x),
                             (x_is_boolean_array | _asarray_2d_null_rows(y)))
    return reduce(_nan_row_maybe_two_inputs, arrs).squeeze()

class ModelData(object):
    """
    Class responsible for handling input data and extracting metadata into the
    appropriate form
    """
    def __init__(self, endog, exog=None, missing='none', hasconst=None,
                 **kwargs):
        if missing != 'none':
            arrays, nan_idx = self.handle_missing(endog, exog, missing,
                                                  **kwargs)
            self.missing_row_idx = nan_idx
            self.__dict__.update(arrays)  # attach all the data arrays
            self.orig_endog = self.endog
            self.orig_exog = self.exog
            self.endog, self.exog = self._convert_endog_exog(self.endog,
                                                             self.exog)
        else:
            self.__dict__.update(kwargs)  # attach the extra arrays anyway
            self.orig_endog = endog
            self.orig_exog = exog
            self.endog, self.exog = self._convert_endog_exog(endog, exog)

        # this has side-effects, attaches k_constant and const_idx
        self._handle_constant(hasconst)
        self._check_integrity()
        self._cache = resettable_cache()

    def _handle_constant(self, hasconst):
        if hasconst is not None:
            if hasconst:
                self.k_constant = 1
                self.const_idx = None
            else:
                self.k_constant = 0
                self.const_idx = None
        else:
            try:  # to detect where the constant is
                const_idx = np.where(self.exog.var(axis=0) == 0)[0].squeeze()
                self.k_constant = const_idx.size
                if self.k_constant > 1:
                    raise ValueError("More than one constant detected.")
                else:
                    self.const_idx = const_idx
            except:  # should be an index error but who knows, means no const
                self.const_idx = None
                self.k_constant = 0

    @classmethod
    def _drop_nans(cls, x, nan_mask):
        return x[nan_mask]

    @classmethod
    def _drop_nans_2d(cls, x, nan_mask):
        return x[nan_mask][:, nan_mask]

    @classmethod
    def handle_missing(cls, endog, exog, missing, **kwargs):
        """
        This returns a dictionary with keys endog, exog and the keys of
        kwargs. It preserves Nones.
        """
        none_array_names = []

        if exog is not None:
            combined = (endog, exog)
            combined_names = ['endog', 'exog']
        else:
            combined = (endog,)
            combined_names = ['endog']
            none_array_names += ['exog']

        # deal with other arrays
        combined_2d = ()
        combined_2d_names = []
        if len(kwargs):
            for key, value_array in iteritems(kwargs):
                if value_array is None or value_array.ndim == 0:
                    none_array_names += [key]
                    continue
                # grab 1d arrays
                if value_array.ndim == 1:
                    combined += (value_array,)
                    combined_names += [key]
                elif value_array.squeeze().ndim == 1:
                    combined += (value_array,)
                    combined_names += [key]

                # grab 2d arrays that are _assumed_ to be symmetric
                elif value_array.ndim == 2:
                    combined_2d += (value_array,)
                    combined_2d_names += [key]
                else:
                    raise ValueError("Arrays with more than 2 dimensions "
                                     "aren't yet handled")

        nan_mask = _nan_rows(*combined)
        if combined_2d:
            nan_mask = _nan_rows(*(nan_mask[:, None],) + combined_2d)

        if missing == 'raise' and np.any(nan_mask):
            raise MissingDataError("NaNs were encountered in the data")

        elif missing == 'drop':
            nan_mask = ~nan_mask
            drop_nans = lambda x: cls._drop_nans(x, nan_mask)
            drop_nans_2d = lambda x: cls._drop_nans_2d(x, nan_mask)
            combined = dict(zip(combined_names, lmap(drop_nans, combined)))
            if combined_2d:
                combined.update(dict(zip(combined_2d_names,
                                         lmap(drop_nans_2d, combined_2d))))
            if none_array_names:
                combined.update(dict(zip(none_array_names,
                                         [None] * len(none_array_names))))
            return combined, np.where(~nan_mask)[0].tolist()
        else:
            raise ValueError("missing option %s not understood" % missing)

    def _convert_endog_exog(self, endog, exog):

        # for consistent outputs if endog is (n,1)
        yarr = self._get_yarr(endog)
        xarr = None
        if exog is not None:
            xarr = self._get_xarr(exog)
            if xarr.ndim == 1:
                xarr = xarr[:, None]
            if xarr.ndim != 2:
                raise ValueError("exog is not 1d or 2d")

        return yarr, xarr

    @cache_writable()
    def ynames(self):
        endog = self.orig_endog
        ynames = self._get_names(endog)
        if not ynames:
            ynames = _make_endog_names(self.endog)

        if len(ynames) == 1:
            return ynames[0]
        else:
            return list(ynames)

    @cache_writable()
    def xnames(self):
        exog = self.orig_exog
        if exog is not None:
            xnames = self._get_names(exog)
            if not xnames:
                xnames = _make_exog_names(self.exog)
            return list(xnames)
        return None

    @cache_readonly
    def row_labels(self):
        exog = self.orig_exog
        if exog is not None:
            row_labels = self._get_row_labels(exog)
        else:
            endog = self.orig_endog
            row_labels = self._get_row_labels(endog)
        return row_labels

    def _get_row_labels(self, arr):
        return None

    def _get_names(self, arr):
        if isinstance(arr, DataFrame):
            return list(arr.columns)
        elif isinstance(arr, Series):
            if arr.name:
                return [arr.name]
            else:
                return
        else:
            try:
                return arr.dtype.names
            except AttributeError:
                pass

        return None

    def _get_yarr(self, endog):
        if data_util._is_structured_ndarray(endog):
            endog = data_util.struct_to_ndarray(endog)
        endog = np.asarray(endog)
        if len(endog) == 1:  # never squeeze to a scalar
            if endog.ndim == 1:
                return endog
            elif endog.ndim > 1:
                return np.asarray([endog.squeeze()])

        return endog.squeeze()

    def _get_xarr(self, exog):
        if data_util._is_structured_ndarray(exog):
            exog = data_util.struct_to_ndarray(exog)
        return np.asarray(exog)

    def _check_integrity(self):
        if self.exog is not None:
            if len(self.exog) != len(self.endog):
                raise ValueError("endog and exog matrices are different sizes")

    def wrap_output(self, obj, how='columns'):
        if how == 'columns':
            return self.attach_columns(obj)
        elif how == 'rows':
            return self.attach_rows(obj)
        elif how == 'cov':
            return self.attach_cov(obj)
        elif how == 'dates':
            return self.attach_dates(obj)
        elif how == 'columns_eq':
            return self.attach_columns_eq(obj)
        elif how == 'cov_eq':
            return self.attach_cov_eq(obj)
        else:
            return obj

    def attach_columns(self, result):
        return result

    def attach_columns_eq(self, result):
        return result

    def attach_cov(self, result):
        return result

    def attach_cov_eq(self, result):
        return result

    def attach_rows(self, result):
        return result

    def attach_dates(self, result):
        return result

class PatsyData(ModelData):
    def _get_names(self, arr):
        return arr.design_info.column_names

class PandasData(ModelData):
    """
    Data handling class which knows how to reattach pandas metadata to model
    results
    """
    @classmethod
    def _drop_nans(cls, x, nan_mask):
        if hasattr(x, 'ix'):
            return x.ix[nan_mask]
        else:  # extra arguments could be plain ndarrays
            return super(PandasData, cls)._drop_nans(x, nan_mask)

    @classmethod
    def _drop_nans_2d(cls, x, nan_mask):
        if hasattr(x, 'ix'):
            return x.ix[nan_mask].ix[:, nan_mask]
        else:  # extra arguments could be plain ndarrays
            return super(PandasData, cls)._drop_nans_2d(x, nan_mask)

    def _check_integrity(self):
        endog, exog = self.orig_endog, self.orig_exog
        # exog can be None and we could be upcasting one or the other
        if (exog is not None and
                (hasattr(endog, 'index') and hasattr(exog, 'index')) and
                not self.orig_endog.index.equals(self.orig_exog.index)):
            raise ValueError("The indices for endog and exog are not aligned")
        super(PandasData, self)._check_integrity()

    def _get_row_labels(self, arr):
        try:
            return arr.index
        except AttributeError:
            # if we've gotten here it's because endog is pandas and
            # exog is not, so just return the row labels from endog
            return self.orig_endog.index

    def attach_columns(self, result):
        # this can either be a 1d array or a scalar
        # don't squeeze because it might be a 2d row array
        # if it needs a squeeze, the bug is elsewhere
        if result.ndim <= 1:
            return Series(result, index=self.xnames)
        else:  # for e.g., confidence intervals
            return DataFrame(result, index=self.xnames)

    def attach_columns_eq(self, result):
        return DataFrame(result, index=self.xnames, columns=self.ynames)

    def attach_cov(self, result):
        return DataFrame(result, index=self.xnames, columns=self.xnames)

    def attach_cov_eq(self, result):
        return DataFrame(result, index=self.ynames, columns=self.ynames)

    def attach_rows(self, result):
        # assumes if len(row_labels) > len(result) it's bc it was truncated
        # at the front, for AR lags, for example
        if result.squeeze().ndim == 1:
            return Series(result, index=self.row_labels[-len(result):])
        else:  # this is for VAR results, may not be general enough
            return DataFrame(result, index=self.row_labels[-len(result):],
                             columns=self.ynames)

    def attach_dates(self, result):
        return TimeSeries(result, index=self.predict_dates)

def _make_endog_names(endog):
    if endog.ndim == 1 or endog.shape[1] == 1:
        ynames = ['y']
    else:  # for VAR
        ynames = ['y%d' % (i+1) for i in range(endog.shape[1])]

    return ynames

def _make_exog_names(exog):
    exog_var = exog.var(0)
    if (exog_var == 0).any():
        # assumes one constant in first or last position
        # avoid exception if more than one constant
        const_idx = exog_var.argmin()
        exog_names = ['x%d' % i for i in range(1, exog.shape[1])]
        exog_names.insert(const_idx, 'const')
    else:
        exog_names = ['x%d' % i for i in range(1, exog.shape[1]+1)]

    return exog_names

def handle_missing(endog, exog=None, missing='none', **kwargs):
    klass = handle_data_class_factory(endog, exog)
    if missing == 'none':
        ret_dict = dict(endog=endog, exog=exog)
        ret_dict.update(kwargs)
        return ret_dict, None
    return klass.handle_missing(endog, exog, missing=missing, **kwargs)

def handle_data_class_factory(endog, exog):
    """
    Given inputs
    """
    if data_util._is_using_ndarray_type(endog, exog):
        klass = ModelData
    elif data_util._is_using_pandas(endog, exog):
        klass = PandasData
    elif data_util._is_using_patsy(endog, exog):
        klass = PatsyData
    # keep this check last
    elif data_util._is_using_ndarray(endog, exog):
        klass = ModelData
    else:
        raise ValueError('unrecognized data structures: %s / %s' %
                         (type(endog), type(exog)))
    return klass

def handle_data(endog, exog, missing='none', hasconst=None, **kwargs):
    # deal with lists and tuples up-front
    if isinstance(endog, (list, tuple)):
        endog = np.asarray(endog)
    if isinstance(exog, (list, tuple)):
        exog = np.asarray(exog)

    klass = handle_data_class_factory(endog, exog)
    return klass(endog, exog=exog, missing=missing, hasconst=hasconst,
                 **kwargs)
<commit_msg>BUG: Allow missing='raise' and no missing data. Closes #1700<commit_after>"""
Base tools for handling various kinds of data structures, attaching metadata to
results, and doing data cleaning
"""
from statsmodels.compat.python import reduce, iteritems, lmap, zip, range
import numpy as np
from pandas import DataFrame, Series, TimeSeries, isnull
from statsmodels.tools.decorators import (resettable_cache, cache_readonly,
                                          cache_writable)
import statsmodels.tools.data as data_util
from statsmodels.tools.sm_exceptions import MissingDataError

def _asarray_2dcolumns(x):
    if np.asarray(x).ndim > 1 and np.asarray(x).squeeze().ndim == 1:
        return

def _asarray_2d_null_rows(x):
    """
    Makes sure input is an array and is 2d. Makes sure output is 2d. True
    indicates a null in the rows of 2d x.
    """
    #Have to have the asarrays because isnull doesn't account for array-like
    #input
    x = np.asarray(x)
    if x.ndim == 1:
        x = x[:, None]
    return np.any(isnull(x), axis=1)[:, None]

def _nan_rows(*arrs):
    """
    Returns a boolean array which is True where any of the rows in any
    of the _2d_ arrays in arrs are NaNs. Inputs can be any mixture of Series,
    DataFrames or array-like.
    """
    if len(arrs) == 1:
        arrs += ([[False]],)

    def _nan_row_maybe_two_inputs(x, y):
        # check for dtype bc dataframe has dtypes
        x_is_boolean_array = hasattr(x, 'dtype') and x.dtype == bool and x
        return np.logical_or(_asarray_2d_null_rows(x),
                             (x_is_boolean_array | _asarray_2d_null_rows(y)))
    return reduce(_nan_row_maybe_two_inputs, arrs).squeeze()

class ModelData(object):
    """
    Class responsible for handling input data and extracting metadata into the
    appropriate form
    """
    def __init__(self, endog, exog=None, missing='none', hasconst=None,
                 **kwargs):
        if missing != 'none':
            arrays, nan_idx = self.handle_missing(endog, exog, missing,
                                                  **kwargs)
            self.missing_row_idx = nan_idx
            self.__dict__.update(arrays)  # attach all the data arrays
            self.orig_endog = self.endog
            self.orig_exog = self.exog
            self.endog, self.exog = self._convert_endog_exog(self.endog,
                                                             self.exog)
        else:
            self.__dict__.update(kwargs)  # attach the extra arrays anyway
            self.orig_endog = endog
            self.orig_exog = exog
            self.endog, self.exog = self._convert_endog_exog(endog, exog)

        # this has side-effects, attaches k_constant and const_idx
        self._handle_constant(hasconst)
        self._check_integrity()
        self._cache = resettable_cache()

    def _handle_constant(self, hasconst):
        if hasconst is not None:
            if hasconst:
                self.k_constant = 1
                self.const_idx = None
            else:
                self.k_constant = 0
                self.const_idx = None
        else:
            try:  # to detect where the constant is
                const_idx = np.where(self.exog.var(axis=0) == 0)[0].squeeze()
                self.k_constant = const_idx.size
                if self.k_constant > 1:
                    raise ValueError("More than one constant detected.")
                else:
                    self.const_idx = const_idx
            except:  # should be an index error but who knows, means no const
                self.const_idx = None
                self.k_constant = 0

    @classmethod
    def _drop_nans(cls, x, nan_mask):
        return x[nan_mask]

    @classmethod
    def _drop_nans_2d(cls, x, nan_mask):
        return x[nan_mask][:, nan_mask]

    @classmethod
    def handle_missing(cls, endog, exog, missing, **kwargs):
        """
        This returns a dictionary with keys endog, exog and the keys of
        kwargs. It preserves Nones.
        """
        none_array_names = []

        if exog is not None:
            combined = (endog, exog)
            combined_names = ['endog', 'exog']
        else:
            combined = (endog,)
            combined_names = ['endog']
            none_array_names += ['exog']

        # deal with other arrays
        combined_2d = ()
        combined_2d_names = []
        if len(kwargs):
            for key, value_array in iteritems(kwargs):
                if value_array is None or value_array.ndim == 0:
                    none_array_names += [key]
                    continue
                # grab 1d arrays
                if value_array.ndim == 1:
                    combined += (value_array,)
                    combined_names += [key]
                elif value_array.squeeze().ndim == 1:
                    combined += (value_array,)
                    combined_names += [key]

                # grab 2d arrays that are _assumed_ to be symmetric
                elif value_array.ndim == 2:
                    combined_2d += (value_array,)
                    combined_2d_names += [key]
                else:
                    raise ValueError("Arrays with more than 2 dimensions "
                                     "aren't yet handled")

        nan_mask = _nan_rows(*combined)
        if combined_2d:
            nan_mask = _nan_rows(*(nan_mask[:, None],) + combined_2d)

        if not np.any(nan_mask):  # no missing don't do anything
            combined = dict(zip(combined_names, combined))
            if combined_2d:
                combined.update(dict(zip(combined_2d_names, combined_2d)))
            if none_array_names:
                combined.update(dict(zip(none_array_names,
                                         [None] * len(none_array_names))))

            return combined, []

        elif missing == 'raise':
            raise MissingDataError("NaNs were encountered in the data")

        elif missing == 'drop':
            nan_mask = ~nan_mask
            drop_nans = lambda x: cls._drop_nans(x, nan_mask)
            drop_nans_2d = lambda x: cls._drop_nans_2d(x, nan_mask)
            combined = dict(zip(combined_names, lmap(drop_nans, combined)))
            if combined_2d:
                combined.update(dict(zip(combined_2d_names,
                                         lmap(drop_nans_2d, combined_2d))))
            if none_array_names:
                combined.update(dict(zip(none_array_names,
                                         [None] * len(none_array_names))))
            return combined, np.where(~nan_mask)[0].tolist()
        else:
            raise ValueError("missing option %s not understood" % missing)

    def _convert_endog_exog(self, endog, exog):

        # for consistent outputs if endog is (n,1)
        yarr = self._get_yarr(endog)
        xarr = None
        if exog is not None:
            xarr = self._get_xarr(exog)
            if xarr.ndim == 1:
                xarr = xarr[:, None]
            if xarr.ndim != 2:
                raise ValueError("exog is not 1d or 2d")

        return yarr, xarr

    @cache_writable()
    def ynames(self):
        endog = self.orig_endog
        ynames = self._get_names(endog)
        if not ynames:
            ynames = _make_endog_names(self.endog)

        if len(ynames) == 1:
            return ynames[0]
        else:
            return list(ynames)

    @cache_writable()
    def xnames(self):
        exog = self.orig_exog
        if exog is not None:
            xnames = self._get_names(exog)
            if not xnames:
                xnames = _make_exog_names(self.exog)
            return list(xnames)
        return None

    @cache_readonly
    def row_labels(self):
        exog = self.orig_exog
        if exog is not None:
            row_labels = self._get_row_labels(exog)
        else:
            endog = self.orig_endog
            row_labels = self._get_row_labels(endog)
        return row_labels

    def _get_row_labels(self, arr):
        return None

    def _get_names(self, arr):
        if isinstance(arr, DataFrame):
            return list(arr.columns)
        elif isinstance(arr, Series):
            if arr.name:
                return [arr.name]
            else:
                return
        else:
            try:
                return arr.dtype.names
            except AttributeError:
                pass

        return None

    def _get_yarr(self, endog):
        if data_util._is_structured_ndarray(endog):
            endog = data_util.struct_to_ndarray(endog)
        endog = np.asarray(endog)
        if len(endog) == 1:  # never squeeze to a scalar
            if endog.ndim == 1:
                return endog
            elif endog.ndim > 1:
                return np.asarray([endog.squeeze()])

        return endog.squeeze()

    def _get_xarr(self, exog):
        if data_util._is_structured_ndarray(exog):
            exog = data_util.struct_to_ndarray(exog)
        return np.asarray(exog)

    def _check_integrity(self):
        if self.exog is not None:
            if len(self.exog) != len(self.endog):
                raise ValueError("endog and exog matrices are different sizes")

    def wrap_output(self, obj, how='columns'):
        if how == 'columns':
            return self.attach_columns(obj)
        elif how == 'rows':
            return self.attach_rows(obj)
        elif how == 'cov':
            return self.attach_cov(obj)
        elif how == 'dates':
            return self.attach_dates(obj)
        elif how == 'columns_eq':
            return self.attach_columns_eq(obj)
        elif how == 'cov_eq':
            return self.attach_cov_eq(obj)
        else:
            return obj

    def attach_columns(self, result):
        return result

    def attach_columns_eq(self, result):
        return result

    def attach_cov(self, result):
        return result

    def attach_cov_eq(self, result):
        return result

    def attach_rows(self, result):
        return result

    def attach_dates(self, result):
        return result

class PatsyData(ModelData):
    def _get_names(self, arr):
        return arr.design_info.column_names

class PandasData(ModelData):
    """
    Data handling class which knows how to reattach pandas metadata to model
    results
    """
    @classmethod
    def _drop_nans(cls, x, nan_mask):
        if hasattr(x, 'ix'):
            return x.ix[nan_mask]
        else:  # extra arguments could be plain ndarrays
            return super(PandasData, cls)._drop_nans(x, nan_mask)

    @classmethod
    def _drop_nans_2d(cls, x, nan_mask):
        if hasattr(x, 'ix'):
            return x.ix[nan_mask].ix[:, nan_mask]
        else:  # extra arguments could be plain ndarrays
            return super(PandasData, cls)._drop_nans_2d(x, nan_mask)

    def _check_integrity(self):
        endog, exog = self.orig_endog, self.orig_exog
        # exog can be None and we could be upcasting one or the other
        if (exog is not None and
                (hasattr(endog, 'index') and hasattr(exog, 'index')) and
                not self.orig_endog.index.equals(self.orig_exog.index)):
            raise ValueError("The indices for endog and exog are not aligned")
        super(PandasData, self)._check_integrity()

    def _get_row_labels(self, arr):
        try:
            return arr.index
        except AttributeError:
            # if we've gotten here it's because endog is pandas and
            # exog is not, so just return the row labels from endog
            return self.orig_endog.index

    def attach_columns(self, result):
        # this can either be a 1d array or a scalar
        # don't squeeze because it might be a 2d row array
        # if it needs a squeeze, the bug is elsewhere
        if result.ndim <= 1:
            return Series(result, index=self.xnames)
        else:  # for e.g., confidence intervals
            return DataFrame(result, index=self.xnames)

    def attach_columns_eq(self, result):
        return DataFrame(result, index=self.xnames, columns=self.ynames)

    def attach_cov(self, result):
        return DataFrame(result, index=self.xnames, columns=self.xnames)

    def attach_cov_eq(self, result):
        return DataFrame(result, index=self.ynames, columns=self.ynames)

    def attach_rows(self, result):
        # assumes if len(row_labels) > len(result) it's bc it was truncated
        # at the front, for AR lags, for example
        if result.squeeze().ndim == 1:
            return Series(result, index=self.row_labels[-len(result):])
        else:  # this is for VAR results, may not be general enough
            return DataFrame(result, index=self.row_labels[-len(result):],
                             columns=self.ynames)

    def attach_dates(self, result):
        return TimeSeries(result, index=self.predict_dates)

def _make_endog_names(endog):
    if endog.ndim == 1 or endog.shape[1] == 1:
        ynames = ['y']
    else:  # for VAR
        ynames = ['y%d' % (i+1) for i in range(endog.shape[1])]

    return ynames

def _make_exog_names(exog):
    exog_var = exog.var(0)
    if (exog_var == 0).any():
        # assumes one constant in first or last position
        # avoid exception if more than one constant
        const_idx = exog_var.argmin()
        exog_names = ['x%d' % i for i in range(1, exog.shape[1])]
        exog_names.insert(const_idx, 'const')
    else:
        exog_names = ['x%d' % i for i in range(1, exog.shape[1]+1)]

    return exog_names

def handle_missing(endog, exog=None, missing='none', **kwargs):
    klass = handle_data_class_factory(endog, exog)
    if missing == 'none':
        ret_dict = dict(endog=endog, exog=exog)
        ret_dict.update(kwargs)
        return ret_dict, None
    return klass.handle_missing(endog, exog, missing=missing, **kwargs)

def handle_data_class_factory(endog, exog):
    """
    Given inputs
    """
    if data_util._is_using_ndarray_type(endog, exog):
        klass = ModelData
    elif data_util._is_using_pandas(endog, exog):
        klass = PandasData
    elif data_util._is_using_patsy(endog, exog):
        klass = PatsyData
    # keep this check last
    elif data_util._is_using_ndarray(endog, exog):
        klass = ModelData
    else:
        raise ValueError('unrecognized data structures: %s / %s' %
                         (type(endog), type(exog)))
    return klass

def handle_data(endog, exog, missing='none', hasconst=None, **kwargs):
    # deal with lists and tuples up-front
    if isinstance(endog, (list, tuple)):
        endog = np.asarray(endog)
    if isinstance(exog, (list, tuple)):
        exog = np.asarray(exog)

    klass = handle_data_class_factory(endog, exog)
    return klass(endog, exog=exog, missing=missing, hasconst=hasconst,
                 **kwargs)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# This file is part of the pyMor project (http://www.pymor.org).
# Copyright Holders: Felix Albrecht, Rene Milk, Stephan Rave
# License: BSD 2-Clause License (http://opensource.org/licenses/BSD-2-Clause)

from __future__ import absolute_import, division, print_function

from numbers import Number

import numpy as np
from scipy.sparse import issparse

from pymor.core.interfaces import BasicInterface, abstractmethod, abstractproperty, abstractclassmethod
from pymor.core.exceptions import CommunicationError

class Communicable(BasicInterface):

    @abstractmethod
    def _data(self):
        pass

    _communication = 'enable'
    @property
    def communication(self):
        return self._communication

    @communication.setter
    def communication(self, v):
        assert v in set('raise', 'warn', 'enable')
        self._communication = v

    def enable_communication(self):
        self.communication = 'enable'

    def disable_communication(self):
        self.communication = 'raise'

    @property
    def data(self):
        '''Returns a `numpy.ndarray` containing the matrix.

        In case, the content of the `Matrix` cannot be modified via
        the array, the `WRITEABLE` flag has to be set to false in
        order to ensure, that copies are made.
        '''
        if self._communication == 'enable':
            return self._data()
        elif self._communication == 'warn':
            logger = getLogger('pymor.la.vectorarray')
            logger.warn('Communication required for {}'.format(self))
            return self._data()
        else:
            raise CommunicationError

class VectorArrayInterface(BasicInterface):
    '''Interface for vector arrays.

    A vector array should be thought of as a list of (possibly high-dimensional) vectors.
    While the vectors themselves will be inaccessible in general (e.g. because they are
    managed by external code on large systems), operations on the vectors like addition can
    be performed via the interface.

    It is moreover assumed that the count of vectors is small enough such that scalar data
    associated to each vector can be handled on the python side. I.e. methods like `l2_norm()`
    or `gramian()` will always return numpy arrays.

    An implementation of the interface via numpy arrays is given by `NumpyVectorArray`.
    In general, it is the implementors decision how memory is allocated internally (e.g.
    continuous block of memory vs. list of pointers to the individual vectors.) Thus no
    general assumptions can be made on the costs of operations like appending or removing
    vectors from the array. As a hint for 'continuous block of memory' implementations,
    `VectorArray` constructors should provide a `reserve` keyword argument which allows
    to specify to what sizes the array is assumed to grow.

    Most methods provide `ind` and/or `o_ind` arguments which are used to specify on which
    vectors the method is supposed to operate. If `ind` (`o_ind`) is `None` the whole array
    is selected. Otherwise, `ind` can be a single index in `range(len(self))`, a `list`
    of indices or a one-dimensional numpy array of indices. One index can be repeated
    in which case the corresponding vector is selected several times.
    '''

    @abstractclassmethod
    def empty(cls, dim, reserve=0):
        '''Create an empty VectorArray

        Parameters
        ----------
        dim
            The dimension of the array.
        reserve
            Hint for the backend to which length the array will grow.

        Returns
        -------
        An empty `VectorArray`.
        '''
        pass

    @abstractclassmethod
    def zeros(cls, dim, count=1):
        '''Create an VectorArray of null vectors

        Parameters
        ----------
        dim
            The dimension of the array.
        count
            The number of vectors.

        Returns
        -------
        An `VectorArray` containing `count` vectors of dimension `dim`
        whith each component zero.
        '''
        pass

    @abstractmethod
    def __len__(self):
        '''The number of vectors in the array.'''
        pass

    @abstractproperty
    def dim(self):
        '''The dimension of the vectors in the `VectorArray`.

        Each vector must have the same dimension.
        '''
        pass

    @abstractmethod
    def copy(self, ind=None):
        '''Returns a copy of a subarray.

        Parameters
        ----------
        ind
            Indices of the vectors that are to be copied (see class documentation).

        Returns
        -------
        A copy of the `VectorArray`.
        '''
        pass

    @abstractmethod
    def append(self, other, o_ind=None, remove_from_other=False):
        '''Append vectors to the array.

        Parameters
        ----------
        other
            A `VectorArray` containing the vectors to be appended.
        o_ind
            Indices of the vectors that are to be appended (see class documentation).
        remove_from_other
            If `True`, the appended vectors are removed from `other`.
            For list-like implementations of `VectorArray` this can be
            used to prevent unnecessary copies of the involved vectors.
        '''
        pass

    @abstractmethod
    def remove(self, ind):
        '''Remove vectors from the array.

        Parameters
        ----------
        ind
            Indices of the vectors that are to be removed (see class documentation).
        '''
        pass

    @abstractmethod
    def replace(self, other, ind=None, o_ind=None, remove_from_other=False):
        '''Replace vectors of the array.

        Parameters
        ----------
        other
            A `VectorArray` containing the replacement vectors.
        ind
            Indices of the vectors that are to be replaced (see class documentation).
        o_ind
            Indices of the replacement vectors (see class documentation).
            `len(ind)` has to agree with `len(o_ind)`.
        remove_from_other
            If `True`, the new vectors are removed from `other?.
            For list-like implementations of `VectorArray` this can be
            used to prevent unnecessary copies of the involved vectors.
        '''
        pass

    @abstractmethod
    def almost_equal(self, other, ind=None, o_ind=None, rtol=None, atol=None):
        '''Check vectors for equality.

        Equality of two vectors should be defined as in `pymor.tools.float_cmp_all`.

        The dimensions of `self` and `other` have to agree. If the length
        of `self` (`ind`) resp. `other` (`o_ind`) is 1, the one specified
        vector is compared to all vectors of the other summand. Otherwise
        the length of ind and o_ind have to agree.

        Parameters
        ----------
        other
            A `VectorArray` containing the vectors to compare with.
        ind
            Indices of the vectors that are to be compared (see class documentation).
        o_ind
            Indices of the vectors in `other` that are to be compared (see class documentation).
        rtol
            See `pymor.tools.float_cmp_all`
        atol
            See `pymor.tools.float_cmp_all`

        Returns
        -------
        Numpy array of the truth values of the comparison.
        '''
        pass

    @abstractmethod
    def scal(self, alpha, ind=None):
        '''BLAS SCAL operation (in-place sclar multiplication).

        This method calculates ::

            self[ind] = alpha*self[ind]

        Parameters
        ----------
        alpha
            The scalar coefficient with which the vectors in `self` are multiplied
        ind
            Indices of the vectors of `self` that are to be scaled (see class documentation).
        '''
        pass

    @abstractmethod
    def axpy(self, alpha, x, ind=None, x_ind=None):
        '''BLAS AXPY operation.

        This method forms the sum ::

            self[ind] = alpha*x[o_ind] + self[ind]

        If the length of `x` (`o_ind`) is 1, the one specified vector is added to all
        vectors of `self` specified by `ind`. Otherwise, the lengths of `self` (`ind`) and
        `x` (`x_ind`) have to be equal. The dimensions of `self` and `x` have to agree.

        Parameters
        ----------
        alpha
            The scalar coefficient with which the vectors in `x` are multiplied
        x
            A `VectorArray` containing the x-summands.
        ind
            Indices of the vectors of `self` that are to be added (see class documentation).
        x_ind
            Indices of the vectors in `x` that are to be added (see class documentation).
        '''
        pass

    @abstractmethod
    def dot(self, other, pairwise, ind=None, o_ind=None):
        '''Returns the scalar products between `VectorArray` elements.

        Parameters
        ----------
        other
            A `VectorArray` containing the second factors.
        pairwise
            See return value documentation.
        ind
            Indices of the vectors whose scalar products are to be taken
            (see class documentation).
        o_ind
            Indices of the vectors in `other` whose scalar products are to be
            taken (see class documentation).

        Returns
        -------
        If pairwise is True, returns a numpy array `result` such
        that ::

            result[i] = ( self[ind][i], other[o_ind][i] ).

        If pairwise is False, returns a numpy array `result` such
        that ::

            result[i, j] = ( self[ind][i], other[o_ind][j] ).
        '''
        pass

    @abstractmethod
    def lincomb(self, coefficients, ind=None):
        '''Returns linear combinations of the vectors contained in the array.

        Parameters
        ----------
        coefficients
            A numpy array of dimension 1 or 2 containing the linear
            coeffcients. `coefficients.shape[-1]` has to agree with
            `len(self)`.
        ind
            Indices of the vectors which are linear combined (see class documentation).

        Returns
        -------
        A `VectorArray` `result` such that ::

            result[i] = ∑ self[j] * coefficients[i,j]

        in case `coefficients` is of dimension 2, otherwise
        `len(result) == 1` and

            result[1] = ∑ self[j] * coefficients[j].
        '''
        pass

    @abstractmethod
    def l1_norm(self, ind=None):
        '''The l1-norms of the vectors contained in the array.

        Parameters
        ----------
        ind
            Indices of the vectors whose norm is to be calculated (see class documentation).

        Returns
        -------
        A numpy array `result` such that `result[i]` contains the norm
        of `self[ind][i]`.
        '''
        pass

    @abstractmethod
    def l2_norm(self, ind=None):
        '''The l2-norms of the vectors contained in the array.

        Parameters
        ----------
        ind
            Indices of the vectors whose norm is to be calculated (see class documentation).

        Returns
        -------
        A numpy array `result` such that `result[i]` contains the norm
        of `self[ind][i]`.
        '''
        pass

    def sup_norm(self, ind=None):
        '''The l-infintiy--norms of the vectors contained in the array.

        Parameters
        ----------
        ind
            Indices of the vectors whose norm is to be calculated (see class documentation).

        Returns
        -------
        A numpy array `result` such that `result[i]` contains the norm
        of `self[ind][i]`.
        '''
        _, max_val = self.amax(ind)
        return max_val

    @abstractmethod
    def components(self, component_indices, ind=None):
        '''Extract components of the vectors contained in the array.

        Parameters
        ----------
        component_indices
            Indices of the vector components that are to be returned.
        ind
            Indices of the vectors whose components to be calculated (see class documentation).

        Returns
        -------
        A numpy array `result` such that `result[i, j]` is the `component_indices[j]`-th
        component of the `ind[i]`-th vector of the array.
        '''
        pass

    @abstractmethod
    def amax(self, ind=None):
        '''The maximum absolute value of the vectors contained in the array.

        Parameters
        ----------
        ind
            Indices of the vectors whose maximum absolute value is to be calculated
            (see class documentation).

        Returns
        -------
        max_ind
            Numpy array containing for each vector an index at which the maximum is
            attained.
        max_val
            Numpy array containing for each vector the maximum absolute value of its
            components.
        '''
        pass

    def gramian(self, ind=None):
        '''Shorthand for `dot(self, pairwise=False, ind=ind, o_ind=ind)`.'''
        return self.dot(self, pairwise=False, ind=ind, o_ind=ind)

    def __add__(self, other):
        result = self.copy()
        result.axpy(1, other)
        return result

    def __iadd__(self, other):
        self.axpy(1, other)
        return self

    __radd__ = __add__

    def __sub__(self, other):
        result = self.copy()
        result.axpy(-1, other)
        return result

    def __isub__(self, other):
        self.axpy(-1, other)
        return self

    def __mul__(self, other):
        result = self.copy()
        result.scal(other)
        return result

    def __imul__(self, other):
        self.scal(other)
        return self

    def __neg__(self):
        result = self.copy()
        result.scal(-1)
        return result

    def check_ind(self, ind):
        '''Check if `ind` is an admissable list of indices in the sense of the class documentation.'''
        return (ind is None or
                isinstance(ind, (Number, list)) or
                isinstance(ind, np.ndarray) and ind.ndim == 1)
<commit_msg>[la.interfaces] bugfix<commit_after># -*- coding: utf-8 -*-
# This file is part of the pyMor project (http://www.pymor.org).
# Copyright Holders: Felix Albrecht, Rene Milk, Stephan Rave
# License: BSD 2-Clause License (http://opensource.org/licenses/BSD-2-Clause)

from __future__ import absolute_import, division, print_function

from numbers import Number

import numpy as np
from scipy.sparse import issparse

from pymor.core.interfaces import BasicInterface, abstractmethod, abstractproperty, abstractclassmethod
from pymor.core.exceptions import CommunicationError

class Communicable(BasicInterface):

    @abstractmethod
    def _data(self):
        pass

    _communication = 'enable'
    @property
    def communication(self):
        return self._communication

    @communication.setter
    def communication(self, v):
        assert v in set('raise', 'warn', 'enable')
        self._communication = v

    def enable_communication(self):
        self.communication = 'enable'

    def disable_communication(self):
        self.communication = 'raise'

    @property
    def data(self):
        '''Returns a `numpy.ndarray` containing the matrix.

        In case, the content of the `Matrix` cannot be modified via
        the array, the `WRITEABLE` flag has to be set to false in
        order to ensure, that copies are made.
        '''
        if self._communication == 'enable':
            return self._data()
        elif self._communication == 'warn':
            logger = getLogger('pymor.la.vectorarray')
            logger.warn('Communication required for {}'.format(self))
            return self._data()
        else:
            raise CommunicationError

class VectorArrayInterface(BasicInterface):
    '''Interface for vector arrays.

    A vector array should be thought of as a list of (possibly high-dimensional) vectors.
    While the vectors themselves will be inaccessible in general (e.g. because they are
    managed by external code on large systems), operations on the vectors like addition can
    be performed via the interface.

    It is moreover assumed that the count of vectors is small enough such that scalar data
    associated to each vector can be handled on the python side. I.e. methods like `l2_norm()`
    or `gramian()` will always return numpy arrays.

    An implementation of the interface via numpy arrays is given by `NumpyVectorArray`.
    In general, it is the implementors decision how memory is allocated internally (e.g.
    continuous block of memory vs. list of pointers to the individual vectors.) Thus no
    general assumptions can be made on the costs of operations like appending or removing
    vectors from the array. As a hint for 'continuous block of memory' implementations,
    `VectorArray` constructors should provide a `reserve` keyword argument which allows
    to specify to what sizes the array is assumed to grow.

    Most methods provide `ind` and/or `o_ind` arguments which are used to specify on which
    vectors the method is supposed to operate. If `ind` (`o_ind`) is `None` the whole array
    is selected. Otherwise, `ind` can be a single index in `range(len(self))`, a `list`
    of indices or a one-dimensional numpy array of indices. One index can be repeated
    in which case the corresponding vector is selected several times.
    '''

    @abstractclassmethod
    def empty(cls, dim, reserve=0):
        '''Create an empty VectorArray

        Parameters
        ----------
        dim
            The dimension of the array.
        reserve
            Hint for the backend to which length the array will grow.

        Returns
        -------
        An empty `VectorArray`.
        '''
        pass

    @abstractclassmethod
    def zeros(cls, dim, count=1):
        '''Create an VectorArray of null vectors

        Parameters
        ----------
        dim
            The dimension of the array.
        count
            The number of vectors.

        Returns
        -------
        An `VectorArray` containing `count` vectors of dimension `dim`
        whith each component zero.
        '''
        pass

    @abstractmethod
    def __len__(self):
        '''The number of vectors in the array.'''
        pass

    @abstractproperty
    def dim(self):
        '''The dimension of the vectors in the `VectorArray`.

        Each vector must have the same dimension.
        '''
        pass

    @abstractmethod
    def copy(self, ind=None):
        '''Returns a copy of a subarray.

        Parameters
        ----------
        ind
            Indices of the vectors that are to be copied (see class documentation).

        Returns
        -------
        A copy of the `VectorArray`.
        '''
        pass

    @abstractmethod
    def append(self, other, o_ind=None, remove_from_other=False):
        '''Append vectors to the array.

        Parameters
        ----------
        other
            A `VectorArray` containing the vectors to be appended.
        o_ind
            Indices of the vectors that are to be appended (see class documentation).
        remove_from_other
            If `True`, the appended vectors are removed from `other`.
            For list-like implementations of `VectorArray` this can be
            used to prevent unnecessary copies of the involved vectors.
        '''
        pass

    @abstractmethod
    def remove(self, ind):
        '''Remove vectors from the array.

        Parameters
        ----------
        ind
            Indices of the vectors that are to be removed (see class documentation).
        '''
        pass

    @abstractmethod
    def replace(self, other, ind=None, o_ind=None, remove_from_other=False):
        '''Replace vectors of the array.

        Parameters
        ----------
        other
            A `VectorArray` containing the replacement vectors.
        ind
            Indices of the vectors that are to be replaced (see class documentation).
        o_ind
            Indices of the replacement vectors (see class documentation).
            `len(ind)` has to agree with `len(o_ind)`.
        remove_from_other
            If `True`, the new vectors are removed from `other?.
            For list-like implementations of `VectorArray` this can be
            used to prevent unnecessary copies of the involved vectors.
        '''
        pass

    @abstractmethod
    def almost_equal(self, other, ind=None, o_ind=None, rtol=None, atol=None):
        '''Check vectors for equality.

        Equality of two vectors should be defined as in `pymor.tools.float_cmp_all`.

        The dimensions of `self` and `other` have to agree. If the length
        of `self` (`ind`) resp. `other` (`o_ind`) is 1, the one specified
        vector is compared to all vectors of the other summand. Otherwise
        the length of ind and o_ind have to agree.

        Parameters
        ----------
        other
            A `VectorArray` containing the vectors to compare with.
        ind
            Indices of the vectors that are to be compared (see class documentation).
        o_ind
            Indices of the vectors in `other` that are to be compared (see class documentation).
        rtol
            See `pymor.tools.float_cmp_all`
        atol
            See `pymor.tools.float_cmp_all`

        Returns
        -------
        Numpy array of the truth values of the comparison.
        '''
        pass

    @abstractmethod
    def scal(self, alpha, ind=None):
        '''BLAS SCAL operation (in-place sclar multiplication).

        This method calculates ::

            self[ind] = alpha*self[ind]

        Parameters
        ----------
        alpha
            The scalar coefficient with which the vectors in `self` are multiplied
        ind
            Indices of the vectors of `self` that are to be scaled (see class documentation).
        '''
        pass

    @abstractmethod
    def axpy(self, alpha, x, ind=None, x_ind=None):
        '''BLAS AXPY operation.

        This method forms the sum ::

            self[ind] = alpha*x[o_ind] + self[ind]

        If the length of `x` (`o_ind`) is 1, the one specified vector is added to all
        vectors of `self` specified by `ind`. Otherwise, the lengths of `self` (`ind`) and
        `x` (`x_ind`) have to be equal. The dimensions of `self` and `x` have to agree.

        Parameters
        ----------
        alpha
            The scalar coefficient with which the vectors in `x` are multiplied
        x
            A `VectorArray` containing the x-summands.
        ind
            Indices of the vectors of `self` that are to be added (see class documentation).
        x_ind
            Indices of the vectors in `x` that are to be added (see class documentation).
        '''
        pass

    @abstractmethod
    def dot(self, other, pairwise, ind=None, o_ind=None):
        '''Returns the scalar products between `VectorArray` elements.

        Parameters
        ----------
        other
            A `VectorArray` containing the second factors.
        pairwise
            See return value documentation.
        ind
            Indices of the vectors whose scalar products are to be taken
            (see class documentation).
        o_ind
            Indices of the vectors in `other` whose scalar products are to be
            taken (see class documentation).

        Returns
        -------
        If pairwise is True, returns a numpy array `result` such
        that ::

            result[i] = ( self[ind][i], other[o_ind][i] ).

        If pairwise is False, returns a numpy array `result` such
        that ::

            result[i, j] = ( self[ind][i], other[o_ind][j] ).
        '''
        pass

    @abstractmethod
    def lincomb(self, coefficients, ind=None):
        '''Returns linear combinations of the vectors contained in the array.

        Parameters
        ----------
        coefficients
            A numpy array of dimension 1 or 2 containing the linear
            coeffcients. `coefficients.shape[-1]` has to agree with
            `len(self)`.
        ind
            Indices of the vectors which are linear combined (see class documentation).

        Returns
        -------
        A `VectorArray` `result` such that ::

            result[i] = ∑ self[j] * coefficients[i,j]

        in case `coefficients` is of dimension 2, otherwise
        `len(result) == 1` and

            result[1] = ∑ self[j] * coefficients[j].
        '''
        pass

    @abstractmethod
    def l1_norm(self, ind=None):
        '''The l1-norms of the vectors contained in the array.

        Parameters
        ----------
        ind
            Indices of the vectors whose norm is to be calculated (see class documentation).

        Returns
        -------
        A numpy array `result` such that `result[i]` contains the norm
        of `self[ind][i]`.
        '''
        pass

    @abstractmethod
    def l2_norm(self, ind=None):
        '''The l2-norms of the vectors contained in the array.

        Parameters
        ----------
        ind
            Indices of the vectors whose norm is to be calculated (see class documentation).

        Returns
        -------
        A numpy array `result` such that `result[i]` contains the norm
        of `self[ind][i]`.
        '''
        pass

    def sup_norm(self, ind=None):
        '''The l-infintiy--norms of the vectors contained in the array.

        Parameters
        ----------
        ind
            Indices of the vectors whose norm is to be calculated (see class documentation).

        Returns
        -------
        A numpy array `result` such that `result[i]` contains the norm
        of `self[ind][i]`.
        '''
        _, max_val = self.amax(ind)
        return max_val

    @abstractmethod
    def components(self, component_indices, ind=None):
        '''Extract components of the vectors contained in the array.

        Parameters
        ----------
        component_indices
            Indices of the vector components that are to be returned.
        ind
            Indices of the vectors whose components to be calculated (see class documentation).

        Returns
        -------
        A numpy array `result` such that `result[i, j]` is the `component_indices[j]`-th
        component of the `ind[i]`-th vector of the array.
        '''
        pass

    @abstractmethod
    def amax(self, ind=None):
        '''The maximum absolute value of the vectors contained in the array.

        Parameters
        ----------
        ind
            Indices of the vectors whose maximum absolute value is to be calculated
            (see class documentation).

        Returns
        -------
        max_ind
            Numpy array containing for each vector an index at which the maximum is
            attained.
        max_val
            Numpy array containing for each vector the maximum absolute value of its
            components.
        '''
        pass

    def gramian(self, ind=None):
        '''Shorthand for `dot(self, pairwise=False, ind=ind, o_ind=ind)`.'''
        return self.dot(self, pairwise=False, ind=ind, o_ind=ind)

    def __add__(self, other):
        if isinstance(other, Number):
            assert other == 0
            return self.copy()

        result = self.copy()
        result.axpy(1, other)
        return result

    def __iadd__(self, other):
        self.axpy(1, other)
        return self

    __radd__ = __add__

    def __sub__(self, other):
        result = self.copy()
        result.axpy(-1, other)
        return result

    def __isub__(self, other):
        self.axpy(-1, other)
        return self

    def __mul__(self, other):
        result = self.copy()
        result.scal(other)
        return result

    def __imul__(self, other):
        self.scal(other)
        return self

    def __neg__(self):
        result = self.copy()
        result.scal(-1)
        return result

    def check_ind(self, ind):
        '''Check if `ind` is an admissable list of indices in the sense of the class documentation.'''
        return (ind is None or
                isinstance(ind, (Number, list)) or
                isinstance(ind, np.ndarray) and ind.ndim == 1)
<|endoftext|>


<commit_before>from __future__ import unicode_literals

"""
SQLite natively supports only the types TEXT, INTEGER, REAL, BLOB and NULL.
And RQLite always answers 'bytes' values.

Converters transforms RQLite answers to Python native types.
Adapters transforms Python native types to RQLite-aware values.
"""

import numbers
import sqlite3

def _decoder(conv_func):
    """ The Python sqlite3 interface returns always byte strings.
        This function converts the received value to a regular string before
        passing it to the receiver function.
    """
    return lambda s: conv_func(s.decode('utf-8'))

adapters = {
    float: lambda x: x,
    int: lambda x: x,
    bool: lambda x: int(x),
}
converters = {
    'TEXT': str,
    'VARCHAR': lambda x: x.encode('utf-8'),
    'INTEGER': int,
    'BOOL': bool,
    'FLOAT': float,
    'REAL': float,
    'NULL': lambda x: None,
    'BLOB': lambda x: x,
    '': lambda x: x.encode('utf-8'),
}

# Non-native converters will be decoded from base64 before fed into converter
_native_converters = ('BOOL', 'FLOAT', 'INTEGER', 'REAL', 'NUMBER', 'TEXT', 'VARCHAR', 'NULL', '')

def register_converter(type_string, value_string):
    raise NotImplementedError()

def register_adapter(type_, function):
    raise NotImplementedError()

def _convert_to_python(type_, value):
    type_upper = type_.upper()
    if type_upper in converters:
        if type_upper not in _native_converters:
            value = value.decode('base64')
        return converters[type_upper](value)
    return value

def _adapt_from_python(value):
    if not isinstance(value, basestring):
        try:
            adapted = adapters[type(value)](value)
        except KeyError:
            # No adapter registered. Let the object adapt itself via PEP-246.
            # It has been rejected by the BDFL, but is still implemented
            # on stdlib sqlite3 module even on Python 3 !!
            if hasattr(value, '__adapt__'):
                adapted = value.__adapt__(sqlite3.PrepareProtocol)
            elif hasattr(value, '__conform__'):
                adapted = value.__conform__(sqlite3.PrepareProtocol)
            else:
                raise
    else:
        adapted = value

    # The adapter could had returned a string
    if isinstance(adapted, basestring):
        adapted = _escape_string(adapted)
    else:
        adapted = str(adapted)

    return adapted

def _escape_string(value):
    return (type(value)(b"'%s'")) % (value.replace(b"'", b"''"))
<commit_msg>Split type names os ( and ' ', as stock does<commit_after>from __future__ import unicode_literals

"""
SQLite natively supports only the types TEXT, INTEGER, REAL, BLOB and NULL.
And RQLite always answers 'bytes' values.

Converters transforms RQLite answers to Python native types.
Adapters transforms Python native types to RQLite-aware values.
"""

import numbers
import sqlite3

def _decoder(conv_func):
    """ The Python sqlite3 interface returns always byte strings.
        This function converts the received value to a regular string before
        passing it to the receiver function.
    """
    return lambda s: conv_func(s.decode('utf-8'))

adapters = {
    float: lambda x: x,
    int: lambda x: x,
    bool: lambda x: int(x),
}
converters = {
    'TEXT': str,
    'VARCHAR': lambda x: x.encode('utf-8'),
    'INTEGER': int,
    'BOOL': bool,
    'FLOAT': float,
    'REAL': float,
    'NULL': lambda x: None,
    'BLOB': lambda x: x,
    '': lambda x: x.encode('utf-8'),
}

# Non-native converters will be decoded from base64 before fed into converter
_native_converters = ('BOOL', 'FLOAT', 'INTEGER', 'REAL', 'NUMBER', 'TEXT', 'VARCHAR', 'NULL', '')

def register_converter(type_string, value_string):
    raise NotImplementedError()

def register_adapter(type_, function):
    raise NotImplementedError()

def _convert_to_python(type_, value):
    ## From: https://github.com/python/cpython/blob/c72b6008e0578e334f962ee298279a23ba298856/Modules/_sqlite/cursor.c#L167
    # /* Converter names are split at '(' and blanks.
    #  * This allows 'INTEGER NOT NULL' to be treated as 'INTEGER' and
    #  * 'NUMBER(10)' to be treated as 'NUMBER', for example.
    #  * In other words, it will work as people expect it to work.*/
    type_upper = type_.upper().partition('(')[0].partition(' ')[0]
    if type_upper in converters:
        if type_upper not in _native_converters:
            value = value.decode('base64')
        return converters[type_upper](value)
    return value

def _adapt_from_python(value):
    if not isinstance(value, basestring):
        try:
            adapted = adapters[type(value)](value)
        except KeyError:
            # No adapter registered. Let the object adapt itself via PEP-246.
            # It has been rejected by the BDFL, but is still implemented
            # on stdlib sqlite3 module even on Python 3 !!
            if hasattr(value, '__adapt__'):
                adapted = value.__adapt__(sqlite3.PrepareProtocol)
            elif hasattr(value, '__conform__'):
                adapted = value.__conform__(sqlite3.PrepareProtocol)
            else:
                raise
    else:
        adapted = value

    # The adapter could had returned a string
    if isinstance(adapted, basestring):
        adapted = _escape_string(adapted)
    else:
        adapted = str(adapted)

    return adapted

def _escape_string(value):
    return (type(value)(b"'%s'")) % (value.replace(b"'", b"''"))
<|endoftext|>