<commit_before>"""
Copyright (c) 2015 SONATA-NFV
ALL RIGHTS RESERVED.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Neither the name of the SONATA-NFV [, ANY ADDITIONAL AFFILIATION]
nor the names of its contributors may be used to endorse or promote
products derived from this software without specific prior written
permission.

This work has been performed in the framework of the SONATA project,
funded by the European Commission under Grant number 671517 through
the Horizon 2020 and 5G-PPP programmes. The authors would like to
acknowledge the contributions of their colleagues of the SONATA
partner consortium (www.sonata-nfv.eu).
"""

import time
import requests
import json
import os
import unittest
from emuvim.test.base import SimpleTestTopology
from emuvim.api.sonata import SonataDummyGatekeeperEndpoint
from emuvim.api.sonata.dummygatekeeper import initialize_GK, parse_interface
import mininet.clean
from ipaddress import ip_network

PACKAGE_PATH = "misc/sonata-demo-service.son"

class testSonataDummyGatekeeper(SimpleTestTopology):

#    @unittest.skip("disabled")
    def test_GK_Api_start_service(self):
        # create network
        self.createNet(nswitches=0, ndatacenter=2, nhosts=2, ndockers=0, enable_learning=True)
        # setup links
        self.net.addLink(self.dc[0], self.h[0])
        self.net.addLink(self.dc[0], self.dc[1])
        self.net.addLink(self.h[1], self.dc[1])
        # connect dummy GK to data centers
        sdkg1 = SonataDummyGatekeeperEndpoint("0.0.0.0", 5000)
        sdkg1.connectDatacenter(self.dc[0])
        sdkg1.connectDatacenter(self.dc[1])
        # run the dummy gatekeeper (in another thread, don't block)
        sdkg1.start()
        # start Mininet network
        self.startNet()
        time.sleep(1)

        print "starting tests"
        # board package
        files = {"package": open(PACKAGE_PATH, "rb")}
        r = requests.post("http://127.0.0.1:5000/packages", files=files)
        self.assertEqual(r.status_code, 201)
        self.assertTrue(json.loads(r.text).get("service_uuid") is not None)

        # instantiate service
        self.service_uuid = json.loads(r.text).get("service_uuid")
        r2 = requests.post("http://127.0.0.1:5000/instantiations", data=json.dumps({"service_uuid": self.service_uuid}))
        self.assertEqual(r2.status_code, 201)

        # give the emulator some time to instantiate everything
        time.sleep(2)

        # check get request APIs
        r3 = requests.get("http://127.0.0.1:5000/packages")
        self.assertEqual(len(json.loads(r3.text).get("service_uuid_list")), 1)
        r4 = requests.get("http://127.0.0.1:5000/instantiations")
        self.assertEqual(len(json.loads(r4.text).get("service_instantiations_list")), 1)

        # check number of running nodes
        self.assertTrue(len(self.getContainernetContainers()) == 3)
        self.assertTrue(len(self.net.hosts) == 5)
        self.assertTrue(len(self.net.switches) == 2)
        # check compute list result
        self.assertEqual(len(self.dc[0].listCompute()), 2)
        # check connectivity by using ping
        ELAN_list=[]

        # check E-Line connection, by checking the IP addresses
        for link in self.net.deployed_elines:
            vnf_src, intf_src, vnf_sap_docker_name = parse_interface(link['connection_points_reference'][0])
            print vnf_src, intf_src
            src = self.net.getNodeByName(vnf_src)
            if not src:
                continue
            network_list = src.getNetworkStatus()
            src_ip = [intf['ip'] for intf in network_list if intf['intf_name'] == intf_src][0]
            src_mask = [intf['netmask'] for intf in network_list if intf['intf_name'] == intf_src][0]

            vnf_dst,  intf_dst, vnf_sap_docker_name = parse_interface(link['connection_points_reference'][1])
            dst = self.net.getNodeByName(vnf_dst)
            if not dst:
                continue
            network_list = dst.getNetworkStatus()
            dst_ip = [intf['ip'] for intf in network_list if intf['intf_name'] == intf_dst][0]
            dst_mask = [intf['netmask'] for intf in network_list if intf['intf_name'] == intf_dst][0]

            print "src = {0}:{1} ip={2}/{3} ".format(vnf_src, intf_src, src_ip, src_mask)
            print "dst = {0}:{1} ip={2}/{3} ".format(vnf_dst, intf_dst, dst_ip, dst_mask)

            # check if the E-Line IP's are in the same subnet
            ret = ip_network(u'{0}/{1}'.format(src_ip, src_mask), strict=False)\
                .compare_networks(ip_network(u'{0}/{1}'.format(dst_ip, dst_mask),strict=False))
            self.assertTrue(ret == 0)

        for vnf in self.dc[0].listCompute():
            # check E LAN connection
            network_list = vnf.getNetworkStatus()
            mgmt_ip = [intf['ip'] for intf in network_list if intf['intf_name'] == 'mgmt']
            self.assertTrue(len(mgmt_ip) > 0)
            ip_address = mgmt_ip[0]
            ELAN_list.append(ip_address)
            print ip_address

        # check ELAN connection by ping over the mgmt network (needs to be configured as ELAN in the test service)
        for vnf in self.dc[0].listCompute():
            network_list = vnf.getNetworkStatus()
            mgmt_ip = [intf['ip'] for intf in network_list if intf['intf_name'] == 'mgmt']
            self.assertTrue(len(mgmt_ip) > 0)
            ip_address = mgmt_ip[0]
            print ELAN_list
            print ip_address
            test_ip_list = list(ELAN_list)
            test_ip_list.remove(ip_address)
            for ip in test_ip_list:
                p = self.net.ping([vnf],manualdestip=ip)
                print p
                self.assertTrue(p <= 0.0)

        # stop Mininet network
        self.stopNet()
        initialize_GK()

    #@unittest.skip("disabled")
    def test_GK_Api_stop_service(self):
        # create network
        self.createNet(ndatacenter=2, nhosts=2)
        # setup links
        self.net.addLink(self.dc[0], self.h[0])
        self.net.addLink(self.dc[0], self.dc[1])
        self.net.addLink(self.h[1], self.dc[1])
        # connect dummy GK to data centers
        sdkg1 = SonataDummyGatekeeperEndpoint("0.0.0.0", 5000)
        sdkg1.connectDatacenter(self.dc[0])
        sdkg1.connectDatacenter(self.dc[1])
        # run the dummy gatekeeper (in another thread, don't block)
        sdkg1.start()
        # start Mininet network
        self.startNet()
        time.sleep(1)

        print "starting tests"
        # board package
        files = {"package": open(PACKAGE_PATH, "rb")}
        r = requests.post("http://127.0.0.1:5000/packages", files=files)
        self.assertEqual(r.status_code, 201)
        self.assertTrue(json.loads(r.text).get("service_uuid") is not None)

        # instantiate service
        self.service_uuid = json.loads(r.text).get("service_uuid")
        r2 = requests.post("http://127.0.0.1:5000/instantiations", data=json.dumps({"service_uuid": self.service_uuid}))
        self.assertEqual(r2.status_code, 201)

        # give the emulator some time to instantiate everything
        time.sleep(2)

        # check get request APIs
        r3 = requests.get("http://127.0.0.1:5000/packages")
        self.assertEqual(len(json.loads(r3.text).get("service_uuid_list")), 1)
        r4 = requests.get("http://127.0.0.1:5000/instantiations")
        self.assertEqual(len(json.loads(r4.text).get("service_instantiations_list")), 1)

        # check number of running nodes
        self.assertTrue(len(self.getContainernetContainers()) == 3)
        self.assertTrue(len(self.net.hosts) == 5)
        self.assertTrue(len(self.net.switches) == 2)
        # check compute list result
        self.assertEqual(len(self.dc[0].listCompute()), 2)

        # stop the service
        service_instance_uuid = json.loads(r2.text).get("service_instance_uuid")
        self.assertTrue(service_instance_uuid is not None)
        requests.delete("http://127.0.0.1:5000/instantiations", data=json.dumps({"service_uuid": self.service_uuid, "service_instance_uuid":service_instance_uuid}))

        r5 = requests.get("http://127.0.0.1:5000/instantiations")
        self.assertTrue(len(json.loads(r5.text).get("service_instantiations_list")), 0)     # note that there was 1 instance before

        # stop Mininet network
        self.stopNet()
        initialize_GK()

    #@unittest.skip("disabled")
    def test_GK_stress_service(self):
        # create network
        self.createNet(ndatacenter=2, nhosts=2)
        # connect dummy GK to data centers
        sdkg1 = SonataDummyGatekeeperEndpoint("0.0.0.0", 5000)
        sdkg1.connectDatacenter(self.dc[0])
        sdkg1.connectDatacenter(self.dc[1])
        # run the dummy gatekeeper (in another thread, don't block)
        sdkg1.start()
        # start Mininet network
        self.startNet()
        time.sleep(1)

        print "starting tests"
        # board package
        files = {"package": open("misc/sonata-stress-service.son", "rb")}
        r = requests.post("http://127.0.0.1:5000/packages", files=files)
        self.assertEqual(r.status_code, 201)
        self.assertTrue(json.loads(r.text).get("service_uuid") is not None)

        # instantiate service
        self.service_uuid = json.loads(r.text).get("service_uuid")
        r2 = requests.post("http://127.0.0.1:5000/instantiations", data=json.dumps({"service_uuid": self.service_uuid}))
        self.assertEqual(r2.status_code, 201)

        # give the emulator some time to instantiate everything
        time.sleep(2)

        # check get request APIs
        r3 = requests.get("http://127.0.0.1:5000/packages")
        self.assertEqual(len(json.loads(r3.text).get("service_uuid_list")), 1)
        r4 = requests.get("http://127.0.0.1:5000/instantiations")
        self.assertEqual(len(json.loads(r4.text).get("service_instantiations_list")), 1)

        # stop the service
        service_instance_uuid = json.loads(r2.text).get("service_instance_uuid")
        self.assertTrue(service_instance_uuid is not None)
        requests.delete("http://127.0.0.1:5000/instantiations", data=json.dumps({"service_uuid": self.service_uuid, "service_instance_uuid":service_instance_uuid}))

        r5 = requests.get("http://127.0.0.1:5000/instantiations")
        self.assertTrue(len(json.loads(r5.text).get("service_instantiations_list")), 0)     # note that there was 1 instance before

        # stop Mininet network
        self.stopNet()
        initialize_GK()

<commit_msg>fix unittest<commit_after>"""
Copyright (c) 2015 SONATA-NFV
ALL RIGHTS RESERVED.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

Neither the name of the SONATA-NFV [, ANY ADDITIONAL AFFILIATION]
nor the names of its contributors may be used to endorse or promote
products derived from this software without specific prior written
permission.

This work has been performed in the framework of the SONATA project,
funded by the European Commission under Grant number 671517 through
the Horizon 2020 and 5G-PPP programmes. The authors would like to
acknowledge the contributions of their colleagues of the SONATA
partner consortium (www.sonata-nfv.eu).
"""

import time
import requests
import json
import os
import unittest
from emuvim.test.base import SimpleTestTopology
from emuvim.api.sonata import SonataDummyGatekeeperEndpoint
from emuvim.api.sonata.dummygatekeeper import initialize_GK, parse_interface
import mininet.clean
from ipaddress import ip_network

PACKAGE_PATH = "misc/sonata-demo-service.son"

class testSonataDummyGatekeeper(SimpleTestTopology):

#    @unittest.skip("disabled")
    def test_GK_Api_start_service(self):
        # create network
        self.createNet(nswitches=0, ndatacenter=2, nhosts=2, ndockers=0, enable_learning=True)
        # setup links
        self.net.addLink(self.dc[0], self.h[0])
        self.net.addLink(self.dc[0], self.dc[1])
        self.net.addLink(self.h[1], self.dc[1])
        # connect dummy GK to data centers
        sdkg1 = SonataDummyGatekeeperEndpoint("0.0.0.0", 5000)
        sdkg1.connectDatacenter(self.dc[0])
        sdkg1.connectDatacenter(self.dc[1])
        # run the dummy gatekeeper (in another thread, don't block)
        sdkg1.start()
        # start Mininet network
        self.startNet()
        time.sleep(1)

        print "starting tests"
        # board package
        files = {"package": open(PACKAGE_PATH, "rb")}
        r = requests.post("http://127.0.0.1:5000/packages", files=files)
        self.assertEqual(r.status_code, 201)
        self.assertTrue(json.loads(r.text).get("service_uuid") is not None)

        # instantiate service
        self.service_uuid = json.loads(r.text).get("service_uuid")
        r2 = requests.post("http://127.0.0.1:5000/instantiations", data=json.dumps({"service_uuid": self.service_uuid}))
        self.assertEqual(r2.status_code, 201)

        # give the emulator some time to instantiate everything
        time.sleep(2)

        # check get request APIs
        r3 = requests.get("http://127.0.0.1:5000/packages")
        self.assertEqual(len(json.loads(r3.text).get("service_uuid_list")), 1)
        r4 = requests.get("http://127.0.0.1:5000/instantiations")
        self.assertEqual(len(json.loads(r4.text).get("service_instantiations_list")), 1)

        # check number of running nodes
        self.assertTrue(len(self.getContainernetContainers()) == 3)
        self.assertTrue(len(self.net.hosts) == 5)
        self.assertTrue(len(self.net.switches) == 2)
        # check compute list result
        self.assertEqual(len(self.dc[0].listCompute()), 2)
        # check connectivity by using ping
        ELAN_list=[]

        # check E-Line connection, by checking the IP addresses
        for link in self.net.deployed_elines:
            vnf_src, intf_src, vnf_sap_docker_name = parse_interface(link['connection_points_reference'][0])
            print vnf_src, intf_src
            src = self.net.getNodeByName(vnf_src)
            if not src:
                continue
            network_list = src.getNetworkStatus()
            src_ip = [intf['ip'] for intf in network_list if intf['intf_name'] == intf_src][0]
            src_mask = [intf['netmask'] for intf in network_list if intf['intf_name'] == intf_src][0]

            vnf_dst,  intf_dst, vnf_sap_docker_name = parse_interface(link['connection_points_reference'][1])
            dst = self.net.getNodeByName(vnf_dst)
            if not dst:
                continue
            network_list = dst.getNetworkStatus()
            dst_ip = [intf['ip'] for intf in network_list if intf['intf_name'] == intf_dst][0]
            dst_mask = [intf['netmask'] for intf in network_list if intf['intf_name'] == intf_dst][0]

            print "src = {0}:{1} ip={2} ".format(vnf_src, intf_src, src_ip, src_mask)
            print "dst = {0}:{1} ip={2} ".format(vnf_dst, intf_dst, dst_ip, dst_mask)

            # check if the E-Line IP's are in the same subnet
            ret = ip_network(u'{0}'.format(src_ip, src_mask), strict=False)\
                .compare_networks(ip_network(u'{0}'.format(dst_ip, dst_mask),strict=False))
            self.assertTrue(ret == 0)

        for vnf in self.dc[0].listCompute():
            # check E LAN connection
            network_list = vnf.getNetworkStatus()
            mgmt_ip = [intf['ip'] for intf in network_list if intf['intf_name'] == 'mgmt']
            self.assertTrue(len(mgmt_ip) > 0)
            ip_address = mgmt_ip[0]
            ELAN_list.append(ip_address)
            print ip_address

        # check ELAN connection by ping over the mgmt network (needs to be configured as ELAN in the test service)
        for vnf in self.dc[0].listCompute():
            network_list = vnf.getNetworkStatus()
            mgmt_ip = [intf['ip'] for intf in network_list if intf['intf_name'] == 'mgmt']
            self.assertTrue(len(mgmt_ip) > 0)
            ip_address = mgmt_ip[0]
            print ELAN_list
            print ip_address
            test_ip_list = list(ELAN_list)
            test_ip_list.remove(ip_address)
            for ip in test_ip_list:
                p = self.net.ping([vnf],manualdestip=ip)
                print p
                self.assertTrue(p <= 0.0)

        # stop Mininet network
        self.stopNet()
        initialize_GK()

    #@unittest.skip("disabled")
    def test_GK_Api_stop_service(self):
        # create network
        self.createNet(ndatacenter=2, nhosts=2)
        # setup links
        self.net.addLink(self.dc[0], self.h[0])
        self.net.addLink(self.dc[0], self.dc[1])
        self.net.addLink(self.h[1], self.dc[1])
        # connect dummy GK to data centers
        sdkg1 = SonataDummyGatekeeperEndpoint("0.0.0.0", 5000)
        sdkg1.connectDatacenter(self.dc[0])
        sdkg1.connectDatacenter(self.dc[1])
        # run the dummy gatekeeper (in another thread, don't block)
        sdkg1.start()
        # start Mininet network
        self.startNet()
        time.sleep(1)

        print "starting tests"
        # board package
        files = {"package": open(PACKAGE_PATH, "rb")}
        r = requests.post("http://127.0.0.1:5000/packages", files=files)
        self.assertEqual(r.status_code, 201)
        self.assertTrue(json.loads(r.text).get("service_uuid") is not None)

        # instantiate service
        self.service_uuid = json.loads(r.text).get("service_uuid")
        r2 = requests.post("http://127.0.0.1:5000/instantiations", data=json.dumps({"service_uuid": self.service_uuid}))
        self.assertEqual(r2.status_code, 201)

        # give the emulator some time to instantiate everything
        time.sleep(2)

        # check get request APIs
        r3 = requests.get("http://127.0.0.1:5000/packages")
        self.assertEqual(len(json.loads(r3.text).get("service_uuid_list")), 1)
        r4 = requests.get("http://127.0.0.1:5000/instantiations")
        self.assertEqual(len(json.loads(r4.text).get("service_instantiations_list")), 1)

        # check number of running nodes
        self.assertTrue(len(self.getContainernetContainers()) == 3)
        self.assertTrue(len(self.net.hosts) == 5)
        self.assertTrue(len(self.net.switches) == 2)
        # check compute list result
        self.assertEqual(len(self.dc[0].listCompute()), 2)

        # stop the service
        service_instance_uuid = json.loads(r2.text).get("service_instance_uuid")
        self.assertTrue(service_instance_uuid is not None)
        requests.delete("http://127.0.0.1:5000/instantiations", data=json.dumps({"service_uuid": self.service_uuid, "service_instance_uuid":service_instance_uuid}))

        r5 = requests.get("http://127.0.0.1:5000/instantiations")
        self.assertTrue(len(json.loads(r5.text).get("service_instantiations_list")), 0)     # note that there was 1 instance before

        # stop Mininet network
        self.stopNet()
        initialize_GK()

    #@unittest.skip("disabled")
    def test_GK_stress_service(self):
        # create network
        self.createNet(ndatacenter=2, nhosts=2)
        # connect dummy GK to data centers
        sdkg1 = SonataDummyGatekeeperEndpoint("0.0.0.0", 5000)
        sdkg1.connectDatacenter(self.dc[0])
        sdkg1.connectDatacenter(self.dc[1])
        # run the dummy gatekeeper (in another thread, don't block)
        sdkg1.start()
        # start Mininet network
        self.startNet()
        time.sleep(1)

        print "starting tests"
        # board package
        files = {"package": open("misc/sonata-stress-service.son", "rb")}
        r = requests.post("http://127.0.0.1:5000/packages", files=files)
        self.assertEqual(r.status_code, 201)
        self.assertTrue(json.loads(r.text).get("service_uuid") is not None)

        # instantiate service
        self.service_uuid = json.loads(r.text).get("service_uuid")
        r2 = requests.post("http://127.0.0.1:5000/instantiations", data=json.dumps({"service_uuid": self.service_uuid}))
        self.assertEqual(r2.status_code, 201)

        # give the emulator some time to instantiate everything
        time.sleep(2)

        # check get request APIs
        r3 = requests.get("http://127.0.0.1:5000/packages")
        self.assertEqual(len(json.loads(r3.text).get("service_uuid_list")), 1)
        r4 = requests.get("http://127.0.0.1:5000/instantiations")
        self.assertEqual(len(json.loads(r4.text).get("service_instantiations_list")), 1)

        # stop the service
        service_instance_uuid = json.loads(r2.text).get("service_instance_uuid")
        self.assertTrue(service_instance_uuid is not None)
        requests.delete("http://127.0.0.1:5000/instantiations", data=json.dumps({"service_uuid": self.service_uuid, "service_instance_uuid":service_instance_uuid}))

        r5 = requests.get("http://127.0.0.1:5000/instantiations")
        self.assertTrue(len(json.loads(r5.text).get("service_instantiations_list")), 0)     # note that there was 1 instance before

        # stop Mininet network
        self.stopNet()
        initialize_GK()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# gagrab documentation build configuration file

import os
import re

def get_version():
    """
    Extracts the version number from the version.py file.
    """
    VERSION_FILE = '../gagrab/version.py'
    mo = re.search(r'^__version__ = [\'"]([^\'"]*)[\'"]', open(VERSION_FILE, 'rt').read(), re.M)
    if mo:
        return mo.group(1)
    else:
        raise RuntimeError('Unable to find version string in {0}.'.format(VERSION_FILE))

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.intersphinx',
    #'sphinx.ext.viewcode',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'gagrab'
copyright = u'2014, Ambition Inc.'

# The short X.Y version.
version = get_version()
# The full version, including alpha/beta/rc tags.
release = version

exclude_patterns = ['_build']

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

intersphinx_mapping = {
    'python': ('http://python.readthedocs.org/en/v2.7.2/', None),
}

# -- Options for HTML output ----------------------------------------------

html_theme = 'default'

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
if not on_rtd:  # only import and set the theme if we're building docs locally
    import sphinx_rtd_theme
    html_theme = 'sphinx_rtd_theme'
    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
#html_static_path = ['_static']

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
html_show_sphinx = False

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
html_show_copyright = True

# Output file base name for HTML help builder.
htmlhelp_basename = 'gagrabdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  ('index', 'gagrab.tex', u'gagrab Documentation',
   u'Erik Swanson', 'manual'),
]

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'gagrab', u'gagrab Documentation',
     [u'Erik Swanson'], 1)
]

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'gagrab', u'gagrab Documentation',
   u'Erik Swanson', 'gagrab', 'A short description',
   'Miscellaneous'),
]
<commit_msg>Make sure gagrab is importable from sphinx.<commit_after># -*- coding: utf-8 -*-
#
# gagrab documentation build configuration file
import os
import re

import sys
sys.path.insert(0, os.path.abspath('..'))

def get_version():
    """
    Extracts the version number from the version.py file.
    """
    VERSION_FILE = '../gagrab/version.py'
    mo = re.search(r'^__version__ = [\'"]([^\'"]*)[\'"]', open(VERSION_FILE, 'rt').read(), re.M)
    if mo:
        return mo.group(1)
    else:
        raise RuntimeError('Unable to find version string in {0}.'.format(VERSION_FILE))

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.intersphinx',
    #'sphinx.ext.viewcode',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'gagrab'
copyright = u'2014, Ambition Inc.'

# The short X.Y version.
version = get_version()
# The full version, including alpha/beta/rc tags.
release = version

exclude_patterns = ['_build']

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

intersphinx_mapping = {
    'python': ('http://python.readthedocs.org/en/v2.7.2/', None),
}

# -- Options for HTML output ----------------------------------------------

html_theme = 'default'

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
if not on_rtd:  # only import and set the theme if we're building docs locally
    import sphinx_rtd_theme
    html_theme = 'sphinx_rtd_theme'
    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
#html_static_path = ['_static']

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
html_show_sphinx = False

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
html_show_copyright = True

# Output file base name for HTML help builder.
htmlhelp_basename = 'gagrabdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#'papersize': 'letterpaper',

# The font size ('10pt', '11pt' or '12pt').
#'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  ('index', 'gagrab.tex', u'gagrab Documentation',
   u'Erik Swanson', 'manual'),
]

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'gagrab', u'gagrab Documentation',
     [u'Erik Swanson'], 1)
]

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  ('index', 'gagrab', u'gagrab Documentation',
   u'Erik Swanson', 'gagrab', 'A short description',
   'Miscellaneous'),
]
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Tests for the linalg.isolve.lgmres module
"""

from numpy.testing import *

from numpy import zeros, ones, arange, array, abs, max, allclose
from scipy.linalg import norm
from scipy.sparse import spdiags, csr_matrix, triu, tril

from scipy.sparse.linalg.interface import LinearOperator
from scipy.sparse.linalg import splu
from scipy.sparse.linalg.isolve import lgmres, gmres

Am = csr_matrix(array([[-2,1,0,0,0,9],
                       [1,-2,1,0,5,0],
                       [0,1,-2,1,0,0],
                       [0,0,1,-2,1,0],
                       [0,3,0,1,-2,1],
                       [1,0,0,0,1,-2]]))
b = array([1,2,3,4,5,6])
count = [0]
def matvec(v):
    count[0] += 1
    return Am*v
A = LinearOperator(matvec=matvec, shape=Am.shape, dtype=Am.dtype)
def do_solve(**kw):
    count[0] = 0
    x0, flag = lgmres(A, b, x0=zeros(A.shape[0]), inner_m=6, tol=1e-14, **kw)
    count_0 = count[0]
    assert allclose(A*x0, b, rtol=1e-12, atol=1e-12), norm(A*x0-b)
    return x0, count_0

class TestLGMRES(TestCase):
    def test_preconditioner(self):
        # Check that preconditioning works
        pc = splu(Am.tocsc())
        M = LinearOperator(matvec=pc.solve, shape=A.shape, dtype=A.dtype)

        x0, count_0 = do_solve()
        x1, count_1 = do_solve(M=M)

        assert count_1 == 3
        assert count_1 < count_0/2
        assert allclose(x1, x0, rtol=1e-14)

    def test_outer_v(self):
        # Check that the augmentation vectors behave as expected

        outer_v = []
        x0, count_0 = do_solve(outer_k=6, outer_v=outer_v)
        assert len(outer_v) > 0
        assert len(outer_v) <= 6

        x1, count_1 = do_solve(outer_k=6, outer_v=outer_v)
        assert count_1 == 2, count_1
        assert count_1 < count_0/2
        assert allclose(x1, x0, rtol=1e-14)

        # ---

        outer_v = []
        x0, count_0 = do_solve(outer_k=6, outer_v=outer_v, store_outer_Av=False)
        assert array([v[1] is None for v in outer_v]).all()
        assert len(outer_v) > 0
        assert len(outer_v) <= 6

        x1, count_1 = do_solve(outer_k=6, outer_v=outer_v)
        assert count_1 == 3, count_1
        assert count_1 < count_0/2
        assert allclose(x1, x0, rtol=1e-14)

if __name__ == "__main__":
    nose.run(argv=['', __file__])
<commit_msg>test_lgmres: add missing nose import<commit_after>#!/usr/bin/env python
"""Tests for the linalg.isolve.lgmres module
"""

from numpy.testing import *

from numpy import zeros, ones, arange, array, abs, max, allclose
from scipy.linalg import norm
from scipy.sparse import spdiags, csr_matrix, triu, tril

from scipy.sparse.linalg.interface import LinearOperator
from scipy.sparse.linalg import splu
from scipy.sparse.linalg.isolve import lgmres, gmres

Am = csr_matrix(array([[-2,1,0,0,0,9],
                       [1,-2,1,0,5,0],
                       [0,1,-2,1,0,0],
                       [0,0,1,-2,1,0],
                       [0,3,0,1,-2,1],
                       [1,0,0,0,1,-2]]))
b = array([1,2,3,4,5,6])
count = [0]
def matvec(v):
    count[0] += 1
    return Am*v
A = LinearOperator(matvec=matvec, shape=Am.shape, dtype=Am.dtype)
def do_solve(**kw):
    count[0] = 0
    x0, flag = lgmres(A, b, x0=zeros(A.shape[0]), inner_m=6, tol=1e-14, **kw)
    count_0 = count[0]
    assert allclose(A*x0, b, rtol=1e-12, atol=1e-12), norm(A*x0-b)
    return x0, count_0

class TestLGMRES(TestCase):
    def test_preconditioner(self):
        # Check that preconditioning works
        pc = splu(Am.tocsc())
        M = LinearOperator(matvec=pc.solve, shape=A.shape, dtype=A.dtype)

        x0, count_0 = do_solve()
        x1, count_1 = do_solve(M=M)

        assert count_1 == 3
        assert count_1 < count_0/2
        assert allclose(x1, x0, rtol=1e-14)

    def test_outer_v(self):
        # Check that the augmentation vectors behave as expected

        outer_v = []
        x0, count_0 = do_solve(outer_k=6, outer_v=outer_v)
        assert len(outer_v) > 0
        assert len(outer_v) <= 6

        x1, count_1 = do_solve(outer_k=6, outer_v=outer_v)
        assert count_1 == 2, count_1
        assert count_1 < count_0/2
        assert allclose(x1, x0, rtol=1e-14)

        # ---

        outer_v = []
        x0, count_0 = do_solve(outer_k=6, outer_v=outer_v, store_outer_Av=False)
        assert array([v[1] is None for v in outer_v]).all()
        assert len(outer_v) > 0
        assert len(outer_v) <= 6

        x1, count_1 = do_solve(outer_k=6, outer_v=outer_v)
        assert count_1 == 3, count_1
        assert count_1 < count_0/2
        assert allclose(x1, x0, rtol=1e-14)

if __name__ == "__main__":
    import nose
    nose.run(argv=['', __file__])
<|endoftext|>


<commit_before># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2014, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""
Temporal Memory mixin that enables detailed monitoring of history.
"""

from collections import defaultdict

from prettytable import PrettyTable

from nupic.research.monitor_mixin.metric import Metric
from nupic.research.monitor_mixin.monitor_mixin_base import MonitorMixinBase
from nupic.research.monitor_mixin.trace import (IndicesTrace, CountsTrace,
                                                BoolsTrace, StringsTrace)

class TemporalMemoryMonitorMixin(MonitorMixinBase):
  """
  Mixin for TemporalMemory that stores a detailed history, for inspection and
  debugging.
  """

  def __init__(self, *args, **kwargs):
    super(TemporalMemoryMonitorMixin, self).__init__(*args, **kwargs)

    self._mmResetActive = True  # First iteration is always a reset

  def mmGetTraceActiveColumns(self):
    """
    @return (Trace) Trace of active columns
    """
    return self._mmTraces["activeColumns"]

  def mmGetTracePredictiveCells(self):
    """
    @return (Trace) Trace of predictive cells
    """
    return self._mmTraces["predictiveCells"]

  def mmGetTraceNumSegments(self):
    """
    @return (Trace) Trace of # segments
    """
    return self._mmTraces["numSegments"]

  def mmGetTraceNumSynapses(self):
    """
    @return (Trace) Trace of # synapses
    """
    return self._mmTraces["numSynapses"]

  def mmGetTraceSequenceLabels(self):
    """
    @return (Trace) Trace of sequence labels
    """
    return self._mmTraces["sequenceLabels"]

  def mmGetTraceResets(self):
    """
    @return (Trace) Trace of resets
    """
    return self._mmTraces["resets"]

  def mmGetTracePredictedActiveCells(self):
    """
    @return (Trace) Trace of predicted => active cells
    """
    self._mmComputeTransitionTraces()
    return self._mmTraces["predictedActiveCells"]

  def mmGetTracePredictedInactiveCells(self):
    """
    @return (Trace) Trace of predicted => inactive cells
    """
    self._mmComputeTransitionTraces()
    return self._mmTraces["predictedInactiveCells"]

  def mmGetTracePredictedActiveColumns(self):
    """
    @return (Trace) Trace of predicted => active columns
    """
    self._mmComputeTransitionTraces()
    return self._mmTraces["predictedActiveColumns"]

  def mmGetTracePredictedInactiveColumns(self):
    """
    @return (Trace) Trace of predicted => inactive columns
    """
    self._mmComputeTransitionTraces()
    return self._mmTraces["predictedInactiveColumns"]

  def mmGetTraceUnpredictedActiveColumns(self):
    """
    @return (Trace) Trace of unpredicted => active columns
    """
    self._mmComputeTransitionTraces()
    return self._mmTraces["unpredictedActiveColumns"]

  def mmGetMetricFromTrace(self, trace):
    """
    Convenience method to compute a metric over an indices trace, excluding
    resets.

    @param (IndicesTrace) Trace of indices

    @return (Metric) Metric over trace excluding resets
    """
    return Metric.createFromTrace(trace.makeCountsTrace(),
                                  excludeResets=self.mmGetTraceResets())

  def mmGetMetricSequencesPredictedActiveCellsPerColumn(self):
    """
    Metric for number of predicted => active cells per column for each sequence

    @return (Metric) metric
    """
    self._mmComputeTransitionTraces()

    numCellsPerColumn = []

    for predictedActiveCells in (
        self._mmData["predictedActiveCellsForSequence"].values()):
      cellsForColumn = self.mapCellsToColumns(predictedActiveCells)
      numCellsPerColumn += [len(x) for x in cellsForColumn.values()]

    return Metric(self,
                  "# predicted => active cells per column for each sequence",
                  numCellsPerColumn)

  def mmGetMetricSequencesPredictedActiveCellsShared(self):
    """
    Metric for number of sequences each predicted => active cell appears in

    Note: This metric is flawed when it comes to high-order sequences.

    @return (Metric) metric
    """
    self._mmComputeTransitionTraces()

    numSequencesForCell = defaultdict(lambda: 0)

    for predictedActiveCells in (
          self._mmData["predictedActiveCellsForSequence"].values()):
      for cell in predictedActiveCells:
        numSequencesForCell[cell] += 1

    return Metric(self,
                  "# sequences each predicted => active cells appears in",
                  numSequencesForCell.values())

  def mmPrettyPrintConnections(self):
    """
    Pretty print the connections in the temporal memory.

    TODO: Use PrettyTable.

    @return (string) Pretty-printed text
    """
    text = ""

    text += ("Segments: (format => "
             "(#) [(source cell=permanence ...),       ...]\n")
    text += "------------------------------------\n"

    columns = range(self.numberOfColumns())

    for column in columns:
      cells = self.cellsForColumn(column)

      for cell in cells:
        segmentDict = dict()

        for seg in self.connections.segmentsForCell(cell):
          synapseList = []

          for synapse in self.connections.synapsesForSegment(seg):
            (_, sourceCell, permanence) = self.connections.dataForSynapse(
              synapse)

            synapseList.append((sourceCell, permanence))

          synapseList.sort()
          synapseStringList = ["{0:3}={1:.2f}".format(sourceCell, permanence) for
                               sourceCell, permanence in synapseList]
          segmentDict[seg] = "({0})".format(" ".join(synapseStringList))

        text += ("Column {0:3} / Cell {1:3}:\t({2}) {3}\n".format(
          column, cell,
          len(segmentDict.values()),
          "[{0}]".format(",       ".join(segmentDict.values()))))

      if column < len(columns) - 1:  # not last
        text += "\n"

    text += "------------------------------------\n"

    return text

  def mmPrettyPrintSequenceCellRepresentations(self, sortby="Column"):
    """
    Pretty print the cell representations for sequences in the history.

    @param sortby (string) Column of table to sort by

    @return (string) Pretty-printed text
    """
    self._mmComputeTransitionTraces()
    table = PrettyTable(["Pattern", "Column", "predicted=>active cells"])

    for sequenceLabel, predictedActiveCells in (
          self._mmData["predictedActiveCellsForSequence"].iteritems()):
      cellsForColumn = self.mapCellsToColumns(predictedActiveCells)
      for column, cells in cellsForColumn.iteritems():
        table.add_row([sequenceLabel, column, list(cells)])

    return table.get_string(sortby=sortby).encode("utf-8")

  # ==============================
  # Helper methods
  # ==============================

  def _mmComputeTransitionTraces(self):
    """
    Computes the transition traces, if necessary.

    Transition traces are the following:

        predicted => active cells
        predicted => inactive cells
        predicted => active columns
        predicted => inactive columns
        unpredicted => active columns
    """
    if not self._mmTransitionTracesStale:
      return

    self._mmData["predictedActiveCellsForSequence"] = defaultdict(set)

    self._mmTraces["predictedActiveCells"] = IndicesTrace(self,
      "predicted => active cells (correct)")
    self._mmTraces["predictedInactiveCells"] = IndicesTrace(self,
      "predicted => inactive cells (extra)")
    self._mmTraces["predictedActiveColumns"] = IndicesTrace(self,
      "predicted => active columns (correct)")
    self._mmTraces["predictedInactiveColumns"] = IndicesTrace(self,
      "predicted => inactive columns (extra)")
    self._mmTraces["unpredictedActiveColumns"] = IndicesTrace(self,
      "unpredicted => active columns (bursting)")

    predictedCellsTrace = self._mmTraces["predictedCells"]

    for i, activeColumns in enumerate(self.mmGetTraceActiveColumns().data):
      predictedActiveCells = set()
      predictedInactiveCells = set()
      predictedActiveColumns = set()
      predictedInactiveColumns = set()

      for predictedCell in predictedCellsTrace.data[i]:
        predictedColumn = self.columnForCell(predictedCell)

        if predictedColumn  in activeColumns:
          predictedActiveCells.add(predictedCell)
          predictedActiveColumns.add(predictedColumn)

          sequenceLabel = self.mmGetTraceSequenceLabels().data[i]
          if sequenceLabel is not None:
            self._mmData["predictedActiveCellsForSequence"][sequenceLabel].add(
              predictedCell)
        else:
          predictedInactiveCells.add(predictedCell)
          predictedInactiveColumns.add(predictedColumn)

      unpredictedActiveColumns = activeColumns - predictedActiveColumns

      self._mmTraces["predictedActiveCells"].data.append(predictedActiveCells)
      self._mmTraces["predictedInactiveCells"].data.append(predictedInactiveCells)
      self._mmTraces["predictedActiveColumns"].data.append(predictedActiveColumns)
      self._mmTraces["predictedInactiveColumns"].data.append(
        predictedInactiveColumns)
      self._mmTraces["unpredictedActiveColumns"].data.append(
        unpredictedActiveColumns)

    self._mmTransitionTracesStale = False

  # ==============================
  # Overrides
  # ==============================
  def compute(self, activeColumns, sequenceLabel=None, **kwargs):
    # Append last cycle's predictiveCells to *predicTEDCells* trace
    self._mmTraces["predictedCells"].data.append(self.predictiveCells)

    super(TemporalMemoryMonitorMixin, self).compute(activeColumns, **kwargs)

    # Append this cycle's predictiveCells to *predicTIVECells* trace
    self._mmTraces["predictiveCells"].data.append(self.predictiveCells)

    self._mmTraces["activeCells"].data.append(self.activeCells)
    self._mmTraces["activeColumns"].data.append(activeColumns)
    self._mmTraces["numSegments"].data.append(self.connections.numSegments())
    self._mmTraces["numSynapses"].data.append(self.connections.numSynapses())
    self._mmTraces["sequenceLabels"].data.append(sequenceLabel)
    self._mmTraces["resets"].data.append(self._mmResetActive)
    self._mmResetActive = False

    self._mmTransitionTracesStale = True

  def reset(self):
    super(TemporalMemoryMonitorMixin, self).reset()

    self._mmResetActive = True

  def mmGetDefaultTraces(self, verbosity=1):
    traces = [
      self.mmGetTraceActiveColumns(),
      self.mmGetTracePredictedActiveColumns(),
      self.mmGetTracePredictedInactiveColumns(),
      self.mmGetTraceUnpredictedActiveColumns(),
      self.mmGetTracePredictedActiveCells(),
      self.mmGetTracePredictedInactiveCells()
    ]

    if verbosity == 1:
      traces = [trace.makeCountsTrace() for trace in traces]

    traces += [
      self.mmGetTraceNumSegments(),
      self.mmGetTraceNumSynapses()
    ]

    return traces + [self.mmGetTraceSequenceLabels()]

  def mmGetDefaultMetrics(self, verbosity=1):
    resetsTrace = self.mmGetTraceResets()
    return ([Metric.createFromTrace(trace, excludeResets=resetsTrace)
              for trace in self.mmGetDefaultTraces()[:-3]] +
            [Metric.createFromTrace(trace)
              for trace in self.mmGetDefaultTraces()[-3:-1]] +
            [self.mmGetMetricSequencesPredictedActiveCellsPerColumn(),
             self.mmGetMetricSequencesPredictedActiveCellsShared()])

  def mmClearHistory(self):
    super(TemporalMemoryMonitorMixin, self).mmClearHistory()

    self._mmTraces["predictedCells"] = IndicesTrace(self, "predicted cells")
    self._mmTraces["activeColumns"] = IndicesTrace(self, "active columns")
    self._mmTraces["activeCells"] = IndicesTrace(self, "active cells")
    self._mmTraces["predictiveCells"] = IndicesTrace(self, "predictive cells")
    self._mmTraces["numSegments"] = CountsTrace(self, "# segments")
    self._mmTraces["numSynapses"] = CountsTrace(self, "# synapses")
    self._mmTraces["sequenceLabels"] = StringsTrace(self, "sequence labels")
    self._mmTraces["resets"] = BoolsTrace(self, "resets")
    self._mmTransitionTracesStale = True

  def mmGetCellActivityPlot(self, title="", showReset=False,
                            resetShading=0.25, activityType="activeCells"):
    """
    Returns plot of the cell activity.

    @param title        (string)  an optional title for the figure

    @param showReset    (bool)    if true, the first set of cell activities
                                  after a reset will have a gray background

    @param resetShading (float)   if showReset is true, this float specifies the
                                  intensity of the reset background with 0.0
                                  being white and 1.0 being black

    @param activityType (string)  The type of cell activity to display. Valid
                                  types include "activeCells",
                                  "predictiveCells", "predictedCells",
                                  and "predictedActiveCells"

    @return (Plot) plot
    """
    if activityType == "predictedActiveCells":
      self._mmComputeTransitionTraces()

    # If the trace contains ConnectionsCell, convert them to int
    cellTrace = self._mmTraces[activityType].data
    for i in xrange(len(cellTrace)):
      cellTrace[i] = self.getCellIndices(cellTrace[i])

    return self.mmGetCellTracePlot(cellTrace, self.numberOfCells(),
                                   activityType, title, showReset, resetShading)<commit_msg>Fixes bug in mmGetCellActivityPlot<commit_after># ----------------------------------------------------------------------
# Numenta Platform for Intelligent Computing (NuPIC)
# Copyright (C) 2014, Numenta, Inc.  Unless you have an agreement
# with Numenta, Inc., for a separate license for this software code, the
# following terms and conditions apply:
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 3 as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------

"""
Temporal Memory mixin that enables detailed monitoring of history.
"""

from collections import defaultdict

from prettytable import PrettyTable

from nupic.research.monitor_mixin.metric import Metric
from nupic.research.monitor_mixin.monitor_mixin_base import MonitorMixinBase
from nupic.research.monitor_mixin.trace import (IndicesTrace, CountsTrace,
                                                BoolsTrace, StringsTrace)

class TemporalMemoryMonitorMixin(MonitorMixinBase):
  """
  Mixin for TemporalMemory that stores a detailed history, for inspection and
  debugging.
  """

  def __init__(self, *args, **kwargs):
    super(TemporalMemoryMonitorMixin, self).__init__(*args, **kwargs)

    self._mmResetActive = True  # First iteration is always a reset

  def mmGetTraceActiveColumns(self):
    """
    @return (Trace) Trace of active columns
    """
    return self._mmTraces["activeColumns"]

  def mmGetTracePredictiveCells(self):
    """
    @return (Trace) Trace of predictive cells
    """
    return self._mmTraces["predictiveCells"]

  def mmGetTraceNumSegments(self):
    """
    @return (Trace) Trace of # segments
    """
    return self._mmTraces["numSegments"]

  def mmGetTraceNumSynapses(self):
    """
    @return (Trace) Trace of # synapses
    """
    return self._mmTraces["numSynapses"]

  def mmGetTraceSequenceLabels(self):
    """
    @return (Trace) Trace of sequence labels
    """
    return self._mmTraces["sequenceLabels"]

  def mmGetTraceResets(self):
    """
    @return (Trace) Trace of resets
    """
    return self._mmTraces["resets"]

  def mmGetTracePredictedActiveCells(self):
    """
    @return (Trace) Trace of predicted => active cells
    """
    self._mmComputeTransitionTraces()
    return self._mmTraces["predictedActiveCells"]

  def mmGetTracePredictedInactiveCells(self):
    """
    @return (Trace) Trace of predicted => inactive cells
    """
    self._mmComputeTransitionTraces()
    return self._mmTraces["predictedInactiveCells"]

  def mmGetTracePredictedActiveColumns(self):
    """
    @return (Trace) Trace of predicted => active columns
    """
    self._mmComputeTransitionTraces()
    return self._mmTraces["predictedActiveColumns"]

  def mmGetTracePredictedInactiveColumns(self):
    """
    @return (Trace) Trace of predicted => inactive columns
    """
    self._mmComputeTransitionTraces()
    return self._mmTraces["predictedInactiveColumns"]

  def mmGetTraceUnpredictedActiveColumns(self):
    """
    @return (Trace) Trace of unpredicted => active columns
    """
    self._mmComputeTransitionTraces()
    return self._mmTraces["unpredictedActiveColumns"]

  def mmGetMetricFromTrace(self, trace):
    """
    Convenience method to compute a metric over an indices trace, excluding
    resets.

    @param (IndicesTrace) Trace of indices

    @return (Metric) Metric over trace excluding resets
    """
    return Metric.createFromTrace(trace.makeCountsTrace(),
                                  excludeResets=self.mmGetTraceResets())

  def mmGetMetricSequencesPredictedActiveCellsPerColumn(self):
    """
    Metric for number of predicted => active cells per column for each sequence

    @return (Metric) metric
    """
    self._mmComputeTransitionTraces()

    numCellsPerColumn = []

    for predictedActiveCells in (
        self._mmData["predictedActiveCellsForSequence"].values()):
      cellsForColumn = self.mapCellsToColumns(predictedActiveCells)
      numCellsPerColumn += [len(x) for x in cellsForColumn.values()]

    return Metric(self,
                  "# predicted => active cells per column for each sequence",
                  numCellsPerColumn)

  def mmGetMetricSequencesPredictedActiveCellsShared(self):
    """
    Metric for number of sequences each predicted => active cell appears in

    Note: This metric is flawed when it comes to high-order sequences.

    @return (Metric) metric
    """
    self._mmComputeTransitionTraces()

    numSequencesForCell = defaultdict(lambda: 0)

    for predictedActiveCells in (
          self._mmData["predictedActiveCellsForSequence"].values()):
      for cell in predictedActiveCells:
        numSequencesForCell[cell] += 1

    return Metric(self,
                  "# sequences each predicted => active cells appears in",
                  numSequencesForCell.values())

  def mmPrettyPrintConnections(self):
    """
    Pretty print the connections in the temporal memory.

    TODO: Use PrettyTable.

    @return (string) Pretty-printed text
    """
    text = ""

    text += ("Segments: (format => "
             "(#) [(source cell=permanence ...),       ...]\n")
    text += "------------------------------------\n"

    columns = range(self.numberOfColumns())

    for column in columns:
      cells = self.cellsForColumn(column)

      for cell in cells:
        segmentDict = dict()

        for seg in self.connections.segmentsForCell(cell):
          synapseList = []

          for synapse in self.connections.synapsesForSegment(seg):
            (_, sourceCell, permanence) = self.connections.dataForSynapse(
              synapse)

            synapseList.append((sourceCell, permanence))

          synapseList.sort()
          synapseStringList = ["{0:3}={1:.2f}".format(sourceCell, permanence) for
                               sourceCell, permanence in synapseList]
          segmentDict[seg] = "({0})".format(" ".join(synapseStringList))

        text += ("Column {0:3} / Cell {1:3}:\t({2}) {3}\n".format(
          column, cell,
          len(segmentDict.values()),
          "[{0}]".format(",       ".join(segmentDict.values()))))

      if column < len(columns) - 1:  # not last
        text += "\n"

    text += "------------------------------------\n"

    return text

  def mmPrettyPrintSequenceCellRepresentations(self, sortby="Column"):
    """
    Pretty print the cell representations for sequences in the history.

    @param sortby (string) Column of table to sort by

    @return (string) Pretty-printed text
    """
    self._mmComputeTransitionTraces()
    table = PrettyTable(["Pattern", "Column", "predicted=>active cells"])

    for sequenceLabel, predictedActiveCells in (
          self._mmData["predictedActiveCellsForSequence"].iteritems()):
      cellsForColumn = self.mapCellsToColumns(predictedActiveCells)
      for column, cells in cellsForColumn.iteritems():
        table.add_row([sequenceLabel, column, list(cells)])

    return table.get_string(sortby=sortby).encode("utf-8")

  # ==============================
  # Helper methods
  # ==============================

  def _mmComputeTransitionTraces(self):
    """
    Computes the transition traces, if necessary.

    Transition traces are the following:

        predicted => active cells
        predicted => inactive cells
        predicted => active columns
        predicted => inactive columns
        unpredicted => active columns
    """
    if not self._mmTransitionTracesStale:
      return

    self._mmData["predictedActiveCellsForSequence"] = defaultdict(set)

    self._mmTraces["predictedActiveCells"] = IndicesTrace(self,
      "predicted => active cells (correct)")
    self._mmTraces["predictedInactiveCells"] = IndicesTrace(self,
      "predicted => inactive cells (extra)")
    self._mmTraces["predictedActiveColumns"] = IndicesTrace(self,
      "predicted => active columns (correct)")
    self._mmTraces["predictedInactiveColumns"] = IndicesTrace(self,
      "predicted => inactive columns (extra)")
    self._mmTraces["unpredictedActiveColumns"] = IndicesTrace(self,
      "unpredicted => active columns (bursting)")

    predictedCellsTrace = self._mmTraces["predictedCells"]

    for i, activeColumns in enumerate(self.mmGetTraceActiveColumns().data):
      predictedActiveCells = set()
      predictedInactiveCells = set()
      predictedActiveColumns = set()
      predictedInactiveColumns = set()

      for predictedCell in predictedCellsTrace.data[i]:
        predictedColumn = self.columnForCell(predictedCell)

        if predictedColumn  in activeColumns:
          predictedActiveCells.add(predictedCell)
          predictedActiveColumns.add(predictedColumn)

          sequenceLabel = self.mmGetTraceSequenceLabels().data[i]
          if sequenceLabel is not None:
            self._mmData["predictedActiveCellsForSequence"][sequenceLabel].add(
              predictedCell)
        else:
          predictedInactiveCells.add(predictedCell)
          predictedInactiveColumns.add(predictedColumn)

      unpredictedActiveColumns = activeColumns - predictedActiveColumns

      self._mmTraces["predictedActiveCells"].data.append(predictedActiveCells)
      self._mmTraces["predictedInactiveCells"].data.append(predictedInactiveCells)
      self._mmTraces["predictedActiveColumns"].data.append(predictedActiveColumns)
      self._mmTraces["predictedInactiveColumns"].data.append(
        predictedInactiveColumns)
      self._mmTraces["unpredictedActiveColumns"].data.append(
        unpredictedActiveColumns)

    self._mmTransitionTracesStale = False

  # ==============================
  # Overrides
  # ==============================
  def compute(self, activeColumns, sequenceLabel=None, **kwargs):
    # Append last cycle's predictiveCells to *predicTEDCells* trace
    self._mmTraces["predictedCells"].data.append(self.predictiveCells)

    super(TemporalMemoryMonitorMixin, self).compute(activeColumns, **kwargs)

    # Append this cycle's predictiveCells to *predicTIVECells* trace
    self._mmTraces["predictiveCells"].data.append(self.predictiveCells)

    self._mmTraces["activeCells"].data.append(self.activeCells)
    self._mmTraces["activeColumns"].data.append(activeColumns)
    self._mmTraces["numSegments"].data.append(self.connections.numSegments())
    self._mmTraces["numSynapses"].data.append(self.connections.numSynapses())
    self._mmTraces["sequenceLabels"].data.append(sequenceLabel)
    self._mmTraces["resets"].data.append(self._mmResetActive)
    self._mmResetActive = False

    self._mmTransitionTracesStale = True

  def reset(self):
    super(TemporalMemoryMonitorMixin, self).reset()

    self._mmResetActive = True

  def mmGetDefaultTraces(self, verbosity=1):
    traces = [
      self.mmGetTraceActiveColumns(),
      self.mmGetTracePredictedActiveColumns(),
      self.mmGetTracePredictedInactiveColumns(),
      self.mmGetTraceUnpredictedActiveColumns(),
      self.mmGetTracePredictedActiveCells(),
      self.mmGetTracePredictedInactiveCells()
    ]

    if verbosity == 1:
      traces = [trace.makeCountsTrace() for trace in traces]

    traces += [
      self.mmGetTraceNumSegments(),
      self.mmGetTraceNumSynapses()
    ]

    return traces + [self.mmGetTraceSequenceLabels()]

  def mmGetDefaultMetrics(self, verbosity=1):
    resetsTrace = self.mmGetTraceResets()
    return ([Metric.createFromTrace(trace, excludeResets=resetsTrace)
              for trace in self.mmGetDefaultTraces()[:-3]] +
            [Metric.createFromTrace(trace)
              for trace in self.mmGetDefaultTraces()[-3:-1]] +
            [self.mmGetMetricSequencesPredictedActiveCellsPerColumn(),
             self.mmGetMetricSequencesPredictedActiveCellsShared()])

  def mmClearHistory(self):
    super(TemporalMemoryMonitorMixin, self).mmClearHistory()

    self._mmTraces["predictedCells"] = IndicesTrace(self, "predicted cells")
    self._mmTraces["activeColumns"] = IndicesTrace(self, "active columns")
    self._mmTraces["activeCells"] = IndicesTrace(self, "active cells")
    self._mmTraces["predictiveCells"] = IndicesTrace(self, "predictive cells")
    self._mmTraces["numSegments"] = CountsTrace(self, "# segments")
    self._mmTraces["numSynapses"] = CountsTrace(self, "# synapses")
    self._mmTraces["sequenceLabels"] = StringsTrace(self, "sequence labels")
    self._mmTraces["resets"] = BoolsTrace(self, "resets")
    self._mmTransitionTracesStale = True

  def mmGetCellActivityPlot(self, title="", showReset=False,
                            resetShading=0.25, activityType="activeCells"):
    """
    Returns plot of the cell activity.

    @param title        (string)  an optional title for the figure

    @param showReset    (bool)    if true, the first set of cell activities
                                  after a reset will have a gray background

    @param resetShading (float)   if showReset is true, this float specifies the
                                  intensity of the reset background with 0.0
                                  being white and 1.0 being black

    @param activityType (string)  The type of cell activity to display. Valid
                                  types include "activeCells",
                                  "predictiveCells", "predictedCells",
                                  and "predictedActiveCells"

    @return (Plot) plot
    """
    if activityType == "predictedActiveCells":
      self._mmComputeTransitionTraces()

    # If the trace contains ConnectionsCell, convert them to int
    cellTrace = self._mmTraces[activityType].data
    for i in xrange(len(cellTrace))
      if not isinstance(cellTrace[i], int):
        cellTrace[i] = self.getCellIndices(cellTrace[i])

    return self.mmGetCellTracePlot(cellTrace, self.numberOfCells(),
                                   activityType, title, showReset, resetShading)<|endoftext|>


<commit_before>"""
Provides the core event loop
"""

import signal
import copy
from spock.mcp import mcdata
from spock.utils import pl_announce

import logging
logger = logging.getLogger('spock')

class EventCore:
	def __init__(self):
		self.kill_event = False
		self.event_handlers = {}
		signal.signal(signal.SIGINT, self.kill)
		signal.signal(signal.SIGTERM, self.kill)

	def event_loop(self):
		while not self.kill_event:
			self.emit('event_tick')
		logger.info('Event Kill called, shutting down')
		self.emit('kill')

	def reg_event_handler(self, event, handler):
		if event not in self.event_handlers:
			self.event_handlers[event] = []
		self.event_handlers[event].append(handler)

	def emit(self, event, data = None):
		if event not in self.event_handlers:
			self.event_handlers[event] = []
		for handler in self.event_handlers[event]:
			handler(event, (data.clone() if hasattr(data, 'clone')
				else copy.deepcopy(data)
			))

	def kill(self, *args):
		self.kill_event = True

@pl_announce('Event')
class EventPlugin:
	def __init__(self, ploader, settings):
		ploader.provides('Event', EventCore())
<commit_msg>Remove truthy handlers during event emits<commit_after>"""
Provides the core event loop
"""

import signal
import copy
from spock.mcp import mcdata
from spock.utils import pl_announce

import logging
logger = logging.getLogger('spock')

class EventCore:
	def __init__(self):
		self.kill_event = False
		self.event_handlers = {}
		signal.signal(signal.SIGINT, self.kill)
		signal.signal(signal.SIGTERM, self.kill)

	def event_loop(self):
		while not self.kill_event:
			self.emit('event_tick')
		logger.info('Event Kill called, shutting down')
		self.emit('kill')

	def reg_event_handler(self, event, handler):
		if event not in self.event_handlers:
			self.event_handlers[event] = []
		self.event_handlers[event].append(handler)

	def emit(self, event, data = None):
		if event not in self.event_handlers:
			self.event_handlers[event] = []
		to_remove = []
		for handler in self.event_handlers[event]:
			if handler(
				event,
				data.clone() if hasattr(data, 'clone') else copy.deepcopy(data)
			):
				to_remove.append(handler)
		for handler in to_remove:
			self.event_handlers[event].remove(handler)

	def kill(self, *args):
		self.kill_event = True

@pl_announce('Event')
class EventPlugin:
	def __init__(self, ploader, settings):
		ploader.provides('Event', EventCore())
<|endoftext|>


<commit_before>
_ndarray_status_fmt='%(msg)s shape=%(shape)s min=%(min)f max=%(max)f'

def ndarray_status(x, fmt=_ndarray_status_fmt, msg="", **kwargs):
    kwargs.update(dict(
            msg=msg,
            min=x.min(),
            max=x.max(),
            mean=x.mean(),
            var = x.var(),
            shape=x.shape))
    return fmt%kwargs

# XXX: copy-paste out of pylearn
try:
    from pylearn.io.image_tiling import tile_slices_to_image
except ImportError:
    def tile_slices_to_image(*args, **kwargs):
        raise NotImplementedError()

<commit_msg>Remove pylearn1 dependency from packaged_dependencies/theano_linear<commit_after>
from imaging import tile_slices_to_image

_ndarray_status_fmt='%(msg)s shape=%(shape)s min=%(min)f max=%(max)f'

def ndarray_status(x, fmt=_ndarray_status_fmt, msg="", **kwargs):
    kwargs.update(dict(
            msg=msg,
            min=x.min(),
            max=x.max(),
            mean=x.mean(),
            var = x.var(),
            shape=x.shape))
    return fmt%kwargs

<|endoftext|>


<commit_before>"""Handling connection with a local Taxonomy Database."""

__author__      = "Mateusz Korycinski"
__license__     = "GPL"
__version__     = "1.0"
__maintainer__  = "Mateusz Korycinski"
__email__       = "mkorycinski@protonmail.ch"
__status__      = "Testing"

class Node(object):
    """Describes Node object that is stored in the database as a document.
    Each node points to a parent node, unless tax_id = 0.

    """

    def __init__(self, taxid, scientific_name, upper_hierarchy, node_type=None):
        self.taxid = taxid
        self.scientific_name = scientific_name
        self.upper_hierarchy = upper_hierarchy
        self.node_type = node_type

class TaxDb(object):
    """Class containing methods to handle users requests for translating
    tax ids into phylogenies.

    """

    self.database_path = "/dbs/TaxID.mongodb"

    def connect(self):
        """Opens connection to the database"""

        pass

    def disconnect(self):
        """Closes connection to the database"""

        pass

    def add_record(self, node: Node):
        """Method updates database with a new entry.

        Params:
            protein_acc (str): Protein accession Identifier
            tax_id (str): Taxonomy Identifier

        Returns:
            Nothing

        """

        pass

    def organism_to_taxid(self, organism_name):
        """Method retrieves taxonomy ID for a query organism name.

        Params:
            organism_name (str): Binominal name of a query organism_name

        Returns:
            taxid (str): Taxonomy ID for a query organism

        e.g.:
        >>> TaxDb.organism_to_taxid('Archaeoglobus fulgidus DSM 4304')
        '224325'

        """

        pass

    def get_lineage_from_db(self, tax_id):
        """Method retrieves phylogenetic lineage from database based on
        accession.

        Params:
            tax_id (str): NCBI taxonomy identifier.

        Returns:
            lineage (str): Lineage of an organism represented by tax identifier.

        e.g.:
        >>> TaxDb.get_lineage_from_db('224325')
        ['cellular organisms', \
'Archaea', \
'Euryarchaeota', \
'Archaeoglobi', \
'Archaeoglobales', \
'Archaeoglobaceae', \
'Archaeoglobus', \
'Archaeoglobus fulgidus', \
'Archaeoglobus fulgidus DSM 4304']

        >>> TaxDb.get_lineage_from_db('2')
        ['cellular organisms', 'Bacteria']

        >> TaxIdDb.get_lineage_from_db('2759')
        ['cellular organisms', 'Eukaryota']

        """

        pass

    def document_exists(self, node: Node):
        """Method checks whether the document already exists in the database"""

        pass<commit_msg>Small fix regarding class attribute.<commit_after>"""Handling connection with a local Taxonomy Database."""

__author__      = "Mateusz Korycinski"
__license__     = "GPL"
__version__     = "1.0"
__maintainer__  = "Mateusz Korycinski"
__email__       = "mkorycinski@protonmail.ch"
__status__      = "Testing"

class Node(object):
    """Describes Node object that is stored in the database as a document.
    Each node points to a parent node, unless tax_id = 0.

    """

    def __init__(self, taxid, scientific_name, upper_hierarchy, node_type=None):
        self.taxid = taxid
        self.scientific_name = scientific_name
        self.upper_hierarchy = upper_hierarchy
        self.node_type = node_type

class TaxDb(object):
    """Class containing methods to handle users requests for translating
    tax ids into phylogenies.

    """

    database_path = '/dbs/TaxID.mongodb'

    def connect(self):
        """Opens connection to the database"""

        pass

    def disconnect(self):
        """Closes connection to the database"""

        pass

    def add_record(self, node: Node):
        """Method updates database with a new entry.

        Params:
            protein_acc (str): Protein accession Identifier
            tax_id (str): Taxonomy Identifier

        Returns:
            Nothing

        """

        pass

    def organism_to_taxid(self, organism_name):
        """Method retrieves taxonomy ID for a query organism name.

        Params:
            organism_name (str): Binominal name of a query organism_name

        Returns:
            taxid (str): Taxonomy ID for a query organism

        e.g.:
        >>> TaxDb.organism_to_taxid('Archaeoglobus fulgidus DSM 4304')
        '224325'

        """

        pass

    def get_lineage_from_db(self, tax_id):
        """Method retrieves phylogenetic lineage from database based on
        accession.

        Params:
            tax_id (str): NCBI taxonomy identifier.

        Returns:
            lineage (str): Lineage of an organism represented by tax identifier.

        e.g.:
        >>> TaxDb.get_lineage_from_db('224325')
        ['cellular organisms', \
'Archaea', \
'Euryarchaeota', \
'Archaeoglobi', \
'Archaeoglobales', \
'Archaeoglobaceae', \
'Archaeoglobus', \
'Archaeoglobus fulgidus', \
'Archaeoglobus fulgidus DSM 4304']

        >>> TaxDb.get_lineage_from_db('2')
        ['cellular organisms', 'Bacteria']

        >> TaxIdDb.get_lineage_from_db('2759')
        ['cellular organisms', 'Eukaryota']

        """

        pass

    def document_exists(self, node: Node):
        """Method checks whether the document already exists in the database"""

        pass<|endoftext|>


<commit_before>"""Tests for the linalg.isolve.lgmres module
"""

from __future__ import division, print_function, absolute_import

from numpy.testing import assert_, assert_allclose, assert_equal

import pytest
from platform import python_implementation

import numpy as np
from numpy import zeros, array, allclose
from scipy.linalg import norm
from scipy.sparse import csr_matrix, eye, rand

from scipy.sparse.linalg.interface import LinearOperator
from scipy.sparse.linalg import splu
from scipy.sparse.linalg.isolve import lgmres, gmres

from scipy._lib._numpy_compat import suppress_warnings

Am = csr_matrix(array([[-2, 1, 0, 0, 0, 9],
                       [1, -2, 1, 0, 5, 0],
                       [0, 1, -2, 1, 0, 0],
                       [0, 0, 1, -2, 1, 0],
                       [0, 3, 0, 1, -2, 1],
                       [1, 0, 0, 0, 1, -2]]))
b = array([1, 2, 3, 4, 5, 6])
count = [0]

def matvec(v):
    count[0] += 1
    return Am*v

A = LinearOperator(matvec=matvec, shape=Am.shape, dtype=Am.dtype)

def do_solve(**kw):
    count[0] = 0
    with suppress_warnings() as sup:
        sup.filter(DeprecationWarning, ".*called without specifying.*")
        x0, flag = lgmres(A, b, x0=zeros(A.shape[0]),
                          inner_m=6, tol=1e-14, **kw)
    count_0 = count[0]
    assert_(allclose(A*x0, b, rtol=1e-12, atol=1e-12), norm(A*x0-b))
    return x0, count_0

class TestLGMRES(object):
    def test_preconditioner(self):
        # Check that preconditioning works
        pc = splu(Am.tocsc())
        M = LinearOperator(matvec=pc.solve, shape=A.shape, dtype=A.dtype)

        x0, count_0 = do_solve()
        x1, count_1 = do_solve(M=M)

        assert_(count_1 == 3)
        assert_(count_1 < count_0/2)
        assert_(allclose(x1, x0, rtol=1e-14))

    def test_outer_v(self):
        # Check that the augmentation vectors behave as expected

        outer_v = []
        x0, count_0 = do_solve(outer_k=6, outer_v=outer_v)
        assert_(len(outer_v) > 0)
        assert_(len(outer_v) <= 6)

        x1, count_1 = do_solve(outer_k=6, outer_v=outer_v,
                               prepend_outer_v=True)
        assert_(count_1 == 2, count_1)
        assert_(count_1 < count_0/2)
        assert_(allclose(x1, x0, rtol=1e-14))

        # ---

        outer_v = []
        x0, count_0 = do_solve(outer_k=6, outer_v=outer_v,
                               store_outer_Av=False)
        assert_(array([v[1] is None for v in outer_v]).all())
        assert_(len(outer_v) > 0)
        assert_(len(outer_v) <= 6)

        x1, count_1 = do_solve(outer_k=6, outer_v=outer_v,
                               prepend_outer_v=True)
        assert_(count_1 == 3, count_1)
        assert_(count_1 < count_0/2)
        assert_(allclose(x1, x0, rtol=1e-14))

    @pytest.mark.skipif(python_implementation() == 'PyPy',
                        reason="Fails on PyPy CI runs. See #9507")
    def test_arnoldi(self):
        np.random.rand(1234)

        A = eye(10000) + rand(10000, 10000, density=1e-4)
        b = np.random.rand(10000)

        # The inner arnoldi should be equivalent to gmres
        with suppress_warnings() as sup:
            sup.filter(DeprecationWarning, ".*called without specifying.*")
            x0, flag0 = lgmres(A, b, x0=zeros(A.shape[0]),
                               inner_m=15, maxiter=1)
            x1, flag1 = gmres(A, b, x0=zeros(A.shape[0]),
                              restart=15, maxiter=1)

        assert_equal(flag0, 1)
        assert_equal(flag1, 1)
        assert_(np.linalg.norm(A.dot(x0) - b) > 1e-3)

        assert_allclose(x0, x1)

    def test_cornercase(self):
        np.random.seed(1234)

        # Rounding error may prevent convergence with tol=0 --- ensure
        # that the return values in this case are correct, and no
        # exceptions are raised

        for n in [3, 5, 10, 100]:
            A = 2*eye(n)

            with suppress_warnings() as sup:
                sup.filter(DeprecationWarning, ".*called without specifying.*")

                b = np.ones(n)
                x, info = lgmres(A, b, maxiter=10)
                assert_equal(info, 0)
                assert_allclose(A.dot(x) - b, 0, atol=1e-14)

                x, info = lgmres(A, b, tol=0, maxiter=10)
                if info == 0:
                    assert_allclose(A.dot(x) - b, 0, atol=1e-14)

                b = np.random.rand(n)
                x, info = lgmres(A, b, maxiter=10)
                assert_equal(info, 0)
                assert_allclose(A.dot(x) - b, 0, atol=1e-14)

                x, info = lgmres(A, b, tol=0, maxiter=10)
                if info == 0:
                    assert_allclose(A.dot(x) - b, 0, atol=1e-14)

    def test_nans(self):
        A = eye(3, format='lil')
        A[1, 1] = np.nan
        b = np.ones(3)

        with suppress_warnings() as sup:
            sup.filter(DeprecationWarning, ".*called without specifying.*")
            x, info = lgmres(A, b, tol=0, maxiter=10)
            assert_equal(info, 1)

    def test_breakdown_with_outer_v(self):
        A = np.array([[1, 2], [3, 4]], dtype=float)
        b = np.array([1, 2])

        x = np.linalg.solve(A, b)
        v0 = np.array([1, 0])

        # The inner iteration should converge to the correct solution,
        # since it's in the outer vector list
        with suppress_warnings() as sup:
            sup.filter(DeprecationWarning, ".*called without specifying.*")
            xp, info = lgmres(A, b, outer_v=[(v0, None), (x, None)], maxiter=1)

        assert_allclose(xp, x, atol=1e-12)

    def test_breakdown_underdetermined(self):
        # Should find LSQ solution in the Krylov span in one inner
        # iteration, despite solver breakdown from nilpotent A.
        A = np.array([[0, 1, 1, 1],
                      [0, 0, 1, 1],
                      [0, 0, 0, 1],
                      [0, 0, 0, 0]], dtype=float)

        bs = [
            np.array([1, 1, 1, 1]),
            np.array([1, 1, 1, 0]),
            np.array([1, 1, 0, 0]),
            np.array([1, 0, 0, 0]),
        ]

        for b in bs:
            with suppress_warnings() as sup:
                sup.filter(DeprecationWarning, ".*called without specifying.*")
                xp, info = lgmres(A, b, maxiter=1)
            resp = np.linalg.norm(A.dot(xp) - b)

            K = np.c_[b, A.dot(b), A.dot(A.dot(b)), A.dot(A.dot(A.dot(b)))]
            y, _, _, _ = np.linalg.lstsq(A.dot(K), b, rcond=-1)
            x = K.dot(y)
            res = np.linalg.norm(A.dot(x) - b)

            assert_allclose(resp, res, err_msg=repr(b))

    def test_denormals(self):
        # Check that no warnings are emitted if the matrix contains
        # numbers for which 1/x has no float representation, and that
        # the solver behaves properly.
        A = np.array([[1, 2], [3, 4]], dtype=float)
        A *= 100 * np.nextafter(0, 1)

        b = np.array([1, 1])

        with suppress_warnings() as sup:
            sup.filter(DeprecationWarning, ".*called without specifying.*")
            xp, info = lgmres(A, b)

        if info == 0:
            assert_allclose(A.dot(xp), b)

<commit_msg>TST: reduce max memory usage for sparse.linalg.lgmres test<commit_after>"""Tests for the linalg.isolve.lgmres module
"""

from __future__ import division, print_function, absolute_import

from numpy.testing import assert_, assert_allclose, assert_equal

import pytest
from platform import python_implementation

import numpy as np
from numpy import zeros, array, allclose
from scipy.linalg import norm
from scipy.sparse import csr_matrix, eye, rand

from scipy.sparse.linalg.interface import LinearOperator
from scipy.sparse.linalg import splu
from scipy.sparse.linalg.isolve import lgmres, gmres

from scipy._lib._numpy_compat import suppress_warnings

Am = csr_matrix(array([[-2, 1, 0, 0, 0, 9],
                       [1, -2, 1, 0, 5, 0],
                       [0, 1, -2, 1, 0, 0],
                       [0, 0, 1, -2, 1, 0],
                       [0, 3, 0, 1, -2, 1],
                       [1, 0, 0, 0, 1, -2]]))
b = array([1, 2, 3, 4, 5, 6])
count = [0]

def matvec(v):
    count[0] += 1
    return Am*v

A = LinearOperator(matvec=matvec, shape=Am.shape, dtype=Am.dtype)

def do_solve(**kw):
    count[0] = 0
    with suppress_warnings() as sup:
        sup.filter(DeprecationWarning, ".*called without specifying.*")
        x0, flag = lgmres(A, b, x0=zeros(A.shape[0]),
                          inner_m=6, tol=1e-14, **kw)
    count_0 = count[0]
    assert_(allclose(A*x0, b, rtol=1e-12, atol=1e-12), norm(A*x0-b))
    return x0, count_0

class TestLGMRES(object):
    def test_preconditioner(self):
        # Check that preconditioning works
        pc = splu(Am.tocsc())
        M = LinearOperator(matvec=pc.solve, shape=A.shape, dtype=A.dtype)

        x0, count_0 = do_solve()
        x1, count_1 = do_solve(M=M)

        assert_(count_1 == 3)
        assert_(count_1 < count_0/2)
        assert_(allclose(x1, x0, rtol=1e-14))

    def test_outer_v(self):
        # Check that the augmentation vectors behave as expected

        outer_v = []
        x0, count_0 = do_solve(outer_k=6, outer_v=outer_v)
        assert_(len(outer_v) > 0)
        assert_(len(outer_v) <= 6)

        x1, count_1 = do_solve(outer_k=6, outer_v=outer_v,
                               prepend_outer_v=True)
        assert_(count_1 == 2, count_1)
        assert_(count_1 < count_0/2)
        assert_(allclose(x1, x0, rtol=1e-14))

        # ---

        outer_v = []
        x0, count_0 = do_solve(outer_k=6, outer_v=outer_v,
                               store_outer_Av=False)
        assert_(array([v[1] is None for v in outer_v]).all())
        assert_(len(outer_v) > 0)
        assert_(len(outer_v) <= 6)

        x1, count_1 = do_solve(outer_k=6, outer_v=outer_v,
                               prepend_outer_v=True)
        assert_(count_1 == 3, count_1)
        assert_(count_1 < count_0/2)
        assert_(allclose(x1, x0, rtol=1e-14))

    @pytest.mark.skipif(python_implementation() == 'PyPy',
                        reason="Fails on PyPy CI runs. See #9507")
    def test_arnoldi(self):
        np.random.rand(1234)

        A = eye(2000) + rand(2000, 2000, density=5e-4)
        b = np.random.rand(2000)

        # The inner arnoldi should be equivalent to gmres
        with suppress_warnings() as sup:
            sup.filter(DeprecationWarning, ".*called without specifying.*")
            x0, flag0 = lgmres(A, b, x0=zeros(A.shape[0]),
                               inner_m=15, maxiter=1)
            x1, flag1 = gmres(A, b, x0=zeros(A.shape[0]),
                              restart=15, maxiter=1)

        assert_equal(flag0, 1)
        assert_equal(flag1, 1)
        assert_(np.linalg.norm(A.dot(x0) - b) > 4e-4)

        assert_allclose(x0, x1)

    def test_cornercase(self):
        np.random.seed(1234)

        # Rounding error may prevent convergence with tol=0 --- ensure
        # that the return values in this case are correct, and no
        # exceptions are raised

        for n in [3, 5, 10, 100]:
            A = 2*eye(n)

            with suppress_warnings() as sup:
                sup.filter(DeprecationWarning, ".*called without specifying.*")

                b = np.ones(n)
                x, info = lgmres(A, b, maxiter=10)
                assert_equal(info, 0)
                assert_allclose(A.dot(x) - b, 0, atol=1e-14)

                x, info = lgmres(A, b, tol=0, maxiter=10)
                if info == 0:
                    assert_allclose(A.dot(x) - b, 0, atol=1e-14)

                b = np.random.rand(n)
                x, info = lgmres(A, b, maxiter=10)
                assert_equal(info, 0)
                assert_allclose(A.dot(x) - b, 0, atol=1e-14)

                x, info = lgmres(A, b, tol=0, maxiter=10)
                if info == 0:
                    assert_allclose(A.dot(x) - b, 0, atol=1e-14)

    def test_nans(self):
        A = eye(3, format='lil')
        A[1, 1] = np.nan
        b = np.ones(3)

        with suppress_warnings() as sup:
            sup.filter(DeprecationWarning, ".*called without specifying.*")
            x, info = lgmres(A, b, tol=0, maxiter=10)
            assert_equal(info, 1)

    def test_breakdown_with_outer_v(self):
        A = np.array([[1, 2], [3, 4]], dtype=float)
        b = np.array([1, 2])

        x = np.linalg.solve(A, b)
        v0 = np.array([1, 0])

        # The inner iteration should converge to the correct solution,
        # since it's in the outer vector list
        with suppress_warnings() as sup:
            sup.filter(DeprecationWarning, ".*called without specifying.*")
            xp, info = lgmres(A, b, outer_v=[(v0, None), (x, None)], maxiter=1)

        assert_allclose(xp, x, atol=1e-12)

    def test_breakdown_underdetermined(self):
        # Should find LSQ solution in the Krylov span in one inner
        # iteration, despite solver breakdown from nilpotent A.
        A = np.array([[0, 1, 1, 1],
                      [0, 0, 1, 1],
                      [0, 0, 0, 1],
                      [0, 0, 0, 0]], dtype=float)

        bs = [
            np.array([1, 1, 1, 1]),
            np.array([1, 1, 1, 0]),
            np.array([1, 1, 0, 0]),
            np.array([1, 0, 0, 0]),
        ]

        for b in bs:
            with suppress_warnings() as sup:
                sup.filter(DeprecationWarning, ".*called without specifying.*")
                xp, info = lgmres(A, b, maxiter=1)
            resp = np.linalg.norm(A.dot(xp) - b)

            K = np.c_[b, A.dot(b), A.dot(A.dot(b)), A.dot(A.dot(A.dot(b)))]
            y, _, _, _ = np.linalg.lstsq(A.dot(K), b, rcond=-1)
            x = K.dot(y)
            res = np.linalg.norm(A.dot(x) - b)

            assert_allclose(resp, res, err_msg=repr(b))

    def test_denormals(self):
        # Check that no warnings are emitted if the matrix contains
        # numbers for which 1/x has no float representation, and that
        # the solver behaves properly.
        A = np.array([[1, 2], [3, 4]], dtype=float)
        A *= 100 * np.nextafter(0, 1)

        b = np.array([1, 1])

        with suppress_warnings() as sup:
            sup.filter(DeprecationWarning, ".*called without specifying.*")
            xp, info = lgmres(A, b)

        if info == 0:
            assert_allclose(A.dot(xp), b)

<|endoftext|>


<commit_before>"""
An implementation of :class:`XBlockUserStateClient`, which stores XBlock Scope.user_state
data in a Django ORM model.
"""

from xblock_user_state.interface import DjangoXBlockUserStateClient

class DjangoXBlockUserStateClient(DjangoXBlockUserStateClient):
    """
    An interface that uses the Django ORM StudentModule as a backend.
    """

    class ServiceUnavailable(XBlockUserStateClient.ServiceUnavailable):
        """
        This error is raised if the service backing this client is currently unavailable.
        """
        pass

    class PermissionDenied(XBlockUserStateClient.PermissionDenied):
        """
        This error is raised if the caller is not allowed to access the requested data.
        """
        pass

    class DoesNotExist(XBlockUserStateClient.DoesNotExist):
        """
        This error is raised if the caller has requested data that does not exist.
        """
        pass

    def get(username, block_key, scope=Scope.user_state):
        return self.get_many(username, [block_key], scope)

    def set(username, block_key, state, scope=Scope.user_state):
        self.set_many(username, {block_key: state}, scope)

    def get_many(username, block_keys, scope=Scope.user_state):
        """Returns dict of block_id -> state."""
        if scope != Scope.user_state:
            raise ValueError("Only Scope.user_state is supported")
        raise NotImplementedError()

    def set_many(username, block_keys_to_state, scope=Scope.user_state):
        if scope != Scope.user_state:
            raise ValueError("Only Scope.user_state is supported")
        raise NotImplementedError()

    def get_history(username, block_key, scope=Scope.user_state):
        """We don't guarantee that history for many blocks will be fast."""
        if scope != Scope.user_state:
            raise ValueError("Only Scope.user_state is supported")
        raise NotImplementedError()

    def iter_all_for_block(block_key, scope=Scope.user_state, batch_size=None):
        """
        You get no ordering guarantees. Fetching will happen in batch_size
        increments. If you're using this method, you should be running in an
        async task.
        """
        if scope != Scope.user_state:
            raise ValueError("Only Scope.user_state is supported")
        raise NotImplementedError()

    def iter_all_for_course(course_key, block_type=None, scope=Scope.user_state, batch_size=None):
        """
        You get no ordering guarantees. Fetching will happen in batch_size
        increments. If you're using this method, you should be running in an
        async task.
        """
        if scope != Scope.user_state:
            raise ValueError("Only Scope.user_state is supported")
        raise NotImplementedError()
<commit_msg>Add more documentation to XBlockUserStateClient interface<commit_after>"""
An implementation of :class:`XBlockUserStateClient`, which stores XBlock Scope.user_state
data in a Django ORM model.
"""

from xblock_user_state.interface import DjangoXBlockUserStateClient

class DjangoXBlockUserStateClient(DjangoXBlockUserStateClient):
    """
    An interface that uses the Django ORM StudentModule as a backend.
    """

    class ServiceUnavailable(XBlockUserStateClient.ServiceUnavailable):
        """
        This error is raised if the service backing this client is currently unavailable.
        """
        pass

    class PermissionDenied(XBlockUserStateClient.PermissionDenied):
        """
        This error is raised if the caller is not allowed to access the requested data.
        """
        pass

    class DoesNotExist(XBlockUserStateClient.DoesNotExist):
        """
        This error is raised if the caller has requested data that does not exist.
        """
        pass

    def get(username, block_key, scope=Scope.user_state, fields=None):
        """
        Retrieve the stored XBlock state for a single xblock usage.

        Arguments:
            username: The name of the user whose state should be retrieved
            block_key (UsageKey): The UsageKey identifying which xblock state to load.
            scope (Scope): The scope to load data from
            fields: A list of field values to retrieve. If None, retrieve all stored fields.

        Returns:
            dict: A dictionary mapping field names to values

        Raises:
            DoesNotExist if no entry is found.
        """
        try:
            _usage_key, state = next(self.get_many(username, [block_key], scope, fields=fields))
        except StopIteration:
            raise self.DoesNotExist()

        return state

    def set(username, block_key, state, scope=Scope.user_state):
        """
        Set fields for a particular XBlock.

        Arguments:
            username: The name of the user whose state should be retrieved
            block_key (UsageKey): The UsageKey identifying which xblock state to load.
            state (dict): A dictionary mapping field names to values
            scope (Scope): The scope to load data from
        """
        self.set_many(username, {block_key: state}, scope)

    def get_many(username, block_keys, scope=Scope.user_state, fields=None):
        """
        Retrieve the stored XBlock state for a single xblock usage.

        Arguments:
            username: The name of the user whose state should be retrieved
            block_keys ([UsageKey]): A list of UsageKeys identifying which xblock states to load.
            scope (Scope): The scope to load data from
            fields: A list of field values to retrieve. If None, retrieve all stored fields.

        Yields:
            (UsageKey, field_state) tuples for each specified UsageKey in block_keys.
            field_state is a dict mapping field names to values.
        """
        if scope != Scope.user_state:
            raise ValueError("Only Scope.user_state is supported")
        raise NotImplementedError()

    def set_many(username, block_keys_to_state, scope=Scope.user_state):
        """
        Set fields for a particular XBlock.

        Arguments:
            username: The name of the user whose state should be retrieved
            block_keys_to_state (dict): A dict mapping UsageKeys to state dicts.
                Each state dict maps field names to values. These state dicts
                are overlaid over the stored state. To delete fields, use
                :meth:`delete` or :meth:`delete_many`.
            scope (Scope): The scope to load data from
        """
        if scope != Scope.user_state:
            raise ValueError("Only Scope.user_state is supported")
        raise NotImplementedError()

    def get_history(username, block_key, scope=Scope.user_state):
        """We don't guarantee that history for many blocks will be fast."""
        if scope != Scope.user_state:
            raise ValueError("Only Scope.user_state is supported")
        raise NotImplementedError()

    def iter_all_for_block(block_key, scope=Scope.user_state, batch_size=None):
        """
        You get no ordering guarantees. Fetching will happen in batch_size
        increments. If you're using this method, you should be running in an
        async task.
        """
        if scope != Scope.user_state:
            raise ValueError("Only Scope.user_state is supported")
        raise NotImplementedError()

    def iter_all_for_course(course_key, block_type=None, scope=Scope.user_state, batch_size=None):
        """
        You get no ordering guarantees. Fetching will happen in batch_size
        increments. If you're using this method, you should be running in an
        async task.
        """
        if scope != Scope.user_state:
            raise ValueError("Only Scope.user_state is supported")
        raise NotImplementedError()
<|endoftext|>