<commit_before># Copyright (c) 2014, German Neuroinformatics Node (G-Node)
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted under the terms of the BSD License. See
# LICENSE file in the root of the Project.

import unittest

from nix import *

class TestDataArray(unittest.TestCase):

    def setUp(self):
        self.file  = File.open("unittest.h5", FileMode.Overwrite)
        self.block = self.file.create_block("test block", "recordingsession")
        self.array = self.block.create_data_array("test array", "signal")
        self.other = self.block.create_data_array("other array", "signal")

    def tearDown(self):
        del self.file.blocks[self.block.id]
        self.file.close()

    def test_data_array_eq(self):
        assert(self.array == self.array)
        assert(not self.array == self.other)
        assert(not self.array == None)

    def test_data_array_id(self):
        assert(self.array.id is not None)

    def test_data_array_name(self):
        def set_none():
            self.array.name = None

        assert(self.array.name is not None)
        self.assertRaises(Exception, set_none)

        self.array.name = "foo array"
        assert(self.array.name == "foo array")

    def test_data_array_type(self):
        def set_none():
            self.array.type = None

        assert(self.array.type is not None)
        self.assertRaises(Exception, set_none)

        self.array.type = "foo type"
        assert(self.array.type == "foo type")

    def test_data_array_definition(self):
        assert(self.array.definition is None)

        self.array.definition = "definition"
        assert(self.array.definition == "definition")

        self.array.definition = None
        assert(self.array.definition is None)

    def test_data_array_timestamps(self):
        created_at = self.array.created_at
        assert(created_at > 0)

        updated_at = self.array.updated_at
        assert(updated_at > 0)

        self.array.force_created_at(1403530068)
        assert(self.array.created_at == 1403530068)

    def test_data_array_label(self):
        assert(self.array.label is None)

        self.array.label = "label"
        assert(self.array.label == "label")

        self.array.label = None
        assert(self.array.label is None)

    def test_data_array_unit(self):
        assert(self.array.unit is None)

        self.array.unit = "mV"
        assert(self.array.unit == "mV")

        self.array.unit = None
        assert(self.array.unit is None)

    def test_data_array_exp_origin(self):
        assert(self.array.expansion_origin is None)

        self.array.expansion_origin = 10.2
        assert(self.array.expansion_origin == 10.2)

        self.array.expansion_origin = None
        assert(self.array.expansion_origin is None)

    def test_data_array_coefficients(self):
        assert(self.array.polynom_coefficients == ())

        self.array.polynom_coefficients = (1.1, 2.2)
        assert(self.array.polynom_coefficients == (1.1, 2.2))

        # TODO delete does not work

    def test_data_array_data(self):
        import numpy as np

        assert(self.array.polynom_coefficients == ())
        assert(not self.array.has_data())
        assert(self.array.data is None)

        data = np.array([float(i) for i in range(100)])
        dout = np.empty_like(data)
        self.array.create_data(data=data)
        assert(self.array.has_data())
        assert(self.array.data.dtype == np.dtype(float))
        self.array.data.read_direct(dout)
        assert(np.array_equal(data, dout))
        dout = np.array(self.array.data)
        assert(np.array_equal(data, dout))
        assert(self.array.data_extent == data.shape)
        assert(self.array.data_extent == self.array.data.shape)
        self.array.data_extent = (200, )
        assert(self.array.data_extent == (200, ))

        dout = np.array([self.array.data[i] for i in range(100)])
        assert(np.array_equal(data, dout))

        # TODO delete does not work
        data = np.eye(123)
        a1 = self.block.create_data_array("double array", "signal")
        self.assertRaises(ValueError, a1.create_data)
        dset = a1.create_data((123, 123))
        assert(a1.data_extent == (123, 123))
        dset.write_direct(data)
        dout = np.empty_like(data)
        dset.read_direct(dout)
        assert(np.array_equal(data, dout))

        dout = dset[12]
        assert(dout.shape == data[12].shape)
        assert(np.array_equal(dout, data[12]))

        a2 = self.block.create_data_array("identity array", "signal")
        self.assertRaises(ValueError, lambda : a1.create_data(data=data, shape=(1,1)))
        a2.create_data(data=data)
        assert(a2.data_extent == (123, 123))
        dout = np.empty_like(data)
        dset.read_direct(dout)
        assert(np.array_equal(data, dout))

        a3 = self.block.create_data_array("int identity array", "signal")
        a3.create_data(dtype='i4', data=data)
        assert(a3.data_extent == (123, 123))
        assert(a3.data.dtype == np.dtype('i4'))

    def test_data_array_dimensions(self):
        assert(len(self.array.dimensions) == 0)

        setd    = self.array.append_set_dimension()
        ranged  = self.array.append_range_dimension(range(10))
        sampled = self.array.append_sampled_dimension(0.1)

        assert(len(self.array.dimensions) == 3)

        self.assertRaises(TypeError, lambda : self.array.dimensions["notexist"])
        self.assertRaises(KeyError, lambda : self.array.dimensions[-4])
        self.assertRaises(KeyError, lambda : self.array.dimensions[3])

        assert(isinstance(str(self.array.dimensions), basestring))
        assert(isinstance(repr(self.array.dimensions), basestring))

        dims   = list(self.array.dimensions)
        for i in range(3):
            assert(dims[i].index == self.array.dimensions[i].index)
            assert(dims[i].dimension_type == self.array.dimensions[i].dimension_type)

            assert(self.array.dimensions[i].index == self.array.dimensions[i - 3].index)

        del self.array.dimensions[2]
        del self.array.dimensions[1]
        del self.array.dimensions[0]

        assert(len(self.array.dimensions) == 0)

    def test_data_array_sources(self):
        source1 = self.block.create_source("source1", "channel")
        source2 = self.block.create_source("source2", "electrode")

        assert(len(self.array.sources) == 0)

        self.array.sources.append(source1)
        self.array.sources.append(source2)

        self.assertRaises(TypeError, lambda : self.array.sources.append(100))

        assert(len(self.array.sources) == 2)
        assert(source1 in self.array.sources)
        assert(source2 in self.array.sources)

        del self.array.sources[source2]
        assert(self.array.sources[0] == source1)

        del self.array.sources[source1]
        assert(len(self.array.sources) == 0)
<commit_msg>[test] DataSet: Add support for single indexing<commit_after># Copyright (c) 2014, German Neuroinformatics Node (G-Node)
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted under the terms of the BSD License. See
# LICENSE file in the root of the Project.

import unittest

from nix import *

class TestDataArray(unittest.TestCase):

    def setUp(self):
        self.file  = File.open("unittest.h5", FileMode.Overwrite)
        self.block = self.file.create_block("test block", "recordingsession")
        self.array = self.block.create_data_array("test array", "signal")
        self.other = self.block.create_data_array("other array", "signal")

    def tearDown(self):
        del self.file.blocks[self.block.id]
        self.file.close()

    def test_data_array_eq(self):
        assert(self.array == self.array)
        assert(not self.array == self.other)
        assert(not self.array == None)

    def test_data_array_id(self):
        assert(self.array.id is not None)

    def test_data_array_name(self):
        def set_none():
            self.array.name = None

        assert(self.array.name is not None)
        self.assertRaises(Exception, set_none)

        self.array.name = "foo array"
        assert(self.array.name == "foo array")

    def test_data_array_type(self):
        def set_none():
            self.array.type = None

        assert(self.array.type is not None)
        self.assertRaises(Exception, set_none)

        self.array.type = "foo type"
        assert(self.array.type == "foo type")

    def test_data_array_definition(self):
        assert(self.array.definition is None)

        self.array.definition = "definition"
        assert(self.array.definition == "definition")

        self.array.definition = None
        assert(self.array.definition is None)

    def test_data_array_timestamps(self):
        created_at = self.array.created_at
        assert(created_at > 0)

        updated_at = self.array.updated_at
        assert(updated_at > 0)

        self.array.force_created_at(1403530068)
        assert(self.array.created_at == 1403530068)

    def test_data_array_label(self):
        assert(self.array.label is None)

        self.array.label = "label"
        assert(self.array.label == "label")

        self.array.label = None
        assert(self.array.label is None)

    def test_data_array_unit(self):
        assert(self.array.unit is None)

        self.array.unit = "mV"
        assert(self.array.unit == "mV")

        self.array.unit = None
        assert(self.array.unit is None)

    def test_data_array_exp_origin(self):
        assert(self.array.expansion_origin is None)

        self.array.expansion_origin = 10.2
        assert(self.array.expansion_origin == 10.2)

        self.array.expansion_origin = None
        assert(self.array.expansion_origin is None)

    def test_data_array_coefficients(self):
        assert(self.array.polynom_coefficients == ())

        self.array.polynom_coefficients = (1.1, 2.2)
        assert(self.array.polynom_coefficients == (1.1, 2.2))

        # TODO delete does not work

    def test_data_array_data(self):
        import numpy as np

        assert(self.array.polynom_coefficients == ())
        assert(not self.array.has_data())
        assert(self.array.data is None)

        data = np.array([float(i) for i in range(100)])
        dout = np.empty_like(data)
        self.array.create_data(data=data)
        assert(self.array.has_data())
        assert(self.array.data.dtype == np.dtype(float))
        self.array.data.read_direct(dout)
        assert(np.array_equal(data, dout))
        dout = np.array(self.array.data)
        assert(np.array_equal(data, dout))
        assert(self.array.data_extent == data.shape)
        assert(self.array.data_extent == self.array.data.shape)

        #indexing support in 1-d arrays
        self.assertRaises(ValueError, lambda : self.array.data[1:4:5])

        dout = np.array([self.array.data[i] for i in range(100)])
        assert(np.array_equal(data, dout))

        dout = self.array.data[...]
        assert(np.array_equal(data, dout))

        #changing shape via data_extent property
        self.array.data_extent = (200, )
        assert(self.array.data_extent == (200, ))

        # TODO delete does not work
        data = np.eye(123)
        a1 = self.block.create_data_array("double array", "signal")
        self.assertRaises(ValueError, a1.create_data)
        dset = a1.create_data((123, 123))
        assert(a1.data_extent == (123, 123))
        dset.write_direct(data)
        dout = np.empty_like(data)
        dset.read_direct(dout)
        assert(np.array_equal(data, dout))

        #indexing support in 2-d arrays
        dout = dset[12]
        assert(dout.shape == data[12].shape)
        assert(np.array_equal(dout, data[12]))
        dout = dset[...]
        assert(np.array_equal(dout, data))

        a2 = self.block.create_data_array("identity array", "signal")
        self.assertRaises(ValueError, lambda : a1.create_data(data=data, shape=(1,1)))
        a2.create_data(data=data)
        assert(a2.data_extent == (123, 123))
        dout = np.empty_like(data)
        dset.read_direct(dout)
        assert(np.array_equal(data, dout))

        a3 = self.block.create_data_array("int identity array", "signal")
        a3.create_data(dtype='i4', data=data)
        assert(a3.data_extent == (123, 123))
        assert(a3.data.dtype == np.dtype('i4'))

    def test_data_array_dimensions(self):
        assert(len(self.array.dimensions) == 0)

        setd    = self.array.append_set_dimension()
        ranged  = self.array.append_range_dimension(range(10))
        sampled = self.array.append_sampled_dimension(0.1)

        assert(len(self.array.dimensions) == 3)

        self.assertRaises(TypeError, lambda : self.array.dimensions["notexist"])
        self.assertRaises(KeyError, lambda : self.array.dimensions[-4])
        self.assertRaises(KeyError, lambda : self.array.dimensions[3])

        assert(isinstance(str(self.array.dimensions), basestring))
        assert(isinstance(repr(self.array.dimensions), basestring))

        dims   = list(self.array.dimensions)
        for i in range(3):
            assert(dims[i].index == self.array.dimensions[i].index)
            assert(dims[i].dimension_type == self.array.dimensions[i].dimension_type)

            assert(self.array.dimensions[i].index == self.array.dimensions[i - 3].index)

        del self.array.dimensions[2]
        del self.array.dimensions[1]
        del self.array.dimensions[0]

        assert(len(self.array.dimensions) == 0)

    def test_data_array_sources(self):
        source1 = self.block.create_source("source1", "channel")
        source2 = self.block.create_source("source2", "electrode")

        assert(len(self.array.sources) == 0)

        self.array.sources.append(source1)
        self.array.sources.append(source2)

        self.assertRaises(TypeError, lambda : self.array.sources.append(100))

        assert(len(self.array.sources) == 2)
        assert(source1 in self.array.sources)
        assert(source2 in self.array.sources)

        del self.array.sources[source2]
        assert(self.array.sources[0] == source1)

        del self.array.sources[source1]
        assert(len(self.array.sources) == 0)
<|endoftext|>


<commit_before>"""
Base settings for the CMS.

Individual sites may override these settings by creating their own settings
file.  It is recommended that you import these base settings into your custom
settings file using the following syntax:

>>> from cms.settings import *

An example site-specific settings file is included with this distribution.
Please see `example_site.settings` for more information.

For an explanation of these settings, please see the Django documentation at:

<http://docs.djangoproject.com/en/dev/>
"""

import os

# Absolute path to the directory containing the CMS.

CMS_ROOT = os.path.dirname(__file__)

# Absolute path to the directory containing the CMS media files.

STATICFILES_DIRS = (
    ("cms", os.path.join(CMS_ROOT, "media")),
)

# Template settings.

TEMPLATE_LOADERS = (
    "django.template.loaders.filesystem.Loader",
    "django.template.loaders.app_directories.Loader",
)

TEMPLATE_DIRS = (os.path.join(CMS_ROOT, "templates",),)

TEMPLATE_CONTEXT_PROCESSORS = (
    "django.contrib.auth.context_processors.auth",
    "django.core.context_processors.debug",
    "django.core.context_processors.i18n",
    "django.core.context_processors.media",
    "django.core.context_processors.static",
    "django.contrib.messages.context_processors.messages",
    "django.core.context_processors.request",
    "cms.core.context_processors.conf",
    "cms.core.context_processors.pages",
)

# Dispatch settings.

MIDDLEWARE_CLASSES = (
    "django.middleware.transaction.TransactionMiddleware",
    "django.middleware.common.CommonMiddleware",
    "django.contrib.sessions.middleware.SessionMiddleware",
    'django.middleware.csrf.CsrfViewMiddleware',
    "django.contrib.auth.middleware.AuthenticationMiddleware",
    "django.contrib.messages.middleware.MessageMiddleware",
    "cms.apps.historylinks.middleware.PermalinkFallbackMiddleware",
    "cms.core.middleware.PreviewMiddleware",
    "cms.core.middleware.PageMiddleware",
)

ROOT_URLCONF = "cms.urls"

# Application settings.

INSTALLED_APPS = (
    "django.contrib.contenttypes",
    "django.contrib.sessions",
    "django.contrib.auth",
    "django.contrib.admin",
    "django.contrib.sitemaps",
    "django.contrib.messages",
    "django.contrib.staticfiles",
    "reversion",
    "cms.core",
    "cms.apps.historylinks",
    "cms.apps.pages",
    "cms.apps.media",
)

PAGE_BACKEND = "cms.apps.pages.backend.PageBackend"

# Multi-site settings.

SITE_ID = 1

# Publication settings.

PUBLICATION_PREVIEW_KEY = "preview"

# SEO settings.

SEO_PRIORITIES = (
    (1.0, "Very high"),
    (0.8, "High"),
    (0.5, "Medium"),
    (0.3, "Low"),
    (0.0, "Very low"),
)

# Internationalization settings.

USE_I18N = False

USE_L10N = True

# Message settings.

MESSAGE_STORAGE = "django.contrib.messages.storage.session.SessionStorage"

# Staff management settings.

DEFAULT_GROUP_IDS = (1,)

# Logging settings.

LOGGING = {
    "version": 1,
    "disable_existing_loggers": False,
    "handlers": {
        "mail_admins": {
            "level": "ERROR",
            "class": "django.utils.log.AdminEmailHandler"
        }
    },
    "loggers": {
        "django.request": {
            "handlers": ["mail_admins"],
            "level": "ERROR",
            "propagate": True,
        },
    }
}<commit_msg>Reverted back to standard message backend. Transactions didn't work, for some reason.<commit_after>"""
Base settings for the CMS.

Individual sites may override these settings by creating their own settings
file.  It is recommended that you import these base settings into your custom
settings file using the following syntax:

>>> from cms.settings import *

An example site-specific settings file is included with this distribution.
Please see `example_site.settings` for more information.

For an explanation of these settings, please see the Django documentation at:

<http://docs.djangoproject.com/en/dev/>
"""

import os

# Absolute path to the directory containing the CMS.

CMS_ROOT = os.path.dirname(__file__)

# Absolute path to the directory containing the CMS media files.

STATICFILES_DIRS = (
    ("cms", os.path.join(CMS_ROOT, "media")),
)

# Template settings.

TEMPLATE_LOADERS = (
    "django.template.loaders.filesystem.Loader",
    "django.template.loaders.app_directories.Loader",
)

TEMPLATE_DIRS = (os.path.join(CMS_ROOT, "templates",),)

TEMPLATE_CONTEXT_PROCESSORS = (
    "django.contrib.auth.context_processors.auth",
    "django.core.context_processors.debug",
    "django.core.context_processors.i18n",
    "django.core.context_processors.media",
    "django.core.context_processors.static",
    "django.contrib.messages.context_processors.messages",
    "django.core.context_processors.request",
    "cms.core.context_processors.conf",
    "cms.core.context_processors.pages",
)

# Dispatch settings.

MIDDLEWARE_CLASSES = (
    "django.middleware.transaction.TransactionMiddleware",
    "django.middleware.common.CommonMiddleware",
    "django.contrib.sessions.middleware.SessionMiddleware",
    'django.middleware.csrf.CsrfViewMiddleware',
    "django.contrib.auth.middleware.AuthenticationMiddleware",
    "django.contrib.messages.middleware.MessageMiddleware",
    "cms.apps.historylinks.middleware.PermalinkFallbackMiddleware",
    "cms.core.middleware.PreviewMiddleware",
    "cms.core.middleware.PageMiddleware",
)

ROOT_URLCONF = "cms.urls"

# Application settings.

INSTALLED_APPS = (
    "django.contrib.contenttypes",
    "django.contrib.sessions",
    "django.contrib.auth",
    "django.contrib.admin",
    "django.contrib.sitemaps",
    "django.contrib.messages",
    "django.contrib.staticfiles",
    "reversion",
    "cms.core",
    "cms.apps.historylinks",
    "cms.apps.pages",
    "cms.apps.media",
)

PAGE_BACKEND = "cms.apps.pages.backend.PageBackend"

# Multi-site settings.

SITE_ID = 1

# Publication settings.

PUBLICATION_PREVIEW_KEY = "preview"

# SEO settings.

SEO_PRIORITIES = (
    (1.0, "Very high"),
    (0.8, "High"),
    (0.5, "Medium"),
    (0.3, "Low"),
    (0.0, "Very low"),
)

# Internationalization settings.

USE_I18N = False

USE_L10N = True

# Staff management settings.

DEFAULT_GROUP_IDS = (1,)

# Logging settings.

LOGGING = {
    "version": 1,
    "disable_existing_loggers": False,
    "handlers": {
        "mail_admins": {
            "level": "ERROR",
            "class": "django.utils.log.AdminEmailHandler"
        }
    },
    "loggers": {
        "django.request": {
            "handlers": ["mail_admins"],
            "level": "ERROR",
            "propagate": True,
        },
    }
}<|endoftext|>


<commit_before>
import testbase
import unittest, sys, datetime
import tables
db = testbase.db
from sqlalchemy import *

class ExecuteTest(testbase.PersistTest):
    def setUpAll(self):
        global users, metadata
        metadata = BoundMetaData(testbase.db)
        users = Table('users', metadata,
            Column('user_id', INT, primary_key = True),
            Column('user_name', VARCHAR(20)),
            mysql_engine='InnoDB'
        )
        metadata.create_all()

    def tearDown(self):
        testbase.db.connect().execute(users.delete())
    def tearDownAll(self):
        metadata.drop_all()

    @testbase.supported('sqlite')
    def test_raw_qmark(self):
        for conn in (testbase.db, testbase.db.connect()):
            conn.execute("insert into users (user_id, user_name) values (?, ?)", (1,"jack"))
            conn.execute("insert into users (user_id, user_name) values (?, ?)", [2,"fred"])
            conn.execute("insert into users (user_id, user_name) values (?, ?)", [3,"ed"], [4,"horse"])
            conn.execute("insert into users (user_id, user_name) values (?, ?)", (5,"barney"), (6,"donkey"))
            conn.execute("insert into users (user_id, user_name) values (?, ?)", 7, 'sally')
            res = conn.execute("select * from users")
            assert res.fetchall() == [(1, "jack"), (2, "fred"), (3, "ed"), (4, "horse"), (5, "barney"), (6, "donkey"), (7, 'sally')]
            conn.execute("delete from users")

    @testbase.supported('mysql')
    def test_raw_sprintf(self):
        for conn in (testbase.db, testbase.db.connect()):
            conn.execute("insert into users (user_id, user_name) values (%s, %s)", [1,"jack"])
            conn.execute("insert into users (user_id, user_name) values (%s, %s)", [2,"ed"], [3,"horse"])
            conn.execute("insert into users (user_id, user_name) values (%s, %s)", 4, 'sally')
            res = conn.execute("select * from users")
            assert res.fetchall() == [(1, "jack"), (2, "ed"), (3, "horse"), (4, 'sally')]
            conn.execute("delete from users")

    @testbase.supported('postgres')
    def test_raw_python(self):
        for conn in (testbase.db, testbase.db.connect()):
            conn.execute("insert into users (user_id, user_name) values (%(id)s, %(name)s)", {'id':1, 'name':'jack'})
            conn.execute("insert into users (user_id, user_name) values (%(id)s, %(name)s)", {'id':2, 'name':'ed'}, {'id':3, 'name':'horse'})
            conn.execute("insert into users (user_id, user_name) values (%(id)s, %(name)s)", id=4, name='sally')
            res = conn.execute("select * from users")
            assert res.fetchall() == [(1, "jack"), (2, "ed"), (3, "horse"), (4, 'sally')]
            conn.execute("delete from users")

if __name__ == "__main__":
    testbase.main()
<commit_msg>unit test with just one param<commit_after>
import testbase
import unittest, sys, datetime
import tables
db = testbase.db
from sqlalchemy import *

class ExecuteTest(testbase.PersistTest):
    def setUpAll(self):
        global users, metadata
        metadata = BoundMetaData(testbase.db)
        users = Table('users', metadata,
            Column('user_id', INT, primary_key = True),
            Column('user_name', VARCHAR(20)),
            mysql_engine='InnoDB'
        )
        metadata.create_all()

    def tearDown(self):
        testbase.db.connect().execute(users.delete())
    def tearDownAll(self):
        metadata.drop_all()

    @testbase.supported('sqlite')
    def test_raw_qmark(self):
        for conn in (testbase.db, testbase.db.connect()):
            conn.execute("insert into users (user_id, user_name) values (?, ?)", (1,"jack"))
            conn.execute("insert into users (user_id, user_name) values (?, ?)", [2,"fred"])
            conn.execute("insert into users (user_id, user_name) values (?, ?)", [3,"ed"], [4,"horse"])
            conn.execute("insert into users (user_id, user_name) values (?, ?)", (5,"barney"), (6,"donkey"))
            conn.execute("insert into users (user_id, user_name) values (?, ?)", 7, 'sally')
            res = conn.execute("select * from users")
            assert res.fetchall() == [(1, "jack"), (2, "fred"), (3, "ed"), (4, "horse"), (5, "barney"), (6, "donkey"), (7, 'sally')]
            conn.execute("delete from users")

    @testbase.supported('mysql')
    def test_raw_sprintf(self):
        for conn in (testbase.db, testbase.db.connect()):
            conn.execute("insert into users (user_id, user_name) values (%s, %s)", [1,"jack"])
            conn.execute("insert into users (user_id, user_name) values (%s, %s)", [2,"ed"], [3,"horse"])
            conn.execute("insert into users (user_id, user_name) values (%s, %s)", 4, 'sally')
            conn.execute("insert into users (user_id) values (%s)", 5)
            res = conn.execute("select * from users")
            assert res.fetchall() == [(1, "jack"), (2, "ed"), (3, "horse"), (4, 'sally'), (5, None)]
            conn.execute("delete from users")

    @testbase.supported('postgres')
    def test_raw_python(self):
        for conn in (testbase.db, testbase.db.connect()):
            conn.execute("insert into users (user_id, user_name) values (%(id)s, %(name)s)", {'id':1, 'name':'jack'})
            conn.execute("insert into users (user_id, user_name) values (%(id)s, %(name)s)", {'id':2, 'name':'ed'}, {'id':3, 'name':'horse'})
            conn.execute("insert into users (user_id, user_name) values (%(id)s, %(name)s)", id=4, name='sally')
            res = conn.execute("select * from users")
            assert res.fetchall() == [(1, "jack"), (2, "ed"), (3, "horse"), (4, 'sally')]
            conn.execute("delete from users")

if __name__ == "__main__":
    testbase.main()
<|endoftext|>


<commit_before>import gym,time,copy,re
import numpy as np
import tensorflow as tf
from matplotlib import pyplot as plt
import cv2
import matplotlib
from Replay_Memory import Replay_Memory

tf.logging.set_verbosity(tf.logging.INFO)

MEMORY_LENGTH = 4
ACTIONS = 4
LEARNING_RATE_SGD = 0.000025
LEARNING_RATE_RMSPROP = 0.00025
FINAL_EXPLORATION_FRAME = 1000000
TRAINING_STEPS = 10000000
DISCOUNT_RATE = 0.99
RMSPROP_MOMENTUM = 0.95
RMSPROP_DECAY = 0.95
RMSPROP_EPSILON = 0.01
MINIBATCH_SIZE = 32
REPLAY_MEMORY_SIZE = 15000
RANDOM_STEPS_REPLAY_MEMORY_INIT = 15000
ENVIRONMENT = 'Breakout-v0'
NO_OP_CODE = 1
TF_RANDOM_SEED = 7
type = np.dtype(np.float32)

env = gym.make(ENVIRONMENT)
env.reset()
memory = Replay_Memory(REPLAY_MEMORY_SIZE)

def rgb2gray(rgb):
    return np.dot(rgb[..., :3], [0.299, 0.587, 0.114])

def downSample(image):
    return cv2.resize(image, (84, 84), interpolation=cv2.INTER_LINEAR)

def stack(frames):
    return np.stack(frames,2)

def randomSteps(steps=RANDOM_STEPS_REPLAY_MEMORY_INIT,initial_no_ops=4):
    t0 = time.time()
    env.reset()
    i = 0
    frame_stack = []

    for _ in range(0,steps):
        if i < initial_no_ops:
            action = NO_OP_CODE
            observation, reward, done, info = env.step(action)
            greyObservation = rgb2gray(observation)
            downObservation = downSample(greyObservation)
            frame_stack.append(downObservation)
            i+=1

        else:
            s_t = stack(frame_stack)
            action = env.action_space.sample()
            observation, reward, done, info = env.step(action)

            greyObservation = rgb2gray(observation)
            downObservation = downSample(greyObservation)

            frame_stack.pop(0)
            frame_stack.append(downObservation)
            s_t_plus1 = stack(frame_stack)

            if done:
                memory.store_transition(
                    (
                    s_t.astype(type),
                    action,
                    reward,
                    None,
                    True
                )
                )

            else:
                memory.store_transition(
                    (
                        s_t.astype(type),
                        action,
                        reward,
                        s_t_plus1.astype(type),
                        False
                    )
                )

        if done:
            #print("Episode finished after {} timesteps".format(_ + 1))
            env.reset()
            i=0
            frame_stack=[]

    t1 = time.time()
    print("This operation took:",t1-t0,)

    #Sample random transitions and show
    #for _ in range(5):
        #t = np.split(memory.sample_transition()[0],4,2)
        #plt.imshow(t[0].reshape(84, 84), cmap=matplotlib.cm.Greys_r)
        #plt.show()

def getEpsilon(step):
    if step > FINAL_EXPLORATION_FRAME:
        return 0.1
    else:
        return 1 - step*(0.9/FINAL_EXPLORATION_FRAME)

input_tensor = tf.placeholder(tf.float32,(None,84,84,4),name="input")
actions_tensor = tf.placeholder(tf.int32, [None], name="actions")
y_tensor = tf.placeholder(tf.float32, [None], name="r")

def model():
    #Placeholders could be here
    conv_1 = tf.contrib.layers.conv2d(input_tensor,num_outputs=32,kernel_size=[8,8],stride=[4,4],padding='SAME')
    conv_2 = tf.contrib.layers.conv2d(conv_1,num_outputs=64,kernel_size=[4,4],stride=[2,2],padding='SAME')
    conv_3 = tf.contrib.layers.conv2d(conv_2, num_outputs=64, kernel_size=[3,3],stride=[1,1],padding='SAME')
    conv_3_flat = tf.reshape(conv_3,[-1,11*11*64])
    relu_1 = tf.contrib.layers.relu(conv_3_flat, num_outputs=512)
    output = tf.contrib.layers.fully_connected(relu_1,num_outputs=ACTIONS)

    actions_one_hot = tf.one_hot(actions_tensor, ACTIONS, name="actions_one_hot")
    apply_action_mask = tf.multiply(output,actions_one_hot)
    Q_of_selected_action = tf.reduce_sum(apply_action_mask)

    loss = tf.square(tf.subtract(Q_of_selected_action, y_tensor))

    cost = tf.reduce_mean(loss)
    #optimizer = tf.train.GradientDescentOptimizer(learning_rate=LEARNING_RATE_SGD).minimize(cost)
    optimizer = tf.train.RMSPropOptimizer(learning_rate=LEARNING_RATE_RMSPROP,momentum=RMSPROP_MOMENTUM,epsilon=RMSPROP_EPSILON
                                          ,decay=RMSPROP_DECAY).minimize(cost)

    return output,optimizer,cost

def train():
    with tf.Session() as sess:
        tf.set_random_seed(TF_RANDOM_SEED)
        output, optimizer, X = model()

        saver = tf.train.Saver()
        checkpoint = tf.train.get_checkpoint_state("saved_networks")
        if checkpoint and checkpoint.model_checkpoint_path:
            saver.restore(sess, checkpoint.model_checkpoint_path)
            print("Successfully loaded:", checkpoint.model_checkpoint_path)
            game = int(re.match('.*?([0-9]+)$', checkpoint.model_checkpoint_path).group(1))
        else:
            print("Could not find old network weights")

        sess.run(tf.global_variables_initializer())
        frames = np.zeros((MINIBATCH_SIZE, 84, 84, 4), np.float32)
        score = 0
        game_scores = []
        i = 0
        frame_stack = []
        initial_no_op = np.random.randint(4,50)
        game = 1
        for step in range(TRAINING_STEPS):
            if i < initial_no_op:
                # WE PERFORM A RANDOM NUMBER OF NO_OP ACTIONS
                action = NO_OP_CODE
                observation, reward, done, info = env.step(action)
                greyObservation = rgb2gray(observation)
                downObservation = downSample(greyObservation)
                if i > 3:
                    frame_stack.pop(0)
                frame_stack.append(downObservation)
                i += 1

            else:
                # CHOOSING ACTION
                s_t = stack(frame_stack)
                if np.random.rand() < getEpsilon(step):
                    # We make random action with probability epsilon
                    action = env.action_space.sample()
                else:
                    # Pick action in a greedy way
                    action = np.argmax(sess.run([output],{input_tensor:np.array(s_t, ndmin=4)}))

                # STORE TRANSITION

                observation, reward, done, info = env.step(action)

                score += reward

                #Process received frame
                greyObservation = rgb2gray(observation)
                downObservation = downSample(greyObservation)

                #Remove oldest frame
                frame_stack.pop(0)
                frame_stack.append(downObservation)

                #Obtain state at t+1
                s_t_plus1 = stack(frame_stack)

                if done:
                    memory.store_transition(
                        (
                            s_t.astype(type),
                            action,
                            reward,
                            None,
                            True
                        )
                    )

                else:
                    memory.store_transition(
                        (
                            s_t.astype(type),
                            action,
                            reward,
                            s_t_plus1.astype(type),
                            False
                        )
                    )

                # OBTAIN MINIBATCH
                actions = []
                y = []

                for i in range(0, MINIBATCH_SIZE):
                    t = memory.sample_transition()
                    frames[i]=t[0]
                    actions.append(t[1])
                    if t[-1]:
                        y.append(t[2])
                    else:
                        y.append(t[2]+DISCOUNT_RATE * np.max(sess.run([output], {input_tensor: np.array(t[3], ndmin=4)})))

                sess.run([optimizer],{input_tensor:frames,actions_tensor:np.array(actions),y_tensor:np.array(y)})

                #print(y)
                #print(sess.run([X], {input_tensor: frames, actions_tensor: np.array(actions), y_tensor: np.array(y)}))

                if done:
                    frame_stack = []
                    game += 1
                    #print("At step ",step," we have finished game ",game," with score:",score)
                    game_scores.append(score)
                    score = 0
                    if game % 1000 == 0:
                        saver.save(sess, 'saved_networks/' + ENVIRONMENT + '-dqn', global_step=game)
                        print('Network backup done')
                    if (game % 20) == 0:
                        print("The average score of the last 20 games is:", np.mean(game_scores[-20:]),
                              " currently at game ", game, " , step ", step)
                        print("The average score of all games is:", np.mean(game_scores))

                    env.reset()
                    initial_no_op = np.random.randint(4, 50)
                    i=0

randomSteps()
train()
<commit_msg>DQN add NO_OP_MAX<commit_after>import gym,time,copy,re
import numpy as np
import tensorflow as tf
from matplotlib import pyplot as plt
import cv2
import matplotlib
from Replay_Memory import Replay_Memory

tf.logging.set_verbosity(tf.logging.INFO)

MEMORY_LENGTH = 4
ACTIONS = 4
LEARNING_RATE_SGD = 0.000025
LEARNING_RATE_RMSPROP = 0.00025
FINAL_EXPLORATION_FRAME = 1000000
TRAINING_STEPS = 10000000
DISCOUNT_RATE = 0.99
RMSPROP_MOMENTUM = 0.95
RMSPROP_DECAY = 0.95
RMSPROP_EPSILON = 0.01
MINIBATCH_SIZE = 32
REPLAY_MEMORY_SIZE = 15000
RANDOM_STEPS_REPLAY_MEMORY_INIT = 15000
NO_OP_MAX = 30
ENVIRONMENT = 'Breakout-v0'
NO_OP_CODE = 1
TF_RANDOM_SEED = 7
type = np.dtype(np.float32)

env = gym.make(ENVIRONMENT)
env.reset()
memory = Replay_Memory(REPLAY_MEMORY_SIZE)

def rgb2gray(rgb):
    return np.dot(rgb[..., :3], [0.299, 0.587, 0.114])

def downSample(image):
    return cv2.resize(image, (84, 84), interpolation=cv2.INTER_LINEAR)

def stack(frames):
    return np.stack(frames,2)

def randomSteps(steps=RANDOM_STEPS_REPLAY_MEMORY_INIT,initial_no_ops=4):
    t0 = time.time()
    env.reset()
    i = 0
    frame_stack = []

    for _ in range(0,steps):
        if i < initial_no_ops:
            action = NO_OP_CODE
            observation, reward, done, info = env.step(action)
            greyObservation = rgb2gray(observation)
            downObservation = downSample(greyObservation)
            frame_stack.append(downObservation)
            i+=1

        else:
            s_t = stack(frame_stack)
            action = env.action_space.sample()
            observation, reward, done, info = env.step(action)

            greyObservation = rgb2gray(observation)
            downObservation = downSample(greyObservation)

            frame_stack.pop(0)
            frame_stack.append(downObservation)
            s_t_plus1 = stack(frame_stack)

            if done:
                memory.store_transition(
                    (
                    s_t.astype(type),
                    action,
                    reward,
                    None,
                    True
                )
                )

            else:
                memory.store_transition(
                    (
                        s_t.astype(type),
                        action,
                        reward,
                        s_t_plus1.astype(type),
                        False
                    )
                )

        if done:
            #print("Episode finished after {} timesteps".format(_ + 1))
            env.reset()
            i=0
            frame_stack=[]

    t1 = time.time()
    print("This operation took:",t1-t0,)

    #Sample random transitions and show
    #for _ in range(5):
        #t = np.split(memory.sample_transition()[0],4,2)
        #plt.imshow(t[0].reshape(84, 84), cmap=matplotlib.cm.Greys_r)
        #plt.show()

def getEpsilon(step):
    if step > FINAL_EXPLORATION_FRAME:
        return 0.1
    else:
        return 1 - step*(0.9/FINAL_EXPLORATION_FRAME)

input_tensor = tf.placeholder(tf.float32,(None,84,84,4),name="input")
actions_tensor = tf.placeholder(tf.int32, [None], name="actions")
y_tensor = tf.placeholder(tf.float32, [None], name="r")

def model():
    #Placeholders could be here
    conv_1 = tf.contrib.layers.conv2d(input_tensor,num_outputs=32,kernel_size=[8,8],stride=[4,4],padding='SAME')
    conv_2 = tf.contrib.layers.conv2d(conv_1,num_outputs=64,kernel_size=[4,4],stride=[2,2],padding='SAME')
    conv_3 = tf.contrib.layers.conv2d(conv_2, num_outputs=64, kernel_size=[3,3],stride=[1,1],padding='SAME')
    conv_3_flat = tf.reshape(conv_3,[-1,11*11*64])
    relu_1 = tf.contrib.layers.relu(conv_3_flat, num_outputs=512)
    output = tf.contrib.layers.fully_connected(relu_1,num_outputs=ACTIONS)

    actions_one_hot = tf.one_hot(actions_tensor, ACTIONS, name="actions_one_hot")
    apply_action_mask = tf.multiply(output,actions_one_hot)
    Q_of_selected_action = tf.reduce_sum(apply_action_mask)

    loss = tf.square(tf.subtract(Q_of_selected_action, y_tensor))

    cost = tf.reduce_mean(loss)
    #optimizer = tf.train.GradientDescentOptimizer(learning_rate=LEARNING_RATE_SGD).minimize(cost)
    optimizer = tf.train.RMSPropOptimizer(learning_rate=LEARNING_RATE_RMSPROP,momentum=RMSPROP_MOMENTUM,epsilon=RMSPROP_EPSILON
                                          ,decay=RMSPROP_DECAY).minimize(cost)

    return output,optimizer,cost

def train():
    with tf.Session() as sess:
        tf.set_random_seed(TF_RANDOM_SEED)
        output, optimizer, X = model()

        saver = tf.train.Saver()
        checkpoint = tf.train.get_checkpoint_state("saved_networks")
        if checkpoint and checkpoint.model_checkpoint_path:
            saver.restore(sess, checkpoint.model_checkpoint_path)
            print("Successfully loaded:", checkpoint.model_checkpoint_path)
            game = int(re.match('.*?([0-9]+)$', checkpoint.model_checkpoint_path).group(1))
        else:
            print("Could not find old network weights")

        sess.run(tf.global_variables_initializer())
        frames = np.zeros((MINIBATCH_SIZE, 84, 84, 4), np.float32)
        score = 0
        game_scores = []
        i = 0
        frame_stack = []
        initial_no_op = np.random.randint(4,NO_OP_MAX)
        game = 1
        for step in range(TRAINING_STEPS):
            if i < initial_no_op:
                # WE PERFORM A RANDOM NUMBER OF NO_OP ACTIONS
                action = NO_OP_CODE
                observation, reward, done, info = env.step(action)
                greyObservation = rgb2gray(observation)
                downObservation = downSample(greyObservation)
                if i > 3:
                    frame_stack.pop(0)
                frame_stack.append(downObservation)
                i += 1

            else:
                # CHOOSING ACTION
                s_t = stack(frame_stack)
                if np.random.rand() < getEpsilon(step):
                    # We make random action with probability epsilon
                    action = env.action_space.sample()
                else:
                    # Pick action in a greedy way
                    action = np.argmax(sess.run([output],{input_tensor:np.array(s_t, ndmin=4)}))

                # STORE TRANSITION

                observation, reward, done, info = env.step(action)

                score += reward

                #Process received frame
                greyObservation = rgb2gray(observation)
                downObservation = downSample(greyObservation)

                #Remove oldest frame
                frame_stack.pop(0)
                frame_stack.append(downObservation)

                #Obtain state at t+1
                s_t_plus1 = stack(frame_stack)

                if done:
                    memory.store_transition(
                        (
                            s_t.astype(type),
                            action,
                            reward,
                            None,
                            True
                        )
                    )

                else:
                    memory.store_transition(
                        (
                            s_t.astype(type),
                            action,
                            reward,
                            s_t_plus1.astype(type),
                            False
                        )
                    )

                # OBTAIN MINIBATCH
                actions = []
                y = []

                for i in range(0, MINIBATCH_SIZE):
                    t = memory.sample_transition()
                    frames[i]=t[0]
                    actions.append(t[1])
                    if t[-1]:
                        y.append(t[2])
                    else:
                        y.append(t[2]+DISCOUNT_RATE * np.max(sess.run([output], {input_tensor: np.array(t[3], ndmin=4)})))

                sess.run([optimizer],{input_tensor:frames,actions_tensor:np.array(actions),y_tensor:np.array(y)})

                #print(y)
                #print(sess.run([X], {input_tensor: frames, actions_tensor: np.array(actions), y_tensor: np.array(y)}))

                if done:
                    frame_stack = []
                    game += 1
                    #print("At step ",step," we have finished game ",game," with score:",score)
                    game_scores.append(score)
                    score = 0
                    if game % 1000 == 0:
                        saver.save(sess, 'saved_networks/' + ENVIRONMENT + '-dqn', global_step=game)
                        print('Network backup done')
                    if (game % 20) == 0:
                        print("The average score of the last 20 games is:", np.mean(game_scores[-20:]),
                              " currently at game ", game, " , step ", step)
                        print("The average score of all games is:", np.mean(game_scores))

                    env.reset()
                    initial_no_op = np.random.randint(4, 50)
                    i=0

randomSteps()
train()
<|endoftext|>


<commit_before>from collections import OrderedDict

class Roman(object):

  def __init__(self, number):
    self.number = int(number)
    self.convert_table = self.make_table()

    while True:
      choice = raw_input("Type Yes or No for modern Roman Numeral Convert: ").lower()
      if choice == "yes":
        print "converted number = {0}".format(self.modern_roman_convert())
        print
        break
      elif choice == "no":
        print "converted number = {0}".format(self.old_roman_convert())
        print
        break
      else:
        print "Please Type Yes or No!"

    while True:
      print "top"
      play_again = raw_input("Do you want to enter another number? Please type yes or no: ").lower()
      if play_again == "no":
        print "thanks for playing"
        break
      elif play_again == "yes":
        Roman(raw_input("Enter another number! "))
      else:
        print "Please Enter Yes or No!"

  def make_table(self):

    number = self.number
    convert_table = OrderedDict()
    while True:
      if number >= 1000:
        if "M" in convert_table:
          convert_table["M"] += 1
        else:
          convert_table["M"] = 1
        number -= 1000
      elif number >= 500:
        if "D" in convert_table:
          convert_table["D"] += 1
        else:
          convert_table["D"] = 1
        number -= 500
      elif number >= 100:
        if "C" in convert_table:
          convert_table["C"] += 1
        else:
          convert_table["C"] = 1
        number -= 100
      elif number >=50:
        if "L" in convert_table:
          convert_table["L"] += 1
        else:
          convert_table["L"] = 1
        number -= 50
      elif number >= 10:
        if "X" in convert_table:
          convert_table["X"] += 1
        else:
          convert_table["X"] = 1
        number -= 10
      elif number >= 5:
        if "V" in convert_table:
          convert_table["V"] += 1
        else:
          convert_table["V"] = 1
        number -= 5
      elif number >= 1:
        if "I" in convert_table:
          convert_table["I"] += 1
        else:
          convert_table["I"] = 1
        number -= 1
      else:
        break
    return convert_table

  def old_roman_convert(self):
    solution = []
    for key,value in self.convert_table.items():
      for i in range(0,value):
        solution.append(key)
    return ''.join(solution)

  def modern_roman_convert(self):
    solution = []
    reversed_list = OrderedDict(reversed(list(self.convert_table.items())))
    for key,value in reversed_list.items():

      if value < 4 and key == "I":
        for i in range(0,value):
          solution.append(key)
        continue
      elif value == 4 and key == "I":
        if "V" in reversed_list:
          solution.extend(("I","X"))
        else:
          solution.extend(("I","V"))
        continue

      if "V" == key and "V" not in solution and "X" not in solution:
        solution.append("V")
        continue

      if value < 4 and key == "X" :
        for i in range(0,value):
          solution.append(key)
        continue
      elif value == 4 and key == "X":
        if "L" in reversed_list:
          solution.extend(("C","X"))
        else:
          solution.extend(("L","X"))
        continue

      if "L" == key and "L" not in solution and "C" not in solution:
        solution.append("L")
        continue

      if value < 4 and key == "C" :
        for i in range(0,value):
          solution.append(key)
        continue
      elif value == 4 and key == "C":
        if "D" in reversed_list:
          solution.extend(("M","C"))
        else:
          solution.extend(("D","C"))
        continue

      if "D" == key and "M" not in solution and "D" not in solution:
        solution.append("D")
        continue
      elif "M" in reversed_list:
        solution.append("M")
        continue
    return ''.join(solution[::-1])

number = Roman(raw_input("Enter a number to be converted into Roman Numberal Form: "))

<commit_msg>Fix problem with a tail end on recursion<commit_after>from collections import OrderedDict

class Roman(object):

  def __init__(self, number):
    self.number = int(number)
    self.convert_table = self.make_table()

    while True:
      choice = raw_input("Type Yes or No for modern Roman Numeral Convert: ").lower()
      if choice == "yes":
        print "converted number = {0}".format(self.modern_roman_convert())
        print
        break
      elif choice == "no":
        print "converted number = {0}".format(self.old_roman_convert())
        print
        break
      else:
        print "Please Type Yes or No!"

    while True:
      print "top"
      play_again = raw_input("Do you want to enter another number? Please type yes or no: ").lower()
      if play_again == "no":
        print "thanks for playing"
        break
      elif play_again == "yes":
        Roman(raw_input("Enter another number! "))
        break
      else:
        print "Please Enter Yes or No!"

  def make_table(self):

    number = self.number
    convert_table = OrderedDict()
    while True:
      if number >= 1000:
        if "M" in convert_table:
          convert_table["M"] += 1
        else:
          convert_table["M"] = 1
        number -= 1000
      elif number >= 500:
        if "D" in convert_table:
          convert_table["D"] += 1
        else:
          convert_table["D"] = 1
        number -= 500
      elif number >= 100:
        if "C" in convert_table:
          convert_table["C"] += 1
        else:
          convert_table["C"] = 1
        number -= 100
      elif number >=50:
        if "L" in convert_table:
          convert_table["L"] += 1
        else:
          convert_table["L"] = 1
        number -= 50
      elif number >= 10:
        if "X" in convert_table:
          convert_table["X"] += 1
        else:
          convert_table["X"] = 1
        number -= 10
      elif number >= 5:
        if "V" in convert_table:
          convert_table["V"] += 1
        else:
          convert_table["V"] = 1
        number -= 5
      elif number >= 1:
        if "I" in convert_table:
          convert_table["I"] += 1
        else:
          convert_table["I"] = 1
        number -= 1
      else:
        break
    return convert_table

  def old_roman_convert(self):
    solution = []
    for key,value in self.convert_table.items():
      for i in range(0,value):
        solution.append(key)
    return ''.join(solution)

  def modern_roman_convert(self):
    solution = []
    reversed_list = OrderedDict(reversed(list(self.convert_table.items())))
    for key,value in reversed_list.items():

      if value < 4 and key == "I":
        for i in range(0,value):
          solution.append(key)
        continue
      elif value == 4 and key == "I":
        if "V" in reversed_list:
          solution.extend(("I","X"))
        else:
          solution.extend(("I","V"))
        continue

      if "V" == key and "V" not in solution and "X" not in solution:
        solution.append("V")
        continue

      if value < 4 and key == "X" :
        for i in range(0,value):
          solution.append(key)
        continue
      elif value == 4 and key == "X":
        if "L" in reversed_list:
          solution.extend(("C","X"))
        else:
          solution.extend(("L","X"))
        continue

      if "L" == key and "L" not in solution and "C" not in solution:
        solution.append("L")
        continue

      if value < 4 and key == "C" :
        for i in range(0,value):
          solution.append(key)
        continue
      elif value == 4 and key == "C":
        if "D" in reversed_list:
          solution.extend(("M","C"))
        else:
          solution.extend(("D","C"))
        continue

      if "D" == key and "M" not in solution and "D" not in solution:
        solution.append("D")
        continue
      elif "M" in reversed_list:
        solution.append("M")
        continue
    return ''.join(solution[::-1])

number = Roman(raw_input("Enter a number to be converted into Roman Numberal Form: "))

<|endoftext|>


<commit_before>import os
import pymel as pm
import oyAuxiliaryFunctions as oyAux
from oyProjectManager.dataModels import assetModel, projectModel

__version__ = "9.11.17"

#----------------------------------------------------------------------
def save( assetObject ):
    """the save action for maya environment

    uses PyMel to save the file (not necessary but comfortable )
    """

    # set the extension to ma by default
    assetObject.setExtension( 'ma' )

    # set the project to the current environment
    pm.workspace.open( assetObject.getSequenceFullPath() )

    # set the render file name and version
    setRenderFileName( assetObject )

    # set the playblast file name
    setPlayblastFileName( assetObject )

    # create the folder if it doesn't exists
    oyAux.createFolder( assetObject.getPath() )

    # save the file
    pm.saveAs( assetObject.getFullPath(), type='mayaAscii' )

    # append it to the recent file list
    appendToRecentFiles( assetObject.getFullPath() )

    return True

#----------------------------------------------------------------------
def export( assetObject ):
    """the export action for maya environment
    """

    # check if there is something selected
    if len(pm.ls(sl=True)) < 1:
        print "selection error"
        return False

    # set the extension to ma by default
    assetObject.setExtension( 'ma' )

    # create the folder if it doesn't exists
    oyAux.createFolder( assetObject.getPath() )

    # export the file
    pm.exportSelected( assetObject.getFullPath(), type='mayaAscii' )

    return True

#----------------------------------------------------------------------
def open_( assetObject, force=False):
    """the open action for maya environment
    """
    #assert(isinstance(assetObject, assetModel.Asset ) )

    # check for unsaved changes
    assetFullPath = assetObject.getFullPath()

    pm.openFile( assetFullPath, f=force )

    # set the project
    pm.workspace.open( assetObject.getSequenceFullPath() )

    # set the playblast folder
    setPlayblastFileName( assetObject )

    appendToRecentFiles( assetFullPath )

    return True

#----------------------------------------------------------------------
def import_( assetObject ):
    """the import action for maya environment
    """
    #assert( isinstance(assetObject, assetModel.Asset ) )
    pm.importFile( assetObject.getFullPath() )

    return True

#----------------------------------------------------------------------
def reference( assetObject ):
    """the reference action for maya environment
    """

    assert( isinstance(assetObject, assetModel.Asset ) )

    # use the file name without extension as the namespace
    nameSpace = assetObject.getFileNameWithoutExtension()

    pm.createReference( assetObject.getFullPath(), gl=True, loadReferenceDepth='all', namespace=nameSpace, options='v=0')

    return True

#----------------------------------------------------------------------
def getPathVariables():
    """gets the file name from maya environment
    """

    fileName = path = None

    fullPath = pm.env.sceneName()

    print "the fullPath in maya is ", fullPath

    readRecentFile = False

    if fullPath != '':
        fileName = os.path.basename( fullPath )

        # try to create an asset with that info
        db = projectModel.Database()
        projName, seqName = db.getProjectAndSequenceNameFromFilePath( fullPath )

        proj = projectModel.Project( projName )
        seq = projectModel.Sequence( proj, seqName )

        testAsset = assetModel.Asset( proj, seq, fileName )

        if testAsset.isValidAsset():
            fileName = testAsset.getFileName()
            path = testAsset.getPath()
        else:
            readRecentFile = True

    if readRecentFile:
        # read the fileName from recent files list
        # try to get the a valid asset file from starting the last recent file

        recentFiles = pm.optionVar['RecentFilesList']
        for i in range(len(recentFiles)-1, -1,-1):

            fileName = os.path.basename( recentFiles[i] )
            projName, seqName = db.getProjectAndSequenceNameFromFilePath( recentFiles[i] )

            proj = projectModel.Project( projName )
            seq = projectModel.Sequence( proj, seqName )

            testAsset = assetModel.Asset( proj, seq, fileName )
            if testAsset.isValidAsset():
                path = testAsset.getPath()
                foundValidAsset = True
                break

        if not foundValidAsset:
            # just get the path from workspace and return an empty fileName
            fileName = None
            path = getWorkspacePath()

    return fileName, path

#----------------------------------------------------------------------
def setRenderFileName( assetObject ):
    """sets the render file name
    """

    parentSeq = assetObject.getParentSequence()

    renderOutputFolder = parentSeq.getStructure().getOutputFolderPathOf( 'RENDER' ) # _RENDERED_IMAGES_/SHOTS

    # image folder from the workspace.mel
    imageFolderFromWS = pm.workspace.fileRules['image'] # _RENDERED_IMAGES_/

    shotFolder = renderOutputFolder[ len(imageFolderFromWS):] # SHOTS

    assetBaseName = assetObject.getBaseName()

    renderFileName = ''
    if parentSeq.noSubNameField():
        renderFileName = shotFolder + "/" + assetBaseName + "/<Layer>/" + assetBaseName + "_<Layer>_<RenderPass>_<Version>"
    else: # remove later when the support for old project is over
        assetSubName = assetObject.getSubName()
        renderFileName = shotFolder + "/" + assetBaseName + "/<Layer>/" + assetBaseName + "_" + assetSubName + "_<Layer>_<RenderPass>_<Version>"

    # defaultRenderGlobals
    dRG = pm.PyNode('defaultRenderGlobals')
    dRG.setAttr('imageFilePrefix', renderFileName)
    dRG.setAttr('renderVersion', assetObject.getVersionString() )
    dRG.setAttr('animation', 1)
    dRG.setAttr('outFormatControl', 0 )
    dRG.setAttr('extensionPadding', 3 )
    dRG.setAttr('pff', 1)

#----------------------------------------------------------------------
def setPlayblastFileName( assetObject ):
    """sets the playblast file name
    """
    pm.optionVar['playblastFile'] =  assetObject.getType().getPlayblastFolder()

#----------------------------------------------------------------------
def setProject( projectName, sequenceName ):
    """sets the project
    """
    db = projectModel.Database()

    mayaProjectPath = os.path.join( db.getProjectsFullPath(), projectName, sequenceName )

    pm.workspace.open(mayaProjectPath)

#----------------------------------------------------------------------
def getWorkspacePath():
    """returns the workspace path
    tries to fix the path separator for windows
    """

    path = pm.workspace.name

    if os.name == 'nt':
        #myDict = dict()
        #myDict[u'/'] = u'\\'
        #path = oyAux.multiple_replace( path, myDict)
        path = oyAux.fixWindowsPath( path )

    return path

#----------------------------------------------------------------------
def appendToRecentFiles(path):
    """appends the given path to the recent files list
    """

    # add the file to the recent file list
    recentFiles = pm.optionVar['RecentFilesList']

    #assert(isinstance(recentFiles,pm.OptionVarList))

    recentFiles.appendVar( path )
    #pm.optionVar['RecentFilesList'] = recentFiles

    <commit_msg>little fix in oyProjectManager<commit_after>import os
import pymel as pm
import oyAuxiliaryFunctions as oyAux
from oyProjectManager.dataModels import assetModel, projectModel

__version__ = "9.11.17"

#----------------------------------------------------------------------
def save( assetObject ):
    """the save action for maya environment

    uses PyMel to save the file (not necessary but comfortable )
    """

    # set the extension to ma by default
    assetObject.setExtension( 'ma' )

    # set the project to the current environment
    pm.workspace.open( assetObject.getSequenceFullPath() )

    # set the render file name and version
    setRenderFileName( assetObject )

    # set the playblast file name
    setPlayblastFileName( assetObject )

    # create the folder if it doesn't exists
    oyAux.createFolder( assetObject.getPath() )

    # save the file
    pm.saveAs( assetObject.getFullPath(), type='mayaAscii' )

    # append it to the recent file list
    appendToRecentFiles( assetObject.getFullPath() )

    return True

#----------------------------------------------------------------------
def export( assetObject ):
    """the export action for maya environment
    """

    # check if there is something selected
    if len(pm.ls(sl=True)) < 1:
        print "selection error"
        return False

    # set the extension to ma by default
    assetObject.setExtension( 'ma' )

    # create the folder if it doesn't exists
    oyAux.createFolder( assetObject.getPath() )

    # export the file
    pm.exportSelected( assetObject.getFullPath(), type='mayaAscii' )

    return True

#----------------------------------------------------------------------
def open_( assetObject, force=False):
    """the open action for maya environment
    """
    #assert(isinstance(assetObject, assetModel.Asset ) )

    # check for unsaved changes
    assetFullPath = assetObject.getFullPath()

    pm.openFile( assetFullPath, f=force )

    # set the project
    pm.workspace.open( assetObject.getSequenceFullPath() )

    # set the playblast folder
    setPlayblastFileName( assetObject )

    appendToRecentFiles( assetFullPath )

    return True

#----------------------------------------------------------------------
def import_( assetObject ):
    """the import action for maya environment
    """
    #assert( isinstance(assetObject, assetModel.Asset ) )
    pm.importFile( assetObject.getFullPath() )

    return True

#----------------------------------------------------------------------
def reference( assetObject ):
    """the reference action for maya environment
    """

    assert( isinstance(assetObject, assetModel.Asset ) )

    # use the file name without extension as the namespace
    nameSpace = assetObject.getFileNameWithoutExtension()

    pm.createReference( assetObject.getFullPath(), gl=True, loadReferenceDepth='all', namespace=nameSpace, options='v=0')

    return True

#----------------------------------------------------------------------
def getPathVariables():
    """gets the file name from maya environment
    """

    fileName = path = None

    fullPath = pm.env.sceneName()

    print "the fullPath in maya is ", fullPath

    readRecentFile = True

    if fullPath != '':
        fileName = os.path.basename( fullPath )

        # try to create an asset with that info
        db = projectModel.Database()
        projName, seqName = db.getProjectAndSequenceNameFromFilePath( fullPath )

        proj = projectModel.Project( projName )
        seq = projectModel.Sequence( proj, seqName )

        testAsset = assetModel.Asset( proj, seq, fileName )

        if testAsset.isValidAsset():
            fileName = testAsset.getFileName()
            path = testAsset.getPath()
            readRecentFile = False

    if readRecentFile:
        # read the fileName from recent files list
        # try to get the a valid asset file from starting the last recent file

        recentFiles = pm.optionVar['RecentFilesList']
        for i in range(len(recentFiles)-1, -1,-1):

            fileName = os.path.basename( recentFiles[i] )
            projName, seqName = db.getProjectAndSequenceNameFromFilePath( recentFiles[i] )

            proj = projectModel.Project( projName )
            seq = projectModel.Sequence( proj, seqName )

            testAsset = assetModel.Asset( proj, seq, fileName )
            if testAsset.isValidAsset():
                path = testAsset.getPath()
                foundValidAsset = True
                break

        if not foundValidAsset:
            # just get the path from workspace and return an empty fileName
            fileName = None
            path = getWorkspacePath()

    return fileName, path

#----------------------------------------------------------------------
def setRenderFileName( assetObject ):
    """sets the render file name
    """

    parentSeq = assetObject.getParentSequence()

    renderOutputFolder = parentSeq.getStructure().getOutputFolderPathOf( 'RENDER' ) # _RENDERED_IMAGES_/SHOTS

    # image folder from the workspace.mel
    imageFolderFromWS = pm.workspace.fileRules['image'] # _RENDERED_IMAGES_/

    shotFolder = renderOutputFolder[ len(imageFolderFromWS):] # SHOTS

    assetBaseName = assetObject.getBaseName()

    renderFileName = ''
    if parentSeq.noSubNameField():
        renderFileName = shotFolder + "/" + assetBaseName + "/<Layer>/" + assetBaseName + "_<Layer>_<RenderPass>_<Version>"
    else: # remove later when the support for old project is over
        assetSubName = assetObject.getSubName()
        renderFileName = shotFolder + "/" + assetBaseName + "/<Layer>/" + assetBaseName + "_" + assetSubName + "_<Layer>_<RenderPass>_<Version>"

    # defaultRenderGlobals
    dRG = pm.PyNode('defaultRenderGlobals')
    dRG.setAttr('imageFilePrefix', renderFileName)
    dRG.setAttr('renderVersion', assetObject.getVersionString() )
    dRG.setAttr('animation', 1)
    dRG.setAttr('outFormatControl', 0 )
    dRG.setAttr('extensionPadding', 3 )
    dRG.setAttr('pff', 1)

#----------------------------------------------------------------------
def setPlayblastFileName( assetObject ):
    """sets the playblast file name
    """
    pm.optionVar['playblastFile'] =  assetObject.getType().getPlayblastFolder()

#----------------------------------------------------------------------
def setProject( projectName, sequenceName ):
    """sets the project
    """
    db = projectModel.Database()

    mayaProjectPath = os.path.join( db.getProjectsFullPath(), projectName, sequenceName )

    pm.workspace.open(mayaProjectPath)

#----------------------------------------------------------------------
def getWorkspacePath():
    """returns the workspace path
    tries to fix the path separator for windows
    """

    path = pm.workspace.name

    if os.name == 'nt':
        #myDict = dict()
        #myDict[u'/'] = u'\\'
        #path = oyAux.multiple_replace( path, myDict)
        path = oyAux.fixWindowsPath( path )

    return path

#----------------------------------------------------------------------
def appendToRecentFiles(path):
    """appends the given path to the recent files list
    """

    # add the file to the recent file list
    recentFiles = pm.optionVar['RecentFilesList']

    #assert(isinstance(recentFiles,pm.OptionVarList))

    recentFiles.appendVar( path )
    #pm.optionVar['RecentFilesList'] = recentFiles

    <|endoftext|>


<commit_before>import contextlib
import json
import os
from abc import ABC, abstractmethod
from unittest.mock import ANY, patch

import pytest

from geopy import exc
from geopy.adapters import BaseAsyncAdapter
from geopy.location import Location

_env = {}
try:
    with open(".test_keys") as fp:
        _env.update(json.loads(fp.read()))
except OSError:
    _env.update(os.environ)

class SkipIfMissingEnv(dict):
    def __init__(self, env):
        super().__init__(env)
        self.is_internet_access_allowed = None

    def __getitem__(self, key):
        assert self.is_internet_access_allowed is not None
        if key not in self:
            if self.is_internet_access_allowed:
                pytest.skip("Missing geocoder credential: %s" % (key,))
            else:
                # Generate some dummy token. We won't perform a networking
                # request anyways.
                return "dummy"
        return super().__getitem__(key)

env = SkipIfMissingEnv(_env)

class BaseTestGeocoder(ABC):
    """
    Base for geocoder-specific test cases.
    """

    geocoder = None
    delta = 0.5

    @pytest.fixture(scope='class', autouse=True)
    async def class_geocoder(_, request, patch_adapter, is_internet_access_allowed):
        """Prepare a class-level Geocoder instance."""
        cls = request.cls
        env.is_internet_access_allowed = is_internet_access_allowed

        geocoder = cls.make_geocoder()
        cls.geocoder = geocoder

        run_async = isinstance(geocoder.adapter, BaseAsyncAdapter)
        if run_async:
            async with geocoder:
                yield geocoder
        else:
            yield geocoder

    @classmethod
    @contextlib.asynccontextmanager
    async def inject_geocoder(cls, geocoder):
        """An async context manager allowing to inject a custom
        geocoder instance in a single test method which will
        be used by the `geocode_run`/`reverse_run` methods.
        """
        with patch.object(cls, 'geocoder', geocoder):
            run_async = isinstance(geocoder.adapter, BaseAsyncAdapter)
            if run_async:
                async with geocoder:
                    yield geocoder
            else:
                yield geocoder

    @pytest.fixture(autouse=True)
    def ensure_no_geocoder_assignment(self):
        yield
        assert self.geocoder is type(self).geocoder, (
            "Detected `self.geocoder` assignment. "
            "Please use `async with inject_geocoder(my_geocoder):` "
            "instead, which supports async adapters."
        )

    @classmethod
    @abstractmethod
    def make_geocoder(cls, **kwargs):  # pragma: no cover
        pass

    async def geocode_run(
        self, payload, expected,
        *,
        skiptest_on_errors=True,
        expect_failure=False
    ):
        """
        Calls geocoder.geocode(**payload), then checks against `expected`.
        """
        cls = type(self)
        result = await self._make_request(
            self.geocoder, 'geocode',
            skiptest_on_errors=skiptest_on_errors,
            **payload,
        )
        if expect_failure:
            assert result is None
            return
        if result is None:
            pytest.fail('%s: No result found' % cls.__name__)
        if result == []:
            pytest.fail('%s returned an empty list instead of None' % cls.__name__)
        self._verify_request(result, exactly_one=payload.get('exactly_one', True),
                             **expected)
        return result

    async def reverse_run(
        self, payload, expected,
        *,
        skiptest_on_errors=True,
        expect_failure=False
    ):
        """
        Calls geocoder.reverse(**payload), then checks against `expected`.
        """
        cls = type(self)
        result = await self._make_request(
            self.geocoder, 'reverse',
            skiptest_on_errors=skiptest_on_errors,
            **payload,
        )
        if expect_failure:
            assert result is None
            return
        if result is None:
            pytest.fail('%s: No result found' % cls.__name__)
        if result == []:
            pytest.fail('%s returned an empty list instead of None' % cls.__name__)
        self._verify_request(result, exactly_one=payload.get('exactly_one', True),
                             **expected)
        return result

    async def reverse_timezone_run(self, payload, expected, *, skiptest_on_errors=True):
        timezone = await self._make_request(
            self.geocoder, 'reverse_timezone',
            skiptest_on_errors=skiptest_on_errors,
            **payload,
        )
        if expected is None:
            assert timezone is None
        else:
            assert timezone.pytz_timezone == expected

        return timezone

    async def _make_request(self, geocoder, method, *, skiptest_on_errors, **kwargs):
        cls = type(self)
        call = getattr(geocoder, method)
        run_async = isinstance(geocoder.adapter, BaseAsyncAdapter)
        try:
            if run_async:
                result = await call(**kwargs)
            else:
                result = call(**kwargs)
        except exc.GeocoderRateLimited as e:
            if not skiptest_on_errors:
                raise
            pytest.skip(
                "%s: Rate-limited, retry-after %s" % (cls.__name__, e.retry_after)
            )
        except exc.GeocoderQuotaExceeded:
            if not skiptest_on_errors:
                raise
            pytest.skip("%s: Quota exceeded" % cls.__name__)
        except exc.GeocoderTimedOut:
            if not skiptest_on_errors:
                raise
            pytest.skip("%s: Service timed out" % cls.__name__)
        except exc.GeocoderUnavailable:
            if not skiptest_on_errors:
                raise
            pytest.skip("%s: Service unavailable" % cls.__name__)
        return result

    def _verify_request(
            self,
            result,
            latitude=ANY,
            longitude=ANY,
            address=ANY,
            exactly_one=True,
            delta=None,
    ):
        if exactly_one:
            assert isinstance(result, Location)
        else:
            assert isinstance(result, list)

        item = result if exactly_one else result[0]
        delta = delta or self.delta

        expected = (
            pytest.approx(latitude, abs=delta) if latitude is not ANY else ANY,
            pytest.approx(longitude, abs=delta) if longitude is not ANY else ANY,
            address,
        )
        received = (
            item.latitude,
            item.longitude,
            item.address,
        )
        assert received == expected
<commit_msg>tests: treat empty env value as a missing env<commit_after>import contextlib
import json
import os
from abc import ABC, abstractmethod
from unittest.mock import ANY, patch

import pytest

from geopy import exc
from geopy.adapters import BaseAsyncAdapter
from geopy.location import Location

_env = {}
try:
    with open(".test_keys") as fp:
        _env.update(json.loads(fp.read()))
except OSError:
    _env.update(os.environ)

class SkipIfMissingEnv(dict):
    def __init__(self, env):
        super().__init__(env)
        self.is_internet_access_allowed = None

    def __getitem__(self, key):
        assert self.is_internet_access_allowed is not None
        if key not in self or not super().__getitem__(key):
            if self.is_internet_access_allowed:
                pytest.skip("Missing geocoder credential: %s" % (key,))
            else:
                # Generate some dummy token. We won't perform a networking
                # request anyways.
                return "dummy"
        return super().__getitem__(key)

env = SkipIfMissingEnv(_env)

class BaseTestGeocoder(ABC):
    """
    Base for geocoder-specific test cases.
    """

    geocoder = None
    delta = 0.5

    @pytest.fixture(scope='class', autouse=True)
    async def class_geocoder(_, request, patch_adapter, is_internet_access_allowed):
        """Prepare a class-level Geocoder instance."""
        cls = request.cls
        env.is_internet_access_allowed = is_internet_access_allowed

        geocoder = cls.make_geocoder()
        cls.geocoder = geocoder

        run_async = isinstance(geocoder.adapter, BaseAsyncAdapter)
        if run_async:
            async with geocoder:
                yield geocoder
        else:
            yield geocoder

    @classmethod
    @contextlib.asynccontextmanager
    async def inject_geocoder(cls, geocoder):
        """An async context manager allowing to inject a custom
        geocoder instance in a single test method which will
        be used by the `geocode_run`/`reverse_run` methods.
        """
        with patch.object(cls, 'geocoder', geocoder):
            run_async = isinstance(geocoder.adapter, BaseAsyncAdapter)
            if run_async:
                async with geocoder:
                    yield geocoder
            else:
                yield geocoder

    @pytest.fixture(autouse=True)
    def ensure_no_geocoder_assignment(self):
        yield
        assert self.geocoder is type(self).geocoder, (
            "Detected `self.geocoder` assignment. "
            "Please use `async with inject_geocoder(my_geocoder):` "
            "instead, which supports async adapters."
        )

    @classmethod
    @abstractmethod
    def make_geocoder(cls, **kwargs):  # pragma: no cover
        pass

    async def geocode_run(
        self, payload, expected,
        *,
        skiptest_on_errors=True,
        expect_failure=False
    ):
        """
        Calls geocoder.geocode(**payload), then checks against `expected`.
        """
        cls = type(self)
        result = await self._make_request(
            self.geocoder, 'geocode',
            skiptest_on_errors=skiptest_on_errors,
            **payload,
        )
        if expect_failure:
            assert result is None
            return
        if result is None:
            pytest.fail('%s: No result found' % cls.__name__)
        if result == []:
            pytest.fail('%s returned an empty list instead of None' % cls.__name__)
        self._verify_request(result, exactly_one=payload.get('exactly_one', True),
                             **expected)
        return result

    async def reverse_run(
        self, payload, expected,
        *,
        skiptest_on_errors=True,
        expect_failure=False
    ):
        """
        Calls geocoder.reverse(**payload), then checks against `expected`.
        """
        cls = type(self)
        result = await self._make_request(
            self.geocoder, 'reverse',
            skiptest_on_errors=skiptest_on_errors,
            **payload,
        )
        if expect_failure:
            assert result is None
            return
        if result is None:
            pytest.fail('%s: No result found' % cls.__name__)
        if result == []:
            pytest.fail('%s returned an empty list instead of None' % cls.__name__)
        self._verify_request(result, exactly_one=payload.get('exactly_one', True),
                             **expected)
        return result

    async def reverse_timezone_run(self, payload, expected, *, skiptest_on_errors=True):
        timezone = await self._make_request(
            self.geocoder, 'reverse_timezone',
            skiptest_on_errors=skiptest_on_errors,
            **payload,
        )
        if expected is None:
            assert timezone is None
        else:
            assert timezone.pytz_timezone == expected

        return timezone

    async def _make_request(self, geocoder, method, *, skiptest_on_errors, **kwargs):
        cls = type(self)
        call = getattr(geocoder, method)
        run_async = isinstance(geocoder.adapter, BaseAsyncAdapter)
        try:
            if run_async:
                result = await call(**kwargs)
            else:
                result = call(**kwargs)
        except exc.GeocoderRateLimited as e:
            if not skiptest_on_errors:
                raise
            pytest.skip(
                "%s: Rate-limited, retry-after %s" % (cls.__name__, e.retry_after)
            )
        except exc.GeocoderQuotaExceeded:
            if not skiptest_on_errors:
                raise
            pytest.skip("%s: Quota exceeded" % cls.__name__)
        except exc.GeocoderTimedOut:
            if not skiptest_on_errors:
                raise
            pytest.skip("%s: Service timed out" % cls.__name__)
        except exc.GeocoderUnavailable:
            if not skiptest_on_errors:
                raise
            pytest.skip("%s: Service unavailable" % cls.__name__)
        return result

    def _verify_request(
            self,
            result,
            latitude=ANY,
            longitude=ANY,
            address=ANY,
            exactly_one=True,
            delta=None,
    ):
        if exactly_one:
            assert isinstance(result, Location)
        else:
            assert isinstance(result, list)

        item = result if exactly_one else result[0]
        delta = delta or self.delta

        expected = (
            pytest.approx(latitude, abs=delta) if latitude is not ANY else ANY,
            pytest.approx(longitude, abs=delta) if longitude is not ANY else ANY,
            address,
        )
        received = (
            item.latitude,
            item.longitude,
            item.address,
        )
        assert received == expected
<|endoftext|>


<commit_before>from __future__ import division
from __future__ import print_function
from ..util import read_param_file
from sys import exit
import numpy as np
from scipy.stats import norm
from . import common_args

# Perform Morris Analysis on file of model results
# Returns a dictionary with keys 'mu', 'mu_star', 'sigma', and 'mu_star_conf'
# Where each entry is a list of size D (the number of parameters)
# Containing the indices in the same order as the parameter file

def analyze(pfile, input_file, output_file, column=0, delim=' ', num_resamples=1000, conf_level=0.95, print_to_console=False):

    param_file = read_param_file(pfile)
    Y = np.loadtxt(output_file, delimiter=delim, usecols=(column,))
    X = np.loadtxt(input_file, delimiter=delim, ndmin=2)
    if len(X.shape) == 1:
        X = X.reshape((len(X), 1))

    D = param_file['num_vars']

    if Y.size % (D + 1) == 0:
        N = int(Y.size / (D + 1))
    else:
        print("""
                Error: Number of samples in model output file must be a multiple of (D+1),
                where D is the number of parameters in your parameter file.
              """)
        exit()

    ee = np.zeros((N,D))

    # For each of the N trajectories
    for i in range(N):

        # Set up the indices corresponding to this trajectory
        j = np.arange(D + 1) + i * (D + 1)
        j1 = j[0:D]
        j2 = j[1:D + 1]

        # The elementary effect is (change in output)/(change in input)
        # Each parameter has one EE per trajectory, because it is only changed
        # once in each trajectory
        try:
            ee[i, :] = np.linalg.solve(X[j2,:]-X[j1,:], Y[j2]-Y[j1])
        except np.linalg.linalg.LinAlgError:
            for k in j1:
                delta = X[k+1,:] - X[k,:]
                col = np.nonzero(delta)[0]
                ee[i,col] = (Y[k+1] - Y[k]) / (X[k+1,col] - X[k,col])

    # Rescale elementary effects so that the delta step is in the quantile
    # space [0,1]
    for j in range(D):
        lb = param_file['bounds'][j][0]
        ub = param_file['bounds'][j][1]
        ee[:, j] *= (ub - lb)

    # Output the Mu, Mu*, and Sigma Values. Also return them in case this is
    # being called from Python
    Si = dict((k, [None] * D)
              for k in ['mu', 'mu_star', 'sigma', 'mu_star_conf'])
    if print_to_console:
        print("Parameter Mu Sigma Mu_Star Mu_Star_Conf")

    for j in range(D):
        Si['mu'][j] = np.average(ee[:, j])
        Si['mu_star'][j] = np.average(np.abs(ee[:, j]))
        Si['sigma'][j] = np.std(ee[:, j])
        Si['mu_star_conf'][j] = compute_mu_star_confidence(
            ee[:, j], N, num_resamples, conf_level)

        if print_to_console:
            print("%s %f %f %f %f" % (param_file['names'][j], Si['mu'][j], Si[
                  'sigma'][j], Si['mu_star'][j], Si['mu_star_conf'][j]))

    return Si

def compute_mu_star_confidence(ee, N, num_resamples, conf_level):

    ee_resampled = np.empty([N])
    mu_star_resampled = np.empty([num_resamples])

    if conf_level < 0 or conf_level > 1:
        print("Error: Confidence level must be between 0-1.")
        exit()

    for i in range(num_resamples):
        for j in range(N):

            index = np.random.randint(0, N)
            ee_resampled[j] = ee[index]

        mu_star_resampled[i] = np.average(np.abs(ee_resampled))

    return norm.ppf(0.5 + conf_level / 2) * mu_star_resampled.std(ddof=1)

if __name__ == "__main__":

    parser = common_args.create()
    parser.add_argument('-X', '--model-input-file', type=str,
                        required=True, default=None, help='Model input file')
    parser.add_argument('-r', '--resamples', type=int, required=False, default=1000,
                        help='Number of bootstrap resamples for Sobol confidence intervals')

    args = parser.parse_args()
    analyze(args.paramfile, args.model_input_file, args.model_output_file, args.column,
            delim=args.delimiter, num_resamples=args.resamples, print_to_console=True)
<commit_msg>Refactored morris analysis to allow testing and vector computation of elementary effects<commit_after>from __future__ import division
from __future__ import print_function
from ..util import read_param_file
from sys import exit
import numpy as np
from scipy.stats import norm
from . import common_args
from collections import OrderedDict
from ..sample.morris import compute_groups_from_parameter_file

# Perform Morris Analysis on file of model results
# Returns a dictionary with keys 'mu', 'mu_star', 'sigma', and 'mu_star_conf'
# Where each entry is a list of size num_vars (the number of parameters)
# Containing the indices in the same order as the parameter file

def analyze(pfile,
            input_file,
            output_file,
            column=0,
            delim=' ',
            num_resamples=1000,
            conf_level=0.95,
            print_to_console=False,
            group_file=None,
            grid_jump=2,
            num_levels=4):

    delta = grid_jump / (num_levels - 1)

    param_file = read_param_file(pfile,True)
    Y = np.loadtxt(output_file, delimiter=delim, usecols=(column,))
    X = np.loadtxt(input_file, delimiter=delim, ndmin=2)
    if len(X.shape) == 1:
        X = X.reshape((len(X), 1))

    num_vars = param_file['num_vars']

    if (Y.size % (num_vars + 1) == 0):
        num_trajectories = int(Y.size / (num_vars + 1))
    elif group_file is not None:
        group_list = param_file['groups']
        groups, unique_group_names = compute_groups_from_parameter_file(group_list, num_vars)
        number_of_groups = len(unique_group_names)
        num_trajectories = int(Y.size / (number_of_groups + 1))
    else:
        raise ValueError("""Number of samples in model output file must be a multiple of (D+1), \
                            where D is the number of parameters in your parameter file. \
                         """)
    ee = np.zeros((num_vars, num_trajectories))
    ee = compute_effects_vector(X, Y, Y.size / num_trajectories, delta)

    # Output the Mu, Mu*, and Sigma Values. Also return them in case this is
    # being called from Python
    Si = dict((k, [None] * num_vars)
              for k in ['names', 'mu', 'mu_star', 'sigma', 'mu_star_conf'])
    if print_to_console:
        print("Parameter Mu Sigma Mu_Star Mu_Star_Conf")

    Si['mu'] = np.average(ee, 1)
    Si['mu_star'] = np.average(np.abs(ee), 1)
    Si['sigma'] = np.std(ee, 1)

    Si['names'] = param_file['names']

    for j in range(num_vars):
        Si['mu_star_conf'][j] = compute_mu_star_confidence(
            ee[j, :], num_trajectories, num_resamples, conf_level)

        if print_to_console:
            print("%s %f %f %f %f" % (param_file['names'][j], Si['mu'][j], Si[
                  'sigma'][j], Si['mu_star'][j], Si['mu_star_conf'][j]))

    if group_file is None:
        return Si
    elif group_file is not None:
        # if there are groups, then the elementary effects returned need to be
        # computed over the groups of variables, rather than the individual variables
        Si_grouped = dict((k, [None] * num_vars)
                for k in ['mu_star', 'mu_star_conf'])
        Si_grouped['mu_star'] = compute_grouped_mu_star(Si['mu_star'], groups)
        Si_grouped['mu_star_conf'] = compute_grouped_mu_star(Si['mu_star_conf'], groups)
        Si_grouped['names'] = unique_group_names
        return Si_grouped
    else:
        raise RuntimeError("Could determine which parameters should be returned")

def compute_grouped_mu_star(mu_star_ungrouped, group_matrix):

    group_matrix = np.array(group_matrix)
    mu_star_grouped = np.divide(np.dot(mu_star_ungrouped, group_matrix), np.sum(group_matrix, 0))

    return mu_star_grouped.T

def compute_elementary_effect(X, Y, j1, j2):
    # The elementary effect is (change in output)/(change in input)
    # Each parameter has one EE per trajectory, because it is only changed
    # once in each trajectory
    try:

        change_in_input = X[j2, :] - X[j1, :]
        change_in_output = Y[j2] - Y[j1]
        return np.linalg.solve(change_in_input, change_in_output)

    except np.linalg.linalg.LinAlgError:

        ee = np.zeros(X.shape[1])
        for k in j1:
            delta = X[k+1, :] - X[k, :]
            col = np.nonzero(delta)[0]
            ee[k] = np.array((Y[k + 1] - Y[k]) / (X[k + 1, col] - X[k, col]))
        return ee

def get_increased_values(op_vec, up, lo):

    up = np.pad(up, ((0, 0), (1, 0), (0, 0)), 'constant')
    lo = np.pad(lo, ((0, 0), (0, 1), (0, 0)), 'constant')

    res = np.einsum('ik,ikj->ij', op_vec, up+lo)

    return res.T

def get_decreased_values(op_vec, up, lo):

    up = np.pad(up, ((0, 0), (0, 1), (0, 0)), 'constant')
    lo = np.pad(lo, ((0, 0), (1, 0), (0, 0)), 'constant')

    res = np.einsum('ik,ikj->ij', op_vec, up+lo)

    return res.T

def compute_effects_vector(model_inputs, model_outputs, trajectory_size, delta):
    '''
    Arguments:
        - model_inputs - matrix of inputs to the model under analysis.
                         x-by-r where x is the number of variables and
                         r is the number of rows (a function of x and num_trajectories)
        - model_outputs - an r-length vector of model outputs
        - trajectory_size - a scalar indicating the number of rows in a
                            trajectory
    '''
    num_vars = model_inputs.shape[1]
    num_rows = model_inputs.shape[0]
    num_trajectories = int(num_rows / trajectory_size)

    ee = np.zeros((num_trajectories, num_vars), dtype=np.float)

    ip_vec = model_inputs.reshape(num_trajectories,trajectory_size,num_vars)
    ip_cha = np.subtract(ip_vec[:,1:,:], ip_vec[:,0:-1,:])
    up = (ip_cha > 0)
    lo = (ip_cha < 0)

    op_vec = model_outputs.reshape(num_trajectories,trajectory_size)

    result_up = get_increased_values(op_vec, up, lo)
    result_lo = get_decreased_values(op_vec, up, lo)

    ee = np.subtract(result_up, result_lo)
    np.divide(ee, delta, out = ee)

    return ee

def compute_mu_star_confidence(ee, num_trajectories, num_resamples, conf_level):

    ee_resampled = np.empty([num_trajectories])
    mu_star_resampled = np.empty([num_resamples])

    if conf_level < 0 or conf_level > 1:
        raise ValueError("Confidence level must be between 0-1.")

    for i in range(num_resamples):
        for j in range(num_trajectories):

            index = np.random.randint(0, num_trajectories)
            ee_resampled[j] = ee[index]

        mu_star_resampled[i] = np.average(np.abs(ee_resampled))

    return norm.ppf(0.5 + conf_level / 2) * mu_star_resampled.std(ddof=1)

if __name__ == "__main__":

    parser = common_args.create()
    parser.add_argument('-X', '--model-input-file', type=str,
                        required=True, default=None, help='Model input file')
    parser.add_argument('-r', '--resamples', type=int, required=False, default=1000,
                        help='Number of bootstrap resamples for Sobol confidence intervals')

    args = parser.parse_args()
    analyze(args.paramfile, args.model_input_file, args.model_output_file, args.column,
            delim=args.delimiter, num_resamples=args.resamples, print_to_console=True)
<|endoftext|>