<commit_before># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Guewen Baconnier
#    Copyright 2014 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import mock
from openerp.addons.magentoerpconnect.unit.import_synchronizer import (
    import_record)
import openerp.tests.common as common
from .common import (mock_api,
                     mock_urlopen_image)
from .test_data import magento_base_responses
from .test_synchronization import SetUpMagentoSynchronized
from ..sale import export_state_change

DB = common.DB
ADMIN_USER_ID = common.ADMIN_USER_ID

class TestSaleOrder(SetUpMagentoSynchronized):

    def _import_sale_order(self, increment_id):
        backend_id = self.backend_id
        with mock_api(magento_base_responses):
            with mock_urlopen_image():
                import_record(self.session,
                              'magento.sale.order',
                              backend_id, increment_id)
        MagentoOrder = self.registry('magento.sale.order')
        binding_ids = MagentoOrder.search(
            self.cr,
            self.uid,
            [('backend_id', '=', backend_id),
             ('magento_id', '=', str(increment_id))])
        self.assertEqual(len(binding_ids), 1)
        return MagentoOrder.browse(self.cr, self.uid, binding_ids[0])

    def test_copy_quotation(self):
        """ Copy a sales order with copy_quotation move bindings """
        binding = self._import_sale_order(900000691)
        order = binding.openerp_id
        SaleOrder = self.registry('sale.order')
        action = SaleOrder.copy_quotation(self.cr, self.uid, [order.id])
        new_id = action['res_id']
        order.refresh()
        self.assertFalse(order.magento_bind_ids)
        binding.refresh()
        self.assertEqual(binding.openerp_id.id, new_id)
        for mag_line in binding.magento_order_line_ids:
            self.assertEqual(mag_line.order_id.id, new_id)

    def test_cancel_delay_job(self):
        """ Cancel an order, delay a cancel job """
        binding = self._import_sale_order(900000691)
        order = binding.openerp_id
        patched = 'openerp.addons.magentoerpconnect.sale.export_state_change'
        # patch the job so it won't be created and we will be able
        # to check if it is called
        with mock.patch(patched) as mock_export_state_change:
            order.action_cancel()
            called = mock_export_state_change.delay
            called.assert_called_with(mock.ANY,
                                      'magento.sale.order',
                                      binding.id,
                                      allowed_states=['cancel'],
                                      description=mock.ANY)

    def test_cancel_export(self):
        """ Export the cancel state """
        binding = self._import_sale_order(900000691)
        order = binding.openerp_id
        # patch the job so it won't be created
        patched = 'openerp.addons.magentoerpconnect.sale.export_state_change'
        with mock.patch(patched):
            order.action_cancel()
        response = {
            'sales_order.info': {'status': 'new'},
            'sales_order.addComment': True,
        }
        with mock_api(response,
                      key_func=lambda method, args: method) as calls_done:
            # call the job synchronously, so we check the calls
            export_state_change(self.session, 'magento.sale.order',
                                binding.id, allowed_states=['cancel'])

            # call 1: sales_order.info to read the status
            # call 2: sales_order.addComment to add a status comment
            self.assertEqual(len(calls_done), 2)
            method, (magento_id, state, comment, notify) = calls_done[1]
            self.assertEqual(method, 'sales_order.addComment')
            self.assertEqual(magento_id, binding.magento_id)
            self.assertEqual(state, 'canceled')
            self.assertFalse(comment)
            self.assertFalse(notify)

    def test_copy_quotation_delay_export_state(self):
        """ Delay a state export on new copy from canceled order """
        binding = self._import_sale_order(900000691)
        order = binding.openerp_id

        # cancel the order
        patched = 'openerp.addons.magentoerpconnect.sale.export_state_change'
        with mock.patch(patched):
            # cancel the sales order, a job exporting the cancel status
            # to Magento is normally created (muted here)
            order.action_cancel()

        SaleOrder = self.registry('sale.order')
        patched = 'openerp.addons.magentoerpconnect.sale.export_state_change'
        with mock.patch(patched) as mock_export_state_change:
            # create a copy of quotation, the new order should be linked to
            # the Magento sales order
            action = SaleOrder.copy_quotation(self.cr, self.uid, [order.id])
            new_id = action['res_id']
            binding.refresh()
            order = binding.openerp_id
            self.assertEqual(order.id, new_id)

            called = mock_export_state_change.delay
            called.assert_called_with(mock.ANY,
                                      'magento.sale.order',
                                      binding.id,
                                      description=mock.ANY)

    def test_copy_quotation_export_state(self):
        """ Export a new state on new copy from canceled order """
        binding = self._import_sale_order(900000691)
        order = binding.openerp_id
        SaleOrder = self.registry('sale.order')

        # cancel the order
        patched = 'openerp.addons.magentoerpconnect.sale.export_state_change'
        with mock.patch(patched):
            # cancel the sales order, a job exporting the cancel status
            # to Magento is normally created (muted here)
            order.action_cancel()

            # create a copy of quotation, the new order should be linked to
            # the Magento sales order
            action = SaleOrder.copy_quotation(self.cr, self.uid, [order.id])
            new_id = action['res_id']
            binding.refresh()
            order = binding.openerp_id
            self.assertEqual(order.id, new_id)

        # we will check if the correct messages are sent to Magento
        response = {
            'sales_order.info': {'status': 'canceled'},
            'sales_order.addComment': True,
        }
        with mock_api(response,
                      key_func=lambda method, args: method) as calls_done:
            # call the job synchronously, so we check the calls
            export_state_change(self.session, 'magento.sale.order',
                                binding.id)

            # call 1: sales_order.info to read the status
            # call 2: sales_order.addComment to add a status comment
            self.assertEqual(len(calls_done), 2)
            method, (magento_id, state, comment, notify) = calls_done[1]
            self.assertEqual(method, 'sales_order.addComment')
            self.assertEqual(magento_id, binding.magento_id)
            self.assertEqual(state, 'pending')
            self.assertFalse(comment)
            self.assertFalse(notify)
<commit_msg>Tests fail because now the adapter removes the trailing None values<commit_after># -*- coding: utf-8 -*-
##############################################################################
#
#    Author: Guewen Baconnier
#    Copyright 2014 Camptocamp SA
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

import mock
from openerp.addons.magentoerpconnect.unit.import_synchronizer import (
    import_record)
import openerp.tests.common as common
from .common import (mock_api,
                     mock_urlopen_image)
from .test_data import magento_base_responses
from .test_synchronization import SetUpMagentoSynchronized
from ..sale import export_state_change

DB = common.DB
ADMIN_USER_ID = common.ADMIN_USER_ID

class TestSaleOrder(SetUpMagentoSynchronized):

    def _import_sale_order(self, increment_id):
        backend_id = self.backend_id
        with mock_api(magento_base_responses):
            with mock_urlopen_image():
                import_record(self.session,
                              'magento.sale.order',
                              backend_id, increment_id)
        MagentoOrder = self.registry('magento.sale.order')
        binding_ids = MagentoOrder.search(
            self.cr,
            self.uid,
            [('backend_id', '=', backend_id),
             ('magento_id', '=', str(increment_id))])
        self.assertEqual(len(binding_ids), 1)
        return MagentoOrder.browse(self.cr, self.uid, binding_ids[0])

    def test_copy_quotation(self):
        """ Copy a sales order with copy_quotation move bindings """
        binding = self._import_sale_order(900000691)
        order = binding.openerp_id
        SaleOrder = self.registry('sale.order')
        action = SaleOrder.copy_quotation(self.cr, self.uid, [order.id])
        new_id = action['res_id']
        order.refresh()
        self.assertFalse(order.magento_bind_ids)
        binding.refresh()
        self.assertEqual(binding.openerp_id.id, new_id)
        for mag_line in binding.magento_order_line_ids:
            self.assertEqual(mag_line.order_id.id, new_id)

    def test_cancel_delay_job(self):
        """ Cancel an order, delay a cancel job """
        binding = self._import_sale_order(900000691)
        order = binding.openerp_id
        patched = 'openerp.addons.magentoerpconnect.sale.export_state_change'
        # patch the job so it won't be created and we will be able
        # to check if it is called
        with mock.patch(patched) as mock_export_state_change:
            order.action_cancel()
            called = mock_export_state_change.delay
            called.assert_called_with(mock.ANY,
                                      'magento.sale.order',
                                      binding.id,
                                      allowed_states=['cancel'],
                                      description=mock.ANY)

    def test_cancel_export(self):
        """ Export the cancel state """
        binding = self._import_sale_order(900000691)
        order = binding.openerp_id
        # patch the job so it won't be created
        patched = 'openerp.addons.magentoerpconnect.sale.export_state_change'
        with mock.patch(patched):
            order.action_cancel()
        response = {
            'sales_order.info': {'status': 'new'},
            'sales_order.addComment': True,
        }
        with mock_api(response,
                      key_func=lambda method, args: method) as calls_done:
            # call the job synchronously, so we check the calls
            export_state_change(self.session, 'magento.sale.order',
                                binding.id, allowed_states=['cancel'])

            # call 1: sales_order.info to read the status
            # call 2: sales_order.addComment to add a status comment
            self.assertEqual(len(calls_done), 2)
            method, (magento_id, state) = calls_done[1]
            self.assertEqual(method, 'sales_order.addComment')
            self.assertEqual(magento_id, binding.magento_id)
            self.assertEqual(state, 'canceled')

    def test_copy_quotation_delay_export_state(self):
        """ Delay a state export on new copy from canceled order """
        binding = self._import_sale_order(900000691)
        order = binding.openerp_id

        # cancel the order
        patched = 'openerp.addons.magentoerpconnect.sale.export_state_change'
        with mock.patch(patched):
            # cancel the sales order, a job exporting the cancel status
            # to Magento is normally created (muted here)
            order.action_cancel()

        SaleOrder = self.registry('sale.order')
        patched = 'openerp.addons.magentoerpconnect.sale.export_state_change'
        with mock.patch(patched) as mock_export_state_change:
            # create a copy of quotation, the new order should be linked to
            # the Magento sales order
            action = SaleOrder.copy_quotation(self.cr, self.uid, [order.id])
            new_id = action['res_id']
            binding.refresh()
            order = binding.openerp_id
            self.assertEqual(order.id, new_id)

            called = mock_export_state_change.delay
            called.assert_called_with(mock.ANY,
                                      'magento.sale.order',
                                      binding.id,
                                      description=mock.ANY)

    def test_copy_quotation_export_state(self):
        """ Export a new state on new copy from canceled order """
        binding = self._import_sale_order(900000691)
        order = binding.openerp_id
        SaleOrder = self.registry('sale.order')

        # cancel the order
        patched = 'openerp.addons.magentoerpconnect.sale.export_state_change'
        with mock.patch(patched):
            # cancel the sales order, a job exporting the cancel status
            # to Magento is normally created (muted here)
            order.action_cancel()

            # create a copy of quotation, the new order should be linked to
            # the Magento sales order
            action = SaleOrder.copy_quotation(self.cr, self.uid, [order.id])
            new_id = action['res_id']
            binding.refresh()
            order = binding.openerp_id
            self.assertEqual(order.id, new_id)

        # we will check if the correct messages are sent to Magento
        response = {
            'sales_order.info': {'status': 'canceled'},
            'sales_order.addComment': True,
        }
        with mock_api(response,
                      key_func=lambda method, args: method) as calls_done:
            # call the job synchronously, so we check the calls
            export_state_change(self.session, 'magento.sale.order',
                                binding.id)

            # call 1: sales_order.info to read the status
            # call 2: sales_order.addComment to add a status comment
            self.assertEqual(len(calls_done), 2)
            method, (magento_id, state) = calls_done[1]
            self.assertEqual(method, 'sales_order.addComment')
            self.assertEqual(magento_id, binding.magento_id)
            self.assertEqual(state, 'pending')
<|endoftext|>


<commit_before>"""
.. todo::

    WRITEME
"""
from copy import deepcopy
import numpy
np = numpy
from pylearn2.train_extensions import TrainExtension
import theano
import theano.tensor as T
from pylearn2.utils import serial

class KeepBestParams(TrainExtension):
    """
    A callback which keeps track of a model's best parameters based on its
    performance for a given cost on a given dataset.

    Parameters
    ----------
    model : pylearn2.models.model.Model
        the model whose best parameters we want to keep track of
    cost : tensor_like
        cost function used to evaluate the model's performance
    monitoring_dataset : pylearn2.datasets.dataset.Dataset
        dataset on which to compute the cost
    batch_size : int
        size of the batches used to compute the cost
    """

    def __init__(self, model, cost, monitoring_dataset, batch_size):
        self.model = model
        self.cost = cost
        self.dataset = monitoring_dataset
        self.batch_size = batch_size
        self.minibatch = T.matrix('minibatch')
        self.target = T.matrix('target')
        if cost.supervised:
            self.supervised = True
            self.cost_function = theano.function(inputs=[self.minibatch,
                                                          self.target],
                                                  outputs=cost(model,
                                                               self.minibatch,
                                                               self.target))
        else:
            self.supervised = False
            self.cost_function = theano.function(inputs=[self.minibatch],
                                                 outputs=cost(model,
                                                              self.minibatch))
        self.best_cost = numpy.inf
        self.best_params = model.get_param_values()

    def on_monitor(self, model, dataset, algorithm):
        """
        Looks whether the model performs better than earlier. If it's the
        case, records the model's parameters.

        Parameters
        ----------
        model : pylearn2.models.model.Model
            Not used
        dataset : pylearn2.datasets.dataset.Dataset
            Not used
        algorithm : TrainingAlgorithm
            Not used
        """
        if self.supervised:
            it = self.dataset.iterator('sequential',
                                       batch_size=self.batch_size,
                                       targets=True)
            new_cost = numpy.mean([self.cost_function(minibatch, target)
                                   for minibatch, target in it])
        else:
            it = self.dataset.iterator('sequential',
                                       batch_size=self.batch_size,
                                       targets=False)
            new_cost = numpy.mean([self.cost_function(minibatch)
                                   for minibatch in it])
        if new_cost < self.best_cost:
            self.best_cost = new_cost
            self.best_params = self.model.get_param_values()

    def get_best_params(self):
        """Returns the best parameters up to now for the model."""
        return self.best_params

class MonitorBasedSaveBest(TrainExtension):
    """
    A callback that saves a copy of the model every time it achieves
    a new minimal value of a monitoring channel.

    Parameters
    ----------
    channel_name : str
        The name of the channel we want to minimize
    save_path : str
        Path to save the best model to
    higher_is_better : bool, optional
        WRITEME
    """

    def __init__(self, channel_name, save_path,higher_is_better=False):
        self.__dict__.update(locals())
        del self.self
        if higher_is_better:
            self.coeff = -1.
        else:
            self.coeff = 1.
        self.best_cost = np.inf

    def on_monitor(self, model, dataset, algorithm):
        """
        Looks whether the model performs better than earlier. If it's the
        case, saves the model.

        Parameters
        ----------
        model : pylearn2.models.model.Model
            model.monitor must contain a channel with name given by
            self.channel_name
        dataset : pylearn2.datasets.dataset.Dataset
            Not used
        algorithm : TrainingAlgorithm
            Not used
        """

        monitor = model.monitor
        channels = monitor.channels
        channel = channels[self.channel_name]
        val_record = channel.val_record
        new_cost = self.coeff * val_record[-1]

        if new_cost < self.best_cost:
            self.best_cost = new_cost
            serial.save(self.save_path, model, on_overwrite = 'backup')

class MonitorBasedStoreBest(TrainExtension):
    """
    Saves the best model in memory.

    Parameters
    ----------
    channel_name : str
        Channel to monitor.
    save_path : str
        Output filename.
    higher_is_better : bool
        Whether a higher channel value indicates a better model.
    """
    def __init__(self, channel_name, higher_is_better=False):
        self.channel_name = channel_name
        self.higher_is_better = higher_is_better
        self.best_cost = np.inf
        self.best_model = None

    def on_monitor(self, model, dataset, algorithm):
        """
        Check if the model performs better than before. Save best models in
        memory to avoid race conditions.

        Parameters
        ----------
        model : Model
            Model to monitor.
        dataset : Dataset
            Training dataset.
        algorithm : TrainingAlgorithm
            Training algorithm.
        """
        monitor = model.monitor
        channels = monitor.channels
        channel = channels[self.channel_name]
        val_record = channel.val_record
        if self.higher_is_better:
            new_cost = -1 * val_record[-1]
        else:
            new_cost = val_record[-1]

        if new_cost < self.best_cost:
            self.best_cost = new_cost
            self.best_model = deepcopy(model)
<commit_msg>Update docstring<commit_after>"""
.. todo::

    WRITEME
"""
from copy import deepcopy
import numpy
np = numpy
from pylearn2.train_extensions import TrainExtension
import theano
import theano.tensor as T
from pylearn2.utils import serial

class KeepBestParams(TrainExtension):
    """
    A callback which keeps track of a model's best parameters based on its
    performance for a given cost on a given dataset.

    Parameters
    ----------
    model : pylearn2.models.model.Model
        the model whose best parameters we want to keep track of
    cost : tensor_like
        cost function used to evaluate the model's performance
    monitoring_dataset : pylearn2.datasets.dataset.Dataset
        dataset on which to compute the cost
    batch_size : int
        size of the batches used to compute the cost
    """

    def __init__(self, model, cost, monitoring_dataset, batch_size):
        self.model = model
        self.cost = cost
        self.dataset = monitoring_dataset
        self.batch_size = batch_size
        self.minibatch = T.matrix('minibatch')
        self.target = T.matrix('target')
        if cost.supervised:
            self.supervised = True
            self.cost_function = theano.function(inputs=[self.minibatch,
                                                          self.target],
                                                  outputs=cost(model,
                                                               self.minibatch,
                                                               self.target))
        else:
            self.supervised = False
            self.cost_function = theano.function(inputs=[self.minibatch],
                                                 outputs=cost(model,
                                                              self.minibatch))
        self.best_cost = numpy.inf
        self.best_params = model.get_param_values()

    def on_monitor(self, model, dataset, algorithm):
        """
        Looks whether the model performs better than earlier. If it's the
        case, records the model's parameters.

        Parameters
        ----------
        model : pylearn2.models.model.Model
            Not used
        dataset : pylearn2.datasets.dataset.Dataset
            Not used
        algorithm : TrainingAlgorithm
            Not used
        """
        if self.supervised:
            it = self.dataset.iterator('sequential',
                                       batch_size=self.batch_size,
                                       targets=True)
            new_cost = numpy.mean([self.cost_function(minibatch, target)
                                   for minibatch, target in it])
        else:
            it = self.dataset.iterator('sequential',
                                       batch_size=self.batch_size,
                                       targets=False)
            new_cost = numpy.mean([self.cost_function(minibatch)
                                   for minibatch in it])
        if new_cost < self.best_cost:
            self.best_cost = new_cost
            self.best_params = self.model.get_param_values()

    def get_best_params(self):
        """Returns the best parameters up to now for the model."""
        return self.best_params

class MonitorBasedSaveBest(TrainExtension):
    """
    A callback that saves a copy of the model every time it achieves
    a new minimal value of a monitoring channel.

    Parameters
    ----------
    channel_name : str
        The name of the channel we want to minimize
    save_path : str
        Path to save the best model to
    higher_is_better : bool, optional
        WRITEME
    """

    def __init__(self, channel_name, save_path,higher_is_better=False):
        self.__dict__.update(locals())
        del self.self
        if higher_is_better:
            self.coeff = -1.
        else:
            self.coeff = 1.
        self.best_cost = np.inf

    def on_monitor(self, model, dataset, algorithm):
        """
        Looks whether the model performs better than earlier. If it's the
        case, saves the model.

        Parameters
        ----------
        model : pylearn2.models.model.Model
            model.monitor must contain a channel with name given by
            self.channel_name
        dataset : pylearn2.datasets.dataset.Dataset
            Not used
        algorithm : TrainingAlgorithm
            Not used
        """

        monitor = model.monitor
        channels = monitor.channels
        channel = channels[self.channel_name]
        val_record = channel.val_record
        new_cost = self.coeff * val_record[-1]

        if new_cost < self.best_cost:
            self.best_cost = new_cost
            serial.save(self.save_path, model, on_overwrite = 'backup')

class MonitorBasedStoreBest(TrainExtension):
    """
    Saves the best model in memory.

    Parameters
    ----------
    channel_name : str
        Channel to monitor.
    higher_is_better : bool
        Whether a higher channel value indicates a better model.
    """
    def __init__(self, channel_name, higher_is_better=False):
        self.channel_name = channel_name
        self.higher_is_better = higher_is_better
        self.best_cost = np.inf
        self.best_model = None

    def on_monitor(self, model, dataset, algorithm):
        """
        Check if the model performs better than before. Save best models in
        memory to avoid race conditions.

        Parameters
        ----------
        model : Model
            Model to monitor.
        dataset : Dataset
            Training dataset.
        algorithm : TrainingAlgorithm
            Training algorithm.
        """
        monitor = model.monitor
        channels = monitor.channels
        channel = channels[self.channel_name]
        val_record = channel.val_record
        if self.higher_is_better:
            new_cost = -1 * val_record[-1]
        else:
            new_cost = val_record[-1]

        if new_cost < self.best_cost:
            self.best_cost = new_cost
            self.best_model = deepcopy(model)
<|endoftext|>


<commit_before>import os
import sys

def saydo(cmd, echo=True, logFN=None, bLogDate=True):
    if echo:
        print(cmd + '\n')

    if logFN == None or logFN == "":
        os.system(cmd)
    else:
        if bLogDate:
            import datetime
            dtStr = datetime.datetime.isoformat(datetime.datetime.now())

        logF = open(logFN, "at")

        if bLogDate:
            logF.write("\nCOMMAND @ %s: %s\n\n" % (dtStr, cmd))
        else:
            logF.write("\nCOMMAND: %s\n\n" % cmd)

        logF.close()

        os.system("%s 2>&1 | tee -a %s" % (cmd, logFN))

def info_log(info, logFN=None, bWarn=False):
    import datetime

    if not bWarn:
        str = "INFO @ %s: %s" \
              % (datetime.datetime.isoformat(datetime.datetime.now()), info)
    else:
        str = "WARNING @ %s: %s" \
              % (datetime.datetime.isoformat(datetime.datetime.now()), info)

    if logFN != None and logFN != "":
        logF = open(logFN, "at")
        logF.write("%s\n" % str)
        logF.close()

    if not bWarn:
        print(str)
    else:
        print("\033[93m" + str + "\033[0m")

def error_log(errInfo, logFN=None):
    import datetime

    errDTStr = "ERROR @ %s: %s" \
               % (datetime.datetime.isoformat(datetime.datetime.now()),
                  errInfo)

    if logFN != None and logFN != "":
        logF = open(logFN, "at")
        logF.write("\n%s\n" % errDTStr)
        logF.write("%s\n\n" % errInfo)
        logF.close()

    print("\033[91m" + errDTStr + "\033[0m")
    raise Exception, errInfo

def qsubmit(cmd, queue, jobname):
    os.system('ezsub -c "%s" -q %s -n %s'%(cmd, queue, jobname))

def remove_empty_strings(lst):
    while lst.count('') > 0:
        lst.remove('')
    return lst

def read_ctab(ctabfn):
    ctabf = open(ctabfn, 'r')
    ctxt = ctabf.read().split('\n')
    ctabf.close()

    roi_nums = []
    roi_names = []
    for clin in ctxt:
        clin = clin.replace("\t", " ")

        if len(clin) == 0:
            continue
        clin = clin.split(' ')
        while clin.count('') > 0:
            clin.remove('')

        #print(clin) # DEBUG

        if clin[1] == 'unknown' or clin[1] == 'bankssts' \
           or clin[1] == "corpuscallosum" or clin[1] == "Unknown" \
           or clin[1].startswith("None"):
            continue

        roi_nums.append(int(clin[0]))
        roi_names.append(clin[1])

    return (roi_nums, roi_names)

def check_file(fn, logFN=None):
    if not os.path.isfile(fn):
        error_log("Missing file: %s" % fn, logFN=logFN)

def check_dir(dn, bCreate=False, logFN=None):
    if not os.path.isdir(dn):
        if not bCreate:
            error_log("Missing directory: %s" % dn, logFN=logFN)
        else:
            saydo("mkdir -p %s" % dn, logFN=logFN)
            if not os.path.isdir(dn):
                error_log("Failed to create directory: %s" % dn, logFN=logFN)
            else:
                info_log("INFO: Created directory: %s"%dn, logFN=logFN)

def delete_file_if_exists(fn, recursive=False):
    if os.path.isfile(fn):
        if not recursive:
            os.system("rm -f %s" % fn)
        else:
            os.system("rm -rf %s" % fn)

        if os.path.isfile(fn):
            raise Exception, "Failed to remove file: %s" % fn

def read_text_file(txtfn):
    txtf = open(txtfn, "rt")
    t = txtf.read().split("\n")
    txtf.close()

    return t

def write_list_to_text_file(tlist, txtfn):
    txtf = open(txtfn, "wt")
    for (i0, tline) in enumerate(tlist):
        txtf.write("%s\n" % tline)
    txtf.close()

def cmd_stdout(cmd):
    from subprocess import Popen, PIPE

    (sout, serr) = Popen(cmd.split(" "), stdout=PIPE, stderr=PIPE)\
                   .communicate()
    return (sout, serr)

def check_bin_path(bfn, logFN=None):
    (tpath, se) = cmd_stdout("which %s" % bfn)
    assert(len(se) == 0)

    if len(tpath) == 0:
        error_log("Cannot find the path to program: %s" % bfn, logFN=logFN)

    return tpath
<commit_msg>Some minor changes to scai_utils.py<commit_after>import os
import sys

def saydo(cmd, echo=True, logFN=None, bLogDate=True):
    if bLogDate:
        import datetime
        dtStr = datetime.datetime.isoformat(datetime.datetime.now())

    if echo:
        if bLogDate:
            print("Running command @ %s:" % dtStr)
            print(cmd + '\n')

    if logFN == None or logFN == "":

        os.system(cmd)
    else:

        logF = open(logFN, "at")

        if bLogDate:
            logF.write("\nCOMMAND @ %s: %s\n\n" % (dtStr, cmd))
        else:
            logF.write("\nCOMMAND: %s\n\n" % cmd)

        logF.close()

        os.system("%s 2>&1 | tee -a %s" % (cmd, logFN))

def info_log(info, logFN=None, bWarn=False):
    import datetime

    if not bWarn:
        str = "INFO @ %s: %s" \
              % (datetime.datetime.isoformat(datetime.datetime.now()), info)
    else:
        str = "WARNING @ %s: %s" \
              % (datetime.datetime.isoformat(datetime.datetime.now()), info)

    if logFN != None and logFN != "":
        logF = open(logFN, "at")
        logF.write("%s\n" % str)
        logF.close()

    if not bWarn:
        print(str)
    else:
        print("\033[93m" + str + "\033[0m")

def error_log(errInfo, logFN=None):
    import datetime

    errDTStr = "ERROR @ %s: %s" \
               % (datetime.datetime.isoformat(datetime.datetime.now()),
                  errInfo)

    if logFN != None and logFN != "":
        logF = open(logFN, "at")
        logF.write("\n%s\n" % errDTStr)
        logF.write("%s\n\n" % errInfo)
        logF.close()

    print("\033[91m" + errDTStr + "\033[0m")
    raise Exception, errInfo

def qsubmit(cmd, queue, jobname):
    os.system('ezsub -c "%s" -q %s -n %s'%(cmd, queue, jobname))

def remove_empty_strings(lst):
    while lst.count('') > 0:
        lst.remove('')
    return lst

def read_ctab(ctabfn):
    ctabf = open(ctabfn, 'r')
    ctxt = ctabf.read().split('\n')
    ctabf.close()

    roi_nums = []
    roi_names = []
    for clin in ctxt:
        clin = clin.replace("\t", " ")

        if len(clin) == 0:
            continue
        clin = clin.split(' ')
        while clin.count('') > 0:
            clin.remove('')

        #print(clin) # DEBUG

        if clin[1] == 'unknown' or clin[1] == 'bankssts' \
           or clin[1] == "corpuscallosum" or clin[1] == "Unknown" \
           or clin[1].startswith("None"):
            continue

        roi_nums.append(int(clin[0]))
        roi_names.append(clin[1])

    return (roi_nums, roi_names)

def check_file(fn, logFN=None):
    if not os.path.isfile(fn):
        error_log("Missing file: %s" % fn, logFN=logFN)

def check_dir(dn, bCreate=False, logFN=None):
    if not os.path.isdir(dn):
        if not bCreate:
            error_log("Missing directory: %s" % dn, logFN=logFN)
        else:
            saydo("mkdir -p %s" % dn, logFN=logFN)
            if not os.path.isdir(dn):
                error_log("Failed to create directory: %s" % dn, logFN=logFN)
            else:
                info_log("Created directory: %s"%dn, logFN=logFN)

def delete_file_if_exists(fn, recursive=False):
    if os.path.isfile(fn):
        if not recursive:
            os.system("rm -f %s" % fn)
        else:
            os.system("rm -rf %s" % fn)

        if os.path.isfile(fn):
            raise Exception, "Failed to remove file: %s" % fn

def read_text_file(txtfn):
    txtf = open(txtfn, "rt")
    t = txtf.read().split("\n")
    txtf.close()

    return t

def write_list_to_text_file(tlist, txtfn):
    txtf = open(txtfn, "wt")
    for (i0, tline) in enumerate(tlist):
        txtf.write("%s\n" % tline)
    txtf.close()

def cmd_stdout(cmd):
    from subprocess import Popen, PIPE

    (sout, serr) = Popen(cmd.split(" "), stdout=PIPE, stderr=PIPE)\
                   .communicate()
    return (sout, serr)

def check_bin_path(bfn, logFN=None):
    (tpath, se) = cmd_stdout("which %s" % bfn)
    assert(len(se) == 0)

    if len(tpath) == 0:
        error_log("Cannot find the path to program: %s" % bfn, logFN=logFN)

    return tpath
<|endoftext|>


<commit_before>"""Use a directory of zoom/row/column tiles as input."""

import logging
import os
from shapely.geometry import box

from mapchete.config import validate_values
from mapchete.errors import MapcheteConfigError
from mapchete.formats import (
    base,
    data_type_from_extension,
    driver_metadata,
    load_output_writer,
    read_output_metadata,
)
from mapchete.io import path_exists, absolute_path, tile_to_zoom_level
from mapchete.io.vector import reproject_geometry, segmentize_geometry
from mapchete.tile import BufferedTilePyramid

logger = logging.getLogger(__name__)
METADATA = {
    "driver_name": "TileDirectory",
    "data_type": None,
    "mode": "r",
    "file_extensions": None,
}

class InputData(base.InputData):
    """
    Main input class.

    Parameters
    ----------
    input_params : dictionary
        driver specific parameters

    Attributes
    ----------
    path : string
        path to Mapchete file
    pixelbuffer : integer
        buffer around output tiles
    pyramid : ``tilematrix.TilePyramid``
        output ``TilePyramid``
    crs : ``rasterio.crs.CRS``
        object describing the process coordinate reference system
    srid : string
        spatial reference ID of CRS (e.g. "{'init': 'epsg:4326'}")
    """

    METADATA = METADATA

    def __init__(self, input_params, **kwargs):
        """Initialize."""
        super().__init__(input_params, **kwargs)
        if "abstract" in input_params:
            self._params = input_params["abstract"]
            self.path = absolute_path(
                path=self._params["path"], base_dir=input_params["conf_dir"]
            )
            logger.debug("InputData params: %s", input_params)
            # define pyramid
            self.td_pyramid = BufferedTilePyramid(
                self._params["grid"],
                metatiling=self._params.get("metatiling", 1),
                tile_size=self._params.get("tile_size", 256),
                pixelbuffer=self._params.get("pixelbuffer", 0),
            )
            self._read_as_tiledir_func = base._read_as_tiledir
            try:
                self._tiledir_metadata_json = read_output_metadata(
                    os.path.join(self.path, "metadata.json")
                )
                try:
                    self._data_type = self._tiledir_metadata_json["driver"]["data_type"]
                except KeyError:
                    self._data_type = driver_metadata(
                        self._tiledir_metadata_json["driver"]["format"]
                    )["data_type"]
            except FileNotFoundError:
                # in case no metadata.json is available, try to guess data type via the
                # format file extension
                self._data_type = data_type_from_extension(self._params["extension"])

        elif "path" in input_params:
            self.path = absolute_path(
                path=input_params["path"], base_dir=input_params.get("conf_dir")
            )
            try:
                self._tiledir_metadata_json = read_output_metadata(
                    os.path.join(self.path, "metadata.json")
                )
            except FileNotFoundError:
                raise MapcheteConfigError(f"Cannot find metadata.json in {self.path}")
            # define pyramid
            self.td_pyramid = self._tiledir_metadata_json["pyramid"]
            self.output_data = load_output_writer(
                dict(
                    self._tiledir_metadata_json["driver"],
                    metatiling=self.td_pyramid.metatiling,
                    pixelbuffer=self.td_pyramid.pixelbuffer,
                    pyramid=self.td_pyramid,
                    grid=self.td_pyramid.grid,
                    path=self.path,
                ),
                readonly=True,
            )
            self._params = dict(
                path=self.path,
                grid=self.td_pyramid.grid.to_dict(),
                metatiling=self.td_pyramid.metatiling,
                pixelbuffer=self.td_pyramid.pixelbuffer,
                tile_size=self.td_pyramid.tile_size,
                extension=self.output_data.file_extension.split(".")[-1],
                **self._tiledir_metadata_json["driver"],
            )
            self._read_as_tiledir_func = self.output_data._read_as_tiledir
            self._data_type = driver_metadata(
                self._tiledir_metadata_json["driver"]["format"]
            )["data_type"]

        # validate parameters
        validate_values(
            self._params, [("path", str), ("grid", (str, dict)), ("extension", str)]
        )
        self._ext = self._params["extension"]

        # additional params
        self._bounds = self._params.get("bounds", self.td_pyramid.bounds)
        self.METADATA.update(
            data_type=self._data_type, file_extensions=[self._params["extension"]]
        )
        if self.METADATA.get("data_type") == "raster":
            self._params["count"] = self._params.get(
                "count", self._params.get("bands", None)
            )
            validate_values(self._params, [("dtype", str), ("count", int)])
            self._profile = {
                "nodata": self._params.get("nodata", 0),
                "dtype": self._params["dtype"],
                "count": self._params["count"],
            }
        else:
            self._profile = None

    def open(self, tile, **kwargs):
        """
        Return InputTile object.

        Parameters
        ----------
        tile : ``Tile``

        Returns
        -------
        input tile : ``InputTile``
            tile view of input data
        """
        return InputTile(
            tile,
            data_type=self.METADATA.get("data_type"),
            basepath=self.path,
            file_extension=self._ext,
            profile=self._profile,
            td_pyramid=self.td_pyramid,
            read_as_tiledir_func=self._read_as_tiledir_func,
            **kwargs,
        )

    def bbox(self, out_crs=None):
        """
        Return data bounding box.

        Parameters
        ----------
        out_crs : ``rasterio.crs.CRS``
            rasterio CRS object (default: CRS of process pyramid)

        Returns
        -------
        bounding box : geometry
            Shapely geometry object
        """
        return reproject_geometry(
            box(*self._bounds),
            src_crs=self.td_pyramid.crs,
            dst_crs=self.pyramid.crs if out_crs is None else out_crs,
            segmentize_on_clip=True,
        )

def _get_tiles_paths(basepath=None, ext=None, pyramid=None, bounds=None, zoom=None):
    return [
        (_tile, _path)
        for _tile, _path in [
            (
                t,
                "%s.%s"
                % (
                    os.path.join(*([basepath, str(t.zoom), str(t.row), str(t.col)])),
                    ext,
                ),
            )
            for t in pyramid.tiles_from_bounds(bounds, zoom)
        ]
        if path_exists(_path)
    ]

class InputTile(base.InputTile):
    """
    Target Tile representation of input data.

    Parameters
    ----------
    tile : ``Tile``
    kwargs : keyword arguments
        driver specific parameters

    Attributes
    ----------
    tile : tile : ``Tile``
    """

    def __init__(
        self,
        tile,
        data_type=None,
        basepath=None,
        file_extension=None,
        profile=None,
        td_crs=None,
        td_pyramid=None,
        read_as_tiledir_func=None,
    ):
        """Initialize."""
        self.tile = tile
        self._data_type = data_type
        self._basepath = basepath
        self._ext = file_extension
        self._profile = profile
        self._td_pyramid = td_pyramid
        self._read_as_tiledir = read_as_tiledir_func

    def read(
        self,
        indexes=None,
        resampling="nearest",
        tile_directory_zoom=None,
        matching_method="gdal",
        matching_max_zoom=None,
        matching_precision=8,
        fallback_to_higher_zoom=False,
        validity_check=False,
        dst_nodata=None,
        gdal_opts=None,
        **kwargs,
    ):
        """
        Read reprojected & resampled input data.

        Parameters
        ----------
        tile_directory_zoom : None
            If set, data will be read from exactly this zoom level
        matching_method : str ('gdal' or 'min') (default: 'gdal')
            gdal: Uses GDAL's standard method. Here, the target resolution is calculated
                by averaging the extent's pixel sizes over both x and y axes. This
                approach returns a zoom level which may not have the best quality but will
                speed up reading significantly.
            min: Returns the zoom level which matches the minimum resolution of the
                extents four corner pixels. This approach returns the zoom level with the
                best possible quality but with low performance. If the tile extent is
                outside of the destination pyramid, a TopologicalError will be raised.
        matching_max_zoom : int (default: None)
            If set, it will prevent reading from zoom levels above the maximum.
        matching_precision : int
            Round resolutions to n digits before comparing.
        fallback_to_higher_zoom : bool (default: False)
            In case no data is found at zoom level, try to read data from higher zoom
            levels. Enabling this setting can lead to many IO requests in areas with no
            data.
        resampling : string
            raster file: one of "nearest", "average", "bilinear" or "lanczos"
        validity_check : bool
            vector file: also run checks if reprojected geometry is valid,
            otherwise throw RuntimeError (default: True)
        indexes : list or int
            raster file: a list of band numbers; None will read all.
        dst_nodata : int or float, optional
            raster file: if not set, the nodata value from the source dataset
            will be used
        gdal_opts : dict
            raster file: GDAL options passed on to rasterio.Env()

        Returns
        -------
        data : list for vector files or numpy array for raster files
        """
        return self._read_as_tiledir(
            data_type=self._data_type,
            out_tile=self.tile,
            td_crs=self._td_pyramid.crs,
            tiles_paths=self._get_tiles_paths(
                tile_directory_zoom=tile_directory_zoom,
                fallback_to_higher_zoom=fallback_to_higher_zoom,
                matching_method=matching_method,
                matching_precision=matching_precision,
                matching_max_zoom=matching_max_zoom,
            ),
            profile=self._profile,
            validity_check=validity_check,
            indexes=indexes,
            resampling=resampling,
            dst_nodata=dst_nodata,
            gdal_opts=gdal_opts,
            **{k: v for k, v in kwargs.items() if k != "data_type"},
        )

    def is_empty(
        self,
        tile_directory_zoom=None,
        fallback_to_higher_zoom=False,
        matching_method="gdal",
        matching_precision=8,
        matching_max_zoom=None,
    ):
        """
        Check if there is data within this tile.

        Parameters
        ----------
        tile_directory_zoom : None
            If set, data will be read from exactly this zoom level
        matching_method : str ('gdal' or 'min') (default: 'gdal')
            gdal: Uses GDAL's standard method. Here, the target resolution is calculated
                by averaging the extent's pixel sizes over both x and y axes. This
                approach returns a zoom level which may not have the best quality but will
                speed up reading significantly.
            min: Returns the zoom level which matches the minimum resolution of the
                extents four corner pixels. This approach returns the zoom level with the
                best possible quality but with low performance. If the tile extent is
                outside of the destination pyramid, a TopologicalError will be raised.
        matching_max_zoom : int (default: None)
            If set, it will prevent reading from zoom levels above the maximum.
        matching_precision : int
            Round resolutions to n digits before comparing.
        fallback_to_higher_zoom : bool (default: False)
            In case no data is found at zoom level, try to read data from higher zoom
            levels. Enabling this setting can lead to many IO requests in areas with no
            data.

        Returns
        -------
        is empty : bool
        """
        return (
            len(
                self._get_tiles_paths(
                    tile_directory_zoom=tile_directory_zoom,
                    fallback_to_higher_zoom=fallback_to_higher_zoom,
                    matching_method=matching_method,
                    matching_precision=matching_precision,
                    matching_max_zoom=matching_max_zoom,
                )
            )
            == 0
        )  # pragma: no cover

    def _get_tiles_paths(
        self,
        tile_directory_zoom=None,
        fallback_to_higher_zoom=False,
        matching_method="gdal",
        matching_precision=8,
        matching_max_zoom=None,
    ):
        # determine tile bounds in TileDirectory CRS
        td_bounds = reproject_geometry(
            self.tile.bbox, src_crs=self.tile.tp.crs, dst_crs=self._td_pyramid.crs
        ).bounds

        # find target zoom level
        if tile_directory_zoom is None:
            zoom = tile_to_zoom_level(
                self.tile,
                dst_pyramid=self._td_pyramid,
                matching_method=matching_method,
                precision=matching_precision,
            )
            if matching_max_zoom is not None:
                zoom = min([zoom, matching_max_zoom])
        else:
            zoom = tile_directory_zoom

        if fallback_to_higher_zoom:
            tiles_paths = []
            # check if tiles exist otherwise try higher zoom level
            while len(tiles_paths) == 0 and zoom >= 0:
                tiles_paths = _get_tiles_paths(
                    basepath=self._basepath,
                    ext=self._ext,
                    pyramid=self._td_pyramid,
                    bounds=td_bounds,
                    zoom=zoom,
                )
                logger.debug(
                    "%s existing tiles found at zoom %s", len(tiles_paths), zoom
                )
                zoom -= 1
        else:
            tiles_paths = _get_tiles_paths(
                basepath=self._basepath,
                ext=self._ext,
                pyramid=self._td_pyramid,
                bounds=td_bounds,
                zoom=zoom,
            )
            logger.debug("%s existing tiles found at zoom %s", len(tiles_paths), zoom)

        return tiles_paths
<commit_msg>deactivate bbox segmentize to see whether this makes the gh action fail<commit_after>"""Use a directory of zoom/row/column tiles as input."""

import logging
import os
from shapely.geometry import box

from mapchete.config import validate_values
from mapchete.errors import MapcheteConfigError
from mapchete.formats import (
    base,
    data_type_from_extension,
    driver_metadata,
    load_output_writer,
    read_output_metadata,
)
from mapchete.io import path_exists, absolute_path, tile_to_zoom_level
from mapchete.io.vector import reproject_geometry, segmentize_geometry
from mapchete.tile import BufferedTilePyramid

logger = logging.getLogger(__name__)
METADATA = {
    "driver_name": "TileDirectory",
    "data_type": None,
    "mode": "r",
    "file_extensions": None,
}

class InputData(base.InputData):
    """
    Main input class.

    Parameters
    ----------
    input_params : dictionary
        driver specific parameters

    Attributes
    ----------
    path : string
        path to Mapchete file
    pixelbuffer : integer
        buffer around output tiles
    pyramid : ``tilematrix.TilePyramid``
        output ``TilePyramid``
    crs : ``rasterio.crs.CRS``
        object describing the process coordinate reference system
    srid : string
        spatial reference ID of CRS (e.g. "{'init': 'epsg:4326'}")
    """

    METADATA = METADATA

    def __init__(self, input_params, **kwargs):
        """Initialize."""
        super().__init__(input_params, **kwargs)
        if "abstract" in input_params:
            self._params = input_params["abstract"]
            self.path = absolute_path(
                path=self._params["path"], base_dir=input_params["conf_dir"]
            )
            logger.debug("InputData params: %s", input_params)
            # define pyramid
            self.td_pyramid = BufferedTilePyramid(
                self._params["grid"],
                metatiling=self._params.get("metatiling", 1),
                tile_size=self._params.get("tile_size", 256),
                pixelbuffer=self._params.get("pixelbuffer", 0),
            )
            self._read_as_tiledir_func = base._read_as_tiledir
            try:
                self._tiledir_metadata_json = read_output_metadata(
                    os.path.join(self.path, "metadata.json")
                )
                try:
                    self._data_type = self._tiledir_metadata_json["driver"]["data_type"]
                except KeyError:
                    self._data_type = driver_metadata(
                        self._tiledir_metadata_json["driver"]["format"]
                    )["data_type"]
            except FileNotFoundError:
                # in case no metadata.json is available, try to guess data type via the
                # format file extension
                self._data_type = data_type_from_extension(self._params["extension"])

        elif "path" in input_params:
            self.path = absolute_path(
                path=input_params["path"], base_dir=input_params.get("conf_dir")
            )
            try:
                self._tiledir_metadata_json = read_output_metadata(
                    os.path.join(self.path, "metadata.json")
                )
            except FileNotFoundError:
                raise MapcheteConfigError(f"Cannot find metadata.json in {self.path}")
            # define pyramid
            self.td_pyramid = self._tiledir_metadata_json["pyramid"]
            self.output_data = load_output_writer(
                dict(
                    self._tiledir_metadata_json["driver"],
                    metatiling=self.td_pyramid.metatiling,
                    pixelbuffer=self.td_pyramid.pixelbuffer,
                    pyramid=self.td_pyramid,
                    grid=self.td_pyramid.grid,
                    path=self.path,
                ),
                readonly=True,
            )
            self._params = dict(
                path=self.path,
                grid=self.td_pyramid.grid.to_dict(),
                metatiling=self.td_pyramid.metatiling,
                pixelbuffer=self.td_pyramid.pixelbuffer,
                tile_size=self.td_pyramid.tile_size,
                extension=self.output_data.file_extension.split(".")[-1],
                **self._tiledir_metadata_json["driver"],
            )
            self._read_as_tiledir_func = self.output_data._read_as_tiledir
            self._data_type = driver_metadata(
                self._tiledir_metadata_json["driver"]["format"]
            )["data_type"]

        # validate parameters
        validate_values(
            self._params, [("path", str), ("grid", (str, dict)), ("extension", str)]
        )
        self._ext = self._params["extension"]

        # additional params
        self._bounds = self._params.get("bounds", self.td_pyramid.bounds)
        self.METADATA.update(
            data_type=self._data_type, file_extensions=[self._params["extension"]]
        )
        if self.METADATA.get("data_type") == "raster":
            self._params["count"] = self._params.get(
                "count", self._params.get("bands", None)
            )
            validate_values(self._params, [("dtype", str), ("count", int)])
            self._profile = {
                "nodata": self._params.get("nodata", 0),
                "dtype": self._params["dtype"],
                "count": self._params["count"],
            }
        else:
            self._profile = None

    def open(self, tile, **kwargs):
        """
        Return InputTile object.

        Parameters
        ----------
        tile : ``Tile``

        Returns
        -------
        input tile : ``InputTile``
            tile view of input data
        """
        return InputTile(
            tile,
            data_type=self.METADATA.get("data_type"),
            basepath=self.path,
            file_extension=self._ext,
            profile=self._profile,
            td_pyramid=self.td_pyramid,
            read_as_tiledir_func=self._read_as_tiledir_func,
            **kwargs,
        )

    def bbox(self, out_crs=None):
        """
        Return data bounding box.

        Parameters
        ----------
        out_crs : ``rasterio.crs.CRS``
            rasterio CRS object (default: CRS of process pyramid)

        Returns
        -------
        bounding box : geometry
            Shapely geometry object
        """
        return reproject_geometry(
            box(*self._bounds),
            src_crs=self.td_pyramid.crs,
            dst_crs=self.pyramid.crs if out_crs is None else out_crs,
            # segmentize_on_clip=True,
        )

def _get_tiles_paths(basepath=None, ext=None, pyramid=None, bounds=None, zoom=None):
    return [
        (_tile, _path)
        for _tile, _path in [
            (
                t,
                "%s.%s"
                % (
                    os.path.join(*([basepath, str(t.zoom), str(t.row), str(t.col)])),
                    ext,
                ),
            )
            for t in pyramid.tiles_from_bounds(bounds, zoom)
        ]
        if path_exists(_path)
    ]

class InputTile(base.InputTile):
    """
    Target Tile representation of input data.

    Parameters
    ----------
    tile : ``Tile``
    kwargs : keyword arguments
        driver specific parameters

    Attributes
    ----------
    tile : tile : ``Tile``
    """

    def __init__(
        self,
        tile,
        data_type=None,
        basepath=None,
        file_extension=None,
        profile=None,
        td_crs=None,
        td_pyramid=None,
        read_as_tiledir_func=None,
    ):
        """Initialize."""
        self.tile = tile
        self._data_type = data_type
        self._basepath = basepath
        self._ext = file_extension
        self._profile = profile
        self._td_pyramid = td_pyramid
        self._read_as_tiledir = read_as_tiledir_func

    def read(
        self,
        indexes=None,
        resampling="nearest",
        tile_directory_zoom=None,
        matching_method="gdal",
        matching_max_zoom=None,
        matching_precision=8,
        fallback_to_higher_zoom=False,
        validity_check=False,
        dst_nodata=None,
        gdal_opts=None,
        **kwargs,
    ):
        """
        Read reprojected & resampled input data.

        Parameters
        ----------
        tile_directory_zoom : None
            If set, data will be read from exactly this zoom level
        matching_method : str ('gdal' or 'min') (default: 'gdal')
            gdal: Uses GDAL's standard method. Here, the target resolution is calculated
                by averaging the extent's pixel sizes over both x and y axes. This
                approach returns a zoom level which may not have the best quality but will
                speed up reading significantly.
            min: Returns the zoom level which matches the minimum resolution of the
                extents four corner pixels. This approach returns the zoom level with the
                best possible quality but with low performance. If the tile extent is
                outside of the destination pyramid, a TopologicalError will be raised.
        matching_max_zoom : int (default: None)
            If set, it will prevent reading from zoom levels above the maximum.
        matching_precision : int
            Round resolutions to n digits before comparing.
        fallback_to_higher_zoom : bool (default: False)
            In case no data is found at zoom level, try to read data from higher zoom
            levels. Enabling this setting can lead to many IO requests in areas with no
            data.
        resampling : string
            raster file: one of "nearest", "average", "bilinear" or "lanczos"
        validity_check : bool
            vector file: also run checks if reprojected geometry is valid,
            otherwise throw RuntimeError (default: True)
        indexes : list or int
            raster file: a list of band numbers; None will read all.
        dst_nodata : int or float, optional
            raster file: if not set, the nodata value from the source dataset
            will be used
        gdal_opts : dict
            raster file: GDAL options passed on to rasterio.Env()

        Returns
        -------
        data : list for vector files or numpy array for raster files
        """
        return self._read_as_tiledir(
            data_type=self._data_type,
            out_tile=self.tile,
            td_crs=self._td_pyramid.crs,
            tiles_paths=self._get_tiles_paths(
                tile_directory_zoom=tile_directory_zoom,
                fallback_to_higher_zoom=fallback_to_higher_zoom,
                matching_method=matching_method,
                matching_precision=matching_precision,
                matching_max_zoom=matching_max_zoom,
            ),
            profile=self._profile,
            validity_check=validity_check,
            indexes=indexes,
            resampling=resampling,
            dst_nodata=dst_nodata,
            gdal_opts=gdal_opts,
            **{k: v for k, v in kwargs.items() if k != "data_type"},
        )

    def is_empty(
        self,
        tile_directory_zoom=None,
        fallback_to_higher_zoom=False,
        matching_method="gdal",
        matching_precision=8,
        matching_max_zoom=None,
    ):
        """
        Check if there is data within this tile.

        Parameters
        ----------
        tile_directory_zoom : None
            If set, data will be read from exactly this zoom level
        matching_method : str ('gdal' or 'min') (default: 'gdal')
            gdal: Uses GDAL's standard method. Here, the target resolution is calculated
                by averaging the extent's pixel sizes over both x and y axes. This
                approach returns a zoom level which may not have the best quality but will
                speed up reading significantly.
            min: Returns the zoom level which matches the minimum resolution of the
                extents four corner pixels. This approach returns the zoom level with the
                best possible quality but with low performance. If the tile extent is
                outside of the destination pyramid, a TopologicalError will be raised.
        matching_max_zoom : int (default: None)
            If set, it will prevent reading from zoom levels above the maximum.
        matching_precision : int
            Round resolutions to n digits before comparing.
        fallback_to_higher_zoom : bool (default: False)
            In case no data is found at zoom level, try to read data from higher zoom
            levels. Enabling this setting can lead to many IO requests in areas with no
            data.

        Returns
        -------
        is empty : bool
        """
        return (
            len(
                self._get_tiles_paths(
                    tile_directory_zoom=tile_directory_zoom,
                    fallback_to_higher_zoom=fallback_to_higher_zoom,
                    matching_method=matching_method,
                    matching_precision=matching_precision,
                    matching_max_zoom=matching_max_zoom,
                )
            )
            == 0
        )  # pragma: no cover

    def _get_tiles_paths(
        self,
        tile_directory_zoom=None,
        fallback_to_higher_zoom=False,
        matching_method="gdal",
        matching_precision=8,
        matching_max_zoom=None,
    ):
        # determine tile bounds in TileDirectory CRS
        td_bounds = reproject_geometry(
            self.tile.bbox, src_crs=self.tile.tp.crs, dst_crs=self._td_pyramid.crs
        ).bounds

        # find target zoom level
        if tile_directory_zoom is None:
            zoom = tile_to_zoom_level(
                self.tile,
                dst_pyramid=self._td_pyramid,
                matching_method=matching_method,
                precision=matching_precision,
            )
            if matching_max_zoom is not None:
                zoom = min([zoom, matching_max_zoom])
        else:
            zoom = tile_directory_zoom

        if fallback_to_higher_zoom:
            tiles_paths = []
            # check if tiles exist otherwise try higher zoom level
            while len(tiles_paths) == 0 and zoom >= 0:
                tiles_paths = _get_tiles_paths(
                    basepath=self._basepath,
                    ext=self._ext,
                    pyramid=self._td_pyramid,
                    bounds=td_bounds,
                    zoom=zoom,
                )
                logger.debug(
                    "%s existing tiles found at zoom %s", len(tiles_paths), zoom
                )
                zoom -= 1
        else:
            tiles_paths = _get_tiles_paths(
                basepath=self._basepath,
                ext=self._ext,
                pyramid=self._td_pyramid,
                bounds=td_bounds,
                zoom=zoom,
            )
            logger.debug("%s existing tiles found at zoom %s", len(tiles_paths), zoom)

        return tiles_paths
<|endoftext|>


<commit_before>import time
from math import cos, radians, sin

from demosys.opengl.projection import Projection
from pyrr import Vector3, matrix44, vector, vector3

# Direction Definitions
RIGHT = 1
LEFT = 2
FORWARD = 3
BACKWARD = 4
UP = 5
DOWN = 6

# Movement Definitions
STILL = 0
POSITIVE = 1
NEGATIVE = 2

class Camera:
    """Simple camera class containing projection"""
    def __init__(self, fov=60, aspect=1.0, near=1, far=100):
        """
        Initialize camera using a specific projection

        :param fov: Field of view
        :param aspect: Aspect ratio
        :param near: Near plane
        :param far: Far plane
        """
        self.position = Vector3([0.0, 0.0, 0.0])
        # Default camera placement
        self.up = Vector3([0.0, 1.0, 0.0])
        self.right = Vector3([1.0, 0.0, 0.0])
        self.dir = Vector3([0.0, 0.0, -1.0])
        # Yaw and Pitch
        self.yaw = -90.0
        self.pitch = 0.0

        # World up vector
        self._up = Vector3([0.0, 1.0, 0.0])

        # Projection
        self.projection = Projection(aspect, fov, near, far)

    def set_position(self, x, y, z):
        """
        Set the 3D position of the camera

        :param x: float
        :param y: float
        :param z: float
        """
        self.position = Vector3([x, y, z])

    @property
    def view_matrix(self):
        """
        :return: The current view matrix for the camera
        """
        self._update_yaw_and_pitch()
        return self._gl_look_at(self.position, self.position + self.dir, self._up)

    def _update_yaw_and_pitch(self):
        """
        Updates the camera vectors based on the current yaw and pitch
        """
        front = Vector3([0.0, 0.0, 0.0])
        front.x = cos(radians(self.yaw)) * cos(radians(self.pitch))
        front.y = sin(radians(self.pitch))
        front.z = sin(radians(self.yaw)) * cos(radians(self.pitch))

        self.dir = vector.normalise(front)
        self.right = vector.normalise(vector3.cross(self.dir, self._up))
        self.up = vector.normalise(vector3.cross(self.right, self.dir))

    def look_at(self, vec=None, pos=None):
        """
        Look at a specific point

        :param vec: Vector3 position
        :param pos: python list [x, y, x]
        :return: Camera matrix
        """
        if pos:
            vec = Vector3(pos)
        if vec is None:
            raise ValueError("vector or pos must be set")
        return self._gl_look_at(self.position, vec, self._up)

    def _gl_look_at(self, pos, target, up):
        """
        The standard lookAt method

        :param pos: current position
        :param target: target position to look at
        :param up: direction up
        """
        z = vector.normalise(pos - target)
        x = vector.normalise(vector3.cross(vector.normalise(up), z))
        y = vector3.cross(z, x)

        translate = matrix44.create_identity()
        translate[3][0] = -pos.x
        translate[3][1] = -pos.y
        translate[3][2] = -pos.z

        rotate = matrix44.create_identity()
        rotate[0][0] = x[0]  # -- X
        rotate[1][0] = x[1]
        rotate[2][0] = x[2]
        rotate[0][1] = y[0]  # -- Y
        rotate[1][1] = y[1]
        rotate[2][1] = y[2]
        rotate[0][2] = z[0]  # -- Z
        rotate[1][2] = z[1]
        rotate[2][2] = z[2]

        return matrix44.multiply(translate, rotate)

class SystemCamera(Camera):
    """System camera controlled by mouse and keyboard"""
    def __init__(self, fov=60, aspect=1.0, near=1, far=100):
        # Position movement states
        self._xdir = STILL
        self._zdir = STILL
        self._ydir = STILL
        self._last_time = 0

        # Velocity in axis units per second
        self.velocity = 10.0
        self.mouse_sensitivity = 0.5
        self.last_x = None
        self.last_y = None

        super().__init__(fov=fov, aspect=aspect, near=near, far=far)

    def move_left(self, activate):
        self.move_state(LEFT, activate)

    def move_right(self, activate):
        self.move_state(RIGHT, activate)

    def move_forward(self, activate):
        self.move_state(FORWARD, activate)

    def move_backward(self, activate):
        self.move_state(BACKWARD, activate)

    def move_up(self, activate):
        self.move_state(UP, activate)

    def move_down(self, activate):
        self.move_state(DOWN, activate)

    def move_state(self, direction, activate):
        """
        Set the camera position move state

        :param direction: What direction to update
        :param activate: Start or stop moving in the direction
        """
        if direction == RIGHT:
            self._xdir = POSITIVE if activate else STILL
        elif direction == LEFT:
            self._xdir = NEGATIVE if activate else STILL
        elif direction == FORWARD:
            self._zdir = NEGATIVE if activate else STILL
        elif direction == BACKWARD:
            self._zdir = POSITIVE if activate else STILL
        elif direction == UP:
            self._ydir = POSITIVE if activate else STILL
        elif direction == DOWN:
            self._ydir = NEGATIVE if activate else STILL

    def rot_state(self, x, y):
        """
        Set the rotation state of the camera

        :param x: viewport x pos
        :param y: viewport y pos
        """
        if self.last_x is None:
            self.last_x = x
        if self.last_y is None:
            self.last_y = y

        x_offset = self.last_x - x
        y_offset = self.last_y - y

        self.last_x = x
        self.last_y = y

        x_offset *= self.mouse_sensitivity
        y_offset *= self.mouse_sensitivity

        self.yaw -= x_offset
        self.pitch += y_offset

        if self.pitch > 85.0:
            self.pitch = 85.0
        if self.pitch < -85.0:
            self.pitch = -85.0

        self._update_yaw_and_pitch()

    @property
    def view_matrix(self):
        """
        :return: The current view matrix for the camera
        """
        # Use separate time in camera so we can move it when the demo is paused
        now = time.time()
        # If the camera has been inactive for a while, a large time delta
        # can suddenly move the camera far away from the scene
        t = max(now - self._last_time, 0)
        self._last_time = now

        # X Movement
        if self._xdir == POSITIVE:
            self.position += self.right * self.velocity * t
        elif self._xdir == NEGATIVE:
            self.position -= self.right * self.velocity * t

        # Z Movement
        if self._zdir == NEGATIVE:
            self.position += self.dir * self.velocity * t
        elif self._zdir == POSITIVE:
            self.position -= self.dir * self.velocity * t

        # Y Movement
        if self._ydir == POSITIVE:
            self.position += self.up * self.velocity * t
        elif self._ydir == NEGATIVE:
            self.position -= self.up * self.velocity * t

        return self._gl_look_at(self.position, self.position + self.dir, self._up)
<commit_msg>Bug: Camera pos crash if using ndarray type<commit_after>import time
from math import cos, radians, sin

from demosys.opengl.projection import Projection
from pyrr import Vector3, matrix44, vector, vector3

# Direction Definitions
RIGHT = 1
LEFT = 2
FORWARD = 3
BACKWARD = 4
UP = 5
DOWN = 6

# Movement Definitions
STILL = 0
POSITIVE = 1
NEGATIVE = 2

class Camera:
    """Simple camera class containing projection"""
    def __init__(self, fov=60, aspect=1.0, near=1, far=100):
        """
        Initialize camera using a specific projection

        :param fov: Field of view
        :param aspect: Aspect ratio
        :param near: Near plane
        :param far: Far plane
        """
        self.position = Vector3([0.0, 0.0, 0.0])
        # Default camera placement
        self.up = Vector3([0.0, 1.0, 0.0])
        self.right = Vector3([1.0, 0.0, 0.0])
        self.dir = Vector3([0.0, 0.0, -1.0])
        # Yaw and Pitch
        self.yaw = -90.0
        self.pitch = 0.0

        # World up vector
        self._up = Vector3([0.0, 1.0, 0.0])

        # Projection
        self.projection = Projection(aspect, fov, near, far)

    def set_position(self, x, y, z):
        """
        Set the 3D position of the camera

        :param x: float
        :param y: float
        :param z: float
        """
        self.position = Vector3([x, y, z])

    @property
    def view_matrix(self):
        """
        :return: The current view matrix for the camera
        """
        self._update_yaw_and_pitch()
        return self._gl_look_at(self.position, self.position + self.dir, self._up)

    def _update_yaw_and_pitch(self):
        """
        Updates the camera vectors based on the current yaw and pitch
        """
        front = Vector3([0.0, 0.0, 0.0])
        front.x = cos(radians(self.yaw)) * cos(radians(self.pitch))
        front.y = sin(radians(self.pitch))
        front.z = sin(radians(self.yaw)) * cos(radians(self.pitch))

        self.dir = vector.normalise(front)
        self.right = vector.normalise(vector3.cross(self.dir, self._up))
        self.up = vector.normalise(vector3.cross(self.right, self.dir))

    def look_at(self, vec=None, pos=None):
        """
        Look at a specific point

        :param vec: Vector3 position
        :param pos: python list [x, y, x]
        :return: Camera matrix
        """
        if pos is None:
            vec = Vector3(pos)

        if vec is None:
            raise ValueError("vector or pos must be set")

        return self._gl_look_at(self.position, vec, self._up)

    def _gl_look_at(self, pos, target, up):
        """
        The standard lookAt method

        :param pos: current position
        :param target: target position to look at
        :param up: direction up
        """
        z = vector.normalise(pos - target)
        x = vector.normalise(vector3.cross(vector.normalise(up), z))
        y = vector3.cross(z, x)

        translate = matrix44.create_identity()
        translate[3][0] = -pos.x
        translate[3][1] = -pos.y
        translate[3][2] = -pos.z

        rotate = matrix44.create_identity()
        rotate[0][0] = x[0]  # -- X
        rotate[1][0] = x[1]
        rotate[2][0] = x[2]
        rotate[0][1] = y[0]  # -- Y
        rotate[1][1] = y[1]
        rotate[2][1] = y[2]
        rotate[0][2] = z[0]  # -- Z
        rotate[1][2] = z[1]
        rotate[2][2] = z[2]

        return matrix44.multiply(translate, rotate)

class SystemCamera(Camera):
    """System camera controlled by mouse and keyboard"""
    def __init__(self, fov=60, aspect=1.0, near=1, far=100):
        # Position movement states
        self._xdir = STILL
        self._zdir = STILL
        self._ydir = STILL
        self._last_time = 0

        # Velocity in axis units per second
        self.velocity = 10.0
        self.mouse_sensitivity = 0.5
        self.last_x = None
        self.last_y = None

        super().__init__(fov=fov, aspect=aspect, near=near, far=far)

    def move_left(self, activate):
        self.move_state(LEFT, activate)

    def move_right(self, activate):
        self.move_state(RIGHT, activate)

    def move_forward(self, activate):
        self.move_state(FORWARD, activate)

    def move_backward(self, activate):
        self.move_state(BACKWARD, activate)

    def move_up(self, activate):
        self.move_state(UP, activate)

    def move_down(self, activate):
        self.move_state(DOWN, activate)

    def move_state(self, direction, activate):
        """
        Set the camera position move state

        :param direction: What direction to update
        :param activate: Start or stop moving in the direction
        """
        if direction == RIGHT:
            self._xdir = POSITIVE if activate else STILL
        elif direction == LEFT:
            self._xdir = NEGATIVE if activate else STILL
        elif direction == FORWARD:
            self._zdir = NEGATIVE if activate else STILL
        elif direction == BACKWARD:
            self._zdir = POSITIVE if activate else STILL
        elif direction == UP:
            self._ydir = POSITIVE if activate else STILL
        elif direction == DOWN:
            self._ydir = NEGATIVE if activate else STILL

    def rot_state(self, x, y):
        """
        Set the rotation state of the camera

        :param x: viewport x pos
        :param y: viewport y pos
        """
        if self.last_x is None:
            self.last_x = x
        if self.last_y is None:
            self.last_y = y

        x_offset = self.last_x - x
        y_offset = self.last_y - y

        self.last_x = x
        self.last_y = y

        x_offset *= self.mouse_sensitivity
        y_offset *= self.mouse_sensitivity

        self.yaw -= x_offset
        self.pitch += y_offset

        if self.pitch > 85.0:
            self.pitch = 85.0
        if self.pitch < -85.0:
            self.pitch = -85.0

        self._update_yaw_and_pitch()

    @property
    def view_matrix(self):
        """
        :return: The current view matrix for the camera
        """
        # Use separate time in camera so we can move it when the demo is paused
        now = time.time()
        # If the camera has been inactive for a while, a large time delta
        # can suddenly move the camera far away from the scene
        t = max(now - self._last_time, 0)
        self._last_time = now

        # X Movement
        if self._xdir == POSITIVE:
            self.position += self.right * self.velocity * t
        elif self._xdir == NEGATIVE:
            self.position -= self.right * self.velocity * t

        # Z Movement
        if self._zdir == NEGATIVE:
            self.position += self.dir * self.velocity * t
        elif self._zdir == POSITIVE:
            self.position -= self.dir * self.velocity * t

        # Y Movement
        if self._ydir == POSITIVE:
            self.position += self.up * self.velocity * t
        elif self._ydir == NEGATIVE:
            self.position -= self.up * self.velocity * t

        return self._gl_look_at(self.position, self.position + self.dir, self._up)
<|endoftext|>


<commit_before>"""
:mod:`ergodic` --- summary measures for ergodic Markov chains
=============================================================

"""

__author__  = "Sergio J. Rey <srey@asu.edu> "

import numpy as np
import numpy.linalg as la

def steady_state(P):
    """
    Calculates the steady state probability vector for a regular Markov
    transition matrix P

    Parameters
    ----------

    P        : matrix (kxk)
               an ergodic Markov transition probability matrix

    Returns
    -------

    implicit : matrix (kx1)
               steady state distribution

    Examples
    --------
    Taken from Kemeny and Snell. [1]_ Land of Oz example where the states are
    Rain, Nice and Snow - so there is 25 percent chance that if it
    rained in Oz today, it will snow tomorrow, while if it snowed today in
    Oz there is a 50 percent chance of snow again tomorrow and a 25
    percent chance of a nice day (nice, like when the witch with the monkeys
    is melting).

    >>> import numpy as np
    >>> p=np.matrix([[.5, .25, .25],[.5,0,.5],[.25,.25,.5]])
    >>> steady_state(p)
    matrix([[ 0.4],
            [ 0.2],
            [ 0.4]])

    Thus, the long run distribution for Oz is to have 40 percent of the
    days classified as Rain, 20 percent as Nice, and 40 percent as Snow
    (states are mutually exclusive).

    """

    v,d=la.eig(np.transpose(P))

    # for a regular P maximum eigenvalue will be 1
    mv=max(v)
    # find its position
    i=v.tolist().index(mv)

    # normalize eigenvector corresponding to the eigenvalue 1
    return d[:,i]/sum(d[:,i])

def fmpt(P):
    """
    Calculates the matrix of first mean passage times for an
    ergodic transition probability matrix.

    Parameters
    ----------

    P    : matrix (kxk)
           an ergodic Markov transition probability matrix

    Returns
    -------

    M    : matrix (kxk)
           elements are the expected value for the number of intervals
           required for  a chain starting in state i to first enter state j

    Examples
    --------

    >>> import numpy as np
    >>> p=np.matrix([[.5, .25, .25],[.5,0,.5],[.25,.25,.5]])
    >>> fm=fmpt(p)
    >>> fm
    matrix([[ 0.        ,  4.        ,  3.33333333],
            [ 2.66666667,  0.        ,  2.66666667],
            [ 3.33333333,  4.        ,  0.        ]])

    Thus, if it is raining today in Oz we can expect a nice day to come
    along in another 4 days, on average, and snow to hit in 3.33 days. If
    it is nice today in Oz, we would experience a change in the weather
    (either rain or snow) in 2.67 days from today. (That wicked witch can
    only die once so I reckon that is the ultimate absorbing state).

    Notes
    -----

    Uses formulation (and examples on p. 218) in Kemeny and Snell (1976) [1]_

    References
    ----------

    .. [1] Kemeny, John, G. and J. Laurie Snell (1976) Finite Markov
       Chains. Springer-Verlag. Berlin
    """
    A=P**1000
    n,k=A.shape
    I=np.identity(k)
    Z=la.inv(I-P+A)
    M=np.zeros_like(Z)
    for i in range(k):
        for j in range(k):
            M[i,j]=(Z[j,j]-Z[i,j])/A[j,j]
    return M

def _test():
    import doctest
    doctest.testmod()

if __name__ == '__main__':
    _test()
<commit_msg>- variances of fmpt and recurrence times<commit_after>"""
:mod:`ergodic` --- summary measures for ergodic Markov chains
=============================================================

"""

__author__  = "Sergio J. Rey <srey@asu.edu> "

import numpy as np
import numpy.linalg as la

def steady_state(P):
    """
    Calculates the steady state probability vector for a regular Markov
    transition matrix P

    Parameters
    ----------

    P        : matrix (kxk)
               an ergodic Markov transition probability matrix

    Returns
    -------

    implicit : matrix (kx1)
               steady state distribution

    Examples
    --------
    Taken from Kemeny and Snell. [1]_ Land of Oz example where the states are
    Rain, Nice and Snow - so there is 25 percent chance that if it
    rained in Oz today, it will snow tomorrow, while if it snowed today in
    Oz there is a 50 percent chance of snow again tomorrow and a 25
    percent chance of a nice day (nice, like when the witch with the monkeys
    is melting).

    >>> import numpy as np
    >>> p=np.matrix([[.5, .25, .25],[.5,0,.5],[.25,.25,.5]])
    >>> steady_state(p)
    matrix([[ 0.4],
            [ 0.2],
            [ 0.4]])

    Thus, the long run distribution for Oz is to have 40 percent of the
    days classified as Rain, 20 percent as Nice, and 40 percent as Snow
    (states are mutually exclusive).

    """

    v,d=la.eig(np.transpose(P))

    # for a regular P maximum eigenvalue will be 1
    mv=max(v)
    # find its position
    i=v.tolist().index(mv)

    # normalize eigenvector corresponding to the eigenvalue 1
    return d[:,i]/sum(d[:,i])

def fmpt(P):
    """
    Calculates the matrix of first mean passage times for an
    ergodic transition probability matrix.

    Parameters
    ----------

    P    : matrix (kxk)
           an ergodic Markov transition probability matrix

    Returns
    -------

    M    : matrix (kxk)
           elements are the expected value for the number of intervals
           required for  a chain starting in state i to first enter state j
           If i=j then this is the recurrence time.

    Examples
    --------

    >>> import numpy as np
    >>> p=np.matrix([[.5, .25, .25],[.5,0,.5],[.25,.25,.5]])
    >>> fm=fmpt(p)
    >>> fm
    matrix([[ 2.5       ,  4.        ,  3.33333333],
            [ 2.66666667,  5.        ,  2.66666667],
            [ 3.33333333,  4.        ,  2.5       ]])

    Thus, if it is raining today in Oz we can expect a nice day to come
    along in another 4 days, on average, and snow to hit in 3.33 days. We can
    expect another rainy day in 2.5 days. If it is nice today in Oz, we would
    experience a change in the weather (either rain or snow) in 2.67 days from
    today. (That wicked witch can only die once so I reckon that is the
    ultimate absorbing state).

    Notes -----

    Uses formulation (and examples on p. 218) in Kemeny and Snell (1976) [1]_

    References
    ----------

    .. [1] Kemeny, John, G. and J. Laurie Snell (1976) Finite Markov
       Chains. Springer-Verlag. Berlin
    """
    A=P**1000
    n,k=A.shape
    I=np.identity(k)
    Z=la.inv(I-P+A)
    E=np.ones_like(Z)
    D=np.diag(1./np.diag(A))
    Zdg=np.diag(np.diag(Z))
    M=(I-Z+E*Zdg)*D
    return M

def var_fmpt(P):
    """
    Variances of first mean passage times for an ergodic transition
    probability matrix

    Parameters
    ----------

    P    : matrix (kxk)
           an ergodic Markov transition probability matrix

    Returns
    -------

    implic : matrix (kxk)
             elements are the variances for the number of intervals
             required for  a chain starting in state i to first enter state j

    Examples
    --------

    >>> import numpy as np
    >>> p=np.matrix([[.5, .25, .25],[.5,0,.5],[.25,.25,.5]])
    >>> vfm=var_fmpt(p)
    >>> vfm
    matrix([[  5.58333333,  12.        ,   6.88888889],
            [  6.22222222,  12.        ,   6.22222222],
            [  6.88888889,  12.        ,   5.58333333]])

    Notes
    -----

    Uses formulation (and examples on p. 83) in Kemeny and Snell (1976) [1]_

    References
    ----------

    .. [1] Kemeny, John, G. and J. Laurie Snell (1976) Finite Markov
       Chains. Springer-Verlag. Berlin

    """
    A=P**1000
    n,k=A.shape
    I=np.identity(k)
    Z=la.inv(I-P+A)
    E=np.ones_like(Z)
    D=np.diag(1./np.diag(A))
    Zdg=np.diag(np.diag(Z))
    M=(I-Z+E*Zdg)*D
    ZM=Z*M
    ZMdg=np.diag(np.diag(ZM))
    W=M*(2*Zdg*D-I)+2*(ZM-E*ZMdg)
    return W-np.multiply(M,M)

def _test():
    import doctest
    doctest.testmod()

if __name__ == '__main__':
    _test()
<|endoftext|>