<commit_before># -*- coding: utf-8 -*-
# Copyright (c) Facebook, Inc. and its affiliates.

import logging
import numpy as np
import time
import weakref
from typing import List, Mapping, Optional
import torch
from torch.nn.parallel import DataParallel, DistributedDataParallel

import detectron2.utils.comm as comm
from detectron2.utils.events import EventStorage, get_event_storage
from detectron2.utils.logger import _log_api_usage

__all__ = ["HookBase", "TrainerBase", "SimpleTrainer", "AMPTrainer"]

class HookBase:
    """
    Base class for hooks that can be registered with :class:`TrainerBase`.

    Each hook can implement 4 methods. The way they are called is demonstrated
    in the following snippet:
    ::
        hook.before_train()
        for iter in range(start_iter, max_iter):
            hook.before_step()
            trainer.run_step()
            hook.after_step()
        iter += 1
        hook.after_train()

    Notes:
        1. In the hook method, users can access ``self.trainer`` to access more
           properties about the context (e.g., model, current iteration, or config
           if using :class:`DefaultTrainer`).

        2. A hook that does something in :meth:`before_step` can often be
           implemented equivalently in :meth:`after_step`.
           If the hook takes non-trivial time, it is strongly recommended to
           implement the hook in :meth:`after_step` instead of :meth:`before_step`.
           The convention is that :meth:`before_step` should only take negligible time.

           Following this convention will allow hooks that do care about the difference
           between :meth:`before_step` and :meth:`after_step` (e.g., timer) to
           function properly.

    """

    trainer: "TrainerBase" = None
    """
    A weak reference to the trainer object. Set by the trainer when the hook is registered.
    """

    def before_train(self):
        """
        Called before the first iteration.
        """
        pass

    def after_train(self):
        """
        Called after the last iteration.
        """
        pass

    def before_step(self):
        """
        Called before each iteration.
        """
        pass

    def after_step(self):
        """
        Called after each iteration.
        """
        pass

    def state_dict(self):
        """
        Hooks are stateless by default, but can be made checkpointable by
        implementing `state_dict` and `load_state_dict`.
        """
        return {}

class TrainerBase:
    """
    Base class for iterative trainer with hooks.

    The only assumption we made here is: the training runs in a loop.
    A subclass can implement what the loop is.
    We made no assumptions about the existence of dataloader, optimizer, model, etc.

    Attributes:
        iter(int): the current iteration.

        start_iter(int): The iteration to start with.
            By convention the minimum possible value is 0.

        max_iter(int): The iteration to end training.

        storage(EventStorage): An EventStorage that's opened during the course of training.
    """

    def __init__(self) -> None:
        self._hooks: List[HookBase] = []
        self.iter: int = 0
        self.start_iter: int = 0
        self.max_iter: int
        self.storage: EventStorage
        _log_api_usage("trainer." + self.__class__.__name__)

    def register_hooks(self, hooks: List[Optional[HookBase]]) -> None:
        """
        Register hooks to the trainer. The hooks are executed in the order
        they are registered.

        Args:
            hooks (list[Optional[HookBase]]): list of hooks
        """
        hooks = [h for h in hooks if h is not None]
        for h in hooks:
            assert isinstance(h, HookBase)
            # To avoid circular reference, hooks and trainer cannot own each other.
            # This normally does not matter, but will cause memory leak if the
            # involved objects contain __del__:
            # See http://engineering.hearsaysocial.com/2013/06/16/circular-references-in-python/
            h.trainer = weakref.proxy(self)
        self._hooks.extend(hooks)

    def train(self, start_iter: int, max_iter: int):
        """
        Args:
            start_iter, max_iter (int): See docs above
        """
        logger = logging.getLogger(__name__)
        logger.info("Starting training from iteration {}".format(start_iter))

        self.iter = self.start_iter = start_iter
        self.max_iter = max_iter

        with EventStorage(start_iter) as self.storage:
            try:
                self.before_train()
                for self.iter in range(start_iter, max_iter):
                    self.before_step()
                    self.run_step()
                    self.after_step()
                # self.iter == max_iter can be used by `after_train` to
                # tell whether the training successfully finished or failed
                # due to exceptions.
                self.iter += 1
            except Exception:
                logger.exception("Exception during training:")
                raise
            finally:
                self.after_train()

    def before_train(self):
        for h in self._hooks:
            h.before_train()

    def after_train(self):
        self.storage.iter = self.iter
        for h in self._hooks:
            h.after_train()

    def before_step(self):
        # Maintain the invariant that storage.iter == trainer.iter
        # for the entire execution of each step
        self.storage.iter = self.iter

        for h in self._hooks:
            h.before_step()

    def after_step(self):
        for h in self._hooks:
            h.after_step()

    def run_step(self):
        raise NotImplementedError

    def state_dict(self):
        ret = {"iteration": self.iter}
        hooks_state = {}
        for h in self._hooks:
            sd = h.state_dict()
            if sd:
                name = type(h).__qualname__
                if name in hooks_state:
                    # TODO handle repetitive stateful hooks
                    continue
                hooks_state[name] = sd
        if hooks_state:
            ret["hooks"] = hooks_state
        return ret

    def load_state_dict(self, state_dict):
        logger = logging.getLogger(__name__)
        self.iter = state_dict["iteration"]
        for key, value in state_dict.get("hooks", {}).items():
            for h in self._hooks:
                try:
                    name = type(h).__qualname__
                except AttributeError:
                    continue
                if name == key:
                    h.load_state_dict(value)
                    break
            else:
                logger.warning(f"Cannot find the hook '{key}', its state_dict is ignored.")

class SimpleTrainer(TrainerBase):
    """
    A simple trainer for the most common type of task:
    single-cost single-optimizer single-data-source iterative optimization,
    optionally using data-parallelism.
    It assumes that every step, you:

    1. Compute the loss with a data from the data_loader.
    2. Compute the gradients with the above loss.
    3. Update the model with the optimizer.

    All other tasks during training (checkpointing, logging, evaluation, LR schedule)
    are maintained by hooks, which can be registered by :meth:`TrainerBase.register_hooks`.

    If you want to do anything fancier than this,
    either subclass TrainerBase and implement your own `run_step`,
    or write your own training loop.
    """

    def __init__(self, model, data_loader, optimizer):
        """
        Args:
            model: a torch Module. Takes a data from data_loader and returns a
                dict of losses.
            data_loader: an iterable. Contains data to be used to call model.
            optimizer: a torch optimizer.
        """
        super().__init__()

        """
        We set the model to training mode in the trainer.
        However it's valid to train a model that's in eval mode.
        If you want your model (or a submodule of it) to behave
        like evaluation during training, you can overwrite its train() method.
        """
        model.train()

        self.model = model
        self.data_loader = data_loader
        self._data_loader_iter = iter(data_loader)
        self.optimizer = optimizer

    def run_step(self):
        """
        Implement the standard training logic described above.
        """
        assert self.model.training, "[SimpleTrainer] model was changed to eval mode!"
        start = time.perf_counter()
        """
        If you want to do something with the data, you can wrap the dataloader.
        """
        data = next(self._data_loader_iter)
        data_time = time.perf_counter() - start

        """
        If you want to do something with the losses, you can wrap the model.
        """
        loss_dict = self.model(data)
        if isinstance(loss_dict, torch.Tensor):
            losses = loss_dict
            loss_dict = {"total_loss": loss_dict}
        else:
            losses = sum(loss_dict.values())

        """
        If you need to accumulate gradients or do something similar, you can
        wrap the optimizer with your custom `zero_grad()` method.
        """
        self.optimizer.zero_grad()
        losses.backward()

        self._write_metrics(loss_dict, data_time)

        """
        If you need gradient clipping/scaling or other processing, you can
        wrap the optimizer with your custom `step()` method. But it is
        suboptimal as explained in https://arxiv.org/abs/2006.15704 Sec 3.2.4
        """
        self.optimizer.step()

    def _write_metrics(
        self,
        loss_dict: Mapping[str, torch.Tensor],
        data_time: float,
        prefix: str = "",
    ) -> None:
        SimpleTrainer.write_metrics(loss_dict, data_time, prefix)

    @staticmethod
    def write_metrics(
        loss_dict: Mapping[str, torch.Tensor],
        data_time: float,
        prefix: str = "",
    ) -> None:
        """
        Args:
            loss_dict (dict): dict of scalar losses
            data_time (float): time taken by the dataloader iteration
            prefix (str): prefix for logging keys
        """
        metrics_dict = {k: v.detach().cpu().item() for k, v in loss_dict.items()}
        metrics_dict["data_time"] = data_time

        # Gather metrics among all workers for logging
        # This assumes we do DDP-style training, which is currently the only
        # supported method in detectron2.
        all_metrics_dict = comm.gather(metrics_dict)

        if comm.is_main_process():
            storage = get_event_storage()

            # data_time among workers can have high variance. The actual latency
            # caused by data_time is the maximum among workers.
            data_time = np.max([x.pop("data_time") for x in all_metrics_dict])
            storage.put_scalar("data_time", data_time)

            # average the rest metrics
            metrics_dict = {
                k: np.mean([x[k] for x in all_metrics_dict]) for k in all_metrics_dict[0].keys()
            }
            total_losses_reduced = sum(metrics_dict.values())
            if not np.isfinite(total_losses_reduced):
                raise FloatingPointError(
                    f"Loss became infinite or NaN at iteration={storage.iter}!\n"
                    f"loss_dict = {metrics_dict}"
                )

            storage.put_scalar("{}total_loss".format(prefix), total_losses_reduced)
            if len(metrics_dict) > 1:
                storage.put_scalars(**metrics_dict)

    def state_dict(self):
        ret = super().state_dict()
        ret["optimizer"] = self.optimizer.state_dict()
        return ret

    def load_state_dict(self, state_dict):
        super().load_state_dict(state_dict)
        self.optimizer.load_state_dict(state_dict["optimizer"])

class AMPTrainer(SimpleTrainer):
    """
    Like :class:`SimpleTrainer`, but uses PyTorch's native automatic mixed precision
    in the training loop.
    """

    def __init__(self, model, data_loader, optimizer, grad_scaler=None):
        """
        Args:
            model, data_loader, optimizer: same as in :class:`SimpleTrainer`.
            grad_scaler: torch GradScaler to automatically scale gradients.
        """
        unsupported = "AMPTrainer does not support single-process multi-device training!"
        if isinstance(model, DistributedDataParallel):
            assert not (model.device_ids and len(model.device_ids) > 1), unsupported
        assert not isinstance(model, DataParallel), unsupported

        super().__init__(model, data_loader, optimizer)

        if grad_scaler is None:
            from torch.cuda.amp import GradScaler

            grad_scaler = GradScaler()
        self.grad_scaler = grad_scaler

    def run_step(self):
        """
        Implement the AMP training logic.
        """
        assert self.model.training, "[AMPTrainer] model was changed to eval mode!"
        assert torch.cuda.is_available(), "[AMPTrainer] CUDA is required for AMP training!"
        from torch.cuda.amp import autocast

        start = time.perf_counter()
        data = next(self._data_loader_iter)
        data_time = time.perf_counter() - start

        with autocast():
            loss_dict = self.model(data)
            if isinstance(loss_dict, torch.Tensor):
                losses = loss_dict
                loss_dict = {"total_loss": loss_dict}
            else:
                losses = sum(loss_dict.values())

        self.optimizer.zero_grad()
        self.grad_scaler.scale(losses).backward()

        self._write_metrics(loss_dict, data_time)

        self.grad_scaler.step(self.optimizer)
        self.grad_scaler.update()

    def state_dict(self):
        ret = super().state_dict()
        ret["grad_scaler"] = self.grad_scaler.state_dict()
        return ret

    def load_state_dict(self, state_dict):
        super().load_state_dict(state_dict)
        self.grad_scaler.load_state_dict(state_dict["grad_scaler"])
<commit_msg>Only create the data loader iterator when it is being used.<commit_after># -*- coding: utf-8 -*-
# Copyright (c) Facebook, Inc. and its affiliates.

import logging
import numpy as np
import time
import weakref
from typing import List, Mapping, Optional
import torch
from torch.nn.parallel import DataParallel, DistributedDataParallel

import detectron2.utils.comm as comm
from detectron2.utils.events import EventStorage, get_event_storage
from detectron2.utils.logger import _log_api_usage

__all__ = ["HookBase", "TrainerBase", "SimpleTrainer", "AMPTrainer"]

class HookBase:
    """
    Base class for hooks that can be registered with :class:`TrainerBase`.

    Each hook can implement 4 methods. The way they are called is demonstrated
    in the following snippet:
    ::
        hook.before_train()
        for iter in range(start_iter, max_iter):
            hook.before_step()
            trainer.run_step()
            hook.after_step()
        iter += 1
        hook.after_train()

    Notes:
        1. In the hook method, users can access ``self.trainer`` to access more
           properties about the context (e.g., model, current iteration, or config
           if using :class:`DefaultTrainer`).

        2. A hook that does something in :meth:`before_step` can often be
           implemented equivalently in :meth:`after_step`.
           If the hook takes non-trivial time, it is strongly recommended to
           implement the hook in :meth:`after_step` instead of :meth:`before_step`.
           The convention is that :meth:`before_step` should only take negligible time.

           Following this convention will allow hooks that do care about the difference
           between :meth:`before_step` and :meth:`after_step` (e.g., timer) to
           function properly.

    """

    trainer: "TrainerBase" = None
    """
    A weak reference to the trainer object. Set by the trainer when the hook is registered.
    """

    def before_train(self):
        """
        Called before the first iteration.
        """
        pass

    def after_train(self):
        """
        Called after the last iteration.
        """
        pass

    def before_step(self):
        """
        Called before each iteration.
        """
        pass

    def after_step(self):
        """
        Called after each iteration.
        """
        pass

    def state_dict(self):
        """
        Hooks are stateless by default, but can be made checkpointable by
        implementing `state_dict` and `load_state_dict`.
        """
        return {}

class TrainerBase:
    """
    Base class for iterative trainer with hooks.

    The only assumption we made here is: the training runs in a loop.
    A subclass can implement what the loop is.
    We made no assumptions about the existence of dataloader, optimizer, model, etc.

    Attributes:
        iter(int): the current iteration.

        start_iter(int): The iteration to start with.
            By convention the minimum possible value is 0.

        max_iter(int): The iteration to end training.

        storage(EventStorage): An EventStorage that's opened during the course of training.
    """

    def __init__(self) -> None:
        self._hooks: List[HookBase] = []
        self.iter: int = 0
        self.start_iter: int = 0
        self.max_iter: int
        self.storage: EventStorage
        _log_api_usage("trainer." + self.__class__.__name__)

    def register_hooks(self, hooks: List[Optional[HookBase]]) -> None:
        """
        Register hooks to the trainer. The hooks are executed in the order
        they are registered.

        Args:
            hooks (list[Optional[HookBase]]): list of hooks
        """
        hooks = [h for h in hooks if h is not None]
        for h in hooks:
            assert isinstance(h, HookBase)
            # To avoid circular reference, hooks and trainer cannot own each other.
            # This normally does not matter, but will cause memory leak if the
            # involved objects contain __del__:
            # See http://engineering.hearsaysocial.com/2013/06/16/circular-references-in-python/
            h.trainer = weakref.proxy(self)
        self._hooks.extend(hooks)

    def train(self, start_iter: int, max_iter: int):
        """
        Args:
            start_iter, max_iter (int): See docs above
        """
        logger = logging.getLogger(__name__)
        logger.info("Starting training from iteration {}".format(start_iter))

        self.iter = self.start_iter = start_iter
        self.max_iter = max_iter

        with EventStorage(start_iter) as self.storage:
            try:
                self.before_train()
                for self.iter in range(start_iter, max_iter):
                    self.before_step()
                    self.run_step()
                    self.after_step()
                # self.iter == max_iter can be used by `after_train` to
                # tell whether the training successfully finished or failed
                # due to exceptions.
                self.iter += 1
            except Exception:
                logger.exception("Exception during training:")
                raise
            finally:
                self.after_train()

    def before_train(self):
        for h in self._hooks:
            h.before_train()

    def after_train(self):
        self.storage.iter = self.iter
        for h in self._hooks:
            h.after_train()

    def before_step(self):
        # Maintain the invariant that storage.iter == trainer.iter
        # for the entire execution of each step
        self.storage.iter = self.iter

        for h in self._hooks:
            h.before_step()

    def after_step(self):
        for h in self._hooks:
            h.after_step()

    def run_step(self):
        raise NotImplementedError

    def state_dict(self):
        ret = {"iteration": self.iter}
        hooks_state = {}
        for h in self._hooks:
            sd = h.state_dict()
            if sd:
                name = type(h).__qualname__
                if name in hooks_state:
                    # TODO handle repetitive stateful hooks
                    continue
                hooks_state[name] = sd
        if hooks_state:
            ret["hooks"] = hooks_state
        return ret

    def load_state_dict(self, state_dict):
        logger = logging.getLogger(__name__)
        self.iter = state_dict["iteration"]
        for key, value in state_dict.get("hooks", {}).items():
            for h in self._hooks:
                try:
                    name = type(h).__qualname__
                except AttributeError:
                    continue
                if name == key:
                    h.load_state_dict(value)
                    break
            else:
                logger.warning(f"Cannot find the hook '{key}', its state_dict is ignored.")

class SimpleTrainer(TrainerBase):
    """
    A simple trainer for the most common type of task:
    single-cost single-optimizer single-data-source iterative optimization,
    optionally using data-parallelism.
    It assumes that every step, you:

    1. Compute the loss with a data from the data_loader.
    2. Compute the gradients with the above loss.
    3. Update the model with the optimizer.

    All other tasks during training (checkpointing, logging, evaluation, LR schedule)
    are maintained by hooks, which can be registered by :meth:`TrainerBase.register_hooks`.

    If you want to do anything fancier than this,
    either subclass TrainerBase and implement your own `run_step`,
    or write your own training loop.
    """

    def __init__(self, model, data_loader, optimizer):
        """
        Args:
            model: a torch Module. Takes a data from data_loader and returns a
                dict of losses.
            data_loader: an iterable. Contains data to be used to call model.
            optimizer: a torch optimizer.
        """
        super().__init__()

        """
        We set the model to training mode in the trainer.
        However it's valid to train a model that's in eval mode.
        If you want your model (or a submodule of it) to behave
        like evaluation during training, you can overwrite its train() method.
        """
        model.train()

        self.model = model
        self.data_loader = data_loader
        # to access the data loader iterator, call `self._data_loader_iter`
        self._data_loader_iter_obj = None
        self.optimizer = optimizer

    def run_step(self):
        """
        Implement the standard training logic described above.
        """
        assert self.model.training, "[SimpleTrainer] model was changed to eval mode!"
        start = time.perf_counter()
        """
        If you want to do something with the data, you can wrap the dataloader.
        """
        data = next(self._data_loader_iter)
        data_time = time.perf_counter() - start

        """
        If you want to do something with the losses, you can wrap the model.
        """
        loss_dict = self.model(data)
        if isinstance(loss_dict, torch.Tensor):
            losses = loss_dict
            loss_dict = {"total_loss": loss_dict}
        else:
            losses = sum(loss_dict.values())

        """
        If you need to accumulate gradients or do something similar, you can
        wrap the optimizer with your custom `zero_grad()` method.
        """
        self.optimizer.zero_grad()
        losses.backward()

        self._write_metrics(loss_dict, data_time)

        """
        If you need gradient clipping/scaling or other processing, you can
        wrap the optimizer with your custom `step()` method. But it is
        suboptimal as explained in https://arxiv.org/abs/2006.15704 Sec 3.2.4
        """
        self.optimizer.step()

    @property
    def _data_loader_iter(self):
        # only create the data loader iterator when it is used
        if self._data_loader_iter_obj is None:
            self._data_loader_iter_obj = iter(self.data_loader)
        return self._data_loader_iter_obj

    def _write_metrics(
        self,
        loss_dict: Mapping[str, torch.Tensor],
        data_time: float,
        prefix: str = "",
    ) -> None:
        SimpleTrainer.write_metrics(loss_dict, data_time, prefix)

    @staticmethod
    def write_metrics(
        loss_dict: Mapping[str, torch.Tensor],
        data_time: float,
        prefix: str = "",
    ) -> None:
        """
        Args:
            loss_dict (dict): dict of scalar losses
            data_time (float): time taken by the dataloader iteration
            prefix (str): prefix for logging keys
        """
        metrics_dict = {k: v.detach().cpu().item() for k, v in loss_dict.items()}
        metrics_dict["data_time"] = data_time

        # Gather metrics among all workers for logging
        # This assumes we do DDP-style training, which is currently the only
        # supported method in detectron2.
        all_metrics_dict = comm.gather(metrics_dict)

        if comm.is_main_process():
            storage = get_event_storage()

            # data_time among workers can have high variance. The actual latency
            # caused by data_time is the maximum among workers.
            data_time = np.max([x.pop("data_time") for x in all_metrics_dict])
            storage.put_scalar("data_time", data_time)

            # average the rest metrics
            metrics_dict = {
                k: np.mean([x[k] for x in all_metrics_dict]) for k in all_metrics_dict[0].keys()
            }
            total_losses_reduced = sum(metrics_dict.values())
            if not np.isfinite(total_losses_reduced):
                raise FloatingPointError(
                    f"Loss became infinite or NaN at iteration={storage.iter}!\n"
                    f"loss_dict = {metrics_dict}"
                )

            storage.put_scalar("{}total_loss".format(prefix), total_losses_reduced)
            if len(metrics_dict) > 1:
                storage.put_scalars(**metrics_dict)

    def state_dict(self):
        ret = super().state_dict()
        ret["optimizer"] = self.optimizer.state_dict()
        return ret

    def load_state_dict(self, state_dict):
        super().load_state_dict(state_dict)
        self.optimizer.load_state_dict(state_dict["optimizer"])

class AMPTrainer(SimpleTrainer):
    """
    Like :class:`SimpleTrainer`, but uses PyTorch's native automatic mixed precision
    in the training loop.
    """

    def __init__(self, model, data_loader, optimizer, grad_scaler=None):
        """
        Args:
            model, data_loader, optimizer: same as in :class:`SimpleTrainer`.
            grad_scaler: torch GradScaler to automatically scale gradients.
        """
        unsupported = "AMPTrainer does not support single-process multi-device training!"
        if isinstance(model, DistributedDataParallel):
            assert not (model.device_ids and len(model.device_ids) > 1), unsupported
        assert not isinstance(model, DataParallel), unsupported

        super().__init__(model, data_loader, optimizer)

        if grad_scaler is None:
            from torch.cuda.amp import GradScaler

            grad_scaler = GradScaler()
        self.grad_scaler = grad_scaler

    def run_step(self):
        """
        Implement the AMP training logic.
        """
        assert self.model.training, "[AMPTrainer] model was changed to eval mode!"
        assert torch.cuda.is_available(), "[AMPTrainer] CUDA is required for AMP training!"
        from torch.cuda.amp import autocast

        start = time.perf_counter()
        data = next(self._data_loader_iter)
        data_time = time.perf_counter() - start

        with autocast():
            loss_dict = self.model(data)
            if isinstance(loss_dict, torch.Tensor):
                losses = loss_dict
                loss_dict = {"total_loss": loss_dict}
            else:
                losses = sum(loss_dict.values())

        self.optimizer.zero_grad()
        self.grad_scaler.scale(losses).backward()

        self._write_metrics(loss_dict, data_time)

        self.grad_scaler.step(self.optimizer)
        self.grad_scaler.update()

    def state_dict(self):
        ret = super().state_dict()
        ret["grad_scaler"] = self.grad_scaler.state_dict()
        return ret

    def load_state_dict(self, state_dict):
        super().load_state_dict(state_dict)
        self.grad_scaler.load_state_dict(state_dict["grad_scaler"])
<|endoftext|>


<commit_before>__all__ = ['coroutine',
           'iscoroutinefunction', 'iscoroutine']

import functools
import inspect
import opcode
import os
import sys
import traceback
import types

from . import events
from . import futures
from .log import logger

# Opcode of "yield from" instruction
_YIELD_FROM = opcode.opmap['YIELD_FROM']

# If you set _DEBUG to true, @coroutine will wrap the resulting
# generator objects in a CoroWrapper instance (defined below).  That
# instance will log a message when the generator is never iterated
# over, which may happen when you forget to use "yield from" with a
# coroutine call.  Note that the value of the _DEBUG flag is taken
# when the decorator is used, so to be of any use it must be set
# before you define your coroutines.  A downside of using this feature
# is that tracebacks show entries for the CoroWrapper.__next__ method
# when _DEBUG is true.
_DEBUG = (not sys.flags.ignore_environment
          and bool(os.environ.get('PYTHONASYNCIODEBUG')))

# Check for CPython issue #21209
def has_yield_from_bug():
    class MyGen:
        def __init__(self):
            self.send_args = None
        def __iter__(self):
            return self
        def __next__(self):
            return 42
        def send(self, *what):
            self.send_args = what
            return None
    def yield_from_gen(gen):
        yield from gen
    value = (1, 2, 3)
    gen = MyGen()
    coro = yield_from_gen(gen)
    next(coro)
    coro.send(value)
    return gen.send_args != (value,)
_YIELD_FROM_BUG = has_yield_from_bug()
del has_yield_from_bug

class CoroWrapper:
    # Wrapper for coroutine object in _DEBUG mode.

    def __init__(self, gen, func):
        assert inspect.isgenerator(gen), gen
        self.gen = gen
        self.func = func
        self._source_traceback = traceback.extract_stack(sys._getframe(1))
        # __name__, __qualname__, __doc__ attributes are set by the coroutine()
        # decorator

    def __repr__(self):
        coro_repr = _format_coroutine(self)
        if self._source_traceback:
            frame = self._source_traceback[-1]
            coro_repr += ', created at %s:%s' % (frame[0], frame[1])
        return '<%s %s>' % (self.__class__.__name__, coro_repr)

    def __iter__(self):
        return self

    def __next__(self):
        return next(self.gen)

    if _YIELD_FROM_BUG:
        # For for CPython issue #21209: using "yield from" and a custom
        # generator, generator.send(tuple) unpacks the tuple instead of passing
        # the tuple unchanged. Check if the caller is a generator using "yield
        # from" to decide if the parameter should be unpacked or not.
        def send(self, *value):
            frame = sys._getframe()
            caller = frame.f_back
            assert caller.f_lasti >= 0
            if caller.f_code.co_code[caller.f_lasti] != _YIELD_FROM:
                value = value[0]
            return self.gen.send(value)
    else:
        def send(self, value):
            return self.gen.send(value)

    def throw(self, exc):
        return self.gen.throw(exc)

    def close(self):
        return self.gen.close()

    @property
    def gi_frame(self):
        return self.gen.gi_frame

    @property
    def gi_running(self):
        return self.gen.gi_running

    @property
    def gi_code(self):
        return self.gen.gi_code

    def __del__(self):
        # Be careful accessing self.gen.frame -- self.gen might not exist.
        gen = getattr(self, 'gen', None)
        frame = getattr(gen, 'gi_frame', None)
        if frame is not None and frame.f_lasti == -1:
            msg = '%r was never yielded from' % self
            tb = getattr(self, '_source_traceback', ())
            if tb:
                tb = ''.join(traceback.format_list(tb))
                msg += ('\nCoroutine object created at '
                        '(most recent call last):\n')
                msg += tb.rstrip()
            logger.error(msg)

def coroutine(func):
    """Decorator to mark coroutines.

    If the coroutine is not yielded from before it is destroyed,
    an error message is logged.
    """
    if inspect.isgeneratorfunction(func):
        coro = func
    else:
        @functools.wraps(func)
        def coro(*args, **kw):
            res = func(*args, **kw)
            if isinstance(res, futures.Future) or inspect.isgenerator(res):
                res = yield from res
            return res

    if not _DEBUG:
        wrapper = coro
    else:
        @functools.wraps(func)
        def wrapper(*args, **kwds):
            w = CoroWrapper(coro(*args, **kwds), func)
            if w._source_traceback:
                del w._source_traceback[-1]
            w.__name__ = func.__name__
            if hasattr(func, '__qualname__'):
                w.__qualname__ = func.__qualname__
            w.__doc__ = func.__doc__
            return w

    wrapper._is_coroutine = True  # For iscoroutinefunction().
    return wrapper

def iscoroutinefunction(func):
    """Return True if func is a decorated coroutine function."""
    return getattr(func, '_is_coroutine', False)

_COROUTINE_TYPES = (types.GeneratorType, CoroWrapper)

def iscoroutine(obj):
    """Return True if obj is a coroutine object."""
    return isinstance(obj, _COROUTINE_TYPES)

def _format_coroutine(coro):
    assert iscoroutine(coro)
    coro_name = getattr(coro, '__qualname__', coro.__name__)

    filename = coro.gi_code.co_filename
    if (isinstance(coro, CoroWrapper)
    and not inspect.isgeneratorfunction(coro.func)):
        filename, lineno = events._get_function_source(coro.func)
        if coro.gi_frame is None:
            coro_repr = ('%s() done, defined at %s:%s'
                         % (coro_name, filename, lineno))
        else:
            coro_repr = ('%s() running, defined at %s:%s'
                         % (coro_name, filename, lineno))
    elif coro.gi_frame is not None:
        lineno = coro.gi_frame.f_lineno
        coro_repr = ('%s() running at %s:%s'
                     % (coro_name, filename, lineno))
    else:
        lineno = coro.gi_code.co_firstlineno
        coro_repr = ('%s() done, defined at %s:%s'
                     % (coro_name, filename, lineno))

    return coro_repr
<commit_msg>update - 协程装饰器.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
__all__ = ['coroutine',
           'iscoroutinefunction', 'iscoroutine']

import functools   # 关键
import inspect     # 关键
import opcode
import os
import sys
import traceback
import types

from . import events        # events._get_function_source()
from . import futures       # 关键对象 futures.Future
from .log import logger

# Opcode of "yield from" instruction
_YIELD_FROM = opcode.opmap['YIELD_FROM']

# If you set _DEBUG to true, @coroutine will wrap the resulting
# generator objects in a CoroWrapper instance (defined below).  That
# instance will log a message when the generator is never iterated
# over, which may happen when you forget to use "yield from" with a
# coroutine call.  Note that the value of the _DEBUG flag is taken
# when the decorator is used, so to be of any use it must be set
# before you define your coroutines.  A downside of using this feature
# is that tracebacks show entries for the CoroWrapper.__next__ method
# when _DEBUG is true.
_DEBUG = (not sys.flags.ignore_environment
          and bool(os.environ.get('PYTHONASYNCIODEBUG')))

# Check for CPython issue #21209
def has_yield_from_bug():
    class MyGen:
        def __init__(self):
            self.send_args = None
        def __iter__(self):
            return self
        def __next__(self):
            return 42
        def send(self, *what):
            self.send_args = what
            return None
    def yield_from_gen(gen):
        yield from gen
    value = (1, 2, 3)
    gen = MyGen()
    coro = yield_from_gen(gen)
    next(coro)
    coro.send(value)
    return gen.send_args != (value,)
_YIELD_FROM_BUG = has_yield_from_bug()
del has_yield_from_bug

#########################################
#             包裹协程对象
#
# 说明:
#   - 调试模式下使用
#   - 类装饰器
#   - 生成器实现
#
#########################################
class CoroWrapper:
    # Wrapper for coroutine object in _DEBUG mode.

    def __init__(self, gen, func):
        assert inspect.isgenerator(gen), gen
        self.gen = gen
        self.func = func
        self._source_traceback = traceback.extract_stack(sys._getframe(1))
        # __name__, __qualname__, __doc__ attributes are set by the coroutine()
        # decorator

    def __repr__(self):
        coro_repr = _format_coroutine(self)
        if self._source_traceback:
            frame = self._source_traceback[-1]
            coro_repr += ', created at %s:%s' % (frame[0], frame[1])
        return '<%s %s>' % (self.__class__.__name__, coro_repr)

    def __iter__(self):
        return self

    def __next__(self):
        return next(self.gen)

    if _YIELD_FROM_BUG:
        # For for CPython issue #21209: using "yield from" and a custom
        # generator, generator.send(tuple) unpacks the tuple instead of passing
        # the tuple unchanged. Check if the caller is a generator using "yield
        # from" to decide if the parameter should be unpacked or not.
        def send(self, *value):
            frame = sys._getframe()
            caller = frame.f_back
            assert caller.f_lasti >= 0
            if caller.f_code.co_code[caller.f_lasti] != _YIELD_FROM:
                value = value[0]
            return self.gen.send(value)
    else:
        def send(self, value):
            return self.gen.send(value)

    def throw(self, exc):
        return self.gen.throw(exc)

    def close(self):
        return self.gen.close()

    @property
    def gi_frame(self):
        return self.gen.gi_frame

    @property
    def gi_running(self):
        return self.gen.gi_running

    @property
    def gi_code(self):
        return self.gen.gi_code

    def __del__(self):
        # Be careful accessing self.gen.frame -- self.gen might not exist.
        gen = getattr(self, 'gen', None)
        frame = getattr(gen, 'gi_frame', None)
        if frame is not None and frame.f_lasti == -1:
            msg = '%r was never yielded from' % self
            tb = getattr(self, '_source_traceback', ())
            if tb:
                tb = ''.join(traceback.format_list(tb))
                msg += ('\nCoroutine object created at '
                        '(most recent call last):\n')
                msg += tb.rstrip()
            logger.error(msg)

#########################################
#             协程装饰器
#
# 说明:
#
#########################################
def coroutine(func):
    """Decorator to mark coroutines.

    If the coroutine is not yielded from before it is destroyed,
    an error message is logged.
    """
    if inspect.isgeneratorfunction(func):
        coro = func
    else:
        @functools.wraps(func)
        def coro(*args, **kw):
            res = func(*args, **kw)    # 执行被包裹的方法

            #
            # res 返回值类型判断
            #   - futures.Future 对象
            #
            if isinstance(res, futures.Future) or inspect.isgenerator(res):
                res = yield from res     # 异步返回: 被包裹方法执行结果
            return res

    if not _DEBUG:                # 非调试模式
        wrapper = coro
    else:                         # 调试模式
        @functools.wraps(func)
        def wrapper(*args, **kwds):
            w = CoroWrapper(coro(*args, **kwds), func)   # 协程包裹:

            if w._source_traceback:
                del w._source_traceback[-1]
            w.__name__ = func.__name__
            if hasattr(func, '__qualname__'):
                w.__qualname__ = func.__qualname__
            w.__doc__ = func.__doc__
            return w

    wrapper._is_coroutine = True  # For iscoroutinefunction().
    return wrapper

#
# 是否是协程方法(被协程装饰器包裹的方法):
#
def iscoroutinefunction(func):
    """Return True if func is a decorated coroutine function."""
    return getattr(func, '_is_coroutine', False)

_COROUTINE_TYPES = (types.GeneratorType, CoroWrapper)

#
# 是否是协程对象:
#
def iscoroutine(obj):
    """Return True if obj is a coroutine object."""
    return isinstance(obj, _COROUTINE_TYPES)

def _format_coroutine(coro):
    assert iscoroutine(coro)
    coro_name = getattr(coro, '__qualname__', coro.__name__)

    filename = coro.gi_code.co_filename
    if (isinstance(coro, CoroWrapper)
    and not inspect.isgeneratorfunction(coro.func)):
        #
        # 事件集
        #
        filename, lineno = events._get_function_source(coro.func)
        if coro.gi_frame is None:
            coro_repr = ('%s() done, defined at %s:%s'
                         % (coro_name, filename, lineno))
        else:
            coro_repr = ('%s() running, defined at %s:%s'
                         % (coro_name, filename, lineno))
    elif coro.gi_frame is not None:
        lineno = coro.gi_frame.f_lineno
        coro_repr = ('%s() running at %s:%s'
                     % (coro_name, filename, lineno))
    else:
        lineno = coro.gi_code.co_firstlineno
        coro_repr = ('%s() done, defined at %s:%s'
                     % (coro_name, filename, lineno))

    return coro_repr
<|endoftext|>


<commit_before>import itertools

from sympy import (Add, Mul, Pow, Symbol, exp, sqrt, symbols, sympify, cse,
    Matrix)
from sympy.simplify import cse_main, cse_opts
from sympy.utilities.pytest import XFAIL

w,x,y,z = symbols('w,x,y,z')
x0,x1,x2 = list(itertools.islice(cse_main.numbered_symbols(), 0, 3))
negone = sympify(-1)

def test_numbered_symbols():
    ns = cse_main.numbered_symbols(prefix='y')
    assert list(itertools.islice(ns, 0, 10)) == [Symbol('y%s'%i) for i in range(0, 10)]
    ns = cse_main.numbered_symbols(prefix='y')
    assert list(itertools.islice(ns, 10, 20)) == [Symbol('y%s'%i) for i in range(10, 20)]
    ns = cse_main.numbered_symbols()
    assert list(itertools.islice(ns, 0, 10)) == [Symbol('x%s'%i) for i in range(0, 10)]

# Dummy "optimization" functions for testing.

def opt1(expr):
    return expr+y
def opt2(expr):
    return expr*z

def test_preprocess_for_cse():
    assert cse_main.preprocess_for_cse(x, [(opt1, None)]) == x+y
    assert cse_main.preprocess_for_cse(x, [(None, opt1)]) == x
    assert cse_main.preprocess_for_cse(x, [(None, None)]) == x
    assert cse_main.preprocess_for_cse(x, [(opt1, opt2)]) == x+y
    assert cse_main.preprocess_for_cse(x, [(opt1, None), (opt2, None)]) == (x+y)*z

def test_postprocess_for_cse():
    assert cse_main.postprocess_for_cse(x, [(opt1, None)]) == x
    assert cse_main.postprocess_for_cse(x, [(None, opt1)]) == x+y
    assert cse_main.postprocess_for_cse(x, [(None, None)]) == x
    assert cse_main.postprocess_for_cse(x, [(opt1, opt2)]) == x*z
    # Note the reverse order of application.
    assert cse_main.postprocess_for_cse(x, [(None, opt1), (None, opt2)]) == x*z+y

def test_cse_single():
    # Simple substitution.
    e = Add(Pow(x+y,2), sqrt(x+y))
    substs, reduced = cse([e], optimizations=[])
    assert substs == [(x0, x+y)]
    assert reduced == [sqrt(x0) + x0**2]

def test_cse_single2():
    # Simple substitution, test for being able to pass the expression directly
    e = Add(Pow(x+y,2), sqrt(x+y))
    substs, reduced = cse(e, optimizations=[])
    assert substs == [(x0, x+y)]
    assert reduced == sqrt(x0) + x0**2
    assert isinstance(cse(Matrix([[1]]))[1], Matrix)

def test_cse_not_possible():
    # No substitution possible.
    e = Add(x,y)
    substs, reduced = cse([e], optimizations=[])
    assert substs == []
    assert reduced == [x+y]

def test_nested_substitution():
    # Substitution within a substitution.
    e = Add(Pow(w*x+y,2), sqrt(w*x+y))
    substs, reduced = cse([e], optimizations=[])
    assert substs == [(x0, w*x+y)]
    assert reduced == [sqrt(x0) + x0**2]

def test_subtraction_opt():
    # Make sure subtraction is optimized.
    e = (x-y)*(z-y) + exp((x-y)*(z-y))
    substs, reduced = cse([e], optimizations=[(cse_opts.sub_pre,cse_opts.sub_post)])
    assert substs == [(x0, (x - y)*(z - y))]
    assert reduced == [x0 + exp(x0)]

def test_multiple_expressions():
    e1 = (x+y)*z
    e2 = (x+y)*w
    substs, reduced = cse([e1, e2], optimizations=[])
    assert substs == [(x0, x+y)]
    assert reduced == [x0*z, x0*w]
    l = [w*x*y + z, w*y]
    substs, reduced = cse(l)
    rsubsts, _ = cse(reversed(l))
    assert substs == rsubsts
    assert reduced == [z + x*x0, x0]
    l = [w*x*y, w*x*y + z, w*y]
    substs, reduced = cse(l)
    rsubsts, _ = cse(reversed(l))
    assert substs == rsubsts
    assert reduced == [x1, x1 + z, x0]
    l = [(x - z)*(y - z), x - z, y - z]
    substs, reduced = cse(l)
    rsubsts, _ = cse(reversed(l))
    substitutions = [
        [(x0, x - z), (x1, y - z)],
        [(x0, y - z), (x1, x - z)],
    ]
    assert substs in substitutions
    assert rsubsts in substitutions
    assert reduced == [x0*x1, x0, x1]
    l = [w*y + w + x + y + z, w*x*y]
    assert cse(l) == ([(x0, w*y)], [w + x + x0 + y + z, x*x0])
    assert cse([x + y, x + y + z]) == ([(x0, x + y)], [x0, z + x0])
    assert cse([x + y, x + z]) == ([], [x + y, x + z])
    assert cse([x*y, x + x*y , x*y*z + 3]) == \
           ([(x0, x*y)], [x0, x + x0, 3 + x0*z])
    A, B, C = symbols('A B C', commutative=False)
    l = [A*B*C, A*C]
    assert cse(l) == ([], l)
    l = [A*B*C, A*B]
    assert cse(l) == ([(x0, A*B)], [x0*C, x0])

@XFAIL
def test_powers():
    assert cse(x*y**2 + x*y) == ([(x0, x*y)], [x0*y + x0])

<commit_msg>test for issue 1399<commit_after>import itertools

from sympy import (Add, Mul, Pow, Symbol, exp, sqrt, symbols, sympify, cse,
    Matrix)
from sympy.simplify import cse_main, cse_opts
from sympy.utilities.pytest import XFAIL

w,x,y,z = symbols('w,x,y,z')
x0,x1,x2 = list(itertools.islice(cse_main.numbered_symbols(), 0, 3))
negone = sympify(-1)

def test_numbered_symbols():
    ns = cse_main.numbered_symbols(prefix='y')
    assert list(itertools.islice(ns, 0, 10)) == [Symbol('y%s'%i) for i in range(0, 10)]
    ns = cse_main.numbered_symbols(prefix='y')
    assert list(itertools.islice(ns, 10, 20)) == [Symbol('y%s'%i) for i in range(10, 20)]
    ns = cse_main.numbered_symbols()
    assert list(itertools.islice(ns, 0, 10)) == [Symbol('x%s'%i) for i in range(0, 10)]

# Dummy "optimization" functions for testing.

def opt1(expr):
    return expr+y
def opt2(expr):
    return expr*z

def test_preprocess_for_cse():
    assert cse_main.preprocess_for_cse(x, [(opt1, None)]) == x+y
    assert cse_main.preprocess_for_cse(x, [(None, opt1)]) == x
    assert cse_main.preprocess_for_cse(x, [(None, None)]) == x
    assert cse_main.preprocess_for_cse(x, [(opt1, opt2)]) == x+y
    assert cse_main.preprocess_for_cse(x, [(opt1, None), (opt2, None)]) == (x+y)*z

def test_postprocess_for_cse():
    assert cse_main.postprocess_for_cse(x, [(opt1, None)]) == x
    assert cse_main.postprocess_for_cse(x, [(None, opt1)]) == x+y
    assert cse_main.postprocess_for_cse(x, [(None, None)]) == x
    assert cse_main.postprocess_for_cse(x, [(opt1, opt2)]) == x*z
    # Note the reverse order of application.
    assert cse_main.postprocess_for_cse(x, [(None, opt1), (None, opt2)]) == x*z+y

def test_cse_single():
    # Simple substitution.
    e = Add(Pow(x+y,2), sqrt(x+y))
    substs, reduced = cse([e], optimizations=[])
    assert substs == [(x0, x+y)]
    assert reduced == [sqrt(x0) + x0**2]

def test_cse_single2():
    # Simple substitution, test for being able to pass the expression directly
    e = Add(Pow(x+y,2), sqrt(x+y))
    substs, reduced = cse(e, optimizations=[])
    assert substs == [(x0, x+y)]
    assert reduced == sqrt(x0) + x0**2
    assert isinstance(cse(Matrix([[1]]))[1], Matrix)

def test_cse_not_possible():
    # No substitution possible.
    e = Add(x,y)
    substs, reduced = cse([e], optimizations=[])
    assert substs == []
    assert reduced == [x+y]

def test_nested_substitution():
    # Substitution within a substitution.
    e = Add(Pow(w*x+y,2), sqrt(w*x+y))
    substs, reduced = cse([e], optimizations=[])
    assert substs == [(x0, w*x+y)]
    assert reduced == [sqrt(x0) + x0**2]

def test_subtraction_opt():
    # Make sure subtraction is optimized.
    e = (x-y)*(z-y) + exp((x-y)*(z-y))
    substs, reduced = cse([e], optimizations=[(cse_opts.sub_pre,cse_opts.sub_post)])
    assert substs == [(x0, (x - y)*(z - y))]
    assert reduced == [x0 + exp(x0)]

def test_multiple_expressions():
    e1 = (x+y)*z
    e2 = (x+y)*w
    substs, reduced = cse([e1, e2], optimizations=[])
    assert substs == [(x0, x+y)]
    assert reduced == [x0*z, x0*w]
    l = [w*x*y + z, w*y]
    substs, reduced = cse(l)
    rsubsts, _ = cse(reversed(l))
    assert substs == rsubsts
    assert reduced == [z + x*x0, x0]
    l = [w*x*y, w*x*y + z, w*y]
    substs, reduced = cse(l)
    rsubsts, _ = cse(reversed(l))
    assert substs == rsubsts
    assert reduced == [x1, x1 + z, x0]
    l = [(x - z)*(y - z), x - z, y - z]
    substs, reduced = cse(l)
    rsubsts, _ = cse(reversed(l))
    substitutions = [
        [(x0, x - z), (x1, y - z)],
        [(x0, y - z), (x1, x - z)],
    ]
    assert substs in substitutions
    assert rsubsts in substitutions
    assert reduced == [x0*x1, x0, x1]
    l = [w*y + w + x + y + z, w*x*y]
    assert cse(l) == ([(x0, w*y)], [w + x + x0 + y + z, x*x0])
    assert cse([x + y, x + y + z]) == ([(x0, x + y)], [x0, z + x0])
    assert cse([x + y, x + z]) == ([], [x + y, x + z])
    assert cse([x*y, x + x*y , x*y*z + 3]) == \
           ([(x0, x*y)], [x0, x + x0, 3 + x0*z])
    A, B, C = symbols('A B C', commutative=False)
    l = [A*B*C, A*C]
    assert cse(l) == ([], l)
    l = [A*B*C, A*B]
    assert cse(l) == ([(x0, A*B)], [x0*C, x0])

@XFAIL
def test_powers():
    assert cse(x*y**2 + x*y) == ([(x0, x*y)], [x0*y + x0])

def test_issue_1399():
    assert cse(w/(x - y) + z/(y - x)) == ([(x0, 1/(x - y))], w*x0 - x0*z)
<|endoftext|>


<commit_before>#!/usr/bin/env python
""" This module tests the software.

"""
import numpy as np
import os

import sys
sys.path.insert(0, '/home/peisenha/restudToolbox/package/respy')

from respy.python.shared.shared_auxiliary import dist_class_attributes
from respy.python.shared.shared_auxiliary import dist_model_paras
from respy.fortran.fortran_auxiliary import write_resfort_initialization

from respy.tests.codes.random_init import generate_init

from respy import RespyCls
from respy.solve import solve

def get_resfort_init():

    respy_obj = RespyCls('test.respy.ini')
    # Distribute class attributes
    model_paras, num_periods, edu_start, is_debug, edu_max, delta, version, \
        num_draws_emax, seed_emax, is_interpolated, num_points, is_myopic, \
        min_idx, store, tau = dist_class_attributes(
        respy_obj, 'model_paras', 'num_periods', 'edu_start', 'is_debug', 'edu_max',
        'delta', 'version', 'num_draws_emax', 'seed_emax', 'is_interpolated',
        'num_points', 'is_myopic', 'min_idx', 'store', 'tau')

    # Distribute model parameters
    coeffs_a, coeffs_b, coeffs_edu, coeffs_home, shocks_cholesky = dist_model_paras(
        model_paras, is_debug)

    # Collect baseline arguments. These are latter amended to account for
    # each interface.

    args = (coeffs_a, coeffs_b, coeffs_edu, coeffs_home, shocks_cholesky, is_interpolated,
    num_draws_emax, num_periods, num_points, is_myopic, edu_start, is_debug,
    edu_max, min_idx, delta)

    args = args + (1, 1, 1, seed_emax, tau, 'solve')
    write_resfort_initialization(*args)

# Test the functionality of the executable for varying number of slaves and
# varying number of model specifications.
import numpy as np
# TODO: Only checks if executing without problem, later test if same result.
for i in range(10):

    # This generates the initialization file
    constr = dict()
    constr['version'] = 'FORTRAN'
    constr['apply'] = False
    generate_init(constr)

    respy_obj = RespyCls('test.respy.ini')
    respy_obj = solve(respy_obj)

    package = respy_obj.get_attr('periods_emax')[0, 0]
    # This ensures
    get_resfort_init()
    base = None
    # TODO: Just draw a random number of slaves
    num_slaves = np.random.randint(1, 5)

    # cmd = 'mpiexec ./master ' + str(num_slaves)
    # assert os.system(cmd) == 0
    # base = np.loadtxt('.eval.resfort.dat')
    # print('\n\n Visual Debugging')
    # print(base, package)
    # print('\n\n')
    #
    # np.testing.assert_allclose(base, package)
<commit_msg>investigating test problem<commit_after>#!/usr/bin/env python
""" This module tests the software.

"""
import numpy as np
import os

import sys
sys.path.insert(0, '/home/peisenha/restudToolbox/package/respy')

from respy.python.shared.shared_auxiliary import dist_class_attributes
from respy.python.shared.shared_auxiliary import dist_model_paras
from respy.fortran.fortran_auxiliary import write_resfort_initialization

from respy.tests.codes.random_init import generate_init

from respy import RespyCls
from respy.solve import solve

def get_resfort_init():

    respy_obj = RespyCls('test.respy.ini')

    # Distribute class attributes
    model_paras, num_periods, edu_start, is_debug, edu_max, delta, version, \
        num_draws_emax, seed_emax, is_interpolated, num_points, is_myopic, \
        min_idx, store, tau = dist_class_attributes(
        respy_obj, 'model_paras', 'num_periods', 'edu_start', 'is_debug', 'edu_max',
        'delta', 'version', 'num_draws_emax', 'seed_emax', 'is_interpolated',
        'num_points', 'is_myopic', 'min_idx', 'store', 'tau')

    # Distribute model parameters
    coeffs_a, coeffs_b, coeffs_edu, coeffs_home, shocks_cholesky = dist_model_paras(
        model_paras, is_debug)

    # Collect baseline arguments. These are latter amended to account for
    # each interface.

    args = (coeffs_a, coeffs_b, coeffs_edu, coeffs_home, shocks_cholesky, is_interpolated,
    num_draws_emax, num_periods, num_points, is_myopic, edu_start, is_debug,
    edu_max, min_idx, delta)

    args = args + (1, 1, 1, seed_emax, tau, 'solve')
    write_resfort_initialization(*args)

# Test the functionality of the executable for varying number of slaves and
# varying number of model specifications.
for i in range(10):

    # This generates the initialization file
    constr = dict()
    #constr['version'] = 'FORTRAN'
    #constr['apply'] = False
    generate_init(constr)

    respy_obj = RespyCls('test.respy.ini')
    #respy_obj = solve(respy_obj)

    # package = respy_obj.get_attr('periods_emax')[0, 0]
    # # This ensures
    # get_resfort_init()
    # base = None
    # # TODO: Just draw a random number of slaves
    # num_slaves = np.random.randint(1, 5)

    # cmd = 'mpiexec ./master ' + str(num_slaves)
    # assert os.system(cmd) == 0
    # base = np.loadtxt('.eval.resfort.dat')
    # print('\n\n Visual Debugging')
    # print(base, package)
    # print('\n\n')
    #
    # np.testing.assert_allclose(base, package)
<|endoftext|>


<commit_before># Copyright 2012 OpenStack Foundation
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from contextlib import closing
import logging
import tempfile
import zipfile

from django.core.urlresolvers import reverse_lazy
from django import http
from django import shortcuts
from django.template.loader import render_to_string
from django.utils.translation import ugettext_lazy as _

from openstack_auth import utils

from horizon import exceptions
from horizon import forms
from horizon import messages
from horizon import tables
from horizon import views

from openstack_dashboard import api
from openstack_dashboard.dashboards.project.api_access \
    import forms as api_access_forms
from openstack_dashboard.dashboards.project.api_access \
    import tables as api_access_tables

LOG = logging.getLogger(__name__)

def _get_ec2_credentials(request):
    tenant_id = request.user.tenant_id
    all_keys = api.keystone.list_ec2_credentials(request,
                                                 request.user.id)

    key = next((x for x in all_keys if x.tenant_id == tenant_id), None)
    if not key:
        key = api.keystone.create_ec2_credentials(request,
                                                  request.user.id,
                                                  tenant_id)
    try:
        s3_endpoint = api.base.url_for(request,
                                       's3',
                                       endpoint_type='publicURL')
    except exceptions.ServiceCatalogException:
        s3_endpoint = None

    try:
        ec2_endpoint = api.base.url_for(request,
                                        'ec2',
                                        endpoint_type='publicURL')
    except exceptions.ServiceCatalogException:
        ec2_endpoint = None

    return {'ec2_access_key': key.access,
            'ec2_secret_key': key.secret,
            'ec2_endpoint': ec2_endpoint,
            's3_endpoint': s3_endpoint}

def _get_openrc_credentials(request):
    keystone_url = api.base.url_for(request,
                                    'identity',
                                    endpoint_type='publicURL')
    credentials = dict(tenant_id=request.user.tenant_id,
                       tenant_name=request.user.tenant_name,
                       auth_url=keystone_url,
                       user=request.user,
                       interface='public',
                       os_endpoint_type='publicURL',
                       region=getattr(request.user, 'services_region') or "")
    return credentials

def download_ec2_bundle(request):
    tenant_name = request.user.tenant_name

    # Gather or create our EC2 credentials
    try:
        credentials = api.nova.get_x509_credentials(request)
        cacert = api.nova.get_x509_root_certificate(request)
        context = _get_ec2_credentials(request)
    except Exception:
        exceptions.handle(request,
                          _('Unable to fetch EC2 credentials.'),
                          redirect=request.build_absolute_uri())

    # Create our file bundle
    template = 'project/api_access/ec2rc.sh.template'
    try:
        temp_zip = tempfile.NamedTemporaryFile(delete=True)
        with closing(zipfile.ZipFile(temp_zip.name, mode='w')) as archive:
            archive.writestr('pk.pem', credentials.private_key)
            archive.writestr('cert.pem', credentials.data)
            archive.writestr('cacert.pem', cacert.data)
            archive.writestr('ec2rc.sh', render_to_string(template, context))
    except Exception:
        exceptions.handle(request,
                          _('Error writing zipfile: %(exc)s'),
                          redirect=request.build_absolute_uri())

    # Send it back
    response = http.HttpResponse(content_type='application/zip')
    response.write(temp_zip.read())
    response['Content-Disposition'] = ('attachment; '
                                       'filename="%s-x509.zip"'
                                       % tenant_name)
    response['Content-Length'] = temp_zip.tell()
    return response

def download_rc_file_v2(request):
    template = 'project/api_access/openrc_v2.sh.template'
    context = _get_openrc_credentials(request)
    context['os_identity_api_version'] = 2
    context['os_auth_version'] = 2
    return _download_rc_file_for_template(request, context, template)

def download_rc_file(request):
    template = 'project/api_access/openrc.sh.template'
    context = _get_openrc_credentials(request)

    # make v3 specific changes
    context['user_domain_name'] = request.user.user_domain_name
    # sanity fix for removing v2.0 from the url if present
    context['auth_url'], _ = utils.fix_auth_url_version_prefix(
        context['auth_url'])
    context['os_identity_api_version'] = 3
    context['os_auth_version'] = 3
    return _download_rc_file_for_template(request, context, template)

def _download_rc_file_for_template(request, context, template):
    try:
        response = shortcuts.render(request,
                                    template,
                                    context,
                                    content_type="text/plain")
        tenant_name = context['tenant_name']
        disposition = 'attachment; filename="%s-openrc.sh"' % tenant_name
        response['Content-Disposition'] = disposition.encode('utf-8')
        response['Content-Length'] = str(len(response.content))
        return response

    except Exception as e:
        LOG.exception("Exception in DownloadOpenRCForm.")
        messages.error(request, _('Error Downloading RC File: %s') % e)
        return shortcuts.redirect(request.build_absolute_uri())

class CredentialsView(forms.ModalFormMixin, views.HorizonTemplateView):
    template_name = 'project/api_access/credentials.html'
    page_title = _("User Credentials Details")

    def get_context_data(self, **kwargs):
        context = super(CredentialsView, self).get_context_data(**kwargs)
        try:
            context['openrc_creds'] = _get_openrc_credentials(self.request)
        except Exception:
            exceptions.handle(self.request,
                              _('Unable to get openrc credentials'))
        if api.base.is_service_enabled(self.request, 'ec2'):
            try:
                context['ec2_creds'] = _get_ec2_credentials(self.request)
            except Exception:
                exceptions.handle(self.request,
                                  _('Unable to get EC2 credentials'))
        return context

class RecreateCredentialsView(forms.ModalFormView):
    form_class = api_access_forms.RecreateCredentials
    form_id = "recreate_credentials"
    page_title = _("Recreate EC2 Credentials")
    template_name = \
        'project/api_access/recreate_credentials.html'
    submit_label = _("Recreate EC2 Credentials")
    submit_url = reverse_lazy(
        "horizon:project:api_access:recreate_credentials")
    success_url = reverse_lazy('horizon:project:api_access:index')

class IndexView(tables.DataTableView):
    table_class = api_access_tables.EndpointsTable
    page_title = _("API Access")

    def get_data(self):
        services = []
        for i, service in enumerate(self.request.user.service_catalog):
            service['id'] = i
            services.append(
                api.keystone.Service(service,
                                     self.request.user.services_region))

        return services
<commit_msg>Sort service list in API Access<commit_after># Copyright 2012 OpenStack Foundation
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from contextlib import closing
import logging
import tempfile
import zipfile

from django.core.urlresolvers import reverse_lazy
from django import http
from django import shortcuts
from django.template.loader import render_to_string
from django.utils.translation import ugettext_lazy as _

from openstack_auth import utils

from horizon import exceptions
from horizon import forms
from horizon import messages
from horizon import tables
from horizon import views

from openstack_dashboard import api
from openstack_dashboard.dashboards.project.api_access \
    import forms as api_access_forms
from openstack_dashboard.dashboards.project.api_access \
    import tables as api_access_tables

LOG = logging.getLogger(__name__)

def _get_ec2_credentials(request):
    tenant_id = request.user.tenant_id
    all_keys = api.keystone.list_ec2_credentials(request,
                                                 request.user.id)

    key = next((x for x in all_keys if x.tenant_id == tenant_id), None)
    if not key:
        key = api.keystone.create_ec2_credentials(request,
                                                  request.user.id,
                                                  tenant_id)
    try:
        s3_endpoint = api.base.url_for(request,
                                       's3',
                                       endpoint_type='publicURL')
    except exceptions.ServiceCatalogException:
        s3_endpoint = None

    try:
        ec2_endpoint = api.base.url_for(request,
                                        'ec2',
                                        endpoint_type='publicURL')
    except exceptions.ServiceCatalogException:
        ec2_endpoint = None

    return {'ec2_access_key': key.access,
            'ec2_secret_key': key.secret,
            'ec2_endpoint': ec2_endpoint,
            's3_endpoint': s3_endpoint}

def _get_openrc_credentials(request):
    keystone_url = api.base.url_for(request,
                                    'identity',
                                    endpoint_type='publicURL')
    credentials = dict(tenant_id=request.user.tenant_id,
                       tenant_name=request.user.tenant_name,
                       auth_url=keystone_url,
                       user=request.user,
                       interface='public',
                       os_endpoint_type='publicURL',
                       region=getattr(request.user, 'services_region') or "")
    return credentials

def download_ec2_bundle(request):
    tenant_name = request.user.tenant_name

    # Gather or create our EC2 credentials
    try:
        credentials = api.nova.get_x509_credentials(request)
        cacert = api.nova.get_x509_root_certificate(request)
        context = _get_ec2_credentials(request)
    except Exception:
        exceptions.handle(request,
                          _('Unable to fetch EC2 credentials.'),
                          redirect=request.build_absolute_uri())

    # Create our file bundle
    template = 'project/api_access/ec2rc.sh.template'
    try:
        temp_zip = tempfile.NamedTemporaryFile(delete=True)
        with closing(zipfile.ZipFile(temp_zip.name, mode='w')) as archive:
            archive.writestr('pk.pem', credentials.private_key)
            archive.writestr('cert.pem', credentials.data)
            archive.writestr('cacert.pem', cacert.data)
            archive.writestr('ec2rc.sh', render_to_string(template, context))
    except Exception:
        exceptions.handle(request,
                          _('Error writing zipfile: %(exc)s'),
                          redirect=request.build_absolute_uri())

    # Send it back
    response = http.HttpResponse(content_type='application/zip')
    response.write(temp_zip.read())
    response['Content-Disposition'] = ('attachment; '
                                       'filename="%s-x509.zip"'
                                       % tenant_name)
    response['Content-Length'] = temp_zip.tell()
    return response

def download_rc_file_v2(request):
    template = 'project/api_access/openrc_v2.sh.template'
    context = _get_openrc_credentials(request)
    context['os_identity_api_version'] = 2
    context['os_auth_version'] = 2
    return _download_rc_file_for_template(request, context, template)

def download_rc_file(request):
    template = 'project/api_access/openrc.sh.template'
    context = _get_openrc_credentials(request)

    # make v3 specific changes
    context['user_domain_name'] = request.user.user_domain_name
    # sanity fix for removing v2.0 from the url if present
    context['auth_url'], _ = utils.fix_auth_url_version_prefix(
        context['auth_url'])
    context['os_identity_api_version'] = 3
    context['os_auth_version'] = 3
    return _download_rc_file_for_template(request, context, template)

def _download_rc_file_for_template(request, context, template):
    try:
        response = shortcuts.render(request,
                                    template,
                                    context,
                                    content_type="text/plain")
        tenant_name = context['tenant_name']
        disposition = 'attachment; filename="%s-openrc.sh"' % tenant_name
        response['Content-Disposition'] = disposition.encode('utf-8')
        response['Content-Length'] = str(len(response.content))
        return response

    except Exception as e:
        LOG.exception("Exception in DownloadOpenRCForm.")
        messages.error(request, _('Error Downloading RC File: %s') % e)
        return shortcuts.redirect(request.build_absolute_uri())

class CredentialsView(forms.ModalFormMixin, views.HorizonTemplateView):
    template_name = 'project/api_access/credentials.html'
    page_title = _("User Credentials Details")

    def get_context_data(self, **kwargs):
        context = super(CredentialsView, self).get_context_data(**kwargs)
        try:
            context['openrc_creds'] = _get_openrc_credentials(self.request)
        except Exception:
            exceptions.handle(self.request,
                              _('Unable to get openrc credentials'))
        if api.base.is_service_enabled(self.request, 'ec2'):
            try:
                context['ec2_creds'] = _get_ec2_credentials(self.request)
            except Exception:
                exceptions.handle(self.request,
                                  _('Unable to get EC2 credentials'))
        return context

class RecreateCredentialsView(forms.ModalFormView):
    form_class = api_access_forms.RecreateCredentials
    form_id = "recreate_credentials"
    page_title = _("Recreate EC2 Credentials")
    template_name = \
        'project/api_access/recreate_credentials.html'
    submit_label = _("Recreate EC2 Credentials")
    submit_url = reverse_lazy(
        "horizon:project:api_access:recreate_credentials")
    success_url = reverse_lazy('horizon:project:api_access:index')

class IndexView(tables.DataTableView):
    table_class = api_access_tables.EndpointsTable
    page_title = _("API Access")

    def get_data(self):
        services = []
        for i, service in enumerate(self.request.user.service_catalog):
            service['id'] = i
            services.append(
                api.keystone.Service(service,
                                     self.request.user.services_region))
        return sorted(services, key=lambda s: s.type)
<|endoftext|>


<commit_before><commit_msg>add motifembedding model<commit_after><|endoftext|>


<commit_before>import logging
import geojson as gj
import copy
import attrdict as ad
import pandas as pd

import emission.storage.timeseries.abstract_timeseries as esta
import emission.net.usercache.abstract_usercache as enua
import emission.storage.timeseries.timequery as estt

import emission.storage.decorations.trip_queries as esdt
import emission.storage.decorations.analysis_timeseries_queries as esda
import emission.storage.decorations.timeline as esdtl

import emission.core.wrapper.location as ecwl
import emission.core.wrapper.entry as ecwe
import emission.core.common as ecc

# TODO: Move this to the section_features class instead
import emission.analysis.intake.cleaning.location_smoothing as eaicl

def _del_non_derializable(prop_dict, extra_keys):
    for key in extra_keys:
        if key in prop_dict:
            del prop_dict[key]

def _stringify_foreign_key(prop_dict, key_names):
    for key_name in key_names:
        if hasattr(prop_dict, key_name):
            setattr(prop_dict, key_name, str(getattr(prop_dict,key_name)))

def location_to_geojson(location):
    """
    Converts a location wrapper object into geojson format.
    This is pretty easy - it is a point.
    Since we have other properties that we care about, we make it a feature.
    Then, all the other stuff goes directly into the properties since the wrapper is a dict too!
    :param location: the location object
    :return: a geojson version of the location. the object is of type "Feature".
    """
    try:
        ret_feature = gj.Feature()
        ret_feature.id = str(location.get_id())
        ret_feature.geometry = location.data.loc
        ret_feature.properties = copy.copy(location.data)
        ret_feature.properties["feature_type"] = "location"
        _del_non_derializable(ret_feature.properties, ["loc"])
        return ret_feature
    except Exception, e:
        logging.exception(("Error while converting object %s" % location))
        raise e

def place_to_geojson(place):
    """
    Converts a place wrapper object into geojson format.
    This is also pretty easy - it is just a point.
    Since we have other properties that we care about, we make it a feature.
    Then, all the other stuff goes directly into the properties since the wrapper is a dict too!
    :param place: the place object
    :return: a geojson version of the place. the object is of type "Feature".
    """

    ret_feature = gj.Feature()
    ret_feature.id = str(place.get_id())
    ret_feature.geometry = place.data.location
    ret_feature.properties = copy.copy(place.data)
    ret_feature.properties["feature_type"] = "place"
    # _stringify_foreign_key(ret_feature.properties, ["ending_trip", "starting_trip"])
    _del_non_derializable(ret_feature.properties, ["location"])
    return ret_feature

def stop_to_geojson(stop):
    """
    Converts a stop wrapper object into geojson format.
    This is also pretty easy - it is just a point.
    Since we have other properties that we care about, we make it a feature.
    Then, all the other stuff goes directly into the properties since the wrapper is a dict too!
    :param stop: the stop object
    :return: a geojson version of the stop. the object is of type "Feature".
    """

    ret_feature = gj.Feature()
    ret_feature.id = str(stop.get_id())
    ret_feature.geometry = gj.LineString()
    ret_feature.geometry.coordinates = [stop.data.enter_loc.coordinates, stop.data.exit_loc.coordinates]
    ret_feature.properties = copy.copy(stop.data)
    ret_feature.properties["feature_type"] = "stop"

    # _stringify_foreign_key(ret_feature.properties, ["ending_section", "starting_section", "trip_id"])
    _del_non_derializable(ret_feature.properties, ["location"])
    return ret_feature

def section_to_geojson(section, tl):
    """
    This is the trickiest part of the visualization.
    The section is basically a collection of points with a line through them.
    So the representation is a feature in which one feature which is the line, and one feature collection which is the set of point features.
    :param section: the section to be converted
    :return: a feature collection which is the geojson version of the section
    """

    ts = esta.TimeSeries.get_time_series(section.user_id)
    entry_it = ts.find_entries(["analysis/recreated_location"],
                               esda.get_time_query_for_trip_like(
                                   "analysis/cleaned_section",
                                   section.get_id()))

    # TODO: Decide whether we want to use Rewrite to use dataframes throughout instead of python arrays.
    # dataframes insert nans. We could use fillna to fill with default values, but if we are not actually
    # using dataframe features here, it is unclear how much that would help.
    feature_array = []
    section_location_entries = [ecwe.Entry(entry) for entry in entry_it]
    if len(section_location_entries) != 0:
        logging.debug("first element in section_location_array = %s" % section_location_entries[0])

        if not ecc.compare_rounded_arrays(section.data.end_loc.coordinates,
                                      section_location_entries[-1].data.loc.coordinates,
                                      digits=4):
            logging.info("section_location_array[-1].data.loc %s != section.data.end_loc %s even after df.ts fix, filling gap" % \
                    (section_location_entries[-1].data.loc, section.data.end_loc))
            assert(False)
            last_loc_doc = ts.get_entry_at_ts("background/filtered_location", "data.ts", section.data.end_ts)
            if last_loc_doc is None:
                logging.warning("can't find entry to patch gap, leaving gap")
            else:
                last_loc_entry = ecwe.Entry(last_loc_doc)
                logging.debug("Adding new entry %s to fill the end point gap between %s and %s"
                   % (last_loc_entry.data.loc, section_location_entries[-1].data.loc,
                        section.data.end_loc))
                section_location_entries.append(last_loc_entry)

    points_line_feature = point_array_to_line(section_location_entries)
    points_line_feature.id = str(section.get_id())
    points_line_feature.properties.update(copy.copy(section.data))
    points_line_feature.properties["feature_type"] = "section"
    points_line_feature.properties["sensed_mode"] = str(points_line_feature.properties["sensed_mode"])

    _del_non_derializable(points_line_feature.properties, ["start_loc", "end_loc"])

    # feature_array.append(gj.FeatureCollection(points_feature_array))
    feature_array.append(points_line_feature)

    return gj.FeatureCollection(feature_array)

def incident_to_geojson(incident):
    ret_feature = gj.Feature()
    ret_feature.id = str(incident.get_id())
    ret_feature.geometry = gj.Point()
    ret_feature.geometry.coordinates = incident.data.loc.coordinates
    ret_feature.properties = copy.copy(incident.data)
    ret_feature.properties["feature_type"] = "incident"

    # _stringify_foreign_key(ret_feature.properties, ["ending_section", "starting_section", "trip_id"])
    _del_non_derializable(ret_feature.properties, ["loc"])
    return ret_feature

def geojson_incidents_in_range(user_id, start_ts, end_ts):
    MANUAL_INCIDENT_KEY = "manual/incident"
    ts = esta.TimeSeries.get_time_series(user_id)
    uc = enua.UserCache.getUserCache(user_id)
    tq = estt.TimeQuery("data.ts", start_ts, end_ts)
    incident_entry_docs = list(ts.find_entries([MANUAL_INCIDENT_KEY], time_query=tq)) \
        + list(uc.getMessage([MANUAL_INCIDENT_KEY], tq))
    incidents = [ecwe.Entry(doc) for doc in incident_entry_docs]
    return map(incident_to_geojson, incidents)

def point_array_to_line(point_array):
    points_line_string = gj.LineString()
    # points_line_string.coordinates = [l.loc.coordinates for l in filtered_section_location_array]
    points_line_string.coordinates = []
    points_times = []

    for l in point_array:
        # logging.debug("About to add %s to line_string " % l)
        points_line_string.coordinates.append(l.data.loc.coordinates)
        points_times.append(l.data.ts)

    points_line_feature = gj.Feature()
    points_line_feature.geometry = points_line_string
    points_line_feature.properties = {}
    points_line_feature.properties["times"] = points_times
    return points_line_feature

def trip_to_geojson(trip, tl):
    """
    Trips are the main focus of our current visualization, so they are most complex.
    Each trip is represented as a feature collection with the following features:
    - two features for the start and end places
    - features for each stop in the trip
    - features for each section in the trip

    :param trip: the trip object to be converted
    :param tl: the timeline used to retrieve related objects
    :return: the geojson version of the trip
    """

    feature_array = []
    curr_start_place = tl.get_object(trip.data.start_place)
    curr_end_place = tl.get_object(trip.data.end_place)
    start_place_geojson = place_to_geojson(curr_start_place)
    start_place_geojson["properties"]["feature_type"] = "start_place"
    feature_array.append(start_place_geojson)

    end_place_geojson = place_to_geojson(curr_end_place)
    end_place_geojson["properties"]["feature_type"] = "end_place"
    feature_array.append(end_place_geojson)

    trip_tl = esdt.get_cleaned_timeline_for_trip(trip.user_id, trip.get_id())
    stops = trip_tl.places
    for stop in stops:
        feature_array.append(stop_to_geojson(stop))

    for i, section in enumerate(trip_tl.trips):
        section_gj = section_to_geojson(section, tl)
        feature_array.append(section_gj)

    trip_geojson = gj.FeatureCollection(features=feature_array, properties=trip.data)
    trip_geojson.id = str(trip.get_id())

    feature_array.extend(geojson_incidents_in_range(trip.user_id,
                                              curr_start_place.data.exit_ts,
                                              curr_end_place.data.enter_ts))
    if trip.metadata.key == esda.CLEANED_UNTRACKED_KEY:
        # trip_geojson.properties["feature_type"] = "untracked"
        # Since the "untracked" type is not correctly handled on the phone, we just
        # skip these trips until
        # https://github.com/e-mission/e-mission-phone/issues/118
        # is fixed
        # TODO: Once it is fixed, re-introduce the first line in this block
        # and remove the None check in get_geojson_for_timeline
        return None
    else:
        trip_geojson.properties["feature_type"] = "trip"
    return trip_geojson

def get_geojson_for_ts(user_id, start_ts, end_ts):
    tl = esdtl.get_cleaned_timeline(user_id, start_ts, end_ts)
    tl.fill_start_end_places()
    return get_geojson_for_timeline(user_id, tl)

def get_geojson_for_dt(user_id, start_local_dt, end_local_dt):
    logging.debug("Getting geojson for %s -> %s" % (start_local_dt, end_local_dt))
    tl = esdtl.get_cleaned_timeline_from_dt(user_id, start_local_dt, end_local_dt)
    tl.fill_start_end_places()
    return get_geojson_for_timeline(user_id, tl)

def get_geojson_for_timeline(user_id, tl):
    """
    tl represents the "timeline" object that is queried for the trips and locations
    """
    geojson_list = []

    for trip in tl.trips:
        try:
            trip_geojson = trip_to_geojson(trip, tl)
            if trip_geojson is not None:
                geojson_list.append(trip_geojson)
        except Exception, e:
            logging.exception("Found error %s while processing trip %s" % (e, trip))
            raise e
    logging.debug("trip count = %d, geojson count = %d" %
                  (len(tl.trips), len(geojson_list)))
    return geojson_list

def get_all_points_for_range(user_id, key, start_ts, end_ts):
    import emission.storage.timeseries.timequery as estt
#     import emission.core.wrapper.location as ecwl

    tq = estt.TimeQuery("metadata.write_ts", start_ts, end_ts)
    ts = esta.TimeSeries.get_time_series(user_id)
    entry_it = ts.find_entries([key], tq)
    points_array = [ecwe.Entry(entry) for entry in entry_it]

    return get_feature_list_for_point_array(points_array)

def get_feature_list_for_point_array(points_array):
    points_feature_array = [location_to_geojson(le) for le in points_array]
    print ("Found %d features from %d points" %
           (len(points_feature_array), len(points_array)))

    feature_array = []
    feature_array.append(gj.FeatureCollection(points_feature_array))
    feature_array.append(point_array_to_line(points_array))
    feature_coll = gj.FeatureCollection(feature_array)

    return feature_coll

def get_feature_list_from_df(loc_time_df, ts="ts", latitude="latitude", longitude="longitude", fmt_time="fmt_time"):
    """
    Input DF should have columns called "ts", "latitude" and "longitude", or the corresponding
    columns can be passed in using the ts, latitude and longitude parameters
    """
    points_array = get_location_entry_list_from_df(loc_time_df, ts, latitude, longitude, fmt_time)
    return get_feature_list_for_point_array(points_array)

def get_location_entry_list_from_df(loc_time_df, ts="ts", latitude="latitude", longitude="longitude", fmt_time="fmt_time"):
    location_entry_list = []
    for idx, row in loc_time_df.iterrows():
        retVal = {"latitude": row[latitude], "longitude": row[longitude], "ts": row[ts],
                  "_id": str(idx), "fmt_time": row[fmt_time], "loc": gj.Point(coordinates=[row[longitude], row[latitude]])}
        location_entry_list.append(ecwe.Entry.create_entry(
            "dummy_user", "background/location", ecwl.Location(retVal)))
    return location_entry_list
<commit_msg>Restore the geojson properties to a section<commit_after>import logging
import geojson as gj
import copy
import attrdict as ad
import pandas as pd

import emission.storage.timeseries.abstract_timeseries as esta
import emission.net.usercache.abstract_usercache as enua
import emission.storage.timeseries.timequery as estt

import emission.storage.decorations.trip_queries as esdt
import emission.storage.decorations.analysis_timeseries_queries as esda
import emission.storage.decorations.timeline as esdtl

import emission.core.wrapper.location as ecwl
import emission.core.wrapper.cleanedsection as ecwcs
import emission.core.wrapper.entry as ecwe
import emission.core.common as ecc

# TODO: Move this to the section_features class instead
import emission.analysis.intake.cleaning.location_smoothing as eaicl

def _del_non_derializable(prop_dict, extra_keys):
    for key in extra_keys:
        if key in prop_dict:
            del prop_dict[key]

def _stringify_foreign_key(prop_dict, key_names):
    for key_name in key_names:
        if hasattr(prop_dict, key_name):
            setattr(prop_dict, key_name, str(getattr(prop_dict,key_name)))

def location_to_geojson(location):
    """
    Converts a location wrapper object into geojson format.
    This is pretty easy - it is a point.
    Since we have other properties that we care about, we make it a feature.
    Then, all the other stuff goes directly into the properties since the wrapper is a dict too!
    :param location: the location object
    :return: a geojson version of the location. the object is of type "Feature".
    """
    try:
        ret_feature = gj.Feature()
        ret_feature.id = str(location.get_id())
        ret_feature.geometry = location.data.loc
        ret_feature.properties = copy.copy(location.data)
        ret_feature.properties["feature_type"] = "location"
        _del_non_derializable(ret_feature.properties, ["loc"])
        return ret_feature
    except Exception, e:
        logging.exception(("Error while converting object %s" % location))
        raise e

def place_to_geojson(place):
    """
    Converts a place wrapper object into geojson format.
    This is also pretty easy - it is just a point.
    Since we have other properties that we care about, we make it a feature.
    Then, all the other stuff goes directly into the properties since the wrapper is a dict too!
    :param place: the place object
    :return: a geojson version of the place. the object is of type "Feature".
    """

    ret_feature = gj.Feature()
    ret_feature.id = str(place.get_id())
    ret_feature.geometry = place.data.location
    ret_feature.properties = copy.copy(place.data)
    ret_feature.properties["feature_type"] = "place"
    # _stringify_foreign_key(ret_feature.properties, ["ending_trip", "starting_trip"])
    _del_non_derializable(ret_feature.properties, ["location"])
    return ret_feature

def stop_to_geojson(stop):
    """
    Converts a stop wrapper object into geojson format.
    This is also pretty easy - it is just a point.
    Since we have other properties that we care about, we make it a feature.
    Then, all the other stuff goes directly into the properties since the wrapper is a dict too!
    :param stop: the stop object
    :return: a geojson version of the stop. the object is of type "Feature".
    """

    ret_feature = gj.Feature()
    ret_feature.id = str(stop.get_id())
    ret_feature.geometry = gj.LineString()
    ret_feature.geometry.coordinates = [stop.data.enter_loc.coordinates, stop.data.exit_loc.coordinates]
    ret_feature.properties = copy.copy(stop.data)
    ret_feature.properties["feature_type"] = "stop"

    # _stringify_foreign_key(ret_feature.properties, ["ending_section", "starting_section", "trip_id"])
    _del_non_derializable(ret_feature.properties, ["location"])
    return ret_feature

def section_to_geojson(section, tl):
    """
    This is the trickiest part of the visualization.
    The section is basically a collection of points with a line through them.
    So the representation is a feature in which one feature which is the line, and one feature collection which is the set of point features.
    :param section: the section to be converted
    :return: a feature collection which is the geojson version of the section
    """

    ts = esta.TimeSeries.get_time_series(section.user_id)
    entry_it = ts.find_entries(["analysis/recreated_location"],
                               esda.get_time_query_for_trip_like(
                                   "analysis/cleaned_section",
                                   section.get_id()))

    # TODO: Decide whether we want to use Rewrite to use dataframes throughout instead of python arrays.
    # dataframes insert nans. We could use fillna to fill with default values, but if we are not actually
    # using dataframe features here, it is unclear how much that would help.
    feature_array = []
    section_location_entries = [ecwe.Entry(entry) for entry in entry_it]
    if len(section_location_entries) != 0:
        logging.debug("first element in section_location_array = %s" % section_location_entries[0])

        if not ecc.compare_rounded_arrays(section.data.end_loc.coordinates,
                                      section_location_entries[-1].data.loc.coordinates,
                                      digits=4):
            logging.info("section_location_array[-1].data.loc %s != section.data.end_loc %s even after df.ts fix, filling gap" % \
                    (section_location_entries[-1].data.loc, section.data.end_loc))
            assert(False)
            last_loc_doc = ts.get_entry_at_ts("background/filtered_location", "data.ts", section.data.end_ts)
            if last_loc_doc is None:
                logging.warning("can't find entry to patch gap, leaving gap")
            else:
                last_loc_entry = ecwe.Entry(last_loc_doc)
                logging.debug("Adding new entry %s to fill the end point gap between %s and %s"
                   % (last_loc_entry.data.loc, section_location_entries[-1].data.loc,
                        section.data.end_loc))
                section_location_entries.append(last_loc_entry)

    points_line_feature = point_array_to_line(section_location_entries)
    points_line_feature.id = str(section.get_id())
    points_line_feature.properties.update(copy.copy(section.data))
    # Update works on dicts, convert back to a section object to make the modes
    # work properly
    points_line_feature.properties = ecwcs.Cleanedsection(points_line_feature.properties)
    points_line_feature.properties["feature_type"] = "section"
    points_line_feature.properties["sensed_mode"] = str(points_line_feature.properties.sensed_mode)

    _del_non_derializable(points_line_feature.properties, ["start_loc", "end_loc"])

    # feature_array.append(gj.FeatureCollection(points_feature_array))
    feature_array.append(points_line_feature)

    return gj.FeatureCollection(feature_array)

def incident_to_geojson(incident):
    ret_feature = gj.Feature()
    ret_feature.id = str(incident.get_id())
    ret_feature.geometry = gj.Point()
    ret_feature.geometry.coordinates = incident.data.loc.coordinates
    ret_feature.properties = copy.copy(incident.data)
    ret_feature.properties["feature_type"] = "incident"

    # _stringify_foreign_key(ret_feature.properties, ["ending_section", "starting_section", "trip_id"])
    _del_non_derializable(ret_feature.properties, ["loc"])
    return ret_feature

def geojson_incidents_in_range(user_id, start_ts, end_ts):
    MANUAL_INCIDENT_KEY = "manual/incident"
    ts = esta.TimeSeries.get_time_series(user_id)
    uc = enua.UserCache.getUserCache(user_id)
    tq = estt.TimeQuery("data.ts", start_ts, end_ts)
    incident_entry_docs = list(ts.find_entries([MANUAL_INCIDENT_KEY], time_query=tq)) \
        + list(uc.getMessage([MANUAL_INCIDENT_KEY], tq))
    incidents = [ecwe.Entry(doc) for doc in incident_entry_docs]
    return map(incident_to_geojson, incidents)

def point_array_to_line(point_array):
    points_line_string = gj.LineString()
    # points_line_string.coordinates = [l.loc.coordinates for l in filtered_section_location_array]
    points_line_string.coordinates = []
    points_times = []

    for l in point_array:
        # logging.debug("About to add %s to line_string " % l)
        points_line_string.coordinates.append(l.data.loc.coordinates)
        points_times.append(l.data.ts)

    points_line_feature = gj.Feature()
    points_line_feature.geometry = points_line_string
    points_line_feature.properties = {}
    points_line_feature.properties["times"] = points_times
    return points_line_feature

def trip_to_geojson(trip, tl):
    """
    Trips are the main focus of our current visualization, so they are most complex.
    Each trip is represented as a feature collection with the following features:
    - two features for the start and end places
    - features for each stop in the trip
    - features for each section in the trip

    :param trip: the trip object to be converted
    :param tl: the timeline used to retrieve related objects
    :return: the geojson version of the trip
    """

    feature_array = []
    curr_start_place = tl.get_object(trip.data.start_place)
    curr_end_place = tl.get_object(trip.data.end_place)
    start_place_geojson = place_to_geojson(curr_start_place)
    start_place_geojson["properties"]["feature_type"] = "start_place"
    feature_array.append(start_place_geojson)

    end_place_geojson = place_to_geojson(curr_end_place)
    end_place_geojson["properties"]["feature_type"] = "end_place"
    feature_array.append(end_place_geojson)

    trip_tl = esdt.get_cleaned_timeline_for_trip(trip.user_id, trip.get_id())
    stops = trip_tl.places
    for stop in stops:
        feature_array.append(stop_to_geojson(stop))

    for i, section in enumerate(trip_tl.trips):
        section_gj = section_to_geojson(section, tl)
        feature_array.append(section_gj)

    trip_geojson = gj.FeatureCollection(features=feature_array, properties=trip.data)
    trip_geojson.id = str(trip.get_id())

    feature_array.extend(geojson_incidents_in_range(trip.user_id,
                                              curr_start_place.data.exit_ts,
                                              curr_end_place.data.enter_ts))
    if trip.metadata.key == esda.CLEANED_UNTRACKED_KEY:
        # trip_geojson.properties["feature_type"] = "untracked"
        # Since the "untracked" type is not correctly handled on the phone, we just
        # skip these trips until
        # https://github.com/e-mission/e-mission-phone/issues/118
        # is fixed
        # TODO: Once it is fixed, re-introduce the first line in this block
        # and remove the None check in get_geojson_for_timeline
        return None
    else:
        trip_geojson.properties["feature_type"] = "trip"
    return trip_geojson

def get_geojson_for_ts(user_id, start_ts, end_ts):
    tl = esdtl.get_cleaned_timeline(user_id, start_ts, end_ts)
    tl.fill_start_end_places()
    return get_geojson_for_timeline(user_id, tl)

def get_geojson_for_dt(user_id, start_local_dt, end_local_dt):
    logging.debug("Getting geojson for %s -> %s" % (start_local_dt, end_local_dt))
    tl = esdtl.get_cleaned_timeline_from_dt(user_id, start_local_dt, end_local_dt)
    tl.fill_start_end_places()
    return get_geojson_for_timeline(user_id, tl)

def get_geojson_for_timeline(user_id, tl):
    """
    tl represents the "timeline" object that is queried for the trips and locations
    """
    geojson_list = []

    for trip in tl.trips:
        try:
            trip_geojson = trip_to_geojson(trip, tl)
            if trip_geojson is not None:
                geojson_list.append(trip_geojson)
        except Exception, e:
            logging.exception("Found error %s while processing trip %s" % (e, trip))
            raise e
    logging.debug("trip count = %d, geojson count = %d" %
                  (len(tl.trips), len(geojson_list)))
    return geojson_list

def get_all_points_for_range(user_id, key, start_ts, end_ts):
    import emission.storage.timeseries.timequery as estt
#     import emission.core.wrapper.location as ecwl

    tq = estt.TimeQuery("metadata.write_ts", start_ts, end_ts)
    ts = esta.TimeSeries.get_time_series(user_id)
    entry_it = ts.find_entries([key], tq)
    points_array = [ecwe.Entry(entry) for entry in entry_it]

    return get_feature_list_for_point_array(points_array)

def get_feature_list_for_point_array(points_array):
    points_feature_array = [location_to_geojson(le) for le in points_array]
    print ("Found %d features from %d points" %
           (len(points_feature_array), len(points_array)))

    feature_array = []
    feature_array.append(gj.FeatureCollection(points_feature_array))
    feature_array.append(point_array_to_line(points_array))
    feature_coll = gj.FeatureCollection(feature_array)

    return feature_coll

def get_feature_list_from_df(loc_time_df, ts="ts", latitude="latitude", longitude="longitude", fmt_time="fmt_time"):
    """
    Input DF should have columns called "ts", "latitude" and "longitude", or the corresponding
    columns can be passed in using the ts, latitude and longitude parameters
    """
    points_array = get_location_entry_list_from_df(loc_time_df, ts, latitude, longitude, fmt_time)
    return get_feature_list_for_point_array(points_array)

def get_location_entry_list_from_df(loc_time_df, ts="ts", latitude="latitude", longitude="longitude", fmt_time="fmt_time"):
    location_entry_list = []
    for idx, row in loc_time_df.iterrows():
        retVal = {"latitude": row[latitude], "longitude": row[longitude], "ts": row[ts],
                  "_id": str(idx), "fmt_time": row[fmt_time], "loc": gj.Point(coordinates=[row[longitude], row[latitude]])}
        location_entry_list.append(ecwe.Entry.create_entry(
            "dummy_user", "background/location", ecwl.Location(retVal)))
    return location_entry_list
<|endoftext|>