<commit_before># Copyright 2014-2017 The ODL contributors
#
# This file is part of ODL.
#
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at https://mozilla.org/MPL/2.0/.

"""Utilities for converting ODL operators to pytorch layers.

This requires the ``torch`` module from the ``pytorch`` package,
see `the pytorch installation guide
<https://github.com/pytorch/pytorch#installation>`_ for instructions.
"""

from __future__ import division
import numpy as np
import torch
import warnings
from pkg_resources import parse_version

if parse_version(torch.__version__) < parse_version('0.4'):
    warnings.warn("This interface is designed to work"
                  "with Pytorch >= 0.4", RuntimeWarning)

__all__ = ('OperatorAsAutogradFunction', 'OperatorAsModule')

# TODO: ProductSpaceOperator as implementation of channels_in and channels_out?

class OperatorAsAutogradFunction(torch.autograd.Function):

    """Wrapper of an ODL operator as a ``torch.autograd.Function``.

    This wrapper exposes an ``odl.Operator`` object to ``pytorch``'s
    autograd machinery by implementing custom ``forward()`` and
    ``backward()`` methods.

    Do not use this directly in a pytorch ``Module`` since backpropagation
    with batches and channels will not work as expected. For that
    purpose, use `OperatorAsModule` instead.
    """

    def __init__(self, operator):
        """Initialize a new instance.

        Parameters
        ----------
        operator : `Operator`
            The ODL operator to be wrapped. For gradient computations to
            work, ``operator.derivative(x).adjoint`` must be implemented.

        Examples
        --------
        >>> matrix = np.array([[1, 0, 1],
        ...                    [0, 1, 1]], dtype='float32')
        >>> odl_op = odl.MatrixOperator(matrix)
        >>> torch_op = OperatorAsAutogradFunction(odl_op)
        >>> torch_op.operator is odl_op
        True
        """
        super(OperatorAsAutogradFunction, self).__init__()
        self.operator = operator

    def forward(self, input):
        """Evaluate forward pass on the input.

        Parameters
        ----------
        input : `torch.tensor._TensorBase`
            Point at which to evaluate the operator.

        Returns
        -------
        result : `torch.autograd.variable.Variable`
            Variable holding the result of the evaluation.

        Examples
        --------
        Perform a matrix multiplication:

        >>> matrix = np.array([[1, 0, 1],
        ...                    [0, 1, 1]], dtype='float32')
        >>> odl_op = odl.MatrixOperator(matrix)
        >>> torch_op = OperatorAsAutogradFunction(odl_op)
        >>> x = torch.Tensor([1, 2, 3])
        >>> x_var = torch.autograd.Variable(x)
        >>> torch_op(x_var)
        Variable containing:
         4
         5
        [torch.FloatTensor of size 2]

        Evaluate a functional, i.e., an operator with scalar output:

        >>> odl_func = odl.solvers.L2NormSquared(odl.rn(3, dtype='float32'))
        >>> torch_func = OperatorAsAutogradFunction(odl_func)
        >>> x = torch.Tensor([1, 2, 3])
        >>> x_var = torch.autograd.Variable(x)
        >>> torch_func(x_var)
        Variable containing:
         14
        [torch.FloatTensor of size 1]
        """
        # TODO: batched evaluation
        if not self.operator.is_linear:
            # Only needed for nonlinear operators
            self.save_for_backward(input)

        # TODO: use GPU memory directly if possible
        input_arr = input.detach().numpy()
        if any(s == 0 for s in input_arr.strides):
            # TODO: remove when Numpy issue #9165 is fixed
            # https://github.com/numpy/numpy/pull/9177
            input_arr = input_arr.copy()

        op_result = self.operator(input_arr)
        if np.isscalar(op_result):
            # For functionals, the result is funnelled through `float`,
            # so we wrap it into a Numpy array with the same dtype as
            # `operator.domain`
            op_result = np.array(op_result, ndmin=1,
                                 dtype=self.operator.domain.dtype)
        tensor = torch.from_numpy(np.array(op_result, copy=False, ndmin=1))
        if input.is_cuda:
            # Push back to GPU
            tensor = tensor.cuda()
        return tensor

    def backward(self, grad_output):
        """Apply the adjoint of the derivative at ``grad_output``.

        This method is usually not called explicitly but as a part of the
        ``cost.backward()`` pass of a backpropagation step.

        Parameters
        ----------
        grad_output : `torch.tensor._TensorBase`
            Tensor to which the Jacobian should be applied. See Notes
            for details.

        Returns
        -------
        result : `torch.autograd.variable.Variable`
            Variable holding the result of applying the Jacobian to
            ``grad_output``. See Notes for details.

        Examples
        --------
        Compute the Jacobian adjoint of the matrix operator, which is the
        operator of the transposed matrix. We compose with the ``sum``
        functional to be able to evaluate ``grad``:

        >>> matrix = np.array([[1, 0, 1],
        ...                    [0, 1, 1]], dtype='float32')
        >>> odl_op = odl.MatrixOperator(matrix)
        >>> torch_op = OperatorAsAutogradFunction(odl_op)
        >>> x = torch.Tensor([1, 2, 3])
        >>> x_var = torch.autograd.Variable(x, requires_grad=True)
        >>> op_x_var = torch_op(x_var)
        >>> cost = op_x_var.sum()
        >>> cost.backward()
        >>> x_var.grad  # should be matrix.T.dot([1, 1])
        Variable containing:
         1
         1
         2
        [torch.FloatTensor of size 3]

        Compute the gradient of a custom functional:

        >>> odl_func = odl.solvers.L2NormSquared(odl.rn(3, dtype='float32'))
        >>> torch_func = OperatorAsAutogradFunction(odl_func)
        >>> x = torch.Tensor([1, 2, 3])
        >>> x_var = torch.autograd.Variable(x, requires_grad=True)
        >>> func_x_var = torch_func(x_var)
        >>> func_x_var
        Variable containing:
         14
        [torch.FloatTensor of size 1]

        >>> func_x_var.backward()
        >>> x_var.grad  # Should be 2 * x
        Variable containing:
         2
         4
         6
        [torch.FloatTensor of size 3]

        Notes
        -----
        This method applies the contribution of this node, i.e., the
        transpose of the Jacobian of its outputs with respect to its inputs,
        to the gradients of some cost function with respect to the outputs
        of this node.

        Example: Assume that this node computes :math:`x \mapsto C(f(x))`,
        where :math:`x` is a tensor variable and :math:`C` is a scalar-valued
        function. In ODL language, what ``backward`` should compute is

        .. math::
            \\nabla(C \circ f)(x) = f'(x)^*\\big(\\nabla C (f(x))\\big)

        according to the chain rule. In ODL code, this corresponds to ::

            f.derivative(x).adjoint(C.gradient(f(x))).

        Hence, the parameter ``grad_output`` is a tensor variable containing
        :math:`y = \\nabla C(f(x))`. Then, ``backward`` boils down to
        computing ``[f'(x)^*(y)]`` using the input ``x`` stored during
        the previous `forward` pass.
        """
        # TODO: implement directly for GPU data
        if not self.operator.is_linear:
            input_arr = self.saved_variables[0].data.cpu().numpy()
            if any(s == 0 for s in input_arr.strides):
                # TODO: remove when Numpy issue #9165 is fixed
                # https://github.com/numpy/numpy/pull/9177
                input_arr = input_arr.copy()

        grad = None

        # ODL weights spaces, pytorch doesn't, so we need to handle this
        try:
            dom_weight = self.operator.domain.weighting.const
        except AttributeError:
            dom_weight = 1.0

        try:
            ran_weight = self.operator.range.weighting.const
        except AttributeError:
            ran_weight = 1.0

        scaling = dom_weight / ran_weight

        if self.needs_input_grad[0]:
            grad_output_arr = grad_output.cpu().numpy()
            if any(s == 0 for s in grad_output_arr.strides):
                # TODO: remove when Numpy issue #9165 is fixed
                # https://github.com/numpy/numpy/pull/9177
                grad_output_arr = grad_output_arr.copy()

            if self.operator.is_linear:
                adjoint = self.operator.adjoint
            else:
                adjoint = self.operator.derivative(input_arr).adjoint

            grad_odl = adjoint(grad_output_arr)

            if scaling != 1.0:
                grad_odl *= scaling

            grad = torch.from_numpy(np.array(grad_odl, copy=False, ndmin=1))

            if grad_output.is_cuda:
                # Push back to GPU
                grad = grad.cuda()

        return grad

    def __repr__(self):
        """Return ``repr(self)``."""
        return '{}(\n    {!r}    \n)'.format(self.__class__.__name__,
                                             self.operator)

class OperatorAsModule(torch.nn.Module):

    """Wrapper of an ODL operator as a ``torch.nn.Module``.

    This wrapper can be used as a layer in ``pytorch`` Neural Networks.
    It works with arbitrary batches and channels and supports
    backpropagation.

    .. note::
        Currently, batches and channels are supported by simply looping
        over them and stacking the results.
    """

    def __init__(self, operator):
        """Initialize a new instance.

        Parameters
        ----------
        operator : `Operator`
            The ODL operator to be wrapped. For gradient computations to
            work, ``operator.derivative(x).adjoint`` must be implemented.

        Examples
        --------
        >>> matrix = np.array([[1, 0, 1],
        ...                    [0, 1, 1]], dtype='float32')
        >>> odl_op = odl.MatrixOperator(matrix)
        >>> torch_op = OperatorAsModule(odl_op)
        >>> torch_op.op_func.operator is odl_op
        True
        """
        super(OperatorAsModule, self).__init__()
        self.op_func = OperatorAsAutogradFunction(operator)

    @property
    def operator(self):
        """The wrapped ODL operator."""
        return self.op_func.operator

    def forward(self, x):
        """Compute forward-pass of this module on ``x``.

        Parameters
        ----------
        x : `torch.autograd.variable.Variable`
            Input of this layer. The contained tensor must have shape
            ``extra_shape + operator.domain.shape``, and
            ``len(extra_shape)`` must be at least 1 (batch axis).

        Returns
        -------
        out : `torch.autograd.variable.Variable`
            The computed output. Its tensor will have shape
            ``extra_shape + operator.range.shape``, where ``extra_shape``
            are the extra axes of ``x``.

        Examples
        --------
        Evaluating on a 2D tensor, where the operator expects a 1D input,
        i.e., with extra batch axis only:

        >>> matrix = np.array([[1, 0, 0],
        ...                    [0, 1, 1]], dtype='float32')
        >>> odl_op = odl.MatrixOperator(matrix)
        >>> odl_op.domain.shape
        (3,)
        >>> odl_op.range.shape
        (2,)
        >>> op_mod = OperatorAsModule(odl_op)
        >>> t = torch.ones(3)
        >>> x = autograd.Variable(t[None, :]) # "fake" batch axis
        >>> op_mod(x)
        Variable containing:
         1  2
        [torch.FloatTensor of size 1x2]
        >>> t = torch.ones(3)
        >>> x_tensor = torch.stack([0 * t, 1 * t])
        >>> x = autograd.Variable(x_tensor)  # batch of 2 inputs
        >>> op_mod(x)
        Variable containing:
         0  0
         1  2
        [torch.FloatTensor of size 2x2]

        An arbitrary number of axes is supported:

        >>> x = autograd.Variable(t[None, None, :])  # "fake" batch and channel
        >>> op_mod(x)
        Variable containing:
        (0 ,.,.) =
          1  2
        [torch.FloatTensor of size 1x1x2]
        >>> x_tensor = torch.stack([torch.stack([0 * t, 1 * t]),
        ...                         torch.stack([2 * t, 3 * t]),
        ...                         torch.stack([4 * t, 5 * t])])
        >>> x = autograd.Variable(x_tensor)  # batch of 3x2 inputs
        >>> op_mod(x)
        Variable containing:
        (0 ,.,.) =
           0   0
           1   2
        <BLANKLINE>
        (1 ,.,.) =
           2   4
           3   6
        <BLANKLINE>
        (2 ,.,.) =
           4   8
           5  10
        [torch.FloatTensor of size 3x2x2]
        """
        in_shape = x.data.shape
        op_in_shape = self.op_func.operator.domain.shape
        op_out_shape = self.op_func.operator.range.shape

        extra_shape = in_shape[:-len(op_in_shape)]

        if in_shape[-len(op_in_shape):] != op_in_shape or not extra_shape:
            shp_str = str(op_in_shape).strip('()')
            raise ValueError('expected input of shape (N, *, {}), got input '
                             'with shape {}'.format(shp_str, in_shape))

        # Flatten extra axes, then do one entry at a time
        newshape = (int(np.prod(extra_shape)),) + op_in_shape
        x_flat_xtra = x.reshape(*newshape)
        results = []
        for i in range(x_flat_xtra.data.shape[0]):
            results.append(self.op_func(x_flat_xtra[i]))

        # Reshape the resulting stack to the expected output shape
        stack_flat_xtra = torch.stack(results)
        return stack_flat_xtra.view(extra_shape + op_out_shape)

    def __repr__(self):
        """Return ``repr(self)``."""
        op_name = self.op_func.operator.__class__.__name__
        op_dom_shape = self.op_func.operator.domain.shape
        if len(op_dom_shape) == 1:
            op_dom_shape = op_dom_shape[0]
        op_ran_shape = self.op_func.operator.range.shape
        if len(op_ran_shape) == 1:
            op_ran_shape = op_ran_shape[0]

        return '{}({}) ({} -> {})'.format(self.__class__.__name__,
                                          op_name, op_dom_shape, op_ran_shape)

if __name__ == '__main__':
    from odl.util.testutils import run_doctests
    import odl
    from torch import autograd, nn
    run_doctests(extraglobs={'np': np, 'odl': odl, 'torch': torch,
                             'nn': nn, 'autograd': autograd})
<commit_msg>BUG: conversion of Pytorch tensor to numpy array<commit_after># Copyright 2014-2017 The ODL contributors
#
# This file is part of ODL.
#
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at https://mozilla.org/MPL/2.0/.

"""Utilities for converting ODL operators to pytorch layers.

This requires the ``torch`` module from the ``pytorch`` package,
see `the pytorch installation guide
<https://github.com/pytorch/pytorch#installation>`_ for instructions.
"""

from __future__ import division
import numpy as np
import torch
import warnings
from pkg_resources import parse_version

if parse_version(torch.__version__) < parse_version('0.4'):
    warnings.warn("This interface is designed to work"
                  "with Pytorch >= 0.4", RuntimeWarning)

__all__ = ('OperatorAsAutogradFunction', 'OperatorAsModule')

# TODO: ProductSpaceOperator as implementation of channels_in and channels_out?

class OperatorAsAutogradFunction(torch.autograd.Function):

    """Wrapper of an ODL operator as a ``torch.autograd.Function``.

    This wrapper exposes an ``odl.Operator`` object to ``pytorch``'s
    autograd machinery by implementing custom ``forward()`` and
    ``backward()`` methods.

    Do not use this directly in a pytorch ``Module`` since backpropagation
    with batches and channels will not work as expected. For that
    purpose, use `OperatorAsModule` instead.
    """

    def __init__(self, operator):
        """Initialize a new instance.

        Parameters
        ----------
        operator : `Operator`
            The ODL operator to be wrapped. For gradient computations to
            work, ``operator.derivative(x).adjoint`` must be implemented.

        Examples
        --------
        >>> matrix = np.array([[1, 0, 1],
        ...                    [0, 1, 1]], dtype='float32')
        >>> odl_op = odl.MatrixOperator(matrix)
        >>> torch_op = OperatorAsAutogradFunction(odl_op)
        >>> torch_op.operator is odl_op
        True
        """
        super(OperatorAsAutogradFunction, self).__init__()
        self.operator = operator

    def forward(self, input):
        """Evaluate forward pass on the input.

        Parameters
        ----------
        input : `torch.tensor._TensorBase`
            Point at which to evaluate the operator.

        Returns
        -------
        result : `torch.autograd.variable.Variable`
            Variable holding the result of the evaluation.

        Examples
        --------
        Perform a matrix multiplication:

        >>> matrix = np.array([[1, 0, 1],
        ...                    [0, 1, 1]], dtype='float32')
        >>> odl_op = odl.MatrixOperator(matrix)
        >>> torch_op = OperatorAsAutogradFunction(odl_op)
        >>> x = torch.Tensor([1, 2, 3])
        >>> x_var = torch.autograd.Variable(x)
        >>> torch_op(x_var)
        Variable containing:
         4
         5
        [torch.FloatTensor of size 2]

        Evaluate a functional, i.e., an operator with scalar output:

        >>> odl_func = odl.solvers.L2NormSquared(odl.rn(3, dtype='float32'))
        >>> torch_func = OperatorAsAutogradFunction(odl_func)
        >>> x = torch.Tensor([1, 2, 3])
        >>> x_var = torch.autograd.Variable(x)
        >>> torch_func(x_var)
        Variable containing:
         14
        [torch.FloatTensor of size 1]
        """
        # TODO: batched evaluation
        if not self.operator.is_linear:
            # Only needed for nonlinear operators
            self.save_for_backward(input)

        # TODO: use GPU memory directly if possible
        input_arr = input.cpu().detach().numpy()
        if any(s == 0 for s in input_arr.strides):
            # TODO: remove when Numpy issue #9165 is fixed
            # https://github.com/numpy/numpy/pull/9177
            input_arr = input_arr.copy()

        op_result = self.operator(input_arr)
        if np.isscalar(op_result):
            # For functionals, the result is funnelled through `float`,
            # so we wrap it into a Numpy array with the same dtype as
            # `operator.domain`
            op_result = np.array(op_result, ndmin=1,
                                 dtype=self.operator.domain.dtype)
        tensor = torch.from_numpy(np.array(op_result, copy=False, ndmin=1))
        if input.is_cuda:
            # Push back to GPU
            tensor = tensor.cuda()
        return tensor

    def backward(self, grad_output):
        """Apply the adjoint of the derivative at ``grad_output``.

        This method is usually not called explicitly but as a part of the
        ``cost.backward()`` pass of a backpropagation step.

        Parameters
        ----------
        grad_output : `torch.tensor._TensorBase`
            Tensor to which the Jacobian should be applied. See Notes
            for details.

        Returns
        -------
        result : `torch.autograd.variable.Variable`
            Variable holding the result of applying the Jacobian to
            ``grad_output``. See Notes for details.

        Examples
        --------
        Compute the Jacobian adjoint of the matrix operator, which is the
        operator of the transposed matrix. We compose with the ``sum``
        functional to be able to evaluate ``grad``:

        >>> matrix = np.array([[1, 0, 1],
        ...                    [0, 1, 1]], dtype='float32')
        >>> odl_op = odl.MatrixOperator(matrix)
        >>> torch_op = OperatorAsAutogradFunction(odl_op)
        >>> x = torch.Tensor([1, 2, 3])
        >>> x_var = torch.autograd.Variable(x, requires_grad=True)
        >>> op_x_var = torch_op(x_var)
        >>> cost = op_x_var.sum()
        >>> cost.backward()
        >>> x_var.grad  # should be matrix.T.dot([1, 1])
        Variable containing:
         1
         1
         2
        [torch.FloatTensor of size 3]

        Compute the gradient of a custom functional:

        >>> odl_func = odl.solvers.L2NormSquared(odl.rn(3, dtype='float32'))
        >>> torch_func = OperatorAsAutogradFunction(odl_func)
        >>> x = torch.Tensor([1, 2, 3])
        >>> x_var = torch.autograd.Variable(x, requires_grad=True)
        >>> func_x_var = torch_func(x_var)
        >>> func_x_var
        Variable containing:
         14
        [torch.FloatTensor of size 1]

        >>> func_x_var.backward()
        >>> x_var.grad  # Should be 2 * x
        Variable containing:
         2
         4
         6
        [torch.FloatTensor of size 3]

        Notes
        -----
        This method applies the contribution of this node, i.e., the
        transpose of the Jacobian of its outputs with respect to its inputs,
        to the gradients of some cost function with respect to the outputs
        of this node.

        Example: Assume that this node computes :math:`x \mapsto C(f(x))`,
        where :math:`x` is a tensor variable and :math:`C` is a scalar-valued
        function. In ODL language, what ``backward`` should compute is

        .. math::
            \\nabla(C \circ f)(x) = f'(x)^*\\big(\\nabla C (f(x))\\big)

        according to the chain rule. In ODL code, this corresponds to ::

            f.derivative(x).adjoint(C.gradient(f(x))).

        Hence, the parameter ``grad_output`` is a tensor variable containing
        :math:`y = \\nabla C(f(x))`. Then, ``backward`` boils down to
        computing ``[f'(x)^*(y)]`` using the input ``x`` stored during
        the previous `forward` pass.
        """
        # TODO: implement directly for GPU data
        if not self.operator.is_linear:
            input_arr = self.saved_variables[0].data.cpu().numpy()
            if any(s == 0 for s in input_arr.strides):
                # TODO: remove when Numpy issue #9165 is fixed
                # https://github.com/numpy/numpy/pull/9177
                input_arr = input_arr.copy()

        grad = None

        # ODL weights spaces, pytorch doesn't, so we need to handle this
        try:
            dom_weight = self.operator.domain.weighting.const
        except AttributeError:
            dom_weight = 1.0

        try:
            ran_weight = self.operator.range.weighting.const
        except AttributeError:
            ran_weight = 1.0

        scaling = dom_weight / ran_weight

        if self.needs_input_grad[0]:
            grad_output_arr = grad_output.cpu().numpy()
            if any(s == 0 for s in grad_output_arr.strides):
                # TODO: remove when Numpy issue #9165 is fixed
                # https://github.com/numpy/numpy/pull/9177
                grad_output_arr = grad_output_arr.copy()

            if self.operator.is_linear:
                adjoint = self.operator.adjoint
            else:
                adjoint = self.operator.derivative(input_arr).adjoint

            grad_odl = adjoint(grad_output_arr)

            if scaling != 1.0:
                grad_odl *= scaling

            grad = torch.from_numpy(np.array(grad_odl, copy=False, ndmin=1))

            if grad_output.is_cuda:
                # Push back to GPU
                grad = grad.cuda()

        return grad

    def __repr__(self):
        """Return ``repr(self)``."""
        return '{}(\n    {!r}    \n)'.format(self.__class__.__name__,
                                             self.operator)

class OperatorAsModule(torch.nn.Module):

    """Wrapper of an ODL operator as a ``torch.nn.Module``.

    This wrapper can be used as a layer in ``pytorch`` Neural Networks.
    It works with arbitrary batches and channels and supports
    backpropagation.

    .. note::
        Currently, batches and channels are supported by simply looping
        over them and stacking the results.
    """

    def __init__(self, operator):
        """Initialize a new instance.

        Parameters
        ----------
        operator : `Operator`
            The ODL operator to be wrapped. For gradient computations to
            work, ``operator.derivative(x).adjoint`` must be implemented.

        Examples
        --------
        >>> matrix = np.array([[1, 0, 1],
        ...                    [0, 1, 1]], dtype='float32')
        >>> odl_op = odl.MatrixOperator(matrix)
        >>> torch_op = OperatorAsModule(odl_op)
        >>> torch_op.op_func.operator is odl_op
        True
        """
        super(OperatorAsModule, self).__init__()
        self.op_func = OperatorAsAutogradFunction(operator)

    @property
    def operator(self):
        """The wrapped ODL operator."""
        return self.op_func.operator

    def forward(self, x):
        """Compute forward-pass of this module on ``x``.

        Parameters
        ----------
        x : `torch.autograd.variable.Variable`
            Input of this layer. The contained tensor must have shape
            ``extra_shape + operator.domain.shape``, and
            ``len(extra_shape)`` must be at least 1 (batch axis).

        Returns
        -------
        out : `torch.autograd.variable.Variable`
            The computed output. Its tensor will have shape
            ``extra_shape + operator.range.shape``, where ``extra_shape``
            are the extra axes of ``x``.

        Examples
        --------
        Evaluating on a 2D tensor, where the operator expects a 1D input,
        i.e., with extra batch axis only:

        >>> matrix = np.array([[1, 0, 0],
        ...                    [0, 1, 1]], dtype='float32')
        >>> odl_op = odl.MatrixOperator(matrix)
        >>> odl_op.domain.shape
        (3,)
        >>> odl_op.range.shape
        (2,)
        >>> op_mod = OperatorAsModule(odl_op)
        >>> t = torch.ones(3)
        >>> x = autograd.Variable(t[None, :]) # "fake" batch axis
        >>> op_mod(x)
        Variable containing:
         1  2
        [torch.FloatTensor of size 1x2]
        >>> t = torch.ones(3)
        >>> x_tensor = torch.stack([0 * t, 1 * t])
        >>> x = autograd.Variable(x_tensor)  # batch of 2 inputs
        >>> op_mod(x)
        Variable containing:
         0  0
         1  2
        [torch.FloatTensor of size 2x2]

        An arbitrary number of axes is supported:

        >>> x = autograd.Variable(t[None, None, :])  # "fake" batch and channel
        >>> op_mod(x)
        Variable containing:
        (0 ,.,.) =
          1  2
        [torch.FloatTensor of size 1x1x2]
        >>> x_tensor = torch.stack([torch.stack([0 * t, 1 * t]),
        ...                         torch.stack([2 * t, 3 * t]),
        ...                         torch.stack([4 * t, 5 * t])])
        >>> x = autograd.Variable(x_tensor)  # batch of 3x2 inputs
        >>> op_mod(x)
        Variable containing:
        (0 ,.,.) =
           0   0
           1   2
        <BLANKLINE>
        (1 ,.,.) =
           2   4
           3   6
        <BLANKLINE>
        (2 ,.,.) =
           4   8
           5  10
        [torch.FloatTensor of size 3x2x2]
        """
        in_shape = x.data.shape
        op_in_shape = self.op_func.operator.domain.shape
        op_out_shape = self.op_func.operator.range.shape

        extra_shape = in_shape[:-len(op_in_shape)]

        if in_shape[-len(op_in_shape):] != op_in_shape or not extra_shape:
            shp_str = str(op_in_shape).strip('()')
            raise ValueError('expected input of shape (N, *, {}), got input '
                             'with shape {}'.format(shp_str, in_shape))

        # Flatten extra axes, then do one entry at a time
        newshape = (int(np.prod(extra_shape)),) + op_in_shape
        x_flat_xtra = x.reshape(*newshape)
        results = []
        for i in range(x_flat_xtra.data.shape[0]):
            results.append(self.op_func(x_flat_xtra[i]))

        # Reshape the resulting stack to the expected output shape
        stack_flat_xtra = torch.stack(results)
        return stack_flat_xtra.view(extra_shape + op_out_shape)

    def __repr__(self):
        """Return ``repr(self)``."""
        op_name = self.op_func.operator.__class__.__name__
        op_dom_shape = self.op_func.operator.domain.shape
        if len(op_dom_shape) == 1:
            op_dom_shape = op_dom_shape[0]
        op_ran_shape = self.op_func.operator.range.shape
        if len(op_ran_shape) == 1:
            op_ran_shape = op_ran_shape[0]

        return '{}({}) ({} -> {})'.format(self.__class__.__name__,
                                          op_name, op_dom_shape, op_ran_shape)

if __name__ == '__main__':
    from odl.util.testutils import run_doctests
    import odl
    from torch import autograd, nn
    run_doctests(extraglobs={'np': np, 'odl': odl, 'torch': torch,
                             'nn': nn, 'autograd': autograd})
<|endoftext|>


<commit_before>import os

import numpy as np
import pandas as pd
from sklearn.metrics import roc_auc_score, average_precision_score

try:
    import torch
except ImportError:
    torch = None

### Evaluator for graph classification
class Evaluator:
    def __init__(self, name):
        self.name = name

        meta_info = pd.read_csv(os.path.join(os.path.dirname(__file__), 'master.csv'), index_col = 0)
        if not self.name in meta_info:
            print(self.name)
            error_mssg = 'Invalid dataset name {}.\n'.format(self.name)
            error_mssg += 'Available datasets are as follows:\n'
            error_mssg += '\n'.join(meta_info.keys())
            raise ValueError(error_mssg)

        self.num_tasks = int(meta_info[self.name]['num tasks'])
        self.eval_metric = meta_info[self.name]['eval metric']

    def _parse_and_check_input(self, input_dict):
        if self.eval_metric == 'rocauc' or self.eval_metric == 'ap' or self.eval_metric == 'rmse' or self.eval_metric == 'acc':
            if not 'y_true' in input_dict:
                raise RuntimeError('Missing key of y_true')
            if not 'y_pred' in input_dict:
                raise RuntimeError('Missing key of y_pred')

            y_true, y_pred = input_dict['y_true'], input_dict['y_pred']

            '''
                y_true: numpy ndarray or torch tensor of shape (num_graphs, num_tasks)
                y_pred: numpy ndarray or torch tensor of shape (num_graphs, num_tasks)
            '''

            # converting to torch.Tensor to numpy on cpu
            if torch is not None and isinstance(y_true, torch.Tensor):
                y_true = y_true.detach().cpu().numpy()

            if torch is not None and isinstance(y_pred, torch.Tensor):
                y_pred = y_pred.detach().cpu().numpy()

            ## check type
            if not isinstance(y_true, np.ndarray):
                raise RuntimeError('Arguments to Evaluator need to be either numpy ndarray or torch tensor')

            if not y_true.shape == y_pred.shape:
                raise RuntimeError('Shape of y_true and y_pred must be the same')

            if not y_true.ndim == 2:
                raise RuntimeError('y_true and y_pred mush to 2-dim arrray, {}-dim array given'.format(y_true.ndim))

            if not y_true.shape[1] == self.num_tasks:
                raise RuntimeError('Number of tasks for {} should be {} but {} given'.format(self.name, self.num_tasks, y_true.shape[1]))

            return y_true, y_pred

        elif self.eval_metric == 'F1':
            if not 'seq_ref' in input_dict:
                raise RuntimeError('Missing key of seq_ref')
            if not 'seq_pred' in input_dict:
                raise RuntimeError('Missing key of seq_pred')

            seq_ref, seq_pred = input_dict['seq_ref'], input_dict['seq_pred']

            if not isinstance(seq_ref, list):
                raise RuntimeError('seq_ref must be of type list')

            if not isinstance(seq_pred, list):
                raise RuntimeError('seq_pred must be of type list')

            if len(seq_ref) != len(seq_pred):
                raise RuntimeError('Length of seq_true and seq_pred should be the same')

            return seq_ref, seq_pred

        else:
            raise ValueError('Undefined eval metric %s ' % (self.eval_metric))

    def eval(self, input_dict):

        if self.eval_metric == 'rocauc':
            y_true, y_pred = self._parse_and_check_input(input_dict)
            return self._eval_rocauc(y_true, y_pred)
        if self.eval_metric == 'ap':
            y_true, y_pred = self._parse_and_check_input(input_dict)
            return self._eval_ap(y_true, y_pred)
        elif self.eval_metric == 'rmse':
            y_true, y_pred = self._parse_and_check_input(input_dict)
            return self._eval_rmse(y_true, y_pred)
        elif self.eval_metric == 'acc':
            y_true, y_pred = self._parse_and_check_input(input_dict)
            return self._eval_acc(y_true, y_pred)
        elif self.eval_metric == 'F1':
            seq_ref, seq_pred = self._parse_and_check_input(input_dict)
            return self._eval_F1(seq_ref, seq_pred)
        else:
            raise ValueError('Undefined eval metric %s ' % (self.eval_metric))

    @property
    def expected_input_format(self):
        desc = '==== Expected input format of Evaluator for {}\n'.format(self.name)
        if self.eval_metric == 'rocauc' or self.eval_metric == 'ap':
            desc += '{\'y_true\': y_true, \'y_pred\': y_pred}\n'
            desc += '- y_true: numpy ndarray or torch tensor of shape (num_graphs, num_tasks)\n'
            desc += '- y_pred: numpy ndarray or torch tensor of shape (num_graphs, num_tasks)\n'
            desc += 'where y_pred stores score values (for computing AUC score),\n'
            desc += 'num_task is {}, and '.format(self.num_tasks)
            desc += 'each row corresponds to one graph.\n'
            desc += 'nan values in y_true are ignored during evaluation.\n'
        elif self.eval_metric == 'rmse':
            desc += '{\'y_true\': y_true, \'y_pred\': y_pred}\n'
            desc += '- y_true: numpy ndarray or torch tensor of shape (num_graphs, num_tasks)\n'
            desc += '- y_pred: numpy ndarray or torch tensor of shape (num_graphs, num_tasks)\n'
            desc += 'where num_task is {}, and '.format(self.num_tasks)
            desc += 'each row corresponds to one graph.\n'
            desc += 'nan values in y_true are ignored during evaluation.\n'
        elif self.eval_metric == 'acc':
            desc += '{\'y_true\': y_true, \'y_pred\': y_pred}\n'
            desc += '- y_true: numpy ndarray or torch tensor of shape (num_nodes num_tasks)\n'
            desc += '- y_pred: numpy ndarray or torch tensor of shape (num_nodes num_tasks)\n'
            desc += 'where y_pred stores predicted class label (integer),\n'
            desc += 'num_task is {}, and '.format(self.num_tasks)
            desc += 'each row corresponds to one graph.\n'
        elif self.eval_metric == 'F1':
            desc += '{\'seq_ref\': seq_ref, \'seq_pred\': seq_pred}\n'
            desc += '- seq_ref: a list of lists of strings\n'
            desc += '- seq_pred: a list of lists of strings\n'
            desc += 'where seq_ref stores the reference sequences of sub-tokens, and\n'
            desc += 'seq_pred stores the predicted sequences of sub-tokens.\n'
        else:
            raise ValueError('Undefined eval metric %s ' % (self.eval_metric))

        return desc

    @property
    def expected_output_format(self):
        desc = '==== Expected output format of Evaluator for {}\n'.format(self.name)
        if self.eval_metric == 'rocauc':
            desc += '{\'rocauc\': rocauc}\n'
            desc += '- rocauc (float): ROC-AUC score averaged across {} task(s)\n'.format(self.num_tasks)
        elif self.eval_metric == 'ap':
            desc += '{\'ap\': ap}\n'
            desc += '- ap (float): Average Precision (AP) score averaged across {} task(s)\n'.format(self.num_tasks)
        elif self.eval_metric == 'rmse':
            desc += '{\'rmse\': rmse}\n'
            desc += '- rmse (float): root mean squared error averaged across {} task(s)\n'.format(self.num_tasks)
        elif self.eval_metric == 'acc':
            desc += '{\'acc\': acc}\n'
            desc += '- acc (float): Accuracy score averaged across {} task(s)\n'.format(self.num_tasks)
        elif self.eval_metric == 'F1':
            desc += '{\'F1\': F1}\n'
            desc += '- F1 (float): F1 score averaged over samples.\n'
        else:
            raise ValueError('Undefined eval metric %s ' % (self.eval_metric))

        return desc

    def _eval_rocauc(self, y_true, y_pred):
        '''
            compute ROC-AUC averaged across tasks
        '''

        rocauc_list = []

        for i in range(y_true.shape[1]):
            #AUC is only defined when there is at least one positive data.
            if np.sum(y_true[:,i] == 1) > 0 and np.sum(y_true[:,i] == 0) > 0:
                # ignore nan values
                is_labeled = y_true[:,i] == y_true[:,i]
                rocauc_list.append(roc_auc_score(y_true[is_labeled,i], y_pred[is_labeled,i]))

        if len(rocauc_list) == 0:
            raise RuntimeError('No positively labeled data available. Cannot compute ROC-AUC.')

        return {'rocauc': sum(rocauc_list)/len(rocauc_list)}

    def _eval_ap(self, y_true, y_pred):
        '''
            compute Average Precision (AP) averaged across tasks
        '''

        ap_list = []

        for i in range(y_true.shape[1]):
            #AUC is only defined when there is at least one positive data.
            if np.sum(y_true[:,i] == 1) > 0 and np.sum(y_true[:,i] == 0) > 0:
                # ignore nan values
                is_labeled = y_true[:,i] == y_true[:,i]
                ap = average_precision_score(y_true[is_labeled,i], y_pred[is_labeled,i])

                ap_list.append(ap)

        if len(ap_list) == 0:
            raise RuntimeError('No positively labeled data available. Cannot compute Average Precision.')

        return {'ap': sum(ap_list)/len(ap_list)}

    def _eval_rmse(self, y_true, y_pred):
        '''
            compute RMSE score averaged across tasks
        '''
        rmse_list = []

        for i in range(y_true.shape[1]):
            # ignore nan values
            is_labeled = y_true[:,i] == y_true[:,i]
            rmse_list.append(np.sqrt(((y_true[is_labeled] - y_pred[is_labeled])**2).mean()))

        return {'rmse': sum(rmse_list)/len(rmse_list)}

    def _eval_acc(self, y_true, y_pred):
        acc_list = []

        for i in range(y_true.shape[1]):
            is_labeled = y_true[:,i] == y_true[:,i]
            correct = y_true[is_labeled,i] == y_pred[is_labeled,i]
            acc_list.append(float(np.sum(correct))/len(correct))

        return {'acc': sum(acc_list)/len(acc_list)}

    def _eval_F1(self, seq_ref, seq_pred):
        # '''
        #     compute F1 score averaged over samples
        # '''

        precision_list = []
        recall_list = []
        f1_list = []

        for l, p in zip(seq_ref, seq_pred):
            label = set(l)
            prediction = set(p)
            true_positive = len(label.intersection(prediction))
            false_positive = len(prediction - label)
            false_negative = len(label - prediction)

            if true_positive + false_positive > 0:
                precision = true_positive / (true_positive + false_positive)
            else:
                precision = 0

            if true_positive + false_negative > 0:
                recall = true_positive / (true_positive + false_negative)
            else:
                recall = 0
            if precision + recall > 0:
                f1 = 2 * precision * recall / (precision + recall)
            else:
                f1 = 0

            precision_list.append(precision)
            recall_list.append(recall)
            f1_list.append(f1)

        return {'precision': np.average(precision_list),
                'recall': np.average(recall_list),
                'F1': np.average(f1_list)}

if __name__ == '__main__':
    evaluator = Evaluator('ogbg-code2')
    print(evaluator.expected_input_format)
    print(evaluator.expected_output_format)
    seq_ref = [['tom', 'is'], ['he'], ['he'], ['hey', 'fea', 'he'], ['alpha'], ['fe4qfq', 'beta'], ['aa']]
    seq_pred = [['tom', 'is'], ['he'], ['he'], ['hey', 'he', 'fea'], ['alpha'], ['beta', 'fe4qfq'], ['aa']] # [['tom', 'is'] , ['he'], ['the', 'he'], ['hey', 'fea', 'he'], ['alpha'], ['beta', 'fe4qfq', 'c', 'fe4qf'], ['']]
    input_dict = {'seq_ref': seq_ref, 'seq_pred': seq_pred}
    result = evaluator.eval(input_dict)
    print(result)

    # exit(-1)

    evaluator = Evaluator('ogbg-molpcba')
    print(evaluator.expected_input_format)
    print(evaluator.expected_output_format)
    y_true = torch.tensor(np.random.randint(2, size = (100,128)))
    y_pred = torch.tensor(np.random.randn(100,128))
    input_dict = {'y_true': y_true, 'y_pred': y_pred}
    result = evaluator.eval(input_dict)
    print(result)

    evaluator = Evaluator('ogbg-molhiv')
    print(evaluator.expected_input_format)
    print(evaluator.expected_output_format)
    y_true = torch.tensor(np.random.randint(2, size = (100,1)))
    y_pred = torch.tensor(np.random.randn(100,1))
    input_dict = {'y_true': y_true, 'y_pred': y_pred}
    result = evaluator.eval(input_dict)
    print(result)

    ### rmse case
    evaluator = Evaluator('ogbg-mollipo')
    print(evaluator.expected_input_format)
    print(evaluator.expected_output_format)
    y_true = np.random.randn(100,1)
    y_pred = np.random.randn(100,1)
    input_dict = {'y_true': y_true, 'y_pred': y_pred}
    result = evaluator.eval(input_dict)
    print(result)

    ### acc
    evaluator = Evaluator('ogbg-ppa')
    print(evaluator.expected_input_format)
    print(evaluator.expected_output_format)
    y_true = np.random.randint(5, size = (100,1))
    y_pred = np.random.randint(5, size = (100,1))
    input_dict = {'y_true': y_true, 'y_pred': y_pred}
    result = evaluator.eval(input_dict)
    print(result)

<commit_msg>Fix bug in rmse evaluation<commit_after>import os

import numpy as np
import pandas as pd
from sklearn.metrics import roc_auc_score, average_precision_score

try:
    import torch
except ImportError:
    torch = None

### Evaluator for graph classification
class Evaluator:
    def __init__(self, name):
        self.name = name

        meta_info = pd.read_csv(os.path.join(os.path.dirname(__file__), 'master.csv'), index_col = 0)
        if not self.name in meta_info:
            print(self.name)
            error_mssg = 'Invalid dataset name {}.\n'.format(self.name)
            error_mssg += 'Available datasets are as follows:\n'
            error_mssg += '\n'.join(meta_info.keys())
            raise ValueError(error_mssg)

        self.num_tasks = int(meta_info[self.name]['num tasks'])
        self.eval_metric = meta_info[self.name]['eval metric']

    def _parse_and_check_input(self, input_dict):
        if self.eval_metric == 'rocauc' or self.eval_metric == 'ap' or self.eval_metric == 'rmse' or self.eval_metric == 'acc':
            if not 'y_true' in input_dict:
                raise RuntimeError('Missing key of y_true')
            if not 'y_pred' in input_dict:
                raise RuntimeError('Missing key of y_pred')

            y_true, y_pred = input_dict['y_true'], input_dict['y_pred']

            '''
                y_true: numpy ndarray or torch tensor of shape (num_graphs, num_tasks)
                y_pred: numpy ndarray or torch tensor of shape (num_graphs, num_tasks)
            '''

            # converting to torch.Tensor to numpy on cpu
            if torch is not None and isinstance(y_true, torch.Tensor):
                y_true = y_true.detach().cpu().numpy()

            if torch is not None and isinstance(y_pred, torch.Tensor):
                y_pred = y_pred.detach().cpu().numpy()

            ## check type
            if not isinstance(y_true, np.ndarray):
                raise RuntimeError('Arguments to Evaluator need to be either numpy ndarray or torch tensor')

            if not y_true.shape == y_pred.shape:
                raise RuntimeError('Shape of y_true and y_pred must be the same')

            if not y_true.ndim == 2:
                raise RuntimeError('y_true and y_pred mush to 2-dim arrray, {}-dim array given'.format(y_true.ndim))

            if not y_true.shape[1] == self.num_tasks:
                raise RuntimeError('Number of tasks for {} should be {} but {} given'.format(self.name, self.num_tasks, y_true.shape[1]))

            return y_true, y_pred

        elif self.eval_metric == 'F1':
            if not 'seq_ref' in input_dict:
                raise RuntimeError('Missing key of seq_ref')
            if not 'seq_pred' in input_dict:
                raise RuntimeError('Missing key of seq_pred')

            seq_ref, seq_pred = input_dict['seq_ref'], input_dict['seq_pred']

            if not isinstance(seq_ref, list):
                raise RuntimeError('seq_ref must be of type list')

            if not isinstance(seq_pred, list):
                raise RuntimeError('seq_pred must be of type list')

            if len(seq_ref) != len(seq_pred):
                raise RuntimeError('Length of seq_true and seq_pred should be the same')

            return seq_ref, seq_pred

        else:
            raise ValueError('Undefined eval metric %s ' % (self.eval_metric))

    def eval(self, input_dict):

        if self.eval_metric == 'rocauc':
            y_true, y_pred = self._parse_and_check_input(input_dict)
            return self._eval_rocauc(y_true, y_pred)
        if self.eval_metric == 'ap':
            y_true, y_pred = self._parse_and_check_input(input_dict)
            return self._eval_ap(y_true, y_pred)
        elif self.eval_metric == 'rmse':
            y_true, y_pred = self._parse_and_check_input(input_dict)
            return self._eval_rmse(y_true, y_pred)
        elif self.eval_metric == 'acc':
            y_true, y_pred = self._parse_and_check_input(input_dict)
            return self._eval_acc(y_true, y_pred)
        elif self.eval_metric == 'F1':
            seq_ref, seq_pred = self._parse_and_check_input(input_dict)
            return self._eval_F1(seq_ref, seq_pred)
        else:
            raise ValueError('Undefined eval metric %s ' % (self.eval_metric))

    @property
    def expected_input_format(self):
        desc = '==== Expected input format of Evaluator for {}\n'.format(self.name)
        if self.eval_metric == 'rocauc' or self.eval_metric == 'ap':
            desc += '{\'y_true\': y_true, \'y_pred\': y_pred}\n'
            desc += '- y_true: numpy ndarray or torch tensor of shape (num_graphs, num_tasks)\n'
            desc += '- y_pred: numpy ndarray or torch tensor of shape (num_graphs, num_tasks)\n'
            desc += 'where y_pred stores score values (for computing AUC score),\n'
            desc += 'num_task is {}, and '.format(self.num_tasks)
            desc += 'each row corresponds to one graph.\n'
            desc += 'nan values in y_true are ignored during evaluation.\n'
        elif self.eval_metric == 'rmse':
            desc += '{\'y_true\': y_true, \'y_pred\': y_pred}\n'
            desc += '- y_true: numpy ndarray or torch tensor of shape (num_graphs, num_tasks)\n'
            desc += '- y_pred: numpy ndarray or torch tensor of shape (num_graphs, num_tasks)\n'
            desc += 'where num_task is {}, and '.format(self.num_tasks)
            desc += 'each row corresponds to one graph.\n'
            desc += 'nan values in y_true are ignored during evaluation.\n'
        elif self.eval_metric == 'acc':
            desc += '{\'y_true\': y_true, \'y_pred\': y_pred}\n'
            desc += '- y_true: numpy ndarray or torch tensor of shape (num_nodes num_tasks)\n'
            desc += '- y_pred: numpy ndarray or torch tensor of shape (num_nodes num_tasks)\n'
            desc += 'where y_pred stores predicted class label (integer),\n'
            desc += 'num_task is {}, and '.format(self.num_tasks)
            desc += 'each row corresponds to one graph.\n'
        elif self.eval_metric == 'F1':
            desc += '{\'seq_ref\': seq_ref, \'seq_pred\': seq_pred}\n'
            desc += '- seq_ref: a list of lists of strings\n'
            desc += '- seq_pred: a list of lists of strings\n'
            desc += 'where seq_ref stores the reference sequences of sub-tokens, and\n'
            desc += 'seq_pred stores the predicted sequences of sub-tokens.\n'
        else:
            raise ValueError('Undefined eval metric %s ' % (self.eval_metric))

        return desc

    @property
    def expected_output_format(self):
        desc = '==== Expected output format of Evaluator for {}\n'.format(self.name)
        if self.eval_metric == 'rocauc':
            desc += '{\'rocauc\': rocauc}\n'
            desc += '- rocauc (float): ROC-AUC score averaged across {} task(s)\n'.format(self.num_tasks)
        elif self.eval_metric == 'ap':
            desc += '{\'ap\': ap}\n'
            desc += '- ap (float): Average Precision (AP) score averaged across {} task(s)\n'.format(self.num_tasks)
        elif self.eval_metric == 'rmse':
            desc += '{\'rmse\': rmse}\n'
            desc += '- rmse (float): root mean squared error averaged across {} task(s)\n'.format(self.num_tasks)
        elif self.eval_metric == 'acc':
            desc += '{\'acc\': acc}\n'
            desc += '- acc (float): Accuracy score averaged across {} task(s)\n'.format(self.num_tasks)
        elif self.eval_metric == 'F1':
            desc += '{\'F1\': F1}\n'
            desc += '- F1 (float): F1 score averaged over samples.\n'
        else:
            raise ValueError('Undefined eval metric %s ' % (self.eval_metric))

        return desc

    def _eval_rocauc(self, y_true, y_pred):
        '''
            compute ROC-AUC averaged across tasks
        '''

        rocauc_list = []

        for i in range(y_true.shape[1]):
            #AUC is only defined when there is at least one positive data.
            if np.sum(y_true[:,i] == 1) > 0 and np.sum(y_true[:,i] == 0) > 0:
                # ignore nan values
                is_labeled = y_true[:,i] == y_true[:,i]
                rocauc_list.append(roc_auc_score(y_true[is_labeled,i], y_pred[is_labeled,i]))

        if len(rocauc_list) == 0:
            raise RuntimeError('No positively labeled data available. Cannot compute ROC-AUC.')

        return {'rocauc': sum(rocauc_list)/len(rocauc_list)}

    def _eval_ap(self, y_true, y_pred):
        '''
            compute Average Precision (AP) averaged across tasks
        '''

        ap_list = []

        for i in range(y_true.shape[1]):
            #AUC is only defined when there is at least one positive data.
            if np.sum(y_true[:,i] == 1) > 0 and np.sum(y_true[:,i] == 0) > 0:
                # ignore nan values
                is_labeled = y_true[:,i] == y_true[:,i]
                ap = average_precision_score(y_true[is_labeled,i], y_pred[is_labeled,i])

                ap_list.append(ap)

        if len(ap_list) == 0:
            raise RuntimeError('No positively labeled data available. Cannot compute Average Precision.')

        return {'ap': sum(ap_list)/len(ap_list)}

    def _eval_rmse(self, y_true, y_pred):
        '''
            compute RMSE score averaged across tasks
        '''
        rmse_list = []

        for i in range(y_true.shape[1]):
            # ignore nan values
            is_labeled = y_true[:,i] == y_true[:,i]
            rmse_list.append(np.sqrt(((y_true[is_labeled,i] - y_pred[is_labeled,i])**2).mean()))

        return {'rmse': sum(rmse_list)/len(rmse_list)}

    def _eval_acc(self, y_true, y_pred):
        acc_list = []

        for i in range(y_true.shape[1]):
            is_labeled = y_true[:,i] == y_true[:,i]
            correct = y_true[is_labeled,i] == y_pred[is_labeled,i]
            acc_list.append(float(np.sum(correct))/len(correct))

        return {'acc': sum(acc_list)/len(acc_list)}

    def _eval_F1(self, seq_ref, seq_pred):
        # '''
        #     compute F1 score averaged over samples
        # '''

        precision_list = []
        recall_list = []
        f1_list = []

        for l, p in zip(seq_ref, seq_pred):
            label = set(l)
            prediction = set(p)
            true_positive = len(label.intersection(prediction))
            false_positive = len(prediction - label)
            false_negative = len(label - prediction)

            if true_positive + false_positive > 0:
                precision = true_positive / (true_positive + false_positive)
            else:
                precision = 0

            if true_positive + false_negative > 0:
                recall = true_positive / (true_positive + false_negative)
            else:
                recall = 0
            if precision + recall > 0:
                f1 = 2 * precision * recall / (precision + recall)
            else:
                f1 = 0

            precision_list.append(precision)
            recall_list.append(recall)
            f1_list.append(f1)

        return {'precision': np.average(precision_list),
                'recall': np.average(recall_list),
                'F1': np.average(f1_list)}

if __name__ == '__main__':
    evaluator = Evaluator('ogbg-code2')
    print(evaluator.expected_input_format)
    print(evaluator.expected_output_format)
    seq_ref = [['tom', 'is'], ['he'], ['he'], ['hey', 'fea', 'he'], ['alpha'], ['fe4qfq', 'beta'], ['aa']]
    seq_pred = [['tom', 'is'], ['he'], ['he'], ['hey', 'he', 'fea'], ['alpha'], ['beta', 'fe4qfq'], ['aa']] # [['tom', 'is'] , ['he'], ['the', 'he'], ['hey', 'fea', 'he'], ['alpha'], ['beta', 'fe4qfq', 'c', 'fe4qf'], ['']]
    input_dict = {'seq_ref': seq_ref, 'seq_pred': seq_pred}
    result = evaluator.eval(input_dict)
    print(result)

    # exit(-1)

    evaluator = Evaluator('ogbg-molpcba')
    print(evaluator.expected_input_format)
    print(evaluator.expected_output_format)
    y_true = torch.tensor(np.random.randint(2, size = (100,128)))
    y_pred = torch.tensor(np.random.randn(100,128))
    input_dict = {'y_true': y_true, 'y_pred': y_pred}
    result = evaluator.eval(input_dict)
    print(result)

    evaluator = Evaluator('ogbg-molhiv')
    print(evaluator.expected_input_format)
    print(evaluator.expected_output_format)
    y_true = torch.tensor(np.random.randint(2, size = (100,1)))
    y_pred = torch.tensor(np.random.randn(100,1))
    input_dict = {'y_true': y_true, 'y_pred': y_pred}
    result = evaluator.eval(input_dict)
    print(result)

    ### rmse case
    evaluator = Evaluator('ogbg-mollipo')
    print(evaluator.expected_input_format)
    print(evaluator.expected_output_format)
    y_true = np.random.randn(100,1)
    y_pred = np.random.randn(100,1)
    input_dict = {'y_true': y_true, 'y_pred': y_pred}
    result = evaluator.eval(input_dict)
    print(result)

    ### acc
    evaluator = Evaluator('ogbg-ppa')
    print(evaluator.expected_input_format)
    print(evaluator.expected_output_format)
    y_true = np.random.randint(5, size = (100,1))
    y_pred = np.random.randint(5, size = (100,1))
    input_dict = {'y_true': y_true, 'y_pred': y_pred}
    result = evaluator.eval(input_dict)
    print(result)

<|endoftext|>


<commit_before># Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

{
  'variables': {
    'chromium_code': 1,
    'toolkit_views2': 0,  # ui/views/ is an experimental framework on Windows.
  },
  'target_defaults': {
    'sources/': [
      ['exclude', '/(cocoa|gtk|win)/'],
      ['exclude', '_(cocoa|gtk|linux|mac|posix|win|x)\\.(cc|mm?)$'],
      ['exclude', '/(gtk|win|x11)_[^/]*\\.cc$'],
    ],
    'conditions': [
      ['toolkit_uses_gtk == 1', {
        'sources/': [
          ['include', '/gtk/'],
          ['include', '_(gtk|linux|posix|skia|x)\\.cc$'],
          ['include', '/(gtk|x11)_[^/]*\\.cc$'],
        ],
      }],
      ['OS=="mac"', {'sources/': [
        ['include', '/cocoa/'],
        ['include', '_(cocoa|mac|posix)\\.(cc|mm?)$'],
      ]}, { # else: OS != "mac"
        'sources/': [
          ['exclude', '\\.mm?$'],
        ],
      }],
      ['OS=="win"',
        {'sources/': [
          ['include', '_(win)\\.cc$'],
          ['include', '/win/'],
          ['include', '/win_[^/]*\\.cc$'],
        ]},
        {'variables': {'toolkit_views2': 1}},
      ],
      ['toolkit_views2==0', {'sources/': [
        ['exclude', 'views/'],
      ]}],
      ['touchui==0', {'sources/': [
        ['exclude', 'event_x.cc$'],
        ['exclude', 'native_menu_x.cc$'],
        ['exclude', 'native_menu_x.h$'],
        ['exclude', 'touchui/'],
        ['exclude', '_(touch)\\.cc$'],
      ]}],
    ],
  },
  'includes': [
    'ui_base.gypi',
    'ui_gfx.gypi',
    'ui_resources.gypi',
  ],
  'conditions': [
    ['toolkit_views2==1', {
      'includes': [
        'ui_views.gypi',
      ],
    }],
    ['inside_chromium_build==1', {
      'includes': [
        'ui_unittests.gypi',
      ],
    }],
  ],
}

# Local Variables:
# tab-width:2
# indent-tabs-mode:nil
# End:
# vim: set expandtab tabstop=2 shiftwidth=2:
<commit_msg>Simplify ui/ui.gyp<commit_after># Copyright (c) 2011 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

{
  'variables': {
    'chromium_code': 1,
    'toolkit_views2': 0,  # ui/views/ is an experimental framework on Windows.
  },
  'target_defaults': {
    'conditions': [
      ['OS=="win"',
        {'variables': {'toolkit_views2': 1}},
      ],
      ['toolkit_views2==0', {'sources/': [
        ['exclude', 'views/'],
      ]}],
      ['touchui==0', {'sources/': [
        ['exclude', 'event_x.cc$'],
        ['exclude', 'native_menu_x.cc$'],
        ['exclude', 'native_menu_x.h$'],
        ['exclude', 'touchui/'],
        ['exclude', '_(touch)\\.cc$'],
      ]}],
    ],
  },
  'includes': [
    'ui_base.gypi',
    'ui_gfx.gypi',
    'ui_resources.gypi',
  ],
  'conditions': [
    ['toolkit_views2==1', {
      'includes': [
        'ui_views.gypi',
      ],
    }],
    ['inside_chromium_build==1', {
      'includes': [
        'ui_unittests.gypi',
      ],
    }],
  ],
}

# Local Variables:
# tab-width:2
# indent-tabs-mode:nil
# End:
# vim: set expandtab tabstop=2 shiftwidth=2:
<|endoftext|>


<commit_before>import subprocess
import time

def trigger(update_event):
    time.sleep(3600)
    update_event.set()

def pull(repo, branch):
    subprocess.call(("git", "pull", repo, branch))
    subprocess.call(("pip", "install", "-r", "requirements.txt"))
<commit_msg>Add KeyboardInterrupt handling for updater<commit_after>import subprocess
import sys
import time

def trigger(update_event):
    try:
        time.sleep(3600)
        update_event.set()
    except KeyboardInterrupt:
        sys.exit()

def pull(repo, branch):
    subprocess.call(("git", "pull", repo, branch))
    subprocess.call(("pip", "install", "-r", "requirements.txt"))
<|endoftext|>


<commit_before>#!/usr/bin/python
# The MIT License (MIT)
#
# Copyright (c) 2014 Gluu
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import os, subprocess, shutil, sys

def cpfile(src, dst):
  if os.path.exists(src) and os.path.exists(dst):
      shutil.copy2(src, dst)

def serviceinit(servicename, action):
    cmd = ["service", servicename, action]
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
    output, err = p.communicate()
    if err:
        print "Error running %s" % " ".join(cmd)
        print err
        sys.exit(1)
    print output

def check_ver():
    initpath = None
    dest_dir = None
    service_name = None
    if os.path.exists("/etc/init.d/gluu-server24"):
        initpath = "/etc/init.d/gluu-server24"
        dest_dir = "/opt/gluu-server24"
        service_name = "gluu-server24"
    elif os.path.exists("/etc/init.d/gluu-server"):
        initpath = "/etc/init.d/gluu-server"
        dest_dir = "/opt/gluu-server"
        service_name = "gluu-server"
    else:
        print "Gluu Server not found. Exiting..."
        sys.exit(2)
    return initpath, dest_dir, service_name

#NEED HELP WITH UPDATER VARIABLE
initpath, dest_dir, service_name = check_ver()
serviceinit(service_name, "stop")
cpfile("%s/idp.war" % initpath, "%s/opt/idp/war/idp.war" % dest_dir)
cpfile("%s/oxcas.war" % initpath, "%s/opt/dist/oxcas.war" % dest_dir)
cpfile("%s/identity.war" % initpath, "%s/opt/tomcat/webapps/identity.war" % dest_dir)
cpfile("%s/oxauth.war" % initpath, "%s/opt/tomcat/webapps/oxauth.war" % dest_dir)
serviceinit(service_name, "start")
<commit_msg>changed file location<commit_after>#!/usr/bin/python
# The MIT License (MIT)
#
# Copyright (c) 2014 Gluu
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

import os, subprocess, shutil, sys

package_dir = "/var/cache/apt/archives/"

def cpfile(src, dst):
  if os.path.exists(src) and os.path.exists(dst):
      shutil.copy2(src, dst)

def serviceinit(servicename, action):
    cmd = ["service", servicename, action]
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
    output, err = p.communicate()
    if err:
        print "Error running %s" % " ".join(cmd)
        print err
        sys.exit(1)
    print output

def check_ver():
    dest_dir = None
    service_name = None
    if os.path.exists("/etc/init.d/gluu-server24"):
        dest_dir = "/opt/gluu-server24"
        service_name = "gluu-server24"
    elif os.path.exists("/etc/init.d/gluu-server"):
        dest_dir = "/opt/gluu-server"
        service_name = "gluu-server"
    else:
        print "Gluu Server not found. Exiting..."
        sys.exit(2)
    return dest_dir, service_name

# MAIN LOOP
dest_dir, service_name = check_ver()
serviceinit(service_name, "stop")
cpfile("%s/%s/idp.war" % (package_dir, service_name),
       "%s/opt/idp/war/idp.war" % dest_dir)
cpfile("%s/%s/oxcas.war" % (package_dir, service_name),
       "%s/opt/dist/oxcas.war" % dest_dir)
cpfile("%s/%s/identity.war" % (package_dir, service_name),
       "%s/opt/tomcat/webapps/identity.war" % dest_dir)
cpfile("%s/%s/oxauth.war" % (package_dir, service_name),
       "%s/opt/tomcat/webapps/oxauth.war" % dest_dir)
serviceinit(service_name, "start")
<|endoftext|>


<commit_before>#!/usr/bin/env python

import requests

from flask import current_app
from flask import request
from requests.exceptions import ConnectionError, Timeout

from ooiservices.app.m2m import m2m as api
from ooiservices.app.models import User

class BadM2MException(Exception):
    pass

class InvalidPortException(BadM2MException):
    def __init__(self, port):
        super(InvalidPortException, self).__init__()
        self.message = 'Requested end point (%s) is not exposed through the machine to machine interface. ' \
                       'Please contact the system admin for more information.' % port

class InvalidPathException(BadM2MException):
    def __init__(self, path):
        super(InvalidPathException, self).__init__()
        self.message = 'Requested URL (%s) is not properly formatted for the machine to machine interface. ' \
                       'Please contact the system admin for more information.' % path

def build_url(path):
    """
    Given an M2M request path, build the corresponding UFrame URL
    Paths must conform to the following specification:
    <UFRAME PORT>/<UFRAME URL>
    :param path: input path
    :return: URL
    """
    try:
        port, path = path.split('/', 1)
        port = int(port)
    except ValueError:
        raise InvalidPathException(path)

    uframe_host = current_app.config['UFRAME_HOST']
    allowed_ports = current_app.config['UFRAME_ALLOWED_M2M_PORTS']

    if port not in allowed_ports:
        raise InvalidPortException(port)

    base_url = 'http://%s:%d' % (uframe_host, port)
    return '/'.join((base_url, path))

@api.route('/', methods=['GET'], defaults={'path': ''})
@api.route('/<path:path>', methods=['GET'])
def m2m_handler(path):
    """
    :param path:
    :return:
    """
    try:
        current_app.logger.info(path)
        if User.api_verify_token(request.authorization['username'], request.authorization['password']):
            url = build_url(path)
            timeout = current_app.config['UFRAME_TIMEOUT_CONNECT']
            timeout_read = current_app.config['UFRAME_TIMEOUT_READ']
            response = requests.get(url, timeout=(timeout, timeout_read), params=request.args)
            return response.text, response.status_code, dict(response.headers)
        else:
            message = 'Authentication failed.'
            current_app.logger.info(message)
            return message, 401
    except BadM2MException as e:
        current_app.logger.info(e.message)
        return e.message, 403
    except ConnectionError:
        message = 'ConnectionError for get uframe contents.'
        current_app.logger.info(message)
        raise Exception(message)
    except Timeout:
        message = 'Timeout for get uframe contents.'
        current_app.logger.info(message)
        raise Exception(message)
    except Exception:
        raise
<commit_msg>Only forward the date and content-type headers for M2M requests<commit_after>#!/usr/bin/env python

import requests

from flask import current_app
from flask import request
from requests.exceptions import ConnectionError, Timeout

from ooiservices.app.m2m import m2m as api
from ooiservices.app.models import User

class BadM2MException(Exception):
    pass

class InvalidPortException(BadM2MException):
    def __init__(self, port):
        super(InvalidPortException, self).__init__()
        self.message = 'Requested end point (%s) is not exposed through the machine to machine interface. ' \
                       'Please contact the system admin for more information.' % port

class InvalidPathException(BadM2MException):
    def __init__(self, path):
        super(InvalidPathException, self).__init__()
        self.message = 'Requested URL (%s) is not properly formatted for the machine to machine interface. ' \
                       'Please contact the system admin for more information.' % path

def build_url(path):
    """
    Given an M2M request path, build the corresponding UFrame URL
    Paths must conform to the following specification:
    <UFRAME PORT>/<UFRAME URL>
    :param path: input path
    :return: URL
    """
    try:
        port, path = path.split('/', 1)
        port = int(port)
    except ValueError:
        raise InvalidPathException(path)

    uframe_host = current_app.config['UFRAME_HOST']
    allowed_ports = current_app.config['UFRAME_ALLOWED_M2M_PORTS']

    if port not in allowed_ports:
        raise InvalidPortException(port)

    base_url = 'http://%s:%d' % (uframe_host, port)
    return '/'.join((base_url, path))

@api.route('/', methods=['GET'], defaults={'path': ''})
@api.route('/<path:path>', methods=['GET'])
def m2m_handler(path):
    """
    :param path:
    :return:
    """
    transfer_header_fields = ['Date', 'Content-Type']
    try:
        current_app.logger.info(path)
        if User.api_verify_token(request.authorization['username'], request.authorization['password']):
            url = build_url(path)
            timeout = current_app.config['UFRAME_TIMEOUT_CONNECT']
            timeout_read = current_app.config['UFRAME_TIMEOUT_READ']
            response = requests.get(url, timeout=(timeout, timeout_read), params=request.args)
            headers = dict(response.headers)
            headers = {k: headers[k] for k in headers if k in transfer_header_fields}
            return response.text, response.status_code, headers
        else:
            message = 'Authentication failed.'
            current_app.logger.info(message)
            return message, 401
    except BadM2MException as e:
        current_app.logger.info(e.message)
        return e.message, 403
    except ConnectionError:
        message = 'ConnectionError for get uframe contents.'
        current_app.logger.info(message)
        raise Exception(message)
    except Timeout:
        message = 'Timeout for get uframe contents.'
        current_app.logger.info(message)
        raise Exception(message)
    except Exception:
        raise
<|endoftext|>


<commit_before>from adn_news import *
from helpers import *

def get_posts_update_db():
    last = Last.query.first()
    t = tavorite.userStream(count=200, since_id=last.post_id)
    if t.get('meta').get('code') == 200:
        for x in t['data']:
            if x['entities']['links']:
                if x.get('id') == x.get('thread_id'):
                    if not Post.query.filter_by(post_id=x.get('id')).first():
                        a = Post(x)
                        try:
                            db.session.add(a)
                            db.session.commit()
                        except:
                            db.session.rollback()

        db.session.delete(last)
        db.session.commit()

        track_last = Last(t['data'][-1])
        db.session.add(track_last)
        db.session.commit()

def update_posts_comments():
    two_days_ago = datetime.utcnow() - timedelta(days=1)
    posts = Post.query.filter(Post.date > two_days_ago).all()
    for x in posts:
        num_comments = number_of_comments(count_comments(x))
        current_replies = tavorite.repliesToPost(post_id=x.post_id, count=200).get('data')
        if current_replies == None:
            current_replies = ["No replies"]

        num_replies = len(current_replies) - 1
        if num_comments < num_replies:
            x.score = num_replies
            db.session.commit()
            add_comments(str(x.post_id))

def hacker_news(votes, item_hour_age, gravity=1.8):
    return votes/pow((item_hour_age+2), gravity)

def reduce_score_with_time():
    """reduces posts score_with_time for last 5 days"""
    six_days_ago = datetime.utcnow() - timedelta(days=5)
    posts = Post.query.filter(Post.date > six_days_ago).all()
    for x in posts:
        x.score_with_time = hacker_news(x.score, link_age_in_hours(x))
        db.session.commit()
    print "Gravity at work"

def update_every_minute():
    """Automates - every X minutes gets new post and update those posts if they have additional comments"""

    s = sched.scheduler(time.time, time.sleep)
    print "updating feed beginning"
    s.enter(300, 1, get_posts_update_db, ())
    s.run()
    update_posts_comments()
    reduce_score_with_time()
    update_every_minute()
    """To continously loop recursive call update_every_minute()"""

# every two minutes maybe needs to lengthened

if __name__ == '__main__':
    update_every_minute()
<commit_msg>fix update posts function<commit_after>from adn_news import *
from helpers import *

def get_posts_update_db():
    last = Last.query.first()
    t = tavorite.userStream(count=200, since_id=last.post_id)
    if t.get('meta').get('code') == 200:
        for x in t['data']:
            if x['entities']['links']:
                if x.get('id') == x.get('thread_id'):
                    if not Post.query.filter_by(post_id=x.get('id')).first():
                        a = Post(x)
                        try:
                            db.session.add(a)
                            db.session.commit()
                        except:
                            db.session.rollback()

        db.session.delete(last)
        db.session.commit()

        track_last = Last(t['data'][0])
        db.session.add(track_last)
        db.session.commit()

def update_posts_comments():
    two_days_ago = datetime.utcnow() - timedelta(days=1)
    posts = Post.query.filter(Post.date > two_days_ago).all()
    for x in posts:
        num_comments = number_of_comments(count_comments(x))
        current_replies = tavorite.repliesToPost(post_id=x.post_id, count=200).get('data')
        if current_replies == None:
            current_replies = ["No replies"]

        num_replies = len(current_replies) - 1
        if num_comments < num_replies:
            x.score = num_replies
            db.session.commit()
            add_comments(str(x.post_id))

def hacker_news(votes, item_hour_age, gravity=1.8):
    return votes/pow((item_hour_age+2), gravity)

def reduce_score_with_time():
    """reduces posts score_with_time for last 5 days"""
    six_days_ago = datetime.utcnow() - timedelta(days=5)
    posts = Post.query.filter(Post.date > six_days_ago).all()
    for x in posts:
        x.score_with_time = hacker_news(x.score, link_age_in_hours(x))
        db.session.commit()
    print "Gravity at work"

def update_every_minute():
    """Automates - every X minutes gets new post and update those posts if they have additional comments"""

    s = sched.scheduler(time.time, time.sleep)
    print "updating feed beginning"
    s.enter(300, 1, get_posts_update_db, ())
    s.run()
    update_posts_comments()
    reduce_score_with_time()
    update_every_minute()
    """To continously loop recursive call update_every_minute()"""

# every two minutes maybe needs to lengthened

if __name__ == '__main__':
    update_every_minute()
<|endoftext|>


<commit_before>#!/usr/bin/env python

import requests

from flask import current_app
from flask import request
from requests.exceptions import ConnectionError, Timeout

from ooiservices.app.m2m import m2m as api
from ooiservices.app.models import User

class BadM2MException(Exception):
    pass

class InvalidPortException(BadM2MException):
    def __init__(self, port):
        super(InvalidPortException, self).__init__()
        self.message = 'Requested end point (%s) is not exposed through the machine to machine interface. ' \
                       'Please contact the system admin for more information.' % port

class InvalidPathException(BadM2MException):
    def __init__(self, path):
        super(InvalidPathException, self).__init__()
        self.message = 'Requested URL (%s) is not properly formatted for the machine to machine interface. ' \
                       'Please contact the system admin for more information.' % path

def build_url(path):
    """
    Given an M2M request path, build the corresponding UFrame URL
    Paths must conform to the following specification:
    <UFRAME PORT>/<UFRAME URL>
    :param path: input path
    :return: URL
    """
    try:
        port, path = path.split('/', 1)
        port = int(port)
    except ValueError:
        raise InvalidPathException(path)

    uframe_host = current_app.config['UFRAME_HOST']
    allowed_ports = current_app.config['UFRAME_ALLOWED_M2M_PORTS']

    if port not in allowed_ports:
        raise InvalidPortException(port)

    base_url = 'http://%s:%d' % (uframe_host, port)
    return '/'.join((base_url, path))

@api.route('/', methods=['GET'], defaults={'path': ''})
@api.route('/<path:path>', methods=['GET'])
def m2m_handler(path):
    """
    :param path:
    :return:
    """
    transfer_header_fields = ['Date', 'Content-Type']
    try:
        current_app.logger.info(path)
        if User.api_verify_token(request.authorization['username'], request.authorization['password']):
            url = build_url(path)
            timeout = current_app.config['UFRAME_TIMEOUT_CONNECT']
            timeout_read = current_app.config['UFRAME_TIMEOUT_READ']
            response = requests.get(url, timeout=(timeout, timeout_read), params=request.args)
            headers = dict(response.headers)
            headers = {k: headers[k] for k in headers if k in transfer_header_fields}
            return response.text, response.status_code, headers
        else:
            message = 'Authentication failed.'
            current_app.logger.info(message)
            return message, 401
    except BadM2MException as e:
        current_app.logger.info(e.message)
        return e.message, 403
    except ConnectionError:
        message = 'ConnectionError for get uframe contents.'
        current_app.logger.info(message)
        raise Exception(message)
    except Timeout:
        message = 'Timeout for get uframe contents.'
        current_app.logger.info(message)
        raise Exception(message)
    except Exception:
        raise
<commit_msg>Stream M2M responses<commit_after>#!/usr/bin/env python

import requests

from flask import Response
from flask import current_app
from flask import request
from requests.exceptions import ConnectionError, Timeout

from ooiservices.app.m2m import m2m as api
from ooiservices.app.models import User

class BadM2MException(Exception):
    pass

class InvalidPortException(BadM2MException):
    def __init__(self, port):
        super(InvalidPortException, self).__init__()
        self.message = 'Requested end point (%s) is not exposed through the machine to machine interface. ' \
                       'Please contact the system admin for more information.' % port

class InvalidPathException(BadM2MException):
    def __init__(self, path):
        super(InvalidPathException, self).__init__()
        self.message = 'Requested URL (%s) is not properly formatted for the machine to machine interface. ' \
                       'Please contact the system admin for more information.' % path

def build_url(path):
    """
    Given an M2M request path, build the corresponding UFrame URL
    Paths must conform to the following specification:
    <UFRAME PORT>/<UFRAME URL>
    :param path: input path
    :return: URL
    """
    try:
        port, path = path.split('/', 1)
        port = int(port)
    except ValueError:
        raise InvalidPathException(path)

    uframe_host = current_app.config['UFRAME_HOST']
    allowed_ports = current_app.config['UFRAME_ALLOWED_M2M_PORTS']

    if port not in allowed_ports:
        raise InvalidPortException(port)

    base_url = 'http://%s:%d' % (uframe_host, port)
    return '/'.join((base_url, path))

@api.route('/', methods=['GET'], defaults={'path': ''})
@api.route('/<path:path>', methods=['GET'])
def m2m_handler(path):
    """
    :param path:
    :return:
    """
    transfer_header_fields = ['Date', 'Content-Type']
    try:
        current_app.logger.info(path)
        if User.api_verify_token(request.authorization['username'], request.authorization['password']):
            url = build_url(path)
            timeout = current_app.config['UFRAME_TIMEOUT_CONNECT']
            timeout_read = current_app.config['UFRAME_TIMEOUT_READ']
            response = requests.get(url, timeout=(timeout, timeout_read), params=request.args, stream=True)
            headers = dict(response.headers)
            headers = {k: headers[k] for k in headers if k in transfer_header_fields}
            return Response(response.iter_content(1024), response.status_code, headers)
        else:
            message = 'Authentication failed.'
            current_app.logger.info(message)
            return message, 401
    except BadM2MException as e:
        current_app.logger.info(e.message)
        return e.message, 403
    except ConnectionError:
        message = 'ConnectionError for get uframe contents.'
        current_app.logger.info(message)
        raise Exception(message)
    except Timeout:
        message = 'Timeout for get uframe contents.'
        current_app.logger.info(message)
        raise Exception(message)
    except Exception:
        raise
<|endoftext|>