<commit_before>from __future__ import division

import logging

from numpy import all as npall
from numpy import sum as npsum
from numpy import (
    asarray, clip, dot, empty, errstate, full, inf, isfinite, log, nan_to_num,
    zeros
)
from numpy.linalg import LinAlgError
from numpy_sugar import epsilon
from numpy_sugar.linalg import rsolve, solve
from tqdm import tqdm

class FastScanner(object):
    r"""Approximated fast inference over several covariates.

    Let :math:`\mathrm X` be a samples-by-covariates matrix,
    :math:`\mathrm M` a samples-by-markers matrix, and
    :math:`\mathbf y` an array of outcome.
    A covariance :math:`\mathrm K` will be provided via its economic eigen
    decomposition ``((Q0, Q1), S0)`` and ``v`` will define the variance
    due to the identity in the formulae

    .. math::

        \mathbf y \sim \mathcal N\big(~ \mathrm X\boldsymbol\beta
        + \mathrm{M}_i \alpha_i,~
        s (\mathrm K + v \mathrm I) ~\big)

    Note that :math:`\alpha_i` is a scalar multiplying a column-matrix
    :math:`\mathrm{M}_i`.
    The variable :math:`s` is a scaling factor that, if not set, is jointly
    adjusted with :math:`\alpha_i` in order the maximise the marginal
    likelihood, ultimately providing the degree of association between the
    marker :math:`\mathrm{M}_i` with the outcome :math:`\mathbf y` via an
    p-value.

    For performance reasons, we make use of the identity

    .. math::

        (\mathrm K + v\mathrm I)^{-1} = \mathrm Q \left(
            \begin{array}{cc}
                \mathrm S_0 + v\mathrm I_0 & \mathbf 0\\
                \mathbf 0 & v\mathrm I_1
            \end{array}\right)^{-1}
            \mathrm Q^{\intercal}

    in the implementation.
    We can thus write the marginal likelihood as

    .. math::

        \mathcal N\left(\mathrm Q^{\intercal} \mathbf y ~|~
                   \mathrm Q^{\intercal} \mathbf m,~
                   s \left(
                       \begin{array}{cc}
                           \mathrm S_0 + v\mathrm I_0 & \mathbf 0\\
                           \mathbf 0 & v\mathrm I_1
                       \end{array}\right)\right).

    Expanding it gives us

    .. math::

        -\frac{n}{2} \log 2\pi - \frac{1}{2}n \log s
            - \frac{1}{2}\log|\mathrm D|\\
            - \frac{1}{2} (\mathrm Q^{\intercal}\mathbf y)^{\intercal} s^{-1}
            \mathrm D^{-1}(\mathrm Q^{\intercal} \mathbf y)\\
            + (\mathrm Q^{\intercal}\mathbf y)^{\intercal}
            s^{-1} \mathrm D^{-1}
            (\mathrm Q^{\intercal} \mathrm X\boldsymbol\beta)\\
            - \frac{1}{2} (\mathrm Q^{\intercal}
            \mathrm X\boldsymbol\beta)^{\intercal} s^{-1} \mathrm D^{-1}
            (\mathrm Q^{\intercal} \mathrm X\boldsymbol\beta),

    where

    .. math::

        \mathrm D = \left(
            \begin{array}{cc}
              \mathrm S_0 + v\mathrm I_0 & \mathbf 0\\
              \mathbf 0 & v\mathrm I_1
            \end{array}
            \right)

    Attributes
    ----------
    _a : tuple
        Store the arrays

        .. math::

            (\mathrm Q_i^{\intercal} \mathbf y)^{\intercal}
            \mathrm D_i^{-1}
            (\mathrm Q_i^{\intercal} \mathbf y),

        for :math:`i = \{0, 1\}`.

    _b : tuple
        Store the arrays

        .. math::

            (\mathrm Q_i^{\intercal} \mathbf y)^{\intercal}
            \mathrm D_i^{-1}
            (\mathrm Q_i^{\intercal} \mathrm X),

        for :math:`i = \{0, 1\}`.

    _yTQdiag : tuple
        Store the arrays
        :math:`(\mathrm Q_i^{\intercal} \mathbf y)^{\intercal}\mathrm D_i^{-1}`
        , for :math:`i = \{0, 1\}`.

    _XTQdiag : tuple
        Store the arrays
        :math:`(\mathrm Q_i^{\intercal} \mathrm X)^{\intercal}\mathrm D_i^{-1}`
        , for :math:`i = \{0, 1\}`.

    Parameters
    ----------
    y : array_like
        Real-valued outcome.
    X : array_like
        Matrix of covariates.
    QS : tuple
        Economic eigen decomposition ``((Q0, Q1), S0)``.
    v : float
        Variance due to iid effect.
    """

    def __init__(self, y, X, QS, v):

        self._scale = None
        self._QS = QS
        self._D = [QS[1] + v, v]

        yTQ = [dot(y.T, Q) for Q in QS[0]]

        XTQ = [dot(X.T, Q) for Q in QS[0]]

        self._yTQdiag = [l / r for (l, r) in zip(yTQ, self._D)]

        self._a = [(i**2 / j).sum() for (i, j) in zip(yTQ, self._D)]

        self._b = [dot(i, j.T) for (i, j) in zip(self._yTQdiag, XTQ)]

        self._XTQdiag = [i / j for (i, j) in zip(XTQ, self._D)]

        nc = X.shape[1]

        self._C = [empty((nc + 1, nc + 1)), empty((nc + 1, nc + 1))]

        for i in range(2):
            self._C[i][:-1, :-1] = dot(self._XTQdiag[i], XTQ[i].T)

    def _static_lml(self):
        LOG2PI = 1.837877066409345339081937709124758839607238769531250
        n = self._QS[0][0].shape[0]
        p = len(self._D[0])
        static_lml = -n * LOG2PI - n
        static_lml -= npsum(log(self._D[0]))
        static_lml -= (n - p) * log(self._D[1])
        return static_lml

    def _fast_scan_chunk(self, markers):
        markers = asarray(markers, float)

        if not markers.ndim == 2:
            raise ValueError("`markers` array must be bidimensional.")

        if not npall(isfinite(markers)):
            raise ValueError("One or more variants have non-finite value.")

        mTQ = [dot(markers.T, Q) for Q in self._QS[0]]
        bm = [dot(i, j.T) for (i, j) in zip(self._yTQdiag, mTQ)]

        c_01 = [dot(i, j.T) for (i, j) in zip(self._XTQdiag, mTQ)]
        c_11 = [npsum((i / j) * i, axis=1) for (i, j) in zip(mTQ, self._D)]

        nsamples = markers.shape[0]
        nmarkers = markers.shape[1]

        lmls = full(nmarkers, self._static_lml())
        effect_sizes = empty(nmarkers)

        if self._XTQdiag[0].shape[0] == 1:
            return self._fast_scan_chunk_1covariate_loop(
                lmls, effect_sizes, bm, c_01, c_11, nsamples)
        else:
            return self._fast_scan_chunk_multicovariate_loop(
                lmls, effect_sizes, bm, c_01, c_11, nsamples, nmarkers)

    def _fast_scan_chunk_multicovariate_loop(self, lmls, effect_sizes, bm,
                                             c_01, c_11, nsamples, nmarkers):
        b00m = empty(len(self._b[0]) + 1)
        b00m[:-1] = self._b[0]

        b11m = empty(len(self._b[1]) + 1)
        b11m[:-1] = self._b[1]

        for i in range(nmarkers):

            b00m[-1] = bm[0][i]
            b11m[-1] = bm[1][i]

            self._C[0][:-1, -1] = c_01[0][:, i]
            self._C[1][:-1, -1] = c_01[1][:, i]

            self._C[0][-1, :-1] = self._C[0][:-1, -1]
            self._C[1][-1, :-1] = self._C[1][:-1, -1]

            self._C[0][-1, -1] = c_11[0][i]
            self._C[1][-1, -1] = c_11[1][i]

            try:
                beta = solve(self._C[1] - self._C[0], b11m - b00m)
            except LinAlgError:
                try:
                    beta = rsolve(self._C[1] - self._C[0], b11m - b00m)
                except LinAlgError:
                    msg = "Could not converge to the optimal"
                    msg += " effect-size of one of the candidates."
                    msg += " Setting its effect-size to zero."
                    logging.getLogger(__name__).warning(msg)
                    beta = zeros(self._C[1].shape[0])

            effect_sizes[i] = beta[-1]

            if self._scale is None:
                p0 = self._a[0] - 2 * b00m.dot(beta) + beta.dot(
                    self._C[0]).dot(beta)
                p1 = self._a[1] - 2 * b11m.dot(beta) + beta.dot(
                    self._C[1].dot(beta))

                scale = (p0 + p1) / nsamples
            else:
                scale = self._scale

            lmls[i] -= nsamples * log(max(scale, epsilon.super_tiny))

        lmls /= 2
        return lmls, effect_sizes

    def _fast_scan_chunk_1covariate_loop(self, lmls, effect_sizes, bm, c_01,
                                         C11, nsamples):

        C00 = [C[0, 0] for C in self._C]
        C01 = [c[0, :] for c in c_01]

        b = [bi[0] for bi in self._b]

        sC00 = C00[1] - C00[0]
        sC01 = C01[1] - C01[0]
        sC11 = C11[1] - C11[0]

        sb = b[1] - b[0]
        sbm = bm[1] - bm[0]

        with errstate(divide='ignore'):
            beta = beta_1covariate(sb, sbm, sC00, sC01, sC11)

        beta = [nan_to_num(bet) for bet in beta]

        scale = zeros(len(lmls))

        if self._scale is None:
            for i in range(2):
                scale += self._a[i] - 2 * (b[i] * beta[0] + bm[i] * beta[1])
                scale += beta[0] * (C00[i] * beta[0] + C01[i] * beta[1])
                scale += beta[1] * (C01[i] * beta[0] + C11[i] * beta[1])
            scale /= nsamples
        else:
            scale = self._scale

        lmls -= nsamples * log(clip(scale, epsilon.super_tiny, inf))
        lmls /= 2

        effect_sizes = beta[1]

        return lmls, effect_sizes

    def fast_scan(self, markers, verbose=True):
        r"""LML and fixed-effect sizes of each marker.

        If the scaling factor ``s`` is not set by the user via method
        :func:`set_scale`, its optimal value will be found and
        used in the calculation.

        Parameters
        ----------
        markers : array_like
            Matrix of fixed-effects across columns.
        verbose : bool, optional
            ``True`` for progress information; ``False`` otherwise.
            Defaults to ``True``.

        Returns
        -------
        array_like
            Log of the marginal likelihoods.
        array_like
            Fixed-effect sizes.
        """

        if not (markers.ndim == 2):
            raise ValueError("`markers` array must be bidimensional.")
        p = markers.shape[1]

        lmls = empty(p)
        effect_sizes = empty(p)

        if verbose:
            nchunks = min(p, 30)
        else:
            nchunks = min(p, 1)

        chunk_size = (p + nchunks - 1) // nchunks

        for i in tqdm(range(nchunks), desc="Scanning", disable=not verbose):
            start = i * chunk_size
            stop = min(start + chunk_size, markers.shape[1])

            l, e = self._fast_scan_chunk(markers[:, start:stop])

            lmls[start:stop] = l
            effect_sizes[start:stop] = e

        return lmls, effect_sizes

    def null_lml(self):
        r"""Log of the marginal likelihood.

        .. math::

            - \frac{n}{2}\log{2\pi}
                - \frac{1}{2} n \log{s}
                - \frac{1}{2} \log{\left|\mathrm K + v \mathrm I \right|}
                    - \frac{1}{2}
                    \left(\mathbf y -
                    \mathrm X\boldsymbol\beta\right)^{\intercal}
                    s^{-1}\left(\mathrm K + v \mathrm I \right)^{-1}
                    \left(\mathbf y -
                    \mathrm X\boldsymbol\beta\right)
        """
        # TODO
        # def _diag(self, i):
        #     if i == 0:
        #         return self._QS[1] * (1 - self.delta) + self.delta
        #     return self.delta
        LOG2PI = 1.837877066409345339081937709124758839607238769531250
        n = self._QS[0][0].shape[0]
        p = n - self._QS[1].shape[0]
        if self._scale is None:
            pass
        else:
            lml = -n * LOG2PI - n - n * log(self._scale)
        lml += -sum(log(self._D[0])) - p * log(self._D[1])
        lml /= 2
        return lml

    def set_scale(self, scale):
        r"""Set the scaling factor.

        Calling this method disables the automatic scale learning.

        Parameters
        ----------
        scale : float
            Scaling factor.
        """
        self._scale = scale

    def unset_scale(self):
        r"""Unset the scaling factor.

        If called, it enables the scale learning again.
        """
        self._scale = None

def beta_1covariate(sb, sbm, sC00, sC01, sC11):
    d0 = sb / sC00
    d1 = sb / sC01

    d3 = sbm / sC01
    d4 = sbm / sC11

    d5 = sC00 / sC01
    d6 = sC11 / sC01

    return [(d1 - d4) / (d5 - 1 / d6), (-d0 + d3) / (d6 - 1 / d5)]
<commit_msg>sane var name, fastscan<commit_after>from __future__ import division

import logging

from numpy import all as npall
from numpy import sum as npsum
from numpy import (
    asarray, clip, dot, empty, errstate, full, inf, isfinite, log, nan_to_num,
    zeros
)
from numpy.linalg import LinAlgError
from numpy_sugar import epsilon
from numpy_sugar.linalg import rsolve, solve
from tqdm import tqdm

from ..util import log2pi

class FastScanner(object):
    r"""Approximated fast inference over several covariates.

    Let :math:`\mathrm X` be a samples-by-covariates matrix,
    :math:`\mathrm M` a samples-by-markers matrix, and
    :math:`\mathbf y` an array of outcome.
    A covariance :math:`\mathrm K` will be provided via its economic eigen
    decomposition ``((Q0, Q1), S0)`` and ``v`` will define the variance
    due to the identity in the formulae

    .. math::

        \mathbf y \sim \mathcal N\big(~ \mathrm X\boldsymbol\beta
        + \mathrm{M}_i \alpha_i,~
        s (\mathrm K + v \mathrm I) ~\big)

    Note that :math:`\alpha_i` is a scalar multiplying a column-matrix
    :math:`\mathrm{M}_i`.
    The variable :math:`s` is a scaling factor that, if not set, is jointly
    adjusted with :math:`\alpha_i` in order the maximise the marginal
    likelihood, ultimately providing the degree of association between the
    marker :math:`\mathrm{M}_i` with the outcome :math:`\mathbf y` via an
    p-value.

    For performance reasons, we make use of the identity

    .. math::

        (\mathrm K + v\mathrm I)^{-1} = \mathrm Q \left(
            \begin{array}{cc}
                \mathrm S_0 + v\mathrm I_0 & \mathbf 0\\
                \mathbf 0 & v\mathrm I_1
            \end{array}\right)^{-1}
            \mathrm Q^{\intercal}

    in the implementation.
    We can thus write the marginal likelihood as

    .. math::

        \mathcal N\left(\mathrm Q^{\intercal} \mathbf y ~|~
                   \mathrm Q^{\intercal} \mathbf m,~
                   s \left(
                       \begin{array}{cc}
                           \mathrm S_0 + v\mathrm I_0 & \mathbf 0\\
                           \mathbf 0 & v\mathrm I_1
                       \end{array}\right)\right).

    Expanding it gives us

    .. math::

        -\frac{n}{2} \log 2\pi - \frac{1}{2}n \log s
            - \frac{1}{2}\log|\mathrm D|\\
            - \frac{1}{2} (\mathrm Q^{\intercal}\mathbf y)^{\intercal} s^{-1}
            \mathrm D^{-1}(\mathrm Q^{\intercal} \mathbf y)\\
            + (\mathrm Q^{\intercal}\mathbf y)^{\intercal}
            s^{-1} \mathrm D^{-1}
            (\mathrm Q^{\intercal} \mathrm X\boldsymbol\beta)\\
            - \frac{1}{2} (\mathrm Q^{\intercal}
            \mathrm X\boldsymbol\beta)^{\intercal} s^{-1} \mathrm D^{-1}
            (\mathrm Q^{\intercal} \mathrm X\boldsymbol\beta),

    where

    .. math::

        \mathrm D = \left(
            \begin{array}{cc}
              \mathrm S_0 + v\mathrm I_0 & \mathbf 0\\
              \mathbf 0 & v\mathrm I_1
            \end{array}
            \right)

    Attributes
    ----------
    _yTQDiQTy : tuple
        Store the arrays

        .. math::

            (\mathrm Q_i^{\intercal} \mathbf y)^{\intercal}
            \mathrm D_i^{-1}
            (\mathrm Q_i^{\intercal} \mathbf y),

        for :math:`i = \{0, 1\}`.

    _yTQDiQTX : tuple
        Store the arrays

        .. math::

            (\mathrm Q_i^{\intercal} \mathbf y)^{\intercal}
            \mathrm D_i^{-1}
            (\mathrm Q_i^{\intercal} \mathrm X),

        for :math:`i = \{0, 1\}`.

    _yTQDi : tuple
        Store the arrays
        :math:`(\mathrm Q_i^{\intercal} \mathbf y)^{\intercal}\mathrm D_i^{-1}`
        , for :math:`i = \{0, 1\}`.

    _XTQDi : tuple
        Store the arrays
        :math:`(\mathrm Q_i^{\intercal} \mathrm X)^{\intercal}\mathrm D_i^{-1}`
        , for :math:`i = \{0, 1\}`.

    Parameters
    ----------
    y : array_like
        Real-valued outcome.
    X : array_like
        Matrix of covariates.
    QS : tuple
        Economic eigen decomposition ``((Q0, Q1), S0)``.
    v : float
        Variance due to iid effect.
    """

    def __init__(self, y, X, QS, v):

        self._scale = None
        self._QS = QS
        self._D = [QS[1] + v, v]

        yTQ = [dot(y.T, Q) for Q in QS[0]]

        XTQ = [dot(X.T, Q) for Q in QS[0]]

        self._yTQDi = [l / r for (l, r) in zip(yTQ, self._D)]

        self._yTQDiQTy = [(i**2 / j).sum() for (i, j) in zip(yTQ, self._D)]

        self._yTQDiQTX = [dot(i, j.T) for (i, j) in zip(self._yTQDi, XTQ)]

        self._XTQDi = [i / j for (i, j) in zip(XTQ, self._D)]

        nc = X.shape[1]

        self._C = [empty((nc + 1, nc + 1)), empty((nc + 1, nc + 1))]

        for i in range(2):
            self._C[i][:-1, :-1] = dot(self._XTQDi[i], XTQ[i].T)

        self._XTQDiQTX = [self._C[i][:-1, :-1] for i in range(2)]

    def _static_lml(self):
        n = self._QS[0][0].shape[0]
        p = len(self._D[0])
        static_lml = -n * log2pi - n
        static_lml -= npsum(log(self._D[0]))
        static_lml -= (n - p) * log(self._D[1])
        return static_lml

    def _fast_scan_chunk(self, markers):
        markers = asarray(markers, float)

        if not markers.ndim == 2:
            raise ValueError("`markers` array must be bidimensional.")

        if not npall(isfinite(markers)):
            raise ValueError("One or more variants have non-finite value.")

        mTQ = [dot(markers.T, Q) for Q in self._QS[0]]
        yTQDiQTm = [dot(i, j.T) for (i, j) in zip(self._yTQDi, mTQ)]

        XTQDiQTm = [dot(i, j.T) for (i, j) in zip(self._XTQDi, mTQ)]
        mTQDiQTm = [npsum((i / j) * i, axis=1) for (i, j) in zip(mTQ, self._D)]

        nsamples = markers.shape[0]
        nmarkers = markers.shape[1]

        lmls = full(nmarkers, self._static_lml())
        effect_sizes = empty(nmarkers)

        if self._XTQDi[0].shape[0] == 1:
            return self._fast_scan_chunk_1covariate_loop(
                lmls, effect_sizes, yTQDiQTm, XTQDiQTm, mTQDiQTm, nsamples)
        else:
            return self._fast_scan_chunk_multicovariate_loop(
                lmls, effect_sizes, yTQDiQTm, XTQDiQTm, mTQDiQTm, nsamples,
                nmarkers)

    def _fast_scan_chunk_multicovariate_loop(self, lmls, effect_sizes,
                                             yTQDiQTm, XTQDiQTm, mTQDiQTm,
                                             nsamples, nmarkers):
        b00m = empty(len(self._yTQDiQTX[0]) + 1)
        b00m[:-1] = self._yTQDiQTX[0]

        b11m = empty(len(self._yTQDiQTX[1]) + 1)
        b11m[:-1] = self._yTQDiQTX[1]

        for i in range(nmarkers):

            b00m[-1] = yTQDiQTm[0][i]
            b11m[-1] = yTQDiQTm[1][i]

            self._C[0][:-1, -1] = XTQDiQTm[0][:, i]
            self._C[1][:-1, -1] = XTQDiQTm[1][:, i]

            self._C[0][-1, :-1] = self._C[0][:-1, -1]
            self._C[1][-1, :-1] = self._C[1][:-1, -1]

            self._C[0][-1, -1] = mTQDiQTm[0][i]
            self._C[1][-1, -1] = mTQDiQTm[1][i]

            try:
                beta = solve(self._C[1] - self._C[0], b11m - b00m)
            except LinAlgError:
                try:
                    beta = rsolve(self._C[1] - self._C[0], b11m - b00m)
                except LinAlgError:
                    msg = "Could not converge to the optimal"
                    msg += " effect-size of one of the candidates."
                    msg += " Setting its effect-size to zero."
                    logging.getLogger(__name__).warning(msg)
                    beta = zeros(self._C[1].shape[0])

            effect_sizes[i] = beta[-1]

            if self._scale is None:
                # _compute_scale(nsamples, beta, self._yTQDiQTy, self._yTQDiQTX, )
                # (nsamples, beta, yTQDiQTy, yTQDiQTX, yTQDiQTm, XTQDiQTX,
                #                    XTQDiQTm, mTQDiQTm)

                p0 = self._yTQDiQTy[0] - 2 * b00m.dot(beta) + beta.dot(
                    self._C[0]).dot(beta)
                p1 = self._yTQDiQTy[1] - 2 * b11m.dot(beta) + beta.dot(
                    self._C[1].dot(beta))

                scale = (p0 + p1) / nsamples
            else:
                scale = self._scale

            lmls[i] -= nsamples * log(max(scale, epsilon.super_tiny))

        lmls /= 2
        return lmls, effect_sizes

    def _fast_scan_chunk_1covariate_loop(self, lmls, effect_sizes, yTQDiQTm,
                                         XTQDiQTm, mTQDiQTm, nsamples):

        C00 = [C[0, 0] for C in self._C]
        C01 = [c[0, :] for c in XTQDiQTm]

        b = [bi[0] for bi in self._yTQDiQTX]

        sC00 = C00[1] - C00[0]
        sC01 = C01[1] - C01[0]
        sC11 = mTQDiQTm[1] - mTQDiQTm[0]

        sb = b[1] - b[0]
        sbm = yTQDiQTm[1] - yTQDiQTm[0]

        with errstate(divide='ignore'):
            beta = beta_1covariate(sb, sbm, sC00, sC01, sC11)

        beta = [nan_to_num(bet) for bet in beta]

        scale = zeros(len(lmls))

        if self._scale is None:
            for i in range(2):
                scale += self._yTQDiQTy[i] - 2 * (
                    b[i] * beta[0] + yTQDiQTm[i] * beta[1])
                scale += beta[0] * (C00[i] * beta[0] + C01[i] * beta[1])
                scale += beta[1] * (C01[i] * beta[0] + mTQDiQTm[i] * beta[1])
            scale /= nsamples
        else:
            scale = self._scale

        lmls -= nsamples * log(clip(scale, epsilon.super_tiny, inf))
        lmls /= 2

        effect_sizes = beta[1]

        return lmls, effect_sizes

    def fast_scan(self, markers, verbose=True):
        r"""LML and fixed-effect sizes of each marker.

        If the scaling factor ``s`` is not set by the user via method
        :func:`set_scale`, its optimal value will be found and
        used in the calculation.

        Parameters
        ----------
        markers : array_like
            Matrix of fixed-effects across columns.
        verbose : bool, optional
            ``True`` for progress information; ``False`` otherwise.
            Defaults to ``True``.

        Returns
        -------
        array_like
            Log of the marginal likelihoods.
        array_like
            Fixed-effect sizes.
        """

        if not (markers.ndim == 2):
            raise ValueError("`markers` array must be bidimensional.")
        p = markers.shape[1]

        lmls = empty(p)
        effect_sizes = empty(p)

        if verbose:
            nchunks = min(p, 30)
        else:
            nchunks = min(p, 1)

        chunk_size = (p + nchunks - 1) // nchunks

        for i in tqdm(range(nchunks), desc="Scanning", disable=not verbose):
            start = i * chunk_size
            stop = min(start + chunk_size, markers.shape[1])

            l, e = self._fast_scan_chunk(markers[:, start:stop])

            lmls[start:stop] = l
            effect_sizes[start:stop] = e

        return lmls, effect_sizes

    def null_lml(self):
        r"""Log of the marginal likelihood.

        .. math::

            - \frac{n}{2}\log{2\pi}
                - \frac{1}{2} n \log{s}
                - \frac{1}{2} \log{\left|\mathrm D\right|}
                    - \frac{1}{2}
                \left(\mathrm Q^{\intercal}\mathbf y -
              \mathrm Q^{\intercal}\mathrm X\boldsymbol\beta\right)^{\intercal}
                    s^{-1}\mathrm D^{-1}
                \left(\mathrm Q^{\intercal}\mathbf y -
                \mathrm Q^{\intercal}\mathrm X\boldsymbol\beta\right)
        """
        n = self._QS[0][0].shape[0]
        p = len(self._D[0])
        static_lml = -n * log2pi - n
        static_lml -= npsum(log(self._D[0]))
        static_lml -= (n - p) * log(self._D[1])
        return static_lml

    def set_scale(self, scale):
        r"""Set the scaling factor.

        Calling this method disables the automatic scale learning.

        Parameters
        ----------
        scale : float
            Scaling factor.
        """
        self._scale = scale

    def unset_scale(self):
        r"""Unset the scaling factor.

        If called, it enables the scale learning again.
        """
        self._scale = None

def beta_1covariate(sb, sbm, sC00, sC01, sC11):
    d0 = sb / sC00
    d1 = sb / sC01

    d3 = sbm / sC01
    d4 = sbm / sC11

    d5 = sC00 / sC01
    d6 = sC11 / sC01

    return [(d1 - d4) / (d5 - 1 / d6), (-d0 + d3) / (d6 - 1 / d5)]

def _compute_scale(nsamples, beta, yTQDiQTy, yTQDiQTX, yTQDiQTm, XTQDiQTX,
                   XTQDiQTm, mTQDiQTm):

    scale = npsum(yTQDiQTy[i] for i in range(2))
    scale -= npsum(2 * yTQDiQTX[i] * beta[0] for i in range(2))
    scale -= npsum(2 * yTQDiQTm[i] * beta[1] for i in range(2))

    scale += npsum(beta[0] * XTQDiQTX[i] * beta[0] for i in range(2))
    scale += npsum(beta[0] * XTQDiQTm[i] * beta[1] for i in range(2))
    scale += npsum(beta[1] * XTQDiQTm[i] * beta[0] for i in range(2))
    scale += npsum(beta[1] * mTQDiQTm[i] * beta[1] for i in range(2))

    return scale / nsamples
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import unittest

from openprocurement.api.tests.base import snitch

from openprocurement.tender.belowthreshold.tests.base import (
    TenderContentWebTest,
    test_bids,
    test_lots,
    test_organization
)
from openprocurement.tender.belowthreshold.tests.award_blanks import (
    # TenderAwardResourceTest
    create_tender_award_invalid,
    create_tender_award,
    patch_tender_award,
    patch_tender_award_unsuccessful,
    get_tender_award,
    patch_tender_award_Administrator_change,
    # TenderLotAwardResourceTest
    create_tender_lot_award,
    patch_tender_lot_award,
    patch_tender_lot_award_unsuccessful,
    # Tender2LotAwardResourceTest
    create_tender_lots_award,
    patch_tender_lots_award,
    # TenderAwardComplaintResourceTest
    create_tender_award_complaint_invalid,
    create_tender_award_complaint,
    patch_tender_award_complaint,
    review_tender_award_complaint,
    get_tender_award_complaint,
    get_tender_award_complaints,
    # TenderLotAwardComplaintResourceTest
    create_tender_lot_award_complaint,
    patch_tender_lot_award_complaint,
    get_tender_lot_award_complaint,
    get_tender_lot_award_complaints,
    # Tender2LotAwardComplaintResourceTest
    create_tender_lots_award_complaint,
    patch_tender_lots_award_complaint,
    # TenderAwardComplaintDocumentResourceTest
    not_found,
    create_tender_award_complaint_document,
    put_tender_award_complaint_document,
    patch_tender_award_complaint_document,
    # Tender2LotAwardComplaintDocumentResourceTest
    create_tender_lots_award_complaint_document,
    put_tender_lots_award_complaint_document,
    patch_tender_lots_award_complaint_document,
    # TenderAwardDocumentResourceTest
    not_found_award_document,
    create_tender_award_document,
    put_tender_award_document,
    patch_tender_award_document,
    # Tender2LotAwardDocumentResourceTest
    create_tender_lots_award_document,
    put_tender_lots_award_document,
    patch_tender_lots_award_document,
)

class TenderAwardResourceTest(TenderContentWebTest):
    initial_status = 'active.qualification'
    initial_bids = test_bids

    test_create_tender_award_invalid = snitch(create_tender_award_invalid)
    test_create_tender_award = snitch(create_tender_award)
    test_patch_tender_award = snitch(patch_tender_award)
    test_patch_tender_award_unsuccessful = snitch(patch_tender_award_unsuccessful)
    test_get_tender_award = snitch(get_tender_award)
    test_patch_tender_award_Administrator_change = snitch(patch_tender_award_Administrator_change)

class TenderLotAwardResourceTest(TenderContentWebTest):
    initial_status = 'active.qualification'
    initial_lots = test_lots
    initial_bids = test_bids

    test_create_tender_lot_award = snitch(create_tender_lot_award)
    test_patch_tender_lot_award = snitch(patch_tender_lot_award)
    test_patch_tender_lot_award_unsuccessful = snitch(patch_tender_lot_award_unsuccessful)

class Tender2LotAwardResourceTest(TenderContentWebTest):
    initial_status = 'active.qualification'
    initial_lots = 2 * test_lots
    initial_bids = test_bids

    test_create_tender_lots_award = snitch(create_tender_lots_award)
    test_patch_tender_lots_award = snitch(patch_tender_lots_award)

class TenderAwardComplaintResourceTest(TenderContentWebTest):
    initial_status = 'active.qualification'
    initial_bids = test_bids

    def setUp(self):
        super(TenderAwardComplaintResourceTest, self).setUp()
        # Create award
        auth = self.app.authorization
        self.app.authorization = ('Basic', ('token', ''))
        response = self.app.post_json('/tenders/{}/awards'.format(
            self.tender_id), {'data': {'suppliers': [test_organization], 'status': 'pending', 'bid_id': self.initial_bids[0]['id']}})
        award = response.json['data']
        self.award_id = award['id']
        self.app.authorization = auth

    test_create_tender_award_complaint_invalid = snitch(create_tender_award_complaint_invalid)
    test_create_tender_award_complaint = snitch(create_tender_award_complaint)
    test_patch_tender_award_complaint = snitch(patch_tender_award_complaint)
    test_review_tender_award_complaint = snitch(review_tender_award_complaint)
    test_get_tender_award_complaint = snitch(get_tender_award_complaint)
    test_get_tender_award_complaints = snitch(get_tender_award_complaints)

class TenderLotAwardComplaintResourceTest(TenderContentWebTest):
    initial_status = 'active.qualification'
    initial_lots = test_lots
    initial_bids = test_bids

    def setUp(self):
        super(TenderLotAwardComplaintResourceTest, self).setUp()
        # Create award
        auth = self.app.authorization
        self.app.authorization = ('Basic', ('token', ''))
        bid = self.initial_bids[0]
        response = self.app.post_json('/tenders/{}/awards'.format(
            self.tender_id), {'data': {'suppliers': [test_organization], 'status': 'pending', 'bid_id': bid['id'], 'lotID': bid['lotValues'][0]['relatedLot']}})
        award = response.json['data']
        self.award_id = award['id']
        self.app.authorization = auth

    test_create_tender_lot_award_complaint = snitch(create_tender_lot_award_complaint)
    test_patch_tender_lot_award_complaint = snitch(patch_tender_lot_award_complaint)
    test_get_tender_lot_award_complaint = snitch(get_tender_lot_award_complaint)
    test_get_tender_lot_award_complaints = snitch(get_tender_lot_award_complaints)

class Tender2LotAwardComplaintResourceTest(TenderLotAwardComplaintResourceTest):
    initial_lots = 2 * test_lots

    test_create_tender_lots_award_complaint = snitch(create_tender_lots_award_complaint)
    test_patch_tender_lots_award_complaint = snitch(patch_tender_lots_award_complaint)

class TenderAwardComplaintDocumentResourceTest(TenderContentWebTest):
    initial_status = 'active.qualification'
    initial_bids = test_bids

    def setUp(self):
        super(TenderAwardComplaintDocumentResourceTest, self).setUp()
        # Create award
        auth = self.app.authorization
        self.app.authorization = ('Basic', ('token', ''))
        response = self.app.post_json('/tenders/{}/awards'.format(
            self.tender_id), {'data': {'suppliers': [test_organization], 'status': 'pending', 'bid_id': self.initial_bids[0]['id']}})
        award = response.json['data']
        self.award_id = award['id']
        self.app.authorization = auth

        # Create complaint for award
        self.bid_token = self.initial_bids_tokens.values()[0]
        response = self.app.post_json('/tenders/{}/awards/{}/complaints?acc_token={}'.format(
            self.tender_id, self.award_id, self.bid_token), {'data': {'title': 'complaint title', 'description': 'complaint description', 'author': test_organization}})
        complaint = response.json['data']
        self.complaint_id = complaint['id']
        self.complaint_owner_token = response.json['access']['token']

    test_not_found = snitch(not_found)
    test_create_tender_award_complaint_document = snitch(create_tender_award_complaint_document)
    test_put_tender_award_complaint_document = snitch(put_tender_award_complaint_document)
    test_patch_tender_award_complaint_document = snitch(patch_tender_award_complaint_document)

class Tender2LotAwardComplaintDocumentResourceTest(TenderContentWebTest):
    initial_status = 'active.qualification'
    initial_bids = test_bids
    initial_lots = 2 * test_lots

    def setUp(self):
        super(Tender2LotAwardComplaintDocumentResourceTest, self).setUp()
        # Create award
        bid = self.initial_bids[0]
        auth = self.app.authorization
        self.app.authorization = ('Basic', ('token', ''))
        response = self.app.post_json('/tenders/{}/awards'.format(
            self.tender_id), {'data': {'suppliers': [test_organization], 'status': 'pending', 'bid_id': bid['id'], 'lotID': bid['lotValues'][0]['relatedLot']}})
        award = response.json['data']
        self.award_id = award['id']
        self.app.authorization = auth
        # Create complaint for award
        bid_token = self.initial_bids_tokens.values()[0]
        response = self.app.post_json('/tenders/{}/awards/{}/complaints?acc_token={}'.format(
            self.tender_id, self.award_id, bid_token), {'data': {'title': 'complaint title', 'description': 'complaint description', 'author': test_organization}})
        complaint = response.json['data']
        self.complaint_id = complaint['id']
        self.complaint_owner_token = response.json['access']['token']

    test_create_tender_lots_award_complaint_document = snitch(create_tender_lots_award_complaint_document)
    test_put_tender_lots_award_complaint_document = snitch(put_tender_lots_award_complaint_document)
    test_patch_tender_lots_award_complaint_document = snitch(patch_tender_lots_award_complaint_document)

class TenderAwardDocumentResourceTest(TenderContentWebTest):
    initial_status = 'active.qualification'
    initial_bids = test_bids

    def setUp(self):
        super(TenderAwardDocumentResourceTest, self).setUp()
        # Create award
        auth = self.app.authorization
        self.app.authorization = ('Basic', ('token', ''))
        response = self.app.post_json('/tenders/{}/awards'.format(
            self.tender_id), {'data': {'suppliers': [test_organization], 'status': 'pending', 'bid_id': self.initial_bids[0]['id']}})
        award = response.json['data']
        self.award_id = award['id']
        self.app.authorization = auth

    test_not_found_award_document = snitch(not_found_award_document)
    test_create_tender_award_document = snitch(create_tender_award_document)
    test_put_tender_award_document = snitch(put_tender_award_document)
    test_patch_tender_award_document = snitch(patch_tender_award_document)

class TenderAwardDocumentWithDSResourceTest(TenderAwardDocumentResourceTest):
    docservice = True

class Tender2LotAwardDocumentResourceTest(TenderContentWebTest):
    initial_status = 'active.qualification'
    initial_bids = test_bids
    initial_lots = 2 * test_lots

    def setUp(self):
        super(Tender2LotAwardDocumentResourceTest, self).setUp()
        # Create award
        auth = self.app.authorization
        self.app.authorization = ('Basic', ('token', ''))
        bid = self.initial_bids[0]
        response = self.app.post_json('/tenders/{}/awards'.format(
            self.tender_id), {'data': {'suppliers': [test_organization], 'status': 'pending', 'bid_id': bid['id'], 'lotID': bid['lotValues'][0]['relatedLot']}})
        award = response.json['data']
        self.award_id = award['id']
        self.app.authorization = auth

    test_create_tender_lots_award_document = snitch(create_tender_lots_award_document)
    test_put_tender_lots_award_document = snitch(put_tender_lots_award_document)
    test_patch_tender_lots_award_document = snitch(patch_tender_lots_award_document)

class Tender2LotAwardDocumentWithDSResourceTest(Tender2LotAwardDocumentResourceTest):
    docservice = True

def suite():
    suite = unittest.TestSuite()
    suite.addTest(unittest.makeSuite(Tender2LotAwardComplaintDocumentResourceTest))
    suite.addTest(unittest.makeSuite(Tender2LotAwardComplaintResourceTest))
    suite.addTest(unittest.makeSuite(Tender2LotAwardDocumentResourceTest))
    suite.addTest(unittest.makeSuite(Tender2LotAwardResourceTest))
    suite.addTest(unittest.makeSuite(TenderAwardComplaintDocumentResourceTest))
    suite.addTest(unittest.makeSuite(TenderAwardComplaintResourceTest))
    suite.addTest(unittest.makeSuite(TenderAwardDocumentResourceTest))
    suite.addTest(unittest.makeSuite(TenderAwardResourceTest))
    suite.addTest(unittest.makeSuite(TenderLotAwardResourceTest))
    return suite

if __name__ == '__main__':
    unittest.main(defaultTest='suite')
<commit_msg>update award tests<commit_after># -*- coding: utf-8 -*-
import unittest

from openprocurement.api.tests.base import snitch

from openprocurement.tender.belowthreshold.tests.base import (
    TenderContentWebTest,
    test_bids,
    test_lots,
    test_organization
)
from openprocurement.tender.belowthreshold.tests.award_blanks import (
    # TenderAwardResourceTest
    create_tender_award_invalid,
    create_tender_award,
    patch_tender_award,
    patch_tender_award_unsuccessful,
    get_tender_award,
    patch_tender_award_Administrator_change,
    # TenderLotAwardResourceTest
    create_tender_lot_award,
    patch_tender_lot_award,
    patch_tender_lot_award_unsuccessful,
    # Tender2LotAwardResourceTest
    create_tender_lots_award,
    patch_tender_lots_award,
    # TenderAwardComplaintResourceTest
    create_tender_award_complaint_invalid,
    create_tender_award_complaint,
    patch_tender_award_complaint,
    review_tender_award_complaint,
    get_tender_award_complaint,
    get_tender_award_complaints,
    # TenderLotAwardComplaintResourceTest
    create_tender_lot_award_complaint,
    patch_tender_lot_award_complaint,
    get_tender_lot_award_complaint,
    get_tender_lot_award_complaints,
    # Tender2LotAwardComplaintResourceTest
    create_tender_lots_award_complaint,
    patch_tender_lots_award_complaint,
    # TenderAwardComplaintDocumentResourceTest
    not_found,
    create_tender_award_complaint_document,
    put_tender_award_complaint_document,
    patch_tender_award_complaint_document,
    # Tender2LotAwardComplaintDocumentResourceTest
    create_tender_lots_award_complaint_document,
    put_tender_lots_award_complaint_document,
    patch_tender_lots_award_complaint_document,
    # TenderAwardDocumentResourceTest
    not_found_award_document,
    create_tender_award_document,
    put_tender_award_document,
    patch_tender_award_document,
    # Tender2LotAwardDocumentResourceTest
    create_tender_lots_award_document,
    put_tender_lots_award_document,
    patch_tender_lots_award_document,
)

class TenderAwardResourceTest(TenderContentWebTest):
    initial_status = 'active.qualification'
    initial_bids = test_bids

    test_create_tender_award_invalid = snitch(create_tender_award_invalid)
    test_create_tender_award = snitch(create_tender_award)
    test_patch_tender_award = snitch(patch_tender_award)
    test_patch_tender_award_unsuccessful = snitch(patch_tender_award_unsuccessful)
    test_get_tender_award = snitch(get_tender_award)
    test_patch_tender_award_Administrator_change = snitch(patch_tender_award_Administrator_change)

class TenderLotAwardResourceTest(TenderContentWebTest):
    initial_status = 'active.qualification'
    initial_lots = test_lots
    initial_bids = test_bids

    test_create_tender_lot_award = snitch(create_tender_lot_award)
    test_patch_tender_lot_award = snitch(patch_tender_lot_award)
    test_patch_tender_lot_award_unsuccessful = snitch(patch_tender_lot_award_unsuccessful)

class Tender2LotAwardResourceTest(TenderContentWebTest):
    initial_status = 'active.qualification'
    initial_lots = 2 * test_lots
    initial_bids = test_bids

    test_create_tender_lots_award = snitch(create_tender_lots_award)
    test_patch_tender_lots_award = snitch(patch_tender_lots_award)

class TenderAwardComplaintResourceTest(TenderContentWebTest):
    initial_status = 'active.qualification'
    initial_bids = test_bids
    test_complaint_status = 'claim'

    def setUp(self):
        super(TenderAwardComplaintResourceTest, self).setUp()
        # Create award
        auth = self.app.authorization
        self.app.authorization = ('Basic', ('token', ''))
        response = self.app.post_json('/tenders/{}/awards'.format(
            self.tender_id), {'data': {'suppliers': [test_organization], 'status': 'pending', 'bid_id': self.initial_bids[0]['id']}})
        award = response.json['data']
        self.award_id = award['id']
        self.app.authorization = auth

    test_create_tender_award_complaint_invalid = snitch(create_tender_award_complaint_invalid)
    test_create_tender_award_complaint = snitch(create_tender_award_complaint)
    test_patch_tender_award_complaint = snitch(patch_tender_award_complaint)
    test_review_tender_award_complaint = snitch(review_tender_award_complaint)
    test_get_tender_award_complaint = snitch(get_tender_award_complaint)
    test_get_tender_award_complaints = snitch(get_tender_award_complaints)

class TenderLotAwardComplaintResourceTest(TenderContentWebTest):
    initial_status = 'active.qualification'
    initial_lots = test_lots
    initial_bids = test_bids

    def setUp(self):
        super(TenderLotAwardComplaintResourceTest, self).setUp()
        # Create award
        auth = self.app.authorization
        self.app.authorization = ('Basic', ('token', ''))
        bid = self.initial_bids[0]
        response = self.app.post_json('/tenders/{}/awards'.format(
            self.tender_id), {'data': {'suppliers': [test_organization], 'status': 'pending', 'bid_id': bid['id'], 'lotID': bid['lotValues'][0]['relatedLot']}})
        award = response.json['data']
        self.award_id = award['id']
        self.app.authorization = auth

    test_create_tender_lot_award_complaint = snitch(create_tender_lot_award_complaint)
    test_patch_tender_lot_award_complaint = snitch(patch_tender_lot_award_complaint)
    test_get_tender_lot_award_complaint = snitch(get_tender_lot_award_complaint)
    test_get_tender_lot_award_complaints = snitch(get_tender_lot_award_complaints)

class Tender2LotAwardComplaintResourceTest(TenderLotAwardComplaintResourceTest):
    initial_lots = 2 * test_lots

    test_create_tender_lots_award_complaint = snitch(create_tender_lots_award_complaint)
    test_patch_tender_lots_award_complaint = snitch(patch_tender_lots_award_complaint)

class TenderAwardComplaintDocumentResourceTest(TenderContentWebTest):
    initial_status = 'active.qualification'
    initial_bids = test_bids

    def setUp(self):
        super(TenderAwardComplaintDocumentResourceTest, self).setUp()
        # Create award
        auth = self.app.authorization
        self.app.authorization = ('Basic', ('token', ''))
        response = self.app.post_json('/tenders/{}/awards'.format(
            self.tender_id), {'data': {'suppliers': [test_organization], 'status': 'pending', 'bid_id': self.initial_bids[0]['id']}})
        award = response.json['data']
        self.award_id = award['id']
        self.app.authorization = auth

        # Create complaint for award
        self.bid_token = self.initial_bids_tokens.values()[0]
        response = self.app.post_json('/tenders/{}/awards/{}/complaints?acc_token={}'.format(
            self.tender_id, self.award_id, self.bid_token), {'data': {'title': 'complaint title', 'description': 'complaint description', 'author': test_organization}})
        complaint = response.json['data']
        self.complaint_id = complaint['id']
        self.complaint_owner_token = response.json['access']['token']

    test_not_found = snitch(not_found)
    test_create_tender_award_complaint_document = snitch(create_tender_award_complaint_document)
    test_put_tender_award_complaint_document = snitch(put_tender_award_complaint_document)
    test_patch_tender_award_complaint_document = snitch(patch_tender_award_complaint_document)

class Tender2LotAwardComplaintDocumentResourceTest(TenderContentWebTest):
    initial_status = 'active.qualification'
    initial_bids = test_bids
    initial_lots = 2 * test_lots

    def setUp(self):
        super(Tender2LotAwardComplaintDocumentResourceTest, self).setUp()
        # Create award
        bid = self.initial_bids[0]
        auth = self.app.authorization
        self.app.authorization = ('Basic', ('token', ''))
        response = self.app.post_json('/tenders/{}/awards'.format(
            self.tender_id), {'data': {'suppliers': [test_organization], 'status': 'pending', 'bid_id': bid['id'], 'lotID': bid['lotValues'][0]['relatedLot']}})
        award = response.json['data']
        self.award_id = award['id']
        self.app.authorization = auth
        # Create complaint for award
        bid_token = self.initial_bids_tokens.values()[0]
        response = self.app.post_json('/tenders/{}/awards/{}/complaints?acc_token={}'.format(
            self.tender_id, self.award_id, bid_token), {'data': {'title': 'complaint title', 'description': 'complaint description', 'author': test_organization}})
        complaint = response.json['data']
        self.complaint_id = complaint['id']
        self.complaint_owner_token = response.json['access']['token']

    test_create_tender_lots_award_complaint_document = snitch(create_tender_lots_award_complaint_document)
    test_put_tender_lots_award_complaint_document = snitch(put_tender_lots_award_complaint_document)
    test_patch_tender_lots_award_complaint_document = snitch(patch_tender_lots_award_complaint_document)

class TenderAwardDocumentResourceTest(TenderContentWebTest):
    initial_status = 'active.qualification'
    initial_bids = test_bids

    def setUp(self):
        super(TenderAwardDocumentResourceTest, self).setUp()
        # Create award
        auth = self.app.authorization
        self.app.authorization = ('Basic', ('token', ''))
        response = self.app.post_json('/tenders/{}/awards'.format(
            self.tender_id), {'data': {'suppliers': [test_organization], 'status': 'pending', 'bid_id': self.initial_bids[0]['id']}})
        award = response.json['data']
        self.award_id = award['id']
        self.app.authorization = auth

    test_not_found_award_document = snitch(not_found_award_document)
    test_create_tender_award_document = snitch(create_tender_award_document)
    test_put_tender_award_document = snitch(put_tender_award_document)
    test_patch_tender_award_document = snitch(patch_tender_award_document)

class TenderAwardDocumentWithDSResourceTest(TenderAwardDocumentResourceTest):
    docservice = True

class Tender2LotAwardDocumentResourceTest(TenderContentWebTest):
    initial_status = 'active.qualification'
    initial_bids = test_bids
    initial_lots = 2 * test_lots

    def setUp(self):
        super(Tender2LotAwardDocumentResourceTest, self).setUp()
        # Create award
        auth = self.app.authorization
        self.app.authorization = ('Basic', ('token', ''))
        bid = self.initial_bids[0]
        response = self.app.post_json('/tenders/{}/awards'.format(
            self.tender_id), {'data': {'suppliers': [test_organization], 'status': 'pending', 'bid_id': bid['id'], 'lotID': bid['lotValues'][0]['relatedLot']}})
        award = response.json['data']
        self.award_id = award['id']
        self.app.authorization = auth

    test_create_tender_lots_award_document = snitch(create_tender_lots_award_document)
    test_put_tender_lots_award_document = snitch(put_tender_lots_award_document)
    test_patch_tender_lots_award_document = snitch(patch_tender_lots_award_document)

class Tender2LotAwardDocumentWithDSResourceTest(Tender2LotAwardDocumentResourceTest):
    docservice = True

def suite():
    suite = unittest.TestSuite()
    suite.addTest(unittest.makeSuite(Tender2LotAwardComplaintDocumentResourceTest))
    suite.addTest(unittest.makeSuite(Tender2LotAwardComplaintResourceTest))
    suite.addTest(unittest.makeSuite(Tender2LotAwardDocumentResourceTest))
    suite.addTest(unittest.makeSuite(Tender2LotAwardResourceTest))
    suite.addTest(unittest.makeSuite(TenderAwardComplaintDocumentResourceTest))
    suite.addTest(unittest.makeSuite(TenderAwardComplaintResourceTest))
    suite.addTest(unittest.makeSuite(TenderAwardDocumentResourceTest))
    suite.addTest(unittest.makeSuite(TenderAwardResourceTest))
    suite.addTest(unittest.makeSuite(TenderLotAwardResourceTest))
    return suite

if __name__ == '__main__':
    unittest.main(defaultTest='suite')
<|endoftext|>


<commit_before>#
# Copyright 2018 Analytics Zoo Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import ray
from ray.util.sgd.torch import TorchTrainer

class PyTorchTrainer(object):
    def __init__(self, model_creator, data_creator, optimizer_creator,
                 loss_creator=None, scheduler_creator=None, training_operator_cls=None,
                 initialization_hook=None, config=None, num_workers=1,
                 use_fp16=False, use_tqdm=False, scheduler_step_freq="batch"):
        # Lift TorchTrainer to an Actor so that its local worker would be
        # created on the cluster as well.
        RemoteTrainer = ray.remote(TorchTrainer)
        self.trainer = RemoteTrainer.remote(model_creator=model_creator,
                                            data_creator=data_creator,
                                            optimizer_creator=optimizer_creator,
                                            loss_creator=loss_creator,
                                            scheduler_creator=scheduler_creator,
                                            training_operator_cls=training_operator_cls,
                                            initialization_hook=initialization_hook,
                                            config=config,
                                            num_workers=num_workers,
                                            backend="gloo",
                                            use_fp16=use_fp16,
                                            use_tqdm=use_tqdm,
                                            scheduler_step_freq=scheduler_step_freq)

    def train(self, nb_epoch=1):
        """Trains a PyTorch model for several epochs."""
        stats_list = list()
        for i in range(nb_epoch):
            stats = ray.get(self.trainer.train.remote())
            stats_list.append(stats)
        return stats_list

    def validate(self, num_steps=None, profile=False, reduce_results=True, info=None):
        return ray.get(self.trainer.validate.remote(num_steps, profile, reduce_results, info))

    def get_model(self):
        """Returns the learned model(s)."""
        return ray.get(self.trainer.get_model.remote())

    def shutdown(self, force=False):
        ray.get(self.trainer.shutdown.remote(force))
<commit_msg>change default value of num workers in torch trainer (#2786)<commit_after>#
# Copyright 2018 Analytics Zoo Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import ray
from ray.util.sgd.torch import TorchTrainer
from zoo.ray import RayContext

class PyTorchTrainer(object):
    def __init__(self, model_creator, data_creator, optimizer_creator,
                 loss_creator=None, scheduler_creator=None, training_operator_cls=None,
                 initialization_hook=None, config=None, num_workers=None,
                 use_fp16=False, use_tqdm=False, scheduler_step_freq="batch"):
        # Lift TorchTrainer to an Actor so that its local worker would be
        # created on the cluster as well.
        RemoteTrainer = ray.remote(TorchTrainer)

        # check whether there has been an active RayContext and get it.
        ray_ctx = RayContext.get()
        if not num_workers:
            num_workers = ray_ctx.num_ray_nodes
        self.trainer = RemoteTrainer.remote(model_creator=model_creator,
                                            data_creator=data_creator,
                                            optimizer_creator=optimizer_creator,
                                            loss_creator=loss_creator,
                                            scheduler_creator=scheduler_creator,
                                            training_operator_cls=training_operator_cls,
                                            initialization_hook=initialization_hook,
                                            config=config,
                                            num_workers=num_workers,
                                            backend="gloo",
                                            use_fp16=use_fp16,
                                            use_tqdm=use_tqdm,
                                            scheduler_step_freq=scheduler_step_freq)

    def train(self, nb_epoch=1):
        """Trains a PyTorch model for several epochs."""
        stats_list = list()
        for i in range(nb_epoch):
            stats = ray.get(self.trainer.train.remote())
            stats_list.append(stats)
        return stats_list

    def validate(self, num_steps=None, profile=False, reduce_results=True, info=None):
        return ray.get(self.trainer.validate.remote(num_steps, profile, reduce_results, info))

    def get_model(self):
        """Returns the learned model(s)."""
        return ray.get(self.trainer.get_model.remote())

    def shutdown(self, force=False):
        ray.get(self.trainer.shutdown.remote(force))
<|endoftext|>


<commit_before># coding=utf-8
# Copyright 2020 The TensorFlow Datasets Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python3
"""Sequence feature."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import tensorflow.compat.v2 as tf

from tensorflow_datasets.core import utils
from tensorflow_datasets.core.features import feature as feature_lib
from tensorflow_datasets.core.features import features_dict
from tensorflow_datasets.core.features import top_level_feature

class Sequence(top_level_feature.TopLevelFeature):
  """Composite `FeatureConnector` for a `dict` where each value is a list.

  `Sequence` correspond to sequence of `tfds.features.FeatureConnector`. At
  generation time, a list for each of the sequence element is given. The output
  of `tf.data.Dataset` will batch all the elements of the sequence together.

  If the length of the sequence is static and known in advance, it should be
  specified in the constructor using the `length` param.

  Note that `Sequence` does not support features which are of type
  `tf.io.FixedLenSequenceFeature`.

  Example:
  At construction time:

  ```
  tfds.features.Sequence(tfds.features.Image(), length=NB_FRAME)
  ```

  or:

  ```
  tfds.features.Sequence({
      'frame': tfds.features.Image(shape=(64, 64, 3))
      'action': tfds.features.ClassLabel(['up', 'down', 'left', 'right'])
  }, length=NB_FRAME)
  ```

  During data generation:

  ```
  yield {
      'frame': np.ones(shape=(NB_FRAME, 64, 64, 3)),
      'action': ['left', 'left', 'up', ...],
  }
  ```

  Tensor returned by `.as_dataset()`:

  ```
  {
      'frame': tf.Tensor(shape=(NB_FRAME, 64, 64, 3), dtype=tf.uint8),
      'action': tf.Tensor(shape=(NB_FRAME,), dtype=tf.int64),
  }
  ```

  At generation time, you can specify a list of features dict, a dict of list
  values or a stacked numpy array. The lists will automatically be distributed
  into their corresponding `FeatureConnector`.

  """

  def __init__(self, feature, length=None, **kwargs):
    """Construct a sequence dict.

    Args:
      feature: `dict`, the features to wrap
      length: `int`, length of the sequence if static and known in advance
      **kwargs: `dict`, constructor kwargs of `tfds.features.FeaturesDict`
    """
    # Convert {} => FeaturesDict, tf.int32 => Tensor(shape=(), dtype=tf.int32)
    self._feature = features_dict.to_feature(feature)
    self._length = length
    super(Sequence, self).__init__(**kwargs)

  @property
  def feature(self):
    """The inner feature."""
    return self._feature

  def _add_length_dim(self, tensor_info):
    """Add the length dimension to the given tensor_info."""
    tensor_info = feature_lib.TensorInfo.copy_from(tensor_info)
    tensor_info.shape = (self._length,) + tensor_info.shape
    tensor_info.sequence_rank += 1
    return tensor_info

  def get_tensor_info(self):
    """See base class for details."""
    # Add the additional length dimension to every shape
    tensor_info = self._feature.get_tensor_info()
    return tf.nest.map_structure(self._add_length_dim, tensor_info)

  def get_serialized_info(self):
    """See base class for details."""
    # Add the additional length dimension to every serialized features
    tensor_info = self._feature.get_serialized_info()
    return tf.nest.map_structure(self._add_length_dim, tensor_info)

  def encode_example(self, example_dict):
    # Convert nested dict[list] into list[nested dict]
    sequence_elements = _transpose_dict_list(example_dict)

    # If length is static, ensure that the given length match
    if self._length is not None and len(sequence_elements) != self._length:
      raise ValueError(
          'Input sequence length do not match the defined one. Got {} != '
          '{}'.format(len(sequence_elements), self._length)
      )

    # Empty sequences return empty arrays
    if not sequence_elements:
      def _build_empty_np(serialized_info):
        return np.empty(
            shape=tuple(s if s else 0 for s in serialized_info.shape),
            dtype=serialized_info.dtype.as_numpy_dtype,
        )

      return tf.nest.map_structure(_build_empty_np, self.get_serialized_info())

    # Encode each individual elements
    sequence_elements = [
        self.feature.encode_example(sequence_elem)
        for sequence_elem in sequence_elements
    ]

    # Then convert back list[nested dict] => nested dict[list]
    def _stack_nested(sequence_elements):
      """Recursivelly stack the tensors from the same dict field."""
      if isinstance(sequence_elements[0], dict):
        return {
            # Stack along the first dimension
            k: _stack_nested(sub_sequence)
            for k, sub_sequence in utils.zip_dict(*sequence_elements)
        }
      # Note: As each field can be a nested ragged list, we don't check here
      # that all elements from the list have matching dtype/shape.
      # Checking is done in `example_serializer` when elements
      # are converted to numpy array and stacked togethers.
      return list(sequence_elements)

    return _stack_nested(sequence_elements)

  def _flatten(self, x):
    """See base class for details."""
    if isinstance(x, Sequence):
      return self.feature._flatten(x.feature)  # pylint: disable=protected-access
    return self.feature._flatten(x)  # pylint: disable=protected-access

  def _nest(self, list_x):
    """See base class for details."""
    return self.feature._nest(list_x)  # pylint: disable=protected-access

  def save_metadata(self, *args, **kwargs):
    """See base class for details."""
    self._feature.save_metadata(*args, **kwargs)

  def load_metadata(self, *args, **kwargs):
    """See base class for details."""
    self._feature.load_metadata(*args, **kwargs)

  def __getitem__(self, key):
    """Convenience method to access the underlying features."""
    return self._feature[key]

  def __getattr__(self, key):
    """Allow to access the underlying attributes directly."""
    return getattr(self._feature, key)

  # The __getattr__ method triggers an infinite recursion loop when loading a
  # pickled instance. So we override that name in the instance dict, and remove
  # it when unplickling.
  def __getstate__(self):
    state = self.__dict__.copy()
    state['__getattr__'] = 0
    return state

  def __setstate__(self, state):
    del state['__getattr__']
    self.__dict__.update(state)

  def __repr__(self):
    """Display the feature."""
    inner_feature_repr = feature_lib.get_inner_feature_repr(self._feature)
    if inner_feature_repr.startswith('FeaturesDict('):
      # Minor formatting cleaning: 'Sequence(FeaturesDict({' => 'Sequence({'
      inner_feature_repr = inner_feature_repr[len('FeaturesDict('):-len(')')]
    return '{}({})'.format(type(self).__name__, inner_feature_repr)

  @classmethod
  def from_json_content(cls, value) -> 'FeatureConnector':
    if isinstance(value, list):
      assert len(value) == 1
      subclass = cls._REGISTERED_FEATURES.get(value[0]['type'])
      return cls(subclass.from_json_content(value[0]['content']))
    features = dict()
    for feature_key, feature_value in value.items():
      if feature_key == 'type' and feature_value == 'Tensor':
        subclass = cls._REGISTERED_FEATURES.get(feature_value)
        return cls(subclass)
      subclass = cls._REGISTERED_FEATURES.get(feature_value['type'])
      features.update(
          {feature_key: subclass.from_json_content(feature_value['content'])})
    return cls(features)

  def to_json_content(self):
    if isinstance(self.feature, features_dict.FeaturesDict):
      return {k: v.to_json() for k, v in self.feature.items()}
    return [self.feature.to_json()]

def _np_to_list(elem):
  """Returns list from list, tuple or ndarray."""
  if isinstance(elem, list):
    return elem
  elif isinstance(elem, tuple):
    return list(elem)
  elif isinstance(elem, np.ndarray):
    return list(elem)
  else:
    raise ValueError(
        'Input elements of a sequence should be either a numpy array, a '
        'python list or tuple. Got {}'.format(type(elem)))

def _transpose_dict_list(dict_list):
  """Transpose a nested dict[list] into a list[nested dict]."""
  # 1. Unstack numpy arrays into list
  dict_list = utils.map_nested(_np_to_list, dict_list, dict_only=True)

  # 2. Extract the sequence length (and ensure the length is constant for all
  # elements)
  length = {'value': None}  # dict because `nonlocal` is Python3 only

  def update_length(elem):
    if length['value'] is None:
      length['value'] = len(elem)
    elif length['value'] != len(elem):
      raise ValueError(
          'The length of all elements of one sequence should be the same. '
          'Got {} != {}'.format(length['value'], len(elem)))
    return elem
  utils.map_nested(update_length, dict_list, dict_only=True)

  # 3. Extract each individual elements
  return [
      utils.map_nested(
          lambda elem: elem[i], dict_list, dict_only=True)   # pylint: disable=cell-var-from-loop
      for i in range(length['value'])  # pytype: disable=wrong-arg-types
  ]
<commit_msg>Minor Cleanup<commit_after># coding=utf-8
# Copyright 2020 The TensorFlow Datasets Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Lint as: python3
"""Sequence feature."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import tensorflow.compat.v2 as tf

from tensorflow_datasets.core import utils
from tensorflow_datasets.core.features import feature as feature_lib
from tensorflow_datasets.core.features import features_dict
from tensorflow_datasets.core.features import top_level_feature

class Sequence(top_level_feature.TopLevelFeature):
  """Composite `FeatureConnector` for a `dict` where each value is a list.

  `Sequence` correspond to sequence of `tfds.features.FeatureConnector`. At
  generation time, a list for each of the sequence element is given. The output
  of `tf.data.Dataset` will batch all the elements of the sequence together.

  If the length of the sequence is static and known in advance, it should be
  specified in the constructor using the `length` param.

  Note that `Sequence` does not support features which are of type
  `tf.io.FixedLenSequenceFeature`.

  Example:
  At construction time:

  ```
  tfds.features.Sequence(tfds.features.Image(), length=NB_FRAME)
  ```

  or:

  ```
  tfds.features.Sequence({
      'frame': tfds.features.Image(shape=(64, 64, 3))
      'action': tfds.features.ClassLabel(['up', 'down', 'left', 'right'])
  }, length=NB_FRAME)
  ```

  During data generation:

  ```
  yield {
      'frame': np.ones(shape=(NB_FRAME, 64, 64, 3)),
      'action': ['left', 'left', 'up', ...],
  }
  ```

  Tensor returned by `.as_dataset()`:

  ```
  {
      'frame': tf.Tensor(shape=(NB_FRAME, 64, 64, 3), dtype=tf.uint8),
      'action': tf.Tensor(shape=(NB_FRAME,), dtype=tf.int64),
  }
  ```

  At generation time, you can specify a list of features dict, a dict of list
  values or a stacked numpy array. The lists will automatically be distributed
  into their corresponding `FeatureConnector`.

  """

  def __init__(self, feature, length=None, **kwargs):
    """Construct a sequence dict.

    Args:
      feature: `dict`, the features to wrap
      length: `int`, length of the sequence if static and known in advance
      **kwargs: `dict`, constructor kwargs of `tfds.features.FeaturesDict`
    """
    # Convert {} => FeaturesDict, tf.int32 => Tensor(shape=(), dtype=tf.int32)
    self._feature = features_dict.to_feature(feature)
    self._length = length
    super(Sequence, self).__init__(**kwargs)

  @property
  def feature(self):
    """The inner feature."""
    return self._feature

  def _add_length_dim(self, tensor_info):
    """Add the length dimension to the given tensor_info."""
    tensor_info = feature_lib.TensorInfo.copy_from(tensor_info)
    tensor_info.shape = (self._length,) + tensor_info.shape
    tensor_info.sequence_rank += 1
    return tensor_info

  def get_tensor_info(self):
    """See base class for details."""
    # Add the additional length dimension to every shape
    tensor_info = self._feature.get_tensor_info()
    return tf.nest.map_structure(self._add_length_dim, tensor_info)

  def get_serialized_info(self):
    """See base class for details."""
    # Add the additional length dimension to every serialized features
    tensor_info = self._feature.get_serialized_info()
    return tf.nest.map_structure(self._add_length_dim, tensor_info)

  def encode_example(self, example_dict):
    # Convert nested dict[list] into list[nested dict]
    sequence_elements = _transpose_dict_list(example_dict)

    # If length is static, ensure that the given length match
    if self._length is not None and len(sequence_elements) != self._length:
      raise ValueError(
          'Input sequence length do not match the defined one. Got {} != '
          '{}'.format(len(sequence_elements), self._length)
      )

    # Empty sequences return empty arrays
    if not sequence_elements:
      def _build_empty_np(serialized_info):
        return np.empty(
            shape=tuple(s if s else 0 for s in serialized_info.shape),
            dtype=serialized_info.dtype.as_numpy_dtype,
        )

      return tf.nest.map_structure(_build_empty_np, self.get_serialized_info())

    # Encode each individual elements
    sequence_elements = [
        self.feature.encode_example(sequence_elem)
        for sequence_elem in sequence_elements
    ]

    # Then convert back list[nested dict] => nested dict[list]
    def _stack_nested(sequence_elements):
      """Recursivelly stack the tensors from the same dict field."""
      if isinstance(sequence_elements[0], dict):
        return {
            # Stack along the first dimension
            k: _stack_nested(sub_sequence)
            for k, sub_sequence in utils.zip_dict(*sequence_elements)
        }
      # Note: As each field can be a nested ragged list, we don't check here
      # that all elements from the list have matching dtype/shape.
      # Checking is done in `example_serializer` when elements
      # are converted to numpy array and stacked togethers.
      return list(sequence_elements)

    return _stack_nested(sequence_elements)

  def _flatten(self, x):
    """See base class for details."""
    if isinstance(x, Sequence):
      return self.feature._flatten(x.feature)  # pylint: disable=protected-access
    return self.feature._flatten(x)  # pylint: disable=protected-access

  def _nest(self, list_x):
    """See base class for details."""
    return self.feature._nest(list_x)  # pylint: disable=protected-access

  def save_metadata(self, *args, **kwargs):
    """See base class for details."""
    self._feature.save_metadata(*args, **kwargs)

  def load_metadata(self, *args, **kwargs):
    """See base class for details."""
    self._feature.load_metadata(*args, **kwargs)

  def __getitem__(self, key):
    """Convenience method to access the underlying features."""
    return self._feature[key]

  def __getattr__(self, key):
    """Allow to access the underlying attributes directly."""
    return getattr(self._feature, key)

  # The __getattr__ method triggers an infinite recursion loop when loading a
  # pickled instance. So we override that name in the instance dict, and remove
  # it when unplickling.
  def __getstate__(self):
    state = self.__dict__.copy()
    state['__getattr__'] = 0
    return state

  def __setstate__(self, state):
    del state['__getattr__']
    self.__dict__.update(state)

  def __repr__(self):
    """Display the feature."""
    inner_feature_repr = feature_lib.get_inner_feature_repr(self._feature)
    if inner_feature_repr.startswith('FeaturesDict('):
      # Minor formatting cleaning: 'Sequence(FeaturesDict({' => 'Sequence({'
      inner_feature_repr = inner_feature_repr[len('FeaturesDict('):-len(')')]
    return '{}({})'.format(type(self).__name__, inner_feature_repr)

  @classmethod
  def from_json_content(cls, value) -> 'FeatureConnector':
    if value.get('arg', None) == 'feature':
      subclass = cls._REGISTERED_FEATURES.get(value['type'])
      return cls(subclass.from_json_content(value['content']))
    features = dict()
    for feature_key, feature_value in value.items():
      subclass = cls._REGISTERED_FEATURES.get(feature_value['type'])
      features.update(
          {feature_key: subclass.from_json_content(feature_value['content'])})
    return cls(features)

  def to_json_content(self):
    if isinstance(self.feature, features_dict.FeaturesDict):
      return {k: v.to_json() for k, v in self.feature.items()}
    value = self.feature.to_json()
    value['arg'] = 'feature'
    return value

def _np_to_list(elem):
  """Returns list from list, tuple or ndarray."""
  if isinstance(elem, list):
    return elem
  elif isinstance(elem, tuple):
    return list(elem)
  elif isinstance(elem, np.ndarray):
    return list(elem)
  else:
    raise ValueError(
        'Input elements of a sequence should be either a numpy array, a '
        'python list or tuple. Got {}'.format(type(elem)))

def _transpose_dict_list(dict_list):
  """Transpose a nested dict[list] into a list[nested dict]."""
  # 1. Unstack numpy arrays into list
  dict_list = utils.map_nested(_np_to_list, dict_list, dict_only=True)

  # 2. Extract the sequence length (and ensure the length is constant for all
  # elements)
  length = {'value': None}  # dict because `nonlocal` is Python3 only

  def update_length(elem):
    if length['value'] is None:
      length['value'] = len(elem)
    elif length['value'] != len(elem):
      raise ValueError(
          'The length of all elements of one sequence should be the same. '
          'Got {} != {}'.format(length['value'], len(elem)))
    return elem
  utils.map_nested(update_length, dict_list, dict_only=True)

  # 3. Extract each individual elements
  return [
      utils.map_nested(
          lambda elem: elem[i], dict_list, dict_only=True)   # pylint: disable=cell-var-from-loop
      for i in range(length['value'])  # pytype: disable=wrong-arg-types
  ]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import os
import sys
import mock
import unittest
import numpy
import h5py
from nose.plugins.attrib import attr

from openquake.baselib.general import writetmp
from openquake.baselib.parallel import Sequential
from openquake.calculators.views import view
from openquake.calculators.tests import (
    CalculatorTestCase, strip_calc_id, REFERENCE_OS)
from openquake.calculators.export import export
from openquake.qa_tests_data.event_based_risk import (
    case_1, case_2, case_3, case_4, case_4a, case_master, case_miriam,
    occupants, case_1g, case_7a)

# used for a sanity check
def check_total_losses(calc):
    dstore = calc.datastore
    loss_dt = calc.oqparam.loss_dt()
    L1 = len(loss_dt.names)
    L = L1 // 2
    data1 = numpy.zeros(L1, numpy.float32)
    for dset in dstore['agg_loss_table'].values():
        for l, lt in enumerate(loss_dt.names):
            i = lt.endswith('_ins')
            data1[l] += dset['loss'][:, l - L * i, i].sum()

    # check the sums are consistent with the ones coming from losses_by_taxon
    data2 = numpy.zeros(L1, numpy.float32)
    lbt = dstore['losses_by_taxon-rlzs']
    for l in range(L1):
        data2[l] += lbt[:, :, l].sum()
    numpy.testing.assert_allclose(data1, data2, 1E-6)

    # test the asset_loss_table exporter; notice that I need to disable
    # the parallelism to avoid reading bogus data: this is the usual
    # heisenbug when reading in parallel an .hdf5 generated in process
    with mock.patch('openquake.baselib.parallel.Starmap', Sequential):
        [fname] = export(('asset_loss_table', 'hdf5'), dstore)
    print('Generating %s' % fname)
    with h5py.File(fname) as f:
        total = f['asset_loss_table'].attrs['total']

    # check the sums are consistent with the ones coming from asset_loss_table
    numpy.testing.assert_allclose(data1, total, 1E-6)

class EventBasedRiskTestCase(CalculatorTestCase):

    def assert_stats_ok(self, pkg, job_ini):
        out = self.run_calc(pkg.__file__, job_ini, exports='csv',
                            concurrent_tasks='4')
        # NB: it is important to use concurrent_tasks > 1 to test the
        # complications of concurrency (for instance the noncommutativity of
        # numpy.float32 addition when computing the average losses)
        all_csv = []
        for fnames in out.values():
            for fname in fnames:
                if 'rlz' in fname:
                    continue
                elif fname.endswith('.csv') and any(x in fname for x in (
                        'loss_curve', 'loss_map', 'agg_loss', 'avg_loss')):
                    all_csv.append(fname)
        assert all_csv, 'Could not find any CSV file??'
        for fname in all_csv:
            self.assertEqualFiles(
                'expected/%s' % strip_calc_id(fname), fname)

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_1(self):
        self.run_calc(case_1.__file__, 'job.ini')
        ekeys = [('agg_curve-stats', 'xml')]
        for ekey in ekeys:
            for fname in export(ekey, self.calc.datastore):
                self.assertEqualFiles(
                    'expected/%s' % strip_calc_id(fname), fname)

        fnames = export(('loss_maps-stats', 'csv'), self.calc.datastore)
        assert fnames
        if REFERENCE_OS:
            for fname in fnames:
                self.assertEqualFiles('expected/' + strip_calc_id(fname),
                                      fname, delta=1E-5)

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_1g(self):
        # vulnerability function with PMF
        self.run_calc(case_1g.__file__, 'job.ini')
        fname = writetmp(view('mean_avg_losses', self.calc.datastore))
        self.assertEqualFiles('expected/avg_losses.txt', fname)
        os.remove(fname)

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_2(self):
        self.run_calc(case_2.__file__, 'job.ini')
        fname = writetmp(view('mean_avg_losses', self.calc.datastore))
        self.assertEqualFiles('expected/mean_avg_losses.txt', fname)
        os.remove(fname)

        # test the composite_risk_model keys (i.e. slash escaping)
        crm = sorted(self.calc.datastore.getitem('composite_risk_model'))
        self.assertEqual(crm, ['RC%2B', 'RM', 'W%2F1'])

        # test the case when all GMFs are filtered out
        with self.assertRaises(RuntimeError) as ctx:
            self.run_calc(case_2.__file__, 'job.ini', minimum_intensity='10.0')
        self.assertEqual(
            str(ctx.exception),
            'No GMFs were generated, perhaps they were all below the '
            'minimum_intensity threshold')

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_2_correlation(self):
        out = self.run_calc(case_2.__file__, 'job_loss.ini', exports='csv',
                            asset_correlation=1.0)
        [fname] = out['agg_loss_table', 'csv']
        self.assertEqualFiles('expected/agg_losses.csv', fname)

        check_total_losses(self.calc)

    @attr('qa', 'risk', 'event_based_risk')
    def test_missing_taxonomy(self):
        with self.assertRaises(RuntimeError) as ctx:
            self.run_calc(case_2.__file__, 'job_err.ini')
        self.assertIn('not in the risk model', str(ctx.exception))

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_3(self):
        # this is a test with statistics and without conditional_loss_poes
        self.run_calc(case_3.__file__, 'job.ini',
                      exports='xml', concurrent_tasks='4')

        # test the number of bytes saved in the rupture records
        grp00 = self.calc.datastore.get_attr('ruptures/grp-00', 'nbytes')
        grp02 = self.calc.datastore.get_attr('ruptures/grp-02', 'nbytes')
        grp03 = self.calc.datastore.get_attr('ruptures/grp-03', 'nbytes')
        self.assertEqual(grp00, 545)
        self.assertEqual(grp02, 545)
        self.assertEqual(grp03, 218)

        hc_id = self.calc.datastore.calc_id
        self.run_calc(case_3.__file__, 'job.ini',
                      exports='xml', individual_curves='false',
                      hazard_calculation_id=str(hc_id))
        [fname] = export(('agg_curve-stats', 'xml'), self.calc.datastore)
        self.assertEqualFiles('expected/%s' % strip_calc_id(fname), fname)

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_4(self):
        # Turkey with SHARE logic tree
        out = self.run_calc(case_4.__file__, 'job.ini', exports='csv')
        [fname] = export(('avg_losses-stats', 'csv'), self.calc.datastore)
        self.assertEqualFiles('expected/avg_losses-mean.csv', fname)

        fnames = out['agg_loss_table', 'csv']
        assert fnames, 'No agg_losses exported??'
        for fname in fnames:
            self.assertEqualFiles('expected/' + strip_calc_id(fname), fname)

    @attr('qa', 'risk', 'event_based_risk')
    def test_occupants(self):
        self.run_calc(occupants.__file__, 'job.ini')
        fnames = export(('loss_maps-rlzs', 'xml'), self.calc.datastore) + \
                 export(('agg_curve-rlzs', 'xml'), self.calc.datastore)
        self.assertEqual(len(fnames), 3)  # 2 loss_maps + 1 agg_curve
        for fname in fnames:
            self.assertEqualFiles('expected/' + strip_calc_id(fname),
                                  fname, delta=1E-5)

        fnames = export(('loss_maps-rlzs', 'csv'), self.calc.datastore)
        assert fnames, 'loss_maps-rlzs not exported?'
        if REFERENCE_OS:
            for fname in fnames:
                self.assertEqualFiles('expected/' + strip_calc_id(fname),
                                      fname, delta=1E-5)

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_master(self):
        if sys.platform == 'darwin':
            raise unittest.SkipTest('MacOSX')
        self.run_calc(case_master.__file__, 'job.ini', exports='csv')
        fnames = export(('avg_losses-stats', 'csv'), self.calc.datastore)
        assert fnames, 'avg_losses-stats not exported?'
        for fname in fnames:
            self.assertEqualFiles('expected/' + strip_calc_id(fname), fname,
                                  delta=1E-5)

        fnames = export(('loss_maps-rlzs', 'csv'), self.calc.datastore)
        assert fnames, 'loss_maps-rlzs not exported?'
        if REFERENCE_OS:
            for fname in fnames:
                self.assertEqualFiles('expected/' + strip_calc_id(fname),
                                      fname, delta=1E-5)

        fnames = export(('losses_by_taxon-stats', 'csv'), self.calc.datastore)
        assert fnames, 'losses_by_taxon-stats not exported?'
        for fname in fnames:
            self.assertEqualFiles('expected/' + strip_calc_id(fname), fname)

        fname = writetmp(view('portfolio_loss', self.calc.datastore))
        self.assertEqualFiles('expected/portfolio_loss.txt', fname, delta=1E-5)
        os.remove(fname)

        # check ruptures are stored correctly
        fname = writetmp(view('ruptures_events', self.calc.datastore))
        self.assertEqualFiles('expected/ruptures_events.txt', fname)
        os.remove(fname)

        # check job_info is stored
        job_info = dict(self.calc.datastore['job_info'])
        self.assertIn(b'build_loss_maps.sent', job_info)
        self.assertIn(b'build_loss_maps.received', job_info)

        check_total_losses(self.calc)

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_miriam(self):
        # this is a case with a grid and asset-hazard association
        out = self.run_calc(case_miriam.__file__, 'job.ini', exports='csv')

        [fname] = out['agg_loss_table', 'csv']
        self.assertEqualFiles('expected/agg_losses-rlz000-structural.csv',
                              fname)
        fname = writetmp(view('portfolio_loss', self.calc.datastore))
        self.assertEqualFiles(
            'expected/portfolio_loss.txt', fname, delta=1E-5)
        os.remove(fname)

    @attr('qa', 'risk', 'case_7a')
    def test_case_7a(self):
        # case with  <insuranceLimit isAbsolute="false"/>
        self.run_calc(case_7a.__file__,  'job_h.ini')
        self.run_calc(case_7a.__file__,  'job_r.ini',
                      hazard_calculation_id=str(self.calc.datastore.calc_id))
        [fname] = export(('agg_loss_table', 'csv'), self.calc.datastore)
        if REFERENCE_OS:
            self.assertEqualFiles('expected/agg_losses.csv', fname, delta=1E-5)

    @attr('qa', 'hazard', 'event_based')
    def test_case_4_hazard(self):
        # Turkey with SHARE logic tree; TODO: add site model
        out = self.run_calc(case_4.__file__, 'job.ini',
                            calculation_mode='event_based',
                            ground_motion_fields='false', exports='csv')
        [fname] = [f for f in out['hcurves', 'csv'] if 'mean' in f]
        self.assertEqualFiles('expected/hazard_curve-mean.csv', fname)
        [fname] = [f for f in out['hmaps', 'csv'] if 'mean' in f]
        self.assertEqualFiles('expected/hazard_map-mean.csv', fname)

        fnames = export(('hmaps', 'xml'), self.calc.datastore)
        self.assertEqual(len(fnames), 20)  # 2 IMT x 2 poes + 16 files

    @attr('qa', 'hazard', 'event_based')
    def test_case_4a(self):
        # the case of a site_model.xml with 7 sites but only 1 asset
        out = self.run_calc(case_4a.__file__, 'job_hazard.ini',
                            exports='txt')
        [fname] = out['gmf_data', 'txt']
        self.assertEqualFiles(
            'expected/gmf-smltp_b1-gsimltp_b1.txt', fname)
<commit_msg>Added a delta<commit_after># -*- coding: utf-8 -*-
# vim: tabstop=4 shiftwidth=4 softtabstop=4
#
# Copyright (C) 2015-2017 GEM Foundation
#
# OpenQuake is free software: you can redistribute it and/or modify it
# under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OpenQuake is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
import os
import sys
import mock
import unittest
import numpy
import h5py
from nose.plugins.attrib import attr

from openquake.baselib.general import writetmp
from openquake.baselib.parallel import Sequential
from openquake.calculators.views import view
from openquake.calculators.tests import (
    CalculatorTestCase, strip_calc_id, REFERENCE_OS)
from openquake.calculators.export import export
from openquake.qa_tests_data.event_based_risk import (
    case_1, case_2, case_3, case_4, case_4a, case_master, case_miriam,
    occupants, case_1g, case_7a)

# used for a sanity check
def check_total_losses(calc):
    dstore = calc.datastore
    loss_dt = calc.oqparam.loss_dt()
    L1 = len(loss_dt.names)
    L = L1 // 2
    data1 = numpy.zeros(L1, numpy.float32)
    for dset in dstore['agg_loss_table'].values():
        for l, lt in enumerate(loss_dt.names):
            i = lt.endswith('_ins')
            data1[l] += dset['loss'][:, l - L * i, i].sum()

    # check the sums are consistent with the ones coming from losses_by_taxon
    data2 = numpy.zeros(L1, numpy.float32)
    lbt = dstore['losses_by_taxon-rlzs']
    for l in range(L1):
        data2[l] += lbt[:, :, l].sum()
    numpy.testing.assert_allclose(data1, data2, 1E-6)

    # test the asset_loss_table exporter; notice that I need to disable
    # the parallelism to avoid reading bogus data: this is the usual
    # heisenbug when reading in parallel an .hdf5 generated in process
    with mock.patch('openquake.baselib.parallel.Starmap', Sequential):
        [fname] = export(('asset_loss_table', 'hdf5'), dstore)
    print('Generating %s' % fname)
    with h5py.File(fname) as f:
        total = f['asset_loss_table'].attrs['total']

    # check the sums are consistent with the ones coming from asset_loss_table
    numpy.testing.assert_allclose(data1, total, 1E-6)

class EventBasedRiskTestCase(CalculatorTestCase):

    def assert_stats_ok(self, pkg, job_ini):
        out = self.run_calc(pkg.__file__, job_ini, exports='csv',
                            concurrent_tasks='4')
        # NB: it is important to use concurrent_tasks > 1 to test the
        # complications of concurrency (for instance the noncommutativity of
        # numpy.float32 addition when computing the average losses)
        all_csv = []
        for fnames in out.values():
            for fname in fnames:
                if 'rlz' in fname:
                    continue
                elif fname.endswith('.csv') and any(x in fname for x in (
                        'loss_curve', 'loss_map', 'agg_loss', 'avg_loss')):
                    all_csv.append(fname)
        assert all_csv, 'Could not find any CSV file??'
        for fname in all_csv:
            self.assertEqualFiles(
                'expected/%s' % strip_calc_id(fname), fname)

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_1(self):
        self.run_calc(case_1.__file__, 'job.ini')
        ekeys = [('agg_curve-stats', 'xml')]
        for ekey in ekeys:
            for fname in export(ekey, self.calc.datastore):
                self.assertEqualFiles(
                    'expected/%s' % strip_calc_id(fname), fname)

        fnames = export(('loss_maps-stats', 'csv'), self.calc.datastore)
        assert fnames
        if REFERENCE_OS:
            for fname in fnames:
                self.assertEqualFiles('expected/' + strip_calc_id(fname),
                                      fname, delta=1E-5)

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_1g(self):
        # vulnerability function with PMF
        self.run_calc(case_1g.__file__, 'job.ini')
        fname = writetmp(view('mean_avg_losses', self.calc.datastore))
        self.assertEqualFiles('expected/avg_losses.txt', fname)
        os.remove(fname)

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_2(self):
        self.run_calc(case_2.__file__, 'job.ini')
        fname = writetmp(view('mean_avg_losses', self.calc.datastore))
        self.assertEqualFiles('expected/mean_avg_losses.txt', fname)
        os.remove(fname)

        # test the composite_risk_model keys (i.e. slash escaping)
        crm = sorted(self.calc.datastore.getitem('composite_risk_model'))
        self.assertEqual(crm, ['RC%2B', 'RM', 'W%2F1'])

        # test the case when all GMFs are filtered out
        with self.assertRaises(RuntimeError) as ctx:
            self.run_calc(case_2.__file__, 'job.ini', minimum_intensity='10.0')
        self.assertEqual(
            str(ctx.exception),
            'No GMFs were generated, perhaps they were all below the '
            'minimum_intensity threshold')

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_2_correlation(self):
        out = self.run_calc(case_2.__file__, 'job_loss.ini', exports='csv',
                            asset_correlation=1.0)
        [fname] = out['agg_loss_table', 'csv']
        self.assertEqualFiles('expected/agg_losses.csv', fname)

        check_total_losses(self.calc)

    @attr('qa', 'risk', 'event_based_risk')
    def test_missing_taxonomy(self):
        with self.assertRaises(RuntimeError) as ctx:
            self.run_calc(case_2.__file__, 'job_err.ini')
        self.assertIn('not in the risk model', str(ctx.exception))

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_3(self):
        # this is a test with statistics and without conditional_loss_poes
        self.run_calc(case_3.__file__, 'job.ini',
                      exports='xml', concurrent_tasks='4')

        # test the number of bytes saved in the rupture records
        grp00 = self.calc.datastore.get_attr('ruptures/grp-00', 'nbytes')
        grp02 = self.calc.datastore.get_attr('ruptures/grp-02', 'nbytes')
        grp03 = self.calc.datastore.get_attr('ruptures/grp-03', 'nbytes')
        self.assertEqual(grp00, 545)
        self.assertEqual(grp02, 545)
        self.assertEqual(grp03, 218)

        hc_id = self.calc.datastore.calc_id
        self.run_calc(case_3.__file__, 'job.ini',
                      exports='xml', individual_curves='false',
                      hazard_calculation_id=str(hc_id))
        [fname] = export(('agg_curve-stats', 'xml'), self.calc.datastore)
        self.assertEqualFiles('expected/%s' % strip_calc_id(fname), fname)

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_4(self):
        # Turkey with SHARE logic tree
        out = self.run_calc(case_4.__file__, 'job.ini', exports='csv')
        [fname] = export(('avg_losses-stats', 'csv'), self.calc.datastore)
        self.assertEqualFiles('expected/avg_losses-mean.csv', fname)

        fnames = out['agg_loss_table', 'csv']
        assert fnames, 'No agg_losses exported??'
        for fname in fnames:
            self.assertEqualFiles('expected/' + strip_calc_id(fname), fname)

    @attr('qa', 'risk', 'event_based_risk')
    def test_occupants(self):
        self.run_calc(occupants.__file__, 'job.ini')
        fnames = export(('loss_maps-rlzs', 'xml'), self.calc.datastore) + \
                 export(('agg_curve-rlzs', 'xml'), self.calc.datastore)
        self.assertEqual(len(fnames), 3)  # 2 loss_maps + 1 agg_curve
        for fname in fnames:
            self.assertEqualFiles('expected/' + strip_calc_id(fname),
                                  fname, delta=1E-5)

        fnames = export(('loss_maps-rlzs', 'csv'), self.calc.datastore)
        assert fnames, 'loss_maps-rlzs not exported?'
        if REFERENCE_OS:
            for fname in fnames:
                self.assertEqualFiles('expected/' + strip_calc_id(fname),
                                      fname, delta=1E-5)

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_master(self):
        if sys.platform == 'darwin':
            raise unittest.SkipTest('MacOSX')
        self.run_calc(case_master.__file__, 'job.ini', exports='csv')
        fnames = export(('avg_losses-stats', 'csv'), self.calc.datastore)
        assert fnames, 'avg_losses-stats not exported?'
        for fname in fnames:
            self.assertEqualFiles('expected/' + strip_calc_id(fname), fname,
                                  delta=1E-5)

        fnames = export(('loss_maps-rlzs', 'csv'), self.calc.datastore)
        assert fnames, 'loss_maps-rlzs not exported?'
        if REFERENCE_OS:
            for fname in fnames:
                self.assertEqualFiles('expected/' + strip_calc_id(fname),
                                      fname, delta=1E-5)

        fnames = export(('losses_by_taxon-stats', 'csv'), self.calc.datastore)
        assert fnames, 'losses_by_taxon-stats not exported?'
        for fname in fnames:
            self.assertEqualFiles('expected/' + strip_calc_id(fname), fname)

        fname = writetmp(view('portfolio_loss', self.calc.datastore))
        self.assertEqualFiles('expected/portfolio_loss.txt', fname, delta=1E-5)
        os.remove(fname)

        # check ruptures are stored correctly
        fname = writetmp(view('ruptures_events', self.calc.datastore))
        self.assertEqualFiles('expected/ruptures_events.txt', fname)
        os.remove(fname)

        # check job_info is stored
        job_info = dict(self.calc.datastore['job_info'])
        self.assertIn(b'build_loss_maps.sent', job_info)
        self.assertIn(b'build_loss_maps.received', job_info)

        check_total_losses(self.calc)

    @attr('qa', 'risk', 'event_based_risk')
    def test_case_miriam(self):
        # this is a case with a grid and asset-hazard association
        out = self.run_calc(case_miriam.__file__, 'job.ini', exports='csv')

        [fname] = out['agg_loss_table', 'csv']
        self.assertEqualFiles('expected/agg_losses-rlz000-structural.csv',
                              fname, delta=1E-5)
        fname = writetmp(view('portfolio_loss', self.calc.datastore))
        self.assertEqualFiles(
            'expected/portfolio_loss.txt', fname, delta=1E-5)
        os.remove(fname)

    @attr('qa', 'risk', 'case_7a')
    def test_case_7a(self):
        # case with  <insuranceLimit isAbsolute="false"/>
        self.run_calc(case_7a.__file__,  'job_h.ini')
        self.run_calc(case_7a.__file__,  'job_r.ini',
                      hazard_calculation_id=str(self.calc.datastore.calc_id))
        [fname] = export(('agg_loss_table', 'csv'), self.calc.datastore)
        if REFERENCE_OS:
            self.assertEqualFiles('expected/agg_losses.csv', fname, delta=1E-5)

    @attr('qa', 'hazard', 'event_based')
    def test_case_4_hazard(self):
        # Turkey with SHARE logic tree; TODO: add site model
        out = self.run_calc(case_4.__file__, 'job.ini',
                            calculation_mode='event_based',
                            ground_motion_fields='false', exports='csv')
        [fname] = [f for f in out['hcurves', 'csv'] if 'mean' in f]
        self.assertEqualFiles('expected/hazard_curve-mean.csv', fname)
        [fname] = [f for f in out['hmaps', 'csv'] if 'mean' in f]
        self.assertEqualFiles('expected/hazard_map-mean.csv', fname)

        fnames = export(('hmaps', 'xml'), self.calc.datastore)
        self.assertEqual(len(fnames), 20)  # 2 IMT x 2 poes + 16 files

    @attr('qa', 'hazard', 'event_based')
    def test_case_4a(self):
        # the case of a site_model.xml with 7 sites but only 1 asset
        out = self.run_calc(case_4a.__file__, 'job_hazard.ini',
                            exports='txt')
        [fname] = out['gmf_data', 'txt']
        self.assertEqualFiles(
            'expected/gmf-smltp_b1-gsimltp_b1.txt', fname)
<|endoftext|>