<commit_before># All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_utils import fixture as utils_fixture

from nova import exception
from nova.objects import service
from nova.tests import fixtures
from nova.tests.functional.notification_sample_tests \
    import notification_sample_base
from nova.tests.unit.api.openstack.compute import test_services
from nova.tests.unit import fake_notifier

class TestServiceNotificationBase(
        notification_sample_base.NotificationSampleTestBase):

    def _verify_notification(self, sample_file_name, replacements=None,
                             actual=None):
        # This just extends the generic _verify_notification to default the
        # service version to the current service version to avoid sample update
        # after every service version bump.
        if 'version' not in replacements:
            replacements['version'] = service.SERVICE_VERSION
        base = super(TestServiceNotificationBase, self)
        base._verify_notification(sample_file_name, replacements, actual)

class TestServiceUpdateNotificationSamplev2_52(TestServiceNotificationBase):

    # These tests have to be capped at 2.52 since the PUT format changes in
    # the 2.53 microversion.
    MAX_MICROVERSION = '2.52'

    def setUp(self):
        super(TestServiceUpdateNotificationSamplev2_52, self).setUp()
        self.stub_out("nova.db.api.service_get_by_host_and_binary",
                      test_services.fake_service_get_by_host_binary)
        self.stub_out("nova.db.api.service_update",
                      test_services.fake_service_update)
        self.useFixture(utils_fixture.TimeFixture(test_services.fake_utcnow()))
        self.useFixture(fixtures.SingleCellSimple())
        self.service_uuid = test_services.fake_service_get_by_host_binary(
            None, 'host1', 'nova-compute')['uuid']

    def test_service_enable(self):
        body = {'host': 'host1',
                'binary': 'nova-compute'}
        self.admin_api.api_put('os-services/enable', body)
        self._verify_notification('service-update',
                                  replacements={'uuid': self.service_uuid})

    def test_service_disabled(self):
        body = {'host': 'host1',
                'binary': 'nova-compute'}
        self.admin_api.api_put('os-services/disable', body)
        self._verify_notification('service-update',
                                  replacements={'disabled': True,
                                                'uuid': self.service_uuid})

    def test_service_disabled_log_reason(self):
        body = {'host': 'host1',
                'binary': 'nova-compute',
                'disabled_reason': 'test2'}
        self.admin_api.api_put('os-services/disable-log-reason', body)
        self._verify_notification('service-update',
                                  replacements={'disabled': True,
                                                'disabled_reason': 'test2',
                                                'uuid': self.service_uuid})

    def test_service_force_down(self):
        body = {'host': 'host1',
                'binary': 'nova-compute',
                'forced_down': True}
        self.admin_api.api_put('os-services/force-down', body)
        self._verify_notification('service-update',
                                  replacements={'forced_down': True,
                                                'disabled': True,
                                                'disabled_reason': 'test2',
                                                'uuid': self.service_uuid})

class TestServiceUpdateNotificationSampleLatest(
        TestServiceUpdateNotificationSamplev2_52):
    """Tests the PUT /os-services/{service_id} API notifications."""

    MAX_MICROVERSION = 'latest'

    def setUp(self):
        super(TestServiceUpdateNotificationSampleLatest, self).setUp()

        def db_service_get_by_uuid(ctxt, service_uuid):
            for svc in test_services.fake_services_list:
                if svc['uuid'] == service_uuid:
                    return svc
            raise exception.ServiceNotFound(service_id=service_uuid)
        self.stub_out('nova.db.api.service_get_by_uuid',
                      db_service_get_by_uuid)

    def test_service_enable(self):
        body = {'status': 'enabled'}
        self.admin_api.api_put('os-services/%s' % self.service_uuid, body)
        self._verify_notification('service-update',
                                  replacements={'uuid': self.service_uuid})

    def test_service_disabled(self):
        body = {'status': 'disabled'}
        self.admin_api.api_put('os-services/%s' % self.service_uuid, body)
        self._verify_notification('service-update',
                                  replacements={'disabled': True,
                                                'uuid': self.service_uuid})

    def test_service_disabled_log_reason(self):
        body = {'status': 'disabled',
                'disabled_reason': 'test2'}
        self.admin_api.api_put('os-services/%s' % self.service_uuid, body)
        self._verify_notification('service-update',
                                  replacements={'disabled': True,
                                                'disabled_reason': 'test2',
                                                'uuid': self.service_uuid})

    def test_service_force_down(self):
        body = {'forced_down': True}
        self.admin_api.api_put('os-services/%s' % self.service_uuid, body)
        self._verify_notification('service-update',
                                  replacements={'forced_down': True,
                                                'disabled': True,
                                                'disabled_reason': 'test2',
                                                'uuid': self.service_uuid})

class TestServiceNotificationSample(TestServiceNotificationBase):

    def test_service_create(self):
        self.compute2 = self.start_service('compute', host='host2')
        self._verify_notification(
            'service-create',
            replacements={
                'uuid':
                notification_sample_base.NotificationSampleTestBase.ANY})

    def test_service_destroy(self):
        self.compute2 = self.start_service('compute', host='host2')
        # This test runs with the latest microversion by default so we get the
        # service uuid back from the REST API.
        compute2_service_id = self.admin_api.get_services(
            host=self.compute2.host, binary='nova-compute')[0]['id']
        self.admin_api.api_delete('os-services/%s' % compute2_service_id)
        self._verify_notification(
            'service-delete',
            replacements={'uuid': compute2_service_id},
            actual=fake_notifier.VERSIONED_NOTIFICATIONS[1])
<commit_msg>Avoid timeout from service update notification tests<commit_after># All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

from oslo_utils import fixture as utils_fixture

from nova import exception
from nova.objects import service
from nova.tests import fixtures
from nova.tests.functional.notification_sample_tests \
    import notification_sample_base
from nova.tests.unit.api.openstack.compute import test_services
from nova.tests.unit import fake_notifier

class TestServiceNotificationBase(
        notification_sample_base.NotificationSampleTestBase):

    def _verify_notification(self, sample_file_name, replacements=None,
                             actual=None):
        # This just extends the generic _verify_notification to default the
        # service version to the current service version to avoid sample update
        # after every service version bump.
        if 'version' not in replacements:
            replacements['version'] = service.SERVICE_VERSION
        base = super(TestServiceNotificationBase, self)
        base._verify_notification(sample_file_name, replacements, actual)

class TestServiceUpdateNotificationSamplev2_52(TestServiceNotificationBase):

    # These tests have to be capped at 2.52 since the PUT format changes in
    # the 2.53 microversion.
    MAX_MICROVERSION = '2.52'

    def setUp(self):
        super(TestServiceUpdateNotificationSamplev2_52, self).setUp()
        self.stub_out("nova.db.api.service_get_by_host_and_binary",
                      test_services.fake_service_get_by_host_binary)
        self.stub_out("nova.db.api.service_update",
                      test_services.fake_service_update)
        # NOTE(gibi): enable / disable a compute service tries to call
        # the compute service via RPC to update placement. However in these
        # tests the compute services are faked. So stub out the RPC call to
        # avoid waiting for the RPC timeout. The notifications are generated
        # regardless of the result of the RPC call anyhow.
        self.stub_out("nova.compute.rpcapi.ComputeAPI.set_host_enabled",
                      lambda *args, **kwargs: None)
        self.useFixture(utils_fixture.TimeFixture(test_services.fake_utcnow()))
        self.useFixture(fixtures.SingleCellSimple())
        self.service_uuid = test_services.fake_service_get_by_host_binary(
            None, 'host1', 'nova-compute')['uuid']

    def test_service_enable(self):
        body = {'host': 'host1',
                'binary': 'nova-compute'}
        self.admin_api.api_put('os-services/enable', body)
        self._verify_notification('service-update',
                                  replacements={'uuid': self.service_uuid})

    def test_service_disabled(self):
        body = {'host': 'host1',
                'binary': 'nova-compute'}
        self.admin_api.api_put('os-services/disable', body)
        self._verify_notification('service-update',
                                  replacements={'disabled': True,
                                                'uuid': self.service_uuid})

    def test_service_disabled_log_reason(self):
        body = {'host': 'host1',
                'binary': 'nova-compute',
                'disabled_reason': 'test2'}
        self.admin_api.api_put('os-services/disable-log-reason', body)
        self._verify_notification('service-update',
                                  replacements={'disabled': True,
                                                'disabled_reason': 'test2',
                                                'uuid': self.service_uuid})

    def test_service_force_down(self):
        body = {'host': 'host1',
                'binary': 'nova-compute',
                'forced_down': True}
        self.admin_api.api_put('os-services/force-down', body)
        self._verify_notification('service-update',
                                  replacements={'forced_down': True,
                                                'disabled': True,
                                                'disabled_reason': 'test2',
                                                'uuid': self.service_uuid})

class TestServiceUpdateNotificationSampleLatest(
        TestServiceUpdateNotificationSamplev2_52):
    """Tests the PUT /os-services/{service_id} API notifications."""

    MAX_MICROVERSION = 'latest'

    def setUp(self):
        super(TestServiceUpdateNotificationSampleLatest, self).setUp()

        def db_service_get_by_uuid(ctxt, service_uuid):
            for svc in test_services.fake_services_list:
                if svc['uuid'] == service_uuid:
                    return svc
            raise exception.ServiceNotFound(service_id=service_uuid)
        self.stub_out('nova.db.api.service_get_by_uuid',
                      db_service_get_by_uuid)

    def test_service_enable(self):
        body = {'status': 'enabled'}
        self.admin_api.api_put('os-services/%s' % self.service_uuid, body)
        self._verify_notification('service-update',
                                  replacements={'uuid': self.service_uuid})

    def test_service_disabled(self):
        body = {'status': 'disabled'}
        self.admin_api.api_put('os-services/%s' % self.service_uuid, body)
        self._verify_notification('service-update',
                                  replacements={'disabled': True,
                                                'uuid': self.service_uuid})

    def test_service_disabled_log_reason(self):
        body = {'status': 'disabled',
                'disabled_reason': 'test2'}
        self.admin_api.api_put('os-services/%s' % self.service_uuid, body)
        self._verify_notification('service-update',
                                  replacements={'disabled': True,
                                                'disabled_reason': 'test2',
                                                'uuid': self.service_uuid})

    def test_service_force_down(self):
        body = {'forced_down': True}
        self.admin_api.api_put('os-services/%s' % self.service_uuid, body)
        self._verify_notification('service-update',
                                  replacements={'forced_down': True,
                                                'disabled': True,
                                                'disabled_reason': 'test2',
                                                'uuid': self.service_uuid})

class TestServiceNotificationSample(TestServiceNotificationBase):

    def test_service_create(self):
        self.compute2 = self.start_service('compute', host='host2')
        self._verify_notification(
            'service-create',
            replacements={
                'uuid':
                notification_sample_base.NotificationSampleTestBase.ANY})

    def test_service_destroy(self):
        self.compute2 = self.start_service('compute', host='host2')
        # This test runs with the latest microversion by default so we get the
        # service uuid back from the REST API.
        compute2_service_id = self.admin_api.get_services(
            host=self.compute2.host, binary='nova-compute')[0]['id']
        self.admin_api.api_delete('os-services/%s' % compute2_service_id)
        self._verify_notification(
            'service-delete',
            replacements={'uuid': compute2_service_id},
            actual=fake_notifier.VERSIONED_NOTIFICATIONS[1])
<|endoftext|>


<commit_before>from warnings import warn

import numpy

import cupy
from cupy import cuda
from cupy.cuda import cublas
from cupy.cuda import device
from cupy.linalg import util

if cuda.cusolver_enabled:
    from cupy.cuda import cusolver

def lu_factor(a, overwrite_a=False, check_finite=True):
    """LU decomposition.

    Decompose a given two-dimensional square matrix into ``P * L * U``,
    where ``P`` is a permutation matrix,  ``L`` lower-triangular with
    unit diagonal elements, and ``U`` upper-triangular matrix.
    Note that in the current implementation ``a`` must be
    a real matrix, and only float32 and float64 are supported.

    Args:
        a (cupy.ndarray): The input matrix with dimension ``(N, N)``
        overwrite_a (bool): Allow overwriting data in ``a`` (may enhance
            performance)
        check_finite (bool): Whether to check that the input matrices contain
            only finite numbers. Disabling may give a performance gain, but may
            result in problems (crashes, non-termination) if the inputs do
            contain infinities or NaNs.

    Returns:
        tuple:
            ``(lu, piv)`` where ``lu`` is a :class:`cupy.ndarray`
            storing ``U`` in its upper triangle, and ``L`` without
            unit diagonal elements in its lower triangle, and `piv` is
            a :class:`cupy.ndarray` storing pivot indices representing
            permutation matrix ``P``.

    .. seealso:: :func:`scipy.linalg.lu_factor`
    """

    if not cuda.cusolver_enabled:
        raise RuntimeError('Current cupy only supports cusolver in CUDA 8.0')

    a = cupy.asarray(a)
    util._assert_rank2(a)
    util._assert_nd_squareness(a)

    dtype = a.dtype

    if dtype == 'f':
        getrf = cusolver.sgetrf
        getrf_bufferSize = cusolver.sgetrf_bufferSize
    elif dtype.char == 'd':
        getrf = cusolver.dgetrf
        getrf_bufferSize = cusolver.dgetrf_bufferSize
    else:
        raise NotImplementedError('Only float32 and float64 are supported.')

    a = a.astype(dtype, order='F', copy=(not overwrite_a))

    if check_finite:
        if a.dtype.kind == 'f' and not cupy.isfinite(a).all():
            raise ValueError(
                "array must not contain infs or NaNs")

    cusolver_handle = device.get_cusolver_handle()
    dev_info = cupy.empty(1, dtype=numpy.intc)

    ipiv = cupy.empty((a.shape[0],), dtype=numpy.intc)

    m = a.shape[0]

    buffersize = getrf_bufferSize(cusolver_handle, m, m, a.data.ptr, m)
    workspace = cupy.empty(buffersize, dtype=dtype)

    # LU factorization
    getrf(cusolver_handle, m, m, a.data.ptr, m, workspace.data.ptr,
          ipiv.data.ptr, dev_info.data.ptr)

    if dev_info[0] < 0:
        raise ValueError('illegal value in %d-th argument of '
                         'internal getrf (lu_factor)' % -dev_info[0])
    elif dev_info[0] > 0:
        warn("Diagonal number %d is exactly zero. Singular matrix."
             % dev_info[0], RuntimeWarning, stacklevel=2)

    # cuSolver uses 1-origin while SciPy uses 0-origin
    ipiv -= 1

    return (a, ipiv)

def lu_solve(lu_and_piv, b, trans=0, overwrite_b=False, check_finite=True):
    """Solve an equation system, ``a * x = b``, given the LU factorization of ``a``

    Args:
        lu_and_piv (tuple): LU factorization of matrix ``a`` (``(M, N)``)
            together with pivot indices.
        b (cupy.ndarray): The matrix with dimension ``(M,)`` or
            ``(M, N)``.
        trans ({0, 1, 2}): Type of system to solve:
            ========  =========
            trans     system
            ========  =========
            0         a x  = b
            1         a^T x = b
            2         a^H x = b
            ========  =========
        overwrite_b (bool): Allow overwriting data in b (may enhance
            performance)
        check_finite (bool): Whether to check that the input matrices contain
            only finite numbers. Disabling may give a performance gain, but may
            result in problems (crashes, non-termination) if the inputs do
            contain infinities or NaNs.

    Returns:
        cupy.ndarray:
            The matrix with dimension ``(M,)`` or ``(M, N)``.

    .. seealso:: :func:`scipy.linalg.lu_solve`
    """

    if not cuda.cusolver_enabled:
        raise RuntimeError('Current cupy only supports cusolver in CUDA 8.0')

    (lu, ipiv) = lu_and_piv

    util._assert_cupy_array(lu)
    util._assert_rank2(lu)
    util._assert_nd_squareness(lu)

    m = lu.shape[0]
    if m != b.shape[0]:
        raise ValueError("incompatible dimensions.")

    dtype = lu.dtype
    if dtype == 'f':
        getrs = cusolver.sgetrs
    elif dtype.char == 'd':
        getrs = cusolver.dgetrs
    else:
        raise NotImplementedError('Only float32 and float64 are supported.')

    if trans == 0:
        trans = cublas.CUBLAS_OP_N
    elif trans == 1:
        trans = cublas.CUBLAS_OP_T
    elif trans == 2:
        trans = cublas.CUBLAS_OP_C
    else:
        raise ValueError("unknown trans")

    lu = lu.astype(dtype, order='F', copy=False)
    ipiv = ipiv.astype(ipiv.dtype, order='F', copy=True)
    # cuSolver uses 1-origin while SciPy uses 0-origin
    ipiv += 1
    b = b.astype(dtype, order='F', copy=(not overwrite_b))

    if check_finite:
        if lu.dtype.kind == 'f' and not cupy.isfinite(lu).all():
            raise ValueError(
                "array must not contain infs or NaNs")
        if b.dtype.kind == 'f' and not cupy.isfinite(b).all():
            raise ValueError(
                "array must not contain infs or NaNs")

    n = 1 if b.ndim == 1 else b.shape[1]
    cusolver_handle = device.get_cusolver_handle()
    dev_info = cupy.empty(1, dtype=numpy.intc)

    # solve for the inverse
    getrs(cusolver_handle,
          trans,
          m, n, lu.data.ptr, m, ipiv.data.ptr, b.data.ptr,
          m, dev_info.data.ptr)

    if dev_info[0] < 0:
        raise ValueError('illegal value in %d-th argument of '
                         'internal getrs (lu_solve)' % -dev_info[0])

    return b
<commit_msg>Document lu_factor's behavior differnt from SciPy<commit_after>from warnings import warn

import numpy

import cupy
from cupy import cuda
from cupy.cuda import cublas
from cupy.cuda import device
from cupy.linalg import util

if cuda.cusolver_enabled:
    from cupy.cuda import cusolver

def lu_factor(a, overwrite_a=False, check_finite=True):
    """LU decomposition.

    Decompose a given two-dimensional square matrix into ``P * L * U``,
    where ``P`` is a permutation matrix,  ``L`` lower-triangular with
    unit diagonal elements, and ``U`` upper-triangular matrix.
    Note that in the current implementation ``a`` must be
    a real matrix, and only float32 and float64 are supported.

    Args:
        a (cupy.ndarray): The input matrix with dimension ``(N, N)``
        overwrite_a (bool): Allow overwriting data in ``a`` (may enhance
            performance)
        check_finite (bool): Whether to check that the input matrices contain
            only finite numbers. Disabling may give a performance gain, but may
            result in problems (crashes, non-termination) if the inputs do
            contain infinities or NaNs.

    Returns:
        tuple:
            ``(lu, piv)`` where ``lu`` is a :class:`cupy.ndarray`
            storing ``U`` in its upper triangle, and ``L`` without
            unit diagonal elements in its lower triangle, and `piv` is
            a :class:`cupy.ndarray` storing pivot indices representing
            permutation matrix ``P``.

    .. seealso:: :func:`scipy.linalg.lu_factor`

    .. note::

        Current implementation returns result different from SciPy when the
        matrix singular. SciPy returns an array containing ``0.`` while the
        current implementation returns an array containing ``nan``.

        >>> import numpy as np
        >>> import scipy.linalg
        >>> scipy.linalg.lu_factor(np.array([[0, 1], [0, 0]], \
dtype=np.float32))
        __main__:1: LinAlgWarning: Diagonal number 1 is exactly zero. \
Singular matrix.
        (array([[0., 1.],
               [0., 0.]], dtype=float32), array([0, 1], dtype=int32))

        >>> import cupy as cp
        >>> import cupyx.scipy.linalg
        >>> cupyx.scipy.linalg.lu_factor(cp.array([[0, 1], [0, 0]], \
dtype=cp.float32))
        __main__:1: RuntimeWarning: Diagonal number 1 is exactly zero. \
Singular matrix.
        (array([[ 0.,  1.],
               [nan, nan]], dtype=float32), array([0, 1], dtype=int32))
    """

    if not cuda.cusolver_enabled:
        raise RuntimeError('Current cupy only supports cusolver in CUDA 8.0')

    a = cupy.asarray(a)
    util._assert_rank2(a)
    util._assert_nd_squareness(a)

    dtype = a.dtype

    if dtype == 'f':
        getrf = cusolver.sgetrf
        getrf_bufferSize = cusolver.sgetrf_bufferSize
    elif dtype.char == 'd':
        getrf = cusolver.dgetrf
        getrf_bufferSize = cusolver.dgetrf_bufferSize
    else:
        raise NotImplementedError('Only float32 and float64 are supported.')

    a = a.astype(dtype, order='F', copy=(not overwrite_a))

    if check_finite:
        if a.dtype.kind == 'f' and not cupy.isfinite(a).all():
            raise ValueError(
                "array must not contain infs or NaNs")

    cusolver_handle = device.get_cusolver_handle()
    dev_info = cupy.empty(1, dtype=numpy.intc)

    ipiv = cupy.empty((a.shape[0],), dtype=numpy.intc)

    m = a.shape[0]

    buffersize = getrf_bufferSize(cusolver_handle, m, m, a.data.ptr, m)
    workspace = cupy.empty(buffersize, dtype=dtype)

    # LU factorization
    getrf(cusolver_handle, m, m, a.data.ptr, m, workspace.data.ptr,
          ipiv.data.ptr, dev_info.data.ptr)

    if dev_info[0] < 0:
        raise ValueError('illegal value in %d-th argument of '
                         'internal getrf (lu_factor)' % -dev_info[0])
    elif dev_info[0] > 0:
        warn("Diagonal number %d is exactly zero. Singular matrix."
             % dev_info[0], RuntimeWarning, stacklevel=2)

    # cuSolver uses 1-origin while SciPy uses 0-origin
    ipiv -= 1

    return (a, ipiv)

def lu_solve(lu_and_piv, b, trans=0, overwrite_b=False, check_finite=True):
    """Solve an equation system, ``a * x = b``, given the LU factorization of ``a``

    Args:
        lu_and_piv (tuple): LU factorization of matrix ``a`` (``(M, N)``)
            together with pivot indices.
        b (cupy.ndarray): The matrix with dimension ``(M,)`` or
            ``(M, N)``.
        trans ({0, 1, 2}): Type of system to solve:
            ========  =========
            trans     system
            ========  =========
            0         a x  = b
            1         a^T x = b
            2         a^H x = b
            ========  =========
        overwrite_b (bool): Allow overwriting data in b (may enhance
            performance)
        check_finite (bool): Whether to check that the input matrices contain
            only finite numbers. Disabling may give a performance gain, but may
            result in problems (crashes, non-termination) if the inputs do
            contain infinities or NaNs.

    Returns:
        cupy.ndarray:
            The matrix with dimension ``(M,)`` or ``(M, N)``.

    .. seealso:: :func:`scipy.linalg.lu_solve`
    """

    if not cuda.cusolver_enabled:
        raise RuntimeError('Current cupy only supports cusolver in CUDA 8.0')

    (lu, ipiv) = lu_and_piv

    util._assert_cupy_array(lu)
    util._assert_rank2(lu)
    util._assert_nd_squareness(lu)

    m = lu.shape[0]
    if m != b.shape[0]:
        raise ValueError("incompatible dimensions.")

    dtype = lu.dtype
    if dtype == 'f':
        getrs = cusolver.sgetrs
    elif dtype.char == 'd':
        getrs = cusolver.dgetrs
    else:
        raise NotImplementedError('Only float32 and float64 are supported.')

    if trans == 0:
        trans = cublas.CUBLAS_OP_N
    elif trans == 1:
        trans = cublas.CUBLAS_OP_T
    elif trans == 2:
        trans = cublas.CUBLAS_OP_C
    else:
        raise ValueError("unknown trans")

    lu = lu.astype(dtype, order='F', copy=False)
    ipiv = ipiv.astype(ipiv.dtype, order='F', copy=True)
    # cuSolver uses 1-origin while SciPy uses 0-origin
    ipiv += 1
    b = b.astype(dtype, order='F', copy=(not overwrite_b))

    if check_finite:
        if lu.dtype.kind == 'f' and not cupy.isfinite(lu).all():
            raise ValueError(
                "array must not contain infs or NaNs")
        if b.dtype.kind == 'f' and not cupy.isfinite(b).all():
            raise ValueError(
                "array must not contain infs or NaNs")

    n = 1 if b.ndim == 1 else b.shape[1]
    cusolver_handle = device.get_cusolver_handle()
    dev_info = cupy.empty(1, dtype=numpy.intc)

    # solve for the inverse
    getrs(cusolver_handle,
          trans,
          m, n, lu.data.ptr, m, ipiv.data.ptr, b.data.ptr,
          m, dev_info.data.ptr)

    if dev_info[0] < 0:
        raise ValueError('illegal value in %d-th argument of '
                         'internal getrs (lu_solve)' % -dev_info[0])

    return b
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

from django.core.exceptions import ObjectDoesNotExist
from django.db.models import Model

from taiga.projects.models import Project
from taiga.projects.history import services as history_services
from taiga.projects.history.choices import HistoryType
from taiga.projects.history.models import HistoryEntry
from taiga.timeline.models import Timeline
from taiga.timeline.service import (_add_to_object_timeline, _get_impl_key_from_model,
    _timeline_impl_map)
from taiga.timeline.signals import on_new_history_entry, _push_to_timelines
from taiga.users.models import User

from unittest.mock import patch
from django.contrib.contenttypes.models import ContentType

import django_pgjson.fields
import django.utils.timezone

timelime_objects = []
created = None

def custom_add_to_object_timeline(obj:object, instance:object, event_type:str, namespace:str="default", extra_data:dict={}):
    global created
    global timelime_objects
    assert isinstance(obj, Model), "obj must be a instance of Model"
    assert isinstance(instance, Model), "instance must be a instance of Model"
    event_type_key = _get_impl_key_from_model(instance.__class__, event_type)
    impl = _timeline_impl_map.get(event_type_key, None)

    timelime_objects.append(Timeline(
        content_object=obj,
        namespace=namespace,
        event_type=event_type_key,
        project=instance.project,
        data=impl(instance, extra_data=extra_data),
        data_content_type = ContentType.objects.get_for_model(instance.__class__),
        created = created,
    ))

def generate_timeline(apps, schema_editor):
    global created
    global timelime_objects
    with patch('taiga.timeline.service._add_to_object_timeline', new=custom_add_to_object_timeline):
        # Projects api wasn't a HistoryResourceMixin so we can't interate on the HistoryEntries in this case
        for project in Project.objects.order_by("created_date").iterator():
            created = project.created_date
            print("Project:", created)
            extra_data = {
                "values_diff": {},
                "user": {
                    "pk": project.owner.id,
                    "user_name": project.owner.get_full_name(),
                },
            }
            _push_to_timelines(project, project.owner, project, "create", extra_data=extra_data)

        Timeline.objects.bulk_create(timelime_objects, batch_size=10000)
        timelime_objects = []

        for historyEntry in HistoryEntry.objects.order_by("created_at").iterator():
            print("History entry:", historyEntry.created_at)
            try:
                created = historyEntry.created_at
                on_new_history_entry(None, historyEntry, None)
            except ObjectDoesNotExist as e:
                print("Ignoring")

        Timeline.objects.bulk_create(timelime_objects, batch_size=10000)

class Migration(migrations.Migration):

    dependencies = [
        ('projects', '0019_auto_20150311_0821'),
        ('contenttypes', '0001_initial'),
        ('timeline', '0001_initial'),
    ]

    operations = [
        migrations.DeleteModel(
            name='Timeline',
        ),
        migrations.CreateModel(
            name='Timeline',
            fields=[
                ('id', models.AutoField(auto_created=True, verbose_name='ID', primary_key=True, serialize=False)),
                ('object_id', models.PositiveIntegerField()),
                ('namespace', models.SlugField(default='default')),
                ('event_type', models.SlugField()),
                ('project', models.ForeignKey(to='projects.Project')),
                ('data', django_pgjson.fields.JsonField()),
                ('data_content_type', models.ForeignKey(to='contenttypes.ContentType', related_name='data_timelines')),
                ('created', models.DateTimeField(default=django.utils.timezone.now)),
                ('content_type', models.ForeignKey(to='contenttypes.ContentType', related_name='content_type_timelines')),
            ],
            options={
            },
            bases=(models.Model,),
        ),
        migrations.AlterIndexTogether(
            name='timeline',
            index_together=set([('content_type', 'object_id', 'namespace')]),
        ),
        migrations.RunPython(generate_timeline),
    ]
<commit_msg>Fixing migration<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

from django.core.exceptions import ObjectDoesNotExist
from django.db.models import Model

from taiga.projects.models import Project
from taiga.projects.history import services as history_services
from taiga.projects.history.choices import HistoryType
from taiga.projects.history.models import HistoryEntry
from taiga.timeline.models import Timeline
from taiga.timeline.service import (_add_to_object_timeline, _get_impl_key_from_model,
    _timeline_impl_map, extract_user_info)
from taiga.timeline.signals import on_new_history_entry, _push_to_timelines
from taiga.users.models import User

from unittest.mock import patch
from django.contrib.contenttypes.models import ContentType

import django_pgjson.fields
import django.utils.timezone

timelime_objects = []
created = None

def custom_add_to_object_timeline(obj:object, instance:object, event_type:str, namespace:str="default", extra_data:dict={}):
    global created
    global timelime_objects
    assert isinstance(obj, Model), "obj must be a instance of Model"
    assert isinstance(instance, Model), "instance must be a instance of Model"
    event_type_key = _get_impl_key_from_model(instance.__class__, event_type)
    impl = _timeline_impl_map.get(event_type_key, None)

    timelime_objects.append(Timeline(
        content_object=obj,
        namespace=namespace,
        event_type=event_type_key,
        project=instance.project,
        data=impl(instance, extra_data=extra_data),
        data_content_type = ContentType.objects.get_for_model(instance.__class__),
        created = created,
    ))

def generate_timeline(apps, schema_editor):
    global created
    global timelime_objects
    with patch('taiga.timeline.service._add_to_object_timeline', new=custom_add_to_object_timeline):
        # Projects api wasn't a HistoryResourceMixin so we can't interate on the HistoryEntries in this case
        for project in Project.objects.order_by("created_date").iterator():
            created = project.created_date
            print("Project:", created)
            extra_data = {
                "values_diff": {},
                "user": extract_user_info(project.owner),
            }
            _push_to_timelines(project, project.owner, project, "create", extra_data=extra_data)

        Timeline.objects.bulk_create(timelime_objects, batch_size=10000)
        timelime_objects = []

        for historyEntry in HistoryEntry.objects.order_by("created_at").iterator():
            print("History entry:", historyEntry.created_at)
            try:
                created = historyEntry.created_at
                on_new_history_entry(None, historyEntry, None)
            except ObjectDoesNotExist as e:
                print("Ignoring")

        Timeline.objects.bulk_create(timelime_objects, batch_size=10000)

class Migration(migrations.Migration):

    dependencies = [
        ('projects', '0019_auto_20150311_0821'),
        ('contenttypes', '0001_initial'),
        ('timeline', '0001_initial'),
    ]

    operations = [
        migrations.DeleteModel(
            name='Timeline',
        ),
        migrations.CreateModel(
            name='Timeline',
            fields=[
                ('id', models.AutoField(auto_created=True, verbose_name='ID', primary_key=True, serialize=False)),
                ('object_id', models.PositiveIntegerField()),
                ('namespace', models.SlugField(default='default')),
                ('event_type', models.SlugField()),
                ('project', models.ForeignKey(to='projects.Project')),
                ('data', django_pgjson.fields.JsonField()),
                ('data_content_type', models.ForeignKey(to='contenttypes.ContentType', related_name='data_timelines')),
                ('created', models.DateTimeField(default=django.utils.timezone.now)),
                ('content_type', models.ForeignKey(to='contenttypes.ContentType', related_name='content_type_timelines')),
            ],
            options={
            },
            bases=(models.Model,),
        ),
        migrations.AlterIndexTogether(
            name='timeline',
            index_together=set([('content_type', 'object_id', 'namespace')]),
        ),
        migrations.RunPython(generate_timeline),
    ]
<|endoftext|>


<commit_before># Copyright 2012 OpenStack Foundation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import time

from tempest_lib import exceptions as lib_exc

from tempest import exceptions
from tempest import config
from tempest.services.volume.json.volumes_client import BaseVolumesClientJSON
from tempest.services.volume.json.snapshots_client import BaseSnapshotsClientJSON

CONF = config.CONF

class CleanBFVResource(BaseVolumesClientJSON, BaseSnapshotsClientJSON):
    """
    Client class to clean up the resources created due to boot from volume
    feature. This cleanup will delete any snapshot attached to volume from
    which VM was booted and then volume itself.
    """
    def __init__(self, auth_provider, **kwargs):
        BaseVolumesClientJSON.__init__(self, auth_provider,
          default_volume_size=CONF.volume.volume_size, **kwargs)
        BaseSnapshotsClientJSON.__init__(self, auth_provider, **kwargs)

    def _wait_for_snapshot_deletion(self, snapshot_id):
        """Waits for a Snapshot to reach a given status."""
        start_time = time.time()
        while True:
            try:
                self.show_snapshot(snapshot_id)
            except lib_exc.NotFound:
                return
            else:
                time.sleep(self.build_interval)
                dtime = time.time() - start_time
                if dtime > self.build_timeout:
                    message = ('Time Limit Exceeded! (%ds)'
                               'while waiting for snapshot %s to get deleted.'
                               % (self.build_timeout, snapshot_id))
                    raise exceptions.TimeoutException(message)

    def _delete_attached_snapshots(self,volume_id):
        self.bfv_snapshots = self.list_snapshots()
        for snapshot in self.bfv_snapshots:
            if snapshot['volume_id'] == volume_id:
                self.delete_snapshot(snapshot['id'])
                self._wait_for_snapshot_deletion(snapshot['id'])
                self.bfv_snapshots.remove(snapshot)

    def volume_not_deletable(self, volume):
        """This will return true if either volume does not exist
        or in available or error state."""
        try:
            res = self.show_volume(volume)
            if res['status'] not in ['available', 'error']:
                return True
        except lib_exc.NotFound:
            return True
        return False

    def clean_bfv_resource(self, volumes):
        """First clean snapshots attached to volume if any then delete volume.
        """
        for volume in volumes:
            if not volume or self.volume_not_deletable(volume):
                continue
            self._delete_attached_snapshots(volume)
            self.delete_volume(volume)

def set_block_device_mapping_args(image_ref, kwargs):
    """
    Update the kwargs dictionary with block device mapping.
    These arguments are required when booting instance from volume.
    """
    if "block_device_mapping_v2" in kwargs.keys() or \
       "block_device_mapping" in kwargs.keys():
        return kwargs
    if 'volume_size' in kwargs:
        vol_size = kwargs.pop('volume_size')
    else:
        vol_size = CONF.volume.volume_size
    bv_map = [{
            "source_type": "image",
            "destination_type": "volume",
            "delete_on_termination": "1",
            "boot_index": 0,
            "uuid": image_ref,
            "device_name": "vda",
            "volume_size": str(vol_size)}]
    bdm_args = {
    'block_device_mapping_v2' : bv_map,
    }
    kwargs.update(bdm_args)
    return kwargs

def get_cleanBFV_obj(auth_provider):
    default_params = {
        'disable_ssl_certificate_validation':
            CONF.identity.disable_ssl_certificate_validation,
        'ca_certs': CONF.identity.ca_certificates_file,
        'trace_requests': CONF.debug.trace_requests
    }
    params = {
        'service': CONF.volume.catalog_type,
        'region': CONF.volume.region or CONF.identity.region,
        'endpoint_type': CONF.volume.endpoint_type,
        'build_interval': CONF.volume.build_interval,
        'build_timeout': CONF.volume.build_timeout
    }
    params.update(default_params)
    return CleanBFVResource(auth_provider, **params)

<commit_msg>Fix for the issue found in boot from volume changes.<commit_after># Copyright 2012 OpenStack Foundation
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import time

from tempest_lib import exceptions as lib_exc

from tempest import exceptions
from tempest import config
from tempest.services.volume.json.volumes_client import BaseVolumesClientJSON
from tempest.services.volume.json.snapshots_client import BaseSnapshotsClientJSON

CONF = config.CONF

class CleanBFVResource(BaseVolumesClientJSON, BaseSnapshotsClientJSON):
    """
    Client class to clean up the resources created due to boot from volume
    feature. This cleanup will delete any snapshot attached to volume from
    which VM was booted and then volume itself.
    """
    def __init__(self, auth_provider, **kwargs):
        BaseVolumesClientJSON.__init__(self, auth_provider,
          default_volume_size=CONF.volume.volume_size, **kwargs)
        BaseSnapshotsClientJSON.__init__(self, auth_provider, **kwargs)

    def _wait_for_snapshot_deletion(self, snapshot_id):
        """Waits for a Snapshot to reach a given status."""
        start_time = time.time()
        while True:
            try:
                self.show_snapshot(snapshot_id)
            except lib_exc.NotFound:
                return
            else:
                time.sleep(self.build_interval)
                dtime = time.time() - start_time
                if dtime > self.build_timeout:
                    message = ('Time Limit Exceeded! (%ds)'
                               'while waiting for snapshot %s to get deleted.'
                               % (self.build_timeout, snapshot_id))
                    raise exceptions.TimeoutException(message)

    def _delete_attached_snapshots(self,volume_id):
        bfv_snapshots = self.list_snapshots()
        for snapshot in bfv_snapshots:
            if snapshot['volume_id'] == volume_id:
                self.delete_snapshot(snapshot['id'])
                self._wait_for_snapshot_deletion(snapshot['id'])

    def volume_not_deletable(self, volume):
        """This will return true if either volume does not exist
        or in available or error state."""
        try:
            res = self.show_volume(volume)
            if res['status'] not in ['available', 'error']:
                return True
        except lib_exc.NotFound:
            return True
        return False

    def clean_bfv_resource(self, volumes):
        """First clean snapshots attached to volume if any then delete volume.
        """
        for volume in volumes:
            if not volume or self.volume_not_deletable(volume):
                continue
            self._delete_attached_snapshots(volume)
            self.delete_volume(volume)

def set_block_device_mapping_args(image_ref, kwargs):
    """
    Update the kwargs dictionary with block device mapping.
    These arguments are required when booting instance from volume.
    """
    if "block_device_mapping_v2" in kwargs.keys() or \
       "block_device_mapping" in kwargs.keys():
        return kwargs
    if 'volume_size' in kwargs:
        vol_size = kwargs.pop('volume_size')
    else:
        vol_size = CONF.volume.volume_size
    bv_map = [{
            "source_type": "image",
            "destination_type": "volume",
            "delete_on_termination": "1",
            "boot_index": 0,
            "uuid": image_ref,
            "device_name": "vda",
            "volume_size": str(vol_size)}]
    bdm_args = {
    'block_device_mapping_v2' : bv_map,
    }
    kwargs.update(bdm_args)
    return kwargs

def get_cleanBFV_obj(auth_provider):
    default_params = {
        'disable_ssl_certificate_validation':
            CONF.identity.disable_ssl_certificate_validation,
        'ca_certs': CONF.identity.ca_certificates_file,
        'trace_requests': CONF.debug.trace_requests
    }
    params = {
        'service': CONF.volume.catalog_type,
        'region': CONF.volume.region or CONF.identity.region,
        'endpoint_type': CONF.volume.endpoint_type,
        'build_interval': CONF.volume.build_interval,
        'build_timeout': CONF.volume.build_timeout
    }
    params.update(default_params)
    return CleanBFVResource(auth_provider, **params)

<|endoftext|>


<commit_before>#!/bin/env python
#
#     analyse_illumina_run.py: analyse and report on Illumina sequencer runs
#     Copyright (C) University of Manchester 2013 Peter Briggs
#
"""analyse_illumina_run.py

Provides functionality for analysing data from an Illumina sequencer run.

"""

__version__ = "0.1.11.1"

#######################################################################
# Import modules
#######################################################################

import os
import sys
import optparse
import shutil
import logging
logging.basicConfig(format="%(levelname)s %(message)s")

# Put ../share onto Python search path for modules
SHARE_DIR = os.path.abspath(
    os.path.normpath(
        os.path.join(os.path.dirname(sys.argv[0]),'..','share')))
sys.path.append(SHARE_DIR)
import IlluminaData
import FASTQFile
import bcf_utils

#######################################################################
# Functions
#######################################################################

def describe_project(illumina_project):
    """Generate description string for samples in a project

    Description string gives the project name and a human-readable
    summary of the sample names, plus number of samples and whether
    the data is paired end.

    Example output: "Project Control: PhiX_1-2  (2 samples)"

    Arguments
      illumina_project: IlluminaProject instance

    Returns
      Description string.

    """
    # Gather information
    paired_end = illumina_project.paired_end
    n_samples = len(illumina_project.samples)
    multiple_fastqs_per_sample = False
    for s in illumina_project.samples:
        n_fastqs = len(s.fastq)
        if (paired_end and n_fastqs > 2) or (not paired_end and n_fastqs > 1):
            multiple_fastqs_per_sample = True
            break
    # Build description
    description = "%s: %s" % (illumina_project.name,
                              illumina_project.prettyPrintSamples())
    sample_names = project.prettyPrintSamples()
    description += " (%d " % n_samples
    if paired_end:
        description += "paired end "
    if n_samples == 1:
        description += "sample"
    else:
        description += "samples"
    if multiple_fastqs_per_sample:
        description += ", multiple fastqs per sample"
    description += ")"
    return "%s" % description

def summarise_projects(illumina_data):
    """Short summary of projects, suitable for logging file

    The summary description is a one line summary of the project names
    along with the number of samples in each, and an indication if the
    run was paired-ended.

    Arguments:
      illumina_data: a populated IlluminaData directory

    Returns:
      Summary description.

    """
    summary = ""
    project_summaries = []
    if illumina_data.paired_end:
        summary = "Paired end: "
    for project in illumina_data.projects:
        n_samples = len(project.samples)
        project_summaries.append("%s (%d sample%s)" % (project.name,
                                                       n_samples,
                                                       's' if n_samples != 1 else ''))
    summary += "; ".join(project_summaries)
    return summary

def verify_run_against_sample_sheet(illumina_data,sample_sheet):
    """Checks existence of predicted outputs from a sample sheet

    Arguments:
      illumina_data: a populated IlluminaData directory
      sample_sheet : path and name of a CSV sample sheet

    Returns:
      True if all the predicted outputs from the sample sheet are
      found, False otherwise.

    """
    # Get predicted outputs
    predicted_projects = IlluminaData.\
        CasavaSampleSheet(sample_sheet).predict_output()
    # Loop through projects and check that predicted outputs exist
    verified = True
    for proj in predicted_projects:
        # Locate project directory
        proj_dir = os.path.join(illumina_data.unaligned_dir,proj)
        if os.path.isdir(proj_dir):
            predicted_samples = predicted_projects[proj]
            for smpl in predicted_samples:
                # Locate sample directory
                smpl_dir = os.path.join(proj_dir,smpl)
                if os.path.isdir(smpl_dir):
                    # Check for output files
                    predicted_names = predicted_samples[smpl]
                    for name in predicted_names:
                        # Look for R1 file
                        f = os.path.join(smpl_dir,"%s_R1_001.fastq.gz" % name)
                        if not os.path.exists(f):
                            logging.warning("Verify: missing R1 file '%s'" % f)
                            verified = False
                        # Look for R2 file (paired end only)
                        if illumina_data.paired_end:
                            f = os.path.join(smpl_dir,"%s_R2_001.fastq.gz" % name)
                            if not os.path.exists(f):
                                logging.warning("Verify: missing R2 file '%s'" % f)
                                verified = False
                else:
                    # Sample directory not found
                    logging.warning("Verify: missing %s" % smpl_dir)
                    verified = False
        else:
            # Project directory not found
            logging.warning("Verify: missing %s" % proj_dir)
            verified = False
    # Return verification status
    return verified

#######################################################################
# Main program
#######################################################################

if __name__ == "__main__":
    # Create command line parser
    p = optparse.OptionParser(usage="%prog OPTIONS illumina_data_dir",
                              version="%prog "+__version__,
                              description="Utility for performing various checks and "
                              "operations on Illumina data. 'illumina_data_dir' is the "
                              "top-level directory containing the 'Unaligned' directory "
                              "with the fastq.gz files.")
    p.add_option("--report",action="store_true",dest="report",
                 help="report sample names and number of samples for each project")
    p.add_option("--summary",action="store_true",dest="summary",
                 help="short report of samples (suitable for logging file)")
    p.add_option("-l","--list",action="store_true",dest="list",
                 help="list projects, samples and fastq files directories")
    p.add_option("--unaligned",action="store",dest="unaligned_dir",default="Unaligned",
                 help="specify an alternative name for the 'Unaligned' directory "
                 "containing the fastq.gz files")
    p.add_option("--copy",action="store",dest="copy_pattern",default=None,
                 help="copy fastq.gz files matching COPY_PATTERN to current directory")
    p.add_option("--merge-fastqs",action="store_true",dest="merge_fastqs",
                 help="Merge multiple fastqs for samples")
    p.add_option("--verify",action="store",dest="sample_sheet",default=None,
                 help="check CASAVA outputs against those expected for SAMPLE_SHEET")
    p.add_option("--stats",action="store_true",dest="stats",
                 help="Report statistics (read counts etc) for fastq files")
    # Parse command line
    options,args = p.parse_args()

    # Get data directory name
    if len(args) != 1:
        p.error("expected one argument (location of Illumina analysis dir)")
    illumina_analysis_dir = os.path.abspath(args[0])

    # Populate Illumina data object
    try:
        illumina_data = IlluminaData.IlluminaData(illumina_analysis_dir,
                                                  unaligned_dir=options.unaligned_dir)
    except IlluminaData.IlluminaDataError, ex:
        logging.error("Failed to collect data: %s",ex)
        sys.exit(1)

    # Check there's at least one thing to do
    if not (options.report or
            options.summary or
            options.list or
            options.sample_sheet or
            options.merge_fastqs):
        options.report = True

    # List option
    if options.list:
        for project in illumina_data.projects:
            n_samples = len(project.samples)
            print "Project: %s (%d sample%s)" % (project.name,
                                                 n_samples,
                                                 's' if n_samples != 1 else '')
            for sample in project.samples:
                if sample.paired_end:
                    n_fastq_pairs = len(sample.fastq_subset(read_number=1))
                    if n_fastq_pairs == 1:
                        print "\t%s (R1/R2 pair)" % sample.name
                    else:
                        print "\t%s (%d fastq R1/R2 pairs)" % \
                            (sample.name,n_fastq_pairs)
                else:
                    n_fastqs = len(sample.fastq)
                    if n_fastqs == 1:
                        print "\t%s" % sample.name
                    else:
                        print "\t%s (%d fastqs)" % (sample.name,n_fastqs)
                # Print fastq names
                fastqs = sample.fastq_subset(read_number=1) + \
                         sample.fastq_subset(read_number=2)
                for fastq in fastqs:
                    print "\t\t%s" % fastq

    # Report the names of the samples in each project
    if options.report:
        for project in illumina_data.projects:
            print "%s" % describe_project(project)
            # Report statistics for fastq files
            if options.stats:
                # Print number of reads for each file, and file size
                for sample in project.samples:
                    for fastq in sample.fastq:
                        fq = os.path.join(sample.dirn,fastq)
                        nreads = FASTQFile.nreads(fq)
                        fsize = os.path.getsize(fq)
                        print "%s\t%s\t%d" % (fastq,
                                              bcf_utils.format_file_size(fsize),
                                              nreads)
            print ""

    # Summary: short report suitable for logging file
    if options.summary:
        print "%s" % summarise_projects(illumina_data)

    # Print number of undetermined reads
    if options.stats and illumina_data.undetermined is not None:
        print "Undetermined indices"
        for lane in illumina_data.undetermined.samples:
            for fastq in lane.fastq:
                fq = os.path.join(lane.dirn,fastq)
                nreads = FASTQFile.nreads(fq)
                fsize = os.path.getsize(fq)
                print "%s\t%s\t%d" % (fastq,
                                  bcf_utils.format_file_size(fsize),
                                  nreads)

    # Copy fastq.gz files to the current directory
    if options.copy_pattern is not None:
        # Extract project and sample names/patterns
        try:
            project_pattern,sample_pattern = options.copy_pattern.split("/")
            print "Copy: look for samples matching pattern %s" % options.copy_pattern
            print "Data files will be copied to %s" % os.getcwd()
        except ValueError:
            logging.error("ERROR invalid pattern '%s'" % options.copy_pattern)
            sys.exit(1)
        # Loop through projects and samples looking for matches
        for project in illumina_data.projects:
            if bcf_utils.name_matches(project.name,project_pattern):
                # Loop through samples
                for sample in project.samples:
                    if bcf_utils.name_matches(sample.name,sample_pattern):
                        for fastq in sample.fastq:
                            fastq_file = os.path.join(sample.dirn,fastq)
                            print "\tCopying .../%s" % os.path.basename(fastq_file)
                            dst = os.path.abspath(os.path.basename(fastq_file))
                            if os.path.exists(dst):
                                logging.error("File %s already exists! Skipped" % dst)
                            else:
                                shutil.copy(fastq_file,dst)

    # Verify against sample sheet
    if options.sample_sheet is not None:
        if verify_run_against_sample_sheet(illumina_data,options.sample_sheet):
            print "Verification against sample sheet '%s': OK" % \
                options.sample_sheet
            status = 0
        else:
            logging.error("Verification against sample sheet '%s': FAILED" %
                          options.sample_sheet)
            status = 1
        sys.exit(status)

    # Merge multiple fastqs in each sample
    if options.merge_fastqs:
        for project in illumina_data.projects:
            for sample in project.samples:
                for read in (1,2):
                    # Concatenate fastqs for this read
                    fastq_merged = sample.name
                    if sample.paired_end:
                        fastq_merged += "_R%d" % read
                    fastq_merged += ".fastq.gz"
                    bcf_utils.concatenate_fastq_files(fastq_merged,
                                                      sample.fastq_subset(read_number=read,
                                                                          full_path=True),
                                                      bufsize=1024*1024)

<commit_msg>illumina2cluster/analyse_illumina_run.py 0.1.11.2: fix bug in describe_project().<commit_after>#!/bin/env python
#
#     analyse_illumina_run.py: analyse and report on Illumina sequencer runs
#     Copyright (C) University of Manchester 2013 Peter Briggs
#
"""analyse_illumina_run.py

Provides functionality for analysing data from an Illumina sequencer run.

"""

__version__ = "0.1.11.2"

#######################################################################
# Import modules
#######################################################################

import os
import sys
import optparse
import shutil
import logging
logging.basicConfig(format="%(levelname)s %(message)s")

# Put ../share onto Python search path for modules
SHARE_DIR = os.path.abspath(
    os.path.normpath(
        os.path.join(os.path.dirname(sys.argv[0]),'..','share')))
sys.path.append(SHARE_DIR)
import IlluminaData
import FASTQFile
import bcf_utils

#######################################################################
# Functions
#######################################################################

def describe_project(illumina_project):
    """Generate description string for samples in a project

    Description string gives the project name and a human-readable
    summary of the sample names, plus number of samples and whether
    the data is paired end.

    Example output: "Project Control: PhiX_1-2  (2 samples)"

    Arguments
      illumina_project: IlluminaProject instance

    Returns
      Description string.

    """
    # Gather information
    paired_end = illumina_project.paired_end
    n_samples = len(illumina_project.samples)
    multiple_fastqs_per_sample = False
    for s in illumina_project.samples:
        n_fastqs = len(s.fastq)
        if (paired_end and n_fastqs > 2) or (not paired_end and n_fastqs > 1):
            multiple_fastqs_per_sample = True
            break
    # Build description
    description = "%s: %s" % (illumina_project.name,
                              illumina_project.prettyPrintSamples())
    sample_names = illumina_project.prettyPrintSamples()
    description += " (%d " % n_samples
    if paired_end:
        description += "paired end "
    if n_samples == 1:
        description += "sample"
    else:
        description += "samples"
    if multiple_fastqs_per_sample:
        description += ", multiple fastqs per sample"
    description += ")"
    return "%s" % description

def summarise_projects(illumina_data):
    """Short summary of projects, suitable for logging file

    The summary description is a one line summary of the project names
    along with the number of samples in each, and an indication if the
    run was paired-ended.

    Arguments:
      illumina_data: a populated IlluminaData directory

    Returns:
      Summary description.

    """
    summary = ""
    project_summaries = []
    if illumina_data.paired_end:
        summary = "Paired end: "
    for project in illumina_data.projects:
        n_samples = len(project.samples)
        project_summaries.append("%s (%d sample%s)" % (project.name,
                                                       n_samples,
                                                       's' if n_samples != 1 else ''))
    summary += "; ".join(project_summaries)
    return summary

def verify_run_against_sample_sheet(illumina_data,sample_sheet):
    """Checks existence of predicted outputs from a sample sheet

    Arguments:
      illumina_data: a populated IlluminaData directory
      sample_sheet : path and name of a CSV sample sheet

    Returns:
      True if all the predicted outputs from the sample sheet are
      found, False otherwise.

    """
    # Get predicted outputs
    predicted_projects = IlluminaData.\
        CasavaSampleSheet(sample_sheet).predict_output()
    # Loop through projects and check that predicted outputs exist
    verified = True
    for proj in predicted_projects:
        # Locate project directory
        proj_dir = os.path.join(illumina_data.unaligned_dir,proj)
        if os.path.isdir(proj_dir):
            predicted_samples = predicted_projects[proj]
            for smpl in predicted_samples:
                # Locate sample directory
                smpl_dir = os.path.join(proj_dir,smpl)
                if os.path.isdir(smpl_dir):
                    # Check for output files
                    predicted_names = predicted_samples[smpl]
                    for name in predicted_names:
                        # Look for R1 file
                        f = os.path.join(smpl_dir,"%s_R1_001.fastq.gz" % name)
                        if not os.path.exists(f):
                            logging.warning("Verify: missing R1 file '%s'" % f)
                            verified = False
                        # Look for R2 file (paired end only)
                        if illumina_data.paired_end:
                            f = os.path.join(smpl_dir,"%s_R2_001.fastq.gz" % name)
                            if not os.path.exists(f):
                                logging.warning("Verify: missing R2 file '%s'" % f)
                                verified = False
                else:
                    # Sample directory not found
                    logging.warning("Verify: missing %s" % smpl_dir)
                    verified = False
        else:
            # Project directory not found
            logging.warning("Verify: missing %s" % proj_dir)
            verified = False
    # Return verification status
    return verified

#######################################################################
# Main program
#######################################################################

if __name__ == "__main__":
    # Create command line parser
    p = optparse.OptionParser(usage="%prog OPTIONS illumina_data_dir",
                              version="%prog "+__version__,
                              description="Utility for performing various checks and "
                              "operations on Illumina data. 'illumina_data_dir' is the "
                              "top-level directory containing the 'Unaligned' directory "
                              "with the fastq.gz files.")
    p.add_option("--report",action="store_true",dest="report",
                 help="report sample names and number of samples for each project")
    p.add_option("--summary",action="store_true",dest="summary",
                 help="short report of samples (suitable for logging file)")
    p.add_option("-l","--list",action="store_true",dest="list",
                 help="list projects, samples and fastq files directories")
    p.add_option("--unaligned",action="store",dest="unaligned_dir",default="Unaligned",
                 help="specify an alternative name for the 'Unaligned' directory "
                 "containing the fastq.gz files")
    p.add_option("--copy",action="store",dest="copy_pattern",default=None,
                 help="copy fastq.gz files matching COPY_PATTERN to current directory")
    p.add_option("--merge-fastqs",action="store_true",dest="merge_fastqs",
                 help="Merge multiple fastqs for samples")
    p.add_option("--verify",action="store",dest="sample_sheet",default=None,
                 help="check CASAVA outputs against those expected for SAMPLE_SHEET")
    p.add_option("--stats",action="store_true",dest="stats",
                 help="Report statistics (read counts etc) for fastq files")
    # Parse command line
    options,args = p.parse_args()

    # Get data directory name
    if len(args) != 1:
        p.error("expected one argument (location of Illumina analysis dir)")
    illumina_analysis_dir = os.path.abspath(args[0])

    # Populate Illumina data object
    try:
        illumina_data = IlluminaData.IlluminaData(illumina_analysis_dir,
                                                  unaligned_dir=options.unaligned_dir)
    except IlluminaData.IlluminaDataError, ex:
        logging.error("Failed to collect data: %s",ex)
        sys.exit(1)

    # Check there's at least one thing to do
    if not (options.report or
            options.summary or
            options.list or
            options.sample_sheet or
            options.merge_fastqs):
        options.report = True

    # List option
    if options.list:
        for project in illumina_data.projects:
            n_samples = len(project.samples)
            print "Project: %s (%d sample%s)" % (project.name,
                                                 n_samples,
                                                 's' if n_samples != 1 else '')
            for sample in project.samples:
                if sample.paired_end:
                    n_fastq_pairs = len(sample.fastq_subset(read_number=1))
                    if n_fastq_pairs == 1:
                        print "\t%s (R1/R2 pair)" % sample.name
                    else:
                        print "\t%s (%d fastq R1/R2 pairs)" % \
                            (sample.name,n_fastq_pairs)
                else:
                    n_fastqs = len(sample.fastq)
                    if n_fastqs == 1:
                        print "\t%s" % sample.name
                    else:
                        print "\t%s (%d fastqs)" % (sample.name,n_fastqs)
                # Print fastq names
                fastqs = sample.fastq_subset(read_number=1) + \
                         sample.fastq_subset(read_number=2)
                for fastq in fastqs:
                    print "\t\t%s" % fastq

    # Report the names of the samples in each project
    if options.report:
        for project in illumina_data.projects:
            print "%s" % describe_project(project)
            # Report statistics for fastq files
            if options.stats:
                # Print number of reads for each file, and file size
                for sample in project.samples:
                    for fastq in sample.fastq:
                        fq = os.path.join(sample.dirn,fastq)
                        nreads = FASTQFile.nreads(fq)
                        fsize = os.path.getsize(fq)
                        print "%s\t%s\t%d" % (fastq,
                                              bcf_utils.format_file_size(fsize),
                                              nreads)
            print ""

    # Summary: short report suitable for logging file
    if options.summary:
        print "%s" % summarise_projects(illumina_data)

    # Print number of undetermined reads
    if options.stats and illumina_data.undetermined is not None:
        print "Undetermined indices"
        for lane in illumina_data.undetermined.samples:
            for fastq in lane.fastq:
                fq = os.path.join(lane.dirn,fastq)
                nreads = FASTQFile.nreads(fq)
                fsize = os.path.getsize(fq)
                print "%s\t%s\t%d" % (fastq,
                                  bcf_utils.format_file_size(fsize),
                                  nreads)

    # Copy fastq.gz files to the current directory
    if options.copy_pattern is not None:
        # Extract project and sample names/patterns
        try:
            project_pattern,sample_pattern = options.copy_pattern.split("/")
            print "Copy: look for samples matching pattern %s" % options.copy_pattern
            print "Data files will be copied to %s" % os.getcwd()
        except ValueError:
            logging.error("ERROR invalid pattern '%s'" % options.copy_pattern)
            sys.exit(1)
        # Loop through projects and samples looking for matches
        for project in illumina_data.projects:
            if bcf_utils.name_matches(project.name,project_pattern):
                # Loop through samples
                for sample in project.samples:
                    if bcf_utils.name_matches(sample.name,sample_pattern):
                        for fastq in sample.fastq:
                            fastq_file = os.path.join(sample.dirn,fastq)
                            print "\tCopying .../%s" % os.path.basename(fastq_file)
                            dst = os.path.abspath(os.path.basename(fastq_file))
                            if os.path.exists(dst):
                                logging.error("File %s already exists! Skipped" % dst)
                            else:
                                shutil.copy(fastq_file,dst)

    # Verify against sample sheet
    if options.sample_sheet is not None:
        if verify_run_against_sample_sheet(illumina_data,options.sample_sheet):
            print "Verification against sample sheet '%s': OK" % \
                options.sample_sheet
            status = 0
        else:
            logging.error("Verification against sample sheet '%s': FAILED" %
                          options.sample_sheet)
            status = 1
        sys.exit(status)

    # Merge multiple fastqs in each sample
    if options.merge_fastqs:
        for project in illumina_data.projects:
            for sample in project.samples:
                for read in (1,2):
                    # Concatenate fastqs for this read
                    fastq_merged = sample.name
                    if sample.paired_end:
                        fastq_merged += "_R%d" % read
                    fastq_merged += ".fastq.gz"
                    bcf_utils.concatenate_fastq_files(fastq_merged,
                                                      sample.fastq_subset(read_number=read,
                                                                          full_path=True),
                                                      bufsize=1024*1024)

<|endoftext|>


<commit_before>from ..worker import config

from performanceplatform.client import AdminAPI

from .util import encode_id, group_by, is_latest_data

REQUIRED_DATA_POINTS = [
    {'name': "cost_per_transaction", 'ignore': 'quarterly'},
    {'name': "digital_cost_per_transaction", 'ignore': None},
    {'name': "digital_takeup", 'ignore': None},
    {'name': "number_of_digital_transactions", 'ignore': 'quarterly'},
    {'name': "number_of_transactions", 'ignore': 'quarterly'},
    {'name': "total_cost", 'ignore': 'quarterly'},
]

ADDITIONAL_FIELDS = [
    "end_at",
    "period",
    "service_id",
    "type"
]

REQUIRED_FIELDS = [
    "_timestamp",
]

admin_api = AdminAPI(
    config.STAGECRAFT_URL,
    config.STAGECRAFT_OAUTH_TOKEN)

def _get_latest_data_point(data, data_point_name):
    def _use_data_point(data_point, name, ignore):
        has_data = (name in data_point and data_point[name])
        should_not_be_ignored = (ignore != data_point['type'])
        return has_data and should_not_be_ignored

    name = data_point_name['name']
    ignore = data_point_name['ignore']

    data.sort(key=lambda item: item['_timestamp'], reverse=True)
    for data_point in data:
        if _use_data_point(data_point, name, ignore):
            return data_point
    return None

def _get_stripped_down_data_for_data_point_name_only(
        dashboard_config,
        latest_data_points,
        data_point_name):
    """
    Builds up backdrop ready datum for a single transaction explorer metric.

    It does this by iterating through the passed in data_point_name
    and all the REQUIRED_FIELDS and building up a new dict based on
    these key: value pairings. We then loop through additional fields and add
    those if present. If a REQUIRED_FIELD is not found we return None for
    this data_point.
    """
    required_fields = REQUIRED_FIELDS + [data_point_name['name']]
    new_data = {}
    for field in required_fields:
        if field in latest_data_points:
            new_data[field] = latest_data_points[field]
        else:
            return None
    for field in ADDITIONAL_FIELDS:
        if field in latest_data_points:
            new_data[field] = latest_data_points[field]
    new_data['dashboard_slug'] = dashboard_config['slug']
    new_data['_id'] = encode_id(
        new_data['dashboard_slug'],
        data_point_name['name'])
    return new_data

def _service_ids_with_data(data):
    return group_by('service_id', data).items()

def _get_dashboard_config(service_id):
    return admin_api.get_dashboard_by_tx_id(service_id)

def _get_dashboard_configs_with_data(ids_with_data):
    dashboard_configs_with_data = []
    for service_id, data in ids_with_data:
        dashboard_configs = _get_dashboard_config(service_id)
        for dashboard_config in dashboard_configs:
            dashboard_configs_with_data.append((dashboard_config, data))
    return dashboard_configs_with_data

def _get_data_points_for_each_tx_metric(data, transform, data_set_config):
    ids_with_data = _service_ids_with_data(
        data)
    dashboard_configs_with_data = _get_dashboard_configs_with_data(
        ids_with_data)
    for data_point_name in REQUIRED_DATA_POINTS:
        for dashboard_config, dashboard_data in dashboard_configs_with_data:
            latest_data = _get_latest_data_point(
                dashboard_data,
                data_point_name)
            if not latest_data:
                continue
            datum = _get_stripped_down_data_for_data_point_name_only(
                dashboard_config, latest_data, data_point_name)
            # we need to look at whether this is later than the latest
            # data currently present on the output data set as
            # for things like digital-takeup the  transactions explorer
            # dataset is not the only source.
            if datum and is_latest_data(
                    {'data_group': transform['output']['data-group'],
                     'data_type': transform['output']['data-type']},
                    transform,
                    datum,
                    additional_read_params={
                        'filter_by': 'dashboard_slug:{}'.format(
                            datum['dashboard_slug'])}):
                yield datum

def compute(data, transform, data_set_config=None):
    return [datum for datum
            in _get_data_points_for_each_tx_metric(
                data,
                transform,
                data_set_config)]
<commit_msg>Filter by id - there is > 1 data point per slug<commit_after>from ..worker import config

from performanceplatform.client import AdminAPI

from .util import encode_id, group_by, is_latest_data

REQUIRED_DATA_POINTS = [
    {'name': "cost_per_transaction", 'ignore': 'quarterly'},
    {'name': "digital_cost_per_transaction", 'ignore': None},
    {'name': "digital_takeup", 'ignore': None},
    {'name': "number_of_digital_transactions", 'ignore': 'quarterly'},
    {'name': "number_of_transactions", 'ignore': 'quarterly'},
    {'name': "total_cost", 'ignore': 'quarterly'},
]

ADDITIONAL_FIELDS = [
    "end_at",
    "period",
    "service_id",
    "type"
]

REQUIRED_FIELDS = [
    "_timestamp",
]

admin_api = AdminAPI(
    config.STAGECRAFT_URL,
    config.STAGECRAFT_OAUTH_TOKEN)

def _get_latest_data_point(data, data_point_name):
    def _use_data_point(data_point, name, ignore):
        has_data = (name in data_point and data_point[name])
        should_not_be_ignored = (ignore != data_point['type'])
        return has_data and should_not_be_ignored

    name = data_point_name['name']
    ignore = data_point_name['ignore']

    data.sort(key=lambda item: item['_timestamp'], reverse=True)
    for data_point in data:
        if _use_data_point(data_point, name, ignore):
            return data_point
    return None

def _get_stripped_down_data_for_data_point_name_only(
        dashboard_config,
        latest_data_points,
        data_point_name):
    """
    Builds up backdrop ready datum for a single transaction explorer metric.

    It does this by iterating through the passed in data_point_name
    and all the REQUIRED_FIELDS and building up a new dict based on
    these key: value pairings. We then loop through additional fields and add
    those if present. If a REQUIRED_FIELD is not found we return None for
    this data_point.
    """
    required_fields = REQUIRED_FIELDS + [data_point_name['name']]
    new_data = {}
    for field in required_fields:
        if field in latest_data_points:
            new_data[field] = latest_data_points[field]
        else:
            return None
    for field in ADDITIONAL_FIELDS:
        if field in latest_data_points:
            new_data[field] = latest_data_points[field]
    new_data['dashboard_slug'] = dashboard_config['slug']
    new_data['_id'] = encode_id(
        new_data['dashboard_slug'],
        data_point_name['name'])
    return new_data

def _service_ids_with_data(data):
    return group_by('service_id', data).items()

def _get_dashboard_config(service_id):
    return admin_api.get_dashboard_by_tx_id(service_id)

def _get_dashboard_configs_with_data(ids_with_data):
    dashboard_configs_with_data = []
    for service_id, data in ids_with_data:
        dashboard_configs = _get_dashboard_config(service_id)
        for dashboard_config in dashboard_configs:
            dashboard_configs_with_data.append((dashboard_config, data))
    return dashboard_configs_with_data

def _get_data_points_for_each_tx_metric(data, transform, data_set_config):
    ids_with_data = _service_ids_with_data(
        data)
    dashboard_configs_with_data = _get_dashboard_configs_with_data(
        ids_with_data)
    for data_point_name in REQUIRED_DATA_POINTS:
        for dashboard_config, dashboard_data in dashboard_configs_with_data:
            latest_data = _get_latest_data_point(
                dashboard_data,
                data_point_name)
            if not latest_data:
                continue
            datum = _get_stripped_down_data_for_data_point_name_only(
                dashboard_config, latest_data, data_point_name)
            # we need to look at whether this is later than the latest
            # data currently present on the output data set as
            # for things like digital-takeup the  transactions explorer
            # dataset is not the only source.
            if datum and is_latest_data(
                    {'data_group': transform['output']['data-group'],
                     'data_type': transform['output']['data-type']},
                    transform,
                    datum,
                    # filter by record id as this is a hash of dashboard_slug
                    # and data_point_name and is therefore the important
                    # identifier of newer data
                    additional_read_params={
                        'filter_by': '_id:{}'.format(
                            datum['_id'])}):
                yield datum

def compute(data, transform, data_set_config=None):
    return [datum for datum
            in _get_data_points_for_each_tx_metric(
                data,
                transform,
                data_set_config)]
<|endoftext|>


<commit_before>import time

import etcd

from tendrl.monitoring_integration.grafana import create_alert_dashboard
from tendrl.monitoring_integration.grafana import dashboard
from tendrl.monitoring_integration.flows.update_dashboard import alert_utils
from tendrl.monitoring_integration.grafana import create_dashboards
from tendrl.monitoring_integration.flows.update_dashboard import \
    UpdateDashboard
from tendrl.monitoring_integration.grafana import create_dashboards
from tendrl.commons.utils import log_utils as logger

DASHBOARDS = ["volumes", "nodes", "bricks"]

class SyncDashboard():

    def refresh_dashboard(self):
        cluster_detail_list = []
        try:
            cluster_detail_list = create_dashboards.get_cluster_details()
        except etcd.EtcdKeyNotFound:
            logger.log("error", NS.get("publisher_id", None),
                           {'message': "Failed to get cluster "
                            "details"})
            return
        if cluster_detail_list:
            for dashboard_name in DASHBOARDS:
                alert_dashboard = alert_utils.get_alert_dashboard(dashboard_name)
                flag = False

                if alert_dashboard:
                    try:
                        if alert_dashboard["message"] == "Dashboard not found":
                            flag = False
                        else:
                            flag = True
                    except (KeyError, AttributeError):
                        flag = True
                if flag:
                    try:
                        if len(alert_dashboard["dashboard"]["rows"]) == 0 or \
                            len(alert_dashboard["dashboard"]["rows"][0]["panels"]) == 0:
                            alert_utils.delete_alert_dashboard(dashboard_name)
                            self.create_all_dashboard(dashboard_name, cluster_detail_list)
                        else:
                            self.create_resource(
                                cluster_detail_list, dashboard_name)
                    except (KeyError, AttributeError):
                        alert_utils.delete_alert_dashboard(dashboard_name)
                        self.create_all_dashboard(dashboard_name, cluster_detail_list)

                else:
                    self.create_all_dashboard(dashboard_name, cluster_detail_list)

    def create_all_dashboard(self, dashboard_name, cluster_detail_list):
        if dashboard_name == "nodes":
            dashboard_name = "hosts"
        try:
            create_alert_dashboard.CreateAlertDashboard(dashboard_name, cluster_detail_list)
        except (etcd.EtcdKeyNotFound, KeyError) as error:
            logger.log("error", NS.get("publisher_id", None),
                       {'message': "Failed to create dashboard"
                           "with error {0}".format(str(error))})

    def create_resource(self, cluster_detail_list, resource_type):
        update_dashboard = UpdateDashboard()
        self.map = {"cluster": "at-a-glance", "host": "nodes",
                    "volume": "volumes", "brick": "bricks"}
        for cluster in cluster_detail_list:
            integration_id = cluster.integration_id
            if resource_type == "volumes":
                for volume in cluster.volumes:
                    time.sleep(3)
                    try:
                        resource_name = str(volume["name"])
                        response = update_dashboard._add_panel(
                           integration_id,
                           self.map["volume"],
                           resource_name)
                        self.log_message(response, resource_name, "volume")
                    except KeyError:
                        logger.log("error", NS.get("publisher_id", None),
                                   {'message': "Failed to get volume {} "
                                      "details".format(volume)})
            if resource_type == "nodes":
                for host in cluster.hosts:
                    time.sleep(3)
                    try:
                        resource_name = str(host["fqdn"])
                        response = update_dashboard._add_panel(
                            integration_id,
                            self.map["host"],
                            resource_name)
                        self.log_message(response, resource_name, "host")
                    except KeyError:
                        logger.log("error", NS.get("publisher_id", None),
                                   {'message': "Failed to get host {} "
                                       "details".format(host)})

            if resource_type == "bricks":
                for brick in cluster.bricks:
                    time.sleep(3)
                    try:
                        resource_name = "%s|%s:%s" % (
                            str(brick["vol_name"]),
                            brick["hostname"],
                            brick["brick_path"].replace("|", "/")
                        )
                        response = update_dashboard._add_panel(
                            integration_id,
                            self.map["brick"],
                            resource_name)
                        self.log_message(response, resource_name, "brick")
                    except KeyError:
                        logger.log("error", NS.get("publisher_id", None),
                                   {'message': "Failed to get brick {} "
                                       "details".format(brick)})

    def log_message(self, response, resource_name, resource_type):
        try:
            if response.status_code == 200:
                msg = '\n' + "Dashboard for resource {0} uploaded " + \
                    "successfully for resource type {1}".format(resource_name,
                                                                resource_type)
            else:
                msg = '\n' + "Dashboard for resource {0} upload failed  " + \
                    "for resource type {1}".format(resource_name,
                                                   resource_type)

            logger.log("info", NS.get("publisher_id", None),
                       {'message': msg})
        except (KeyError, AttributeError):
            pass
<commit_msg>Update dashbaord_sync.py<commit_after>import time

import etcd

from tendrl.monitoring_integration.grafana import create_alert_dashboard
from tendrl.monitoring_integration.grafana import dashboard
from tendrl.monitoring_integration.flows.update_dashboard import alert_utils
from tendrl.monitoring_integration.grafana import create_dashboards
from tendrl.monitoring_integration.flows.update_dashboard import \
    UpdateDashboard
from tendrl.monitoring_integration.grafana import create_dashboards
from tendrl.commons.utils import log_utils as logger

DASHBOARDS = ["volumes", "nodes", "bricks"]

class SyncDashboard():

    def refresh_dashboard(self):
        cluster_detail_list = []
        try:
            cluster_detail_list = create_dashboards.get_cluster_details()
        except etcd.EtcdKeyNotFound:
            logger.log("error", NS.get("publisher_id", None),
                           {'message': "Failed to get cluster "
                            "details"})
            return
        if cluster_detail_list:
            for dashboard_name in DASHBOARDS:
                alert_dashboard = alert_utils.get_alert_dashboard(dashboard_name)
                flag = False

                if alert_dashboard:
                    try:
                        if alert_dashboard["message"] == "Dashboard not found":
                            flag = False
                        else:
                            flag = True
                    except (KeyError, AttributeError):
                        flag = True
                if flag:
                    try:
                        if len(alert_dashboard["dashboard"]["rows"]) == 0 or \
                            len(alert_dashboard["dashboard"]["rows"][0]["panels"]) == 0:
                            alert_utils.delete_alert_dashboard(dashboard_name)
                            self.create_all_dashboard(dashboard_name, cluster_detail_list)
                        else:
                            self.create_resource(
                                cluster_detail_list, dashboard_name)
                    except (KeyError, AttributeError):
                        alert_utils.delete_alert_dashboard(dashboard_name)
                        self.create_all_dashboard(dashboard_name, cluster_detail_list)

                else:
                    self.create_all_dashboard(dashboard_name, cluster_detail_list)

    def create_all_dashboard(self, dashboard_name, cluster_detail_list):
        if dashboard_name == "nodes":
            dashboard_name = "hosts"
        try:
            create_alert_dashboard.CreateAlertDashboard(dashboard_name, cluster_detail_list)
        except (etcd.EtcdKeyNotFound, KeyError) as error:
            logger.log("error", NS.get("publisher_id", None),
                       {'message': "Failed to create dashboard"
                           "with error {0}".format(str(error))})

    def create_resource(self, cluster_detail_list, resource_type):
        update_dashboard = UpdateDashboard()
        self.map = {"cluster": "at-a-glance", "host": "nodes",
                    "volume": "volumes", "brick": "bricks"}
        for cluster in cluster_detail_list:
            integration_id = cluster.integration_id
            if resource_type == "volumes":
                for volume in cluster.volumes:
                    try:
                        resource_name = str(volume["name"])
                        response = update_dashboard._add_panel(
                           integration_id,
                           self.map["volume"],
                           resource_name)
                        self.log_message(response, resource_name, "volume")
                    except KeyError:
                        logger.log("error", NS.get("publisher_id", None),
                                   {'message': "Failed to get volume {} "
                                      "details".format(volume)})
            if resource_type == "nodes":
                for host in cluster.hosts:
                    try:
                        resource_name = str(host["fqdn"])
                        response = update_dashboard._add_panel(
                            integration_id,
                            self.map["host"],
                            resource_name)
                        self.log_message(response, resource_name, "host")
                    except KeyError:
                        logger.log("error", NS.get("publisher_id", None),
                                   {'message': "Failed to get host {} "
                                       "details".format(host)})

            if resource_type == "bricks":
                for brick in cluster.bricks:
                    try:
                        resource_name = "%s|%s:%s" % (
                            str(brick["vol_name"]),
                            brick["hostname"],
                            brick["brick_path"].replace("|", "/")
                        )
                        response = update_dashboard._add_panel(
                            integration_id,
                            self.map["brick"],
                            resource_name)
                        self.log_message(response, resource_name, "brick")
                    except KeyError:
                        logger.log("error", NS.get("publisher_id", None),
                                   {'message': "Failed to get brick {} "
                                       "details".format(brick)})

    def log_message(self, response, resource_name, resource_type):
        try:
            if response.status_code == 200:
                msg = '\n' + "Dashboard for resource {0} uploaded " + \
                    "successfully for resource type {1}".format(resource_name,
                                                                resource_type)
            else:
                msg = '\n' + "Dashboard for resource {0} upload failed  " + \
                    "for resource type {1}".format(resource_name,
                                                   resource_type)

            logger.log("info", NS.get("publisher_id", None),
                       {'message': msg})
        except (KeyError, AttributeError):
            pass
<|endoftext|>


<commit_before><commit_msg>Added couch_id column to auditcare tables<commit_after><|endoftext|>