<commit_before>from __future__ import division, print_function, absolute_import

import numpy as np
from numpy.testing import assert_array_almost_equal, assert_array_equal
from pytest import raises as assert_raises
from scipy.sparse.csgraph import (shortest_path, dijkstra, johnson,
                                  bellman_ford, construct_dist_matrix,
                                  NegativeCycleError)
import scipy.sparse
import pytest

directed_G = np.array([[0, 3, 3, 0, 0],
                       [0, 0, 0, 2, 4],
                       [0, 0, 0, 0, 0],
                       [1, 0, 0, 0, 0],
                       [2, 0, 0, 2, 0]], dtype=float)

undirected_G = np.array([[0, 3, 3, 1, 2],
                         [3, 0, 0, 2, 4],
                         [3, 0, 0, 0, 0],
                         [1, 2, 0, 0, 2],
                         [2, 4, 0, 2, 0]], dtype=float)

unweighted_G = (directed_G > 0).astype(float)

directed_SP = [[0, 3, 3, 5, 7],
               [3, 0, 6, 2, 4],
               [np.inf, np.inf, 0, np.inf, np.inf],
               [1, 4, 4, 0, 8],
               [2, 5, 5, 2, 0]]

directed_sparse_zero_G = scipy.sparse.csr_matrix(([0, 1, 2, 3, 1],
                                            ([0, 1, 2, 3, 4],
                                             [1, 2, 0, 4, 3])),
                                            shape = (5, 5))

directed_sparse_zero_SP = [[0, 0, 1, np.inf, np.inf],
                      [3, 0, 1, np.inf, np.inf],
                      [2, 2, 0, np.inf, np.inf],
                      [np.inf, np.inf, np.inf, 0, 3],
                      [np.inf, np.inf, np.inf, 1, 0]]

undirected_sparse_zero_G = scipy.sparse.csr_matrix(([0, 0, 1, 1, 2, 2, 1, 1],
                                              ([0, 1, 1, 2, 2, 0, 3, 4],
                                               [1, 0, 2, 1, 0, 2, 4, 3])),
                                              shape = (5, 5))

undirected_sparse_zero_SP = [[0, 0, 1, np.inf, np.inf],
                        [0, 0, 1, np.inf, np.inf],
                        [1, 1, 0, np.inf, np.inf],
                        [np.inf, np.inf, np.inf, 0, 1],
                        [np.inf, np.inf, np.inf, 1, 0]]

directed_pred = np.array([[-9999, 0, 0, 1, 1],
                          [3, -9999, 0, 1, 1],
                          [-9999, -9999, -9999, -9999, -9999],
                          [3, 0, 0, -9999, 1],
                          [4, 0, 0, 4, -9999]], dtype=float)

undirected_SP = np.array([[0, 3, 3, 1, 2],
                          [3, 0, 6, 2, 4],
                          [3, 6, 0, 4, 5],
                          [1, 2, 4, 0, 2],
                          [2, 4, 5, 2, 0]], dtype=float)

undirected_SP_limit_2 = np.array([[0, np.inf, np.inf, 1, 2],
                                  [np.inf, 0, np.inf, 2, np.inf],
                                  [np.inf, np.inf, 0, np.inf, np.inf],
                                  [1, 2, np.inf, 0, 2],
                                  [2, np.inf, np.inf, 2, 0]], dtype=float)

undirected_SP_limit_0 = np.ones((5, 5), dtype=float) - np.eye(5)
undirected_SP_limit_0[undirected_SP_limit_0 > 0] = np.inf

undirected_pred = np.array([[-9999, 0, 0, 0, 0],
                            [1, -9999, 0, 1, 1],
                            [2, 0, -9999, 0, 0],
                            [3, 3, 0, -9999, 3],
                            [4, 4, 0, 4, -9999]], dtype=float)

methods = ['auto', 'FW', 'D', 'BF', 'J']

def test_dijkstra_limit():
    limits = [0, 2, np.inf]
    results = [undirected_SP_limit_0,
               undirected_SP_limit_2,
               undirected_SP]

    def check(limit, result):
        SP = dijkstra(undirected_G, directed=False, limit=limit)
        assert_array_almost_equal(SP, result)

    for limit, result in zip(limits, results):
        check(limit, result)

def test_directed():
    def check(method):
        SP = shortest_path(directed_G, method=method, directed=True,
                           overwrite=False)
        assert_array_almost_equal(SP, directed_SP)

    for method in methods:
        check(method)

def test_undirected():
    def check(method, directed_in):
        if directed_in:
            SP1 = shortest_path(directed_G, method=method, directed=False,
                                overwrite=False)
            assert_array_almost_equal(SP1, undirected_SP)
        else:
            SP2 = shortest_path(undirected_G, method=method, directed=True,
                                overwrite=False)
            assert_array_almost_equal(SP2, undirected_SP)

    for method in methods:
        for directed_in in (True, False):
            check(method, directed_in)

def test_directed_sparse_zero():
    # test directed sparse graph with zero-weight edge and two connected components
    def check(method):
        SP = shortest_path(directed_sparse_zero_G, method=method, directed=True,
                           overwrite=False)
        assert_array_almost_equal(SP, directed_sparse_zero_SP)

    for method in methods:
        check(method)

def test_undirected_sparse_zero():
    def check(method, directed_in):
        if directed_in:
            SP1 = shortest_path(directed_sparse_zero_G, method=method, directed=False,
                                overwrite=False)
            assert_array_almost_equal(SP1, undirected_sparse_zero_SP)
        else:
            SP2 = shortest_path(undirected_sparse_zero_G, method=method, directed=True,
                                overwrite=False)
            assert_array_almost_equal(SP2, undirected_sparse_zero_SP)

    for method in methods:
        for directed_in in (True, False):
            check(method, directed_in)

@pytest.mark.parametrize('directed, SP_ans',
                         ((True, directed_SP),
                          (False, undirected_SP)))
@pytest.mark.parametrize('indices', ([0, 2, 4], [0, 4], [3, 4], [0, 0]))
def test_dijkstra_indices_min_only(directed, SP_ans, indices):
    SP_ans = np.array(SP_ans)
    indices = np.array(indices, dtype=np.int64)
    min_ind_ans = indices[np.argmin(SP_ans[indices, :], axis=0)]
    min_d_ans = np.zeros(SP_ans.shape[0], SP_ans.dtype)
    for k in range(SP_ans.shape[0]):
        min_d_ans[k] = SP_ans[min_ind_ans[k], k]
    min_ind_ans[np.isinf(min_d_ans)] = -9999

    SP, pred, sources = dijkstra(directed_G,
                                 directed=directed,
                                 indices=indices,
                                 min_only=True,
                                 return_predecessors=True)
    assert_array_almost_equal(SP, min_d_ans)
    assert_array_equal(min_ind_ans, sources)
    SP = dijkstra(directed_G,
                  directed=directed,
                  indices=indices,
                  min_only=True,
                  return_predecessors=False)
    assert_array_almost_equal(SP, min_d_ans)

@pytest.mark.parametrize('n', (10, 100, 1000))
def test_shortest_path_min_only_random(n):
    np.random.seed(1234)
    data = scipy.sparse.rand(n, n, density=0.5, format='lil',
                             random_state=42, dtype=np.float)
    data.setdiag(np.zeros(n, dtype=np.bool))
    # choose some random vertices
    v = np.arange(n)
    np.random.shuffle(v)
    indices = v[:int(n*.1)]
    ds, pred, sources = dijkstra(data,
                                 directed=False,
                                 indices=indices,
                                 min_only=True,
                                 return_predecessors=True)
    for k in range(n):
        p = pred[k]
        s = sources[k]
        while(p != -9999):
            assert(sources[p] == s)
            p = pred[p]

def test_shortest_path_indices():
    indices = np.arange(4)

    def check(func, indshape):
        outshape = indshape + (5,)
        SP = func(directed_G, directed=False,
                  indices=indices.reshape(indshape))
        assert_array_almost_equal(SP, undirected_SP[indices].reshape(outshape))

    for indshape in [(4,), (4, 1), (2, 2)]:
        for func in (dijkstra, bellman_ford, johnson, shortest_path):
            check(func, indshape)

    assert_raises(ValueError, shortest_path, directed_G, method='FW',
                  indices=indices)

def test_predecessors():
    SP_res = {True: directed_SP,
              False: undirected_SP}
    pred_res = {True: directed_pred,
                False: undirected_pred}

    def check(method, directed):
        SP, pred = shortest_path(directed_G, method, directed=directed,
                                 overwrite=False,
                                 return_predecessors=True)
        assert_array_almost_equal(SP, SP_res[directed])
        assert_array_almost_equal(pred, pred_res[directed])

    for method in methods:
        for directed in (True, False):
            check(method, directed)

def test_construct_shortest_path():
    def check(method, directed):
        SP1, pred = shortest_path(directed_G,
                                  directed=directed,
                                  overwrite=False,
                                  return_predecessors=True)
        SP2 = construct_dist_matrix(directed_G, pred, directed=directed)
        assert_array_almost_equal(SP1, SP2)

    for method in methods:
        for directed in (True, False):
            check(method, directed)

def test_unweighted_path():
    def check(method, directed):
        SP1 = shortest_path(directed_G,
                            directed=directed,
                            overwrite=False,
                            unweighted=True)
        SP2 = shortest_path(unweighted_G,
                            directed=directed,
                            overwrite=False,
                            unweighted=False)
        assert_array_almost_equal(SP1, SP2)

    for method in methods:
        for directed in (True, False):
            check(method, directed)

def test_negative_cycles():
    # create a small graph with a negative cycle
    graph = np.ones([5, 5])
    graph.flat[::6] = 0
    graph[1, 2] = -2

    def check(method, directed):
        assert_raises(NegativeCycleError, shortest_path, graph, method,
                      directed)

    for method in ['FW', 'J', 'BF']:
        for directed in (True, False):
            check(method, directed)

def test_masked_input():
    G = np.ma.masked_equal(directed_G, 0)

    def check(method):
        SP = shortest_path(directed_G, method=method, directed=True,
                           overwrite=False)
        assert_array_almost_equal(SP, directed_SP)

    for method in methods:
        check(method)

def test_overwrite():
    G = np.array([[0, 3, 3, 1, 2],
                  [3, 0, 0, 2, 4],
                  [3, 0, 0, 0, 0],
                  [1, 2, 0, 0, 2],
                  [2, 4, 0, 2, 0]], dtype=float)
    foo = G.copy()
    shortest_path(foo, overwrite=False)
    assert_array_equal(foo, G)

@pytest.mark.parametrize('method', methods)
def test_buffer(method):
    # Smoke test that sparse matrices with read-only buffers (e.g., those from
    # joblib workers) do not cause::
    #
    #     ValueError: buffer source array is read-only
    #
    G = scipy.sparse.csr_matrix([[1.]])
    G.data.flags['WRITEABLE'] = False
    shortest_path(G, method=method)

def test_NaN_warnings():
    with pytest.warns(None) as record:
        shortest_path(np.array([[0, 1], [np.nan, 0]]))
    for i in range(len(record)):
        assert record[i].category is not RuntimeWarning
<commit_msg>MAINT: refactor for loop<commit_after>from __future__ import division, print_function, absolute_import

import numpy as np
from numpy.testing import assert_array_almost_equal, assert_array_equal
from pytest import raises as assert_raises
from scipy.sparse.csgraph import (shortest_path, dijkstra, johnson,
                                  bellman_ford, construct_dist_matrix,
                                  NegativeCycleError)
import scipy.sparse
import pytest

directed_G = np.array([[0, 3, 3, 0, 0],
                       [0, 0, 0, 2, 4],
                       [0, 0, 0, 0, 0],
                       [1, 0, 0, 0, 0],
                       [2, 0, 0, 2, 0]], dtype=float)

undirected_G = np.array([[0, 3, 3, 1, 2],
                         [3, 0, 0, 2, 4],
                         [3, 0, 0, 0, 0],
                         [1, 2, 0, 0, 2],
                         [2, 4, 0, 2, 0]], dtype=float)

unweighted_G = (directed_G > 0).astype(float)

directed_SP = [[0, 3, 3, 5, 7],
               [3, 0, 6, 2, 4],
               [np.inf, np.inf, 0, np.inf, np.inf],
               [1, 4, 4, 0, 8],
               [2, 5, 5, 2, 0]]

directed_sparse_zero_G = scipy.sparse.csr_matrix(([0, 1, 2, 3, 1],
                                            ([0, 1, 2, 3, 4],
                                             [1, 2, 0, 4, 3])),
                                            shape = (5, 5))

directed_sparse_zero_SP = [[0, 0, 1, np.inf, np.inf],
                      [3, 0, 1, np.inf, np.inf],
                      [2, 2, 0, np.inf, np.inf],
                      [np.inf, np.inf, np.inf, 0, 3],
                      [np.inf, np.inf, np.inf, 1, 0]]

undirected_sparse_zero_G = scipy.sparse.csr_matrix(([0, 0, 1, 1, 2, 2, 1, 1],
                                              ([0, 1, 1, 2, 2, 0, 3, 4],
                                               [1, 0, 2, 1, 0, 2, 4, 3])),
                                              shape = (5, 5))

undirected_sparse_zero_SP = [[0, 0, 1, np.inf, np.inf],
                        [0, 0, 1, np.inf, np.inf],
                        [1, 1, 0, np.inf, np.inf],
                        [np.inf, np.inf, np.inf, 0, 1],
                        [np.inf, np.inf, np.inf, 1, 0]]

directed_pred = np.array([[-9999, 0, 0, 1, 1],
                          [3, -9999, 0, 1, 1],
                          [-9999, -9999, -9999, -9999, -9999],
                          [3, 0, 0, -9999, 1],
                          [4, 0, 0, 4, -9999]], dtype=float)

undirected_SP = np.array([[0, 3, 3, 1, 2],
                          [3, 0, 6, 2, 4],
                          [3, 6, 0, 4, 5],
                          [1, 2, 4, 0, 2],
                          [2, 4, 5, 2, 0]], dtype=float)

undirected_SP_limit_2 = np.array([[0, np.inf, np.inf, 1, 2],
                                  [np.inf, 0, np.inf, 2, np.inf],
                                  [np.inf, np.inf, 0, np.inf, np.inf],
                                  [1, 2, np.inf, 0, 2],
                                  [2, np.inf, np.inf, 2, 0]], dtype=float)

undirected_SP_limit_0 = np.ones((5, 5), dtype=float) - np.eye(5)
undirected_SP_limit_0[undirected_SP_limit_0 > 0] = np.inf

undirected_pred = np.array([[-9999, 0, 0, 0, 0],
                            [1, -9999, 0, 1, 1],
                            [2, 0, -9999, 0, 0],
                            [3, 3, 0, -9999, 3],
                            [4, 4, 0, 4, -9999]], dtype=float)

methods = ['auto', 'FW', 'D', 'BF', 'J']

def test_dijkstra_limit():
    limits = [0, 2, np.inf]
    results = [undirected_SP_limit_0,
               undirected_SP_limit_2,
               undirected_SP]

    def check(limit, result):
        SP = dijkstra(undirected_G, directed=False, limit=limit)
        assert_array_almost_equal(SP, result)

    for limit, result in zip(limits, results):
        check(limit, result)

def test_directed():
    def check(method):
        SP = shortest_path(directed_G, method=method, directed=True,
                           overwrite=False)
        assert_array_almost_equal(SP, directed_SP)

    for method in methods:
        check(method)

def test_undirected():
    def check(method, directed_in):
        if directed_in:
            SP1 = shortest_path(directed_G, method=method, directed=False,
                                overwrite=False)
            assert_array_almost_equal(SP1, undirected_SP)
        else:
            SP2 = shortest_path(undirected_G, method=method, directed=True,
                                overwrite=False)
            assert_array_almost_equal(SP2, undirected_SP)

    for method in methods:
        for directed_in in (True, False):
            check(method, directed_in)

def test_directed_sparse_zero():
    # test directed sparse graph with zero-weight edge and two connected components
    def check(method):
        SP = shortest_path(directed_sparse_zero_G, method=method, directed=True,
                           overwrite=False)
        assert_array_almost_equal(SP, directed_sparse_zero_SP)

    for method in methods:
        check(method)

def test_undirected_sparse_zero():
    def check(method, directed_in):
        if directed_in:
            SP1 = shortest_path(directed_sparse_zero_G, method=method, directed=False,
                                overwrite=False)
            assert_array_almost_equal(SP1, undirected_sparse_zero_SP)
        else:
            SP2 = shortest_path(undirected_sparse_zero_G, method=method, directed=True,
                                overwrite=False)
            assert_array_almost_equal(SP2, undirected_sparse_zero_SP)

    for method in methods:
        for directed_in in (True, False):
            check(method, directed_in)

@pytest.mark.parametrize('directed, SP_ans',
                         ((True, directed_SP),
                          (False, undirected_SP)))
@pytest.mark.parametrize('indices', ([0, 2, 4], [0, 4], [3, 4], [0, 0]))
def test_dijkstra_indices_min_only(directed, SP_ans, indices):
    SP_ans = np.array(SP_ans)
    indices = np.array(indices, dtype=np.int64)
    min_ind_ans = indices[np.argmin(SP_ans[indices, :], axis=0)]
    min_d_ans = np.zeros(SP_ans.shape[0], SP_ans.dtype)
    for k in range(SP_ans.shape[0]):
        min_d_ans[k] = SP_ans[min_ind_ans[k], k]
    min_ind_ans[np.isinf(min_d_ans)] = -9999

    SP, pred, sources = dijkstra(directed_G,
                                 directed=directed,
                                 indices=indices,
                                 min_only=True,
                                 return_predecessors=True)
    assert_array_almost_equal(SP, min_d_ans)
    assert_array_equal(min_ind_ans, sources)
    SP = dijkstra(directed_G,
                  directed=directed,
                  indices=indices,
                  min_only=True,
                  return_predecessors=False)
    assert_array_almost_equal(SP, min_d_ans)

@pytest.mark.parametrize('n', (10, 100, 1000))
def test_shortest_path_min_only_random(n):
    np.random.seed(1234)
    data = scipy.sparse.rand(n, n, density=0.5, format='lil',
                             random_state=42, dtype=np.float)
    data.setdiag(np.zeros(n, dtype=np.bool))
    # choose some random vertices
    v = np.arange(n)
    np.random.shuffle(v)
    indices = v[:int(n*.1)]
    ds, pred, sources = dijkstra(data,
                                 directed=False,
                                 indices=indices,
                                 min_only=True,
                                 return_predecessors=True)
    for k in range(n):
        p = pred[k]
        s = sources[k]
        while(p != -9999):
            assert(sources[p] == s)
            p = pred[p]

def test_shortest_path_indices():
    indices = np.arange(4)

    def check(func, indshape):
        outshape = indshape + (5,)
        SP = func(directed_G, directed=False,
                  indices=indices.reshape(indshape))
        assert_array_almost_equal(SP, undirected_SP[indices].reshape(outshape))

    for indshape in [(4,), (4, 1), (2, 2)]:
        for func in (dijkstra, bellman_ford, johnson, shortest_path):
            check(func, indshape)

    assert_raises(ValueError, shortest_path, directed_G, method='FW',
                  indices=indices)

def test_predecessors():
    SP_res = {True: directed_SP,
              False: undirected_SP}
    pred_res = {True: directed_pred,
                False: undirected_pred}

    def check(method, directed):
        SP, pred = shortest_path(directed_G, method, directed=directed,
                                 overwrite=False,
                                 return_predecessors=True)
        assert_array_almost_equal(SP, SP_res[directed])
        assert_array_almost_equal(pred, pred_res[directed])

    for method in methods:
        for directed in (True, False):
            check(method, directed)

def test_construct_shortest_path():
    def check(method, directed):
        SP1, pred = shortest_path(directed_G,
                                  directed=directed,
                                  overwrite=False,
                                  return_predecessors=True)
        SP2 = construct_dist_matrix(directed_G, pred, directed=directed)
        assert_array_almost_equal(SP1, SP2)

    for method in methods:
        for directed in (True, False):
            check(method, directed)

def test_unweighted_path():
    def check(method, directed):
        SP1 = shortest_path(directed_G,
                            directed=directed,
                            overwrite=False,
                            unweighted=True)
        SP2 = shortest_path(unweighted_G,
                            directed=directed,
                            overwrite=False,
                            unweighted=False)
        assert_array_almost_equal(SP1, SP2)

    for method in methods:
        for directed in (True, False):
            check(method, directed)

def test_negative_cycles():
    # create a small graph with a negative cycle
    graph = np.ones([5, 5])
    graph.flat[::6] = 0
    graph[1, 2] = -2

    def check(method, directed):
        assert_raises(NegativeCycleError, shortest_path, graph, method,
                      directed)

    for method in ['FW', 'J', 'BF']:
        for directed in (True, False):
            check(method, directed)

def test_masked_input():
    G = np.ma.masked_equal(directed_G, 0)

    def check(method):
        SP = shortest_path(directed_G, method=method, directed=True,
                           overwrite=False)
        assert_array_almost_equal(SP, directed_SP)

    for method in methods:
        check(method)

def test_overwrite():
    G = np.array([[0, 3, 3, 1, 2],
                  [3, 0, 0, 2, 4],
                  [3, 0, 0, 0, 0],
                  [1, 2, 0, 0, 2],
                  [2, 4, 0, 2, 0]], dtype=float)
    foo = G.copy()
    shortest_path(foo, overwrite=False)
    assert_array_equal(foo, G)

@pytest.mark.parametrize('method', methods)
def test_buffer(method):
    # Smoke test that sparse matrices with read-only buffers (e.g., those from
    # joblib workers) do not cause::
    #
    #     ValueError: buffer source array is read-only
    #
    G = scipy.sparse.csr_matrix([[1.]])
    G.data.flags['WRITEABLE'] = False
    shortest_path(G, method=method)

def test_NaN_warnings():
    with pytest.warns(None) as record:
        shortest_path(np.array([[0, 1], [np.nan, 0]]))
    for r in record:
        assert r.category is not RuntimeWarning
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Tests for the linalg.isolve.gcrotmk module
"""

from __future__ import division, print_function, absolute_import

from numpy.testing import assert_, assert_allclose, assert_equal
from scipy._lib._numpy_compat import suppress_warnings

import numpy as np
from numpy import zeros, array, allclose
from scipy.linalg import norm
from scipy.sparse import csr_matrix, eye, rand

from scipy.sparse.linalg.interface import LinearOperator
from scipy.sparse.linalg import splu
from scipy.sparse.linalg.isolve import gcrotmk, gmres

Am = csr_matrix(array([[-2,1,0,0,0,9],
                       [1,-2,1,0,5,0],
                       [0,1,-2,1,0,0],
                       [0,0,1,-2,1,0],
                       [0,3,0,1,-2,1],
                       [1,0,0,0,1,-2]]))
b = array([1,2,3,4,5,6])
count = [0]

def matvec(v):
    count[0] += 1
    return Am*v

A = LinearOperator(matvec=matvec, shape=Am.shape, dtype=Am.dtype)

def do_solve(**kw):
    count[0] = 0
    with suppress_warnings() as sup:
        sup.filter(DeprecationWarning, ".*called without specifying.*")
        x0, flag = gcrotmk(A, b, x0=zeros(A.shape[0]), tol=1e-14, **kw)
    count_0 = count[0]
    assert_(allclose(A*x0, b, rtol=1e-12, atol=1e-12), norm(A*x0-b))
    return x0, count_0

class TestGCROTMK(object):
    def test_preconditioner(self):
        # Check that preconditioning works
        pc = splu(Am.tocsc())
        M = LinearOperator(matvec=pc.solve, shape=A.shape, dtype=A.dtype)

        x0, count_0 = do_solve()
        x1, count_1 = do_solve(M=M)

        assert_equal(count_1, 3)
        assert_(count_1 < count_0/2)
        assert_(allclose(x1, x0, rtol=1e-14))

    def test_arnoldi(self):
        np.random.rand(1234)

        A = eye(10000) + rand(10000,10000,density=1e-4)
        b = np.random.rand(10000)

        # The inner arnoldi should be equivalent to gmres
        with suppress_warnings() as sup:
            sup.filter(DeprecationWarning, ".*called without specifying.*")
            x0, flag0 = gcrotmk(A, b, x0=zeros(A.shape[0]), m=15, k=0, maxiter=1)
            x1, flag1 = gmres(A, b, x0=zeros(A.shape[0]), restart=15, maxiter=1)

        assert_equal(flag0, 1)
        assert_equal(flag1, 1)
        assert_(np.linalg.norm(A.dot(x0) - b) > 1e-3)

        assert_allclose(x0, x1)

    def test_cornercase(self):
        np.random.seed(1234)

        # Rounding error may prevent convergence with tol=0 --- ensure
        # that the return values in this case are correct, and no
        # exceptions are raised

        for n in [3, 5, 10, 100]:
            A = 2*eye(n)

            with suppress_warnings() as sup:
                sup.filter(DeprecationWarning, ".*called without specifying.*")
                b = np.ones(n)
                x, info = gcrotmk(A, b, maxiter=10)
                assert_equal(info, 0)
                assert_allclose(A.dot(x) - b, 0, atol=1e-14)

                x, info = gcrotmk(A, b, tol=0, maxiter=10)
                if info == 0:
                    assert_allclose(A.dot(x) - b, 0, atol=1e-14)

                b = np.random.rand(n)
                x, info = gcrotmk(A, b, maxiter=10)
                assert_equal(info, 0)
                assert_allclose(A.dot(x) - b, 0, atol=1e-14)

                x, info = gcrotmk(A, b, tol=0, maxiter=10)
                if info == 0:
                    assert_allclose(A.dot(x) - b, 0, atol=1e-14)

    def test_nans(self):
        A = eye(3, format='lil')
        A[1,1] = np.nan
        b = np.ones(3)

        with suppress_warnings() as sup:
            sup.filter(DeprecationWarning, ".*called without specifying.*")
            x, info = gcrotmk(A, b, tol=0, maxiter=10)
            assert_equal(info, 1)

    def test_truncate(self):
        np.random.seed(1234)
        A = np.random.rand(30, 30) + np.eye(30)
        b = np.random.rand(30)

        for truncate in ['oldest', 'smallest']:
            with suppress_warnings() as sup:
                sup.filter(DeprecationWarning, ".*called without specifying.*")
                x, info = gcrotmk(A, b, m=10, k=10, truncate=truncate, tol=1e-4,
                                  maxiter=200)
            assert_equal(info, 0)
            assert_allclose(A.dot(x) - b, 0, atol=1e-3)

    def test_CU(self):
        for discard_C in (True, False):
            # Check that C,U behave as expected
            CU = []
            x0, count_0 = do_solve(CU=CU, discard_C=discard_C)
            assert_(len(CU) > 0)
            assert_(len(CU) <= 6)

            if discard_C:
                for c, u in CU:
                    assert_(c is None)

            # should converge immediately
            x1, count_1 = do_solve(CU=CU, discard_C=discard_C)
            if discard_C:
                assert_equal(count_1, 2 + len(CU))
            else:
                assert_equal(count_1, 3)
            assert_(count_1 <= count_0/2)
            assert_allclose(x1, x0, atol=1e-14)

    def test_denormals(self):
        # Check that no warnings are emitted if the matrix contains
        # numbers for which 1/x has no float representation, and that
        # the solver behaves properly.
        A = np.array([[1, 2], [3, 4]], dtype=float)
        A *= 100 * np.nextafter(0, 1)

        b = np.array([1, 1])

        with suppress_warnings() as sup:
            sup.filter(DeprecationWarning, ".*called without specifying.*")
            xp, info = gcrotmk(A, b)

        if info == 0:
            assert_allclose(A.dot(xp), b)
<commit_msg>TST: reduce max memory usage for sparse.linalg.gcrotmk test.<commit_after>#!/usr/bin/env python
"""Tests for the linalg.isolve.gcrotmk module
"""

from __future__ import division, print_function, absolute_import

from numpy.testing import assert_, assert_allclose, assert_equal
from scipy._lib._numpy_compat import suppress_warnings

import numpy as np
from numpy import zeros, array, allclose
from scipy.linalg import norm
from scipy.sparse import csr_matrix, eye, rand

from scipy.sparse.linalg.interface import LinearOperator
from scipy.sparse.linalg import splu
from scipy.sparse.linalg.isolve import gcrotmk, gmres

Am = csr_matrix(array([[-2,1,0,0,0,9],
                       [1,-2,1,0,5,0],
                       [0,1,-2,1,0,0],
                       [0,0,1,-2,1,0],
                       [0,3,0,1,-2,1],
                       [1,0,0,0,1,-2]]))
b = array([1,2,3,4,5,6])
count = [0]

def matvec(v):
    count[0] += 1
    return Am*v

A = LinearOperator(matvec=matvec, shape=Am.shape, dtype=Am.dtype)

def do_solve(**kw):
    count[0] = 0
    with suppress_warnings() as sup:
        sup.filter(DeprecationWarning, ".*called without specifying.*")
        x0, flag = gcrotmk(A, b, x0=zeros(A.shape[0]), tol=1e-14, **kw)
    count_0 = count[0]
    assert_(allclose(A*x0, b, rtol=1e-12, atol=1e-12), norm(A*x0-b))
    return x0, count_0

class TestGCROTMK(object):
    def test_preconditioner(self):
        # Check that preconditioning works
        pc = splu(Am.tocsc())
        M = LinearOperator(matvec=pc.solve, shape=A.shape, dtype=A.dtype)

        x0, count_0 = do_solve()
        x1, count_1 = do_solve(M=M)

        assert_equal(count_1, 3)
        assert_(count_1 < count_0/2)
        assert_(allclose(x1, x0, rtol=1e-14))

    def test_arnoldi(self):
        np.random.rand(1234)

        A = eye(2000) + rand(2000, 2000, density=5e-4)
        b = np.random.rand(2000)

        # The inner arnoldi should be equivalent to gmres
        with suppress_warnings() as sup:
            sup.filter(DeprecationWarning, ".*called without specifying.*")
            x0, flag0 = gcrotmk(A, b, x0=zeros(A.shape[0]), m=15, k=0, maxiter=1)
            x1, flag1 = gmres(A, b, x0=zeros(A.shape[0]), restart=15, maxiter=1)

        assert_equal(flag0, 1)
        assert_equal(flag1, 1)
        assert_(np.linalg.norm(A.dot(x0) - b) > 1e-3)

        assert_allclose(x0, x1)

    def test_cornercase(self):
        np.random.seed(1234)

        # Rounding error may prevent convergence with tol=0 --- ensure
        # that the return values in this case are correct, and no
        # exceptions are raised

        for n in [3, 5, 10, 100]:
            A = 2*eye(n)

            with suppress_warnings() as sup:
                sup.filter(DeprecationWarning, ".*called without specifying.*")
                b = np.ones(n)
                x, info = gcrotmk(A, b, maxiter=10)
                assert_equal(info, 0)
                assert_allclose(A.dot(x) - b, 0, atol=1e-14)

                x, info = gcrotmk(A, b, tol=0, maxiter=10)
                if info == 0:
                    assert_allclose(A.dot(x) - b, 0, atol=1e-14)

                b = np.random.rand(n)
                x, info = gcrotmk(A, b, maxiter=10)
                assert_equal(info, 0)
                assert_allclose(A.dot(x) - b, 0, atol=1e-14)

                x, info = gcrotmk(A, b, tol=0, maxiter=10)
                if info == 0:
                    assert_allclose(A.dot(x) - b, 0, atol=1e-14)

    def test_nans(self):
        A = eye(3, format='lil')
        A[1,1] = np.nan
        b = np.ones(3)

        with suppress_warnings() as sup:
            sup.filter(DeprecationWarning, ".*called without specifying.*")
            x, info = gcrotmk(A, b, tol=0, maxiter=10)
            assert_equal(info, 1)

    def test_truncate(self):
        np.random.seed(1234)
        A = np.random.rand(30, 30) + np.eye(30)
        b = np.random.rand(30)

        for truncate in ['oldest', 'smallest']:
            with suppress_warnings() as sup:
                sup.filter(DeprecationWarning, ".*called without specifying.*")
                x, info = gcrotmk(A, b, m=10, k=10, truncate=truncate, tol=1e-4,
                                  maxiter=200)
            assert_equal(info, 0)
            assert_allclose(A.dot(x) - b, 0, atol=1e-3)

    def test_CU(self):
        for discard_C in (True, False):
            # Check that C,U behave as expected
            CU = []
            x0, count_0 = do_solve(CU=CU, discard_C=discard_C)
            assert_(len(CU) > 0)
            assert_(len(CU) <= 6)

            if discard_C:
                for c, u in CU:
                    assert_(c is None)

            # should converge immediately
            x1, count_1 = do_solve(CU=CU, discard_C=discard_C)
            if discard_C:
                assert_equal(count_1, 2 + len(CU))
            else:
                assert_equal(count_1, 3)
            assert_(count_1 <= count_0/2)
            assert_allclose(x1, x0, atol=1e-14)

    def test_denormals(self):
        # Check that no warnings are emitted if the matrix contains
        # numbers for which 1/x has no float representation, and that
        # the solver behaves properly.
        A = np.array([[1, 2], [3, 4]], dtype=float)
        A *= 100 * np.nextafter(0, 1)

        b = np.array([1, 1])

        with suppress_warnings() as sup:
            sup.filter(DeprecationWarning, ".*called without specifying.*")
            xp, info = gcrotmk(A, b)

        if info == 0:
            assert_allclose(A.dot(xp), b)
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
__author__ = 'ar'

"""
status:
    - ok  : fiished
    - in  : in progress
    - err :
"""
class StageInfo:

    def __init__(self):
        pass

class SeriesProcStages:
    def __init__(self, ):

if __name__ == '__main__':
    pass<commit_msg>stub branch<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
__author__ = 'ar'

import enum
import json

"""
status for stage/total:
    - new : all stages is incompleted
    - ok  : is finished
    - in  : in progress
    - err : error, and
"""
class StageStatus(enum.Enum):
    STATE_NEW   = 'new'
    STATE_OK    = 'ok'
    STATE_IN    = 'in'
    STATE_ERR   = 'err'
    STATES = [STATE_NEW, STATE_OK, STATE_IN, STATE_ERR]
    def __init__(self, pdict=None):
        if pdict is None:
            self._state = StageStatus.STATE_NEW
            self._serr = None
        else:
            if 'state' in pdict:
                pass
            else:
                self._state = StageStatus.STATE_ERR
                self._serr = ''

class SeriesProcStages:
    FILE_STATUS = 'status.json'
    def __init__(self, pdir=None):
        pass
    def buildNew(self):
        ret = dict()

if __name__ == '__main__':
    pass<|endoftext|>


<commit_before>import matplotlib.pyplot as plt
from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import StratifiedKFold
from sklearn.feature_selection import RFECV
from sklearn.datasets import make_classification
from sklearn.datasets import load_iris
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import mutual_info_classif

from nalaf.learning.lib.sklsvm import SklSVM
from nalaf.structures.data import Dataset
from loctext.learning.train import read_corpus
from loctext.learning.annotators import LocTextSSmodelRelationExtractor
from util import *
from loctext.util import *
import time

print(__doc__)

SCORING_FUNCS = [mutual_info_classif]
SCORING_NAMES = ['f1_macro']

annotator, X, y = get_model_and_data()

num_instances, num_features = X.shape

search_space = [
    {
        'k': list(range(1, num_features + 1)),
    },
]

for scoring_name in SCORING_NAMES:
    for scoring_func in SCORING_FUNCS:

        kbest = SelectKBest(scoring_func, k="all")
        kbest.fit(X, y)
        selected_feat_keys = get_kbest_feature_keys()

        scores = []

        for num_seletected_kbest_features in range(1, num_features + 1):

            allowed_feat_keys = selected_feat_keys[:num_seletected_kbest_features]

            svc = SVC(kernel='linear', C=1, verbose=False)  # TODO C=1 linear / rbf ??

            # do cv through allowing only allowed_feat_keys
            # scores.append(cv.score)

        print()
        # print(print_selected_features(selected_feat_keys, annotator.pipeline.feature_set, file_prefix="rfe"))
        print()
        print("Max performance for {}: {}".format(scoring_name, max(scores)))
        print()

        plot_recursive_features(scoring_name, scores)
<commit_msg>almost there<commit_after>import matplotlib.pyplot as plt
from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import StratifiedKFold
from sklearn.feature_selection import RFECV
from sklearn.datasets import make_classification
from sklearn.datasets import load_iris
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import mutual_info_classif

from nalaf.learning.lib.sklsvm import SklSVM
from nalaf.structures.data import Dataset
from loctext.learning.train import read_corpus
from loctext.learning.annotators import LocTextSSmodelRelationExtractor
from util import *
from loctext.util import *
import time
from sklearn.model_selection import cross_val_score
from sklearn.pipeline import make_pipeline

print(__doc__)

SCORING_FUNCS = ["stub"]
SCORING_NAMES = ['f1_macro']

annotator, X, y = get_model_and_data()

num_instances, num_features = X.shape

search_space = [
    {
        'k': list(range(1, num_features + 1)),
    },
]

for scoring_name in SCORING_NAMES:
    for scoring_func in SCORING_FUNCS:

        kbest = SelectKBest(scoring_func, k="all")
        kbest.fit(X, y)
        selected_feat_keys = get_kbest_feature_keys()

        scores = []

        for num_seletected_kbest_features in range(1, num_features + 1):

            allowed_feat_keys = selected_feat_keys[:num_seletected_kbest_features]

            estimator = make_pipeline(MY_TRANSFORMATION, SVC(kernel='linear', C=1, verbose=False))  # TODO C=1 linear / rbf ??

            cv_scores = cross_val_score(estimator, X, y, scoring=scoring_name, cv=my_cv_generator(num_instances))
            scores.append(cv_scores.append(cv_scores.mean()))

        assert(len(scores) == num_features)

        print()
        # print(print_selected_features(selected_feat_keys, annotator.pipeline.feature_set, file_prefix="rfe"))
        print()
        print("Max performance for {}: {}".format(scoring_name, max(scores)))
        print()

        plot_recursive_features(scoring_name, scores)
<|endoftext|>


<commit_before># --------------------------------------------------------------------------
#
# Copyright (c) Microsoft Corporation. All rights reserved.
#
# The MIT License (MIT)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the ""Software""), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#
# --------------------------------------------------------------------------
import threading
import uuid
try:
    from urlparse import urlparse # type: ignore # pylint: disable=unused-import
except ImportError:
    from urllib.parse import urlparse

from typing import Any, Callable, Union, List, Optional, TYPE_CHECKING
from azure.core.pipeline.transport.base import HttpResponse  # type: ignore

if TYPE_CHECKING:
    import requests
    from msrest.serialization import Model # type: ignore # pylint: disable=unused-import
    DeserializationCallbackType = Union[Model, Callable[[requests.Response], Model]]

class PollingMethod(object):
    """ABC class for polling method.
    """
    def initialize(self, client, initial_response, deserialization_callback):
        # type: (Any, Any, Any) -> None
        raise NotImplementedError("This method needs to be implemented")

    def run(self):
        # type: () -> None
        raise NotImplementedError("This method needs to be implemented")

    def status(self):
        # type: () -> str
        raise NotImplementedError("This method needs to be implemented")

    def finished(self):
        # type: () -> bool
        raise NotImplementedError("This method needs to be implemented")

    def resource(self):
        # type: () -> Any
        raise NotImplementedError("This method needs to be implemented")

class NoPolling(PollingMethod):
    """An empty poller that returns the deserialized initial response.
    """
    def __init__(self):
        self._initial_response = None
        self._deserialization_callback = None

    def initialize(self, _, initial_response, deserialization_callback):
        # type: (Any, requests.Response, Callable) -> None
        self._initial_response = initial_response
        self._deserialization_callback = deserialization_callback

    def run(self):
        # type: () -> None
        """Empty run, no polling.
        """

    def status(self):
        # type: () -> str
        """Return the current status as a string.

        :rtype: str
        """
        return "succeeded"

    def finished(self):
        # type: () -> bool
        """Is this polling finished?

        :rtype: bool
        """
        return True

    def resource(self):
        # type: () -> Any
        return self._deserialization_callback(self._initial_response)

class LROPoller(object):
    """Poller for long running operations.

    :param client: A pipeline service client
    :type client: ~azure.core.pipeline.PipelineClient
    :param initial_response: The initial call response
    :type initial_response: ~azure.core.pipeline.HttpResponse
    :param deserialization_callback: A callback that takes a Response and return a deserialized object.
                                     If a subclass of Model is given, this passes "deserialize" as callback.
    :type deserialization_callback: callable or msrest.serialization.Model
    :param polling_method: The polling strategy to adopt
    :type polling_method: ~msrest.polling.PollingMethod
    """

    def __init__(self, client, initial_response, deserialization_callback, polling_method):
        # type: (Any, HttpResponse, DeserializationCallbackType, PollingMethod) -> None
        self._client = client
        self._response = initial_response
        self._callbacks = []  # type: List[Callable]
        self._polling_method = polling_method

        # This implicit test avoids bringing in an explicit dependency on Model directly
        try:
            deserialization_callback = deserialization_callback.deserialize # type: ignore
        except AttributeError:
            pass

        # Might raise a CloudError
        self._polling_method.initialize(self._client, self._response, deserialization_callback)

        # Prepare thread execution
        self._thread = None
        self._done = None
        self._exception = None
        if not self._polling_method.finished():
            self._done = threading.Event()
            self._thread = threading.Thread(
                target=self._start,
                name="LROPoller({})".format(uuid.uuid4()))
            self._thread.daemon = True
            self._thread.start()

    def _start(self):
        """Start the long running operation.
        On completion, runs any callbacks.

        :param callable update_cmd: The API request to check the status of
         the operation.
        """
        try:
            self._polling_method.run()
        except Exception as err: #pylint: disable=broad-except
            self._exception = err

        finally:
            self._done.set()

        callbacks, self._callbacks = self._callbacks, []
        while callbacks:
            for call in callbacks:
                call(self._polling_method)
            callbacks, self._callbacks = self._callbacks, []

    def status(self):
        # type: () -> str
        """Returns the current status string.

        :returns: The current status string
        :rtype: str
        """
        return self._polling_method.status()

    def result(self, timeout=None):
        # type: (Optional[int]) -> Model
        """Return the result of the long running operation, or
        the result available after the specified timeout.

        :returns: The deserialized resource of the long running operation,
         if one is available.
        :raises CloudError: Server problem with the query.
        """
        self.wait(timeout)
        return self._polling_method.resource()

    def wait(self, timeout=None):
        # type: (Optional[int]) -> None
        """Wait on the long running operation for a specified length
        of time. You can check if this call as ended with timeout with the
        "done()" method.

        :param int timeout: Period of time to wait for the long running
         operation to complete (in seconds).
        :raises CloudError: Server problem with the query.
        """
        if self._thread is None:
            return
        self._thread.join(timeout=timeout)
        try:
            # Let's handle possible None in forgiveness here
            raise self._exception  # type: ignore
        except TypeError: # Was None
            pass

    def done(self):
        # type: () -> bool
        """Check status of the long running operation.

        :returns: 'True' if the process has completed, else 'False'.
        """
        return self._thread is None or not self._thread.is_alive()

    def add_done_callback(self, func):
        # type: (Callable) -> None
        """Add callback function to be run once the long running operation
        has completed - regardless of the status of the operation.

        :param callable func: Callback function that takes at least one
         argument, a completed LongRunningOperation.
        """
        # Still use "_done" and not "done", since CBs are executed inside the thread.
        if self._done is None or self._done.is_set():
            func(self._polling_method)
        # Let's add them still, for consistency (if you wish to access to it for some reasons)
        self._callbacks.append(func)

    def remove_done_callback(self, func):
        # type: (Callable) -> None
        """Remove a callback from the long running operation.

        :param callable func: The function to be removed from the callbacks.
        :raises: ValueError if the long running operation has already
         completed.
        """
        if self._done is None or self._done.is_set():
            raise ValueError("Process is complete.")
        self._callbacks = [c for c in self._callbacks if c != func]
<commit_msg>Trace core lro (#6456)<commit_after># --------------------------------------------------------------------------
#
# Copyright (c) Microsoft Corporation. All rights reserved.
#
# The MIT License (MIT)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the ""Software""), to
# deal in the Software without restriction, including without limitation the
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
# sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
# IN THE SOFTWARE.
#
# --------------------------------------------------------------------------
import threading
import uuid
try:
    from urlparse import urlparse # type: ignore # pylint: disable=unused-import
except ImportError:
    from urllib.parse import urlparse

from typing import Any, Callable, Union, List, Optional, TYPE_CHECKING
from azure.core.pipeline.transport.base import HttpResponse  # type: ignore
from azure.core.tracing.context import tracing_context
from azure.core.tracing.decorator import distributed_trace

if TYPE_CHECKING:
    import requests
    from msrest.serialization import Model # type: ignore # pylint: disable=unused-import
    DeserializationCallbackType = Union[Model, Callable[[requests.Response], Model]]

class PollingMethod(object):
    """ABC class for polling method.
    """
    def initialize(self, client, initial_response, deserialization_callback):
        # type: (Any, Any, Any) -> None
        raise NotImplementedError("This method needs to be implemented")

    def run(self):
        # type: () -> None
        raise NotImplementedError("This method needs to be implemented")

    def status(self):
        # type: () -> str
        raise NotImplementedError("This method needs to be implemented")

    def finished(self):
        # type: () -> bool
        raise NotImplementedError("This method needs to be implemented")

    def resource(self):
        # type: () -> Any
        raise NotImplementedError("This method needs to be implemented")

class NoPolling(PollingMethod):
    """An empty poller that returns the deserialized initial response.
    """
    def __init__(self):
        self._initial_response = None
        self._deserialization_callback = None

    def initialize(self, _, initial_response, deserialization_callback):
        # type: (Any, requests.Response, Callable) -> None
        self._initial_response = initial_response
        self._deserialization_callback = deserialization_callback

    def run(self):
        # type: () -> None
        """Empty run, no polling.
        """

    def status(self):
        # type: () -> str
        """Return the current status as a string.

        :rtype: str
        """
        return "succeeded"

    def finished(self):
        # type: () -> bool
        """Is this polling finished?

        :rtype: bool
        """
        return True

    def resource(self):
        # type: () -> Any
        return self._deserialization_callback(self._initial_response)

class LROPoller(object):
    """Poller for long running operations.

    :param client: A pipeline service client
    :type client: ~azure.core.pipeline.PipelineClient
    :param initial_response: The initial call response
    :type initial_response: ~azure.core.pipeline.HttpResponse
    :param deserialization_callback: A callback that takes a Response and return a deserialized object.
                                     If a subclass of Model is given, this passes "deserialize" as callback.
    :type deserialization_callback: callable or msrest.serialization.Model
    :param polling_method: The polling strategy to adopt
    :type polling_method: ~msrest.polling.PollingMethod
    """

    def __init__(self, client, initial_response, deserialization_callback, polling_method):
        # type: (Any, HttpResponse, DeserializationCallbackType, PollingMethod) -> None
        self._client = client
        self._response = initial_response
        self._callbacks = []  # type: List[Callable]
        self._polling_method = polling_method

        # This implicit test avoids bringing in an explicit dependency on Model directly
        try:
            deserialization_callback = deserialization_callback.deserialize # type: ignore
        except AttributeError:
            pass

        # Might raise a CloudError
        self._polling_method.initialize(self._client, self._response, deserialization_callback)

        # Prepare thread execution
        self._thread = None
        self._done = None
        self._exception = None
        if not self._polling_method.finished():
            self._done = threading.Event()
            self._thread = threading.Thread(
                target=tracing_context.with_current_context(self._start),
                name="LROPoller({})".format(uuid.uuid4()))
            self._thread.daemon = True
            self._thread.start()

    def _start(self):
        """Start the long running operation.
        On completion, runs any callbacks.

        :param callable update_cmd: The API request to check the status of
         the operation.
        """
        try:
            self._polling_method.run()
        except Exception as err: #pylint: disable=broad-except
            self._exception = err

        finally:
            self._done.set()

        callbacks, self._callbacks = self._callbacks, []
        while callbacks:
            for call in callbacks:
                call(self._polling_method)
            callbacks, self._callbacks = self._callbacks, []

    def status(self):
        # type: () -> str
        """Returns the current status string.

        :returns: The current status string
        :rtype: str
        """
        return self._polling_method.status()

    def result(self, timeout=None):
        # type: (Optional[int]) -> Model
        """Return the result of the long running operation, or
        the result available after the specified timeout.

        :returns: The deserialized resource of the long running operation,
         if one is available.
        :raises CloudError: Server problem with the query.
        """
        self.wait(timeout)
        return self._polling_method.resource()

    @distributed_trace
    def wait(self, timeout=None):
        # type: (Optional[int]) -> None
        """Wait on the long running operation for a specified length
        of time. You can check if this call as ended with timeout with the
        "done()" method.

        :param int timeout: Period of time to wait for the long running
         operation to complete (in seconds).
        :raises CloudError: Server problem with the query.
        """
        if self._thread is None:
            return
        self._thread.join(timeout=timeout)
        try:
            # Let's handle possible None in forgiveness here
            raise self._exception  # type: ignore
        except TypeError: # Was None
            pass

    def done(self):
        # type: () -> bool
        """Check status of the long running operation.

        :returns: 'True' if the process has completed, else 'False'.
        """
        return self._thread is None or not self._thread.is_alive()

    def add_done_callback(self, func):
        # type: (Callable) -> None
        """Add callback function to be run once the long running operation
        has completed - regardless of the status of the operation.

        :param callable func: Callback function that takes at least one
         argument, a completed LongRunningOperation.
        """
        # Still use "_done" and not "done", since CBs are executed inside the thread.
        if self._done is None or self._done.is_set():
            func(self._polling_method)
        # Let's add them still, for consistency (if you wish to access to it for some reasons)
        self._callbacks.append(func)

    def remove_done_callback(self, func):
        # type: (Callable) -> None
        """Remove a callback from the long running operation.

        :param callable func: The function to be removed from the callbacks.
        :raises: ValueError if the long running operation has already
         completed.
        """
        if self._done is None or self._done.is_set():
            raise ValueError("Process is complete.")
        self._callbacks = [c for c in self._callbacks if c != func]
<|endoftext|>


<commit_before>import os
import h5py
import numpy as np
from collections import defaultdict
from .mongoquery import JSONCollection
from .base import MDSTemplate, MDSROTemplate
from ..utils import ensure_path_exists

def append(dataset, data):
    old_size = dataset.size
    new_size = old_size + len(data)
    dataset.resize((new_size,))
    dataset[old_size:] = data

class RunStartCollection(JSONCollection):
    def __init__(self, event_col, *args, **kwargs):
        self._event_col = event_col
        super(RunStartCollection, self).__init__(*args, **kwargs)

    def insert_one(self, doc):
        self._event_col.new_runstart(doc)
        super(RunStartCollection, self).insert_one(doc, fk='uid')

class DescriptorCollection(JSONCollection):
    def __init__(self, event_col, *args, **kwargs):
        self._event_col = event_col
        super(DescriptorCollection, self).__init__(*args, **kwargs)

    def insert_one(self, doc):
        self._event_col.new_descriptor(doc)
        super(DescriptorCollection, self).insert_one(doc)

class EventCollection(object):
    def __init__(self, dirpath):
        self._runstarts = {}
        self._descriptors = {}
        self._dirpath = dirpath

    def new_runstart(self, doc):
        uid = doc['uid']
        fp = os.path.join(self._dirpath, '{}.h5'.format(uid))
        self._runstarts[uid] = fp

    def new_descriptor(self, doc):
        uid = doc['uid']
        run_start_uid = doc['run_start']
        groupname = 'desc_' + uid.replace('-', '_')
        fp = self._runstarts[run_start_uid]
        with h5py.File(fp, 'a') as f:
            g = f.create_group(groupname)
            g.create_dataset('uid', shape=(0,), maxshape=(None,), dtype='S36')
            g.create_dataset('time', shape=(0,), maxshape=(None,),
                             dtype='float64')
            g.create_dataset('seq_num', shape=(0,), maxshape=(None,))
            g.create_group('data')
            g.create_group('timestamps')
            for key in doc['data_keys']:
                # Create an empty, resizable dataset for each key.
                g['timestamps'].create_dataset(key, shape=(0,),
                                               maxshape=(None,),
                                               dtype='float64')
        self._descriptors[uid] = run_start_uid

    def find(self, query, sort=None):
        if list(query.keys()) != ['descriptor']:
            raise NotImplementedError("Only queries based on descriptor uid "
                                      "are supported.")
        desc_uid = query['descriptor']
        groupname = 'desc_' + desc_uid.replace('-', '_')
        fp = self._runstarts[self._descriptors[desc_uid]]
        with h5py.File(fp, 'r') as f:
            g = f[groupname]
            transposed_uid = list(g['uid'][:])
            transposed_seq_num = list(g['seq_num'][:])
            transposed_time = list(g['time'][:])
            transposed_data = {}
            transposed_ts = {}
            for key in g['data']:
                transposed_data[key] = list(g['data'][key][:])
                transposed_ts[key] = list(g['timestamps'][key][:])
        for _ in range(len(transposed_uid)):
            event = {}
            event['uid'] = transposed_uid.pop(0).decode()
            event['seq_num'] = transposed_seq_num.pop(0)
            event['time'] = transposed_time.pop(0)
            event['data'] = {}
            event['timestamps'] = {}
            for key in transposed_data:
                data = transposed_data[key].pop(0)
                if data.dtype.kind == 'S':
                    data = bytes(data).decode('utf-8')
                event['data'][key] = data
                event['timestamps'][key] = transposed_ts[key].pop(0)
            yield event

    def find_one(self, query):
        # not used on event_col
        raise NotImplementedError()

    def insert_one(self, doc):
        # not used on event_col
        return self.insert([doc])

    def insert(self, docs):
        # Sort docs by descriptor.
        descs = defaultdict(list)
        for doc in docs:
            descs[doc['descriptor']].append(doc)
        # Process one descriptor at a time.
        for uid, docs in descs.items():
            keys = docs[0]['data']
            transposed_data = {k: [] for k in keys}
            transposed_ts = {k: [] for k in keys}
            transposed_uid = []
            transposed_seq_num = []
            transposed_time = []
            for doc in docs:
                transposed_uid.append(doc['uid'])
                transposed_time.append(doc['time'])
                transposed_seq_num.append(doc['seq_num'])
                data = doc['data']
                ts = doc['timestamps']
                for k in keys:
                    transposed_data[k].append(data[k])
                    transposed_ts[k].append(ts[k])
            fp = self._runstarts[self._descriptors[uid]]
            groupname = 'desc_' + uid.replace('-', '_')
            with h5py.File(fp, 'a') as f:
                g = f[groupname]
                uids = [s.encode('ascii', 'ignore') for s in transposed_uid]
                append(g['uid'], uids)
                append(g['time'], transposed_time)
                append(g['seq_num'], transposed_seq_num)
                for k in keys:
                    data = np.asarray(transposed_data[k])
                    dtype = data.dtype
                    if dtype.kind == 'U':
                        data = data.astype('S')
                        dtype = data.dtype
                    if k not in g['data']:

                        g['data'].create_dataset(k, shape=(0,),
                                                 maxshape=(None,),
                                                 dtype=dtype)

                    append(g['data'][k], data)
                    append(g['timestamps'][k], transposed_ts[k])

class _CollectionMixin(object):
    def __init__(self, *args, **kwargs):
        self._config = None
        super(_CollectionMixin, self).__init__(*args, **kwargs)
        self.__event_col = None
        self.__descriptor_col = None
        self.__runstart_col = None
        self.__runstop_col = None
        ensure_path_exists(self._config['directory'])

    @property
    def config(self):
        return self._config

    @config.setter
    def config(self, val):
        self._config = val
        self.__event_col = None
        self.__descriptor_col = None
        self.__runstart_col = None
        self.__runstop_col = None

    @property
    def _runstart_col(self):
        if self.__runstart_col is None:
            fp = os.path.join(self.config['directory'], 'run_starts.json')
            self.__runstart_col = RunStartCollection(self._event_col, fp)
        return self.__runstart_col

    @property
    def _runstop_col(self):
        if self.__runstop_col is None:
            fp = os.path.join(self.config['directory'], 'run_stops.json')
            self.__runstop_col = JSONCollection(fp)
        return self.__runstop_col

    @property
    def _descriptor_col(self):
        self._event_col
        if self.__descriptor_col is None:
            fp = os.path.join(self.config['directory'],
                              'event_descriptors.json')
            self.__descriptor_col = DescriptorCollection(self._event_col, fp)
        return self.__descriptor_col

    @property
    def _event_col(self):
        if self.__event_col is None:
            self.__event_col = EventCollection(self.config['directory'])
        return self.__event_col

class MDSRO(_CollectionMixin, MDSROTemplate):
    pass

class MDS(_CollectionMixin, MDSTemplate):
    pass
<commit_msg>FIX: Accept nonscalars in HDF5 event source.<commit_after>import os
import h5py
import numpy as np
from collections import defaultdict
from .mongoquery import JSONCollection
from .base import MDSTemplate, MDSROTemplate
from ..utils import ensure_path_exists

def append(dataset, data):
    data = np.asanyarray(data)
    cur_shape = dataset.shape
    num_events = data.shape[0]
    new_shape = (dataset.shape[0] + num_events,) + cur_shape[1:]
    dataset.resize(new_shape)
    dataset[cur_shape[0]:, ...] = data

class RunStartCollection(JSONCollection):
    def __init__(self, event_col, *args, **kwargs):
        self._event_col = event_col
        super(RunStartCollection, self).__init__(*args, **kwargs)

    def insert_one(self, doc):
        self._event_col.new_runstart(doc)
        super(RunStartCollection, self).insert_one(doc, fk='uid')

class DescriptorCollection(JSONCollection):
    def __init__(self, event_col, *args, **kwargs):
        self._event_col = event_col
        super(DescriptorCollection, self).__init__(*args, **kwargs)

    def insert_one(self, doc):
        self._event_col.new_descriptor(doc)
        super(DescriptorCollection, self).insert_one(doc)

class EventCollection(object):
    def __init__(self, dirpath):
        self._runstarts = {}
        self._descriptors = {}
        self._dirpath = dirpath

    def new_runstart(self, doc):
        uid = doc['uid']
        fp = os.path.join(self._dirpath, '{}.h5'.format(uid))
        self._runstarts[uid] = fp

    def new_descriptor(self, doc):
        uid = doc['uid']
        run_start_uid = doc['run_start']
        groupname = 'desc_' + uid.replace('-', '_')
        fp = self._runstarts[run_start_uid]
        with h5py.File(fp, 'a') as f:
            g = f.create_group(groupname)
            g.create_dataset('uid', shape=(0,), maxshape=(None,), dtype='S36')
            g.create_dataset('time', shape=(0,), maxshape=(None,),
                             dtype='float64')
            g.create_dataset('seq_num', shape=(0,), maxshape=(None,))
            g.create_group('data')
            g.create_group('timestamps')
            for key in doc['data_keys']:
                # Create an empty, resizable dataset for each key.
                g['timestamps'].create_dataset(key, shape=(0,),
                                               maxshape=(None,),
                                               dtype='float64')
        self._descriptors[uid] = run_start_uid

    def find(self, query, sort=None):
        if list(query.keys()) != ['descriptor']:
            raise NotImplementedError("Only queries based on descriptor uid "
                                      "are supported.")
        desc_uid = query['descriptor']
        groupname = 'desc_' + desc_uid.replace('-', '_')
        fp = self._runstarts[self._descriptors[desc_uid]]
        with h5py.File(fp, 'r') as f:
            g = f[groupname]
            transposed_uid = list(g['uid'][:])
            transposed_seq_num = list(g['seq_num'][:])
            transposed_time = list(g['time'][:])
            transposed_data = {}
            transposed_ts = {}
            for key in g['data']:
                transposed_data[key] = list(g['data'][key][:])
                transposed_ts[key] = list(g['timestamps'][key][:])
        for _ in range(len(transposed_uid)):
            event = {}
            event['uid'] = transposed_uid.pop(0).decode()
            event['seq_num'] = transposed_seq_num.pop(0)
            event['time'] = transposed_time.pop(0)
            event['data'] = {}
            event['timestamps'] = {}
            for key in transposed_data:
                data = transposed_data[key].pop(0)
                if data.dtype.kind == 'S':
                    data = bytes(data).decode('utf-8')
                event['data'][key] = data
                event['timestamps'][key] = transposed_ts[key].pop(0)
            yield event

    def find_one(self, query):
        # not used on event_col
        raise NotImplementedError()

    def insert_one(self, doc):
        # not used on event_col
        return self.insert([doc])

    def insert(self, docs):
        # Sort docs by descriptor.
        descs = defaultdict(list)
        for doc in docs:
            descs[doc['descriptor']].append(doc)
        # Process one descriptor at a time.
        for uid, docs in descs.items():
            keys = docs[0]['data']
            transposed_data = {k: [] for k in keys}
            transposed_ts = {k: [] for k in keys}
            transposed_uid = []
            transposed_seq_num = []
            transposed_time = []
            for doc in docs:
                transposed_uid.append(doc['uid'])
                transposed_time.append(doc['time'])
                transposed_seq_num.append(doc['seq_num'])
                data = doc['data']
                ts = doc['timestamps']
                for k in keys:
                    transposed_data[k].append(data[k])
                    transposed_ts[k].append(ts[k])
            fp = self._runstarts[self._descriptors[uid]]
            groupname = 'desc_' + uid.replace('-', '_')
            with h5py.File(fp, 'a') as f:
                g = f[groupname]
                uids = [s.encode('ascii', 'ignore') for s in transposed_uid]
                append(g['uid'], uids)
                append(g['time'], transposed_time)
                append(g['seq_num'], transposed_seq_num)
                for k in keys:
                    data = np.asarray(transposed_data[k])
                    dtype = data.dtype
                    if dtype.kind == 'U':
                        data = data.astype('S')
                        dtype = data.dtype
                    if k not in g['data']:
                        shape = np.asarray(data).shape[1:]
                        g['data'].create_dataset(k, shape=(0,) + shape,
                                                 maxshape=(None,) + shape,
                                                 dtype=dtype)

                    append(g['data'][k], data)
                    append(g['timestamps'][k], transposed_ts[k])

class _CollectionMixin(object):
    def __init__(self, *args, **kwargs):
        self._config = None
        super(_CollectionMixin, self).__init__(*args, **kwargs)
        self.__event_col = None
        self.__descriptor_col = None
        self.__runstart_col = None
        self.__runstop_col = None
        ensure_path_exists(self._config['directory'])

    @property
    def config(self):
        return self._config

    @config.setter
    def config(self, val):
        self._config = val
        self.__event_col = None
        self.__descriptor_col = None
        self.__runstart_col = None
        self.__runstop_col = None

    @property
    def _runstart_col(self):
        if self.__runstart_col is None:
            fp = os.path.join(self.config['directory'], 'run_starts.json')
            self.__runstart_col = RunStartCollection(self._event_col, fp)
        return self.__runstart_col

    @property
    def _runstop_col(self):
        if self.__runstop_col is None:
            fp = os.path.join(self.config['directory'], 'run_stops.json')
            self.__runstop_col = JSONCollection(fp)
        return self.__runstop_col

    @property
    def _descriptor_col(self):
        self._event_col
        if self.__descriptor_col is None:
            fp = os.path.join(self.config['directory'],
                              'event_descriptors.json')
            self.__descriptor_col = DescriptorCollection(self._event_col, fp)
        return self.__descriptor_col

    @property
    def _event_col(self):
        if self.__event_col is None:
            self.__event_col = EventCollection(self.config['directory'])
        return self.__event_col

class MDSRO(_CollectionMixin, MDSROTemplate):
    pass

class MDS(_CollectionMixin, MDSTemplate):
    pass
<|endoftext|>


<commit_before># --------------------------------------------------------------------------
#
# Copyright (c) Microsoft Corporation. All rights reserved.
#
# The MIT License (MIT)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the ""Software""), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# --------------------------------------------------------------------------

from azure.core.pipeline import Pipeline
from azure.core import PipelineClient
from azure.core.pipeline.transport import HttpRequest
from azure.core.pipeline.policies import (
    UserAgentPolicy,
    RedirectPolicy,
    RetryPolicy
)

def test_example_requests():
    request = HttpRequest("GET", "https://bing.com")
    policies = [
        UserAgentPolicy("myuseragent"),
        RedirectPolicy(),
        RetryPolicy()
    ]
    # [START requests]
    from azure.core.pipeline.transport import RequestsTransport

    with Pipeline(transport=RequestsTransport(), policies=policies) as pipeline:
        response = pipeline.run(request)
    # [END requests]
    assert pipeline._transport.session is None
    assert response.http_response.status_code == 200

def test_example_pipeline():
    # [START build_pipeline]
    from azure.core.pipeline import Pipeline
    from azure.core.pipeline.policies import RedirectPolicy, UserAgentPolicy
    from azure.core.pipeline.transport import RequestsTransport, HttpRequest

    # example: create request and policies
    request = HttpRequest("GET", "https://bing.com")
    policies = [
        UserAgentPolicy("myuseragent"),
        RedirectPolicy()
    ]

    # run the pipeline
    with Pipeline(transport=RequestsTransport(), policies=policies) as pipeline:
        response = pipeline.run(request)
    # [END build_pipeline]
    assert pipeline._transport.session is None
    assert response.http_response.status_code == 200

def test_example_pipeline_client():
    url = "https://bing.com"
    # [START build_pipeline_client]
    from azure.core import PipelineClient
    from azure.core.pipeline.policies import RedirectPolicy, UserAgentPolicy

    # example configuration with some policies
    policies = [
        UserAgentPolicy("myuseragent"),
        RedirectPolicy()
    ]

    client = PipelineClient(base_url=url, policies=policies)
    request = client.get("https://bing.com")

    pipeline_response = client._pipeline.run(request)
    # [END build_pipeline_client]

    response = pipeline_response.http_response
    assert response.status_code == 200

def test_example_redirect_policy():

    url = "https://bing.com"

    # [START redirect_policy]
    from azure.core.pipeline.policies import RedirectPolicy

    redirect_policy = RedirectPolicy()

    # Client allows redirects. Defaults to True.
    redirect_policy.allow = True

    # The maximum allowed redirects. The default value is 30
    redirect_policy.max_redirects = 10

    # Alternatively you can disable redirects entirely
    redirect_policy = RedirectPolicy.no_redirects()

    # It can also be overridden per operation.
    client = PipelineClient(base_url=url, policies=[redirect_policy])
    request = client.get(url)
    pipeline_response = client._pipeline.run(request, permit_redirects=True, redirect_max=5)
    # [END redirect_policy]

    response = pipeline_response.http_response
    assert response.status_code == 200

def test_example_no_redirects():
    url = "https://bing.com"

    redirect_policy = RedirectPolicy.no_redirects()

    client = PipelineClient(base_url=url, policies=[redirect_policy])
    request = client.get(url)
    pipeline_response = client._pipeline.run(request)

    response = pipeline_response.http_response
    # bing returns 301 if not redirected
    assert response.status_code == 301

def test_example_retry_policy():

    url = "https://bing.com"

    policies = [
        UserAgentPolicy("myuseragent"),
        RedirectPolicy()
    ]

    # [START retry_policy]
    from azure.core.pipeline.policies import RetryPolicy

    retry_policy = RetryPolicy()

    # Total number of retries to allow. Takes precedence over other counts.
    # Default value is 10.
    retry_policy.total_retries = 5

    # How many connection-related errors to retry on.
    # These are errors raised before the request is sent to the remote server,
    # which we assume has not triggered the server to process the request. Default value is 3
    retry_policy.connect_retries = 2

    # How many times to retry on read errors.
    # These errors are raised after the request was sent to the server, so the
    # request may have side-effects. Default value is 3.
    retry_policy.read_retries = 4

    # How many times to retry on bad status codes. Default value is 3.
    retry_policy.status_retries = 3

    # A backoff factor to apply between attempts after the second try
    # (most errors are resolved immediately by a second try without a delay).
    # Retry policy will sleep for:
    #    {backoff factor} * (2 ** ({number of total retries} - 1))
    # seconds. If the backoff_factor is 0.1, then the retry will sleep
    # for [0.0s, 0.2s, 0.4s, ...] between retries.
    # The default value is 0.8.
    retry_policy.backoff_factor = 0.5

    # The maximum back off time. Default value is 120 seconds (2 minutes).
    retry_policy.backoff_max = 120

    # Alternatively you can disable redirects entirely
    retry_policy = RetryPolicy.no_retries()

    # All of these settings can also be configured per operation.
    policies.append(retry_policy)
    client = PipelineClient(base_url=url, policies=policies)
    request = client.get(url)
    pipeline_response = client._pipeline.run(
        request,
        retry_total=10,
        retry_connect=1,
        retry_read=1,
        retry_status=5,
        retry_backoff_factor=0.5,
        retry_backoff_max=120,
        retry_on_methods=['GET']
    )
    # [END retry_policy]

    response = pipeline_response.http_response
    assert response.status_code == 200

def test_example_no_retries():
    url = "https://bing.com"

    retry_policy = RetryPolicy.no_retries()

    client = PipelineClient(base_url=url, policies=[retry_policy])
    request = client.get(url)
    pipeline_response = client._pipeline.run(request)

    response = pipeline_response.http_response
    # bing returns 301 if not retried
    assert response.status_code == 301
<commit_msg>check response code is an int rather than 200 (#9192)<commit_after># --------------------------------------------------------------------------
#
# Copyright (c) Microsoft Corporation. All rights reserved.
#
# The MIT License (MIT)
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the ""Software""), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# --------------------------------------------------------------------------

from azure.core.pipeline import Pipeline
from azure.core import PipelineClient
from azure.core.pipeline.transport import HttpRequest
from azure.core.pipeline.policies import (
    UserAgentPolicy,
    RedirectPolicy,
    RetryPolicy
)

def test_example_requests():
    request = HttpRequest("GET", "https://bing.com")
    policies = [
        UserAgentPolicy("myuseragent"),
        RedirectPolicy(),
        RetryPolicy()
    ]
    # [START requests]
    from azure.core.pipeline.transport import RequestsTransport

    with Pipeline(transport=RequestsTransport(), policies=policies) as pipeline:
        response = pipeline.run(request)
    # [END requests]
    assert pipeline._transport.session is None
    assert response.http_response.status_code == 200

def test_example_pipeline():
    # [START build_pipeline]
    from azure.core.pipeline import Pipeline
    from azure.core.pipeline.policies import RedirectPolicy, UserAgentPolicy
    from azure.core.pipeline.transport import RequestsTransport, HttpRequest

    # example: create request and policies
    request = HttpRequest("GET", "https://bing.com")
    policies = [
        UserAgentPolicy("myuseragent"),
        RedirectPolicy()
    ]

    # run the pipeline
    with Pipeline(transport=RequestsTransport(), policies=policies) as pipeline:
        response = pipeline.run(request)
    # [END build_pipeline]
    assert pipeline._transport.session is None
    assert response.http_response.status_code == 200

def test_example_pipeline_client():
    url = "https://bing.com"
    # [START build_pipeline_client]
    from azure.core import PipelineClient
    from azure.core.pipeline.policies import RedirectPolicy, UserAgentPolicy

    # example configuration with some policies
    policies = [
        UserAgentPolicy("myuseragent"),
        RedirectPolicy()
    ]

    client = PipelineClient(base_url=url, policies=policies)
    request = client.get("https://bing.com")

    pipeline_response = client._pipeline.run(request)
    # [END build_pipeline_client]

    response = pipeline_response.http_response
    assert isinstance(response.status_code, int)

def test_example_redirect_policy():

    url = "https://bing.com"

    # [START redirect_policy]
    from azure.core.pipeline.policies import RedirectPolicy

    redirect_policy = RedirectPolicy()

    # Client allows redirects. Defaults to True.
    redirect_policy.allow = True

    # The maximum allowed redirects. The default value is 30
    redirect_policy.max_redirects = 10

    # Alternatively you can disable redirects entirely
    redirect_policy = RedirectPolicy.no_redirects()

    # It can also be overridden per operation.
    client = PipelineClient(base_url=url, policies=[redirect_policy])
    request = client.get(url)
    pipeline_response = client._pipeline.run(request, permit_redirects=True, redirect_max=5)
    # [END redirect_policy]

    response = pipeline_response.http_response
    assert response.status_code == 200

def test_example_no_redirects():
    url = "https://bing.com"

    redirect_policy = RedirectPolicy.no_redirects()

    client = PipelineClient(base_url=url, policies=[redirect_policy])
    request = client.get(url)
    pipeline_response = client._pipeline.run(request)

    response = pipeline_response.http_response
    # bing returns 301 if not redirected
    assert response.status_code == 301

def test_example_retry_policy():

    url = "https://bing.com"

    policies = [
        UserAgentPolicy("myuseragent"),
        RedirectPolicy()
    ]

    # [START retry_policy]
    from azure.core.pipeline.policies import RetryPolicy

    retry_policy = RetryPolicy()

    # Total number of retries to allow. Takes precedence over other counts.
    # Default value is 10.
    retry_policy.total_retries = 5

    # How many connection-related errors to retry on.
    # These are errors raised before the request is sent to the remote server,
    # which we assume has not triggered the server to process the request. Default value is 3
    retry_policy.connect_retries = 2

    # How many times to retry on read errors.
    # These errors are raised after the request was sent to the server, so the
    # request may have side-effects. Default value is 3.
    retry_policy.read_retries = 4

    # How many times to retry on bad status codes. Default value is 3.
    retry_policy.status_retries = 3

    # A backoff factor to apply between attempts after the second try
    # (most errors are resolved immediately by a second try without a delay).
    # Retry policy will sleep for:
    #    {backoff factor} * (2 ** ({number of total retries} - 1))
    # seconds. If the backoff_factor is 0.1, then the retry will sleep
    # for [0.0s, 0.2s, 0.4s, ...] between retries.
    # The default value is 0.8.
    retry_policy.backoff_factor = 0.5

    # The maximum back off time. Default value is 120 seconds (2 minutes).
    retry_policy.backoff_max = 120

    # Alternatively you can disable redirects entirely
    retry_policy = RetryPolicy.no_retries()

    # All of these settings can also be configured per operation.
    policies.append(retry_policy)
    client = PipelineClient(base_url=url, policies=policies)
    request = client.get(url)
    pipeline_response = client._pipeline.run(
        request,
        retry_total=10,
        retry_connect=1,
        retry_read=1,
        retry_status=5,
        retry_backoff_factor=0.5,
        retry_backoff_max=120,
        retry_on_methods=['GET']
    )
    # [END retry_policy]

    response = pipeline_response.http_response
    assert response.status_code == 200

def test_example_no_retries():
    url = "https://bing.com"

    retry_policy = RetryPolicy.no_retries()

    client = PipelineClient(base_url=url, policies=[retry_policy])
    request = client.get(url)
    pipeline_response = client._pipeline.run(request)

    response = pipeline_response.http_response
    # bing returns 301 if not retried
    assert response.status_code == 301
<|endoftext|>


<commit_before>from SimPEG.EM.Static.DC import Src
from SimPEG import Props
from SimPEG.Utils import sdiag
import scipy.sparse as sp
import numpy as np
from SimPEG.EM.Static.DC import Survey

class StreamingCurrents(Src.BaseSrc):

    L = None
    mesh = None
    # "Hydraulic Head (m)"
    # "Streaming current source (A/m^3)"
    # "Streaming current density (A/m^2)"
    modelType = None

    def __init__(self, rxList, **kwargs):
        Src.BaseSrc.__init__(self, rxList, **kwargs)
        if self.L is None:
            raise Exception("SP source requires cross coupling coefficient L")
        if self.mesh is None:
            raise Exception("SP source requires mesh")
        self.mesh.setCellGradBC("neumann")
        # TODO: remove below when confirmed
        # self.V = sp.block_diag([sdiag(self.mesh.vol)]*3)
        # self.Div = -self.mesh.cellGrad.T

    def eval(self, prob):
        """

            Computing source term using:

            - Hydraulic head: h
            - Cross coupling coefficient: L

            .. math::

                -\nabla \cdot \vec{j}^s = \nabla \cdot L \nabla \phi \\

        """
        if prob._formulation == 'HJ':
            if self.modelType == "Head":
                q = -self.Div*self.MfLiI*prob.Grad*prob.h
            elif self.modelType == "CurrentSource":
                q = prob.q
            elif self.modelType == "CurrentDensity":
                # q = -self.Div*prob.mesh.aveF2CCV.T* self.V *np.r_[prob.jsx, prob.jsy, prob.jsz]
                q = prob.Grad.T*prob.mesh.aveCC2FV*np.r_[prob.jsx, prob.jsy, prob.jsz]
            else:
                raise NotImplementedError()
        elif prob._formulation == 'EB':
            raise NotImplementedError()
        return q

    def evalDeriv(self, prob, v=None, adjoint=False):
        if prob._formulation == 'HJ':
            if adjoint:
                if self.modelType == "Head":
                    srcDeriv = - prob.hDeriv.T * prob.Grad.T * self.MfLiI.T * (self.Div.T * v)
                elif self.modelType == "CurrentSource":
                    srcDeriv = prob.qDeriv.T * v
                elif self.modelType == "CurrentDensity":
                    jsDeriv = sp.vstack((prob.jsxDeriv, prob.jsyDeriv, prob.jszDeriv))
                    # srcDeriv = - jsDeriv.T * prob.mesh.aveF2CCV * self.V * (self.Div.T*v)
                    srcDeriv = jsDeriv.T * prob.mesh.aveCC2FV.T * (prob.Grad*v)
                else:
                    raise NotImplementedError()
            else:
                if self.modelType == "Head":
                    srcDeriv = -self.Div*self.MfLiI*prob.Grad*(prob.hDeriv*v)
                elif self.modelType == "CurrentSource":
                    srcDeriv = prob.qDeriv * v
                elif self.modelType == "CurrentDensity":
                    jsDeriv = sp.vstack((prob.jsxDeriv, prob.jsyDeriv, prob.jszDeriv))
                    # srcDeriv = -self.Div * prob.mesh.aveF2CCV.T* self.V * (jsDeriv*v)
                    srcDeriv = prob.Grad.T * prob.mesh.aveCC2FV*(jsDeriv*v)
                else:
                    raise NotImplementedError()
        elif prob._formulation == 'EB':
            raise NotImplementedError()
        return srcDeriv

    @property
    def MfLi(self):
        """
            :code:`MfLi`
        """
        if getattr(self, '_MfLi', None) is None:
            self._MfLi = self.mesh.getFaceInnerProduct(1./self.L)
        return seself.lf._MfLi

    @property
    def MfLiI(self):
        """
            Inverse of :code:`_MfLiI`
        """
        if getattr(self, '_MfLiI', None) is None:
            self._MfLiI = self.mesh.getFaceInnerProduct(1./self.L, invMat=True)
        return self._MfLiI

if __name__ == '__main__':
    from SimPEG import Mesh, np
    mesh = Mesh.TensorMesh([10, 10])
    L = np.ones(mesh.nC)
    src = StreamingCurrents([], L=L, mesh=mesh)
    thing = src.MfLiI
    if thing is not None:
        pass
<commit_msg>use `prob.Grad` by letting streaming current, js, at boundary is zero.<commit_after>from SimPEG.EM.Static.DC import Src
from SimPEG import Props
from SimPEG.Utils import sdiag
import scipy.sparse as sp
import numpy as np
from SimPEG.EM.Static.DC import Survey

class StreamingCurrents(Src.BaseSrc):

    L = None
    mesh = None
    # "Hydraulic Head (m)"
    # "Streaming current source (A/m^3)"
    # "Streaming current density (A/m^2)"
    modelType = None

    def __init__(self, rxList, **kwargs):
        Src.BaseSrc.__init__(self, rxList, **kwargs)
        if self.L is None:
            raise Exception("SP source requires cross coupling coefficient L")
        if self.mesh is None:
            raise Exception("SP source requires mesh")
        # TODO: remove below when confirmed
        # self.mesh.setCellGradBC("neumann")
        # self.V = sp.block_diag([sdiag(self.mesh.vol)]*3)
        # self.Div = -self.mesh.cellGrad.T

    def eval(self, prob):
        """

            Computing source term using:

            - Hydraulic head: h
            - Cross coupling coefficient: L

            .. math::

                -\nabla \cdot \vec{j}^s = \nabla \cdot L \nabla \phi \\

        """
        if prob._formulation == 'HJ':
            if self.modelType == "Head":
                q = prob.Grad.T*self.MfLiI*prob.Grad*prob.h
            elif self.modelType == "CurrentSource":
                q = prob.q
            elif self.modelType == "CurrentDensity":
                # q = -self.Div*prob.mesh.aveF2CCV.T* self.V *np.r_[prob.jsx, prob.jsy, prob.jsz]
                q = prob.Grad.T*prob.mesh.aveCC2FV*np.r_[prob.jsx, prob.jsy, prob.jsz]
            else:
                raise NotImplementedError()
        elif prob._formulation == 'EB':
            raise NotImplementedError()
        return q

    def evalDeriv(self, prob, v=None, adjoint=False):
        if prob._formulation == 'HJ':
            if adjoint:
                if self.modelType == "Head":
                    srcDeriv = prob.hDeriv.T * prob.Grad.T * self.MfLiI.T * (prob.Grad * v)
                elif self.modelType == "CurrentSource":
                    srcDeriv = prob.qDeriv.T * v
                elif self.modelType == "CurrentDensity":
                    jsDeriv = sp.vstack((prob.jsxDeriv, prob.jsyDeriv, prob.jszDeriv))
                    # srcDeriv = - jsDeriv.T * prob.mesh.aveF2CCV * self.V * (self.Div.T*v)
                    srcDeriv = jsDeriv.T * prob.mesh.aveCC2FV.T * (prob.Grad*v)
                else:
                    raise NotImplementedError()
            else:
                if self.modelType == "Head":
                    srcDeriv = prob.Grad.T*self.MfLiI*prob.Grad*(prob.hDeriv*v)
                elif self.modelType == "CurrentSource":
                    srcDeriv = prob.qDeriv * v
                elif self.modelType == "CurrentDensity":
                    jsDeriv = sp.vstack((prob.jsxDeriv, prob.jsyDeriv, prob.jszDeriv))
                    # srcDeriv = -self.Div * prob.mesh.aveF2CCV.T* self.V * (jsDeriv*v)
                    srcDeriv = prob.Grad.T * prob.mesh.aveCC2FV*(jsDeriv*v)
                else:
                    raise NotImplementedError()
        elif prob._formulation == 'EB':
            raise NotImplementedError()
        return srcDeriv

    @property
    def MfLi(self):
        """
            :code:`MfLi`
        """
        if getattr(self, '_MfLi', None) is None:
            self._MfLi = self.mesh.getFaceInnerProduct(1./self.L)
        return seself.lf._MfLi

    @property
    def MfLiI(self):
        """
            Inverse of :code:`_MfLiI`
        """
        if getattr(self, '_MfLiI', None) is None:
            self._MfLiI = self.mesh.getFaceInnerProduct(1./self.L, invMat=True)
        return self._MfLiI

if __name__ == '__main__':
    from SimPEG import Mesh, np
    mesh = Mesh.TensorMesh([10, 10])
    L = np.ones(mesh.nC)
    src = StreamingCurrents([], L=L, mesh=mesh)
    thing = src.MfLiI
    if thing is not None:
        pass
<|endoftext|>