<commit_before>"Test reduce functions."

import warnings
import traceback
#  from itertools import permutations

import numpy as np
from numpy.testing import (assert_equal, assert_raises,
                           assert_array_almost_equal)

import bottleneck as bn
from .util import arrays, array_order, DTYPES
import pytest

@pytest.mark.parametrize("func", bn.get_functions("reduce"),
                         ids=lambda x: x.__name__)
def test_reduce(func):
    "test reduce functions"
    return unit_maker(func)

def unit_maker(func, decimal=5, skip_dtype=['nansum', 'ss']):
    "Test that bn.xxx gives the same output as bn.slow.xxx."
    fmt = '\nfunc %s | input %s (%s) | shape %s | axis %s | order %s\n'
    fmt += '\nInput array:\n%s\n'
    name = func.__name__
    func0 = eval('bn.slow.%s' % name)
    for i, a in enumerate(arrays(name)):
        if a.ndim == 0:
            axes = [None]  # numpy can't handle e.g. np.nanmean(9, axis=-1)
        else:
            axes = list(range(-1, a.ndim)) + [None]
        for axis in axes:
            actual = 'Crashed'
            desired = 'Crashed'
            actualraised = False
            try:
                # do not use a.copy() here because it will C order the array
                actual = func(a, axis=axis)
            except:  # noqa
                actualraised = True
            desiredraised = False
            try:
                with warnings.catch_warnings():
                    warnings.simplefilter("ignore")
                    desired = func0(a, axis=axis)
            except:  # noqa
                desiredraised = True
            if actualraised and desiredraised:
                pass
            else:
                tup = (name, 'a' + str(i), str(a.dtype), str(a.shape),
                       str(axis), array_order(a), a)
                err_msg = fmt % tup
                if actualraised != desiredraised:
                    if actualraised:
                        fmt2 = '\nbn.%s raised\nbn.slow.%s ran\n\n%s'
                    else:
                        fmt2 = '\nbn.%s ran\nbn.slow.%s raised\n\n%s'
                    msg = fmt2 % (name, name, traceback.format_exc())
                    err_msg += msg
                    assert False, err_msg
                assert_array_almost_equal(actual, desired, decimal, err_msg)
                err_msg += '\n dtype mismatch %s %s'
                if name not in skip_dtype:
                    if hasattr(actual, 'dtype') and hasattr(desired, 'dtype'):
                        da = actual.dtype
                        dd = desired.dtype
                        assert_equal(da, dd, err_msg % (da, dd))

# ---------------------------------------------------------------------------
# Test argument parsing

@pytest.mark.parametrize("func", bn.get_functions("reduce"),
                         ids=lambda x: x.__name__)
def test_arg_parsing(func):
    "test argument parsing"
    return unit_maker_argparse(func)

def unit_maker_argparse(func, decimal=5):
    "test argument parsing."

    name = func.__name__
    func0 = eval('bn.slow.%s' % name)

    a = np.array([1., 2, 3])

    fmt = '\n%s' % func
    fmt += '%s\n'
    fmt += '\nInput array:\n%s\n' % a

    actual = func(a)
    desired = func0(a)
    err_msg = fmt % "(a)"
    assert_array_almost_equal(actual, desired, decimal, err_msg)

    actual = func(a, 0)
    desired = func0(a, 0)
    err_msg = fmt % "(a, 0)"
    assert_array_almost_equal(actual, desired, decimal, err_msg)

    actual = func(a, None)
    desired = func0(a, None)
    err_msg = fmt % "(a, None)"
    assert_array_almost_equal(actual, desired, decimal, err_msg)

    actual = func(a, axis=0)
    desired = func0(a, axis=0)
    err_msg = fmt % "(a, axis=0)"
    assert_array_almost_equal(actual, desired, decimal, err_msg)

    actual = func(a, axis=None)
    desired = func0(a, axis=None)
    err_msg = fmt % "(a, axis=None)"
    assert_array_almost_equal(actual, desired, decimal, err_msg)

    actual = func(a=a)
    desired = func0(a=a)
    err_msg = fmt % "(a)"
    assert_array_almost_equal(actual, desired, decimal, err_msg)

    # regression test: make sure len(kwargs) == 0 doesn't raise
    args = (a, 0)
    kwargs = {}
    func(*args, **kwargs)

@pytest.mark.parametrize("func", bn.get_functions("reduce"))
def test_arg_parse_raises(func):
    "test argument parsing raises in reduce"
    return unit_maker_argparse_raises(func)

def unit_maker_argparse_raises(func):
    "test argument parsing raises in reduce"
    a = np.array([1., 2, 3])
    assert_raises(TypeError, func)
    assert_raises(TypeError, func, axis=a)
    assert_raises(TypeError, func, a, axis=0, extra=0)
    assert_raises(TypeError, func, a, axis=0, a=a)
    assert_raises(TypeError, func, a, 0, 0, 0, 0, 0)
    assert_raises(TypeError, func, a, axis='0')
    if func.__name__ not in ('nanstd', 'nanvar'):
        assert_raises(TypeError, func, a, ddof=0)
    assert_raises(TypeError, func, a, a)
    # assert_raises(TypeError, func, None) results vary

# ---------------------------------------------------------------------------
# Check that exceptions are raised

def test_nanmax_size_zero(dtypes=DTYPES):
    "Test nanmax for size zero input arrays."
    shapes = [(0,), (2, 0), (1, 2, 0)]
    for shape in shapes:
        for dtype in dtypes:
            a = np.zeros(shape, dtype=dtype)
            assert_raises(ValueError, bn.nanmax, a)
            assert_raises(ValueError, bn.slow.nanmax, a)

def test_nanmin_size_zero(dtypes=DTYPES):
    "Test nanmin for size zero input arrays."
    shapes = [(0,), (2, 0), (1, 2, 0)]
    for shape in shapes:
        for dtype in dtypes:
            a = np.zeros(shape, dtype=dtype)
            assert_raises(ValueError, bn.nanmin, a)
            assert_raises(ValueError, bn.slow.nanmin, a)

# ---------------------------------------------------------------------------
# nanstd and nanvar regression test (issue #60)

def test_nanstd_issue60():
    "nanstd regression test (issue #60)"

    f = bn.nanstd([1.0], ddof=1)
    with np.errstate(invalid='ignore'):
        s = bn.slow.nanstd([1.0], ddof=1)
    assert_equal(f, s, err_msg="bn.nanstd([1.0], ddof=1) wrong")

    f = bn.nanstd([1], ddof=1)
    with np.errstate(invalid='ignore'):
        s = bn.slow.nanstd([1], ddof=1)
    assert_equal(f, s, err_msg="bn.nanstd([1], ddof=1) wrong")

    f = bn.nanstd([1, np.nan], ddof=1)
    with np.errstate(invalid='ignore'):
        s = bn.slow.nanstd([1, np.nan], ddof=1)
    assert_equal(f, s, err_msg="bn.nanstd([1, nan], ddof=1) wrong")

    f = bn.nanstd([[1, np.nan], [np.nan, 1]], axis=0, ddof=1)
    with np.errstate(invalid='ignore'):
        s = bn.slow.nanstd([[1, np.nan], [np.nan, 1]], axis=0, ddof=1)
    assert_equal(f, s, err_msg="issue #60 regression")

def test_nanvar_issue60():
    "nanvar regression test (issue #60)"

    f = bn.nanvar([1.0], ddof=1)
    with np.errstate(invalid='ignore'):
        s = bn.slow.nanvar([1.0], ddof=1)
    assert_equal(f, s, err_msg="bn.nanvar([1.0], ddof=1) wrong")

    f = bn.nanvar([1], ddof=1)
    with np.errstate(invalid='ignore'):
        s = bn.slow.nanvar([1], ddof=1)
    assert_equal(f, s, err_msg="bn.nanvar([1], ddof=1) wrong")

    f = bn.nanvar([1, np.nan], ddof=1)
    with np.errstate(invalid='ignore'):
        s = bn.slow.nanvar([1, np.nan], ddof=1)
    assert_equal(f, s, err_msg="bn.nanvar([1, nan], ddof=1) wrong")

    f = bn.nanvar([[1, np.nan], [np.nan, 1]], axis=0, ddof=1)
    with np.errstate(invalid='ignore'):
        s = bn.slow.nanvar([[1, np.nan], [np.nan, 1]], axis=0, ddof=1)
    assert_equal(f, s, err_msg="issue #60 regression")
<commit_msg>TST: add test cases for n < ddof for nanstd and nanvar<commit_after>"Test reduce functions."

import warnings
import traceback
#  from itertools import permutations

import numpy as np
from numpy.testing import (assert_equal, assert_raises,
                           assert_array_almost_equal)

import bottleneck as bn
from .util import arrays, array_order, DTYPES
import pytest

@pytest.mark.parametrize("func", bn.get_functions("reduce"),
                         ids=lambda x: x.__name__)
def test_reduce(func):
    "test reduce functions"
    return unit_maker(func)

def unit_maker(func, decimal=5, skip_dtype=['nansum', 'ss']):
    "Test that bn.xxx gives the same output as bn.slow.xxx."
    fmt = '\nfunc %s | input %s (%s) | shape %s | axis %s | order %s\n'
    fmt += '\nInput array:\n%s\n'
    name = func.__name__
    func0 = eval('bn.slow.%s' % name)
    for i, a in enumerate(arrays(name)):
        if a.ndim == 0:
            axes = [None]  # numpy can't handle e.g. np.nanmean(9, axis=-1)
        else:
            axes = list(range(-1, a.ndim)) + [None]
        for axis in axes:
            actual = 'Crashed'
            desired = 'Crashed'
            actualraised = False
            try:
                # do not use a.copy() here because it will C order the array
                actual = func(a, axis=axis)
            except:  # noqa
                actualraised = True
            desiredraised = False
            try:
                with warnings.catch_warnings():
                    warnings.simplefilter("ignore")
                    desired = func0(a, axis=axis)
            except:  # noqa
                desiredraised = True
            if actualraised and desiredraised:
                pass
            else:
                tup = (name, 'a' + str(i), str(a.dtype), str(a.shape),
                       str(axis), array_order(a), a)
                err_msg = fmt % tup
                if actualraised != desiredraised:
                    if actualraised:
                        fmt2 = '\nbn.%s raised\nbn.slow.%s ran\n\n%s'
                    else:
                        fmt2 = '\nbn.%s ran\nbn.slow.%s raised\n\n%s'
                    msg = fmt2 % (name, name, traceback.format_exc())
                    err_msg += msg
                    assert False, err_msg
                assert_array_almost_equal(actual, desired, decimal, err_msg)
                err_msg += '\n dtype mismatch %s %s'
                if name not in skip_dtype:
                    if hasattr(actual, 'dtype') and hasattr(desired, 'dtype'):
                        da = actual.dtype
                        dd = desired.dtype
                        assert_equal(da, dd, err_msg % (da, dd))

# ---------------------------------------------------------------------------
# Test argument parsing

@pytest.mark.parametrize("func", bn.get_functions("reduce"),
                         ids=lambda x: x.__name__)
def test_arg_parsing(func):
    "test argument parsing"
    return unit_maker_argparse(func)

def unit_maker_argparse(func, decimal=5):
    "test argument parsing."

    name = func.__name__
    func0 = eval('bn.slow.%s' % name)

    a = np.array([1., 2, 3])

    fmt = '\n%s' % func
    fmt += '%s\n'
    fmt += '\nInput array:\n%s\n' % a

    actual = func(a)
    desired = func0(a)
    err_msg = fmt % "(a)"
    assert_array_almost_equal(actual, desired, decimal, err_msg)

    actual = func(a, 0)
    desired = func0(a, 0)
    err_msg = fmt % "(a, 0)"
    assert_array_almost_equal(actual, desired, decimal, err_msg)

    actual = func(a, None)
    desired = func0(a, None)
    err_msg = fmt % "(a, None)"
    assert_array_almost_equal(actual, desired, decimal, err_msg)

    actual = func(a, axis=0)
    desired = func0(a, axis=0)
    err_msg = fmt % "(a, axis=0)"
    assert_array_almost_equal(actual, desired, decimal, err_msg)

    actual = func(a, axis=None)
    desired = func0(a, axis=None)
    err_msg = fmt % "(a, axis=None)"
    assert_array_almost_equal(actual, desired, decimal, err_msg)

    actual = func(a=a)
    desired = func0(a=a)
    err_msg = fmt % "(a)"
    assert_array_almost_equal(actual, desired, decimal, err_msg)

    # regression test: make sure len(kwargs) == 0 doesn't raise
    args = (a, 0)
    kwargs = {}
    func(*args, **kwargs)

@pytest.mark.parametrize("func", bn.get_functions("reduce"))
def test_arg_parse_raises(func):
    "test argument parsing raises in reduce"
    return unit_maker_argparse_raises(func)

def unit_maker_argparse_raises(func):
    "test argument parsing raises in reduce"
    a = np.array([1., 2, 3])
    assert_raises(TypeError, func)
    assert_raises(TypeError, func, axis=a)
    assert_raises(TypeError, func, a, axis=0, extra=0)
    assert_raises(TypeError, func, a, axis=0, a=a)
    assert_raises(TypeError, func, a, 0, 0, 0, 0, 0)
    assert_raises(TypeError, func, a, axis='0')
    if func.__name__ not in ('nanstd', 'nanvar'):
        assert_raises(TypeError, func, a, ddof=0)
    assert_raises(TypeError, func, a, a)
    # assert_raises(TypeError, func, None) results vary

# ---------------------------------------------------------------------------
# Check that exceptions are raised

def test_nanmax_size_zero(dtypes=DTYPES):
    "Test nanmax for size zero input arrays."
    shapes = [(0,), (2, 0), (1, 2, 0)]
    for shape in shapes:
        for dtype in dtypes:
            a = np.zeros(shape, dtype=dtype)
            assert_raises(ValueError, bn.nanmax, a)
            assert_raises(ValueError, bn.slow.nanmax, a)

def test_nanmin_size_zero(dtypes=DTYPES):
    "Test nanmin for size zero input arrays."
    shapes = [(0,), (2, 0), (1, 2, 0)]
    for shape in shapes:
        for dtype in dtypes:
            a = np.zeros(shape, dtype=dtype)
            assert_raises(ValueError, bn.nanmin, a)
            assert_raises(ValueError, bn.slow.nanmin, a)

# ---------------------------------------------------------------------------
# nanstd and nanvar regression test (issue #60)

def test_nanstd_issue60():
    "nanstd regression test (issue #60)"

    f = bn.nanstd([1.0], ddof=1)
    with np.errstate(invalid='ignore'):
        s = bn.slow.nanstd([1.0], ddof=1)
    assert_equal(f, s, err_msg="bn.nanstd([1.0], ddof=1) wrong")

    f = bn.nanstd([1], ddof=1)
    with np.errstate(invalid='ignore'):
        s = bn.slow.nanstd([1], ddof=1)
    assert_equal(f, s, err_msg="bn.nanstd([1], ddof=1) wrong")

    f = bn.nanstd([1, np.nan], ddof=1)
    with np.errstate(invalid='ignore'):
        s = bn.slow.nanstd([1, np.nan], ddof=1)
    assert_equal(f, s, err_msg="bn.nanstd([1, nan], ddof=1) wrong")

    f = bn.nanstd([[1, np.nan], [np.nan, 1]], axis=0, ddof=1)
    with np.errstate(invalid='ignore'):
        s = bn.slow.nanstd([[1, np.nan], [np.nan, 1]], axis=0, ddof=1)
    assert_equal(f, s, err_msg="issue #60 regression")

def test_nanvar_issue60():
    "nanvar regression test (issue #60)"

    f = bn.nanvar([1.0], ddof=1)
    with np.errstate(invalid='ignore'):
        s = bn.slow.nanvar([1.0], ddof=1)
    assert_equal(f, s, err_msg="bn.nanvar([1.0], ddof=1) wrong")

    f = bn.nanvar([1], ddof=1)
    with np.errstate(invalid='ignore'):
        s = bn.slow.nanvar([1], ddof=1)
    assert_equal(f, s, err_msg="bn.nanvar([1], ddof=1) wrong")

    f = bn.nanvar([1, np.nan], ddof=1)
    with np.errstate(invalid='ignore'):
        s = bn.slow.nanvar([1, np.nan], ddof=1)
    assert_equal(f, s, err_msg="bn.nanvar([1, nan], ddof=1) wrong")

    f = bn.nanvar([[1, np.nan], [np.nan, 1]], axis=0, ddof=1)
    with np.errstate(invalid='ignore'):
        s = bn.slow.nanvar([[1, np.nan], [np.nan, 1]], axis=0, ddof=1)
    assert_equal(f, s, err_msg="issue #60 regression")

@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("func", (bn.nanstd, bn.nanvar),
                         ids=lambda x: x.__name__)
def test_ddof_nans(func, dtype):
    array = np.ones((1, 1), dtype=dtype)
    for axis in [None, 0, 1, -1]:
        result = func(array, axis=axis, ddof=3)
        assert np.isnan(result)
<|endoftext|>


<commit_before>#   Copyright (c) 2006-2007 Open Source Applications Foundation
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

"""This module provides the communication and management between the various
server interfaces and the browser's js interface"""

import copy, os
import simplejson
import logging
from uuid import uuid1
import windmill
from time import sleep
from dateutil.parser import parse as dateutil_parse

test_results_logger = logging.getLogger('test_results')

class ControllerQueue(object):

    def __init__(self, command_resolution_suite, test_resolution_suite):
        self.crs = command_resolution_suite
        self.trs = test_resolution_suite
        self.command_queue = []
        self.test_queue = []
        self.current_suite = None

    def add_command(self, command, suite_name=None):
        if suite_name is None and not command.get('suite_name'):
            suite_name = self.current_suite
        command['suite_name'] = suite_name
        self.command_queue.append(command)

    def add_test(self, test, suite_name=None):
        if suite_name is None and not test.get('suite_name'):
            suite_name = self.current_suite
        test['suite_name'] = suite_name
        self.test_queue.append(test)

    def start_suite(self, suite_name):
        self.current_suite = suite_name

    def stop_suite(self):
        self.current_suite = None

    def command(self, command):

        self.command_queue.insert(0, command)

    def next_action(self):

        if len(self.command_queue) is not 0:
            command = self.command_queue.pop(0)
            self.crs.add(command)
            return command
        elif len(self.test_queue) is not 0:
            test = self.test_queue.pop(0)
            self.trs.add(test)
            return test
        else:
            return None

callback = {'version':'0.1'}

class TestResolutionSuite(object):
    """Collection of tests run and results"""
    result_processor = None

    def __init__(self):
        self.unresolved = {}
        self.resolved = {}
        self.current_suite = None

    def resolve(self, result, uuid, starttime, endtime, debug=None):
        """Resolve test by uuid"""
        starttime = dateutil_parse(starttime)
        endtime = dateutil_parse(endtime)
        test = self.unresolved.pop(uuid)
        test['result'] = result
        test['starttime'] = starttime
        test['endtime'] = endtime
        self.resolved[uuid] = test

        if result is False:
            test_results_logger.error('Test Failue in test %s' % test)
        elif result is True:
            test_results_logger.debug('Test Success in test %s' % test)

        if self.result_processor is not None:
            if result is False:
                self.result_processor.failure(test, debug=debug)
            elif result is True:
                self.result_processor.success(test, debug=debug)

        if test.has_key('result_callback'):
            test['result_callback'](result, debug)

    def start_suite(self, suite_name):
        self.current_suite = suite_name

    def stop_suite(self):
        self.current_suite = None

    def add(self, test, suite_name=None):
        if suite_name is None and not test.get('suite_name'):
            suite_name = self.current_suite
        test['suite_name'] = suite_name
        self.unresolved[test['params']['uuid']] = test

class CommandResolutionSuite(object):

    def __init__(self):
        self.unresolved = {}
        self.resolved ={}

    def resolve(self, status, uuid, result):
        """Resolve command by uuid"""
        command = self.unresolved.pop(uuid, None)
        command['status'] = status
        command['result'] = result
        self.resolved[uuid] = command

        if status is False:
            test_results_logger.error('Command Failure in command %s' % command)
        elif status is True:
            test_results_logger.debug('Command Succes in command %s' % command)

        if command.has_key('result_callback'):
            command['result_callback'](status, result)

    def add(self, command, suite_name=None):
        self.unresolved[command['params']['uuid']] = command

class RecursiveRPC(object):

    def __init__(self, execution_method):
        self.execution_method = execution_method

    def __getattr__(self, key):
        """Call a method on the controller as if it was a local method"""
        class ExecuteJSONRecursiveAttribute(object):
            def __init__(self, name, execution_method):
                self.name = name
                self.execution_method = execution_method
            def __call__(self, **kwargs):
                rpc = {'method':self.name, 'params':kwargs}
                return self.execution_method(rpc)
            def __getattr__(self, key):
                return ExecuteRecursiveAttribute(self.name+'.'+key, self.execution_method)

        return ExecuteRecursiveAttribute(key, self.execution_method)

class RPCMethods(object):

    def __init__(self, queue, test_resolution_suite, command_resolution_suite):
        self._queue = queue
        self._logger = logging.getLogger('jsonrpc_methods_instance')
        self._test_resolution_suite = test_resolution_suite
        self._command_resolution_suite = command_resolution_suite

    def start_suite(self, suite_name):
        self._test_resolution_suite.start_suite(suite_name)
        self._queue.start_suite(suite_name)
        return 200

    def stop_suite(self):
        self._test_resolution_suite.stop_suite()
        return 200

    def add_object(self, queue_method, action_object, suite_name=None):
        """Procedue neutral addition method"""
        callback_object = copy.copy(callback)
        callback_object.update(action_object)
        if not callback_object['params'].get('uuid'):
            callback_object['params']['uuid'] = str(uuid1())
        self._logger.debug('Adding object %s' % str(callback_object))
        queue_method(callback_object, suite_name=suite_name)
        return callback_object['params']['uuid']

    def add_json_test(self, json, suite_name=None):
        """Add test from json object with 'method' and 'params' defined"""
        action_object = simplejson.loads(json)
        self.add_object(self._queue.add_test,  action_object, suite_name)
        return 200

    def add_test(self, test_object, suite_name=None):
        self.add_object(self._queue.add_test,  test_object, suite_name)
        return 200

    def add_json_command(self, json):
        """Add command from json object with 'method' and 'params' defined"""
        action_object = simplejson.loads(json)
        self.add_object(self._queue.add_command, action_object)
        return 200

    def add_command(self, command_object):
        """Add command from object"""
        self.add_object(self._queue.add_command, command_object)
        return 200

    def execute_object(self, queue_method, resolution_suite, action_object):
        """Procedure neutral blocking exeution of a given object."""
        uuid = self.add_object(queue_method, resolution_suite, action_object)

        while not resolution_suite.resolved.get(uuid):
            sleep(1)

        result = resolution_suite.resolved[uuid]
        result.pop('totaltime', None)
        return result

    def execute_json_command(self, json):
        """Add command from json object with 'method' and 'params' defined, block until it returns, return the result"""
        action_object = simplejson.loads(json)
        return self.execute_object(self._queue.add_command, self._command_resolution_suite, action_object)

    def execute_json_test(self, json):
        """Add test from json object with 'method' and 'params' defined, block until it returns, return the result"""
        action_object = simplejson.loads(json)
        return self.execute_object(self._queue.add_test, self._test_resolution_suite, action_object)

    def execute_command(self, action_object):
        """Add command from dict object with 'method' and 'params' defined, block until it returns, return the result"""
        return self.execute_object(self._queue.add_command, self._command_resolution_suite, action_object)

    def execute_test(self, action_object):
        """Add test from dict object with 'method' and 'params' defined, block until it returns, return the result"""
        return self.execute_object(self._queue.add_test, self._test_resolution_suite, action_object)

    def run_json_tests(self, tests):
        """Run list of json tests"""
        for test in tests:
            self.add_json_test(test)
        return 200

    def run_tests(self, tests):
        """Run list of tests"""
        for test in tests:
            self.add_test(test)
        return 200

class JSONRPCMethods(RPCMethods):

    def next_action(self):
        """The next action for the browser to execute"""
        windmill.ide_is_awake = True
        action = self._queue.next_action()
        if action is not None:
            self._logger.debug('queue has next_action %s' % str(action))
            return action
        else:
            self._logger.debug('queue has no next_action, returning "pause" method')
            action = copy.copy(callback)
            action.update({'method':'defer'})
            return action

    def report(self, uuid, result, starttime, endtime, debug=None):
        """Report fass/fail for a test"""
        self._test_resolution_suite.resolve(result, uuid, starttime, endtime, debug)

    def command_result(self, status, uuid, result):
        self._command_resolution_suite.resolve(status, uuid, result)

    def status_change(self, status):
        pass

    def restart_test_run(self, tests):
        self.clear_queue()
        self._test_resolution_suite.unresolved = {}
        self._command_resolution_suite.unresolved ={}
        for test in tests:
            self.add_test(test, suite_name=test.get('suite_name'))

    def create_save_file(self, transformer, suite_name, tests):
        for test in tests:
            if test.get('suite_name'):
                test.pop('suite_name')
            if test['params'].get('uuid'):
                test['params'].pop('uuid')
        return windmill.authoring.transforms.registry[transformer](suite_name, tests)

    def clear_queue(self):
        """Clear the server queue"""
        self._queue.command_queue = []
        self._queue.test_queue = []

class XMLRPCMethods(RPCMethods):
    pass

            <commit_msg>Fixed the suite_name issue, for real this time.<commit_after>#   Copyright (c) 2006-2007 Open Source Applications Foundation
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

"""This module provides the communication and management between the various
server interfaces and the browser's js interface"""

import copy, os
import simplejson
import logging
from uuid import uuid1
import windmill
from time import sleep
from dateutil.parser import parse as dateutil_parse

test_results_logger = logging.getLogger('test_results')

class ControllerQueue(object):

    def __init__(self, command_resolution_suite, test_resolution_suite):
        self.crs = command_resolution_suite
        self.trs = test_resolution_suite
        self.command_queue = []
        self.test_queue = []
        self.current_suite = None

    def add_command(self, command, suite_name=None):
        if suite_name is None and not command.get('suite_name'):
            suite_name = self.current_suite
        command['suite_name'] = suite_name
        self.command_queue.append(command)

    def add_test(self, test, suite_name=None):
        if suite_name is None and not test.get('suite_name'):
            suite_name = self.current_suite
        test['suite_name'] = suite_name
        self.test_queue.append(test)

    def start_suite(self, suite_name):
        self.current_suite = suite_name

    def stop_suite(self):
        self.current_suite = None

    def command(self, command):

        self.command_queue.insert(0, command)

    def next_action(self):

        if len(self.command_queue) is not 0:
            command = self.command_queue.pop(0)
            self.crs.add(command)
            return command
        elif len(self.test_queue) is not 0:
            test = self.test_queue.pop(0)
            self.trs.add(test)
            return test
        else:
            return None

callback = {'version':'0.1'}

class TestResolutionSuite(object):
    """Collection of tests run and results"""
    result_processor = None

    def __init__(self):
        self.unresolved = {}
        self.resolved = {}
        self.current_suite = None

    def resolve(self, result, uuid, starttime, endtime, debug=None):
        """Resolve test by uuid"""
        starttime = dateutil_parse(starttime)
        endtime = dateutil_parse(endtime)
        test = self.unresolved.pop(uuid)
        test['result'] = result
        test['starttime'] = starttime
        test['endtime'] = endtime
        self.resolved[uuid] = test

        if result is False:
            test_results_logger.error('Test Failue in test %s' % test)
        elif result is True:
            test_results_logger.debug('Test Success in test %s' % test)

        if self.result_processor is not None:
            if result is False:
                self.result_processor.failure(test, debug=debug)
            elif result is True:
                self.result_processor.success(test, debug=debug)

        if test.has_key('result_callback'):
            test['result_callback'](result, debug)

    def start_suite(self, suite_name):
        self.current_suite = suite_name

    def stop_suite(self):
        self.current_suite = None

    def add(self, test, suite_name=None):
        self.unresolved[test['params']['uuid']] = test

class CommandResolutionSuite(object):

    def __init__(self):
        self.unresolved = {}
        self.resolved ={}

    def resolve(self, status, uuid, result):
        """Resolve command by uuid"""
        command = self.unresolved.pop(uuid, None)
        command['status'] = status
        command['result'] = result
        self.resolved[uuid] = command

        if status is False:
            test_results_logger.error('Command Failure in command %s' % command)
        elif status is True:
            test_results_logger.debug('Command Succes in command %s' % command)

        if command.has_key('result_callback'):
            command['result_callback'](status, result)

    def add(self, command, suite_name=None):
        self.unresolved[command['params']['uuid']] = command

class RecursiveRPC(object):

    def __init__(self, execution_method):
        self.execution_method = execution_method

    def __getattr__(self, key):
        """Call a method on the controller as if it was a local method"""
        class ExecuteJSONRecursiveAttribute(object):
            def __init__(self, name, execution_method):
                self.name = name
                self.execution_method = execution_method
            def __call__(self, **kwargs):
                rpc = {'method':self.name, 'params':kwargs}
                return self.execution_method(rpc)
            def __getattr__(self, key):
                return ExecuteRecursiveAttribute(self.name+'.'+key, self.execution_method)

        return ExecuteRecursiveAttribute(key, self.execution_method)

class RPCMethods(object):

    def __init__(self, queue, test_resolution_suite, command_resolution_suite):
        self._queue = queue
        self._logger = logging.getLogger('jsonrpc_methods_instance')
        self._test_resolution_suite = test_resolution_suite
        self._command_resolution_suite = command_resolution_suite

    def start_suite(self, suite_name):
        self._test_resolution_suite.start_suite(suite_name)
        self._queue.start_suite(suite_name)
        return 200

    def stop_suite(self):
        self._test_resolution_suite.stop_suite()
        return 200

    def add_object(self, queue_method, action_object, suite_name=None):
        """Procedue neutral addition method"""
        callback_object = copy.copy(callback)
        callback_object.update(action_object)
        if not callback_object['params'].get('uuid'):
            callback_object['params']['uuid'] = str(uuid1())
        self._logger.debug('Adding object %s' % str(callback_object))
        queue_method(callback_object, suite_name=suite_name)
        return callback_object['params']['uuid']

    def add_json_test(self, json, suite_name=None):
        """Add test from json object with 'method' and 'params' defined"""
        action_object = simplejson.loads(json)
        self.add_object(self._queue.add_test,  action_object, suite_name)
        return 200

    def add_test(self, test_object, suite_name=None):
        self.add_object(self._queue.add_test,  test_object, suite_name)
        return 200

    def add_json_command(self, json):
        """Add command from json object with 'method' and 'params' defined"""
        action_object = simplejson.loads(json)
        self.add_object(self._queue.add_command, action_object)
        return 200

    def add_command(self, command_object):
        """Add command from object"""
        self.add_object(self._queue.add_command, command_object)
        return 200

    def execute_object(self, queue_method, resolution_suite, action_object):
        """Procedure neutral blocking exeution of a given object."""
        uuid = self.add_object(queue_method, resolution_suite, action_object)

        while not resolution_suite.resolved.get(uuid):
            sleep(1)

        result = resolution_suite.resolved[uuid]
        result.pop('totaltime', None)
        return result

    def execute_json_command(self, json):
        """Add command from json object with 'method' and 'params' defined, block until it returns, return the result"""
        action_object = simplejson.loads(json)
        return self.execute_object(self._queue.add_command, self._command_resolution_suite, action_object)

    def execute_json_test(self, json):
        """Add test from json object with 'method' and 'params' defined, block until it returns, return the result"""
        action_object = simplejson.loads(json)
        return self.execute_object(self._queue.add_test, self._test_resolution_suite, action_object)

    def execute_command(self, action_object):
        """Add command from dict object with 'method' and 'params' defined, block until it returns, return the result"""
        return self.execute_object(self._queue.add_command, self._command_resolution_suite, action_object)

    def execute_test(self, action_object):
        """Add test from dict object with 'method' and 'params' defined, block until it returns, return the result"""
        return self.execute_object(self._queue.add_test, self._test_resolution_suite, action_object)

    def run_json_tests(self, tests):
        """Run list of json tests"""
        for test in tests:
            self.add_json_test(test)
        return 200

    def run_tests(self, tests):
        """Run list of tests"""
        for test in tests:
            self.add_test(test)
        return 200

class JSONRPCMethods(RPCMethods):

    def next_action(self):
        """The next action for the browser to execute"""
        windmill.ide_is_awake = True
        action = self._queue.next_action()
        if action is not None:
            self._logger.debug('queue has next_action %s' % str(action))
            return action
        else:
            self._logger.debug('queue has no next_action, returning "pause" method')
            action = copy.copy(callback)
            action.update({'method':'defer'})
            return action

    def report(self, uuid, result, starttime, endtime, debug=None):
        """Report fass/fail for a test"""
        self._test_resolution_suite.resolve(result, uuid, starttime, endtime, debug)

    def command_result(self, status, uuid, result):
        self._command_resolution_suite.resolve(status, uuid, result)

    def status_change(self, status):
        pass

    def restart_test_run(self, tests):
        self.clear_queue()
        self._test_resolution_suite.unresolved = {}
        self._command_resolution_suite.unresolved ={}
        for test in tests:
            self.add_test(test, suite_name=test.get('suite_name'))

    def create_save_file(self, transformer, suite_name, tests):
        for test in tests:
            if test.get('suite_name'):
                test.pop('suite_name')
            if test['params'].get('uuid'):
                test['params'].pop('uuid')
        return windmill.authoring.transforms.registry[transformer](suite_name, tests)

    def clear_queue(self):
        """Clear the server queue"""
        self._queue.command_queue = []
        self._queue.test_queue = []

class XMLRPCMethods(RPCMethods):
    pass

            <|endoftext|>


<commit_before>"""

Copyright 2015, Institute for Systems Biology

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

"""

from functools import wraps
import logging
from re import compile as re_compile
from time import time

from api.api_helpers import authorize_credentials_with_Google
from api.schema.tcga_clinical import schema as clinical_schema

from django.conf import settings

from bq_data_access.errors import FeatureNotFoundException
from bq_data_access.feature_value_types import DataTypes, BigQuerySchemaToValueTypeConverter
from bq_data_access.utils import DurationLogged

CLINICAL_FEATURE_TYPE = 'CLIN'

class InvalidClinicalFeatureIDException(Exception):
    def __init__(self, feature_id, reason):
        self.feature_id = feature_id
        self.reason = reason

    def __str__(self):
        return "Invalid internal clinical feature ID '{feature_id}', reason '{reason}'".format(
            feature_id=self.feature_id,
            reason=self.reason
        )

class ClinicalFeatureDef(object):
    def __init__(self, table_field, value_type):
        self.table_field = table_field
        self.value_type = value_type

    @classmethod
    def get_table_field_and_value_type(cls, column_id):
        table_field = None
        value_type = None
        for clinical_field in clinical_schema:
            name, schema_field_type = clinical_field['name'], clinical_field['type']
            if name == column_id:
                table_field= name
                value_type = BigQuerySchemaToValueTypeConverter.get_value_type(schema_field_type)

        return table_field, value_type

    @classmethod
    def from_feature_id(cls, feature_id):
        # Example ID: CLIN:vital_status
        regex = re_compile("^CLIN:"
                           # column name
                           "([a-zA-Z0-9_\-]+)$")

        feature_fields = regex.findall(feature_id)
        if len(feature_fields) == 0:
            raise FeatureNotFoundException(feature_id)
        column_name = feature_fields[0]

        table_field, value_type = cls.get_table_field_and_value_type(column_name)
        if table_field is None:
            raise FeatureNotFoundException(feature_id)

        return cls(table_field, value_type)

class ClinicalFeatureProvider(object):
    TABLES = [
        {
            'name': 'Clinical',
            'info': 'Clinical',
            'id': 'tcga_clinical'
        }
    ]

    def __init__(self, feature_id):
        self.table_name = ''
        self.feature_def = None
        self.parse_internal_feature_id(feature_id)

    def get_value_type(self):
        return self.feature_def.value_type

    def get_feature_type(self):
        return DataTypes.CLIN

    @classmethod
    def process_data_point(cls, data_point):
        return data_point['value']

    def build_query(self, project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array):
        # Generate the 'IN' statement string: (%s, %s, ..., %s)
        cohort_id_stmt = ', '.join([str(cohort_id) for cohort_id in cohort_id_array])

        query_template = \
            ("SELECT clin.ParticipantBarcode, biospec.sample_id, clin.{column_name} "
             "FROM ( "
             " SELECT ParticipantBarcode, {column_name} "
             " FROM [{project_name}:{dataset_name}.{table_name}] "
             " ) AS clin "
             " JOIN ( "
             " SELECT ParticipantBarcode, SampleBarcode as sample_id "
             " FROM [{project_name}:tcga_data_open.Biospecimen] "
             " ) AS biospec "
             " ON clin.ParticipantBarcode = biospec.ParticipantBarcode "
             "WHERE biospec.sample_id IN ( "
             "    SELECT sample_barcode "
             "    FROM [{project_name}:{cohort_dataset}.{cohort_table}] "
             "    WHERE cohort_id IN ({cohort_id_list}) "
             ")"
             "GROUP BY clin.ParticipantBarcode, biospec.sample_id, clin.{column_name}")

        query = query_template.format(dataset_name=dataset_name, project_name=project_name, table_name=table_name,
                                      column_name=feature_def.table_field,
                                      cohort_dataset=cohort_dataset, cohort_table=cohort_table,
                                      cohort_id_list=cohort_id_stmt)

        logging.debug("BQ_QUERY_CLIN: " + query)
        return query

    @DurationLogged('CLIN', 'AUTH')
    def get_bq_service(self):
        bigquery_service = authorize_credentials_with_Google()
        return bigquery_service

    @DurationLogged('CLIN', 'BQ_QUERY')
    def execute_bq_query(self, bigquery, project_id, query_body):
        table_data = bigquery.jobs()
        query_response = table_data.query(projectId=project_id, body=query_body).execute()
        return query_response

    @DurationLogged('CLIN', 'UNPACK')
    def unpack_query_response(self, query_response):
        """
        Unpacks values from a BigQuery response object into a flat array. The array will contain dicts with
        the following fields:
        - 'patient_id': Patient barcode
        - 'sample_id': Sample barcode
        - 'aliquot_id': Always None
        - 'value': Value of the selected column from the clinical data table

        Args:
            query_response: A BigQuery query response object

        Returns:
            Array of dict objects.
        """
        result = []
        num_result_rows = int(query_response['totalRows'])
        if num_result_rows == 0:
            return result

        for row in query_response['rows']:
            result.append({
                'patient_id': row['f'][0]['v'],
                'sample_id': row['f'][1]['v'],
                'aliquot_id': None,
                'value': row['f'][2]['v']
            })

        return result

    def do_query(self, project_id, project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array):
        bigquery_service = self.get_bq_service()

        query = self.build_query(project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array)

        query_body = {
            'query': query
        }

        query_response = self.execute_bq_query(bigquery_service, project_id, query_body)
        result = self.unpack_query_response(query_response)
        return result

    def get_data_from_bigquery(self, cohort_id_array, cohort_dataset, cohort_table):
        project_id = settings.BQ_PROJECT_ID
        project_name = settings.BIGQUERY_PROJECT_NAME
        dataset_name = settings.BIGQUERY_DATASET2
        result = self.do_query(project_id, project_name, dataset_name, self.table_name, self.feature_def, cohort_dataset, cohort_table, cohort_id_array)
        return result

    def get_data(self, cohort_id_array, cohort_dataset, cohort_table):
        result = self.get_data_from_bigquery(cohort_id_array, cohort_dataset, cohort_table)
        return result

    def get_table_name(self):
        return self.TABLES[0]['name']

    def parse_internal_feature_id(self, feature_id):
        self.feature_def = ClinicalFeatureDef.from_feature_id(feature_id)
        self.table_name = self.get_table_name()

    @classmethod
    def is_valid_feature_id(cls, feature_id):
        is_valid = False
        try:
            ClinicalFeatureDef.from_feature_id(feature_id)
            is_valid = True
        except Exception:
            # ClinicalFeatureDef.from_feature_id raises Exception if the feature identifier
            # is not valid. Nothing needs to be done here, since is_valid is already False.
            pass
        finally:
            return is_valid
<commit_msg>Prototype asynchronous BigQuery job for clinical data<commit_after>"""

Copyright 2015, Institute for Systems Biology

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

"""

import logging
from re import compile as re_compile
from uuid import uuid4
from time import sleep

from api.api_helpers import authorize_credentials_with_Google
from api.schema.tcga_clinical import schema as clinical_schema

from django.conf import settings

from bq_data_access.errors import FeatureNotFoundException
from bq_data_access.feature_value_types import DataTypes, BigQuerySchemaToValueTypeConverter
from bq_data_access.utils import DurationLogged

CLINICAL_FEATURE_TYPE = 'CLIN'

class InvalidClinicalFeatureIDException(Exception):
    def __init__(self, feature_id, reason):
        self.feature_id = feature_id
        self.reason = reason

    def __str__(self):
        return "Invalid internal clinical feature ID '{feature_id}', reason '{reason}'".format(
            feature_id=self.feature_id,
            reason=self.reason
        )

class ClinicalFeatureDef(object):
    def __init__(self, table_field, value_type):
        self.table_field = table_field
        self.value_type = value_type

    @classmethod
    def get_table_field_and_value_type(cls, column_id):
        table_field = None
        value_type = None
        for clinical_field in clinical_schema:
            name, schema_field_type = clinical_field['name'], clinical_field['type']
            if name == column_id:
                table_field= name
                value_type = BigQuerySchemaToValueTypeConverter.get_value_type(schema_field_type)

        return table_field, value_type

    @classmethod
    def from_feature_id(cls, feature_id):
        # Example ID: CLIN:vital_status
        regex = re_compile("^CLIN:"
                           # column name
                           "([a-zA-Z0-9_\-]+)$")

        feature_fields = regex.findall(feature_id)
        if len(feature_fields) == 0:
            raise FeatureNotFoundException(feature_id)
        column_name = feature_fields[0]

        table_field, value_type = cls.get_table_field_and_value_type(column_name)
        if table_field is None:
            raise FeatureNotFoundException(feature_id)

        return cls(table_field, value_type)

class ClinicalFeatureProvider(object):
    TABLES = [
        {
            'name': 'Clinical',
            'info': 'Clinical',
            'id': 'tcga_clinical'
        }
    ]

    def __init__(self, feature_id):
        self.table_name = ''
        self.feature_def = None
        self.parse_internal_feature_id(feature_id)

    def get_value_type(self):
        return self.feature_def.value_type

    def get_feature_type(self):
        return DataTypes.CLIN

    @classmethod
    def process_data_point(cls, data_point):
        return data_point['value']

    def build_query(self, project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array):
        # Generate the 'IN' statement string: (%s, %s, ..., %s)
        cohort_id_stmt = ', '.join([str(cohort_id) for cohort_id in cohort_id_array])

        query_template = \
            ("SELECT clin.ParticipantBarcode, biospec.sample_id, clin.{column_name} "
             "FROM ( "
             " SELECT ParticipantBarcode, {column_name} "
             " FROM [{project_name}:{dataset_name}.{table_name}] "
             " ) AS clin "
             " JOIN ( "
             " SELECT ParticipantBarcode, SampleBarcode as sample_id "
             " FROM [{project_name}:tcga_data_open.Biospecimen] "
             " ) AS biospec "
             " ON clin.ParticipantBarcode = biospec.ParticipantBarcode "
             "WHERE biospec.sample_id IN ( "
             "    SELECT sample_barcode "
             "    FROM [{project_name}:{cohort_dataset}.{cohort_table}] "
             "    WHERE cohort_id IN ({cohort_id_list}) "
             ")"
             "GROUP BY clin.ParticipantBarcode, biospec.sample_id, clin.{column_name}")

        query = query_template.format(dataset_name=dataset_name, project_name=project_name, table_name=table_name,
                                      column_name=feature_def.table_field,
                                      cohort_dataset=cohort_dataset, cohort_table=cohort_table,
                                      cohort_id_list=cohort_id_stmt)

        logging.debug("BQ_QUERY_CLIN: " + query)
        return query

    @DurationLogged('CLIN', 'AUTH')
    def get_bq_service(self):
        bigquery_service = authorize_credentials_with_Google()
        return bigquery_service

    @DurationLogged('CLIN', 'BQ_SUBMIT')
    def submit_bigquery_job(self, bigquery, project_id, query_body, batch=False):
        job_data = {
            'jobReference': {
                'projectId': project_id,
                'job_id': str(uuid4())
            },
            'configuration': {
                'query': {
                    'query': query_body,
                    'priority': 'BATCH' if batch else 'INTERACTIVE'
                }
            }
        }

        return bigquery.jobs().insert(
            projectId=project_id,
            body=job_data).execute(num_retries=5)

    @DurationLogged('CLIN', 'BQ_POLL')
    def poll_async_job(self, bigquery_service, project_id, job_id, poll_interval=5):
        job_collection = bigquery_service.jobs()

        poll = True

        while poll:
            sleep(poll_interval)
            job = job_collection.get(projectId=project_id,
                                     jobId=job_id).execute()

            if job['status']['state'] == 'DONE':
                poll = False

            if 'errorResult' in job['status']:
                raise Exception(job['status'])

    def download_query_result(self, bigquery, query_job):
        result = []
        page_token = None
        total_rows = 0

        while True:
            page = bigquery.jobs().getQueryResults(
                    pageToken=page_token,
                    **query_job['jobReference']).execute(num_retries=2)

            rows = page['rows']
            result.extend(rows)

            total_rows += len(rows)
            logging.debug(total_rows)

            page_token = page.get('pageToken')
            if not page_token:
                break

        return result

    @DurationLogged('CLIN', 'UNPACK')
    def unpack_query_response(self, query_result_array):
        """
        Unpacks values from a BigQuery response object into a flat array. The array will contain dicts with
        the following fields:
        - 'patient_id': Patient barcode
        - 'sample_id': Sample barcode
        - 'aliquot_id': Always None
        - 'value': Value of the selected column from the clinical data table

        Args:
            query_result_array: A BigQuery query response object

        Returns:
            Array of dict objects.
        """
        result = []

        for row in query_result_array:
            result.append({
                'patient_id': row['f'][0]['v'],
                'sample_id': row['f'][1]['v'],
                'aliquot_id': None,
                'value': row['f'][2]['v']
            })

        return result

    def do_query(self, project_id, project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array):
        bigquery_service = self.get_bq_service()

        query_body = self.build_query(project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array)
        query_job = self.submit_bigquery_job(bigquery_service, project_id, query_body)

        # Poll for completion of the query
        job_id = query_job['jobReference']['jobId']
        logging.debug("JOBID {id}".format(id=job_id))

        self.poll_async_job(bigquery_service, project_id, job_id)
        query_result_array = self.download_query_result(bigquery_service, query_job)

        result = self.unpack_query_response(query_result_array)
        return result

    def get_data_from_bigquery(self, cohort_id_array, cohort_dataset, cohort_table):
        project_id = settings.BQ_PROJECT_ID
        project_name = settings.BIGQUERY_PROJECT_NAME
        dataset_name = settings.BIGQUERY_DATASET2
        result = self.do_query(project_id, project_name, dataset_name, self.table_name, self.feature_def, cohort_dataset, cohort_table, cohort_id_array)
        return result

    def get_data(self, cohort_id_array, cohort_dataset, cohort_table):
        result = self.get_data_from_bigquery(cohort_id_array, cohort_dataset, cohort_table)
        return result

    def get_table_name(self):
        return self.TABLES[0]['name']

    def parse_internal_feature_id(self, feature_id):
        self.feature_def = ClinicalFeatureDef.from_feature_id(feature_id)
        self.table_name = self.get_table_name()

    @classmethod
    def is_valid_feature_id(cls, feature_id):
        is_valid = False
        try:
            ClinicalFeatureDef.from_feature_id(feature_id)
            is_valid = True
        except Exception:
            # ClinicalFeatureDef.from_feature_id raises Exception if the feature identifier
            # is not valid. Nothing needs to be done here, since is_valid is already False.
            pass
        finally:
            return is_valid
<|endoftext|>


<commit_before>import base64
try:
    import cPickle as pickle
except ImportError:
    import pickle
import datetime
import logging
import sys
import traceback
import warnings

from django.conf import settings as dj_settings
from django.core.exceptions import ImproperlyConfigured
from django.db import models, transaction
from django.http import Http404
from django.utils.encoding import smart_unicode
from django.utils.translation import ugettext_lazy as _

from sentry import settings
from sentry.client.base import SentryClient
from sentry.helpers import cached_property, construct_checksum, get_db_engine, get_installed_apps, transform
from sentry.manager import GroupedMessageManager, SentryManager
from sentry.reporter import FakeRequest

_reqs = ('paging', 'indexer')
for r in _reqs:
    if r not in dj_settings.INSTALLED_APPS:
        raise ImproperlyConfigured("Put '%s' in your "
            "INSTALLED_APPS setting in order to use the sentry application." % r)

try:
    from idmapper.models import SharedMemoryModel as Model
except ImportError:
    Model = models.Model

logger = logging.getLogger('sentry')

__all__ = ('Message', 'GroupedMessage')

STATUS_LEVELS = (
    (0, _('unresolved')),
    (1, _('resolved')),
)

class GzippedDictField(models.TextField):
    """
    Slightly different from a JSONField in the sense that the default
    value is a dictionary.
    """
    __metaclass__ = models.SubfieldBase

    def to_python(self, value):
        if isinstance(value, basestring) and value:
            value = pickle.loads(base64.b64decode(value).decode('zlib'))
        elif not value:
            return {}
        return value

    def get_prep_value(self, value):
        if value is None: return
        return base64.b64encode(pickle.dumps(transform(value)).encode('zlib'))

    def value_to_string(self, obj):
        value = self._get_val_from_obj(obj)
        return self.get_db_prep_value(value)

    def south_field_triple(self):
        "Returns a suitable description of this field for South."
        from south.modelsinspector import introspector
        field_class = "django.db.models.fields.TextField"
        args, kwargs = introspector(self)
        return (field_class, args, kwargs)

class MessageBase(Model):
    logger          = models.CharField(max_length=64, blank=True, default='root', db_index=True)
    class_name      = models.CharField(_('type'), max_length=128, blank=True, null=True, db_index=True)
    level           = models.PositiveIntegerField(choices=settings.LOG_LEVELS, default=logging.ERROR, blank=True, db_index=True)
    message         = models.TextField()
    traceback       = models.TextField(blank=True, null=True)
    view            = models.CharField(max_length=255, db_index=True, blank=True, null=True)
    checksum        = models.CharField(max_length=32, db_index=True)

    objects         = SentryManager()

    class Meta:
        abstract = True

    def shortened_traceback(self):
        return '\n'.join(self.traceback.split('\n')[-5:])
    shortened_traceback.short_description = _('traceback')
    shortened_traceback.admin_order_field = 'traceback'

    def error(self):
        if self.message:
            message = smart_unicode(self.message)
            if len(message) > 100:
                message = message[:97] + '...'
            if self.class_name:
                return "%s: %s" % (self.class_name, message)
        else:
            self.message = self._class_name or ''
        return message
    error.short_description = _('error')

    def description(self):
        return self.traceback or ''
    description.short_description = _('description')

class GroupedMessage(MessageBase):
    status          = models.PositiveIntegerField(default=0, choices=STATUS_LEVELS)
    times_seen      = models.PositiveIntegerField(default=1)
    last_seen       = models.DateTimeField(default=datetime.datetime.now, db_index=True)
    first_seen      = models.DateTimeField(default=datetime.datetime.now, db_index=True)

    objects         = GroupedMessageManager()

    class Meta:
        unique_together = (('logger', 'view', 'checksum'),)
        verbose_name_plural = _('grouped messages')
        verbose_name = _('grouped message')
        permissions = (
            ("can_view", "Can view"),
        )
        db_table = 'sentry_groupedmessage'

    def __unicode__(self):
        return "(%s) %s" % (self.times_seen, self.error())

    def natural_key(self):
        return (self.logger, self.view, self.checksum)

    @classmethod
    def get_score_clause(cls):
        engine = get_db_engine()
        if engine.startswith('postgresql'):
            return 'times_seen / (pow((floor(extract(epoch from now() - last_seen) / 3600) + 2), 1.25) + 1)'
        if engine.startswith('mysql'):
            return 'times_seen / (pow((floor((unix_timestamp(now()) - unix_timestamp(last_seen)) / 3600) + 2), 1.25) + 1)'
        return 'times_seen'

    def mail_admins(self, request=None, fail_silently=True):
        if not settings.ADMINS:
            return

        from django.core.mail import send_mail
        from django.template.loader import render_to_string

        message = self.message_set.order_by('-id')[0]

        obj_request = message.request

        subject = 'Error (%s IP): %s' % ((obj_request.META.get('REMOTE_ADDR') in dj_settings.INTERNAL_IPS and 'internal' or 'EXTERNAL'), obj_request.path)
        try:
            request_repr = repr(obj_request)
        except:
            request_repr = "Request repr() unavailable"

        if request:
            link = request.build_absolute_url(self.get_absolute_url())
        else:
            link = None

        body = render_to_string('sentry/emails/error.txt', {
            'request_repr': request_repr,
            'request': obj_request,
            'group': self,
            'traceback': message.traceback,
            'link': link,
        })

        send_mail(dj_settings.EMAIL_SUBJECT_PREFIX + subject, body,
                  dj_settings.SERVER_EMAIL, settings.ADMINS,
                  fail_silently=fail_silently)

class Message(MessageBase):
    group           = models.ForeignKey(GroupedMessage, blank=True, null=True, related_name="message_set")
    datetime        = models.DateTimeField(default=datetime.datetime.now, db_index=True)
    data            = GzippedDictField(blank=True, null=True)
    url             = models.URLField(verify_exists=False, null=True, blank=True)
    server_name     = models.CharField(max_length=128, db_index=True)

    class Meta:
        verbose_name = _('message')
        verbose_name_plural = _('messages')
        db_table = 'sentry_message'

    def __unicode__(self):
        return self.error()

    def save(self, *args, **kwargs):
        if not self.checksum:
            self.checksum = construct_checksum(**self.__dict__)
        super(Message, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return self.url

    def shortened_url(self):
        if not self.url:
            return _('no data')
        url = self.url
        if len(url) > 60:
            url = url[:60] + '...'
        return url
    shortened_url.short_description = _('url')
    shortened_url.admin_order_field = 'url'

    def full_url(self):
        return self.data.get('url') or self.url
    full_url.short_description = _('url')
    full_url.admin_order_field = 'url'

    @cached_property
    def request(self):
        fake_request = FakeRequest()
        fake_request.META = self.data.get('META', {})
        fake_request.GET = self.data.get('GET', {})
        fake_request.POST = self.data.get('POST', {})
        fake_request.FILES = self.data.get('FILES', {})
        fake_request.COOKIES = self.data.get('COOKIES', {})
        fake_request.url = self.url
        if self.url:
            fake_request.path_info = '/' + self.url.split('/', 3)[-1]
        else:
            fake_request.path_info = ''
        fake_request.path = fake_request.path_info
        return fake_request
<commit_msg>Fix description of error() when no message present<commit_after>import base64
try:
    import cPickle as pickle
except ImportError:
    import pickle
import datetime
import logging
import sys
import traceback
import warnings

from django.conf import settings as dj_settings
from django.core.exceptions import ImproperlyConfigured
from django.db import models, transaction
from django.http import Http404
from django.utils.encoding import smart_unicode
from django.utils.translation import ugettext_lazy as _

from sentry import settings
from sentry.client.base import SentryClient
from sentry.helpers import cached_property, construct_checksum, get_db_engine, get_installed_apps, transform
from sentry.manager import GroupedMessageManager, SentryManager
from sentry.reporter import FakeRequest

_reqs = ('paging', 'indexer')
for r in _reqs:
    if r not in dj_settings.INSTALLED_APPS:
        raise ImproperlyConfigured("Put '%s' in your "
            "INSTALLED_APPS setting in order to use the sentry application." % r)

try:
    from idmapper.models import SharedMemoryModel as Model
except ImportError:
    Model = models.Model

logger = logging.getLogger('sentry')

__all__ = ('Message', 'GroupedMessage')

STATUS_LEVELS = (
    (0, _('unresolved')),
    (1, _('resolved')),
)

class GzippedDictField(models.TextField):
    """
    Slightly different from a JSONField in the sense that the default
    value is a dictionary.
    """
    __metaclass__ = models.SubfieldBase

    def to_python(self, value):
        if isinstance(value, basestring) and value:
            value = pickle.loads(base64.b64decode(value).decode('zlib'))
        elif not value:
            return {}
        return value

    def get_prep_value(self, value):
        if value is None: return
        return base64.b64encode(pickle.dumps(transform(value)).encode('zlib'))

    def value_to_string(self, obj):
        value = self._get_val_from_obj(obj)
        return self.get_db_prep_value(value)

    def south_field_triple(self):
        "Returns a suitable description of this field for South."
        from south.modelsinspector import introspector
        field_class = "django.db.models.fields.TextField"
        args, kwargs = introspector(self)
        return (field_class, args, kwargs)

class MessageBase(Model):
    logger          = models.CharField(max_length=64, blank=True, default='root', db_index=True)
    class_name      = models.CharField(_('type'), max_length=128, blank=True, null=True, db_index=True)
    level           = models.PositiveIntegerField(choices=settings.LOG_LEVELS, default=logging.ERROR, blank=True, db_index=True)
    message         = models.TextField()
    traceback       = models.TextField(blank=True, null=True)
    view            = models.CharField(max_length=255, db_index=True, blank=True, null=True)
    checksum        = models.CharField(max_length=32, db_index=True)

    objects         = SentryManager()

    class Meta:
        abstract = True

    def shortened_traceback(self):
        return '\n'.join(self.traceback.split('\n')[-5:])
    shortened_traceback.short_description = _('traceback')
    shortened_traceback.admin_order_field = 'traceback'

    def error(self):
        if self.message:
            message = smart_unicode(self.message)
            if len(message) > 100:
                message = message[:97] + '...'
            if self.class_name:
                return "%s: %s" % (self.class_name, message)
        else:
            self.message = self.class_name or ''
        return message
    error.short_description = _('error')

    def description(self):
        return self.traceback or ''
    description.short_description = _('description')

class GroupedMessage(MessageBase):
    status          = models.PositiveIntegerField(default=0, choices=STATUS_LEVELS)
    times_seen      = models.PositiveIntegerField(default=1)
    last_seen       = models.DateTimeField(default=datetime.datetime.now, db_index=True)
    first_seen      = models.DateTimeField(default=datetime.datetime.now, db_index=True)

    objects         = GroupedMessageManager()

    class Meta:
        unique_together = (('logger', 'view', 'checksum'),)
        verbose_name_plural = _('grouped messages')
        verbose_name = _('grouped message')
        permissions = (
            ("can_view", "Can view"),
        )
        db_table = 'sentry_groupedmessage'

    def __unicode__(self):
        return "(%s) %s" % (self.times_seen, self.error())

    def natural_key(self):
        return (self.logger, self.view, self.checksum)

    @classmethod
    def get_score_clause(cls):
        engine = get_db_engine()
        if engine.startswith('postgresql'):
            return 'times_seen / (pow((floor(extract(epoch from now() - last_seen) / 3600) + 2), 1.25) + 1)'
        if engine.startswith('mysql'):
            return 'times_seen / (pow((floor((unix_timestamp(now()) - unix_timestamp(last_seen)) / 3600) + 2), 1.25) + 1)'
        return 'times_seen'

    def mail_admins(self, request=None, fail_silently=True):
        if not settings.ADMINS:
            return

        from django.core.mail import send_mail
        from django.template.loader import render_to_string

        message = self.message_set.order_by('-id')[0]

        obj_request = message.request

        subject = 'Error (%s IP): %s' % ((obj_request.META.get('REMOTE_ADDR') in dj_settings.INTERNAL_IPS and 'internal' or 'EXTERNAL'), obj_request.path)
        try:
            request_repr = repr(obj_request)
        except:
            request_repr = "Request repr() unavailable"

        if request:
            link = request.build_absolute_url(self.get_absolute_url())
        else:
            link = None

        body = render_to_string('sentry/emails/error.txt', {
            'request_repr': request_repr,
            'request': obj_request,
            'group': self,
            'traceback': message.traceback,
            'link': link,
        })

        send_mail(dj_settings.EMAIL_SUBJECT_PREFIX + subject, body,
                  dj_settings.SERVER_EMAIL, settings.ADMINS,
                  fail_silently=fail_silently)

class Message(MessageBase):
    group           = models.ForeignKey(GroupedMessage, blank=True, null=True, related_name="message_set")
    datetime        = models.DateTimeField(default=datetime.datetime.now, db_index=True)
    data            = GzippedDictField(blank=True, null=True)
    url             = models.URLField(verify_exists=False, null=True, blank=True)
    server_name     = models.CharField(max_length=128, db_index=True)

    class Meta:
        verbose_name = _('message')
        verbose_name_plural = _('messages')
        db_table = 'sentry_message'

    def __unicode__(self):
        return self.error()

    def save(self, *args, **kwargs):
        if not self.checksum:
            self.checksum = construct_checksum(**self.__dict__)
        super(Message, self).save(*args, **kwargs)

    def get_absolute_url(self):
        return self.url

    def shortened_url(self):
        if not self.url:
            return _('no data')
        url = self.url
        if len(url) > 60:
            url = url[:60] + '...'
        return url
    shortened_url.short_description = _('url')
    shortened_url.admin_order_field = 'url'

    def full_url(self):
        return self.data.get('url') or self.url
    full_url.short_description = _('url')
    full_url.admin_order_field = 'url'

    @cached_property
    def request(self):
        fake_request = FakeRequest()
        fake_request.META = self.data.get('META', {})
        fake_request.GET = self.data.get('GET', {})
        fake_request.POST = self.data.get('POST', {})
        fake_request.FILES = self.data.get('FILES', {})
        fake_request.COOKIES = self.data.get('COOKIES', {})
        fake_request.url = self.url
        if self.url:
            fake_request.path_info = '/' + self.url.split('/', 3)[-1]
        else:
            fake_request.path_info = ''
        fake_request.path = fake_request.path_info
        return fake_request
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys

from sasila.slow_system.pipeline.kafa_pipeline import KafkaPipeline
from sasila.slow_system.core.request_spider import RequestSpider
from sasila.slow_system.pipeline.console_pipeline import ConsolePipeline
from sasila.slow_system.pipeline.text_pipeline import TextPipeline
from base_processor import BaseProcessor
from sasila.slow_system.downloader.http.spider_request import Request
from bs4 import BeautifulSoup as bs
import time

import traceback

reload(sys)
sys.setdefaultencoding('utf-8')

class QccProcessor(BaseProcessor):
    spider_id = 'qcc'
    spider_name = 'qcc'
    allowed_domains = ['qichacha.com']

    start_requests = [
        Request(url='http://www.qichacha.com/search?key=%E5%B0%8F%E9%A2%9D%E8%B4%B7%E6%AC%BE')]

    def process(self, response):
        soup = bs(response.m_response.content, "lxml")
        province_list = soup.select_one("dl#provinceOld").select("div.pull-left")[1].select("dd a")
        for province in province_list:
            province_name = province.string.strip()
            province_id = province["data-value"].strip()
            request = Request(
                    url="http://www.qichacha.com/search_getCityListHtml?province=" + province_id + "&q_type=1",
                    callback="get_city", priority=0)
            request.meta["province_name"] = province_name
            request.meta["province_id"] = province_id
            yield request

    def get_city(self, response):
        if response.m_response.content == "":
            request = Request(
                    url="http://www.qichacha.com/search_index?key=%25E5%25B0%258F%25E9%25A2%259D%25E8%25B4%25B7%25E6%25AC%25BE&ajaxflag=1&province=" +
                        response.request.meta["province_id"] + "&",
                    callback="get_all_page", priority=1)
            request.meta["city_name"] = ""
            request.meta["city_id"] = ""
            request.meta["province_name"] = response.request.meta["province_name"]
            request.meta["province_id"] = response.request.meta["province_id"]
            request.meta["only_province_sign"] = "0"
            yield request
        else:
            soup = bs(response.m_response.content, "lxml")
            city_list = soup.select("a")
            for city in city_list:
                city_name = city.string.strip()
                city_id = city["data-value"].strip()
                request = Request(
                        url="http://www.qichacha.com/search_index?key=%25E5%25B0%258F%25E9%25A2%259D%25E8%25B4%25B7%25E6%25AC%25BE&ajaxflag=1&province=" +
                            response.request.meta["province_id"] + "&city=" + city_id + "&",
                        callback="get_all_page", priority=1)
                request.meta["city_name"] = city_name
                request.meta["city_id"] = city_id
                request.meta["province_name"] = response.request.meta["province_name"]
                request.meta["province_id"] = response.request.meta["province_id"]
                yield request

    def get_all_page(self, response):
        soup = bs(response.m_response.content, "lxml")
        try:
            temp_page = soup.find(lambda tag: tag.name == 'a' and '>' == tag.text).parent.findNextSibling()
            if temp_page:
                page = temp_page.select_one("a")
                if page:
                    total_page = int(page.string.strip().replace("...", ""))
                else:
                    total_page = 1
            else:
                temp_page = soup.find(lambda tag: tag.name == 'a' and '>' == tag.text).parent.findPreviousSibling()
                if temp_page:
                    page = temp_page.select_one("a")
                    if page:
                        total_page = int(page.string.strip().replace("...", ""))
                    else:
                        total_page = 1
                else:
                    total_page = 1
        except:
            total_page = 1

        now_page = 1
        while now_page <= total_page:
            if "only_province_sign" in response.request.meta.keys():
                request = Request(
                        url="http://www.qichacha.com/search_index?key=%25E5%25B0%258F%25E9%25A2%259D%25E8%25B4%25B7%25E6%25AC%25BE&ajaxflag=1&province=" +
                            response.request.meta["province_id"] + "&p=" + str(now_page) + "&",
                        callback="get_content", priority=2)
                request.meta["city_name"] = response.request.meta["city_name"]
                request.meta["city_id"] = response.request.meta["city_id"]
                request.meta["province_name"] = response.request.meta["province_name"]
                request.meta["province_id"] = response.request.meta["province_id"]
                yield request
            else:
                request = Request(
                        url="http://www.qichacha.com/search_index?key=%25E5%25B0%258F%25E9%25A2%259D%25E8%25B4%25B7%25E6%25AC%25BE&ajaxflag=1&&p=" + str(
                                now_page) + "&province=" +
                            response.request.meta["province_id"] + "&city=" + response.request.meta["city_id"] + "&",
                        callback="get_content", priority=2)
                request.meta["city_name"] = response.request.meta["city_name"]
                request.meta["city_id"] = response.request.meta["city_id"]
                request.meta["province_name"] = response.request.meta["province_name"]
                request.meta["province_id"] = response.request.meta["province_id"]
                yield request
            now_page += 1

    def get_content(self, response):
        soup = bs(response.m_response.content, "lxml")
        content_list = soup.select("table.m_srchList tbody tr")
        for content in content_list:
            try:
                result_item = dict()
                result_item["province"] = response.request.meta["province_name"]
                result_item["city"] = response.request.meta["city_name"]
                result_item["company_name"] = content.select("td")[1].text.split('\n')[0].strip()
                result_item["company_man"] = content.select("td")[1].text.split('\n')[1].strip().replace("企业法人：", "")
                result_item["company_telephone"] = content.select("td")[1].text.split('\n')[2].strip().replace("联系方式：",
                                                                                "")
                result_item["company_address"] = content.select("td")[1].text.split('\n')[3].strip()
                if "地址：" in result_item["company_address"]:
                    result_item["company_address"] = result_item["company_address"].replace("地址：", "")
                else:
                    result_item["company_address"] = ""
                result_item["company_registered_capital"] = content.select("td")[2].text.strip()
                result_item["company_registered_time"] = content.select("td")[3].text.strip()
                result_item["company_status"] = content.select("td")[4].text.strip()
                result_item["source"] = "企查查"
                result_item["update_time"] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
                yield result_item
            except Exception:
                print traceback.format_exc()

qcc_spider = RequestSpider(QccProcessor(), time_sleep=6).set_pipeline(KafkaPipeline()).set_pipeline(
    TextPipeline()).set_pipeline(ConsolePipeline())
if __name__ == '__main__':
    qcc_spider.start()
<commit_msg>sasila<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
import sys

from sasila.slow_system.pipeline.kafa_pipeline import KafkaPipeline
from sasila.slow_system.core.request_spider import RequestSpider
from sasila.slow_system.pipeline.console_pipeline import ConsolePipeline
from sasila.slow_system.pipeline.text_pipeline import TextPipeline
from base_processor import BaseProcessor
from sasila.slow_system.downloader.http.spider_request import Request
from bs4 import BeautifulSoup as bs
import time

import traceback

reload(sys)
sys.setdefaultencoding('utf-8')

class QccProcessor(BaseProcessor):
    spider_id = 'qcc'
    spider_name = 'qcc'
    allowed_domains = ['qichacha.com']

    start_requests = [
        Request(url='http://www.qichacha.com/search?key=%E5%B0%8F%E9%A2%9D%E8%B4%B7%E6%AC%BE')]

    def process(self, response):
        soup = bs(response.m_response.content, "lxml")
        province_list = soup.select_one("dl#provinceOld").select("div.pull-left")[1].select("dd a")
        for province in province_list:
            province_name = province.string.strip()
            province_id = province["data-value"].strip()
            request = Request(
                    url="http://www.qichacha.com/search_getCityListHtml?province=" + province_id + "&q_type=1",
                    callback="get_city", priority=0)
            request.meta["province_name"] = province_name
            request.meta["province_id"] = province_id
            yield request

    def get_city(self, response):
        if response.m_response.content == "":
            request = Request(
                    url="http://www.qichacha.com/search_index?key=%25E5%25B0%258F%25E9%25A2%259D%25E8%25B4%25B7%25E6%25AC%25BE&ajaxflag=1&province=" +
                        response.request.meta["province_id"] + "&",
                    callback="get_all_page", priority=1)
            request.meta["city_name"] = ""
            request.meta["city_id"] = ""
            request.meta["province_name"] = response.request.meta["province_name"]
            request.meta["province_id"] = response.request.meta["province_id"]
            yield request
        else:
            soup = bs(response.m_response.content, "lxml")
            city_list = soup.select("a")
            for city in city_list:
                city_name = city.string.strip()
                city_id = city["data-value"].strip()
                request = Request(
                        url="http://www.qichacha.com/search_index?key=%25E5%25B0%258F%25E9%25A2%259D%25E8%25B4%25B7%25E6%25AC%25BE&ajaxflag=1&province=" +
                            response.request.meta["province_id"] + "&city=" + city_id + "&",
                        callback="get_all_page", priority=1)
                request.meta["city_name"] = city_name
                request.meta["city_id"] = city_id
                request.meta["province_name"] = response.request.meta["province_name"]
                request.meta["province_id"] = response.request.meta["province_id"]
                yield request

    def get_all_page(self, response):
        soup = bs(response.m_response.content, "lxml")
        try:
            temp_page = soup.find(lambda tag: tag.name == 'a' and '>' == tag.text).parent.findNextSibling()
            if temp_page:
                page = temp_page.select_one("a")
                if page:
                    total_page = int(page.string.strip().replace("...", ""))
                else:
                    total_page = 1
            else:
                temp_page = soup.find(lambda tag: tag.name == 'a' and '>' == tag.text).parent.findPreviousSibling()
                if temp_page:
                    page = temp_page.select_one("a")
                    if page:
                        total_page = int(page.string.strip().replace("...", ""))
                    else:
                        total_page = 1
                else:
                    total_page = 1
        except:
            total_page = 1

        now_page = 1
        while now_page <= total_page:
            if response.request.meta["city_id"] == "":
                request = Request(
                        url="http://www.qichacha.com/search_index?key=%25E5%25B0%258F%25E9%25A2%259D%25E8%25B4%25B7%25E6%25AC%25BE&ajaxflag=1&province=" +
                            response.request.meta["province_id"] + "&p=" + str(now_page) + "&",
                        callback="get_content", priority=2)
                request.meta["city_name"] = response.request.meta["city_name"]
                request.meta["city_id"] = response.request.meta["city_id"]
                request.meta["province_name"] = response.request.meta["province_name"]
                request.meta["province_id"] = response.request.meta["province_id"]
                yield request
            else:
                request = Request(
                        url="http://www.qichacha.com/search_index?key=%25E5%25B0%258F%25E9%25A2%259D%25E8%25B4%25B7%25E6%25AC%25BE&ajaxflag=1&&p=" + str(
                                now_page) + "&province=" +
                            response.request.meta["province_id"] + "&city=" + response.request.meta["city_id"] + "&",
                        callback="get_content", priority=2)
                request.meta["city_name"] = response.request.meta["city_name"]
                request.meta["city_id"] = response.request.meta["city_id"]
                request.meta["province_name"] = response.request.meta["province_name"]
                request.meta["province_id"] = response.request.meta["province_id"]
                yield request
            now_page += 1

    def get_content(self, response):
        soup = bs(response.m_response.content, "lxml")
        content_list = soup.select("table.m_srchList tbody tr")
        for content in content_list:
            try:
                result_item = dict()
                result_item["province"] = response.request.meta["province_name"]
                result_item["city"] = response.request.meta["city_name"]
                result_item["company_name"] = content.select("td")[1].text.split('\n')[0].strip()
                result_item["company_man"] = content.select("td")[1].text.split('\n')[1].strip().replace("企业法人：", "")
                result_item["company_telephone"] = content.select("td")[1].text.split('\n')[2].strip().replace("联系方式：",
                                                                                "")
                result_item["company_address"] = content.select("td")[1].text.split('\n')[3].strip()
                if "地址：" in result_item["company_address"]:
                    result_item["company_address"] = result_item["company_address"].replace("地址：", "")
                else:
                    result_item["company_address"] = ""
                result_item["company_registered_capital"] = content.select("td")[2].text.strip()
                result_item["company_registered_time"] = content.select("td")[3].text.strip()
                result_item["company_status"] = content.select("td")[4].text.strip()
                result_item["source"] = "企查查"
                result_item["update_time"] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))
                yield result_item
            except Exception:
                print traceback.format_exc()

qcc_spider = RequestSpider(QccProcessor(), time_sleep=1).set_pipeline(KafkaPipeline()).set_pipeline(
    TextPipeline()).set_pipeline(ConsolePipeline())
if __name__ == '__main__':
    qcc_spider.start()
<|endoftext|>


<commit_before>import Sofa
import SofaPython.Tools
import SofaTest

def createScene(node):
    ## testing Data<vector<T>> resize from python binding

    # dummy components, just to use their Data<vector<T>>
    dof = node.createObject("MechanicalObject",template="Vec3d",name="dof",position="0 0 0  1 1 1  2 2 2  3 3 3  4 4 4  5 5 5")
    fc = node.createObject("FixedConstraint", name="fc", indices="0 1 2 3 4")
    node.createObject('PythonScriptController', filename=__file__, classname='VerifController')

class VerifController(SofaTest.Controller):

    def onEndAnimationStep(self,dt):

        self.dof = self.node.getObject("dof")
        self.fc = self.node.getObject("fc")

        # two-dimensional vector resize

        self.ASSERT( len(self.dof.position)==6, "test1" )
        self.dof.position=[9,9,9] # smaller
        self.ASSERT( len(self.dof.position)==1, "test2" )
        self.dof.position=[] # empty list
        self.ASSERT( len(self.dof.position)==0, "test3" )
        self.dof.position=[9,9,9]*9 # larger
        self.ASSERT( len(self.dof.position)==9, "test4" )
        self.dof.position="" # empty string
        self.ASSERT( len(self.dof.position)==0, "test5" )
        self.dof.position=SofaPython.Tools.listToStr( [9,9,9]*9 ) # larger string
        self.ASSERT( len(self.dof.position)==9, "test6" )

        # one-dimensional vector resize

        self.ASSERT( len(self.fc.indices)==5, "test7" )
        self.fc.indices=[8]
        self.ASSERT( len(self.fc.indices)==1, "test8" )
        self.fc.indices=[]
        self.ASSERT( len(self.fc.indices)==0, "test9" )
        self.fc.indices=[0,1,6,3,4,5,3,1,2]
        self.ASSERT( len(self.fc.indices)==9, "test10" )
        self.fc.indices=""
        self.ASSERT( len(self.fc.indices)==0, "test11" )
        self.fc.indices="0 1 6 3 4 5 3 1 2"
        self.ASSERT( len(self.fc.indices)==9, "test12" )
        self.fc.indices="8"
        self.ASSERT( len(self.fc.indices)==1, "test13" )

        self.sendSuccess()<commit_msg>SofaPython: improved Data resizing test with non-resizable Data<commit_after>import Sofa
import SofaPython.Tools
import SofaTest

def createScene(node):
    ## testing Data<vector<T>> resize from python binding

    # dummy components, just to use their Data<vector<T>>
    dof = node.createObject("MechanicalObject", template="Vec3d", name="dof", position="0 0 0  1 1 1  2 2 2  3 3 3  4 4 4  5 5 5")
    fc = node.createObject("FixedConstraint", name="fc", indices="0 1 2 3 4")
    node.createObject('PythonScriptController', filename=__file__, classname='VerifController')

class VerifController(SofaTest.Controller):

    def onEndAnimationStep(self, dt):
        self.dof = self.node.getObject("dof")
        self.fc = self.node.getObject("fc")

        # two-dimensional vector resize

        self.ASSERT(len(self.dof.position) == 6, "test1")
        self.dof.position = [9, 9, 9]  # smaller
        self.ASSERT(len(self.dof.position) == 1, "test2")
        self.dof.position = []  # empty list
        self.ASSERT(len(self.dof.position) == 0, "test3")
        self.dof.position = [9, 9, 9] * 9  # larger
        self.ASSERT(len(self.dof.position) == 9, "test4")
        self.dof.position = ""  # empty string
        self.ASSERT(len(self.dof.position) == 0, "test5")
        self.dof.position = SofaPython.Tools.listToStr([9, 9, 9] * 9)  # larger string
        self.ASSERT(len(self.dof.position) == 9, "test6")

        # one-dimensional vector resize

        self.ASSERT(len(self.fc.indices) == 5, "test7")
        self.fc.indices = [8]
        self.ASSERT(len(self.fc.indices) == 1, "test8")
        self.fc.indices = []
        self.ASSERT(len(self.fc.indices) == 0, "test9")
        self.fc.indices = [0, 1, 6, 3, 4, 5, 3, 1, 2]
        self.ASSERT(len(self.fc.indices) == 9, "test10")
        self.fc.indices = ""
        self.ASSERT(len(self.fc.indices) == 0, "test11")
        self.fc.indices = "0 1 6 3 4 5 3 1 2"
        self.ASSERT(len(self.fc.indices) == 9, "test12")
        self.fc.indices = "8"
        self.ASSERT(len(self.fc.indices) == 1, "test13")

        # non resizable boolean Data

        self.fc.fixAll = True
        self.fc.fixAll = ""
        self.ASSERT(self.fc.fixAll == True, "test14")

        # non resizable scalar Data

        self.dof.showObjectScale = 9
        self.dof.showObjectScale = ""
        self.ASSERT(self.dof.showObjectScale == 9, "test15")

        self.sendSuccess()
<|endoftext|>