<commit_before>#coding :utf-8
#
# The MIT License (MIT)
#
# Copyright (c) 2016-2018 yutiansut/QUANTAXIS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""
QUANTAXIS

Quantitative Financial Strategy Framework

by yutiansut

2017/4/8
"""

__version__ = '1.3.0.dev16'
__author__ = 'yutiansut'

# fetch methods
from QUANTAXIS.QAFetch.Fetcher import QA_quotation

from QUANTAXIS.QAFetch import (
    QA_fetch_get_stock_day,
    QA_fetch_get_trade_date,
    QA_fetch_get_stock_min,
    QA_fetch_get_stock_xdxr,
    QA_fetch_get_stock_indicator,
    QA_fetch_get_stock_realtime,
    QA_fetch_get_stock_transaction,
    QA_fetch_get_index_day,
    QA_fetch_get_index_min,
    QA_fetch_get_stock_list,
    QA_fetch_get_stock_info,
    QA_fetch_get_stock_block,
    QA_fetch_get_stock_transaction_realtime,
    QA_fetch_get_security_bars,
    QA_fetch_get_future_day,
    QA_fetch_get_future_min,
    QA_fetch_get_future_list,
    QA_fetch_get_future_transaction,
    QA_fetch_get_future_transaction_realtime,
    QA_fetch_get_future_realtime,
    QA_fetch_get_bond_list,
    QA_fetch_get_index_list,
    QA_fetch_get_hkfund_list,
    QA_fetch_get_hkfund_day,
    QA_fetch_get_hkfund_min,
    QA_fetch_get_hkindex_list,
    QA_fetch_get_hkindex_day,
    QA_fetch_get_hkindex_min,
    QA_fetch_get_hkstock_list,
    QA_fetch_get_hkstock_day,
    QA_fetch_get_hkstock_min,
    QA_fetch_get_usstock_list,
    QA_fetch_get_usstock_day,
    QA_fetch_get_usstock_min,
    QA_fetch_get_option_list,
    QA_fetch_get_option_day,
    QA_fetch_get_option_min,
    QA_fetch_get_globalindex_day,
    QA_fetch_get_globalindex_min,
    QA_fetch_get_globalindex_list,
    QA_fetch_get_macroindex_list,
    QA_fetch_get_macroindex_day,
    QA_fetch_get_macroindex_min,
    QA_fetch_get_exchangerate_list,
    QA_fetch_get_exchangerate_day,
    QA_fetch_get_exchangerate_min,
    QA_fetch_get_globalfuture_list,
    QA_fetch_get_globalfuture_day,
    QA_fetch_get_globalfuture_min,
    QA_fetch_get_chibor,
    get_stock_market
)
from QUANTAXIS.QAFetch.QAQuery import (
    QA_fetch_trade_date,
    QA_fetch_account,
    QA_fetch_financial_report,
    QA_fetch_stock_day,
    QA_fetch_stock_min,
    QA_fetch_ctp_tick,
    QA_fetch_index_day,
    QA_fetch_index_min,
    QA_fetch_index_list,
    QA_fetch_future_min,
    QA_fetch_future_day,
    QA_fetch_future_list,
    QA_fetch_future_tick,
    QA_fetch_stock_list,
    QA_fetch_stock_full,
    QA_fetch_stock_xdxr,
    QA_fetch_backtest_info,
    QA_fetch_backtest_history,
    QA_fetch_stock_block,
    QA_fetch_stock_info,
    QA_fetch_stock_name,
    QA_fetch_quotation,
    QA_fetch_quotations
)
from QUANTAXIS.QAFetch.QACrawler import QA_fetch_get_sh_margin, QA_fetch_get_sz_margin

from QUANTAXIS.QAFetch.QAQuery_Advance import *

# save
from QUANTAXIS.QASU.main import (
    QA_SU_save_stock_list,
    QA_SU_save_stock_day,
    QA_SU_save_index_day,
    QA_SU_save_index_min,
    QA_SU_save_stock_info_tushare,
    QA_SU_save_stock_min,
    QA_SU_save_stock_xdxr,
    QA_SU_save_stock_info,
    QA_SU_save_stock_min_5,
    QA_SU_save_index_list,
    QA_SU_save_future_list,
    QA_SU_save_stock_block,
    QA_SU_save_etf_day,
    QA_SU_save_etf_min,
    QA_SU_save_financialfiles
)

# from QUANTAXIS.QASU.save_backtest import (
#     QA_SU_save_account_message, QA_SU_save_backtest_message, QA_SU_save_account_to_csv)

from QUANTAXIS.QASU.user import (QA_user_sign_in, QA_user_sign_up)
from QUANTAXIS.QASU.save_strategy import QA_SU_save_strategy
# event driver

# market
from QUANTAXIS.QAMarket import (
    QA_Order,
    QA_OrderQueue,
    QA_OrderHandler,
    QA_Market,
    QA_Dealer,
    QA_Broker,
    QA_RandomBroker,
    QA_SimulatedBroker,
    QA_RealBroker,
    QA_BacktestBroker,
    QA_TTSBroker,
)

# Account,Risk,Portfolio,User,Strategy

from QUANTAXIS.QAARP.QAAccount import QA_Account
from QUANTAXIS.QAARP.QAPortfolio import QA_Portfolio, QA_PortfolioView
from QUANTAXIS.QAARP.QARisk import QA_Performance, QA_Risk
from QUANTAXIS.QAARP.QAUser import QA_User
from QUANTAXIS.QAARP.QAStrategy import QA_Strategy
# Backtest
from QUANTAXIS.QAApplication.QABacktest import QA_Backtest

from QUANTAXIS.QAApplication.QAAnalysis import QA_backtest_analysis_backtest
from QUANTAXIS.QAApplication.QAResult import backtest_result_analyzer

# ENGINE
from QUANTAXIS.QAEngine import QA_Thread, QA_Event, QA_Worker, QA_Task, QA_Engine

# Data
from QUANTAXIS.QAData import (
    QA_data_tick_resample_1min,
    QA_data_tick_resample,
    QA_data_day_resample,
    QA_data_min_resample,
    QA_data_ctptick_resample,
    QA_data_calc_marketvalue,
    QA_data_marketvalue,
    QA_data_stock_to_fq,
    QA_DataStruct_Stock_day,
    QA_DataStruct_Stock_min,
    QA_DataStruct_Day,
    QA_DataStruct_Min,
    QA_DataStruct_Future_day,
    QA_DataStruct_Future_min,
    QA_DataStruct_Index_day,
    QA_DataStruct_Index_min,
    QA_DataStruct_Indicators,
    QA_DataStruct_Stock_realtime,
    QA_DataStruct_Stock_transaction,
    QA_DataStruct_Stock_block,
    QA_DataStruct_Series,
    QA_DataStruct_Financial,
    from_tushare,
    QDS_StockMinWarpper,
    QDS_StockDayWarpper,
    QDS_IndexDayWarpper,
    QDS_IndexMinWarpper
)
from QUANTAXIS.QAData.dsmethods import *
# Analysis
from QUANTAXIS.QAAnalysis import *

# Setting

from QUANTAXIS.QASetting.QALocalize import qa_path, setting_path, cache_path, download_path, log_path

# Util

from QUANTAXIS.QAUtil import (
    QA_util_date_stamp,
    QA_util_time_stamp,
    QA_util_ms_stamp,
    QA_util_date_valid,
    QA_util_calc_time,
    QA_util_realtime,
    QA_util_id2date,
    QA_util_is_trade,
    QA_util_get_date_index,
    QA_util_get_last_day,
    QA_util_get_next_day,
    QA_util_get_order_datetime,
    QA_util_get_trade_datetime,
    QA_util_get_index_date,
    QA_util_select_hours,
    QA_util_date_gap,
    QA_util_time_gap,
    QA_util_get_last_datetime,
    QA_util_get_next_datetime,
    QA_util_select_min,
    QA_util_time_delay,
    QA_util_time_now,
    QA_util_date_str2int,
    QA_util_date_int2str,
    QA_util_date_today,
    QA_util_to_datetime,
    QA_util_sql_mongo_setting,
    QA_util_sql_async_mongo_setting,
    QA_util_sql_mongo_sort_ASCENDING,
    QA_util_sql_mongo_sort_DESCENDING,
    QA_util_log_debug,
    QA_util_log_expection,
    QA_util_log_info,
    QA_util_cfg_initial,
    QA_util_get_cfg,
    QA_Setting,
    DATABASE,
    info_ip_list,
    stock_ip_list,
    future_ip_list,
    QA_util_web_ping,
    QA_util_send_mail,
    trade_date_sse,
    QA_util_if_trade,
    QA_util_if_tradetime,
    QA_util_get_real_datelist,
    QA_util_get_real_date,
    QA_util_get_trade_range,
    QA_util_get_trade_gap,
    QA_util_save_csv,
    QA_util_code_tostr,
    QA_util_code_tolist,
    QA_util_dict_remove_key,
    QA_util_multi_demension_list,
    QA_util_diff_list,
    QA_util_to_json_from_pandas,
    QA_util_to_list_from_numpy,
    QA_util_to_list_from_pandas,
    QA_util_to_pandas_from_json,
    QA_util_to_pandas_from_list,
    QA_util_mongo_initial,
    QA_util_mongo_status,
    QA_util_mongo_infos,
    QA_util_make_min_index,
    QA_util_make_hour_index,
    QA_util_random_with_topic,
    QA_util_file_md5,
    MARKET_TYPE,
    ORDER_STATUS,
    TRADE_STATUS,
    MARKET_ERROR,
    AMOUNT_MODEL,
    ORDER_DIRECTION,
    ORDER_MODEL,
    ORDER_EVENT,
    MARKET_EVENT,
    ENGINE_EVENT,
    RUNNING_ENVIRONMENT,
    FREQUENCE,
    BROKER_EVENT,
    BROKER_TYPE,
    DATASOURCE,
    OUTPUT_FORMAT
)                                      # QAPARAMETER

from QUANTAXIS.QAIndicator import *
#from QUANTAXIS.QAFetch.QATdx_adv import bat

# CMD and Cli
import QUANTAXIS.QACmd
from QUANTAXIS.QACmd import QA_cmd

import argparse

# check
import sys
if sys.version_info.major != 3 or sys.version_info.minor not in [4, 5, 6, 7, 8]:
    print('wrong version, should be 3.4/3.5/3.6/3.7/3.8 version')
    sys.exit()

#QA_util_log_info('Welcome to QUANTAXIS, the Version is {}'.format(__version__))
def __repr__():
    return ' \n \
            ```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````` \n \
            ``########`````##````````##``````````##`````````####````````##```##########````````#``````##``````###```##`````######`` \n \
            `##``````## ```##````````##`````````####````````##`##```````##```````##```````````###``````##````##`````##```##`````##` \n \
            ##````````##```##````````##````````##`##````````##``##``````##```````##``````````####```````#```##``````##```##``````## \n \
            ##````````##```##````````##```````##```##```````##```##`````##```````##`````````##`##```````##`##```````##````##``````` \n \
            ##````````##```##````````##``````##`````##``````##````##````##```````##````````##``###```````###````````##`````##`````` \n \
            ##````````##```##````````##``````##``````##`````##`````##```##```````##```````##````##```````###````````##``````###```` \n \
            ##````````##```##````````##`````##````````##````##``````##``##```````##``````##``````##`````##`##```````##````````##``` \n \
            ##````````##```##````````##````#############````##```````##`##```````##`````###########`````##``##``````##`````````##`` \n \
            ###```````##```##````````##```##```````````##```##```````##`##```````##````##`````````##```##```##``````##```##`````##` \n \
            `##``````###````##``````###``##`````````````##``##````````####```````##```##``````````##``###````##`````##````##`````## \n \
            ``#########``````########```##``````````````###`##``````````##```````##``##````````````##`##``````##````##`````###``### \n \
            ````````#####`````````````````````````````````````````````````````````````````````````````````````````````````````##``  \n \
            ``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````` \n \
            ``````````````````````````Copyright``yutiansut``2018``````QUANTITATIVE FINANCIAL FRAMEWORK````````````````````````````` \n \
            ``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````` \n \
            ```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````` \n \
            ```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````` \n '

__str__= __repr__
# QA_util_log_info(Logo)
<commit_msg>#update 1.3.0<commit_after>#coding :utf-8
#
# The MIT License (MIT)
#
# Copyright (c) 2016-2018 yutiansut/QUANTAXIS
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
"""
QUANTAXIS

Quantitative Financial Strategy Framework

by yutiansut

2017/4/8
"""

__version__ = '1.3.0'
__author__ = 'yutiansut'

# fetch methods
from QUANTAXIS.QAFetch.Fetcher import QA_quotation

from QUANTAXIS.QAFetch import (
    QA_fetch_get_stock_day,
    QA_fetch_get_trade_date,
    QA_fetch_get_stock_min,
    QA_fetch_get_stock_xdxr,
    QA_fetch_get_stock_indicator,
    QA_fetch_get_stock_realtime,
    QA_fetch_get_stock_transaction,
    QA_fetch_get_index_day,
    QA_fetch_get_index_min,
    QA_fetch_get_stock_list,
    QA_fetch_get_stock_info,
    QA_fetch_get_stock_block,
    QA_fetch_get_stock_transaction_realtime,
    QA_fetch_get_security_bars,
    QA_fetch_get_future_day,
    QA_fetch_get_future_min,
    QA_fetch_get_future_list,
    QA_fetch_get_future_transaction,
    QA_fetch_get_future_transaction_realtime,
    QA_fetch_get_future_realtime,
    QA_fetch_get_bond_list,
    QA_fetch_get_index_list,
    QA_fetch_get_hkfund_list,
    QA_fetch_get_hkfund_day,
    QA_fetch_get_hkfund_min,
    QA_fetch_get_hkindex_list,
    QA_fetch_get_hkindex_day,
    QA_fetch_get_hkindex_min,
    QA_fetch_get_hkstock_list,
    QA_fetch_get_hkstock_day,
    QA_fetch_get_hkstock_min,
    QA_fetch_get_usstock_list,
    QA_fetch_get_usstock_day,
    QA_fetch_get_usstock_min,
    QA_fetch_get_option_list,
    QA_fetch_get_option_day,
    QA_fetch_get_option_min,
    QA_fetch_get_globalindex_day,
    QA_fetch_get_globalindex_min,
    QA_fetch_get_globalindex_list,
    QA_fetch_get_macroindex_list,
    QA_fetch_get_macroindex_day,
    QA_fetch_get_macroindex_min,
    QA_fetch_get_exchangerate_list,
    QA_fetch_get_exchangerate_day,
    QA_fetch_get_exchangerate_min,
    QA_fetch_get_globalfuture_list,
    QA_fetch_get_globalfuture_day,
    QA_fetch_get_globalfuture_min,
    QA_fetch_get_chibor,
    get_stock_market
)
from QUANTAXIS.QAFetch.QAQuery import (
    QA_fetch_trade_date,
    QA_fetch_account,
    QA_fetch_financial_report,
    QA_fetch_stock_day,
    QA_fetch_stock_min,
    QA_fetch_ctp_tick,
    QA_fetch_index_day,
    QA_fetch_index_min,
    QA_fetch_index_list,
    QA_fetch_future_min,
    QA_fetch_future_day,
    QA_fetch_future_list,
    QA_fetch_future_tick,
    QA_fetch_stock_list,
    QA_fetch_stock_full,
    QA_fetch_stock_xdxr,
    QA_fetch_backtest_info,
    QA_fetch_backtest_history,
    QA_fetch_stock_block,
    QA_fetch_stock_info,
    QA_fetch_stock_name,
    QA_fetch_quotation,
    QA_fetch_quotations
)
from QUANTAXIS.QAFetch.QACrawler import QA_fetch_get_sh_margin, QA_fetch_get_sz_margin

from QUANTAXIS.QAFetch.QAQuery_Advance import *

# save
from QUANTAXIS.QASU.main import (
    QA_SU_save_stock_list,
    QA_SU_save_stock_day,
    QA_SU_save_index_day,
    QA_SU_save_index_min,
    QA_SU_save_stock_info_tushare,
    QA_SU_save_stock_min,
    QA_SU_save_stock_xdxr,
    QA_SU_save_stock_info,
    QA_SU_save_stock_min_5,
    QA_SU_save_index_list,
    QA_SU_save_future_list,
    QA_SU_save_stock_block,
    QA_SU_save_etf_day,
    QA_SU_save_etf_min,
    QA_SU_save_financialfiles
)

# from QUANTAXIS.QASU.save_backtest import (
#     QA_SU_save_account_message, QA_SU_save_backtest_message, QA_SU_save_account_to_csv)

from QUANTAXIS.QASU.user import (QA_user_sign_in, QA_user_sign_up)
from QUANTAXIS.QASU.save_strategy import QA_SU_save_strategy
# event driver

# market
from QUANTAXIS.QAMarket import (
    QA_Order,
    QA_OrderQueue,
    QA_OrderHandler,
    QA_Market,
    QA_Dealer,
    QA_Broker,
    QA_RandomBroker,
    QA_SimulatedBroker,
    QA_RealBroker,
    QA_BacktestBroker,
    QA_TTSBroker,
)

# Account,Risk,Portfolio,User,Strategy

from QUANTAXIS.QAARP.QAAccount import QA_Account
from QUANTAXIS.QAARP.QAPortfolio import QA_Portfolio, QA_PortfolioView
from QUANTAXIS.QAARP.QARisk import QA_Performance, QA_Risk
from QUANTAXIS.QAARP.QAUser import QA_User
from QUANTAXIS.QAARP.QAStrategy import QA_Strategy
# Backtest
from QUANTAXIS.QAApplication.QABacktest import QA_Backtest

from QUANTAXIS.QAApplication.QAAnalysis import QA_backtest_analysis_backtest
from QUANTAXIS.QAApplication.QAResult import backtest_result_analyzer

# ENGINE
from QUANTAXIS.QAEngine import QA_Thread, QA_Event, QA_Worker, QA_Task, QA_Engine

# Data
from QUANTAXIS.QAData import (
    QA_data_tick_resample_1min,
    QA_data_tick_resample,
    QA_data_day_resample,
    QA_data_min_resample,
    QA_data_ctptick_resample,
    QA_data_calc_marketvalue,
    QA_data_marketvalue,
    QA_data_stock_to_fq,
    QA_DataStruct_Stock_day,
    QA_DataStruct_Stock_min,
    QA_DataStruct_Day,
    QA_DataStruct_Min,
    QA_DataStruct_Future_day,
    QA_DataStruct_Future_min,
    QA_DataStruct_Index_day,
    QA_DataStruct_Index_min,
    QA_DataStruct_Indicators,
    QA_DataStruct_Stock_realtime,
    QA_DataStruct_Stock_transaction,
    QA_DataStruct_Stock_block,
    QA_DataStruct_Series,
    QA_DataStruct_Financial,
    from_tushare,
    QDS_StockMinWarpper,
    QDS_StockDayWarpper,
    QDS_IndexDayWarpper,
    QDS_IndexMinWarpper
)
from QUANTAXIS.QAData.dsmethods import *
# Analysis
from QUANTAXIS.QAAnalysis import *

# Setting

from QUANTAXIS.QASetting.QALocalize import qa_path, setting_path, cache_path, download_path, log_path

# Util

from QUANTAXIS.QAUtil import (
    QA_util_date_stamp,
    QA_util_time_stamp,
    QA_util_ms_stamp,
    QA_util_date_valid,
    QA_util_calc_time,
    QA_util_realtime,
    QA_util_id2date,
    QA_util_is_trade,
    QA_util_get_date_index,
    QA_util_get_last_day,
    QA_util_get_next_day,
    QA_util_get_order_datetime,
    QA_util_get_trade_datetime,
    QA_util_get_index_date,
    QA_util_select_hours,
    QA_util_date_gap,
    QA_util_time_gap,
    QA_util_get_last_datetime,
    QA_util_get_next_datetime,
    QA_util_select_min,
    QA_util_time_delay,
    QA_util_time_now,
    QA_util_date_str2int,
    QA_util_date_int2str,
    QA_util_date_today,
    QA_util_to_datetime,
    QA_util_sql_mongo_setting,
    QA_util_sql_async_mongo_setting,
    QA_util_sql_mongo_sort_ASCENDING,
    QA_util_sql_mongo_sort_DESCENDING,
    QA_util_log_debug,
    QA_util_log_expection,
    QA_util_log_info,
    QA_util_cfg_initial,
    QA_util_get_cfg,
    QA_Setting,
    DATABASE,
    info_ip_list,
    stock_ip_list,
    future_ip_list,
    QA_util_web_ping,
    QA_util_send_mail,
    trade_date_sse,
    QA_util_if_trade,
    QA_util_if_tradetime,
    QA_util_get_real_datelist,
    QA_util_get_real_date,
    QA_util_get_trade_range,
    QA_util_get_trade_gap,
    QA_util_save_csv,
    QA_util_code_tostr,
    QA_util_code_tolist,
    QA_util_dict_remove_key,
    QA_util_multi_demension_list,
    QA_util_diff_list,
    QA_util_to_json_from_pandas,
    QA_util_to_list_from_numpy,
    QA_util_to_list_from_pandas,
    QA_util_to_pandas_from_json,
    QA_util_to_pandas_from_list,
    QA_util_mongo_initial,
    QA_util_mongo_status,
    QA_util_mongo_infos,
    QA_util_make_min_index,
    QA_util_make_hour_index,
    QA_util_random_with_topic,
    QA_util_file_md5,
    MARKET_TYPE,
    ORDER_STATUS,
    TRADE_STATUS,
    MARKET_ERROR,
    AMOUNT_MODEL,
    ORDER_DIRECTION,
    ORDER_MODEL,
    ORDER_EVENT,
    MARKET_EVENT,
    ENGINE_EVENT,
    RUNNING_ENVIRONMENT,
    FREQUENCE,
    BROKER_EVENT,
    BROKER_TYPE,
    DATASOURCE,
    OUTPUT_FORMAT
)                                      # QAPARAMETER

from QUANTAXIS.QAIndicator import *
#from QUANTAXIS.QAFetch.QATdx_adv import bat

# CMD and Cli
import QUANTAXIS.QACmd
from QUANTAXIS.QACmd import QA_cmd

import argparse

# check
import sys
if sys.version_info.major != 3 or sys.version_info.minor not in [4, 5, 6, 7, 8]:
    print('wrong version, should be 3.4/3.5/3.6/3.7/3.8 version')
    sys.exit()

#QA_util_log_info('Welcome to QUANTAXIS, the Version is {}'.format(__version__))
def __repr__():
    return ' \n \
            ```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````` \n \
            ``########`````##````````##``````````##`````````####````````##```##########````````#``````##``````###```##`````######`` \n \
            `##``````## ```##````````##`````````####````````##`##```````##```````##```````````###``````##````##`````##```##`````##` \n \
            ##````````##```##````````##````````##`##````````##``##``````##```````##``````````####```````#```##``````##```##``````## \n \
            ##````````##```##````````##```````##```##```````##```##`````##```````##`````````##`##```````##`##```````##````##``````` \n \
            ##````````##```##````````##``````##`````##``````##````##````##```````##````````##``###```````###````````##`````##`````` \n \
            ##````````##```##````````##``````##``````##`````##`````##```##```````##```````##````##```````###````````##``````###```` \n \
            ##````````##```##````````##`````##````````##````##``````##``##```````##``````##``````##`````##`##```````##````````##``` \n \
            ##````````##```##````````##````#############````##```````##`##```````##`````###########`````##``##``````##`````````##`` \n \
            ###```````##```##````````##```##```````````##```##```````##`##```````##````##`````````##```##```##``````##```##`````##` \n \
            `##``````###````##``````###``##`````````````##``##````````####```````##```##``````````##``###````##`````##````##`````## \n \
            ``#########``````########```##``````````````###`##``````````##```````##``##````````````##`##``````##````##`````###``### \n \
            ````````#####`````````````````````````````````````````````````````````````````````````````````````````````````````##``  \n \
            ``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````` \n \
            ``````````````````````````Copyright``yutiansut``2018``````QUANTITATIVE FINANCIAL FRAMEWORK````````````````````````````` \n \
            ``````````````````````````````````````````````````````````````````````````````````````````````````````````````````````` \n \
            ```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````` \n \
            ```````````````````````````````````````````````````````````````````````````````````````````````````````````````````````` \n '

__str__= __repr__
# QA_util_log_info(Logo)
<|endoftext|>


<commit_before>#
# Copyright 2013 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import requests

from collections import OrderedDict
import xml.etree.ElementTree as ET

from loader_utils import (
    guarded_conversion,
    safe_int,
    Mapping,
    date_conversion,
    source_to_records
)

def get_treasury_date(dstring):
    return date_conversion(dstring.split("T")[0], date_pattern='%Y-%m-%d',
                           to_utc=False)

def get_treasury_rate(string_val):
    val = guarded_conversion(float, string_val)
    if val is not None:
        val = round(val / 100.0, 4)
    return val

_CURVE_MAPPINGS = {
    'tid': (safe_int, "Id"),
    'date': (get_treasury_date, "NEW_DATE"),
    '1month': (get_treasury_rate, "BC_1MONTH"),
    '3month': (get_treasury_rate, "BC_3MONTH"),
    '6month': (get_treasury_rate, "BC_6MONTH"),
    '1year': (get_treasury_rate, "BC_1YEAR"),
    '2year': (get_treasury_rate, "BC_2YEAR"),
    '3year': (get_treasury_rate, "BC_3YEAR"),
    '5year': (get_treasury_rate, "BC_5YEAR"),
    '7year': (get_treasury_rate, "BC_7YEAR"),
    '10year': (get_treasury_rate, "BC_10YEAR"),
    '20year': (get_treasury_rate, "BC_20YEAR"),
    '30year': (get_treasury_rate, "BC_30YEAR"),
}

def treasury_mappings():
    return {key: Mapping(*value)
            for key, value
            in _CURVE_MAPPINGS.iteritems()}

class iter_to_stream(object):
    """
    Exposes an iterable as an i/o stream
    """
    def __init__(self, iterable):
        self.buffered = ""
        self.iter = iter(iterable)

    def read(self, size):
        result = ""
        while size > 0:
            data = self.buffered or next(self.iter, None)
            self.buffered = ""
            if data is None:
                break
            size -= len(data)
            if size < 0:
                data, self.buffered = data[:size], data[size:]
            result += data
        return result

def get_treasury_source():
    url = """\
http://data.treasury.gov/feed.svc/DailyTreasuryYieldCurveRateData\
"""
    res = requests.get(url, stream=True)
    stream = iter_to_stream(res.iter_lines())

    elements = ET.iterparse(stream, ('end', 'start-ns', 'end-ns'))

    namespaces = OrderedDict()
    properties_xpath = ['']

    def updated_namespaces():
        if '' in namespaces and 'm' in namespaces:
            properties_xpath[0] = "{%s}content/{%s}properties" % (
                namespaces[''], namespaces['m']
            )
        else:
            properties_xpath[0] = ''

    for event, element in elements:
        if event == 'end':
            tag = ET.QName(element.tag).localname
            if tag == "entry":
                properties = element.find(properties_xpath[0])
                datum = {ET.QName(node.tag).localname: node.text
                         for node in properties.iterchildren()
                         if ET.iselement(node)}
                # clear the element after we've dealt with it:
                element.clear()
                yield datum

        elif event == 'start-ns':
            namespaces[element[0]] = element[1]
            updated_namespaces()

        elif event == 'end-ns':
            namespaces.popitem()
            updated_namespaces()

def get_treasury_data():
    mappings = treasury_mappings()
    source = get_treasury_source()
    return source_to_records(mappings, source)
<commit_msg>BUG: Fix treasury loading.<commit_after>#
# Copyright 2013 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import re

import requests

from collections import OrderedDict
import xml.etree.ElementTree as ET

from loader_utils import (
    guarded_conversion,
    safe_int,
    Mapping,
    date_conversion,
    source_to_records
)

def get_treasury_date(dstring):
    return date_conversion(dstring.split("T")[0], date_pattern='%Y-%m-%d',
                           to_utc=False)

def get_treasury_rate(string_val):
    val = guarded_conversion(float, string_val)
    if val is not None:
        val = round(val / 100.0, 4)
    return val

_CURVE_MAPPINGS = {
    'tid': (safe_int, "Id"),
    'date': (get_treasury_date, "NEW_DATE"),
    '1month': (get_treasury_rate, "BC_1MONTH"),
    '3month': (get_treasury_rate, "BC_3MONTH"),
    '6month': (get_treasury_rate, "BC_6MONTH"),
    '1year': (get_treasury_rate, "BC_1YEAR"),
    '2year': (get_treasury_rate, "BC_2YEAR"),
    '3year': (get_treasury_rate, "BC_3YEAR"),
    '5year': (get_treasury_rate, "BC_5YEAR"),
    '7year': (get_treasury_rate, "BC_7YEAR"),
    '10year': (get_treasury_rate, "BC_10YEAR"),
    '20year': (get_treasury_rate, "BC_20YEAR"),
    '30year': (get_treasury_rate, "BC_30YEAR"),
}

def treasury_mappings():
    return {key: Mapping(*value)
            for key, value
            in _CURVE_MAPPINGS.iteritems()}

class iter_to_stream(object):
    """
    Exposes an iterable as an i/o stream
    """
    def __init__(self, iterable):
        self.buffered = ""
        self.iter = iter(iterable)

    def read(self, size):
        result = ""
        while size > 0:
            data = self.buffered or next(self.iter, None)
            self.buffered = ""
            if data is None:
                break
            size -= len(data)
            if size < 0:
                data, self.buffered = data[:size], data[size:]
            result += data
        return result

def get_localname(element):
    qtag = ET.QName(element.tag).text
    return re.match("(\{.*\})(.*)", qtag).group(2)

def get_treasury_source():
    url = """\
http://data.treasury.gov/feed.svc/DailyTreasuryYieldCurveRateData\
"""
    res = requests.get(url, stream=True)
    stream = iter_to_stream(res.iter_lines())

    elements = ET.iterparse(stream, ('end', 'start-ns', 'end-ns'))

    namespaces = OrderedDict()
    properties_xpath = ['']

    def updated_namespaces():
        if '' in namespaces and 'm' in namespaces:
            properties_xpath[0] = "{%s}content/{%s}properties" % (
                namespaces[''], namespaces['m']
            )
        else:
            properties_xpath[0] = ''

    for event, element in elements:
        if event == 'end':
            tag = get_localname(element)
            if tag == "entry":
                properties = element.find(properties_xpath[0])
                datum = {get_localname(node): node.text
                         for node in properties.getchildren()
                         if ET.iselement(node)}
                # clear the element after we've dealt with it:
                element.clear()
                yield datum

        elif event == 'start-ns':
            namespaces[element[0]] = element[1]
            updated_namespaces()

        elif event == 'end-ns':
            namespaces.popitem()
            updated_namespaces()

def get_treasury_data():
    mappings = treasury_mappings()
    source = get_treasury_source()
    return source_to_records(mappings, source)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import logging

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.db import models
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext as _
from vkontakte_api.models import VkontakteManager, VkontaktePKModel

from .mixins import ParseGroupsMixin, PhotableModelMixin, UserableModelMixin, VideoableModelMixin

if 'field_history' in settings.INSTALLED_APPS:
    from field_history.tracker import FieldHistoryTracker
    using_field_history = True
else:
    using_field_history = False

log = logging.getLogger('vkontakte_groups')

GROUP_TYPE_CHOICES = (
    ('group',  u'Группа'),
    ('page',  u'Страница'),
    ('event',  u'Событие'),
)

class CheckMembersCountFailed(Exception):
    pass

class GroupRemoteManager(VkontakteManager):

    def api_call(self, *args, **kwargs):
        if 'ids' in kwargs:
            kwargs['group_ids'] = ','.join(map(lambda i: str(i), kwargs.pop('ids')))
        return super(GroupRemoteManager, self).api_call(*args, **kwargs)

    def search(self, q, offset=None, count=None):
        kwargs = {'q': q}
        if offset:
            kwargs.update(offset=offset)
        if count:
            kwargs.update(count=count)

        return self.get(method='search', **kwargs)

    def fetch(self, *args, **kwargs):
        """
        Add additional fields to parent fetch request
        """
        if 'fields' not in kwargs:
            kwargs['fields'] = 'members_count'
        return super(GroupRemoteManager, self).fetch(*args, **kwargs)

    def get_members_ids(self, group, check_count=True, **kwargs):
        ids = set()
        attempts = 0
        kwargs['offset'] = 0
        kwargs['group_id'] = group.remote_id

        while True:
            response = self.api_call('get_members', **kwargs)
            ids_iteration = response.get('items', [])
            for user_id in ids_iteration:
                ids.add(int(user_id))
            ids_iteration_count = len(ids_iteration)
            ids_count = len(ids)
            log.debug('Get members of group %s. Got %s, total %s, actual ammount %s, offset %s' % (
                group, ids_iteration_count, ids_count, group.members_count, kwargs['offset']))
            if ids_iteration_count != 0:
                attempts = 0
                kwargs['offset'] += ids_iteration_count
            else:
                try:
                    if check_count:
                        self.check_members_count(group, ids_count)
                    break
                except CheckMembersCountFailed as e:
                    attempts += 1
                    if attempts <= 5:
                        log.warning('%s, offset %s, attempts %s' % (e, kwargs['offset'], attempts))
                        continue
                    else:
                        log.error(e)
                        raise

        return list(ids)

    def check_members_count(self, group, count):
        if group.members_count and count > 0:
            division = float(group.members_count) / count
            if 0.99 > division or 1.01 < division:
                raise CheckMembersCountFailed("Suspicious ammount of members fetched for group %s. "
                                              "Actual ammount is %d, fetched %d, division is %s" % (
                    group, group.members_count, count, division))

@python_2_unicode_compatible
class Group(PhotableModelMixin, VideoableModelMixin, UserableModelMixin, VkontaktePKModel):

    resolve_screen_name_types = ['group', 'page', 'event']
    slug_prefix = 'club'

    name = models.CharField(max_length=800)
    screen_name = models.CharField(u'Короткое имя группы', max_length=50, db_index=True)
    is_closed = models.NullBooleanField(u'Флаг закрытой группы')
    is_admin = models.NullBooleanField(u'Пользователь является администратором')
    members_count = models.IntegerField(u'Всего участников', null=True)
    verified = models.NullBooleanField(u'Флаг официальной группы')
    type = models.CharField(u'Тип объекта', max_length=10, choices=GROUP_TYPE_CHOICES)

    photo = models.URLField()
    photo_big = models.URLField()
    photo_medium = models.URLField()

    remote = GroupRemoteManager(remote_pk=('remote_id',), methods_namespace='groups', version=5.28, methods={
        'get': 'getById',
        'search': 'search',
        'get_members': 'getMembers',
    })

    if using_field_history:
        field_history = FieldHistoryTracker(['members_count'])

    class Meta:
        verbose_name = _('Vkontakte group')
        verbose_name_plural = _('Vkontakte groups')

    def __str__(self):
        return self.name

    @property
    def refresh_kwargs(self):
        return {'ids': [self.remote_id]}

    @property
    def wall_comments(self):
        if 'vkontakte_wall' not in settings.INSTALLED_APPS:
            raise ImproperlyConfigured("Application 'vkontakte_wall' not in INSTALLED_APPS")

        from vkontakte_wall.models import Comment
        # TODO: improve schema and queries with using owner_id field
        return Comment.objects.filter(remote_id__startswith='-%s_' % self.remote_id)

    @property
    def topics_comments(self):
        if 'vkontakte_board' not in settings.INSTALLED_APPS:
            raise ImproperlyConfigured("Application 'vkontakte_board' not in INSTALLED_APPS")

        from vkontakte_board.models import Comment
        # TODO: improve schema and queries with using owner_id field
        return Comment.objects.filter(remote_id__startswith='-%s_' % self.remote_id)

    def fetch_posts(self, *args, **kwargs):
        if 'vkontakte_wall' not in settings.INSTALLED_APPS:
            raise ImproperlyConfigured("Application 'vkontakte_wall' not in INSTALLED_APPS")

        from vkontakte_wall.models import Post
        return Post.remote.fetch_wall(owner=self, *args, **kwargs)

    def fetch_topics(self, *args, **kwargs):
        if 'vkontakte_board' not in settings.INSTALLED_APPS:
            raise ImproperlyConfigured("Application 'vkontakte_board' not in INSTALLED_APPS")

        from vkontakte_board.models import Topic
        return Topic.remote.fetch(group=self, *args, **kwargs)

    def fetch_statistic(self, *args, **kwargs):
        if 'vkontakte_groups_statistic' not in settings.INSTALLED_APPS:
            raise ImproperlyConfigured("Application 'vkontakte_groups_statistic' not in INSTALLED_APPS")

        from vkontakte_groups_statistic.models import fetch_statistic_for_group
        return fetch_statistic_for_group(group=self, *args, **kwargs)

from . import signals
<commit_msg>removed field_history<commit_after># -*- coding: utf-8 -*-
import logging

from django.conf import settings
from django.core.exceptions import ImproperlyConfigured
from django.db import models
from django.utils.encoding import python_2_unicode_compatible
from django.utils.translation import ugettext as _
from vkontakte_api.models import VkontakteManager, VkontaktePKModel

from .mixins import ParseGroupsMixin, PhotableModelMixin, UserableModelMixin, VideoableModelMixin

log = logging.getLogger('vkontakte_groups')

GROUP_TYPE_CHOICES = (
    ('group',  u'Группа'),
    ('page',  u'Страница'),
    ('event',  u'Событие'),
)

class CheckMembersCountFailed(Exception):
    pass

class GroupRemoteManager(VkontakteManager):

    def api_call(self, *args, **kwargs):
        if 'ids' in kwargs:
            kwargs['group_ids'] = ','.join(map(lambda i: str(i), kwargs.pop('ids')))
        return super(GroupRemoteManager, self).api_call(*args, **kwargs)

    def search(self, q, offset=None, count=None):
        kwargs = {'q': q}
        if offset:
            kwargs.update(offset=offset)
        if count:
            kwargs.update(count=count)

        return self.get(method='search', **kwargs)

    def fetch(self, *args, **kwargs):
        """
        Add additional fields to parent fetch request
        """
        if 'fields' not in kwargs:
            kwargs['fields'] = 'members_count'
        return super(GroupRemoteManager, self).fetch(*args, **kwargs)

    def get_members_ids(self, group, check_count=True, **kwargs):
        ids = set()
        attempts = 0
        kwargs['offset'] = 0
        kwargs['group_id'] = group.remote_id

        while True:
            response = self.api_call('get_members', **kwargs)
            ids_iteration = response.get('items', [])
            for user_id in ids_iteration:
                ids.add(int(user_id))
            ids_iteration_count = len(ids_iteration)
            ids_count = len(ids)
            log.debug('Get members of group %s. Got %s, total %s, actual ammount %s, offset %s' % (
                group, ids_iteration_count, ids_count, group.members_count, kwargs['offset']))
            if ids_iteration_count != 0:
                attempts = 0
                kwargs['offset'] += ids_iteration_count
            else:
                try:
                    if check_count:
                        self.check_members_count(group, ids_count)
                    break
                except CheckMembersCountFailed as e:
                    attempts += 1
                    if attempts <= 5:
                        log.warning('%s, offset %s, attempts %s' % (e, kwargs['offset'], attempts))
                        continue
                    else:
                        log.error(e)
                        raise

        return list(ids)

    def check_members_count(self, group, count):
        if group.members_count and count > 0:
            division = float(group.members_count) / count
            if 0.99 > division or 1.01 < division:
                raise CheckMembersCountFailed("Suspicious ammount of members fetched for group %s. "
                                              "Actual ammount is %d, fetched %d, division is %s" % (
                    group, group.members_count, count, division))

@python_2_unicode_compatible
class Group(PhotableModelMixin, VideoableModelMixin, UserableModelMixin, VkontaktePKModel):

    resolve_screen_name_types = ['group', 'page', 'event']
    slug_prefix = 'club'

    name = models.CharField(max_length=800)
    screen_name = models.CharField(u'Короткое имя группы', max_length=50, db_index=True)
    is_closed = models.NullBooleanField(u'Флаг закрытой группы')
    is_admin = models.NullBooleanField(u'Пользователь является администратором')
    members_count = models.IntegerField(u'Всего участников', null=True)
    verified = models.NullBooleanField(u'Флаг официальной группы')
    type = models.CharField(u'Тип объекта', max_length=10, choices=GROUP_TYPE_CHOICES)

    photo = models.URLField()
    photo_big = models.URLField()
    photo_medium = models.URLField()

    remote = GroupRemoteManager(remote_pk=('remote_id',), methods_namespace='groups', version=5.28, methods={
        'get': 'getById',
        'search': 'search',
        'get_members': 'getMembers',
    })

    class Meta:
        verbose_name = _('Vkontakte group')
        verbose_name_plural = _('Vkontakte groups')

    def __str__(self):
        return self.name

    @property
    def refresh_kwargs(self):
        return {'ids': [self.remote_id]}

    @property
    def wall_comments(self):
        if 'vkontakte_wall' not in settings.INSTALLED_APPS:
            raise ImproperlyConfigured("Application 'vkontakte_wall' not in INSTALLED_APPS")

        from vkontakte_wall.models import Comment
        # TODO: improve schema and queries with using owner_id field
        return Comment.objects.filter(remote_id__startswith='-%s_' % self.remote_id)

    @property
    def topics_comments(self):
        if 'vkontakte_board' not in settings.INSTALLED_APPS:
            raise ImproperlyConfigured("Application 'vkontakte_board' not in INSTALLED_APPS")

        from vkontakte_board.models import Comment
        # TODO: improve schema and queries with using owner_id field
        return Comment.objects.filter(remote_id__startswith='-%s_' % self.remote_id)

    def fetch_posts(self, *args, **kwargs):
        if 'vkontakte_wall' not in settings.INSTALLED_APPS:
            raise ImproperlyConfigured("Application 'vkontakte_wall' not in INSTALLED_APPS")

        from vkontakte_wall.models import Post
        return Post.remote.fetch_wall(owner=self, *args, **kwargs)

    def fetch_topics(self, *args, **kwargs):
        if 'vkontakte_board' not in settings.INSTALLED_APPS:
            raise ImproperlyConfigured("Application 'vkontakte_board' not in INSTALLED_APPS")

        from vkontakte_board.models import Topic
        return Topic.remote.fetch(group=self, *args, **kwargs)

    def fetch_statistic(self, *args, **kwargs):
        if 'vkontakte_groups_statistic' not in settings.INSTALLED_APPS:
            raise ImproperlyConfigured("Application 'vkontakte_groups_statistic' not in INSTALLED_APPS")

        from vkontakte_groups_statistic.models import fetch_statistic_for_group
        return fetch_statistic_for_group(group=self, *args, **kwargs)

from . import signals
<|endoftext|>


<commit_before><commit_msg>NEW: New script to query Orthanc<commit_after><|endoftext|>


<commit_before>"""
WSGI config for demo project.

It exposes the WSGI callable as a module-level variable named ``application``.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/howto/deployment/wsgi/
"""

import os

from django.core.wsgi import get_wsgi_application

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "demo.settings")

from whitenoise.django import DjangoWhiteNoise
application = DjangoWhiteNoise(application)
<commit_msg>Fixed wsgi.py<commit_after>"""
WSGI config for demo project.

It exposes the WSGI callable as a module-level variable named ``application``.

For more information on this file, see
https://docs.djangoproject.com/en/1.9/howto/deployment/wsgi/
"""

import os

from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()

os.environ.setdefault("DJANGO_SETTINGS_MODULE", "demo.settings")

from whitenoise.django import DjangoWhiteNoise
application = DjangoWhiteNoise(application)
<|endoftext|>


<commit_before><commit_msg>Amazing.  A very subtle change in policy in descr-branch actually found a bug here.  Here's the deal:<commit_after><|endoftext|>


<commit_before>import uuid

import pytest
from flask import g
from freezegun import freeze_time

from app import notify_celery

# requiring notify_api ensures notify_celery.init_app has been called
@pytest.fixture(scope='session')
def celery_task(notify_api):
    @notify_celery.task(name=uuid.uuid4(), base=notify_celery.task_cls)
    def test_task(delivery_info=None): pass
    return test_task

@pytest.fixture
def async_task(celery_task):
    celery_task.push_request(delivery_info={'routing_key': 'test-queue'})
    yield celery_task
    celery_task.pop_request()

def test_success_should_log_and_call_statsd(mocker, notify_api, async_task):
    statsd = mocker.patch.object(notify_api.statsd_client, 'timing')
    logger = mocker.patch.object(notify_api.logger, 'info')

    with freeze_time() as frozen:
        async_task()
        frozen.tick(5)

        async_task.on_success(
            retval=None, task_id=1234, args=[], kwargs={}
        )

    statsd.assert_called_once_with(f'celery.test-queue.{async_task.name}.success', 5.0)
    logger.assert_called_once_with(f'Celery task {async_task.name} (queue: test-queue) took 5.0000')

def test_success_queue_when_applied_synchronously(mocker, notify_api, celery_task):
    statsd = mocker.patch.object(notify_api.statsd_client, 'timing')
    logger = mocker.patch.object(notify_api.logger, 'info')

    with freeze_time() as frozen:
        celery_task()
        frozen.tick(5)

        celery_task.on_success(
            retval=None, task_id=1234, args=[], kwargs={}
        )

    statsd.assert_called_once_with(f'celery.none.{celery_task.name}.success', 5.0)
    logger.assert_called_once_with(f'Celery task {celery_task.name} (queue: none) took 5.0000')

def test_failure_should_log_and_call_statsd(mocker, notify_api, async_task):
    statsd = mocker.patch.object(notify_api.statsd_client, 'incr')
    logger = mocker.patch.object(notify_api.logger, 'exception')

    async_task.on_failure(
        exc=Exception, task_id=1234, args=[], kwargs={}, einfo=None
    )

    statsd.assert_called_once_with(f'celery.test-queue.{async_task.name}.failure')
    logger.assert_called_once_with(f'Celery task {async_task.name} (queue: test-queue) failed')

def test_failure_queue_when_applied_synchronously(mocker, notify_api, celery_task):
    statsd = mocker.patch.object(notify_api.statsd_client, 'incr')
    logger = mocker.patch.object(notify_api.logger, 'exception')

    celery_task.on_failure(
        exc=Exception, task_id=1234, args=[], kwargs={}, einfo=None
    )

    statsd.assert_called_once_with(f'celery.none.{celery_task.name}.failure')
    logger.assert_called_once_with(f'Celery task {celery_task.name} (queue: none) failed')

def test_call_exports_request_id_from_kwargs(mocker, celery_task):
    g = mocker.patch('app.celery.celery.g')
    # this would fail if the kwarg was passed through unexpectedly
    celery_task(request_id='1234')
    assert g.request_id == '1234'
<commit_msg>Add tests for apply_async injecting request_id<commit_after>import uuid

import pytest
from flask import g
from freezegun import freeze_time

from app import notify_celery

# requiring notify_api ensures notify_celery.init_app has been called
@pytest.fixture(scope='session')
def celery_task(notify_api):
    @notify_celery.task(name=uuid.uuid4(), base=notify_celery.task_cls)
    def test_task(delivery_info=None): pass
    return test_task

@pytest.fixture
def async_task(celery_task):
    celery_task.push_request(delivery_info={'routing_key': 'test-queue'})
    yield celery_task
    celery_task.pop_request()

def test_success_should_log_and_call_statsd(mocker, notify_api, async_task):
    statsd = mocker.patch.object(notify_api.statsd_client, 'timing')
    logger = mocker.patch.object(notify_api.logger, 'info')

    with freeze_time() as frozen:
        async_task()
        frozen.tick(5)

        async_task.on_success(
            retval=None, task_id=1234, args=[], kwargs={}
        )

    statsd.assert_called_once_with(f'celery.test-queue.{async_task.name}.success', 5.0)
    logger.assert_called_once_with(f'Celery task {async_task.name} (queue: test-queue) took 5.0000')

def test_success_queue_when_applied_synchronously(mocker, notify_api, celery_task):
    statsd = mocker.patch.object(notify_api.statsd_client, 'timing')
    logger = mocker.patch.object(notify_api.logger, 'info')

    with freeze_time() as frozen:
        celery_task()
        frozen.tick(5)

        celery_task.on_success(
            retval=None, task_id=1234, args=[], kwargs={}
        )

    statsd.assert_called_once_with(f'celery.none.{celery_task.name}.success', 5.0)
    logger.assert_called_once_with(f'Celery task {celery_task.name} (queue: none) took 5.0000')

def test_failure_should_log_and_call_statsd(mocker, notify_api, async_task):
    statsd = mocker.patch.object(notify_api.statsd_client, 'incr')
    logger = mocker.patch.object(notify_api.logger, 'exception')

    async_task.on_failure(
        exc=Exception, task_id=1234, args=[], kwargs={}, einfo=None
    )

    statsd.assert_called_once_with(f'celery.test-queue.{async_task.name}.failure')
    logger.assert_called_once_with(f'Celery task {async_task.name} (queue: test-queue) failed')

def test_failure_queue_when_applied_synchronously(mocker, notify_api, celery_task):
    statsd = mocker.patch.object(notify_api.statsd_client, 'incr')
    logger = mocker.patch.object(notify_api.logger, 'exception')

    celery_task.on_failure(
        exc=Exception, task_id=1234, args=[], kwargs={}, einfo=None
    )

    statsd.assert_called_once_with(f'celery.none.{celery_task.name}.failure')
    logger.assert_called_once_with(f'Celery task {celery_task.name} (queue: none) failed')

def test_call_exports_request_id_from_kwargs(mocker, celery_task):
    g = mocker.patch('app.celery.celery.g')
    # this would fail if the kwarg was passed through unexpectedly
    celery_task(request_id='1234')
    assert g.request_id == '1234'

def test_apply_async_injects_global_request_id_into_kwargs(mocker, celery_task):
    super_apply = mocker.patch('celery.app.task.Task.apply_async')
    g.request_id = '1234'
    celery_task.apply_async()

    super_apply.assert_called_with(
        None,
        {'request_id': '1234'},
        None,
        None,
        None,
        None
    )

def test_apply_async_injects_id_into_kwargs_from_request(mocker, notify_api, celery_task):
    super_apply = mocker.patch('celery.app.task.Task.apply_async')
    request_id_header = notify_api.config['NOTIFY_TRACE_ID_HEADER']
    request_headers = {request_id_header: '1234'}

    with notify_api.test_request_context(headers=request_headers):
        celery_task.apply_async()

    super_apply.assert_called_with(
        None,
        {'request_id': '1234'},
        None,
        None,
        None,
        None
    )
<|endoftext|>


<commit_before>import logging
from datetime import datetime
from string import Template

import luigi
# from luigi import six

from googleapiclient.errors import HttpError
import time
from luigi_gcloud.gcore import get_default_client, _GCloudTask
from luigi_gcloud.storage import GCSTarget

logger = logging.getLogger('luigi-gcloud')

default_bigquery_udf = None

def _table_with_default(name, client):
    return _split_tablename(name, client.project_id())

def _split_tablename(name, default_project=None):
    cmpt = name.split(':')
    if len(cmpt) == 1:
        project_id = default_project
        rest = cmpt[0]
    elif len(cmpt) == 2:
        project_id = cmpt[0]
        rest = cmpt[1]
    else:
        raise RuntimeError

    cmpt = rest.split('.')
    if len(cmpt) == 2:
        dataset_id = cmpt[0]
        table_id = cmpt[1]
    else:
        raise RuntimeError

    return {
        'projectId': project_id,
        'datasetId': dataset_id,
        'tableId': table_id
    }

class _BqJob:
    def __init__(self, bq, project_id, job):
        self.bq = bq
        self.project_id = project_id
        self.job = self.bq.jobs().insert(projectId=self.project_id, body=job).execute()
        self.job_id = self.job['jobReference']['jobId']
        logger.info('BigQuery job %s is %s', self.job_id, str(self.job['status']))

    def wait_for_done(self):

        while True:
            self.job = self.bq.jobs().get(
                projectId=self.project_id,
                jobId=self.job_id).execute()
            if 'DONE' == self.job['status']['state']:
                if self.job['status'].has_key('errors'):
                    logger.warning('BigQuery job %s has errors', self.job_id)
                    logger.debug(str(self.job))
                    return False
                else:
                    logger.info('BigQuery job %s is DONE', self.job_id)
                    return True
            logger.debug('BigQuery job %s is %s', self.job_id, str(self.job['status']))
            time.sleep(5)

    def raise_error(self, message):
        if self.job['status'].has_key('errors'):
            if message is None:
                message = "Google BigQuery job has error"
            raise Exception(message + ": " + str(self.job['status']['errors']))

    def get(self):
        return self.job

class BigQueryTarget(luigi.Target):
    def __init__(self, table, query=None, client=None):
        self.client = client or get_default_client()
        self.table = _table_with_default(table, self.client)
        self.query = query
        http = self.client.http_authorized()
        self.bigquery_api = self.client.bigquery_api(http)

    def query(self):
        raise NotImplementedError

    def exists(self):
        tables = self.bigquery_api.tables()
        try:
            table = tables.get(projectId=self.table['projectId'], datasetId=self.table['datasetId'],
                               tableId=self.table['tableId']).execute()
            jobs = self.bigquery_api.jobs()
            job = {
                'projectId': self.table['projectId'],
                'configuration': {
                    'query': {
                        'useQueryCache': 'True',
                        'query': self.query
                    }
                }
            }
            logger.debug(table)
            if self.query is None:
                return True

            insert_job = _BqJob(self.bigquery_api, self.table['projectId'], job=job)
            if insert_job.wait_for_done():
                result = insert_job.get()
                result = jobs.getQueryResults(projectId=self.table['projectId'],
                                              jobId=result['jobReference']['jobId']).execute()
                return result['rows'][0]['f'][0]['v'] in ['true', 'True']
        except HttpError as err:
            if err.resp.status == 404:
                return False
            logger.error("HttpError, when checking existence of " + self.table['projectId'] + ":" + self.table[
                'datasetId'] + "." + self.table['tableId'] + ".")
            logger.debug(str(err))
            return False

class _BqTask(_GCloudTask):
    service_name = 'bigquery'
    client = None
    bigquery_api = None

    def __init__(self, *args, **kwargs):
        self.client = kwargs.get("client") or get_default_client()
        http = self.client.http_authorized()
        self.bigquery_api = self.client.bigquery_api(http)
        super(_BqTask, self).__init__(*args, **kwargs)

class BigQueryLoadTask(_BqTask):
    """ Load/Append data into a BigQuery table """

    def __init__(self, *args, **kwargs):
        super(BigQueryLoadTask, self).__init__(*args, **kwargs)

    def output(self):
        return BigQueryTarget(self.table(), None, self.client)

    def schema(self):
        raise NotImplementedError

    def source(self):
        raise NotImplementedError

    def table(self):
        """
        Format 123456789:DataSet.Table
        :return:
        """
        raise NotImplementedError

    def run(self):
        table = _table_with_default(self.table(), self.client)
        # https://cloud.google.com/bigquery/docs/reference/v2/jobs
        job = {
            'projectId': table['projectId'],
            'configuration': {
                'load': {
                    # 'quote':'',
                    'destinationTable': {
                        'projectId': table['projectId'],
                        'datasetId': table['datasetId'],
                        'tableId': table['tableId']
                    },
                    'sourceFormat': self.get_service_value("sourceFormat", "NEWLINE_DELIMITED_JSON"),
                    'createDisposition': self.get_service_value("createDisposition", "CREATE_IF_NEEDED"),
                    'writeDisposition': self.get_service_value("writeDisposition", "WRITE_APPEND"),
                    'sourceUris': [self.source()],
                    'maxBadRecords': 1000,
                    'schema': {
                        'fields': self.schema()
                    }
                }
            }
        }
        load_job = _BqJob(self.bigquery_api, table['projectId'], job=job)
        if load_job.wait_for_done():
            marker = self.output()
            if hasattr(marker, "touch") and callable(getattr(marker, "touch")):
                logger.info("Writing marker file " + str(marker))
                marker.touch()
            return
        else:
            load_job.raise_error("Error loading data in Google BigQuery")

class BigQueryTask(_BqTask):
    """ Query a BigQuery table into another table or storage file """

    def __init__(self, *args, **kwargs):
        super(BigQueryTask, self).__init__(*args, **kwargs)

    def output(self):
        if self.table() is not None:
            return BigQueryTarget(self.table())
        if self.destination() is not None:
            destination = self.destination()
            if destination.find('*') != -1:
                destination = destination[:destination.rindex('/') + 1]
            return GCSTarget(destination)

    def schema(self):
        return None

    def query(self):
        with open(self.query_file(), 'r') as f:
            content = f.read()
        return content

    def query_file(self):
        return NotImplementedError

    def table(self):
        """
        Format 123456789:DataSet.Table
        :return:
        """
        return None

    def destination(self):
        return None

    def lib_uris(self):
        return []

    def _success(self):
        marker = self.output()
        if hasattr(marker, "touch") and callable(getattr(marker, "touch")):
            logger.info("Writing marker file " + str(marker))
            marker.touch()
        return

    def _js_file_uris(self):
        global default_bigquery_udf
        artifacts = []
        uris = self.lib_uris()
        artifacts.extend(uris)
        if default_bigquery_udf is not None:
            if not default_bigquery_udf.get('append') and len(uris) == 0:
                artifacts.extend(default_bigquery_udf.get('uris'))
            elif default_bigquery_udf.get('append'):
                artifacts.extend(default_bigquery_udf.get('uris'))
        return artifacts

    def run(self):
        query = Template(self.query()).substitute(self.variables())
        job = {
            'projectId': self.client.project_id(),
            "configuration": {
                "query": {
                    "query": query,
                }
            }
        }
        schema = self.schema()
        if schema is not None:
            job["configuration"]["schema"] = {
                'fields': schema
            }

        udf = self._js_file_uris()
        if len(udf) > 0:
            job["configuration"]["query"]["userDefinedFunctionResources"] = []
            for uri in udf:
                job["configuration"]["query"]["userDefinedFunctionResources"].append(
                    {
                        "resourceUri": uri
                    }
                )

        if self.table() is None and self.destination() is None:
            raise RuntimeError("At least table or destination need to be supplied.")
        if self.table() is not None and self.destination() is not None:
            raise RuntimeError("Set either table or destination.")

        if self.table() is not None:
            table = _table_with_default(self.table(), self.client)
            job["configuration"]["query"]["destinationTable"] = {
                'projectId': table['projectId'],
                'datasetId': table['datasetId'],
                'tableId': table['tableId']
            }
            job["configuration"]["query"]["createDisposition"] = \
                self.get_service_value("createDisposition", "CREATE_IF_NEEDED")
            job["configuration"]["query"]["writeDisposition"] = \
                self.get_service_value("writeDisposition", "WRITE_APPEND")
            job["configuration"]["query"]["allowLargeResults"] = \
                self.get_service_value("allowLargeResults", "false")

            insert_job = _BqJob(self.bigquery_api, self.client.project_id(), job=job)
            if insert_job.wait_for_done():
                self._success()
            else:
                insert_job.raise_error("Error executing query in Google BigQuery")

        if self.destination() is not None:
            large = self.get_service_value("allowLargeResults", "false")
            if str(large) in ("true", "1", "True"):
                # This is a workaround for a BigQuery limitation, if the result is to large
                # we need to set a destination table and do allowLargeResults
                # we need a DataSet that has an expiration to make it work properly in production
                job["configuration"]["query"]["destinationTable"] = {
                    'projectId': self.client.project_id(),
                    'datasetId': self.get_service_value("tempDataset", "work_temp"),
                    'tableId': datetime.now().strftime("xlbq_%Y%m%d%H%M%S")
                }
                job["configuration"]["query"]["allowLargeResults"] = "true"

            insert_job = _BqJob(self.bigquery_api, self.client.project_id(), job=job)
            if insert_job.wait_for_done():
                result = insert_job.get()

                job = {
                    'projectId': self.client.project_id(),
                    'configuration': {
                        'extract': {
                            'sourceTable': {
                                'projectId': result["configuration"]["query"]["destinationTable"]["projectId"],
                                'datasetId': result["configuration"]["query"]["destinationTable"]["datasetId"],
                                'tableId': result["configuration"]["query"]["destinationTable"]["tableId"]
                            },
                            'destinationUris': [
                                self.destination()
                            ],
                            'destinationFormat': self.get_service_value("destinationFormat", "NEWLINE_DELIMITED_JSON")
                        }
                    }
                }
                extract_job = _BqJob(self.bigquery_api, self.client.project_id(), job=job)
                if extract_job.wait_for_done():
                    self._success()
                else:
                    extract_job.raise_error("Error extracting query data from Google BigQuery")
            else:
                insert_job.raise_error("Error querying to temp table in Google BigQuery")
<commit_msg>python3 compat checking if dict has key<commit_after>import logging
from datetime import datetime
from string import Template

import luigi
# from luigi import six

from googleapiclient.errors import HttpError
import time
from luigi_gcloud.gcore import get_default_client, _GCloudTask
from luigi_gcloud.storage import GCSTarget

logger = logging.getLogger('luigi-gcloud')

default_bigquery_udf = None

def _table_with_default(name, client):
    return _split_tablename(name, client.project_id())

def _split_tablename(name, default_project=None):
    cmpt = name.split(':')
    if len(cmpt) == 1:
        project_id = default_project
        rest = cmpt[0]
    elif len(cmpt) == 2:
        project_id = cmpt[0]
        rest = cmpt[1]
    else:
        raise RuntimeError

    cmpt = rest.split('.')
    if len(cmpt) == 2:
        dataset_id = cmpt[0]
        table_id = cmpt[1]
    else:
        raise RuntimeError

    return {
        'projectId': project_id,
        'datasetId': dataset_id,
        'tableId': table_id
    }

class _BqJob:
    def __init__(self, bq, project_id, job):
        self.bq = bq
        self.project_id = project_id
        self.job = self.bq.jobs().insert(projectId=self.project_id, body=job).execute()
        self.job_id = self.job['jobReference']['jobId']
        logger.info('BigQuery job %s is %s', self.job_id, str(self.job['status']))

    def wait_for_done(self):

        while True:
            self.job = self.bq.jobs().get(
                projectId=self.project_id,
                jobId=self.job_id).execute()
            if 'DONE' == self.job['status']['state']:
                if 'errors' in self.job['status']:
                    logger.warning('BigQuery job %s has errors', self.job_id)
                    logger.debug(str(self.job))
                    return False
                else:
                    logger.info('BigQuery job %s is DONE', self.job_id)
                    return True
            logger.debug('BigQuery job %s is %s', self.job_id, str(self.job['status']))
            time.sleep(5)

    def raise_error(self, message):
        if 'errors' in self.job['status']:
            if message is None:
                message = "Google BigQuery job has error"
            raise Exception(message + ": " + str(self.job['status']['errors']))

    def get(self):
        return self.job

class BigQueryTarget(luigi.Target):
    def __init__(self, table, query=None, client=None):
        self.client = client or get_default_client()
        self.table = _table_with_default(table, self.client)
        self.query = query
        http = self.client.http_authorized()
        self.bigquery_api = self.client.bigquery_api(http)

    def query(self):
        raise NotImplementedError

    def exists(self):
        tables = self.bigquery_api.tables()
        try:
            table = tables.get(projectId=self.table['projectId'], datasetId=self.table['datasetId'],
                               tableId=self.table['tableId']).execute()
            jobs = self.bigquery_api.jobs()
            job = {
                'projectId': self.table['projectId'],
                'configuration': {
                    'query': {
                        'useQueryCache': 'True',
                        'query': self.query
                    }
                }
            }
            logger.debug(table)
            if self.query is None:
                return True

            insert_job = _BqJob(self.bigquery_api, self.table['projectId'], job=job)
            if insert_job.wait_for_done():
                result = insert_job.get()
                result = jobs.getQueryResults(projectId=self.table['projectId'],
                                              jobId=result['jobReference']['jobId']).execute()
                return result['rows'][0]['f'][0]['v'] in ['true', 'True']
        except HttpError as err:
            if err.resp.status == 404:
                return False
            logger.error("HttpError, when checking existence of " + self.table['projectId'] + ":" + self.table[
                'datasetId'] + "." + self.table['tableId'] + ".")
            logger.debug(str(err))
            return False

class _BqTask(_GCloudTask):
    service_name = 'bigquery'
    client = None
    bigquery_api = None

    def __init__(self, *args, **kwargs):
        self.client = kwargs.get("client") or get_default_client()
        http = self.client.http_authorized()
        self.bigquery_api = self.client.bigquery_api(http)
        super(_BqTask, self).__init__(*args, **kwargs)

class BigQueryLoadTask(_BqTask):
    """ Load/Append data into a BigQuery table """

    def __init__(self, *args, **kwargs):
        super(BigQueryLoadTask, self).__init__(*args, **kwargs)

    def output(self):
        return BigQueryTarget(self.table(), None, self.client)

    def schema(self):
        raise NotImplementedError

    def source(self):
        raise NotImplementedError

    def table(self):
        """
        Format 123456789:DataSet.Table
        :return:
        """
        raise NotImplementedError

    def run(self):
        table = _table_with_default(self.table(), self.client)
        # https://cloud.google.com/bigquery/docs/reference/v2/jobs
        job = {
            'projectId': table['projectId'],
            'configuration': {
                'load': {
                    # 'quote':'',
                    'destinationTable': {
                        'projectId': table['projectId'],
                        'datasetId': table['datasetId'],
                        'tableId': table['tableId']
                    },
                    'sourceFormat': self.get_service_value("sourceFormat", "NEWLINE_DELIMITED_JSON"),
                    'createDisposition': self.get_service_value("createDisposition", "CREATE_IF_NEEDED"),
                    'writeDisposition': self.get_service_value("writeDisposition", "WRITE_APPEND"),
                    'sourceUris': [self.source()],
                    'maxBadRecords': 1000,
                    'schema': {
                        'fields': self.schema()
                    }
                }
            }
        }
        load_job = _BqJob(self.bigquery_api, table['projectId'], job=job)
        if load_job.wait_for_done():
            marker = self.output()
            if hasattr(marker, "touch") and callable(getattr(marker, "touch")):
                logger.info("Writing marker file " + str(marker))
                marker.touch()
            return
        else:
            load_job.raise_error("Error loading data in Google BigQuery")

class BigQueryTask(_BqTask):
    """ Query a BigQuery table into another table or storage file """

    def __init__(self, *args, **kwargs):
        super(BigQueryTask, self).__init__(*args, **kwargs)

    def output(self):
        if self.table() is not None:
            return BigQueryTarget(self.table())
        if self.destination() is not None:
            destination = self.destination()
            if destination.find('*') != -1:
                destination = destination[:destination.rindex('/') + 1]
            return GCSTarget(destination)

    def schema(self):
        return None

    def query(self):
        with open(self.query_file(), 'r') as f:
            content = f.read()
        return content

    def query_file(self):
        return NotImplementedError

    def table(self):
        """
        Format 123456789:DataSet.Table
        :return:
        """
        return None

    def destination(self):
        return None

    def lib_uris(self):
        return []

    def _success(self):
        marker = self.output()
        if hasattr(marker, "touch") and callable(getattr(marker, "touch")):
            logger.info("Writing marker file " + str(marker))
            marker.touch()
        return

    def _js_file_uris(self):
        global default_bigquery_udf
        artifacts = []
        uris = self.lib_uris()
        artifacts.extend(uris)
        if default_bigquery_udf is not None:
            if not default_bigquery_udf.get('append') and len(uris) == 0:
                artifacts.extend(default_bigquery_udf.get('uris'))
            elif default_bigquery_udf.get('append'):
                artifacts.extend(default_bigquery_udf.get('uris'))
        return artifacts

    def run(self):
        query = Template(self.query()).substitute(self.variables())
        job = {
            'projectId': self.client.project_id(),
            "configuration": {
                "query": {
                    "query": query,
                }
            }
        }
        schema = self.schema()
        if schema is not None:
            job["configuration"]["schema"] = {
                'fields': schema
            }

        udf = self._js_file_uris()
        if len(udf) > 0:
            job["configuration"]["query"]["userDefinedFunctionResources"] = []
            for uri in udf:
                job["configuration"]["query"]["userDefinedFunctionResources"].append(
                    {
                        "resourceUri": uri
                    }
                )

        if self.table() is None and self.destination() is None:
            raise RuntimeError("At least table or destination need to be supplied.")
        if self.table() is not None and self.destination() is not None:
            raise RuntimeError("Set either table or destination.")

        if self.table() is not None:
            table = _table_with_default(self.table(), self.client)
            job["configuration"]["query"]["destinationTable"] = {
                'projectId': table['projectId'],
                'datasetId': table['datasetId'],
                'tableId': table['tableId']
            }
            job["configuration"]["query"]["createDisposition"] = \
                self.get_service_value("createDisposition", "CREATE_IF_NEEDED")
            job["configuration"]["query"]["writeDisposition"] = \
                self.get_service_value("writeDisposition", "WRITE_APPEND")
            job["configuration"]["query"]["allowLargeResults"] = \
                self.get_service_value("allowLargeResults", "false")

            insert_job = _BqJob(self.bigquery_api, self.client.project_id(), job=job)
            if insert_job.wait_for_done():
                self._success()
            else:
                insert_job.raise_error("Error executing query in Google BigQuery")

        if self.destination() is not None:
            large = self.get_service_value("allowLargeResults", "false")
            if str(large) in ("true", "1", "True"):
                # This is a workaround for a BigQuery limitation, if the result is to large
                # we need to set a destination table and do allowLargeResults
                # we need a DataSet that has an expiration to make it work properly in production
                job["configuration"]["query"]["destinationTable"] = {
                    'projectId': self.client.project_id(),
                    'datasetId': self.get_service_value("tempDataset", "work_temp"),
                    'tableId': datetime.now().strftime("xlbq_%Y%m%d%H%M%S")
                }
                job["configuration"]["query"]["allowLargeResults"] = "true"

            insert_job = _BqJob(self.bigquery_api, self.client.project_id(), job=job)
            if insert_job.wait_for_done():
                result = insert_job.get()

                job = {
                    'projectId': self.client.project_id(),
                    'configuration': {
                        'extract': {
                            'sourceTable': {
                                'projectId': result["configuration"]["query"]["destinationTable"]["projectId"],
                                'datasetId': result["configuration"]["query"]["destinationTable"]["datasetId"],
                                'tableId': result["configuration"]["query"]["destinationTable"]["tableId"]
                            },
                            'destinationUris': [
                                self.destination()
                            ],
                            'destinationFormat': self.get_service_value("destinationFormat", "NEWLINE_DELIMITED_JSON")
                        }
                    }
                }
                extract_job = _BqJob(self.bigquery_api, self.client.project_id(), job=job)
                if extract_job.wait_for_done():
                    self._success()
                else:
                    extract_job.raise_error("Error extracting query data from Google BigQuery")
            else:
                insert_job.raise_error("Error querying to temp table in Google BigQuery")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
===============================================================================
Voronoi --Subclass of GenericGeometry for a standard Geometry created from a
Voronoi Diagram Used with Delaunay Network but could work for others (not tested)
===============================================================================

"""

import scipy as sp
from OpenPNM.Geometry import models as gm
from OpenPNM.Geometry import GenericGeometry

class Voronoi(GenericGeometry):
    r"""
    Voronoi subclass of GenericGeometry.

    Parameters
    ----------

    """

    def __init__(self, fibre_rad = 3e-06, load_gen='gen', **kwargs):
        r"""
        Initialize
        """
        if int(sp.__version__.split('.')[1]) < 13:
            raise Exception('The installed version of Scipy is too old, Voronoi cannot run')
        super(Voronoi,self).__init__(**kwargs)
        if load_gen == 'gen':
            self._generate(fibre_rad)

    def _generate(self,fibre_rad):
        r"""
        """

        self.models.add(propname='pore.vertices',
                        model=gm.pore_vertices.voronoi)
        self.models.add(propname='throat.vertices',
                        model=gm.throat_vertices.voronoi)
        self.models.add(propname='throat.normal',
                        model=gm.throat_normal.voronoi)
        self.models.add(propname='throat.offset_vertices',
                        model=gm.throat_offset_vertices.distance_transform,
                        offset=fibre_rad,
                        set_dependent = True)
        "Remove throats that are fully occluded and pores with no connections"
        self._net.trim_occluded_throats()

        self.models.add(propname='pore.seed',
                        model=gm.pore_misc.random,
                        seed=self._seed)
        self.models.add(propname='throat.seed',
                        model=gm.throat_misc.neighbor,
                        pore_prop='pore.seed',
                        mode='min')
        self.models.add(propname='pore.volume',
                        model=gm.pore_volume.voronoi)
        self.models.add(propname='pore.diameter',
                        model=gm.pore_diameter.voronoi)
        self.models.add(propname='pore.area',
                        model=gm.pore_area.spherical)
        self.models.add(propname='throat.diameter',
                        model=gm.throat_diameter.voronoi)
        self.models.add(propname='throat.length',
                        model=gm.throat_length.constant,
                        const=fibre_rad*2)
        self.models.add(propname='throat.volume',
                        model=gm.throat_volume.extrusion)
        self.models.add(propname='throat.surface_area',
                        model=gm.throat_surface_area.extrusion)

        "Old Methods Replaced by Distance Transform"
        #self.add_model(propname='throat.area',
        #               model=gm.throat_area.voronoi)
        #self.add_model(propname='throat.perimeter',
        #               model=gm.throat_perimeter.voronoi)
        #self.add_model(propname='throat.centroid',
        #               model=gm.throat_centroid.centre_of_mass)
        #self.add_model(propname='pore.centroid',
        #               model=gm.pore_centroid.centre_of_mass)
        #self.add_model(propname='pore.indiameter',
        #               model=gm.pore_diameter.insphere)
        #self.add_model(propname='throat.indiameter',
        #               model=gm.throat_diameter.incircle)
        "Shift the pore coords to the centroids"
        #vo.pore2centroid(self._net)

if __name__ == '__main__':
    import OpenPNM
    pn = OpenPNM.Network.Delaunay(name='test_net')
    pn.generate(num_pores=100, domain_size=[0.0001,0.0001,0.0001])
    pn.add_boundaries()
    test = OpenPNM.Geometry.Voronoi(pores=pn.Ps,throats=pn.Ts,network=pn)
    pn.regenerate_geometries()<commit_msg>Pep8ified Voronoi<commit_after># -*- coding: utf-8 -*-
"""
===============================================================================
Voronoi --Subclass of GenericGeometry for a standard Geometry created from a
Voronoi Diagram Used with Delaunay Network but could work for others (not tested)
===============================================================================

"""

import scipy as sp
from OpenPNM.Geometry import models as gm
from OpenPNM.Geometry import GenericGeometry

class Voronoi(GenericGeometry):
    r"""
    Voronoi subclass of GenericGeometry.

    Parameters
    ----------

    """

    def __init__(self, fibre_rad=3e-06, load_gen='gen', **kwargs):
        super().__init__(**kwargs)
        if load_gen == 'gen':
            self._generate(fibre_rad)

    def _generate(self, fibre_rad):
        self.models.add(propname='pore.vertices',
                        model=gm.pore_vertices.voronoi)
        self.models.add(propname='throat.vertices',
                        model=gm.throat_vertices.voronoi)
        self.models.add(propname='throat.normal',
                        model=gm.throat_normal.voronoi)
        self.models.add(propname='throat.offset_vertices',
                        model=gm.throat_offset_vertices.distance_transform,
                        offset=fibre_rad,
                        set_dependent=True)
        # Remove throats that are fully occluded and pores with no connections
        self._net.trim_occluded_throats()

        self.models.add(propname='pore.seed',
                        model=gm.pore_misc.random,
                        seed=self._seed)
        self.models.add(propname='throat.seed',
                        model=gm.throat_misc.neighbor,
                        pore_prop='pore.seed',
                        mode='min')
        self.models.add(propname='pore.volume',
                        model=gm.pore_volume.voronoi)
        self.models.add(propname='pore.diameter',
                        model=gm.pore_diameter.voronoi)
        self.models.add(propname='pore.area',
                        model=gm.pore_area.spherical)
        self.models.add(propname='throat.diameter',
                        model=gm.throat_diameter.voronoi)
        self.models.add(propname='throat.length',
                        model=gm.throat_length.constant,
                        const=fibre_rad*2)
        self.models.add(propname='throat.volume',
                        model=gm.throat_volume.extrusion)
        self.models.add(propname='throat.surface_area',
                        model=gm.throat_surface_area.extrusion)

if __name__ == '__main__':
    import OpenPNM
    pn = OpenPNM.Network.Delaunay(name='test_net')
    pn.generate(num_pores=100, domain_size=[0.0001, 0.0001, 0.0001])
    pn.add_boundaries()
    test = OpenPNM.Geometry.Voronoi(pores=pn.Ps, throats=pn.Ts, network=pn)
    pn.regenerate_geometries()
<|endoftext|>