diff --git a/cluster/src/test/scala/org/apache/spark/sql/NWQueries.scala b/cluster/src/test/scala/org/apache/spark/sql/NWQueries.scala index 3f93f39a60..6f3fc159a0 100644 --- a/cluster/src/test/scala/org/apache/spark/sql/NWQueries.scala +++ b/cluster/src/test/scala/org/apache/spark/sql/NWQueries.scala @@ -787,80 +787,37 @@ object NWQueries extends SnappyFunSuite { "Q23" -> Q23, "Q24" -> Q24, "Q25" -> Q25, - "Q25_1" -> Q25_1, - "Q25_2" -> Q25_2, "Q26" -> Q26, - "Q26_1" -> Q26_1, - "Q26_2" -> Q26_2, "Q27" -> Q27, - "Q27_1" -> Q27_1, - "Q27_2" -> Q27_2, - "Q27_3" -> Q27_3, - "Q27_4" -> Q27_4, "Q28" -> Q28, - "Q28_1" -> Q28_1, - "Q28_2" -> Q28_2, "Q29" -> Q29, - "Q29_1" -> Q29_1, - "Q29_2" -> Q29_2, "Q30" -> Q30, - "Q30_1" -> Q30_1, - "Q30_2" -> Q30_2, "Q31" -> Q31, - "Q31_1" -> Q31_1, - "Q31_2" -> Q31_2, - "Q31_3" -> Q31_3, - "Q31_4" -> Q31_4, "Q32" -> Q32, - "Q32_1" -> Q32_1, "Q33" -> Q33, - "Q33_1" -> Q33_1, "Q34" -> Q34, - "Q34_1" -> Q34_1, - "Q34_2" -> Q34_2, "Q35" -> Q35, - "Q35_1" -> Q35_1, - "Q35_2" -> Q35_2, "Q36" -> Q36, - "Q36_1" -> Q36_1, - "Q36_2" -> Q36_2, "Q37" -> Q37, "Q38" -> Q38, - "Q38_1" -> Q38_1, - "Q38_2" -> Q38_2, "Q39" -> Q39, "Q40" -> Q40, - "Q40_1" -> Q40_1, - "Q40_2" -> Q40_2, "Q41" -> Q41, "Q42" -> Q42, - "Q42_1" -> Q42_1, - "Q42_2" -> Q42_2, "Q43" -> Q43, - "Q43_1" -> Q43_1, - "Q43_2" -> Q43_2, "Q44" -> Q44, "Q45" -> Q45, "Q46" -> Q46, "Q47" -> Q47, "Q48" -> Q48, "Q49" -> Q49, - "Q49_1" -> Q49_1, - "Q49_2" -> Q49_2, "Q50" -> Q50, "Q51" -> Q51, - "Q51_1" -> Q51_1, - "Q51_2" -> Q51_2, "Q52" -> Q52, "Q53" -> Q53, "Q54" -> Q54, "Q55" -> Q55, - "Q55_1" -> Q55_1, - "Q55_2" -> Q55_2, - "Q56" -> Q56, - "Q56_1" -> Q56_1, - "Q56_2" -> Q56_2, - "Q56_3" -> Q56_3 + "Q56" -> Q56 ) def regions(sqlContext: SQLContext): DataFrame = diff --git a/dtests/src/resources/scripts/ct/ct_create_and_load_evictionOverflow_row_table.sql b/dtests/src/resources/scripts/ct/ct_create_and_load_evictionOverflow_row_table.sql new file mode 100644 index 0000000000..fdc1fec1d7 --- /dev/null +++ b/dtests/src/resources/scripts/ct/ct_create_and_load_evictionOverflow_row_table.sql @@ -0,0 +1,105 @@ +-- DROP TABLE IF ALREADY EXISTS -- +DROP TABLE IF EXISTS ORDERS_DETAILS; +DROP TABLE IF EXISTS staging_orders_details; + +-- CREATE PARTITIONED ROW TABLE ORDERS_DETAILS -- +CREATE EXTERNAL TABLE staging_orders_details USING com.databricks.spark.csv + OPTIONS (path ':dataLocation/ORDERS_DETAILS.dat', header 'true', inferSchema 'false', + nullValue 'NULL'); + +CREATE TABLE ORDERS_DETAILS + (SINGLE_ORDER_DID BIGINT ,SYS_ORDER_ID VARCHAR(64) ,SYS_ORDER_VER INTEGER ,DATA_SNDG_SYS_NM VARCHAR(128) , + SRC_SYS VARCHAR(20) ,SYS_PARENT_ORDER_ID VARCHAR(64) ,SYS_PARENT_ORDER_VER SMALLINT ,PARENT_ORDER_TRD_DATE VARCHAR(20), + PARENT_ORDER_SYS_NM VARCHAR(128) ,SYS_ALT_ORDER_ID VARCHAR(64) ,TRD_DATE VARCHAR(20),GIVE_UP_BROKER VARCHAR(20) , + EVENT_RCV_TS TIMESTAMP ,SYS_ROOT_ORDER_ID VARCHAR(64) ,GLB_ROOT_ORDER_ID VARCHAR(64) ,GLB_ROOT_ORDER_SYS_NM VARCHAR(128) , + GLB_ROOT_ORDER_RCV_TS TIMESTAMP ,SYS_ORDER_STAT_CD VARCHAR(20) ,SYS_ORDER_STAT_DESC_TXT VARCHAR(120) ,DW_STAT_CD VARCHAR(20) , + EVENT_TS TIMESTAMP,ORDER_OWNER_FIRM_ID VARCHAR(20),RCVD_ORDER_ID VARCHAR(64) ,EVENT_INITIATOR_ID VARCHAR(64), + TRDR_SYS_LOGON_ID VARCHAR(64),SOLICITED_FG VARCHAR(1),RCVD_FROM_FIRMID_CD VARCHAR(20),RCV_DESK VARCHAR(20), + SYS_ACCT_ID_SRC VARCHAR(64) ,CUST_ACCT_MNEMONIC VARCHAR(128),CUST_SLANG VARCHAR(20) ,SYS_ACCT_TYPE VARCHAR(20) , + CUST_EXCH_ACCT_ID VARCHAR(64) ,SYS_SECURITY_ALT_ID VARCHAR(64) ,TICKER_SYMBOL VARCHAR(32) ,TICKER_SYMBOL_SUFFIX VARCHAR(20) , + PRODUCT_CAT_CD VARCHAR(20) ,SIDE VARCHAR(20) ,LIMIT_PRICE DECIMAL(38, 18),STOP_PRICE DECIMAL(38, 18),ORDER_QTY DECIMAL(18, 4) , + TOTAL_EXECUTED_QTY DECIMAL(18, 4) ,AVG_PRICE DECIMAL(38, 18) ,DAY_EXECUTED_QTY DECIMAL(18, 4) ,DAY_AVG_PRICE DECIMAL(38, 18) , + REMNG_QTY DECIMAL(18, 4) ,CNCL_QTY DECIMAL(18, 4) ,CNCL_BY_FG VARCHAR(1) ,EXPIRE_TS TIMESTAMP ,EXEC_INSTR VARCHAR(64) , + TIME_IN_FORCE VARCHAR(20) ,RULE80AF VARCHAR(1) ,DEST_FIRMID_CD VARCHAR(20) ,SENT_TO_CONDUIT VARCHAR(20) ,SENT_TO_MPID VARCHAR(20) , + RCV_METHOD_CD VARCHAR(20) ,LIMIT_ORDER_DISP_IND VARCHAR(1) ,MERGED_ORDER_FG VARCHAR(1) ,MERGED_TO_ORDER_ID VARCHAR(64) , + RCV_DEPT_ID VARCHAR(20) ,ROUTE_METHOD_CD VARCHAR(20) ,LOCATE_ID VARCHAR(256) ,LOCATE_TS TIMESTAMP ,LOCATE_OVERRIDE_REASON VARCHAR(2000) , + LOCATE_BROKER VARCHAR(256) ,ORDER_BRCH_SEQ_TXT VARCHAR(20) ,IGNORE_CD VARCHAR(20) ,CLIENT_ORDER_REFID VARCHAR(64) , + CLIENT_ORDER_ORIG_REFID VARCHAR(64) ,ORDER_TYPE_CD VARCHAR(20) ,SENT_TO_ORDER_ID VARCHAR(64) ,ASK_PRICE DECIMAL(38, 18) , + ASK_QTY DECIMAL(18, 4) ,BID_PRICE DECIMAL(38, 18) ,BID_QTY DECIMAL(18, 4) ,REG_NMS_EXCEP_CD VARCHAR(20) ,REG_NMS_EXCEP_TXT VARCHAR(2000) , + REG_NMS_LINK_ID VARCHAR(64) ,REG_NMS_PRINTS VARCHAR(1) ,REG_NMS_STOP_TIME TIMESTAMP ,SENT_TS TIMESTAMP ,RULE92 VARCHAR(1) , + RULE92_OVERRIDE_TXT VARCHAR(2000) ,RULE92_RATIO DECIMAL(25, 10) ,EXMPT_STGY_BEGIN_TIME TIMESTAMP ,EXMPT_STGY_END_TIME TIMESTAMP , + EXMPT_STGY_PRICE_INST VARCHAR(2000) ,EXMPT_STGY_QTY DECIMAL(18, 4) ,CAPACITY VARCHAR(20) ,DISCRETION_QTY DECIMAL(18, 4) , + DISCRETION_PRICE VARCHAR(64) ,BRCHID_CD VARCHAR(20) ,BASKET_ORDER_ID VARCHAR(64) ,PT_STRTGY_CD VARCHAR(20) , + SETL_DATE VARCHAR(20),SETL_TYPE VARCHAR(20) ,SETL_CURR_CD VARCHAR(20) ,SETL_INSTRS VARCHAR(2000) ,COMMENT_TXT VARCHAR(2000) , + CHANNEL_NM VARCHAR(128) ,FLOW_CAT VARCHAR(20) ,FLOW_CLASS VARCHAR(20) ,FLOW_TGT VARCHAR(20) ,ORDER_FLOW_ENTRY VARCHAR(20) , + ORDER_FLOW_CHANNEL VARCHAR(20) ,ORDER_FLOW_DESK VARCHAR(20) ,FLOW_SUB_CAT VARCHAR(20) ,STRTGY_CD VARCHAR(20) ,RCVD_FROM_VENDOR VARCHAR(20) , + RCVD_FROM_CONDUIT VARCHAR(20) ,SLS_PERSON_ID VARCHAR(64) ,SYNTHETIC_FG VARCHAR(1) ,SYNTHETIC_TYPE VARCHAR(20) ,FXRT DECIMAL(25, 8) , + PARENT_CLREFID VARCHAR(64) ,REF_TIME_ID INTEGER ,OPT_CONTRACT_QTY DECIMAL(18, 4) ,OCEAN_PRODUCT_ID BIGINT ,CREATED_BY VARCHAR(64) , + CREATED_DATE TIMESTAMP ,FIRM_ACCT_ID BIGINT ,DEST VARCHAR(20) ,CNTRY_CD VARCHAR(20) ,DW_SINGLE_ORDER_CAT VARCHAR(20) ,CLIENT_ACCT_ID BIGINT , + EXTERNAL_TRDR_ID VARCHAR(64) ,ANONYMOUS_ORDER_FG VARCHAR(1) ,SYS_SECURITY_ALT_SRC VARCHAR(20) ,CURR_CD VARCHAR(20) , + EVENT_TYPE_CD VARCHAR(20) ,SYS_CLIENT_ACCT_ID VARCHAR(64) ,SYS_FIRM_ACCT_ID VARCHAR(20) ,SYS_TRDR_ID VARCHAR(64) ,DEST_ID INTEGER , + OPT_PUT_OR_CALL VARCHAR(20) ,SRC_FEED_REF_CD VARCHAR(64) ,DIGEST_KEY VARCHAR(128) ,EFF_TS TIMESTAMP ,ENTRY_TS TIMESTAMP , + OPT_STRIKE_PRICE DECIMAL(38, 18) ,OPT_MATURITY_DATE VARCHAR(20) ,ORDER_RESTR VARCHAR(4) ,SHORT_SELL_EXEMPT_CD VARCHAR(4) , + QUOTE_TIME TIMESTAMP ,SLS_CREDIT VARCHAR(20) ,SYS_SECURITY_ID VARCHAR(64) ,SYS_SECURITY_ID_SRC VARCHAR(20) ,SYS_SRC_SYS_ID VARCHAR(20) , + SYS_ORDER_ID_UNIQUE_SUFFIX VARCHAR(20) ,DEST_ID_SRC VARCHAR(4) ,GLB_ROOT_SRC_SYS_ID VARCHAR(20) ,GLB_ROOT_ORDER_ID_SUFFIX VARCHAR(64) , + SYS_ROOT_ORDER_ID_SUFFIX VARCHAR(20) ,SYS_PARENT_ORDER_ID_SUFFIX VARCHAR(20) ,CREDIT_BREACH_PERCENT DECIMAL(25, 10) , + CREDIT_BREACH_OVERRIDE VARCHAR(256) ,INFO_BARRIER_ID VARCHAR(256) ,EXCH_PARTICIPANT_ID VARCHAR(64) ,REJECT_REASON_CD VARCHAR(4) , + DIRECTED_DEST VARCHAR(20) ,REG_NMS_LINK_TYPE VARCHAR(20) ,CONVER_RATIO DECIMAL(18, 9) ,STOCK_REF_PRICE DECIMAL(38, 18) , + CB_SWAP_ORDER_FG VARCHAR(1) ,EV DECIMAL(38, 18) ,SYS_DATA_MODIFIED_TS TIMESTAMP ,CMSN_TYPE VARCHAR(20) ,SYS_CREDIT_TRDR_ID VARCHAR(20) , + SYS_ENTRY_USER_ID VARCHAR(20) ,OPEN_CLOSE_CD VARCHAR(20) ,AS_OF_TRD_FG VARCHAR(1) ,HANDLING_INSTR VARCHAR(20) ,SECURITY_DESC VARCHAR(512) , + MINIMUM_QTY DECIMAL(21, 6) ,CUST_OR_FIRM VARCHAR(20) ,MAXIMUM_SHOW DECIMAL(21, 6) ,SECURITY_SUB_TYPE VARCHAR(20) ,MULTILEG_RPT_TYPE VARCHAR(4) , + ORDER_ACTION_TYPE VARCHAR(4) ,BARRIER_STYLE VARCHAR(4) ,AUTO_IOI_REF_TYPE VARCHAR(4) ,PEG_OFFSET_VAL DECIMAL(10, 2) ,AUTO_IOI_OFFSET DECIMAL(28, 12) , + IOI_PRICE DECIMAL(28, 12) ,TGT_PRICE DECIMAL(28, 12) ,IOI_QTY VARCHAR(64) ,IOI_ORDER_QTY DECIMAL(18, 4) ,CMSN VARCHAR(64) ,SYS_LEG_REF_ID VARCHAR(64) , + TRADING_TYPE VARCHAR(4) ,EXCH_ORDER_ID VARCHAR(64) ,DEAL_ID VARCHAR(64) ,ORDER_TRD_TYPE VARCHAR(4) ,CXL_REASON VARCHAR(64)) + USING ROW OPTIONS (partition_by 'SINGLE_ORDER_DID', buckets '11',redundancy ':redundancy', EVICTION_BY ':evictionByOption', OVERFLOW 'true'); + +INSERT INTO ORDERS_DETAILS SELECT * FROM staging_orders_details; + +-- DROP TABLE IF ALREADY EXISTS -- +DROP TABLE IF EXISTS EXEC_DETAILS; +DROP TABLE IF EXISTS staging_exec_details; + +-- CREATE PARTITONED ROW TABLE EXEC_DETAILS -- +CREATE EXTERNAL TABLE staging_exec_details USING com.databricks.spark.csv + OPTIONS (path ':dataLocation/EXEC_DETAILS.dat', header 'true', inferSchema 'false', nullValue 'NULL'); + +CREATE TABLE EXEC_DETAILS + (EXEC_DID BIGINT,SYS_EXEC_VER INTEGER,SYS_EXEC_ID VARCHAR(64),TRD_DATE VARCHAR(20),ALT_EXEC_ID VARCHAR(64),SYS_EXEC_STAT VARCHAR(20), + DW_EXEC_STAT VARCHAR(20),ORDER_OWNER_FIRM_ID VARCHAR(20),TRDR_SYS_LOGON_ID VARCHAR(64),CONTRA_BROKER_MNEMONIC VARCHAR(20),SIDE VARCHAR(20), + TICKER_SYMBOL VARCHAR(32),SYS_SECURITY_ALT_ID VARCHAR(64),PRODUCT_CAT_CD VARCHAR(20),LAST_MKT VARCHAR(20),EXECUTED_QTY DECIMAL(18, 4), + EXEC_PRICE DECIMAL( 38, 18),EXEC_PRICE_CURR_CD VARCHAR(20),EXEC_CAPACITY VARCHAR(20),CLIENT_ACCT_ID BIGINT,FIRM_ACCT_ID BIGINT, + AVG_PRICE_ACCT_ID BIGINT,OCEAN_ACCT_ID BIGINT,EXEC_CNTRY_CD VARCHAR(20),CMSN VARCHAR(20),COMMENT_TXT VARCHAR(2000), + ACT_BRCH_SEQ_TXT VARCHAR(20),IGNORE_CD VARCHAR(20),SRC_SYS VARCHAR(20),EXEC_TYPE_CD VARCHAR(20),LIQUIDITY_CD VARCHAR(20), + ASK_PRICE DECIMAL( 38, 18),ASK_QTY DECIMAL(18, 4),TRD_REPORT_ASOF_DATE VARCHAR(20),BID_PRICE DECIMAL( 38, 18),BID_QTY DECIMAL(18, 4), + CROSS_ID VARCHAR(64),NYSE_SUBREPORT_TYPE VARCHAR(20),QUOTE_COORDINATOR VARCHAR(20),QUOTE_TIME TIMESTAMP,REG_NMS_EXCEPT_CD VARCHAR(20), + REG_NMS_EXCEPT_TXT VARCHAR(2000),REG_NMS_LINK_ID VARCHAR(64),REG_NMS_MKT_CENTER_ID VARCHAR(64),REG_NMS_OVERRIDE VARCHAR(20),REG_NMS_PRINTS VARCHAR(1), + EXECUTED_BY VARCHAR(20),TICKER_SYMBOL_SUFFIX VARCHAR(20),PREREGNMS_TRD_MOD1 VARCHAR(1),PREREGNMS_TRD_MOD2 VARCHAR(1),PREREGNMS_TRD_MOD3 VARCHAR(1), + PREREGNMS_TRD_MOD4 VARCHAR(1),NMS_FG VARCHAR(1),GIVEUP_BROKER VARCHAR(20),CHANNEL_NM VARCHAR(128),ORDER_FLOW_ENTRY VARCHAR(20),FLOW_CAT VARCHAR(20), + FLOW_CLASS VARCHAR(20),FLOW_TGT VARCHAR(20),ORDER_FLOW_CHANNEL VARCHAR(20),FLOW_SUBCAT VARCHAR(20),SYS_ACCT_ID_SRC VARCHAR(64),STRTGY_CD VARCHAR(20), + EXECUTING_BROKER_CD VARCHAR(20),LEAF_EXEC_FG VARCHAR(1),RCVD_EXEC_ID VARCHAR(64),RCVD_EXEC_VER INTEGER,ORDER_FLOW_DESK VARCHAR(20), + SYS_ROOT_ORDER_ID VARCHAR(64),SYS_ROOT_ORDER_VER INTEGER,GLB_ROOT_ORDER_ID VARCHAR(64),TOTAL_EXECUTED_QTY DECIMAL(18, 4),AVG_PRICE DECIMAL( 38, 18), + DEST_CD VARCHAR(20),CLIENT_ORDER_REFID VARCHAR(64),CLIENT_ORDER_ORIG_REFID VARCHAR(64),CROSS_EXEC_FG VARCHAR(1),OCEAN_PRODUCT_ID BIGINT, + TRDR_ID BIGINT,REF_TIME_ID INTEGER,CREATED_BY VARCHAR(64),CREATED_DATE TIMESTAMP,FIX_EXEC_ID VARCHAR(64),FIX_ORIGINAL_EXEC_ID VARCHAR(64), + RELATED_MKT_CENTER VARCHAR(20),TRANS_TS TIMESTAMP,SYS_SECURITY_ALT_SRC VARCHAR(20),EVENT_TYPE_CD VARCHAR(20),SYS_CLIENT_ACCT_ID VARCHAR(64), + SYS_FIRM_ACCT_ID VARCHAR(20),SYS_AVG_PRICE_ACCT_ID VARCHAR(20),SYS_TRDR_ID VARCHAR(64),ACT_BRCH_SEQ VARCHAR(20),SYS_ORDER_ID VARCHAR(64), + SYS_ORDER_VER INTEGER,SRC_FEED_REF_CD VARCHAR(64),DIGEST_KEY VARCHAR(128),TRUE_LAST_MKT VARCHAR(20),ENTRY_TS TIMESTAMP,OPT_STRIKE_PRICE DECIMAL( 38, 18), + OPT_MATURITY_DATE VARCHAR(20),EXPIRE_TS TIMESTAMP,OPT_PUT_OR_CALL VARCHAR(20),SYS_ORDER_STAT_CD VARCHAR(20),CONTRA_ACCT VARCHAR(64),CONTRA_ACCT_SRC VARCHAR(20), + CONTRA_BROKER_SRC VARCHAR(20),SYS_SECURITY_ID VARCHAR(64),SYS_SECURITY_ID_SRC VARCHAR(20),SYS_SRC_SYS_ID VARCHAR(20),SYS_ORDER_ID_UNIQUE_SUFFIX VARCHAR(20), + DEST VARCHAR(20),DEST_ID_SRC VARCHAR(4),CONVER_RATIO DECIMAL(18, 9),STOCK_REF_PRICE DECIMAL( 38, 18),AS_OF_TRD_FG VARCHAR(1),MULTILEG_RPT_TYPE VARCHAR(4), + REG_NMS_LINK_TYPE VARCHAR(20),EXEC_SUB_TYPE VARCHAR(4),CMSN_TYPE VARCHAR(20),QUOTE_CONDITION_IND VARCHAR(20),TRD_THROUGH_FG VARCHAR(1), + REGNMS_ORDER_LINK_ID VARCHAR(64),REGNMS_ORDER_LINK_TYPE VARCHAR(20),DK_IND VARCHAR(20),NBBO_QUOTE_TIME VARCHAR(20),GLB_ROOT_SRC_SYS_ID VARCHAR(20), + TRD_REPORT_TYPE VARCHAR(20),REPORT_TO_EXCH_FG VARCHAR(1),CMPLN_COMMENT VARCHAR(256),DEAL_TYPE VARCHAR(4),EXEC_COMMENTS VARCHAR(256), + OPTAL_FIELDS VARCHAR(120),SPOT_REF_PRICE VARCHAR(20),DELTA_OVERRIDE VARCHAR(20),UNDERLYING_PRICE VARCHAR(20),PRICE_DELTA VARCHAR(20), + NORMALIZED_LIQUIDITY_IND VARCHAR(4),USER_AVG_PRICE VARCHAR(20),LAST_EXEC_TS TIMESTAMP,LULD_LOWER_PRICE_BAND VARCHAR(20),LULD_UPPER_PRICE_BAND VARCHAR(20), + LULD_PRICE_BAND_TS TIMESTAMP,REMNG_QTY DECIMAL(18, 4),ORDER_QTY DECIMAL(18, 4),AMD_TS TIMESTAMP,SETL_CODE VARCHAR(50),SETL_DATE VARCHAR(20), + CUST_NM VARCHAR(50),EXEC_TYPE VARCHAR(50),TRDR_KEY VARCHAR(50),TRDR_NM VARCHAR(50),FX_RATE VARCHAR(50),CUST_FX_RATE VARCHAR(50), + PARENT_ORDER_SYS_NM VARCHAR(10),CNC_TYPE VARCHAR(50),FEE_AMT DECIMAL(20, 2),FEE_CCY VARCHAR(10),BRKG_AMT DECIMAL(20, 2),BRKG_CCY VARCHAR(10), + CLEAR VARCHAR(50),PMT_FIX_DATE VARCHAR(20),FOLLOW_ON_FG VARCHAR(1),FX_RATE_CCY_TO VARCHAR(10),FX_RATE_CCY_FROM VARCHAR(10),CUST_FX_RATE_CCY_TO VARCHAR(10), + CUST_FX_RATE_CCY_FROM VARCHAR(10),SYS_GFCID VARCHAR(20),CONTRA_SIDE VARCHAR(20),OPT_CONTRACT_MULTIPLIER DECIMAL(10, 2),PRIOR_REF_PRICE_TS TIMESTAMP, + SECURITY_SUB_TYPE VARCHAR(20),MSG_DIRECTION VARCHAR(20),LEAF_SYS_EXEC_ID VARCHAR(64),LEAF_SRC_SYS VARCHAR(20),FIX_LAST_MKT VARCHAR(20), + FIX_CONTRA_BROKER_MNEMONIC VARCHAR(20),RIO_MSG_SRC VARCHAR(64),SNAPSHOT_TS TIMESTAMP,EXTERNAL_TRANS_TS TIMESTAMP,PRICE_CATEGORY VARCHAR(32), + UNDERLYING_FX_RATE DECIMAL(36, 8),CONVERSION_RATE DECIMAL(36, 8),TRANS_COMMENT VARCHAR(256),AGGRESSOR_FLAG VARCHAR(1)) + USING ROW OPTIONS (partition_by 'EXEC_DID', buckets '11', redundancy ':redundancy', EVICTION_BY ':evictionByOption', OVERFLOW 'true') ; + +INSERT INTO EXEC_DETAILS SELECT * FROM staging_exec_details; + diff --git a/dtests/src/resources/scripts/ct/ct_create_and_load_persistentEvictionOverflow_row_table.sql b/dtests/src/resources/scripts/ct/ct_create_and_load_persistentEvictionOverflow_row_table.sql new file mode 100644 index 0000000000..13a8dbc865 --- /dev/null +++ b/dtests/src/resources/scripts/ct/ct_create_and_load_persistentEvictionOverflow_row_table.sql @@ -0,0 +1,111 @@ +DROP DISKSTORE IF EXISTS OverflowDiskStore; +CREATE DISKSTORE OverflowDiskStore; + +-- DROP TABLE IF ALREADY EXISTS -- +DROP TABLE IF EXISTS ORDERS_DETAILS; +DROP TABLE IF EXISTS staging_orders_details; + +-- CREATE PERSISTENT COLOCATED ROW TABLE ORDERS_DETAILS -- +CREATE EXTERNAL TABLE staging_orders_details USING com.databricks.spark.csv + OPTIONS (path ':dataLocation/ORDERS_DETAILS.dat', header 'true', inferSchema 'false', + nullValue 'NULL'); + +CREATE TABLE ORDERS_DETAILS + (SINGLE_ORDER_DID BIGINT ,SYS_ORDER_ID VARCHAR(64) ,SYS_ORDER_VER INTEGER ,DATA_SNDG_SYS_NM VARCHAR(128) , + SRC_SYS VARCHAR(20) ,SYS_PARENT_ORDER_ID VARCHAR(64) ,SYS_PARENT_ORDER_VER SMALLINT ,PARENT_ORDER_TRD_DATE VARCHAR(20), + PARENT_ORDER_SYS_NM VARCHAR(128) ,SYS_ALT_ORDER_ID VARCHAR(64) ,TRD_DATE VARCHAR(20),GIVE_UP_BROKER VARCHAR(20) , + EVENT_RCV_TS TIMESTAMP ,SYS_ROOT_ORDER_ID VARCHAR(64) ,GLB_ROOT_ORDER_ID VARCHAR(64) ,GLB_ROOT_ORDER_SYS_NM VARCHAR(128) , + GLB_ROOT_ORDER_RCV_TS TIMESTAMP ,SYS_ORDER_STAT_CD VARCHAR(20) ,SYS_ORDER_STAT_DESC_TXT VARCHAR(120) ,DW_STAT_CD VARCHAR(20) , + EVENT_TS TIMESTAMP,ORDER_OWNER_FIRM_ID VARCHAR(20),RCVD_ORDER_ID VARCHAR(64) ,EVENT_INITIATOR_ID VARCHAR(64), + TRDR_SYS_LOGON_ID VARCHAR(64),SOLICITED_FG VARCHAR(1),RCVD_FROM_FIRMID_CD VARCHAR(20),RCV_DESK VARCHAR(20), + SYS_ACCT_ID_SRC VARCHAR(64) ,CUST_ACCT_MNEMONIC VARCHAR(128),CUST_SLANG VARCHAR(20) ,SYS_ACCT_TYPE VARCHAR(20) , + CUST_EXCH_ACCT_ID VARCHAR(64) ,SYS_SECURITY_ALT_ID VARCHAR(64) ,TICKER_SYMBOL VARCHAR(32) ,TICKER_SYMBOL_SUFFIX VARCHAR(20) , + PRODUCT_CAT_CD VARCHAR(20) ,SIDE VARCHAR(20) ,LIMIT_PRICE DECIMAL(38, 18),STOP_PRICE DECIMAL(38, 18),ORDER_QTY DECIMAL(18, 4) , + TOTAL_EXECUTED_QTY DECIMAL(18, 4) ,AVG_PRICE DECIMAL(38, 18) ,DAY_EXECUTED_QTY DECIMAL(18, 4) ,DAY_AVG_PRICE DECIMAL(38, 18) , + REMNG_QTY DECIMAL(18, 4) ,CNCL_QTY DECIMAL(18, 4) ,CNCL_BY_FG VARCHAR(1) ,EXPIRE_TS TIMESTAMP ,EXEC_INSTR VARCHAR(64) , + TIME_IN_FORCE VARCHAR(20) ,RULE80AF VARCHAR(1) ,DEST_FIRMID_CD VARCHAR(20) ,SENT_TO_CONDUIT VARCHAR(20) ,SENT_TO_MPID VARCHAR(20) , + RCV_METHOD_CD VARCHAR(20) ,LIMIT_ORDER_DISP_IND VARCHAR(1) ,MERGED_ORDER_FG VARCHAR(1) ,MERGED_TO_ORDER_ID VARCHAR(64) , + RCV_DEPT_ID VARCHAR(20) ,ROUTE_METHOD_CD VARCHAR(20) ,LOCATE_ID VARCHAR(256) ,LOCATE_TS TIMESTAMP ,LOCATE_OVERRIDE_REASON VARCHAR(2000) , + LOCATE_BROKER VARCHAR(256) ,ORDER_BRCH_SEQ_TXT VARCHAR(20) ,IGNORE_CD VARCHAR(20) ,CLIENT_ORDER_REFID VARCHAR(64) , + CLIENT_ORDER_ORIG_REFID VARCHAR(64) ,ORDER_TYPE_CD VARCHAR(20) ,SENT_TO_ORDER_ID VARCHAR(64) ,ASK_PRICE DECIMAL(38, 18) , + ASK_QTY DECIMAL(18, 4) ,BID_PRICE DECIMAL(38, 18) ,BID_QTY DECIMAL(18, 4) ,REG_NMS_EXCEP_CD VARCHAR(20) ,REG_NMS_EXCEP_TXT VARCHAR(2000) , + REG_NMS_LINK_ID VARCHAR(64) ,REG_NMS_PRINTS VARCHAR(1) ,REG_NMS_STOP_TIME TIMESTAMP ,SENT_TS TIMESTAMP ,RULE92 VARCHAR(1) , + RULE92_OVERRIDE_TXT VARCHAR(2000) ,RULE92_RATIO DECIMAL(25, 10) ,EXMPT_STGY_BEGIN_TIME TIMESTAMP ,EXMPT_STGY_END_TIME TIMESTAMP , + EXMPT_STGY_PRICE_INST VARCHAR(2000) ,EXMPT_STGY_QTY DECIMAL(18, 4) ,CAPACITY VARCHAR(20) ,DISCRETION_QTY DECIMAL(18, 4) , + DISCRETION_PRICE VARCHAR(64) ,BRCHID_CD VARCHAR(20) ,BASKET_ORDER_ID VARCHAR(64) ,PT_STRTGY_CD VARCHAR(20) , + SETL_DATE VARCHAR(20),SETL_TYPE VARCHAR(20) ,SETL_CURR_CD VARCHAR(20) ,SETL_INSTRS VARCHAR(2000) ,COMMENT_TXT VARCHAR(2000) , + CHANNEL_NM VARCHAR(128) ,FLOW_CAT VARCHAR(20) ,FLOW_CLASS VARCHAR(20) ,FLOW_TGT VARCHAR(20) ,ORDER_FLOW_ENTRY VARCHAR(20) , + ORDER_FLOW_CHANNEL VARCHAR(20) ,ORDER_FLOW_DESK VARCHAR(20) ,FLOW_SUB_CAT VARCHAR(20) ,STRTGY_CD VARCHAR(20) ,RCVD_FROM_VENDOR VARCHAR(20) , + RCVD_FROM_CONDUIT VARCHAR(20) ,SLS_PERSON_ID VARCHAR(64) ,SYNTHETIC_FG VARCHAR(1) ,SYNTHETIC_TYPE VARCHAR(20) ,FXRT DECIMAL(25, 8) , + PARENT_CLREFID VARCHAR(64) ,REF_TIME_ID INTEGER ,OPT_CONTRACT_QTY DECIMAL(18, 4) ,OCEAN_PRODUCT_ID BIGINT ,CREATED_BY VARCHAR(64) , + CREATED_DATE TIMESTAMP ,FIRM_ACCT_ID BIGINT ,DEST VARCHAR(20) ,CNTRY_CD VARCHAR(20) ,DW_SINGLE_ORDER_CAT VARCHAR(20) ,CLIENT_ACCT_ID BIGINT , + EXTERNAL_TRDR_ID VARCHAR(64) ,ANONYMOUS_ORDER_FG VARCHAR(1) ,SYS_SECURITY_ALT_SRC VARCHAR(20) ,CURR_CD VARCHAR(20) , + EVENT_TYPE_CD VARCHAR(20) ,SYS_CLIENT_ACCT_ID VARCHAR(64) ,SYS_FIRM_ACCT_ID VARCHAR(20) ,SYS_TRDR_ID VARCHAR(64) ,DEST_ID INTEGER , + OPT_PUT_OR_CALL VARCHAR(20) ,SRC_FEED_REF_CD VARCHAR(64) ,DIGEST_KEY VARCHAR(128) ,EFF_TS TIMESTAMP ,ENTRY_TS TIMESTAMP , + OPT_STRIKE_PRICE DECIMAL(38, 18) ,OPT_MATURITY_DATE VARCHAR(20) ,ORDER_RESTR VARCHAR(4) ,SHORT_SELL_EXEMPT_CD VARCHAR(4) , + QUOTE_TIME TIMESTAMP ,SLS_CREDIT VARCHAR(20) ,SYS_SECURITY_ID VARCHAR(64) ,SYS_SECURITY_ID_SRC VARCHAR(20) ,SYS_SRC_SYS_ID VARCHAR(20) , + SYS_ORDER_ID_UNIQUE_SUFFIX VARCHAR(20) ,DEST_ID_SRC VARCHAR(4) ,GLB_ROOT_SRC_SYS_ID VARCHAR(20) ,GLB_ROOT_ORDER_ID_SUFFIX VARCHAR(64) , + SYS_ROOT_ORDER_ID_SUFFIX VARCHAR(20) ,SYS_PARENT_ORDER_ID_SUFFIX VARCHAR(20) ,CREDIT_BREACH_PERCENT DECIMAL(25, 10) , + CREDIT_BREACH_OVERRIDE VARCHAR(256) ,INFO_BARRIER_ID VARCHAR(256) ,EXCH_PARTICIPANT_ID VARCHAR(64) ,REJECT_REASON_CD VARCHAR(4) , + DIRECTED_DEST VARCHAR(20) ,REG_NMS_LINK_TYPE VARCHAR(20) ,CONVER_RATIO DECIMAL(18, 9) ,STOCK_REF_PRICE DECIMAL(38, 18) , + CB_SWAP_ORDER_FG VARCHAR(1) ,EV DECIMAL(38, 18) ,SYS_DATA_MODIFIED_TS TIMESTAMP ,CMSN_TYPE VARCHAR(20) ,SYS_CREDIT_TRDR_ID VARCHAR(20) , + SYS_ENTRY_USER_ID VARCHAR(20) ,OPEN_CLOSE_CD VARCHAR(20) ,AS_OF_TRD_FG VARCHAR(1) ,HANDLING_INSTR VARCHAR(20) ,SECURITY_DESC VARCHAR(512) , + MINIMUM_QTY DECIMAL(21, 6) ,CUST_OR_FIRM VARCHAR(20) ,MAXIMUM_SHOW DECIMAL(21, 6) ,SECURITY_SUB_TYPE VARCHAR(20) ,MULTILEG_RPT_TYPE VARCHAR(4) , + ORDER_ACTION_TYPE VARCHAR(4) ,BARRIER_STYLE VARCHAR(4) ,AUTO_IOI_REF_TYPE VARCHAR(4) ,PEG_OFFSET_VAL DECIMAL(10, 2) ,AUTO_IOI_OFFSET DECIMAL(28, 12) , + IOI_PRICE DECIMAL(28, 12) ,TGT_PRICE DECIMAL(28, 12) ,IOI_QTY VARCHAR(64) ,IOI_ORDER_QTY DECIMAL(18, 4) ,CMSN VARCHAR(64) ,SYS_LEG_REF_ID VARCHAR(64) , + TRADING_TYPE VARCHAR(4) ,EXCH_ORDER_ID VARCHAR(64) ,DEAL_ID VARCHAR(64) ,ORDER_TRD_TYPE VARCHAR(4) ,CXL_REASON VARCHAR(64)) + USING ROW OPTIONS (partition_by 'SINGLE_ORDER_DID', buckets '11', PERSISTENT ':persistenceMode', + redundancy ':redundancy', EVICTION_BY ':evictionByOption', OVERFLOW 'true', diskstore 'OverflowDiskStore'); + +INSERT INTO ORDERS_DETAILS SELECT * FROM staging_orders_details; + +-- DROP TABLE IF ALREADY EXISTS -- +DROP TABLE IF EXISTS EXEC_DETAILS; +DROP TABLE IF EXISTS staging_exec_details; + +-- CREATE PERSISTENT COLCOATED ROW TABLE EXEC_DETAILS -- +CREATE EXTERNAL TABLE staging_exec_details USING com.databricks.spark.csv + OPTIONS (path ':dataLocation/EXEC_DETAILS.dat', header 'true', inferSchema 'false', nullValue 'NULL'); + +CREATE TABLE EXEC_DETAILS + (EXEC_DID BIGINT,SYS_EXEC_VER INTEGER,SYS_EXEC_ID VARCHAR(64),TRD_DATE VARCHAR(20),ALT_EXEC_ID VARCHAR(64),SYS_EXEC_STAT VARCHAR(20), + DW_EXEC_STAT VARCHAR(20),ORDER_OWNER_FIRM_ID VARCHAR(20),TRDR_SYS_LOGON_ID VARCHAR(64),CONTRA_BROKER_MNEMONIC VARCHAR(20),SIDE VARCHAR(20), + TICKER_SYMBOL VARCHAR(32),SYS_SECURITY_ALT_ID VARCHAR(64),PRODUCT_CAT_CD VARCHAR(20),LAST_MKT VARCHAR(20),EXECUTED_QTY DECIMAL(18, 4), + EXEC_PRICE DECIMAL( 38, 18),EXEC_PRICE_CURR_CD VARCHAR(20),EXEC_CAPACITY VARCHAR(20),CLIENT_ACCT_ID BIGINT,FIRM_ACCT_ID BIGINT, + AVG_PRICE_ACCT_ID BIGINT,OCEAN_ACCT_ID BIGINT,EXEC_CNTRY_CD VARCHAR(20),CMSN VARCHAR(20),COMMENT_TXT VARCHAR(2000), + ACT_BRCH_SEQ_TXT VARCHAR(20),IGNORE_CD VARCHAR(20),SRC_SYS VARCHAR(20),EXEC_TYPE_CD VARCHAR(20),LIQUIDITY_CD VARCHAR(20), + ASK_PRICE DECIMAL( 38, 18),ASK_QTY DECIMAL(18, 4),TRD_REPORT_ASOF_DATE VARCHAR(20),BID_PRICE DECIMAL( 38, 18),BID_QTY DECIMAL(18, 4), + CROSS_ID VARCHAR(64),NYSE_SUBREPORT_TYPE VARCHAR(20),QUOTE_COORDINATOR VARCHAR(20),QUOTE_TIME TIMESTAMP,REG_NMS_EXCEPT_CD VARCHAR(20), + REG_NMS_EXCEPT_TXT VARCHAR(2000),REG_NMS_LINK_ID VARCHAR(64),REG_NMS_MKT_CENTER_ID VARCHAR(64),REG_NMS_OVERRIDE VARCHAR(20),REG_NMS_PRINTS VARCHAR(1), + EXECUTED_BY VARCHAR(20),TICKER_SYMBOL_SUFFIX VARCHAR(20),PREREGNMS_TRD_MOD1 VARCHAR(1),PREREGNMS_TRD_MOD2 VARCHAR(1),PREREGNMS_TRD_MOD3 VARCHAR(1), + PREREGNMS_TRD_MOD4 VARCHAR(1),NMS_FG VARCHAR(1),GIVEUP_BROKER VARCHAR(20),CHANNEL_NM VARCHAR(128),ORDER_FLOW_ENTRY VARCHAR(20),FLOW_CAT VARCHAR(20), + FLOW_CLASS VARCHAR(20),FLOW_TGT VARCHAR(20),ORDER_FLOW_CHANNEL VARCHAR(20),FLOW_SUBCAT VARCHAR(20),SYS_ACCT_ID_SRC VARCHAR(64),STRTGY_CD VARCHAR(20), + EXECUTING_BROKER_CD VARCHAR(20),LEAF_EXEC_FG VARCHAR(1),RCVD_EXEC_ID VARCHAR(64),RCVD_EXEC_VER INTEGER,ORDER_FLOW_DESK VARCHAR(20), + SYS_ROOT_ORDER_ID VARCHAR(64),SYS_ROOT_ORDER_VER INTEGER,GLB_ROOT_ORDER_ID VARCHAR(64),TOTAL_EXECUTED_QTY DECIMAL(18, 4),AVG_PRICE DECIMAL( 38, 18), + DEST_CD VARCHAR(20),CLIENT_ORDER_REFID VARCHAR(64),CLIENT_ORDER_ORIG_REFID VARCHAR(64),CROSS_EXEC_FG VARCHAR(1),OCEAN_PRODUCT_ID BIGINT, + TRDR_ID BIGINT,REF_TIME_ID INTEGER,CREATED_BY VARCHAR(64),CREATED_DATE TIMESTAMP,FIX_EXEC_ID VARCHAR(64),FIX_ORIGINAL_EXEC_ID VARCHAR(64), + RELATED_MKT_CENTER VARCHAR(20),TRANS_TS TIMESTAMP,SYS_SECURITY_ALT_SRC VARCHAR(20),EVENT_TYPE_CD VARCHAR(20),SYS_CLIENT_ACCT_ID VARCHAR(64), + SYS_FIRM_ACCT_ID VARCHAR(20),SYS_AVG_PRICE_ACCT_ID VARCHAR(20),SYS_TRDR_ID VARCHAR(64),ACT_BRCH_SEQ VARCHAR(20),SYS_ORDER_ID VARCHAR(64), + SYS_ORDER_VER INTEGER,SRC_FEED_REF_CD VARCHAR(64),DIGEST_KEY VARCHAR(128),TRUE_LAST_MKT VARCHAR(20),ENTRY_TS TIMESTAMP,OPT_STRIKE_PRICE DECIMAL( 38, 18), + OPT_MATURITY_DATE VARCHAR(20),EXPIRE_TS TIMESTAMP,OPT_PUT_OR_CALL VARCHAR(20),SYS_ORDER_STAT_CD VARCHAR(20),CONTRA_ACCT VARCHAR(64),CONTRA_ACCT_SRC VARCHAR(20), + CONTRA_BROKER_SRC VARCHAR(20),SYS_SECURITY_ID VARCHAR(64),SYS_SECURITY_ID_SRC VARCHAR(20),SYS_SRC_SYS_ID VARCHAR(20),SYS_ORDER_ID_UNIQUE_SUFFIX VARCHAR(20), + DEST VARCHAR(20),DEST_ID_SRC VARCHAR(4),CONVER_RATIO DECIMAL(18, 9),STOCK_REF_PRICE DECIMAL( 38, 18),AS_OF_TRD_FG VARCHAR(1),MULTILEG_RPT_TYPE VARCHAR(4), + REG_NMS_LINK_TYPE VARCHAR(20),EXEC_SUB_TYPE VARCHAR(4),CMSN_TYPE VARCHAR(20),QUOTE_CONDITION_IND VARCHAR(20),TRD_THROUGH_FG VARCHAR(1), + REGNMS_ORDER_LINK_ID VARCHAR(64),REGNMS_ORDER_LINK_TYPE VARCHAR(20),DK_IND VARCHAR(20),NBBO_QUOTE_TIME VARCHAR(20),GLB_ROOT_SRC_SYS_ID VARCHAR(20), + TRD_REPORT_TYPE VARCHAR(20),REPORT_TO_EXCH_FG VARCHAR(1),CMPLN_COMMENT VARCHAR(256),DEAL_TYPE VARCHAR(4),EXEC_COMMENTS VARCHAR(256), + OPTAL_FIELDS VARCHAR(120),SPOT_REF_PRICE VARCHAR(20),DELTA_OVERRIDE VARCHAR(20),UNDERLYING_PRICE VARCHAR(20),PRICE_DELTA VARCHAR(20), + NORMALIZED_LIQUIDITY_IND VARCHAR(4),USER_AVG_PRICE VARCHAR(20),LAST_EXEC_TS TIMESTAMP,LULD_LOWER_PRICE_BAND VARCHAR(20),LULD_UPPER_PRICE_BAND VARCHAR(20), + LULD_PRICE_BAND_TS TIMESTAMP,REMNG_QTY DECIMAL(18, 4),ORDER_QTY DECIMAL(18, 4),AMD_TS TIMESTAMP,SETL_CODE VARCHAR(50),SETL_DATE VARCHAR(20), + CUST_NM VARCHAR(50),EXEC_TYPE VARCHAR(50),TRDR_KEY VARCHAR(50),TRDR_NM VARCHAR(50),FX_RATE VARCHAR(50),CUST_FX_RATE VARCHAR(50), + PARENT_ORDER_SYS_NM VARCHAR(10),CNC_TYPE VARCHAR(50),FEE_AMT DECIMAL(20, 2),FEE_CCY VARCHAR(10),BRKG_AMT DECIMAL(20, 2),BRKG_CCY VARCHAR(10), + CLEAR VARCHAR(50),PMT_FIX_DATE VARCHAR(20),FOLLOW_ON_FG VARCHAR(1),FX_RATE_CCY_TO VARCHAR(10),FX_RATE_CCY_FROM VARCHAR(10),CUST_FX_RATE_CCY_TO VARCHAR(10), + CUST_FX_RATE_CCY_FROM VARCHAR(10),SYS_GFCID VARCHAR(20),CONTRA_SIDE VARCHAR(20),OPT_CONTRACT_MULTIPLIER DECIMAL(10, 2),PRIOR_REF_PRICE_TS TIMESTAMP, + SECURITY_SUB_TYPE VARCHAR(20),MSG_DIRECTION VARCHAR(20),LEAF_SYS_EXEC_ID VARCHAR(64),LEAF_SRC_SYS VARCHAR(20),FIX_LAST_MKT VARCHAR(20), + FIX_CONTRA_BROKER_MNEMONIC VARCHAR(20),RIO_MSG_SRC VARCHAR(64),SNAPSHOT_TS TIMESTAMP,EXTERNAL_TRANS_TS TIMESTAMP,PRICE_CATEGORY VARCHAR(32), + UNDERLYING_FX_RATE DECIMAL(36, 8),CONVERSION_RATE DECIMAL(36, 8),TRANS_COMMENT VARCHAR(256),AGGRESSOR_FLAG VARCHAR(1)) + USING ROW OPTIONS (partition_by 'EXEC_DID', buckets '11', PERSISTENT ':persistenceMode', + redundancy ':redundancy', EVICTION_BY ':evictionByOption', OVERFLOW 'true', diskstore 'OverflowDiskStore'); + +INSERT INTO EXEC_DETAILS SELECT * FROM staging_exec_details; + + diff --git a/dtests/src/test/java/io/snappydata/hydra/adAnalytics/SnappyAdAnalyticsTest.java b/dtests/src/test/java/io/snappydata/hydra/adAnalytics/SnappyAdAnalyticsTest.java index 1f0bfff09d..8d9d456d69 100644 --- a/dtests/src/test/java/io/snappydata/hydra/adAnalytics/SnappyAdAnalyticsTest.java +++ b/dtests/src/test/java/io/snappydata/hydra/adAnalytics/SnappyAdAnalyticsTest.java @@ -379,6 +379,10 @@ protected void executeSnappyStreamingJob(Vector jobClassNames, String logFileNam throw new TestException("Got Exception while executing streaming job. Please check " + "the job status output."); } + try { + Thread.sleep(10 * 1000); + } catch (InterruptedException ie) { + } } } } @@ -576,8 +580,8 @@ public boolean getJobStatus(String jobID){ leadHost = getLeadHost(); String leadPort = (String) SnappyBB.getBB().getSharedMap().get("primaryLeadPort"); try { - String dest = getCurrentDirPath() + File.separator + "jobStatus_" + RemoteTestModule - .getCurrentThread().getThreadId() + "_" + System.currentTimeMillis() + ".log"; + String dest = getCurrentDirPath() + File.separator + "jobStatus_" + getMyTid() + "_" + jobID + + ".log"; File commandOutput = new File(dest); String command = snappyJobScript + " status --lead " + leadHost + ":" + leadPort + " " + "--job-id " + jobID + " > " + commandOutput; @@ -591,7 +595,7 @@ public boolean getJobStatus(String jobID){ return false; break; } - } try { Thread.sleep(10*1000);} catch(InterruptedException ie) { } + } } catch (IOException ie){ Log.getLogWriter().info("Got exception while accessing current dir"); } diff --git a/dtests/src/test/java/io/snappydata/hydra/adAnalytics/adAnalytics.conf b/dtests/src/test/java/io/snappydata/hydra/adAnalytics/adAnalytics.conf index b35ac416c2..ed2676564d 100644 --- a/dtests/src/test/java/io/snappydata/hydra/adAnalytics/adAnalytics.conf +++ b/dtests/src/test/java/io/snappydata/hydra/adAnalytics/adAnalytics.conf @@ -120,9 +120,8 @@ CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = Hy io.snappydata.hydra.cluster.SnappyPrms-sqlScriptNames = adAnalytics_queries.sql threadGroups = snappyThreads; -/*ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_dumpStacks +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_dumpStacks clientNames = locator1; -*/ ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_stopSnappyCluster clientNames = locator1; diff --git a/dtests/src/test/java/io/snappydata/hydra/adAnalytics/adAnalyticsRestartStreaming.conf b/dtests/src/test/java/io/snappydata/hydra/adAnalytics/adAnalyticsRestartStreaming.conf index cfbc0deb00..158ab7e706 100644 --- a/dtests/src/test/java/io/snappydata/hydra/adAnalytics/adAnalyticsRestartStreaming.conf +++ b/dtests/src/test/java/io/snappydata/hydra/adAnalytics/adAnalyticsRestartStreaming.conf @@ -31,4 +31,9 @@ CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = Hy io.snappydata.hydra.cluster.SnappyPrms-sqlScriptNames = adAnalytics_queries.sql threadGroups = snappyThreads; +CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSQLScripts + io.snappydata.hydra.cluster.SnappyPrms-logFileName = sqlScriptResults1.log + io.snappydata.hydra.cluster.SnappyPrms-sqlScriptNames = adAnalytics_queries.sql + threadGroups = snappyThreads; + INCLUDE $JTESTS/io/snappydata/hydra/adAnalytics/adAnalytics.conf; diff --git a/dtests/src/test/java/io/snappydata/hydra/alterTable/alterTable.bt b/dtests/src/test/java/io/snappydata/hydra/alterTable/alterTable.bt index 0aa4ccb900..c4614d2217 100644 --- a/dtests/src/test/java/io/snappydata/hydra/alterTable/alterTable.bt +++ b/dtests/src/test/java/io/snappydata/hydra/alterTable/alterTable.bt @@ -1,7 +1,7 @@ io/snappydata/hydra/alterTable/nwAlterTablesTestUsingSnappyJob.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="PartitionedRow","ReplicatedRow" @@ -13,7 +13,7 @@ io/snappydata/hydra/alterTable/nwAlterTablesTestUsingSnappyJob.conf io/snappydata/hydra/alterTable/nwAlterPartitionedRowTablesTestUsingSnappyShell.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="PartitionedRow" @@ -24,7 +24,7 @@ io/snappydata/hydra/alterTable/nwAlterPartitionedRowTablesTestUsingSnappyShell.c io/snappydata/hydra/alterTable/nwAlterReplicatedRowTablesTestUsingSnappyShell.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="ReplicatedRow" @@ -35,7 +35,7 @@ io/snappydata/hydra/alterTable/nwAlterReplicatedRowTablesTestUsingSnappyShell.co io/snappydata/hydra/alterTable/nwAlterPartitionedRowTablesTestUsingJDBCClient.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="PartitionedRow" @@ -46,7 +46,7 @@ io/snappydata/hydra/alterTable/nwAlterPartitionedRowTablesTestUsingJDBCClient.co io/snappydata/hydra/alterTable/nwAlterReplicatedRowTablesTestUsingJDBCClient.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="ReplicatedRow" diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyPrms.java b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyPrms.java index da593b2fc1..3756216954 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyPrms.java +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyPrms.java @@ -18,6 +18,8 @@ import org.apache.commons.lang.StringUtils; +import java.util.Collections; +import java.util.List; import java.util.Vector; import hydra.BasePrms; @@ -693,7 +695,7 @@ public class SnappyPrms extends BasePrms { public static Long numRowsList; /** - * (int)Number of stack dumps to be taken for each thread of locators, servers, leads + * Number of stack dump for each thread of locators, servers, leads */ public static Long numOfStackDumpItr; @@ -702,6 +704,81 @@ public static int getNumOfStackDumpItrs(){ return tasktab().intAt(key, tab().intAt(key, 3)); } + /** + * + */ + + public static Long reRunWithDebugLogging; + + public static boolean isReRunWithDebugEnabled() { + Long key = reRunWithDebugLogging; + return tasktab().booleanAt(key, tab().booleanAt(key, false)); + + } + /** + * Parameter to pass for which logger the loglevel settings should be changed + */ + public static Long logger; + + public static String getLogger(){ + Long key = logger; + return tasktab().stringAt(key, tab().stringAt(key, "")); + } + + /** + * Parameter to specificy the required loglevel setting. + */ + + public static Long newLogLevel; + + public static String getNewLogLevel(){ + Long key = newLogLevel; + return tasktab().stringAt(key, tab().stringAt(key, "CONFIG")); + } + + + /** + * Parameter used to get the List of expected exceptions in the test run. + * Array of values for expected exceptions + */ + public static Long expectedExceptions; + + public static List getExpectedExceptionList() { + Long key = expectedExceptions; + Vector statements = tab().vecAt(key, new HydraVector()); + List strArr = Collections.list(statements.elements()); + return strArr; + } + + /** + * Parameter used to get the List of expected exceptions in the test run in case of HA. + * Array of values for expected exceptions + */ + public static Long expectedExceptionsForHA; + + public static List getExpectedExceptionListForHA() { + Long key = expectedExceptionsForHA; + Vector statements = tab().vecAt(key, new HydraVector()); + List strArr = Collections.list(statements.elements()); + return strArr; + } + + /** + * Parameter used to get the List of expected exceptions in the test run in case of HA. + * Array of values for expected exceptions + */ + public static Long knownExceptionsForHA; + + public static String[] getKnownExceptionListForHA() { + Long key = knownExceptionsForHA; + Vector statements = tab().vecAt(key, new HydraVector()); + String[] strArr = new String[statements.size()]; + for (int i = 0; i < statements.size(); i++) { + strArr[i] = (String)statements.elementAt(i); + } + return strArr; + } + /** * (int)Sleep time in secs between 2 thread dumps. */ @@ -712,12 +789,21 @@ public static int getSleepBtwnStackDumps(){ return tasktab().intAt(key, tab().intAt(key, 5)); } + /** + * (boolean) Flag to notify on whether to keep the clusters running after hydra tasks are done. + */ + public static Long keepClusterRunning; + + public static boolean isKeepClusterRunning(){ + Long key = keepClusterRunning; + return tasktab().booleanAt(key, tab().booleanAt(key, false)); + } + public static String getCredentialFile() { Long key = credentialFile; return BasePrms.tasktab().stringAt(key, BasePrms.tab().stringAt(key, null)); } - public static boolean isSecurityOn() { Long key = isSecurity; return tasktab().booleanAt(key, tab().booleanAt(key, false)); diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java index 924c17a019..6bf28141e3 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/SnappyTest.java @@ -23,6 +23,7 @@ import java.rmi.RemoteException; import java.sql.Connection; import java.sql.DriverManager; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.text.SimpleDateFormat; @@ -51,7 +52,11 @@ import sql.SQLPrms; import sql.dmlStatements.DMLStmtIF; import sql.sqlutil.DMLStmtsFactory; -import util.*; +import util.PRObserver; +import util.StopStartPrms; +import util.StopStartVMs; +import util.TestException; +import util.TestHelper; import static hydra.Prms.maxResultWaitSec; @@ -62,6 +67,7 @@ public class SnappyTest implements Serializable { protected static SnappyTest snappyTest; protected static HostDescription hd = TestConfig.getInstance().getMasterDescription() .getVmDescription().getHostDescription(); + protected static char sep = hd.getFileSep(); private static String gemfireHome = hd.getGemFireHome() + sep; protected static String productDir = gemfireHome + ".." + sep + "snappy" + sep; @@ -767,7 +773,6 @@ private static synchronized List getEndpoints(String type) { return endpoints; } - /** * Returns PIDs for all the processes started in the test, e.g. locator, server, lead . */ @@ -784,7 +789,6 @@ private static synchronized Set getPidList() { return pidList; } - /** * Returns hostname of the process */ @@ -800,7 +804,6 @@ protected static synchronized String getPidHost(String pid) { return pidHost; } - /** * Returns primary lead port . */ @@ -817,7 +820,6 @@ private static synchronized String getPrimaryLeadPort(String clientName) { return port; } - protected void initHydraThreadLocals() { this.connection = getConnection(); } @@ -1820,7 +1822,7 @@ protected static String getPrimaryLocatorPort() { return primaryLocatorPort; } - public void executeProcess(ProcessBuilder pb, File logFile) { + public int executeProcess(ProcessBuilder pb, File logFile) { Process p = null; try { if (logFile != null) { @@ -1835,10 +1837,13 @@ public void executeProcess(ProcessBuilder pb, File logFile) { assert p.getInputStream().read() == -1; } int rc = p.waitFor(); - if ((rc == 0) || (pb.command().contains("grep") && rc == 1)) { - Log.getLogWriter().info("Executed successfully"); + String pbCmd = Arrays.toString(pb.command().toArray()); + if ((rc == 0) || (pbCmd.contains("grep -v -F") && rc == 1)) { + Log.getLogWriter().info("Process executed successfully"); + return 0; } else { - Log.getLogWriter().info("Failed with exit code: " + rc); + Log.getLogWriter().info("Process failed with exit code: " + rc); + return 1; } } catch (IOException e) { throw new TestException("Exception occurred while starting the process:" + pb + @@ -1949,6 +1954,9 @@ public static String getCurrentDirPath() { * Task(ENDTASK) for cleaning up snappy processes, because they are not stopped by Hydra in case of Test failure. */ public static void HydraTask_cleanUpSnappyProcessesOnFailure() { + if(SnappyPrms.isKeepClusterRunning()) { + return; + } Process pr = null; ProcessBuilder pb = null; File logFile = null, log = null, nukeRunOutput = null; @@ -2253,9 +2261,18 @@ public void executeSnappyJob(Vector jobClassNames, String logFileName, String logFile = new File(dest); snappyTest.executeProcess(pb, logFile); pb = new ProcessBuilder("/bin/bash", "-c", curlCommand2); - snappyTest.executeProcess(pb, logFile); + int status = snappyTest.executeProcess(pb, logFile); } boolean retry = snappyTest.getSnappyJobsStatus(snappyJobScript, logFile, leadPort); + if(!checkJobStatus(getJobIDs(logFile)) && !cycleVms){ + if(SnappyPrms.isReRunWithDebugEnabled()) + { + HydraTask_changeLogLevel(); + HydraTask_executeSnappyJob(); + } + throw new TestException + ("Snappy job execution has failed. Please check the logs."); + } if (retry) { if (jobSubmissionCount <= SnappyPrms.getRetryCountForJob()) { jobSubmissionCount++; @@ -2316,7 +2333,6 @@ public String setCDCSparkAppCmds(String userAppArgs, String commonArgs, String s public void executeSparkJob(Vector jobClassNames, String logFileName) { String snappyJobScript = getScriptLocation("spark-submit"); - boolean isCDCStream = SnappyCDCPrms.getIsCDCStream(); ProcessBuilder pb = null; File log = null, logFile = null; userAppJar = SnappyPrms.getUserAppJar(); @@ -2343,17 +2359,18 @@ public void executeSparkJob(Vector jobClassNames, String logFileName) { userAppArgs = userAppArgs + " " + dmlProps; } if (SnappyCDCPrms.getIsCDC()) { - command = setCDCSparkAppCmds(userAppArgs, commonArgs, snappyJobScript, userJob, masterHost, masterPort, logFile); + command = setCDCSparkAppCmds(userAppArgs, commonArgs, snappyJobScript, userJob, + masterHost, masterPort, logFile); } else { command = snappyJobScript + " --class " + userJob + " --master spark://" + masterHost + ":" + masterPort + " " + SnappyPrms.getExecutorMemory() + " " + - SnappyPrms.getSparkSubmitExtraPrms() + " " + commonArgs + " " + snappyTest.getUserAppJarLocation(userAppJar, jarPath) + " " + - userAppArgs + " " + primaryLocatorHost + ":" + primaryLocatorPort; + SnappyPrms.getSparkSubmitExtraPrms() + " " + commonArgs + " " + snappyTest + .getUserAppJarLocation(userAppJar, jarPath) + " " + userAppArgs; } Log.getLogWriter().info("spark-submit command is : " + command); pb = new ProcessBuilder("/bin/bash", "-c", command); - snappyTest.executeProcess(pb, logFile); + int status = snappyTest.executeProcess(pb, logFile); Log.getLogWriter().info("CDC stream is : " + SnappyCDCPrms.getIsCDCStream()); if (SnappyCDCPrms.getIsCDCStream()) { //wait for 2 min until the cdc streams starts off. @@ -2365,6 +2382,9 @@ public void executeSparkJob(Vector jobClassNames, String logFileName) { Log.getLogWriter().info("Inside getIsCDCStream : " + SnappyCDCPrms.getIsCDCStream()); return; } + if(status==1 && !cycleVms){ + throw new TestException("Spark Application has failed. Please check the logs."); + } String searchString = "Spark ApplicationEnd: "; String expression = "cat " + logFile + " | grep -e Exception -e '" + searchString + "' |" + " grep -v java.net.BindException" + " | wc -l)\""; @@ -2493,6 +2513,56 @@ protected void simulateStream() { } } + public String getJobIDs(File logFile) { + String line = null; + String jobIDs = null; + try { + BufferedReader inputFile = new BufferedReader(new FileReader(logFile)); + while ((line = inputFile.readLine()) != null) { + if (line.contains("jobId")) { + String temp = line.split(":")[1].trim(); + jobIDs = temp.substring(1, temp.length() - 2); + jobIDs += ","; + } + } + inputFile.close(); + } catch (IOException ie) { + throw new TestException("Exception while reading job status file."); + } + if (jobIDs == null) { + throw new TestException("Failed to start the snappy job. Please check the logs."); + } + Log.getLogWriter().info("JobID is : " + jobIDs); + return jobIDs; + } + + public boolean checkJobStatus(String jobIDs){ + String[] jobIDArr = jobIDs.split(","); + for(int i = 0; i< jobIDArr.length; i++){ + try { + String dest = getCurrentDirPath() + File.separator + "jobStatus_" + getMyTid() + "_" + + jobIDArr[i] + ".log"; + File commandOutput = new File(dest); + String line = null; + BufferedReader inputFile = new BufferedReader(new FileReader(commandOutput)); + while ((line = inputFile.readLine()) != null) { + if (line.contains("status")) { + if (line.contains("ERROR")) + return false; + break; + } + } + try { + Thread.sleep(10 * 1000); + } catch (InterruptedException ie) { + } + } catch (IOException ie) { + Log.getLogWriter().info("Got exception while accessing current dir", ie); + } + } + return true; + } + public boolean getSnappyJobsStatus(String snappyJobScript, File logFile, String leadPort) { boolean found = false; try { @@ -2514,8 +2584,8 @@ public boolean getSnappyJobsStatus(String snappyJobScript, File logFile, String inputFile.close(); for (String str : jobIds) { File log = new File("."); - String dest = log.getCanonicalPath() + File.separator + "jobStatus_" + RemoteTestModule - .getCurrentThread().getThreadId() + "_" + System.currentTimeMillis() + ".log"; + String dest = log.getCanonicalPath() + File.separator + "jobStatus_" + getMyTid() + + "_" + str + ".log"; File commandOutput = new File(dest); String expression = snappyJobScript + " status --lead " + leadHost + ":" + leadPort + " " + "--job-id " + str + " > " + commandOutput + " 2>&1 ; grep -e '\"status\": " + @@ -2771,6 +2841,84 @@ public static synchronized void HydraTask_recordProcessIDWithHostWithUserConfsTe } } + /* + * Check if there is any suspect strings in the test. To be executed at the end of test. + */ + + public static synchronized void HydraTask_checkSuspectStrings() { + snappyTest.checkSuspectStrings(); + } + + public void checkSuspectStrings() { + String hasOOMEOrJVMCrash = checkForJVMCrashOrOOME(); + String checkSuspectOutPut = getCurrentDirPath() + File.separator + "suspectStrings.txt"; + File suspectStringFile = new File(checkSuspectOutPut); + StringBuilder cmd = new StringBuilder(); + StringBuilder exceptedExcep = new StringBuilder(); + + List expectedExceptions = SnappyPrms.getExpectedExceptionList(); + if(cycleVms) { + expectedExceptions.addAll(SnappyPrms.getExpectedExceptionListForHA()); + } + for (int i = 0; i < expectedExceptions.size(); i++) + exceptedExcep.append(" | grep -v \"").append(expectedExceptions.get(i) + "\""); + + cmd.setLength(0); + cmd.append("find " + getCurrentDirPath() + " -type f \\( -name \"*.log\" -not -iname " + + "\"*aster*.log\" -or -name \"*.out\" \\) "); + cmd.append(" | xargs grep Exception "); + cmd.append(exceptedExcep.toString()); + cmd. append(" | grep -v \\.java:"); + Log.getLogWriter().info("grep command is : " + cmd); + ProcessBuilder pb = new ProcessBuilder("/bin/bash", "-c", cmd.toString()); + executeProcess(pb, suspectStringFile); + + if(suspectStringFile.length() != 0){ + try { + StringBuilder exceptionList = new StringBuilder(); + BufferedReader reader = new BufferedReader(new FileReader(suspectStringFile)); + String line = ""; + while((line = reader.readLine()) != null) + exceptionList.append(line).append("\n"); + throw new TestException("Unknown Exceptions observed in the run " + exceptionList + .toString()); + } catch(FileNotFoundException fe) { + throw new TestException("Got exception while checking for suspect strings." , fe); + } catch(IOException ie) { + throw new TestException("Got exception while checking for suspect strings." , ie); + } + } + if(hasOOMEOrJVMCrash.length()>0){ + throw new TestException(hasOOMEOrJVMCrash); + } + } + + + public String checkForJVMCrashOrOOME() { + StringBuilder msg = new StringBuilder(); + String oomeOutPut = getCurrentDirPath() + File.separator + "checkOOME.txt"; + File oomeOutPutFile = new File(oomeOutPut); + + StringBuilder cmd = new StringBuilder(); + cmd.append("find " + getCurrentDirPath() + " -type f \\( -name \"*.hprof\" \\)"); + ProcessBuilder pb = new ProcessBuilder("/bin/bash", "-c", cmd.toString()); + executeProcess(pb, oomeOutPutFile); + if(oomeOutPutFile.length() != 0) { + msg.append("There is OOME observed in the test. Please check logs for more details.\n"); + } + cmd.setLength(0); + String jvmCrashOutput = getCurrentDirPath() + File.separator + "checkJVMCrash.txt"; + File jvmCrashOutputFile = new File(jvmCrashOutput); + cmd.append("find " + getCurrentDirPath() + " -type f \\( -name \"hs_err*.log\" \\)"); + pb = new ProcessBuilder("/bin/bash", "-c", cmd.toString()); + executeProcess(pb, jvmCrashOutputFile); + if(jvmCrashOutputFile.length() != 0) { + msg.append("There is HOTSPOT error observed in the test. Please check logs for more details" + + ". \n"); + } + return msg.toString(); + } + /** * Create and start snappy locator using snappy-locators.sh script. */ @@ -2828,6 +2976,9 @@ public static synchronized void HydraTask_startSparkCluster() { * Stops Spark Cluster. */ public static synchronized void HydraTask_stopSparkCluster() { + if (SnappyPrms.isKeepClusterRunning()) { + return; + } File log = null; try { initSnappyArtifacts(); @@ -2909,6 +3060,9 @@ public static synchronized void HydraTask_stopSnappyLocator() { } public static synchronized void HydraTask_stopSnappyCluster() { + if (SnappyPrms.isKeepClusterRunning()) { + return; + } File log = null; try { initSnappyArtifacts(); @@ -3017,10 +3171,10 @@ protected void cycleVM(int numToKill, int stopStartVMs, String cycledVM, Long la try { if (vmName.equalsIgnoreCase("lead")) vms = stopStartVMs(numToKill, "lead", isDmlOp, restart, rebalance); - else if (vmName.equalsIgnoreCase("server")) vms = stopStartVMs(numToKill, "server", - isDmlOp, restart, rebalance); - else if (vmName.equalsIgnoreCase("locator")) vms = stopStartVMs(numToKill, - "locator", isDmlOp, restart, rebalance); + else if (vmName.equalsIgnoreCase("server")) + vms = stopStartVMs(numToKill, "server", isDmlOp, restart, rebalance); + else if (vmName.equalsIgnoreCase("locator")) + vms = stopStartVMs(numToKill, "locator", isDmlOp, restart, rebalance); break; } catch (TestException te) { throw te; @@ -3696,7 +3850,6 @@ protected static String getMasterHost() { return masterHost; } - private String printStackTrace(Exception e) { StringWriter error = new StringWriter(); e.printStackTrace(new PrintWriter(error)); @@ -3724,11 +3877,31 @@ public List getHostNameFromConf(String nodeName) { return hostNames; } + public static void HydraTask_changeLogLevel() { + setLogLevel(SnappyPrms.getLogger(), SnappyPrms.getNewLogLevel()); + } + + public static void setLogLevel(String logger, String loglevel){ + Connection conn = null; + try { + conn = getLocatorConnection(); + } catch (SQLException se) { + throw new TestException("Got exception while getting connection", se); + } + try { + PreparedStatement ps = conn.prepareStatement("call sys.set_log_level(?,?)"); + ps.setString(0, logger); + ps.setString(1, loglevel); + ps.execute(); + } catch(SQLException se) { + throw new TestException("Got exception while executing set log level procedure.", se); + } + } protected void startSnappyLocator() { File log = null; - ProcessBuilder pb = null; List hostNames = getHostNameFromConf("locators"); + ProcessBuilder pb = null; try { if (useRowStore) { Log.getLogWriter().info("Starting locator/s using rowstore option..."); @@ -3751,8 +3924,8 @@ protected void startSnappyLocator() { protected void startSnappyServer() { File log = null; - ProcessBuilder pb = null; List hostNames = getHostNameFromConf("servers"); + ProcessBuilder pb = null; try { if (useRowStore) { Log.getLogWriter().info("Starting server/s using rowstore option..."); diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/expectedExceptionList.inc b/dtests/src/test/java/io/snappydata/hydra/cluster/expectedExceptionList.inc new file mode 100644 index 0000000000..d1ad388493 --- /dev/null +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/expectedExceptionList.inc @@ -0,0 +1,36 @@ +io.snappydata.hydra.cluster.SnappyPrms-expectedExceptions = +"java.net.BindException" +"NoSuchObjectException" +"NucleusObjectNotFoundException" +"RegionDestroyedException" +"DistributedSystemDisconnectedException" +"newDisconnectedException" +"CacheClosedException" +"java.io.FileNotFoundException" +"org.apache.spark.shuffle.FetchFailedException" +"org.glassfish.jersey.server.internal.MappableExceptionWrapperInterceptor" +"java.lang.reflect.InvocationTargetException" +"org.apache.spark.storage.ShuffleBlockFetcherIterator.throwFetchFailedException" +"org.apache.spark.SparkException: Exception thrown in awaitResult" +"com.gemstone.gemfire.distributed.LockServiceDestroyedException" +"Failed to retrieve information for" +"WARN LeadImpl: Exception while starting lead node" +; + +io.snappydata.hydra.cluster.SnappyPrms-expectedExceptionsForHA = +"java.lang.reflect.InvocationTargetException" +"java.lang.IllegalStateException: Cannot call methods on a stopped SparkContext" +; + +io.snappydata.hydra.cluster.SnappyPrms-knownExceptionsForHA = +"java.net.ConnectException: Connection refused" +"Caused by: org.apache.thrift.transport.TTransportException: Failed to create or configure socket." +; + + +/* +io.snappydata.hydra.cluster.SnappyPrms-suspectStrings = +"java.lang.IllegalStateException" +"NullPointerException" +; +*/ \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/startLongRunningCluster.bt b/dtests/src/test/java/io/snappydata/hydra/cluster/startLongRunningCluster.bt index 68f25d8a52..5c37ac7869 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cluster/startLongRunningCluster.bt +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/startLongRunningCluster.bt @@ -1,6 +1,6 @@ io/snappydata/hydra/cluster/startLongRunningCluster.conf A=snappyStore snappyStoreHosts=10 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=10 workerVMsPerHost=1 workerThreadsPerVM=1 numVMsToStop=1 \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/stopDualModeCluster.conf b/dtests/src/test/java/io/snappydata/hydra/cluster/stopDualModeCluster.conf index e249542db4..d2c316b171 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cluster/stopDualModeCluster.conf +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/stopDualModeCluster.conf @@ -18,7 +18,7 @@ INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = H runMode = always threadGroups = snappyThreads; -ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_dumpStacks +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_dumpStacks clientNames = snappy1; ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_stopSparkCluster diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/stopRowStoreCluster.conf b/dtests/src/test/java/io/snappydata/hydra/cluster/stopRowStoreCluster.conf index ba8f0e36a5..dd5e927ea9 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cluster/stopRowStoreCluster.conf +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/stopRowStoreCluster.conf @@ -11,11 +11,14 @@ INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = runMode = always threadGroups = snappyThreads; -CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_stopSnappyServers - threadGroups = snappyThreads; +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_dumpStacks + clientNames = locator1; -CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_stopSnappyLocator - threadGroups = snappyThreads; +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_stopSnappyServers + clientNames = locator1; + +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_stopSnappyLocator + clientNames = locator1; io.snappydata.hydra.cluster.SnappyPrms-isStopMode = true; //io.snappydata.hydra.cluster.SnappyPrms-useRowStore = true; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/cluster/stopSnappyCluster.conf b/dtests/src/test/java/io/snappydata/hydra/cluster/stopSnappyCluster.conf index a3bb2f43fc..2ea1163874 100644 --- a/dtests/src/test/java/io/snappydata/hydra/cluster/stopSnappyCluster.conf +++ b/dtests/src/test/java/io/snappydata/hydra/cluster/stopSnappyCluster.conf @@ -24,11 +24,14 @@ CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_stopSnappyLocator threadGroups = snappyThreads;*/ +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_dumpStacks + clientNames = locator1; + ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_stopSnappyCluster - clientNames = locator1; + clientNames = locator1; ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_deleteSnappyConfig - clientNames = locator1; + clientNames = locator1; ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_cleanUpSnappyProcessesOnFailure clientNames = locator1; diff --git a/dtests/src/test/java/io/snappydata/hydra/connectionPool/HikariConnectionPool.java b/dtests/src/test/java/io/snappydata/hydra/connectionPool/HikariConnectionPool.java index e459e2c712..dcb5a84258 100644 --- a/dtests/src/test/java/io/snappydata/hydra/connectionPool/HikariConnectionPool.java +++ b/dtests/src/test/java/io/snappydata/hydra/connectionPool/HikariConnectionPool.java @@ -18,12 +18,12 @@ package io.snappydata.hydra.connectionPool; import java.sql.Connection; +import java.util.Properties; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import hydra.Log; import io.snappydata.hydra.cluster.SnappyTest; -import org.apache.tomcat.jdbc.pool.PoolProperties; import util.TestException; public class HikariConnectionPool { @@ -35,9 +35,14 @@ private HikariConnectionPool() { Log.getLogWriter().info(" Creating instance of HikariConnectionPool"); String url = SnappyConnectionPoolPrms.getUrl() + SnappyTest.validateLocatorEndpointData().get (0); - PoolProperties p = new PoolProperties(); - - HikariConfig jdbcConfig = new HikariConfig(); + Properties connProperties = new Properties(); + //user connection properties + String[] strArr = SnappyConnectionPoolPrms.getConnProperties(); + for(int i = 0; i < strArr.length; i++) { + String prop[] = strArr[i].split("="); + connProperties.setProperty(prop[0],prop[1]); + } + HikariConfig jdbcConfig = new HikariConfig(connProperties); jdbcConfig.setPoolName(SnappyConnectionPoolPrms.getPoolName()); jdbcConfig.setMaximumPoolSize(SnappyConnectionPoolPrms.getInitialSize()); jdbcConfig.setJdbcUrl(url); diff --git a/dtests/src/test/java/io/snappydata/hydra/connectionPool/SnappyConnectionPoolPrms.java b/dtests/src/test/java/io/snappydata/hydra/connectionPool/SnappyConnectionPoolPrms.java index 76af2df9c1..c47f67c106 100644 --- a/dtests/src/test/java/io/snappydata/hydra/connectionPool/SnappyConnectionPoolPrms.java +++ b/dtests/src/test/java/io/snappydata/hydra/connectionPool/SnappyConnectionPoolPrms.java @@ -17,6 +17,8 @@ package io.snappydata.hydra.connectionPool; +import java.util.Vector; + import hydra.BasePrms; import hydra.TestConfig; @@ -38,6 +40,9 @@ public class SnappyConnectionPoolPrms extends BasePrms { public static Long poolName; + /* space seperated connection properties to be set while creating a connection */ + public static Long connectionProperties; + public static String getString(Long key){ return TestConfig.tasktab().stringAt(key, TestConfig.tab().stringAt(key, null)); } @@ -87,6 +92,18 @@ else if (connPoolType.equalsIgnoreCase("Tomcat")) return -1; //don't use connection pool } + public static String[] getConnProperties() { + Long key = connectionProperties; + Vector connProperties = TestConfig.tasktab().vecAt(key, TestConfig.tab().vecAt(key, null)); + String[] strArr = new String[connProperties.size()]; + if (connProperties != null) { + for (int i = 0; i < connProperties.size(); i++) { + strArr[i] = (String)connProperties.get(i); + } + } + return strArr; + } + static { BasePrms.setValues(SnappyConnectionPoolPrms.class); } diff --git a/dtests/src/test/java/io/snappydata/hydra/connectionPool/TomcatConnectionPool.java b/dtests/src/test/java/io/snappydata/hydra/connectionPool/TomcatConnectionPool.java index 658465c72c..faca93f2d0 100644 --- a/dtests/src/test/java/io/snappydata/hydra/connectionPool/TomcatConnectionPool.java +++ b/dtests/src/test/java/io/snappydata/hydra/connectionPool/TomcatConnectionPool.java @@ -42,6 +42,14 @@ private TomcatConnectionPool() { p.setPassword(SnappyConnectionPoolPrms.getPassword()); p.setInitialSize(SnappyConnectionPoolPrms.getInitialSize()); p.setMaxWait(SnappyConnectionPoolPrms.getMaxWait()); + //user connection properties + String[] connProperties = SnappyConnectionPoolPrms.getConnProperties(); + String connProp = ""; + for(int i = 0; i < connProperties.length; i++) + connProp = connProp + connProperties[i] + ";"; + int lastIndex = connProp.lastIndexOf(";"); + if (lastIndex != -1) connProp = connProp.substring(0,lastIndex); + p.setConnectionProperties(connProp); datasource = new DataSource(); datasource.setPoolProperties(p); } diff --git a/dtests/src/test/java/io/snappydata/hydra/ct/ct.bt b/dtests/src/test/java/io/snappydata/hydra/ct/ct.bt index 19a7f35d37..746b8b423b 100644 --- a/dtests/src/test/java/io/snappydata/hydra/ct/ct.bt +++ b/dtests/src/test/java/io/snappydata/hydra/ct/ct.bt @@ -9,7 +9,7 @@ io/snappydata/hydra/ct/ctSplitMode.conf redundantCopies=0 persistenceMode="null" //not required for replicated table, but required for hydra config dataFilesLocation="/export/shared/QA_DATA/ct" tableType="Replicated" - fullResultSetValidation= true + numRowsValidation=true fullResultSetValidation=true io/snappydata/hydra/ct/ctSplitModeCreateWithScript.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 @@ -20,31 +20,9 @@ io/snappydata/hydra/ct/ctSplitModeCreateWithScript.conf redundantCopies=0 persistenceMode="sync" //redundancy not required for replicated table, but required for hydra config tableType="PersistentReplicated" scriptName="ct_create_and_load_persistentReplicated_row_tables.sql" - fullResultSetValidation= true - -io/snappydata/hydra/ct/ctSplitModeCreateWithScriptLeadHA.conf - A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 - B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 - D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 - redundantCopies=0 persistenceMode="sync" - dataFilesLocation="/export/shared/QA_DATA/ct" - tableType="PersistentReplicated" - scriptName="ct_create_and_load_persistentReplicated_row_tables.sql" - numVMsToStop=1 - fullResultSetValidation= true + numRowsValidation=true fullResultSetValidation=true io/snappydata/hydra/ct/ctSplitMode.conf - A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 - B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 - D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 - redundantCopies=0,1,2 persistenceMode="sync" - dataFilesLocation="/export/shared/QA_DATA/ct" - tableType="PartitionedRow" - fullResultSetValidation= true - -io/snappydata/hydra/ct/ctSplitModeServerHA.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 @@ -52,73 +30,61 @@ io/snappydata/hydra/ct/ctSplitModeServerHA.conf redundantCopies=1 persistenceMode="sync" dataFilesLocation="/export/shared/QA_DATA/ct" tableType="PartitionedRow" - numVMsToStop=1 - fullResultSetValidation= true + numRowsValidation=true fullResultSetValidation=true io/snappydata/hydra/ct/ctSplitModeCreateWithScript.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 - redundantCopies=0,1 persistenceMode="sync" + redundantCopies=1 persistenceMode="sync" dataFilesLocation="/export/shared/QA_DATA/ct" tableType="PersistentPartitionRow" scriptName="ct_create_and_load_persistentPartitioned_row_table.sql" - fullResultSetValidation= true + numRowsValidation=true fullResultSetValidation=true io/snappydata/hydra/ct/ctSplitMode.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 - redundantCopies=0,1 persistenceMode="sync" - dataFilesLocation="/export/shared/QA_DATA/ct" - tableType="ColocatedRow" - fullResultSetValidation= true - -io/snappydata/hydra/ct/ctSplitModeCreateWithSparkAppLeadHA.conf - A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 - B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 - D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 - redundantCopies=0,1 persistenceMode="sync" + redundantCopies=1 persistenceMode="sync" dataFilesLocation="/export/shared/QA_DATA/ct" tableType="ColocatedRow" - numVMsToStop=1 - fullResultSetValidation= true + numRowsValidation=true fullResultSetValidation=true io/snappydata/hydra/ct/ctSplitModeCreateWithScript.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 - redundantCopies=0,1 persistenceMode="sync" + redundantCopies=1 persistenceMode="sync" dataFilesLocation="/export/shared/QA_DATA/ct" tableType="EvictionRow" scriptName="ct_create_and_load_eviction_row_table.sql" - fullResultSetValidation= true + numRowsValidation=true fullResultSetValidation=true io/snappydata/hydra/ct/ctSplitModeCreateWithScript.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 - redundantCopies=0,1 persistenceMode="sync" + redundantCopies=1 persistenceMode="sync" dataFilesLocation="/export/shared/QA_DATA/ct" tableType="PersistentColocatedRow" scriptName="ct_create_and_load_persistentColocated_row_table.sql" - fullResultSetValidation= true + numRowsValidation=true fullResultSetValidation=true io/snappydata/hydra/ct/ctSplitModeCreateWithScript.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 - redundantCopies=0,1 persistenceMode="sync" + redundantCopies=1 persistenceMode="sync" dataFilesLocation="/export/shared/QA_DATA/ct" tableType="ColocatedWithEvictionRow" scriptName="ct_create_and_load_colocatedWithEviction_row_table.sql" - fullResultSetValidation= true + numRowsValidation=true fullResultSetValidation=true //Column tables io/snappydata/hydra/ct/ctSplitMode.conf @@ -126,104 +92,47 @@ io/snappydata/hydra/ct/ctSplitMode.conf B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 - redundantCopies=0,1 persistenceMode="sync" + redundantCopies=1 persistenceMode="sync" dataFilesLocation="/export/shared/QA_DATA/ct" tableType="Column" - fullResultSetValidation= true + numRowsValidation=true fullResultSetValidation=true io/snappydata/hydra/ct/ctSplitModeCreateWithSparkApp.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 - redundantCopies=0,1,2 persistenceMode="sync" + redundantCopies=1 persistenceMode="sync" dataFilesLocation="/export/shared/QA_DATA/ct" tableType="PersistentColumn" - fullResultSetValidation= true + numRowsValidation=true fullResultSetValidation=true io/snappydata/hydra/ct/ctSplitMode.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 - redundantCopies=0,1 persistenceMode="sync" - dataFilesLocation="/export/shared/QA_DATA/ct" - tableType="ColocatedColumn" - fullResultSetValidation= true - -io/snappydata/hydra/ct/ctSplitModeCreateWithScriptLeadHA.conf - A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 - B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 - D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 - redundantCopies=0,1 persistenceMode="sync" + redundantCopies=1 persistenceMode="sync" dataFilesLocation="/export/shared/QA_DATA/ct" tableType="ColocatedColumn" - scriptName="ct_create_and_load_colocated_column_table.sql" - numVMsToStop=1 - fullResultSetValidation= true - -io/snappydata/hydra/ct/ctSplitModeCreateWithScriptServerHA.conf - A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 - B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 - D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 - redundantCopies=1,2 persistenceMode="sync" - dataFilesLocation="/export/shared/QA_DATA/ct" - tableType="PersistentColumn" - numVMsToStop=1 - scriptName="ct_create_and_load_persistent_column_table.sql" - fullResultSetValidation= true + numRowsValidation=true fullResultSetValidation=true io/snappydata/hydra/ct/ctSplitMode.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 - redundantCopies=0,1,2 persistenceMode="sync" - dataFilesLocation="/export/shared/QA_DATA/ct" - tableType="PersistentColocatedColumn" - fullResultSetValidation= true - -io/snappydata/hydra/ct/ctSplitModeLeadHA.conf - A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 - B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 - D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 - redundantCopies=0,1 persistenceMode="sync" - dataFilesLocation="/export/shared/QA_DATA/ct" - tableType="PersistentColocatedColumn" - numVMsToStop=1 - fullResultSetValidation= true - -io/snappydata/hydra/ct/ctSplitModeServerHA.conf - A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 - B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 - D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 - redundantCopies=1,2 persistenceMode="sync" + redundantCopies=1 persistenceMode="sync" dataFilesLocation="/export/shared/QA_DATA/ct" tableType="PersistentColocatedColumn" - numVMsToStop=1 - fullResultSetValidation= true + numRowsValidation=true fullResultSetValidation=true io/snappydata/hydra/ct/ctSplitModeCreateWithSparkApp.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 - redundantCopies=0,1 persistenceMode="sync" - dataFilesLocation="/export/shared/QA_DATA/ct" - tableType="Column" - fullResultSetValidation= true - -io/snappydata/hydra/ct/ctSplitModeCreateWithSparkAppServerHA.conf - A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 - B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 - D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 - redundantCopies=1,2 persistenceMode="sync" + redundantCopies=1 persistenceMode="sync" dataFilesLocation="/export/shared/QA_DATA/ct" tableType="Column" - numVMsToStop=1 - fullResultSetValidation= true + numRowsValidation=true fullResultSetValidation=true diff --git a/dtests/src/test/java/io/snappydata/hydra/ct/ctHA.bt b/dtests/src/test/java/io/snappydata/hydra/ct/ctHA.bt new file mode 100644 index 0000000000..94ebdecd38 --- /dev/null +++ b/dtests/src/test/java/io/snappydata/hydra/ct/ctHA.bt @@ -0,0 +1,90 @@ +io/snappydata/hydra/ct/ctSplitModeCreateWithScriptLeadHA.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 + redundantCopies=0 persistenceMode="sync" //redundanctCopies not required in case of replicated table. + dataFilesLocation="/export/shared/QA_DATA/ct" + tableType="PersistentReplicated" + scriptName="ct_create_and_load_persistentReplicated_row_tables.sql" + numVMsToStop=1 + numRowsValidation=true fullResultSetValidation=true + +io/snappydata/hydra/ct/ctSplitModeServerHA.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 + redundantCopies=1 persistenceMode="sync" + dataFilesLocation="/export/shared/QA_DATA/ct" + tableType="PartitionedRow" + numVMsToStop=1 + numRowsValidation=true fullResultSetValidation=true + +io/snappydata/hydra/ct/ctSplitModeCreateWithSparkAppLeadHA.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 + redundantCopies=1 persistenceMode="sync" + dataFilesLocation="/export/shared/QA_DATA/ct" + tableType="ColocatedRow" + numVMsToStop=1 + numRowsValidation=true fullResultSetValidation=true + +io/snappydata/hydra/ct/ctSplitModeCreateWithScriptLeadHA.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 + redundantCopies=1 persistenceMode="sync" + dataFilesLocation="/export/shared/QA_DATA/ct" + tableType="ColocatedColumn" + scriptName="ct_create_and_load_colocated_column_table.sql" + numVMsToStop=1 + numRowsValidation=true fullResultSetValidation=true + +io/snappydata/hydra/ct/ctSplitModeCreateWithScriptServerHA.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 + redundantCopies=1 persistenceMode="sync" + dataFilesLocation="/export/shared/QA_DATA/ct" + tableType="PersistentColumn" + numVMsToStop=1 + scriptName="ct_create_and_load_persistent_column_table.sql" + numRowsValidation=true fullResultSetValidation=true + +io/snappydata/hydra/ct/ctSplitModeLeadHA.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 + redundantCopies=1 persistenceMode="sync" + dataFilesLocation="/export/shared/QA_DATA/ct" + tableType="PersistentColocatedColumn" + numVMsToStop=1 + numRowsValidation=true fullResultSetValidation=true + +io/snappydata/hydra/ct/ctSplitModeServerHA.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 + redundantCopies=1 persistenceMode="sync" + dataFilesLocation="/export/shared/QA_DATA/ct" + tableType="PersistentColocatedColumn" + numVMsToStop=1 + numRowsValidation=true fullResultSetValidation=true + +io/snappydata/hydra/ct/ctSplitModeCreateWithSparkAppServerHA.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 + redundantCopies=1 persistenceMode="sync" + dataFilesLocation="/export/shared/QA_DATA/ct" + tableType="Column" + numVMsToStop=1 + numRowsValidation=true fullResultSetValidation=true diff --git a/dtests/src/test/java/io/snappydata/hydra/ct/ctMisc.bt b/dtests/src/test/java/io/snappydata/hydra/ct/ctMisc.bt new file mode 100644 index 0000000000..75cb9fa263 --- /dev/null +++ b/dtests/src/test/java/io/snappydata/hydra/ct/ctMisc.bt @@ -0,0 +1,169 @@ +io/snappydata/hydra/ct/ctSplitMode.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 + redundantCopies=0,2 persistenceMode="sync" + dataFilesLocation="/export/shared/QA_DATA/ct" + tableType="PartitionedRow" + numRowsValidation=true fullResultSetValidation=true + +io/snappydata/hydra/ct/ctSplitModeCreateWithScript.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 + redundantCopies=0 persistenceMode="sync" + dataFilesLocation="/export/shared/QA_DATA/ct" + tableType="PersistentPartitionRow" + scriptName="ct_create_and_load_persistentPartitioned_row_table.sql" + numRowsValidation=true fullResultSetValidation=true + +io/snappydata/hydra/ct/ctSplitModeCreateWithScript.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 + redundantCopies=0 persistenceMode="sync" + dataFilesLocation="/export/shared/QA_DATA/ct" + tableType="PersistentPartitionRow" + scriptName="ct_create_and_load_persistentPartitioned_row_table.sql" + numRowsValidation=true fullResultSetValidation=true + +io/snappydata/hydra/ct/ctSplitMode.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 + redundantCopies=0 persistenceMode="sync" + dataFilesLocation="/export/shared/QA_DATA/ct" + tableType="ColocatedRow" + numRowsValidation=true fullResultSetValidation=true + +io/snappydata/hydra/ct/ctSplitModeCreateWithSparkAppLeadHA.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 + redundantCopies=0 persistenceMode="sync" + dataFilesLocation="/export/shared/QA_DATA/ct" + tableType="ColocatedRow" + numVMsToStop=1 + numRowsValidation=true fullResultSetValidation=true + +io/snappydata/hydra/ct/ctSplitModeCreateWithScript.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 + redundantCopies=0 persistenceMode="sync" + dataFilesLocation="/export/shared/QA_DATA/ct" + tableType="EvictionRow" + scriptName="ct_create_and_load_eviction_row_table.sql" + numRowsValidation=true fullResultSetValidation=true + +io/snappydata/hydra/ct/ctSplitModeCreateWithScript.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 + redundantCopies=0 persistenceMode="sync" + dataFilesLocation="/export/shared/QA_DATA/ct" + tableType="PersistentColocatedRow" + scriptName="ct_create_and_load_persistentColocated_row_table.sql" + numRowsValidation=true fullResultSetValidation=true + +io/snappydata/hydra/ct/ctSplitModeCreateWithScript.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 + redundantCopies=0 persistenceMode="sync" + dataFilesLocation="/export/shared/QA_DATA/ct" + tableType="ColocatedWithEvictionRow" + scriptName="ct_create_and_load_colocatedWithEviction_row_table.sql" + numRowsValidation=true fullResultSetValidation=true + +io/snappydata/hydra/ct/ctSplitMode.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 + redundantCopies=0 persistenceMode="sync" + dataFilesLocation="/export/shared/QA_DATA/ct" + tableType="Column" + numRowsValidation=true fullResultSetValidation=true + +io/snappydata/hydra/ct/ctSplitModeCreateWithSparkApp.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 + redundantCopies=0 persistenceMode="sync" + dataFilesLocation="/export/shared/QA_DATA/ct" + tableType="PersistentColumn" + numRowsValidation=true fullResultSetValidation=true + +io/snappydata/hydra/ct/ctSplitMode.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 + redundantCopies=0,2 persistenceMode="sync" + dataFilesLocation="/export/shared/QA_DATA/ct" + tableType="ColocatedColumn" + numRowsValidation=true fullResultSetValidation=true + +io/snappydata/hydra/ct/ctSplitMode.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 + redundantCopies=0 persistenceMode="sync" + dataFilesLocation="/export/shared/QA_DATA/ct" + tableType="PersistentColocatedColumn" + numRowsValidation=true fullResultSetValidation=true + +io/snappydata/hydra/ct/ctSplitModeCreateWithSparkApp.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 + redundantCopies=2 persistenceMode="sync" + dataFilesLocation="/export/shared/QA_DATA/ct" + tableType="Column" + numRowsValidation=true fullResultSetValidation=true + +io/snappydata/hydra/ct/ctSplitModeCreateWithScriptServerHA.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 + redundantCopies=2 persistenceMode="sync" + dataFilesLocation="/export/shared/QA_DATA/ct" + tableType="PersistentColumn" + numVMsToStop=1 + scriptName="ct_create_and_load_persistent_column_table.sql" + numRowsValidation=true fullResultSetValidation=true + +io/snappydata/hydra/ct/ctSplitModeServerHA.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 + redundantCopies=2 persistenceMode="sync" + dataFilesLocation="/export/shared/QA_DATA/ct" + tableType="PersistentColocatedColumn" + numVMsToStop=1 + numRowsValidation=true fullResultSetValidation=true + +io/snappydata/hydra/ct/ctSplitModeCreateWithSparkAppServerHA.conf + A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 + redundantCopies=2 persistenceMode="sync" + dataFilesLocation="/export/shared/QA_DATA/ct" + tableType="Column" + numVMsToStop=1 + numRowsValidation=true fullResultSetValidation=true \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/ct/ctSplitMode.conf b/dtests/src/test/java/io/snappydata/hydra/ct/ctSplitMode.conf index d5b7b53199..da3b31e56e 100644 --- a/dtests/src/test/java/io/snappydata/hydra/ct/ctSplitMode.conf +++ b/dtests/src/test/java/io/snappydata/hydra/ct/ctSplitMode.conf @@ -15,7 +15,7 @@ INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.hydra.ct.ValidateCTQueriesJob - io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "dataFilesLocation=${dataFilesLocation},tableType=${tableType},fullResultSetValidation=${fullResultSetValidation}" + io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "dataFilesLocation=${dataFilesLocation},tableType=${tableType},fullResultSetValidation=${fullResultSetValidation},numRowsValidation=${numRowsValidation}" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar maxThreads = 1 threadGroups = leadThreads; @@ -29,7 +29,7 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = H TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSparkJob io.snappydata.hydra.cluster.SnappyPrms-sparkJobClassNames = io.snappydata.hydra.ct.ValidateCTQueriesApp - io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "${dataFilesLocation} ${tableType} ${fullResultSetValidation}" + io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "${dataFilesLocation} ${tableType} ${fullResultSetValidation} ${numRowsValidation}" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar io.snappydata.hydra.cluster.SnappyPrms-executorMemory = 4g maxThreads = 1 diff --git a/dtests/src/test/java/io/snappydata/hydra/ct/ctSplitModeCreateWithScript.conf b/dtests/src/test/java/io/snappydata/hydra/ct/ctSplitModeCreateWithScript.conf index c5df3b4428..1eeacbd85d 100644 --- a/dtests/src/test/java/io/snappydata/hydra/ct/ctSplitModeCreateWithScript.conf +++ b/dtests/src/test/java/io/snappydata/hydra/ct/ctSplitModeCreateWithScript.conf @@ -5,6 +5,7 @@ hydra.Prms-testDescription = "This test starts the snappy cluster and spark clus INCLUDE $JTESTS/io/snappydata/hydra/northwind/startDualModeCluster.conf; + //Create and load table using sql script INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSQLScripts io.snappydata.hydra.cluster.SnappyPrms-sqlScriptNames = ${scriptName} @@ -16,7 +17,7 @@ INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.hydra.ct.ValidateCTQueriesJob - io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "dataFilesLocation=${dataFilesLocation},tableType=${tableType},fullResultSetValidation=${fullResultSetValidation}" + io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "dataFilesLocation=${dataFilesLocation},tableType=${tableType},fullResultSetValidation=${fullResultSetValidation},numRowsValidation=${numRowsValidation}" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar maxThreads = 1 threadGroups = leadThreads; @@ -30,7 +31,7 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = H TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSparkJob io.snappydata.hydra.cluster.SnappyPrms-sparkJobClassNames = io.snappydata.hydra.ct.ValidateCTQueriesApp - io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "${dataFilesLocation} ${tableType} ${fullResultSetValidation}" + io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "${dataFilesLocation} ${tableType} ${fullResultSetValidation} ${numRowsValidation}" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar io.snappydata.hydra.cluster.SnappyPrms-executorMemory = 4g maxThreads = 1 diff --git a/dtests/src/test/java/io/snappydata/hydra/ct/ctSplitModeCreateWithScriptLeadHA.conf b/dtests/src/test/java/io/snappydata/hydra/ct/ctSplitModeCreateWithScriptLeadHA.conf index f728055197..ba207d2f8f 100644 --- a/dtests/src/test/java/io/snappydata/hydra/ct/ctSplitModeCreateWithScriptLeadHA.conf +++ b/dtests/src/test/java/io/snappydata/hydra/ct/ctSplitModeCreateWithScriptLeadHA.conf @@ -1,6 +1,14 @@ hydra.Prms-testRequirement += " with lead HA."; hydra.Prms-testDescription += " When the tasks are being peformed, a lead node will be restarted."; +CLOSETASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSparkJob + io.snappydata.hydra.cluster.SnappyPrms-sparkJobClassNames = io.snappydata.hydra.ct.ValidateCTQueriesApp + io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "${dataFilesLocation} ${tableType} ${fullResultSetValidation} ${numRowsValidation}" + io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar + io.snappydata.hydra.cluster.SnappyPrms-executorMemory = 4g + maxThreads = 1 + threadGroups = snappyStoreThreads + ; INCLUDE $JTESTS/io/snappydata/hydra/northwind/leadHA.inc; INCLUDE $JTESTS/io/snappydata/hydra/ct/ctSplitModeCreateWithScript.conf; INCLUDE $JTESTS/io/snappydata/hydra/northwind/backUpLeadConfig.inc; diff --git a/dtests/src/test/java/io/snappydata/hydra/ct/ctSplitModeCreateWithSparkApp.conf b/dtests/src/test/java/io/snappydata/hydra/ct/ctSplitModeCreateWithSparkApp.conf index c5d3561948..ff7412e563 100644 --- a/dtests/src/test/java/io/snappydata/hydra/ct/ctSplitModeCreateWithSparkApp.conf +++ b/dtests/src/test/java/io/snappydata/hydra/ct/ctSplitModeCreateWithSparkApp.conf @@ -15,7 +15,7 @@ INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = H TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.hydra.ct.ValidateCTQueriesJob - io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "dataFilesLocation=${dataFilesLocation},tableType=${tableType},fullResultSetValidation=${fullResultSetValidation}" + io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "dataFilesLocation=${dataFilesLocation},tableType=${tableType},fullResultSetValidation=${fullResultSetValidation},numRowsValidation=${numRowsValidation}" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar maxThreads = 1 threadGroups = leadThreads; @@ -29,7 +29,7 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = H TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSparkJob io.snappydata.hydra.cluster.SnappyPrms-sparkJobClassNames = io.snappydata.hydra.ct.ValidateCTQueriesApp - io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "${dataFilesLocation} ${tableType} ${fullResultSetValidation}" + io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "${dataFilesLocation} ${tableType} ${fullResultSetValidation} ${numRowsValidation}" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar io.snappydata.hydra.cluster.SnappyPrms-executorMemory = 4g maxThreads = 1 diff --git a/dtests/src/test/java/io/snappydata/hydra/ct/local.ct.conf b/dtests/src/test/java/io/snappydata/hydra/ct/local.ct.conf new file mode 100644 index 0000000000..0b4820f662 --- /dev/null +++ b/dtests/src/test/java/io/snappydata/hydra/ct/local.ct.conf @@ -0,0 +1,14 @@ +hydra.HostPrms-hostNames = +fcn "hydra.TestConfigFcns.pool(\" \", ${snappyStoreHosts})" ncf +fcn "hydra.TestConfigFcns.pool(\" \", ${leadHosts})" ncf +fcn "hydra.TestConfigFcns.pool(\" \", ${locatorHosts})" ncf +fcn "hydra.TestConfigFcns.pool(\" \", ${workerHosts})" ncf +; + +io.snappydata.hydra.cluster.SnappyPrms-serverLauncherProps = " -heap-size=6g -memory-size=6g -critical-heap-percentage=95 -J-XX:+UnlockDiagnosticVMOptions -J-XX:ParGCCardsPerStrideChunk=32768 -member-timeout=60000 -J-Dp2p.socket_timeout=60000 "; +io.snappydata.hydra.cluster.SnappyPrms-locatorLauncherProps = " -member-timeout=60000 -J-Dp2p.socket_timeout=60000 "; + +io.snappydata.hydra.cluster.SnappyPrms-executorMemory = 4g; +io.snappydata.hydra.cluster.SnappyPrms-leadMemory = 4g; + + diff --git a/dtests/src/test/java/io/snappydata/hydra/distIndex/distIndex.bt b/dtests/src/test/java/io/snappydata/hydra/distIndex/distIndex.bt index b5f92ae67c..e795f86e55 100644 --- a/dtests/src/test/java/io/snappydata/hydra/distIndex/distIndex.bt +++ b/dtests/src/test/java/io/snappydata/hydra/distIndex/distIndex.bt @@ -1,7 +1,7 @@ io/snappydata/hydra/distIndex/distIndexResultValidation.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 redundantCopies=1 buckets_Order_Lineitem=113 buckets_Cust_Part_PartSupp=113 @@ -16,7 +16,7 @@ io/snappydata/hydra/distIndex/distIndexResultValidation.conf io/snappydata/hydra/distIndex/distIndexPerformance.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 redundantCopies=1 buckets_Order_Lineitem=113 buckets_Cust_Part_PartSupp=113 diff --git a/dtests/src/test/java/io/snappydata/hydra/distJoin/distJoin.bt b/dtests/src/test/java/io/snappydata/hydra/distJoin/distJoin.bt index 3ff1911734..0d9e11f3c9 100644 --- a/dtests/src/test/java/io/snappydata/hydra/distJoin/distJoin.bt +++ b/dtests/src/test/java/io/snappydata/hydra/distJoin/distJoin.bt @@ -2,32 +2,48 @@ io/snappydata/hydra/distJoin/distJoinWithLeadHA.conf A=snappyStore snappyStoreHosts=2 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 + dataLocation="$GEMFIRE/../../../tests/common/src/main/resources/TPCH" + buckets_Order_Lineitem=128 + buckets_Cust_Part_PartSupp=128 + isSupplierColumnTable=true + buckets_Supplier=128 redundantCopies=1 - buckets_Order_Lineitem=29 - buckets_Cust_Part_PartSupp=7 + persistence=true + persistence_type="SYNCHRONOUS" + numberOfLoadStages=1 + isParquet=false + createParquet=false + traceEvents=false queries="1-3-4-5-6-7-8-10-11-12-13-14-15-16-17-18-19-20-22" queryPlan=false - shufflePartitions=4 - inMemoryColumnarStorageCompressed=false - dataLocation="$GEMFIRE/../../../tests/common/src/main/resources/TPCH" - useIndex=true resultCollection=true - warmUpIterations=5 + warmUps=5 actualRuns=5 + isDynamic=true + resultCollection=false + randomSeed=42 io/snappydata/hydra/distJoin/distJoinWithServerHA.conf A=snappyStore snappyStoreHosts=2 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 + dataLocation="$GEMFIRE/../../../tests/common/src/main/resources/TPCH" + buckets_Order_Lineitem=128 + buckets_Cust_Part_PartSupp=128 + isSupplierColumnTable=true + buckets_Supplier=128 redundantCopies=1 - buckets_Order_Lineitem=29 - buckets_Cust_Part_PartSupp=7 + persistence=true + persistence_type="SYNCHRONOUS" + numberOfLoadStages=1 + isParquet=false + createParquet=false + traceEvents=false queries="1-3-4-5-6-7-8-10-11-12-13-14-15-16-17-18-19-20-22" queryPlan=false - shufflePartitions=4 - inMemoryColumnarStorageCompressed=false - dataLocation="$GEMFIRE/../../../tests/common/src/main/resources/TPCH" - useIndex=true resultCollection=true warmUpIterations=5 actualRuns=5 + isDynamic=true + resultCollection=false + randomSeed=42 diff --git a/dtests/src/test/java/io/snappydata/hydra/distJoin/distJoinWithLeadHA.conf b/dtests/src/test/java/io/snappydata/hydra/distJoin/distJoinWithLeadHA.conf index 9af1b6f238..a83199e30c 100644 --- a/dtests/src/test/java/io/snappydata/hydra/distJoin/distJoinWithLeadHA.conf +++ b/dtests/src/test/java/io/snappydata/hydra/distJoin/distJoinWithLeadHA.conf @@ -89,25 +89,25 @@ INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = b threadGroups = snappyThreads; INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob - io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.benchmark.snappy.TPCH_Snappy_Tables + io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.benchmark.snappy.tpch.TableCreationJob io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = - "dataLocation=${dataLocation},Buckets_Order_Lineitem=${buckets_Order_Lineitem},Buckets_Cust_Part_PartSupp=${buckets_Cust_Part_PartSupp},useIndex=${useIndex}" + "dataLocation=$dataLocation,Buckets_Order_Lineitem=$buckets_Order_Lineitem,Buckets_Cust_Part_PartSupp=$buckets_Cust_Part_PartSupp,IsSupplierColumnTable=$isSupplierColumnTable,Buckets_Supplier=$buckets_Supplier,Redundancy=$redundantCopies,Persistence=$persistence,Persistence_Type=$persistence_type,numberOfLoadStages=$numberOfLoadStages,isParquet=$isParquet,createParquet=$createParquet,traceEvents=$traceEvents" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-cluster*tests.jar threadGroups = snappyThreads ; INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob - io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.benchmark.snappy.TPCH_Snappy_Query + io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.benchmark.snappy.tpch.QueryExecutionJob io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = - "queries=${queries},queryPlan=${queryPlan},spark.sql.shuffle.partitions=${shufflePartitions},spark.sql.inMemoryColumnarStorage.compressed=${inMemoryColumnarStorageCompressed},useIndex=${useIndex},resultCollection=${resultCollection},warmUpIterations=${warmUpIterations},actualRuns=${actualRuns}" + "queries=$queries,isDynamic=$isDynamic,resultCollection=$resultCollection,warmUpIterations=$warmUps,actualRuns=$actualRuns,threadNumber=1,traceEvents=$traceEvents,randomSeed=$randomSeed" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappy-cluster*tests.jar threadGroups = snappyThreads ; TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob - io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.benchmark.snappy.TPCH_Snappy_Query + io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.benchmark.snappy.tpch.QueryExecutionJob io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = - "queries=${queries},queryPlan=${queryPlan},spark.sql.shuffle.partitions=${shufflePartitions},spark.sql.inMemoryColumnarStorage.compressed=${inMemoryColumnarStorageCompressed},useIndex=${useIndex},resultCollection=${resultCollection},warmUpIterations=${warmUpIterations},actualRuns=${actualRuns}" + "queries=$queries,isDynamic=$isDynamic,resultCollection=$resultCollection,warmUpIterations=$warmUps,actualRuns=$actualRuns,threadNumber=1,traceEvents=$traceEvents,randomSeed=$randomSeed" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-cluster*tests.jar threadGroups = snappyThreads weight = 60 @@ -115,9 +115,9 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = Hy ; TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob - io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.benchmark.snappy.TPCH_Snappy_Query + io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.benchmark.snappy.tpch.QueryExecutionJob io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = - "queries=11-12-13-14-15-16-17,queryPlan=${queryPlan},spark.sql.shuffle.partitions=${shufflePartitions},spark.sql.inMemoryColumnarStorage.compressed=${inMemoryColumnarStorageCompressed},useIndex=${useIndex},resultCollection=${resultCollection},warmUpIterations=${warmUpIterations},actualRuns=${actualRuns}" + "queries=11-12-13-14-15-16-17,isDynamic=$isDynamic,resultCollection=$resultCollection,warmUpIterations=$warmUps,actualRuns=$actualRuns,threadNumber=1,traceEvents=$traceEvents,randomSeed=$randomSeed" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-cluster*tests.jar threadGroups = leadThreads maxThreads = 1 diff --git a/dtests/src/test/java/io/snappydata/hydra/distJoin/distJoinWithServerHA.conf b/dtests/src/test/java/io/snappydata/hydra/distJoin/distJoinWithServerHA.conf index aab85ef05b..e16e17da77 100644 --- a/dtests/src/test/java/io/snappydata/hydra/distJoin/distJoinWithServerHA.conf +++ b/dtests/src/test/java/io/snappydata/hydra/distJoin/distJoinWithServerHA.conf @@ -89,26 +89,26 @@ INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = b threadGroups = snappyThreads; INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob - io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.benchmark.snappy.TPCH_Snappy_Tables + io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.benchmark.snappy.tpch.TableCreationJob io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = - "dataLocation=${dataLocation},Buckets_Order_Lineitem=${buckets_Order_Lineitem},Buckets_Cust_Part_PartSupp=${buckets_Cust_Part_PartSupp},useIndex=${useIndex}" + "dataLocation=$dataLocation,Buckets_Order_Lineitem=$buckets_Order_Lineitem,Buckets_Cust_Part_PartSupp=$buckets_Cust_Part_PartSupp,IsSupplierColumnTable=$isSupplierColumnTable,Buckets_Supplier=$buckets_Supplier,Redundancy=$redundantCopies,Persistence=$persistence,Persistence_Type=$persistence_type,numberOfLoadStages=$numberOfLoadStages,isParquet=$isParquet,createParquet=$createParquet,traceEvents=$traceEvents" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-cluster*tests.jar threadGroups = snappyThreads ; INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob - io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.benchmark.snappy.TPCH_Snappy_Query + io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.benchmark.snappy.tpch.QueryExecutionJob io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = - "queries=${queries},queryPlan=${queryPlan},spark.sql.shuffle.partitions=${shufflePartitions},spark.sql.inMemoryColumnarStorage.compressed=${inMemoryColumnarStorageCompressed},useIndex=${useIndex},resultCollection=${resultCollection},warmUpIterations=${warmUpIterations},actualRuns=${actualRuns}" + "queries=$queries,isDynamic=$isDynamic,resultCollection=$resultCollection,warmUpIterations=$warmUps,actualRuns=$actualRuns,threadNumber=1,traceEvents=$traceEvents,randomSeed=$randomSeed" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-cluster*tests.jar threadGroups = snappyThreads ; TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob - io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.benchmark.snappy.TPCH_Snappy_Query + io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.benchmark.snappy.tpch.QueryExecutionJob io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = - "queries=${queries},queryPlan=${queryPlan},spark.sql.shuffle.partitions=${shufflePartitions},spark.sql.inMemoryColumnarStorage.compressed=${inMemoryColumnarStorageCompressed},useIndex=${useIndex},resultCollection=${resultCollection},warmUpIterations=${warmUpIterations},actualRuns=${actualRuns}" + "queries=$queries,isDynamic=$isDynamic,resultCollection=$resultCollection,warmUpIterations=$warmUps,actualRuns=$actualRuns,threadNumber=1,traceEvents=$traceEvents,randomSeed=$randomSeed" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-cluster*tests.jar threadGroups = snappyThreads weight = 60 @@ -116,9 +116,9 @@ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = Hy ; TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob - io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.benchmark.snappy.TPCH_Snappy_Query + io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.benchmark.snappy.tpch.QueryExecutionJob io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = - "queries=11-12-13-14-15-16-17,queryPlan=${queryPlan},spark.sql.shuffle.partitions=${shufflePartitions},spark.sql.inMemoryColumnarStorage.compressed=${inMemoryColumnarStorageCompressed},useIndex=${useIndex},resultCollection=${resultCollection},warmUpIterations=${warmUpIterations},actualRuns=${actualRuns}" + "queries=11-12-13-14-15-16-17,isDynamic=$isDynamic,resultCollection=$resultCollection,warmUpIterations=$warmUps,actualRuns=$actualRuns,threadNumber=1,traceEvents=$traceEvents,randomSeed=$randomSeed" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-cluster*tests.jar threadGroups = snappyStoreThreads maxThreads = 1 diff --git a/dtests/src/test/java/io/snappydata/hydra/installJar/installJar.bt b/dtests/src/test/java/io/snappydata/hydra/installJar/installJar.bt index 029ac52d39..bd60acacf3 100644 --- a/dtests/src/test/java/io/snappydata/hydra/installJar/installJar.bt +++ b/dtests/src/test/java/io/snappydata/hydra/installJar/installJar.bt @@ -1,25 +1,25 @@ io/snappydata/hydra/installJar/dynamicJarLoadingUsingSnappyJob.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 io/snappydata/hydra/installJar/dynamicJarLoadingUsingSnappyShell.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 io/snappydata/hydra/installJar/dynamicJarLoadingUsingSparkApp.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 io/snappydata/hydra/installJar/dynamicJarLoadingUsingSnappyJobsWithIdenticalNames.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 io/snappydata/hydra/installJar/verifyCleanupAfterSnappyJobExecution.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 \ No newline at end of file + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/longRun/longRunTest.bt b/dtests/src/test/java/io/snappydata/hydra/longRun/longRunTest.bt index 4a3e6dcbc1..2875106516 100644 --- a/dtests/src/test/java/io/snappydata/hydra/longRun/longRunTest.bt +++ b/dtests/src/test/java/io/snappydata/hydra/longRun/longRunTest.bt @@ -7,7 +7,7 @@ io/snappydata/hydra/longRun/longRunTestEmbededMode.conf A=snappyStore snappyStoreHosts=4 snappyStoreVMsPerHost=2 snappyStoreThreadsPerVM=4 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 redundantCopies=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="Colocated" @@ -17,7 +17,7 @@ io/snappydata/hydra/longRun/longRunTestEmbededMode.conf io/snappydata/hydra/longRun/longRunTestEmbededModeWithHA.conf A=snappyStore snappyStoreHosts=4 snappyStoreVMsPerHost=2 snappyStoreThreadsPerVM=4 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 redundantCopies=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="Colocated" @@ -27,7 +27,7 @@ io/snappydata/hydra/longRun/longRunTestEmbededModeWithHA.conf io/snappydata/hydra/longRun/longRunTestSplitMode.conf A=snappyStore snappyStoreHosts=4 snappyStoreVMsPerHost=2 snappyStoreThreadsPerVM=4 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 redundantCopies=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" diff --git a/dtests/src/test/java/io/snappydata/hydra/northwind/local.northWind.conf b/dtests/src/test/java/io/snappydata/hydra/northwind/local.northWind.conf index 91dffdfae1..b9e84b0884 100644 --- a/dtests/src/test/java/io/snappydata/hydra/northwind/local.northWind.conf +++ b/dtests/src/test/java/io/snappydata/hydra/northwind/local.northWind.conf @@ -1,7 +1,14 @@ +/* hydra.HostPrms-hostNames = fcn "hydra.TestConfigFcns.pool(\" \", ${snappyStoreHosts})" ncf -fcn "hydra.TestConfigFcns.pool(\" \", ${leadHosts})" ncf -fcn "hydra.TestConfigFcns.pool(\" \", ${locatorHosts})" ncf +fcn "hydra.TestConfigFcns.pool(\" \", ${leadHosts})" ncf +fcn "hydra.TestConfigFcns.pool(\" \", ${locatorHosts})" ncf fcn "hydra.TestConfigFcns.pool(\" \", ${workerHosts})" ncf ; +*/ +io.snappydata.hydra.cluster.SnappyPrms-serverLauncherProps = " -heap-size=4g -memory-size=8g -J-XX:+UnlockDiagnosticVMOptions -J-XX:ParGCCardsPerStrideChunk=32768 -member-timeout=60000 -J-Dp2p.socket_timeout=60000 "; +io.snappydata.hydra.cluster.SnappyPrms-locatorLauncherProps = " -member-timeout=60000 -J-Dp2p.socket_timeout=60000 "; + +io.snappydata.hydra.cluster.SnappyPrms-executorMemory = 4g; +io.snappydata.hydra.cluster.SnappyPrms-leadMemory = 4g; diff --git a/dtests/src/test/java/io/snappydata/hydra/northwind/northWind.bt b/dtests/src/test/java/io/snappydata/hydra/northwind/northWind.bt index db71b11497..c2b1b2e99e 100644 --- a/dtests/src/test/java/io/snappydata/hydra/northwind/northWind.bt +++ b/dtests/src/test/java/io/snappydata/hydra/northwind/northWind.bt @@ -1,7 +1,7 @@ io/snappydata/hydra/northwind/nwReplicatedRowTablesTest.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="ReplicatedRow" @@ -13,7 +13,7 @@ io/snappydata/hydra/northwind/nwReplicatedRowTablesTest.conf io/snappydata/hydra/northwind/nwPartitionedRowTablesTest.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="PartitionedRow" @@ -25,7 +25,7 @@ io/snappydata/hydra/northwind/nwPartitionedRowTablesTest.conf io/snappydata/hydra/northwind/nwColumnTablesTest.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="Column" @@ -37,7 +37,7 @@ io/snappydata/hydra/northwind/nwColumnTablesTest.conf io/snappydata/hydra/northwind/nwColocatedTablesTest.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="Colocated" @@ -49,7 +49,7 @@ io/snappydata/hydra/northwind/nwColocatedTablesTest.conf io/snappydata/hydra/northwind/nwPersistentReplicatedRowTablesTest.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="ReplicatedRow" @@ -60,7 +60,7 @@ io/snappydata/hydra/northwind/nwPersistentReplicatedRowTablesTest.conf io/snappydata/hydra/northwind/nwPersistentPartitionedRowTablesTest.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="PartitionedRow" @@ -71,7 +71,7 @@ io/snappydata/hydra/northwind/nwPersistentPartitionedRowTablesTest.conf io/snappydata/hydra/northwind/nwPersistentColumnTablesTest.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="Column" @@ -82,7 +82,7 @@ io/snappydata/hydra/northwind/nwPersistentColumnTablesTest.conf io/snappydata/hydra/northwind/nwPersistentColocatedTablesTest.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="Colocated" @@ -93,7 +93,7 @@ io/snappydata/hydra/northwind/nwPersistentColocatedTablesTest.conf io/snappydata/hydra/northwind/nwPartitionedRowTablesWithEvictionTest.conf A=snappyStore snappyStoreHosts=2 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="PartitionedRow" @@ -104,7 +104,7 @@ io/snappydata/hydra/northwind/nwPartitionedRowTablesWithEvictionTest.conf io/snappydata/hydra/northwind/nwPartitionedRowTablesWithEvictionOverflowTest.conf A=snappyStore snappyStoreHosts=2 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="PartitionedRow" @@ -115,7 +115,7 @@ io/snappydata/hydra/northwind/nwPartitionedRowTablesWithEvictionOverflowTest.con io/snappydata/hydra/northwind/nwPersistentPartitionedRowTablesWithEvictionOverflowTest.conf A=snappyStore snappyStoreHosts=2 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="PartitionedRow" diff --git a/dtests/src/test/java/io/snappydata/hydra/northwind/northWindHA.bt b/dtests/src/test/java/io/snappydata/hydra/northwind/northWindHA.bt index 6512a8e3ba..914c6260d4 100644 --- a/dtests/src/test/java/io/snappydata/hydra/northwind/northWindHA.bt +++ b/dtests/src/test/java/io/snappydata/hydra/northwind/northWindHA.bt @@ -1,7 +1,7 @@ io/snappydata/hydra/northwind/nwReplicatedRowTablesTestWithServerHA.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="ReplicatedRow" @@ -14,7 +14,7 @@ io/snappydata/hydra/northwind/nwReplicatedRowTablesTestWithServerHA.conf io/snappydata/hydra/northwind/nwPartitionedRowTablesTestWithServerHA.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="PartitionedRow" @@ -27,7 +27,7 @@ io/snappydata/hydra/northwind/nwPartitionedRowTablesTestWithServerHA.conf io/snappydata/hydra/northwind/nwColumnTablesTestWithServerHA.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="Column" @@ -40,7 +40,7 @@ io/snappydata/hydra/northwind/nwColumnTablesTestWithServerHA.conf io/snappydata/hydra/northwind/nwColocatedTablesTestWithServerHA.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="Colocated" @@ -53,7 +53,7 @@ io/snappydata/hydra/northwind/nwColocatedTablesTestWithServerHA.conf io/snappydata/hydra/northwind/nwReplicatedRowTablesTestWithLeadHA.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="ReplicatedRow" @@ -66,7 +66,7 @@ io/snappydata/hydra/northwind/nwReplicatedRowTablesTestWithLeadHA.conf io/snappydata/hydra/northwind/nwPartitionedRowTablesTestWithLeadHA.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="PartitionedRow" @@ -79,7 +79,7 @@ io/snappydata/hydra/northwind/nwPartitionedRowTablesTestWithLeadHA.conf io/snappydata/hydra/northwind/nwColumnTablesTestWithLeadHA.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="Column" @@ -92,7 +92,7 @@ io/snappydata/hydra/northwind/nwColumnTablesTestWithLeadHA.conf io/snappydata/hydra/northwind/nwColocatedTablesTestWithLeadHA.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="Colocated" @@ -105,7 +105,7 @@ io/snappydata/hydra/northwind/nwColocatedTablesTestWithLeadHA.conf io/snappydata/hydra/northwind/nwPersistentReplicatedRowTablesTestWithServerHA.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="ReplicatedRow" @@ -117,7 +117,7 @@ io/snappydata/hydra/northwind/nwPersistentReplicatedRowTablesTestWithServerHA.co io/snappydata/hydra/northwind/nwPersistentReplicatedRowTablesTestWithLeadHA.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="ReplicatedRow" @@ -129,7 +129,7 @@ io/snappydata/hydra/northwind/nwPersistentReplicatedRowTablesTestWithLeadHA.conf io/snappydata/hydra/northwind/nwPersistentPartitionedRowTablesTestWithServerHA.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="PartitionedRow" @@ -141,7 +141,7 @@ io/snappydata/hydra/northwind/nwPersistentPartitionedRowTablesTestWithServerHA.c io/snappydata/hydra/northwind/nwPersistentPartitionedRowTablesTestWithLeadHA.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="PartitionedRow" @@ -153,7 +153,7 @@ io/snappydata/hydra/northwind/nwPersistentPartitionedRowTablesTestWithLeadHA.con io/snappydata/hydra/northwind/nwPersistentColumnTablesTestWithServerHA.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="Column" @@ -165,7 +165,7 @@ io/snappydata/hydra/northwind/nwPersistentColumnTablesTestWithServerHA.conf io/snappydata/hydra/northwind/nwPersistentColumnTablesTestWithLeadHA.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="Column" @@ -177,7 +177,7 @@ io/snappydata/hydra/northwind/nwPersistentColumnTablesTestWithLeadHA.conf io/snappydata/hydra/northwind/nwPersistentColocatedTablesTestWithServerHA.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="Colocated" @@ -189,7 +189,7 @@ io/snappydata/hydra/northwind/nwPersistentColocatedTablesTestWithServerHA.conf io/snappydata/hydra/northwind/nwPersistentColocatedTablesTestWithLeadHA.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="Colocated" @@ -201,7 +201,7 @@ io/snappydata/hydra/northwind/nwPersistentColocatedTablesTestWithLeadHA.conf io/snappydata/hydra/northwind/nwReplicatedRowTablesTestWithLocatorHA.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="ReplicatedRow" @@ -277,7 +277,7 @@ io/snappydata/hydra/northwind/nwPersistentPartitionedRowTablesTestWithLocatorHA. io/snappydata/hydra/northwind/nwPersistentColumnTablesTestWithLocatorHA.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" tableType="Column" diff --git a/dtests/src/test/java/io/snappydata/hydra/northwind/northWindUseCase.bt b/dtests/src/test/java/io/snappydata/hydra/northwind/northWindUseCase.bt index 088f0db31c..6f6dc0e337 100644 --- a/dtests/src/test/java/io/snappydata/hydra/northwind/northWindUseCase.bt +++ b/dtests/src/test/java/io/snappydata/hydra/northwind/northWindUseCase.bt @@ -1,6 +1,6 @@ io/snappydata/hydra/northwind/northwindUseCase.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="$GEMFIRE/../../../tests/common/src/main/resources/northwind/" diff --git a/dtests/src/test/java/io/snappydata/hydra/northwind/nwPartitionedRowTablesWithEvictionOverflowTest.conf b/dtests/src/test/java/io/snappydata/hydra/northwind/nwPartitionedRowTablesWithEvictionOverflowTest.conf index b331abfcae..689c390c82 100644 --- a/dtests/src/test/java/io/snappydata/hydra/northwind/nwPartitionedRowTablesWithEvictionOverflowTest.conf +++ b/dtests/src/test/java/io/snappydata/hydra/northwind/nwPartitionedRowTablesWithEvictionOverflowTest.conf @@ -19,15 +19,17 @@ INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = H io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "dataFilesLocation=${dataFilesLocation},tableType=${tableType},fullResultSetValidation=${fullResultSetValidation},isSmokeRun=${isSmokeRun},numRowsValidation=${numRowsValidation}" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar - threadGroups = snappyStoreThreads + threadGroups = snappyThreads ; +/* TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSparkJob io.snappydata.hydra.cluster.SnappyPrms-sparkJobClassNames = io.snappydata.hydra.northwind.ValidateNWQueriesApp io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "${dataFilesLocation} ${tableType} ${fullResultSetValidation} ${isSmokeRun} ${numRowsValidation}" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar threadGroups = leadThreads ; +*/ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSQLScripts io.snappydata.hydra.cluster.SnappyPrms-sqlScriptNames = nw_queries.sql diff --git a/dtests/src/test/java/io/snappydata/hydra/northwind/nwPartitionedRowTablesWithEvictionTest.conf b/dtests/src/test/java/io/snappydata/hydra/northwind/nwPartitionedRowTablesWithEvictionTest.conf index 2a02f027c2..1e0de68b53 100644 --- a/dtests/src/test/java/io/snappydata/hydra/northwind/nwPartitionedRowTablesWithEvictionTest.conf +++ b/dtests/src/test/java/io/snappydata/hydra/northwind/nwPartitionedRowTablesWithEvictionTest.conf @@ -19,15 +19,16 @@ INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = H io.snappydata.hydra.cluster.SnappyPrms-appPropsForJobServer = "dataFilesLocation=${dataFilesLocation},tableType=${tableType},fullResultSetValidation=${fullResultSetValidation},isSmokeRun=${isSmokeRun},numRowsValidation=${numRowsValidation}" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar - threadGroups = snappyStoreThreads + threadGroups = snappyThreads ; - +/* TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSparkJob io.snappydata.hydra.cluster.SnappyPrms-sparkJobClassNames = io.snappydata.hydra.northwind.ValidateNWQueriesApp io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "${dataFilesLocation} ${tableType} ${fullResultSetValidation} ${isSmokeRun} ${numRowsValidation}" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar threadGroups = leadThreads ; +*/ TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSQLScripts io.snappydata.hydra.cluster.SnappyPrms-sqlScriptNames = nw_queries.sql diff --git a/dtests/src/test/java/io/snappydata/hydra/northwind/nwPersistentPartitionedRowTablesWithEvictionOverflowTest.conf b/dtests/src/test/java/io/snappydata/hydra/northwind/nwPersistentPartitionedRowTablesWithEvictionOverflowTest.conf index b002c44bda..1833f32cfa 100644 --- a/dtests/src/test/java/io/snappydata/hydra/northwind/nwPersistentPartitionedRowTablesWithEvictionOverflowTest.conf +++ b/dtests/src/test/java/io/snappydata/hydra/northwind/nwPersistentPartitionedRowTablesWithEvictionOverflowTest.conf @@ -18,7 +18,7 @@ INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = H io.snappydata.hydra.cluster.SnappyPrms-sparkJobClassNames = io.snappydata.hydra.northwind.ValidateNWQueriesApp io.snappydata.hydra.cluster.SnappyPrms-userAppArgs = "${dataFilesLocation} ${tableType} ${fullResultSetValidation} ${isSmokeRun} ${numRowsValidation}" io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar - threadGroups = leadThreads + threadGroups = snappyThreads ; TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob diff --git a/dtests/src/test/java/io/snappydata/hydra/northwind/startDualModeCluster.conf b/dtests/src/test/java/io/snappydata/hydra/northwind/startDualModeCluster.conf index 08e49b37fa..4344c510ba 100644 --- a/dtests/src/test/java/io/snappydata/hydra/northwind/startDualModeCluster.conf +++ b/dtests/src/test/java/io/snappydata/hydra/northwind/startDualModeCluster.conf @@ -88,3 +88,5 @@ INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = H INITTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_recordProcessIDWithHost runMode = always threadGroups = snappyThreads, locatorThreads, snappyStoreThreads, leadThreads, workerThreads; + +INCLUDE $JTESTS/io/snappydata/hydra/cluster/expectedExceptionList.inc; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/northwind/stopDualModeCluster.conf b/dtests/src/test/java/io/snappydata/hydra/northwind/stopDualModeCluster.conf index b473f1fa79..45aeb69631 100644 --- a/dtests/src/test/java/io/snappydata/hydra/northwind/stopDualModeCluster.conf +++ b/dtests/src/test/java/io/snappydata/hydra/northwind/stopDualModeCluster.conf @@ -1,7 +1,7 @@ hydra.Prms-testRequirement = "Test to stop the snappy embedded and split mode cluster started using startDualModeCluster.conf"; hydra.Prms-testDescription = "This test stops the snappy embedded as well as split mode cluster started using startDualModeCluster.conf."; -ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_dumpStacks +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_dumpStacks clientNames = locator1; ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_stopSparkCluster @@ -16,6 +16,9 @@ ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = H ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_cleanUpSnappyProcessesOnFailure clientNames = locator1; +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_checkSuspectStrings + clientNames = locator1; + hydra.Prms-totalTaskTimeSec = 100; hydra.Prms-maxResultWaitSec = 3600; diff --git a/dtests/src/test/java/io/snappydata/hydra/recovery/clusterRestartWithPersistentRecovery.bt b/dtests/src/test/java/io/snappydata/hydra/recovery/clusterRestartWithPersistentRecovery.bt index ac8ebf7cef..795e212813 100644 --- a/dtests/src/test/java/io/snappydata/hydra/recovery/clusterRestartWithPersistentRecovery.bt +++ b/dtests/src/test/java/io/snappydata/hydra/recovery/clusterRestartWithPersistentRecovery.bt @@ -1,7 +1,7 @@ io/snappydata/hydra/recovery/clusterRestartWithPersistentRecovery.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=1 workerVMsPerHost=1 workerThreadsPerVM=1 dataLocation="/export/shared/QA_DATA/RCORE_DATA/" diff --git a/dtests/src/test/java/io/snappydata/hydra/security/stopSnappyCluster.conf b/dtests/src/test/java/io/snappydata/hydra/security/stopSnappyCluster.conf index cd80c330b8..c8e4761112 100644 --- a/dtests/src/test/java/io/snappydata/hydra/security/stopSnappyCluster.conf +++ b/dtests/src/test/java/io/snappydata/hydra/security/stopSnappyCluster.conf @@ -1,7 +1,7 @@ hydra.Prms-testRequirement = "Test to stop the snappy embedded and split mode cluster started using startDualModeCluster.conf"; hydra.Prms-testDescription = "This test stops the snappy embedded as well as split mode cluster started using startDualModeCluster.conf."; -ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_dumpStacks +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_dumpStacks clientNames = locator1; ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_stopSnappyCluster diff --git a/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/SnapshotIsolationTest.java b/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/SnapshotIsolationTest.java index ed48144818..7207c73a27 100644 --- a/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/SnapshotIsolationTest.java +++ b/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/SnapshotIsolationTest.java @@ -783,10 +783,15 @@ public static void HydraTask_verifyResults() { public void verifyResults() { try { if (SnapshotIsolationDMLOpsBB.getBB().getSharedMap().containsKey("dmlThreads")) { + ArrayList selectThreads = null; ArrayList dmlthreads = (ArrayList)SnapshotIsolationDMLOpsBB.getBB().getSharedMap().get("dmlThreads"); for (int i : dmlthreads) { - if(hasDuplicateSchemas) - replayOpsInDerby(i + "_" + getMyTid()); + if(hasDuplicateSchemas) { + if (SnapshotIsolationDMLOpsBB.getBB().getSharedMap().containsKey("selectThreads")) + selectThreads = (ArrayList)SnapshotIsolationDMLOpsBB.getBB().getSharedMap().get("selectThreads"); + for (int selectTid : selectThreads) + replayOpsInDerby(i + "_" + selectTid); + } else replayOpsInDerby(String.valueOf(i)); } diff --git a/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/local.snapshotIsolation.conf b/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/local.snapshotIsolation.conf new file mode 100644 index 0000000000..f2be760208 --- /dev/null +++ b/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/local.snapshotIsolation.conf @@ -0,0 +1,12 @@ +hydra.Prms-removeDiskFilesAfterTest = true; +hydra.Prms-extraDerbyServerVMArgs += " -Xmx4096m -Dderby.storage.pageCacheSize=32000 -Dderby.locks.waitTimeout=60 -Dderby.locks.deadlockTimeout=40 "; +hydra.Prms-extraDerbyServerVMArgs += " -XX:MaxPermSize=128m "; + +io.snappydata.hydra.cluster.SnappyPrms-serverLauncherProps = " -heap-size=4g -memory-size=3g "; +//io.snappydata.hydra.cluster.SnappyPrms-serverLauncherProps += " -J-Dgemfire.VersionVector.VERBOSE=true -J-Dgemfirexd.debug.true=TraceTran,TraceQuery "; +io.snappydata.hydra.cluster.SnappyPrms-locatorLauncherProps = " "; + +io.snappydata.hydra.cluster.SnappyPrms-executorMemory = 4g; +io.snappydata.hydra.cluster.SnappyPrms-leadMemory = 4g; + +//io.snappydata.hydra.cluster.SnappyPrms-logLevel = fine; diff --git a/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/testSnapshotWithoutDMLOPsJDBCClient_ColocatedTables.conf b/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/testSnapshotWithoutDMLOPsJDBCClient_ColocatedTables.conf index 010a17bba6..bac76878ea 100644 --- a/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/testSnapshotWithoutDMLOPsJDBCClient_ColocatedTables.conf +++ b/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/testSnapshotWithoutDMLOPsJDBCClient_ColocatedTables.conf @@ -6,5 +6,4 @@ INCLUDE $JTESTS/io/snappydata/hydra/snapshotIsolation/colocatedTablesExtn.inc; //task for running select queries TASK taskClass = io.snappydata.hydra.snapshotIsolation.SnapshotIsolationTest taskMethod = HydraTask_executeQueries - maxThreads = 1 threadGroups = leadThreads,snappyThreads,snappyStoreThreads; diff --git a/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/testSnapshotWithoutDMLOPsJDBCClient_ColumnTables.conf b/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/testSnapshotWithoutDMLOPsJDBCClient_ColumnTables.conf index ed1ca179a2..fd1d2c899d 100644 --- a/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/testSnapshotWithoutDMLOPsJDBCClient_ColumnTables.conf +++ b/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/testSnapshotWithoutDMLOPsJDBCClient_ColumnTables.conf @@ -6,5 +6,4 @@ INCLUDE $JTESTS/io/snappydata/hydra/snapshotIsolation/columnTableExtn.inc; //task for running select queries TASK taskClass = io.snappydata.hydra.snapshotIsolation.SnapshotIsolationTest taskMethod = HydraTask_executeQueries - maxThreads = 1 threadGroups = leadThreads,snappyThreads,snappyStoreThreads; diff --git a/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/testSnapshotWithoutDMLOPsJDBCClient_PartitionedRow.conf b/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/testSnapshotWithoutDMLOPsJDBCClient_PartitionedRow.conf index 0ea888ec54..d81c17072c 100644 --- a/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/testSnapshotWithoutDMLOPsJDBCClient_PartitionedRow.conf +++ b/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/testSnapshotWithoutDMLOPsJDBCClient_PartitionedRow.conf @@ -6,5 +6,4 @@ INCLUDE $JTESTS/io/snappydata/hydra/snapshotIsolation/partitionedRowExtn.inc; //task for running select queries TASK taskClass = io.snappydata.hydra.snapshotIsolation.SnapshotIsolationTest taskMethod = HydraTask_executeQueries - maxThreads = 1 threadGroups = leadThreads,snappyThreads,snappyStoreThreads; diff --git a/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/testSnapshotWithoutDMLOPsJDBCClient_ReplicatedRow.conf b/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/testSnapshotWithoutDMLOPsJDBCClient_ReplicatedRow.conf index c63bb0081c..0cc8d25059 100644 --- a/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/testSnapshotWithoutDMLOPsJDBCClient_ReplicatedRow.conf +++ b/dtests/src/test/java/io/snappydata/hydra/snapshotIsolation/testSnapshotWithoutDMLOPsJDBCClient_ReplicatedRow.conf @@ -6,5 +6,4 @@ INCLUDE $JTESTS/io/snappydata/hydra/snapshotIsolation/replicatedRowExtn.inc; //task for running select queries TASK taskClass = io.snappydata.hydra.snapshotIsolation.SnapshotIsolationTest taskMethod = HydraTask_executeQueries - maxThreads = 1 threadGroups = leadThreads,snappyThreads,snappyStoreThreads; diff --git a/dtests/src/test/java/io/snappydata/hydra/startUp/nwTestWithAllServerHA_meanKill_clusterRestartWithRandomOrderForServerStartUp.bt b/dtests/src/test/java/io/snappydata/hydra/startUp/nwTestWithAllServerHA_meanKill_clusterRestartWithRandomOrderForServerStartUp.bt index 20a2b4652a..121714625e 100644 --- a/dtests/src/test/java/io/snappydata/hydra/startUp/nwTestWithAllServerHA_meanKill_clusterRestartWithRandomOrderForServerStartUp.bt +++ b/dtests/src/test/java/io/snappydata/hydra/startUp/nwTestWithAllServerHA_meanKill_clusterRestartWithRandomOrderForServerStartUp.bt @@ -1,7 +1,7 @@ io/snappydata/hydra/startUp/nwTestWithAllServerHA_meanKill_clusterRestartWithRandomOrderForServerStartUp.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="/export/shared/QA_DATA/northWind_1GB/" tableType="Column" diff --git a/dtests/src/test/java/io/snappydata/hydra/startUp/startUp.bt b/dtests/src/test/java/io/snappydata/hydra/startUp/startUp.bt index f3f052a5aa..4002036eab 100644 --- a/dtests/src/test/java/io/snappydata/hydra/startUp/startUp.bt +++ b/dtests/src/test/java/io/snappydata/hydra/startUp/startUp.bt @@ -1,7 +1,7 @@ io/snappydata/hydra/startUp/nwTestWithServerHA_clusterRestart.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="/export/shared/QA_DATA/northWind_1GB/" tableType="Column","PartitionedRow","ReplicatedRow","Colocated" @@ -14,7 +14,7 @@ io/snappydata/hydra/startUp/nwTestWithServerHA_clusterRestart.conf io/snappydata/hydra/startUp/nwTestWithLeadHA_clusterRestart.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="/export/shared/QA_DATA/northWind_1GB/" tableType="Column","PartitionedRow","ReplicatedRow","Colocated" @@ -66,7 +66,7 @@ io/snappydata/hydra/startUp/nwColumnTablesTestWithDdlOpAfterLocatorStop_ClusterR io/snappydata/hydra/startUp/nwColumnTablesTestWithDdlOpDuringLocatorHA.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="/export/shared/QA_DATA/northWind_1GB/" tableType="Column" @@ -91,7 +91,7 @@ io/snappydata/hydra/startUp/nwPersistentColumnTablesTestWithDdlOpDuringLocatorHA io/snappydata/hydra/startUp/nwPersistentColumnTablesTestWithDdlOpAfterLocatorStop_ClusterRestart.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="/export/shared/QA_DATA/northWind_1GB/" tableType="Column" @@ -103,7 +103,7 @@ io/snappydata/hydra/startUp/nwPersistentColumnTablesTestWithDdlOpAfterLocatorSto io/snappydata/hydra/startUp/nwPersistentColumnTablesTestWithDdlOpAfterAllLocatorStop_ClusterRestart.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="/export/shared/QA_DATA/northWind_1GB/" tableType="Column" @@ -115,7 +115,7 @@ io/snappydata/hydra/startUp/nwPersistentColumnTablesTestWithDdlOpAfterAllLocator io/snappydata/hydra/startUp/executeAndVerifyOpsAfterAllLocatorsStopAndClusterRestart.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 dataFilesLocation="/export/shared/QA_DATA/northWind_1GB/" tableType="Column" @@ -127,7 +127,7 @@ io/snappydata/hydra/startUp/executeAndVerifyOpsAfterAllLocatorsStopAndClusterRes io/snappydata/hydra/startUp/nwTestWithServerHAWithRebalance_clusterRestart.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 E=newStore newStoreHosts=1 newStoreVMsPerHost=1 newStoreThreadsPerVM=1 dataFilesLocation="/export/shared/QA_DATA/northWind_1GB/" @@ -140,7 +140,7 @@ io/snappydata/hydra/startUp/nwTestWithServerHAWithRebalance_clusterRestart.conf io/snappydata/hydra/startUp/nwTestWithOpsDuringServerHA_clusterRestart.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 E=newStore newStoreHosts=1 newStoreVMsPerHost=1 newStoreThreadsPerVM=1 dataFilesLocation="/export/shared/QA_DATA/northWind_1GB/" @@ -153,7 +153,7 @@ io/snappydata/hydra/startUp/nwTestWithOpsDuringServerHA_clusterRestart.conf io/snappydata/hydra/startUp/nwTestWithNewServerRebalance_clusterRestart.conf A=snappyStore snappyStoreHosts=3 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=1 B=lead leadHosts=2 leadVMsPerHost=1 leadThreadsPerVM=2 - C=locator locatorHosts=3 locatorVMsPerHost=1 locatorThreadsPerVM=1 + C=locator locatorHosts=2 locatorVMsPerHost=1 locatorThreadsPerVM=1 D=worker workerHosts=3 workerVMsPerHost=1 workerThreadsPerVM=1 E=newStore newStoreHosts=1 newStoreVMsPerHost=1 newStoreThreadsPerVM=1 dataFilesLocation="/export/shared/QA_DATA/northWind_1GB/" diff --git a/dtests/src/test/java/io/snappydata/hydra/storeTests/stopSnappyCluster.inc b/dtests/src/test/java/io/snappydata/hydra/storeTests/stopSnappyCluster.inc index 9e26104166..57ddbc9b76 100644 --- a/dtests/src/test/java/io/snappydata/hydra/storeTests/stopSnappyCluster.inc +++ b/dtests/src/test/java/io/snappydata/hydra/storeTests/stopSnappyCluster.inc @@ -1,3 +1,6 @@ +ENDTASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_dumpStacks + clientNames = locator1; + ENDTASK taskClass = sql.snappy.SnappyTest taskMethod = HydraTask_stopSnappyCluster clientNames = locator1; diff --git a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/SnappyDMLOpsUtil.java b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/SnappyDMLOpsUtil.java index 2e1cc24520..e3215e4ec7 100644 --- a/dtests/src/test/java/io/snappydata/hydra/testDMLOps/SnappyDMLOpsUtil.java +++ b/dtests/src/test/java/io/snappydata/hydra/testDMLOps/SnappyDMLOpsUtil.java @@ -967,9 +967,9 @@ public void performUpdate() { Log.getLogWriter().info("Updated " + numRows + " rows in snappy."); if (hasDerbyServer) { dConn = derbyTestUtils.getDerbyConnection(); - Log.getLogWriter().info("Executing " + stmt + " on derby."); if (stmt.toUpperCase().contains("SELECT")) - getAndExecuteSelect(dConn, stmt, true); + getAndExecuteSelect(dConn,stmt,true); + Log.getLogWriter().info("Executing " + stmt + " on derby."); int derbyRows = dConn.createStatement().executeUpdate(stmt); Log.getLogWriter().info("Updated " + derbyRows + " rows in derby."); if (numRows != derbyRows) { @@ -1016,9 +1016,9 @@ public void performDelete() { Log.getLogWriter().info("Deleted " + numRows + " rows in snappy."); if (hasDerbyServer) { dConn = derbyTestUtils.getDerbyConnection(); - Log.getLogWriter().info("Executing " + stmt + " on derby."); if (stmt.toUpperCase().contains("SELECT")) - getAndExecuteSelect(dConn, stmt, true); + getAndExecuteSelect(dConn,stmt,true); + Log.getLogWriter().info("Executing " + stmt + " on derby."); int derbyRows = dConn.createStatement().executeUpdate(stmt); Log.getLogWriter().info("Deleted " + derbyRows + " rows in derby."); if (numRows != derbyRows) { diff --git a/dtests/src/test/java/io/snappydata/hydra/testEndToEndValidation/testEndToEndValidation.bt b/dtests/src/test/java/io/snappydata/hydra/testEndToEndValidation/testEndToEndValidation.bt new file mode 100644 index 0000000000..493473137e --- /dev/null +++ b/dtests/src/test/java/io/snappydata/hydra/testEndToEndValidation/testEndToEndValidation.bt @@ -0,0 +1,23 @@ +io/snappydata/hydra/testEndToEndValidation/testSnappyJobFailure.conf + A=snappyStore snappyStoreHosts=1 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=1 workerVMsPerHost=1 workerThreadsPerVM=1 + +io/snappydata/hydra/testEndToEndValidation/testSnappyJobHang.conf + A=snappyStore snappyStoreHosts=1 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=1 workerVMsPerHost=1 workerThreadsPerVM=1 + +io/snappydata/hydra/testEndToEndValidation/testSparkAppFailure.conf + A=snappyStore snappyStoreHosts=1 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=1 workerVMsPerHost=1 workerThreadsPerVM=1 + +io/snappydata/hydra/testEndToEndValidation/testSparkAppHang.conf + A=snappyStore snappyStoreHosts=1 snappyStoreVMsPerHost=1 snappyStoreThreadsPerVM=2 + B=lead leadHosts=1 leadVMsPerHost=1 leadThreadsPerVM=2 + C=locator locatorHosts=1 locatorVMsPerHost=1 locatorThreadsPerVM=1 + D=worker workerHosts=1 workerVMsPerHost=1 workerThreadsPerVM=1 diff --git a/dtests/src/test/java/io/snappydata/hydra/testEndToEndValidation/testSnappyJobFailure.conf b/dtests/src/test/java/io/snappydata/hydra/testEndToEndValidation/testSnappyJobFailure.conf new file mode 100644 index 0000000000..9addaffb20 --- /dev/null +++ b/dtests/src/test/java/io/snappydata/hydra/testEndToEndValidation/testSnappyJobFailure.conf @@ -0,0 +1,18 @@ +hydra.Prms-testRequirement = "Test to verify hydra reports failure when the snappy job fails"; +hydra.Prms-testDescription = " +This test starts the snappy cluster and spark cluster. +Test then runs the snappy job that throws a TestException and eventually cause hydra test to fail."; + +INCLUDE $JTESTS/io/snappydata/hydra/northwind/startDualModeCluster.conf; + +TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob + io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.hydra.testEndToEndValidation.ValidateFailureScenarioJob + io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar + threadGroups = snappyThreads + maxTimesToRun = 1 + ; + + +INCLUDE $JTESTS/io/snappydata/hydra/northwind/stopDualModeCluster.conf; +hydra.Prms-totalTaskTimeSec = 100; +hydra.Prms-maxResultWaitSec = 120; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/testEndToEndValidation/testSnappyJobHang.conf b/dtests/src/test/java/io/snappydata/hydra/testEndToEndValidation/testSnappyJobHang.conf new file mode 100644 index 0000000000..cffb0705c5 --- /dev/null +++ b/dtests/src/test/java/io/snappydata/hydra/testEndToEndValidation/testSnappyJobHang.conf @@ -0,0 +1,19 @@ +hydra.Prms-testRequirement = "Test to verify hydra reports failure when the snappy job hangs"; +hydra.Prms-testDescription = " +This test starts the snappy cluster and spark cluster. +Test then runs the snappy job that does not complete execution before hydra times out and eventually +cause hydra test to fail."; + +INCLUDE $JTESTS/io/snappydata/hydra/northwind/startDualModeCluster.conf; + +TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSnappyJob + io.snappydata.hydra.cluster.SnappyPrms-jobClassNames = io.snappydata.hydra.testEndToEndValidation.ValidateHangScenarioJob + io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar + threadGroups = snappyThreads + maxTimesToRun = 1 + ; + +INCLUDE $JTESTS/io/snappydata/hydra/northwind/stopDualModeCluster.conf; + +hydra.Prms-totalTaskTimeSec = 100; +hydra.Prms-maxResultWaitSec = 120; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/testEndToEndValidation/testSparkAppFailure.conf b/dtests/src/test/java/io/snappydata/hydra/testEndToEndValidation/testSparkAppFailure.conf new file mode 100644 index 0000000000..15054e7b38 --- /dev/null +++ b/dtests/src/test/java/io/snappydata/hydra/testEndToEndValidation/testSparkAppFailure.conf @@ -0,0 +1,18 @@ +hydra.Prms-testRequirement = "Test to verify hydra reports failure when the spark app fails"; +hydra.Prms-testDescription = " +This test starts the snappy cluster and spark cluster. +Test then runs a spark app that throws an Exception and eventually cause hydra test to fail."; + +INCLUDE $JTESTS/io/snappydata/hydra/northwind/startDualModeCluster.conf; + +TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSparkJob + io.snappydata.hydra.cluster.SnappyPrms-sparkJobClassNames = io.snappydata.hydra.testEndToEndValidation.ValidateFailureScenarioApp + io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar + threadGroups = snappyThreads + maxTimesToRun = 1 + ; + +INCLUDE $JTESTS/io/snappydata/hydra/northwind/stopDualModeCluster.conf; + +hydra.Prms-totalTaskTimeSec = 100; +hydra.Prms-maxResultWaitSec = 120; \ No newline at end of file diff --git a/dtests/src/test/java/io/snappydata/hydra/testEndToEndValidation/testSparkAppHang.conf b/dtests/src/test/java/io/snappydata/hydra/testEndToEndValidation/testSparkAppHang.conf new file mode 100644 index 0000000000..9c70f28440 --- /dev/null +++ b/dtests/src/test/java/io/snappydata/hydra/testEndToEndValidation/testSparkAppHang.conf @@ -0,0 +1,19 @@ +hydra.Prms-testRequirement = "Test to verify hydra reports failure when the spark app hangs"; +hydra.Prms-testDescription = " +This test starts the snappy cluster and spark cluster. +Test then runs a spark app that that does not complete execution before hydra times out and + eventually cause hydra test to fail."; + +INCLUDE $JTESTS/io/snappydata/hydra/northwind/startDualModeCluster.conf; + +TASK taskClass = io.snappydata.hydra.cluster.SnappyTest taskMethod = HydraTask_executeSparkJob + io.snappydata.hydra.cluster.SnappyPrms-sparkJobClassNames = io.snappydata.hydra.testEndToEndValidation.ValidateHangScenarioApp + io.snappydata.hydra.cluster.SnappyPrms-userAppJar = snappydata-store-scala-tests*tests.jar + threadGroups = snappyThreads + maxTimesToRun = 1 + ; + +INCLUDE $JTESTS/io/snappydata/hydra/northwind/stopDualModeCluster.conf; + +hydra.Prms-totalTaskTimeSec = 100; +hydra.Prms-maxResultWaitSec = 120; diff --git a/dtests/src/test/scala/io/snappydata/hydra/AirlineDataSparkJobApp.scala b/dtests/src/test/scala/io/snappydata/hydra/AirlineDataSparkJobApp.scala index 46439a7b53..e6926d7aad 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/AirlineDataSparkJobApp.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/AirlineDataSparkJobApp.scala @@ -29,10 +29,9 @@ object AirlineDataSparkJobApp { def main(args: Array[String]) { // scalastyle:off println - val connectionURL = args(args.length - 1) val conf = new SparkConf(). - setAppName("Airline Data Application_" + System.currentTimeMillis()). - set("snappydata.connection", connectionURL) + setAppName("Airline Data Application_" + System.currentTimeMillis()) + val sc = SparkContext.getOrCreate(conf) val snc = SnappyContext(sc) snc.sql("set spark.sql.shuffle.partitions=6") diff --git a/dtests/src/test/scala/io/snappydata/hydra/InstallJarTestSparkApp.scala b/dtests/src/test/scala/io/snappydata/hydra/InstallJarTestSparkApp.scala index 4470c93598..bbc39e119f 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/InstallJarTestSparkApp.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/InstallJarTestSparkApp.scala @@ -25,16 +25,15 @@ import org.apache.spark.{SparkConf, SparkContext} import scala.util.{Failure, Success, Try} object InstallJarTestSparkApp { + val conf = new SparkConf(). setAppName("InstallJarTest Application_" + System.currentTimeMillis()) val sc = new SparkContext(conf) val snc = SnappyContext(sc) def main(args: Array[String]): Unit = { - val connectionURL = args(args.length - 1) val conf = new SparkConf(). - setAppName("InstallJarTest Application"). - set("snappydata.connection", connectionURL) + setAppName("InstallJarTest Application") val sc = SparkContext.getOrCreate(conf) val snc = SnappyContext(sc) val threadID = Thread.currentThread().getId diff --git a/dtests/src/test/scala/io/snappydata/hydra/SnappyTestUtils.scala b/dtests/src/test/scala/io/snappydata/hydra/SnappyTestUtils.scala index a9693de565..9c4e1646f3 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/SnappyTestUtils.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/SnappyTestUtils.scala @@ -14,32 +14,195 @@ * permissions and limitations under the License. See accompanying * LICENSE file. */ + package io.snappydata.hydra -import java.io.{File, PrintWriter} +import java.io.{File, FileNotFoundException, IOException, PrintWriter} +import java.sql.Timestamp +import java.util +import java.util.Collections + +import scala.io.Source import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.encoders.RowEncoder -import scala.io.Source +object SnappyTestUtils { + var validateFullResultSet: Boolean = false; + var numRowsValidation: Boolean = false; + var tableType: String = null -object SnappyTestUtils { + def logTime: String = { + "[" + new Timestamp(System.currentTimeMillis()).toString + "] " + } - def assertJoinFullResultSet(snc: SnappyContext, sqlString: String, queryNum: String, - tableType: String, pw: PrintWriter, sqlContext: SQLContext, - planCachingEnabled: Boolean): Any = { - snc.sql("set spark.sql.crossJoin.enabled = true") - sqlContext.sql("set spark.sql.crossJoin.enabled = true") - assertQueryFullResultSet(snc, sqlString, queryNum, tableType, pw, sqlContext, - planCachingEnabled); + /* + Executes the join query, matches only the full result with expected result, returns false if the + query validation has failed. + */ + def assertJoin(snc: SnappyContext, sqlString: String, queryNum: String, pw: PrintWriter, + sqlContext: SQLContext): Boolean = { + var validationFailed = false + numRowsValidation = false + validationFailed = assertJoin(snc, sqlString, 0, queryNum, pw, sqlContext) + return validationFailed } - def assertJoinFullResultSet(snc: SnappyContext, sqlString: String, queryNum: String, - tableType: String, pw: PrintWriter, sqlContext: SQLContext): Any = { + /* + Executes the join query, matches the result with expected result, returns false if the query + validation has failed. + */ + def assertJoin(snc: SnappyContext, sqlString: String, numRows: Int, queryNum: String, + pw: PrintWriter, sqlContext: SQLContext): Boolean = { + var validationFailed = false snc.sql("set spark.sql.crossJoin.enabled = true") - sqlContext.sql("set spark.sql.crossJoin.enabled = true") - assertQueryFullResultSet(snc, sqlString, queryNum, tableType, pw, sqlContext) + if (validateFullResultSet) { + sqlContext.sql("set spark.sql.crossJoin.enabled = true") + } + validationFailed = assertQuery(snc, sqlString, numRows, queryNum, pw, sqlContext) + return validationFailed + } + + /* + Executes the query, matches only the full resultSet with expected result, returns false if the + query validation has failed. + */ + def assertQuery(snc: SnappyContext, sqlString: String, queryNum: String, + pw: PrintWriter, sqlContext: SQLContext): Boolean = { + numRowsValidation = false + assertQuery(snc, sqlString, 0, queryNum, pw, sqlContext) + } + + /* + Executes the query, matches the result with expected result, returns false if the query + validation has failed. + */ + def assertQuery(snc: SnappyContext, sqlString: String, numRows: Int, queryNum: String, + pw: PrintWriter, sqlContext: SQLContext): Boolean = { + var validationFailed = false + var snappyDF: DataFrame = null + snappyDF = snc.sql(sqlString).cache() + val snappyDFCount = snappyDF.count + // scalastyle:off println + pw.println(s"\n${logTime} Executing Query $queryNum ...") + println(s"Query $queryNum") + snappyDF.explain(true) + if (numRowsValidation) { + pw.println(s"${logTime} Query ${queryNum} returned ${snappyDFCount} rows for ${tableType}") + if (snappyDFCount != numRows) { + pw.println(s"${logTime} Result mismatch for query ${queryNum} found ${snappyDFCount} rows" + + s"but expected ${numRows} rows.") + validationFailed = true + } + pw.flush() + } + var fullRSValidationFailed: Boolean = false + if (validateFullResultSet) { + val sparkDF = sqlContext.sql(sqlString).cache() + val sparkDFCount = sparkDF.count() + if(snappyDFCount != sparkDFCount) { + pw.println(s"Count difference observed in snappy and spark resultset for query " + + s"${queryNum}. Snappy returned ${snappyDFCount} and spark returned ${sparkDFCount}.") + fullRSValidationFailed = true + } + fullRSValidationFailed = assertQuery(snc, snappyDF, sparkDF, queryNum, pw) + } + if (validationFailed) { + pw.println(s"\n${logTime} NumRows validation failed for query ${queryNum} on ${tableType} " + + s"table.") + } + if (fullRSValidationFailed) { + pw.println(s"\n${logTime} Full resultset validation failed for query ${queryNum} on " + + s"${tableType} table.") + validationFailed = true + } + pw.println(s"${logTime} Execution completed for query ${queryNum}") + pw.flush() + return validationFailed + } + + def assertQuery(snc: SnappyContext, snappyDF: DataFrame, sparkDF: DataFrame, queryNum: String, + pw: PrintWriter): Boolean = { + var fullRSValidationFailed = false + + val snappyResFileName = s"Snappy_${queryNum}" + val snappyDest: String = getQueryResultDir("snappyResults") + File.separator + snappyResFileName + // scalastyle:off println + val snappyFile: File = new java.io.File(snappyDest) + + val sparkResFileName = s"Spark_${queryNum}" + val sparkDest: String = getQueryResultDir("sparkResults") + File.separator + sparkResFileName + val sparkFile: File = new java.io.File(sparkDest) + + try { + if (!snappyFile.exists()) { + // val snap_col1 = snappyDF.schema.fieldNames(0) + // val snap_col = snappyDF.schema.fieldNames.filter(!_.equals(snap_col1)).toSeq + // snappyDF.repartition(1).sortWithinPartitions(snap_col1, snap_col: _*) + writeToFile(snappyDF.repartition((1)), snappyDest, snc) + pw.println(s"${logTime} Snappy result collected in : ${snappyDest}") + } + + if (!sparkFile.exists()) { + // val col1 = sparkDF.schema.fieldNames(0) + // val col = sparkDF.schema.fieldNames.filter(!_.equals(col1)).toSeq + // sparkDF.repartition(1).sortWithinPartitions(col1, col: _*) + writeToFile(sparkDF.repartition(1), sparkDest, snc) + pw.println(s"${logTime} Spark result collected in : ${sparkDest}") + } + val expectedFile = sparkFile.listFiles.filter(_.getName.endsWith(".csv")) + val sparkDF2 = snc.read.format("com.databricks.spark.csv") + .option("header", "false") + .option("inferSchema", "false") + .option("nullValue", "NULL") + .option("maxCharsPerColumn", "4096") + .load(s"${expectedFile.iterator.next().getAbsolutePath}") + + val missingDF: Array[Row] = sparkDF2.except(snappyDF).sort(sparkDF2.columns(0)).collect() + val unexpectedDF: Array[Row] = snappyDF.except(sparkDF2).sort(sparkDF2.columns(0)).collect() + + val aStr = new StringBuilder + if(missingDF.length > 0 || unexpectedDF.length > 0) { + pw.println(s"Found mismatch in resultset for query ${queryNum}... ") + if(missingDF.length > 0) { + aStr.append(s"The following ${missingDF.size} rows were missing in snappyDF:\n ") + for(i <- 0 to missingDF.size) + aStr.append(missingDF(i) + "\n") + } + if(unexpectedDF.length > 0) { + aStr.append(s"The following ${unexpectedDF.size} rows were unexpected in snappyDF:\n") + for(i <- 0 to unexpectedDF.size) + aStr.append(unexpectedDF(i) + "\n") + } + + // check if the mismatch is due to decimal, and can be ignored + if (unexpectedDF.length == missingDF.length) { + for (i <- 0 until missingDF.size) { + if (!isIgnorable(missingDF(i).toString, unexpectedDF(i).toString)) { + fullRSValidationFailed = true + } + } + pw.println("This mismatch can be ignored.") + aStr.setLength(0) // data mismatch can be ignored + } + if(aStr.length > 0) { + pw.println(aStr) + fullRSValidationFailed = true + } + } + // fullRSValidationFailed + // = compareFiles(snappyFile, sparkFile, pw, queryNum, fullRSValidationFailed) + } catch { + case ex: Exception => { + fullRSValidationFailed = true + pw.println(s"${logTime} Full resultSet validation for ${queryNum} got the following " + + s"exception:\n") + ex.printStackTrace(pw) + } + } + pw.flush() + fullRSValidationFailed } def dataTypeConverter(row: Row): Row = { @@ -57,6 +220,9 @@ object SnappyTestUtils { Row.fromSeq(md) } + /* + Writes the query resultset to a csv file. + */ def writeToFile(df: DataFrame, dest: String, snc: SnappyContext): Unit = { import snc.implicits._ df.map(dataTypeConverter)(RowEncoder(df.schema)) @@ -72,10 +238,14 @@ object SnappyTestUtils { }) sb.toString() }).write.format("org.apache.spark.sql.execution.datasources.csv.CSVFileFormat").option( - "header", false).save(dest) + "header", true).save(dest) } - def getTempDir(dirName: String): String = { + /* + Returns the path for the directory where the output of resultset of queries have been saved. + Creates a new directory, if not already existing + */ + def getQueryResultDir(dirName: String): String = { val log: File = new File(".") var dest: String = null val dirString = log.getCanonicalPath; @@ -83,21 +253,25 @@ object SnappyTestUtils { val logDir = log.listFiles.filter(_.getName.equals("snappyleader.log")) if (!logDir.isEmpty) { val leaderLogFile: File = logDir.iterator.next() - if (leaderLogFile.exists()) dest = dirString + File.separator + ".." + File.separator + - ".." + File.separator + dirName + if (leaderLogFile.exists()) { + dest = dirString + File.separator + ".." + File.separator + ".." + File.separator + + dirName + } } else dest = dirString + File.separator + ".." + File.separator + dirName } else dest = log.getCanonicalPath + File.separator + dirName - val tempDir: File = new File(dest) - if (!tempDir.exists) tempDir.mkdir() - return tempDir.getAbsolutePath + val queryResultDir: File = new File(dest) + if (!queryResultDir.exists) { + queryResultDir.mkdir() + } + return queryResultDir.getAbsolutePath } /* - In case of round-off, there is a difference of .1 in snappy and spark results. We can ignore - such differences - */ + In case of round-off, there is a difference of .1 in snappy and spark results. We can ignore + such differences + */ def isIgnorable(actualRow: String, expectedRow: String): Boolean = { var isIgnorable = false if (actualRow != null && actualRow.size > 0 && expectedRow != null && expectedRow.size > 0) { @@ -128,187 +302,261 @@ object SnappyTestUtils { isIgnorable } - def assertQueryFullResultSet(snc: SnappyContext, sqlString: String, queryNum: String, - tableType: String, pw: PrintWriter, sqlContext: SQLContext): Any = { - assertQueryFullResultSet(snc, sqlString, queryNum, tableType, pw, sqlContext, true) + def executeProcess(pb: ProcessBuilder, logFile: File, pw: PrintWriter): Int = { + var p: Process = null + try { + if (logFile != null) { + pb.redirectErrorStream(true) + pb.redirectError(ProcessBuilder.Redirect.PIPE) + pb.redirectOutput(ProcessBuilder.Redirect.appendTo(logFile)) + } + p = pb.start + if (logFile != null) { + assert(pb.redirectInput eq ProcessBuilder.Redirect.PIPE) + assert(pb.redirectOutput.file eq logFile) + assert(p.getInputStream.read == -1) + } + val rc = p.waitFor + val pbCmd = util.Arrays.toString(pb.command.toArray) + if ((rc == 0) || (pbCmd.contains("grep -v -F") && rc == 1)) { + println("Process executed successfully") + 0 + } + else { + println("Process execution failed with exit code: " + rc) + 1 + } + } catch { + case e: IOException => + println("Exception occurred while starting the process:" + pb + "\nError Message:" + e + .getMessage) + 1 + case e: InterruptedException => + println("Exception occurred while waiting for the process execution:" + p + "\nError " + + "Message:" + e.getMessage) + 1 + } } - def assertQueryFullResultSet(snc: SnappyContext, sqlString: String, queryNum: String, - tableType: String, pw: PrintWriter, sqlContext: SQLContext, - usePlanCaching: Boolean): Any = { - var snappyDF: DataFrame = null - if(!usePlanCaching) { - snappyDF = snc.sqlUncached(sqlString) - } else { - snappyDF = snc.sql(sqlString) - } - var sparkDF = sqlContext.sql(sqlString); - val snappyQueryFileName = s"Snappy_${queryNum}.out" - val sparkQueryFileName = s"Spark_${queryNum}.out" - val snappyDest: String = getTempDir("snappyQueryFiles_" + tableType) + File.separator + - snappyQueryFileName - val sparkDest: String = getTempDir("sparkQueryFiles") + File.separator + sparkQueryFileName - val sparkFile: File = new java.io.File(sparkDest) - val snappyFile = new java.io.File(snappyDest) - if (snappyFile.listFiles() == null) { - val col1 = snappyDF.schema.fieldNames(0) - val col = snappyDF.schema.fieldNames.tail - snappyDF = snappyDF.repartition(1).sortWithinPartitions(col1, col: _*) - writeToFile(snappyDF, snappyDest, snc) - // scalastyle:off println - pw.println(s"${queryNum} Result Collected in file $snappyDest") - } - if (sparkFile.listFiles() == null) { - val col1 = sparkDF.schema.fieldNames(0) - val col = sparkDF.schema.fieldNames.tail - sparkDF = sparkDF.repartition(1).sortWithinPartitions(col1, col: _*) - writeToFile(sparkDF, sparkDest, snc) - pw.println(s"${queryNum} Result Collected in file $sparkDest") - } + def compareFiles(snappyFile: File, sparkFile: File, pw: PrintWriter, queryNum: + String, validationFailed: Boolean) + : Boolean = { + var hasValidationFailed = validationFailed val expectedFile = sparkFile.listFiles.filter(_.getName.endsWith(".csv")) val actualFile = snappyFile.listFiles.filter(_.getName.endsWith(".csv")) + + hasValidationFailed = compareFiles(getQueryResultDir("snappyQueryFiles"), actualFile.iterator + .next().getAbsolutePath, expectedFile.iterator.next().getAbsolutePath, pw, queryNum) + + /* val expectedLineSet = Source.fromFile(expectedFile.iterator.next()).getLines() - val actualLineSet = Source.fromFile(actualFile.iterator.next()).getLines - // var numLines = 0 + val actualLineSet = Source.fromFile(actualFile.iterator.next()).getLines() + while (expectedLineSet.hasNext && actualLineSet.hasNext) { val expectedLine = expectedLineSet.next() val actualLine = actualLineSet.next() if (!actualLine.equals(expectedLine)) { if (!isIgnorable(actualLine, expectedLine)) { - pw.println(s"\n** For ${queryNum} result mismatch observed**") - pw.println(s"\nExpected Result:\n $expectedLine") - pw.println(s"\nActual Result:\n $actualLine") - pw.println(s"\nQuery =" + sqlString + " Table Type : " + tableType) + hasValidationFailed = true + pw.println(s"Expected Result : $expectedLine") + pw.println(s"Actual Result : $actualLine") + } else { + hasValidationFailed = false } - /* assert(assertion = false, s"\n** For $queryNum result mismatch observed** \n" + - s"Expected Result \n: $expectedLine \n" + - s"Actual Result \n: $actualLine \n" + - s"Query =" + sqlString + " Table Type : " + tableType) */ - // Commented due to Q37 failure by just the difference of 0.1 in actual and expected value + } - // numLines += 1 } + // scalastyle:off println if (actualLineSet.hasNext || expectedLineSet.hasNext) { - pw.println(s"\nFor ${queryNum} result count mismatch observed") - pw.flush() - // assert(assertion = false, s"\nFor $queryNum result count mismatch observed") + hasValidationFailed = true + if (actualLineSet.hasNext) { + pw.println(s"Following ${actualLineSet.size} rows are unexpected in Snappy:") + while (actualLineSet.hasNext) { + pw.println(s"${actualLineSet.next()}") + } + } + if (expectedLineSet.hasNext) { + pw.println(s"Following ${expectedLineSet.size} rows are missing in Snappy:") + while (expectedLineSet.hasNext) { + pw.println(s"${expectedLineSet.next()}") + } + } } - // scalastyle:on println - pw.flush() + */ + hasValidationFailed } - def assertQueryFullResultSet(snc: SnappyContext, snDF : DataFrame, - spDF : DataFrame, queryNum: String, - tableType: String, pw: PrintWriter, sqlContext: SQLContext): Any = { - var snappyDF: DataFrame = snDF - var sparkDF = spDF - val snappyQueryFileName = s"Snappy_${queryNum}.out" - val sparkQueryFileName = s"Spark_${queryNum}.out" - val snappyDest: String = getTempDir("snappyQueryFiles_" + tableType) + File.separator + - snappyQueryFileName - val sparkDest: String = getTempDir("sparkQueryFiles") + File.separator + sparkQueryFileName - val sparkFile: File = new java.io.File(sparkDest) - val snappyFile = new java.io.File(snappyDest) - if (snappyFile.listFiles() == null) { - val col1 = snappyDF.schema.fieldNames(0) - val col = snappyDF.schema.fieldNames.tail - snappyDF = snappyDF.repartition(1).sortWithinPartitions(col1, col: _*) - writeToFile(snappyDF, snappyDest, snc) - // scalastyle:off println - pw.println(s"${queryNum} Result Collected in file $snappyDest") + def compareFiles(dir: String, snappyResultsFile: String, sparkResultsFile: String, + pw: PrintWriter, queryNum: String): Boolean = { + val aStr = new StringBuilder + var pb: ProcessBuilder = null + var command: String = null + val missingFileName = dir + File.separator + "missing_" + queryNum + ".txt" + val unexpectedFileName = dir + File.separator + "unexpected_" + queryNum + ".txt" + try { + var writer = new PrintWriter(missingFileName) + writer.print("") + writer.close() + writer = new PrintWriter(unexpectedFileName) + writer.print("") + writer.close() + } catch { + case fe: FileNotFoundException => + pw.println(s"${logTime} Exception while overwriting the result mismatch files", fe) + return true } - if (sparkFile.listFiles() == null) { - val col1 = sparkDF.schema.fieldNames(0) - val col = sparkDF.schema.fieldNames.tail - sparkDF = sparkDF.repartition(1).sortWithinPartitions(col1, col: _*) - writeToFile(sparkDF, sparkDest, snc) - pw.println(s"${queryNum} Result Collected in file $sparkDest") + val unexpectedResultsFile = new File(unexpectedFileName) + val missingResultsFile = new File(missingFileName) + command = "grep -v -F -x -f " + sparkResultsFile + " " + snappyResultsFile + pb = new ProcessBuilder("/bin/bash", "-c", command) + println("Executing command : " + command) + // get the unexpected rows in snappy + executeProcess(pb, unexpectedResultsFile, pw) + command = "grep -v -F -x -f " + snappyResultsFile + " " + sparkResultsFile + pb = new ProcessBuilder("/bin/bash", "-c", command) + println("Executing command : " + command) + // get the missing rows in snappy + executeProcess(pb, missingResultsFile, pw) + var unexpectedRsReader: Iterator[String] = null + var missingRsReader: Iterator[String] = null + try { + unexpectedRsReader = Source.fromFile(unexpectedResultsFile).getLines() + missingRsReader = Source.fromFile(missingResultsFile).getLines() + } catch { + case fe: FileNotFoundException => + pw.println(s"${logTime} Could not find file to compare results.", fe) + return true } - val expectedFile = sparkFile.listFiles.filter(_.getName.endsWith(".csv")) - val actualFile = snappyFile.listFiles.filter(_.getName.endsWith(".csv")) - val expectedLineSet = Source.fromFile(expectedFile.iterator.next()).getLines() - val actualLineSet = Source.fromFile(actualFile.iterator.next()).getLines - while (expectedLineSet.hasNext && actualLineSet.hasNext) { - val expectedLine = expectedLineSet.next() - val actualLine = actualLineSet.next() - if (!actualLine.equals(expectedLine)) { - if (!isIgnorable(actualLine, expectedLine)) { - pw.println(s"\n** For ${queryNum} result mismatch observed**") - pw.println(s"\nExpected Result:\n $expectedLine") - pw.println(s"\nActual Result:\n $actualLine") - - } - } + val unexpected = new util.ArrayList[String] + val missing = new util.ArrayList[String] + try { + while (unexpectedRsReader.hasNext) + unexpected.add("\n " + unexpectedRsReader.next()) + while (missingRsReader.hasNext) + missing.add("\n " + missingRsReader.next()) + } catch { + case ie: IOException => + pw.println(s"${logTime} Got exception while reading resultset" + + s" files", ie) } - if (actualLineSet.hasNext || expectedLineSet.hasNext) { - pw.println(s"\nFor ${queryNum} result count mismatch observed") - pw.flush() - } - pw.println(s"Validation for ${queryNum} finished.") - pw.println() - // scalastyle:on println - pw.flush() - } - def assertQueryFullResultSet(snc: SnappyContext, snDF : DataFrame, - spDF : DataFrame, queryNum: String, - tableType: String, pw: PrintWriter, sqlContext: SQLContext, isJoin : Boolean ): Any = { - var snappyDF: DataFrame = snDF - var sparkDF = spDF - val snappyQueryFileName = s"Snappy_${queryNum}.out" - val sparkQueryFileName = s"Spark_${queryNum}.out" - val snappyDest: String = getTempDir("snappyQueryFiles_" + tableType) + File.separator + - snappyQueryFileName - val sparkDest: String = getTempDir("sparkQueryFiles") + File.separator + sparkQueryFileName - val sparkFile: File = new java.io.File(sparkDest) - val snappyFile = new java.io.File(snappyDest) - if (snappyFile.listFiles() == null) { - val col1 = snappyDF.schema.fieldNames(0) - val col = snappyDF.schema.fieldNames.tail - if(isJoin) { - snappyDF = snappyDF.repartition(1) + if (missing.size > 0) { + if (missing.size < 20) { + aStr.append(s"\nThe following ${missing.size} rows are missing from snappy resultset: \n") + aStr.append(missing.toString) } else { - snappyDF = snappyDF.repartition(1).sortWithinPartitions(col1, col: _*) + aStr.append(s"There are ${missing.size} rows missing from snappy for $queryNum. Please " + + s"check $missingFileName") } - writeToFile(snappyDF, snappyDest, snc) - // scalastyle:off println - pw.println(s"${queryNum} Result Collected in file $snappyDest") + aStr.append("\n") } - if (sparkFile.listFiles() == null) { - val col1 = sparkDF.schema.fieldNames(0) - val col = sparkDF.schema.fieldNames.tail - if(isJoin) { - sparkDF = sparkDF.repartition(1) + if (unexpected.size > 0) { + if (unexpected.size < 20) { + aStr.append(s"\nThe following ${unexpected.size} rows are unexpected in snappy " + + s"resultset:\n") + aStr.append(unexpected.toString) } else { - sparkDF = sparkDF.repartition(1).sortWithinPartitions(col1, col: _*) + aStr.append(s"There are ${unexpected.size} rows unexpected in snappy for $queryNum. " + + s"Please check $unexpectedFileName") } - writeToFile(sparkDF, sparkDest, snc) - pw.println(s"${queryNum} Result Collected in file $sparkDest") + aStr.append("\n") } - val expectedFile = sparkFile.listFiles.filter(_.getName.endsWith(".csv")) - val actualFile = snappyFile.listFiles.filter(_.getName.endsWith(".csv")) - val expectedLineSet = Source.fromFile(expectedFile.iterator.next()).getLines() - val actualLineSet = Source.fromFile(actualFile.iterator.next()).getLines - while (expectedLineSet.hasNext && actualLineSet.hasNext) { - val expectedLine = expectedLineSet.next() - val actualLine = actualLineSet.next() - if (!actualLine.equals(expectedLine)) { - if (!isIgnorable(actualLine, expectedLine)) { - pw.println(s"\n** For ${queryNum} result mismatch observed**") - pw.println(s"\nExpected Result:\n $expectedLine") - pw.println(s"\nActual Result:\n $actualLine") - } + + // check if the mismatch is due to decimal, and can be ignored + if ((missing.size() > 0) && (missing.size() == unexpected.size())) { + Collections.sort(missing) + Collections.sort(unexpected) + for (i <- 0 until missing.size()) { + if (!isIgnorable(missing.get(i), unexpected.get(i))) true } + // pw.println("This mismatch can be ignored.") + aStr.setLength(0) // data mismatch can be ignored } - if (actualLineSet.hasNext || expectedLineSet.hasNext) { - pw.println(s"\nFor ${queryNum} result count mismatch observed") - pw.flush() + + if (aStr.length() > 0) { + pw.println(s"${logTime} ${aStr.toString}") + true + } else { + false + } + } + + /* + If validation has failed for a query, add the query number to failedQueries String + */ + def addToFailedQueryList(failedQueries: String, queryNum: String): String = { + var str = failedQueries + if (str.isEmpty) { + str = queryNum + } + else { + str = str + "," + queryNum + } + return str + } + + + /* + Performs full resultSet validation from snappy for a select query against results in a + goldenFile. + */ + def assertValidateFullResultSetFromGoldenFile(sqlString: String, queryNum: String, tableType: + String, snc: SnappyContext, pw: PrintWriter, validationFailed: Boolean, goldenFileDest: String): + Boolean = { + var hasValidationFailed = validationFailed + + val snappyQueryFileName = s"Snappy_${queryNum}" + val snappyDest: String = getQueryResultDir("snappyQueryFiles") + + File.separator + snappyQueryFileName + pw.println(s" ${logTime} ${snappyDest}") + val snappyFile: File = new java.io.File(snappyDest) + var snappyDF = snc.sql(sqlString) + + pw.println(s"${logTime} ${goldenFileDest}") + val goldenFileName = goldenFileDest + File.separator + s"Spark_$queryNum" + val sortedGoldenDest = goldenFileDest + File.separator + s"Sorted_$queryNum" + val sortedGoldenFile: File = new java.io.File(sortedGoldenDest) + val goldenFile: File = new java.io.File(goldenFileName) + + try { + if (!snappyFile.exists()) { + val snap_col1 = snappyDF.schema.fieldNames(0) + val snap_col = snappyDF.schema.fieldNames.filter(!_.equals(snap_col1)).toSeq + snappyDF = snappyDF.repartition(1).sortWithinPartitions(snap_col1, snap_col: _*) + writeToFile(snappyDF, snappyDest, snc) + pw.println(s"${logTime} ${queryNum} Result Collected in file $snappyDest") + } + if (!goldenFile.exists()) { + pw.println(s"${logTime} Did not find any golden file for query $queryNum") + throw new Exception(s"Did not find any golden file for query $queryNum") + } else if (goldenFile.length() > 0) { + // sort the contents of golden file before comparing results + var goldenDF = snc.read.format("com.databricks.spark.csv") + .option("header", "false").option("inferSchema", "true").option("nullValue", "NULL") + .load(goldenFileName) + val col1 = goldenDF.schema.fieldNames(0) + val col = goldenDF.schema.fieldNames.filter(!_.equals(col1)).toSeq + goldenDF = goldenDF.repartition(1).sortWithinPartitions(col1, col: _*) + writeToFile(goldenDF, sortedGoldenDest, snc) + pw.println(s"${logTime} ${queryNum} Result Collected in file ${sortedGoldenDest}") + } else { + pw.println(s"${logTime} No results in query result file for $queryNum.") + } + hasValidationFailed = compareFiles(snappyFile, sortedGoldenFile, pw, queryNum, + hasValidationFailed) + + } catch { + case ex: Exception => { + hasValidationFailed = true + pw.println(s"${logTime} Full resultSet Validation failed for ${queryNum} with following " + + s"exception:\n") + ex.printStackTrace(pw) + } } - pw.println(s"Validation for ${queryNum} finished.") - pw.println() - // scalastyle:on println pw.flush() + return hasValidationFailed } + } diff --git a/dtests/src/test/scala/io/snappydata/hydra/TestUtil.scala b/dtests/src/test/scala/io/snappydata/hydra/TestUtil.scala deleted file mode 100644 index b243441929..0000000000 --- a/dtests/src/test/scala/io/snappydata/hydra/TestUtil.scala +++ /dev/null @@ -1,304 +0,0 @@ -/* - * Copyright (c) 2017-2019 TIBCO Software Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You - * may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. See accompanying - * LICENSE file. - */ - -package io.snappydata.hydra - -import java.io.{File, PrintWriter} - -import scala.io.Source - -import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.encoders.RowEncoder - -object TestUtil { - var validateFullResultSet: Boolean = false; - var tableType: String = null - - /* - Executes the join query, matches the result with expected result, returns false if the query - validation has failed. - */ - def assertJoin(snc: SnappyContext, sqlString: String, numRows: Int, queryNum: String, pw: - PrintWriter, sqlContext: SQLContext, skipNumRowsValidation: Boolean): Boolean = { - var hasValidationFailed = false - snc.sql("set spark.sql.crossJoin.enabled = true") - if (!skipNumRowsValidation) { - val df = snc.sql(sqlString) - pw.println(s"No. rows in resultset for join query ${queryNum} is : ${df.count} for ${tableType} table") - if (df.count() != numRows) { - pw.println(s"Result mismatch for join query ${queryNum} : found ${df.count} rows but expected ${numRows} rows.") - hasValidationFailed = true - } - pw.flush() - } - if (validateFullResultSet) - hasValidationFailed = assertValidateFullResultSet(snc, sqlString, queryNum, pw, sqlContext, - hasValidationFailed) - - if (hasValidationFailed) - pw.println(s"Failed Query =" + sqlString + " Table Type : " + tableType + "\n") - pw.flush() - return hasValidationFailed - } - - /* - Executes the query, matches the result with expected result, returns false if the query - validation has failed. - */ - def assertQuery(snc: SnappyContext, sqlString: String, numRows: Int, queryNum: String, pw: - PrintWriter, sqlContext: SQLContext, skipNumRowsValidation: Boolean): Boolean = { - - var hasValidationFailed = false - if (!skipNumRowsValidation) { - val df = snc.sql(sqlString) - pw.println(s"No. rows in resultset for query ${queryNum} is : ${df.count} for ${tableType} table") - if (df.count() != numRows) { - pw.println(s"Result mismatch for query ${queryNum} : found ${df.count} rows but expected ${numRows} rows.") - hasValidationFailed = true - } - pw.flush() - } - if (validateFullResultSet) - hasValidationFailed = assertValidateFullResultSet(snc, sqlString, queryNum, pw, sqlContext, - hasValidationFailed) - - if (hasValidationFailed) - pw.println(s"Failed Query : " + sqlString + "\n Table Type : " + tableType + "\n") - pw.flush() - return hasValidationFailed - } - - /* - Performs full resultSet validation for snappy results for the query against snappy spark - resultset. - */ - def assertValidateFullResultSet(snc: SnappyContext, sqlString: String, queryNum: String, pw: - PrintWriter, sqlContext: SQLContext, validationFailed: Boolean): Boolean = { - var hasValidationFailed = validationFailed - - val snappyQueryFileName = s"Snappy_${queryNum}" - val snappyDest: String = SnappyTestUtils.getTempDir("snappyQueryFiles") + File.separator + - snappyQueryFileName - pw.println(snappyDest) - val snappyFile: File = new java.io.File(snappyDest) - var snappyDF = snc.sql(sqlString) - - val sparkQueryFileName = s"Spark_${queryNum}" - val sparkDest: String = SnappyTestUtils.getTempDir("sparkQueryFiles") + File.separator + sparkQueryFileName - pw.println(sparkDest) - val sparkFile: File = new java.io.File(sparkDest) - var sparkDF = sqlContext.sql(sqlString) - - try { - if (!snappyFile.exists()) { - val snap_col1 = snappyDF.schema.fieldNames(0) - val snap_col = snappyDF.schema.fieldNames.filter(!_.equals(snap_col1)).toSeq - snappyDF = snappyDF.repartition(1).sortWithinPartitions(snap_col1, snap_col: _*) - SnappyTestUtils.writeToFile(snappyDF, snappyDest, snc) - //writeResultSetToCsv(snappyDF, snappyFile) - pw.println(s"${queryNum} Result Collected in file ${snappyDest}") - } - if (!sparkFile.exists()) { - val col1 = sparkDF.schema.fieldNames(0) - val col = sparkDF.schema.fieldNames.filter(!_.equals(col1)).toSeq - sparkDF = sparkDF.repartition(1).sortWithinPartitions(col1, col: _*) - SnappyTestUtils.writeToFile(sparkDF, sparkDest, snc) - //writeResultSetToCsv(sparkDF, sparkFile) - pw.println(s"${queryNum} Result Collected in file ${sparkDest}") - } - hasValidationFailed = compareFiles(snappyFile, sparkFile, pw, hasValidationFailed) - } catch { - case ex: Exception => { - hasValidationFailed = true - pw.println(s"Full resultSet Validation failed for ${queryNum} with following exception:\n") - ex.printStackTrace(pw) - } - } - pw.flush() - return hasValidationFailed - } - - /* - Writes the query resultset to a csv file. - */ - def writeResultSetToCsv(df: DataFrame, destFile: File): Unit = { - val parent = destFile.getParentFile - if (!parent.exists()) { - parent.mkdirs() - } - implicit val encoder = RowEncoder(df.schema) - df.mapPartitions(rows => { - val sb: StringBuilder = new StringBuilder() - val pw = new PrintWriter(destFile) - try { - rows.foreach { row => - row.toSeq.foreach { - case d: Double => - // round to one decimal digit - sb.append(math.floor(d * 10.0 + 0.5) / 10.0).append(',') - case bd: java.math.BigDecimal => - sb.append(bd.setScale(2, java.math.RoundingMode.HALF_UP)).append(',') - case v => sb.append(v).append(',') - } - val len = sb.length - if (len > 0) sb.setLength(len - 1) - sb.append('\n') - if (sb.length >= 1048576) { - pw.append(sb) - pw.flush() - sb.clear() - } - } - if (sb.nonEmpty) { - pw.append(sb) - pw.flush() - } - } - finally { - pw.close() - } - Iterator.empty - }).collect() - } - - /* - Returns the path for the directory where the output of resultset of queries have been saved. - Creates a new directory, if not already existing - */ - def getQueryResultDir(dirName: String): String = { - val log: File = new File(".") - var dest: String = null - val dirString = log.getCanonicalPath; - if (dirName.equals("sparkQueryFiles")) { - val logDir = log.listFiles.filter(_.getName.equals("snappyleader.log")) - if (!logDir.isEmpty) { - val leaderLogFile: File = logDir.iterator.next() - if (leaderLogFile.exists()) - dest = dirString + File.separator + ".." + File.separator + ".." + File.separator + dirName - } - else dest = dirString + File.separator + ".." + File.separator + dirName - } - else dest = log.getCanonicalPath + File.separator + dirName - val queryResultDir: File = new File(dest) - if (!queryResultDir.exists) - queryResultDir.mkdir() - return queryResultDir.getAbsolutePath - } - - /* - Performs full resultSet validation from snappy for a select query against results in a goldenFile. - */ - def assertValidateFullResultSetFromGoldenFile(sqlString: String, queryNum: String, tableType: - String, snc: SnappyContext, pw: PrintWriter, validationFailed: Boolean, goldenFileDest: String): - Boolean = { - var hasValidationFailed = validationFailed - - val snappyQueryFileName = s"Snappy_${queryNum}" - val snappyDest: String = SnappyTestUtils.getTempDir("snappyQueryFiles") + File.separator + snappyQueryFileName - pw.println(snappyDest) - val snappyFile: File = new java.io.File(snappyDest) - var snappyDF = snc.sql(sqlString) - - pw.println(goldenFileDest) - val goldenFileName = goldenFileDest + File.separator + s"Spark_$queryNum" - val sortedGoldenDest = goldenFileDest + File.separator + s"Sorted_$queryNum" - val sortedGoldenFile: File = new java.io.File(sortedGoldenDest) - val goldenFile: File = new java.io.File(goldenFileName) - - try { - if (!snappyFile.exists()) { - val snap_col1 = snappyDF.schema.fieldNames(0) - val snap_col = snappyDF.schema.fieldNames.filter(!_.equals(snap_col1)).toSeq - snappyDF = snappyDF.repartition(1).sortWithinPartitions(snap_col1, snap_col: _*) - SnappyTestUtils.writeToFile(snappyDF, snappyDest, snc) - //writeResultSetToCsv(snappyDF, snappyFile) - pw.println(s"${queryNum} Result Collected in file $snappyDest") - } - if (!goldenFile.exists()) { - pw.println(s"Did not find any golden file for query $queryNum") - throw new Exception(s"Did not find any golden file for query $queryNum") - } else if (goldenFile.length() > 0) { - // sort the contents of golden file before comparing results - var goldenDF = snc.read.format("com.databricks.spark.csv") - .option("header", "false").option("inferSchema", "true").option("nullValue", "NULL") - .load(goldenFileName) - val col1 = goldenDF.schema.fieldNames(0) - val col = goldenDF.schema.fieldNames.filter(!_.equals(col1)).toSeq - goldenDF = goldenDF.repartition(1).sortWithinPartitions(col1, col: _*) - SnappyTestUtils.writeToFile(goldenDF, sortedGoldenDest, snc) - //writeResultSetToCsv(goldenDF, sortedGoldenFile) - pw.println(s"${queryNum} Result Collected in file ${sortedGoldenDest}") - } else { - pw.println(s"zero results in query $queryNum.") - } - hasValidationFailed = compareFiles(snappyFile, sortedGoldenFile, pw, hasValidationFailed) - - } catch { - case ex: Exception => { - hasValidationFailed = true - pw.println(s"Full resultSet Validation failed for ${queryNum} with following exception:\n") - ex.printStackTrace(pw) - } - } - pw.flush() - return hasValidationFailed - } - - def compareFiles(snappyFile: File, sparkFile: File, pw: PrintWriter, validationFailed: Boolean): - Boolean = { - var hasValidationFailed = validationFailed - val expectedFile = sparkFile.listFiles.filter(_.getName.endsWith(".csv")) - val actualFile = snappyFile.listFiles.filter(_.getName.endsWith(".csv")) - val expectedLineSet = Source.fromFile(expectedFile.iterator.next()).getLines() - val actualLineSet = Source.fromFile(actualFile.iterator.next()).getLines() - - while (expectedLineSet.hasNext && actualLineSet.hasNext) { - val expectedLine = expectedLineSet.next() - val actualLine = actualLineSet.next() - if (!actualLine.equals(expectedLine)) { - hasValidationFailed = true - pw.println(s"Expected Result : $expectedLine") - pw.println(s"Actual Result : $actualLine") - } - } - if (actualLineSet.hasNext || expectedLineSet.hasNext) { - hasValidationFailed = true - if (actualLineSet.hasNext) - pw.println(s"Following ${actualLineSet.size} rows are unexpected in Snappy:") - while (actualLineSet.hasNext) - pw.println(actualLineSet.next()) - if (expectedLineSet.hasNext) - pw.println(s"Following ${expectedLineSet.size} rows are missing in Snappy:") - while (expectedLineSet.hasNext) - pw.println(expectedLineSet.next()) - } - hasValidationFailed - } - - /* - If validation has failed for a query, add the query number to failedQueries String - */ - def addToFailedQueryList(failedQueries: String, queryNum: String): String = { - var str = failedQueries - if (str.isEmpty) - str = queryNum - else - str = str + "," + queryNum - return str - } - -} diff --git a/dtests/src/test/scala/io/snappydata/hydra/ao/ConvertToParquet.scala b/dtests/src/test/scala/io/snappydata/hydra/ao/ConvertToParquet.scala new file mode 100644 index 0000000000..7a13573560 --- /dev/null +++ b/dtests/src/test/scala/io/snappydata/hydra/ao/ConvertToParquet.scala @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ + +package io.snappydata.hydra.ao + +import java.io.{File, PrintWriter} +import java.sql.Timestamp + +import scala.util.{Failure, Success, Try} + +import com.typesafe.config.Config + +import org.apache.spark.sql._ + + +object ConvertToParquet extends SnappySQLJob { + + val currDir: String = new java.io.File(".").getCanonicalPath + val fileSep: String = File.separator + override def runSnappyJob(snSession: SnappySession, jobConfig: Config): Any = { + val outputFileName = s"ConvertToParquet_${System.currentTimeMillis()}.out" + val pw = new PrintWriter(outputFileName) + Try { + // scalastyle:off println + val snc = snSession.sqlContext + val tableName = jobConfig.getString("tableName") + // where to store the parquet file and read from csv + val dataFileLocation = jobConfig.getString("dataFileLocation") + val fromType: String = jobConfig.getString("fromType") + writeToParquetAndCsv(tableName, pw, snc, dataFileLocation, fromType) + } match { + case Success(v) => pw.close() + s"See ${currDir}/$outputFileName" + case Failure(e) => pw.close(); + throw e; + } + } + + def writeToParquetAndCsv(tableName: String, pw: PrintWriter, snc: SQLContext, dataFileLocation: + String, fromType: String): Unit = { + pw.println("Writing data to parquet and/or csv..") + val parquetFileLoc = dataFileLocation + fileSep + "parquetFiles" + val csvFileLoc = dataFileLocation + fileSep + "csvFiles" + var df: DataFrame = null + if (fromType.equals("csv")) { + df = snc.read.format("com.databricks.spark.csv") + .option("header", "false") + .option("inferSchema", "false") + .option("maxCharsPerColumn", "4096") + .option("nullValue", "") + .csv(s"${dataFileLocation}/${tableName}.csv") + df.write.parquet(s"${parquetFileLoc}/${tableName}") + } else if (fromType.equals("table")) { + df = snc.sql(s"select * from ${tableName}") + df.write.parquet(s"${parquetFileLoc}/${tableName}") + df.write.csv(s"${csvFileLoc}/$tableName") + } + pw.flush() + } + + /** + * Validate if the data files are available, else throw SparkJobInvalid + * + */ + override def isValidJob(sc: SnappySession, config: Config): SnappyJobValidation = SnappyJobValid() + +} \ No newline at end of file diff --git a/dtests/src/test/scala/io/snappydata/hydra/ao/LoadData.scala b/dtests/src/test/scala/io/snappydata/hydra/ao/LoadData.scala index 5279449392..edf2b0a0c6 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/ao/LoadData.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/ao/LoadData.scala @@ -115,6 +115,9 @@ object LoadData extends SnappySQLJob { pw.flush() } + /* + Generate duplicate data from csv. + */ def generateDupData(tableName: String, pw: PrintWriter, numIter: Int, snc: SQLContext, dataLocation: String, csvFileLocation: String, parquetFileLocation: String): Any = { @@ -129,20 +132,24 @@ object LoadData extends SnappySQLJob { // generating 10 million records from 2 lac records pw.println("Duplicating data to temp table...") for (i <- 0 until numIter) { - temp_DF.write.format("column").mode(SaveMode.Append).saveAsTable(s"tempTable") + temp_DF.write.format("column").mode(SaveMode.Append).saveAsTable(s"${tableName}Temp") } // writing dat to parquet and csv - pw.println("creating df with duplicate records and writing it to csv and parquet..") - val df = snc.sql("select * from meterReadingstemp") - df.write.parquet(s"${parquetFileLocation}/${tableName}") - df.write.csv(s"${csvFileLocation}/$tableName") + writeToCsvAndParquet(tableName + "Temp", pw, snc, csvFileLocation, parquetFileLocation) pw.println("Dropping temp table") snc.sql(s"drop table if exists ${tableName}Temp") - pw.flush() } + def writeToCsvAndParquet(tableName: String, pw: PrintWriter, snc: SQLContext, csvFileLocation: + String, parquetFileLocation: String): Unit = { + pw.println("creating df with duplicate records and writing it to csv and parquet..") + val df = snc.sql(s"select * from ${tableName}") + df.write.parquet(s"${parquetFileLocation}/${tableName}") + df.write.csv(s"${csvFileLocation}/$tableName") + pw.flush() + } /** * Validate if the data files are available, else throw SparkJobInvalid * diff --git a/dtests/src/test/scala/io/snappydata/hydra/cdcConnector/CDCValidationApp.scala b/dtests/src/test/scala/io/snappydata/hydra/cdcConnector/CDCValidationApp.scala index c59e384c11..d39a4bd6cc 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/cdcConnector/CDCValidationApp.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/cdcConnector/CDCValidationApp.scala @@ -17,18 +17,16 @@ package io.snappydata.hydra.cdcConnector -import java.io.{File, PrintWriter} -import java.sql.{Connection, DriverManager, ResultSet} +import java.io.PrintWriter +import java.sql.{Connection, DriverManager} import java.util.Properties +import scala.util.{Failure, Random, Success, Try} + import breeze.numerics.abs -import io.snappydata.hydra.SnappyTestUtils.{getTempDir, writeToFile} -import org.apache.spark.sql.{SQLContext, SnappyContext} -import org.apache.spark.{SparkConf, SparkContext} -import scala.collection.mutable -import scala.io.Source -import scala.util.{Failure, Random, Success, Try} +import org.apache.spark.sql.SnappyContext +import org.apache.spark.{SparkConf, SparkContext} object CDCValidationApp { diff --git a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/AllMixedTypes.scala b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/AllMixedTypes.scala index 17d0dd91b9..26bab7b7d4 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/AllMixedTypes.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/AllMixedTypes.scala @@ -53,12 +53,6 @@ class AllMixedTypes extends SnappySQLJob{ snc.sql("CREATE TABLE IF NOT EXISTS TwentyTwenty USING COLUMN " + "AS (SELECT * FROM TempTwenty)") - snc.sql(ComplexTypeUtils.Mixed_Q1) - snc.sql(ComplexTypeUtils.Mixed_Q2) - snc.sql(ComplexTypeUtils.Mixed_Q3) - snc.sql(ComplexTypeUtils.Mixed_Q4) - snc.sql(ComplexTypeUtils.Mixed_Q5) - if(printContent) { println("snc : Mixed_Q1 " + (snc.sql(ComplexTypeUtils.Mixed_Q1).show)) println("snc : Mixed_Q2 " + (snc.sql(ComplexTypeUtils.Mixed_Q2).show)) @@ -67,6 +61,7 @@ class AllMixedTypes extends SnappySQLJob{ println("snc : Mixed_Q5 " + (snc.sql(ComplexTypeUtils.Mixed_Q5).show)) } + SnappyTestUtils.tableType = "column" /* --- Create the Spark Tables / Views for Verification --- */ val allMixed : DataFrame = spark.read.json(dataLocation) allMixed.createOrReplaceTempView("TwentyTwenty") @@ -74,16 +69,12 @@ class AllMixedTypes extends SnappySQLJob{ /* --- Verification --- */ // TODO Due to SNAP-2782 Below line is commented, Hydra Framework required changes. - // SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Mixed_Q1, - // "Mixed_Q1", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Mixed_Q2, - "Mixed_Q2", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Mixed_Q3, - "Mixed_Q3", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Mixed_Q4, - "Mixed_Q4", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Mixed_Q5, - "Mixed_Q5", "column", pw, sqlContext) + // SnappyTestUtils.assertQueryFullResultSet(snc, Mixed_Q1, "Mixed_Q1", "column", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Mixed_Q2, "Mixed_Q2", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Mixed_Q3, "Mixed_Q3", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Mixed_Q4, "Mixed_Q4", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Mixed_Q5, "Mixed_Q5", pw, sqlContext) + /* --- Clean up --- */ snc.sql("DROP TABLE IF EXISTS TwentyTwenty") diff --git a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/ArrayType.scala b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/ArrayType.scala index cf6ebf813d..19ae06b6e7 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/ArrayType.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/ArrayType.scala @@ -54,12 +54,7 @@ class ArrayType extends SnappySQLJob { "OPTIONS(path '" + dataLocation + "')") snc.sql("CREATE TABLE Student USING COLUMN AS (SELECT * FROM TempArray)") - snc.sql(ComplexTypeUtils.Array_Q1) - snc.sql(ComplexTypeUtils.Array_Q2) - snc.sql(ComplexTypeUtils.Array_Q3) snc.sql(ComplexTypeUtils.Array_View) - snc.sql(ComplexTypeUtils.Array_Q4) - snc.sql(ComplexTypeUtils.Array_Q5) if(printContent) { println(snc.sql(ComplexTypeUtils.Array_Q1).show()) @@ -77,16 +72,13 @@ class ArrayType extends SnappySQLJob { /* --- Verification --- */ - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Array_Q1, - "Array_Q1", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Array_Q2, - "Array_Q2", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Array_Q3, - "Array_Q3", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Array_Q4, - "Array_Q4", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Array_Q5, - "Array_Q5", "column", pw, sqlContext) + SnappyTestUtils.tableType = "column" + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Array_Q1, "Array_Q1", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Array_Q2, "Array_Q2", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Array_Q3, "Array_Q3", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Array_Q4, "Array_Q4", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Array_Q5, "Array_Q5", pw, sqlContext) + /* --- Clean up --- */ spark.sql("DROP VIEW IF EXISTS Student") diff --git a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/ArrayTypeAPI.scala b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/ArrayTypeAPI.scala index 1009d9ed81..25bb97df24 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/ArrayTypeAPI.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/ArrayTypeAPI.scala @@ -105,20 +105,21 @@ class ArrayTypeAPI extends SnappySQLJob{ println("sncArrQuery5DF data = " + sncArrQuery5DF.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, sncSelectDF, sparkSelectDF, - "ArrayTypeAPIQuery1", "column" , pw, sqlContext) + SnappyTestUtils.tableType = "column" + SnappyTestUtils.assertQuery(snc, sncSelectDF, sparkSelectDF, + "ArrayTypeAPIQuery1" , pw) println("Finished the ArrayType Query 1....") - SnappyTestUtils.assertQueryFullResultSet(snc, sncArrQuery2DF, sparkArrQuery2DF, - "ArrayTypeAPIQuery2", "column", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, sncArrQuery2DF, sparkArrQuery2DF, + "ArrayTypeAPIQuery2", pw) println("Finished the ArrayType Query 2....") - SnappyTestUtils.assertQueryFullResultSet(snc, sncArrQuery3DF, sparkArrQuery3DF, - "ArrayTypeAPIQuery3", "column", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, sncArrQuery3DF, sparkArrQuery3DF, + "ArrayTypeAPIQuery3", pw) println("Finished the ArrayType Query 3....") - SnappyTestUtils.assertQueryFullResultSet(snc, sncArrQuery4DF, sparkArrQuery4DF, - "ArrayTypeAPIQuery4", "column", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, sncArrQuery4DF, sparkArrQuery4DF, + "ArrayTypeAPIQuery4", pw) println("Finished the ArrayType Query 4....") - SnappyTestUtils.assertQueryFullResultSet(snc, sncArrQuery5DF, sparkArrQuery5DF, - "ArrayTypeAPIQuery5", "column", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, sncArrQuery5DF, sparkArrQuery5DF, + "ArrayTypeAPIQuery5", pw) println("Finished the ArrayType Query 5....") pw.close() diff --git a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/ArraysOfStringInMapAsValue.scala b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/ArraysOfStringInMapAsValue.scala index cac94aa74c..619c964dcf 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/ArraysOfStringInMapAsValue.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/ArraysOfStringInMapAsValue.scala @@ -57,9 +57,6 @@ class ArraysOfStringInMapAsValue extends SnappySQLJob{ "USING COLUMN AS (SELECT * FROM TempFamousPeople)") snc.sql(ComplexTypeUtils.Array_Map_TempView) - snc.sql(ComplexTypeUtils.Array_Map_Q1) - snc.sql(ComplexTypeUtils.Array_Map_Q2) - snc.sql(ComplexTypeUtils.Array_Map_Q3) if(printContent) { println("snc : Array_Map_Q1 " + snc.sql(ComplexTypeUtils.Array_Map_Q1).show) @@ -74,13 +71,10 @@ class ArraysOfStringInMapAsValue extends SnappySQLJob{ "AS SELECT country, explode(celebrities) FROM FamousPeople") /* --- Verification --- */ - - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Array_Map_Q1, "Array_Map_Q1", - "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Array_Map_Q2, "Array_Map_Q2", - "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Array_Map_Q3, "Array_Map_Q3", - "column", pw, sqlContext) + SnappyTestUtils.tableType = "column" + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Array_Map_Q1, "Array_Map_Q1", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Array_Map_Q2, "Array_Map_Q2", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Array_Map_Q3, "Array_Map_Q3", pw, sqlContext) /* --- Clean up --- */ snc.sql("DROP TABLE IF EXISTS TempFamousPeople") diff --git a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/ArraysOfStructType.scala b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/ArraysOfStructType.scala index 67c2dd58bb..d29546858a 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/ArraysOfStructType.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/ArraysOfStructType.scala @@ -51,12 +51,6 @@ class ArraysOfStructType extends SnappySQLJob{ snc.sql("CREATE TABLE IF NOT EXISTS TwoWheeler USING COLUMN " + "AS (SELECT * FROM TempBike)") - snc.sql(ComplexTypeUtils.ArraysOfStruct_Q1) - snc.sql(ComplexTypeUtils.ArraysOfStruct_Q2) - snc.sql(ComplexTypeUtils.ArraysOfStruct_Q3) - snc.sql(ComplexTypeUtils.ArraysOfStruct_Q4) - snc.sql(ComplexTypeUtils.ArraysOfStruct_Q5) - if(printContent) { println("snc : ArraysOfStruct_Q1 " + (snc.sql(ComplexTypeUtils.ArraysOfStruct_Q1).show())) println("snc : ArraysOfStruct_Q2 " + (snc.sql(ComplexTypeUtils.ArraysOfStruct_Q2).show())) @@ -70,18 +64,19 @@ class ArraysOfStructType extends SnappySQLJob{ arrOfStruct.createOrReplaceTempView("TwoWheeler") /* --- Verification --- */ + SnappyTestUtils.tableType = "column" + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.ArraysOfStruct_Q1, "ArraysOfStruct_Q1", pw, + sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.ArraysOfStruct_Q1, - "ArraysOfStruct_Q1", "column", pw, sqlContext) - // TODO Due to SNAP-2782 Below line is commented, Hydra Framework required changes. -// SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.ArraysOfStruct_Q2, -// "ArraysOfStruct_Q2", "column", pw, sqlContext) -// SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.ArraysOfStruct_Q3, -// "ArraysOfStruct_Q3","column", pw, sqlContext) -// SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.ArraysOfStruct_Q4, -// "ArraysOfStruct_Q4","column", pw, sqlContext) -// SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.ArraysOfStruct_Q5, -// "ArraysOfStruct_Q5","column", pw, sqlContext) +// TODO Due to SNAP-2782 Below line is commented, Hydra Framework required changes. +// SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.ArraysOfStruct_Q2, +// "ArraysOfStruct_Q2", pw, sqlContext) +// SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.ArraysOfStruct_Q3, +// "ArraysOfStruct_Q3", pw, sqlContext) +// SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.ArraysOfStruct_Q4, +// "ArraysOfStruct_Q4", pw, sqlContext) +// SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.ArraysOfStruct_Q5, +// "ArraysOfStruct_Q5", pw, sqlContext) /* --- Clean up --- */ snc.sql("DROP TABLE IF EXISTS TwoWheeler") diff --git a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/MapType.scala b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/MapType.scala index a7e0ebb83c..a463321dd0 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/MapType.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/MapType.scala @@ -53,13 +53,6 @@ class MapType extends SnappySQLJob{ snc.sql("CREATE TABLE IF NOT EXISTS StudentMarksRecord USING COLUMN " + "AS (SELECT * FROM TempStRecord)") - snc.sql(ComplexTypeUtils.Map_Q1) - snc.sql(ComplexTypeUtils.Map_Q2) - snc.sql(ComplexTypeUtils.Map_Q3) - snc.sql(ComplexTypeUtils.Map_Q4) - snc.sql(ComplexTypeUtils.Map_Q5) - snc.sql(ComplexTypeUtils.Map_Q6) - if(printContent) { println("snc Map_Q1:" + snc.sql(ComplexTypeUtils.Map_Q1).show) println("snc Map_Q2:" + snc.sql(ComplexTypeUtils.Map_Q2).show) @@ -74,19 +67,15 @@ class MapType extends SnappySQLJob{ mapType.createTempView("StudentMarksRecord") /* --- Verification --- */ + SnappyTestUtils.tableType = "column" // TODO Due to SNAP-2782 Below line is commented, Hydra Framework required changes. - // SnappyTestUtils.assertQueryFullResultSet(snc, Map_Q1, "Map_Q1", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Map_Q2, - "Map_Q2", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Map_Q3, - "Map_Q3", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Map_Q4, - "Map_Q4", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Map_Q5, - "Map_Q5", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Map_Q6, - "Map_Q6", "column", pw, sqlContext) + // SnappyTestUtils.assertQuery(snc, Map_Q1, "Map_Q1", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Map_Q2, "Map_Q2", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Map_Q3, "Map_Q3", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Map_Q4, "Map_Q4", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Map_Q5, "Map_Q5", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Map_Q6, "Map_Q6", pw, sqlContext) /* --- Clean Up --- */ snc.sql("DROP TABLE IF EXISTS TempStRecord") diff --git a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/MapTypeAPI.scala b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/MapTypeAPI.scala index fe5b6ff51f..7d14fd6069 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/MapTypeAPI.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/MapTypeAPI.scala @@ -129,21 +129,16 @@ class MapTypeAPI extends SnappySQLJob { * 100.0/600.0).as("Percentage")) .orderBy(desc("Percentage")) - SnappyTestUtils.assertQueryFullResultSet(snc, sncSelectDF, sparkSelectDF, - "MapTypeQuery1", "column", pw, sqlContext ) + SnappyTestUtils.tableType = "column" + SnappyTestUtils.assertQuery(snc, sncSelectDF, sparkSelectDF, "MapTypeQuery1", pw ) println("Finish the Map Type Query1") - SnappyTestUtils.assertQueryFullResultSet(snc, sncMapQuery2DF, sparkMapQuery2DF, - "MapTypeQuery2", "column", pw,sqlContext) + SnappyTestUtils.assertQuery(snc, sncMapQuery2DF, sparkMapQuery2DF, "MapTypeQuery2", pw) println("Finish the Map Type Query2") - SnappyTestUtils.assertQueryFullResultSet(snc, sncMapQuery3DF, sparkMapQuery3DF, - "MapTypeQuery3", "column", pw,sqlContext) + SnappyTestUtils.assertQuery(snc, sncMapQuery3DF, sparkMapQuery3DF, "MapTypeQuery3", pw) println("Finish the Map Type Query3") - - SnappyTestUtils.assertQueryFullResultSet(snc, sncMapQuery4DF, sparkMapQuery4DF, - "MapTypeQuery4", "column", pw,sqlContext) + SnappyTestUtils.assertQuery(snc, sncMapQuery4DF, sparkMapQuery4DF, "MapTypeQuery4", pw) println("Finish the Map Type Query4") - SnappyTestUtils.assertQueryFullResultSet(snc, sncMapQuery5DF, sparkMapQuery5DF, - "MapTypeQuery5", "column", pw,sqlContext) + SnappyTestUtils.assertQuery(snc, sncMapQuery5DF, sparkMapQuery5DF, "MapTypeQuery5", pw) println("Finish the Map Type Query5") println("Query MapType Via API, Job Completed....") diff --git a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorAllMixedType.scala b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorAllMixedType.scala index f7d40ce089..876dd2c51d 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorAllMixedType.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorAllMixedType.scala @@ -53,12 +53,6 @@ object SmartConnectorAllMixedType { snc.sql("CREATE TABLE IF NOT EXISTS TwentyTwenty USING COLUMN " + "AS (SELECT * FROM TempTwenty)") - snc.sql(ComplexTypeUtils.Mixed_Q1) - snc.sql(ComplexTypeUtils.Mixed_Q2) - snc.sql(ComplexTypeUtils.Mixed_Q3) - snc.sql(ComplexTypeUtils.Mixed_Q4) - snc.sql(ComplexTypeUtils.Mixed_Q5) - if(printContent) { println("snc : Mixed_Q1 " + (snc.sql(ComplexTypeUtils.Mixed_Q1).show)) println("snc : Mixed_Q2 " + (snc.sql(ComplexTypeUtils.Mixed_Q2).show)) @@ -73,17 +67,18 @@ object SmartConnectorAllMixedType { /* --- Verification --- */ + SnappyTestUtils.tableType = "column" // TODO Due to SNAP-2782 Below line is commented, Hydra Framework required changes. // SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Mixed_Q1, // "Mixed_Q1", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Mixed_Q2, - "Mixed_Q2", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Mixed_Q3, - "Mixed_Q3", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Mixed_Q4, - "Mixed_Q4", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Mixed_Q5, - "Mixed_Q5", "column", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Mixed_Q2, + "Mixed_Q2", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Mixed_Q3, + "Mixed_Q3", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Mixed_Q4, + "Mixed_Q4", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Mixed_Q5, + "Mixed_Q5", pw, sqlContext) /* --- Clean up --- */ snc.sql("DROP TABLE IF EXISTS TwentyTwenty") diff --git a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorArrayType.scala b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorArrayType.scala index 9b72c0e283..e1fb164a1f 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorArrayType.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorArrayType.scala @@ -54,12 +54,7 @@ object SmartConnectorArrayType { "OPTIONS(path '" + dataLocation + "')") snc.sql("CREATE TABLE Student USING COLUMN AS (SELECT * FROM TempArray)") - snc.sql(ComplexTypeUtils.Array_Q1) - snc.sql(ComplexTypeUtils.Array_Q2) - snc.sql(ComplexTypeUtils.Array_Q3) snc.sql(ComplexTypeUtils.Array_View) - snc.sql(ComplexTypeUtils.Array_Q4) - snc.sql(ComplexTypeUtils.Array_Q5) if(printContent) { println(snc.sql(ComplexTypeUtils.Array_Q1).show()) @@ -76,17 +71,12 @@ object SmartConnectorArrayType { "AS SELECT id,name,explode(marks) AS Marks FROM Student") /* --- Verification --- */ - - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Array_Q1, - "Array_Q1", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Array_Q2, - "Array_Q2", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Array_Q3, - "Array_Q3", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Array_Q4, - "Array_Q4", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Array_Q5, - "Array_Q5", "column", pw, sqlContext) + SnappyTestUtils.tableType = "column" + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Array_Q1, "Array_Q1", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Array_Q2, "Array_Q2", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Array_Q3, "Array_Q3", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Array_Q4, "Array_Q4", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Array_Q5, "Array_Q5", pw, sqlContext) /* --- Clean up --- */ spark.sql("DROP VIEW IF EXISTS Student") diff --git a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorArrayTypeAPI1.scala b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorArrayTypeAPI1.scala index a8d5d6e811..053507daa2 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorArrayTypeAPI1.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorArrayTypeAPI1.scala @@ -114,21 +114,21 @@ object SmartConnectorArrayTypeAPI1 { println("sncArrQuery5DF count = " + sncArrQuery5DF.count()) println("sncArrQuery5DF data = " + sncArrQuery5DF.show()) } - - SnappyTestUtils.assertQueryFullResultSet(snc, sncSelectDF, sparkSelectDF, - "Smart Connector ArrayTypeAPIQuery1", "column" , pw, sqlContext) + SnappyTestUtils.tableType = "column" + SnappyTestUtils.assertQuery(snc, sncSelectDF, sparkSelectDF, + "Smart Connector ArrayTypeAPIQuery1" , pw) println("Finished the Smart Connector ArrayType Query 1....") - SnappyTestUtils.assertQueryFullResultSet(snc, sncArrQuery2DF, sparkArrQuery2DF, - "Smart Connector ArrayTypeAPIQuery2", "column", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, sncArrQuery2DF, sparkArrQuery2DF, + "Smart Connector ArrayTypeAPIQuery2", pw) println("Finished the Smart Connector ArrayType Query 2....") - SnappyTestUtils.assertQueryFullResultSet(snc, sncArrQuery3DF, sparkArrQuery3DF, - "Smart Connector ArrayTypeAPIQuery3", "column", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, sncArrQuery3DF, sparkArrQuery3DF, + "Smart Connector ArrayTypeAPIQuery3", pw) println("Finished the Smart Connector ArrayType Query 3....") - SnappyTestUtils.assertQueryFullResultSet(snc, sncArrQuery4DF, sparkArrQuery4DF, - "Smart Connector ArrayTypeAPIQuery4", "column", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, sncArrQuery4DF, sparkArrQuery4DF, + "Smart Connector ArrayTypeAPIQuery4", pw) println("Finished the Smart Connector ArrayType Query 4....") - SnappyTestUtils.assertQueryFullResultSet(snc, sncArrQuery5DF, sparkArrQuery5DF, - "Smart Connector ArrayTypeAPIQuery5", "column", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, sncArrQuery5DF, sparkArrQuery5DF, + "Smart Connector ArrayTypeAPIQuery5", pw) println("Finished the Smart Connector ArrayType Query 5....") pw.close() diff --git a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorArraysOfStringInMapAsValue.scala b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorArraysOfStringInMapAsValue.scala index d46120b53e..742dbf8c22 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorArraysOfStringInMapAsValue.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorArraysOfStringInMapAsValue.scala @@ -57,9 +57,6 @@ object SmartConnectorArraysOfStringInMapAsValue { "USING COLUMN AS (SELECT * FROM TempFamousPeople)") snc.sql(ComplexTypeUtils.Array_Map_TempView) - snc.sql(ComplexTypeUtils.Array_Map_Q1) - snc.sql(ComplexTypeUtils.Array_Map_Q2) - snc.sql(ComplexTypeUtils.Array_Map_Q3) if(printContent) { println("snc : Array_Map_Q1 " + snc.sql(ComplexTypeUtils.Array_Map_Q1).show) @@ -74,13 +71,13 @@ object SmartConnectorArraysOfStringInMapAsValue { "AS SELECT country, explode(celebrities) FROM FamousPeople") /* --- Verification --- */ - - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Array_Map_Q1, "Array_Map_Q1", - "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Array_Map_Q2, "Array_Map_Q2", - "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Array_Map_Q3, "Array_Map_Q3", - "column", pw, sqlContext) + SnappyTestUtils.tableType = "column" + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Array_Map_Q1, "Array_Map_Q1", + pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Array_Map_Q2, "Array_Map_Q2", + pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Array_Map_Q3, "Array_Map_Q3", + pw, sqlContext) /* --- Clean up --- */ snc.sql("DROP TABLE IF EXISTS TempFamousPeople") diff --git a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorArraysOfStructType.scala b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorArraysOfStructType.scala index 0917807591..e2e12cfc8a 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorArraysOfStructType.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorArraysOfStructType.scala @@ -54,12 +54,6 @@ object SmartConnectorArraysOfStructType { snc.sql("CREATE TABLE IF NOT EXISTS TwoWheeler USING COLUMN " + "AS (SELECT * FROM TempBike)") - snc.sql(ComplexTypeUtils.ArraysOfStruct_Q1) - snc.sql(ComplexTypeUtils.ArraysOfStruct_Q2) - snc.sql(ComplexTypeUtils.ArraysOfStruct_Q3) - snc.sql(ComplexTypeUtils.ArraysOfStruct_Q4) - snc.sql(ComplexTypeUtils.ArraysOfStruct_Q5) - if(printContent) { println("snc : ArraysOfStruct_Q1 " + (snc.sql(ComplexTypeUtils.ArraysOfStruct_Q1).show())) println("snc : ArraysOfStruct_Q2 " + (snc.sql(ComplexTypeUtils.ArraysOfStruct_Q2).show())) @@ -73,18 +67,18 @@ object SmartConnectorArraysOfStructType { arrOfStruct.createOrReplaceTempView("TwoWheeler") /* --- Verification --- */ - - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.ArraysOfStruct_Q1, - "ArraysOfStruct_Q1", "column", pw, sqlContext) + SnappyTestUtils.tableType = "column" + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.ArraysOfStruct_Q1, + "ArraysOfStruct_Q1", pw, sqlContext) // TODO Due to SNAP-2782 Below line is commented, Hydra Framework required changes. - // SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.ArraysOfStruct_Q2, - // "ArraysOfStruct_Q2", "column", pw, sqlContext) - // SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.ArraysOfStruct_Q3, - // "ArraysOfStruct_Q3","column", pw, sqlContext) - // SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.ArraysOfStruct_Q4, - // "ArraysOfStruct_Q4","column", pw, sqlContext) - // SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.ArraysOfStruct_Q5, - // "ArraysOfStruct_Q5","column", pw, sqlContext) + // SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.ArraysOfStruct_Q2, + // "ArraysOfStruct_Q2", pw, sqlContext) + // SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.ArraysOfStruct_Q3, + // "ArraysOfStruct_Q3", pw, sqlContext) + // SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.ArraysOfStruct_Q4, + // "ArraysOfStruct_Q4", pw, sqlContext) + // SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.ArraysOfStruct_Q5, + // "ArraysOfStruct_Q5", pw, sqlContext) /* --- Clean up --- */ snc.sql("DROP TABLE IF EXISTS TwoWheeler") diff --git a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorMapType.scala b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorMapType.scala index 00bc1b9a21..85a33b1ee3 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorMapType.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorMapType.scala @@ -54,13 +54,6 @@ object SmartConnectorMapType { snc.sql("CREATE TABLE IF NOT EXISTS StudentMarksRecord USING COLUMN " + "AS (SELECT * FROM TempStRecord)") - snc.sql(ComplexTypeUtils.Map_Q1) - snc.sql(ComplexTypeUtils.Map_Q2) - snc.sql(ComplexTypeUtils.Map_Q3) - snc.sql(ComplexTypeUtils.Map_Q4) - snc.sql(ComplexTypeUtils.Map_Q5) - snc.sql(ComplexTypeUtils.Map_Q6) - if(printContent) { println("snc Map_Q1:" + snc.sql(ComplexTypeUtils.Map_Q1).show) println("snc Map_Q2:" + snc.sql(ComplexTypeUtils.Map_Q2).show) @@ -75,19 +68,14 @@ object SmartConnectorMapType { mapType.createTempView("StudentMarksRecord") /* --- Verification --- */ - + SnappyTestUtils.tableType = "column" // TODO Due to SNAP-2782 Below line is commented, Hydra Framework required changes. - // SnappyTestUtils.assertQueryFullResultSet(snc, Map_Q1, "Map_Q1", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Map_Q2, - "Map_Q2", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Map_Q3, - "Map_Q3", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Map_Q4, - "Map_Q4", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Map_Q5, - "Map_Q5", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Map_Q6, - "Map_Q6", "column", pw, sqlContext) + // SnappyTestUtils.assertQueryFullResultSet(snc, Map_Q1, "Map_Q1", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Map_Q2, "Map_Q2", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Map_Q3, "Map_Q3", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Map_Q4, "Map_Q4", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Map_Q5, "Map_Q5", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Map_Q6, "Map_Q6", pw, sqlContext) /* --- Clean Up --- */ snc.sql("DROP TABLE IF EXISTS TempStRecord") diff --git a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorMapTypeAPI.scala b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorMapTypeAPI.scala index 7c6805bbf7..cadf781ed9 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorMapTypeAPI.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorMapTypeAPI.scala @@ -139,21 +139,17 @@ object SmartConnectorMapTypeAPI { * 100.0/600.0).as("Percentage")) .orderBy(desc("Percentage")) - SnappyTestUtils.assertQueryFullResultSet(snc, sncSelectDF, sparkSelectDF, - "MapTypeQuery1", "column", pw, sqlContext ) + SnappyTestUtils.tableType = "column" + SnappyTestUtils.assertQuery(snc, sncSelectDF, sparkSelectDF, "MapTypeQuery1", pw ) println("Finish the Map Type Query1") - SnappyTestUtils.assertQueryFullResultSet(snc, sncMapQuery2DF, sparkMapQuery2DF, - "MapTypeQuery2", "column", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, sncMapQuery2DF, sparkMapQuery2DF, "MapTypeQuery2", pw) println("Finish the Map Type Query2") - SnappyTestUtils.assertQueryFullResultSet(snc, sncMapQuery3DF, sparkMapQuery3DF, - "MapTypeQuery3", "column", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, sncMapQuery3DF, sparkMapQuery3DF, "MapTypeQuery3", pw) println("Finish the Map Type Query3") - SnappyTestUtils.assertQueryFullResultSet(snc, sncMapQuery4DF, sparkMapQuery4DF, - "MapTypeQuery4", "column", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, sncMapQuery4DF, sparkMapQuery4DF, "MapTypeQuery4", pw) println("Finish the Map Type Query4") - SnappyTestUtils.assertQueryFullResultSet(snc, sncMapQuery5DF, sparkMapQuery5DF, - "MapTypeQuery5", "column", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, sncMapQuery5DF, sparkMapQuery5DF, "MapTypeQuery5", pw) println("Finish the Map Type Query6") println("Query MapType Via API, Job Completed....") diff --git a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorStructType.scala b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorStructType.scala index 5a8d81fced..9a8a7947f5 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorStructType.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorStructType.scala @@ -53,11 +53,6 @@ object SmartConnectorStructType { snc.sql("CREATE TABLE IF NOT EXISTS CricketRecord USING COLUMN " + "AS (SELECT * FROM TempCRRecord)") - snc.sql(ComplexTypeUtils.Struct_Q1) - snc.sql(ComplexTypeUtils.Struct_Q2) - snc.sql(ComplexTypeUtils.Struct_Q3) - snc.sql(ComplexTypeUtils.Struct_Q4) - if(printContent) { println("Struct_Q1 : " + snc.sql(ComplexTypeUtils.Struct_Q1).show()) println("Struct_Q2 : " + snc.sql(ComplexTypeUtils.Struct_Q2).show()) @@ -68,16 +63,12 @@ object SmartConnectorStructType { /* --- Create the Spark Tables / Views for Verification --- */ val structType = spark.read.json(dataLocation) structType.createTempView("CricketRecord") - + SnappyTestUtils.tableType = "column" /* --- Verification --- */ - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Struct_Q1, - "Struct_Q1", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Struct_Q2, - "Struct_Q2", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Struct_Q3, - "Struct_Q3", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Struct_Q4, - "Struct_Q4", "column", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Struct_Q1, "Struct_Q1", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Struct_Q2, "Struct_Q2", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Struct_Q3, "Struct_Q3", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Struct_Q4, "Struct_Q4", pw, sqlContext) /* --- Clean up --- */ snc.sql("DROP TABLE IF EXISTS CricketRecord") diff --git a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorStructTypeAPI.scala b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorStructTypeAPI.scala index 8945c56963..fe023b5570 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorStructTypeAPI.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/SmartConnectorStructTypeAPI.scala @@ -83,17 +83,18 @@ object SmartConnectorStructTypeAPI { sparkReadDF("TestRecord").getField("Avg")) .orderBy(desc("Matches")) - SnappyTestUtils.assertQueryFullResultSet(snc, sncStructQuery1DF, sparkStructQuery1DF, - "StructTypeQuery1", "column", pw, sqlContext) + SnappyTestUtils.tableType = "column" + SnappyTestUtils.assertQuery(snc, sncStructQuery1DF, sparkStructQuery1DF, + "StructTypeQuery1", pw) println("Finished the StructType Query-1") - SnappyTestUtils.assertQueryFullResultSet(snc, sncStructQuery2DF, sparkStructQuery2DF, - "StructTypeQuery2", "column", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, sncStructQuery2DF, sparkStructQuery2DF, + "StructTypeQuery2", pw) println("Finished the StructType Query-2") - SnappyTestUtils.assertQueryFullResultSet(snc, sncStructQuery3DF, sparkStructQuery3DF, - "StructTypeQuery3", "column", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, sncStructQuery3DF, sparkStructQuery3DF, + "StructTypeQuery3", pw) println("Finished the StructType Query-3") - SnappyTestUtils.assertQueryFullResultSet(snc, sncStructQuery4DF, sparkStructQuery4DF, - "StructTypeQuery4", "column", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, sncStructQuery4DF, sparkStructQuery4DF, + "StructTypeQuery4", pw) println("Finished the StructType Query-4") diff --git a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/StructType.scala b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/StructType.scala index dfd00b00c2..6b4a19953b 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/StructType.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/StructType.scala @@ -52,11 +52,6 @@ class StructType extends SnappySQLJob{ snc.sql("CREATE TABLE IF NOT EXISTS CricketRecord USING COLUMN " + "AS (SELECT * FROM TempCRRecord)") - snc.sql(ComplexTypeUtils.Struct_Q1) - snc.sql(ComplexTypeUtils.Struct_Q2) - snc.sql(ComplexTypeUtils.Struct_Q3) - snc.sql(ComplexTypeUtils.Struct_Q4) - if(printContent) { println("Struct_Q1 : " + snc.sql(ComplexTypeUtils.Struct_Q1).show()) println("Struct_Q2 : " + snc.sql(ComplexTypeUtils.Struct_Q2).show()) @@ -68,15 +63,12 @@ class StructType extends SnappySQLJob{ val structType = spark.read.json(dataLocation) structType.createTempView("CricketRecord") + SnappyTestUtils.tableType = "column" /* --- Verification --- */ - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Struct_Q1, - "Struct_Q1", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Struct_Q2, - "Struct_Q2", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Struct_Q3, - "Struct_Q3", "column", pw, sqlContext) - SnappyTestUtils.assertQueryFullResultSet(snc, ComplexTypeUtils.Struct_Q4, - "Struct_Q4", "column", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Struct_Q1, "Struct_Q1", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Struct_Q2, "Struct_Q2", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Struct_Q3, "Struct_Q3", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, ComplexTypeUtils.Struct_Q4, "Struct_Q4", pw, sqlContext) /* --- Clean up --- */ snc.sql("DROP TABLE IF EXISTS CricketRecord") diff --git a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/StructTypeAPI.scala b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/StructTypeAPI.scala index 19bfc3477a..37c85860e1 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/StructTypeAPI.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/complexdatatypes/StructTypeAPI.scala @@ -86,19 +86,19 @@ class StructTypeAPI extends SnappySQLJob { println("snc count : " + sncStructQuery4DF.count()) println("spark count : " + sparkStructQuery4DF.count()) - - SnappyTestUtils.assertQueryFullResultSet(snc, sncStructQuery1DF, sparkStructQuery1DF, - "StructTypeQuery1", "column", pw, sqlContext) + SnappyTestUtils.tableType = "column" + SnappyTestUtils.assertQuery(snc, sncStructQuery1DF, sparkStructQuery1DF, + "StructTypeQuery1", pw) println("Finished the StructType Query-1") - SnappyTestUtils.assertQueryFullResultSet(snc, sncStructQuery2DF, sparkStructQuery2DF, - "StructTypeQuery2", "column", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, sncStructQuery2DF, sparkStructQuery2DF, + "StructTypeQuery2", pw) println("Finished the StructType Query-2") - SnappyTestUtils.assertQueryFullResultSet(snc, sncStructQuery3DF, sparkStructQuery3DF, - "StructTypeQuery3", "column", pw, sqlContext) + SnappyTestUtils.assertQuery(snc, sncStructQuery3DF, sparkStructQuery3DF, + "StructTypeQuery3", pw) println("Finished the StructType Query-3") - SnappyTestUtils.assertQueryFullResultSet(snc, sncStructQuery4DF, sparkStructQuery4DF, - "StructTypeQuery4", "column", pw, sqlContext) - println("Finished the StructType Query-4") + SnappyTestUtils.assertQuery(snc, sncStructQuery4DF, sparkStructQuery4DF, + "StructTypeQuery4", pw) + println("Finished the StructType Query-4") println("Query StructType Via API, Job completed....") } diff --git a/dtests/src/test/scala/io/snappydata/hydra/concurrency/ConcTestUtils.scala b/dtests/src/test/scala/io/snappydata/hydra/concurrency/ConcTestUtils.scala index a894d07ffb..da03e28a2f 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/concurrency/ConcTestUtils.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/concurrency/ConcTestUtils.scala @@ -25,18 +25,19 @@ import org.apache.spark.sql.{SQLContext, SnappyContext} object ConcTestUtils { def validateAnalyticalQueriesFullResultSet(snc: SnappyContext, tableType: String, pw: PrintWriter, sqlContext: SQLContext): Unit = { + SnappyTestUtils.validateFullResultSet = true; for (q <- NWQueries.queries) { q._1 match { - case "Q37" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q37, "Q37", - tableType, pw, sqlContext) - case "Q55" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q55, "Q55", - tableType, pw, sqlContext) - case "Q36" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q36, "Q36", - tableType, pw, sqlContext) - case "Q56" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q56, "Q56", - tableType, pw, sqlContext) - case "Q38" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q38, "Q38", - tableType, pw, sqlContext) + case "Q37" => SnappyTestUtils.assertJoin(snc, NWQueries.Q37, "Q37", + pw, sqlContext) + case "Q55" => SnappyTestUtils.assertJoin(snc, NWQueries.Q55, "Q55", + pw, sqlContext) + case "Q36" => SnappyTestUtils.assertJoin(snc, NWQueries.Q36, "Q36", + pw, sqlContext) + case "Q56" => SnappyTestUtils.assertJoin(snc, NWQueries.Q56, "Q56", + pw, sqlContext) + case "Q38" => SnappyTestUtils.assertJoin(snc, NWQueries.Q38, "Q38", + pw, sqlContext) // scalastyle:off println case _ => println("OK") // scalastyle:on println @@ -46,114 +47,115 @@ object ConcTestUtils { def validatePointLookUPQueriesFullResultSet(snc: SnappyContext, tableType: String, pw: PrintWriter, sqlContext: SQLContext): Unit = { + SnappyTestUtils.validateFullResultSet = true; for (q <- NWPLQueries.queries) { q._1 match { - case "Q1" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q1, "Q1", - tableType, pw, sqlContext) - case "Q2" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q2, "Q2", - tableType, pw, sqlContext) - case "Q3" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q3, "Q3", - tableType, pw, sqlContext) - case "Q4" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q4, "Q4", - tableType, pw, sqlContext) - case "Q5" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q5, "Q5", - tableType, pw, sqlContext) - case "Q6" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q6, "Q6", - tableType, pw, sqlContext) - case "Q7" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q7, "Q7", - tableType, pw, sqlContext) - case "Q8" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q8, "Q8", - tableType, pw, sqlContext) - case "Q9" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q9, "Q9", - tableType, pw, sqlContext) - case "Q10" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q5, "Q10", - tableType, pw, sqlContext) - case "Q11" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q11, "Q11", - tableType, pw, sqlContext) - case "Q12" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q12, "Q12", - tableType, pw, sqlContext) - case "Q13" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q13, "Q13", - tableType, pw, sqlContext) - case "Q14" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q14, "Q14", - tableType, pw, sqlContext) - case "Q15" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q15, "Q15", - tableType, pw, sqlContext) - case "Q16" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q16, "Q16", - tableType, pw, sqlContext) - case "Q17" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q17, "Q17", - tableType, pw, sqlContext) - case "Q18" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q18, "Q18", - tableType, pw, sqlContext) - case "Q19" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q19, "Q19", - tableType, pw, sqlContext) - case "Q20" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q20, "Q20", - tableType, pw, sqlContext) - case "Q21" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q21, "Q21", - tableType, pw, sqlContext) - case "Q22" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q22, "Q22", - tableType, pw, sqlContext) - case "Q23" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q23, "Q23", - tableType, pw, sqlContext) - case "Q24" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q24, "Q24", - tableType, pw, sqlContext) - case "Q25" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q25, "Q25", - tableType, pw, sqlContext) - case "Q26" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q26, "Q28", - tableType, pw, sqlContext) - case "Q27" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q27, "Q27", - tableType, pw, sqlContext) - case "Q28" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q28, "Q28", - tableType, pw, sqlContext) - case "Q29" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q29, "Q29", - tableType, pw, sqlContext) - case "Q30" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q30, "Q30", - tableType, pw, sqlContext) - case "Q31" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q31, "Q31", - tableType, pw, sqlContext) - case "Q32" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q32, "Q32", - tableType, pw, sqlContext) - case "Q33" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q33, "Q33", - tableType, pw, sqlContext) - case "Q34" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q34, "Q34", - tableType, pw, sqlContext) - case "Q35" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q35, "Q35", - tableType, pw, sqlContext) - case "Q36" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q36, "Q36", - tableType, pw, sqlContext) - case "Q37" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q37, "Q37", - tableType, pw, sqlContext) - case "Q38" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q38, "Q38", - tableType, pw, sqlContext) - case "Q39" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q39, "Q39", - tableType, pw, sqlContext) - case "Q40" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q40, "Q40", - tableType, pw, sqlContext) - case "Q41" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q41, "Q41", - tableType, pw, sqlContext) - case "Q42" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q42, "Q42", - tableType, pw, sqlContext) - case "Q43" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q43, "Q43", - tableType, pw, sqlContext) - case "Q44" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q44, "Q44", - tableType, pw, sqlContext) - case "Q45" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q45, "Q45", - tableType, pw, sqlContext) - case "Q46" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q46, "Q46", - tableType, pw, sqlContext) - case "Q47" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q47, "Q47", - tableType, pw, sqlContext) - case "Q48" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q48, "Q48", - tableType, pw, sqlContext) - case "Q49" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q49, "Q49", - tableType, pw, sqlContext) - case "Q50" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q50, "Q50", - tableType, pw, sqlContext) - case "Q51" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q51, "Q51", - tableType, pw, sqlContext) - /* case "Q52" => SnappyTestUtils.assertQueryFullResultSet(snc, NWPLQueries.Q52, "Q52", - tableType, pw, sqlContext) */ + case "Q1" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q1, "Q1", + pw, sqlContext) + case "Q2" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q2, "Q2", + pw, sqlContext) + case "Q3" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q3, "Q3", + pw, sqlContext) + case "Q4" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q4, "Q4", + pw, sqlContext) + case "Q5" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q5, "Q5", + pw, sqlContext) + case "Q6" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q6, "Q6", + pw, sqlContext) + case "Q7" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q7, "Q7", + pw, sqlContext) + case "Q8" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q8, "Q8", + pw, sqlContext) + case "Q9" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q9, "Q9", + pw, sqlContext) + case "Q10" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q5, "Q10", + pw, sqlContext) + case "Q11" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q11, "Q11", + pw, sqlContext) + case "Q12" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q12, "Q12", + pw, sqlContext) + case "Q13" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q13, "Q13", + pw, sqlContext) + case "Q14" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q14, "Q14", + pw, sqlContext) + case "Q15" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q15, "Q15", + pw, sqlContext) + case "Q16" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q16, "Q16", + pw, sqlContext) + case "Q17" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q17, "Q17", + pw, sqlContext) + case "Q18" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q18, "Q18", + pw, sqlContext) + case "Q19" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q19, "Q19", + pw, sqlContext) + case "Q20" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q20, "Q20", + pw, sqlContext) + case "Q21" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q21, "Q21", + pw, sqlContext) + case "Q22" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q22, "Q22", + pw, sqlContext) + case "Q23" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q23, "Q23", + pw, sqlContext) + case "Q24" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q24, "Q24", + pw, sqlContext) + case "Q25" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q25, "Q25", + pw, sqlContext) + case "Q26" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q26, "Q28", + pw, sqlContext) + case "Q27" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q27, "Q27", + pw, sqlContext) + case "Q28" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q28, "Q28", + pw, sqlContext) + case "Q29" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q29, "Q29", + pw, sqlContext) + case "Q30" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q30, "Q30", + pw, sqlContext) + case "Q31" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q31, "Q31", + pw, sqlContext) + case "Q32" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q32, "Q32", + pw, sqlContext) + case "Q33" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q33, "Q33", + pw, sqlContext) + case "Q34" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q34, "Q34", + pw, sqlContext) + case "Q35" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q35, "Q35", + pw, sqlContext) + case "Q36" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q36, "Q36", + pw, sqlContext) + case "Q37" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q37, "Q37", + pw, sqlContext) + case "Q38" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q38, "Q38", + pw, sqlContext) + case "Q39" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q39, "Q39", + pw, sqlContext) + case "Q40" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q40, "Q40", + pw, sqlContext) + case "Q41" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q41, "Q41", + pw, sqlContext) + case "Q42" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q42, "Q42", + pw, sqlContext) + case "Q43" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q43, "Q43", + pw, sqlContext) + case "Q44" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q44, "Q44", + pw, sqlContext) + case "Q45" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q45, "Q45", + pw, sqlContext) + case "Q46" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q46, "Q46", + pw, sqlContext) + case "Q47" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q47, "Q47", + pw, sqlContext) + case "Q48" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q48, "Q48", + pw, sqlContext) + case "Q49" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q49, "Q49", + pw, sqlContext) + case "Q50" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q50, "Q50", + pw, sqlContext) + case "Q51" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q51, "Q51", + pw, sqlContext) + /* case "Q52" => SnappyTestUtils.assertQuery(snc, NWPLQueries.Q52, "Q52", + pw, sqlContext) */ // scalastyle:off println - case _ => println("OK") + case _ => pw.println(s"Query ${q._1} has not been executed.") // scalastyle:on println } } diff --git a/dtests/src/test/scala/io/snappydata/hydra/ct/CTQueries.scala b/dtests/src/test/scala/io/snappydata/hydra/ct/CTQueries.scala index a5b2a2fc5d..9fb9a58193 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/ct/CTQueries.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/ct/CTQueries.scala @@ -41,17 +41,20 @@ object CTQueries { val query8: String = "select SUM(TOTAL_EXECUTED_QTY),SRC_SYS from ORDERS_DETAILS GROUP BY SRC_SYS" - val query9: String = "select SUM(TOTAL_EXECUTED_QTY),MIN(TOTAL_EXECUTED_QTY),MAX(TOTAL_EXECUTED_QTY),SRC_SYS " + + val query9: String = "select SUM(TOTAL_EXECUTED_QTY),MIN(TOTAL_EXECUTED_QTY)," + + "MAX(TOTAL_EXECUTED_QTY),SRC_SYS " + "from ORDERS_DETAILS WHERE SRC_SYS='APFF' GROUP BY SRC_SYS" val query10: String = "select count(*) from ORDERS_DETAILS where Src_sys='OATC'" - val query11: String = "select '5-CTFIX_ORDER' as SrcFl, a.* from ORDERS_DETAILS a , ORDERS_DETAILS b " + - "where a.glb_root_order_id = b.glb_root_order_id and a.trd_date >='20160413' and b.trd_date >='20160413' " + + val query11: String = "select '5-CTFIX_ORDER' as SrcFl, a.* " + + "from ORDERS_DETAILS a , ORDERS_DETAILS b " + + "where a.glb_root_order_id = b.glb_root_order_id " + + "and a.trd_date >='20160413' and b.trd_date >='20160413' " + "and b.src_sys ='CRIO' order by a.glb_root_order_id, a.trd_datE" - val query12: String = "select '4-CTFIX_ORDER' as SrcFl, a.glb_root_order_id, a.src_sys, count(*) " + - "from ORDERS_DETAILS a , ORDERS_DETAILS b " + + val query12: String = "select '4-CTFIX_ORDER' as SrcFl, a.glb_root_order_id, a.src_sys, " + + "count(*) from ORDERS_DETAILS a , ORDERS_DETAILS b " + "where a.glb_root_order_id = b.glb_root_order_id and a.trd_date ='20160413' " + "and b.trd_date ='20160413' and b.src_sys ='CRIO' " + "group by a.glb_root_order_id, a.src_sys order by a.glb_root_order_id, a.src_sys" @@ -64,13 +67,15 @@ object CTQueries { val query15: String = "select '5-CTFIX_ORDER' as SrcFl, * from ORDERS_DETAILS " + "where trd_date>='20160413' and glb_root_order_id in " + - "( select glb_root_order_id from ORDERS_DETAILS where trd_date>='20160413' and src_sys='CRIO' ) " + + "( select glb_root_order_id from ORDERS_DETAILS where trd_date>='20160413' " + + "and src_sys='CRIO' ) " + "order by glb_root_order_id, trd_datE" val query16: String = "select '4-CTFIX_ORDER' as SrcFl, glb_root_order_id, src_sys, count(*) " + "from ORDERS_DETAILS " + "where trd_date='20160413' and glb_root_order_id in " + - "( select glb_root_order_id from ORDERS_DETAILS where trd_date='20160413' and src_sys='CRIO') " + + "( select glb_root_order_id from ORDERS_DETAILS where trd_date='20160413' " + + "and src_sys='CRIO') " + "group by glb_root_order_id, src_sys order by glb_root_order_id, src_sys" val query17: String = "select Event_type_cd, count(1) from ORDERS_DETAILS " + @@ -83,73 +88,90 @@ object CTQueries { "WHERE ESOD.TRD_DATE = '20160413' AND ESOD.EVENT_TYPE_CD = 'NEW_CONF' " + "AND ESOD.EXEC_INSTR like '%A%' GROUP BY ESOD.EXEC_INSTR" - val query20: String = "select EVENT_RCV_TS, EVENT_TS, src_sys,glb_root_src_sys_id,glb_root_order_id, " + - "ticker_symbol,SIDE,order_qty,EVENT_TYPE_CD,product_cat_cd,cntry_cd " + + val query20: String = "select EVENT_RCV_TS, EVENT_TS, src_sys, glb_root_src_sys_id, " + + "glb_root_order_id, ticker_symbol,SIDE,order_qty,EVENT_TYPE_CD,product_cat_cd,cntry_cd " + "from ORDERS_DETAILS " + - "where trd_date > '20160212' and src_sys='CAIQS' and event_ts not like '%.%' order by EVENT_RCV_TS limit 100 " + "where trd_date > '20160212' and src_sys='CAIQS' " + + "and event_ts not like '%.%' order by EVENT_RCV_TS limit 100 " val query21: String = "select event_type_cd,event_rcv_ts,event_ts,sent_ts " + "from ORDERS_DETAILS " + - "where trd_date='20160413' and glb_root_order_id='15344x8c7' and sys_order_id='20151210.92597'" + "where trd_date='20160413' and glb_root_order_id='15344x8c7' " + + "and sys_order_id='20151210.92597'" - val query22: String = "select count(*) from EXEC_DETAILS a LEFT JOIN ORDERS_DETAILS b using (sys_root_order_id)" + val query22: String = "select count(*) from EXEC_DETAILS a " + + "LEFT JOIN ORDERS_DETAILS b using (sys_root_order_id)" - val query23: String = "(select TRD_DATE, ROOT_FLOW_CAT, sum(Notional) as notional, count(*) as trades, sum(shares) as shares " + + val query23: String = "(select TRD_DATE, ROOT_FLOW_CAT, sum(Notional) as notional, " + + "count(*) as trades, sum(shares) as shares " + "from " + - "(select execs.sys_order_id, execs.EXECUTED_QTY * execs.EXEC_PRICE as notional, execs.EXECUTED_QTY as shares, " + - "execs.TRD_DATE, case when coalesce(root_exec.flow_cat,root.flow_cat) is null then 'UNKNOWN' else " + + "(select execs.sys_order_id, execs.EXECUTED_QTY * execs.EXEC_PRICE as notional, " + + "execs.EXECUTED_QTY as shares, execs.TRD_DATE, " + + "case when coalesce(root_exec.flow_cat,root.flow_cat) is null then 'UNKNOWN' else " + "coalesce(root_exec.flow_cat,root.flow_cat) end as ROOT_FLOW_CAT " + "from EXEC_DETAILS as execs left join " + "( select distinct TRD_DATE,glb_root_order_id,flow_cat " + "from EXEC_DETAILS where TRD_DATE in ('20160325','20160413' ) " + "and (PRODUCT_CAT_CD is null or PRODUCT_CAT_CD not in ('OPT','FUT','MLEG')) " + "and (exec_price_curr_cd = 'USD' OR exec_price_curr_cd is null) " + - "and sys_src_sys_id in ('93', '7', '70', '115' ,'6','150','189','31','157','185','7','153','163133','80','51','139','137') " + - "and sys_order_id = glb_root_order_id and sys_src_sys_id = glb_root_src_sys_id )root_exec on " + - "execs.trd_date=root_exec.trd_date and execs.glb_root_order_id=root_exec.glb_root_order_id left join " + - "( select distinct TRD_DATE, glb_root_order_id,flow_cat " + + "and sys_src_sys_id in ('93', '7', '70', '115' ,'6','150','189','31','157','185'," + + "'7','153','163133','80','51','139','137') " + + "and sys_order_id = glb_root_order_id " + + "and sys_src_sys_id = glb_root_src_sys_id )root_exec on " + + "execs.trd_date=root_exec.trd_date and execs.glb_root_order_id=root_exec.glb_root_order_id " + + "left join " + + "(select distinct TRD_DATE, glb_root_order_id,flow_cat " + "from ORDERS_DETAILS T " + "where T.sys_order_id = T.glb_root_order_id " + "and T.sys_src_sys_id = T.glb_root_src_sys_id " + - "and T.sys_src_sys_id in ('93', '7', '70', '115' ,'6','150','189','31','157','185','7','153','163133','80','51','139','137') " + + "and T.sys_src_sys_id in ('93', '7', '70', '115' ,'6','150','189','31','157','185','7'," + + "'153','163133','80','51','139','137') " + "and T.TRD_DATE in ('20160325','20160413' ) " + "and (T.CURR_CD = 'USD' or T.CURR_CD is null) " + "and (T.PRODUCT_CAT_CD is null or T.PRODUCT_CAT_CD not in ('OPT', 'FUT','MLEG')) ) root on " + "execs.trd_date=root.trd_date and execs.glb_root_order_id=root.glb_root_order_id " + "where execs.LEAF_EXEC_FG = 'Y' " + "and execs.event_type_cd = 'FILLED_CONF' " + - "and execs.sys_src_sys_id in ('93', '7', '70', '115' ,'6','150','189','31','157','185','7','153','163133','80','51','139','137') " + + "and execs.sys_src_sys_id in ('93', '7', '70', '115' ,'6','150','189','31','157','185','7'," + + "'153','163133','80','51','139','137') " + "and execs.SYS_ORDER_STAT_CD in ('2','1') " + "and execs.TRD_DATE in ('20160325','20160413' ) " + "and (execs.PRODUCT_CAT_CD is null or execs.PRODUCT_CAT_CD not in ('OPT', 'FUT','MLEG')) " + "and (execs.exec_price_curr_cd = 'USD' or execs.exec_price_curr_cd = null) ) " + "Aggregated group by TRD_DATE, ROOT_FLOW_CAT order by TRD_DATE ) " + "union all " + - "( select TRD_DATE, ROOT_FLOW_CAT, sum(Notional) as notional, count(*) as trades, sum (shares) as shares " + + "(select TRD_DATE, ROOT_FLOW_CAT, sum(Notional) as notional, " + + "count(*) as trades, sum (shares) as shares " + "from " + - "(select execs.sys_order_id, execs.EXECUTED_QTY * execs.EXEC_PRICE as notional, execs.EXECUTED_QTY as shares, " + + "(select execs.sys_order_id, execs.EXECUTED_QTY * execs.EXEC_PRICE as notional, " + + "execs.EXECUTED_QTY as shares, " + "execs.TRD_DATE, 'ALL' as ROOT_FLOW_CAT " + "from EXEC_DETAILS as execs " + "left join ( select distinct TRD_DATE,glb_root_order_id,flow_cat " + "from EXEC_DETAILS where TRD_DATE in ('20160325','20160413' ) " + "and (PRODUCT_CAT_CD is null or PRODUCT_CAT_CD not in ('OPT','FUT','MLEG')) " + "and (exec_price_curr_cd = 'USD' OR exec_price_curr_cd is null) " + - "and sys_src_sys_id in ('93', '7', '70', '115' ,'6','150','189','31','157','185','7','153','163133','80','51','139','137') " + - "and sys_order_id = glb_root_order_id and sys_src_sys_id = glb_root_src_sys_id )root_exec on " + - "execs.trd_date=root_exec.trd_date and execs.glb_root_order_id=root_exec.glb_root_order_id left join " + + "and sys_src_sys_id in ('93', '7', '70', '115' ,'6','150','189','31','157','185','7'," + + "'153','163133','80','51','139','137') " + + "and sys_order_id = glb_root_order_id and sys_src_sys_id = glb_root_src_sys_id) " + + "root_exec on " + + "execs.trd_date=root_exec.trd_date and " + + "execs.glb_root_order_id=root_exec.glb_root_order_id left join " + "( select distinct TRD_DATE, glb_root_order_id,flow_cat " + "from ORDERS_DETAILS T " + "where T.sys_order_id = T.glb_root_order_id " + "and T.sys_src_sys_id = T.glb_root_src_sys_id " + - "and T.sys_src_sys_id in ('93', '7', '70', '115' ,'6','150','189','31','157','185','7','153','163133','80','51','139','137') " + + "and T.sys_src_sys_id in ('93', '7', '70', '115' ,'6','150','189','31','157','185','7'," + + "'153','163133','80','51','139','137') " + "and T.TRD_DATE in ('20160325','20160413' ) " + "and (T.CURR_CD = 'USD' or T.CURR_CD is null) " + "and (T.PRODUCT_CAT_CD is null or T.PRODUCT_CAT_CD not in ('OPT', 'FUT','MLEG')) ) root on " + "execs.trd_date=root.trd_date and execs.glb_root_order_id=root.glb_root_order_id " + "where execs.LEAF_EXEC_FG = 'Y' " + "and execs.event_type_cd = 'FILLED_CONF' " + - "and execs.sys_src_sys_id in ('93', '7', '70', '115' ,'6','150','189','31','157','185','7','153','163133','80','51','139','137') " + + "and execs.sys_src_sys_id in ('93', '7', '70', '115' ,'6','150','189','31','157','185','7'," + + "'153','163133','80','51','139','137') " + "and execs.SYS_ORDER_STAT_CD in ('2','1') " + "and execs.TRD_DATE in ('20160325','20160413' ) " + "and (execs.PRODUCT_CAT_CD is null or execs.PRODUCT_CAT_CD not in ('OPT', 'FUT','MLEG')) " + diff --git a/dtests/src/test/scala/io/snappydata/hydra/ct/CTTestUtil.scala b/dtests/src/test/scala/io/snappydata/hydra/ct/CTTestUtil.scala index 6753768d3f..c68c91bfaa 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/ct/CTTestUtil.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/ct/CTTestUtil.scala @@ -19,22 +19,16 @@ package io.snappydata.hydra.ct import java.io.PrintWriter -import io.snappydata.hydra.TestUtil +import io.snappydata.hydra.SnappyTestUtils import org.apache.spark.sql.snappy._ import org.apache.spark.sql.{SQLContext, SnappyContext} object CTTestUtil { - def getCurrentDirectory: String = new java.io.File(".").getCanonicalPath + var insertUniqueRecords: Boolean = false; - def assertQuery(snc: SnappyContext, sqlString: String, queryNum: String, pw: PrintWriter): - Any = { - // scalastyle:off println - pw.println(s"Query execution for $queryNum") - val df = snc.sql(sqlString) - pw.println("Number of Rows for : " + sqlString + " is :" + df.count()) - } + def getCurrentDirectory: String = new java.io.File(".").getCanonicalPath def createReplicatedRowTables(snc: SnappyContext): Unit = { snc.sql(CTQueries.orders_details_create_ddl) @@ -230,11 +224,15 @@ object CTTestUtil { /* Create and load tables in Spark */ - def createAndLoadSparkTables(sqlContext: SQLContext, insertUniqueRecords: Boolean): Unit = { + + def createAndLoadSparkTables(sqlContext: SQLContext): Unit = { + // scalastyle:off println if (insertUniqueRecords) { - CTQueries.orders_details_df(sqlContext).dropDuplicates("SINGLE_ORDER_DID").createOrReplaceTempView("orders_details") + CTQueries.orders_details_df(sqlContext). + dropDuplicates("SINGLE_ORDER_DID").createOrReplaceTempView("orders_details") println(s"orders_details Table created successfully in spark") - CTQueries.exec_details_df(sqlContext).dropDuplicates("EXEC_DID").createOrReplaceTempView("exec_details") + CTQueries.exec_details_df(sqlContext). + dropDuplicates("EXEC_DID").createOrReplaceTempView("exec_details") println(s"exec_details Table created successfully in spark") } else { @@ -246,73 +244,87 @@ object CTTestUtil { } /* - Performs validation for tables with the queries. Returns failed queries in a string. + Method for validating queires results for ct schema for 1.5GB data. + For any other data size, this method can be used for comparing full resultsets by setting + numRowsValidation to false. + Returns failed queries in a string. */ - def executeQueries(snc: SnappyContext, tblType: String, pw: PrintWriter, - fullResultSetValidation: Boolean, sqlContext: SQLContext, - insertUniqueRecords: Boolean, skipNumRowsValidation: Boolean): String = { - TestUtil.validateFullResultSet = fullResultSetValidation - TestUtil.tableType = tblType + def executeQueries(snc: SnappyContext, tblType: String, pw: PrintWriter, sqlContext: + SQLContext): String = { + SnappyTestUtils.tableType = tblType var failedQueries = "" - if (TestUtil.validateFullResultSet) { - CTTestUtil.createAndLoadSparkTables(sqlContext, insertUniqueRecords) + if (SnappyTestUtils.validateFullResultSet) { + // scalastyle:off println + pw.println(s"createAndLoadSparkTables started ...") + + val startTime = System.currentTimeMillis + CTTestUtil.createAndLoadSparkTables(sqlContext) + val finishTime = System.currentTimeMillis() + pw.println(s"createAndLoadSparkTables completed successfully in : " + ((finishTime - + startTime)/1000) + " seconds") } - for (q <- CTQueries.queries) { + var queryExecuted = true; var hasValidationFailed = false; q._1 match { - case "Q1" => hasValidationFailed = TestUtil.assertQuery(snc, CTQueries.query1, 1, "Q1", - pw, sqlContext, skipNumRowsValidation) - case "Q2" => hasValidationFailed = TestUtil.assertQuery(snc, CTQueries.query2, 1, "Q2", - pw, sqlContext, skipNumRowsValidation) - case "Q3" => hasValidationFailed = TestUtil.assertQuery(snc, CTQueries.query3, 1, "Q3", - pw, sqlContext, skipNumRowsValidation) - case "Q4" => hasValidationFailed = TestUtil.assertQuery(snc, CTQueries.query4, 1, "Q4", - pw, sqlContext, skipNumRowsValidation) - case "Q5" => hasValidationFailed = TestUtil.assertQuery(snc, CTQueries.query5, 1, "Q5", - pw, sqlContext, skipNumRowsValidation) - case "Q6" => hasValidationFailed = TestUtil.assertQuery(snc, CTQueries.query6, 5, "Q6", - pw, sqlContext, skipNumRowsValidation) - case "Q7" => hasValidationFailed = TestUtil.assertQuery(snc, CTQueries.query7, 5, "Q7", - pw, sqlContext, skipNumRowsValidation) - case "Q8" => hasValidationFailed = TestUtil.assertQuery(snc, CTQueries.query8, 5, "Q8", - pw, sqlContext, skipNumRowsValidation) - case "Q9" => hasValidationFailed = TestUtil.assertQuery(snc, CTQueries.query9, 1, "Q9", - pw, sqlContext, skipNumRowsValidation) - case "Q10" => hasValidationFailed = TestUtil.assertQuery(snc, CTQueries.query10, 1, "Q10", - pw, sqlContext, skipNumRowsValidation) - case "Q11" => hasValidationFailed = TestUtil.assertJoin(snc, CTQueries.query11, 2706, "Q11", - pw, sqlContext, skipNumRowsValidation) - case "Q12" => hasValidationFailed = TestUtil.assertJoin(snc, CTQueries.query12, 150, "Q12", - pw, sqlContext, skipNumRowsValidation) - case "Q13" => hasValidationFailed = TestUtil.assertQuery(snc, CTQueries.query13, 149, "Q13", - pw, sqlContext, skipNumRowsValidation) - case "Q14" => hasValidationFailed = TestUtil.assertQuery(snc, CTQueries.query14, 149, "Q14", - pw, sqlContext, skipNumRowsValidation) - case "Q15" => hasValidationFailed = TestUtil.assertJoin(snc, CTQueries.query15, 2620, "Q15", - pw, sqlContext, skipNumRowsValidation) - case "Q16" => hasValidationFailed = TestUtil.assertJoin(snc, CTQueries.query16, 150, "Q16", - pw, sqlContext, skipNumRowsValidation) - case "Q17" => hasValidationFailed = TestUtil.assertQuery(snc, CTQueries.query17, 2, "Q17", - pw, sqlContext, skipNumRowsValidation) - case "Q18" => hasValidationFailed = TestUtil.assertQuery(snc, CTQueries.query18, 0, "Q18", - pw, sqlContext, skipNumRowsValidation) - case "Q19" => hasValidationFailed = TestUtil.assertQuery(snc, CTQueries.query19, 47, "Q19", - pw, sqlContext, skipNumRowsValidation) - case "Q20" => hasValidationFailed = TestUtil.assertQuery(snc, CTQueries.query20, 100, "Q20", - pw, sqlContext, skipNumRowsValidation) - case "Q21" => hasValidationFailed = TestUtil.assertQuery(snc, CTQueries.query21, 2, "Q21", - pw, sqlContext, skipNumRowsValidation) - case "Q22" => hasValidationFailed = TestUtil.assertJoin(snc, CTQueries.query22, 1, "Q22", - pw, sqlContext, skipNumRowsValidation) - // case "Q23" => hasValidationFailed = TestUtil.assertJoin(snc, CTQueries.query23,0,"Q23", - // pw,sqlContext) - case "Q24" => hasValidationFailed = TestUtil.assertQuery(snc, CTQueries.query24, 999, "Q24", - pw, sqlContext, skipNumRowsValidation) - case _ => pw.println(s"Query not be executed ${q._1}") + case "Q1" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, CTQueries.query1, 1, + "Q1", pw, sqlContext) + case "Q2" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, CTQueries.query2, 1, + "Q2", pw, sqlContext) + case "Q3" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, CTQueries.query3, 1, + "Q3", pw, sqlContext) + case "Q4" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, CTQueries.query4, 1, + "Q4", pw, sqlContext) + case "Q5" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, CTQueries.query5, 1, + "Q5", pw, sqlContext) + case "Q6" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, CTQueries.query6, 5, + "Q6", pw, sqlContext) + case "Q7" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, CTQueries.query7, 5, + "Q7", pw, sqlContext) + case "Q8" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, CTQueries.query8, 5, + "Q8", pw, sqlContext) + case "Q9" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, CTQueries.query9, 1, + "Q9", pw, sqlContext) + case "Q10" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, CTQueries.query10, 1, + "Q10", pw, sqlContext) + case "Q11" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, CTQueries.query11, 2706, + "Q11", pw, sqlContext) + case "Q12" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, CTQueries.query12, 150, + "Q12", pw, sqlContext) + case "Q13" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, CTQueries.query13, 149, + "Q13", pw, sqlContext) + case "Q14" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, CTQueries.query14, 149, + "Q14", pw, sqlContext) + case "Q15" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, CTQueries.query15, 2620, + "Q15", pw, sqlContext) + case "Q16" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, CTQueries.query16, 150, + "Q16", pw, sqlContext) + case "Q17" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, CTQueries.query17, 2, + "Q17", pw, sqlContext) + case "Q18" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, CTQueries.query18, 0, + "Q18", pw, sqlContext) + case "Q19" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, CTQueries.query19, 47, + "Q19", pw, sqlContext) + case "Q20" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, CTQueries.query20, 100, + "Q20", pw, sqlContext) + case "Q21" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, CTQueries.query21, 2, + "Q21", pw, sqlContext) + case "Q22" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, CTQueries.query22, 1, + "Q22", pw, sqlContext) + // case "Q23" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, CTQueries.query23,0, + // "Q23", pw,sqlContext) + case "Q24" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, CTQueries.query24, 999, + "Q24", pw, sqlContext) + case _ => + // scalastyle:off println + pw.println(s"Query ${q._1} will not be executed.") + queryExecuted = false + } + if (queryExecuted) { + pw.println(s"Execution completed for query ${q._1}") } if (hasValidationFailed) { - failedQueries = TestUtil.addToFailedQueryList(failedQueries, q._1) + failedQueries = SnappyTestUtils.addToFailedQueryList(failedQueries, q._1) } } return failedQueries; @@ -322,6 +334,6 @@ object CTTestUtil { snc.sql("drop table if exists exec_details") snc.sql("drop table if exists orders_details") } - } + diff --git a/dtests/src/test/scala/io/snappydata/hydra/ct/CreateAndLoadCTTablesApp.scala b/dtests/src/test/scala/io/snappydata/hydra/ct/CreateAndLoadCTTablesApp.scala index 1be6e28eab..8b35fb6954 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/ct/CreateAndLoadCTTablesApp.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/ct/CreateAndLoadCTTablesApp.scala @@ -19,16 +19,16 @@ package io.snappydata.hydra.ct import java.io.{File, FileOutputStream, PrintWriter} +import io.snappydata.hydra.SnappyTestUtils + import org.apache.spark.sql.SnappyContext -import org.apache.spark.{SparkContext, SparkConf} +import org.apache.spark.{SparkConf, SparkContext} object CreateAndLoadCTTablesApp { def main(args: Array[String]) { - val connectionURL = args(args.length - 1) val conf = new SparkConf(). - setAppName("CreateAndLoadCTTables Application"). - set("snappydata.connection", connectionURL) + setAppName("CreateAndLoadCTTables Application") val sc = SparkContext.getOrCreate(conf) val snc = SnappyContext(sc) val dataFilesLocation = args(0) @@ -39,9 +39,10 @@ object CreateAndLoadCTTablesApp { val persistenceMode = args(3) val pw = new PrintWriter(new FileOutputStream(new File("CreateAndLoadCTTablesApp.out"), true)) // scalastyle:off println - pw.println(s"dataFilesLocation : ${dataFilesLocation}") + pw.println(s"${SnappyTestUtils.logTime} dataFilesLocation : ${dataFilesLocation}") CTTestUtil.dropTables(snc) - pw.println(s"Create and load for ${tableType} tables has started") + pw.println(s"${SnappyTestUtils.logTime} Create and load for ${tableType} tables " + + s"has started") pw.flush() tableType match { // replicated row tables @@ -73,12 +74,13 @@ object CreateAndLoadCTTablesApp { throw new Exception(s"Did not find any match for ${tableType} to create tables." + s" See ${CTTestUtil.getCurrentDirectory}/CreateAndLoadCTTablesApp.out") } - pw.println("Tables are created. Now loading data.") + pw.println(s"${SnappyTestUtils.logTime} Tables are created. Now loading data.") pw.flush() CTTestUtil.loadTables(snc) println(s"Create and load for ${tableType} tables has completed successfully. " + s"See ${CTTestUtil.getCurrentDirectory}/CreateAndLoadCTTablesApp.out") - pw.println(s"Create and load for ${tableType} tables has completed successfully") + pw.println(s"${SnappyTestUtils.logTime} Create and load for ${tableType} tables " + + s"has completed successfully") pw.close() } } diff --git a/dtests/src/test/scala/io/snappydata/hydra/ct/CreateAndLoadCTTablesJob.scala b/dtests/src/test/scala/io/snappydata/hydra/ct/CreateAndLoadCTTablesJob.scala index 96c5682b9e..70115e892b 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/ct/CreateAndLoadCTTablesJob.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/ct/CreateAndLoadCTTablesJob.scala @@ -18,59 +18,71 @@ package io.snappydata.hydra.ct import java.io.{File, FileOutputStream, PrintWriter} + import scala.util.{Failure, Success, Try} + import com.typesafe.config.Config +import io.snappydata.hydra.SnappyTestUtils import util.TestException -import org.apache.spark.sql.{SnappySession, SnappyJobValid, SnappyJobValidation, SnappySQLJob} +import org.apache.spark.sql.{SnappyJobValid, SnappyJobValidation, SnappySQLJob, SnappySession} class CreateAndLoadCTTablesJob extends SnappySQLJob { override def runSnappyJob(snSession: SnappySession, jobConfig: Config): Any = { val pw = new PrintWriter(new FileOutputStream(new File("CreateAndLoadCTTablesJob.out"), true)); val tableType = jobConfig.getString("tableType") - pw.println("In create and load tables Job") + // scalastyle:off println Try { val snc = snSession.sqlContext snc.sql("set spark.sql.shuffle.partitions=6") val dataFilesLocation = jobConfig.getString("dataFilesLocation") val redundancy = jobConfig.getString("redundancy") - pw.println(s"Data files are at : ${dataFilesLocation}") - snc.setConf("dataFilesLocation",dataFilesLocation) + pw.println(s"${SnappyTestUtils.logTime} dataFilesLocation : ${dataFilesLocation}") + snc.setConf("dataFilesLocation", dataFilesLocation) CTQueries.snc = snc CTTestUtil.dropTables(snc) - pw.println(s"Create and load for ${tableType} tables has started...") + pw.println(s"${SnappyTestUtils.logTime} Create and load for ${tableType} tables" + + s" has started...") pw.flush() tableType match { - //replicated row tables + // replicated row tables case "Replicated" => CTTestUtil.createReplicatedRowTables(snc) - case "PersistentReplicated" => CTTestUtil.createPersistReplicatedRowTables(snc,jobConfig.getString("persistenceMode")) - //partitioned row tables - case "PartitionedRow" => CTTestUtil.createPartitionedRowTables(snc,redundancy) - case "PersistentPartitionRow" => CTTestUtil.createPersistPartitionedRowTables(snc,redundancy,jobConfig.getString("persistenceMode")) - case "ColocatedRow" => CTTestUtil.createColocatedRowTables(snc,redundancy) - case "EvictionRow"=> CTTestUtil.createRowTablesWithEviction(snc,redundancy) - case "PersistentColocatedRow" => CTTestUtil.createPersistColocatedTables(snc,redundancy,jobConfig.getString("persistenceMode")) - case "ColocatedWithEvictionRow" => CTTestUtil.createColocatedRowTablesWithEviction(snc,redundancy,jobConfig.getString("persistenceMode")) - //column tables - case "Column" => CTTestUtil.createColumnTables(snc,redundancy) - case "PersistentColumn" => CTTestUtil.createPersistColumnTables(snc,jobConfig.getString("persistenceMode")) - case "ColocatedColumn" => CTTestUtil.createColocatedColumnTables(snc,redundancy) - case "EvictionColumn" => CTTestUtil.createColumnTablesWithEviction(snc,redundancy) - case "PersistentColocatedColumn" => CTTestUtil.createPersistColocatedColumnTables(snc,redundancy,jobConfig.getString("persistenceMode")) - case "ColocatedWithEvictionColumn" => CTTestUtil.createColocatedColumnTablesWithEviction(snc,redundancy) + case "PersistentReplicated" => + CTTestUtil.createPersistReplicatedRowTables(snc, jobConfig.getString("persistenceMode")) + // partitioned row tables + case "PartitionedRow" => CTTestUtil.createPartitionedRowTables(snc, redundancy) + case "PersistentPartitionRow" => CTTestUtil.createPersistPartitionedRowTables(snc, + redundancy, jobConfig.getString("persistenceMode")) + case "ColocatedRow" => CTTestUtil.createColocatedRowTables(snc, redundancy) + case "EvictionRow" => CTTestUtil.createRowTablesWithEviction(snc, redundancy) + case "PersistentColocatedRow" => CTTestUtil.createPersistColocatedTables(snc, redundancy, + jobConfig.getString("persistenceMode")) + case "ColocatedWithEvictionRow" => CTTestUtil.createColocatedRowTablesWithEviction(snc, + redundancy, jobConfig.getString("persistenceMode")) + // column tables + case "Column" => CTTestUtil.createColumnTables(snc, redundancy) + case "PersistentColumn" => + CTTestUtil.createPersistColumnTables(snc, jobConfig.getString("persistenceMode")) + case "ColocatedColumn" => CTTestUtil.createColocatedColumnTables(snc, redundancy) + case "EvictionColumn" => CTTestUtil.createColumnTablesWithEviction(snc, redundancy) + case "PersistentColocatedColumn" => CTTestUtil.createPersistColocatedColumnTables(snc, + redundancy, jobConfig.getString("persistenceMode")) + case "ColocatedWithEvictionColumn" => + CTTestUtil.createColocatedColumnTablesWithEviction(snc, redundancy) case _ => pw.println(s"Did not find any match for ${tableType} to create tables") pw.close() throw new TestException(s"Did not find any match for ${tableType} to create tables." + s" See ${CTTestUtil.getCurrentDirectory}/CreateAndLoadCTTablesJob.out") } - pw.println("Tables are created. Now loading data.") + pw.println(s"${SnappyTestUtils.logTime} Tables are created. Now loading data.") pw.flush() CTTestUtil.loadTables(snc); println(s"Create and load for ${tableType} tables has completed successfully. " + s"See ${CTTestUtil.getCurrentDirectory}/CreateAndLoadCTTablesJob.out") - pw.println(s"Create and load for ${tableType} tables has completed successfully") + pw.println(s"${SnappyTestUtils.logTime} Create and load for ${tableType} tables" + + s" has completed successfully") pw.close() } match { case Success(v) => pw.close() diff --git a/dtests/src/test/scala/io/snappydata/hydra/ct/ValidateCTQueriesApp.scala b/dtests/src/test/scala/io/snappydata/hydra/ct/ValidateCTQueriesApp.scala index 5f9f1006c3..f6e1fea268 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/ct/ValidateCTQueriesApp.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/ct/ValidateCTQueriesApp.scala @@ -19,57 +19,58 @@ package io.snappydata.hydra.ct import java.io.{File, FileOutputStream, PrintWriter} +import io.snappydata.hydra.SnappyTestUtils + import org.apache.spark.sql.{SQLContext, SnappyContext} -import org.apache.spark.{SparkContext, SparkConf} +import org.apache.spark.{SparkConf, SparkContext} object ValidateCTQueriesApp { def main(args: Array[String]) { - val connectionURL = args(args.length - 1) + + val threadID = Thread.currentThread().getId val conf = new SparkConf(). - setAppName("ValidateCTQueriesApp Application_" + System.currentTimeMillis()). - set("snappydata.connection", connectionURL) + setAppName("ValidateCTQueriesApp_" + threadID + "_" + System.currentTimeMillis()) val sc = SparkContext.getOrCreate(conf) val sqlContext = SQLContext.getOrCreate(sc) val snc = SnappyContext(sc) def getCurrentDirectory = new java.io.File(".").getCanonicalPath - val threadID = Thread.currentThread().getId val outputFile = "ValidateCTQueriesApp_thread_" + threadID + "_" + System.currentTimeMillis + ".out" val pw = new PrintWriter(new FileOutputStream(new File(outputFile), true)) val dataFilesLocation = args(0) snc.setConf("dataFilesLocation", dataFilesLocation) // scalastyle:off println - pw.println(s"dataFilesLocation : ${dataFilesLocation}") CTQueries.snc = snc val tableType = args(1) val fullResultSetValidation: Boolean = args(2).toBoolean - pw.println(s"Validation for queries with ${tableType} tables started") - if (fullResultSetValidation) { - pw.println(s"Test will perform fullResultSetValidation") - } - else { - pw.println(s"Test will not perform fullResultSetValidation") - } + val numRowsValidation: Boolean = args(3).toBoolean + SnappyTestUtils.numRowsValidation = numRowsValidation + SnappyTestUtils.validateFullResultSet = fullResultSetValidation + pw.println(s"${SnappyTestUtils.logTime} Validation for queries with ${tableType} " + + s"tables started") + val startTime = System.currentTimeMillis - val failedQueries = CTTestUtil.executeQueries(snc, tableType, pw, fullResultSetValidation, - sqlContext, false, false) + val failedQueries = CTTestUtil.executeQueries(snc, tableType, pw, sqlContext) val endTime = System.currentTimeMillis val totalTime = (endTime - startTime) / 1000 - pw.println(s"Total time for execution is :: ${totalTime} seconds.") + pw.println(s"${SnappyTestUtils.logTime} Total time for execution is :" + + s" ${totalTime} seconds.") if (!failedQueries.isEmpty) { - println(s"Validation failed for ${tableType} for queries ${failedQueries}.. See " + + println(s"Validation failed for ${tableType} tables for queries ${failedQueries}. See " + s"${getCurrentDirectory}/${outputFile}") - pw.println(s"Validation failed for ${tableType} for queries ${failedQueries}. ") + pw.println(s"${SnappyTestUtils.logTime} Validation failed for ${tableType} " + + s"tables for queries ${failedQueries}. ") pw.close() throw new Exception(s"Validation task failed for ${tableType}. See " + s"${getCurrentDirectory}/${outputFile}") } println(s"Validation for queries with ${tableType} tables completed successfully. See " + s"${getCurrentDirectory}/${outputFile}") - pw.println(s"Validation for queries with ${tableType} tables completed successfully.") + pw.println(s"${SnappyTestUtils.logTime} Validation for queries with ${tableType} " + + s"tables completed successfully.") pw.close() } } diff --git a/dtests/src/test/scala/io/snappydata/hydra/ct/ValidateCTQueriesJob.scala b/dtests/src/test/scala/io/snappydata/hydra/ct/ValidateCTQueriesJob.scala index 4681af33bc..49178e4a75 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/ct/ValidateCTQueriesJob.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/ct/ValidateCTQueriesJob.scala @@ -18,21 +18,24 @@ package io.snappydata.hydra.ct import java.io.{File, FileOutputStream, PrintWriter} + import scala.util.{Failure, Success, Try} import com.typesafe.config.Config +import io.snappydata.hydra.SnappyTestUtils import util.TestException import org.apache.spark.SparkContext -import org.apache.spark.sql.{SnappySession, SQLContext, SnappyJobValid, SnappyJobValidation, SnappySQLJob} +import org.apache.spark.sql.{SQLContext, SnappyJobValid, SnappyJobValidation, SnappySQLJob, SnappySession} class ValidateCTQueriesJob extends SnappySQLJob { override def runSnappyJob(snSession: SnappySession, jobConfig: Config): Any = { def getCurrentDirectory = new java.io.File(".").getCanonicalPath val threadID = Thread.currentThread().getId - val outputFile = "ValidateCTQueriesJob_thread_" + threadID + "_" + System.currentTimeMillis + ".out" - val pw = new PrintWriter(new FileOutputStream(new File(outputFile), true)) + val outputFile = + "ValidateCTQueriesJob_thread_" + threadID + "_" + System.currentTimeMillis + ".out" + val pw = new PrintWriter(new FileOutputStream(new File(outputFile), true)); val tableType = jobConfig.getString("tableType") Try { @@ -41,34 +44,40 @@ class ValidateCTQueriesJob extends SnappySQLJob { val dataFilesLocation = jobConfig.getString("dataFilesLocation") snc.setConf("dataFilesLocation", dataFilesLocation) CTQueries.snc = snc - pw.println(s"Validation for $tableType tables started in snappy Job") + // scalastyle:off println + pw.println(s"${SnappyTestUtils.logTime} Validation for $tableType tables " + + s"started in snappy Job") + val numRowsValidation: Boolean = jobConfig.getBoolean("numRowsValidation") val fullResultSetValidation: Boolean = jobConfig.getBoolean("fullResultSetValidation") + SnappyTestUtils.validateFullResultSet = fullResultSetValidation + SnappyTestUtils.numRowsValidation = numRowsValidation val sc = SparkContext.getOrCreate() val sqlContext = SQLContext.getOrCreate(sc) - if (fullResultSetValidation) - pw.println(s"Test will perform fullResultSetValidation") - else - pw.println(s"Test will not perform fullResultSetValidation") val startTime = System.currentTimeMillis - val failedQueries = CTTestUtil.executeQueries(snc, tableType, pw, fullResultSetValidation, - sqlContext, false, false) + val failedQueries = CTTestUtil.executeQueries(snc, tableType, pw, sqlContext) val endTime = System.currentTimeMillis val totalTime = (endTime - startTime) / 1000 - pw.println(s"Total time for execution is :: ${totalTime} seconds.") + pw.println(s"${SnappyTestUtils.logTime} Total execution took ${totalTime} " + + s"seconds.") if(!failedQueries.isEmpty) { - println(s"Validation failed for ${tableType} for queries ${failedQueries}. See ${getCurrentDirectory}/${outputFile}") - pw.println(s"Validation failed for ${tableType} for queries ${failedQueries}. ") + println(s"Validation failed for ${tableType} tables for queries ${failedQueries}. " + + s"See ${getCurrentDirectory}/${outputFile}") + pw.println(s"${SnappyTestUtils.logTime} Validation failed for ${tableType} " + + s"tables for queries ${failedQueries}. ") pw.close() - throw new TestException(s"Validation task failed for ${tableType}. See ${getCurrentDirectory}/${outputFile}") + throw new TestException(s"Validation task failed for ${tableType}. " + + s"See ${getCurrentDirectory}/${outputFile}") } - println(s"Validation for $tableType tables completed. See ${getCurrentDirectory}/${outputFile}") - pw.println(s"Validation for $tableType tables completed.") + println(s"Validation for $tableType tables completed sucessfully. " + + s"See ${getCurrentDirectory}/${outputFile}") + pw.println(s"${SnappyTestUtils.logTime} ValidateQueries for ${tableType} tables" + + s" completed successfully.") pw.close() } match { case Success(v) => pw.close() - s"See ${getCurrentDirectory}/${outputFile}" + s"Validation passed. See ${getCurrentDirectory}/${outputFile}" case Failure(e) => pw.close(); - throw e; + throw new TestException(s"Validation failed. See ${getCurrentDirectory}/${outputFile}"); } } diff --git a/dtests/src/test/scala/io/snappydata/hydra/ct/ValidateCTQueriesWithPutIntoJob.scala b/dtests/src/test/scala/io/snappydata/hydra/ct/ValidateCTQueriesWithPutIntoJob.scala index c13762b3d8..96d7ab26f8 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/ct/ValidateCTQueriesWithPutIntoJob.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/ct/ValidateCTQueriesWithPutIntoJob.scala @@ -18,13 +18,15 @@ package io.snappydata.hydra.ct import java.io.{File, FileOutputStream, PrintWriter} + import scala.util.{Failure, Success, Try} import com.typesafe.config.Config +import io.snappydata.hydra.SnappyTestUtils import util.TestException import org.apache.spark.SparkContext -import org.apache.spark.sql.{SnappySession, SQLContext, SnappyJobValid, SnappyJobValidation, SnappySQLJob} +import org.apache.spark.sql.{SQLContext, SnappyJobValid, SnappyJobValidation, SnappySQLJob, SnappySession} class ValidateCTQueriesWithPutIntoJob extends SnappySQLJob { @@ -32,7 +34,8 @@ class ValidateCTQueriesWithPutIntoJob extends SnappySQLJob { def getCurrentDirectory = new java.io.File(".").getCanonicalPath val threadID = Thread.currentThread().getId - val outputFile = "ValidateCTQueriesJob_thread_" + threadID + "_" + System.currentTimeMillis + ".out" + val outputFile = + "ValidateCTQueriesJob_thread_" + threadID + "_" + System.currentTimeMillis + ".out" val pw = new PrintWriter(new FileOutputStream(new File(outputFile), true)) val tableType = jobConfig.getString("tableType") val insertUniqueRecords = jobConfig.getString("insertUniqueRecords").toBoolean @@ -44,27 +47,35 @@ class ValidateCTQueriesWithPutIntoJob extends SnappySQLJob { val dataFilesLocation = jobConfig.getString("dataFilesLocation") snc.setConf("dataFilesLocation", dataFilesLocation) CTQueries.snc = snc + // scalastyle:off println pw.println(s"Validation for $tableType tables started in snappy Job") val fullResultSetValidation: Boolean = jobConfig.getBoolean("fullResultSetValidation") val sc = SparkContext.getOrCreate() val sqlContext = SQLContext.getOrCreate(sc) - if (fullResultSetValidation) + if (fullResultSetValidation) { pw.println(s"Test will perform fullResultSetValidation") - else + } + else { pw.println(s"Test will not perform fullResultSetValidation") + } val startTime = System.currentTimeMillis - val failedQueries = CTTestUtil.executeQueries(snc, tableType, pw, fullResultSetValidation, - sqlContext, insertUniqueRecords, skipNumRowsValidation) + CTTestUtil.insertUniqueRecords = insertUniqueRecords + SnappyTestUtils.numRowsValidation = skipNumRowsValidation + SnappyTestUtils.validateFullResultSet = fullResultSetValidation + val failedQueries = CTTestUtil.executeQueries(snc, tableType, pw, sqlContext) val endTime = System.currentTimeMillis val totalTime = (endTime - startTime) / 1000 pw.println(s"Total time for execution is :: ${totalTime} seconds.") if (!failedQueries.isEmpty) { - println(s"Validation failed for ${tableType} for queries ${failedQueries}. See ${getCurrentDirectory}/${outputFile}") + println(s"Validation failed for ${tableType} for queries ${failedQueries}. " + + s"See ${getCurrentDirectory}/${outputFile}") pw.println(s"Validation failed for ${tableType} for queries ${failedQueries}. ") pw.close() - throw new TestException(s"Validation task failed for ${tableType}. See ${getCurrentDirectory}/${outputFile}") + throw new TestException(s"Validation task failed for ${tableType}. " + + s"See ${getCurrentDirectory}/${outputFile}") } - println(s"Validation for $tableType tables completed. See ${getCurrentDirectory}/${outputFile}") + println(s"Validation for $tableType tables completed. " + + s"See ${getCurrentDirectory}/${outputFile}") pw.println(s"Validation for $tableType tables completed.") pw.close() } match { diff --git a/dtests/src/test/scala/io/snappydata/hydra/externaltables/ExternalTablesAPINorthWind.scala b/dtests/src/test/scala/io/snappydata/hydra/externaltables/ExternalTablesAPINorthWind.scala index 419fb17672..3bb30432bf 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/externaltables/ExternalTablesAPINorthWind.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/externaltables/ExternalTablesAPINorthWind.scala @@ -150,8 +150,9 @@ class ExternalTablesAPINorthWind extends SnappySQLJob{ println("***** <1>snc_namesWithTOC : " + snc_namesWithTOC.show()) println("##### <1>spark_namesWithTOC : " + spark_namesWithTOC.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_namesWithTOC, - spark_namesWithTOC, "EmbeddedNWAPI1", "column", pw, sqlContext, false) + + SnappyTestUtils.tableType = "externalTables" + SnappyTestUtils.assertQuery(snc, snc_namesWithTOC, spark_namesWithTOC, "EmbeddedNWAPI1", pw) /* <2> SELECT FirstName, LastName FROM Employees; */ val snc_names = sncEmpDF.select("FirstName" , "LastName") @@ -160,8 +161,7 @@ class ExternalTablesAPINorthWind extends SnappySQLJob{ println("***** <2>snc_names : " + snc_names.show()) println("##### <2>spark_names : " + spark_names.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_names, spark_names, - "EmbeddedNWAPI2", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_names, spark_names, "EmbeddedNWAPI2", pw) /* <3> SELECT FirstName, LastName FROM Employees ORDER BY LastName; */ val snc_namesSortByLastName = sncEmpDF.select("FirstName", "LastName").orderBy(sncEmpDF("LastName").desc) @@ -171,8 +171,8 @@ class ExternalTablesAPINorthWind extends SnappySQLJob{ println("***** <3>snc_namesSortByLastName : " + snc_namesSortByLastName.show()) println("##### <3>spark_namesSortByLastName : " + spark_namesSortByLastName.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_namesSortByLastName, - spark_namesSortByLastName, "EmbeddedNWAPI3", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_namesSortByLastName, + spark_namesSortByLastName, "EmbeddedNWAPI3", pw) /* <4> SELECT Title, FirstName, LastName FROM Employees WHERE Title = 'Sales Representative'; */ val snc_salesRep = sncEmpDF.select(("Title"), "FirstName", "LastName").filter(sncEmpDF("Title") === "Sales Representative") @@ -182,8 +182,7 @@ class ExternalTablesAPINorthWind extends SnappySQLJob{ println("***** <4>snc_salesRep : " + snc_salesRep.show()) println("##### <4>spark_salesRep : " + spark_SalesRep.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_salesRep, spark_SalesRep, - "EmbeddedNWAPI4", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_salesRep, spark_SalesRep, "EmbeddedNWAPI4", pw) /* <5> SELECT FirstName, LastName FROM Employees WHERE Title <> 'Sales Representative'; // TODO : Test the where(String) or filter("String) condition @@ -196,8 +195,8 @@ class ExternalTablesAPINorthWind extends SnappySQLJob{ println("***** <5>snc_titleOtherThanSalsRep : " + snc_titleOtherThanSalsRep.show()) println("##### <5>spark_titleOtherThanSalsRep : " + spark_titleOtherThanSalsRep.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_titleOtherThanSalsRep, - spark_titleOtherThanSalsRep, "EmbeddedNWAPI5", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_titleOtherThanSalsRep, spark_titleOtherThanSalsRep, + "EmbeddedNWAPI5", pw) /* <6> SELECT FirstName, LastName FROM Employees WHERE LastName >= 'N' ORDER BY LastName DESC; */ @@ -209,8 +208,8 @@ class ExternalTablesAPINorthWind extends SnappySQLJob{ println("***** <6>snc_empName : " + snc_EmpNameDesc.show()) println("##### <6>spark_empName : " + spark_EmpNameDesc.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_EmpNameDesc, spark_EmpNameDesc, - "EmbeddedNWAPI6", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_EmpNameDesc, spark_EmpNameDesc, + "EmbeddedNWAPI6", pw) /* <7> SELECT OrderID, Freight, Freight * 1.1 AS FreightTotal FROM Orders WHERE Freight >= 500; */ @@ -226,8 +225,8 @@ class ExternalTablesAPINorthWind extends SnappySQLJob{ println("***** <7>snc_freightgeq500 : " + snc_Freightgeq500.show()) println("##### <7>spark_freightgeq500 : " + spark_Freightgeq500.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_Freightgeq500, spark_Freightgeq500, - "EmbeddedNWAPI7", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_Freightgeq500, spark_Freightgeq500, + "EmbeddedNWAPI7", pw) /* <8> SELECT SUM(Quantity) AS TotalUnits FROM Order_Details WHERE ProductID=3; */ import org.apache.spark.sql.functions._ @@ -239,8 +238,8 @@ class ExternalTablesAPINorthWind extends SnappySQLJob{ println("***** <8>snc_totalUnits : " + snc_TotalUnits.show()) println("##### <8>spark_totalUnits : " + spark_TotalUnits.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_TotalUnits, spark_TotalUnits, - "EmbeddedNWAPI8", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_TotalUnits, spark_TotalUnits, + "EmbeddedNWAPI8", pw) /* <9> SELECT COUNT(DISTINCT City) AS NumCities FROM Employees; */ val snc_DistinctCity = sncEmpDF.select("City").distinct().withColumnRenamed("City", "NumCities") @@ -254,8 +253,8 @@ class ExternalTablesAPINorthWind extends SnappySQLJob{ println("***** <9.2>snc_DistinctCityCount : " + snc_DistinctCityCount.show()) println("##### <9.2>spark_DistinctCityCount : " + spark_DistinctCityCount.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_DistinctCity, spark_DistinctCity, - "EmbeddedNWAPI9", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_DistinctCity, spark_DistinctCity, + "EmbeddedNWAPI9", pw) /* <10> SELECT CONCAT(FirstName, ' ', LastName) FROM Employees; */ val snc_Name = sncEmpDF.select(concat_ws(" ", col("FirstName"), col("LastName"))) @@ -268,10 +267,8 @@ class ExternalTablesAPINorthWind extends SnappySQLJob{ println("***** <10.2>snc_Name1 : " + snc_Name1.show()) println("##### <10.2>spark_Name1 : " + spark_Name1.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_Name, spark_Name, - "EmbeddedNWAPI10_1", "column", pw, sqlContext, false) - SnappyTestUtils.assertQueryFullResultSet(snc, snc_Name1, spark_Name1, - "EmbeddedNWAPI10_1", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_Name, spark_Name, "EmbeddedNWAPI10_1", pw) + SnappyTestUtils.assertQuery(snc, snc_Name1, spark_Name1, "EmbeddedNWAPI10_1", pw) /* <11> SELECT count(*) FROM orders FULL JOIN order_details; */ val snc_FullJoinCnt = sncOrdersDF.crossJoin(sncOrderDetailsDF) @@ -280,8 +277,7 @@ class ExternalTablesAPINorthWind extends SnappySQLJob{ println("***** <11>snc_FullJoinCount : " + snc_FullJoinCnt.count()) println("##### <11>spark_FullJoinCount : " + spark_FullJoinCnt.count()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_FullJoinCnt, spark_FullJoinCnt, - "EmbeddedNWAPI11", "column", pw, sqlContext, true) + SnappyTestUtils.assertQuery(snc, snc_FullJoinCnt, spark_FullJoinCnt, "EmbeddedNWAPI11", pw) /* <12> SELECT OrderDate, count(1) from Orders group by OrderDate order by OrderDate asc; */ import org.apache.spark.sql.functions.{count, lit} @@ -296,9 +292,8 @@ class ExternalTablesAPINorthWind extends SnappySQLJob{ println("***** <12>snc_dateWiseOrderCountASC : " + snc_dateWiseOrderCountASC.show(480)) println("##### <12>spark_dateWiseOrderCountASC : " + spark_dateWiseOrderCountASC.show(480)) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_dateWiseOrderCountASC, - spark_dateWiseOrderCountASC, "EmbeddedNWAPI12", - "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_dateWiseOrderCountASC, spark_dateWiseOrderCountASC, + "EmbeddedNWAPI12", pw) /* <13> SELECT OrderDate, count(1) from Orders group by OrderDate order by OrderDate; */ val snc_dateWiseOrderCnt = sncOrdersDF.select(col("OrderDt")).groupBy(col("OrderDt")) @@ -311,8 +306,8 @@ class ExternalTablesAPINorthWind extends SnappySQLJob{ println("***** <13>snc_dateWiseOrderCnt : " + snc_dateWiseOrderCnt.show(480)) println("##### <13>spark_dateWiseOrderCnt : " + spark_dateWiseOrderCnt.show(480)) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_dateWiseOrderCnt, spark_dateWiseOrderCnt, - "EmbeddedNWAPI13", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_dateWiseOrderCnt, spark_dateWiseOrderCnt, + "EmbeddedNWAPI13", pw) /* <14> SELECT FirstName, LastName FROM Employees WHERE LastName >= 'N'; */ val snc_EmpName = sncEmpDF.select(col("FirstName"), col("LastName")) @@ -323,8 +318,8 @@ class ExternalTablesAPINorthWind extends SnappySQLJob{ println("***** <14>snc_EmpName : " + snc_EmpName.show()) println("##### <14>spark_EmpName " + spark_EmpName.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_EmpName, spark_EmpName, - "EmbeddedNWAPI14", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_EmpName, spark_EmpName, + "EmbeddedNWAPI14", pw) /* <15> SELECT FirstName, LastName FROM Employees WHERE Region IS NULL; */ val snc_EmpNameWhereRegIsNull = sncEmpDF.select(col("FirstName"), col("LastName")) @@ -335,10 +330,10 @@ class ExternalTablesAPINorthWind extends SnappySQLJob{ println("***** <15>snc_EmpNameWhereRegIsNull : " + snc_EmpNameWhereRegIsNull.show()) println("##### <15>spark_EmpNameWhereRegIsNull : " + spark_EmpNameWhereRegIsNull.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_EmpNameWhereRegIsNull, + SnappyTestUtils.assertQuery(snc, snc_EmpNameWhereRegIsNull, spark_EmpNameWhereRegIsNull, - "EmbeddedNWAPI15", "column", - pw, sqlContext, false) + "EmbeddedNWAPI15", + pw) /* <16> SELECT Title, FirstName, LastName FROM Employees ORDER BY 1,3; */ val snc_EmpNameOrderByColumnPos = sncEmpDF.select(col("Title"), col("FirstName"), col("LastName")) @@ -350,9 +345,8 @@ class ExternalTablesAPINorthWind extends SnappySQLJob{ println("***** <16>snc_EmpNameOrderByColumnPos : " + snc_EmpNameOrderByColumnPos.show()) println("##### <16>spark_EmpNameOrderByColumnPos : " + spark_EmpNameOrderByColumnPos.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_EmpNameOrderByColumnPos, - spark_EmpNameOrderByColumnPos, "EmbeddedNWAPI16", - "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_EmpNameOrderByColumnPos, spark_EmpNameOrderByColumnPos, + "EmbeddedNWAPI16", pw) /* <17> SELECT Title, FirstName, LastName FROM Employees ORDER BY Title ASC, LastName DESC; */ val snc_EmpNameOrderByTitleLastName = sncEmpDF.select(col("Title"), col("FirstName"), col("LastName")) @@ -366,9 +360,8 @@ class ExternalTablesAPINorthWind extends SnappySQLJob{ println("##### <17>spark_EmpNameOrderByTitleLastName : " + spark_EmpNameOrderByTitleLastName.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_EmpNameOrderByTitleLastName, - spark_EmpNameOrderByTitleLastName, "EmbeddedNWAPI17", - "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_EmpNameOrderByTitleLastName, + spark_EmpNameOrderByTitleLastName, "EmbeddedNWAPI17", pw) /* Will add all the NorthWind Queries */ diff --git a/dtests/src/test/scala/io/snappydata/hydra/externaltables/SmartConnectorExternalTableAPINW.scala b/dtests/src/test/scala/io/snappydata/hydra/externaltables/SmartConnectorExternalTableAPINW.scala index 0192e0e88d..615114f450 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/externaltables/SmartConnectorExternalTableAPINW.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/externaltables/SmartConnectorExternalTableAPINW.scala @@ -152,8 +152,9 @@ object SmartConnectorExternalTableAPINW { println("***** <1>snc_namesWithTOC : " + snc_namesWithTOC.show()) println("##### <1>spark_namesWithTOC : " + spark_namesWithTOC.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_namesWithTOC, - spark_namesWithTOC, "NWAPI1", "column", pw, sqlContext, false) + SnappyTestUtils.tableType = "column" + SnappyTestUtils.assertQuery(snc, snc_namesWithTOC, + spark_namesWithTOC, "NWAPI1", pw) /* <2> SELECT FirstName, LastName FROM Employees; */ val snc_names = sncEmpDF.select("FirstName" , "LastName") @@ -162,8 +163,8 @@ object SmartConnectorExternalTableAPINW { println("***** <2>snc_names : " + snc_names.show()) println("##### <2>spark_names : " + spark_names.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_names, spark_names, - "NWAPI2", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_names, spark_names, + "NWAPI2", pw) /* <3> SELECT FirstName, LastName FROM Employees ORDER BY LastName; */ val snc_namesSortByLastName = sncEmpDF.select("FirstName", "LastName").orderBy(sncEmpDF("LastName").desc) @@ -173,8 +174,8 @@ object SmartConnectorExternalTableAPINW { println("***** <3>snc_namesSortByLastName : " + snc_namesSortByLastName.show()) println("##### <3>spark_namesSortByLastName : " + spark_namesSortByLastName.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_namesSortByLastName, - spark_namesSortByLastName, "NWAPI3", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_namesSortByLastName, + spark_namesSortByLastName, "NWAPI3", pw) /* <4> SELECT Title, FirstName, LastName FROM Employees WHERE Title = 'Sales Representative'; */ val snc_salesRep = sncEmpDF.select(("Title"), "FirstName", "LastName").filter(sncEmpDF("Title") === "Sales Representative") @@ -184,8 +185,8 @@ object SmartConnectorExternalTableAPINW { println("***** <4>snc_salesRep : " + snc_salesRep.show()) println("##### <4>spark_salesRep : " + spark_SalesRep.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_salesRep, spark_SalesRep, - "NWAPI4", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_salesRep, spark_SalesRep, + "NWAPI4", pw) /* <5> SELECT FirstName, LastName FROM Employees WHERE Title <> 'Sales Representative'; // TODO : Test the where(String) or filter("String) condition @@ -198,8 +199,8 @@ object SmartConnectorExternalTableAPINW { println("***** <5>snc_titleOtherThanSalsRep : " + snc_titleOtherThanSalsRep.show()) println("##### <5>spark_titleOtherThanSalsRep : " + spark_titleOtherThanSalsRep.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_titleOtherThanSalsRep, - spark_titleOtherThanSalsRep, "NWAPI5", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_titleOtherThanSalsRep, + spark_titleOtherThanSalsRep, "NWAPI5", pw) /* <6> SELECT FirstName, LastName FROM Employees WHERE LastName >= 'N' ORDER BY LastName DESC; */ @@ -211,8 +212,8 @@ object SmartConnectorExternalTableAPINW { println("***** <6>snc_empName : " + snc_EmpNameDesc.show()) println("##### <6>spark_empName : " + spark_EmpNameDesc.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_EmpNameDesc, spark_EmpNameDesc, - "NWAPI6", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_EmpNameDesc, spark_EmpNameDesc, + "NWAPI6", pw) /* <7> SELECT OrderID, Freight, Freight * 1.1 AS FreightTotal FROM Orders WHERE Freight >= 500; */ @@ -228,8 +229,8 @@ object SmartConnectorExternalTableAPINW { println("***** <7>snc_freightgeq500 : " + snc_Freightgeq500.show()) println("##### <7>spark_freightgeq500 : " + spark_Freightgeq500.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_Freightgeq500, spark_Freightgeq500, - "NWAPI7", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_Freightgeq500, spark_Freightgeq500, + "NWAPI7", pw) /* <8> SELECT SUM(Quantity) AS TotalUnits FROM Order_Details WHERE ProductID=3; */ import org.apache.spark.sql.functions._ @@ -241,8 +242,8 @@ object SmartConnectorExternalTableAPINW { println("***** <8>snc_totalUnits : " + snc_TotalUnits.show()) println("##### <8>spark_totalUnits : " + spark_TotalUnits.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_TotalUnits, spark_TotalUnits, - "NWAPI8", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_TotalUnits, spark_TotalUnits, + "NWAPI8", pw) // Print the result in file /* <9> SELECT COUNT(DISTINCT City) AS NumCities FROM Employees; */ @@ -257,8 +258,8 @@ object SmartConnectorExternalTableAPINW { println("***** <9.2>snc_DistinctCityCount : " + snc_DistinctCityCount.show()) println("##### <9.2>spark_DistinctCityCount : " + spark_DistinctCityCount.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_DistinctCity, spark_DistinctCity, - "NWAPI9", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_DistinctCity, spark_DistinctCity, + "NWAPI9", pw) /* <10> SELECT CONCAT(FirstName, ' ', LastName) FROM Employees; */ val snc_Name = sncEmpDF.select(concat_ws(" ", col("FirstName"), col("LastName"))) @@ -271,10 +272,10 @@ object SmartConnectorExternalTableAPINW { println("***** <10.2>snc_Name1 : " + snc_Name1.show()) println("##### <10.2>spark_Name1 : " + spark_Name1.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_Name, spark_Name, - "NWAPI10_1", "column", pw, sqlContext, false) - SnappyTestUtils.assertQueryFullResultSet(snc, snc_Name1, spark_Name1, - "NWAPI10_2", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_Name, spark_Name, + "NWAPI10_1", pw) + SnappyTestUtils.assertQuery(snc, snc_Name1, spark_Name1, + "NWAPI10_2", pw) /* <11> SELECT count(*) FROM orders FULL JOIN order_details; */ val snc_FullJoinCnt = sncOrdersDF.crossJoin(sncOrderDetailsDF) @@ -283,8 +284,7 @@ object SmartConnectorExternalTableAPINW { println("***** <11>snc_FullJoinCount : " + snc_FullJoinCnt.count()) println("##### <11>spark_FullJoinCount : " + spark_FullJoinCnt.count()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_FullJoinCnt, spark_FullJoinCnt, - "NWAPI11", "column", pw, sqlContext, true) + SnappyTestUtils.assertQuery(snc, snc_FullJoinCnt, spark_FullJoinCnt, "NWAPI11", pw) /* <12> SELECT OrderDate, count(1) from Orders group by OrderDate order by OrderDate asc; */ import org.apache.spark.sql.functions.{count, lit} @@ -299,8 +299,8 @@ object SmartConnectorExternalTableAPINW { println("***** <12>snc_dateWiseOrderCountASC : " + snc_dateWiseOrderCountASC.show(480)) println("##### <12>spark_dateWiseOrderCountASC : " + spark_dateWiseOrderCountASC.show(480)) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_dateWiseOrderCountASC, - spark_dateWiseOrderCountASC, "NWAPI12", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_dateWiseOrderCountASC, + spark_dateWiseOrderCountASC, "NWAPI12", pw) /* <13> SELECT OrderDate, count(1) from Orders group by OrderDate order by OrderDate; */ val snc_dateWiseOrderCnt = sncOrdersDF.select(col("OrderDt")).groupBy(col("OrderDt")) @@ -313,8 +313,8 @@ object SmartConnectorExternalTableAPINW { println("***** <13>snc_dateWiseOrderCnt : " + snc_dateWiseOrderCnt.show(480)) println("##### <13>spark_dateWiseOrderCnt : " + spark_dateWiseOrderCnt.show(480)) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_dateWiseOrderCnt, spark_dateWiseOrderCnt, - "NWAPI13", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_dateWiseOrderCnt, spark_dateWiseOrderCnt, + "NWAPI13", pw) /* <14> SELECT FirstName, LastName FROM Employees WHERE LastName >= 'N'; */ val snc_EmpName = sncEmpDF.select(col("FirstName"), col("LastName")) @@ -325,8 +325,8 @@ object SmartConnectorExternalTableAPINW { println("***** <14>snc_EmpName : " + snc_EmpName.show()) println("##### <14>spark_EmpName " + spark_EmpName.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_EmpName, spark_EmpName, - "NWAPI14", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_EmpName, spark_EmpName, + "NWAPI14", pw) /* <15> SELECT FirstName, LastName FROM Employees WHERE Region IS NULL; */ val snc_EmpNameWhereRegIsNull = sncEmpDF.select(col("FirstName"), col("LastName")) @@ -337,8 +337,8 @@ object SmartConnectorExternalTableAPINW { println("***** <15>snc_EmpNameWhereRegIsNull : " + snc_EmpNameWhereRegIsNull.show()) println("##### <15>spark_EmpNameWhereRegIsNull : " + spark_EmpNameWhereRegIsNull.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_EmpNameWhereRegIsNull, - spark_EmpNameWhereRegIsNull, "NWAPI15", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_EmpNameWhereRegIsNull, + spark_EmpNameWhereRegIsNull, "NWAPI15", pw) /* <16> SELECT Title, FirstName, LastName FROM Employees ORDER BY 1,3; */ val snc_EmpNameOrderByColumnPos = sncEmpDF.select(col("Title"), col("FirstName"), col("LastName")) @@ -350,8 +350,8 @@ object SmartConnectorExternalTableAPINW { println("***** <16>snc_EmpNameOrderByColumnPos : " + snc_EmpNameOrderByColumnPos.show()) println("##### <16>spark_EmpNameOrderByColumnPos : " + spark_EmpNameOrderByColumnPos.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_EmpNameOrderByColumnPos, - spark_EmpNameOrderByColumnPos, "NWAPI16", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_EmpNameOrderByColumnPos, + spark_EmpNameOrderByColumnPos, "NWAPI16", pw) /* <17> SELECT Title, FirstName, LastName FROM Employees ORDER BY Title ASC, LastName DESC; */ val snc_EmpNameOrderByTitleLastName = sncEmpDF.select(col("Title"), col("FirstName"), col("LastName")) @@ -365,8 +365,8 @@ object SmartConnectorExternalTableAPINW { println("##### <17>spark_EmpNameOrderByTitleLastName : " + spark_EmpNameOrderByTitleLastName.show()) } - SnappyTestUtils.assertQueryFullResultSet(snc, snc_EmpNameOrderByTitleLastName, - spark_EmpNameOrderByTitleLastName, "NWAPI17", "column", pw, sqlContext, false) + SnappyTestUtils.assertQuery(snc, snc_EmpNameOrderByTitleLastName, + spark_EmpNameOrderByTitleLastName, "NWAPI17", pw) /* Will add all the NorthWind Queries */ diff --git a/dtests/src/test/scala/io/snappydata/hydra/hiveThriftServer/HiveThriftServer.scala b/dtests/src/test/scala/io/snappydata/hydra/hiveThriftServer/HiveThriftServer.scala index f3a10f20e9..653543f8a1 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/hiveThriftServer/HiveThriftServer.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/hiveThriftServer/HiveThriftServer.scala @@ -282,8 +282,8 @@ class HiveThriftServer extends SnappySQLJob { command : String, pw : PrintWriter, sqlContext : SQLContext) : Unit = { hts.stmt = hts.connection.createStatement() hts.rs = hts.stmt.executeQuery(command) - SnappyTestUtils.assertQueryFullResultSet (snc, command, command, - "showtbls_sys", pw, sqlContext, true) + SnappyTestUtils.tableType = "showTables" + SnappyTestUtils.assertQuery (snc, command, command, pw, sqlContext) } def executeShowSchemas(hts : HiveThriftServer, command : String, @@ -292,8 +292,8 @@ class HiveThriftServer extends SnappySQLJob { hts.stmt = hts.connection.createStatement() hts.rs = hts.stmt.executeQuery(command) hts.rsMetaData = hts.rs.getMetaData - SnappyTestUtils.assertQueryFullResultSet (snc, command, command, - "showtbls", pw, sqlContext, true) + SnappyTestUtils.tableType = "showtbls" + SnappyTestUtils.assertQuery (snc, command, command, pw, sqlContext) } def executeShowTablesInSys(hts : HiveThriftServer, command : String, snc : SnappyContext, @@ -346,44 +346,47 @@ class HiveThriftServer extends SnappySQLJob { // com.google.common.util.concurrent.UncheckedExecutionException: // java.lang.ClassCastException: // org.apache.spark.sql.SparkSession cannot be cast to org.apache.spark.sql.SnappySession - SnappyTestUtils.assertQueryFullResultSet (snc, insertChk1, - "insertCheck1" + System.currentTimeMillis(), tblType, pw, sqlContext, true) - SnappyTestUtils.assertQueryFullResultSet (snc, insertChk2, - "insertCheck2" + System.currentTimeMillis(), tblType, pw, sqlContext, true) + SnappyTestUtils.tableType = tblType + SnappyTestUtils.assertQuery (snc, insertChk1, "insertCheck1" + System.currentTimeMillis(), + pw, sqlContext) + SnappyTestUtils.assertQuery (snc, insertChk2, "insertCheck2" + System.currentTimeMillis(), + pw, sqlContext) } - def updateTableFromBeeline(hts : HiveThriftServer, command : String, - snc : SnappyContext, spark : SparkSession, pw : PrintWriter, - sqlContext : SQLContext, table : String, - tblType: String) : Unit = { - pw.println("Update the table from beeline...") - var updateChk1 : String = null - var updateChk2 : String = null - hts.stmt = hts.connection.createStatement() - hts.stmt.executeQuery(command) - updateChk1 = "select count(*) as Total from " + table - updateChk2 = "select * from " + table + " where id < 500 order by id ASC" - SnappyTestUtils.assertQueryFullResultSet (snc, updateChk1, - "updateCheck1" + System.currentTimeMillis(), tblType, pw, sqlContext, true) - SnappyTestUtils.assertQueryFullResultSet (snc, updateChk2, - "updateCheck2" + System.currentTimeMillis(), tblType, pw, sqlContext, true) - } - - def deleteFromTableFromBeeline(hts : HiveThriftServer, command : String, - snc : SnappyContext, spark : SparkSession, pw : PrintWriter, - sqlContext : SQLContext, table : String, - tblType: String) : Unit = { - pw.println("Delete from table from beeline...") - var deleteChk1 : String = null - var deleteChk2 : String = null - hts.stmt = hts.connection.createStatement() - hts.stmt.executeQuery(command) - deleteChk1 = "select count(*) as Total from " + table - deleteChk2 = "select * from " + table - SnappyTestUtils.assertQueryFullResultSet (snc, deleteChk1, - "deleteCheck1" + System.currentTimeMillis(), tblType, pw, sqlContext, true) - SnappyTestUtils.assertQueryFullResultSet (snc, deleteChk2, - "deleteCheck2" + System.currentTimeMillis(), tblType, pw, sqlContext, true) + def updateTableFromBeeline(hts: HiveThriftServer, command: String, + snc: SnappyContext, spark: SparkSession, pw: PrintWriter, + sqlContext: SQLContext, table: String, + tblType: String): Unit = { + pw.println("Update the table from beeline...") + var updateChk1: String = null + var updateChk2: String = null + hts.stmt = hts.connection.createStatement() + hts.stmt.executeQuery(command) + updateChk1 = "select count(*) as Total from " + table + updateChk2 = "select * from " + table + " where id < 500 order by id ASC" + SnappyTestUtils.tableType = tblType + SnappyTestUtils.assertQuery(snc, updateChk1, + "updateCheck1" + System.currentTimeMillis(), pw, sqlContext) + SnappyTestUtils.assertQuery(snc, updateChk2, + "updateCheck2" + System.currentTimeMillis(), pw, sqlContext) + } + + def deleteFromTableFromBeeline(hts: HiveThriftServer, command: String, + snc: SnappyContext, spark: SparkSession, pw: PrintWriter, + sqlContext: SQLContext, table: String, + tblType: String): Unit = { + pw.println("Delete from table from beeline...") + var deleteChk1: String = null + var deleteChk2: String = null + hts.stmt = hts.connection.createStatement() + hts.stmt.executeQuery(command) + deleteChk1 = "select count(*) as Total from " + table + deleteChk2 = "select * from " + table + SnappyTestUtils.tableType = tblType + SnappyTestUtils.assertQuery(snc, deleteChk1, + "deleteCheck1" + System.currentTimeMillis(), pw, sqlContext) + SnappyTestUtils.assertQuery(snc, deleteChk2, + "deleteCheck2" + System.currentTimeMillis(), pw, sqlContext) } def insertIntoTableFromSnappy(hts : HiveThriftServer, command : String, @@ -399,10 +402,11 @@ class HiveThriftServer extends SnappySQLJob { insertChk2 = "select id, name from " + table + // " where id > 3000000 order by id DESC" " where id > 300 order by id DESC" - SnappyTestUtils.assertQueryFullResultSet (snc, insertChk1, - "insertCheck1" + System.currentTimeMillis(), tblType, pw, sqlContext, true) - SnappyTestUtils.assertQueryFullResultSet (snc, insertChk2, - "insertCheck2" + System.currentTimeMillis(), tblType, pw, sqlContext, true) + SnappyTestUtils.tableType = tblType + SnappyTestUtils.assertQuery (snc, insertChk1, + "insertCheck1" + System.currentTimeMillis(), pw, sqlContext) + SnappyTestUtils.assertQuery (snc, insertChk2, + "insertCheck2" + System.currentTimeMillis(), pw, sqlContext) } def updateTableFromSnappy(hts : HiveThriftServer, command : String, @@ -415,10 +419,11 @@ class HiveThriftServer extends SnappySQLJob { snc.sql(command) updateChk1 = "select count(*) as Total from " + table updateChk2 = "select id, name from " + table + " where id < 500 order by id ASC" - SnappyTestUtils.assertQueryFullResultSet (snc, updateChk1, - "updateCheck1" + System.currentTimeMillis(), tblType, pw, sqlContext, true) - SnappyTestUtils.assertQueryFullResultSet (snc, updateChk2, - "updateCheck2" + System.currentTimeMillis() , tblType, pw, sqlContext, true) + SnappyTestUtils.tableType = tblType + SnappyTestUtils.assertQuery (snc, updateChk1, + "updateCheck1" + System.currentTimeMillis(), pw, sqlContext) + SnappyTestUtils.assertQuery (snc, updateChk2, + "updateCheck2" + System.currentTimeMillis() , pw, sqlContext) } def deleteFromTableFromSnappy(hts : HiveThriftServer, command : String, @@ -431,10 +436,11 @@ class HiveThriftServer extends SnappySQLJob { snc.sql(command) deleteChk1 = "select count(*) as Total from " + table deleteChk2 = "select * from " + table - SnappyTestUtils.assertQueryFullResultSet (snc, deleteChk1, - "deleteCheck1" + System.currentTimeMillis(), tblType, pw, sqlContext, true) - SnappyTestUtils.assertQueryFullResultSet (snc, deleteChk2, - "deleteCheck2" + System.currentTimeMillis(), tblType, pw, sqlContext, true) + SnappyTestUtils.tableType = tblType + SnappyTestUtils.assertQuery (snc, deleteChk1, + "deleteCheck1" + System.currentTimeMillis(), pw, sqlContext) + SnappyTestUtils.assertQuery (snc, deleteChk2, + "deleteCheck2" + System.currentTimeMillis(), pw, sqlContext) } def executeShowTables(hts : HiveThriftServer, command : String, pw : PrintWriter) : Unit = { diff --git a/dtests/src/test/scala/io/snappydata/hydra/northwind/CreateAndLoadNWTablesJob.scala b/dtests/src/test/scala/io/snappydata/hydra/northwind/CreateAndLoadNWTablesJob.scala index 37cfb4f6ef..dc02c7a177 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/northwind/CreateAndLoadNWTablesJob.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/northwind/CreateAndLoadNWTablesJob.scala @@ -19,9 +19,9 @@ package io.snappydata.hydra.northwind import java.io.{File, FileOutputStream, PrintWriter} import com.typesafe.config.Config -import io.snappydata.hydra.northwind -import org.apache.spark.sql._ +import io.snappydata.hydra.{SnappyTestUtils, northwind} +import org.apache.spark.sql._ import scala.util.{Failure, Success, Try} class CreateAndLoadNWTablesJob extends SnappySQLJob { @@ -33,18 +33,15 @@ class CreateAndLoadNWTablesJob extends SnappySQLJob { // scalastyle:off println println("jobConfig.entrySet().size() : " + jobConfig.entrySet().size()) val dataFilesLocation = jobConfig.getString("dataFilesLocation") - pw.println(s"dataFilesLocation is : ${dataFilesLocation}") - println(s"dataFilesLocation is : ${dataFilesLocation}") + pw.println(s"${SnappyTestUtils.logTime} dataFilesLocation : ${dataFilesLocation}") val tableType = jobConfig.getString("tableType") - pw.println(s"tableType : " + tableType) - println(s"tableType : " + tableType) snc.setConf("dataFilesLocation", dataFilesLocation) val createLargeOrderTable = jobConfig.getString("createLargeOrderTable").toBoolean northwind.NWQueries.snc = snc NWQueries.dataFilesLocation = dataFilesLocation NWTestUtil.dropTables(snc) - pw.println(s"Create and load ${tableType} tables Test started at : " + System - .currentTimeMillis) + pw.println(s"${SnappyTestUtils.logTime} Create and load ${tableType} tables " + + s"Test has started") tableType match { case "ReplicatedRow" => NWTestUtil.createAndLoadReplicatedTables(snc) case "PartitionedRow" => @@ -53,14 +50,13 @@ class CreateAndLoadNWTablesJob extends SnappySQLJob { case "Colocated" => NWTestUtil.createAndLoadColocatedTables(snc) case _ => // the default, catch-all } - pw.println(s"Create and load ${tableType} tables Test completed successfully at : " + - System.currentTimeMillis) + pw.println(s"${SnappyTestUtils.logTime} Create and load ${tableType} tables " + + s"Test completed successfully") pw.flush() if (createLargeOrderTable) { NWTestUtil.ingestMoreData(snc, 10) } - pw.println(s"Loaded more data successfully at : " + - System.currentTimeMillis) + pw.println(s"${SnappyTestUtils.logTime} Loaded more data successfully") pw.close() } match { case Success(v) => pw.close() diff --git a/dtests/src/test/scala/io/snappydata/hydra/northwind/CreateAndLoadNWTablesSparkApp.scala b/dtests/src/test/scala/io/snappydata/hydra/northwind/CreateAndLoadNWTablesSparkApp.scala index d3956f2769..db937a607d 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/northwind/CreateAndLoadNWTablesSparkApp.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/northwind/CreateAndLoadNWTablesSparkApp.scala @@ -18,16 +18,16 @@ package io.snappydata.hydra.northwind import java.io.{File, FileOutputStream, PrintWriter} +import io.snappydata.hydra.SnappyTestUtils + import org.apache.spark.sql.SnappyContext -import org.apache.spark.{SparkContext, SparkConf} +import org.apache.spark.{SparkConf, SparkContext} object CreateAndLoadNWTablesSparkApp { def main(args: Array[String]) { - val connectionURL = args(args.length - 1) val conf = new SparkConf(). - setAppName("CreateAndLoadNWTablesSpark Application"). - set("snappydata.connection", connectionURL) + setAppName("CreateAndLoadNWTablesSpark Application") val sc = SparkContext.getOrCreate(conf) val snc = SnappyContext(sc) val dataFilesLocation = args(0) @@ -40,9 +40,10 @@ object CreateAndLoadNWTablesSparkApp { // scalastyle:off println val pw = new PrintWriter(new FileOutputStream(new File("CreateAndLoadNWTablesSparkApp.out"), true)); - pw.println(s"dataFilesLocation : ${dataFilesLocation}") + pw.println(s"${SnappyTestUtils.logTime} dataFilesLocation : ${dataFilesLocation}") NWTestUtil.dropTables(snc) - pw.println(s"Create and load ${tableType} tables Test started at : " + System.currentTimeMillis) + pw.println(s"${SnappyTestUtils.logTime} Create and load ${tableType} tables Test " + + s"started") tableType match { case "ReplicatedRow" => NWTestUtil.createAndLoadReplicatedTables(snc) case "PartitionedRow" => NWTestUtil.createAndLoadPartitionedTables(snc, createLargeOrderTable) @@ -50,14 +51,13 @@ object CreateAndLoadNWTablesSparkApp { case "Colocated" => NWTestUtil.createAndLoadColocatedTables(snc) case _ => // the default, catch-all } - pw.println(s"Create and load ${tableType} tables Test completed successfully at : " + System - .currentTimeMillis) + pw.println(s"${SnappyTestUtils.logTime} Create and load ${tableType} tables Test " + + s"completed successfully.") pw.flush() if (createLargeOrderTable) { NWTestUtil.ingestMoreData(snc, 10) } - pw.println(s"Loaded more data successfully at : " + System - .currentTimeMillis) + pw.println(s"${SnappyTestUtils.logTime} Loaded more data successfully.") pw.close() } } diff --git a/dtests/src/test/scala/io/snappydata/hydra/northwind/EvictionTestNWQueriesApp.scala b/dtests/src/test/scala/io/snappydata/hydra/northwind/EvictionTestNWQueriesApp.scala index 4efb259482..bd6b9d0ff2 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/northwind/EvictionTestNWQueriesApp.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/northwind/EvictionTestNWQueriesApp.scala @@ -24,10 +24,8 @@ import org.apache.spark.{SparkConf, SparkContext} object EvictionTestNWQueriesApp { def main(args: Array[String]) { - val connectionURL = args(args.length - 1) val conf = new SparkConf(). - setAppName("EvictionTestNWQueriesApp Application_" + System.currentTimeMillis()). - set("snappydata.connection", connectionURL) + setAppName("EvictionTestNWQueriesApp Application_" + System.currentTimeMillis()) val sc = SparkContext.getOrCreate(conf) val snc = SnappyContext(sc) val tableType = args(0) diff --git a/dtests/src/test/scala/io/snappydata/hydra/northwind/NWQueries.scala b/dtests/src/test/scala/io/snappydata/hydra/northwind/NWQueries.scala index 13d2efa1e7..506cf3ee64 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/northwind/NWQueries.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/northwind/NWQueries.scala @@ -851,82 +851,39 @@ object NWQueries { "Q23" -> Q23, "Q24" -> Q24, "Q25" -> Q25, - "Q25_1" -> Q25_1, - "Q25_2" -> Q25_2, "Q26" -> Q26, - "Q26_1" -> Q26_1, - "Q26_2" -> Q26_2, "Q27" -> Q27, - "Q27_1" -> Q27_1, - "Q27_2" -> Q27_2, - "Q27_3" -> Q27_3, - "Q27_4" -> Q27_4, "Q28" -> Q28, - "Q28_1" -> Q28_1, - "Q28_2" -> Q28_2, "Q29" -> Q29, - "Q29_1" -> Q29_1, - "Q29_2" -> Q29_2, "Q30" -> Q30, - "Q30_1" -> Q30_1, - "Q30_2" -> Q30_2, "Q61" -> Q61, "Q62" -> Q62, "Q31" -> Q31, - "Q31_1" -> Q31_1, - "Q31_2" -> Q31_2, - "Q31_3" -> Q31_3, - "Q31_4" -> Q31_4, "Q32" -> Q32, - "Q32_1" -> Q32_1, "Q33" -> Q33, - "Q33_1" -> Q33_1, "Q34" -> Q34, - "Q34_1" -> Q34_1, - "Q34_2" -> Q34_2, "Q35" -> Q35, - "Q35_1" -> Q35_1, - "Q35_2" -> Q35_2, "Q36" -> Q36, - "Q36_1" -> Q36_1, - "Q36_2" -> Q36_2, "Q37" -> Q37, "Q38" -> Q38, - "Q38_1" -> Q38_1, - "Q38_2" -> Q38_2, "Q39" -> Q39, "Q40" -> Q40, - "Q40_1" -> Q40_1, - "Q40_2" -> Q40_2, "Q41" -> Q41, "Q42" -> Q42, - "Q42_1" -> Q42_1, - "Q42_2" -> Q42_2, "Q43" -> Q43, - "Q43_1" -> Q43_1, - "Q43_2" -> Q43_2, "Q44" -> Q44, "Q45" -> Q45, "Q46" -> Q46, "Q47" -> Q47, "Q48" -> Q48, "Q49" -> Q49, - "Q49_1" -> Q49_1, - "Q49_2" -> Q49_2, "Q50" -> Q50, "Q51" -> Q51, - "Q51_1" -> Q51_1, - "Q51_2" -> Q51_2, "Q52" -> Q52, "Q53" -> Q53, "Q54" -> Q54, "Q55" -> Q55, - "Q55_1" -> Q55_1, - "Q55_2" -> Q55_2, "Q56" -> Q56, - "Q56_1" -> Q56_1, - "Q56_2" -> Q56_2, - "Q56_3" -> Q56_3, "Q57" -> Q57, "Q58" -> Q58, "Q59" -> Q59, diff --git a/dtests/src/test/scala/io/snappydata/hydra/northwind/NWSparkTablesAndQueriesApp.scala b/dtests/src/test/scala/io/snappydata/hydra/northwind/NWSparkTablesAndQueriesApp.scala index 1ec75cc98c..b54512c010 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/northwind/NWSparkTablesAndQueriesApp.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/northwind/NWSparkTablesAndQueriesApp.scala @@ -24,10 +24,8 @@ import org.apache.spark.{SparkConf, SparkContext} object NWSparkTablesAndQueriesApp { def main(args: Array[String]) { - val connectionURL = args(args.length - 1) val conf = new SparkConf(). - setAppName("NWSparkTablesAndQueriesApp Application_" + System.currentTimeMillis()). - set("snappydata.connection", connectionURL) + setAppName("NWSparkTablesAndQueriesApp Application_" + System.currentTimeMillis()) val sc = SparkContext.getOrCreate(conf) val sqlContext = SQLContext.getOrCreate(sc) val snc = SnappyContext(sc) diff --git a/dtests/src/test/scala/io/snappydata/hydra/northwind/NWTestJob.scala b/dtests/src/test/scala/io/snappydata/hydra/northwind/NWTestJob.scala index bd5abb7cf4..8fe2cd3622 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/northwind/NWTestJob.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/northwind/NWTestJob.scala @@ -19,13 +19,15 @@ package io.snappydata.hydra.northwind import java.io.{File, FileOutputStream, PrintWriter} import com.typesafe.config.Config -import org.apache.spark.sql._ +import org.apache.spark.sql._ import scala.util.{Failure, Success, Try} +import io.snappydata.hydra.SnappyTestUtils + +import org.apache.spark.SparkContext + object NWTestJob extends SnappySQLJob { - var regions, categories, shippers, employees, customers, orders, order_details, products, - suppliers, territories, employee_territories: DataFrame = null def getCurrentDirectory: String = new java.io.File(".").getCanonicalPath @@ -33,63 +35,35 @@ object NWTestJob extends SnappySQLJob { val pw = new PrintWriter(new FileOutputStream(new File("NWTestSnappyJob.out"), true)); Try { val snc = snappySession.sqlContext + val sc = SparkContext.getOrCreate() + val sqlContext = SQLContext.getOrCreate(sc) snc.sql("set spark.sql.shuffle.partitions=6") val dataLocation = jobConfig.getString("dataFilesLocation") snc.setConf("dataFilesLocation", dataLocation) NWQueries.snc = snc NWQueries.dataFilesLocation = dataLocation - regions = snc.read.format("com.databricks.spark.csv") - .option("header", "true") - .load(s"$dataLocation/regions.csv") - categories = snc.read.format("com.databricks.spark.csv") - .option("header", "true") - .load(s"$dataLocation/categories.csv") - shippers = snc.read.format("com.databricks.spark.csv") - .option("header", "true") - .load(s"$dataLocation/shippers.csv") - employees = snc.read.format("com.databricks.spark.csv") - .option("header", "true") - .load(s"$dataLocation/employees.csv") - customers = snc.read.format("com.databricks.spark.csv") - .option("header", "true") - .load(s"$dataLocation/customers.csv") - orders = snc.read.format("com.databricks.spark.csv") - .option("header", "true") - .load(s"$dataLocation/orders.csv") - order_details = snc.read.format("com.databricks.spark.csv") - .option("header", "true") - .load(s"$dataLocation/order-details.csv") - products = snc.read.format("com.databricks.spark.csv") - .option("header", "true") - .load(s"$dataLocation/products.csv") - suppliers = snc.read.format("com.databricks.spark.csv") - .option("header", "true") - .load(s"$dataLocation/suppliers.csv") - territories = snc.read.format("com.databricks.spark.csv") - .option("header", "true") - .load(s"$dataLocation/territories.csv") - employee_territories = snc.read.format("com.databricks.spark.csv") - .option("header", "true") - .load(s"$dataLocation/employee-territories.csv") + SnappyTestUtils.numRowsValidation = true + SnappyTestUtils.validateFullResultSet = true NWTestUtil.dropTables(snc) // scalastyle:off println println("Test replicated row tables queries started") NWTestUtil.createAndLoadReplicatedTables(snc) - NWTestUtil.validateQueries(snc, "Replicated Row Table", pw) + NWTestUtil.validateQueries(snc, "Replicated Row Table", pw, sqlContext) println("Test replicated row tables queries completed successfully") NWTestUtil.dropTables(snc) println("Test partitioned row tables queries started") NWTestUtil.createAndLoadPartitionedTables(snc) - NWTestUtil.validateQueries(snc, "Partitioned Row Table", pw) + + NWTestUtil.validateQueries(snc, "Partitioned Row Table", pw, sqlContext) println("Test partitioned row tables queries completed successfully") NWTestUtil.dropTables(snc) println("Test column tables queries started") NWTestUtil.createAndLoadColumnTables(snc) - NWTestUtil.validateQueries(snc, "Column Table", pw) + NWTestUtil.validateQueries(snc, "Column Table", pw, sqlContext) println("Test column tables queries completed successfully") NWTestUtil.dropTables(snc) NWTestUtil.createAndLoadColocatedTables(snc) - NWTestUtil.validateQueries(snc, "Colocated Table", pw) + NWTestUtil.validateQueries(snc, "Colocated Table", pw, sqlContext) pw.close() } match { case Success(v) => pw.close() diff --git a/dtests/src/test/scala/io/snappydata/hydra/northwind/NWTestSparkApp.scala b/dtests/src/test/scala/io/snappydata/hydra/northwind/NWTestSparkApp.scala index 09403d2cdb..1bba040f98 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/northwind/NWTestSparkApp.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/northwind/NWTestSparkApp.scala @@ -18,18 +18,19 @@ package io.snappydata.hydra.northwind import java.io.{File, FileOutputStream, PrintWriter} -import org.apache.spark.sql.SnappyContext +import io.snappydata.hydra.SnappyTestUtils + +import org.apache.spark.sql.{SQLContext, SnappyContext} import org.apache.spark.{SparkConf, SparkContext} object NWTestSparkApp { def main(args: Array[String]) { - val connectionURL = args(args.length - 1) val conf = new SparkConf(). - setAppName("NWTestSpark Application_" + System.currentTimeMillis()). - set("snappydata.connection", connectionURL) + setAppName("NWTestSpark Application_" + System.currentTimeMillis()) val sc = SparkContext.getOrCreate(conf) + val sqlContext = SQLContext.getOrCreate(sc) val snc = SnappyContext(sc) val dataFilesLocation = args(0) // scalastyle:off println @@ -39,24 +40,26 @@ object NWTestSparkApp { NWQueries.snc = snc NWQueries.dataFilesLocation = dataFilesLocation val pw = new PrintWriter(new FileOutputStream(new File("NWTestSparkApp.out"), true)); + SnappyTestUtils.numRowsValidation = true + SnappyTestUtils.validateFullResultSet = true NWTestUtil.dropTables(snc) println("Test replicated row tables queries started") NWTestUtil.createAndLoadReplicatedTables(snc) - NWTestUtil.validateQueries(snc, "Replicated Row Table", pw) + NWTestUtil.validateQueries(snc, "Replicated Row Table", pw, sqlContext) println("Test replicated row tables queries completed successfully") NWTestUtil.dropTables(snc) println("Test partitioned row tables queries started") NWTestUtil.createAndLoadPartitionedTables(snc) - NWTestUtil.validateQueries(snc, "Partitioned Row Table", pw) + NWTestUtil.validateQueries(snc, "Partitioned Row Table", pw, sqlContext) println("Test partitioned row tables queries completed successfully") NWTestUtil.dropTables(snc) println("Test column tables queries started") NWTestUtil.createAndLoadColumnTables(snc) - NWTestUtil.validateQueries(snc, "Column Table", pw) + NWTestUtil.validateQueries(snc, "Column Table", pw, sqlContext) println("Test column tables queries completed successfully") NWTestUtil.dropTables(snc) NWTestUtil.createAndLoadColocatedTables(snc) - NWTestUtil.validateQueries(snc, "Colocated Table", pw) + NWTestUtil.validateQueries(snc, "Colocated Table", pw, sqlContext) pw.close() } diff --git a/dtests/src/test/scala/io/snappydata/hydra/northwind/NWTestUtil.scala b/dtests/src/test/scala/io/snappydata/hydra/northwind/NWTestUtil.scala index f7b1d44954..36d1cfb14e 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/northwind/NWTestUtil.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/northwind/NWTestUtil.scala @@ -16,166 +16,16 @@ */ package io.snappydata.hydra.northwind -import java.io.{File, PrintWriter} +import java.io.PrintWriter + import io.snappydata.hydra.SnappyTestUtils -import scala.io.Source -import org.apache.spark.sql.catalyst.encoders.RowEncoder import org.apache.spark.sql._ object NWTestUtil { var executeQueriesByChangingConstants: Boolean = false; - def assertJoin(snc: SnappyContext, sqlString: String, numRows: Int, queryNum: String, - tableType: String, pw: PrintWriter): Any = { - snc.sql("set spark.sql.crossJoin.enabled = true") - val df = snc.sql(sqlString) - // scalastyle:off println - println(s"Query $queryNum") - df.explain(true) - pw.println(s"Query ${queryNum} \n df.count for join query is : ${df.count} \n Expected " + - s"numRows : ${numRows} \n Table Type : ${tableType}") - println(s"Query ${queryNum} \n df.count for join query is : ${df.count} \n Expected numRows :" + - s" ${numRows} \n Table Type : ${tableType}") - if(df.count() != numRows){ - pw.println(s"Count mismatch for query ${queryNum} : df.count -> ${df.count()} but " + - s"expected numRows -> $numRows. Query => $sqlString Table Type => $tableType") - } - // scalastyle:on println - pw.flush() - } - - def assertQuery(snc: SnappyContext, sqlString: String, numRows: Int, queryNum: String, - tableType: String, pw: PrintWriter): Any = { - val df = snc.sql(sqlString) - // scalastyle:off println - println(s"Query $queryNum") - df.explain(true) - pw.println(s"Query ${queryNum} \n df.count is : ${df.count} \n Expected numRows : ${numRows} " + - s"\n Table Type : ${tableType}") - println(s"Query ${queryNum} \n df.count is : ${df.count} \n Expected numRows : ${numRows} \n " + - s"Table Type : ${tableType}") - if(df.count() != numRows){ - pw.println(s"Count mismatch for query ${queryNum} : df.count -> ${df.count()} but " + - s"expected numRows -> $numRows. Query => $sqlString Table Type => $tableType") - } - // scalastyle:on println - pw.flush() - } - - def assertJoinFullResultSet(snc: SnappyContext, sqlString: String, numRows: Int, queryNum: - String, tableType: String, pw: PrintWriter, sqlContext: SQLContext): Any = { - snc.sql("set spark.sql.crossJoin.enabled = true") - sqlContext.sql("set spark.sql.crossJoin.enabled = true") - assertQueryFullResultSet(snc, sqlString, numRows, queryNum, tableType, pw, sqlContext) - } - - def dataTypeConverter(row: Row): Row = { - val md = row.toSeq.map { - // case d: Double => "%18.1f".format(d).trim().toDouble - case d: Double => math.floor(d * 10.0 + 0.5) / 10.0 - case de: BigDecimal => { - de.setScale(2, BigDecimal.RoundingMode.HALF_UP) - } - case i: Integer => { - i - } - case v => v - } - Row.fromSeq(md) - } - - def writeToFile(df: DataFrame, dest: String, snc: SnappyContext): Unit = { - import snc.implicits._ - df.map(dataTypeConverter)(RowEncoder(df.schema)) - .map(row => { - val sb = new StringBuilder - row.toSeq.foreach { - case e if e == null => - sb.append("NULL").append(",") - case e => - sb.append(e.toString).append(",") - } - sb.toString() - }).write.format("org.apache.spark.sql.execution.datasources.csv.CSVFileFormat").option( - "header", false).save(dest) - } - - protected def getTempDir(dirName: String): String = { - val log: File = new File(".") - var dest: String = null - val dirString = log.getCanonicalPath; - if (dirName.equals("sparkQueryFiles")) { - val logDir = log.listFiles.filter(_.getName.equals("snappyleader.log")) - if (!logDir.isEmpty) { - val leaderLogFile: File = logDir.iterator.next() - if (leaderLogFile.exists()) dest = dirString + File.separator + ".." + File.separator + "" + - ".." + File.separator + dirName - } - else dest = dirString + File.separator + ".." + File.separator + dirName - } - else dest = log.getCanonicalPath + File.separator + dirName - val tempDir: File = new File(dest) - if (!tempDir.exists) tempDir.mkdir() - return tempDir.getAbsolutePath - } - - def assertQueryFullResultSet(snc: SnappyContext, sqlString: String, numRows: Int, queryNum: - String, tableType: String, pw: PrintWriter, sqlContext: SQLContext): Any = { - // scalastyle:off println - var snappyDF = snc.sql(sqlString) - var sparkDF = sqlContext.sql(sqlString); - val snappyQueryFileName = s"Snappy_${queryNum}.out" - val sparkQueryFileName = s"Spark_${queryNum}.out" - val snappyDest: String = getTempDir("snappyQueryFiles") + File.separator + snappyQueryFileName - val sparkDest: String = getTempDir("sparkQueryFiles") + File.separator + sparkQueryFileName - val sparkFile: File = new java.io.File(sparkDest) - val snappyFile = new java.io.File(snappyDest) - val col1 = sparkDF.schema.fieldNames(0) - val col = sparkDF.schema.fieldNames.filter(!_.equals(col1)).toSeq - if (snappyFile.listFiles() == null) { - snappyDF = snappyDF.repartition(1).sortWithinPartitions(col1, col: _*) - writeToFile(snappyDF, snappyDest, snc) - pw.println(s"${queryNum} Result Collected in file $snappyDest") - } - if (sparkFile.listFiles() == null) { - sparkDF = sparkDF.repartition(1).sortWithinPartitions(col1, col: _*) - writeToFile(sparkDF, sparkDest, snc) - pw.println(s"${queryNum} Result Collected in file $sparkDest") - } - val expectedFile = sparkFile.listFiles.filter(_.getName.endsWith(".csv")) - val actualFile = snappyFile.listFiles.filter(_.getName.endsWith(".csv")) - val expectedLineSet = Source.fromFile(expectedFile.iterator.next()).getLines() - val actualLineSet = Source.fromFile(actualFile.iterator.next()).getLines - var numLines = 0 - while (expectedLineSet.hasNext && actualLineSet.hasNext) { - val expectedLine = expectedLineSet.next() - val actualLine = actualLineSet.next() - if (!actualLine.equals(expectedLine)) { - pw.println(s"\n** For ${queryNum} result mismatch observed**") - pw.println(s"\nExpected Result:\n $expectedLine") - pw.println(s"\nActual Result:\n $actualLine") - pw.println(s"\nQuery =" + sqlString + " Table Type : " + tableType) - /* assert(assertion = false, s"\n** For $queryNum result mismatch observed** \n" + - s"Expected Result \n: $expectedLine \n" + - s"Actual Result \n: $actualLine \n" + - s"Query =" + sqlString + " Table Type : " + tableType) - */ - // Commented due to Q37 failure by just the difference of 0.1 in actual and expected value - } - numLines += 1 - } - if (actualLineSet.hasNext || expectedLineSet.hasNext) { - pw.println(s"\nFor ${queryNum} result count mismatch observed") - assert(assertion = false, s"\nFor $queryNum result count mismatch observed") - } - assert(numLines == numRows, s"\nFor $queryNum result count mismatch " + - s"observed: Expected=$numRows, Got=$numLines") - pw.flush() - // scalastyle:on println - } - def createAndLoadReplicatedTables(snc: SnappyContext): Unit = { snc.sql(NWQueries.regions_table) @@ -212,602 +62,481 @@ object NWTestUtil { NWQueries.employee_territories(snc).write.insertInto("employee_territories") } - def validateQueries(snc: SnappyContext, tableType: String, pw: PrintWriter): Unit = { + /* + Method for validating queires results for northwind schema with default size data. + For larger data size, this method can be used for comparing full resultsets by setting + numRowsValidation to false. + Returns failed queries in a string. + */ + def validateQueries(snc: SnappyContext, tableType: String, pw: PrintWriter, sqlContext: + SQLContext): String = { + var failedQueries = "" + if (SnappyTestUtils.validateFullResultSet) { + // scalastyle:off println + pw.println(s"${SnappyTestUtils.logTime} CreateAndLoadSparkTables started ...") + val startTime = System.currentTimeMillis + NWTestUtil.createAndLoadSparkTables(sqlContext) + val finishTime = System.currentTimeMillis() + pw.println(s"${SnappyTestUtils.logTime} CreateAndLoadSparkTables completed successfully in " + + s": " + ((finishTime - startTime) / 1000) + " seconds") + } for (q <- NWQueries.queries) { + var queryExecuted = false; + var hasValidationFailed = false; + if (!executeQueriesByChangingConstants) { q._1 match { - case "Q1" => assertQuery(snc, NWQueries.Q1, 8, "Q1", tableType, pw) - case "Q2" => assertQuery(snc, NWQueries.Q2, 91, "Q2", tableType, pw) - case "Q3" => assertQuery(snc, NWQueries.Q3, 830, "Q3", tableType, pw) - case "Q4" => assertQuery(snc, NWQueries.Q4, 9, "Q4", tableType, pw) - case "Q5" => assertQuery(snc, NWQueries.Q5, 9, "Q5", tableType, pw) - case "Q6" => assertQuery(snc, NWQueries.Q6, 9, "Q6", tableType, pw) - case "Q7" => assertQuery(snc, NWQueries.Q7, 9, "Q7", tableType, pw) - case "Q8" => assertQuery(snc, NWQueries.Q8, 6, "Q8", tableType, pw) - case "Q9" => assertQuery(snc, NWQueries.Q9, 3, "Q9", tableType, pw) - case "Q10" => assertQuery(snc, NWQueries.Q10, 2, "Q10", tableType, pw) - case "Q11" => assertQuery(snc, NWQueries.Q11, 4, "Q11", tableType, pw) - case "Q12" => assertQuery(snc, NWQueries.Q12, 2, "Q12", tableType, pw) - case "Q13" => assertQuery(snc, NWQueries.Q13, 2, "Q13", tableType, pw) - case "Q14" => assertQuery(snc, NWQueries.Q14, 69, "Q14", tableType, pw) - case "Q15" => assertQuery(snc, NWQueries.Q15, 5, "Q15", tableType, pw) - case "Q16" => assertQuery(snc, NWQueries.Q16, 8, "Q16", tableType, pw) - case "Q17" => assertQuery(snc, NWQueries.Q17, 3, "Q17", tableType, pw) - case "Q18" => assertQuery(snc, NWQueries.Q18, 9, "Q18", tableType, pw) - case "Q19" => assertQuery(snc, NWQueries.Q19, 13, "Q19", tableType, pw) - case "Q20" => assertQuery(snc, NWQueries.Q20, 1, "Q20", tableType, pw) - case "Q21" => assertQuery(snc, NWQueries.Q21, 1, "Q21", tableType, pw) - case "Q22" => assertQuery(snc, NWQueries.Q22, 1, "Q22", tableType, pw) - case "Q23" => assertQuery(snc, NWQueries.Q23, 1, "Q23", tableType, pw) - case "Q24" => assertQuery(snc, NWQueries.Q24, 4, "Q24", tableType, pw) - case "Q37" => assertJoin(snc, NWQueries.Q37, 77, "Q37", tableType, pw) - case "Q39" => assertJoin(snc, NWQueries.Q39, 9, "Q39", tableType, pw) - case "Q41" => assertJoin(snc, NWQueries.Q41, 2155, "Q41", tableType, pw) - case "Q44" => assertJoin(snc, NWQueries.Q44, 830, "Q44", tableType, pw)// LeftSemiJoinHash - case "Q45" => assertJoin(snc, NWQueries.Q45, 1788650, "Q45", tableType, pw) - case "Q46" => assertJoin(snc, NWQueries.Q46, 1788650, "Q46", tableType, pw) - case "Q47" => assertJoin(snc, NWQueries.Q47, 1788650, "Q47", tableType, pw) - case "Q48" => assertJoin(snc, NWQueries.Q48, 1788650, "Q48", tableType, pw) - case "Q50" => assertJoin(snc, NWQueries.Q50, 2155, "Q50", tableType, pw) - case "Q52" => assertJoin(snc, NWQueries.Q52, 2155, "Q52", tableType, pw) - case "Q53" => assertJoin(snc, NWQueries.Q53, 2155, "Q53", tableType, pw) - case "Q54" => assertJoin(snc, NWQueries.Q54, 2155, "Q54", tableType, pw) - case "Q57" => assertJoin(snc, NWQueries.Q57, 120, "Q57", tableType, pw) - case "Q58" => assertJoin(snc, NWQueries.Q58, 1, "Q58", tableType, pw) - case "Q59" => assertJoin(snc, NWQueries.Q59, 1, "Q59", tableType, pw) - case "Q60" => assertJoin(snc, NWQueries.Q60, 947, "Q60", tableType, pw) - case "Q61" => assertJoin(snc, NWQueries.Q61, 480, "Q61", tableType, pw) - case "Q62" => assertJoin(snc, NWQueries.Q62, 480, "Q62", tableType, pw) + case "Q1" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, NWQueries.Q1, 8, "Q1", + pw, sqlContext); queryExecuted = true + case "Q2" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, NWQueries.Q2, 91, + "Q2", pw, sqlContext); queryExecuted = true + case "Q3" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, NWQueries.Q3, 830, + "Q3", pw, sqlContext); queryExecuted = true + case "Q4" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, NWQueries.Q4, 9, "Q4", + pw, sqlContext); queryExecuted = true + case "Q5" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, NWQueries.Q5, 9, "Q5", + pw, sqlContext); queryExecuted = true + case "Q6" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, NWQueries.Q6, 9, "Q6", + pw, sqlContext); queryExecuted = true + case "Q7" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, NWQueries.Q7, 9, "Q7", + pw, sqlContext); queryExecuted = true + case "Q8" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, NWQueries.Q8, 6, "Q8", + pw, sqlContext); queryExecuted = true + case "Q9" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, NWQueries.Q9, 3, "Q9", + pw, sqlContext); queryExecuted = true + case "Q10" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, NWQueries.Q10, 2, + "Q10", pw, sqlContext); queryExecuted = true + case "Q11" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, NWQueries.Q11, 4, + "Q11", pw, sqlContext); queryExecuted = true + case "Q12" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, NWQueries.Q12, 2, + "Q12", pw, sqlContext); queryExecuted = true + case "Q13" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, NWQueries.Q13, 2, + "Q13", pw, sqlContext); queryExecuted = true + case "Q14" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, NWQueries.Q14, 69, + "Q14", pw, sqlContext); queryExecuted = true + case "Q15" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, NWQueries.Q15, 5, + "Q15", pw, sqlContext); queryExecuted = true + case "Q16" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, NWQueries.Q16, 8, + "Q16", pw, sqlContext); queryExecuted = true + case "Q17" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, NWQueries.Q17, 3, + "Q17", pw, sqlContext); queryExecuted = true + case "Q18" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, NWQueries.Q18, 9, + "Q18", pw, sqlContext); queryExecuted = true + case "Q19" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, NWQueries.Q19, 13, + "Q19", pw, sqlContext); queryExecuted = true + case "Q20" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, NWQueries.Q20, 1, + "Q20", pw, sqlContext); queryExecuted = true + case "Q21" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, NWQueries.Q21, 1, + "Q21", pw, sqlContext); queryExecuted = true + case "Q22" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, NWQueries.Q22, 1, + "Q22", pw, sqlContext); queryExecuted = true + case "Q23" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, NWQueries.Q23, 1, + "Q23", pw, sqlContext); queryExecuted = true + case "Q24" => hasValidationFailed = SnappyTestUtils.assertQuery(snc, NWQueries.Q24, 4, + "Q24", pw, sqlContext); queryExecuted = true + case "Q37" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q37, 77, + "Q37", pw, sqlContext); queryExecuted = true + case "Q39" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q39, 9, + "Q39", pw, sqlContext); queryExecuted = true + case "Q41" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q41, 2155, + "Q41", pw, sqlContext); queryExecuted = true + case "Q44" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q44, 830, + "Q44", pw, sqlContext); queryExecuted = true // LeftSemiJoinHash + case "Q45" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q45, + 1788650, "Q45", pw, sqlContext); queryExecuted = true + case "Q46" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q46, + 1788650, "Q46", pw, sqlContext); queryExecuted = true + case "Q47" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q47, + 1788650, "Q47", pw, sqlContext); queryExecuted = true + case "Q48" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q48, + 1788650, "Q48", pw, sqlContext); queryExecuted = true + case "Q50" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q50, + 2155, "Q50", pw, sqlContext); queryExecuted = true + case "Q52" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q52, + 2155, "Q52", pw, sqlContext); queryExecuted = true + case "Q53" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q53, + 2155, "Q53", pw, sqlContext); queryExecuted = true + case "Q54" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q54, + 2155, "Q54", pw, sqlContext); queryExecuted = true + case "Q57" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q57, + 120, "Q57", pw, sqlContext); queryExecuted = true + case "Q58" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q58, + 1, "Q58", pw, sqlContext); queryExecuted = true + case "Q59" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q59, 1, + "Q59", pw, sqlContext); queryExecuted = true + case "Q60" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q60, 947, + "Q60", pw, sqlContext); queryExecuted = true + case "Q61" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q61, 480, + "Q61", pw, sqlContext); queryExecuted = true + case "Q62" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q62, + 480, "Q62", pw, sqlContext); queryExecuted = true case _ => // do nothing } } q._1 match { - case "Q25" => assertJoin(snc, NWQueries.Q25, 1, "Q25", tableType, pw) - if (executeQueriesByChangingConstants) { - assertJoin(snc, NWQueries.Q25_1, 1, "Q25_1", tableType, pw) - assertJoin(snc, NWQueries.Q25_2, 1, "Q25_2", tableType, pw) - } - case "Q26" => assertJoin(snc, NWQueries.Q26, 86, "Q26", tableType, pw) - if (executeQueriesByChangingConstants) { - assertJoin(snc, NWQueries.Q26_1, 54, "Q26_1", tableType, pw) - assertJoin(snc, NWQueries.Q26_2, 60, "Q26_2", tableType, pw) - } - case "Q27" => assertJoin(snc, NWQueries.Q27, 9, "Q27", tableType, pw) - if (executeQueriesByChangingConstants) { - assertJoin(snc, NWQueries.Q27_1, 5, "Q27_1", tableType, pw) - assertJoin(snc, NWQueries.Q27_2, 8, "Q27_2", tableType, pw) - assertJoin(snc, NWQueries.Q27_3, 3, "Q27_3", tableType, pw) - assertJoin(snc, NWQueries.Q27_4, 6, "Q27_4", tableType, pw) - } - case "Q28" => assertJoin(snc, NWQueries.Q28, 12, "Q28", tableType, pw) - if (executeQueriesByChangingConstants) { - assertJoin(snc, NWQueries.Q28_1, 12, "Q28_1", tableType, pw) - assertJoin(snc, NWQueries.Q28_2, 5, "Q28_2", tableType, pw) - } - case "Q29" => assertJoin(snc, NWQueries.Q29, 8, "Q29", tableType, pw) - if (executeQueriesByChangingConstants) { - assertJoin(snc, NWQueries.Q29_1, 5, "Q29_1", tableType, pw) - assertJoin(snc, NWQueries.Q29_2, 6, "Q29_2", tableType, pw) - } - case "Q30" => assertJoin(snc, NWQueries.Q30, 8, "Q30", tableType, pw) - if (executeQueriesByChangingConstants) { - assertJoin(snc, NWQueries.Q30_1, 8, "Q30_1", tableType, pw) - assertJoin(snc, NWQueries.Q30_2, 6, "Q30_2", tableType, pw) - } - case "Q31" => assertJoin(snc, NWQueries.Q31, 830, "Q31", tableType, pw) - if (executeQueriesByChangingConstants) { - assertJoin(snc, NWQueries.Q31_1, 502, "Q31_1", tableType, pw) - assertJoin(snc, NWQueries.Q31_2, 286, "Q31_2", tableType, pw) - assertJoin(snc, NWQueries.Q31_3, 219, "Q31_3", tableType, pw) - assertJoin(snc, NWQueries.Q31_4, 484, "Q31_4", tableType, pw) - } - case "Q32" => assertJoin(snc, NWQueries.Q32, 8, "Q32", tableType, pw) - if (executeQueriesByChangingConstants) { - assertJoin(snc, NWQueries.Q32_1, 282, "Q32_1", tableType, pw) - } - case "Q33" => assertJoin(snc, NWQueries.Q33, 37, "Q33", tableType, pw) - if (executeQueriesByChangingConstants) { - assertJoin(snc, NWQueries.Q33_1, 769, "Q33_1", tableType, pw) - } - case "Q34" => assertJoin(snc, NWQueries.Q34, 5, "Q34", tableType, pw) - if (executeQueriesByChangingConstants) { - assertJoin(snc, NWQueries.Q34_1, 1, "Q34_1", tableType, pw) - assertJoin(snc, NWQueries.Q34_2, 4, "Q34_2", tableType, pw) - } - case "Q35" => assertJoin(snc, NWQueries.Q35, 3, "Q35", tableType, pw) - if (executeQueriesByChangingConstants) { - assertJoin(snc, NWQueries.Q35_1, 2, "Q35_1", tableType, pw) - assertJoin(snc, NWQueries.Q35_2, 3, "Q35_2", tableType, pw) - } - case "Q36" => assertJoin(snc, NWQueries.Q36, 290, "Q36", tableType, pw) - if (executeQueriesByChangingConstants) { - assertJoin(snc, NWQueries.Q36_1, 232, "Q36_1", tableType, pw) - assertJoin(snc, NWQueries.Q36_2, 61, "Q36_2", tableType, pw) - } - case "Q38" => assertJoin(snc, NWQueries.Q38, 2155, "Q38", tableType, pw) - if (executeQueriesByChangingConstants) { - assertJoin(snc, NWQueries.Q38_1, 2080, "Q38_1", tableType, pw) - assertJoin(snc, NWQueries.Q38_2, 2041, "Q38_2", tableType, pw) - } - - case "Q40" => assertJoin(snc, NWQueries.Q40, 830, "Q40", tableType, pw) - if (executeQueriesByChangingConstants) { - assertJoin(snc, NWQueries.Q40_1, 12, "Q40_1", tableType, pw) - assertJoin(snc, NWQueries.Q40_2, 9, "Q40_2", tableType, pw) - } - case "Q42" => assertJoin(snc, NWQueries.Q42, 22, "Q42", tableType, pw) - if (executeQueriesByChangingConstants) { - assertJoin(snc, NWQueries.Q42_1, 22, "Q42_1", tableType, pw) - assertJoin(snc, NWQueries.Q42_2, 7, "Q42_2", tableType, pw) - } - case "Q43" => assertJoin(snc, NWQueries.Q43, 830, "Q43", tableType, pw) - if (executeQueriesByChangingConstants) { - assertJoin(snc, NWQueries.Q43_1, 10, "Q43_1", tableType, pw) - assertJoin(snc, NWQueries.Q43_2, 2, "Q43_2", tableType, pw) - } - case "Q49" => assertJoin(snc, NWQueries.Q49, 1788650, "Q49", tableType, pw) - if (executeQueriesByChangingConstants) { - assertJoin(snc, NWQueries.Q49_1, 1713225, "Q49_1", tableType, pw) - assertJoin(snc, NWQueries.Q49_2, 1741240, "Q49_2", tableType, pw) - } - case "Q51" => assertJoin(snc, NWQueries.Q51, 2155, "Q51", tableType, pw) - if (executeQueriesByChangingConstants) { - assertJoin(snc, NWQueries.Q51_1, 2080, "Q51_1", tableType, pw) - assertJoin(snc, NWQueries.Q51_2, 2041, "Q51_2", tableType, pw) - } - case "Q55" => assertJoin(snc, NWQueries.Q55, 21, "Q55", tableType, pw) - if (executeQueriesByChangingConstants) { - assertJoin(snc, NWQueries.Q55_1, 7, "Q55_1", tableType, pw) - assertJoin(snc, NWQueries.Q55_2, 6, "Q55_2", tableType, pw) - } - case "Q56" => assertJoin(snc, NWQueries.Q56, 8, "Q56", tableType, pw) - if (executeQueriesByChangingConstants) { - assertJoin(snc, NWQueries.Q56, 8, "Q56_1", tableType, pw) - assertJoin(snc, NWQueries.Q56, 8, "Q56_2", tableType, pw) - assertJoin(snc, NWQueries.Q56, 8, "Q56_3", tableType, pw) - } - // scalastyle:off println - case _ => println("OK") - // scalastyle:on println - + case "Q25" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q25, 1, + "Q25", pw, sqlContext) + if (executeQueriesByChangingConstants) { + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q25_1, 1, "Q25_1", pw, + sqlContext) + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q25_2, 1, "Q25_2", pw, + sqlContext) + }; queryExecuted = true + case "Q26" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q26, 86, + "Q26", pw, sqlContext) + if (executeQueriesByChangingConstants) { + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q26_1, 54, "Q26_1", pw, + sqlContext) + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q26_2, 60, "Q26_2", pw, + sqlContext) + }; queryExecuted = true + case "Q27" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q27, 9, "Q27", + pw, sqlContext) + if (executeQueriesByChangingConstants) { + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q27_1, 5, "Q27_1", pw, + sqlContext) + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q27_2, 8, "Q27_2", pw, + sqlContext) + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q27_3, 3, "Q27_3", pw, + sqlContext) + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q27_4, 6, "Q27_4", pw, + sqlContext) + }; queryExecuted = true + case "Q28" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q28, 12, + "Q28", pw, sqlContext) + if (executeQueriesByChangingConstants) { + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q28_1, 12, "Q28_1", + pw, sqlContext) + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q28_2, 5, "Q28_2", + pw, sqlContext) + }; queryExecuted = true + case "Q29" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q29, 8, + "Q29", pw, sqlContext) + if (executeQueriesByChangingConstants) { + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q29_1, 5, "Q29_1", pw, + sqlContext) + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q29_2, 6, "Q29_2", pw, + sqlContext) + }; queryExecuted = true + case "Q30" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q30, 8, "Q30", + pw, sqlContext) + if (executeQueriesByChangingConstants) { + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q30_1, 8, "Q30_1", pw, + sqlContext) + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q30_2, 6, "Q30_2", pw, + sqlContext) + }; queryExecuted = true + case "Q31" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q31, 830, + "Q31", pw, sqlContext) + if (executeQueriesByChangingConstants) { + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q31_1, 502, "Q31_1", + pw, sqlContext) + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q31_2, 286, "Q31_2", pw, + sqlContext) + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q31_3, 219, "Q31_3", pw, + sqlContext) + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q31_4, 484, "Q31_4", pw, + sqlContext) + }; queryExecuted = true + case "Q32" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q32, 8, "Q32", + pw, sqlContext) + if (executeQueriesByChangingConstants) { + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q32_1, 282, "Q32_1", pw, + sqlContext) + }; queryExecuted = true + case "Q33" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q33, 37, + "Q33", pw, sqlContext) + if (executeQueriesByChangingConstants) { + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q33_1, 769, "Q33_1", + pw, sqlContext) + }; queryExecuted = true + case "Q34" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q34, 5, + "Q34", pw, sqlContext) + if (executeQueriesByChangingConstants) { + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q34_1, 1, "Q34_1", + pw, sqlContext) + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q34_2, 4, "Q34_2", + pw, sqlContext) + }; queryExecuted = true + case "Q35" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q35, 3, "Q35", + pw, sqlContext) + if (executeQueriesByChangingConstants) { + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q35_1, 2, "Q35_1", pw, + sqlContext) + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q35_2, 3, "Q35_2", pw, + sqlContext) + }; queryExecuted = true + case "Q36" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q36, 290, + "Q36", pw, sqlContext) + if (executeQueriesByChangingConstants) { + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q36_1, 232, "Q36_1", + pw, sqlContext) + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q36_2, 61, "Q36_2", pw, + sqlContext) + }; queryExecuted = true + case "Q38" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q38, 2155, + "Q38", pw, sqlContext) + if (executeQueriesByChangingConstants) { + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q38_1, 2080, "Q38_1", + pw, sqlContext) + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q38_2, 2041, "Q38_2", + pw, sqlContext) + }; queryExecuted = true + case "Q40" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q40, 830, + "Q40", pw, sqlContext) + if (executeQueriesByChangingConstants) { + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q40_1, 12, "Q40_1", pw, + sqlContext) + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q40_2, 9, "Q40_2", pw, + sqlContext) + }; queryExecuted = true + case "Q42" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q42, 22, + "Q42", pw, sqlContext) + if (executeQueriesByChangingConstants) { + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q42_1, 22, "Q42_1", + pw, sqlContext) + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q42_2, 7, "Q42_2", + pw, sqlContext) + }; queryExecuted = true + case "Q43" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q43, 830, + "Q43", pw, sqlContext) + if (executeQueriesByChangingConstants) { + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q43_1, 10, "Q43_1", + pw, sqlContext) + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q43_2, 2, "Q43_2", pw, + sqlContext) + }; queryExecuted = true + case "Q49" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q49, 1788650, + "Q49", pw, sqlContext) + if (executeQueriesByChangingConstants) { + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q49_1, 1713225, "Q49_1", + pw, sqlContext) + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q49_2, 1741240, "Q49_2", + pw, sqlContext) + }; queryExecuted = true + case "Q51" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q51, 2155, + "Q51", pw, sqlContext) + if (executeQueriesByChangingConstants) { + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q51_1, 2080, "Q51_1", + pw, sqlContext) + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q51_2, 2041, "Q51_2", + pw, sqlContext) + }; queryExecuted = true + case "Q55" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q55, 21, + "Q55", pw, sqlContext) + if (executeQueriesByChangingConstants) { + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q55_1, 7, "Q55_1", pw, + sqlContext) + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q55_2, 6, "Q55_2", pw, + sqlContext) + }; queryExecuted = true + case "Q56" => hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q56, 8, "Q56", + pw, sqlContext) + if (executeQueriesByChangingConstants) { + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q56_1, 8, "Q56_1", pw, + sqlContext) + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q56_2, 8, "Q56_2", pw, + sqlContext) + hasValidationFailed = SnappyTestUtils.assertJoin(snc, NWQueries.Q56_3, 8, "Q56_3", pw, + sqlContext) + }; queryExecuted = true + case _ => // do nothing + } + if (!queryExecuted) { + pw.println(s"${SnappyTestUtils.logTime} Did not execute ${q._1}.") + } + if (hasValidationFailed) { + failedQueries = SnappyTestUtils.addToFailedQueryList(failedQueries, q._1) } } + return failedQueries; + } + + def executeAndValidateQueriesByChangingConstants(snc: SnappyContext, tableType: String, + pw: PrintWriter, sqlContext: SQLContext): String = { + executeQueriesByChangingConstants = true + return validateQueries(snc, tableType, pw, sqlContext) } - def validateQueriesFullResultSet(snc: SnappyContext, tableType: String, pw: PrintWriter, - sqlContext: SQLContext): Unit = { - val usePlanCaching: Boolean = false + def validateSelectiveQueriesFullResultSet(snc: SnappyContext, tableType: String, pw: + PrintWriter, sqlContext: SQLContext): String = { + SnappyTestUtils.numRowsValidation = false + SnappyTestUtils.validateFullResultSet = true + var failedQueries = "" + val usePlanCaching = false; + if (SnappyTestUtils.validateFullResultSet) { + // scalastyle:off println + pw.println(s"createAndLoadSparkTables started ...") + val startTime = System.currentTimeMillis + NWTestUtil.createAndLoadSparkTables(sqlContext) + val finishTime = System.currentTimeMillis() + println(s"createAndLoadSparkTables completed successfully in : " + (finishTime - startTime) + + " seconds") + } for (q <- NWQueries.queries) { - if (!executeQueriesByChangingConstants) { - q._1 match { - case "Q1" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q1, "Q1", tableType, + var queryExecuted = true; + var hasValidationFailed = false; + q._1 match { + case "Q6" => SnappyTestUtils.assertQuery(snc, NWQueries.Q6, "Q6", + pw, sqlContext) + case "Q7" => SnappyTestUtils.assertQuery(snc, NWQueries.Q7, "Q7", + pw, sqlContext) + case "Q9" => SnappyTestUtils.assertQuery(snc, NWQueries.Q9, "Q9", + pw, sqlContext) + case "Q11" => SnappyTestUtils.assertQuery(snc, NWQueries.Q11, "Q11", + pw, sqlContext) + case "Q12" => SnappyTestUtils.assertQuery(snc, NWQueries.Q12, "Q12", + pw, sqlContext) + case "Q13" => SnappyTestUtils.assertQuery(snc, NWQueries.Q13, "Q13", + pw, sqlContext) + case "Q14" => SnappyTestUtils.assertQuery(snc, NWQueries.Q14, "Q14", + pw, sqlContext) + case "Q15" => SnappyTestUtils.assertQuery(snc, NWQueries.Q15, "Q15", + pw, sqlContext) + case "Q16" => SnappyTestUtils.assertQuery(snc, NWQueries.Q16, "Q16", + pw, sqlContext) + case "Q17" => SnappyTestUtils.assertQuery(snc, NWQueries.Q17, "Q17", + pw, sqlContext) + case "Q18" => SnappyTestUtils.assertQuery(snc, NWQueries.Q18, "Q18", + pw, sqlContext) + case "Q19" => SnappyTestUtils.assertQuery(snc, NWQueries.Q19, "Q19", + pw, sqlContext) + case "Q20" => SnappyTestUtils.assertQuery(snc, NWQueries.Q20, "Q20", + pw, sqlContext) + case "Q21" => SnappyTestUtils.assertQuery(snc, NWQueries.Q21, "Q21", + pw, sqlContext) + case "Q22" => SnappyTestUtils.assertQuery(snc, NWQueries.Q22, "Q22", + pw, sqlContext) + case "Q24" => SnappyTestUtils.assertQuery(snc, NWQueries.Q24, "Q24", + pw, sqlContext) + case "Q25" => SnappyTestUtils.assertJoin(snc, NWQueries.Q25, "Q25", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q25_1, "Q25_1", pw, sqlContext) - case "Q2" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q2, "Q2", tableType, + SnappyTestUtils.assertJoin(snc, NWQueries.Q25_2, "Q25_2", pw, sqlContext) - case "Q3" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q3, "Q3", tableType, + case "Q26" => SnappyTestUtils.assertJoin(snc, NWQueries.Q26, "Q26", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q26_1, "Q26_1", pw, sqlContext) - case "Q4" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q4, "Q4", tableType, + SnappyTestUtils.assertJoin(snc, NWQueries.Q26_2, "Q26_2", pw, sqlContext) - case "Q5" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q5, "Q5", tableType, + case "Q27" => SnappyTestUtils.assertJoin(snc, NWQueries.Q27, "Q27", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q27_1, "Q27_1", pw, sqlContext) - case "Q6" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q6, "Q6", tableType, + SnappyTestUtils.assertJoin(snc, NWQueries.Q27_2, "Q27_2", pw, sqlContext) - case "Q7" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q7, "Q7", tableType, + SnappyTestUtils.assertJoin(snc, NWQueries.Q27_3, "Q27_3", pw, sqlContext) - case "Q8" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q8, "Q8", tableType, + SnappyTestUtils.assertJoin(snc, NWQueries.Q27_4, "Q27_4", pw, sqlContext) - case "Q9" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q9, "Q9", tableType, + case "Q28" => SnappyTestUtils.assertJoin(snc, NWQueries.Q28, "Q28", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q28_1, "Q28_1", pw, sqlContext) - case "Q10" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q10, "Q10", - tableType, pw, sqlContext) - case "Q11" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q11, "Q11", - tableType, pw, sqlContext) - case "Q12" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q12, "Q12", - tableType, pw, sqlContext) - case "Q13" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q13, "Q13", - tableType, pw, sqlContext) - case "Q14" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q14, "Q14", - tableType, pw, sqlContext) - case "Q15" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q15, "Q15", - tableType, pw, sqlContext) - case "Q16" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q16, "Q16", - tableType, pw, sqlContext) - case "Q17" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q17, "Q17", - tableType, pw, sqlContext) - case "Q18" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q18, "Q18", - tableType, pw, sqlContext) - case "Q19" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q19, "Q19", - tableType, pw, sqlContext) - case "Q20" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q20, "Q20", - tableType, pw, sqlContext) - case "Q21" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q21, "Q21", - tableType, pw, sqlContext) - case "Q22" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q22, "Q22", - tableType, pw, sqlContext) - case "Q23" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q23, "Q23", - tableType, pw, sqlContext) - case "Q24" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q24, "Q24", - tableType, pw, sqlContext) - case "Q37" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q37, "Q37", - tableType, pw, sqlContext) - case "Q39" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q39, "Q39", - tableType, pw, sqlContext) - case "Q41" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q41, "Q41", - tableType, pw, sqlContext) - case "Q44" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q44, "Q44", - tableType, pw, sqlContext) // LeftSemiJoinHash - case "Q45" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q45, "Q45", - tableType, pw, sqlContext) - case "Q46" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q46, "Q46", - tableType, pw, sqlContext) - case "Q47" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q47, "Q47", - tableType, pw, sqlContext) - case "Q48" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q48, "Q48", - tableType, pw, sqlContext) - case "Q50" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q50, "Q50", - tableType, pw, sqlContext) - case "Q52" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q52, "Q52", - tableType, pw, sqlContext) - case "Q53" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q53, "Q53", - tableType, pw, sqlContext) - case "Q54" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q54, "Q54", - tableType, pw, sqlContext) - case "Q57" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q57, "Q57", - tableType, pw, sqlContext) - case "Q58" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q58, "Q58", - tableType, pw, sqlContext) - case "Q59" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q59, "Q59", - tableType, pw, sqlContext) - case "Q60" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q60, "Q60", - tableType, pw, sqlContext) - case "Q61" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q61, "Q61", - tableType, pw, sqlContext) - case "Q62" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q62, "Q62", - tableType, pw, sqlContext) - case _ => // do nothing - } - } - q._1 match { - case "Q25" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q25, "Q25", - tableType, pw, sqlContext) - if (executeQueriesByChangingConstants) { - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q25_1, "Q25_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q25_2, "Q25_2", - tableType, pw, sqlContext, usePlanCaching) - } - case "Q26" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q26, "Q26", - tableType, pw, sqlContext) - if (executeQueriesByChangingConstants) { - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q26_1, "Q26_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q26_2, "Q26_2", - tableType, pw, sqlContext, usePlanCaching) - } - case "Q27" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q27, "Q27", - tableType, pw, sqlContext) - if (executeQueriesByChangingConstants) { - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q27_1, "Q27_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q27_2, "Q27_2", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q27_3, "Q27_3", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q27_4, "Q27_4", - tableType, pw, sqlContext, usePlanCaching) - } - case "Q28" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q28, "Q28", - tableType, pw, sqlContext) - if (executeQueriesByChangingConstants) { - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q28_1, "Q28_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q28_2, "Q28_2", - tableType, pw, sqlContext, usePlanCaching) - } - case "Q29" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q29, "Q29", - tableType, pw, sqlContext) - if (executeQueriesByChangingConstants) { - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q29_1, "Q29_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q29_2, "Q29_2", - tableType, pw, sqlContext, usePlanCaching) - } - case "Q30" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q30, "Q30", - tableType, pw, sqlContext) - if (executeQueriesByChangingConstants) { - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q30_1, "Q30_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q30_2, "Q30_2", - tableType, pw, sqlContext, usePlanCaching) - } - case "Q31" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q31, "Q31", - tableType, pw, sqlContext) - if (executeQueriesByChangingConstants) { - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q31_1, "Q31_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q31_2, "Q31_2", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q31_3, "Q31_3", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q31_4, "Q31_4", - tableType, pw, sqlContext, usePlanCaching) - } - case "Q32" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q32, "Q32", - tableType, pw, sqlContext) - if (executeQueriesByChangingConstants) { - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q32_1, "Q32_1", - tableType, pw, sqlContext, usePlanCaching) - } - case "Q33" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q33, "Q33", - tableType, pw, sqlContext) - if (executeQueriesByChangingConstants) { - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q33_1, "Q33_1", - tableType, pw, sqlContext, usePlanCaching) - } - case "Q34" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q34, "Q34", - tableType, pw, sqlContext) - if (executeQueriesByChangingConstants) { - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q34_1, "Q34_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q34_2, "Q34_2", - tableType, pw, sqlContext, usePlanCaching) - } - case "Q35" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q35, "Q35", - tableType, pw, sqlContext) - if (executeQueriesByChangingConstants) { - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q35_1, "Q35_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q35_2, "Q35_2", - tableType, pw, sqlContext, usePlanCaching) - } - case "Q36" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q36, "Q36", - tableType, pw, sqlContext) - if (executeQueriesByChangingConstants) { - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q36_1, "Q36_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q36_2, "Q36_2", - tableType, pw, sqlContext, usePlanCaching) - } - case "Q38" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q38, "Q38", - tableType, pw, sqlContext) - if (executeQueriesByChangingConstants) { - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q38_1, "Q38_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q38_2, "Q38_2", - tableType, pw, sqlContext, usePlanCaching) - } - case "Q40" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q40, "Q40", - tableType, pw, sqlContext) - if (executeQueriesByChangingConstants) { - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q40_1, "Q40_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q40_2, "Q40_2", - tableType, pw, sqlContext, usePlanCaching) - } - case "Q42" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q42, "Q42", - tableType, pw, sqlContext) - if (executeQueriesByChangingConstants) { - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q42_1, "Q42_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q42_2, "Q42_2", - tableType, pw, sqlContext, usePlanCaching) - } - case "Q43" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q43, "Q43", - tableType, pw, sqlContext) - if (executeQueriesByChangingConstants) { - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q43_1, "Q43_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q43_2, "Q43_2", - tableType, pw, sqlContext, usePlanCaching) - } - case "Q49" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q49, "Q49", - tableType, pw, sqlContext) - if (executeQueriesByChangingConstants) { - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q49_1, "Q49_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q49_2, "Q49_2", - tableType, pw, sqlContext, usePlanCaching) - } - case "Q51" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q51, "Q51", - tableType, pw, sqlContext) - if (executeQueriesByChangingConstants) { - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q51_1, "Q51_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q51_2, "Q51_2", - tableType, pw, sqlContext, usePlanCaching) - } - case "Q55" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q55, "Q55", - tableType, pw, sqlContext) - if (executeQueriesByChangingConstants) { - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q55_1, "Q55_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q55_2, "Q55_2", - tableType, pw, sqlContext, usePlanCaching) - } - case "Q56" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q56, "Q56", - tableType, pw, sqlContext) - if (executeQueriesByChangingConstants) { - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q56_1, "Q56_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q56_2, "Q56_2", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q56_3, "Q56_3", - tableType, pw, sqlContext, usePlanCaching) - } + SnappyTestUtils.assertJoin(snc, NWQueries.Q28_2, "Q28_2", + pw, sqlContext) + case "Q30" => SnappyTestUtils.assertJoin(snc, NWQueries.Q30, "Q30", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q30_2, "Q30_1", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q30_2, "Q30_2", + pw, sqlContext) + case "Q61" => SnappyTestUtils.assertQuery(snc, NWQueries.Q61, "Q61", + pw, sqlContext) + case "Q62" => SnappyTestUtils.assertQuery(snc, NWQueries.Q62, "Q62", + pw, sqlContext) + case "Q31" => SnappyTestUtils.assertJoin(snc, NWQueries.Q31, "Q31", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q31_1, "Q31_1", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q31_2, "Q31_2", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q31_3, "Q31_3", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q31_4, "Q31_4", + pw, sqlContext) + case "Q32" => SnappyTestUtils.assertJoin(snc, NWQueries.Q32, "Q32", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q32_1, "Q32_1", + pw, sqlContext) + case "Q33" => SnappyTestUtils.assertJoin(snc, NWQueries.Q33, "Q33", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q33_1, "Q33_1", + pw, sqlContext) + case "Q34" => SnappyTestUtils.assertJoin(snc, NWQueries.Q34, "Q34", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q34_1, "Q34_1", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q34_2, "Q34_2", + pw, sqlContext) + case "Q36" => SnappyTestUtils.assertJoin(snc, NWQueries.Q36, "Q36", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q36_1, "Q36_1", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q36_2, "Q36_2", + pw, sqlContext) + case "Q37" => SnappyTestUtils.assertJoin(snc, NWQueries.Q37, "Q37", + pw, sqlContext) + case "Q38" => SnappyTestUtils.assertJoin(snc, NWQueries.Q38, "Q38", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q38_1, "Q38_1", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q38_2, "Q38_2", + pw, sqlContext) + case "Q39" => SnappyTestUtils.assertJoin(snc, NWQueries.Q39, "Q39", + pw, sqlContext) + case "Q40" => SnappyTestUtils.assertJoin(snc, NWQueries.Q40, "Q40", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q40_1, "Q40_1", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q40_2, "Q40_2", + pw, sqlContext) + case "Q41" => SnappyTestUtils.assertJoin(snc, NWQueries.Q41, "Q41", + pw, sqlContext) + case "Q42" => SnappyTestUtils.assertJoin(snc, NWQueries.Q42, "Q42", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q42_1, "Q42_1", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q42_2, "Q42_2", + pw, sqlContext) + case "Q43" => SnappyTestUtils.assertJoin(snc, NWQueries.Q43, "Q43", + pw, sqlContext) + case "Q51" => SnappyTestUtils.assertJoin(snc, NWQueries.Q51, "Q51", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q51_1, "Q51_1", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q51_2, "Q51_2", + pw, sqlContext) + case "Q52" => SnappyTestUtils.assertJoin(snc, NWQueries.Q52, "Q52", + pw, sqlContext) + case "Q55" => SnappyTestUtils.assertJoin(snc, NWQueries.Q55, "Q55", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q55_1, "Q55_1", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q55_2, "Q55_2", + pw, sqlContext) + case "Q56" => SnappyTestUtils.assertJoin(snc, NWQueries.Q56, "Q56", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q56_1, "Q56_1", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q56_2, "Q56_2", + pw, sqlContext) + SnappyTestUtils.assertJoin(snc, NWQueries.Q56_3, "Q56_3", + pw, sqlContext) + case "Q58" => SnappyTestUtils.assertQuery(snc, NWQueries.Q58, "Q58", + pw, sqlContext) + case "Q59" => SnappyTestUtils.assertQuery(snc, NWQueries.Q59, "Q59", + pw, sqlContext) // scalastyle:off println - case _ => println("OK") + pw.println(s"Execution completed for query ${q._1}") } - } - } - - def executeAndValidateQueriesByChangingConstants(snc: SnappyContext, tableType: String, - pw: PrintWriter, sqlContext: SQLContext): Unit = { - executeQueriesByChangingConstants = true - validateQueries(snc, tableType, pw) - validateQueriesFullResultSet(snc, tableType, pw, sqlContext) - } - - def validateSelectiveQueriesFullResultSet(snc: SnappyContext, tableType: String, pw: - PrintWriter, sqlContext: SQLContext): Unit = { - val usePlanCaching = false; - for (q <- NWQueries.queries) { - q._1 match { - case "Q6" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q6, "Q6", - tableType, pw, sqlContext) - case "Q7" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q7, "Q7", - tableType, pw, sqlContext) - case "Q9" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q9, "Q9", - tableType, pw, sqlContext) - case "Q11" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q11, "Q11", - tableType, pw, sqlContext) - case "Q12" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q12, "Q12", - tableType, pw, sqlContext) - case "Q13" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q13, "Q13", - tableType, pw, sqlContext) - case "Q14" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q14, "Q14", - tableType, pw, sqlContext) - case "Q15" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q15, "Q15", - tableType, pw, sqlContext) - case "Q16" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q16, "Q16", - tableType, pw, sqlContext) - case "Q17" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q17, "Q17", - tableType, pw, sqlContext) - case "Q18" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q18, "Q18", - tableType, pw, sqlContext) - case "Q19" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q19, "Q19", - tableType, pw, sqlContext) - case "Q20" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q20, "Q20", - tableType, pw, sqlContext) - case "Q21" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q21, "Q21", - tableType, pw, sqlContext) - case "Q22" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q22, "Q22", - tableType, pw, sqlContext) - case "Q24" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q24, "Q24", - tableType, pw, sqlContext) - case "Q25" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q25, "Q25", - tableType, pw, sqlContext) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q25_1, "Q25_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q25_2, "Q25_2", - tableType, pw, sqlContext, usePlanCaching) - case "Q26" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q26, "Q26", - tableType, pw, sqlContext) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q26_1, "Q26_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q26_2, "Q26_2", - tableType, pw, sqlContext, usePlanCaching) - case "Q27" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q27, "Q27", - tableType, pw, sqlContext) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q27_1, "Q27_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q27_2, "Q27_2", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q27_3, "Q27_3", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q27_4, "Q27_4", - tableType, pw, sqlContext, usePlanCaching) - case "Q28" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q28, "Q28", - tableType, pw, sqlContext) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q28_1, "Q28_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q28_2, "Q28_2", - tableType, pw, sqlContext, usePlanCaching) - case "Q30" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q30, "Q30", - tableType, pw, sqlContext) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q30_2, "Q30_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q30_2, "Q30_2", - tableType, pw, sqlContext, usePlanCaching) - case "Q61" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q61, "Q61", - tableType, pw, sqlContext) - case "Q62" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q62, "Q62", - tableType, pw, sqlContext) - case "Q31" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q31, "Q31", - tableType, pw, sqlContext) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q31_1, "Q31_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q31_2, "Q31_2", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q31_3, "Q31_3", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q31_4, "Q31_4", - tableType, pw, sqlContext, usePlanCaching) - case "Q32" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q32, "Q32", - tableType, pw, sqlContext) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q32_1, "Q32_1", - tableType, pw, sqlContext, usePlanCaching) - case "Q33" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q33, "Q33", - tableType, pw, sqlContext) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q33_1, "Q33_1", - tableType, pw, sqlContext, usePlanCaching) - case "Q34" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q34, "Q34", - tableType, pw, sqlContext) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q34_1, "Q34_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q34_2, "Q34_2", - tableType, pw, sqlContext, usePlanCaching) - case "Q36" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q36, "Q36", - tableType, pw, sqlContext) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q36_1, "Q36_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q36_2, "Q36_2", - tableType, pw, sqlContext, usePlanCaching) - case "Q37" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q37, "Q37", - tableType, pw, sqlContext) - case "Q38" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q38, "Q38", - tableType, pw, sqlContext) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q38_1, "Q38_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q38_2, "Q38_2", - tableType, pw, sqlContext, usePlanCaching) - case "Q39" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q39, "Q39", - tableType, pw, sqlContext) - case "Q40" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q40, "Q40", - tableType, pw, sqlContext) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q40_1, "Q40_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q40_2, "Q40_2", - tableType, pw, sqlContext, usePlanCaching) - case "Q41" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q41, "Q41", - tableType, pw, sqlContext) - case "Q42" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q42, "Q42", - tableType, pw, sqlContext) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q42_1, "Q42_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q42_2, "Q42_2", - tableType, pw, sqlContext, usePlanCaching) - case "Q43" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q43, "Q43", - tableType, pw, sqlContext) - case "Q51" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q51, "Q51", - tableType, pw, sqlContext) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q51_1, "Q51_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q51_2, "Q51_2", - tableType, pw, sqlContext, usePlanCaching) - case "Q52" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q52, "Q52", - tableType, pw, sqlContext) - case "Q55" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q55, "Q55", - tableType, pw, sqlContext) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q55_1, "Q55_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q55_2, "Q55_2", - tableType, pw, sqlContext, usePlanCaching) - case "Q56" => SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q56, "Q56", - tableType, pw, sqlContext) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q56_1, "Q56_1", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q56_2, "Q56_2", - tableType, pw, sqlContext, usePlanCaching) - SnappyTestUtils.assertJoinFullResultSet(snc, NWQueries.Q56_3, "Q56_3", - tableType, pw, sqlContext, usePlanCaching) - case "Q58" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q58, "Q58", - tableType, pw, sqlContext) - case "Q59" => SnappyTestUtils.assertQueryFullResultSet(snc, NWQueries.Q59, "Q59", - tableType, pw, sqlContext) - // scalastyle:off println - case _ => println("OK") + if (hasValidationFailed) { + failedQueries = SnappyTestUtils.addToFailedQueryList(failedQueries, q._1) } } + return failedQueries; } def createAndLoadPartitionedTables(snc: SnappyContext, @@ -1069,5 +798,4 @@ object NWTestUtil { println("territories table dropped successfully."); // scalastyle:on println } - } diff --git a/dtests/src/test/scala/io/snappydata/hydra/northwind/ValidateNWQueriesApp.scala b/dtests/src/test/scala/io/snappydata/hydra/northwind/ValidateNWQueriesApp.scala index ba6204e85a..faa0333721 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/northwind/ValidateNWQueriesApp.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/northwind/ValidateNWQueriesApp.scala @@ -18,16 +18,17 @@ package io.snappydata.hydra.northwind import java.io.{File, FileOutputStream, PrintWriter} +import io.snappydata.hydra.SnappyTestUtils + import org.apache.spark.sql.{SQLContext, SnappyContext} import org.apache.spark.{SparkConf, SparkContext} object ValidateNWQueriesApp { def main(args: Array[String]) { - val connectionURL = args(args.length - 1) + val threadID = Thread.currentThread().getId val conf = new SparkConf(). - setAppName("ValidateNWQueries Application_" + System.currentTimeMillis()). - set("snappydata.connection", connectionURL) + setAppName("ValidateNWQueriesApplication_" + threadID + "_" + System.currentTimeMillis()) val sc = SparkContext.getOrCreate(conf) val sqlContext = SQLContext.getOrCreate(sc) val snc = SnappyContext(sc) @@ -39,38 +40,42 @@ object ValidateNWQueriesApp { val tableType = args(1) val fullResultSetValidation: Boolean = args(2).toBoolean val numRowsValidation: Boolean = args(4).toBoolean + SnappyTestUtils.validateFullResultSet = fullResultSetValidation + SnappyTestUtils.numRowsValidation = numRowsValidation + SnappyTestUtils.tableType = tableType val isSmokeRun: Boolean = args(3).toBoolean - val threadID = Thread.currentThread().getId + + def getCurrentDirectory = new java.io.File(".").getCanonicalPath + val outputFile = "ValidateNWQueriesApp_thread_" + threadID + "_" + System.currentTimeMillis + ".out" val pw = new PrintWriter(new FileOutputStream(new File(outputFile), true)); - if (numRowsValidation) { - // scalastyle:off println - pw.println(s"Validate ${tableType} tables Queries Test started at : " + System - .currentTimeMillis) - pw.println(s"dataFilesLocation : ${dataFilesLocation}") - NWTestUtil.validateQueries(snc, tableType, pw) - pw.println(s"Validate ${tableType} tables Queries Test completed successfully at : " + - System.currentTimeMillis) + var failedQueries = "" + // scalastyle:off println + val startTime = System.currentTimeMillis() + pw.println(s"${SnappyTestUtils.logTime} ValidateQueries for ${tableType} tables " + + s"started ..") + if (isSmokeRun) { + NWTestUtil.validateSelectiveQueriesFullResultSet(snc, tableType, pw, sqlContext) + } + else { + failedQueries = NWTestUtil.validateQueries(snc, tableType, pw, sqlContext) } - if (fullResultSetValidation) { - pw.println(s"createAndLoadSparkTables Test started at : " + System.currentTimeMillis) - NWTestUtil.createAndLoadSparkTables(sqlContext) - println(s"createAndLoadSparkTables Test completed successfully at : " + System - .currentTimeMillis) - pw.println(s"createAndLoadSparkTables Test completed successfully at : " + System - .currentTimeMillis) - pw.println(s"ValidateQueriesFullResultSet for ${tableType} tables Queries Test started at :" + - s" " + System.currentTimeMillis) - if (isSmokeRun) { - NWTestUtil.validateSelectiveQueriesFullResultSet(snc, tableType, pw, sqlContext) - } - else { - NWTestUtil.validateQueriesFullResultSet(snc, tableType, pw, sqlContext) - } - pw.println(s"validateQueriesFullResultSet ${tableType} tables Queries Test completed " + - s"successfully at : " + System.currentTimeMillis) + val finishTime = System.currentTimeMillis() + val totalTime = (finishTime -startTime)/1000 + if (!failedQueries.isEmpty) { + println(s"Validation failed for ${tableType} tables for queries ${failedQueries}. See " + + s"${getCurrentDirectory}/${outputFile}") + pw.println(s"${SnappyTestUtils.logTime} Total execution took ${totalTime} " + + s"seconds.") + pw.println(s"${SnappyTestUtils.logTime} Validation failed for ${tableType} " + + s"tables for queries ${failedQueries}. ") + pw.close() + throw new Exception(s"Validation task failed for ${tableType} tables. See " + + s"${getCurrentDirectory}/${outputFile}") } + pw.println(s"${SnappyTestUtils.logTime} ValidateQueries for ${tableType} tables " + + s"completed successfully in ${totalTime} seconds ") pw.close() } } diff --git a/dtests/src/test/scala/io/snappydata/hydra/northwind/ValidateNWQueriesJob.scala b/dtests/src/test/scala/io/snappydata/hydra/northwind/ValidateNWQueriesJob.scala index fbf51f38b3..9fcbdf8229 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/northwind/ValidateNWQueriesJob.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/northwind/ValidateNWQueriesJob.scala @@ -18,11 +18,12 @@ package io.snappydata.hydra.northwind import java.io.{File, FileOutputStream, PrintWriter} +import util.TestException import com.typesafe.config.Config -import io.snappydata.hydra.northwind +import io.snappydata.hydra.{SnappyTestUtils, northwind} + import org.apache.spark.SparkContext import org.apache.spark.sql._ - import scala.util.{Failure, Success, Try} class ValidateNWQueriesJob extends SnappySQLJob { @@ -37,46 +38,51 @@ class ValidateNWQueriesJob extends SnappySQLJob { val isSmokeRun: Boolean = jobConfig.getString("isSmokeRun").toBoolean val fullResultSetValidation: Boolean = jobConfig.getString("fullResultSetValidation").toBoolean val numRowsValidation: Boolean = jobConfig.getString("numRowsValidation").toBoolean + SnappyTestUtils.validateFullResultSet = fullResultSetValidation + SnappyTestUtils.numRowsValidation = numRowsValidation + SnappyTestUtils.tableType = tableType val sc = SparkContext.getOrCreate() val sqlContext = SQLContext.getOrCreate(sc) Try { + var failedQueries: String = ""; snc.sql("set spark.sql.shuffle.partitions=23") val dataFilesLocation = jobConfig.getString("dataFilesLocation") snc.setConf("dataFilesLocation", dataFilesLocation) northwind.NWQueries.snc = snc NWQueries.dataFilesLocation = dataFilesLocation - if (numRowsValidation) { - // scalastyle:off println - pw.println(s"Validate ${tableType} tables Queries Test started at : " + System - .currentTimeMillis) - NWTestUtil.validateQueries(snc, tableType, pw) - pw.println(s"Validate ${tableType} tables Queries Test completed successfully at : " + - System.currentTimeMillis) + // scalastyle:off println + val startTime = System.currentTimeMillis() + pw.println(s"${SnappyTestUtils.logTime} ValidateQueries for ${tableType} tables" + + s" started ..") + if (isSmokeRun) { + failedQueries = NWTestUtil.validateSelectiveQueriesFullResultSet(snc, tableType, pw, + sqlContext) + } + else { + failedQueries = NWTestUtil.validateQueries(snc, tableType, pw, sqlContext) } - if (fullResultSetValidation) { - pw.println(s"createAndLoadSparkTables Test started at : " + System.currentTimeMillis) - NWTestUtil.createAndLoadSparkTables(sqlContext) - println(s"createAndLoadSparkTables Test completed successfully at : " + System - .currentTimeMillis) - pw.println(s"createAndLoadSparkTables Test completed successfully at : " + System - .currentTimeMillis) - pw.println(s"ValidateQueriesFullResultSet for ${tableType} tables Queries Test started at" + - s" : " + System.currentTimeMillis) - if (isSmokeRun) { - NWTestUtil.validateSelectiveQueriesFullResultSet(snc, tableType, pw, sqlContext) - } - else { - NWTestUtil.validateQueriesFullResultSet(snc, tableType, pw, sqlContext) - } - pw.println(s"validateQueriesFullResultSet ${tableType} tables Queries Test completed " + - s"successfully at : " + System.currentTimeMillis) + val finishTime = System.currentTimeMillis() + val totalTime = (finishTime -startTime)/1000 + pw.println(s"${SnappyTestUtils.logTime} Total execution took ${totalTime} " + + s"seconds.") + if (!failedQueries.isEmpty) { + println(s"Validation failed for ${tableType} tables for queries ${failedQueries}. " + + s"See ${getCurrentDirectory}/${outputFile}") + pw.println(s"${SnappyTestUtils.logTime} Validation failed for ${tableType} " + + s"tables for queries ${failedQueries}. ") + pw.close() + throw new TestException(s"Validation task failed for ${tableType}. " + + s"See ${getCurrentDirectory}/${outputFile}") } + pw.println(s"ValidateQueries for ${tableType} tables completed successfully.") pw.close() } match { case Success(v) => pw.close() - s"See ${getCurrentDirectory}/${outputFile}" - case Failure(e) => pw.close(); - throw e; + s"Validation passed. See ${getCurrentDirectory}/${outputFile}" + throw new Exception() + case Failure(e) => + pw.close(); + throw new TestException(s"Validation failed. See ${getCurrentDirectory}/${outputFile}"); } } diff --git a/dtests/src/test/scala/io/snappydata/hydra/northwind/ValidateNWQueriesWithChangingConstantsApp.scala b/dtests/src/test/scala/io/snappydata/hydra/northwind/ValidateNWQueriesWithChangingConstantsApp.scala index c9e823f711..b97cf0f8b6 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/northwind/ValidateNWQueriesWithChangingConstantsApp.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/northwind/ValidateNWQueriesWithChangingConstantsApp.scala @@ -18,16 +18,16 @@ package io.snappydata.hydra.northwind import java.io.{File, FileOutputStream, PrintWriter} +import io.snappydata.hydra.SnappyTestUtils + import org.apache.spark.sql.{SQLContext, SnappyContext} import org.apache.spark.{SparkConf, SparkContext} object ValidateNWQueriesWithChangingConstantsApp { def main(args: Array[String]) { - val connectionURL = args(args.length - 1) val conf = new SparkConf(). - setAppName("ValidateNWQueries Application_" + System.currentTimeMillis()). - set("snappydata.connection", connectionURL) + setAppName("ValidateNWQueries Application_" + System.currentTimeMillis()) val sc = SparkContext.getOrCreate(conf) val sqlContext = SQLContext.getOrCreate(sc) val snc = SnappyContext(sc) @@ -38,21 +38,41 @@ object ValidateNWQueriesWithChangingConstantsApp { NWQueries.dataFilesLocation = dataFilesLocation val tableType = args(1) val threadID = Thread.currentThread().getId + SnappyTestUtils.validateFullResultSet = true + SnappyTestUtils.numRowsValidation = true + SnappyTestUtils.tableType = tableType + + def getCurrentDirectory = new java.io.File(".").getCanonicalPath + val outputFile = "ValidateNWQueriesApp_thread_" + threadID + "_" + System.currentTimeMillis + ".out" val pw = new PrintWriter(new FileOutputStream(new File(outputFile), true)); // scalastyle:off println - pw.println(s"createAndLoadSparkTables Test started at : " + System.currentTimeMillis) - NWTestUtil.createAndLoadSparkTables(sqlContext) - println(s"createAndLoadSparkTables Test completed successfully at : " + System - .currentTimeMillis) - pw.println(s"createAndLoadSparkTables Test completed successfully at : " + System - .currentTimeMillis) - pw.println(s"ValidateQueries for ${tableType} tables Queries Test started at :" + - s" " + System.currentTimeMillis) - NWTestUtil.executeAndValidateQueriesByChangingConstants(snc, tableType, pw, sqlContext) - pw.println(s"validateQueries for ${tableType} tables Queries Test completed " + - s"successfully at : " + System.currentTimeMillis) + var startTime = System.currentTimeMillis() + pw.println(s"${SnappyTestUtils.logTime} createAndLoadSparkTables started.. ") + NWTestUtil.createAndLoadSparkTables(sqlContext) + var finishTime = System.currentTimeMillis() + var totalTime = (finishTime - startTime) / 1000 + pw.println(s"${SnappyTestUtils.logTime} createAndLoadSparkTables completed successfully in " + + s"$totalTime secs.") + pw.flush() + pw.println(s"${SnappyTestUtils.logTime} Validation for ${tableType} tables queries started..") + startTime = System.currentTimeMillis() + val failedQueries: String = NWTestUtil.executeAndValidateQueriesByChangingConstants(snc, + tableType, pw, sqlContext) + finishTime = System.currentTimeMillis() + totalTime = (finishTime - startTime) / 1000 + if (!failedQueries.isEmpty) { + println(s"Validation failed for ${tableType} tables for queries ${failedQueries}. See " + + s"${getCurrentDirectory}/${outputFile}") + pw.println(s"${SnappyTestUtils.logTime} Total execution took ${totalTime} seconds.") + pw.println(s"${SnappyTestUtils.logTime} Validation failed for ${tableType} tables for " + + s"queries ${failedQueries}. ") + pw.close() + throw new Exception(s"Validation failed for ${tableType} tables. See " + + s"${getCurrentDirectory}/${outputFile}") + } + pw.println(s"ValidateQueries for $tableType tables completed successfully in $totalTime secs.") pw.close() } } diff --git a/dtests/src/test/scala/io/snappydata/hydra/northwind/ValidateNWQueriesWithChangingConstantsJob.scala b/dtests/src/test/scala/io/snappydata/hydra/northwind/ValidateNWQueriesWithChangingConstantsJob.scala index d6e7676290..262cd78638 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/northwind/ValidateNWQueriesWithChangingConstantsJob.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/northwind/ValidateNWQueriesWithChangingConstantsJob.scala @@ -21,10 +21,11 @@ import java.io.{File, FileOutputStream, PrintWriter} import scala.util.{Failure, Success, Try} import com.typesafe.config.Config -import io.snappydata.hydra.northwind +import io.snappydata.hydra.{SnappyTestUtils, northwind} import org.apache.spark.SparkContext import org.apache.spark.sql._ +import _root_.util.TestException class ValidateNWQueriesWithChangingConstantsJob extends SnappySQLJob { override def runSnappyJob(snappySession: SnappySession, jobConfig: Config): Any = { @@ -37,6 +38,9 @@ class ValidateNWQueriesWithChangingConstantsJob extends SnappySQLJob { val pw = new PrintWriter(new FileOutputStream(new File(outputFile), true)); val sc = SparkContext.getOrCreate() val sqlContext = SQLContext.getOrCreate(sc) + SnappyTestUtils.validateFullResultSet = true + SnappyTestUtils.numRowsValidation = true + SnappyTestUtils.tableType = tableType Try { snc.sql("set spark.sql.shuffle.partitions=23") val dataFilesLocation = jobConfig.getString("dataFilesLocation") @@ -44,23 +48,38 @@ class ValidateNWQueriesWithChangingConstantsJob extends SnappySQLJob { northwind.NWQueries.snc = snc NWQueries.dataFilesLocation = dataFilesLocation // scalastyle:off println - pw.println(s"createAndLoadSparkTables Test started at : " + System.currentTimeMillis) + var startTime = System.currentTimeMillis() + pw.println(s"${SnappyTestUtils.logTime} createAndLoadSparkTables started.") NWTestUtil.createAndLoadSparkTables(sqlContext) - println(s"createAndLoadSparkTables Test completed successfully at : " + System - .currentTimeMillis) - pw.println(s"createAndLoadSparkTables Test completed successfully at : " + System - .currentTimeMillis) - pw.println(s"ValidateQueriesFullResultSet for ${tableType} tables Queries Test started at" + - s" : " + System.currentTimeMillis) - NWTestUtil.executeAndValidateQueriesByChangingConstants(snc, tableType, pw, sqlContext) - pw.println(s"validateQueries for ${tableType} tables Queries Test completed " + - s"successfully at : " + System.currentTimeMillis) + var finishTime = System.currentTimeMillis() + var totalTime = (finishTime - startTime) / 1000 + pw.println(s"${SnappyTestUtils.logTime} createAndLoadSparkTables completed successfully in " + + s"$totalTime secs.") + pw.flush() + pw.println(s"${SnappyTestUtils.logTime} Validation for ${tableType} tables queries started..") + startTime = System.currentTimeMillis() + val failedQueries: String = NWTestUtil.executeAndValidateQueriesByChangingConstants(snc, + tableType, pw, sqlContext) + finishTime = System.currentTimeMillis() + totalTime = (finishTime - startTime) / 1000 + if (!failedQueries.isEmpty) { + println(s"Validation failed for ${tableType} tables for queries ${failedQueries}. " + + s"See ${getCurrentDirectory}/${outputFile}") + pw.println(s"${SnappyTestUtils.logTime} Total execution took ${totalTime} seconds.") + pw.println(s"${SnappyTestUtils.logTime} Validation failed for ${tableType} tables for " + + s"queries ${failedQueries}. ") + pw.close() + throw new TestException(s"Validation failed for ${tableType}. " + + s"See ${getCurrentDirectory}/${outputFile}") + } + pw.println(s"${SnappyTestUtils.logTime} ValidateQueries for $tableType tables completed " + + s"successfully in $totalTime secs.") pw.close() } match { case Success(v) => pw.close() s"See ${getCurrentDirectory}/${outputFile}" case Failure(e) => pw.close(); - throw e; + throw new TestException(s"Validation failed. See ${getCurrentDirectory}/${outputFile}"); } } diff --git a/dtests/src/test/scala/io/snappydata/hydra/security/CreateAndLoadTablesSparkApp.scala b/dtests/src/test/scala/io/snappydata/hydra/security/CreateAndLoadTablesSparkApp.scala index fe68fc4175..f8e5c34db7 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/security/CreateAndLoadTablesSparkApp.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/security/CreateAndLoadTablesSparkApp.scala @@ -28,11 +28,8 @@ object CreateAndLoadTablesSparkApp { def main(args: Array[String]) { // scalastyle:off println Thread.sleep(60000L) - val connectionURL = args(args.length - 1) - println("The connection url is " + connectionURL) val conf = new SparkConf(). - setAppName("CreateAndLoadNWTablesSpark Application"). - set("snappydata.connection", connectionURL) + setAppName("CreateAndLoadNWTablesSpark Application") val sc = SparkContext.getOrCreate(conf) val snc = SnappyContext(sc) diff --git a/dtests/src/test/scala/io/snappydata/hydra/snapshotIsolation/SnapshotIsolationTestUtils.scala b/dtests/src/test/scala/io/snappydata/hydra/snapshotIsolation/SnapshotIsolationTestUtils.scala index 05c6fdab64..9d1710e7e3 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/snapshotIsolation/SnapshotIsolationTestUtils.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/snapshotIsolation/SnapshotIsolationTestUtils.scala @@ -20,8 +20,6 @@ package io.snappydata.hydra.snapshotIsolation import java.io.{File, PrintWriter} import java.util -import io.snappydata.hydra.TestUtil - import org.apache.spark.sql.{Row, DataFrame, SnappyContext} class SnapshotIsolationTestUtils { @@ -35,9 +33,10 @@ class SnapshotIsolationTestUtils { val rowList = snc.sql(sqlString).collect() try { verifyDuplicateRows(rowList, pw) - //TestUtil.compareFiles(snappyFile, newDFFile, pw, false) + // TestUtil.compareFiles(snappyFile, newDFFile, pw, false) } catch { case ex: Exception => { + // scalastyle:off println pw.println(s"Verification failed for ${queryNum} with following exception:\n") ex.printStackTrace(pw) } @@ -48,8 +47,9 @@ class SnapshotIsolationTestUtils { def verifyDuplicateRows(rowList: Array[Row], pw: PrintWriter): Unit = { val numRows = rowList.length pw.println(s"Num rows in resultSet are ${numRows}.") - if (numRows == 0) + if (numRows == 0) { pw.println(s"There are no duplicate rows in resultSet. ") + } else { pw.println(s"Below duplicate rows found in resultSet: ") for (row <- rowList) diff --git a/dtests/src/test/scala/io/snappydata/hydra/spva/SPVATestUtil.scala b/dtests/src/test/scala/io/snappydata/hydra/spva/SPVATestUtil.scala index 9d9f97d59b..2af57e23a4 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/spva/SPVATestUtil.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/spva/SPVATestUtil.scala @@ -29,34 +29,26 @@ object SPVATestUtil { def createAndLoadReplicatedTables(snc: SnappyContext): Unit = { snc.sql(SPVAQueries.patients_table) - SPVAQueries.patients(snc).write.insertInto("patient") snc.sql(SPVAQueries.encounters_table) - SPVAQueries.encounters(snc).write.insertInto("encounters") snc.sql(SPVAQueries.allergies_table) - SPVAQueries.allergies(snc).write.insertInto("allergies") snc.sql(SPVAQueries.careplans_table) - SPVAQueries.careplans(snc).write.insertInto("careplans") snc.sql(SPVAQueries.conditions_table) - SPVAQueries.conditions(snc).write.insertInto("conditions") snc.sql(SPVAQueries.imaging_studies_table) - SPVAQueries.imaging_studies(snc).write.insertInto("imaging_studies") snc.sql(SPVAQueries.immunizations_table) - SPVAQueries.immunizations(snc).write.insertInto("immunizations") snc.sql(SPVAQueries.medications_table) - SPVAQueries.medications(snc).write.insertInto("medications") snc.sql(SPVAQueries.observations_table) - SPVAQueries.observations(snc).write.insertInto("observations") snc.sql(SPVAQueries.procedures_table) - SPVAQueries.procedures(snc).write.insertInto("procedures") + + loadTables(snc) } def createAndLoadPartitionedTables(snc: SnappyContext): Unit = { @@ -91,16 +83,7 @@ object SPVATestUtil { " colocate_with 'PATIENTS', buckets '12', redundancy '1', PERSISTENT 'sync', " + " EVICTION_BY 'LRUHEAPPERCENT')") - SPVAQueries.patients(snc).write.insertInto("patients") - SPVAQueries.encounters(snc).write.insertInto("encounters") - SPVAQueries.allergies(snc).write.insertInto("allergies") - SPVAQueries.careplans(snc).write.insertInto("careplans") - SPVAQueries.conditions(snc).write.insertInto("conditions") - SPVAQueries.imaging_studies(snc).write.insertInto("imaging_studies") - SPVAQueries.immunizations(snc).write.insertInto("immunizations") - SPVAQueries.medications(snc).write.insertInto("medications") - SPVAQueries.observations(snc).write.insertInto("observations") - SPVAQueries.procedures(snc).write.insertInto("procedures") + loadTables(snc) } def createAndLoadColumnTables(snc: SnappyContext): Unit = { @@ -134,7 +117,10 @@ object SPVATestUtil { snc.sql(SPVAQueries.procedures_table + " using column options(PARTITION_BY 'PATIENT', " + " colocate_with 'PATIENTS', buckets '12', redundancy '1', PERSISTENT 'sync', " + " EVICTION_BY 'LRUHEAPPERCENT')") + loadTables(snc) + } + def loadTables(snc: SnappyContext): Unit = { SPVAQueries.patients(snc).write.insertInto("patients") SPVAQueries.encounters(snc).write.insertInto("encounters") SPVAQueries.allergies(snc).write.insertInto("allergies") @@ -148,131 +134,153 @@ object SPVATestUtil { } def validateQueriesFullResultSet(snc: SnappyContext, tableType: String, pw: PrintWriter, - sqlContext: SQLContext): Unit = { + sqlContext: SQLContext): String = { + var failedQueries = "" + SnappyTestUtils.tableType = tableType for (q <- SPVAQueries.queries) { + var queryExecuted = true; + var validationFailed = false; + if (SnappyTestUtils.validateFullResultSet) { + // scalastyle:off println + pw.println(s"createAndLoadSparkTables started ...") + val startTime = System.currentTimeMillis + createAndLoadSparkTables(sqlContext) + val finishTime = System.currentTimeMillis() + pw.println(s"createAndLoadSparkTables completed successfully in : " + ((finishTime - + startTime)/1000) + " seconds") + } q._1 match { - case "Q1_1" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q1_1, "Q1_1", tableType, - pw, sqlContext) - case "Q1_2" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q1_2, "Q1_2", tableType, + case "Q1_1" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q1_1, + "Q1_1", pw, sqlContext) + case "Q1_2" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q1_2, "Q1_2", pw, sqlContext) - case "Q1_3" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q1_3, "Q1_3", tableType, + case "Q1_3" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q1_3, "Q1_3", pw, sqlContext) - case "Q1_4" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q1_4, "Q1_4", tableType, + case "Q1_4" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q1_4, "Q1_4", pw, sqlContext) - case "Q1_5" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q1_5, "Q1_5", tableType, + case "Q1_5" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q1_5, "Q1_5", pw, sqlContext) - case "Q1_6" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q1_6, "Q1_6", tableType, + case "Q1_6" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q1_6, "Q1_6", pw, sqlContext) - case "Q1_7" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q1_7, "Q1_7", tableType, + case "Q1_7" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q1_7, "Q1_7", pw, sqlContext) - case "Q1_8" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q1_8, "Q1_8", tableType, + case "Q1_8" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q1_8, "Q1_8", pw, sqlContext) - case "Q1_9" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q1_9, "Q1_9", tableType, + case "Q1_9" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q1_9, "Q1_9", pw, sqlContext) - case "Q2_1" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q2_1, "Q2_1", tableType, + case "Q2_1" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q2_1, "Q2_1", pw, sqlContext) - case "Q2_2" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q2_2, "Q2_2", tableType, + case "Q2_2" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q2_2, "Q2_2", pw, sqlContext) - case "Q2_3" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q2_3, "Q2_3", tableType, + case "Q2_3" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q2_3, "Q2_3", pw, sqlContext) - case "Q2_4" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q2_4, "Q2_4", tableType, + case "Q2_4" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q2_4, "Q2_4", pw, sqlContext) - case "Q2_5" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q2_5, "Q2_5", tableType, + case "Q2_5" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q2_5, "Q2_5", pw, sqlContext) - case "Q2_6" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q2_6, "Q2_6", tableType, + case "Q2_6" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q2_6, "Q2_6", pw, sqlContext) - case "Q2_7" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q2_7, "Q2_7", tableType, + case "Q2_7" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q2_7, "Q2_7", pw, sqlContext) - case "Q2_8" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q2_8, "Q2_8", tableType, + case "Q2_8" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q2_8, "Q2_8", pw, sqlContext) - case "Q2_9" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q2_9, "Q2_9", tableType, + case "Q2_9" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q2_9, "Q2_9", pw, sqlContext) - case "Q2_10" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q2_10, "Q2_10", tableType, + case "Q2_10" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q2_10, + "Q2_10", pw, sqlContext) + case "Q2_11" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q2_11, + "Q2_11", pw, sqlContext) + case "Q2_12" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q2_12, + "Q2_12", pw, sqlContext) + case "Q2_13" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q2_13, + "Q2_13", pw, sqlContext) + case "Q3_1_2" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q3_1_2, + "Q3_1_2", pw, sqlContext) + case "Q3_2_2" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q3_2_2, + "Q3_2_2", pw, sqlContext) + case "Q3_3" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q3_3, + "Q3_3", pw, sqlContext) + case "Q3_4_1" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q3_4_1, + "Q3_4_1", pw, sqlContext) + case "Q3_4_2" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q3_4_2, + "Q3_4_2", pw, sqlContext) + case "Q3_4_3" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q3_4_3, + "Q3_4_3", pw, sqlContext) + case "Q3_5" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q3_5, "Q3_5", pw, sqlContext) - case "Q2_11" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q2_11, "Q2_11", tableType, + case "Q3_6" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q3_6, "Q3_6", pw, sqlContext) - case "Q2_12" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q2_12, "Q2_12", tableType, + case "Q3_7_1" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q3_7_1, + "Q3_7_1", pw, sqlContext) + case "Q3_7_2" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q3_7_2, + "Q3_7_2", pw, sqlContext) + case "Q3_7_3" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q3_7_3, + "Q3_7_3", pw, sqlContext) + case "Q3_7_4" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q3_7_4, + "Q3_7_4", pw, sqlContext) + case "Q3_7_5" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q3_7_5, + "Q3_7_5", pw, sqlContext) + case "Q3_8" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q3_8, "Q3_8", pw, sqlContext) - case "Q2_13" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q2_13, "Q2_13", tableType, + case "Q3_9" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q3_9, "Q3_9", pw, sqlContext) - case "Q3_1_2" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q3_1_2, "Q3_1_2", tableType, + case "Q3_10" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q3_10, + "Q3_10", pw, sqlContext) + case "Q3_11" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q3_11, + "Q3_11", pw, sqlContext) + case "Q3_12" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q3_12, + "Q3_12", pw, sqlContext) + case "Q3_13" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q3_13, + "Q3_13", pw, sqlContext) + case "Q3_14" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q3_14, + "Q3_14", pw, sqlContext) + case "Q4_1" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q4_1, "Q4_1", pw, sqlContext) - case "Q3_2_2" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q3_2_2, "Q3_2_2", tableType, + case "Q4_2" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q4_2, "Q4_2", pw, sqlContext) - case "Q3_3" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q3_3, "Q3_3", tableType, + case "Q5_1_1" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q5_1_1, + "Q5_1_1", pw, sqlContext) + case "Q5_1_2" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q5_1_2, + "Q5_1_2", pw, sqlContext) + case "Q5_2" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q5_2, "Q5_2", pw, sqlContext) - case "Q3_4_1" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q3_4_1, "Q3_4_1", tableType, + case "Q6_1" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q6_1, "Q6_1", pw, sqlContext) - case "Q3_4_2" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q3_4_2, "Q3_4_2", tableType, + case "Q6_2" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q6_2, "Q6_2", pw, sqlContext) - case "Q3_4_3" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q3_4_3, "Q3_4_3", tableType, + case "Q7_1" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q7_1, "Q7_1", pw, sqlContext) - case "Q3_5" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q3_5, "Q3_5", tableType, + case "Q7_2" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q7_2, "Q7_2", pw, sqlContext) - case "Q3_6" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q3_6, "Q3_6", tableType, + case "Q8_1" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q8_1, "Q8_1", pw, sqlContext) - case "Q3_7_1" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q3_7_1, "Q3_7_1", tableType, + case "Q8_2" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q8_2, "Q8_2", pw, sqlContext) - case "Q3_7_2" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q3_7_2, "Q3_7_2", tableType, + case "Q9_1" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q9_1, "Q9_1", pw, sqlContext) - case "Q3_7_3" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q3_7_3, "Q3_7_3", tableType, + case "Q9_2" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q9_2, "Q9_2", pw, sqlContext) - case "Q3_7_4" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q3_7_4, "Q3_7_4", tableType, + case "Q10" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q10, "Q10", pw, sqlContext) - case "Q3_7_5" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q3_7_5, "Q3_7_5", tableType, + case "Q11_1" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q11_1, + "Q11_1", pw, sqlContext) + case "Q11_2" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q11_2, + "Q11_2", pw, sqlContext) + case "Q12" => validationFailed = SnappyTestUtils.assertQuery(snc, SPVAQueries.Q12, "Q12", pw, sqlContext) - case "Q3_8" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q3_8, "Q3_8", tableType, - pw, sqlContext) - case "Q3_9" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q3_9, "Q3_9", tableType, - pw, sqlContext) - case "Q3_10" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q3_10, "Q3_10", tableType, - pw, sqlContext) - case "Q3_11" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q3_11, "Q3_11", tableType, - pw, sqlContext) - case "Q3_12" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q3_12, "Q3_12", tableType, - pw, sqlContext) - case "Q3_13" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q3_13, "Q3_13", tableType, - pw, sqlContext) - case "Q3_14" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q3_14, "Q3_14", tableType, - pw, sqlContext) - case "Q4_1" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q4_1, "Q4_1", tableType, - pw, sqlContext) - case "Q4_2" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q4_2, "Q4_2", tableType, - pw, sqlContext) - case "Q5_1_1" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q5_1_1, "Q5_1_1", tableType, - pw, sqlContext) - case "Q5_1_2" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q5_1_2, "Q5_1_2", tableType, - pw, sqlContext) - case "Q5_2" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q5_2, "Q5_2", tableType, - pw, sqlContext) - case "Q6_1" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q6_1, "Q6_1", tableType, - pw, sqlContext) - case "Q6_2" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q6_2, "Q6_2", tableType, - pw, sqlContext) - case "Q7_1" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q7_1, "Q7_1", tableType, - pw, sqlContext) - case "Q7_2" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q7_2, "Q7_2", tableType, - pw, sqlContext) - case "Q8_1" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q8_1, "Q8_1", tableType, - pw, sqlContext) - case "Q8_2" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q8_2, "Q8_2", tableType, - pw, sqlContext) - case "Q9_1" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q9_1, "Q9_1", tableType, - pw, sqlContext) - case "Q9_2" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q9_2, "Q9_2", tableType, - pw, sqlContext) - case "Q10" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q10, "Q10", - tableType, pw, sqlContext) - case "Q11_1" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q11_1, "Q11_1", - tableType, pw, sqlContext) - case "Q11_2" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q11_2, "Q11_2", tableType, - pw, sqlContext) - case "Q12" => SnappyTestUtils.assertQueryFullResultSet(snc, SPVAQueries.Q12, "Q12", - tableType, pw, sqlContext) // scalastyle:off println - case _ => println("OK") + case _ => // scalastyle:off println + pw.println(s"Query ${q._1} will not be executed.") + queryExecuted = false + } + if (queryExecuted) { + pw.println(s"Execution completed for query ${q._1}") + } + if (validationFailed) { + failedQueries = SnappyTestUtils.addToFailedQueryList(failedQueries, q._1) } } + return failedQueries; } def dropTables(snc: SnappyContext): Unit = { diff --git a/dtests/src/test/scala/io/snappydata/hydra/spva/ValidateSPVAQueriesJob.scala b/dtests/src/test/scala/io/snappydata/hydra/spva/ValidateSPVAQueriesJob.scala index 38e1b09b91..7ae29fcdeb 100644 --- a/dtests/src/test/scala/io/snappydata/hydra/spva/ValidateSPVAQueriesJob.scala +++ b/dtests/src/test/scala/io/snappydata/hydra/spva/ValidateSPVAQueriesJob.scala @@ -19,10 +19,10 @@ package io.snappydata.hydra.spva import java.io.{File, FileOutputStream, PrintWriter} import com.typesafe.config.Config -import io.snappydata.hydra.spva +import io.snappydata.hydra.{SnappyTestUtils, spva} + import org.apache.spark.SparkContext import org.apache.spark.sql._ - import scala.util.{Failure, Success, Try} @@ -31,24 +31,20 @@ object ValidateSPVAQueriesJob extends SnappySQLJob { val snc = snappySession.sqlContext def getCurrentDirectory = new java.io.File(".").getCanonicalPath - + // scalastyle:off println val tableType = jobConfig.getString("tableType") val outputFile = "ValidateSPVAQueries_" + tableType + "_" + jobConfig.getString("logFileName") val pw = new PrintWriter(new FileOutputStream(new File(outputFile), true)); val sc = SparkContext.getOrCreate() val sqlContext = SQLContext.getOrCreate(sc) + SnappyTestUtils.validateFullResultSet = true + SnappyTestUtils.numRowsValidation = false Try { val dataFilesLocation = jobConfig.getString("dataFilesLocation") snc.setConf("dataFilesLocation", dataFilesLocation) spva.SPVAQueries.snc = snc SPVAQueries.dataFilesLocation = dataFilesLocation - pw.println(s"createAndLoadSparkTables Test started at : " + System.currentTimeMillis) - // sqlContext.sql("CREATE SCHEMA IF NOT EXISTS SPD") - SPVATestUtil.createAndLoadSparkTables(sqlContext) - println(s"createAndLoadSparkTables Test completed successfully at : " + System - .currentTimeMillis) - pw.println(s"createAndLoadSparkTables Test completed successfully at : " + System - .currentTimeMillis) + pw.println(s"ValidateQueriesFullResultSet for ${tableType} tables Queries Test started at" + s" : " + System.currentTimeMillis) SPVATestUtil.validateQueriesFullResultSet(snc, tableType, pw, sqlContext) diff --git a/dtests/src/test/scala/io/snappydata/hydra/testEndToEndValidation/ValidateFailureScenarioApp.scala b/dtests/src/test/scala/io/snappydata/hydra/testEndToEndValidation/ValidateFailureScenarioApp.scala new file mode 100644 index 0000000000..ff778b5498 --- /dev/null +++ b/dtests/src/test/scala/io/snappydata/hydra/testEndToEndValidation/ValidateFailureScenarioApp.scala @@ -0,0 +1,38 @@ + +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package io.snappydata.hydra.testEndToEndValidation + +import java.io.{File, FileOutputStream, PrintWriter} + +import org.apache.spark.sql._ +import org.apache.spark.{SparkConf, SparkContext} + +object ValidateFailureScenarioApp { + def main(args: Array[String]) { + val conf = new SparkConf(). + setAppName("TestHangInSparkApp_" + System.currentTimeMillis()) + val sc = SparkContext.getOrCreate(conf) + val snc = SnappyContext(sc) + val pw: PrintWriter + = new PrintWriter(new FileOutputStream(new File("testAppFailure.out")), true) + // scalastyle:off println + pw.println("Throwing test exception...") + pw.flush() + throw new Exception("Throwing test exception...") + } +} diff --git a/dtests/src/test/scala/io/snappydata/hydra/testEndToEndValidation/ValidateFailureScenarioJob.scala b/dtests/src/test/scala/io/snappydata/hydra/testEndToEndValidation/ValidateFailureScenarioJob.scala new file mode 100644 index 0000000000..8279794c03 --- /dev/null +++ b/dtests/src/test/scala/io/snappydata/hydra/testEndToEndValidation/ValidateFailureScenarioJob.scala @@ -0,0 +1,52 @@ + +/* + * Copyright (c) 2016 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package io.snappydata.hydra.testEndToEndValidation + +import java.io.{File, FileOutputStream, PrintWriter} + +import scala.util.{Failure, Success, Try} + +import com.typesafe.config.Config +import util.TestException + +import org.apache.spark.sql._ + +class ValidateFailureScenarioJob extends SnappySQLJob { + override def runSnappyJob(snSession: SnappySession, jobConfig: Config): Any = { + val snc = snSession.sqlContext + val pw: PrintWriter + = new PrintWriter(new FileOutputStream(new File("testJobFailure.out")), true) + Try { + // scalastyle:off println + pw.println("Throwing test exception...") + pw.flush() + throw new TestException("Throwing test exception...") + } match { + case Success(v) => pw.close() + s"See logs" + case Failure(e) => + pw.println("Exception occurred while executing the job " + + "\nError Message:" + e.getMessage) + pw.close(); + throw new TestException(s"Job failed. Check logs."); + } + pw.close(); + } + + override def isValidJob(sc: SnappySession, config: Config): SnappyJobValidation = SnappyJobValid() +} diff --git a/dtests/src/test/scala/io/snappydata/hydra/testEndToEndValidation/ValidateHangScenarioApp.scala b/dtests/src/test/scala/io/snappydata/hydra/testEndToEndValidation/ValidateHangScenarioApp.scala new file mode 100644 index 0000000000..6e459fb625 --- /dev/null +++ b/dtests/src/test/scala/io/snappydata/hydra/testEndToEndValidation/ValidateHangScenarioApp.scala @@ -0,0 +1,43 @@ + +/* + * Copyright (c) 2018 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package io.snappydata.hydra.testEndToEndValidation + +import java.io.{File, FileOutputStream, PrintWriter} + +import org.apache.spark.sql._ +import org.apache.spark.{SparkConf, SparkContext} + +object ValidateHangScenarioApp { + def main(args: Array[String]) { + val conf = new SparkConf(). + setAppName("TestFailureInSparkApp_" + System.currentTimeMillis()) + val sc = SparkContext.getOrCreate(conf) + val snc = SnappyContext(sc) + + def getCurrentDirectory = new java.io.File(".").getCanonicalPath + + val outputFile = "testAppHang.out" + val pw = new PrintWriter(new FileOutputStream(new File(outputFile), true)); + // scalastyle:off println + pw.println("Thread goes to sleep now...") + pw.flush() + Thread.sleep(300 * 1000) + pw.println("Done sleeping") + pw.flush() + } +} diff --git a/dtests/src/test/scala/io/snappydata/hydra/testEndToEndValidation/ValidateHangScenarioJob.scala b/dtests/src/test/scala/io/snappydata/hydra/testEndToEndValidation/ValidateHangScenarioJob.scala new file mode 100644 index 0000000000..40f57c5a38 --- /dev/null +++ b/dtests/src/test/scala/io/snappydata/hydra/testEndToEndValidation/ValidateHangScenarioJob.scala @@ -0,0 +1,53 @@ + +/* + * Copyright (c) 2018 SnappyData, Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You + * may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. See accompanying + * LICENSE file. + */ +package io.snappydata.hydra.testEndToEndValidation + +import java.io.{File, FileOutputStream, PrintWriter} + +import util.TestException +import com.typesafe.config.Config +import io.snappydata.hydra.{SnappyTestUtils, northwind} + +import org.apache.spark.SparkContext +import org.apache.spark.sql._ +import scala.util.{Failure, Success, Try} + +class ValidateHangScenarioJob extends SnappySQLJob { + override def runSnappyJob(snappySession: SnappySession, jobConfig: Config): Any = { + val snc = snappySession.sqlContext + + def getCurrentDirectory = new java.io.File(".").getCanonicalPath + val outputFile = "testJobHang.out" + val pw = new PrintWriter(new FileOutputStream(new File(outputFile), true)); + Try { + // scalastyle:off println + pw.println("Thread goes to sleep now...") + pw.flush() + Thread.sleep(300*1000) + pw.println("Done sleeping") + pw.flush() + } match { + case Success(v) => pw.close() + s"See ${getCurrentDirectory}/${outputFile}" + case Failure(e) => pw.close(); + throw e; + } + } + + override def isValidJob(sc: SnappySession, config: Config): SnappyJobValidation = SnappyJobValid() +}