From 8ee17805a3f92896083330df31e5a5e4bb512e2c Mon Sep 17 00:00:00 2001 From: pengyu Date: Wed, 25 Dec 2024 17:57:00 +0800 Subject: [PATCH 1/2] test: trigger compaction with table_name instead of tablets --- ...date_compaction_with_higher_version.groovy | 8 +- ...ompaction_exception_fault_injection.groovy | 55 +------------ ...st_index_compaction_fault_injection.groovy | 82 ++++--------------- .../test_index_compaction_dup_keys_arr.groovy | 57 +------------ .../test_index_compaction_null_arr.groovy | 67 ++------------- .../test_index_compaction_null.groovy | 67 ++------------- 6 files changed, 44 insertions(+), 292 deletions(-) diff --git a/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_compaction_with_higher_version.groovy b/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_compaction_with_higher_version.groovy index b5ae14957a706f..c90d1462ffb6d5 100644 --- a/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_compaction_with_higher_version.groovy +++ b/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_compaction_with_higher_version.groovy @@ -54,7 +54,7 @@ suite("test_partial_update_compaction_with_higher_version", "nonConcurrent") { } logger.info("tablet ${tabletId} on backend ${tabletBackend.Host} with backendId=${tabletBackend.BackendId}"); - def check_rs_metas = { expected_rs_meta_size, check_func -> + def check_rs_metas = { expected_rs_meta_size, check_func -> if (isCloudMode()) { return } @@ -88,7 +88,7 @@ suite("test_partial_update_compaction_with_higher_version", "nonConcurrent") { } }) - def enable_publish_spin_wait = { tokenName -> + def enable_publish_spin_wait = { tokenName -> if (isCloudMode()) { GetDebugPoint().enableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.enable_spin_wait", [token: "${tokenName}"]) } else { @@ -104,7 +104,7 @@ suite("test_partial_update_compaction_with_higher_version", "nonConcurrent") { } } - def enable_block_in_publish = { passToken -> + def enable_block_in_publish = { passToken -> if (isCloudMode()) { GetDebugPoint().enableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.block", [pass_token: "${passToken}"]) } else { @@ -209,7 +209,7 @@ suite("test_partial_update_compaction_with_higher_version", "nonConcurrent") { Assert.assertEquals(numRows, 4) // 4 = 2 + 2 } }) - + } catch(Exception e) { logger.info(e.getMessage()) throw e diff --git a/regression-test/suites/fault_injection_p0/test_index_compaction_exception_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_index_compaction_exception_fault_injection.groovy index 8a9c6d71609a2e..08a5dfd4788ffd 100644 --- a/regression-test/suites/fault_injection_p0/test_index_compaction_exception_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_index_compaction_exception_fault_injection.groovy @@ -36,49 +36,6 @@ suite("test_index_compaction_exception_fault_injection", "nonConcurrent") { } } - def trigger_full_compaction_on_tablets = { tablets -> - for (def tablet : tablets) { - String tablet_id = tablet.TabletId - String backend_id = tablet.BackendId - int times = 1 - - String compactionStatus; - do{ - def (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - ++times - sleep(2000) - compactionStatus = parseJson(out.trim()).status.toLowerCase(); - } while (compactionStatus!="success" && times<=10 && compactionStatus!="e-6010") - - - if (compactionStatus == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction && compactionStatus!="e-6010") { - assertEquals("success", compactionStatus) - } - } - } - - def wait_full_compaction_done = { tablets -> - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - String backend_id = tablet.BackendId - def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - } - def get_rowset_count = { tablets -> int rowsetCount = 0 for (def tablet in tablets) { @@ -201,7 +158,7 @@ suite("test_index_compaction_exception_fault_injection", "nonConcurrent") { "Compaction::construct_skip_inverted_index_index_file_reader_open_error" ] - def run_test = { tablets, debug_point, abnormal -> + def run_test = { tablets, debug_point, abnormal, table_name -> insert_data.call() run_sql.call() @@ -226,8 +183,7 @@ suite("test_index_compaction_exception_fault_injection", "nonConcurrent") { try { GetDebugPoint().enableDebugPointForAllBEs(debug_point) logger.info("trigger_full_compaction_on_tablets with fault injection: ${debug_point}") - trigger_full_compaction_on_tablets.call(tablets) - wait_full_compaction_done.call(tablets) + trigger_and_wait_compaction(table_name, "full") } finally { GetDebugPoint().disableDebugPointForAllBEs(debug_point) } @@ -240,10 +196,7 @@ suite("test_index_compaction_exception_fault_injection", "nonConcurrent") { logger.info("trigger_full_compaction_on_tablets normally") // trigger full compactions for all tablets in ${tableName} // this time, index compaction of some columns will be skipped because of the fault injection - trigger_full_compaction_on_tablets.call(tablets) - - // wait for full compaction done - wait_full_compaction_done.call(tablets) + trigger_and_wait_compaction(table_name, "full") } // after full compaction, there is only 1 rowset. @@ -291,7 +244,7 @@ suite("test_index_compaction_exception_fault_injection", "nonConcurrent") { """ def tablets = sql_return_maparray """ show tablets from ${table_name}; """ - run_test.call(tablets, debug_point, is_abnormal) + run_test.call(tablets, debug_point, is_abnormal, table_name) } } diff --git a/regression-test/suites/fault_injection_p0/test_index_compaction_fault_injection.groovy b/regression-test/suites/fault_injection_p0/test_index_compaction_fault_injection.groovy index 6c9cac37f0b27f..0181f51b4e5027 100644 --- a/regression-test/suites/fault_injection_p0/test_index_compaction_fault_injection.groovy +++ b/regression-test/suites/fault_injection_p0/test_index_compaction_fault_injection.groovy @@ -25,7 +25,7 @@ suite("test_index_compaction_failure_injection", "nonConcurrent") { getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); boolean disableAutoCompaction = false - + def set_be_config = { key, value -> for (String backend_id: backendId_to_backendIP.keySet()) { def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) @@ -33,49 +33,6 @@ suite("test_index_compaction_failure_injection", "nonConcurrent") { } } - def trigger_full_compaction_on_tablets = { tablets -> - for (def tablet : tablets) { - String tablet_id = tablet.TabletId - String backend_id = tablet.BackendId - int times = 1 - - String compactionStatus; - do{ - def (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - ++times - sleep(2000) - compactionStatus = parseJson(out.trim()).status.toLowerCase(); - } while (compactionStatus!="success" && times<=10 && compactionStatus!="e-6010") - - - if (compactionStatus == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction && compactionStatus!="e-6010") { - assertEquals("success", compactionStatus) - } - } - } - - def wait_full_compaction_done = { tablets -> - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - String backend_id = tablet.BackendId - def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - } - def get_rowset_count = { tablets -> int rowsetCount = 0 for (def tablet in tablets) { @@ -105,7 +62,7 @@ suite("test_index_compaction_failure_injection", "nonConcurrent") { } } - def insert_data = { -> + def insert_data = { -> sql """ INSERT INTO ${tableName} VALUES (1, "andy", "andy love apple", 100); """ sql """ INSERT INTO ${tableName} VALUES (1, "bason", "bason hate pear", 99); """ sql """ INSERT INTO ${tableName} VALUES (2, "andy", "andy love apple", 100); """ @@ -114,14 +71,14 @@ suite("test_index_compaction_failure_injection", "nonConcurrent") { sql """ INSERT INTO ${tableName} VALUES (3, "bason", "bason hate pear", 99); """ } - def run_sql = { -> + def run_sql = { -> qt_sql """ select * from ${tableName} order by id, name, hobbies, score """ qt_sql """ select * from ${tableName} where name match "andy" order by id, name, hobbies, score """ qt_sql """ select * from ${tableName} where hobbies match "pear" order by id, name, hobbies, score """ qt_sql """ select * from ${tableName} where score < 100 order by id, name, hobbies, score """ } - def run_test = { tablets -> + def run_test = { tablets, table_name -> insert_data.call() run_sql.call() @@ -143,8 +100,7 @@ suite("test_index_compaction_failure_injection", "nonConcurrent") { try { GetDebugPoint().enableDebugPointForAllBEs("index_compaction_compact_column_throw_error") logger.info("trigger_full_compaction_on_tablets with fault injection: index_compaction_compact_column_throw_error") - trigger_full_compaction_on_tablets.call(tablets) - wait_full_compaction_done.call(tablets) + trigger_and_wait_compaction(table_name, "full") } finally { GetDebugPoint().disableDebugPointForAllBEs("index_compaction_compact_column_throw_error") } @@ -155,10 +111,7 @@ suite("test_index_compaction_failure_injection", "nonConcurrent") { logger.info("trigger_full_compaction_on_tablets normally") // trigger full compactions for all tablets in ${tableName} // this time, index compaction of some columns will be skipped because of the fault injection - trigger_full_compaction_on_tablets.call(tablets) - - // wait for full compaction done - wait_full_compaction_done.call(tablets) + trigger_and_wait_compaction(table_name, "full") // after full compaction, there is only 1 rowset. rowsetCount = get_rowset_count.call(tablets); @@ -187,8 +140,7 @@ suite("test_index_compaction_failure_injection", "nonConcurrent") { try { GetDebugPoint().enableDebugPointForAllBEs("index_compaction_compact_column_status_not_ok") logger.info("trigger_full_compaction_on_tablets with fault injection: index_compaction_compact_column_status_not_ok") - trigger_full_compaction_on_tablets.call(tablets) - wait_full_compaction_done.call(tablets) + trigger_and_wait_compaction(table_name, "full") } finally { GetDebugPoint().disableDebugPointForAllBEs("index_compaction_compact_column_status_not_ok") } @@ -210,10 +162,7 @@ suite("test_index_compaction_failure_injection", "nonConcurrent") { logger.info("trigger_full_compaction_on_tablets normally") // trigger full compactions for all tablets in ${tableName} // this time, index compaction of some columns will be skipped because of the fault injection - trigger_full_compaction_on_tablets.call(tablets) - - // wait for full compaction done - wait_full_compaction_done.call(tablets) + trigger_and_wait_compaction(table_name, "full") // after full compaction, there is only 1 rowset. rowsetCount = get_rowset_count.call(tablets); @@ -227,7 +176,7 @@ suite("test_index_compaction_failure_injection", "nonConcurrent") { // insert more data and trigger full compaction again insert_data.call() - + sql """ select * from ${tableName} """ // insert 6 rows, so there are 7 rowsets. @@ -240,10 +189,7 @@ suite("test_index_compaction_failure_injection", "nonConcurrent") { // tigger full compaction for all tablets normally // this time, index compaction will be done successfully logger.info("trigger_full_compaction_on_tablets normally") - trigger_full_compaction_on_tablets.call(tablets) - - // wait for full compaction done - wait_full_compaction_done.call(tablets) + trigger_and_wait_compaction(table_name, "full") // after full compaction, there is only 1 rowset. rowsetCount = get_rowset_count.call(tablets); @@ -262,7 +208,7 @@ suite("test_index_compaction_failure_injection", "nonConcurrent") { String backend_id; backend_id = backendId_to_backendIP.keySet()[0] def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) def configList = parseJson(out.trim()) @@ -308,7 +254,7 @@ suite("test_index_compaction_failure_injection", "nonConcurrent") { //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus def tablets = sql_return_maparray """ show tablets from ${tableName}; """ - run_test.call(tablets) + run_test.call(tablets, tableName) /** * test for unique key table @@ -329,7 +275,7 @@ suite("test_index_compaction_failure_injection", "nonConcurrent") { UNIQUE KEY(`id`) COMMENT 'OLAP' DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES ( + PROPERTIES ( "replication_num" = "1", "disable_auto_compaction" = "true", "enable_unique_key_merge_on_write" = "true", @@ -338,7 +284,7 @@ suite("test_index_compaction_failure_injection", "nonConcurrent") { """ tablets = sql_return_maparray """ show tablets from ${tableName}; """ - run_test.call(tablets) + run_test.call(tablets, tableName) } finally { if (has_update_be_config) { diff --git a/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_dup_keys_arr.groovy b/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_dup_keys_arr.groovy index c7d41e68cdd112..5d409eb3fbdf9a 100644 --- a/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_dup_keys_arr.groovy +++ b/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_dup_keys_arr.groovy @@ -32,7 +32,7 @@ suite("test_index_compaction_dup_keys_array", "array_contains_inverted_index, no getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); boolean disableAutoCompaction = false - + def set_be_config = { key, value -> for (String backend_id: backendId_to_backendIP.keySet()) { def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) @@ -40,49 +40,6 @@ suite("test_index_compaction_dup_keys_array", "array_contains_inverted_index, no } } - def trigger_full_compaction_on_tablets = { tablets -> - for (def tablet : tablets) { - String tablet_id = tablet.TabletId - String backend_id = tablet.BackendId - int times = 1 - - String compactionStatus; - do{ - def (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - ++times - sleep(2000) - compactionStatus = parseJson(out.trim()).status.toLowerCase(); - } while (compactionStatus!="success" && times<=10) - - - if (compactionStatus == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction) { - assertEquals("success", compactionStatus) - } - } - } - - def wait_full_compaction_done = { tablets -> - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - String backend_id = tablet.BackendId - def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - } - def get_rowset_count = { tablets -> int rowsetCount = 0 for (def tablet in tablets) { @@ -118,7 +75,7 @@ suite("test_index_compaction_dup_keys_array", "array_contains_inverted_index, no String backend_id; backend_id = backendId_to_backendIP.keySet()[0] def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) def configList = parseJson(out.trim()) @@ -186,10 +143,7 @@ suite("test_index_compaction_dup_keys_array", "array_contains_inverted_index, no assert (rowsetCount == 7 * replicaNum) // trigger full compactions for all tablets in ${tableName} - trigger_full_compaction_on_tablets.call(tablets) - - // wait for full compaction done - wait_full_compaction_done.call(tablets) + trigger_and_wait_compaction(tableName, "full") // after full compaction, there is only 1 rowset. rowsetCount = get_rowset_count.call(tablets); @@ -224,10 +178,7 @@ suite("test_index_compaction_dup_keys_array", "array_contains_inverted_index, no assert (rowsetCount == 7 * replicaNum) } // trigger full compactions for all tablets in ${tableName} - trigger_full_compaction_on_tablets.call(tablets) - - // wait for full compaction done - wait_full_compaction_done.call(tablets) + trigger_and_wait_compaction(tableName, "full") // after full compaction, there is only 1 rowset. rowsetCount = get_rowset_count.call(tablets); diff --git a/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_null_arr.groovy b/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_null_arr.groovy index dfc38998c4d4bc..df957b2e5638e1 100644 --- a/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_null_arr.groovy +++ b/regression-test/suites/inverted_index_p0/array_contains/test_index_compaction_null_arr.groovy @@ -32,7 +32,7 @@ suite("test_index_compaction_null_arr", "array_contains_inverted_index, nonConcu getBackendIpHttpPort(backendId_to_backendIP, backendId_to_backendHttpPort); boolean disableAutoCompaction = false - + def set_be_config = { key, value -> for (String backend_id: backendId_to_backendIP.keySet()) { def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) @@ -40,49 +40,6 @@ suite("test_index_compaction_null_arr", "array_contains_inverted_index, nonConcu } } - def trigger_full_compaction_on_tablets = { tablets -> - for (def tablet : tablets) { - String tablet_id = tablet.TabletId - String backend_id = tablet.BackendId - int times = 1 - - String compactionStatus; - do{ - def (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - ++times - sleep(2000) - compactionStatus = parseJson(out.trim()).status.toLowerCase(); - } while (compactionStatus!="success" && times<=10 && compactionStatus!="e-6010") - - - if (compactionStatus == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction && compactionStatus!="e-6010") { - assertEquals("success", compactionStatus) - } - } - } - - def wait_full_compaction_done = { tablets -> - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - String backend_id = tablet.BackendId - def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - } - def get_rowset_count = { tablets -> int rowsetCount = 0 for (def tablet in tablets) { @@ -112,7 +69,7 @@ suite("test_index_compaction_null_arr", "array_contains_inverted_index, nonConcu } } - def insert_data = { -> + def insert_data = { -> sql """insert into ${tableName} values (1,null,['addr qie3'],'yy','lj',[100]), (2,null,['hehe'],null,'lala',[200]), @@ -125,7 +82,7 @@ suite("test_index_compaction_null_arr", "array_contains_inverted_index, nonConcu """ } - def run_sql = { -> + def run_sql = { -> // select all data qt_select_0 "SELECT * FROM ${tableName} ORDER BY id" @@ -163,7 +120,7 @@ suite("test_index_compaction_null_arr", "array_contains_inverted_index, nonConcu qt_select_match_2 "SELECT * FROM ${tableName} WHERE array_contains(addr, 'addr') ORDER BY id" } - def run_test = { tablets -> + def run_test = { tablets, table_name -> insert_data.call() insert_data.call() @@ -183,10 +140,7 @@ suite("test_index_compaction_null_arr", "array_contains_inverted_index, nonConcu assert (rowsetCount == 3 * replicaNum) // tigger full compaction for all tablets - trigger_full_compaction_on_tablets.call(tablets) - - // wait for full compaction done - wait_full_compaction_done.call(tablets) + trigger_and_wait_compaction(table_name, "full") // after full compaction, there is only 1 rowset. rowsetCount = get_rowset_count.call(tablets); @@ -211,10 +165,7 @@ suite("test_index_compaction_null_arr", "array_contains_inverted_index, nonConcu } // tigger full compaction for all tablets - trigger_full_compaction_on_tablets.call(tablets) - - // wait for full compaction done - wait_full_compaction_done.call(tablets) + trigger_and_wait_compaction(table_name, "full") // after full compaction, there is only 1 rowset. rowsetCount = get_rowset_count.call(tablets); @@ -233,7 +184,7 @@ suite("test_index_compaction_null_arr", "array_contains_inverted_index, nonConcu String backend_id; backend_id = backendId_to_backendIP.keySet()[0] def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) def configList = parseJson(out.trim()) @@ -286,7 +237,7 @@ suite("test_index_compaction_null_arr", "array_contains_inverted_index, nonConcu //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus def tablets = sql_return_maparray """ show tablets from ${tableName}; """ - run_test.call(tablets) + run_test.call(tablets, tableName) /** * test for unique key table @@ -319,7 +270,7 @@ suite("test_index_compaction_null_arr", "array_contains_inverted_index, nonConcu """ tablets = sql_return_maparray """ show tablets from ${tableName}; """ - run_test.call(tablets) + run_test.call(tablets, tableName) } finally { if (has_update_be_config) { diff --git a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_null.groovy b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_null.groovy index 5bf1ec1773b559..8f349d1ef0b024 100644 --- a/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_null.groovy +++ b/regression-test/suites/inverted_index_p0/index_compaction/test_index_compaction_null.groovy @@ -26,7 +26,7 @@ suite("test_index_compaction_null", "nonConcurrent") { sql """ set global enable_match_without_inverted_index = false """ boolean disableAutoCompaction = false - + def set_be_config = { key, value -> for (String backend_id: backendId_to_backendIP.keySet()) { def (code, out, err) = update_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), key, value) @@ -34,49 +34,6 @@ suite("test_index_compaction_null", "nonConcurrent") { } } - def trigger_full_compaction_on_tablets = { tablets -> - for (def tablet : tablets) { - String tablet_id = tablet.TabletId - String backend_id = tablet.BackendId - int times = 1 - - String compactionStatus; - do{ - def (code, out, err) = be_run_full_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err) - ++times - sleep(2000) - compactionStatus = parseJson(out.trim()).status.toLowerCase(); - } while (compactionStatus!="success" && times<=10 && compactionStatus!="e-6010") - - - if (compactionStatus == "fail") { - assertEquals(disableAutoCompaction, false) - logger.info("Compaction was done automatically!") - } - if (disableAutoCompaction && compactionStatus!="e-6010") { - assertEquals("success", compactionStatus) - } - } - } - - def wait_full_compaction_done = { tablets -> - for (def tablet in tablets) { - boolean running = true - do { - Thread.sleep(1000) - String tablet_id = tablet.TabletId - String backend_id = tablet.BackendId - def (code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id) - logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err) - assertEquals(code, 0) - def compactionStatus = parseJson(out.trim()) - assertEquals("success", compactionStatus.status.toLowerCase()) - running = compactionStatus.run_status - } while (running) - } - } - def get_rowset_count = { tablets -> int rowsetCount = 0 for (def tablet in tablets) { @@ -106,7 +63,7 @@ suite("test_index_compaction_null", "nonConcurrent") { } } - def insert_data = { -> + def insert_data = { -> sql """insert into ${tableName} values (1,null,'addr qie3','yy','lj',100), (2,null,'hehe',null,'lala',200), @@ -119,7 +76,7 @@ suite("test_index_compaction_null", "nonConcurrent") { """ } - def run_sql = { -> + def run_sql = { -> sql """ set enable_common_expr_pushdown=true """ // select all data qt_select_0 "SELECT * FROM ${tableName} ORDER BY id" @@ -154,7 +111,7 @@ suite("test_index_compaction_null", "nonConcurrent") { qt_select_match_2 "SELECT * FROM ${tableName} WHERE addr MATCH_ALL 'addr fengtai' ORDER BY id" } - def run_test = { tablets -> + def run_test = { tablets, table_name -> insert_data.call() insert_data.call() @@ -174,10 +131,7 @@ suite("test_index_compaction_null", "nonConcurrent") { assert (rowsetCount == 3 * replicaNum) // tigger full compaction for all tablets - trigger_full_compaction_on_tablets.call(tablets) - - // wait for full compaction done - wait_full_compaction_done.call(tablets) + trigger_and_wait_compaction(table_name, "full") // after full compaction, there is only 1 rowset. rowsetCount = get_rowset_count.call(tablets); @@ -202,10 +156,7 @@ suite("test_index_compaction_null", "nonConcurrent") { } // tigger full compaction for all tablets - trigger_full_compaction_on_tablets.call(tablets) - - // wait for full compaction done - wait_full_compaction_done.call(tablets) + trigger_and_wait_compaction(table_name, "full") // after full compaction, there is only 1 rowset. rowsetCount = get_rowset_count.call(tablets); @@ -224,7 +175,7 @@ suite("test_index_compaction_null", "nonConcurrent") { String backend_id; backend_id = backendId_to_backendIP.keySet()[0] def (code, out, err) = show_be_config(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id)) - + logger.info("Show config: code=" + code + ", out=" + out + ", err=" + err) assertEquals(code, 0) def configList = parseJson(out.trim()) @@ -278,7 +229,7 @@ suite("test_index_compaction_null", "nonConcurrent") { //TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus def tablets = sql_return_maparray """ show tablets from ${tableName}; """ - run_test.call(tablets) + run_test.call(tablets, tableName) /** * test for unique key table @@ -313,7 +264,7 @@ suite("test_index_compaction_null", "nonConcurrent") { sql """ set enable_common_expr_pushdown = true """ tablets = sql_return_maparray """ show tablets from ${tableName}; """ - run_test.call(tablets) + run_test.call(tablets, tableName) } finally { if (has_update_be_config) { From eb04dbea8beda8b0dec8ae08a3f8f5523673372b Mon Sep 17 00:00:00 2001 From: pengyu Date: Mon, 30 Dec 2024 15:45:53 +0800 Subject: [PATCH 2/2] fix: ignore no suitable version --- regression-test/plugins/plugin_compaction.groovy | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/regression-test/plugins/plugin_compaction.groovy b/regression-test/plugins/plugin_compaction.groovy index 385292eb1fe86d..2bf08246fabcd0 100644 --- a/regression-test/plugins/plugin_compaction.groovy +++ b/regression-test/plugins/plugin_compaction.groovy @@ -105,7 +105,13 @@ Suite.metaClass.trigger_and_wait_compaction = { String table_name, String compac if (trigger_status.status.toLowerCase() == "already_exist") { triggered_tablets.add(tablet) // compaction already in queue, treat it as successfully triggered } else if (!auto_compaction_disabled) { - // ignore the error if auto compaction enabled + if ((compaction_type == "cumulative" && trigger_status.status.toLowerCase() == "e-2000") || + (compaction_type == "base" && trigger_status.status.toLowerCase() == "e-808") || + (compaction_type == "full" && trigger_status.status.toLowerCase() == "e-2008")) { + // no suitable rowsets for compaction, ignore + } else { + throw new Exception("trigger compaction failed, be host: ${be_host}, tablet id: ${tablet.TabletId}, status: ${trigger_status}") + } } else { throw new Exception("trigger compaction failed, be host: ${be_host}, tablet id: ${tablet.TabletId}, status: ${trigger_status.status}") }