diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/txn/Transaction.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/txn/Transaction.java index 82031e32ed787f..b8152c1e89e595 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/txn/Transaction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/txn/Transaction.java @@ -133,13 +133,12 @@ public void executeInsertIntoTableCommand(StmtExecutor executor) { filteredRows = Integer.parseInt(coordinator.getLoadCounters().get(LoadEtlTask.DPP_ABNORMAL_ALL)); } - // if in strict mode, insert will fail if there are filtered rows - if (ctx.getSessionVariable().getEnableInsertStrict()) { - if (filteredRows > 0) { - ctx.getState().setError(ErrorCode.ERR_FAILED_WHEN_INSERT, - "Insert has filtered data in strict mode, tracking_url=" + coordinator.getTrackingUrl()); - return; - } + if (1.0 * filteredRows / (filteredRows + loadedRows) > ctx.getSessionVariable().getMaxFilterRatio()) { + ctx.getState().setError(ErrorCode.ERR_FAILED_WHEN_INSERT, + "Insert has filtered ratio > max_filter_ratio:" + + ctx.getSessionVariable().getMaxFilterRatio() + + "tracking_url=" + coordinator.getTrackingUrl()); + return; } if (table.getType() != TableType.OLAP && table.getType() != TableType.MATERIALIZED_VIEW) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 7a4cef6e40b51e..26d2dadc26534b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -119,6 +119,8 @@ public class SessionVariable implements Serializable, Writable { public static final String PROFILE_LEVEL = "profile_level"; public static final String MAX_INSTANCE_NUM = "max_instance_num"; public static final String ENABLE_INSERT_STRICT = "enable_insert_strict"; + + public static final String MAX_FILTER_RATIO = "max_filter_ratio"; public static final String ENABLE_SPILLING = "enable_spilling"; public static final String ENABLE_EXCHANGE_NODE_PARALLEL_MERGE = "enable_exchange_node_parallel_merge"; public static final String PREFER_JOIN_METHOD = "prefer_join_method"; @@ -669,6 +671,9 @@ public class SessionVariable implements Serializable, Writable { @VariableMgr.VarAttr(name = ENABLE_INSERT_STRICT, needForward = true) public boolean enableInsertStrict = true; + @VariableMgr.VarAttr(name = MAX_FILTER_RATIO, needForward = true) + public double maxFilterRatio = 0; + @VariableMgr.VarAttr(name = ENABLE_ODBC_TRANSCATION) public boolean enableOdbcTransaction = false; @@ -1829,6 +1834,10 @@ public void setEnableInsertStrict(boolean enableInsertStrict) { this.enableInsertStrict = enableInsertStrict; } + public double getMaxFilterRatio() { + return maxFilterRatio; + } + public boolean isEnableSqlCache() { return enableSqlCache; }