Sync with upstream (#18)

* Add TLPWhereGenerator interface

* Add generic TLPWhere oracle

* Implement TLPWhereGenerator interfaces for SQLite3

* Use generic TLPWhere oracle for SQLite3

* [Databend] Refactor tests

* [Databend] Update to v1.2.542

* Remove duplicated logs in NoREC oracle (#953)

* [ClickHouse] Added pattern for changed error message (#955)

* [ClickHouse] Added pattern for changed error message

https://fiddle.clickhouse.com/a3a95024-4da7-4275-baff-86f116014744

```
Received exception from server (version 24.2.3):
Code: 403. DB::Exception: Received from localhost:9000. DB::Exception: Cannot get JOIN keys from JOIN ON section: '476505718 = `_--right_2.c0`', found keys: [Left keys: [] Right keys [] Condition columns: '', 'equals(476505718, _--right_2.c0)']. (INVALID_JOIN_ON_EXPRESSION)
(query: SELECT SUM(check <> 0) FROM ((SELECT right_0.c0 AS `check` FROM t0 AS left FULL OUTER JOIN t0 AS right_0 ON ((left.c0)=(right_0.c0)) LEFT OUTER JOIN t0 AS right_1 ON ((left.c0)=(right_1.c0)) LEFT ANTI JOIN t0 AS right_2 ON ((476505718)=(right_2.c0)))) as res;)
```

* formatter

* Avoid storage parameters during CREATE TABLE for partitioned tables (#956)

CREATE TABLE WITH() allows storage parameters, but partitioned tables
emit an error (given below) if attempted. This is because partitioned
non-leaf tables are virutal tables [1] and don't accept
storage parameters.

Sample Error:
"ERROR: cannot specify storage parameters for a partitioned table"

This patch skips storage parameter generation for partitioned tables.

Ref:
1. https://www.postgresql.org/docs/current/ddl-partitioning.html

* Review feedback

* Ensure INHERITS() only uses tables (not views)

In Postgres, CREATE TABLE INHERITS () throws an error when VIEWs are
provided, sample given below.

ERROR: inherited relation "pg_buffercache" is not a table or foreign table

Ref:
1. https://www.postgresql.org/docs/current/ddl-inherit.html

* Revert "Ensure INHERITS() only uses tables (not views)"

This reverts commit 0316c1c.

* Ensure INHERITS() only uses tables (not views)

In Postgres, CREATE TABLE INHERITS () throws an error when VIEWs are
provided, sample given below.

ERROR: inherited relation "pg_buffercache" is not a table or foreign table

Ref:
1. https://www.postgresql.org/docs/current/ddl-inherit.html

* Another fix

---------

Co-authored-by: malwaregarry <[email protected]>
Co-authored-by: ming wei <[email protected]>
Co-authored-by: Manuel Rigger <[email protected]>
Co-authored-by: Ilya Yatsishin <[email protected]>
Co-authored-by: Robins <[email protected]>

src/sqlancer/clickhouse/ClickHouseErrors.java

@@ -45,7 +45,8 @@ public static List<String> getExpectedExpressionErrors() {
                 "Positional argument numeric constant expression is not representable as",
                 "Positional argument must be constant with numeric type", " is out of bounds. Expected in range",
                 "with constants is not supported. (INVALID_JOIN_ON_EXPRESSION)",
-                "Unexpected inf or nan to integer conversion", "Unsigned type must not contain",
+                "Cannot get JOIN keys from JOIN ON section", "Unexpected inf or nan to integer conversion",
+                "Cannot determine join keys in", "Unsigned type must not contain",
                 "Unexpected inf or nan to integer conversion",
 
                 // The way we generate JOINs we can have ambiguous left table column without

src/sqlancer/common/gen/PartitionGenerator.java

@@ -0,0 +1,27 @@
+package sqlancer.common.gen;
+
+import sqlancer.common.ast.newast.Expression;
+import sqlancer.common.schema.AbstractTableColumn;
+
+public interface PartitionGenerator<E extends Expression<C>, C extends AbstractTableColumn<?, ?>> {
+
+    /**
+     * Negates a predicate (i.e., uses a NOT operator).
+     *
+     * @param predicate
+     *            the boolean predicate.
+     *
+     * @return the negated predicate.
+     */
+    E negatePredicate(E predicate);
+
+    /**
+     * Checks if an expression evaluates to NULL (i.e., implements the IS NULL operator).
+     *
+     * @param expr
+     *            the expression
+     *
+     * @return an expression that checks whether the expression evaluates to NULL.
+     */
+    E isNull(E expr);
+}

src/sqlancer/common/gen/TLPWhereGenerator.java

@@ -0,0 +1,28 @@
+package sqlancer.common.gen;
+
+import java.util.List;
+
+import sqlancer.common.ast.newast.Expression;
+import sqlancer.common.ast.newast.Join;
+import sqlancer.common.ast.newast.Select;
+import sqlancer.common.schema.AbstractTable;
+import sqlancer.common.schema.AbstractTableColumn;
+import sqlancer.common.schema.AbstractTables;
+
+public interface TLPWhereGenerator<S extends Select<J, E, T, C>, J extends Join<E, T, C>, E extends Expression<C>, T extends AbstractTable<C, ?, ?>, C extends AbstractTableColumn<?, ?>>
+        extends PartitionGenerator<E, C> {
+
+    TLPWhereGenerator<S, J, E, T, C> setTablesAndColumns(AbstractTables<T, C> tables);
+
+    E generateBooleanExpression();
+
+    S generateSelect();
+
+    List<J> getRandomJoinClauses();
+
+    List<E> getTableRefs();
+
+    List<E> generateFetchColumns(boolean shouldCreateDummy);
+
+    List<E> generateOrderBys();
+}

src/sqlancer/common/oracle/NoRECOracle.java

@@ -123,10 +123,6 @@ public Reproducer<G> getLastReproducer() {
     private int countRows(String queryString, ExpectedErrors errors, SQLGlobalState<?, ?> state) {
         SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors);
 
-        if (state.getOptions().logEachSelect()) {
-            state.getLogger().writeCurrent(queryString);
-        }
-
         int count = 0;
         try (SQLancerResultSet rs = q.executeAndGet(state)) {
             if (rs == null) {
@@ -151,10 +147,6 @@ private int countRows(String queryString, ExpectedErrors errors, SQLGlobalState<
 
     private int extractCounts(String queryString, ExpectedErrors errors, SQLGlobalState<?, ?> state) {
         SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors);
-        if (state.getOptions().logEachSelect()) {
-            state.getLogger().writeCurrent(queryString);
-        }
-
         int count = 0;
         try (SQLancerResultSet rs = q.executeAndGet(state)) {
             if (rs == null) {

src/sqlancer/common/oracle/TLPWhereOracle.java

@@ -0,0 +1,84 @@
+package sqlancer.common.oracle;
+
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+
+import sqlancer.ComparatorHelper;
+import sqlancer.Randomly;
+import sqlancer.SQLGlobalState;
+import sqlancer.common.ast.newast.Expression;
+import sqlancer.common.ast.newast.Join;
+import sqlancer.common.ast.newast.Select;
+import sqlancer.common.gen.TLPWhereGenerator;
+import sqlancer.common.query.ExpectedErrors;
+import sqlancer.common.schema.AbstractSchema;
+import sqlancer.common.schema.AbstractTable;
+import sqlancer.common.schema.AbstractTableColumn;
+import sqlancer.common.schema.AbstractTables;
+
+public class TLPWhereOracle<Z extends Select<J, E, T, C>, J extends Join<E, T, C>, E extends Expression<C>, S extends AbstractSchema<?, T>, T extends AbstractTable<C, ?, ?>, C extends AbstractTableColumn<?, ?>, G extends SQLGlobalState<?, S>>
+        implements TestOracle<G> {
+
+    private final G state;
+
+    private TLPWhereGenerator<Z, J, E, T, C> gen;
+    private final ExpectedErrors errors;
+
+    private String generatedQueryString;
+
+    public TLPWhereOracle(G state, TLPWhereGenerator<Z, J, E, T, C> gen, ExpectedErrors expectedErrors) {
+        if (state == null || gen == null || expectedErrors == null) {
+            throw new IllegalArgumentException("Null variables used to initialize test oracle.");
+        }
+        this.state = state;
+        this.gen = gen;
+        this.errors = expectedErrors;
+    }
+
+    @Override
+    public void check() throws SQLException {
+        S s = state.getSchema();
+        AbstractTables<T, C> targetTables = TestOracleUtils.getRandomTableNonEmptyTables(s);
+        gen = gen.setTablesAndColumns(targetTables);
+
+        Select<J, E, T, C> select = gen.generateSelect();
+
+        boolean shouldCreateDummy = true;
+        select.setFetchColumns(gen.generateFetchColumns(shouldCreateDummy));
+        select.setJoinClauses(gen.getRandomJoinClauses());
+        select.setFromList(gen.getTableRefs());
+        select.setWhereClause(null);
+
+        String originalQueryString = select.asString();
+        generatedQueryString = originalQueryString;
+        List<String> firstResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors,
+                state);
+
+        boolean orderBy = Randomly.getBooleanWithSmallProbability();
+        if (orderBy) {
+            select.setOrderByClauses(gen.generateOrderBys());
+        }
+
+        TestOracleUtils.PredicateVariants<E, C> predicates = TestOracleUtils.initializeTernaryPredicateVariants(gen,
+                gen.generateBooleanExpression());
+        select.setWhereClause(predicates.predicate);
+        String firstQueryString = select.asString();
+        select.setWhereClause(predicates.negatedPredicate);
+        String secondQueryString = select.asString();
+        select.setWhereClause(predicates.isNullPredicate);
+        String thirdQueryString = select.asString();
+
+        List<String> combinedString = new ArrayList<>();
+        List<String> secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString,
+                thirdQueryString, combinedString, !orderBy, state, errors);
+
+        ComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, originalQueryString, combinedString,
+                state);
+    }
+
+    @Override
+    public String getLastQueryString() {
+        return generatedQueryString;
+    }
+}

src/sqlancer/common/oracle/TestOracleUtils.java

@@ -2,6 +2,8 @@
 
 import sqlancer.IgnoreMeException;
 import sqlancer.Randomly;
+import sqlancer.common.ast.newast.Expression;
+import sqlancer.common.gen.PartitionGenerator;
 import sqlancer.common.schema.AbstractSchema;
 import sqlancer.common.schema.AbstractTable;
 import sqlancer.common.schema.AbstractTableColumn;
@@ -12,11 +14,42 @@ public final class TestOracleUtils {
     private TestOracleUtils() {
     }
 
+    public static final class PredicateVariants<E extends Expression<C>, C extends AbstractTableColumn<?, ?>> {
+        public E predicate;
+        public E negatedPredicate;
+        public E isNullPredicate;
+
+        PredicateVariants(E predicate, E negatedPredicate, E isNullPredicate) {
+            this.predicate = predicate;
+            this.negatedPredicate = negatedPredicate;
+            this.isNullPredicate = isNullPredicate;
+        }
+    }
+
     public static <T extends AbstractTable<C, ?, ?>, C extends AbstractTableColumn<?, ?>> AbstractTables<T, C> getRandomTableNonEmptyTables(
             AbstractSchema<?, T> schema) {
         if (schema.getDatabaseTables().isEmpty()) {
             throw new IgnoreMeException();
         }
         return new AbstractTables<>(Randomly.nonEmptySubset(schema.getDatabaseTables()));
     }
+
+    public static <E extends Expression<C>, T extends AbstractTable<C, ?, ?>, C extends AbstractTableColumn<?, ?>> PredicateVariants<E, C> initializeTernaryPredicateVariants(
+            PartitionGenerator<E, C> gen, E predicate) {
+        if (gen == null) {
+            throw new IllegalStateException();
+        }
+        if (predicate == null) {
+            throw new IllegalStateException();
+        }
+        E negatedPredicate = gen.negatePredicate(predicate);
+        if (negatedPredicate == null) {
+            throw new IllegalStateException();
+        }
+        E isNullPredicate = gen.isNull(predicate);
+        if (isNullPredicate == null) {
+            throw new IllegalStateException();
+        }
+        return new PredicateVariants<>(predicate, negatedPredicate, isNullPredicate);
+    }
 }

src/sqlancer/datafusion/DataFusionProvider.java

@@ -1,7 +1,7 @@
 package sqlancer.datafusion;
 
-import static java.lang.System.exit;
 import static sqlancer.datafusion.DataFusionUtil.DataFusionLogger.DataFusionLogType.DML;
+import static sqlancer.datafusion.DataFusionUtil.dfAssert;
 import static sqlancer.datafusion.DataFusionUtil.displayTables;
 
 import java.sql.Connection;
@@ -52,13 +52,12 @@ public void generateDatabase(DataFusionGlobalState globalState) throws Exception
         List<DataFusionTable> allTables = globalState.getSchema().getDatabaseTables();
         List<String> allTablesName = allTables.stream().map(t -> t.getName()).collect(Collectors.toList());
         if (allTablesName.isEmpty()) {
-            System.out.println("Generate database failed");
-            exit(1);
+            dfAssert(false, "Generate Database failed.");
         }
 
         // Randomly insert some data into existing tables
         for (DataFusionTable table : allTables) {
-            int nInsertQuery = globalState.getRandomly().getInteger(0, 8); // [0, 10)
+            int nInsertQuery = globalState.getRandomly().getInteger(0, globalState.getOptions().getMaxNumberInserts());
 
             for (int i = 0; i < nInsertQuery; i++) {
                 SQLQueryAdapter insertQuery = null;

src/sqlancer/datafusion/DataFusionUtil.java

@@ -1,7 +1,5 @@
 package sqlancer.datafusion;
 
-import static java.lang.System.exit;
-
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileReader;
@@ -66,11 +64,15 @@ public static String displayTables(DataFusionGlobalState state, List<String> fro
         return resultStringBuilder.toString();
     }
 
+    // During development, you might want to manually let this function call exit(1) to fail fast
     public static void dfAssert(boolean condition, String message) {
         if (!condition) {
-            String methodName = Thread.currentThread().getStackTrace()[2].getMethodName();
-            System.err.println("DataFusion assertion failed in function '" + methodName + "': " + message);
-            exit(1);
+            // // Development mode assertion failure
+            // String methodName = Thread.currentThread().getStackTrace()[2]// .getMethodName();
+            // System.err.println("DataFusion assertion failed in function '" + methodName + "': " + message);
+            // exit(1);
+
+            throw new AssertionError(message);
         }
     }
 
@@ -149,9 +151,7 @@ public void appendToLog(DataFusionLogType logType, String logContent) {
                 try {
                     logFileWriter = new FileWriter(errorLogFile, true);
                 } catch (IOException e) {
-                    System.out.println("Failed to create FileWriter for errorLogFIle");
-                    e.printStackTrace();
-                    exit(1);
+                    dfAssert(false, "Failed to create FileWriter for errorLogFIle");
                 }
                 DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
                 String formattedDateTime = LocalDateTime.now().format(formatter);
@@ -175,14 +175,11 @@ public void appendToLog(DataFusionLogType logType, String logContent) {
                     logFileWriter.write(logContent);
                     logFileWriter.flush();
                 } catch (IOException e) {
-                    System.out.println("Failed to write to " + logType + " log: " + e.getMessage());
-                    e.printStackTrace();
-                    exit(1);
+                    String err = "Failed to write to " + logType + " log: " + e.getMessage();
+                    dfAssert(false, err);
                 }
             } else {
-                System.out.println("appending to log failed");
-                Thread.currentThread().getStackTrace();
-                exit(1);
+                dfAssert(false, "appending to log failed");
             }
         }
 

src/sqlancer/datafusion/server/datafusion_server/Cargo.toml

@@ -18,9 +18,10 @@ async-trait = "0.1.73"
 bytes = "1.4"
 chrono = { version = "0.4.34", default-features = false }
 dashmap = "5.5.0"
+# This version is for SQLancer CI run
 datafusion = { version = "40.0.0" }
-# datafusion = { git = "https://github.com/apache/datafusion.git", branch = "main"}
-datafusion_dev = { package = "datafusion", git = "https://github.com/apache/datafusion.git", branch = "main", optional = true }
+# Use following line if you want to test against the latest main branch of DataFusion
+# datafusion = { git = "https://github.com/apache/datafusion.git", branch = "main" }
 env_logger = "0.11"
 futures = "0.3"
 half = { version = "2.2.1", default-features = false }

src/sqlancer/postgres/gen/PostgresTableGenerator.java

@@ -113,7 +113,9 @@ private void createStandard() throws AssertionError {
         generateInherits();
         generatePartitionBy();
         generateUsing();
-        PostgresCommon.generateWith(sb, globalState, errors);
+        if (!isPartitionedTable) {
+            PostgresCommon.generateWith(sb, globalState, errors);
+        }
         if (Randomly.getBoolean() && isTemporaryTable) {
             sb.append(" ON COMMIT ");
             sb.append(Randomly.fromOptions("PRESERVE ROWS", "DELETE ROWS", "DROP"));
@@ -205,7 +207,7 @@ private void generateUsing() {
     }
 
     private void generateInherits() {
-        if (Randomly.getBoolean() && !newSchema.getDatabaseTables().isEmpty()) {
+        if (Randomly.getBoolean() && !newSchema.getDatabaseTablesWithoutViews().isEmpty()) {
             sb.append(" INHERITS(");
             sb.append(newSchema.getDatabaseTablesRandomSubsetNotEmpty().stream().map(t -> t.getName())
                     .collect(Collectors.joining(", ")));

src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java

@@ -9,6 +9,7 @@
 import sqlancer.Randomly;
 import sqlancer.common.gen.ExpressionGenerator;
 import sqlancer.common.gen.NoRECGenerator;
+import sqlancer.common.gen.TLPWhereGenerator;
 import sqlancer.common.schema.AbstractTables;
 import sqlancer.sqlite3.SQLite3GlobalState;
 import sqlancer.sqlite3.ast.SQLite3Aggregate;
@@ -50,7 +51,8 @@
 import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table;
 
 public class SQLite3ExpressionGenerator implements ExpressionGenerator<SQLite3Expression>,
-        NoRECGenerator<SQLite3Select, Join, SQLite3Expression, SQLite3Table, SQLite3Column> {
+        NoRECGenerator<SQLite3Select, Join, SQLite3Expression, SQLite3Table, SQLite3Column>,
+        TLPWhereGenerator<SQLite3Select, Join, SQLite3Expression, SQLite3Table, SQLite3Column> {
 
     private SQLite3RowValue rw;
     private final SQLite3GlobalState globalState;
@@ -133,6 +135,7 @@ public static SQLite3Expression getRandomLiteralValue(SQLite3GlobalState globalS
         return new SQLite3ExpressionGenerator(globalState).getRandomLiteralValueInternal(globalState.getRandomly());
     }
 
+    @Override
     public List<SQLite3Expression> generateOrderBys() {
         List<SQLite3Expression> expressions = new ArrayList<>();
         for (int i = 0; i < Randomly.smallNumber() + 1; i++) {
@@ -747,6 +750,18 @@ public List<SQLite3Expression> getTableRefs() {
         return tableRefs;
     }
 
+    @Override
+    public List<SQLite3Expression> generateFetchColumns(boolean shouldCreateDummy) {
+        List<SQLite3Expression> columns = new ArrayList<>();
+        if (shouldCreateDummy && Randomly.getBoolean()) {
+            columns.add(new SQLite3ColumnName(SQLite3Column.createDummy("*"), null));
+        } else {
+            columns = Randomly.nonEmptySubset(this.columns).stream().map(c -> new SQLite3ColumnName(c, null))
+                    .collect(Collectors.toList());
+        }
+        return columns;
+    }
+
     @Override
     public String generateOptimizedQueryString(SQLite3Select select, SQLite3Expression whereCondition,
             boolean shouldUseAggregate) {