[test](jdbc catalog) add some jdbc query tvf test (#45938)

apache · Dec 26, 2024 · 57a7b75 · 57a7b75
1 parent e24e6d1
commit 57a7b75
Show file tree

Hide file tree

Showing 10 changed files with 56 additions and 26 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcPostgreSQLClient.java
@@ -25,6 +25,8 @@
 import org.apache.doris.datasource.jdbc.util.JdbcFieldSchema;
 
 import com.google.common.collect.Lists;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 
 import java.sql.Connection;
 import java.sql.DatabaseMetaData;
@@ -37,6 +39,7 @@
 import java.util.Optional;
 
 public class JdbcPostgreSQLClient extends JdbcClient {
+    private static final Logger LOG = LogManager.getLogger(JdbcPostgreSQLClient.class);
 
     private static final String[] supportedInnerType = new String[] {
             "int2", "int4", "int8", "smallserial", "serial",
@@ -168,6 +171,7 @@ protected Type jdbcTypeToDoris(JdbcFieldSchema fieldSchema) {
     private Type convertArrayType(JdbcFieldSchema fieldSchema) {
         int arrayDimensions = fieldSchema.getArrayDimensions().orElse(0);
         if (arrayDimensions == 0) {
+            LOG.warn("postgres array type without dimensions");
             return Type.UNSUPPORTED;
         }
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/util/JdbcFieldSchema.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/util/JdbcFieldSchema.java
@@ -104,6 +104,7 @@ public JdbcFieldSchema(ResultSetMetaData metaData, int columnIndex) throws SQLEx
         this.dataTypeName = Optional.ofNullable(metaData.getColumnTypeName(columnIndex));
         this.columnSize = Optional.of(metaData.getPrecision(columnIndex));
         this.decimalDigits = Optional.of(metaData.getScale(columnIndex));
+        this.arrayDimensions = Optional.of(0);
     }
 
     public int requiredColumnSize() {

diff --git a/regression-test/data/external_table_p0/jdbc/test_db2_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_db2_jdbc_catalog.out
@@ -6,33 +6,38 @@ internal
 db2_jdbc_catalog
 
 -- !sample_table_desc --
-BIGINT_COLUMN	BIGINT	Yes	true	\N	
-CHAR_COLUMN	CHARACTER	Yes	true	\N	
-CHAR_VARYING_COLUMN	VARCHAR(255)	Yes	true	\N	
-CLOB_COLUMN	TEXT	Yes	true	\N	
-DATE_COLUMN	DATE	Yes	true	\N	
-DECFLOAT_COLUMN	DECIMAL(34, 0)	Yes	true	\N	
-DECIMAL_COLUMN	DECIMAL(31, 10)	Yes	true	\N	
-DOUBLE_COLUMN	DOUBLE	Yes	true	\N	
-DOUBLE_PRECISION_COLUMN	DOUBLE	Yes	true	\N	
-FLOAT_COLUMN	DOUBLE	Yes	true	\N	
-ID_COLUMN	INT	No	true	\N	
-INT_COLUMN	INT	Yes	true	\N	
-LONG_VARCHARPHIC_COLUMN	TEXT	Yes	true	\N	
-LONG_VARCHAR_COLUMN	VARCHAR(32700)	Yes	true	\N	
-NUMERIC_COLUMN	DECIMAL(5, 0)	Yes	true	\N	
-REAL_COLUMN	FLOAT	Yes	true	\N	
-SMALLINT_COLUMN	SMALLINT	Yes	true	\N	
-TIMESTAMP_COLUMN	DATETIME(6)	Yes	true	\N	
-TIME_COLUMN	TEXT	Yes	true	\N	
-VARCHARPHIC_COLUMN	TEXT	Yes	true	\N	
-VARCHAR_COLUMN	VARCHAR(255)	Yes	true	\N	
+BIGINT_COLUMN	bigint	Yes	true	\N	
+CHAR_COLUMN	character(255)	Yes	true	\N	
+CHAR_VARYING_COLUMN	varchar(255)	Yes	true	\N	
+CLOB_COLUMN	text	Yes	true	\N	
+DATE_COLUMN	date	Yes	true	\N	
+DECFLOAT_COLUMN	decimal(34,0)	Yes	true	\N	
+DECIMAL_COLUMN	decimal(31,10)	Yes	true	\N	
+DOUBLE_COLUMN	double	Yes	true	\N	
+DOUBLE_PRECISION_COLUMN	double	Yes	true	\N	
+FLOAT_COLUMN	double	Yes	true	\N	
+ID_COLUMN	int	No	true	\N	
+INT_COLUMN	int	Yes	true	\N	
+LONG_VARCHARPHIC_COLUMN	text	Yes	true	\N	
+LONG_VARCHAR_COLUMN	varchar(32700)	Yes	true	\N	
+NUMERIC_COLUMN	decimal(5,0)	Yes	true	\N	
+REAL_COLUMN	float	Yes	true	\N	
+SMALLINT_COLUMN	smallint	Yes	true	\N	
+TIMESTAMP_COLUMN	datetime(6)	Yes	true	\N	
+TIME_COLUMN	text	Yes	true	\N	
+VARCHARPHIC_COLUMN	text	Yes	true	\N	
+VARCHAR_COLUMN	varchar(255)	Yes	true	\N	
 
 -- !sample_table_select --
 \N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N
 123	1234567890.1234567890	1	12345.6789	12345.679	1.2345678901234567E9	1.2345678901234567E9	123	12345	123456789012345	Varchar text	Varcharphic text	Long varchar text	Long varcharphic text	Char varying text	Char text	2024-01-24	2024-01-24T12:34:56.789	12:34:56	Sample CLOB text
 123	1234567890.1234567890	1	12345.6789	12345.679	1.2345678901234567E9	1.2345678901234567E9	123	12345	123456789012345	中文一	中文二	中文三	中文四	中文五	中文六	2024-01-24	2024-01-24T12:34:56.789	12:34:56	中文七
 
+-- !sample_table_select_tvf --
+1	123	1234567890.1234567890	1	12345.6789	12345.679	1.2345678901234567E9	1.2345678901234567E9	123	12345	123456789012345	Varchar text	Varcharphic text	Long varchar text	Long varcharphic text	Char varying text	Char text	2024-01-24	2024-01-24T12:34:56.789	12:34:56	Sample CLOB text
+2	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N
+3	123	1234567890.1234567890	1	12345.6789	12345.679	1.2345678901234567E9	1.2345678901234567E9	123	12345	123456789012345	中文一	中文二	中文三	中文四	中文五	中文六	2024-01-24	2024-01-24T12:34:56.789	12:34:56	中文七
+
 -- !sample_table_insert --
 \N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N
 \N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N
@@ -62,3 +67,6 @@ mysql
 -- !select_xml --
 1000	<catalog><book><author> Gambardella Matthew</author><title>XML Developers Guide</title><genre>Computer</genre><price>44.95</price><publish_date>2000-10-01</publish_date><description>An in-depth look at creating application\n                          with XML</description></book></catalog>
 
+-- !select_xml_tvf --
+1000	<catalog><book><author> Gambardella Matthew</author><title>XML Developers Guide</title><genre>Computer</genre><price>44.95</price><publish_date>2000-10-01</publish_date><description>An in-depth look at creating application\n                          with XML</description></book></catalog>
+
diff --git a/regression-test/data/external_table_p0/jdbc/test_oracle_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_oracle_jdbc_catalog.out
@@ -157,6 +157,10 @@ doris3	20
 1	111	123	7456123.89	573	34	673.43	34.1264	60.0	23.231	99	9999	999999999	999999999999999999	999	99999	9999999999	9999999999999999999	1	china 	beijing	alice	abcdefghrjkmnopq	123.45	12300	0.0012345	2022-01-21T05:23:01	2019-11-12T20:33:57.999	2019-11-12T20:33:57.999998	2019-11-12T20:33:57.999996	2019-11-12T20:33:57.999997	223-9	12 10:23:1.123457
 2	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N
 
+-- !select_all_types_tvf --
+1	111	123	7456123.89	573	34	673.43	34.1264	60	23.231	99	9999	999999999	999999999999999999	999	99999	9999999999	9999999999999999999	1	china 	beijing	alice	abcdefghrjkmnopq	123.45	12300	0.0012345	2022-01-21T05:23:01	2019-11-12T20:33:57.999	2019-11-12T20:33:57.999998	2019-11-12T20:33:57.999996	2019-11-12T20:33:57.999997	223-9	12 10:23:1.123457
+2	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N
+
 -- !ctas --
 1	111	123	7456123.89	573	34	673.43	34.1264	60.0	23.231	99	9999	999999999	999999999999999999	999	99999	9999999999	9999999999999999999	1	china 	beijing	alice	abcdefghrjkmnopq	123.45	12300	0.0012345	2022-01-21T05:23:01	2019-11-12T20:33:57.999	2019-11-12T20:33:57.999998	2019-11-12T20:33:57.999996	2019-11-12T20:33:57.999997	223-9	12 10:23:1.123457
 2	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N

diff --git a/regression-test/data/external_table_p0/jdbc/test_pg_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_pg_jdbc_catalog.out
@@ -2234,6 +2234,10 @@ doris3	20
 1	abc	def	2022-10-11	1	2	3	2022-10-22T10:59:59	34.123	false	12.123456	10.16.10.14/32	10.16.10.14	ff:ff:ff:ff:ff:aa	1010101010	01010	1	{"id": 1}	(1.0,1.0)	{1.0,1.0,1.0}	[(1.0,1.0),(2.0,2.0)]	(2.0,2.0),(1.0,1.0)	((1.0,1.0),(2.0,2.0),(2.0,1.0))	((1.0,1.0),(2.0,2.0),(2.0,1.0))	<(0.0,0.0),1.0>
 2	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	2	\N	\N	\N	\N	\N	\N	\N	\N
 
+-- !select_all_types_tvf --
+1	abc	def	2022-10-11	1	2	3	2022-10-22T10:59:59	34.123	false	12.123456	10.16.10.14/32	10.16.10.14	ff:ff:ff:ff:ff:aa	1010101010	01010	1	{"id": 1}	(1.0,1.0)	{1.0,1.0,1.0}	[(1.0,1.0),(2.0,2.0)]	(2.0,2.0),(1.0,1.0)	((1.0,1.0),(2.0,2.0),(2.0,1.0))	((1.0,1.0),(2.0,2.0),(2.0,1.0))	<(0.0,0.0),1.0>
+2	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	2	\N	\N	\N	\N	\N	\N	\N	\N
+
 -- !select_all_arr_types --
 1	["A", "B", "C"]	["Hello", "World"]	["Text1", "Text2"]	["2024-01-01", "2024-01-02"]	[10, 20]	[100, 200]	[1000, 2000]	["2024-01-01 10:00:00.000000", "2024-01-02 12:00:00.000000"]	["2024-01-01 18:00:00.000000", "2024-01-02 20:00:00.000000"]	[1, 0]	[1.1, 2.2]	[3.3, 4.4]
 2	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N

diff --git a/regression-test/data/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.out b/regression-test/data/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.out
@@ -73,10 +73,6 @@ sys
 2	alice	19
 3	bob	20
 
--- !filter4_old --
-2	alice	19
-3	bob	20
-
 -- !id --
 2
 
@@ -130,6 +126,10 @@ timestamp_col	text	Yes	true	\N
 -- !query_timestamp --
 1
 
+-- !all_types_tvf --
+1	doris	18	0	1	1	123.123	123.123	123.123	12345678901234567890123456789012345678	12345678901234567890123456789012345678	1234567890123456789012345678.0123456789	1234567890123456789012345678.0123456789	Make Doris Great!   	Make Doris Great!	Make Doris Great!	Make Doris Great!   	Make Doris Great!	Make Doris Great!	2023-01-17	16:49:05.123	2023-01-17T16:49:05	2023-01-17T16:49:05.123456	2023-01-17T16:49	2023-01-17 16:49:05 +08:00	Make Doris Great!	Make Doris Great!	922337203685477.5807	214748.3647	false
+2	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N	\N
+
 -- !sql --
 db_accessadmin
 db_backupoperator

diff --git a/regression-test/suites/external_table_p0/jdbc/test_db2_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_db2_jdbc_catalog.groovy
@@ -202,7 +202,7 @@ suite("test_db2_jdbc_catalog", "p0,external,db2,external_docker,external_docker_
                 "type"="jdbc",
                 "user"="db2inst1",
                 "password"="123456",
-                "jdbc_url" = "jdbc:db2://${externalEnvIp}:${db2_port}/doris:allowNextOnExhaustedResultSet=1;resultSetHoldability=1",
+                "jdbc_url" = "jdbc:db2://${externalEnvIp}:${db2_port}/doris:allowNextOnExhaustedResultSet=1;resultSetHoldability=1;",
                 "driver_url" = "${driver_url}",
                 "driver_class" = "com.ibm.db2.jcc.DB2Driver"
             );"""
@@ -215,6 +215,8 @@ suite("test_db2_jdbc_catalog", "p0,external,db2,external_docker,external_docker_
 
             order_qt_sample_table_select  """ select * except(ID_COLUMN) from ${sample_table} order by 1; """
 
+            order_qt_sample_table_select_tvf """ select * from query('catalog' = 'db2_jdbc_catalog', 'query' = 'select * from DORIS_TEST.SAMPLE_TABLE;') order by 1; """
+
             sql """INSERT INTO ${sample_table} (
                        numeric_column,
                        decimal_column,
@@ -263,6 +265,8 @@ suite("test_db2_jdbc_catalog", "p0,external,db2,external_docker,external_docker_
 
             order_qt_select_xml "select * from TEST.BOOKS;"
 
+            order_qt_select_xml_tvf "select * from query('catalog' = '${catalog_name}', 'query' = 'select * from TEST.BOOKS;') order by 1;"
+
             sql """ drop catalog if exists ${catalog_name} """
 
             db2_docker "DROP TABLE IF EXISTS doris_test.sample_table;"

diff --git a/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy
@@ -159,6 +159,8 @@ suite("test_oracle_jdbc_catalog", "p0,external,oracle,external_docker,external_d
         // test select all types
         order_qt_select_all_types """select * from ${test_all_types}; """
 
+        order_qt_select_all_types_tvf """select * from query('catalog' = '${catalog_name}', 'query' ='select * from ${test_all_types}') order by 1; """
+
         // test test ctas
         sql """ drop table if exists internal.${internal_db_name}.${test_ctas} """
         sql """ create table internal.${internal_db_name}.${test_ctas}

diff --git a/regression-test/suites/external_table_p0/jdbc/test_pg_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_pg_jdbc_catalog.groovy
@@ -146,6 +146,8 @@ suite("test_pg_jdbc_catalog", "p0,external,pg,external_docker,external_docker_pg
         // test select all types
         order_qt_select_all_types """select * from ${test_all_types}; """
 
+        order_qt_select_all_types_tvf """ select * from query('catalog' = '${catalog_name}', 'query' = 'select * from catalog_pg_test.${test_all_types};') order by 1"""
+
         // test select all array types
         order_qt_select_all_arr_types """select *  from test_all_support_types_array order by 1;"""
 

diff --git a/regression-test/suites/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.groovy b/regression-test/suites/external_table_p0/jdbc/test_sqlserver_jdbc_catalog.groovy
@@ -81,6 +81,7 @@ suite("test_sqlserver_jdbc_catalog", "p0,external,sqlserver,external_docker,exte
         order_qt_desc_timestamp """desc dbo.test_timestamp; """
         order_qt_query_timestamp """select count(timestamp_col) from dbo.test_timestamp; """
 
+        order_qt_all_types_tvf """ select * from query('catalog' = '${catalog_name}', 'query' = 'select * from all_type;') order by 1"""
 
         sql """ drop catalog if exists ${catalog_name} """