Skip to content

Commit

Permalink
Support local transactions of ClickHouse under GraalVM Native Image
Browse files Browse the repository at this point in the history
  • Loading branch information
linghengqian committed Nov 25, 2024
1 parent 2973418 commit e6571ae
Show file tree
Hide file tree
Showing 26 changed files with 91 additions and 144 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ CREATE DATABASE demo_ds_2;
```sql
-- noinspection SqlNoDataSourceInspectionForFile
create table IF NOT EXISTS t_order (
order_id Int64 NOT NULL DEFAULT rand(),
order_id Int64 NOT NULL,
order_type Int32,
user_id Int32 NOT NULL,
address_id Int64 NOT NULL,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ to connect to ClickHouse and execute the following SQL.
```sql
-- noinspection SqlNoDataSourceInspectionForFile
create table IF NOT EXISTS t_order (
order_id Int64 NOT NULL DEFAULT rand(),
order_id Int64 NOT NULL,
order_type Int32,
user_id Int32 NOT NULL,
address_id Int64 NOT NULL,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
"name":"[Lcom.github.dockerjava.api.model.VolumesFrom;"
},
{
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.connection.DriverDatabaseConnectionManager$$Lambda/0x00007fb05bce1828"},
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.connection.DriverDatabaseConnectionManager$$Lambda/0x00007f4b07ce0c30"},
"name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;"
},
{
Expand Down Expand Up @@ -75,10 +75,6 @@
"condition":{"typeReachable":"org.apache.shardingsphere.mode.repository.standalone.jdbc.JDBCRepository"},
"name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;"
},
{
"condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.connector.jdbc.datasource.JDBCBackendDataSource"},
"name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;"
},
{
"condition":{"typeReachable":"org.apache.shardingsphere.proxy.frontend.postgresql.command.query.extended.Portal"},
"name":"[Lcom.zaxxer.hikari.util.ConcurrentBag$IConcurrentBagEntry;"
Expand Down Expand Up @@ -1071,7 +1067,7 @@
"name":"org.apache.shardingsphere.driver.yaml.YamlJDBCConfiguration",
"allDeclaredFields":true,
"queryAllPublicMethods":true,
"methods":[{"name":"<init>","parameterTypes":[] }, {"name":"setDataSources","parameterTypes":["java.util.Map"] }, {"name":"setMode","parameterTypes":["org.apache.shardingsphere.infra.yaml.config.pojo.mode.YamlModeConfiguration"] }, {"name":"setProps","parameterTypes":["java.util.Properties"] }, {"name":"setRules","parameterTypes":["java.util.Collection"] }, {"name":"setSqlParser","parameterTypes":["org.apache.shardingsphere.parser.yaml.config.YamlSQLParserRuleConfiguration"] }, {"name":"setTransaction","parameterTypes":["org.apache.shardingsphere.transaction.yaml.config.YamlTransactionRuleConfiguration"] }]
"methods":[{"name":"<init>","parameterTypes":[] }, {"name":"setDataSources","parameterTypes":["java.util.Map"] }, {"name":"setMode","parameterTypes":["org.apache.shardingsphere.infra.yaml.config.pojo.mode.YamlModeConfiguration"] }, {"name":"setRules","parameterTypes":["java.util.Collection"] }, {"name":"setSqlParser","parameterTypes":["org.apache.shardingsphere.parser.yaml.config.YamlSQLParserRuleConfiguration"] }, {"name":"setTransaction","parameterTypes":["org.apache.shardingsphere.transaction.yaml.config.YamlTransactionRuleConfiguration"] }]
},
{
"condition":{"typeReachable":"org.apache.shardingsphere.infra.util.yaml.YamlEngine"},
Expand Down Expand Up @@ -2078,7 +2074,7 @@
"queryAllDeclaredMethods":true
},
{
"condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.listener.DatabaseMetaDataChangedListener$$Lambda/0x00007fb05bb277d8"},
"condition":{"typeReachable":"org.apache.shardingsphere.mode.manager.cluster.listener.DatabaseMetaDataChangedListener$$Lambda/0x00007f4b07b277d8"},
"name":"org.apache.shardingsphere.mode.manager.cluster.event.subscriber.dispatch.MetaDataChangedSubscriber"
},
{
Expand Down Expand Up @@ -3131,6 +3127,16 @@
"name":"org.apache.shardingsphere.single.yaml.config.YamlSingleRuleConfiguration",
"allDeclaredFields":true
},
{
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSphereStatement"},
"name":"org.apache.shardingsphere.sql.parser.clickhouse.parser.ClickHouseLexer",
"methods":[{"name":"<init>","parameterTypes":["org.antlr.v4.runtime.CharStream"] }]
},
{
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSphereStatement"},
"name":"org.apache.shardingsphere.sql.parser.clickhouse.parser.ClickHouseParser",
"methods":[{"name":"<init>","parameterTypes":["org.antlr.v4.runtime.TokenStream"] }]
},
{
"condition":{"typeReachable":"org.apache.shardingsphere.proxy.backend.handler.ProxySQLComQueryParser"},
"name":"org.apache.shardingsphere.sql.parser.clickhouse.parser.ClickHouseParserFacade"
Expand All @@ -3140,6 +3146,11 @@
"name":"org.apache.shardingsphere.sql.parser.clickhouse.visitor.statement.ClickHouseStatementVisitorFacade",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
{
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSphereStatement"},
"name":"org.apache.shardingsphere.sql.parser.clickhouse.visitor.statement.type.ClickHouseDMLStatementVisitor",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
{
"condition":{"typeReachable":"org.apache.shardingsphere.sql.parser.core.database.cache.ParseTreeCacheBuilder"},
"name":"org.apache.shardingsphere.sql.parser.core.database.cache.ParseTreeCacheLoader"
Expand Down Expand Up @@ -3327,6 +3338,11 @@
"name":"org.apache.shardingsphere.sql.parser.statement.clickhouse.dml.ClickHouseSelectStatement",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
{
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSphereStatement"},
"name":"org.apache.shardingsphere.sql.parser.statement.clickhouse.dml.ClickHouseSelectStatement",
"methods":[{"name":"<init>","parameterTypes":[] }]
},
{
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.statement.ShardingSpherePreparedStatement"},
"name":"org.apache.shardingsphere.sql.parser.statement.hive.dml.HiveInsertStatement",
Expand Down Expand Up @@ -3607,5 +3623,10 @@
{
"condition":{"typeReachable":"org.apache.shardingsphere.proxy.initializer.BootstrapInitializer"},
"name":"org.apache.shardingsphere.transaction.yaml.config.YamlTransactionRuleConfigurationCustomizer"
},
{
"condition":{"typeReachable":"org.apache.shardingsphere.infra.executor.kernel.ExecutorEngine"},
"name":"sun.security.provider.SecureRandom",
"methods":[{"name":"<init>","parameterTypes":[] }, {"name":"<init>","parameterTypes":["java.security.SecureRandomParameters"] }]
}
]
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,12 @@
}, {
"condition":{"typeReachable":"org.apache.shardingsphere.transaction.xa.atomikos.manager.AtomikosTransactionManagerProvider"},
"pattern":"\\QMETA-INF/services/com.atomikos.recovery.OltpLogFactory\\E"
}, {
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.connection.DriverDatabaseConnectionManager"},
"pattern":"\\QMETA-INF/services/com.clickhouse.client.ClickHouseClient\\E"
}, {
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.connection.DriverDatabaseConnectionManager$$Lambda/0x00007f4b07b34b38"},
"pattern":"\\QMETA-INF/services/com.clickhouse.client.ClickHouseClient\\E"
}, {
"condition":{"typeReachable":"org.apache.shardingsphere.driver.jdbc.core.datasource.ShardingSphereDataSource"},
"pattern":"\\QMETA-INF/services/com.clickhouse.client.ClickHouseClient\\E"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,12 @@
import org.apache.shardingsphere.test.natived.commons.repository.AddressRepository;
import org.apache.shardingsphere.test.natived.commons.repository.OrderItemRepository;
import org.apache.shardingsphere.test.natived.commons.repository.OrderRepository;
import org.awaitility.Awaitility;

import javax.sql.DataSource;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
Expand Down Expand Up @@ -84,6 +86,7 @@ public void processSuccessInClickHouse() throws SQLException {
assertThat(orderRepository.selectAll(), equalTo(Collections.emptyList()));
assertThat(orderItemRepository.selectAll(), equalTo(Collections.emptyList()));
assertThat(addressRepository.selectAll(), equalTo(Collections.emptyList()));
orderItemRepository.assertRollbackWithTransactions();
}

private void extracted() throws SQLException {
Expand Down Expand Up @@ -173,11 +176,14 @@ public void deleteData(final Collection<Long> orderIds) throws SQLException {

/**
* Delete data in ClickHouse.
* TODO It is necessary to avoid the use of {@code Awaitility.await().pollDelay(Duration.ofSeconds(9L)).until(()->true)}.
* After ClickHouse enables experimental transactions, performance drops significantly.
*
* @param orderIds orderId of the insert statement.
* @throws SQLException An exception that provides information on a database access error or other errors.
*/
public void deleteDataInClickHouse(final Collection<Long> orderIds) throws SQLException {
Awaitility.await().pollDelay(Duration.ofSeconds(9L)).until(() -> true);
long count = 1L;
for (Long each : orderIds) {
orderRepository.deleteInClickHouse(each);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ public void createTableInSQLServer() throws SQLException {
*/
public void createTableIfNotExistsInClickHouse() throws SQLException {
String sql = "create table IF NOT EXISTS t_order_item( \n"
+ "order_item_id Int64 NOT NULL DEFAULT rand(), \n"
+ "order_item_id Int64 NOT NULL, \n"
+ "order_id Int64 NOT NULL, \n"
+ "user_id Int32 NOT NULL, \n"
+ "phone String,\n"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ public void createTableInSQLServer() throws SQLException {
*/
public void createTableIfNotExistsInClickHouse() throws SQLException {
String sql = "create table IF NOT EXISTS t_order( \n"
+ "order_id Int64 NOT NULL DEFAULT rand(), \n"
+ "order_id Int64 NOT NULL, \n"
+ "order_type Int32, \n"
+ "user_id Int32 NOT NULL, \n"
+ "address_id Int64 NOT NULL,\n"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,11 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledInNativeImage;
import org.testcontainers.clickhouse.ClickHouseContainer;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.Network;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.utility.MountableFile;

import javax.sql.DataSource;
import java.sql.Connection;
Expand All @@ -46,13 +49,24 @@
* Cannot use testcontainers-java style jdbcURL for Clickhouse Server due to unresolved
* <a href="https://github.com/testcontainers/testcontainers-java/issues/8736">testcontainers/testcontainers-java#8736</a>.
*/
@SuppressWarnings("SqlNoDataSourceInspection")
@SuppressWarnings({"SqlNoDataSourceInspection", "resource"})
@EnabledInNativeImage
@Testcontainers
class ClickHouseTest {

private static final Network NETWORK = Network.newNetwork();

@Container
private static final GenericContainer<?> ZOOKEEPER_CONTAINER = new GenericContainer<>("zookeeper:3.9.3-jre-17")
.withNetwork(NETWORK)
.withNetworkAliases("foo");

@Container
public static final ClickHouseContainer CONTAINER = new ClickHouseContainer("clickhouse/clickhouse-server:24.10.2.80");
public static final ClickHouseContainer CONTAINER = new ClickHouseContainer("clickhouse/clickhouse-server:24.10.2.80")
.withCopyFileToContainer(MountableFile.forClasspathResource("test-native/xml/clickhouse-transactions.xml"),
"/etc/clickhouse-server/config.d/transactions.xml")
.withNetwork(NETWORK)
.dependsOn(ZOOKEEPER_CONTAINER);

private static final String SYSTEM_PROP_KEY_PREFIX = "fixture.test-native.yaml.database.clickhouse.";

Expand All @@ -69,6 +83,7 @@ static void beforeAll() {

@AfterAll
static void afterAll() {
NETWORK.close();
System.clearProperty(SYSTEM_PROP_KEY_PREFIX + "ds0.jdbc-url");
System.clearProperty(SYSTEM_PROP_KEY_PREFIX + "ds1.jdbc-url");
System.clearProperty(SYSTEM_PROP_KEY_PREFIX + "ds2.jdbc-url");
Expand Down Expand Up @@ -125,9 +140,9 @@ private DataSource createDataSource() throws SQLException {
HikariConfig config = new HikariConfig();
config.setDriverClassName("org.apache.shardingsphere.driver.ShardingSphereDriver");
config.setJdbcUrl("jdbc:shardingsphere:classpath:test-native/yaml/jdbc/databases/clickhouse.yaml?placeholder-type=system_props");
System.setProperty(SYSTEM_PROP_KEY_PREFIX + "ds0.jdbc-url", jdbcUrlPrefix + "demo_ds_0");
System.setProperty(SYSTEM_PROP_KEY_PREFIX + "ds1.jdbc-url", jdbcUrlPrefix + "demo_ds_1");
System.setProperty(SYSTEM_PROP_KEY_PREFIX + "ds2.jdbc-url", jdbcUrlPrefix + "demo_ds_2");
System.setProperty(SYSTEM_PROP_KEY_PREFIX + "ds0.jdbc-url", jdbcUrlPrefix + "demo_ds_0?transactionSupport=true");
System.setProperty(SYSTEM_PROP_KEY_PREFIX + "ds1.jdbc-url", jdbcUrlPrefix + "demo_ds_1?transactionSupport=true");
System.setProperty(SYSTEM_PROP_KEY_PREFIX + "ds2.jdbc-url", jdbcUrlPrefix + "demo_ds_2?transactionSupport=true");
return new HikariDataSource(config);
}

Expand All @@ -137,7 +152,7 @@ private void initTable(final String databaseName) {
Statement statement = connection.createStatement()) {
statement.executeUpdate("create table IF NOT EXISTS t_order\n"
+ "(\n"
+ " order_id Int64 NOT NULL DEFAULT rand(),\n"
+ " order_id Int64 NOT NULL,\n"
+ " order_type Int32,\n"
+ " user_id Int32 NOT NULL,\n"
+ " address_id Int64 NOT NULL,\n"
Expand All @@ -147,7 +162,7 @@ private void initTable(final String databaseName) {
+ " order by (order_id)");
statement.executeUpdate("create table IF NOT EXISTS t_order_item\n"
+ "(\n"
+ " order_item_id Int64 NOT NULL DEFAULT rand(),\n"
+ " order_item_id Int64 NOT NULL,\n"
+ " order_id Int64 NOT NULL,\n"
+ " user_id Int32 NOT NULL,\n"
+ " phone String,\n"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<clickhouse>
<allow_experimental_transactions>1</allow_experimental_transactions>
<zookeeper>
<node index="1">
<host>foo</host>
<port>2181</port>
</node>
</zookeeper>
</clickhouse>
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,6 @@
# limitations under the License.
#

mode:
type: Standalone
repository:
type: JDBC

dataSources:
ds_0:
dataSourceClassName: com.zaxxer.hikari.HikariDataSource
Expand Down Expand Up @@ -73,6 +68,3 @@ rules:
- !BROADCAST
tables:
- t_address

props:
sql-show: false
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,6 @@
# limitations under the License.
#

mode:
type: Standalone
repository:
type: JDBC

dataSources:
ds_0:
dataSourceClassName: com.zaxxer.hikari.HikariDataSource
Expand Down Expand Up @@ -67,6 +62,3 @@ rules:
- !BROADCAST
tables:
- t_address

props:
sql-show: false
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,6 @@
# limitations under the License.
#

mode:
type: Standalone
repository:
type: JDBC

dataSources:
ds_0:
dataSourceClassName: com.zaxxer.hikari.HikariDataSource
Expand Down Expand Up @@ -67,6 +62,3 @@ rules:
- !BROADCAST
tables:
- t_address

props:
sql-show: false
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,6 @@
# limitations under the License.
#

mode:
type: Standalone
repository:
type: JDBC

dataSources:
ds_0:
dataSourceClassName: com.zaxxer.hikari.HikariDataSource
Expand Down Expand Up @@ -73,6 +68,3 @@ rules:
- !BROADCAST
tables:
- t_address

props:
sql-show: false
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,6 @@
# limitations under the License.
#

mode:
type: Standalone
repository:
type: JDBC

dataSources:
ds_0:
dataSourceClassName: com.zaxxer.hikari.HikariDataSource
Expand Down Expand Up @@ -73,6 +68,3 @@ rules:
- !BROADCAST
tables:
- t_address

props:
sql-show: false
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,6 @@
# limitations under the License.
#

mode:
type: Standalone
repository:
type: JDBC

dataSources:
ds_0:
dataSourceClassName: com.zaxxer.hikari.HikariDataSource
Expand Down Expand Up @@ -67,6 +62,3 @@ rules:
- !BROADCAST
tables:
- t_address

props:
sql-show: false
Loading

0 comments on commit e6571ae

Please sign in to comment.