Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Feature][scaleph-engine-sql-gateway] add flink catalog implementions sakura and sql examples #589

Merged
merged 15 commits into from
Aug 6, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion scaleph-engine/scaleph-engine-sql-gateway/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
<artifactId>scaleph-common</artifactId>
</dependency>
<dependency>
<groupId>cn.sliew</groupId>
<groupId>${project.parent.groupId}</groupId>
<artifactId>scaleph-kubernetes</artifactId>
</dependency>

Expand Down Expand Up @@ -71,5 +71,10 @@
<groupId>org.apache.paimon</groupId>
<artifactId>paimon-flink-${flink.base.version}</artifactId>
</dependency>
<dependency>
<groupId>cn.sliew</groupId>
<artifactId>sakura-catalog</artifactId>
<version>0.0.1-SNAPSHOT</version>
</dependency>
</dependencies>
</project>
6 changes: 0 additions & 6 deletions tools/docker/mysql/init.d/sakura-mysq.sql
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,7 @@ CREATE TABLE `catalog_database`
`name` VARCHAR(256) NOT NULL,
properties TEXT,
remark VARCHAR(256),
creator VARCHAR(32),
create_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
editor VARCHAR(32),
update_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (id),
UNIQUE KEY uniq_name (catalog, `name`)
Expand All @@ -29,9 +27,7 @@ CREATE TABLE `catalog_table`
original_query TEXT,
expanded_query TEXT,
remark VARCHAR(256),
creator VARCHAR(32),
create_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
editor VARCHAR(32),
update_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (id),
UNIQUE KEY uniq_name (database_id, kind, `name`)
Expand All @@ -46,9 +42,7 @@ CREATE TABLE `catalog_function`
class_name VARCHAR(256) NOT NULL,
function_language VARCHAR(8) NOT NULL,
remark VARCHAR(256),
creator VARCHAR(32),
create_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
editor VARCHAR(32),
update_time DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (id),
UNIQUE KEY uniq_name (database_id, `name`)
Expand Down
15 changes: 15 additions & 0 deletions tools/docker/mysql/init.d/scaleph-ws-mysql.sql
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,10 @@ INSERT INTO `ws_flink_artifact`(`id`, `project_id`, `type`, `name`, `remark`, `c
VALUES (6, 1, '0', 'catalog-example', NULL, 'sys', 'sys');
INSERT INTO `ws_flink_artifact` (`id`, `project_id`, `type`, `name`, `remark`, `creator`, `editor`)
VALUES (7, 1, '0', 'select-example', NULL, 'sys', 'sys');
INSERT INTO `ws_flink_artifact` (`id`, `project_id`, `type`, `name`, `remark`, `creator`, `editor`)
VALUES (8, 1, '0', 'jdbc&paimon-example', 'jdbc 和 paimon catalog example', 'sys', 'sys');
INSERT INTO `ws_flink_artifact` (`id`, `project_id`, `type`, `name`, `remark`, `creator`, `editor`)
VALUES (9, 1, '0', 'sakura-example', 'sakura catalog example', 'sys', 'sys');

drop table if exists ws_flink_artifact_jar;
create table ws_flink_artifact_jar
Expand Down Expand Up @@ -111,6 +115,17 @@ VALUES (5, 7, '1.17.1',
'CREATE CATALOG my_catalog WITH (\n \'type\' = \'generic_in_memory\'\n);\n\nCREATE DATABASE my_catalog.my_database;\n\n\nCREATE TABLE my_catalog.my_database.source_table (\n `id` bigint,\n `name` string,\n `age` int,\n `address` string,\n `create_time`TIMESTAMP(3),\n `update_time`TIMESTAMP(3),\n WATERMARK FOR `update_time` AS update_time - INTERVAL \'1\' MINUTE\n)\nCOMMENT \'\'\nWITH (\n \'connector\' = \'datagen\'\n);\n\nSELECT * FROM my_catalog.my_database.source_table;',
'1', 'sys', 'sys');

INSERT INTO `ws_flink_artifact_sql` (`id`, `flink_artifact_id`, `flink_version`, `script`, `current`, `creator`,
`editor`)
VALUES (6, 8, '1.17.1',
'CREATE CATALOG jdbc_scaleph WITH(\n \'type\' = \'jdbc\',\n \'base-url\' = \'jdbc:mysql://localhost:3306/data_service\',\n \'username\' = \'root\',\n \'password\' = \'123456\',\n \'default-database\' = \'data_service\'\n);\n\nCREATE CATALOG paimon_minio WITH (\n \'type\'=\'paimon\',\n \'warehouse\'=\'s3a://scaleph/paimon\',\n \'s3.endpoint\' = \'http://localhost:9000\',\n \'s3.access-key\' = \'admin\',\n \'s3.secret-key\' = \'password\',\n \'s3.path.style.access\' = \'true\'\n);\n\nCREATE DATABASE paimon_minio.jdbc_data_service;\n\nCREATE TABLE paimon_minio.jdbc_data_service.sample_data_e_commerce (\n `id` BIGINT COMMENT \'自增主键\',\n `invoice_no` STRING COMMENT \'发票号码,每笔交易分配唯一的6位整数,而退货订单的代码以字母\'\'c\'\'开头\',\n `stock_code` STRING COMMENT \'产品代码,每个不同的产品分配唯一的5位整数\',\n `description` STRING COMMENT \'产品描述,对每件产品的简略描述\',\n `quantity` INT COMMENT \'产品数量,每笔交易的每件产品的数量\',\n `invoice_date` STRING COMMENT \'发票日期和时间,每笔交易发生的日期和时间\',\n `unit_price` DECIMAL(20, 2) COMMENT \'单价(英镑),单位产品价格\',\n `customer_id`STRING COMMENT \'顾客号码,每个客户分配唯一的5位整数\',\n `country`STRING COMMENT \'国家的名字,每个客户所在国家/地区的名称\',\n PRIMARY KEY(`invoice_no`) NOT ENFORCED\n)\nCOMMENT \'E-Commerce 数据集\'\n;\n\nINSERT INTO paimon_minio.jdbc_data_service.sample_data_e_commerce\nSELECT * FROM jdbc_scaleph.data_service.sample_data_e_commerce;',
'1', 'sys', 'sys');
INSERT INTO `ws_flink_artifact_sql` (`id`, `flink_artifact_id`, `flink_version`, `script`, `current`, `creator`,
`editor`)
VALUES (7, 9, '1.17.1',
'CREATE CATALOG sakura WITH(\n \'type\' = \'sakura\',\n \'jdbcUrl\' = \'jdbc:mysql://localhost:3306/sakura\',\n \'username\' = \'root\',\n \'password\' = \'123456\',\n \'driver\' = \'com.mysql.cj.jdbc.Driver\'\n);\n\nCREATE DATABASE sakura.dev;\n\nCREATE TABLE sakura.dev.orders (\n order_number BIGINT,\n price DECIMAL(32,2),\n buyer ROW<first_name STRING, last_name STRING>,\n order_time TIMESTAMP(3)\n) WITH (\n \'connector\' = \'datagen\'\n);\n\nCREATE TABLE sakura.dev.print_table WITH (\'connector\' = \'print\')\n LIKE orders;\n\nINSERT INTO sakura.dev.print_table \nSELECT * FROM sakura.dev.orders;',
'1', 'sysc', 'sys');

drop table if exists ws_di_job;
create table ws_di_job
(
Expand Down