feat: hot start support benchmark results

DTStack · Sep 9, 2024 · aebd031 · aebd031
1 parent 8b93313
commit aebd031
Show file tree

Hide file tree

Showing 16 changed files with 411 additions and 116 deletions.
diff --git a/benchmark/sqlBenchmark.ts b/benchmark/sqlBenchmark.ts
@@ -128,8 +128,9 @@ class SqlBenchmark {
      */
     clearATNCache() {
         const caches = Object.keys(require.cache);
+        const sourcePath = path.join(__dirname, '../src');
         caches
-            .filter((cache) => cache.includes('dt-sql-parser/src'))
+            .filter((cache) => cache.includes(sourcePath))
             .forEach((moduleName) => {
                 const module = require.cache[moduleName]!;
                 // Fix Memory Leak
@@ -168,6 +169,10 @@ class SqlBenchmark {
             this.clearATNCache();
         }
 
+        if (this.isHot && loopTimes < 2) {
+            throw new Error('Hot start should run at least 2 times');
+        }
+
         for (let i = 0; i < loopTimes; i++) {
             const parser = this.getSqlParser();
             if (!parser[type] || typeof parser[type] !== 'function') return;
@@ -179,14 +184,10 @@ class SqlBenchmark {
             costTimes.push(Math.round(costTime));
         }
 
-        if (this.isHot) {
-            avgTime = costTimes[0];
-        } else {
-            const filteredData = removeOutliers(costTimes);
-            avgTime = Math.round(
-                filteredData.reduce((prev, curr) => prev + curr, 0) / filteredData.length
-            );
-        }
+        const filteredData = removeOutliers(this.isHot ? costTimes.slice(1) : costTimes);
+        avgTime = Math.round(
+            filteredData.reduce((prev, curr) => prev + curr, 0) / filteredData.length
+        );
 
         const result = {
             name,
@@ -202,7 +203,12 @@ class SqlBenchmark {
     }
 
     getLastResults() {
-        const reportPath = path.join(__dirname, `./reports/${this.language}.benchmark.md`);
+        const reportPath = path.join(
+            __dirname,
+            './reports',
+            this.isHot ? 'hot_start' : 'cold_start',
+            `${this.language}.benchmark.md`
+        );
         if (this.isRelease || !fs.existsSync(reportPath)) return null;
 
         const report = fs.readFileSync(reportPath, { encoding: 'utf-8' });
@@ -265,8 +271,11 @@ class SqlBenchmark {
         );
         const currentVersion = require('../package.json').version;
         const parsedEnvInfo = JSON.parse(envInfo);
-
-        const baseDir = path.join(__dirname, this.isRelease ? '../benchmark_reports' : './reports');
+        const baseDir = path.join(
+            __dirname,
+            this.isRelease ? '../benchmark_reports' : './reports',
+            this.isHot ? 'hot_start' : 'cold_start'
+        );
 
         if (!fs.existsSync(baseDir)) {
             fs.mkdirSync(baseDir, { recursive: true });
@@ -293,9 +302,9 @@ class SqlBenchmark {
 
         writter.writeHeader('Version', 3);
         writter.writeText(`\`nodejs\`: ${process.version}`);
-        writter.writeText(`\`dt-sql-parser\`: ${currentVersion}`);
-        writter.writeText(`\`antlr4-c3\`: ${parsedEnvInfo.npmPackages['antlr4-c3']?.installed}`);
-        writter.writeText(`\`antlr4ng\`: ${parsedEnvInfo.npmPackages['antlr4ng']?.installed}`);
+        writter.writeText(`\`dt-sql-parser\`: v${currentVersion}`);
+        writter.writeText(`\`antlr4-c3\`: v${parsedEnvInfo.npmPackages['antlr4-c3']?.installed}`);
+        writter.writeText(`\`antlr4ng\`: v${parsedEnvInfo.npmPackages['antlr4ng']?.installed}`);
         writter.writeLine();
 
         writter.writeHeader('Running Mode', 3);

diff --git a/benchmark_reports/flink.benchmark.md → ...ark_reports/cold_start/flink.benchmark.md b/benchmark_reports/flink.benchmark.md → ...ark_reports/cold_start/flink.benchmark.md
@@ -4,7 +4,7 @@
 FlinkSQL
 
 ### Report Time
-2024/8/28 11:33:36
+2024/9/9 19:55:03
 
 ### Device
 macOS 14.4.1
@@ -13,21 +13,24 @@ macOS 14.4.1
 
 ### Version
 `nodejs`: v21.6.1
-`dt-sql-parser`: 4.0.2
-`antlr4-c3`: 3.3.7
-`antlr4ng`: 2.0.11
+`dt-sql-parser`: v4.0.2
+`antlr4-c3`: v3.3.7
+`antlr4ng`: v2.0.11
+
+### Running Mode
+Cold Start
 
 ### Report
 | Benchmark Name |         Method Name        |SQL Rows|Average Time(ms)| 
 |----------------|----------------------------|--------|----------------| 
-|Query Collection|        getAllTokens        |  1015  |       225      | 
-|Query Collection|          validate          |  1015  |       235      | 
-| Insert Columns |        getAllTokens        |  1001  |       67       | 
-| Insert Columns |          validate          |  1001  |       78       | 
+|Query Collection|        getAllTokens        |  1015  |       227      | 
+|Query Collection|          validate          |  1015  |       221      | 
+| Insert Columns |        getAllTokens        |  1001  |       65       | 
+| Insert Columns |          validate          |  1001  |       65       | 
 |  Create Table  |        getAllTokens        |  1004  |       27       | 
 |  Create Table  |          validate          |  1004  |       26       | 
-|    Split SQL   |     splitSQLByStatement    |   999  |       55       | 
-|Collect Entities|       getAllEntities       |  1056  |       140      | 
-|   Suggestion   |getSuggestionAtCaretPosition|  1056  |       135      | 
+|    Split SQL   |     splitSQLByStatement    |   999  |       52       | 
+|Collect Entities|       getAllEntities       |  1056  |       141      | 
+|   Suggestion   |getSuggestionAtCaretPosition|  1056  |       131      | 
 
 
diff --git a/benchmark_reports/hive.benchmark.md → ...mark_reports/cold_start/hive.benchmark.md b/benchmark_reports/hive.benchmark.md → ...mark_reports/cold_start/hive.benchmark.md
@@ -4,7 +4,7 @@
 HiveSQL
 
 ### Report Time
-2024/8/28 11:33:36
+2024/9/9 19:55:03
 
 ### Device
 macOS 14.4.1
@@ -13,23 +13,26 @@ macOS 14.4.1
 
 ### Version
 `nodejs`: v21.6.1
-`dt-sql-parser`: 4.0.2
-`antlr4-c3`: 3.3.7
-`antlr4ng`: 2.0.11
+`dt-sql-parser`: v4.0.2
+`antlr4-c3`: v3.3.7
+`antlr4ng`: v2.0.11
+
+### Running Mode
+Cold Start
 
 ### Report
 | Benchmark Name |         Method Name        |SQL Rows|Average Time(ms)| 
 |----------------|----------------------------|--------|----------------| 
-|Query Collection|        getAllTokens        |  1015  |       180      | 
+|Query Collection|        getAllTokens        |  1015  |       185      | 
 |Query Collection|          validate          |  1015  |       179      | 
-|  Update Table  |        getAllTokens        |  1011  |       117      | 
-|  Update Table  |          validate          |  1011  |       117      | 
-| Insert Columns |        getAllTokens        |  1001  |       330      | 
-| Insert Columns |          validate          |  1001  |       323      | 
-|  Create Table  |        getAllTokens        |  1002  |       20       | 
+|  Update Table  |        getAllTokens        |  1011  |       112      | 
+|  Update Table  |          validate          |  1011  |       109      | 
+| Insert Columns |        getAllTokens        |  1001  |       329      | 
+| Insert Columns |          validate          |  1001  |       329      | 
+|  Create Table  |        getAllTokens        |  1002  |       21       | 
 |  Create Table  |          validate          |  1002  |       20       | 
-|    Split SQL   |     splitSQLByStatement    |  1001  |       71       | 
-|Collect Entities|       getAllEntities       |  1066  |       105      | 
-|   Suggestion   |getSuggestionAtCaretPosition|  1066  |       98       | 
+|    Split SQL   |     splitSQLByStatement    |  1001  |       72       | 
+|Collect Entities|       getAllEntities       |  1066  |       106      | 
+|   Suggestion   |getSuggestionAtCaretPosition|  1066  |       100      | 
 
 
diff --git a/benchmark_reports/impala.benchmark.md → ...rk_reports/cold_start/impala.benchmark.md b/benchmark_reports/impala.benchmark.md → ...rk_reports/cold_start/impala.benchmark.md
@@ -4,7 +4,7 @@
 ImpalaSQL
 
 ### Report Time
-2024/8/28 11:33:36
+2024/9/9 19:55:03
 
 ### Device
 macOS 14.4.1
@@ -13,23 +13,26 @@ macOS 14.4.1
 
 ### Version
 `nodejs`: v21.6.1
-`dt-sql-parser`: 4.0.2
-`antlr4-c3`: 3.3.7
-`antlr4ng`: 2.0.11
+`dt-sql-parser`: v4.0.2
+`antlr4-c3`: v3.3.7
+`antlr4ng`: v2.0.11
+
+### Running Mode
+Cold Start
 
 ### Report
 | Benchmark Name |         Method Name        |SQL Rows|Average Time(ms)| 
 |----------------|----------------------------|--------|----------------| 
-|Query Collection|        getAllTokens        |  1015  |       75       | 
-|Query Collection|          validate          |  1015  |       74       | 
-|  Update Table  |        getAllTokens        |  1011  |       115      | 
-|  Update Table  |          validate          |  1011  |       116      | 
-| Insert Columns |        getAllTokens        |  1001  |       213      | 
-| Insert Columns |          validate          |  1001  |       215      | 
+|Query Collection|        getAllTokens        |  1015  |       71       | 
+|Query Collection|          validate          |  1015  |       71       | 
+|  Update Table  |        getAllTokens        |  1011  |       113      | 
+|  Update Table  |          validate          |  1011  |       108      | 
+| Insert Columns |        getAllTokens        |  1001  |       208      | 
+| Insert Columns |          validate          |  1001  |       213      | 
 |  Create Table  |        getAllTokens        |  1002  |       23       | 
 |  Create Table  |          validate          |  1002  |       23       | 
-|    Split SQL   |     splitSQLByStatement    |  1001  |       68       | 
-|Collect Entities|       getAllEntities       |  1066  |       81       | 
-|   Suggestion   |getSuggestionAtCaretPosition|  1066  |       75       | 
+|    Split SQL   |     splitSQLByStatement    |  1001  |       65       | 
+|Collect Entities|       getAllEntities       |  1066  |       82       | 
+|   Suggestion   |getSuggestionAtCaretPosition|  1066  |       83       | 
 
 
diff --git a/benchmark_reports/mysql.benchmark.md → ...ark_reports/cold_start/mysql.benchmark.md b/benchmark_reports/mysql.benchmark.md → ...ark_reports/cold_start/mysql.benchmark.md
@@ -4,7 +4,7 @@
 MySQL
 
 ### Report Time
-2024/8/28 11:33:36
+2024/9/9 19:55:03
 
 ### Device
 macOS 14.4.1
@@ -13,23 +13,26 @@ macOS 14.4.1
 
 ### Version
 `nodejs`: v21.6.1
-`dt-sql-parser`: 4.0.2
-`antlr4-c3`: 3.3.7
-`antlr4ng`: 2.0.11
+`dt-sql-parser`: v4.0.2
+`antlr4-c3`: v3.3.7
+`antlr4ng`: v2.0.11
+
+### Running Mode
+Cold Start
 
 ### Report
 | Benchmark Name |         Method Name        |SQL Rows|Average Time(ms)| 
 |----------------|----------------------------|--------|----------------| 
-|Query Collection|        getAllTokens        |  1015  |      1299      | 
-|Query Collection|          validate          |  1015  |      1290      | 
-|  Update Table  |        getAllTokens        |  1011  |       831      | 
-|  Update Table  |          validate          |  1011  |       817      | 
-| Insert Columns |        getAllTokens        |  1001  |       256      | 
-| Insert Columns |          validate          |  1001  |       261      | 
-|  Create Table  |        getAllTokens        |  1002  |       45       | 
-|  Create Table  |          validate          |  1002  |       44       | 
-|    Split SQL   |     splitSQLByStatement    |  1001  |       286      | 
-|Collect Entities|       getAllEntities       |  1066  |       517      | 
-|   Suggestion   |getSuggestionAtCaretPosition|  1066  |       486      | 
+|Query Collection|        getAllTokens        |  1015  |      1281      | 
+|Query Collection|          validate          |  1015  |      1254      | 
+|  Update Table  |        getAllTokens        |  1011  |       876      | 
+|  Update Table  |          validate          |  1011  |       842      | 
+| Insert Columns |        getAllTokens        |  1001  |       261      | 
+| Insert Columns |          validate          |  1001  |       266      | 
+|  Create Table  |        getAllTokens        |  1002  |       48       | 
+|  Create Table  |          validate          |  1002  |       45       | 
+|    Split SQL   |     splitSQLByStatement    |  1001  |       287      | 
+|Collect Entities|       getAllEntities       |  1066  |       474      | 
+|   Suggestion   |getSuggestionAtCaretPosition|  1066  |       462      | 
 
 
diff --git a/benchmark_reports/postgresql.benchmark.md → ...eports/cold_start/postgresql.benchmark.md b/benchmark_reports/postgresql.benchmark.md → ...eports/cold_start/postgresql.benchmark.md
@@ -4,7 +4,7 @@
 PostgreSQL
 
 ### Report Time
-2024/8/28 11:33:36
+2024/9/9 19:55:03
 
 ### Device
 macOS 14.4.1
@@ -13,23 +13,26 @@ macOS 14.4.1
 
 ### Version
 `nodejs`: v21.6.1
-`dt-sql-parser`: 4.0.2
-`antlr4-c3`: 3.3.7
-`antlr4ng`: 2.0.11
+`dt-sql-parser`: v4.0.2
+`antlr4-c3`: v3.3.7
+`antlr4ng`: v2.0.11
+
+### Running Mode
+Cold Start
 
 ### Report
 | Benchmark Name |         Method Name        |SQL Rows|Average Time(ms)| 
 |----------------|----------------------------|--------|----------------| 
-|Query Collection|        getAllTokens        |  1015  |      1063      | 
-|Query Collection|          validate          |  1015  |      1071      | 
-|  Update Table  |        getAllTokens        |  1011  |      1216      | 
-|  Update Table  |          validate          |  1011  |      1171      | 
-| Insert Columns |        getAllTokens        |  1001  |       538      | 
-| Insert Columns |          validate          |  1001  |       573      | 
-|  Create Table  |        getAllTokens        |  1002  |       270      | 
-|  Create Table  |          validate          |  1002  |       288      | 
-|    Split SQL   |     splitSQLByStatement    |  1001  |       579      | 
-|Collect Entities|       getAllEntities       |  1066  |       829      | 
-|   Suggestion   |getSuggestionAtCaretPosition|  1066  |       810      | 
+|Query Collection|        getAllTokens        |  1015  |      1086      | 
+|Query Collection|          validate          |  1015  |      1078      | 
+|  Update Table  |        getAllTokens        |  1011  |      1193      | 
+|  Update Table  |          validate          |  1011  |      1183      | 
+| Insert Columns |        getAllTokens        |  1001  |       539      | 
+| Insert Columns |          validate          |  1001  |       565      | 
+|  Create Table  |        getAllTokens        |  1002  |       294      | 
+|  Create Table  |          validate          |  1002  |       275      | 
+|    Split SQL   |     splitSQLByStatement    |  1001  |       597      | 
+|Collect Entities|       getAllEntities       |  1066  |       797      | 
+|   Suggestion   |getSuggestionAtCaretPosition|  1066  |       776      | 
 
 
diff --git a/benchmark_reports/spark.benchmark.md → ...ark_reports/cold_start/spark.benchmark.md b/benchmark_reports/spark.benchmark.md → ...ark_reports/cold_start/spark.benchmark.md
@@ -4,7 +4,7 @@
 SparkSQL
 
 ### Report Time
-2024/8/28 11:33:36
+2024/9/9 19:55:03
 
 ### Device
 macOS 14.4.1
@@ -13,23 +13,26 @@ macOS 14.4.1
 
 ### Version
 `nodejs`: v21.6.1
-`dt-sql-parser`: 4.0.2
-`antlr4-c3`: 3.3.7
-`antlr4ng`: 2.0.11
+`dt-sql-parser`: v4.0.2
+`antlr4-c3`: v3.3.7
+`antlr4ng`: v2.0.11
+
+### Running Mode
+Cold Start
 
 ### Report
 | Benchmark Name |         Method Name        |SQL Rows|Average Time(ms)| 
 |----------------|----------------------------|--------|----------------| 
-|Query Collection|        getAllTokens        |  1015  |       271      | 
-|Query Collection|          validate          |  1015  |       262      | 
-|  Update Table  |        getAllTokens        |  1011  |       226      | 
-|  Update Table  |          validate          |  1011  |       227      | 
-| Insert Columns |        getAllTokens        |  1001  |       202      | 
+|Query Collection|        getAllTokens        |  1015  |       268      | 
+|Query Collection|          validate          |  1015  |       259      | 
+|  Update Table  |        getAllTokens        |  1011  |       232      | 
+|  Update Table  |          validate          |  1011  |       226      | 
+| Insert Columns |        getAllTokens        |  1001  |       198      | 
 | Insert Columns |          validate          |  1001  |       200      | 
 |  Create Table  |        getAllTokens        |  1002  |       29       | 
-|  Create Table  |          validate          |  1002  |       29       | 
+|  Create Table  |          validate          |  1002  |       30       | 
 |    Split SQL   |     splitSQLByStatement    |  1001  |       111      | 
-|Collect Entities|       getAllEntities       |  1066  |       173      | 
-|   Suggestion   |getSuggestionAtCaretPosition|  1066  |       158      | 
+|Collect Entities|       getAllEntities       |  1066  |       170      | 
+|   Suggestion   |getSuggestionAtCaretPosition|  1066  |       164      |