feat: hot start will clear atn cache every benchmark test and add ver…

…sion limit
DTStack · Aug 28, 2024 · 8b93313 · 8b93313
1 parent cb8a1d1
commit 8b93313
Show file tree

Hide file tree

Showing 4 changed files with 66 additions and 33 deletions.
diff --git a/benchmark/run.ts b/benchmark/run.ts
@@ -10,8 +10,8 @@ import { Table } from 'console-table-printer';
 
 const argv = argsParser(process.argv.slice(2));
 const isChooseAll = argv.lang === 'all';
-const isHot = !!argv.hot;
-const isRelease = !!argv.release;
+const isHot = argv.hot !== undefined;
+const isRelease = argv.release !== undefined;
 const testFiles = config.testFiles;
 
 let benchmarkResults: SqlBenchmark[] = [];
@@ -146,7 +146,7 @@ function run() {
     } else {
         benchmark(argv.lang);
     }
-    !isHot && askForSaveResult();
+    askForSaveResult();
 }
 
 process.on('uncaughtException', (err) => console.log(err));

diff --git a/benchmark/sqlBenchmark.ts b/benchmark/sqlBenchmark.ts
@@ -101,6 +101,8 @@ class SqlBenchmark {
 
     private _DEFAULT_LOOP_TIMES = 5;
 
+    private _RELEASE_LOOP_TIMES = 15;
+
     /**
      * If current average time difference from last time grater than DIFF_RATIO, we will highlight that record.
      */
@@ -110,30 +112,34 @@ class SqlBenchmark {
 
     /**
      * Returns SqlParser instance of specific language.
-     *
-     * Due to the presence of ATN cache in antlr4, we will clear the module cache to ensure that each parser is brand new and with no cache.
      */
     getSqlParser(): BasicSQL {
-        const caches = Object.keys(require.cache);
         if (!this.isHot) {
-            caches
-                .filter((cache) => cache.includes('dt-sql-parser/src'))
-                .forEach((moduleName) => {
-                    const module = require.cache[moduleName]!;
-                    // Fix Memory Leak
-                    if (module.parent) {
-                        module.parent.children = [];
-                    }
-                    delete require.cache[moduleName];
-                });
+            this.clearATNCache();
         }
-
         const Parser = require(path.resolve(`src/parser/${this.language}/index.ts`))[
             languageNameMap[this.language]
         ];
         return new Parser();
     }
 
+    /**
+     * Due to the presence of ATN cache in antlr4, we will clear the module cache to ensure that each parser is brand new and with no cache.
+     */
+    clearATNCache() {
+        const caches = Object.keys(require.cache);
+        caches
+            .filter((cache) => cache.includes('dt-sql-parser/src'))
+            .forEach((moduleName) => {
+                const module = require.cache[moduleName]!;
+                // Fix Memory Leak
+                if (module.parent) {
+                    module.parent.children = [];
+                }
+                delete require.cache[moduleName];
+            });
+    }
+
     getColumns = () => {
         return this.isRelease ? releaseColumns : this.isHot ? hotRunColumns : tableColumns;
     };
@@ -150,13 +156,18 @@ class SqlBenchmark {
         name: string,
         params: any[],
         sqlRows: number,
-        loopTimes: number = this._DEFAULT_LOOP_TIMES
+        loops: number = this._DEFAULT_LOOP_TIMES
     ) {
         let avgTime = 0;
+        let loopTimes = this.isRelease ? this._RELEASE_LOOP_TIMES : loops;
         const costTimes: number[] = [];
         const lastResult =
             this._lastResultsCache?.find((item) => item.type === type && item.name === name) ?? {};
 
+        if (this.isHot) {
+            this.clearATNCache();
+        }
+
         for (let i = 0; i < loopTimes; i++) {
             const parser = this.getSqlParser();
             if (!parser[type] || typeof parser[type] !== 'function') return;
@@ -233,8 +244,8 @@ class SqlBenchmark {
             table.addRow(
                 {
                     ...record,
-                    lastCostTime: !this.isHot ? record.lastCostTime ?? '-' : '-',
-                    avgTime: !this.isHot ? record.avgTime + icon : '-',
+                    lastCostTime: record.lastCostTime ?? '-',
+                    avgTime: record.avgTime + icon,
                 },
                 {
                     color,
@@ -287,6 +298,10 @@ class SqlBenchmark {
         writter.writeText(`\`antlr4ng\`: ${parsedEnvInfo.npmPackages['antlr4ng']?.installed}`);
         writter.writeLine();
 
+        writter.writeHeader('Running Mode', 3);
+        writter.writeText(this.isHot ? 'Hot Start' : 'Cold Start');
+        writter.writeLine();
+
         writter.writeHeader('Report', 3);
 
         const columns = this.getColumns();

diff --git a/package.json b/package.json
@@ -34,7 +34,7 @@
     "format": "prettier --write .",
     "format-g4": "antlr-format -c ./antlr.format.json -v ./src/grammar/**/*.g4",
     "cleanComment": "node ./scripts/cleanCommentCli.js",
-    "benchmark": "node --no-warnings ./scripts/benchmark.js",
+    "benchmark": "NODE_OPTIONS=--max_old_space_size=4096 && node --no-warnings ./scripts/benchmark.js",
     "benchmark:release": "node --no-warnings ./scripts/benchmark.js --release"
   },
   "license": "MIT",

diff --git a/scripts/benchmark.js b/scripts/benchmark.js
@@ -14,17 +14,21 @@ const languages = fs.readdirSync(outputPath).filter((item) => {
 const isRelease = !!argv.release;
 const cmd = 'tsx';
 
+const MIN_VERSION = '16.14.0';
+const RELEASE_VERSION = '21.6.1';
+const RECOMMENDED_VERSION = '18.20.3';
+
 function runBenchmark(language, mode) {
     const lang = language === 'All Languages' ? 'all' : language;
-    const isHotRuns = mode === 'Hot runs';
+    const isHotStart = mode === 'Hot start';
     const tsx = spawn(
         cmd,
         [
+            '--no-warnings',
             'benchmark/run.ts',
             '--lang=' + lang,
-            isHotRuns ? '--hot' : '',
+            isHotStart ? '--hot' : '',
             isRelease ? '--release' : '',
-            '--no-warnings',
         ],
         {
             cwd: process.cwd(),
@@ -36,27 +40,28 @@ function runBenchmark(language, mode) {
 
 function checkVersion() {
     const currentVersion = process.versions.node;
-    const minVersion = '16.14.0';
-    const recommendedVersion = '21.6.1';
-    if (semver.lt(currentVersion, minVersion)) {
+    if (semver.lt(currentVersion, MIN_VERSION)) {
         console.error(
             chalk.bold.red(
-                `Current Node.js version (v${currentVersion}) is lower than required version (v${minVersion})`
+                `Current Node.js version (v${currentVersion}) is lower than required version (v${MIN_VERSION}+)`
             )
         );
         return false;
     } else {
-        if (semver.lt(currentVersion, recommendedVersion))
-            console.warn(
-                chalk.bold.bgCyan(
-                    `Node.js version v${recommendedVersion} is recommended, otherwise there may be a memory leak!`
+        if (isRelease && semver.lt(currentVersion, RELEASE_VERSION)) {
+            console.error(
+                chalk.bold.red(
+                    `Node.js version higher than v${RELEASE_VERSION} is required for release benchmark!`
                 )
             );
+            return false;
+        }
         return true;
     }
 }
 
 function prompt() {
+    const isNodeVersionOk = semver.gte(process.versions.node, RECOMMENDED_VERSION);
     inquirer
         .prompt([
             {
@@ -70,7 +75,20 @@ function prompt() {
                 type: 'list',
                 name: 'mode',
                 message: 'Which mode you want to run',
-                choices: [{ name: 'Cold runs' }, { name: 'Hot runs', disabled: !!isRelease }],
+                choices: [
+                    {
+                        name:
+                            'Cold start' +
+                            (isNodeVersionOk
+                                ? ''
+                                : ` (Only supported on Node.js v${RECOMMENDED_VERSION}+)`),
+                        disabled: !isNodeVersionOk,
+                    },
+                    {
+                        name: 'Hot start',
+                        disabled: isRelease,
+                    },
+                ],
             },
         ])
         .then((result) => {