Add compressing

renovatebot · Jan 15, 2024 · 247c32b · 247c32b
1 parent 3014171
commit 247c32b
Show file tree

Hide file tree

Showing 3 changed files with 49 additions and 33 deletions.
diff --git a/lib/util/cache/package/index.ts b/lib/util/cache/package/index.ts
@@ -66,21 +66,7 @@ export async function init(config: AllConfig): Promise<void> {
 
   // istanbul ignore if
   if (config.cacheDir && config.useSqliteCache) {
-    const sqlite = await SqlitePackageCache.init(config.cacheDir);
-    cacheProxy = {
-      get: (namespace: string, key: string) =>
-        Promise.resolve(sqlite.get(namespace, key)),
-      set: (
-        namespace: string,
-        key: string,
-        value: unknown,
-        ttlMinutes: number,
-      ) => Promise.resolve(sqlite.set(namespace, key, value, ttlMinutes)),
-      cleanup: () => {
-        sqlite.close();
-        return Promise.resolve();
-      },
-    };
+    cacheProxy = await SqlitePackageCache.init(config.cacheDir);
     return;
   }
 

diff --git a/lib/util/cache/package/sqlite.spec.ts b/lib/util/cache/package/sqlite.spec.ts
@@ -2,13 +2,13 @@ import { withDir } from 'tmp-promise';
 import { SqlitePackageCache } from './sqlite';
 
 function withSqlite<T>(
-  fn: (sqlite: SqlitePackageCache) => T | Promise<T>,
+  fn: (sqlite: SqlitePackageCache) => Promise<T>,
 ): Promise<T> {
   return withDir(
     async ({ path }) => {
       const sqlite = await SqlitePackageCache.init(path);
       const res = await fn(sqlite);
-      sqlite.close();
+      await sqlite.cleanup();
       return res;
     },
     { unsafeCleanup: true },
@@ -22,10 +22,10 @@ describe('util/cache/package/sqlite', () => {
   });
 
   it('should set and get', async () => {
-    const res = await withSqlite((sqlite) => {
-      sqlite.set('foo', 'bar', { foo: 'foo' });
-      sqlite.set('foo', 'bar', { bar: 'bar' });
-      sqlite.set('foo', 'bar', { baz: 'baz' });
+    const res = await withSqlite(async (sqlite) => {
+      await sqlite.set('foo', 'bar', { foo: 'foo' });
+      await sqlite.set('foo', 'bar', { bar: 'bar' });
+      await sqlite.set('foo', 'bar', { baz: 'baz' });
       return sqlite.get('foo', 'bar');
     });
     expect(res).toEqual({ baz: 'baz' });

diff --git a/lib/util/cache/package/sqlite.ts b/lib/util/cache/package/sqlite.ts
@@ -1,13 +1,30 @@
+import { brotliCompressSync, brotliDecompressSync, constants } from 'node:zlib';
 import Sqlite from 'better-sqlite3';
 import type { Database, Statement } from 'better-sqlite3';
 import * as upath from 'upath';
 import { logger } from '../../../logger';
 import { ensureDir } from '../../fs';
 
+function compress(input: unknown): Buffer {
+  const jsonStr = JSON.stringify(input);
+  return brotliCompressSync(jsonStr, {
+    params: {
+      [constants.BROTLI_PARAM_MODE]: constants.BROTLI_MODE_TEXT,
+      [constants.BROTLI_PARAM_QUALITY]: 8,
+    },
+  });
+}
+
+function decompress<T>(input: Buffer): T {
+  const buf = brotliDecompressSync(input);
+  const jsonStr = buf.toString('utf8');
+  return JSON.parse(jsonStr) as T;
+}
+
 export class SqlitePackageCache {
   private readonly upsertStatement: Statement<unknown[]>;
   private readonly getStatement: Statement<unknown[]>;
-  private readonly cleanupStatement: Statement<unknown[]>;
+  private readonly deleteExpiredRows: Statement<unknown[]>;
   private readonly countStatement: Statement<unknown[]>;
 
   static async init(cacheDir: string): Promise<SqlitePackageCache> {
@@ -29,8 +46,8 @@ export class SqlitePackageCache {
           CREATE TABLE IF NOT EXISTS package_cache (
             namespace TEXT NOT NULL,
             key TEXT NOT NULL,
-            data TEXT NOT NULL,
             expiry INTEGER NOT NULL,
+            data BLOB NOT NULL,
             PRIMARY KEY (namespace, key)
           )
         `,
@@ -63,7 +80,7 @@ export class SqlitePackageCache {
       )
       .pluck(true);
 
-    this.cleanupStatement = client.prepare(`
+    this.deleteExpiredRows = client.prepare(`
       DELETE FROM package_cache
       WHERE expiry <= unixepoch()
     `);
@@ -73,30 +90,43 @@ export class SqlitePackageCache {
       .pluck(true);
   }
 
-  set(namespace: string, key: string, value: unknown, ttlMinutes = 5): void {
-    const data = JSON.stringify(value);
+  set(
+    namespace: string,
+    key: string,
+    value: unknown,
+    ttlMinutes = 5,
+  ): Promise<void> {
+    const data = compress(value);
     const ttlSeconds = ttlMinutes * 60;
     this.upsertStatement.run({ namespace, key, data, ttlSeconds });
+    return Promise.resolve();
   }
 
-  get<T = never>(namespace: string, key: string): T | undefined {
-    const res = this.getStatement.get({ namespace, key }) as string | undefined;
-    return res ? JSON.parse(res) : undefined;
+  get<T = never>(namespace: string, key: string): Promise<T | undefined> {
+    const data = this.getStatement.get({ namespace, key }) as
+      | Buffer
+      | undefined;
+    if (!data) {
+      return Promise.resolve(undefined);
+    }
+    const res = decompress<T>(data);
+    return Promise.resolve(res);
   }
 
-  private cleanup(): void {
+  private cleanupExpired(): void {
     const start = Date.now();
     const totalCount = this.countStatement.get() as number;
-    const { changes: deletedCount } = this.cleanupStatement.run();
+    const { changes: deletedCount } = this.deleteExpiredRows.run();
     const finish = Date.now();
     const durationMs = finish - start;
     logger.debug(
       `SQLite package cache: deleted ${deletedCount} of ${totalCount} entries in ${durationMs}ms`,
     );
   }
 
-  close(): void {
-    this.cleanup();
+  cleanup(): Promise<void> {
+    this.cleanupExpired();
     this.client.close();
+    return Promise.resolve();
   }
 }