Merge pull request #2057 from cardstack/invalidation-improvement

Invalidation performance improvement

packages/host/app/lib/current-run.ts

@@ -284,21 +284,27 @@ export class CurrentRun {
   ): Promise<string[]> {
     log.debug(`discovering invalidations in dir ${url.href}`);
     perfLog.debug(`discovering invalidations in dir ${url.href}`);
-    let ignoreStart = Date.now();
-    let ignorePatterns = await this.#reader.readFile(
-      new URL('.gitignore', url),
-    );
-    perfLog.debug(`time to get ignore rules ${Date.now() - ignoreStart} ms`);
-    if (ignorePatterns && ignorePatterns.content) {
-      this.ignoreMap.set(url.href, ignore().add(ignorePatterns.content));
-      this.#ignoreData[url.href] = ignorePatterns.content;
-    }
-
     let mtimesStart = Date.now();
     let filesystemMtimes = await this.#reader.mtimes();
     perfLog.debug(
       `time to get file system mtimes ${Date.now() - mtimesStart} ms`,
     );
+
+    let ignoreFile = new URL('.gitignore', url).href;
+    // it costs about 10 sec to try to get the ignore file when it doesn't
+    // exist, so don't get it if it's not there.
+    if (filesystemMtimes[ignoreFile]) {
+      let ignoreStart = Date.now();
+      let ignorePatterns = await this.#reader.readFile(new URL(ignoreFile));
+      perfLog.debug(`time to get ignore rules ${Date.now() - ignoreStart} ms`);
+      if (ignorePatterns && ignorePatterns.content) {
+        this.ignoreMap.set(url.href, ignore().add(ignorePatterns.content));
+        this.#ignoreData[url.href] = ignorePatterns.content;
+      }
+    } else {
+      perfLog.debug(`skip getting the ignore file--there is nothing to ignore`);
+    }
+
     let invalidationList: string[] = [];
     let skipList: string[] = [];
     for (let [url, lastModified] of Object.entries(filesystemMtimes)) {

packages/host/config/schema/1736950557343_schema.sql → packages/host/config/schema/1737128666066_schema.sql

@@ -10,7 +10,7 @@
    pristine_doc BLOB,
    search_doc BLOB,
    error_doc BLOB,
-   deps BLOB,
+   deps BLOB DEFAULT '[]',
    types BLOB,
    isolated_html TEXT,
    indexed_at,
@@ -36,7 +36,7 @@
    pristine_doc BLOB,
    search_doc BLOB,
    error_doc BLOB,
-   deps BLOB,
+   deps BLOB DEFAULT '[]',
    types BLOB,
    icon_html TEXT,
    isolated_html TEXT,

packages/postgres/migrations/1737128666066_deps-default-value.js

@@ -0,0 +1,25 @@
+exports.up = (pgm) => {
+  pgm.alterColumn('boxel_index', 'deps', {
+    default: `[]`,
+  });
+  pgm.alterColumn('boxel_index_working', 'deps', {
+    default: `[]`,
+  });
+  pgm.createIndex('boxel_index', ['realm_url', 'type']);
+  pgm.createIndex('boxel_index_working', ['realm_url', 'type']);
+  pgm.sql(`UPDATE boxel_index SET deps = '[]'::jsonb WHERE deps IS NULL`);
+  pgm.sql(
+    `UPDATE boxel_index_working SET deps = '[]'::jsonb WHERE deps IS NULL`,
+  );
+};
+
+exports.down = (pgm) => {
+  pgm.dropIndex('boxel_index', ['realm_url', 'type']);
+  pgm.dropIndex('boxel_index_working', ['realm_url', 'type']);
+  pgm.alterColumn('boxel_index', 'deps', {
+    default: null,
+  });
+  pgm.alterColumn('boxel_index_working', 'deps', {
+    default: null,
+  });
+};

packages/postgres/scripts/convert-to-sqlite.ts

@@ -111,6 +111,13 @@ function createColumns(
           if ('value' in constraint.expr) {
             column.push('DEFAULT', String(constraint.expr.value));
             break;
+          } else if (
+            constraint.expr.type === 'cast_operator_expr' &&
+            'expr' in constraint.expr &&
+            'text' in constraint.expr.expr
+          ) {
+            column.push('DEFAULT', String(constraint.expr.expr.text));
+            break;
           } else {
             throw new Error(
               `Don't know how to serialize default value constraint for expression type '${constraint.expr.type}'`,

packages/runtime-common/index-writer.ts

@@ -413,6 +413,8 @@ export class Batch {
 
   async invalidate(url: URL): Promise<string[]> {
     await this.ready;
+    let start = Date.now();
+    this.#perfLog.debug(`starting invalidation of ${url.href}`);
     let alias = trimExecutableExtension(url).href;
     let visited = new Set<string>();
     let invalidations = [
@@ -447,6 +449,7 @@ export class Batch {
     );
 
     let names = flattenDeep(columns);
+    let insertStart = Date.now();
     await this.#query([
       'INSERT INTO boxel_index_working',
       ...addExplicitParens(separatedByCommas(columns)),
@@ -457,6 +460,15 @@ export class Batch {
       'ON CONFLICT ON CONSTRAINT boxel_index_working_pkey DO UPDATE SET',
       ...separatedByCommas(names.map((name) => [`${name}=EXCLUDED.${name}`])),
     ] as Expression);
+    this.#perfLog.debug(
+      `inserted invalidated rows for  ${url.href} in ${
+        Date.now() - insertStart
+      } ms`,
+    );
+
+    this.#perfLog.debug(
+      `completed invalidation of ${url.href} in ${Date.now() - start} ms`,
+    );
 
     this.#invalidations = new Set([...this.#invalidations, ...invalidations]);
     return invalidations;
@@ -492,7 +504,7 @@ export class Batch {
           [
             dbExpression({
               sqlite: `deps_array_element =`,
-              pg: `COALESCE(i.deps, '[]'::jsonb) @>`,
+              pg: `i.deps @>`,
             }),
             param({ sqlite: resolvedPath, pg: `["${resolvedPath}"]` }),
           ],
@@ -503,7 +515,6 @@ export class Batch {
           // realm invalidation
           [`i.realm_url =`, param(this.realmURL.href)],
         ]),
-        'ORDER BY i.url COLLATE "POSIX"',
         `LIMIT ${pageSize} OFFSET ${pageNumber * pageSize}`,
       ] as Expression)) as (Pick<BoxelIndexTable, 'url' | 'file_alias'> & {
         type: 'instance' | 'module' | 'error';
@@ -514,7 +525,7 @@ export class Batch {
     this.#perfLog.debug(
       `time to determine items that reference ${resolvedPath} ${
         Date.now() - start
-      } ms`,
+      } ms (page count=${pageNumber})`,
     );
     return results.map(({ url, file_alias, type }) => ({
       url,