fix(gate): cache deno imports on deploy (#925)

<!--
Pull requests are squashed and merged using:
- their title as the commit message
- their description as the commit body

Having a good title and description is important for the users to get
readable changelog.
-->

<!-- 1. Explain WHAT the change is about -->

- Fixes
[MET-766](https://linear.app/metatypedev/issue/MET-766/typescript-import-caching-and-validation-on-deploy)
and
[MET-746](https://linear.app/metatypedev/issue/MET-746/deno-runtime-timeouts-on-error)

<!-- 3. Explain HOW users should update their code -->

#### Migration notes

---

- [ ] The change comes with new or modified tests
- [ ] Hard-to-understand functions have explanatory comments
- [ ] End-user documentation is updated to reflect the change

---------

Co-authored-by: Natoandro <[email protected]>

examples/typegraphs/func.py

@@ -5,6 +5,7 @@
 from typegraph.graph.params import Auth
 from typegraph.providers.prisma import PrismaRuntime
 from typegraph.graph.params import Cors
+
 # skip:end
 
 
@@ -83,7 +84,7 @@ def roadmap(g: Graph):
         parse_markdown=deno.import_(
             t.struct({"raw": t.string()}),
             t.string(),
-            module="scripts/md2html.ts.src",
+            module="scripts/md2html.ts",
             name="parse",
         ),
     )

examples/typegraphs/func.ts

@@ -78,7 +78,7 @@ await typegraph(
         create_vote: db.create(vote),
         // skip:end
         parse_markdown: deno.import(t.struct({ raw: t.string() }), t.string(), {
-          module: "scripts/md2html.ts.src",
+          module: "scripts/md2html.ts",
           name: "parse",
         }),
       },

src/typegate/src/runtimes/deno/deno.ts

@@ -13,6 +13,7 @@ import { DenoMessenger } from "./deno_messenger.ts";
 import type { Task } from "./shared_types.ts";
 import { path } from "compress/deps.ts";
 import { globalConfig as config } from "../../config.ts";
+import { createArtifactMeta } from "../utils/deno.ts";
 
 const predefinedFuncs: Record<string, Resolver<Record<string, unknown>>> = {
   identity: ({ _, ...args }) => args,
@@ -81,23 +82,10 @@ export class DenoRuntime extends Runtime {
       } else if (mat.name === "module") {
         const matData = mat.data;
         const entryPoint = artifacts[matData.entryPoint as string];
-        const deps = (matData.deps as string[]).map((dep) => artifacts[dep]);
-
-        const moduleMeta = {
-          typegraphName: typegraphName,
-          relativePath: entryPoint.path,
-          hash: entryPoint.hash,
-          sizeInBytes: entryPoint.size,
-        };
-
-        const depMetas = deps.map((dep) => {
-          return {
-            typegraphName: typegraphName,
-            relativePath: dep.path,
-            hash: dep.hash,
-            sizeInBytes: dep.size,
-          };
-        });
+        const depMetas = (matData.deps as string[]).map((dep) =>
+          createArtifactMeta(typegraphName, artifacts[dep]),
+        );
+        const moduleMeta = createArtifactMeta(typegraphName, entryPoint);
 
         // Note:
         // Worker destruction seems to have no effect on the import cache? (deinit() => stop(worker))

src/typegate/src/runtimes/deno/hooks/mod.ts

@@ -0,0 +1,78 @@
+// Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0.
+// SPDX-License-Identifier: MPL-2.0
+
+import { getLogger } from "../../../log.ts";
+import { PushFailure, PushHandler } from "../../../typegate/hooks.ts";
+import { createArtifactMeta } from "../../utils/deno.ts";
+
+const logger = getLogger("typegate");
+
+export class DenoFailure extends Error {
+  failure: PushFailure;
+
+  constructor(message: string) {
+    super(message);
+    this.failure = { reason: "DenoImportError", message };
+  }
+}
+
+function sandboxImport(modulePath: string) {
+  return new Promise<void>((resolve, reject) => {
+    const worker = new Worker(new URL("./worker.ts", import.meta.url).href, {
+      type: "module",
+    });
+
+    worker.postMessage({ import: modulePath });
+
+    worker.onmessage = ({ data }: MessageEvent<{ error?: any }>) => {
+      if (data.error) {
+        reject(data.error);
+      } else {
+        resolve();
+      }
+    };
+
+    worker.onerror = (error) => {
+      reject(error);
+    };
+  });
+}
+
+export const cacheModules: PushHandler = async (
+  typegraph,
+  _secretManager,
+  _response,
+  artifactStore,
+) => {
+  const { title } = typegraph.types[0];
+  const { artifacts } = typegraph.meta;
+
+  for (const mat of typegraph.materializers) {
+    if (mat.name === "module") {
+      const matData = mat.data;
+      const entryPoint = artifacts[matData.entryPoint as string];
+      const moduleMeta = createArtifactMeta(title, entryPoint);
+      const depMetas = (matData.deps as string[]).map((dep) =>
+        createArtifactMeta(title, artifacts[dep]),
+      );
+      const entryModulePath = await artifactStore.getLocalPath(
+        moduleMeta,
+        depMetas,
+      );
+
+      try {
+        logger.info(`Caching deno imports for ${title} (${entryPoint.path})`);
+        await sandboxImport(entryModulePath);
+        logger.info(`'${entryPoint.path}' was cached`);
+      } catch (error) {
+        console.error(error.stack);
+
+        throw new DenoFailure(
+          `An error occured when trying to import '${entryPoint.path}'`,
+        );
+      }
+    }
+  }
+
+  return typegraph;
+};

src/typegate/src/runtimes/deno/hooks/worker.ts

@@ -0,0 +1,13 @@
+// Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0.
+// SPDX-License-Identifier: MPL-2.0
+
+self.onmessage = async ({ data }: MessageEvent<{ import: string }>) => {
+  try {
+    await import(data.import);
+    self.postMessage({ success: true });
+  } catch (error) {
+    self.postMessage({ error });
+  }
+
+  self.close();
+};

src/typegate/src/runtimes/utils/deno.ts

@@ -0,0 +1,17 @@
+// Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0.
+// SPDX-License-Identifier: MPL-2.0
+
+import { ArtifactMeta } from "../../typegate/artifacts/mod.ts";
+import { Artifact } from "../../typegraph/types.ts";
+
+export function createArtifactMeta(
+  typegraphName: string,
+  artifact: Artifact,
+): ArtifactMeta {
+  return {
+    typegraphName,
+    hash: artifact.hash,
+    sizeInBytes: artifact.size,
+    relativePath: artifact.path,
+  };
+}

src/typegate/src/typegate/hooks.ts

@@ -3,27 +3,35 @@
 
 import type { MessageEntry, Migrations } from "../typegate/register.ts";
 import type { SecretManager, TypeGraphDS } from "../typegraph/mod.ts";
+import { ArtifactStore } from "./artifacts/mod.ts";
 
 const Message = {
   INFO: "info",
   WARNING: "warning",
   ERROR: "error",
 } as const;
 
-export type PushFailure = {
-  reason: "DatabaseResetRequired";
-  message: string;
-  runtimeName: string;
-} | {
-  reason: "NullConstraintViolation";
-  message: string;
-  runtimeName: string;
-  column: string;
-  table: string;
-} | {
-  reason: "Unknown";
-  message: string;
-};
+export type PushFailure =
+  | {
+      reason: "DatabaseResetRequired";
+      message: string;
+      runtimeName: string;
+    }
+  | {
+      reason: "NullConstraintViolation";
+      message: string;
+      runtimeName: string;
+      column: string;
+      table: string;
+    }
+  | {
+      reason: "DenoImportError";
+      message: string;
+    }
+  | {
+      reason: "Unknown";
+      message: string;
+    };
 
 export class PushResponse {
   tgName?: string;
@@ -74,5 +82,6 @@ export interface PushHandler {
     tg: TypeGraphDS,
     secretManager: SecretManager,
     response: PushResponse,
+    artifactStore: ArtifactStore,
   ): Promise<TypeGraphDS>;
 }

src/typegate/src/typegate/mod.ts

@@ -19,6 +19,7 @@ import { type PushHandler, PushResponse } from "../typegate/hooks.ts";
 import { upgradeTypegraph } from "../typegraph/versions.ts";
 import { parseGraphQLTypeGraph } from "../transports/graphql/typegraph.ts";
 import * as PrismaHooks from "../runtimes/prisma/hooks/mod.ts";
+import * as DenoHooks from "../runtimes/deno/hooks/mod.ts";
 import {
   type RuntimeResolver,
   SecretManager,
@@ -31,9 +32,8 @@ import { resolveIdentifier } from "../services/middlewares.ts";
 import { handleGraphQL } from "../services/graphql_service.ts";
 import { getLogger } from "../log.ts";
 import { MigrationFailure } from "../runtimes/prisma/hooks/run_migrations.ts";
-import introspectionJson from "../typegraphs/introspection.json" with {
-  type: "json",
-};
+import { DenoFailure } from "../runtimes/deno/hooks/mod.ts";
+import introspectionJson from "../typegraphs/introspection.json" with { type: "json" };
 import { ArtifactService } from "../services/artifact_service.ts";
 import type { ArtifactStore } from "./artifacts/mod.ts";
 // TODO move from tests (MET-497)
@@ -170,6 +170,7 @@ export class Typegate implements AsyncDisposable {
     this.#onPush((tg) => Promise.resolve(parseGraphQLTypeGraph(tg)));
     this.#onPush(PrismaHooks.generateSchema);
     this.#onPush(PrismaHooks.runMigrations);
+    this.#onPush(DenoHooks.cacheModules);
     this.#artifactService = new ArtifactService(artifactStore);
   }
 
@@ -192,13 +193,15 @@ export class Typegate implements AsyncDisposable {
 
     for (const handler of this.#onPushHooks) {
       try {
-        res = await handler(res, secretManager, response);
+        res = await handler(res, secretManager, response, this.artifactStore);
       } catch (e) {
         logger.error(`Error in onPush hook: ${e}`);
         // FIXME: MigrationFailur err message parser doesn't support all errors like
         // can't reach database errs
         if (e instanceof MigrationFailure && e.errors[0]) {
           response.setFailure(e.errors[0]);
+        } else if (e instanceof DenoFailure) {
+          response.setFailure(e.failure);
         } else {
           response.setFailure({
             reason: "Unknown",
@@ -399,14 +402,14 @@ export class Typegate implements AsyncDisposable {
 
     const introspection = enableIntrospection
       ? await TypeGraph.init(
-        this,
-        introspectionDef,
-        new SecretManager(introspectionDef, {}),
-        {
-          typegraph: TypeGraphRuntime.init(tgDS, [], {}),
-        },
-        null,
-      )
+          this,
+          introspectionDef,
+          new SecretManager(introspectionDef, {}),
+          {
+            typegraph: TypeGraphRuntime.init(tgDS, [], {}),
+          },
+          null,
+        )
       : null;
 
     const tg = await TypeGraph.init(