feat: support using output from previous plugins

.eslintrc

@@ -15,7 +15,7 @@
     "constructor-super": "error",
     "no-invalid-this": "off",
     "@typescript-eslint/no-invalid-this": ["error"],
-    "no-restricted-syntax": ["error", "ForInStatement"],
+    "no-restricted-syntax": ["error"],
     "use-isnan": "error",
     "@typescript-eslint/no-unused-vars": [
       "error",

package.json

@@ -19,7 +19,7 @@
     "prepare": "husky install",
     "deploy-dev": "wrangler deploy --env dev",
     "deploy-production": "wrangler deploy --env production",
-    "worker": "wrangler dev --port 8787",
+    "worker": "wrangler dev --env dev --port 8787",
     "proxy": "tsx src/proxy.ts"
   },
   "keywords": [

src/github/handlers/index.ts

@@ -19,7 +19,7 @@ function tryCatchWrapper(fn: (event: EmitterWebhookEvent) => unknown) {
 export function bindHandlers(eventHandler: GitHubEventHandler) {
   eventHandler.on("issue_comment.created", issueCommentCreated);
   eventHandler.on("repository_dispatch", repositoryDispatch);
-  eventHandler.onAny(tryCatchWrapper((event) => handleEvent(event, eventHandler)));
+  eventHandler.onAny(tryCatchWrapper((event) => handleEvent(event, eventHandler))); // onAny should also receive GithubContext but the types in octokit/webhooks are weird
 }
 
 async function handleEvent(event: EmitterWebhookEvent, eventHandler: InstanceType<typeof GitHubEventHandler>) {
@@ -54,7 +54,7 @@ async function handleEvent(event: EmitterWebhookEvent, eventHandler: InstanceTyp
     const state = {
       eventId: context.id,
       eventName: context.key,
-      event: event,
+      eventPayload: event.payload,
       currentPlugin: 0,
       pluginChain: pluginChain.uses,
       outputs: new Array(pluginChain.uses.length),
@@ -63,7 +63,7 @@ async function handleEvent(event: EmitterWebhookEvent, eventHandler: InstanceTyp
 
     const ref = plugin.ref ?? (await getDefaultBranch(context, plugin.owner, plugin.repo));
     const token = await eventHandler.getToken(event.payload.installation.id);
-    const inputs = new DelegatedComputeInputs(stateId, context.key, event, settings, token, ref);
+    const inputs = new DelegatedComputeInputs(stateId, context.key, event.payload, settings, token, ref);
 
     state.inputs[0] = inputs;
     await eventHandler.pluginChainState.put(stateId, state);

src/github/handlers/repository-dispatch.ts

@@ -1,26 +1,34 @@
 import { GitHubContext } from "../github-context";
 import { dispatchWorkflow, getDefaultBranch } from "../utils/workflow-dispatch";
 import { Value } from "@sinclair/typebox/value";
-import { DelegatedComputeInputs, PluginOutput, pluginOutputSchema } from "../types/plugin";
+import { DelegatedComputeInputs, PluginChainState, expressionRegex, pluginOutputSchema } from "../types/plugin";
+import { PluginChain } from "../types/config";
 
 export async function repositoryDispatch(context: GitHubContext<"repository_dispatch">) {
   console.log("Repository dispatch event received", context.payload.client_payload);
 
-  const pluginOutput = context.payload.client_payload as PluginOutput;
+  if (context.payload.action !== "return_data_to_ubiquibot_kernel") {
+    console.log("Skipping non-ubiquibot event");
+    return;
+  }
+
+  let pluginOutput;
 
-  if (!Value.Decode(pluginOutputSchema, pluginOutput)) {
-    const errors = [...Value.Errors(pluginOutputSchema, pluginOutput)];
-    console.error("Invalid environment variables", errors);
-    throw new Error("Invalid environment variables");
+  try {
+    pluginOutput = Value.Decode(pluginOutputSchema, context.payload.client_payload);
+  } catch (error) {
+    console.error("Cannot decode plugin output", error);
+    throw error;
   }
+  console.log("Plugin output", pluginOutput);
 
-  const state = await context.eventHandler.pluginChainState.get(pluginOutput.stateId);
+  const state = await context.eventHandler.pluginChainState.get(pluginOutput.state_id);
   if (!state) {
     console.error("No state found for plugin chain");
     return;
   }
 
-  if (!("installation" in state.event.payload) || state.event.payload.installation?.id === undefined) {
+  if (!("installation" in state.eventPayload) || state.eventPayload.installation?.id === undefined) {
     console.error("No installation found");
     return;
   }
@@ -30,23 +38,25 @@ export async function repositoryDispatch(context: GitHubContext<"repository_disp
     console.error("Plugin chain state does not match payload");
     return;
   }
+  state.outputs[state.currentPlugin] = pluginOutput;
+  console.log("State", state);
 
-  const nextPlugin = state.pluginChain[state.currentPlugin];
+  const nextPlugin = state.pluginChain[state.currentPlugin + 1];
   if (!nextPlugin) {
     console.log("No more plugins to call");
+    await context.eventHandler.pluginChainState.put(pluginOutput.state_id, state);
     return;
   }
-
   console.log("Dispatching next plugin", nextPlugin);
 
-  const token = await context.eventHandler.getToken(state.event.payload.installation.id);
+  const token = await context.eventHandler.getToken(state.eventPayload.installation.id);
   const ref = nextPlugin.plugin.ref ?? (await getDefaultBranch(context, nextPlugin.plugin.owner, nextPlugin.plugin.repo));
-  const inputs = new DelegatedComputeInputs(pluginOutput.stateId, state.eventName, state.event, nextPlugin.with, token, ref);
+  const settings = findAndReplaceExpressions(nextPlugin, state);
+  const inputs = new DelegatedComputeInputs(pluginOutput.state_id, state.eventName, state.eventPayload, settings, token, ref);
 
-  state.outputs[state.currentPlugin] = pluginOutput;
   state.currentPlugin++;
   state.inputs[state.currentPlugin] = inputs;
-  await context.eventHandler.pluginChainState.put(pluginOutput.stateId, state);
+  await context.eventHandler.pluginChainState.put(pluginOutput.state_id, state);
 
   await dispatchWorkflow(context, {
     owner: nextPlugin.plugin.owner,
@@ -56,3 +66,52 @@ export async function repositoryDispatch(context: GitHubContext<"repository_disp
     inputs: inputs.getInputs(),
   });
 }
+
+function findAndReplaceExpressions(plugin: PluginChain[0], state: PluginChainState): Record<string, unknown> {
+  const settings: Record<string, unknown> = {};
+
+  for (const key in plugin.with) {
+    const value = plugin.with[key];
+
+    if (typeof value === "string") {
+      const matches = value.match(expressionRegex);
+      if (!matches) {
+        settings[key] = value;
+        continue;
+      }
+      const parts = matches[1].split(".");
+      if (parts.length !== 3) {
+        throw new Error(`Invalid expression: ${value}`);
+      }
+      const pluginId = parts[0];
+
+      if (parts[1] === "output") {
+        const outputProperty = parts[2];
+        settings[key] = getPluginOutputValue(state, pluginId, outputProperty);
+      } else {
+        throw new Error(`Invalid expression: ${value}`);
+      }
+    } else {
+      settings[key] = value;
+    }
+  }
+
+  return settings;
+}
+
+function getPluginOutputValue(state: PluginChainState, pluginId: string, outputKey: string): unknown {
+  const pluginIdx = state.pluginChain.findIndex((plugin) => plugin.id === pluginId);
+  if (pluginIdx === -1) {
+    throw new Error(`Plugin ${pluginId} not found in the chain`);
+  }
+  if (pluginIdx > state.currentPlugin) {
+    throw new Error(`You cannot use output values from plugin ${pluginId} because it's not been called yet`);
+  }
+
+  const outputValue = state.outputs[pluginIdx].output[outputKey];
+  if (outputValue === undefined) {
+    throw new Error(`Output key '${outputKey}' not found for plugin ${pluginId}`);
+  }
+
+  return outputValue;
+}

src/github/types/config.ts

@@ -2,7 +2,7 @@ import { Type as T } from "@sinclair/typebox";
 import { StaticDecode } from "@sinclair/typebox";
 import { githubWebhookEvents } from "./webhook-events";
 
-const pluginNameRegex = new RegExp("^([0-9a-zA-Z-._]+)/([0-9a-zA-Z-._]+)(?::([0-9a-zA-Z-._]+))?(?:@([0-9a-zA-Z]+))?$");
+const pluginNameRegex = new RegExp("^([0-9a-zA-Z-._]+)/([0-9a-zA-Z-._]+)(?::([0-9a-zA-Z-._]+))?(?:@([0-9a-zA-Z-._]+))?$");
 
 type GithubPlugin = {
   owner: string;
@@ -32,9 +32,10 @@ function githubPluginType() {
 
 const pluginChainSchema = T.Array(
   T.Object({
+    id: T.Optional(T.String()),
     plugin: githubPluginType(),
     type: T.Union([T.Literal("github")], { default: "github" }),
-    with: T.Optional(T.Unknown()),
+    with: T.Record(T.String(), T.Unknown()),
   }),
   { minItems: 1 }
 );

src/github/types/plugin.ts

@@ -2,14 +2,16 @@ import { EmitterWebhookEvent, EmitterWebhookEventName } from "@octokit/webhooks"
 import { PluginChain } from "./config";
 import { StaticDecode, Type } from "@sinclair/typebox";
 
+export const expressionRegex = /^\s*\${{\s*(\S+)\s*}}\s*$/;
+
 function jsonString() {
   return Type.Transform(Type.String())
-    .Decode((value) => JSON.parse(value) as unknown)
+    .Decode((value) => JSON.parse(value) as Record<string, unknown>)
     .Encode((value) => JSON.stringify(value));
 }
 
 export const pluginOutputSchema = Type.Object({
-  stateId: Type.String(),
+  state_id: Type.String(), // Github forces snake_case
   output: jsonString(),
 });
 
@@ -18,15 +20,15 @@ export type PluginOutput = StaticDecode<typeof pluginOutputSchema>;
 export class DelegatedComputeInputs<T extends EmitterWebhookEventName = EmitterWebhookEventName> {
   public stateId: string;
   public eventName: T;
-  public event: EmitterWebhookEvent<T>;
+  public eventPayload: EmitterWebhookEvent<T>["payload"];
   public settings: unknown;
   public authToken: string;
   public ref: string;
 
-  constructor(stateId: string, eventName: T, event: EmitterWebhookEvent<T>, settings: unknown, authToken: string, ref: string) {
+  constructor(stateId: string, eventName: T, eventPayload: EmitterWebhookEvent<T>["payload"], settings: unknown, authToken: string, ref: string) {
     this.stateId = stateId;
     this.eventName = eventName;
-    this.event = event;
+    this.eventPayload = eventPayload;
     this.settings = settings;
     this.authToken = authToken;
     this.ref = ref;
@@ -36,7 +38,7 @@ export class DelegatedComputeInputs<T extends EmitterWebhookEventName = EmitterW
     return {
       stateId: this.stateId,
       eventName: this.eventName,
-      event: JSON.stringify(this.event),
+      eventPayload: JSON.stringify(this.eventPayload),
       settings: JSON.stringify(this.settings),
       authToken: this.authToken,
       ref: this.ref,
@@ -47,7 +49,7 @@ export class DelegatedComputeInputs<T extends EmitterWebhookEventName = EmitterW
 export type PluginChainState<T extends EmitterWebhookEventName = EmitterWebhookEventName> = {
   eventId: string;
   eventName: T;
-  event: EmitterWebhookEvent<T>;
+  eventPayload: EmitterWebhookEvent<T>["payload"];
   currentPlugin: number;
   pluginChain: PluginChain;
   inputs: DelegatedComputeInputs[];

src/github/types/webhook-events.ts

@@ -1,6 +1,7 @@
 import { emitterEventNames, EmitterWebhookEventName as GitHubEventClassName } from "@octokit/webhooks";
 
-type EventName = GitHubEventClassName | "*";
+export type EventName = GitHubEventClassName | "*";
+export const eventNames: EventName[] = [...emitterEventNames, "*"];
 
 type Formatted<T extends string> = T extends `${infer Prefix}.${infer Rest}` ? `${Prefix}_${Formatted<Rest>}` : T;
 

src/github/utils/config.ts

@@ -2,6 +2,8 @@ import { Value } from "@sinclair/typebox/value";
 import { GitHubContext } from "../github-context";
 import YAML from "yaml";
 import { Config, configSchema } from "../types/config";
+import { expressionRegex } from "../types/plugin";
+import { eventNames } from "../types/webhook-events";
 
 const UBIQUIBOT_CONFIG_FULL_PATH = ".github/ubiquibot-config.yml";
 
@@ -16,13 +18,66 @@ export async function getConfig(context: GitHubContext): Promise<Config | null>
       owner: payload.repository.owner.login,
     })
   );
+  if (!_repoConfig) return null;
 
+  let config: Config;
   try {
-    return Value.Decode(configSchema, Value.Default(configSchema, _repoConfig));
+    config = Value.Decode(configSchema, Value.Default(configSchema, _repoConfig));
   } catch (error) {
     console.error("Error decoding config", error);
     return null;
   }
+
+  checkPluginChains(config);
+
+  return config;
+}
+
+// eslint-disable-next-line sonarjs/cognitive-complexity
+function checkPluginChains(config: Config) {
+  for (const eventName of eventNames) {
+    const plugins = config.plugins[eventName];
+    for (const plugin of plugins) {
+      // make sure ids are unique
+      const allIds = new Set();
+      for (const use of plugin.uses) {
+        if (use.id) {
+          if (allIds.has(use.id)) {
+            throw new Error(`Duplicate id ${use.id} in plugin chain`);
+          }
+          allIds.add(use.id);
+        }
+      }
+      // check expressions
+      const calledIds = new Set();
+      for (const use of plugin.uses) {
+        for (const key in use.with) {
+          const value = use.with[key];
+          if (typeof value === "string" && value.match(expressionRegex)) {
+            const matches = value.match(expressionRegex);
+            if (!matches) {
+              throw new Error(`Invalid expression: ${value}`);
+            }
+            const parts = matches[1].split(".");
+            if (parts.length !== 3) {
+              throw new Error(`Invalid expression: ${value}`);
+            }
+            const id = parts[0];
+            if (!allIds.has(id)) {
+              throw new Error(`Expression ${value} refers to non-existent id ${id}`);
+            }
+            if (!calledIds.has(id)) {
+              throw new Error(`Expression ${value} refers to plugin id ${id} before it is called`);
+            }
+            if (parts[1] !== "output") {
+              throw new Error(`Invalid expression: ${value}`);
+            }
+          }
+        }
+        calledIds.add(use.id);
+      }
+    }
+  }
 }
 
 async function download({ context, repository, owner }: { context: GitHubContext; repository: string; owner: string }): Promise<string | null> {

src/github/utils/workflow-dispatch.ts

@@ -33,7 +33,8 @@ export async function dispatchWorkflow(context: GitHubContext, options: Workflow
 }
 
 export async function getDefaultBranch(context: GitHubContext, owner: string, repository: string) {
-  const repo = await context.octokit.repos.get({
+  const octokit = await getInstallationOctokitForOrg(context, owner); // we cannot access other repos with the context's octokit
+  const repo = await octokit.repos.get({
     owner: owner,
     repo: repository,
   });