From 40256f88f7a43109f26d4be538a26cd43134403c Mon Sep 17 00:00:00 2001 From: ponderingdemocritus Date: Tue, 28 Jan 2025 18:33:15 +1100 Subject: [PATCH 1/2] testing for goals --- package.json | 5 +- .../src/core/__tests__/goal-manager.test.ts | 377 +++++++++ .../{__test__ => __tests__}/vector-db.test.ts | 0 packages/core/src/core/chain-of-thought.ts | 723 +++++++++--------- packages/core/src/core/goal-manager.ts | 4 +- 5 files changed, 745 insertions(+), 364 deletions(-) create mode 100644 packages/core/src/core/__tests__/goal-manager.test.ts rename packages/core/src/core/{__test__ => __tests__}/vector-db.test.ts (100%) diff --git a/package.json b/package.json index 9473c00c..e229481a 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,10 @@ "devDependencies": { "@types/node": "^22.10.5", "knip": "^5.43.1", - "typescript": "~5.6.2" + "typescript": "~5.6.2", + "@types/jest": "^29.0.0", + "jest": "^29.0.0", + "ts-jest": "^29.0.0" }, "packageManager": "pnpm@9.15.2+sha512.93e57b0126f0df74ce6bff29680394c0ba54ec47246b9cf321f0121d8d9bb03f750a705f24edc3c1180853afd7c2c3b94196d0a3d53d3e069d9e2793ef11f321" } diff --git a/packages/core/src/core/__tests__/goal-manager.test.ts b/packages/core/src/core/__tests__/goal-manager.test.ts new file mode 100644 index 00000000..fb464288 --- /dev/null +++ b/packages/core/src/core/__tests__/goal-manager.test.ts @@ -0,0 +1,377 @@ +import { describe, it, expect, beforeEach } from "vitest"; +import { GoalManager } from "../goal-manager"; +import type { Goal, GoalStatus } from "../types"; + +describe("GoalManager", () => { + let manager: GoalManager; + const now = Date.now(); + + // Helper function to create a valid base goal + const createBaseGoal = (status: GoalStatus = "pending") => ({ + description: "Test Goal", + horizon: "short" as const, + status: status as GoalStatus, + priority: 1, + success_criteria: ["Goal completed successfully"], + created_at: now, + }); + + beforeEach(() => { + manager = new GoalManager(); + }); + + describe("addGoal", () => { + it("should create a goal with generated ID", () => { + const goal = manager.addGoal(createBaseGoal()); + + expect(goal.id).toBeDefined(); + expect(goal.description).toBe("Test Goal"); + expect(goal.progress).toBe(0); + }); + + it("should handle subgoal relationships", () => { + const parent = manager.addGoal({ + ...createBaseGoal(), + description: "Parent Goal", + horizon: "medium", + }); + + const child = manager.addGoal({ + ...createBaseGoal(), + description: "Child Goal", + parentGoal: parent.id, + }); + + const parentFromStore = manager.getGoalById(parent.id); + expect(parentFromStore?.subgoals).toContain(child.id); + }); + }); + + describe("goal status management", () => { + it("should update goal status correctly", () => { + const goal = manager.addGoal(createBaseGoal()); + + manager.updateGoalStatus(goal.id, "completed"); + const updated = manager.getGoalById(goal.id); + + expect(updated?.status).toBe("completed"); + expect(updated?.progress).toBe(100); + expect(updated?.completed_at).toBeDefined(); + }); + + it("should handle goal failure", async () => { + const goal = manager.addGoal(createBaseGoal()); + + await manager.processGoalFailure(goal); + const failed = manager.getGoalById(goal.id); + + expect(failed?.status).toBe("failed"); + }); + }); + + describe("goal dependencies", () => { + it("should check prerequisites correctly", () => { + const dep = manager.addGoal({ + ...createBaseGoal(), + description: "Dependency", + }); + + const goal = manager.addGoal({ + ...createBaseGoal(), + description: "Main Goal", + dependencies: [dep.id], + }); + + expect(manager.arePrerequisitesMet(goal.id)).toBe(false); + + manager.updateGoalStatus(dep.id, "completed"); + expect(manager.arePrerequisitesMet(goal.id)).toBe(true); + }); + + it("should update dependent goals when dependency is completed", () => { + const dep = manager.addGoal({ + ...createBaseGoal(), + description: "Dependency", + }); + + const goal = manager.addGoal({ + ...createBaseGoal(), + description: "Main Goal", + status: "pending", + dependencies: [dep.id], + }); + + manager.updateGoalStatus(dep.id, "completed"); + const updated = manager.getGoalById(goal.id); + expect(updated?.status).toBe("ready"); + }); + }); + + describe("goal hierarchy", () => { + it("should get goal hierarchy correctly", () => { + const parent = manager.addGoal({ + ...createBaseGoal(), + description: "Parent", + horizon: "medium", + }); + + const child1 = manager.addGoal({ + ...createBaseGoal(), + description: "Child 1", + parentGoal: parent.id, + }); + + const child2 = manager.addGoal({ + ...createBaseGoal(), + description: "Child 2", + parentGoal: parent.id, + }); + + const hierarchy = manager.getGoalHierarchy(parent.id); + expect(hierarchy).toHaveLength(3); + expect(hierarchy.map((g) => g.id)).toContain(child1.id); + expect(hierarchy.map((g) => g.id)).toContain(child2.id); + }); + + it("should update parent progress when child is completed", () => { + const parent = manager.addGoal({ + ...createBaseGoal(), + description: "Parent", + horizon: "medium", + }); + + const child1 = manager.addGoal({ + ...createBaseGoal(), + description: "Child 1", + parentGoal: parent.id, + }); + + const child2 = manager.addGoal({ + ...createBaseGoal(), + description: "Child 2", + parentGoal: parent.id, + }); + + manager.updateGoalStatus(child1.id, "completed"); + expect(manager.getGoalById(parent.id)?.progress).toBe(50); + + manager.updateGoalStatus(child2.id, "completed"); + const updatedParent = manager.getGoalById(parent.id); + expect(updatedParent?.progress).toBe(100); + expect(updatedParent?.status).toBe("ready"); + }); + }); + + describe("goal filtering and sorting", () => { + it("should get goals by horizon", () => { + manager.addGoal({ + ...createBaseGoal(), + description: "Short 1", + priority: 2, + }); + + manager.addGoal({ + ...createBaseGoal(), + description: "Short 2", + priority: 1, + }); + + manager.addGoal({ + ...createBaseGoal(), + description: "Medium", + horizon: "medium", + }); + + const shortGoals = manager.getGoalsByHorizon("short"); + expect(shortGoals).toHaveLength(2); + expect(shortGoals[0].priority).toBe(2); // Check priority sorting + }); + + it("should get ready goals correctly", () => { + // Create a ready goal + const readyGoal = manager.addGoal({ + ...createBaseGoal("ready"), + description: "Ready Goal", + }); + + // Create a blocked goal + const blockedGoal = manager.addGoal(createBaseGoal("pending")); + manager.updateGoalStatus(blockedGoal.id, "blocked"); + + const readyGoals = manager.getReadyGoals(); + expect(readyGoals).toHaveLength(1); + expect(readyGoals[0].description).toBe("Ready Goal"); + }); + }); + + describe("goal outcomes", () => { + it("should record goal outcomes correctly", () => { + const goal = manager.addGoal(createBaseGoal()); + + manager.recordGoalOutcome(goal.id, 0.8, "Good performance"); + const updated = manager.getGoalById(goal.id); + + expect(updated?.outcomeScore).toBe(0.8); + expect(updated?.scoreHistory).toHaveLength(1); + expect(updated?.status).toBe("completed"); + }); + + it("should record goal failures correctly", () => { + const goal = manager.addGoal(createBaseGoal()); + + manager.recordGoalFailure(goal.id, "Resource unavailable"); + const failed = manager.getGoalById(goal.id); + + expect(failed?.status).toBe("failed"); + expect(failed?.meta?.failReason).toBe("Resource unavailable"); + expect(failed?.scoreHistory).toHaveLength(1); + }); + }); + + describe("additional goal management", () => { + it("should update goal dependencies", () => { + const goal = manager.addGoal(createBaseGoal()); + const dep = manager.addGoal(createBaseGoal()); + + manager.updateGoalDependencies(goal.id, [dep.id]); + const updated = manager.getGoalById(goal.id); + + expect(updated?.dependencies).toContain(dep.id); + }); + + it("should get child goals", () => { + const parent = manager.addGoal(createBaseGoal()); + const child1 = manager.addGoal({ + ...createBaseGoal(), + parentGoal: parent.id, + }); + const child2 = manager.addGoal({ + ...createBaseGoal(), + parentGoal: parent.id, + }); + + const children = manager.getChildGoals(parent.id); + expect(children).toHaveLength(2); + expect(children.map((g) => g.id)).toContain(child1.id); + expect(children.map((g) => g.id)).toContain(child2.id); + }); + + it("should get dependent goals", () => { + const dep = manager.addGoal(createBaseGoal()); + const goal1 = manager.addGoal({ + ...createBaseGoal(), + dependencies: [dep.id], + }); + const goal2 = manager.addGoal({ + ...createBaseGoal(), + dependencies: [dep.id], + }); + + const dependents = manager.getDependentGoals(dep.id); + expect(dependents).toHaveLength(2); + expect(dependents.map((g) => g.id)).toContain(goal1.id); + expect(dependents.map((g) => g.id)).toContain(goal2.id); + }); + + it("should update goal progress", () => { + const goal = manager.addGoal(createBaseGoal()); + manager.updateGoalProgress(goal.id, 75); + + const updated = manager.getGoalById(goal.id); + expect(updated?.progress).toBe(75); + }); + + it("should get goals by status", () => { + const goal1 = manager.addGoal({ + ...createBaseGoal(), + status: "completed", + }); + const goal2 = manager.addGoal({ + ...createBaseGoal(), + status: "completed", + }); + + const completed = manager.getGoalsByStatus("completed"); + expect(completed).toHaveLength(2); + expect(completed.map((g) => g.id)).toContain(goal1.id); + expect(completed.map((g) => g.id)).toContain(goal2.id); + }); + + it("should check if goal can be refined", () => { + const shortGoal = manager.addGoal({ + ...createBaseGoal(), + horizon: "short", + }); + const mediumGoal = manager.addGoal({ + ...createBaseGoal(), + horizon: "medium", + }); + + expect(manager.canBeRefined(shortGoal.id)).toBe(false); + expect(manager.canBeRefined(mediumGoal.id)).toBe(true); + }); + + it("should block goal hierarchy", () => { + const parent = manager.addGoal(createBaseGoal()); + const child = manager.addGoal({ + ...createBaseGoal(), + parentGoal: parent.id, + }); + + manager.blockGoalHierarchy(parent.id, "Resource unavailable"); + + const blockedParent = manager.getGoalById(parent.id); + const blockedChild = manager.getGoalById(child.id); + + expect(blockedParent?.status).toBe("blocked"); + expect(blockedChild?.status).toBe("blocked"); + }); + + it("should get goal path", () => { + const root = manager.addGoal(createBaseGoal()); + const mid = manager.addGoal({ + ...createBaseGoal(), + parentGoal: root.id, + }); + const leaf = manager.addGoal({ + ...createBaseGoal(), + parentGoal: mid.id, + }); + + const path = manager.getGoalPath(leaf.id); + expect(path).toHaveLength(3); + expect(path[0].id).toBe(root.id); + expect(path[1].id).toBe(mid.id); + expect(path[2].id).toBe(leaf.id); + }); + + it("should estimate completion time", () => { + const goal = manager.addGoal({ + ...createBaseGoal(), + horizon: "medium", + }); + const dep = manager.addGoal({ + ...createBaseGoal(), + horizon: "short", + }); + manager.updateGoalDependencies(goal.id, [dep.id]); + + const estimate = manager.estimateCompletionTime(goal.id); + expect(estimate).toBeGreaterThan(0); + }); + + it("should get goals by score", () => { + const goal1 = manager.addGoal(createBaseGoal()); + const goal2 = manager.addGoal(createBaseGoal()); + + manager.recordGoalOutcome(goal1.id, 0.8); + manager.recordGoalOutcome(goal2.id, 0.9); + + const scored = manager.getGoalsByScore(); + expect(scored).toHaveLength(2); + expect(scored[0].outcomeScore).toBeGreaterThan( + scored[1].outcomeScore as number + ); + }); + }); +}); diff --git a/packages/core/src/core/__test__/vector-db.test.ts b/packages/core/src/core/__tests__/vector-db.test.ts similarity index 100% rename from packages/core/src/core/__test__/vector-db.test.ts rename to packages/core/src/core/__tests__/vector-db.test.ts diff --git a/packages/core/src/core/chain-of-thought.ts b/packages/core/src/core/chain-of-thought.ts index fa592721..d2173d9d 100644 --- a/packages/core/src/core/chain-of-thought.ts +++ b/packages/core/src/core/chain-of-thought.ts @@ -79,33 +79,12 @@ export class ChainOfThought extends EventEmitter { this.logger.debug( "decomposeObjectiveIntoGoals", "Planning strategy for objective", - { objective } - ); - - const context = await this.gatherRelevantContext(objective); - this.logger.info( - "decomposeObjectiveIntoGoals", - "Gathered relevant context", - { context } - ); - - const goals = await this.generateGoalHierarchy(objective, context); - this.logger.info( - "decomposeObjectiveIntoGoals", - "Generated goal hierarchy", - { goals } - ); - - await this.createAndLinkGoals(goals); - this.recordReasoningStep( - `Strategy planned for objective: ${objective}`, - "planning", - ["strategy-planning"], - { goals: this.getCreatedGoals() } + { + objective, + } ); - } - private async gatherRelevantContext(objective: string) { + // Fetch relevant documents and experiences related to the objective const [relevantDocs, relevantExperiences] = await Promise.all([ this.memory.findSimilarDocuments(objective, 5), this.memory.findSimilarEpisodes(objective, 3), @@ -120,6 +99,7 @@ export class ChainOfThought extends EventEmitter { } ); + // Build context from relevant documents const gameStateContext = relevantDocs .map( (doc) => ` @@ -131,6 +111,7 @@ export class ChainOfThought extends EventEmitter { ) .join("\n\n"); + // Build context from past experiences const experienceContext = relevantExperiences .map( (exp) => ` @@ -143,23 +124,16 @@ export class ChainOfThought extends EventEmitter { ) .join("\n\n"); - return { gameStateContext, experienceContext }; - } - - private async generateGoalHierarchy( - objective: string, - context: { gameStateContext: string; experienceContext: string } - ) { const prompt = ` "${objective}" - ${context.gameStateContext} + ${gameStateContext} - ${context.experienceContext} + ${experienceContext} @@ -207,7 +181,7 @@ export class ChainOfThought extends EventEmitter { }); try { - return await validateLLMResponseSchema({ + const goals = await validateLLMResponseSchema({ prompt, systemPrompt: "You are a strategic planning system that creates hierarchical goal structures.", @@ -217,12 +191,91 @@ export class ChainOfThought extends EventEmitter { this.logger.error( "decomposeObjectiveIntoGoals", `Attempt ${attempt} failed`, - { error } + { + error, + } ); }, llmClient: this.llmClient, logger: this.logger, }); + + const allLLMGoals = [ + ...goals.long_term.map((g) => ({ + horizon: "long" as const, + ...g, + })), + ...goals.medium_term.map((g) => ({ + horizon: "medium" as const, + ...g, + })), + ...goals.short_term.map((g) => ({ + horizon: "short" as const, + ...g, + })), + ]; + + // Link: LLM’s "id" -> goal manager’s "goal-xyz" ID + const llmIdToRealId = new Map(); + + // Keep track of newly created goal IDs so we can fetch them after pass #2 + const createdGoalIds: string[] = []; + + // Pass #1: Create each goal (with empty dependencies) + for (const llmGoal of allLLMGoals) { + const { + id: llmId, + horizon, + dependencies: _, + ...rest + } = llmGoal; + + // Create a new goal, letting GoalManager generate the random ID + const newGoal = this.goalManager.addGoal({ + horizon, + status: "pending", + created_at: Date.now(), + dependencies: [], // empty for now, will fill in pass #2 + ...rest, + }); + + // Map LLM’s temp ID -> our new random ID + llmIdToRealId.set(llmId, newGoal.id); + createdGoalIds.push(newGoal.id); + + this.emit("goal:created", { + id: newGoal.id, + description: newGoal.description, + priority: newGoal.priority, + }); + } + + // PASS #2: Update dependencies with real IDs + for (const llmGoal of allLLMGoals) { + // Grab the real ID for this LLM goal + const realGoalId = llmIdToRealId.get(llmGoal.id); + if (!realGoalId) continue; + + // Convert LLM dependencies to our manager IDs + const realDeps = llmGoal.dependencies + .map((dep) => llmIdToRealId.get(dep)) + .filter((id): id is string => !!id); + + this.goalManager.updateGoalDependencies(realGoalId, realDeps); + } + + // Get all the goals we just created + const finalGoals = createdGoalIds + .map((id) => this.goalManager.getGoalById(id)) + .filter((g): g is Goal => !!g); + + // Add a planning step + this.recordReasoningStep( + `Strategy planned for objective: ${objective}`, + "planning", + ["strategy-planning"], + { goals: finalGoals } + ); } catch (error) { this.logger.error( "decomposeObjectiveIntoGoals", @@ -233,64 +286,6 @@ export class ChainOfThought extends EventEmitter { } } - private async createAndLinkGoals(goals: { - long_term: any[]; - medium_term: any[]; - short_term: any[]; - }) { - const allLLMGoals = [ - ...goals.long_term.map((g) => ({ horizon: "long" as const, ...g })), - ...goals.medium_term.map((g) => ({ - horizon: "medium" as const, - ...g, - })), - ...goals.short_term.map((g) => ({ - horizon: "short" as const, - ...g, - })), - ]; - - const llmIdToRealId = new Map(); - const createdGoalIds: string[] = []; - - // Pass #1: Create goals - for (const llmGoal of allLLMGoals) { - const { id: llmId, horizon, dependencies: _, ...rest } = llmGoal; - const newGoal = this.goalManager.addGoal({ - horizon, - status: "pending", - created_at: Date.now(), - dependencies: [], - ...rest, - }); - - llmIdToRealId.set(llmId, newGoal.id); - createdGoalIds.push(newGoal.id); - - this.emit("goal:created", { - id: newGoal.id, - description: newGoal.description, - priority: newGoal.priority, - }); - } - - // Pass #2: Link dependencies - for (const llmGoal of allLLMGoals) { - const realGoalId = llmIdToRealId.get(llmGoal.id); - if (!realGoalId) continue; - - const realDeps = llmGoal.dependencies - .map((dep: string) => llmIdToRealId.get(dep)) - .filter((id: string | undefined): id is string => !!id); - - this.goalManager.updateGoalDependencies(realGoalId, realDeps); - } - } - - private getCreatedGoals(): Goal[] { - return Array.from(this.goalManager.goals.values()); - } - /** * Checks if a goal can be executed based on current state and requirements. * @@ -310,14 +305,55 @@ export class ChainOfThought extends EventEmitter { missing_requirements: string[]; incompleteState?: boolean; }> { - try { - const [relevantDocs, relevantExperiences, blackboardState] = - await Promise.all([ - this.memory.findSimilarDocuments(goal.description, 5), - this.memory.findSimilarEpisodes(goal.description, 3), - this.getBlackboardState(), - ]); + const [relevantDocs, relevantExperiences, blackboardState] = + await Promise.all([ + this.memory.findSimilarDocuments(goal.description, 5), + this.memory.findSimilarEpisodes(goal.description, 3), + this.getBlackboardState(), + ]); + + const prompt = ` + + + + ${goal.description} + + + + ${relevantDocs + .map((doc) => `Document: ${doc.title}\n${doc.content}`) + .join("\n\n")} + + + + ${relevantExperiences + .map((exp) => `Experience: ${exp.action}\n${exp.outcome}`) + .join("\n\n")} + + + ${JSON.stringify(blackboardState, null, 2)} + + + # Required dependencies: + ${JSON.stringify(goal.dependencies || {}, null, 2)} + + # Analyze if this goal can be executed right now. Consider: + + 1. Are all required resources available in the current game state? + 2. Are environmental conditions met? + 3. Are there any blocking conditions? + 4. Do we have the necessary game state requirements? + + If you need to query then you could potentially complete the goal. + + + Think about this goal and the context here. + + + `; + + try { const schema = z .object({ possible: z.boolean(), @@ -332,13 +368,6 @@ export class ChainOfThought extends EventEmitter { }) .strict(); - const prompt = this.buildValidationPrompt( - goal, - relevantDocs, - relevantExperiences, - blackboardState - ); - const response = await validateLLMResponseSchema<{ possible: boolean; reason: string; @@ -353,7 +382,9 @@ export class ChainOfThought extends EventEmitter { this.logger.warn( "validateGoalPrerequisites", `Retry attempt ${attempt}`, - { error } + { + error, + } ); }, llmClient: this.llmClient, @@ -381,52 +412,6 @@ export class ChainOfThought extends EventEmitter { }; } } - - private buildValidationPrompt( - goal: Goal, - relevantDocs: any[], - relevantExperiences: any[], - blackboardState: any - ): string { - return ` - - ${goal.description} - - - - ${relevantDocs - .map((doc) => `Document: ${doc.title}\n${doc.content}`) - .join("\n\n")} - - - - ${relevantExperiences - .map((exp) => `Experience: ${exp.action}\n${exp.outcome}`) - .join("\n\n")} - - - - ${JSON.stringify(blackboardState, null, 2)} - - - # Required dependencies: - ${JSON.stringify(goal.dependencies || {}, null, 2)} - - # Analyze if this goal can be executed right now. Consider: - - 1. Are all required resources available in the current game state? - 2. Are environmental conditions met? - 3. Are there any blocking conditions? - 4. Do we have the necessary game state requirements? - - If you need to query then you could potentially complete the goal. - - - Think about this goal and the context here. - - `; - } - /** * Refines a high-level goal into more specific, actionable sub-goals by analyzing relevant context. * @@ -444,7 +429,7 @@ export class ChainOfThought extends EventEmitter { private async breakdownGoalIntoSubtasks( goal: Goal, maxRetries: number = 3 - ): Promise { + ): Promise { const [relevantDocs, relevantExperiences, blackboardState] = await Promise.all([ this.memory.findSimilarDocuments(goal.description, 5), @@ -539,13 +524,9 @@ export class ChainOfThought extends EventEmitter { // Update original goal status this.goalManager.updateGoalStatus(goal.id, "active"); } catch (error) { - this.logger.error( - "breakdownGoalIntoSubtasks", - "Failed to refine goal", - { error } + throw new Error( + `Failed to refine goal after ${maxRetries} attempts: ${error}` ); - - return JSON.stringify({ error }); } } @@ -674,12 +655,7 @@ export class ChainOfThought extends EventEmitter { error, } ); - await this.goalManager.processGoalFailure(currentGoal); - - this.emit("goal:failed", { - id: currentGoal.id, - error, - }); + await this.processGoalFailure(currentGoal, error); // Go on to the next goal continue; } @@ -789,6 +765,35 @@ export class ChainOfThought extends EventEmitter { } } + /** + * Handles the failure of a goal by updating its status and notifying relevant systems. + * + * This method: + * 1. Updates the failed goal's status + * 2. If the goal has a parent, marks the parent as blocked + * 3. Emits a goal:failed event + * + * @param goal - The goal that failed + * @param error - The error that caused the failure + * @internal + */ + private async processGoalFailure( + goal: Goal, + error: Error | unknown + ): Promise { + this.goalManager.updateGoalStatus(goal.id, "failed"); + + // If this was a sub-goal, mark parent as blocked + if (goal.parentGoal) { + this.goalManager.updateGoalStatus(goal.parentGoal, "blocked"); + } + + this.emit("goal:failed", { + id: goal.id, + error, + }); + } + /** * Validates whether a goal has been successfully achieved by analyzing the current context * against the goal's success criteria. @@ -1125,27 +1130,6 @@ export class ChainOfThought extends EventEmitter { } } - /** - * Returns a formatted string listing all available outputs registered in the outputs registry. - * The string includes each output name on a new line prefixed with a bullet point. - * @returns A formatted string containing all registered output names - * @example - * ```ts - * // If outputs contains "console" and "file" - * getAvailableOutputs() // Returns: - * // Available outputs: - * // - console - * // - file - * ``` - * @internal - */ - private getAvailableOutputs(): string { - const outputs = Array.from(this.outputs.keys()); - return `Available outputs:\n${outputs - .map((output) => `- ${output}`) - .join("\n")}`; - } - private buildPrompt(tags: Record = {}): string { this.logger.debug("buildPrompt", "Building LLM prompt"); @@ -1236,16 +1220,68 @@ ${availableOutputs public async think( userQuery: string, maxIterations: number = 10 - ): Promise { + ): Promise { this.emit("think:start", { query: userQuery }); try { - await this.initializeThinkingContext(userQuery); - let pendingActions = await this.getInitialActions(userQuery); + // Consult single memory instance for both types of memories + const [similarExperiences, relevantDocs] = await Promise.all([ + this.memory.findSimilarEpisodes(userQuery, 1), + this.memory.findSimilarDocuments(userQuery, 1), + ]); + + this.logger.debug("think", "Retrieved memory context", { + experienceCount: similarExperiences.length, + docCount: relevantDocs.length, + }); + + this.logger.debug("think", "Beginning to think", { + userQuery, + maxIterations, + }); + + // Initialize with user query + this.recordReasoningStep(`User Query: ${userQuery}`, "task", [ + "user-query", + ]); let currentIteration = 0; let isComplete = false; + // Get initial plan and actions + const initialResponse = await validateLLMResponseSchema({ + prompt: this.buildPrompt({ query: userQuery }), + schema: z.object({ + plan: z.string().optional(), + meta: z.any().optional(), + actions: z.array( + z.object({ + type: z.string(), + payload: z.any(), + }) + ), + }), + systemPrompt: + "You are a reasoning system that outputs structured JSON only.", + maxRetries: 3, + llmClient: this.llmClient, + logger: this.logger, + }); + + // Initialize pending actions queue with initial actions + let pendingActions: CoTAction[] = [ + ...initialResponse.actions, + ] as CoTAction[]; + + // Add initial plan as a step if provided + if (initialResponse.plan) { + this.recordReasoningStep( + `Initial plan: ${initialResponse.plan}`, + "planning", + ["initial-plan"] + ); + } + while ( !isComplete && currentIteration < maxIterations && @@ -1256,162 +1292,31 @@ ${availableOutputs pendingActionsCount: pendingActions.length, }); + // Process one action at a time const currentAction = pendingActions.shift()!; + this.logger.debug("think", "Processing action", { + action: currentAction, + remainingActions: pendingActions.length, + }); try { const result = await this.executeAction(currentAction); - await this.handleActionResult(currentAction, result); - - const completion = await this.verifyCompletion( - userQuery, - result, - currentAction - ); - isComplete = completion.complete; + // Store the experience + await this.storeEpisode(currentAction.type, result); - if (completion.newActions?.length > 0) { - pendingActions.push( - ...this.extractNewActions(completion.newActions) + // If the result seems important, store as knowledge + if (calculateImportance(result) > 0.7) { + await this.storeKnowledge( + `Learning from ${currentAction.type}`, + result, + "action_learning", + [currentAction.type, "automated_learning"] ); } - if (isComplete || !completion.shouldContinue) { - await this.handleCompletion( - isComplete, - completion.reason, - userQuery - ); - return; - } - - this.recordReasoningStep( - `Action completed, continuing execution: ${completion.reason}`, - "system", - ["continuation"] - ); - } catch (error) { - this.emit("think:error", { query: userQuery, error }); - return JSON.stringify({ error }); - } - - currentIteration++; - } - - if (currentIteration >= maxIterations) { - const error = `Failed to solve query "${userQuery}" within ${maxIterations} iterations`; - this.logger.error("think", error); - this.emit("think:timeout", { query: userQuery }); - } - } catch (error) { - this.emit("think:error", { query: userQuery, error }); - return JSON.stringify({ error }); - } - } - - private async initializeThinkingContext(userQuery: string): Promise { - const [similarExperiences, relevantDocs] = await Promise.all([ - this.memory.findSimilarEpisodes(userQuery, 1), - this.memory.findSimilarDocuments(userQuery, 1), - ]); - - this.logger.debug("think", "Retrieved memory context", { - experienceCount: similarExperiences.length, - docCount: relevantDocs.length, - }); - - this.logger.debug("think", "Beginning to think", { - userQuery, - }); - - this.recordReasoningStep(`User Query: ${userQuery}`, "task", [ - "user-query", - ]); - } - - private async getInitialActions(userQuery: string): Promise { - const initialResponse = await validateLLMResponseSchema({ - prompt: this.buildPrompt({ query: userQuery }), - schema: z.object({ - plan: z.string().optional(), - meta: z.any().optional(), - actions: z.array( - z.object({ - type: z.string(), - payload: z.any(), - }) - ), - }), - systemPrompt: - "You are a reasoning system that outputs structured JSON only.", - maxRetries: 3, - llmClient: this.llmClient, - logger: this.logger, - }); - - if (initialResponse.plan) { - this.recordReasoningStep( - `Initial plan: ${initialResponse.plan}`, - "planning", - ["initial-plan"] - ); - } - - return [...initialResponse.actions] as CoTAction[]; - } - - private async handleActionResult( - currentAction: CoTAction, - result: any - ): Promise { - await this.storeEpisode(currentAction.type, result); - - if (calculateImportance(result) > 0.7) { - await this.storeKnowledge( - `Learning from ${currentAction.type}`, - result, - "action_learning", - [currentAction.type, "automated_learning"] - ); - } - } - - private async verifyCompletion( - userQuery: string, - result: any, - currentAction: CoTAction - ): Promise<{ - complete: boolean; - reason: string; - shouldContinue: boolean; - newActions: any[]; - }> { - return await validateLLMResponseSchema({ - prompt: this.buildVerificationPrompt( - userQuery, - result, - currentAction - ), - schema: z.object({ - complete: z.boolean(), - reason: z.string(), - shouldContinue: z.boolean(), - newActions: z.array(z.any()), - }), - systemPrompt: - "You are a goal completion analyzer using Chain of Verification...", - maxRetries: 3, - llmClient: this.llmClient, - logger: this.logger, - }); - } - - private buildVerificationPrompt( - userQuery: string, - result: any, - currentAction: CoTAction - ): string { - return `${this.buildPrompt({ result })} + const completion = await validateLLMResponseSchema({ + prompt: `${this.buildPrompt({ result })} ${JSON.stringify({ query: userQuery, currentSteps: this.stepManager.getSteps(), @@ -1449,33 +1354,86 @@ ${availableOutputs Think in detail here - `; - } + + `, + schema: z.object({ + complete: z.boolean(), + reason: z.string(), + shouldContinue: z.boolean(), + newActions: z.array(z.any()), + }), + systemPrompt: + "You are a goal completion analyzer using Chain of Verification...", + maxRetries: 3, + llmClient: this.llmClient, + logger: this.logger, + }); - private extractNewActions(newActions: any[]): CoTAction[] { - const extractedActions = newActions.flatMap( - (plan: any) => plan.actions || [] - ); + try { + isComplete = completion.complete; + + if (completion.newActions?.length > 0) { + // Add new actions to the end of the pending queue + const extractedActions = + completion.newActions.flatMap( + (plan: any) => plan.actions || [] + ); + pendingActions.push(...extractedActions); + + this.logger.debug("think", "Added new actions", { + newActionsCount: extractedActions.length, + totalPendingCount: pendingActions.length, + }); + } - this.logger.debug("think", "Added new actions", { - newActionsCount: extractedActions.length, - totalPendingCount: extractedActions.length, - }); + if (isComplete || !completion.shouldContinue) { + this.recordReasoningStep( + `Goal ${isComplete ? "achieved" : "failed"}: ${ + completion.reason + }`, + "system", + ["completion"] + ); + this.emit("think:complete", { query: userQuery }); + return; + } else { + this.recordReasoningStep( + `Action completed, continuing execution: ${completion.reason}`, + "system", + ["continuation"] + ); + } + } catch (error) { + this.logger.error( + "think", + "Error parsing completion check", + { + error: + error instanceof Error + ? error.message + : String(error), + completion, + } + ); + continue; + } + } catch (error) { + this.emit("think:error", { query: userQuery, error }); + throw error; + } - return extractedActions; - } + currentIteration++; + } - private async handleCompletion( - isComplete: boolean, - reason: string, - userQuery: string - ): Promise { - this.recordReasoningStep( - `Goal ${isComplete ? "achieved" : "failed"}: ${reason}`, - "system", - ["completion"] - ); - this.emit("think:complete", { query: userQuery }); + if (currentIteration >= maxIterations) { + const error = `Failed to solve query "${userQuery}" within ${maxIterations} iterations`; + this.logger.error("think", error); + this.emit("think:timeout", { query: userQuery }); + } + } catch (error) { + this.emit("think:error", { query: userQuery, error }); + throw error; + } } /** @@ -1564,6 +1522,10 @@ ${availableOutputs id: crypto.randomUUID(), }, }); + + this.logger.info("storeEpisode", "Stored experience", { + experience, + }); } catch (error) { this.logger.error("storeEpisode", "Failed to store experience", { error, @@ -1598,6 +1560,10 @@ ${availableOutputs await this.memory.storeDocument(document); + this.logger.info("storeKnowledge", "Stored knowledge", { + document, + }); + this.emit("memory:knowledge_stored", { document }); } catch (error) { this.logger.error("storeKnowledge", "Failed to store knowledge", { @@ -1678,6 +1644,10 @@ ${availableOutputs state[update.type][update.key] = update.value; }); + this.logger.info("getBlackboardState", "Found blackboard state", { + state, + }); + return state; } catch (error) { this.logger.error( @@ -1719,6 +1689,14 @@ ${availableOutputs const docs = await this.memory.searchDocumentsByTag(tags, limit); + this.logger.info( + "getBlackboardHistory", + "Found blackboard history", + { + docs, + } + ); + return docs .map((doc) => ({ ...JSON.parse(doc.content), @@ -1737,4 +1715,25 @@ ${availableOutputs return []; } } + + /** + * Returns a formatted string listing all available outputs registered in the outputs registry. + * The string includes each output name on a new line prefixed with a bullet point. + * @returns A formatted string containing all registered output names + * @example + * ```ts + * // If outputs contains "console" and "file" + * getAvailableOutputs() // Returns: + * // Available outputs: + * // - console + * // - file + * ``` + * @internal + */ + private getAvailableOutputs(): string { + const outputs = Array.from(this.outputs.keys()); + return `Available outputs:\n${outputs + .map((output) => `- ${output}`) + .join("\n")}`; + } } diff --git a/packages/core/src/core/goal-manager.ts b/packages/core/src/core/goal-manager.ts index 1f8359a1..11129eb3 100644 --- a/packages/core/src/core/goal-manager.ts +++ b/packages/core/src/core/goal-manager.ts @@ -178,13 +178,15 @@ export class GoalManager { return Array.from(this.goals.values()) .filter((goal) => { const horizonMatch = !horizon || goal.horizon === horizon; + // Only consider goals that are explicitly in "ready" status const isReady = goal.status === "ready"; const dependenciesMet = !goal.dependencies?.length || goal.dependencies.every( (depId) => this.goals.get(depId)?.status === "completed" ); - return horizonMatch && (isReady || dependenciesMet); + // A goal is only ready if it's explicitly in ready status AND dependencies are met + return horizonMatch && isReady && dependenciesMet; }) .sort((a, b) => b.priority - a.priority); } From 83d3d8453c5a2ca865c8a4078f360a25087606a7 Mon Sep 17 00:00:00 2001 From: ponderingdemocritus Date: Tue, 28 Jan 2025 18:37:45 +1100 Subject: [PATCH 2/2] cleanup --- package.json | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/package.json b/package.json index e229481a..9473c00c 100644 --- a/package.json +++ b/package.json @@ -33,10 +33,7 @@ "devDependencies": { "@types/node": "^22.10.5", "knip": "^5.43.1", - "typescript": "~5.6.2", - "@types/jest": "^29.0.0", - "jest": "^29.0.0", - "ts-jest": "^29.0.0" + "typescript": "~5.6.2" }, "packageManager": "pnpm@9.15.2+sha512.93e57b0126f0df74ce6bff29680394c0ba54ec47246b9cf321f0121d8d9bb03f750a705f24edc3c1180853afd7c2c3b94196d0a3d53d3e069d9e2793ef11f321" }