diff --git a/src/commands/onboard-memory.test.ts b/src/commands/onboard-memory.test.ts new file mode 100644 index 00000000000..fb5b6035852 --- /dev/null +++ b/src/commands/onboard-memory.test.ts @@ -0,0 +1,439 @@ +import { describe, expect, it, vi, beforeEach } from "vitest"; +import type { OpenClawConfig } from "../config/config.js"; +import type { RuntimeEnv } from "../runtime.js"; +import type { WizardPrompter } from "../wizard/prompts.js"; +import { applyNonInteractiveMemoryDefaults, setupMemoryOptimization } from "./onboard-memory.js"; + +describe("onboard-memory", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + const createMockPrompter = (multiselectValue: string[]): WizardPrompter => ({ + confirm: vi.fn().mockResolvedValue(true), + note: vi.fn().mockResolvedValue(undefined), + intro: vi.fn().mockResolvedValue(undefined), + outro: vi.fn().mockResolvedValue(undefined), + text: vi.fn().mockResolvedValue(""), + select: vi.fn().mockResolvedValue(""), + multiselect: vi.fn().mockResolvedValue(multiselectValue), + progress: vi.fn().mockReturnValue({ + stop: vi.fn(), + update: vi.fn(), + }), + }); + + const createMockRuntime = (): RuntimeEnv => ({ + log: vi.fn(), + error: vi.fn(), + exit: vi.fn() as unknown as RuntimeEnv["exit"], + }); + + describe("setupMemoryOptimization", () => { + it("should enable all options when all are selected", async () => { + const cfg: OpenClawConfig = {}; + const prompter = createMockPrompter([ + "hybrid-search", + "embedding-cache", + "memory-flush", + "session-transcripts", + ]); + const runtime = createMockRuntime(); + + const result = await setupMemoryOptimization(cfg, runtime, prompter); + + // Hybrid search + expect(result.agents?.defaults?.memorySearch?.query?.hybrid?.enabled).toBe(true); + expect(result.agents?.defaults?.memorySearch?.query?.hybrid?.vectorWeight).toBe(0.7); + expect(result.agents?.defaults?.memorySearch?.query?.hybrid?.textWeight).toBe(0.3); + expect(result.agents?.defaults?.memorySearch?.query?.hybrid?.candidateMultiplier).toBe(4); + + // Embedding cache + expect(result.agents?.defaults?.memorySearch?.cache?.enabled).toBe(true); + expect(result.agents?.defaults?.memorySearch?.cache?.maxEntries).toBe(50_000); + + // Memory flush + expect(result.agents?.defaults?.compaction?.mode).toBe("safeguard"); + expect(result.agents?.defaults?.compaction?.memoryFlush?.enabled).toBe(true); + + // Session transcripts + expect(result.agents?.defaults?.memorySearch?.enabled).toBe(true); + expect(result.agents?.defaults?.memorySearch?.experimental?.sessionMemory).toBe(true); + expect(result.agents?.defaults?.memorySearch?.sync?.sessions?.deltaBytes).toBe(50_000); + expect(result.agents?.defaults?.memorySearch?.sync?.sessions?.deltaMessages).toBe(25); + }); + + it("should enable only hybrid search when selected alone", async () => { + const cfg: OpenClawConfig = {}; + const prompter = createMockPrompter(["hybrid-search"]); + const runtime = createMockRuntime(); + + const result = await setupMemoryOptimization(cfg, runtime, prompter); + + expect(result.agents?.defaults?.memorySearch?.query?.hybrid?.enabled).toBe(true); + expect(result.agents?.defaults?.memorySearch?.query?.hybrid?.vectorWeight).toBe(0.7); + expect(result.agents?.defaults?.memorySearch?.query?.hybrid?.textWeight).toBe(0.3); + expect(result.agents?.defaults?.memorySearch?.query?.hybrid?.candidateMultiplier).toBe(4); + + // Other options should not be set + expect(result.agents?.defaults?.memorySearch?.cache).toBeUndefined(); + expect(result.agents?.defaults?.compaction).toBeUndefined(); + expect(result.agents?.defaults?.memorySearch?.experimental).toBeUndefined(); + }); + + it("should enable only embedding cache when selected alone", async () => { + const cfg: OpenClawConfig = {}; + const prompter = createMockPrompter(["embedding-cache"]); + const runtime = createMockRuntime(); + + const result = await setupMemoryOptimization(cfg, runtime, prompter); + + expect(result.agents?.defaults?.memorySearch?.cache?.enabled).toBe(true); + expect(result.agents?.defaults?.memorySearch?.cache?.maxEntries).toBe(50_000); + + // Other options should not be set + expect(result.agents?.defaults?.memorySearch?.query).toBeUndefined(); + expect(result.agents?.defaults?.compaction).toBeUndefined(); + expect(result.agents?.defaults?.memorySearch?.experimental).toBeUndefined(); + }); + + it("should enable only memory flush when selected alone", async () => { + const cfg: OpenClawConfig = {}; + const prompter = createMockPrompter(["memory-flush"]); + const runtime = createMockRuntime(); + + const result = await setupMemoryOptimization(cfg, runtime, prompter); + + expect(result.agents?.defaults?.compaction?.mode).toBe("safeguard"); + expect(result.agents?.defaults?.compaction?.memoryFlush?.enabled).toBe(true); + + // Other options should not be set + expect(result.agents?.defaults?.memorySearch).toBeUndefined(); + }); + + it("should enable only session transcripts when selected alone", async () => { + const cfg: OpenClawConfig = {}; + const prompter = createMockPrompter(["session-transcripts"]); + const runtime = createMockRuntime(); + + const result = await setupMemoryOptimization(cfg, runtime, prompter); + + expect(result.agents?.defaults?.memorySearch?.enabled).toBe(true); + expect(result.agents?.defaults?.memorySearch?.experimental?.sessionMemory).toBe(true); + expect(result.agents?.defaults?.memorySearch?.sync?.sessions?.deltaBytes).toBe(50_000); + expect(result.agents?.defaults?.memorySearch?.sync?.sessions?.deltaMessages).toBe(25); + + // Other options should not be set + expect(result.agents?.defaults?.memorySearch?.query).toBeUndefined(); + expect(result.agents?.defaults?.memorySearch?.cache).toBeUndefined(); + expect(result.agents?.defaults?.compaction).toBeUndefined(); + }); + + it("should return config unchanged when user skips", async () => { + const cfg: OpenClawConfig = { + agents: { defaults: { workspace: "/my-workspace" } }, + }; + const prompter = createMockPrompter(["__skip__"]); + const runtime = createMockRuntime(); + + const result = await setupMemoryOptimization(cfg, runtime, prompter); + + expect(result).toEqual(cfg); + // Only the intro note should be shown, no confirmation note + expect(prompter.note).toHaveBeenCalledTimes(1); + }); + + it("should NOT overwrite existing hybrid search config values", async () => { + const cfg: OpenClawConfig = { + agents: { + defaults: { + memorySearch: { + query: { + hybrid: { + enabled: true, + vectorWeight: 0.5, + textWeight: 0.5, + candidateMultiplier: 8, + }, + }, + }, + }, + }, + }; + const prompter = createMockPrompter(["hybrid-search"]); + const runtime = createMockRuntime(); + + const result = await setupMemoryOptimization(cfg, runtime, prompter); + + // Existing values must be preserved (nullish coalescing) + expect(result.agents?.defaults?.memorySearch?.query?.hybrid?.vectorWeight).toBe(0.5); + expect(result.agents?.defaults?.memorySearch?.query?.hybrid?.textWeight).toBe(0.5); + expect(result.agents?.defaults?.memorySearch?.query?.hybrid?.candidateMultiplier).toBe(8); + }); + + it("should NOT overwrite existing embedding cache config values", async () => { + const cfg: OpenClawConfig = { + agents: { + defaults: { + memorySearch: { + cache: { + enabled: false, + maxEntries: 10_000, + }, + }, + }, + }, + }; + const prompter = createMockPrompter(["embedding-cache"]); + const runtime = createMockRuntime(); + + const result = await setupMemoryOptimization(cfg, runtime, prompter); + + expect(result.agents?.defaults?.memorySearch?.cache?.enabled).toBe(false); + expect(result.agents?.defaults?.memorySearch?.cache?.maxEntries).toBe(10_000); + }); + + it("should NOT overwrite existing compaction config values", async () => { + const cfg: OpenClawConfig = { + agents: { + defaults: { + compaction: { + mode: "default", + memoryFlush: { + enabled: false, + }, + }, + }, + }, + }; + const prompter = createMockPrompter(["memory-flush"]); + const runtime = createMockRuntime(); + + const result = await setupMemoryOptimization(cfg, runtime, prompter); + + // Existing values must be preserved + expect(result.agents?.defaults?.compaction?.mode).toBe("default"); + expect(result.agents?.defaults?.compaction?.memoryFlush?.enabled).toBe(false); + }); + + it("should NOT overwrite existing session transcript config values", async () => { + const cfg: OpenClawConfig = { + agents: { + defaults: { + memorySearch: { + enabled: false, + experimental: { + sessionMemory: false, + }, + sync: { + sessions: { + deltaBytes: 100_000, + deltaMessages: 50, + }, + }, + }, + }, + }, + }; + const prompter = createMockPrompter(["session-transcripts"]); + const runtime = createMockRuntime(); + + const result = await setupMemoryOptimization(cfg, runtime, prompter); + + expect(result.agents?.defaults?.memorySearch?.enabled).toBe(false); + expect(result.agents?.defaults?.memorySearch?.experimental?.sessionMemory).toBe(false); + expect(result.agents?.defaults?.memorySearch?.sync?.sessions?.deltaBytes).toBe(100_000); + expect(result.agents?.defaults?.memorySearch?.sync?.sessions?.deltaMessages).toBe(50); + }); + + it("should preserve unrelated existing config", async () => { + const cfg: OpenClawConfig = { + agents: { + defaults: { + workspace: "/existing-workspace", + model: { primary: "anthropic/claude-opus-4-5" }, + }, + }, + gateway: { mode: "local", port: 3000 }, + }; + const prompter = createMockPrompter(["hybrid-search"]); + const runtime = createMockRuntime(); + + const result = await setupMemoryOptimization(cfg, runtime, prompter); + + expect(result.agents?.defaults?.workspace).toBe("/existing-workspace"); + expect((result.agents?.defaults?.model as { primary?: string })?.primary).toBe( + "anthropic/claude-opus-4-5", + ); + expect(result.gateway?.mode).toBe("local"); + expect(result.gateway?.port).toBe(3000); + // And the new config should still be applied + expect(result.agents?.defaults?.memorySearch?.query?.hybrid?.enabled).toBe(true); + }); + + it("should show correct multiselect options", async () => { + const cfg: OpenClawConfig = {}; + const prompter = createMockPrompter(["__skip__"]); + const runtime = createMockRuntime(); + + await setupMemoryOptimization(cfg, runtime, prompter); + + expect(prompter.multiselect).toHaveBeenCalledWith({ + message: "Enable memory optimizations?", + options: [ + { value: "__skip__", label: "Skip for now" }, + { + value: "hybrid-search", + label: "🔍 Hybrid search (BM25 + vector)", + hint: "70/30 vector/text blend with 4x candidate pool — improves recall for exact terms", + }, + { + value: "embedding-cache", + label: "💾 Embedding cache", + hint: "Caches embeddings in SQLite — saves API calls on reindex", + }, + { + value: "memory-flush", + label: "🧠 Pre-compaction memory flush", + hint: "Auto-saves notes before context compaction — prevents amnesia", + }, + { + value: "session-transcripts", + label: "📜 Session transcript search", + hint: "Indexes past transcripts via memory_search (experimental)", + }, + ], + }); + }); + + it("should show intro and confirmation notes", async () => { + const cfg: OpenClawConfig = {}; + const prompter = createMockPrompter(["hybrid-search", "memory-flush"]); + const runtime = createMockRuntime(); + + await setupMemoryOptimization(cfg, runtime, prompter); + + const noteCalls = (prompter.note as ReturnType).mock.calls; + expect(noteCalls).toHaveLength(2); + + // Intro note + expect(noteCalls[0][0]).toContain("Memory optimization"); + expect(noteCalls[0][1]).toBe("Memory Optimization"); + + // Confirmation note + expect(noteCalls[1][0]).toContain("Enabled 2 optimizations: hybrid search, memory flush"); + expect(noteCalls[1][1]).toBe("Memory Configured"); + }); + + it("should not mutate the original config", async () => { + const cfg: OpenClawConfig = { + agents: { defaults: { workspace: "/ws" } }, + }; + const original = JSON.stringify(cfg); + const prompter = createMockPrompter(["hybrid-search"]); + const runtime = createMockRuntime(); + + await setupMemoryOptimization(cfg, runtime, prompter); + + expect(JSON.stringify(cfg)).toBe(original); + }); + }); + + describe("applyNonInteractiveMemoryDefaults", () => { + it("should enable hybrid search, embedding cache, and memory flush", () => { + const cfg: OpenClawConfig = {}; + + const result = applyNonInteractiveMemoryDefaults(cfg); + + // Hybrid search + expect(result.agents?.defaults?.memorySearch?.query?.hybrid?.enabled).toBe(true); + expect(result.agents?.defaults?.memorySearch?.query?.hybrid?.vectorWeight).toBe(0.7); + expect(result.agents?.defaults?.memorySearch?.query?.hybrid?.textWeight).toBe(0.3); + expect(result.agents?.defaults?.memorySearch?.query?.hybrid?.candidateMultiplier).toBe(4); + + // Embedding cache + expect(result.agents?.defaults?.memorySearch?.cache?.enabled).toBe(true); + expect(result.agents?.defaults?.memorySearch?.cache?.maxEntries).toBe(50_000); + + // Memory flush + expect(result.agents?.defaults?.compaction?.mode).toBe("safeguard"); + expect(result.agents?.defaults?.compaction?.memoryFlush?.enabled).toBe(true); + }); + + it("should NOT enable session transcript search (experimental, opt-in only)", () => { + const cfg: OpenClawConfig = {}; + + const result = applyNonInteractiveMemoryDefaults(cfg); + + expect(result.agents?.defaults?.memorySearch?.experimental?.sessionMemory).toBeUndefined(); + expect(result.agents?.defaults?.memorySearch?.sync?.sessions).toBeUndefined(); + }); + + it("should NOT overwrite existing values", () => { + const cfg: OpenClawConfig = { + agents: { + defaults: { + memorySearch: { + query: { + hybrid: { + enabled: false, + vectorWeight: 0.9, + }, + }, + cache: { + enabled: false, + }, + }, + compaction: { + mode: "default", + memoryFlush: { + enabled: false, + }, + }, + }, + }, + }; + + const result = applyNonInteractiveMemoryDefaults(cfg); + + expect(result.agents?.defaults?.memorySearch?.query?.hybrid?.enabled).toBe(false); + expect(result.agents?.defaults?.memorySearch?.query?.hybrid?.vectorWeight).toBe(0.9); + // textWeight was not set, so it should get the default + expect(result.agents?.defaults?.memorySearch?.query?.hybrid?.textWeight).toBe(0.3); + expect(result.agents?.defaults?.memorySearch?.cache?.enabled).toBe(false); + expect(result.agents?.defaults?.compaction?.mode).toBe("default"); + expect(result.agents?.defaults?.compaction?.memoryFlush?.enabled).toBe(false); + }); + + it("should not mutate the original config", () => { + const cfg: OpenClawConfig = { + agents: { defaults: { workspace: "/ws" } }, + }; + const original = JSON.stringify(cfg); + + applyNonInteractiveMemoryDefaults(cfg); + + expect(JSON.stringify(cfg)).toBe(original); + }); + + it("should preserve unrelated config", () => { + const cfg: OpenClawConfig = { + agents: { + defaults: { + workspace: "/existing", + model: { primary: "openai/gpt-4o" }, + }, + }, + gateway: { port: 4000 }, + }; + + const result = applyNonInteractiveMemoryDefaults(cfg); + + expect(result.agents?.defaults?.workspace).toBe("/existing"); + expect((result.agents?.defaults?.model as { primary?: string })?.primary).toBe( + "openai/gpt-4o", + ); + expect(result.gateway?.port).toBe(4000); + }); + }); +}); diff --git a/src/commands/onboard-memory.ts b/src/commands/onboard-memory.ts new file mode 100644 index 00000000000..59cf1f58889 --- /dev/null +++ b/src/commands/onboard-memory.ts @@ -0,0 +1,176 @@ +import type { OpenClawConfig } from "../config/config.js"; +import type { RuntimeEnv } from "../runtime.js"; +import type { WizardPrompter } from "../wizard/prompts.js"; + +const MEMORY_OPTIONS = { + hybridSearch: "hybrid-search", + embeddingCache: "embedding-cache", + memoryFlush: "memory-flush", + sessionTranscripts: "session-transcripts", +} as const; + +export async function setupMemoryOptimization( + cfg: OpenClawConfig, + _runtime: RuntimeEnv, + prompter: WizardPrompter, +): Promise { + await prompter.note( + [ + "Memory optimization surfaces powerful but non-obvious features", + "that dramatically improve recall, caching, and context persistence.", + "", + "All options use safe defaults and never overwrite your existing config.", + ].join("\n"), + "Memory Optimization", + ); + + const selected = await prompter.multiselect({ + message: "Enable memory optimizations?", + options: [ + { value: "__skip__", label: "Skip for now" }, + { + value: MEMORY_OPTIONS.hybridSearch, + label: "🔍 Hybrid search (BM25 + vector)", + hint: "70/30 vector/text blend with 4x candidate pool — improves recall for exact terms", + }, + { + value: MEMORY_OPTIONS.embeddingCache, + label: "💾 Embedding cache", + hint: "Caches embeddings in SQLite — saves API calls on reindex", + }, + { + value: MEMORY_OPTIONS.memoryFlush, + label: "🧠 Pre-compaction memory flush", + hint: "Auto-saves notes before context compaction — prevents amnesia", + }, + { + value: MEMORY_OPTIONS.sessionTranscripts, + label: "📜 Session transcript search", + hint: "Indexes past transcripts via memory_search (experimental)", + }, + ], + }); + + const choices = new Set((selected ?? []).filter((v) => v !== "__skip__")); + if (choices.size === 0) { + return cfg; + } + + let next = structuredClone(cfg); + + if (choices.has(MEMORY_OPTIONS.hybridSearch)) { + next = applyHybridSearch(next); + } + + if (choices.has(MEMORY_OPTIONS.embeddingCache)) { + next = applyEmbeddingCache(next); + } + + if (choices.has(MEMORY_OPTIONS.memoryFlush)) { + next = applyMemoryFlush(next); + } + + if (choices.has(MEMORY_OPTIONS.sessionTranscripts)) { + next = applySessionTranscripts(next); + } + + const labels: string[] = []; + if (choices.has(MEMORY_OPTIONS.hybridSearch)) { + labels.push("hybrid search"); + } + if (choices.has(MEMORY_OPTIONS.embeddingCache)) { + labels.push("embedding cache"); + } + if (choices.has(MEMORY_OPTIONS.memoryFlush)) { + labels.push("memory flush"); + } + if (choices.has(MEMORY_OPTIONS.sessionTranscripts)) { + labels.push("session transcripts"); + } + + await prompter.note( + [ + `Enabled ${labels.length} optimization${labels.length > 1 ? "s" : ""}: ${labels.join(", ")}`, + "", + "You can tune these later in your config under:", + " agents.defaults.memorySearch", + " agents.defaults.compaction", + ].join("\n"), + "Memory Configured", + ); + + return next; +} + +// ── Helpers (safe deep-set with nullish coalescing) ────────────────── + +function ensureAgentsDefaults(cfg: OpenClawConfig): OpenClawConfig { + cfg.agents ??= {}; + cfg.agents.defaults ??= {}; + return cfg; +} + +function ensureMemorySearch(cfg: OpenClawConfig): OpenClawConfig { + cfg = ensureAgentsDefaults(cfg); + cfg.agents!.defaults!.memorySearch ??= {}; + return cfg; +} + +function applyHybridSearch(cfg: OpenClawConfig): OpenClawConfig { + cfg = ensureMemorySearch(cfg); + const ms = cfg.agents!.defaults!.memorySearch!; + ms.query ??= {}; + ms.query.hybrid ??= {}; + ms.query.hybrid.enabled ??= true; + ms.query.hybrid.vectorWeight ??= 0.7; + ms.query.hybrid.textWeight ??= 0.3; + ms.query.hybrid.candidateMultiplier ??= 4; + return cfg; +} + +function applyEmbeddingCache(cfg: OpenClawConfig): OpenClawConfig { + cfg = ensureMemorySearch(cfg); + const ms = cfg.agents!.defaults!.memorySearch!; + ms.cache ??= {}; + ms.cache.enabled ??= true; + ms.cache.maxEntries ??= 50_000; + return cfg; +} + +function applyMemoryFlush(cfg: OpenClawConfig): OpenClawConfig { + cfg = ensureAgentsDefaults(cfg); + const d = cfg.agents!.defaults!; + d.compaction ??= {}; + d.compaction.mode ??= "safeguard"; + d.compaction.memoryFlush ??= {}; + d.compaction.memoryFlush.enabled ??= true; + return cfg; +} + +function applySessionTranscripts(cfg: OpenClawConfig): OpenClawConfig { + cfg = ensureMemorySearch(cfg); + const ms = cfg.agents!.defaults!.memorySearch!; + ms.enabled ??= true; + ms.experimental ??= {}; + ms.experimental.sessionMemory ??= true; + ms.sync ??= {}; + ms.sync.sessions ??= {}; + ms.sync.sessions.deltaBytes ??= 50_000; + ms.sync.sessions.deltaMessages ??= 25; + return cfg; +} + +// ── Non-interactive defaults ───────────────────────────────────────── + +/** + * Apply sensible memory optimization defaults for non-interactive onboarding. + * Enables hybrid search, embedding cache, and pre-compaction memory flush. + * Session transcript search is experimental and opt-in only. + */ +export function applyNonInteractiveMemoryDefaults(cfg: OpenClawConfig): OpenClawConfig { + let next = structuredClone(cfg); + next = applyHybridSearch(next); + next = applyEmbeddingCache(next); + next = applyMemoryFlush(next); + return next; +} diff --git a/src/commands/onboard-non-interactive/local.ts b/src/commands/onboard-non-interactive/local.ts index 5e26bf50d24..40e660570ff 100644 --- a/src/commands/onboard-non-interactive/local.ts +++ b/src/commands/onboard-non-interactive/local.ts @@ -12,6 +12,7 @@ import { resolveControlUiLinks, waitForGatewayReachable, } from "../onboard-helpers.js"; +import { applyNonInteractiveMemoryDefaults } from "../onboard-memory.js"; import type { OnboardOptions } from "../onboard-types.js"; import { inferAuthChoiceFromFlags } from "./local/auth-choice-inference.js"; import { applyNonInteractiveGatewayConfig } from "./local/gateway-config.js"; @@ -125,6 +126,8 @@ export async function runNonInteractiveOnboardingLocal(params: { nextConfig = applyNonInteractiveSkillsConfig({ nextConfig, opts, runtime }); + nextConfig = applyNonInteractiveMemoryDefaults(nextConfig); + nextConfig = applyWizardMetadata(nextConfig, { command: "onboard", mode }); await writeConfigFile(nextConfig); logConfigUpdated(runtime); diff --git a/src/wizard/onboarding.ts b/src/wizard/onboarding.ts index e8265efd49e..2664dcaef43 100644 --- a/src/wizard/onboarding.ts +++ b/src/wizard/onboarding.ts @@ -545,6 +545,10 @@ export async function runOnboardingWizard( const { setupInternalHooks } = await import("../commands/onboard-hooks.js"); nextConfig = await setupInternalHooks(nextConfig, runtime, prompter); + // Memory optimization (hybrid search, caching, compaction flush) + const { setupMemoryOptimization } = await import("../commands/onboard-memory.js"); + nextConfig = await setupMemoryOptimization(nextConfig, runtime, prompter); + nextConfig = onboardHelpers.applyWizardMetadata(nextConfig, { command: "onboard", mode }); await writeConfigFile(nextConfig);