From 79348f73c8b6db45f66284cf85654e15601c8a16 Mon Sep 17 00:00:00 2001 From: Mariano <132747814+mbelinky@users.noreply.github.com> Date: Mon, 6 Apr 2026 00:32:38 +0200 Subject: [PATCH] feat(memory-core): add REM preview and safe promotion replay (#61540) * memory: add REM preview and safe promotion replay thanks @mbelinky * changelog: note REM preview and promotion replay --------- Co-authored-by: Vignesh --- CHANGELOG.md | 1 + extensions/memory-core/src/cli.runtime.ts | 239 +++++++++++++++++- extensions/memory-core/src/cli.test.ts | 79 +++++- extensions/memory-core/src/cli.ts | 44 ++++ extensions/memory-core/src/cli.types.ts | 8 + extensions/memory-core/src/dreaming-phases.ts | 90 ++++++- .../src/short-term-promotion.test.ts | 67 +++++ .../memory-core/src/short-term-promotion.ts | 41 ++- 8 files changed, 554 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1532f442531..9edab45f6a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Docs: https://docs.openclaw.ai - Memory/dreaming (experimental): add weighted short-term recall promotion, a `/dreaming` command, Dreams UI, multilingual conceptual tagging, and doctor/status repair support, while refactoring dreaming from competing modes into three cooperative phases (light, deep, REM) with independent schedules and recovery behavior so durable memory promotion can run in the background with less manual setup. (#60569, #60697) Thanks @vignesh07. - Memory/dreaming: add configurable aging controls (`recencyHalfLifeDays`, `maxAgeDays`) plus optional verbose logging so operators can tune recall decay and inspect promotion decisions more easily. +- Memory/dreaming: add REM preview tooling (`openclaw memory rem-harness`, `promote-explain`), surface possible lasting truths during REM staging, and make deep promotion replay-safe so reruns reconcile instead of duplicating `MEMORY.md` entries. - Agents/video generation: add the built-in `video_generate` tool so agents can create videos through configured providers and return the generated media directly in the reply. - Control UI/multilingual: add localized control UI support for Simplified Chinese, Traditional Chinese, Brazilian Portuguese, German, Spanish, Japanese, Korean, French, Turkish, Indonesian, Polish, and Ukrainian. Thanks @vincentkoc. - iOS/exec approvals: add generic APNs approval notifications that open an in-app exec approval modal, fetch command details only after authenticated operator reconnect, and clear stale notification state when the approval resolves. (#60239) Thanks @ngutman. diff --git a/extensions/memory-core/src/cli.runtime.ts b/extensions/memory-core/src/cli.runtime.ts index ec1b7f13388..a8a3ce15553 100644 --- a/extensions/memory-core/src/cli.runtime.ts +++ b/extensions/memory-core/src/cli.runtime.ts @@ -2,6 +2,7 @@ import fsSync from "node:fs"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import { resolveMemoryRemDreamingConfig } from "openclaw/plugin-sdk/memory-core-host-status"; import { buildAgentSessionKey } from "openclaw/plugin-sdk/routing"; import { colorize, @@ -28,13 +29,17 @@ import { import type { MemoryCommandOptions, MemoryPromoteCommandOptions, + MemoryPromoteExplainOptions, + MemoryRemHarnessOptions, MemorySearchCommandOptions, } from "./cli.types.js"; +import { previewRemDreaming } from "./dreaming-phases.js"; import { resolveShortTermPromotionDreamingConfig } from "./dreaming.js"; import { applyShortTermPromotions, auditShortTermPromotionArtifacts, repairShortTermPromotionArtifacts, + readShortTermRecallEntries, recordShortTermRecalls, rankShortTermPromotionCandidates, resolveShortTermRecallLockPath, @@ -208,6 +213,26 @@ function formatExtraPaths(workspaceDir: string, extraPaths: string[]): string[] return normalizeExtraMemoryPaths(workspaceDir, extraPaths).map((entry) => shortenHomePath(entry)); } +function matchesPromotionSelector( + candidate: { + key: string; + path: string; + snippet: string; + }, + selector: string, +): boolean { + const trimmed = selector.trim().toLowerCase(); + if (!trimmed) { + return false; + } + return ( + candidate.key.toLowerCase() === trimmed || + candidate.key.toLowerCase().includes(trimmed) || + candidate.path.toLowerCase().includes(trimmed) || + candidate.snippet.toLowerCase().includes(trimmed) + ); +} + async function withMemoryManagerForAgent(params: { cfg: OpenClawConfig; agentId: string; @@ -1000,6 +1025,8 @@ export async function runMemoryPromote(opts: MemoryPromoteCommandOptions) { apply: applyResult ? { applied: applyResult.applied, + appended: applyResult.appended, + reconciledExisting: applyResult.reconciledExisting, memoryPath: applyResult.memoryPath, appliedCandidates: applyResult.appliedCandidates, } @@ -1068,7 +1095,14 @@ export async function runMemoryPromote(opts: MemoryPromoteCommandOptions) { colorize( rich, theme.success, - `Promoted ${applyResult.applied} candidate(s) to ${shortenHomePath(applyResult.memoryPath)}.`, + `Processed ${applyResult.applied} candidate(s) for ${shortenHomePath(applyResult.memoryPath)}.`, + ), + ); + lines.push( + colorize( + rich, + theme.muted, + `appended=${applyResult.appended} reconciledExisting=${applyResult.reconciledExisting}`, ), ); } else { @@ -1079,3 +1113,206 @@ export async function runMemoryPromote(opts: MemoryPromoteCommandOptions) { }, }); } + +export async function runMemoryPromoteExplain( + selectorArg: string | undefined, + opts: MemoryPromoteExplainOptions, +) { + const selector = selectorArg?.trim(); + if (!selector) { + defaultRuntime.error("Memory promote-explain requires a non-empty selector."); + process.exitCode = 1; + return; + } + + const { config: cfg, diagnostics } = await loadMemoryCommandConfig("memory promote-explain"); + emitMemorySecretResolveDiagnostics(diagnostics, { json: Boolean(opts.json) }); + const agentId = resolveAgent(cfg, opts.agent); + + await withMemoryManagerForAgent({ + cfg, + agentId, + purpose: "status", + run: async (manager) => { + const status = manager.status(); + const workspaceDir = status.workspaceDir?.trim(); + const dreaming = resolveShortTermPromotionDreamingConfig({ + pluginConfig: resolveMemoryPluginConfig(cfg), + cfg, + }); + if (!workspaceDir) { + defaultRuntime.error("Memory promote-explain requires a resolvable workspace directory."); + process.exitCode = 1; + return; + } + + let candidates: Awaited>; + try { + candidates = await rankShortTermPromotionCandidates({ + workspaceDir, + minScore: 0, + minRecallCount: 0, + minUniqueQueries: 0, + includePromoted: Boolean(opts.includePromoted), + recencyHalfLifeDays: dreaming.recencyHalfLifeDays, + maxAgeDays: dreaming.maxAgeDays, + }); + } catch (err) { + defaultRuntime.error(`Memory promote-explain failed: ${formatErrorMessage(err)}`); + process.exitCode = 1; + return; + } + + const candidate = candidates.find((entry) => matchesPromotionSelector(entry, selector)); + if (!candidate) { + defaultRuntime.error(`No promotion candidate matched "${selector}".`); + process.exitCode = 1; + return; + } + + const thresholds = { + minScore: dreaming.minScore, + minRecallCount: dreaming.minRecallCount, + minUniqueQueries: dreaming.minUniqueQueries, + maxAgeDays: dreaming.maxAgeDays ?? null, + }; + + if (opts.json) { + defaultRuntime.writeJson({ + workspaceDir, + thresholds, + candidate, + passes: { + score: candidate.score >= thresholds.minScore, + recallCount: candidate.recallCount >= thresholds.minRecallCount, + uniqueQueries: candidate.uniqueQueries >= thresholds.minUniqueQueries, + maxAge: + thresholds.maxAgeDays === null ? true : candidate.ageDays <= thresholds.maxAgeDays, + }, + }); + return; + } + + const rich = isRich(); + const lines = [ + `${colorize(rich, theme.heading, "Promotion Explain")} ${colorize(rich, theme.muted, `(${agentId})`)}`, + `${colorize(rich, theme.accent, candidate.key)}`, + `${colorize(rich, theme.muted, `${shortenHomePath(candidate.path)}:${candidate.startLine}-${candidate.endLine}`)}`, + candidate.snippet, + colorize( + rich, + theme.muted, + `score=${candidate.score.toFixed(3)} recallCount=${candidate.recallCount} uniqueQueries=${candidate.uniqueQueries} ageDays=${candidate.ageDays.toFixed(1)}`, + ), + colorize( + rich, + theme.muted, + `components: frequency=${candidate.components.frequency.toFixed(2)} relevance=${candidate.components.relevance.toFixed(2)} diversity=${candidate.components.diversity.toFixed(2)} recency=${candidate.components.recency.toFixed(2)} consolidation=${candidate.components.consolidation.toFixed(2)} conceptual=${candidate.components.conceptual.toFixed(2)}`, + ), + colorize( + rich, + theme.muted, + `thresholds: minScore=${thresholds.minScore} minRecallCount=${thresholds.minRecallCount} minUniqueQueries=${thresholds.minUniqueQueries} maxAgeDays=${thresholds.maxAgeDays ?? "none"}`, + ), + ]; + if (candidate.conceptTags.length > 0) { + lines.push(colorize(rich, theme.muted, `concepts=${candidate.conceptTags.join(", ")}`)); + } + defaultRuntime.log(lines.join("\n")); + }, + }); +} + +export async function runMemoryRemHarness(opts: MemoryRemHarnessOptions) { + const { config: cfg, diagnostics } = await loadMemoryCommandConfig("memory rem-harness"); + emitMemorySecretResolveDiagnostics(diagnostics, { json: Boolean(opts.json) }); + const agentId = resolveAgent(cfg, opts.agent); + + await withMemoryManagerForAgent({ + cfg, + agentId, + purpose: "status", + run: async (manager) => { + const status = manager.status(); + const workspaceDir = status.workspaceDir?.trim(); + const pluginConfig = resolveMemoryPluginConfig(cfg); + const deep = resolveShortTermPromotionDreamingConfig({ + pluginConfig, + cfg, + }); + if (!workspaceDir) { + defaultRuntime.error("Memory rem-harness requires a resolvable workspace directory."); + process.exitCode = 1; + return; + } + const remConfig = resolveMemoryRemDreamingConfig({ + pluginConfig, + cfg, + }); + const nowMs = Date.now(); + const cutoffMs = nowMs - Math.max(0, remConfig.lookbackDays) * 24 * 60 * 60 * 1000; + const recallEntries = (await readShortTermRecallEntries({ workspaceDir, nowMs })).filter( + (entry) => Date.parse(entry.lastRecalledAt) >= cutoffMs, + ); + const remPreview = previewRemDreaming({ + entries: recallEntries, + limit: remConfig.limit, + minPatternStrength: remConfig.minPatternStrength, + }); + const deepCandidates = await rankShortTermPromotionCandidates({ + workspaceDir, + minScore: 0, + minRecallCount: 0, + minUniqueQueries: 0, + includePromoted: Boolean(opts.includePromoted), + recencyHalfLifeDays: deep.recencyHalfLifeDays, + maxAgeDays: deep.maxAgeDays, + }); + + if (opts.json) { + defaultRuntime.writeJson({ + workspaceDir, + remConfig, + deepConfig: { + minScore: deep.minScore, + minRecallCount: deep.minRecallCount, + minUniqueQueries: deep.minUniqueQueries, + recencyHalfLifeDays: deep.recencyHalfLifeDays, + maxAgeDays: deep.maxAgeDays ?? null, + }, + rem: remPreview, + deep: { + candidateCount: deepCandidates.length, + candidates: deepCandidates, + }, + }); + return; + } + + const rich = isRich(); + const lines = [ + `${colorize(rich, theme.heading, "REM Harness")} ${colorize(rich, theme.muted, `(${agentId})`)}`, + colorize(rich, theme.muted, `workspace=${shortenHomePath(workspaceDir)}`), + colorize( + rich, + theme.muted, + `recentRecallEntries=${recallEntries.length} deepCandidates=${deepCandidates.length}`, + ), + "", + colorize(rich, theme.heading, "REM Preview"), + ...remPreview.bodyLines, + "", + colorize(rich, theme.heading, "Deep Candidates"), + ...(deepCandidates.length > 0 + ? deepCandidates + .slice(0, 10) + .map( + (candidate) => + `${candidate.score.toFixed(3)} ${candidate.snippet} [${shortenHomePath(candidate.path)}:${candidate.startLine}-${candidate.endLine}]`, + ) + : ["- No deep candidates."]), + ]; + defaultRuntime.log(lines.join("\n")); + }, + }); +} diff --git a/extensions/memory-core/src/cli.test.ts b/extensions/memory-core/src/cli.test.ts index 5c427658018..275a6a62235 100644 --- a/extensions/memory-core/src/cli.test.ts +++ b/extensions/memory-core/src/cli.test.ts @@ -295,6 +295,12 @@ describe("memory cli", () => { expect(helpText).toContain("Limit results for focused troubleshooting."); expect(helpText).toContain("openclaw memory promote --apply"); expect(helpText).toContain("Append top-ranked short-term candidates into MEMORY.md."); + expect(helpText).toContain('openclaw memory promote-explain "router vlan"'); + expect(helpText).toContain("Explain why a specific candidate would or would not promote."); + expect(helpText).toContain("openclaw memory rem-harness --json"); + expect(helpText).toContain( + "Preview REM reflections, candidate truths, and deep promotion output.", + ); }); it("prints vector error when unavailable", async () => { @@ -881,6 +887,75 @@ describe("memory cli", () => { }); }); + it("explains a specific promote candidate as json", async () => { + await withTempWorkspace(async (workspaceDir) => { + await recordShortTermRecalls({ + workspaceDir, + query: "router notes", + results: [ + { + path: "memory/2026-04-03.md", + startLine: 4, + endLine: 8, + score: 0.86, + snippet: "Configured VLAN 10 for IoT on router", + source: "memory", + }, + ], + }); + + const close = vi.fn(async () => {}); + mockManager({ + status: () => makeMemoryStatus({ workspaceDir }), + close, + }); + + const writeJson = spyRuntimeJson(defaultRuntime); + await runMemoryCli(["promote-explain", "router", "--json", "--include-promoted"]); + + const payload = firstWrittenJsonArg<{ candidate?: { snippet?: string } }>(writeJson); + expect(payload?.candidate?.snippet).toContain("Configured VLAN 10"); + expect(close).toHaveBeenCalled(); + }); + }); + + it("previews rem harness output as json", async () => { + await withTempWorkspace(async (workspaceDir) => { + await recordShortTermRecalls({ + workspaceDir, + query: "weather plans", + nowMs: Date.parse("2026-04-03T10:00:00.000Z"), + results: [ + { + path: "memory/2026-04-03.md", + startLine: 2, + endLine: 3, + score: 0.92, + snippet: "Always check weather before suggesting outdoor plans.", + source: "memory", + }, + ], + }); + + const close = vi.fn(async () => {}); + mockManager({ + status: () => makeMemoryStatus({ workspaceDir }), + close, + }); + + const writeJson = spyRuntimeJson(defaultRuntime); + await runMemoryCli(["rem-harness", "--json"]); + + const payload = firstWrittenJsonArg<{ + rem?: { candidateTruths?: Array<{ snippet?: string }> }; + deep?: { candidates?: Array<{ snippet?: string }> }; + }>(writeJson); + expect(payload?.rem?.candidateTruths?.[0]?.snippet).toContain("Always check weather"); + expect(payload?.deep?.candidates?.[0]?.snippet).toContain("Always check weather"); + expect(close).toHaveBeenCalled(); + }); + }); + it("applies top promote candidates into MEMORY.md", async () => { await withTempWorkspace(async (workspaceDir) => { await writeDailyMemoryNote(workspaceDir, "2026-04-01", [ @@ -935,8 +1010,10 @@ describe("memory cli", () => { const memoryPath = path.join(workspaceDir, "MEMORY.md"); const memoryText = await fs.readFile(memoryPath, "utf-8"); expect(memoryText).toContain("Promoted From Short-Term Memory"); + expect(memoryText).toContain("openclaw-memory-promotion:"); expect(memoryText).toContain("memory/2026-04-01.md:10-10"); - expect(log).toHaveBeenCalledWith(expect.stringContaining("Promoted 1 candidate(s) to")); + expect(log).toHaveBeenCalledWith(expect.stringContaining("Processed 1 candidate(s) for")); + expect(log).toHaveBeenCalledWith(expect.stringContaining("appended=1 reconciledExisting=0")); expect(close).toHaveBeenCalled(); }); }); diff --git a/extensions/memory-core/src/cli.ts b/extensions/memory-core/src/cli.ts index 07c5fb47a89..f0d5f8ac5d0 100644 --- a/extensions/memory-core/src/cli.ts +++ b/extensions/memory-core/src/cli.ts @@ -7,6 +7,8 @@ import { import type { MemoryCommandOptions, MemoryPromoteCommandOptions, + MemoryPromoteExplainOptions, + MemoryRemHarnessOptions, MemorySearchCommandOptions, } from "./cli.types.js"; import { @@ -44,6 +46,19 @@ async function runMemoryPromote(opts: MemoryPromoteCommandOptions) { await runtime.runMemoryPromote(opts); } +async function runMemoryPromoteExplain( + selectorArg: string | undefined, + opts: MemoryPromoteExplainOptions, +) { + const runtime = await loadMemoryCliRuntime(); + await runtime.runMemoryPromoteExplain(selectorArg, opts); +} + +async function runMemoryRemHarness(opts: MemoryRemHarnessOptions) { + const runtime = await loadMemoryCliRuntime(); + await runtime.runMemoryRemHarness(opts); +} + export function registerMemoryCli(program: Command) { const memory = program .command("memory") @@ -72,6 +87,14 @@ export function registerMemoryCli(program: Command) { "openclaw memory promote --apply", "Append top-ranked short-term candidates into MEMORY.md.", ], + [ + 'openclaw memory promote-explain "router vlan"', + "Explain why a specific candidate would or would not promote.", + ], + [ + "openclaw memory rem-harness --json", + "Preview REM reflections, candidate truths, and deep promotion output.", + ], ["openclaw memory status --json", "Output machine-readable JSON (good for scripts)."], ])}\n\n${theme.muted("Docs:")} ${formatDocsLink("/cli/memory", "docs.openclaw.ai/cli/memory")}\n`, ); @@ -138,4 +161,25 @@ export function registerMemoryCli(program: Command) { .action(async (opts: MemoryPromoteCommandOptions) => { await runMemoryPromote(opts); }); + + memory + .command("promote-explain") + .description("Explain a specific promotion candidate and its score breakdown") + .argument("", "Candidate key, path fragment, or snippet fragment") + .option("--agent ", "Agent id (default: default agent)") + .option("--include-promoted", "Include already promoted candidates", false) + .option("--json", "Print JSON") + .action(async (selectorArg: string | undefined, opts: MemoryPromoteExplainOptions) => { + await runMemoryPromoteExplain(selectorArg, opts); + }); + + memory + .command("rem-harness") + .description("Preview REM reflections, candidate truths, and deep promotions without writing") + .option("--agent ", "Agent id (default: default agent)") + .option("--include-promoted", "Include already promoted deep candidates", false) + .option("--json", "Print JSON") + .action(async (opts: MemoryRemHarnessOptions) => { + await runMemoryRemHarness(opts); + }); } diff --git a/extensions/memory-core/src/cli.types.ts b/extensions/memory-core/src/cli.types.ts index 6bb52ee2b8b..5e5f8262367 100644 --- a/extensions/memory-core/src/cli.types.ts +++ b/extensions/memory-core/src/cli.types.ts @@ -22,3 +22,11 @@ export type MemoryPromoteCommandOptions = MemoryCommandOptions & { apply?: boolean; includePromoted?: boolean; }; + +export type MemoryPromoteExplainOptions = MemoryCommandOptions & { + includePromoted?: boolean; +}; + +export type MemoryRemHarnessOptions = MemoryCommandOptions & { + includePromoted?: boolean; +}; diff --git a/extensions/memory-core/src/dreaming-phases.ts b/extensions/memory-core/src/dreaming-phases.ts index a012b3d6235..9ce6d4068f8 100644 --- a/extensions/memory-core/src/dreaming-phases.ts +++ b/extensions/memory-core/src/dreaming-phases.ts @@ -380,7 +380,55 @@ function buildLightDreamingBody(entries: ShortTermRecallEntry[]): string[] { return lines; } -function buildRemDreamingBody( +type RemTruthCandidate = { + snippet: string; + confidence: number; + evidence: string; +}; + +export type RemDreamingPreview = { + sourceEntryCount: number; + reflections: string[]; + candidateTruths: RemTruthCandidate[]; + bodyLines: string[]; +}; + +function calculateCandidateTruthConfidence(entry: ShortTermRecallEntry): number { + const recallStrength = Math.min(1, Math.log1p(entry.recallCount) / Math.log1p(6)); + const averageScore = entryAverageScore(entry); + const consolidation = Math.min(1, (entry.recallDays?.length ?? 0) / 3); + const conceptual = Math.min(1, (entry.conceptTags?.length ?? 0) / 6); + return Math.max( + 0, + Math.min( + 1, + averageScore * 0.45 + recallStrength * 0.25 + consolidation * 0.2 + conceptual * 0.1, + ), + ); +} + +function selectRemCandidateTruths( + entries: ShortTermRecallEntry[], + limit: number, +): RemTruthCandidate[] { + if (limit <= 0) { + return []; + } + return dedupeEntries( + entries.filter((entry) => !entry.promotedAt), + 0.88, + ) + .map((entry) => ({ + snippet: entry.snippet || "(no snippet captured)", + confidence: calculateCandidateTruthConfidence(entry), + evidence: `${entry.path}:${entry.startLine}-${entry.endLine}`, + })) + .filter((entry) => entry.confidence >= 0.45) + .toSorted((a, b) => b.confidence - a.confidence || a.snippet.localeCompare(b.snippet)) + .slice(0, limit); +} + +function buildRemReflections( entries: ShortTermRecallEntry[], limit: number, minPatternStrength: number, @@ -424,6 +472,36 @@ function buildRemDreamingBody( return lines; } +export function previewRemDreaming(params: { + entries: ShortTermRecallEntry[]; + limit: number; + minPatternStrength: number; +}): RemDreamingPreview { + const reflections = buildRemReflections(params.entries, params.limit, params.minPatternStrength); + const candidateTruths = selectRemCandidateTruths( + params.entries, + Math.max(1, Math.min(3, params.limit)), + ); + const bodyLines = [ + "### Reflections", + ...reflections, + "", + "### Possible Lasting Truths", + ...(candidateTruths.length > 0 + ? candidateTruths.map( + (entry) => + `- ${entry.snippet} [confidence=${entry.confidence.toFixed(2)} evidence=${entry.evidence}]`, + ) + : ["- No strong candidate truths surfaced."]), + ]; + return { + sourceEntryCount: params.entries.length, + reflections, + candidateTruths, + bodyLines, + }; +} + async function runLightDreaming(params: { workspaceDir: string; config: MemoryLightDreamingConfig & { @@ -478,15 +556,15 @@ async function runRemDreaming(params: { const entries = ( await readShortTermRecallEntries({ workspaceDir: params.workspaceDir, nowMs }) ).filter((entry) => Date.parse(entry.lastRecalledAt) >= cutoffMs); - const bodyLines = buildRemDreamingBody( + const preview = previewRemDreaming({ entries, - params.config.limit, - params.config.minPatternStrength, - ); + limit: params.config.limit, + minPatternStrength: params.config.minPatternStrength, + }); await writeDailyDreamingPhaseBlock({ workspaceDir: params.workspaceDir, phase: "rem", - bodyLines, + bodyLines: preview.bodyLines, nowMs, timezone: params.config.timezone, storage: params.config.storage, diff --git a/extensions/memory-core/src/short-term-promotion.test.ts b/extensions/memory-core/src/short-term-promotion.test.ts index c166f480936..aaa006e5d87 100644 --- a/extensions/memory-core/src/short-term-promotion.test.ts +++ b/extensions/memory-core/src/short-term-promotion.test.ts @@ -249,6 +249,73 @@ describe("short-term promotion", () => { }); }); + it("reconciles existing promotion markers instead of appending duplicates", async () => { + await withTempWorkspace(async (workspaceDir) => { + await writeDailyMemoryNote(workspaceDir, "2026-04-01", [ + "line 1", + "line 2", + "The gateway should stay loopback-only on port 18789.", + ]); + await recordShortTermRecalls({ + workspaceDir, + query: "gateway loopback", + results: [ + { + path: "memory/2026-04-01.md", + startLine: 3, + endLine: 3, + score: 0.95, + snippet: "The gateway should stay loopback-only on port 18789.", + source: "memory", + }, + ], + }); + + const ranked = await rankShortTermPromotionCandidates({ + workspaceDir, + minScore: 0, + minRecallCount: 0, + minUniqueQueries: 0, + }); + const firstApply = await applyShortTermPromotions({ + workspaceDir, + candidates: ranked, + minScore: 0, + minRecallCount: 0, + minUniqueQueries: 0, + }); + expect(firstApply.applied).toBe(1); + expect(firstApply.appended).toBe(1); + expect(firstApply.reconciledExisting).toBe(0); + + const storePath = resolveShortTermRecallStorePath(workspaceDir); + const rawStore = JSON.parse(await fs.readFile(storePath, "utf-8")) as { + entries: Record; + }; + for (const entry of Object.values(rawStore.entries)) { + delete entry.promotedAt; + } + await fs.writeFile(storePath, `${JSON.stringify(rawStore, null, 2)}\n`, "utf-8"); + + const secondApply = await applyShortTermPromotions({ + workspaceDir, + candidates: ranked, + minScore: 0, + minRecallCount: 0, + minUniqueQueries: 0, + }); + expect(secondApply.applied).toBe(1); + expect(secondApply.appended).toBe(0); + expect(secondApply.reconciledExisting).toBe(1); + + const memoryText = await fs.readFile(path.join(workspaceDir, "MEMORY.md"), "utf-8"); + expect(memoryText.match(/openclaw-memory-promotion:/g)?.length).toBe(1); + expect( + memoryText.match(/The gateway should stay loopback-only on port 18789\./g)?.length, + ).toBe(1); + }); + }); + it("filters out candidates older than maxAgeDays during ranking", async () => { await withTempWorkspace(async (workspaceDir) => { await recordShortTermRecalls({ diff --git a/extensions/memory-core/src/short-term-promotion.ts b/extensions/memory-core/src/short-term-promotion.ts index 27a09ffbeab..f920626fc10 100644 --- a/extensions/memory-core/src/short-term-promotion.ts +++ b/extensions/memory-core/src/short-term-promotion.ts @@ -17,6 +17,7 @@ const DEFAULT_RECENCY_HALF_LIFE_DAYS = 14; export const DEFAULT_PROMOTION_MIN_SCORE = 0.75; export const DEFAULT_PROMOTION_MIN_RECALL_COUNT = 3; export const DEFAULT_PROMOTION_MIN_UNIQUE_QUERIES = 2; +const PROMOTION_MARKER_PREFIX = "openclaw-memory-promotion:"; const MAX_QUERY_HASHES = 32; const MAX_RECALL_DAYS = 16; const SHORT_TERM_STORE_RELATIVE_PATH = path.join("memory", ".dreams", "short-term-recall.json"); @@ -168,6 +169,8 @@ export type ApplyShortTermPromotionsOptions = { export type ApplyShortTermPromotionsResult = { memoryPath: string; applied: number; + appended: number; + reconciledExisting: number; appliedCandidates: PromotionCandidate[]; }; @@ -935,6 +938,7 @@ function buildPromotionSection( for (const candidate of candidates) { const source = `${candidate.path}:${candidate.startLine}-${candidate.endLine}`; const snippet = candidate.snippet || "(no snippet captured)"; + lines.push(``); lines.push( `- ${snippet} [score=${candidate.score.toFixed(3)} recalls=${candidate.recallCount} avg=${candidate.avgScore.toFixed(3)} source=${source}]`, ); @@ -951,6 +955,18 @@ function withTrailingNewline(content: string): string { return content.endsWith("\n") ? content : `${content}\n`; } +function extractPromotionMarkers(memoryText: string): Set { + const markers = new Set(); + const matches = memoryText.matchAll(//gi); + for (const match of matches) { + const key = match[1]?.trim(); + if (key) { + markers.add(key); + } + } + return markers; +} + export async function applyShortTermPromotions( options: ApplyShortTermPromotionsOptions, ): Promise { @@ -1011,6 +1027,8 @@ export async function applyShortTermPromotions( return { memoryPath, applied: 0, + appended: 0, + reconciledExisting: 0, appliedCandidates: [], }; } @@ -1021,14 +1039,21 @@ export async function applyShortTermPromotions( } throw err; }); - - const header = existingMemory.trim().length > 0 ? "" : "# Long-Term Memory\n\n"; - const section = buildPromotionSection(rehydratedSelected, nowMs, options.timezone); - await fs.writeFile( - memoryPath, - `${header}${withTrailingNewline(existingMemory)}${section}`, - "utf-8", + const existingMarkers = extractPromotionMarkers(existingMemory); + const alreadyWritten = rehydratedSelected.filter((candidate) => + existingMarkers.has(candidate.key), ); + const toAppend = rehydratedSelected.filter((candidate) => !existingMarkers.has(candidate.key)); + + if (toAppend.length > 0) { + const header = existingMemory.trim().length > 0 ? "" : "# Long-Term Memory\n\n"; + const section = buildPromotionSection(toAppend, nowMs, options.timezone); + await fs.writeFile( + memoryPath, + `${header}${withTrailingNewline(existingMemory)}${section}`, + "utf-8", + ); + } for (const candidate of rehydratedSelected) { const entry = store.entries[candidate.key]; @@ -1046,6 +1071,8 @@ export async function applyShortTermPromotions( return { memoryPath, applied: rehydratedSelected.length, + appended: toAppend.length, + reconciledExisting: alreadyWritten.length, appliedCandidates: rehydratedSelected, }; });