From 4e74e7e26cfdff5474533da5bba73252754ac7c2 Mon Sep 17 00:00:00 2001 From: Vignesh Natarajan Date: Sat, 28 Mar 2026 16:24:08 -0700 Subject: [PATCH] fix(memory): resolve slugified qmd search paths (#50313) --- CHANGELOG.md | 4 + .../qmd-manager.slugified-paths.test.ts | 365 ++++++++++++++++++ .../memory-core/src/memory/qmd-manager.ts | 102 ++++- 3 files changed, 468 insertions(+), 3 deletions(-) create mode 100644 extensions/memory-core/src/memory/qmd-manager.slugified-paths.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 690c2b8b30f..f67167911c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ Docs: https://docs.openclaw.ai ## Unreleased +### Fixes + +- Memory/QMD: resolve slugified `memory_search` file hints back to the indexed filesystem path before returning search hits, so `memory_get` works again for mixed-case and spaced paths. (#50313) Thanks @erra9x. + ## 2026.3.28-beta.1 ### Breaking diff --git a/extensions/memory-core/src/memory/qmd-manager.slugified-paths.test.ts b/extensions/memory-core/src/memory/qmd-manager.slugified-paths.test.ts new file mode 100644 index 00000000000..3c69cdcd262 --- /dev/null +++ b/extensions/memory-core/src/memory/qmd-manager.slugified-paths.test.ts @@ -0,0 +1,365 @@ +import { EventEmitter } from "node:events"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import type { Mock } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const { logWarnMock, logDebugMock, logInfoMock } = vi.hoisted(() => ({ + logWarnMock: vi.fn(), + logDebugMock: vi.fn(), + logInfoMock: vi.fn(), +})); + +type MockChild = EventEmitter & { + stdout: EventEmitter; + stderr: EventEmitter; + kill: (signal?: NodeJS.Signals) => void; + closeWith: (code?: number | null) => void; +}; + +function createMockChild(params?: { autoClose?: boolean }): MockChild { + const stdout = new EventEmitter(); + const stderr = new EventEmitter(); + const child = new EventEmitter() as MockChild; + child.stdout = stdout; + child.stderr = stderr; + child.closeWith = (code = 0) => { + child.emit("close", code); + }; + child.kill = () => {}; + if (params?.autoClose !== false) { + queueMicrotask(() => { + child.emit("close", 0); + }); + } + return child; +} + +function emitAndClose( + child: MockChild, + stream: "stdout" | "stderr", + data: string, + code: number = 0, +) { + queueMicrotask(() => { + child[stream].emit("data", data); + child.closeWith(code); + }); +} + +vi.mock("openclaw/plugin-sdk/memory-core-host-engine-foundation", async () => { + const actual = await vi.importActual< + typeof import("openclaw/plugin-sdk/memory-core-host-engine-foundation") + >("openclaw/plugin-sdk/memory-core-host-engine-foundation"); + return { + ...actual, + createSubsystemLogger: () => { + const logger = { + warn: logWarnMock, + debug: logDebugMock, + info: logInfoMock, + child: () => logger, + }; + return logger; + }, + }; +}); + +vi.mock("node:child_process", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + spawn: vi.fn(), + }; +}); + +import { spawn as mockedSpawn } from "node:child_process"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/memory-core-host-engine-foundation"; +import { resolveMemoryBackendConfig } from "openclaw/plugin-sdk/memory-core-host-engine-storage"; +import { QmdMemoryManager } from "./qmd-manager.js"; + +const spawnMock = mockedSpawn as unknown as Mock; + +describe("QmdMemoryManager slugified path resolution", () => { + let tmpRoot: string; + let workspaceDir: string; + let stateDir: string; + let cfg: OpenClawConfig; + const agentId = "main"; + const openManagers = new Set(); + + function trackManager(manager: T): T { + if (manager) { + openManagers.add(manager); + } + return manager; + } + + async function createManager(params?: { cfg?: OpenClawConfig }) { + const cfgToUse = params?.cfg ?? cfg; + const resolved = resolveMemoryBackendConfig({ cfg: cfgToUse, agentId }); + const manager = trackManager( + await QmdMemoryManager.create({ + cfg: cfgToUse, + agentId, + resolved, + mode: "status", + }), + ); + if (!manager) { + throw new Error("manager missing"); + } + return { manager }; + } + + function installIndexedPathStub(params: { + manager: QmdMemoryManager; + collection: string; + normalizedPath: string; + actualPath?: string; + exactPaths?: string[]; + allPaths?: string[]; + }) { + const inner = params.manager as unknown as { + db: { + prepare: (query: string) => { all: (...args: unknown[]) => unknown }; + close: () => void; + }; + }; + inner.db = { + prepare: (query: string) => ({ + all: (...args: unknown[]) => { + if (query.includes("collection = ? AND path = ?")) { + expect(args).toEqual([params.collection, params.normalizedPath]); + return (params.exactPaths ?? []).map((pathValue) => ({ path: pathValue })); + } + if (query.includes("collection = ? AND active = 1")) { + expect(args).toEqual([params.collection]); + return (params.allPaths ?? [params.actualPath]).map((pathValue) => ({ + path: pathValue, + })); + } + throw new Error(`unexpected sqlite query: ${query}`); + }, + }), + close: () => {}, + }; + } + + beforeEach(async () => { + spawnMock.mockReset(); + spawnMock.mockImplementation(() => createMockChild()); + logWarnMock.mockClear(); + logDebugMock.mockClear(); + logInfoMock.mockClear(); + + tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-qmd-slugified-")); + workspaceDir = path.join(tmpRoot, "workspace"); + stateDir = path.join(tmpRoot, "state"); + await fs.mkdir(workspaceDir, { recursive: true }); + process.env.OPENCLAW_STATE_DIR = stateDir; + + cfg = { + agents: { + list: [{ id: agentId, default: true, workspace: workspaceDir }], + }, + memory: { + backend: "qmd", + qmd: { + includeDefaultMemory: false, + update: { interval: "0s", debounceMs: 60_000, onBoot: false }, + paths: [{ path: workspaceDir, pattern: "**/*.md", name: "workspace" }], + }, + }, + } as OpenClawConfig; + }); + + afterEach(async () => { + await Promise.all( + Array.from(openManagers, async (manager) => { + await manager.close(); + }), + ); + openManagers.clear(); + await fs.rm(tmpRoot, { recursive: true, force: true }); + delete process.env.OPENCLAW_STATE_DIR; + }); + + it("maps slugified workspace qmd URIs back to the indexed filesystem path", async () => { + const actualRelative = "extra-docs/Category/Sub Category/Topic Name/Topic Name.md"; + const actualFile = path.join(workspaceDir, actualRelative); + await fs.mkdir(path.dirname(actualFile), { recursive: true }); + await fs.writeFile(actualFile, "line-1\nline-2\nline-3", "utf-8"); + + spawnMock.mockImplementation((_cmd: string, args: string[]) => { + if (args[0] === "search") { + const child = createMockChild({ autoClose: false }); + emitAndClose( + child, + "stdout", + JSON.stringify([ + { + file: "qmd://workspace-main/extra-docs/category/sub-category/topic-name/topic-name.md", + score: 0.73, + snippet: "@@ -2,1\nline-2", + }, + ]), + ); + return child; + } + return createMockChild(); + }); + + const { manager } = await createManager(); + installIndexedPathStub({ + manager, + collection: "workspace-main", + normalizedPath: "extra-docs/category/sub-category/topic-name/topic-name.md", + actualPath: actualRelative, + }); + + const results = await manager.search("line-2", { + sessionKey: "agent:main:slack:dm:u123", + }); + expect(results).toEqual([ + { + path: actualRelative, + startLine: 2, + endLine: 2, + score: 0.73, + snippet: "@@ -2,1\nline-2", + source: "memory", + }, + ]); + + await expect(manager.readFile({ relPath: results[0]!.path })).resolves.toEqual({ + path: actualRelative, + text: "line-1\nline-2\nline-3", + }); + }); + + it("maps slugified extra collection qmd URIs back to qmd// paths", async () => { + const extraRoot = path.join(tmpRoot, "vault"); + await fs.mkdir(extraRoot, { recursive: true }); + cfg = { + ...cfg, + memory: { + backend: "qmd", + qmd: { + includeDefaultMemory: false, + update: { interval: "0s", debounceMs: 60_000, onBoot: false }, + paths: [{ path: extraRoot, pattern: "**/*.md", name: "vault" }], + }, + }, + } as OpenClawConfig; + + const actualRelative = "Topics/Sub Category/Topic Name.md"; + const actualFile = path.join(extraRoot, actualRelative); + await fs.mkdir(path.dirname(actualFile), { recursive: true }); + await fs.writeFile(actualFile, "vault memory", "utf-8"); + + spawnMock.mockImplementation((_cmd: string, args: string[]) => { + if (args[0] === "search") { + const child = createMockChild({ autoClose: false }); + emitAndClose( + child, + "stdout", + JSON.stringify([ + { + file: "qmd://vault-main/topics/sub-category/topic-name.md", + score: 0.81, + snippet: "@@ -1,1\nvault memory", + }, + ]), + ); + return child; + } + return createMockChild(); + }); + + const { manager } = await createManager({ cfg }); + installIndexedPathStub({ + manager, + collection: "vault-main", + normalizedPath: "topics/sub-category/topic-name.md", + actualPath: actualRelative, + }); + + const results = await manager.search("vault memory", { + sessionKey: "agent:main:slack:dm:u123", + }); + expect(results).toEqual([ + { + path: `qmd/vault-main/${actualRelative}`, + startLine: 1, + endLine: 1, + score: 0.81, + snippet: "@@ -1,1\nvault memory", + source: "memory", + }, + ]); + + await expect(manager.readFile({ relPath: results[0]!.path })).resolves.toEqual({ + path: `qmd/vault-main/${actualRelative}`, + text: "vault memory", + }); + }); + + it("prefers an exact indexed path over normalized slug recovery", async () => { + const exactRelative = "notes/topic-name.md"; + const slugCollisionRelative = "notes/Topic Name.md"; + const exactFile = path.join(workspaceDir, exactRelative); + const collisionFile = path.join(workspaceDir, slugCollisionRelative); + await fs.mkdir(path.dirname(exactFile), { recursive: true }); + await fs.writeFile(exactFile, "exact slugified path", "utf-8"); + await fs.writeFile(collisionFile, "mixed case path", "utf-8"); + + spawnMock.mockImplementation((_cmd: string, args: string[]) => { + if (args[0] === "search") { + const child = createMockChild({ autoClose: false }); + emitAndClose( + child, + "stdout", + JSON.stringify([ + { + file: "qmd://workspace-main/notes/topic-name.md", + score: 0.79, + snippet: "@@ -1,1\nexact slugified path", + }, + ]), + ); + return child; + } + return createMockChild(); + }); + + const { manager } = await createManager(); + installIndexedPathStub({ + manager, + collection: "workspace-main", + normalizedPath: exactRelative, + exactPaths: [exactRelative], + allPaths: [exactRelative, slugCollisionRelative], + }); + + const results = await manager.search("exact slugified path", { + sessionKey: "agent:main:slack:dm:u123", + }); + expect(results).toEqual([ + { + path: exactRelative, + startLine: 1, + endLine: 1, + score: 0.79, + snippet: "@@ -1,1\nexact slugified path", + source: "memory", + }, + ]); + + await expect(manager.readFile({ relPath: results[0]!.path })).resolves.toEqual({ + path: exactRelative, + text: "exact slugified path", + }); + }); +}); diff --git a/extensions/memory-core/src/memory/qmd-manager.ts b/extensions/memory-core/src/memory/qmd-manager.ts index 839198cf3cb..75c5e3f0704 100644 --- a/extensions/memory-core/src/memory/qmd-manager.ts +++ b/extensions/memory-core/src/memory/qmd-manager.ts @@ -1548,6 +1548,13 @@ export class QmdMemoryManager implements MemorySearchManager { if (!hints.preferredCollection || !hints.preferredFile) { return null; } + const indexedLocation = this.resolveIndexedDocLocationFromHint( + hints.preferredCollection, + hints.preferredFile, + ); + if (indexedLocation) { + return indexedLocation; + } const collectionRelativePath = this.toCollectionRelativePath( hints.preferredCollection, hints.preferredFile, @@ -1558,6 +1565,45 @@ export class QmdMemoryManager implements MemorySearchManager { return this.toDocLocation(hints.preferredCollection, collectionRelativePath); } + private resolveIndexedDocLocationFromHint( + collection: string, + preferredFile: string, + ): { rel: string; abs: string; source: MemorySource } | null { + const trimmedCollection = collection.trim(); + const trimmedFile = preferredFile.trim(); + if (!trimmedCollection || !trimmedFile) { + return null; + } + const exactPath = path.normalize(trimmedFile).replace(/\\/g, "/"); + let rows: Array<{ path: string }> = []; + try { + const db = this.ensureDb(); + const exactRows = db + .prepare("SELECT path FROM documents WHERE collection = ? AND path = ? AND active = 1") + .all(trimmedCollection, exactPath) as Array<{ path: string }>; + if (exactRows.length > 0) { + return this.toDocLocation(trimmedCollection, exactRows[0].path); + } + rows = db + .prepare("SELECT path FROM documents WHERE collection = ? AND active = 1") + .all(trimmedCollection) as Array<{ path: string }>; + } catch (err) { + if (this.isSqliteBusyError(err)) { + log.debug(`qmd index is busy while resolving hinted path: ${String(err)}`); + throw this.createQmdBusyError(err); + } + // Hint-based lookup is best effort. Fall back to the raw hinted path when + // the index is unavailable or still warming. + log.debug(`qmd index hint lookup skipped: ${String(err)}`); + return null; + } + const matches = rows.filter((row) => this.matchesPreferredFileHint(row.path, trimmedFile)); + if (matches.length !== 1) { + return null; + } + return this.toDocLocation(trimmedCollection, matches[0].path); + } + private normalizeDocHints(hints?: { preferredCollection?: string; preferredFile?: string }): { preferredCollection?: string; preferredFile?: string; @@ -1637,10 +1683,8 @@ export class QmdMemoryManager implements MemorySearchManager { } } if (hints?.preferredFile) { - const preferred = path.normalize(hints.preferredFile); for (const row of rows) { - const rowPath = path.normalize(row.path); - if (rowPath !== preferred && !rowPath.endsWith(path.sep + preferred)) { + if (!this.matchesPreferredFileHint(row.path, hints.preferredFile)) { continue; } const location = this.toDocLocation(row.collection, row.path); @@ -1658,6 +1702,58 @@ export class QmdMemoryManager implements MemorySearchManager { return null; } + private matchesPreferredFileHint(rowPath: string, preferredFile: string): boolean { + const preferred = path.normalize(preferredFile).replace(/\\/g, "/"); + const normalizedRowPath = path.normalize(rowPath).replace(/\\/g, "/"); + if (normalizedRowPath === preferred || normalizedRowPath.endsWith(`/${preferred}`)) { + return true; + } + const normalizedPreferredLookup = this.normalizeQmdLookupPath(preferredFile); + if (!normalizedPreferredLookup) { + return false; + } + const normalizedRowLookup = this.normalizeQmdLookupPath(rowPath); + return ( + normalizedRowLookup === normalizedPreferredLookup || + normalizedRowLookup.endsWith(`/${normalizedPreferredLookup}`) + ); + } + + private normalizeQmdLookupPath(filePath: string): string { + return filePath + .replace(/\\/g, "/") + .split("/") + .filter((segment) => segment.length > 0 && segment !== ".") + .map((segment) => this.normalizeQmdLookupSegment(segment)) + .filter(Boolean) + .join("/"); + } + + private normalizeQmdLookupSegment(segment: string): string { + const trimmed = segment.trim(); + if (!trimmed) { + return ""; + } + if (trimmed === "." || trimmed === "..") { + return trimmed; + } + const parsed = path.posix.parse(trimmed); + const normalizePart = (value: string): string => + value + .normalize("NFKD") + .toLowerCase() + .replace(/[^\p{Letter}\p{Number}]+/gu, "-") + .replace(/-{2,}/g, "-") + .replace(/^-+|-+$/g, ""); + const normalizedName = normalizePart(parsed.name); + const normalizedExt = parsed.ext + .normalize("NFKD") + .toLowerCase() + .replace(/[^\p{Letter}\p{Number}.]+/gu, ""); + const fallbackName = parsed.name.normalize("NFKD").toLowerCase().replace(/\s+/g, "-").trim(); + return `${normalizedName || fallbackName || "file"}${normalizedExt}`; + } + private extractSnippetLines(snippet: string): { startLine: number; endLine: number } { const match = SNIPPET_HEADER_RE.exec(snippet); if (match) {