mirror of https://github.com/openclaw/openclaw.git
fix(memory): use explicit qmd snippet line metadata (#58181)
* fix(memory): preserve qmd snippet line metadata * Memory/QMD: preserve snippet span with partial line metadata
This commit is contained in:
parent
fcc2488579
commit
075645f5cb
|
|
@ -47,6 +47,7 @@ Docs: https://docs.openclaw.ai
|
|||
- Memory/doctor: suppress the orphan transcript cleanup warning when QMD session indexing is enabled, so doctor no longer suggests deleting transcript history that QMD still uses for recall. (#40584) Thanks @Gyarados4157 and @vincentkoc.
|
||||
- CI/dev checks: default local `pnpm check` to a lower-memory typecheck/lint path while keeping CI on the normal parallel path, and harden Telegram test typing/literals around native TypeScript-Go tooling crashes.
|
||||
- Agents/sandbox: honor `tools.sandbox.tools.alsoAllow`, let explicit sandbox re-allows remove matching built-in default-deny tools, and keep sandbox explain/error guidance aligned with the effective sandbox tool policy. (#54492) Thanks @ngutman.
|
||||
- Memory/QMD: preserve explicit `start_line` and `end_line` metadata from mcporter query results so `memory search` hits keep the real snippet offsets instead of falling back to the snippet header. (#47960) Thanks @vincentkoc.
|
||||
- LINE/ACP: add current-conversation binding and inbound binding-routing parity so `/acp spawn ... --thread here`, configured ACP bindings, and active conversation-bound ACP sessions work on LINE like the other conversation channels.
|
||||
- LINE/markdown: preserve underscores inside Latin, Cyrillic, and CJK words when stripping markdown, while still removing standalone `_italic_` markers on the shared text-runtime path used by LINE and TTS. (#47465) Thanks @jackjin1997.
|
||||
- TTS/Microsoft: auto-switch the default Edge voice to Chinese for CJK-dominant text without overriding explicitly selected Microsoft voices. (#52355) Thanks @extrasmall0.
|
||||
|
|
|
|||
|
|
@ -2080,6 +2080,153 @@ describe("QmdMemoryManager", () => {
|
|||
await manager.close();
|
||||
});
|
||||
|
||||
it("prefers mcporter start and end lines over snippet header offsets", async () => {
|
||||
const expectedDocId = "line-123";
|
||||
cfg = {
|
||||
...cfg,
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
includeDefaultMemory: false,
|
||||
searchMode: "query",
|
||||
update: { interval: "0s", debounceMs: 60_000, onBoot: false },
|
||||
paths: [{ path: workspaceDir, pattern: "**/*.md", name: "workspace" }],
|
||||
mcporter: { enabled: true, serverName: "qmd", startDaemon: false },
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
|
||||
spawnMock.mockImplementation((cmd: string, args: string[]) => {
|
||||
const child = createMockChild({ autoClose: false });
|
||||
if (isMcporterCommand(cmd) && args[0] === "call") {
|
||||
expect(args[1]).toBe("qmd.query");
|
||||
emitAndClose(
|
||||
child,
|
||||
"stdout",
|
||||
JSON.stringify({
|
||||
results: [
|
||||
{
|
||||
docid: expectedDocId,
|
||||
score: 0.91,
|
||||
collection: "workspace-main",
|
||||
start_line: 8,
|
||||
end_line: 10,
|
||||
snippet: "@@ -20,3\nline one\nline two\nline three",
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
return child;
|
||||
}
|
||||
emitAndClose(child, "stdout", "[]");
|
||||
return child;
|
||||
});
|
||||
|
||||
const { manager } = await createManager();
|
||||
const inner = manager as unknown as {
|
||||
db: { prepare: (query: string) => { all: (arg: unknown) => unknown }; close: () => void };
|
||||
};
|
||||
inner.db = {
|
||||
prepare: (_query: string) => ({
|
||||
all: (arg: unknown) => {
|
||||
if (typeof arg === "string" && arg.startsWith(expectedDocId)) {
|
||||
return [{ collection: "workspace-main", path: "notes/welcome.md" }];
|
||||
}
|
||||
return [];
|
||||
},
|
||||
}),
|
||||
close: () => {},
|
||||
};
|
||||
|
||||
await expect(
|
||||
manager.search("line one", { sessionKey: "agent:main:slack:dm:u123" }),
|
||||
).resolves.toEqual([
|
||||
{
|
||||
path: "notes/welcome.md",
|
||||
startLine: 8,
|
||||
endLine: 10,
|
||||
score: 0.91,
|
||||
snippet: "@@ -20,3\nline one\nline two\nline three",
|
||||
source: "memory",
|
||||
},
|
||||
]);
|
||||
|
||||
await manager.close();
|
||||
});
|
||||
|
||||
it("uses snippet header width when mcporter only returns a start line", async () => {
|
||||
const expectedDocId = "line-456";
|
||||
cfg = {
|
||||
...cfg,
|
||||
memory: {
|
||||
backend: "qmd",
|
||||
qmd: {
|
||||
includeDefaultMemory: false,
|
||||
searchMode: "query",
|
||||
update: { interval: "0s", debounceMs: 60_000, onBoot: false },
|
||||
paths: [{ path: workspaceDir, pattern: "**/*.md", name: "workspace" }],
|
||||
mcporter: { enabled: true, serverName: "qmd", startDaemon: false },
|
||||
},
|
||||
},
|
||||
} as OpenClawConfig;
|
||||
|
||||
spawnMock.mockImplementation((cmd: string, args: string[]) => {
|
||||
const child = createMockChild({ autoClose: false });
|
||||
if (isMcporterCommand(cmd) && args[0] === "call") {
|
||||
expect(args[1]).toBe("qmd.query");
|
||||
emitAndClose(
|
||||
child,
|
||||
"stdout",
|
||||
JSON.stringify({
|
||||
results: [
|
||||
{
|
||||
docid: expectedDocId,
|
||||
score: 0.73,
|
||||
collection: "workspace-main",
|
||||
start_line: 8,
|
||||
snippet: "@@ -20,3\nline one\nline two\nline three",
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
return child;
|
||||
}
|
||||
emitAndClose(child, "stdout", "[]");
|
||||
return child;
|
||||
});
|
||||
|
||||
const { manager } = await createManager();
|
||||
const inner = manager as unknown as {
|
||||
db: { prepare: (query: string) => { all: (arg: unknown) => unknown }; close: () => void };
|
||||
};
|
||||
inner.db = {
|
||||
prepare: (_query: string) => ({
|
||||
all: (arg: unknown) => {
|
||||
if (typeof arg === "string" && arg.startsWith(expectedDocId)) {
|
||||
return [{ collection: "workspace-main", path: "notes/welcome.md" }];
|
||||
}
|
||||
return [];
|
||||
},
|
||||
}),
|
||||
close: () => {},
|
||||
};
|
||||
|
||||
await expect(
|
||||
manager.search("line one", { sessionKey: "agent:main:slack:dm:u123" }),
|
||||
).resolves.toEqual([
|
||||
{
|
||||
path: "notes/welcome.md",
|
||||
startLine: 8,
|
||||
endLine: 10,
|
||||
score: 0.73,
|
||||
snippet: "@@ -20,3\nline one\nline two\nline three",
|
||||
source: "memory",
|
||||
},
|
||||
]);
|
||||
|
||||
await manager.close();
|
||||
});
|
||||
|
||||
it('uses unified v2 args when the explicit mcporter search tool override is "query"', async () => {
|
||||
cfg = {
|
||||
...cfg,
|
||||
|
|
|
|||
|
|
@ -983,7 +983,7 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||
continue;
|
||||
}
|
||||
const snippet = entry.snippet?.slice(0, this.qmd.limits.maxSnippetChars) ?? "";
|
||||
const lines = this.extractSnippetLines(snippet);
|
||||
const lines = this.resolveSnippetLines(entry, snippet);
|
||||
const score = typeof entry.score === "number" ? entry.score : 0;
|
||||
const minScore = opts?.minScore ?? 0;
|
||||
if (score < minScore) {
|
||||
|
|
@ -1681,7 +1681,16 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||
const scoreRaw = item.score;
|
||||
const score = typeof scoreRaw === "number" ? scoreRaw : Number(scoreRaw);
|
||||
const snippet = typeof item.snippet === "string" ? item.snippet : "";
|
||||
out.push({ docid, score: Number.isFinite(score) ? score : 0, snippet });
|
||||
out.push({
|
||||
docid,
|
||||
score: Number.isFinite(score) ? score : 0,
|
||||
snippet,
|
||||
collection: typeof item.collection === "string" ? item.collection : undefined,
|
||||
file: typeof item.file === "string" ? item.file : undefined,
|
||||
body: typeof item.body === "string" ? item.body : undefined,
|
||||
startLine: this.normalizeSnippetLine(item.start_line ?? item.startLine),
|
||||
endLine: this.normalizeSnippetLine(item.end_line ?? item.endLine),
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
|
@ -2099,18 +2108,69 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||
}
|
||||
|
||||
private extractSnippetLines(snippet: string): { startLine: number; endLine: number } {
|
||||
const match = SNIPPET_HEADER_RE.exec(snippet);
|
||||
if (match) {
|
||||
const start = Number(match[1]);
|
||||
const count = Number(match[2]);
|
||||
if (Number.isFinite(start) && Number.isFinite(count)) {
|
||||
return { startLine: start, endLine: start + count - 1 };
|
||||
}
|
||||
const headerLines = this.parseSnippetHeaderLines(snippet);
|
||||
if (headerLines) {
|
||||
return headerLines;
|
||||
}
|
||||
const lines = snippet.split("\n").length;
|
||||
return { startLine: 1, endLine: lines };
|
||||
}
|
||||
|
||||
private resolveSnippetLines(
|
||||
entry: QmdQueryResult,
|
||||
snippet: string,
|
||||
): { startLine: number; endLine: number } {
|
||||
const explicitStart = this.normalizeSnippetLine(entry.startLine);
|
||||
const explicitEnd = this.normalizeSnippetLine(entry.endLine);
|
||||
const headerLines = this.parseSnippetHeaderLines(snippet);
|
||||
if (explicitStart !== undefined && explicitEnd !== undefined) {
|
||||
return explicitStart <= explicitEnd
|
||||
? { startLine: explicitStart, endLine: explicitEnd }
|
||||
: { startLine: explicitEnd, endLine: explicitStart };
|
||||
}
|
||||
if (explicitStart !== undefined) {
|
||||
if (headerLines) {
|
||||
const width = headerLines.endLine - headerLines.startLine;
|
||||
return {
|
||||
startLine: explicitStart,
|
||||
endLine: explicitStart + Math.max(0, width),
|
||||
};
|
||||
}
|
||||
return { startLine: explicitStart, endLine: explicitStart };
|
||||
}
|
||||
if (explicitEnd !== undefined) {
|
||||
if (headerLines) {
|
||||
const width = headerLines.endLine - headerLines.startLine;
|
||||
return {
|
||||
startLine: Math.max(1, explicitEnd - Math.max(0, width)),
|
||||
endLine: explicitEnd,
|
||||
};
|
||||
}
|
||||
return { startLine: explicitEnd, endLine: explicitEnd };
|
||||
}
|
||||
if (headerLines) {
|
||||
return headerLines;
|
||||
}
|
||||
return { startLine: 1, endLine: snippet.split("\n").length };
|
||||
}
|
||||
|
||||
private normalizeSnippetLine(value: unknown): number | undefined {
|
||||
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : undefined;
|
||||
}
|
||||
|
||||
private parseSnippetHeaderLines(snippet: string): { startLine: number; endLine: number } | null {
|
||||
const match = SNIPPET_HEADER_RE.exec(snippet);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
const start = Number(match[1]);
|
||||
const count = Number(match[2]);
|
||||
if (Number.isFinite(start) && Number.isFinite(count)) {
|
||||
return { startLine: start, endLine: start + count - 1 };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private readCounts(): {
|
||||
totalDocuments: number;
|
||||
sourceCounts: Array<{ source: MemorySource; files: number; chunks: number }>;
|
||||
|
|
|
|||
|
|
@ -24,6 +24,22 @@ complete`,
|
|||
expect(results).toEqual([{ docid: "abc", score: 0.5 }]);
|
||||
});
|
||||
|
||||
it("preserves explicit qmd line metadata when present", () => {
|
||||
const results = parseQmdQueryJson(
|
||||
'[{"docid":"abc","score":0.5,"start_line":4,"end_line":6,"snippet":"@@ -10,1\\nignored"}]',
|
||||
"",
|
||||
);
|
||||
expect(results).toEqual([
|
||||
{
|
||||
docid: "abc",
|
||||
score: 0.5,
|
||||
snippet: "@@ -10,1\nignored",
|
||||
startLine: 4,
|
||||
endLine: 6,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("treats plain-text no-results from stderr as an empty result set", () => {
|
||||
const results = parseQmdQueryJson("", "No results found\n");
|
||||
expect(results).toEqual([]);
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ export type QmdQueryResult = {
|
|||
file?: string;
|
||||
snippet?: string;
|
||||
body?: string;
|
||||
startLine?: number;
|
||||
endLine?: number;
|
||||
};
|
||||
|
||||
export function parseQmdQueryJson(stdout: string, stderr: string): QmdQueryResult[] {
|
||||
|
|
@ -73,12 +75,40 @@ function parseQmdQueryResultArray(raw: string): QmdQueryResult[] | null {
|
|||
if (!Array.isArray(parsed)) {
|
||||
return null;
|
||||
}
|
||||
return parsed as QmdQueryResult[];
|
||||
return parsed.map((item) => {
|
||||
if (typeof item !== "object" || item === null) {
|
||||
return item as QmdQueryResult;
|
||||
}
|
||||
const record = item as Record<string, unknown>;
|
||||
const docid = typeof record.docid === "string" ? record.docid : undefined;
|
||||
const score =
|
||||
typeof record.score === "number" && Number.isFinite(record.score)
|
||||
? record.score
|
||||
: undefined;
|
||||
const collection = typeof record.collection === "string" ? record.collection : undefined;
|
||||
const file = typeof record.file === "string" ? record.file : undefined;
|
||||
const snippet = typeof record.snippet === "string" ? record.snippet : undefined;
|
||||
const body = typeof record.body === "string" ? record.body : undefined;
|
||||
return {
|
||||
docid,
|
||||
score,
|
||||
collection,
|
||||
file,
|
||||
snippet,
|
||||
body,
|
||||
startLine: parseQmdLineNumber(record.start_line ?? record.startLine),
|
||||
endLine: parseQmdLineNumber(record.end_line ?? record.endLine),
|
||||
} as QmdQueryResult;
|
||||
});
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function parseQmdLineNumber(value: unknown): number | undefined {
|
||||
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : undefined;
|
||||
}
|
||||
|
||||
function extractFirstJsonArray(raw: string): string | null {
|
||||
const start = raw.indexOf("[");
|
||||
if (start < 0) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue