mirror of https://github.com/openclaw/openclaw.git
feat(memory-wiki): compile related backlinks blocks
This commit is contained in:
Vincent Koc
parent
2f72363984
commit
08492dfeee
|
|
@ -67,4 +67,105 @@ describe("compileMemoryWikiVault", () => {
|
|||
"[[sources/alpha|Alpha]]",
|
||||
);
|
||||
});
|
||||
|
||||
it("writes related blocks from source ids and shared sources", async () => {
|
||||
const rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-compile-"));
|
||||
tempDirs.push(rootDir);
|
||||
const config = resolveMemoryWikiConfig(
|
||||
{ vault: { path: rootDir } },
|
||||
{ homedir: "/Users/tester" },
|
||||
);
|
||||
await initializeMemoryWikiVault(config);
|
||||
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "sources", "alpha.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: { pageType: "source", id: "source.alpha", title: "Alpha" },
|
||||
body: "# Alpha\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "entities", "beta.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "entity",
|
||||
id: "entity.beta",
|
||||
title: "Beta",
|
||||
sourceIds: ["source.alpha"],
|
||||
},
|
||||
body: "# Beta\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "concepts", "gamma.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: {
|
||||
pageType: "concept",
|
||||
id: "concept.gamma",
|
||||
title: "Gamma",
|
||||
sourceIds: ["source.alpha"],
|
||||
},
|
||||
body: "# Gamma\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
await compileMemoryWikiVault(config);
|
||||
|
||||
await expect(fs.readFile(path.join(rootDir, "entities", "beta.md"), "utf8")).resolves.toContain(
|
||||
"## Related",
|
||||
);
|
||||
await expect(fs.readFile(path.join(rootDir, "entities", "beta.md"), "utf8")).resolves.toContain(
|
||||
"[Alpha](sources/alpha.md)",
|
||||
);
|
||||
await expect(fs.readFile(path.join(rootDir, "entities", "beta.md"), "utf8")).resolves.toContain(
|
||||
"[Gamma](concepts/gamma.md)",
|
||||
);
|
||||
await expect(fs.readFile(path.join(rootDir, "sources", "alpha.md"), "utf8")).resolves.toContain(
|
||||
"[Beta](entities/beta.md)",
|
||||
);
|
||||
await expect(fs.readFile(path.join(rootDir, "sources", "alpha.md"), "utf8")).resolves.toContain(
|
||||
"[Gamma](concepts/gamma.md)",
|
||||
);
|
||||
});
|
||||
|
||||
it("ignores generated related links when computing backlinks on repeated compile", async () => {
|
||||
const rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "memory-wiki-compile-"));
|
||||
tempDirs.push(rootDir);
|
||||
const config = resolveMemoryWikiConfig(
|
||||
{ vault: { path: rootDir } },
|
||||
{ homedir: "/Users/tester" },
|
||||
);
|
||||
await initializeMemoryWikiVault(config);
|
||||
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "entities", "beta.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: { pageType: "entity", id: "entity.beta", title: "Beta" },
|
||||
body: "# Beta\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
await fs.writeFile(
|
||||
path.join(rootDir, "concepts", "gamma.md"),
|
||||
renderWikiMarkdown({
|
||||
frontmatter: { pageType: "concept", id: "concept.gamma", title: "Gamma" },
|
||||
body: "# Gamma\n\nSee [Beta](entities/beta.md).\n",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
await compileMemoryWikiVault(config);
|
||||
const second = await compileMemoryWikiVault(config);
|
||||
|
||||
expect(second.updatedFiles).toEqual([]);
|
||||
await expect(fs.readFile(path.join(rootDir, "entities", "beta.md"), "utf8")).resolves.toContain(
|
||||
"[Gamma](concepts/gamma.md)",
|
||||
);
|
||||
await expect(
|
||||
fs.readFile(path.join(rootDir, "concepts", "gamma.md"), "utf8"),
|
||||
).resolves.not.toContain("### Referenced By");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -11,6 +11,8 @@ import {
|
|||
toWikiPageSummary,
|
||||
type WikiPageKind,
|
||||
type WikiPageSummary,
|
||||
WIKI_RELATED_END_MARKER,
|
||||
WIKI_RELATED_START_MARKER,
|
||||
} from "./markdown.js";
|
||||
import { initializeMemoryWikiVault } from "./vault.js";
|
||||
|
||||
|
|
@ -73,6 +75,162 @@ function buildPageCounts(pages: WikiPageSummary[]): Record<WikiPageKind, number>
|
|||
};
|
||||
}
|
||||
|
||||
function normalizeComparableTarget(value: string): string {
|
||||
return value
|
||||
.trim()
|
||||
.replace(/\\/g, "/")
|
||||
.replace(/\.md$/i, "")
|
||||
.replace(/^\.\/+/, "")
|
||||
.replace(/\/+$/, "")
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
function uniquePages(pages: WikiPageSummary[]): WikiPageSummary[] {
|
||||
const seen = new Set<string>();
|
||||
const unique: WikiPageSummary[] = [];
|
||||
for (const page of pages) {
|
||||
const key = page.id ?? page.relativePath;
|
||||
if (seen.has(key)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(key);
|
||||
unique.push(page);
|
||||
}
|
||||
return unique;
|
||||
}
|
||||
|
||||
function buildPageLookupKeys(page: WikiPageSummary): Set<string> {
|
||||
const keys = new Set<string>();
|
||||
keys.add(normalizeComparableTarget(page.relativePath));
|
||||
keys.add(normalizeComparableTarget(page.relativePath.replace(/\.md$/i, "")));
|
||||
keys.add(normalizeComparableTarget(page.title));
|
||||
if (page.id) {
|
||||
keys.add(normalizeComparableTarget(page.id));
|
||||
}
|
||||
return keys;
|
||||
}
|
||||
|
||||
function renderWikiPageLinks(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
pages: WikiPageSummary[];
|
||||
}): string {
|
||||
return params.pages
|
||||
.map(
|
||||
(page) =>
|
||||
`- ${formatWikiLink({
|
||||
renderMode: params.config.vault.renderMode,
|
||||
relativePath: page.relativePath,
|
||||
title: page.title,
|
||||
})}`,
|
||||
)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function buildRelatedBlockBody(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
page: WikiPageSummary;
|
||||
allPages: WikiPageSummary[];
|
||||
}): string {
|
||||
const pagesById = new Map(
|
||||
params.allPages.flatMap((candidate) =>
|
||||
candidate.id ? [[candidate.id, candidate] as const] : [],
|
||||
),
|
||||
);
|
||||
const sourcePages = uniquePages(
|
||||
params.page.sourceIds.flatMap((sourceId) => {
|
||||
const page = pagesById.get(sourceId);
|
||||
return page ? [page] : [];
|
||||
}),
|
||||
);
|
||||
const backlinkKeys = buildPageLookupKeys(params.page);
|
||||
const backlinks = uniquePages(
|
||||
params.allPages.filter((candidate) => {
|
||||
if (candidate.relativePath === params.page.relativePath) {
|
||||
return false;
|
||||
}
|
||||
if (candidate.sourceIds.includes(params.page.id ?? "")) {
|
||||
return true;
|
||||
}
|
||||
return candidate.linkTargets.some((target) =>
|
||||
backlinkKeys.has(normalizeComparableTarget(target)),
|
||||
);
|
||||
}),
|
||||
);
|
||||
const relatedPages = uniquePages(
|
||||
params.allPages.filter((candidate) => {
|
||||
if (candidate.relativePath === params.page.relativePath) {
|
||||
return false;
|
||||
}
|
||||
if (sourcePages.some((sourcePage) => sourcePage.relativePath === candidate.relativePath)) {
|
||||
return false;
|
||||
}
|
||||
if (backlinks.some((backlink) => backlink.relativePath === candidate.relativePath)) {
|
||||
return false;
|
||||
}
|
||||
if (params.page.sourceIds.length === 0 || candidate.sourceIds.length === 0) {
|
||||
return false;
|
||||
}
|
||||
return params.page.sourceIds.some((sourceId) => candidate.sourceIds.includes(sourceId));
|
||||
}),
|
||||
);
|
||||
|
||||
const sections: string[] = [];
|
||||
if (sourcePages.length > 0) {
|
||||
sections.push(
|
||||
"### Sources",
|
||||
renderWikiPageLinks({ config: params.config, pages: sourcePages }),
|
||||
);
|
||||
}
|
||||
if (backlinks.length > 0) {
|
||||
sections.push(
|
||||
"### Referenced By",
|
||||
renderWikiPageLinks({ config: params.config, pages: backlinks }),
|
||||
);
|
||||
}
|
||||
if (relatedPages.length > 0) {
|
||||
sections.push(
|
||||
"### Related Pages",
|
||||
renderWikiPageLinks({ config: params.config, pages: relatedPages }),
|
||||
);
|
||||
}
|
||||
if (sections.length === 0) {
|
||||
return "- No related pages yet.";
|
||||
}
|
||||
return sections.join("\n\n");
|
||||
}
|
||||
|
||||
async function refreshPageRelatedBlocks(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
pages: WikiPageSummary[];
|
||||
}): Promise<string[]> {
|
||||
if (!params.config.render.createBacklinks) {
|
||||
return [];
|
||||
}
|
||||
const updatedFiles: string[] = [];
|
||||
for (const page of params.pages) {
|
||||
const original = await fs.readFile(page.absolutePath, "utf8");
|
||||
const updated = withTrailingNewline(
|
||||
replaceManagedMarkdownBlock({
|
||||
original,
|
||||
heading: "## Related",
|
||||
startMarker: WIKI_RELATED_START_MARKER,
|
||||
endMarker: WIKI_RELATED_END_MARKER,
|
||||
body: buildRelatedBlockBody({
|
||||
config: params.config,
|
||||
page,
|
||||
allPages: params.pages,
|
||||
}),
|
||||
}),
|
||||
);
|
||||
if (updated === original) {
|
||||
continue;
|
||||
}
|
||||
await fs.writeFile(page.absolutePath, updated, "utf8");
|
||||
updatedFiles.push(page.absolutePath);
|
||||
}
|
||||
return updatedFiles;
|
||||
}
|
||||
|
||||
function renderSectionList(params: {
|
||||
config: ResolvedMemoryWikiConfig;
|
||||
pages: WikiPageSummary[];
|
||||
|
|
@ -162,9 +320,12 @@ export async function compileMemoryWikiVault(
|
|||
): Promise<CompileMemoryWikiResult> {
|
||||
await initializeMemoryWikiVault(config);
|
||||
const rootDir = config.vault.path;
|
||||
const pages = await readPageSummaries(rootDir);
|
||||
let pages = await readPageSummaries(rootDir);
|
||||
const updatedFiles = await refreshPageRelatedBlocks({ config, pages });
|
||||
if (updatedFiles.length > 0) {
|
||||
pages = await readPageSummaries(rootDir);
|
||||
}
|
||||
const counts = buildPageCounts(pages);
|
||||
const updatedFiles: string[] = [];
|
||||
|
||||
const rootIndexPath = path.join(rootDir, "index.md");
|
||||
if (
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@ import path from "node:path";
|
|||
import YAML from "yaml";
|
||||
|
||||
export const WIKI_PAGE_KINDS = ["entity", "concept", "source", "synthesis", "report"] as const;
|
||||
export const WIKI_RELATED_START_MARKER = "<!-- openclaw:wiki:related:start -->";
|
||||
export const WIKI_RELATED_END_MARKER = "<!-- openclaw:wiki:related:end -->";
|
||||
|
||||
export type WikiPageKind = (typeof WIKI_PAGE_KINDS)[number];
|
||||
|
||||
|
|
@ -38,6 +40,11 @@ function normalizeOptionalString(value: unknown): string | undefined {
|
|||
|
||||
const FRONTMATTER_PATTERN = /^---\n([\s\S]*?)\n---\n?/;
|
||||
const OBSIDIAN_LINK_PATTERN = /\[\[([^\]|]+)(?:\|[^\]]+)?\]\]/g;
|
||||
const MARKDOWN_LINK_PATTERN = /\[[^\]]+\]\(([^)]+)\)/g;
|
||||
const RELATED_BLOCK_PATTERN = new RegExp(
|
||||
`${WIKI_RELATED_START_MARKER}[\\s\\S]*?${WIKI_RELATED_END_MARKER}`,
|
||||
"g",
|
||||
);
|
||||
|
||||
export function slugifyWikiSegment(raw: string): string {
|
||||
const slug = raw
|
||||
|
|
@ -98,13 +105,24 @@ function normalizeStringList(value: unknown): string[] {
|
|||
}
|
||||
|
||||
export function extractWikiLinks(markdown: string): string[] {
|
||||
const searchable = markdown.replace(RELATED_BLOCK_PATTERN, "");
|
||||
const links: string[] = [];
|
||||
for (const match of markdown.matchAll(OBSIDIAN_LINK_PATTERN)) {
|
||||
for (const match of searchable.matchAll(OBSIDIAN_LINK_PATTERN)) {
|
||||
const target = match[1]?.trim();
|
||||
if (target) {
|
||||
links.push(target);
|
||||
}
|
||||
}
|
||||
for (const match of searchable.matchAll(MARKDOWN_LINK_PATTERN)) {
|
||||
const rawTarget = match[1]?.trim();
|
||||
if (!rawTarget || rawTarget.startsWith("#") || /^[a-z]+:/i.test(rawTarget)) {
|
||||
continue;
|
||||
}
|
||||
const target = rawTarget.split("#")[0]?.split("?")[0]?.replace(/\\/g, "/").trim();
|
||||
if (target) {
|
||||
links.push(target);
|
||||
}
|
||||
}
|
||||
return links;
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue