Matrix: harden media handling and summaries

This commit is contained in:
Gustavo Madeira Santana 2026-03-12 19:40:30 +00:00
parent 309c600770
commit f35fcb89b4
9 changed files with 728 additions and 5 deletions

View File

@ -0,0 +1,87 @@
import { describe, expect, it } from "vitest";
import { summarizeMatrixRawEvent } from "./summary.js";
describe("summarizeMatrixRawEvent", () => {
it("replaces bare media filenames with a media marker", () => {
const summary = summarizeMatrixRawEvent({
event_id: "$image",
sender: "@gum:matrix.example.org",
type: "m.room.message",
origin_server_ts: 123,
content: {
msgtype: "m.image",
body: "photo.jpg",
},
});
expect(summary).toMatchObject({
eventId: "$image",
msgtype: "m.image",
attachment: {
kind: "image",
filename: "photo.jpg",
},
});
expect(summary.body).toBeUndefined();
});
it("preserves captions while marking media summaries", () => {
const summary = summarizeMatrixRawEvent({
event_id: "$image",
sender: "@gum:matrix.example.org",
type: "m.room.message",
origin_server_ts: 123,
content: {
msgtype: "m.image",
body: "can you see this?",
filename: "photo.jpg",
},
});
expect(summary).toMatchObject({
body: "can you see this?",
attachment: {
kind: "image",
caption: "can you see this?",
filename: "photo.jpg",
},
});
});
it("does not treat a sentence ending in a file extension as a bare filename", () => {
const summary = summarizeMatrixRawEvent({
event_id: "$image",
sender: "@gum:matrix.example.org",
type: "m.room.message",
origin_server_ts: 123,
content: {
msgtype: "m.image",
body: "see image.png",
},
});
expect(summary).toMatchObject({
body: "see image.png",
attachment: {
kind: "image",
caption: "see image.png",
},
});
});
it("leaves text messages unchanged", () => {
const summary = summarizeMatrixRawEvent({
event_id: "$text",
sender: "@gum:matrix.example.org",
type: "m.room.message",
origin_server_ts: 123,
content: {
msgtype: "m.text",
body: "hello",
},
});
expect(summary.body).toBe("hello");
expect(summary.attachment).toBeUndefined();
});
});

View File

@ -1,3 +1,4 @@
import { resolveMatrixMessageAttachment, resolveMatrixMessageBody } from "../media-text.js";
import { fetchMatrixPollMessageSummary } from "../poll-summary.js";
import type { MatrixClient } from "../sdk.js";
import {
@ -31,8 +32,17 @@ export function summarizeMatrixRawEvent(event: MatrixRawEvent): MatrixMessageSum
return {
eventId: event.event_id,
sender: event.sender,
body: content.body,
body: resolveMatrixMessageBody({
body: content.body,
filename: content.filename,
msgtype: content.msgtype,
}),
msgtype: content.msgtype,
attachment: resolveMatrixMessageAttachment({
body: content.body,
filename: content.filename,
msgtype: content.msgtype,
}),
timestamp: event.origin_server_ts,
relatesTo,
};

View File

@ -59,6 +59,7 @@ export type MatrixMessageSummary = {
sender?: string;
body?: string;
msgtype?: string;
attachment?: MatrixMessageAttachmentSummary;
timestamp?: number;
relatesTo?: {
relType?: string;
@ -67,6 +68,14 @@ export type MatrixMessageSummary = {
};
};
export type MatrixMessageAttachmentKind = "audio" | "file" | "image" | "sticker" | "video";
export type MatrixMessageAttachmentSummary = {
kind: MatrixMessageAttachmentKind;
caption?: string;
filename?: string;
};
export type MatrixActionClient = {
client: MatrixClient;
stopOnDone: boolean;

View File

@ -0,0 +1,147 @@
import path from "node:path";
import type {
MatrixMessageAttachmentKind,
MatrixMessageAttachmentSummary,
MatrixMessageSummary,
} from "./actions/types.js";
const MATRIX_MEDIA_KINDS: Record<string, MatrixMessageAttachmentKind> = {
"m.audio": "audio",
"m.file": "file",
"m.image": "image",
"m.sticker": "sticker",
"m.video": "video",
};
function resolveMatrixMediaKind(msgtype: string | undefined): MatrixMessageAttachmentKind | null {
return MATRIX_MEDIA_KINDS[msgtype ?? ""] ?? null;
}
function resolveMatrixMediaLabel(
kind: MatrixMessageAttachmentKind | undefined,
fallback = "media",
): string {
return `${kind ?? fallback} attachment`;
}
function formatMatrixAttachmentMarker(params: {
kind?: MatrixMessageAttachmentKind;
unavailable?: boolean;
}): string {
const label = resolveMatrixMediaLabel(params.kind);
return params.unavailable ? `[matrix ${label} unavailable]` : `[matrix ${label}]`;
}
export function isLikelyBareFilename(text: string): boolean {
const trimmed = text.trim();
if (!trimmed || trimmed.includes("\n") || /\s/.test(trimmed)) {
return false;
}
if (path.basename(trimmed) !== trimmed) {
return false;
}
return path.extname(trimmed).length > 1;
}
function resolveCaptionOrFilename(params: { body?: string; filename?: string }): {
caption?: string;
filename?: string;
} {
const body = params.body?.trim() ?? "";
const filename = params.filename?.trim() ?? "";
if (filename) {
if (!body || body === filename) {
return { filename };
}
return { caption: body, filename };
}
if (!body) {
return {};
}
if (isLikelyBareFilename(body)) {
return { filename: body };
}
return { caption: body };
}
export function resolveMatrixMessageAttachment(params: {
body?: string;
filename?: string;
msgtype?: string;
}): MatrixMessageAttachmentSummary | undefined {
const kind = resolveMatrixMediaKind(params.msgtype);
if (!kind) {
return undefined;
}
const resolved = resolveCaptionOrFilename(params);
return {
kind,
caption: resolved.caption,
filename: resolved.filename,
};
}
export function resolveMatrixMessageBody(params: {
body?: string;
filename?: string;
msgtype?: string;
}): string | undefined {
const attachment = resolveMatrixMessageAttachment(params);
if (!attachment) {
const body = params.body?.trim() ?? "";
return body || undefined;
}
return attachment.caption;
}
export function formatMatrixAttachmentText(params: {
attachment?: MatrixMessageAttachmentSummary;
unavailable?: boolean;
}): string | undefined {
if (!params.attachment) {
return undefined;
}
return formatMatrixAttachmentMarker({
kind: params.attachment.kind,
unavailable: params.unavailable,
});
}
export function formatMatrixMessageText(params: {
body?: string;
attachment?: MatrixMessageAttachmentSummary;
unavailable?: boolean;
}): string | undefined {
const body = params.body?.trim() ?? "";
const marker = formatMatrixAttachmentText({
attachment: params.attachment,
unavailable: params.unavailable,
});
if (!marker) {
return body || undefined;
}
if (!body) {
return marker;
}
return `${body}\n\n${marker}`;
}
export function formatMatrixMessageSummaryText(
summary: Pick<MatrixMessageSummary, "body" | "attachment">,
): string | undefined {
return formatMatrixMessageText(summary);
}
export function formatMatrixMediaUnavailableText(params: {
body?: string;
filename?: string;
msgtype?: string;
}): string {
return (
formatMatrixMessageText({
body: resolveMatrixMessageBody(params),
attachment: resolveMatrixMessageAttachment(params),
unavailable: true,
}) ?? ""
);
}

View File

@ -0,0 +1,239 @@
import type { PluginRuntime, RuntimeEnv, RuntimeLogger } from "openclaw/plugin-sdk/matrix";
import { beforeEach, describe, expect, it, vi } from "vitest";
import { setMatrixRuntime } from "../../runtime.js";
import type { MatrixClient } from "../sdk.js";
import type { MatrixRawEvent } from "./types.js";
import { EventType } from "./types.js";
const { downloadMatrixMediaMock } = vi.hoisted(() => ({
downloadMatrixMediaMock: vi.fn(),
}));
vi.mock("./media.js", () => ({
downloadMatrixMedia: (...args: unknown[]) => downloadMatrixMediaMock(...args),
}));
import { createMatrixRoomMessageHandler } from "./handler.js";
function createHandlerHarness() {
const recordInboundSession = vi.fn().mockResolvedValue(undefined);
const logger = {
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
} as unknown as RuntimeLogger;
const runtime = {
error: vi.fn(),
} as unknown as RuntimeEnv;
const core = {
channel: {
pairing: {
readAllowFromStore: vi.fn().mockResolvedValue([]),
upsertPairingRequest: vi.fn().mockResolvedValue(undefined),
buildPairingReply: vi.fn().mockReturnValue("pairing"),
},
routing: {
resolveAgentRoute: vi.fn().mockReturnValue({
agentId: "main",
accountId: undefined,
sessionKey: "agent:main:matrix:channel:!room:example.org",
mainSessionKey: "agent:main:main",
}),
},
session: {
resolveStorePath: vi.fn().mockReturnValue("/tmp/openclaw-test-session.json"),
readSessionUpdatedAt: vi.fn().mockReturnValue(123),
recordInboundSession,
},
reply: {
resolveEnvelopeFormatOptions: vi.fn().mockReturnValue({}),
formatAgentEnvelope: vi.fn().mockImplementation((params: { body: string }) => params.body),
finalizeInboundContext: vi.fn().mockImplementation((ctx: Record<string, unknown>) => ctx),
createReplyDispatcherWithTyping: vi.fn().mockReturnValue({
dispatcher: {},
replyOptions: {},
markDispatchIdle: vi.fn(),
}),
resolveHumanDelayConfig: vi.fn().mockReturnValue(undefined),
dispatchReplyFromConfig: vi
.fn()
.mockResolvedValue({ queuedFinal: false, counts: { final: 0, block: 0, tool: 0 } }),
},
commands: {
shouldHandleTextCommands: vi.fn().mockReturnValue(true),
},
text: {
hasControlCommand: vi.fn().mockReturnValue(false),
resolveMarkdownTableMode: vi.fn().mockReturnValue("code"),
},
reactions: {
shouldAckReaction: vi.fn().mockReturnValue(false),
},
},
system: {
enqueueSystemEvent: vi.fn(),
},
} as unknown as PluginRuntime;
const client = {
getUserId: vi.fn().mockResolvedValue("@bot:matrix.example.org"),
} as unknown as MatrixClient;
const handler = createMatrixRoomMessageHandler({
client,
core,
cfg: {},
accountId: "ops",
runtime,
logger,
logVerboseMessage: vi.fn(),
allowFrom: [],
groupAllowFrom: [],
roomsConfig: undefined,
mentionRegexes: [],
groupPolicy: "open",
replyToMode: "first",
threadReplies: "inbound",
dmEnabled: true,
dmPolicy: "open",
textLimit: 4000,
mediaMaxBytes: 5 * 1024 * 1024,
startupMs: Date.now() - 120_000,
startupGraceMs: 60_000,
directTracker: {
isDirectMessage: vi.fn().mockResolvedValue(true),
},
getRoomInfo: vi.fn().mockResolvedValue({
name: "Media Room",
canonicalAlias: "#media:example.org",
altAliases: [],
}),
getMemberDisplayName: vi.fn().mockResolvedValue("Gum"),
needsRoomAliasesForConfig: false,
});
return { handler, recordInboundSession, logger, runtime };
}
function createImageEvent(content: Record<string, unknown>): MatrixRawEvent {
return {
type: EventType.RoomMessage,
event_id: "$event1",
sender: "@gum:matrix.example.org",
origin_server_ts: Date.now(),
content: {
...content,
"m.mentions": { user_ids: ["@bot:matrix.example.org"] },
},
} as MatrixRawEvent;
}
describe("createMatrixRoomMessageHandler media failures", () => {
beforeEach(() => {
downloadMatrixMediaMock.mockReset();
setMatrixRuntime({
channel: {
mentions: {
matchesMentionPatterns: (text: string, patterns: RegExp[]) =>
patterns.some((pattern) => pattern.test(text)),
},
media: {
saveMediaBuffer: vi.fn(),
},
},
config: {
loadConfig: () => ({}),
},
state: {
resolveStateDir: () => "/tmp",
},
} as unknown as PluginRuntime);
});
it("replaces bare image filenames with an unavailable marker when unencrypted download fails", async () => {
downloadMatrixMediaMock.mockRejectedValue(new Error("download failed"));
const { handler, recordInboundSession, logger, runtime } = createHandlerHarness();
await handler(
"!room:example.org",
createImageEvent({
msgtype: "m.image",
body: "image.png",
url: "mxc://example/image",
}),
);
expect(recordInboundSession).toHaveBeenCalledWith(
expect.objectContaining({
ctx: expect.objectContaining({
RawBody: "[matrix image attachment unavailable]",
CommandBody: "[matrix image attachment unavailable]",
MediaPath: undefined,
}),
}),
);
expect(logger.warn).toHaveBeenCalledWith(
"matrix media download failed",
expect.objectContaining({
eventId: "$event1",
msgtype: "m.image",
encrypted: false,
}),
);
expect(runtime.error).not.toHaveBeenCalled();
});
it("replaces bare image filenames with an unavailable marker when encrypted download fails", async () => {
downloadMatrixMediaMock.mockRejectedValue(new Error("decrypt failed"));
const { handler, recordInboundSession } = createHandlerHarness();
await handler(
"!room:example.org",
createImageEvent({
msgtype: "m.image",
body: "photo.jpg",
file: {
url: "mxc://example/encrypted",
key: { kty: "oct", key_ops: ["encrypt"], alg: "A256CTR", k: "secret", ext: true },
iv: "iv",
hashes: { sha256: "hash" },
v: "v2",
},
}),
);
expect(recordInboundSession).toHaveBeenCalledWith(
expect.objectContaining({
ctx: expect.objectContaining({
RawBody: "[matrix image attachment unavailable]",
CommandBody: "[matrix image attachment unavailable]",
MediaPath: undefined,
}),
}),
);
});
it("preserves a real caption while marking the attachment unavailable", async () => {
downloadMatrixMediaMock.mockRejectedValue(new Error("download failed"));
const { handler, recordInboundSession } = createHandlerHarness();
await handler(
"!room:example.org",
createImageEvent({
msgtype: "m.image",
body: "can you see this image?",
filename: "image.png",
url: "mxc://example/image",
}),
);
expect(recordInboundSession).toHaveBeenCalledWith(
expect.objectContaining({
ctx: expect.objectContaining({
RawBody: "can you see this image?\n\n[matrix image attachment unavailable]",
CommandBody: "can you see this image?\n\n[matrix image attachment unavailable]",
}),
}),
);
});
});

View File

@ -0,0 +1,159 @@
import type { PluginRuntime, RuntimeEnv, RuntimeLogger } from "openclaw/plugin-sdk/matrix";
import { describe, expect, it, vi } from "vitest";
import { setMatrixRuntime } from "../../runtime.js";
import type { MatrixClient } from "../sdk.js";
import { createMatrixRoomMessageHandler } from "./handler.js";
import { EventType, type MatrixRawEvent } from "./types.js";
describe("createMatrixRoomMessageHandler thread root media", () => {
it("keeps image-only thread roots visible via attachment markers", async () => {
setMatrixRuntime({
channel: {
mentions: {
matchesMentionPatterns: (text: string, patterns: RegExp[]) =>
patterns.some((pattern) => pattern.test(text)),
},
media: {
saveMediaBuffer: vi.fn(),
},
},
config: {
loadConfig: () => ({}),
},
state: {
resolveStateDir: () => "/tmp",
},
} as unknown as PluginRuntime);
const recordInboundSession = vi.fn().mockResolvedValue(undefined);
const formatAgentEnvelope = vi
.fn()
.mockImplementation((params: { body: string }) => params.body);
const core = {
channel: {
pairing: {
readAllowFromStore: vi.fn().mockResolvedValue([]),
upsertPairingRequest: vi.fn().mockResolvedValue(undefined),
buildPairingReply: vi.fn().mockReturnValue("pairing"),
},
routing: {
resolveAgentRoute: vi.fn().mockReturnValue({
agentId: "main",
accountId: undefined,
sessionKey: "agent:main:matrix:channel:!room:example.org",
mainSessionKey: "agent:main:main",
}),
},
session: {
resolveStorePath: vi.fn().mockReturnValue("/tmp/openclaw-test-session.json"),
readSessionUpdatedAt: vi.fn().mockReturnValue(undefined),
recordInboundSession,
},
reply: {
resolveEnvelopeFormatOptions: vi.fn().mockReturnValue({}),
formatAgentEnvelope,
finalizeInboundContext: vi.fn().mockImplementation((ctx: Record<string, unknown>) => ctx),
createReplyDispatcherWithTyping: vi.fn().mockReturnValue({
dispatcher: {},
replyOptions: {},
markDispatchIdle: vi.fn(),
}),
resolveHumanDelayConfig: vi.fn().mockReturnValue(undefined),
dispatchReplyFromConfig: vi
.fn()
.mockResolvedValue({ queuedFinal: false, counts: { final: 0, block: 0, tool: 0 } }),
},
commands: {
shouldHandleTextCommands: vi.fn().mockReturnValue(true),
},
text: {
hasControlCommand: vi.fn().mockReturnValue(false),
resolveMarkdownTableMode: vi.fn().mockReturnValue("code"),
},
reactions: {
shouldAckReaction: vi.fn().mockReturnValue(false),
},
},
system: {
enqueueSystemEvent: vi.fn(),
},
} as unknown as PluginRuntime;
const client = {
getUserId: vi.fn().mockResolvedValue("@bot:matrix.example.org"),
getEvent: vi.fn().mockResolvedValue({
event_id: "$thread-root",
sender: "@gum:matrix.example.org",
type: "m.room.message",
origin_server_ts: 123,
content: {
msgtype: "m.image",
body: "photo.jpg",
},
}),
} as unknown as MatrixClient;
const handler = createMatrixRoomMessageHandler({
client,
core,
cfg: {},
accountId: "ops",
runtime: { error: vi.fn() } as unknown as RuntimeEnv,
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() } as unknown as RuntimeLogger,
logVerboseMessage: vi.fn(),
allowFrom: [],
groupAllowFrom: [],
roomsConfig: undefined,
mentionRegexes: [],
groupPolicy: "open",
replyToMode: "first",
threadReplies: "inbound",
dmEnabled: true,
dmPolicy: "open",
textLimit: 4000,
mediaMaxBytes: 5 * 1024 * 1024,
startupMs: Date.now() - 120_000,
startupGraceMs: 60_000,
directTracker: {
isDirectMessage: vi.fn().mockResolvedValue(true),
},
getRoomInfo: vi.fn().mockResolvedValue({
name: "Media Room",
canonicalAlias: "#media:example.org",
altAliases: [],
}),
getMemberDisplayName: vi.fn().mockResolvedValue("Gum"),
needsRoomAliasesForConfig: false,
});
await handler("!room:example.org", {
type: EventType.RoomMessage,
event_id: "$reply",
sender: "@bu:matrix.example.org",
origin_server_ts: Date.now(),
content: {
msgtype: "m.text",
body: "replying",
"m.mentions": { user_ids: ["@bot:matrix.example.org"] },
"m.relates_to": {
rel_type: "m.thread",
event_id: "$thread-root",
},
},
} as MatrixRawEvent);
expect(formatAgentEnvelope).toHaveBeenCalledWith(
expect.objectContaining({
body: expect.stringContaining("replying"),
}),
);
expect(recordInboundSession).toHaveBeenCalledWith(
expect.objectContaining({
ctx: expect.objectContaining({
ThreadStarterBody: expect.stringContaining("[matrix image attachment]"),
}),
}),
);
});
});

View File

@ -13,6 +13,7 @@ import {
type RuntimeLogger,
} from "openclaw/plugin-sdk/matrix";
import type { CoreConfig, MatrixRoomConfig, ReplyToMode } from "../../types.js";
import { formatMatrixMediaUnavailableText } from "../media-text.js";
import { fetchMatrixPollSnapshot } from "../poll-summary.js";
import {
formatPollAsText,
@ -102,6 +103,27 @@ function resolveMatrixMentionPrecheckText(params: {
return "";
}
function resolveMatrixInboundBodyText(params: {
rawBody: string;
filename?: string;
mediaPlaceholder?: string;
msgtype?: string;
hadMediaUrl: boolean;
mediaDownloadFailed: boolean;
}): string {
if (params.mediaPlaceholder) {
return params.rawBody || params.mediaPlaceholder;
}
if (!params.mediaDownloadFailed || !params.hadMediaUrl) {
return params.rawBody;
}
return formatMatrixMediaUnavailableText({
body: params.rawBody,
filename: params.filename,
msgtype: params.msgtype,
});
}
export function createMatrixRoomMessageHandler(params: MatrixMonitorHandlerParams) {
const {
client,
@ -519,6 +541,7 @@ export function createMatrixRoomMessageHandler(params: MatrixMonitorHandlerParam
contentType?: string;
placeholder: string;
} | null = null;
let mediaDownloadFailed = false;
const finalContentUrl =
"url" in content && typeof content.url === "string" ? content.url : undefined;
const finalContentFile =
@ -543,13 +566,31 @@ export function createMatrixRoomMessageHandler(params: MatrixMonitorHandlerParam
file: finalContentFile,
});
} catch (err) {
logVerboseMessage(`matrix: media download failed: ${String(err)}`);
mediaDownloadFailed = true;
const errorText = err instanceof Error ? err.message : String(err);
logVerboseMessage(
`matrix: media download failed room=${roomId} id=${event.event_id ?? "unknown"} type=${content.msgtype} error=${errorText}`,
);
logger.warn("matrix media download failed", {
roomId,
eventId: event.event_id,
msgtype: content.msgtype,
encrypted: Boolean(finalContentFile),
error: errorText,
});
}
}
const rawBody =
locationPayload?.text ?? (typeof content.body === "string" ? content.body.trim() : "");
const bodyText = rawBody || media?.placeholder || "";
const bodyText = resolveMatrixInboundBodyText({
rawBody,
filename: typeof content.filename === "string" ? content.filename : undefined,
mediaPlaceholder: media?.placeholder,
msgtype: content.msgtype,
hadMediaUrl: Boolean(finalMediaUrl),
mediaDownloadFailed,
});
if (!bodyText) {
return;
}

View File

@ -21,6 +21,21 @@ describe("matrix thread context", () => {
).toBe("Thread starter body");
});
it("marks media-only thread starter events instead of returning bare filenames", () => {
expect(
summarizeMatrixThreadStarterEvent({
event_id: "$root",
sender: "@alice:example.org",
type: "m.room.message",
origin_server_ts: Date.now(),
content: {
msgtype: "m.image",
body: "photo.jpg",
},
} as MatrixRawEvent),
).toBe("[matrix image attachment]");
});
it("resolves and caches thread starter context", async () => {
const getEvent = vi.fn(async () => ({
event_id: "$root",

View File

@ -1,3 +1,8 @@
import {
formatMatrixMessageText,
resolveMatrixMessageAttachment,
resolveMatrixMessageBody,
} from "../media-text.js";
import type { MatrixClient } from "../sdk.js";
import type { MatrixRawEvent } from "./types.js";
@ -24,8 +29,19 @@ function truncateThreadStarterBody(value: string): string {
}
export function summarizeMatrixThreadStarterEvent(event: MatrixRawEvent): string | undefined {
const content = event.content as { body?: unknown; msgtype?: unknown };
const body = trimMaybeString(content.body);
const content = event.content as { body?: unknown; filename?: unknown; msgtype?: unknown };
const body = formatMatrixMessageText({
body: resolveMatrixMessageBody({
body: trimMaybeString(content.body),
filename: trimMaybeString(content.filename),
msgtype: trimMaybeString(content.msgtype),
}),
attachment: resolveMatrixMessageAttachment({
body: trimMaybeString(content.body),
filename: trimMaybeString(content.filename),
msgtype: trimMaybeString(content.msgtype),
}),
});
if (body) {
return truncateThreadStarterBody(body);
}