diff --git a/extensions/qa-channel/api.ts b/extensions/qa-channel/api.ts index d3f235c7226..813fe53ae3a 100644 --- a/extensions/qa-channel/api.ts +++ b/extensions/qa-channel/api.ts @@ -2,3 +2,4 @@ export * from "./src/accounts.js"; export * from "./src/channel.js"; export * from "./src/channel-actions.js"; export * from "./src/runtime.js"; +export * from "./test-api.js"; diff --git a/extensions/qa-lab/api.ts b/extensions/qa-lab/api.ts new file mode 100644 index 00000000000..5f2fd7473e6 --- /dev/null +++ b/extensions/qa-lab/api.ts @@ -0,0 +1,10 @@ +export * from "./src/bus-queries.js"; +export * from "./src/bus-server.js"; +export * from "./src/bus-state.js"; +export * from "./src/bus-waiters.js"; +export * from "./src/harness-runtime.js"; +export * from "./src/lab-server.js"; +export * from "./src/report.js"; +export * from "./src/scenario.js"; +export * from "./src/self-check-scenario.js"; +export * from "./src/self-check.js"; diff --git a/extensions/qa-lab/index.ts b/extensions/qa-lab/index.ts new file mode 100644 index 00000000000..4ade2a4e87f --- /dev/null +++ b/extensions/qa-lab/index.ts @@ -0,0 +1,24 @@ +import { definePluginEntry } from "./runtime-api.js"; +import { registerQaLabCli } from "./src/cli.js"; + +export default definePluginEntry({ + id: "qa-lab", + name: "QA Lab", + description: "Private QA automation harness and debugger UI", + register(api) { + api.registerCli( + async ({ program }) => { + registerQaLabCli(program); + }, + { + descriptors: [ + { + name: "qa", + description: "Run QA scenarios and launch the private QA debugger UI", + hasSubcommands: true, + }, + ], + }, + ); + }, +}); diff --git a/extensions/qa-lab/openclaw.plugin.json b/extensions/qa-lab/openclaw.plugin.json new file mode 100644 index 00000000000..3f61b2435ce --- /dev/null +++ b/extensions/qa-lab/openclaw.plugin.json @@ -0,0 +1,8 @@ +{ + "id": "qa-lab", + "configSchema": { + "type": "object", + "additionalProperties": false, + "properties": {} + } +} diff --git a/extensions/qa-lab/package.json b/extensions/qa-lab/package.json new file mode 100644 index 00000000000..8c236a65e65 --- /dev/null +++ b/extensions/qa-lab/package.json @@ -0,0 +1,31 @@ +{ + "name": "@openclaw/qa-lab", + "version": "2026.4.4", + "private": true, + "description": "OpenClaw QA lab plugin with private debugger UI and scenario runner", + "type": "module", + "devDependencies": { + "openclaw": "workspace:*" + }, + "peerDependencies": { + "openclaw": ">=2026.4.4" + }, + "peerDependenciesMeta": { + "openclaw": { + "optional": true + } + }, + "openclaw": { + "extensions": [ + "./index.ts" + ], + "install": { + "npmSpec": "@openclaw/qa-lab", + "defaultChoice": "npm", + "minHostVersion": ">=2026.4.4" + }, + "compat": { + "pluginApi": ">=2026.4.4" + } + } +} diff --git a/extensions/qa-lab/runtime-api.ts b/extensions/qa-lab/runtime-api.ts new file mode 100644 index 00000000000..801051438fb --- /dev/null +++ b/extensions/qa-lab/runtime-api.ts @@ -0,0 +1 @@ +export * from "./src/runtime-api.js"; diff --git a/extensions/qa-lab/src/bus-queries.ts b/extensions/qa-lab/src/bus-queries.ts new file mode 100644 index 00000000000..ec6d449dab3 --- /dev/null +++ b/extensions/qa-lab/src/bus-queries.ts @@ -0,0 +1,137 @@ +import type { + QaBusConversation, + QaBusEvent, + QaBusMessage, + QaBusPollInput, + QaBusPollResult, + QaBusReadMessageInput, + QaBusSearchMessagesInput, + QaBusStateSnapshot, + QaBusThread, +} from "./runtime-api.js"; + +export const DEFAULT_ACCOUNT_ID = "default"; + +export function normalizeAccountId(raw?: string): string { + const trimmed = raw?.trim(); + return trimmed || DEFAULT_ACCOUNT_ID; +} + +export function normalizeConversationFromTarget(target: string): { + conversation: QaBusConversation; + threadId?: string; +} { + const trimmed = target.trim(); + if (trimmed.startsWith("thread:")) { + const rest = trimmed.slice("thread:".length); + const slash = rest.indexOf("/"); + if (slash > 0) { + return { + conversation: { id: rest.slice(0, slash), kind: "channel" }, + threadId: rest.slice(slash + 1), + }; + } + } + if (trimmed.startsWith("channel:")) { + return { + conversation: { id: trimmed.slice("channel:".length), kind: "channel" }, + }; + } + if (trimmed.startsWith("dm:")) { + return { + conversation: { id: trimmed.slice("dm:".length), kind: "direct" }, + }; + } + return { + conversation: { id: trimmed, kind: "direct" }, + }; +} + +export function cloneMessage(message: QaBusMessage): QaBusMessage { + return { + ...message, + conversation: { ...message.conversation }, + reactions: message.reactions.map((reaction) => ({ ...reaction })), + }; +} + +export function cloneEvent(event: QaBusEvent): QaBusEvent { + switch (event.kind) { + case "inbound-message": + case "outbound-message": + case "message-edited": + case "message-deleted": + case "reaction-added": + return { ...event, message: cloneMessage(event.message) }; + case "thread-created": + return { ...event, thread: { ...event.thread } }; + } +} + +export function buildQaBusSnapshot(params: { + cursor: number; + conversations: Map; + threads: Map; + messages: Map; + events: QaBusEvent[]; +}): QaBusStateSnapshot { + return { + cursor: params.cursor, + conversations: Array.from(params.conversations.values()).map((conversation) => ({ + ...conversation, + })), + threads: Array.from(params.threads.values()).map((thread) => ({ ...thread })), + messages: Array.from(params.messages.values()).map((message) => cloneMessage(message)), + events: params.events.map((event) => cloneEvent(event)), + }; +} + +export function readQaBusMessage(params: { + messages: Map; + input: QaBusReadMessageInput; +}) { + const message = params.messages.get(params.input.messageId); + if (!message) { + throw new Error(`qa-bus message not found: ${params.input.messageId}`); + } + return cloneMessage(message); +} + +export function searchQaBusMessages(params: { + messages: Map; + input: QaBusSearchMessagesInput; +}) { + const accountId = normalizeAccountId(params.input.accountId); + const limit = Math.max(1, Math.min(params.input.limit ?? 20, 100)); + const query = params.input.query?.trim().toLowerCase(); + return Array.from(params.messages.values()) + .filter((message) => message.accountId === accountId) + .filter((message) => + params.input.conversationId ? message.conversation.id === params.input.conversationId : true, + ) + .filter((message) => + params.input.threadId ? message.threadId === params.input.threadId : true, + ) + .filter((message) => (query ? message.text.toLowerCase().includes(query) : true)) + .slice(-limit) + .map((message) => cloneMessage(message)); +} + +export function pollQaBusEvents(params: { + events: QaBusEvent[]; + cursor: number; + input?: QaBusPollInput; +}): QaBusPollResult { + const accountId = normalizeAccountId(params.input?.accountId); + const startCursor = params.input?.cursor ?? 0; + const effectiveStartCursor = params.cursor < startCursor ? 0 : startCursor; + const limit = Math.max(1, Math.min(params.input?.limit ?? 100, 500)); + const matches = params.events + .filter((event) => event.accountId === accountId && event.cursor > effectiveStartCursor) + .slice(0, limit) + .map((event) => cloneEvent(event)); + return { + cursor: params.cursor, + events: matches, + }; +} diff --git a/extensions/qa-lab/src/bus-server.ts b/extensions/qa-lab/src/bus-server.ts new file mode 100644 index 00000000000..32580f4ab4f --- /dev/null +++ b/extensions/qa-lab/src/bus-server.ts @@ -0,0 +1,179 @@ +import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; +import type { QaBusState } from "./bus-state.js"; +import type { + QaBusCreateThreadInput, + QaBusDeleteMessageInput, + QaBusEditMessageInput, + QaBusInboundMessageInput, + QaBusOutboundMessageInput, + QaBusPollInput, + QaBusReactToMessageInput, + QaBusReadMessageInput, + QaBusSearchMessagesInput, + QaBusWaitForInput, +} from "./runtime-api.js"; + +async function readJson(req: IncomingMessage): Promise { + const chunks: Buffer[] = []; + for await (const chunk of req) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + const text = Buffer.concat(chunks).toString("utf8").trim(); + return text ? (JSON.parse(text) as unknown) : {}; +} + +export function writeJson(res: ServerResponse, statusCode: number, body: unknown) { + const payload = JSON.stringify(body); + res.writeHead(statusCode, { + "content-type": "application/json; charset=utf-8", + "content-length": Buffer.byteLength(payload), + }); + res.end(payload); +} + +export function writeError(res: ServerResponse, statusCode: number, error: unknown) { + writeJson(res, statusCode, { + error: error instanceof Error ? error.message : String(error), + }); +} + +export async function handleQaBusRequest(params: { + req: IncomingMessage; + res: ServerResponse; + state: QaBusState; +}): Promise { + const method = params.req.method ?? "GET"; + const url = new URL(params.req.url ?? "/", "http://127.0.0.1"); + + if (method === "GET" && url.pathname === "/health") { + writeJson(params.res, 200, { ok: true }); + return true; + } + + if (method === "GET" && url.pathname === "/v1/state") { + writeJson(params.res, 200, params.state.getSnapshot()); + return true; + } + + if (!url.pathname.startsWith("/v1/")) { + return false; + } + + if (method !== "POST") { + writeError(params.res, 405, "method not allowed"); + return true; + } + + const body = (await readJson(params.req)) as Record; + + try { + switch (url.pathname) { + case "/v1/reset": + params.state.reset(); + writeJson(params.res, 200, { ok: true }); + return true; + case "/v1/inbound/message": + writeJson(params.res, 200, { + message: params.state.addInboundMessage(body as unknown as QaBusInboundMessageInput), + }); + return true; + case "/v1/outbound/message": + writeJson(params.res, 200, { + message: params.state.addOutboundMessage(body as unknown as QaBusOutboundMessageInput), + }); + return true; + case "/v1/actions/thread-create": + writeJson(params.res, 200, { + thread: params.state.createThread(body as unknown as QaBusCreateThreadInput), + }); + return true; + case "/v1/actions/react": + writeJson(params.res, 200, { + message: params.state.reactToMessage(body as unknown as QaBusReactToMessageInput), + }); + return true; + case "/v1/actions/edit": + writeJson(params.res, 200, { + message: params.state.editMessage(body as unknown as QaBusEditMessageInput), + }); + return true; + case "/v1/actions/delete": + writeJson(params.res, 200, { + message: params.state.deleteMessage(body as unknown as QaBusDeleteMessageInput), + }); + return true; + case "/v1/actions/read": + writeJson(params.res, 200, { + message: params.state.readMessage(body as unknown as QaBusReadMessageInput), + }); + return true; + case "/v1/actions/search": + writeJson(params.res, 200, { + messages: params.state.searchMessages(body as unknown as QaBusSearchMessagesInput), + }); + return true; + case "/v1/poll": { + const input = body as unknown as QaBusPollInput; + const timeoutMs = Math.max(0, Math.min(input.timeoutMs ?? 0, 30_000)); + const initial = params.state.poll(input); + if (initial.events.length > 0 || timeoutMs === 0) { + writeJson(params.res, 200, initial); + return true; + } + try { + await params.state.waitFor({ + kind: "event-kind", + eventKind: "inbound-message", + timeoutMs, + }); + } catch { + // timeout ok for long-poll + } + writeJson(params.res, 200, params.state.poll(input)); + return true; + } + case "/v1/wait": + writeJson(params.res, 200, { + match: await params.state.waitFor(body as unknown as QaBusWaitForInput), + }); + return true; + default: + writeError(params.res, 404, "not found"); + return true; + } + } catch (error) { + writeError(params.res, 400, error); + return true; + } +} + +export function createQaBusServer(state: QaBusState): Server { + return createServer(async (req, res) => { + const handled = await handleQaBusRequest({ req, res, state }); + if (!handled) { + writeError(res, 404, "not found"); + } + }); +} + +export async function startQaBusServer(params: { state: QaBusState; port?: number }) { + const server = createQaBusServer(params.state); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(params.port ?? 0, "127.0.0.1", () => resolve()); + }); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("qa-bus failed to bind"); + } + return { + server, + port: address.port, + baseUrl: `http://127.0.0.1:${address.port}`, + async stop() { + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + }, + }; +} diff --git a/extensions/qa-lab/src/bus-state.ts b/extensions/qa-lab/src/bus-state.ts new file mode 100644 index 00000000000..6beddca6734 --- /dev/null +++ b/extensions/qa-lab/src/bus-state.ts @@ -0,0 +1,257 @@ +import { randomUUID } from "node:crypto"; +import { + buildQaBusSnapshot, + cloneMessage, + normalizeAccountId, + normalizeConversationFromTarget, + pollQaBusEvents, + readQaBusMessage, + searchQaBusMessages, +} from "./bus-queries.js"; +import { createQaBusWaiterStore } from "./bus-waiters.js"; +import type { + QaBusConversation, + QaBusCreateThreadInput, + QaBusDeleteMessageInput, + QaBusEditMessageInput, + QaBusEvent, + QaBusInboundMessageInput, + QaBusMessage, + QaBusOutboundMessageInput, + QaBusPollInput, + QaBusReadMessageInput, + QaBusReactToMessageInput, + QaBusSearchMessagesInput, + QaBusThread, + QaBusWaitForInput, +} from "./runtime-api.js"; + +const DEFAULT_BOT_ID = "openclaw"; +const DEFAULT_BOT_NAME = "OpenClaw QA"; + +type QaBusEventSeed = + | Omit, "cursor"> + | Omit, "cursor"> + | Omit, "cursor"> + | Omit, "cursor"> + | Omit, "cursor"> + | Omit, "cursor">; + +export function createQaBusState() { + const conversations = new Map(); + const threads = new Map(); + const messages = new Map(); + const events: QaBusEvent[] = []; + let cursor = 0; + const waiters = createQaBusWaiterStore(() => + buildQaBusSnapshot({ + cursor, + conversations, + threads, + messages, + events, + }), + ); + + const pushEvent = (event: QaBusEventSeed | ((cursor: number) => QaBusEventSeed)): QaBusEvent => { + cursor += 1; + const next = typeof event === "function" ? event(cursor) : event; + const finalized = { cursor, ...next } as QaBusEvent; + events.push(finalized); + waiters.settle(); + return finalized; + }; + + const ensureConversation = (conversation: QaBusConversation): QaBusConversation => { + const existing = conversations.get(conversation.id); + if (existing) { + if (!existing.title && conversation.title) { + existing.title = conversation.title; + } + return existing; + } + const created = { ...conversation }; + conversations.set(created.id, created); + return created; + }; + + const createMessage = (params: { + direction: QaBusMessage["direction"]; + accountId: string; + conversation: QaBusConversation; + senderId: string; + senderName?: string; + text: string; + timestamp?: number; + threadId?: string; + threadTitle?: string; + replyToId?: string; + }): QaBusMessage => { + const conversation = ensureConversation(params.conversation); + const message: QaBusMessage = { + id: randomUUID(), + accountId: params.accountId, + direction: params.direction, + conversation, + senderId: params.senderId, + senderName: params.senderName, + text: params.text, + timestamp: params.timestamp ?? Date.now(), + threadId: params.threadId, + threadTitle: params.threadTitle, + replyToId: params.replyToId, + reactions: [], + }; + messages.set(message.id, message); + return message; + }; + + return { + reset() { + conversations.clear(); + threads.clear(); + messages.clear(); + events.length = 0; + // Keep the cursor monotonic across resets so long-poll clients do not + // miss fresh events after the bus is cleared mid-session. + waiters.reset(); + }, + getSnapshot() { + return buildQaBusSnapshot({ + cursor, + conversations, + threads, + messages, + events, + }); + }, + addInboundMessage(input: QaBusInboundMessageInput) { + const accountId = normalizeAccountId(input.accountId); + const message = createMessage({ + direction: "inbound", + accountId, + conversation: input.conversation, + senderId: input.senderId, + senderName: input.senderName, + text: input.text, + timestamp: input.timestamp, + threadId: input.threadId, + threadTitle: input.threadTitle, + replyToId: input.replyToId, + }); + pushEvent({ + kind: "inbound-message", + accountId, + message: cloneMessage(message), + }); + return cloneMessage(message); + }, + addOutboundMessage(input: QaBusOutboundMessageInput) { + const accountId = normalizeAccountId(input.accountId); + const { conversation, threadId } = normalizeConversationFromTarget(input.to); + const message = createMessage({ + direction: "outbound", + accountId, + conversation, + senderId: input.senderId?.trim() || DEFAULT_BOT_ID, + senderName: input.senderName?.trim() || DEFAULT_BOT_NAME, + text: input.text, + timestamp: input.timestamp, + threadId: input.threadId ?? threadId, + replyToId: input.replyToId, + }); + pushEvent({ + kind: "outbound-message", + accountId, + message: cloneMessage(message), + }); + return cloneMessage(message); + }, + createThread(input: QaBusCreateThreadInput) { + const accountId = normalizeAccountId(input.accountId); + const thread: QaBusThread = { + id: `thread-${randomUUID()}`, + accountId, + conversationId: input.conversationId, + title: input.title, + createdAt: input.timestamp ?? Date.now(), + createdBy: input.createdBy?.trim() || DEFAULT_BOT_ID, + }; + threads.set(thread.id, thread); + ensureConversation({ + id: input.conversationId, + kind: "channel", + }); + pushEvent({ + kind: "thread-created", + accountId, + thread: { ...thread }, + }); + return { ...thread }; + }, + reactToMessage(input: QaBusReactToMessageInput) { + const accountId = normalizeAccountId(input.accountId); + const message = messages.get(input.messageId); + if (!message) { + throw new Error(`qa-bus message not found: ${input.messageId}`); + } + const reaction = { + emoji: input.emoji, + senderId: input.senderId?.trim() || DEFAULT_BOT_ID, + timestamp: input.timestamp ?? Date.now(), + }; + message.reactions.push(reaction); + pushEvent({ + kind: "reaction-added", + accountId, + message: cloneMessage(message), + emoji: reaction.emoji, + senderId: reaction.senderId, + }); + return cloneMessage(message); + }, + editMessage(input: QaBusEditMessageInput) { + const accountId = normalizeAccountId(input.accountId); + const message = messages.get(input.messageId); + if (!message) { + throw new Error(`qa-bus message not found: ${input.messageId}`); + } + message.text = input.text; + message.editedAt = input.timestamp ?? Date.now(); + pushEvent({ + kind: "message-edited", + accountId, + message: cloneMessage(message), + }); + return cloneMessage(message); + }, + deleteMessage(input: QaBusDeleteMessageInput) { + const accountId = normalizeAccountId(input.accountId); + const message = messages.get(input.messageId); + if (!message) { + throw new Error(`qa-bus message not found: ${input.messageId}`); + } + message.deleted = true; + pushEvent({ + kind: "message-deleted", + accountId, + message: cloneMessage(message), + }); + return cloneMessage(message); + }, + readMessage(input: QaBusReadMessageInput) { + return readQaBusMessage({ messages, input }); + }, + searchMessages(input: QaBusSearchMessagesInput) { + return searchQaBusMessages({ messages, input }); + }, + poll(input: QaBusPollInput = {}) { + return pollQaBusEvents({ events, cursor, input }); + }, + async waitFor(input: QaBusWaitForInput) { + return await waiters.waitFor(input); + }, + }; +} + +export type QaBusState = ReturnType; diff --git a/extensions/qa-lab/src/bus-waiters.ts b/extensions/qa-lab/src/bus-waiters.ts new file mode 100644 index 00000000000..1d6c15671f6 --- /dev/null +++ b/extensions/qa-lab/src/bus-waiters.ts @@ -0,0 +1,87 @@ +import type { + QaBusEvent, + QaBusMessage, + QaBusStateSnapshot, + QaBusThread, + QaBusWaitForInput, +} from "./runtime-api.js"; + +export const DEFAULT_WAIT_TIMEOUT_MS = 5_000; + +export type QaBusWaitMatch = QaBusEvent | QaBusMessage | QaBusThread; + +type Waiter = { + resolve: (event: QaBusWaitMatch) => void; + reject: (error: Error) => void; + timer: NodeJS.Timeout; + matcher: (snapshot: QaBusStateSnapshot) => QaBusWaitMatch | null; +}; + +function createQaBusMatcher( + input: QaBusWaitForInput, +): (snapshot: QaBusStateSnapshot) => QaBusWaitMatch | null { + return (snapshot) => { + if (input.kind === "event-kind") { + return snapshot.events.find((event) => event.kind === input.eventKind) ?? null; + } + if (input.kind === "thread-id") { + return snapshot.threads.find((thread) => thread.id === input.threadId) ?? null; + } + return ( + snapshot.messages.find( + (message) => + (!input.direction || message.direction === input.direction) && + message.text.includes(input.textIncludes), + ) ?? null + ); + }; +} + +export function createQaBusWaiterStore(getSnapshot: () => QaBusStateSnapshot) { + const waiters = new Set(); + + return { + reset(reason = "qa-bus reset") { + for (const waiter of waiters) { + clearTimeout(waiter.timer); + waiter.reject(new Error(reason)); + } + waiters.clear(); + }, + settle() { + if (waiters.size === 0) { + return; + } + const snapshot = getSnapshot(); + for (const waiter of Array.from(waiters)) { + const match = waiter.matcher(snapshot); + if (!match) { + continue; + } + clearTimeout(waiter.timer); + waiters.delete(waiter); + waiter.resolve(match); + } + }, + async waitFor(input: QaBusWaitForInput) { + const matcher = createQaBusMatcher(input); + const immediate = matcher(getSnapshot()); + if (immediate) { + return immediate; + } + return await new Promise((resolve, reject) => { + const timeoutMs = input.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS; + const waiter: Waiter = { + resolve, + reject, + matcher, + timer: setTimeout(() => { + waiters.delete(waiter); + reject(new Error(`qa-bus wait timeout after ${timeoutMs}ms`)); + }, timeoutMs), + }; + waiters.add(waiter); + }); + }, + }; +} diff --git a/extensions/qa-lab/src/cli.runtime.ts b/extensions/qa-lab/src/cli.runtime.ts new file mode 100644 index 00000000000..fef5043ff44 --- /dev/null +++ b/extensions/qa-lab/src/cli.runtime.ts @@ -0,0 +1,37 @@ +import { startQaLabServer } from "./lab-server.js"; + +export async function runQaLabSelfCheckCommand(opts: { output?: string }) { + const server = await startQaLabServer({ + outputPath: opts.output, + }); + try { + const result = await server.runSelfCheck(); + process.stdout.write(`QA self-check report: ${result.outputPath}\n`); + } finally { + await server.stop(); + } +} + +export async function runQaLabUiCommand(opts: { host?: string; port?: number }) { + const server = await startQaLabServer({ + host: opts.host, + port: Number.isFinite(opts.port) ? opts.port : undefined, + }); + process.stdout.write(`QA Lab UI: ${server.baseUrl}\n`); + process.stdout.write("Press Ctrl+C to stop.\n"); + + const shutdown = async () => { + process.off("SIGINT", onSignal); + process.off("SIGTERM", onSignal); + await server.stop(); + process.exit(0); + }; + + const onSignal = () => { + void shutdown(); + }; + + process.on("SIGINT", onSignal); + process.on("SIGTERM", onSignal); + await new Promise(() => undefined); +} diff --git a/extensions/qa-lab/src/cli.ts b/extensions/qa-lab/src/cli.ts new file mode 100644 index 00000000000..0600d9fc1e4 --- /dev/null +++ b/extensions/qa-lab/src/cli.ts @@ -0,0 +1,41 @@ +import type { Command } from "commander"; + +type QaLabCliRuntime = typeof import("./cli.runtime.js"); + +let qaLabCliRuntimePromise: Promise | null = null; + +async function loadQaLabCliRuntime(): Promise { + qaLabCliRuntimePromise ??= import("./cli.runtime.js"); + return await qaLabCliRuntimePromise; +} + +async function runQaSelfCheck(opts: { output?: string }) { + const runtime = await loadQaLabCliRuntime(); + await runtime.runQaLabSelfCheckCommand(opts); +} + +async function runQaUi(opts: { host?: string; port?: number }) { + const runtime = await loadQaLabCliRuntime(); + await runtime.runQaLabUiCommand(opts); +} + +export function registerQaLabCli(program: Command) { + const qa = program + .command("qa") + .description("Run private QA automation flows and launch the QA debugger"); + + qa.command("run") + .description("Run the bundled QA self-check and write a Markdown report") + .option("--output ", "Report output path") + .action(async (opts: { output?: string }) => { + await runQaSelfCheck(opts); + }); + + qa.command("ui") + .description("Start the private QA debugger UI and local QA bus") + .option("--host ", "Bind host", "127.0.0.1") + .option("--port ", "Bind port", (value: string) => Number(value)) + .action(async (opts: { host?: string; port?: number }) => { + await runQaUi(opts); + }); +} diff --git a/extensions/qa-lab/src/extract-tool-payload.ts b/extensions/qa-lab/src/extract-tool-payload.ts new file mode 100644 index 00000000000..29ceee6d9dc --- /dev/null +++ b/extensions/qa-lab/src/extract-tool-payload.ts @@ -0,0 +1,31 @@ +type ResultWithDetails = { + details?: unknown; + content?: unknown; +}; + +export function extractQaToolPayload(result: ResultWithDetails | null | undefined): unknown { + if (!result) { + return undefined; + } + if (result.details !== undefined) { + return result.details; + } + const textBlock = Array.isArray(result.content) + ? result.content.find( + (block) => + block && + typeof block === "object" && + (block as { type?: unknown }).type === "text" && + typeof (block as { text?: unknown }).text === "string", + ) + : undefined; + const text = (textBlock as { text?: string } | undefined)?.text; + if (!text) { + return result.content ?? result; + } + try { + return JSON.parse(text); + } catch { + return text; + } +} diff --git a/extensions/qa-lab/src/harness-runtime.ts b/extensions/qa-lab/src/harness-runtime.ts new file mode 100644 index 00000000000..a32a26bfdfc --- /dev/null +++ b/extensions/qa-lab/src/harness-runtime.ts @@ -0,0 +1,75 @@ +import type { PluginRuntime } from "openclaw/plugin-sdk/core"; + +type SessionRecord = { + sessionKey: string; + body: string; +}; + +export function createQaRunnerRuntime(): PluginRuntime { + const sessions = new Map(); + return { + channel: { + routing: { + resolveAgentRoute({ + accountId, + peer, + }: { + accountId?: string | null; + peer?: { kind?: string; id?: string } | null; + }) { + return { + agentId: "qa-agent", + accountId: accountId ?? "default", + sessionKey: `qa-agent:${peer?.kind ?? "direct"}:${peer?.id ?? "default"}`, + mainSessionKey: "qa-agent:main", + lastRoutePolicy: "session", + matchedBy: "default", + channel: "qa-channel", + }; + }, + }, + session: { + resolveStorePath(_store: string | undefined, { agentId }: { agentId: string }) { + return agentId; + }, + readSessionUpdatedAt({ sessionKey }: { sessionKey: string }) { + return sessions.has(sessionKey) ? Date.now() : undefined; + }, + recordInboundSession({ + sessionKey, + ctx, + }: { + sessionKey: string; + ctx: { BodyForAgent?: string; Body?: string }; + }) { + sessions.set(sessionKey, { + sessionKey, + body: String(ctx.BodyForAgent ?? ctx.Body ?? ""), + }); + }, + }, + reply: { + resolveEnvelopeFormatOptions() { + return {}; + }, + formatAgentEnvelope({ body }: { body: string }) { + return body; + }, + finalizeInboundContext(ctx: Record) { + return ctx as typeof ctx & { CommandAuthorized: boolean }; + }, + async dispatchReplyWithBufferedBlockDispatcher({ + ctx, + dispatcherOptions, + }: { + ctx: { BodyForAgent?: string; Body?: string }; + dispatcherOptions: { deliver: (payload: { text: string }) => Promise }; + }) { + await dispatcherOptions.deliver({ + text: `qa-echo: ${String(ctx.BodyForAgent ?? ctx.Body ?? "")}`, + }); + }, + }, + }, + } as unknown as PluginRuntime; +} diff --git a/extensions/qa-lab/src/lab-server.test.ts b/extensions/qa-lab/src/lab-server.test.ts new file mode 100644 index 00000000000..9d5d6413229 --- /dev/null +++ b/extensions/qa-lab/src/lab-server.test.ts @@ -0,0 +1,67 @@ +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { startQaLabServer } from "./lab-server.js"; + +const cleanups: Array<() => Promise> = []; + +afterEach(async () => { + while (cleanups.length > 0) { + await cleanups.pop()?.(); + } +}); + +describe("qa-lab server", () => { + it("serves bootstrap state and writes a self-check report", async () => { + const tempDir = await mkdtemp(path.join(os.tmpdir(), "qa-lab-test-")); + cleanups.push(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + const outputPath = path.join(tempDir, "self-check.md"); + + const lab = await startQaLabServer({ + host: "127.0.0.1", + port: 0, + outputPath, + }); + cleanups.push(async () => { + await lab.stop(); + }); + + const bootstrapResponse = await fetch(`${lab.baseUrl}/api/bootstrap`); + expect(bootstrapResponse.status).toBe(200); + const bootstrap = (await bootstrapResponse.json()) as { + defaults: { conversationId: string; senderId: string }; + }; + expect(bootstrap.defaults.conversationId).toBe("alice"); + expect(bootstrap.defaults.senderId).toBe("alice"); + + const messageResponse = await fetch(`${lab.baseUrl}/api/inbound/message`, { + method: "POST", + headers: { + "content-type": "application/json", + }, + body: JSON.stringify({ + conversation: { id: "bob", kind: "direct" }, + senderId: "bob", + senderName: "Bob", + text: "hello from test", + }), + }); + expect(messageResponse.status).toBe(200); + + const stateResponse = await fetch(`${lab.baseUrl}/api/state`); + expect(stateResponse.status).toBe(200); + const snapshot = (await stateResponse.json()) as { + messages: Array<{ direction: string; text: string }>; + }; + expect(snapshot.messages.some((message) => message.text === "hello from test")).toBe(true); + + const result = await lab.runSelfCheck(); + expect(result.scenarioResult.status).toBe("pass"); + const markdown = await readFile(outputPath, "utf8"); + expect(markdown).toContain("Synthetic Slack-class roundtrip"); + expect(markdown).toContain("- Status: pass"); + }); +}); diff --git a/extensions/qa-lab/src/lab-server.ts b/extensions/qa-lab/src/lab-server.ts new file mode 100644 index 00000000000..b6736cc1b29 --- /dev/null +++ b/extensions/qa-lab/src/lab-server.ts @@ -0,0 +1,289 @@ +import fs from "node:fs"; +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { handleQaBusRequest, writeError, writeJson } from "./bus-server.js"; +import { createQaBusState, type QaBusState } from "./bus-state.js"; +import { createQaRunnerRuntime } from "./harness-runtime.js"; +import { qaChannelPlugin, setQaChannelRuntime, type OpenClawConfig } from "./runtime-api.js"; +import { runQaSelfCheckAgainstState, type QaSelfCheckResult } from "./self-check.js"; + +type QaLabLatestReport = { + outputPath: string; + markdown: string; + generatedAt: string; +}; + +async function readJson(req: IncomingMessage): Promise { + const chunks: Buffer[] = []; + for await (const chunk of req) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + const text = Buffer.concat(chunks).toString("utf8").trim(); + return text ? (JSON.parse(text) as unknown) : {}; +} + +function detectContentType(filePath: string): string { + if (filePath.endsWith(".css")) { + return "text/css; charset=utf-8"; + } + if (filePath.endsWith(".js")) { + return "text/javascript; charset=utf-8"; + } + if (filePath.endsWith(".json")) { + return "application/json; charset=utf-8"; + } + if (filePath.endsWith(".svg")) { + return "image/svg+xml"; + } + return "text/html; charset=utf-8"; +} + +function missingUiHtml() { + return ` + + + + + QA Lab UI Missing + + + +
+

QA Lab UI not built

+

Build the private debugger bundle, then reload this page.

+

pnpm qa:lab:build

+
+ +`; +} + +function resolveUiDistDir() { + return fileURLToPath(new URL("../web/dist", import.meta.url)); +} + +function tryResolveUiAsset(pathname: string): string | null { + const distDir = resolveUiDistDir(); + if (!fs.existsSync(distDir)) { + return null; + } + const safePath = pathname === "/" ? "/index.html" : pathname; + const decoded = decodeURIComponent(safePath); + const candidate = path.normalize(path.join(distDir, decoded)); + if (!candidate.startsWith(distDir)) { + return null; + } + if (fs.existsSync(candidate) && fs.statSync(candidate).isFile()) { + return candidate; + } + const fallback = path.join(distDir, "index.html"); + return fs.existsSync(fallback) ? fallback : null; +} + +function createQaLabConfig(baseUrl: string): OpenClawConfig { + return { + channels: { + "qa-channel": { + enabled: true, + baseUrl, + botUserId: "openclaw", + botDisplayName: "OpenClaw QA", + allowFrom: ["*"], + }, + }, + }; +} + +async function startQaGatewayLoop(params: { state: QaBusState; baseUrl: string }) { + const runtime = createQaRunnerRuntime(); + setQaChannelRuntime(runtime); + const cfg = createQaLabConfig(params.baseUrl); + const account = qaChannelPlugin.config.resolveAccount(cfg, "default"); + const abort = new AbortController(); + const task = qaChannelPlugin.gateway?.startAccount?.({ + accountId: account.accountId, + account, + cfg, + runtime: { + log: () => undefined, + error: () => undefined, + exit: () => undefined, + }, + abortSignal: abort.signal, + log: { + info: () => undefined, + warn: () => undefined, + error: () => undefined, + debug: () => undefined, + }, + getStatus: () => ({ + accountId: account.accountId, + configured: true, + enabled: true, + running: true, + }), + setStatus: () => undefined, + }); + return { + cfg, + async stop() { + abort.abort(); + await task; + }, + }; +} + +export async function startQaLabServer(params?: { + host?: string; + port?: number; + outputPath?: string; +}) { + const state = createQaBusState(); + let latestReport: QaLabLatestReport | null = null; + let gateway: + | { + cfg: OpenClawConfig; + stop: () => Promise; + } + | undefined; + + const server = createServer(async (req, res) => { + const url = new URL(req.url ?? "/", "http://127.0.0.1"); + + if (await handleQaBusRequest({ req, res, state })) { + return; + } + + try { + if (req.method === "GET" && url.pathname === "/api/bootstrap") { + writeJson(res, 200, { + baseUrl, + latestReport, + defaults: { + conversationKind: "direct", + conversationId: "alice", + senderId: "alice", + senderName: "Alice", + }, + }); + return; + } + if (req.method === "GET" && url.pathname === "/api/state") { + writeJson(res, 200, state.getSnapshot()); + return; + } + if (req.method === "GET" && url.pathname === "/api/report") { + writeJson(res, 200, { report: latestReport }); + return; + } + if (req.method === "POST" && url.pathname === "/api/reset") { + state.reset(); + writeJson(res, 200, { ok: true }); + return; + } + if (req.method === "POST" && url.pathname === "/api/inbound/message") { + const body = await readJson(req); + writeJson(res, 200, { + message: state.addInboundMessage(body as Parameters[0]), + }); + return; + } + if (req.method === "POST" && url.pathname === "/api/scenario/self-check") { + const result = await runQaSelfCheckAgainstState({ + state, + cfg: gateway?.cfg ?? createQaLabConfig(baseUrl), + outputPath: params?.outputPath, + }); + latestReport = { + outputPath: result.outputPath, + markdown: result.report, + generatedAt: new Date().toISOString(), + }; + writeJson(res, 200, serializeSelfCheck(result)); + return; + } + + if (req.method !== "GET" && req.method !== "HEAD") { + writeError(res, 404, "not found"); + return; + } + + const asset = tryResolveUiAsset(url.pathname); + if (!asset) { + const html = missingUiHtml(); + res.writeHead(200, { + "content-type": "text/html; charset=utf-8", + "content-length": Buffer.byteLength(html), + }); + if (req.method === "HEAD") { + res.end(); + return; + } + res.end(html); + return; + } + + const body = fs.readFileSync(asset); + res.writeHead(200, { + "content-type": detectContentType(asset), + "content-length": body.byteLength, + }); + if (req.method === "HEAD") { + res.end(); + return; + } + res.end(body); + } catch (error) { + writeError(res, 500, error); + } + }); + + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(params?.port ?? 0, params?.host ?? "127.0.0.1", () => resolve()); + }); + const address = server.address(); + if (!address || typeof address === "string") { + throw new Error("qa-lab failed to bind"); + } + const baseUrl = `http://${params?.host ?? "127.0.0.1"}:${address.port}`; + gateway = await startQaGatewayLoop({ state, baseUrl }); + + return { + baseUrl, + state, + async runSelfCheck() { + const result = await runQaSelfCheckAgainstState({ + state, + cfg: gateway!.cfg, + outputPath: params?.outputPath, + }); + latestReport = { + outputPath: result.outputPath, + markdown: result.report, + generatedAt: new Date().toISOString(), + }; + return result; + }, + async stop() { + await gateway?.stop(); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + }, + }; +} + +function serializeSelfCheck(result: QaSelfCheckResult) { + return { + outputPath: result.outputPath, + report: result.report, + checks: result.checks, + scenario: result.scenarioResult, + }; +} diff --git a/extensions/qa-lab/src/report.ts b/extensions/qa-lab/src/report.ts new file mode 100644 index 00000000000..ff8c254d92d --- /dev/null +++ b/extensions/qa-lab/src/report.ts @@ -0,0 +1,91 @@ +export type QaReportCheck = { + name: string; + status: "pass" | "fail" | "skip"; + details?: string; +}; + +export type QaReportScenario = { + name: string; + status: "pass" | "fail" | "skip"; + details?: string; + steps?: QaReportCheck[]; +}; + +export function renderQaMarkdownReport(params: { + title: string; + startedAt: Date; + finishedAt: Date; + checks?: QaReportCheck[]; + scenarios?: QaReportScenario[]; + timeline?: string[]; + notes?: string[]; +}) { + const checks = params.checks ?? []; + const scenarios = params.scenarios ?? []; + const passCount = + checks.filter((check) => check.status === "pass").length + + scenarios.filter((scenario) => scenario.status === "pass").length; + const failCount = + checks.filter((check) => check.status === "fail").length + + scenarios.filter((scenario) => scenario.status === "fail").length; + + const lines = [ + `# ${params.title}`, + "", + `- Started: ${params.startedAt.toISOString()}`, + `- Finished: ${params.finishedAt.toISOString()}`, + `- Duration ms: ${params.finishedAt.getTime() - params.startedAt.getTime()}`, + `- Passed: ${passCount}`, + `- Failed: ${failCount}`, + "", + ]; + + if (checks.length > 0) { + lines.push("## Checks", ""); + for (const check of checks) { + lines.push(`- [${check.status === "pass" ? "x" : " "}] ${check.name}`); + if (check.details) { + lines.push(` - ${check.details}`); + } + } + } + + if (scenarios.length > 0) { + lines.push("", "## Scenarios", ""); + for (const scenario of scenarios) { + lines.push(`### ${scenario.name}`); + lines.push(""); + lines.push(`- Status: ${scenario.status}`); + if (scenario.details) { + lines.push(`- Details: ${scenario.details}`); + } + if (scenario.steps?.length) { + lines.push("- Steps:"); + for (const step of scenario.steps) { + lines.push(` - [${step.status === "pass" ? "x" : " "}] ${step.name}`); + if (step.details) { + lines.push(` - ${step.details}`); + } + } + } + lines.push(""); + } + } + + if (params.timeline && params.timeline.length > 0) { + lines.push("## Timeline", ""); + for (const item of params.timeline) { + lines.push(`- ${item}`); + } + } + + if (params.notes && params.notes.length > 0) { + lines.push("", "## Notes", ""); + for (const note of params.notes) { + lines.push(`- ${note}`); + } + } + + lines.push(""); + return lines.join("\n"); +} diff --git a/extensions/qa-lab/src/runtime-api.ts b/extensions/qa-lab/src/runtime-api.ts new file mode 100644 index 00000000000..e6abef5a3cd --- /dev/null +++ b/extensions/qa-lab/src/runtime-api.ts @@ -0,0 +1,38 @@ +export type { Command } from "commander"; +export type { OpenClawConfig, PluginRuntime } from "openclaw/plugin-sdk/core"; +export { definePluginEntry } from "openclaw/plugin-sdk/core"; +export { + buildQaTarget, + createQaBusThread, + deleteQaBusMessage, + editQaBusMessage, + getQaBusState, + injectQaBusInboundMessage, + normalizeQaTarget, + parseQaTarget, + pollQaBus, + qaChannelPlugin, + reactToQaBusMessage, + readQaBusMessage, + searchQaBusMessages, + sendQaBusMessage, + setQaChannelRuntime, +} from "../../qa-channel/api.js"; +export type { + QaBusConversation, + QaBusCreateThreadInput, + QaBusDeleteMessageInput, + QaBusEditMessageInput, + QaBusEvent, + QaBusInboundMessageInput, + QaBusMessage, + QaBusOutboundMessageInput, + QaBusPollInput, + QaBusPollResult, + QaBusReactToMessageInput, + QaBusReadMessageInput, + QaBusSearchMessagesInput, + QaBusStateSnapshot, + QaBusThread, + QaBusWaitForInput, +} from "../../qa-channel/api.js"; diff --git a/extensions/qa-lab/src/scenario.ts b/extensions/qa-lab/src/scenario.ts new file mode 100644 index 00000000000..6e839875aeb --- /dev/null +++ b/extensions/qa-lab/src/scenario.ts @@ -0,0 +1,65 @@ +import type { QaBusState } from "./bus-state.js"; + +export type QaScenarioStepContext = { + state: QaBusState; +}; + +export type QaScenarioStep = { + name: string; + run: (ctx: QaScenarioStepContext) => Promise; +}; + +export type QaScenarioDefinition = { + name: string; + steps: QaScenarioStep[]; +}; + +export type QaScenarioStepResult = { + name: string; + status: "pass" | "fail"; + details?: string; +}; + +export type QaScenarioResult = { + name: string; + status: "pass" | "fail"; + steps: QaScenarioStepResult[]; + details?: string; +}; + +export async function runQaScenario( + definition: QaScenarioDefinition, + ctx: QaScenarioStepContext, +): Promise { + const steps: QaScenarioStepResult[] = []; + + for (const step of definition.steps) { + try { + const details = await step.run(ctx); + steps.push({ + name: step.name, + status: "pass", + ...(details ? { details } : {}), + }); + } catch (error) { + const details = error instanceof Error ? error.message : String(error); + steps.push({ + name: step.name, + status: "fail", + details, + }); + return { + name: definition.name, + status: "fail", + steps, + details, + }; + } + } + + return { + name: definition.name, + status: "pass", + steps, + }; +} diff --git a/extensions/qa-lab/src/self-check-scenario.ts b/extensions/qa-lab/src/self-check-scenario.ts new file mode 100644 index 00000000000..bdcd6026605 --- /dev/null +++ b/extensions/qa-lab/src/self-check-scenario.ts @@ -0,0 +1,121 @@ +import { extractQaToolPayload } from "./extract-tool-payload.js"; +import { qaChannelPlugin, type OpenClawConfig } from "./runtime-api.js"; +import type { QaScenarioDefinition } from "./scenario.js"; + +export function createQaSelfCheckScenario(cfg: OpenClawConfig): QaScenarioDefinition { + return { + name: "Synthetic Slack-class roundtrip", + steps: [ + { + name: "DM echo roundtrip", + async run({ state }) { + state.addInboundMessage({ + conversation: { id: "alice", kind: "direct" }, + senderId: "alice", + senderName: "Alice", + text: "hello from qa", + }); + await state.waitFor({ + kind: "message-text", + textIncludes: "qa-echo: hello from qa", + direction: "outbound", + timeoutMs: 5_000, + }); + }, + }, + { + name: "Thread create and threaded echo", + async run({ state }) { + const threadResult = await qaChannelPlugin.actions?.handleAction?.({ + channel: "qa-channel", + action: "thread-create", + cfg, + accountId: "default", + params: { + channelId: "qa-room", + title: "QA thread", + }, + }); + const threadPayload = extractQaToolPayload(threadResult) as + | { thread?: { id?: string } } + | undefined; + const threadId = threadPayload?.thread?.id; + if (!threadId) { + throw new Error("thread-create did not return thread id"); + } + + state.addInboundMessage({ + conversation: { id: "qa-room", kind: "channel", title: "QA Room" }, + senderId: "alice", + senderName: "Alice", + text: "inside thread", + threadId, + threadTitle: "QA thread", + }); + await state.waitFor({ + kind: "message-text", + textIncludes: "qa-echo: inside thread", + direction: "outbound", + timeoutMs: 5_000, + }); + return threadId; + }, + }, + { + name: "Reaction, edit, delete lifecycle", + async run({ state }) { + const outbound = state + .searchMessages({ query: "qa-echo: inside thread", conversationId: "qa-room" }) + .at(-1); + if (!outbound) { + throw new Error("threaded outbound message not found"); + } + + await qaChannelPlugin.actions?.handleAction?.({ + channel: "qa-channel", + action: "react", + cfg, + accountId: "default", + params: { + messageId: outbound.id, + emoji: "white_check_mark", + }, + }); + const reacted = state.readMessage({ messageId: outbound.id }); + if (reacted.reactions.length === 0) { + throw new Error("reaction not recorded"); + } + + await qaChannelPlugin.actions?.handleAction?.({ + channel: "qa-channel", + action: "edit", + cfg, + accountId: "default", + params: { + messageId: outbound.id, + text: "qa-echo: inside thread (edited)", + }, + }); + const edited = state.readMessage({ messageId: outbound.id }); + if (!edited.text.includes("(edited)")) { + throw new Error("edit not recorded"); + } + + await qaChannelPlugin.actions?.handleAction?.({ + channel: "qa-channel", + action: "delete", + cfg, + accountId: "default", + params: { + messageId: outbound.id, + }, + }); + const deleted = state.readMessage({ messageId: outbound.id }); + if (!deleted.deleted) { + throw new Error("delete not recorded"); + } + }, + }, + ], + }; +} diff --git a/extensions/qa-lab/src/self-check.ts b/extensions/qa-lab/src/self-check.ts new file mode 100644 index 00000000000..472cc35123e --- /dev/null +++ b/extensions/qa-lab/src/self-check.ts @@ -0,0 +1,91 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import type { OpenClawConfig } from "openclaw/plugin-sdk/core"; +import type { QaBusState } from "./bus-state.js"; +import { startQaLabServer } from "./lab-server.js"; +import { renderQaMarkdownReport } from "./report.js"; +import { runQaScenario, type QaScenarioResult } from "./scenario.js"; +import { createQaSelfCheckScenario } from "./self-check-scenario.js"; + +export type QaSelfCheckResult = { + outputPath: string; + report: string; + checks: Array<{ name: string; status: "pass" | "fail"; details?: string }>; + scenarioResult: QaScenarioResult; +}; + +export async function runQaSelfCheckAgainstState(params: { + state: QaBusState; + cfg: OpenClawConfig; + outputPath?: string; + notes?: string[]; +}): Promise { + const startedAt = new Date(); + params.state.reset(); + const scenarioResult = await runQaScenario(createQaSelfCheckScenario(params.cfg), { + state: params.state, + }); + const checks = [ + { + name: "QA self-check scenario", + status: scenarioResult.status, + details: `${scenarioResult.steps.filter((step) => step.status === "pass").length}/${scenarioResult.steps.length} steps passed`, + }, + ] satisfies Array<{ name: string; status: "pass" | "fail"; details?: string }>; + const finishedAt = new Date(); + const snapshot = params.state.getSnapshot(); + const timeline = snapshot.events.map((event) => { + switch (event.kind) { + case "thread-created": + return `${event.cursor}. ${event.kind} ${event.thread.conversationId}/${event.thread.id}`; + case "reaction-added": + return `${event.cursor}. ${event.kind} ${event.message.id} ${event.emoji}`; + default: + return `${event.cursor}. ${event.kind} ${"message" in event ? event.message.id : ""}`.trim(); + } + }); + const report = renderQaMarkdownReport({ + title: "OpenClaw QA E2E Self-Check", + startedAt, + finishedAt, + checks, + scenarios: [ + { + name: scenarioResult.name, + status: scenarioResult.status, + details: scenarioResult.details, + steps: scenarioResult.steps, + }, + ], + timeline, + notes: params.notes ?? [ + "Vertical slice: qa-channel + qa-lab bus + private debugger surface.", + "Docker orchestration, matrix runs, and auto-fix loops remain follow-up work.", + ], + }); + + const outputPath = + params.outputPath ?? path.join(process.cwd(), ".artifacts", "qa-e2e", "self-check.md"); + await fs.mkdir(path.dirname(outputPath), { recursive: true }); + await fs.writeFile(outputPath, report, "utf8"); + + return { + outputPath, + report, + checks, + scenarioResult, + }; +} + +export async function runQaLabSelfCheck(params?: { outputPath?: string }) { + const server = await startQaLabServer({ + outputPath: params?.outputPath, + }); + try { + return await server.runSelfCheck(); + } finally { + await server.stop(); + } +} + +export const runQaE2eSelfCheck = runQaLabSelfCheck; diff --git a/extensions/qa-lab/web/index.html b/extensions/qa-lab/web/index.html new file mode 100644 index 00000000000..71b4ca80bf0 --- /dev/null +++ b/extensions/qa-lab/web/index.html @@ -0,0 +1,12 @@ + + + + + + QA Lab + + +
+ + + diff --git a/extensions/qa-lab/web/src/app.ts b/extensions/qa-lab/web/src/app.ts new file mode 100644 index 00000000000..7c9dbb0ba0b --- /dev/null +++ b/extensions/qa-lab/web/src/app.ts @@ -0,0 +1,498 @@ +type Conversation = { + id: string; + kind: "direct" | "channel"; + title?: string; +}; + +type Thread = { + id: string; + conversationId: string; + title: string; +}; + +type Message = { + id: string; + direction: "inbound" | "outbound"; + conversation: Conversation; + senderId: string; + senderName?: string; + text: string; + timestamp: number; + threadId?: string; + threadTitle?: string; + deleted?: boolean; + editedAt?: number; + reactions: Array<{ emoji: string; senderId: string }>; +}; + +type BusEvent = + | { cursor: number; kind: "thread-created"; thread: Thread } + | { cursor: number; kind: string; message?: Message; emoji?: string }; + +type Snapshot = { + conversations: Conversation[]; + threads: Thread[]; + messages: Message[]; + events: BusEvent[]; +}; + +type ReportEnvelope = { + report: null | { + outputPath: string; + markdown: string; + generatedAt: string; + }; +}; + +type Bootstrap = { + baseUrl: string; + latestReport: ReportEnvelope["report"]; + defaults: { + conversationKind: "direct" | "channel"; + conversationId: string; + senderId: string; + senderName: string; + }; +}; + +type UiState = { + bootstrap: Bootstrap | null; + snapshot: Snapshot | null; + latestReport: ReportEnvelope["report"]; + selectedConversationId: string | null; + selectedThreadId: string | null; + composer: { + conversationKind: "direct" | "channel"; + conversationId: string; + senderId: string; + senderName: string; + text: string; + }; + busy: boolean; + error: string | null; +}; + +async function getJson(path: string): Promise { + const response = await fetch(path); + if (!response.ok) { + throw new Error(`${response.status} ${response.statusText}`); + } + return (await response.json()) as T; +} + +async function postJson(path: string, body: unknown): Promise { + const response = await fetch(path, { + method: "POST", + headers: { + "content-type": "application/json", + }, + body: JSON.stringify(body), + }); + if (!response.ok) { + const payload = (await response.json().catch(() => ({}))) as { error?: string }; + throw new Error(payload.error || `${response.status} ${response.statusText}`); + } + return (await response.json()) as T; +} + +function formatTime(timestamp: number) { + return new Date(timestamp).toLocaleTimeString([], { + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + }); +} + +function escapeHtml(text: string) { + return text + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """); +} + +function filteredMessages(state: UiState) { + const messages = state.snapshot?.messages ?? []; + return messages.filter((message) => { + if (state.selectedConversationId && message.conversation.id !== state.selectedConversationId) { + return false; + } + if (state.selectedThreadId && message.threadId !== state.selectedThreadId) { + return false; + } + return true; + }); +} + +function deriveSelectedConversation(state: UiState): string | null { + if (state.selectedConversationId) { + return state.selectedConversationId; + } + return state.snapshot?.conversations[0]?.id ?? null; +} + +function deriveSelectedThread(state: UiState): string | null { + if (state.selectedThreadId) { + return state.selectedThreadId; + } + return null; +} + +export async function createQaLabApp(root: HTMLDivElement) { + const state: UiState = { + bootstrap: null, + snapshot: null, + latestReport: null, + selectedConversationId: null, + selectedThreadId: null, + composer: { + conversationKind: "direct", + conversationId: "alice", + senderId: "alice", + senderName: "Alice", + text: "", + }, + busy: false, + error: null, + }; + + async function refresh() { + try { + const [bootstrap, snapshot, report] = await Promise.all([ + getJson("/api/bootstrap"), + getJson("/api/state"), + getJson("/api/report"), + ]); + state.bootstrap = bootstrap; + state.snapshot = snapshot; + state.latestReport = report.report ?? bootstrap.latestReport; + if (!state.selectedConversationId) { + state.selectedConversationId = snapshot.conversations[0]?.id ?? null; + } + if (!state.composer.conversationId) { + state.composer = { + ...state.composer, + conversationKind: bootstrap.defaults.conversationKind, + conversationId: bootstrap.defaults.conversationId, + senderId: bootstrap.defaults.senderId, + senderName: bootstrap.defaults.senderName, + }; + } + state.error = null; + } catch (error) { + state.error = error instanceof Error ? error.message : String(error); + } + render(); + } + + async function runSelfCheck() { + state.busy = true; + state.error = null; + render(); + try { + const result = await postJson<{ report: string; outputPath: string }>( + "/api/scenario/self-check", + {}, + ); + state.latestReport = { + outputPath: result.outputPath, + markdown: result.report, + generatedAt: new Date().toISOString(), + }; + await refresh(); + } catch (error) { + state.error = error instanceof Error ? error.message : String(error); + render(); + } finally { + state.busy = false; + render(); + } + } + + async function resetState() { + state.busy = true; + render(); + try { + await postJson("/api/reset", {}); + state.latestReport = null; + state.selectedThreadId = null; + await refresh(); + } catch (error) { + state.error = error instanceof Error ? error.message : String(error); + render(); + } finally { + state.busy = false; + render(); + } + } + + async function sendInbound() { + const conversationId = state.composer.conversationId.trim(); + const text = state.composer.text.trim(); + if (!conversationId || !text) { + state.error = "Conversation id and text are required."; + render(); + return; + } + state.busy = true; + state.error = null; + render(); + try { + await postJson("/api/inbound/message", { + conversation: { + id: conversationId, + kind: state.composer.conversationKind, + ...(state.composer.conversationKind === "channel" ? { title: conversationId } : {}), + }, + senderId: state.composer.senderId.trim() || "alice", + senderName: state.composer.senderName.trim() || undefined, + text, + ...(state.selectedThreadId ? { threadId: state.selectedThreadId } : {}), + }); + state.selectedConversationId = conversationId; + state.composer.text = ""; + await refresh(); + } catch (error) { + state.error = error instanceof Error ? error.message : String(error); + render(); + } finally { + state.busy = false; + render(); + } + } + + function downloadReport() { + if (!state.latestReport?.markdown) { + return; + } + const blob = new Blob([state.latestReport.markdown], { type: "text/markdown;charset=utf-8" }); + const href = URL.createObjectURL(blob); + const anchor = document.createElement("a"); + anchor.href = href; + anchor.download = "qa-report.md"; + anchor.click(); + URL.revokeObjectURL(href); + } + + function bindEvents() { + root.querySelectorAll("[data-conversation-id]").forEach((node) => { + node.onclick = () => { + state.selectedConversationId = node.dataset.conversationId ?? null; + state.selectedThreadId = null; + render(); + }; + }); + root.querySelectorAll("[data-thread-id]").forEach((node) => { + node.onclick = () => { + state.selectedConversationId = node.dataset.conversationId ?? null; + state.selectedThreadId = node.dataset.threadId ?? null; + render(); + }; + }); + root.querySelector("[data-action='refresh']")!.onclick = () => { + void refresh(); + }; + root.querySelector("[data-action='reset']")!.onclick = () => { + void resetState(); + }; + root.querySelector("[data-action='self-check']")!.onclick = () => { + void runSelfCheck(); + }; + root.querySelector("[data-action='send']")!.onclick = () => { + void sendInbound(); + }; + root.querySelector("[data-action='download-report']")!.onclick = () => { + downloadReport(); + }; + + root.querySelector("#conversation-kind")!.onchange = (event) => { + const target = event.currentTarget as HTMLSelectElement; + state.composer.conversationKind = target.value === "channel" ? "channel" : "direct"; + }; + root.querySelector("#conversation-id")!.oninput = (event) => { + state.composer.conversationId = (event.currentTarget as HTMLInputElement).value; + }; + root.querySelector("#sender-id")!.oninput = (event) => { + state.composer.senderId = (event.currentTarget as HTMLInputElement).value; + }; + root.querySelector("#sender-name")!.oninput = (event) => { + state.composer.senderName = (event.currentTarget as HTMLInputElement).value; + }; + root.querySelector("#composer-text")!.oninput = (event) => { + state.composer.text = (event.currentTarget as HTMLTextAreaElement).value; + }; + } + + function render() { + const selectedConversationId = deriveSelectedConversation(state); + const selectedThreadId = deriveSelectedThread(state); + const conversations = state.snapshot?.conversations ?? []; + const threads = (state.snapshot?.threads ?? []).filter( + (thread) => !selectedConversationId || thread.conversationId === selectedConversationId, + ); + const messages = filteredMessages({ + ...state, + selectedConversationId, + selectedThreadId, + }); + const events = (state.snapshot?.events ?? []).slice(-20).reverse(); + + root.innerHTML = ` +
+
+
+

Private QA Workspace

+

QA Lab

+

Synthetic Slack-style debugger for qa-channel.

+
+
+ + + +
+
+
+ Bus ${state.bootstrap ? "online" : "booting"} + Conversation ${selectedConversationId ?? "none"} + Thread ${selectedThreadId ?? "root"} + ${state.latestReport ? `Report ${escapeHtml(state.latestReport.outputPath)}` : 'No report yet'} + ${state.error ? `${escapeHtml(state.error)}` : ""} +
+
+ +
+
+

Transcript

+
+ ${ + messages.length === 0 + ? '

No messages in this slice yet.

' + : messages + .map( + (message) => ` +
+
+ ${escapeHtml(message.senderName || message.senderId)} + ${message.direction} + +
+

${escapeHtml(message.text)}

+
+ ${escapeHtml(message.id)} + ${message.threadId ? `thread ${escapeHtml(message.threadId)}` : ""} + ${message.editedAt ? "edited" : ""} + ${message.deleted ? "deleted" : ""} + ${message.reactions.length ? `${message.reactions.map((reaction) => reaction.emoji).join(" ")}` : ""} +
+
`, + ) + .join("") + } +
+
+
+

Inject inbound

+
+ + + + +
+ +
+ +
+
+
+ +
+
`; + bindEvents(); + } + + render(); + await refresh(); + setInterval(() => { + void refresh(); + }, 1_000); +} diff --git a/extensions/qa-lab/web/src/main.ts b/extensions/qa-lab/web/src/main.ts new file mode 100644 index 00000000000..be1f49b887d --- /dev/null +++ b/extensions/qa-lab/web/src/main.ts @@ -0,0 +1,10 @@ +import "./styles.css"; +import { createQaLabApp } from "./app"; + +const root = document.querySelector("#app"); + +if (!root) { + throw new Error("QA Lab app root missing"); +} + +void createQaLabApp(root); diff --git a/extensions/qa-lab/web/src/styles.css b/extensions/qa-lab/web/src/styles.css new file mode 100644 index 00000000000..5c19eca4cb5 --- /dev/null +++ b/extensions/qa-lab/web/src/styles.css @@ -0,0 +1,305 @@ +:root { + color-scheme: dark; + --bg: #0b0f14; + --bg-alt: #11161d; + --panel: rgba(17, 22, 29, 0.92); + --panel-strong: rgba(22, 29, 38, 0.98); + --line: rgba(145, 170, 197, 0.16); + --text: #eff4fb; + --muted: #8c98a8; + --accent: #65d6bf; + --accent-2: #ff9b57; + --danger: #ff7b88; + --shadow: 0 30px 80px rgba(0, 0, 0, 0.35); + --radius: 20px; + font-family: + ui-sans-serif, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; +} + +* { + box-sizing: border-box; +} + +body { + margin: 0; + min-height: 100vh; + background: + radial-gradient(circle at top left, rgba(101, 214, 191, 0.14), transparent 30%), + radial-gradient(circle at top right, rgba(255, 155, 87, 0.12), transparent 28%), + linear-gradient(180deg, #0a0f14, #0c1218 58%, #0a0f14); + color: var(--text); +} + +button, +input, +select, +textarea { + font: inherit; +} + +button, +input, +select, +textarea { + border-radius: 14px; + border: 1px solid var(--line); + background: rgba(255, 255, 255, 0.02); + color: var(--text); +} + +button { + cursor: pointer; + padding: 0.7rem 1rem; +} + +button.accent { + background: linear-gradient(135deg, var(--accent), #7bdad0); + border-color: transparent; + color: #04130f; + font-weight: 700; +} + +button:disabled { + opacity: 0.55; + cursor: not-allowed; +} + +input, +select, +textarea { + width: 100%; + padding: 0.75rem 0.85rem; +} + +textarea { + resize: vertical; +} + +.shell { + padding: 1.2rem; +} + +.topbar, +.statusbar, +.workspace { + max-width: 1600px; + margin: 0 auto; +} + +.topbar { + display: flex; + justify-content: space-between; + gap: 1rem; + align-items: end; + margin-bottom: 0.8rem; +} + +.topbar h1, +.panel h2 { + margin: 0; +} + +.eyebrow { + margin: 0 0 0.25rem; + color: var(--accent); + text-transform: uppercase; + letter-spacing: 0.12em; + font-size: 0.72rem; +} + +.subtle { + margin: 0.25rem 0 0; + color: var(--muted); +} + +.toolbar { + display: flex; + gap: 0.7rem; + flex-wrap: wrap; +} + +.statusbar { + display: flex; + gap: 0.65rem; + flex-wrap: wrap; + margin-bottom: 1rem; +} + +.pill { + padding: 0.45rem 0.75rem; + border-radius: 999px; + background: rgba(255, 255, 255, 0.05); + border: 1px solid var(--line); + color: var(--muted); +} + +.pill.success { + color: var(--accent); +} + +.pill.error { + color: var(--danger); +} + +.workspace { + display: grid; + grid-template-columns: 280px minmax(0, 1fr) 360px; + gap: 1rem; +} + +.rail, +.center, +.right { + display: flex; + flex-direction: column; + gap: 1rem; + min-height: 0; +} + +.panel { + background: var(--panel); + border: 1px solid var(--line); + border-radius: var(--radius); + box-shadow: var(--shadow); + padding: 1rem; + min-height: 0; +} + +.panel-header { + display: flex; + justify-content: space-between; + gap: 0.7rem; + align-items: center; + margin-bottom: 0.7rem; +} + +.stack { + display: flex; + flex-direction: column; + gap: 0.65rem; +} + +.list-item { + display: flex; + flex-direction: column; + align-items: start; + gap: 0.15rem; + text-align: left; +} + +.list-item.selected { + background: rgba(101, 214, 191, 0.12); + border-color: rgba(101, 214, 191, 0.4); +} + +.list-item span, +.message footer, +.event-row span { + color: var(--muted); + font-size: 0.82rem; +} + +.transcript, +.events { + flex: 1; +} + +.messages, +.report { + min-height: 0; + max-height: 46vh; + overflow: auto; +} + +.message { + padding: 0.9rem; + border-radius: 16px; + margin-bottom: 0.8rem; + border: 1px solid rgba(255, 255, 255, 0.04); + background: var(--panel-strong); +} + +.message.inbound { + border-left: 3px solid var(--accent-2); +} + +.message.outbound { + border-left: 3px solid var(--accent); +} + +.message header, +.message footer, +.event-row { + display: flex; + gap: 0.55rem; + align-items: center; + flex-wrap: wrap; +} + +.message p { + margin: 0.65rem 0; + white-space: pre-wrap; +} + +.composer-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 0.8rem; +} + +label span { + display: block; + margin-bottom: 0.35rem; + color: var(--muted); + font-size: 0.84rem; +} + +.textarea-label { + display: block; + margin-top: 0.85rem; +} + +.lower { + margin-top: 0.85rem; +} + +.report { + margin: 0; + padding: 0.8rem; + border-radius: 16px; + background: rgba(7, 10, 14, 0.7); + border: 1px solid rgba(255, 255, 255, 0.04); + white-space: pre-wrap; + font-family: ui-monospace, SFMono-Regular, Menlo, monospace; + font-size: 0.78rem; + line-height: 1.45; +} + +.event-row { + padding: 0.75rem; + border-radius: 14px; + background: rgba(255, 255, 255, 0.03); +} + +.event-row code { + display: block; + width: 100%; + color: #c7d2e3; + white-space: pre-wrap; + word-break: break-word; +} + +.empty { + color: var(--muted); +} + +@media (max-width: 1180px) { + .workspace { + grid-template-columns: 1fr; + } + + .messages, + .report { + max-height: none; + } +} diff --git a/extensions/qa-lab/web/vite.config.ts b/extensions/qa-lab/web/vite.config.ts new file mode 100644 index 00000000000..b6f958cc6fb --- /dev/null +++ b/extensions/qa-lab/web/vite.config.ts @@ -0,0 +1,11 @@ +import path from "node:path"; +import { defineConfig } from "vite"; + +export default defineConfig({ + root: path.resolve(import.meta.dirname), + base: "./", + build: { + outDir: path.resolve(import.meta.dirname, "dist"), + emptyOutDir: true, + }, +}); diff --git a/package.json b/package.json index 12eca03ceec..f4d226f9a84 100644 --- a/package.json +++ b/package.json @@ -723,6 +723,10 @@ "types": "./dist/plugin-sdk/nostr.d.ts", "default": "./dist/plugin-sdk/nostr.js" }, + "./plugin-sdk/qa-channel": { + "types": "./dist/plugin-sdk/qa-channel.d.ts", + "default": "./dist/plugin-sdk/qa-channel.js" + }, "./plugin-sdk/provider-auth": { "types": "./dist/plugin-sdk/provider-auth.d.ts", "default": "./dist/plugin-sdk/provider-auth.js" @@ -1030,6 +1034,8 @@ "protocol:gen": "node --import tsx scripts/protocol-gen.ts", "protocol:gen:swift": "node --import tsx scripts/protocol-gen-swift.ts", "qa:e2e": "node --import tsx scripts/qa-e2e.ts", + "qa:lab:build": "vite build --config extensions/qa-lab/web/vite.config.ts", + "qa:lab:ui": "pnpm openclaw qa ui", "release:check": "pnpm check:base-config-schema && pnpm check:bundled-channel-config-metadata && pnpm config:docs:check && pnpm plugin-sdk:check-exports && pnpm plugin-sdk:api:check && node --import tsx scripts/release-check.ts", "release:openclaw:npm:check": "node --import tsx scripts/openclaw-npm-release-check.ts", "release:openclaw:npm:verify-published": "node --import tsx scripts/openclaw-npm-postpublish-verify.ts", diff --git a/scripts/lib/plugin-sdk-entrypoints.json b/scripts/lib/plugin-sdk-entrypoints.json index 75392bf4c82..fc555415603 100644 --- a/scripts/lib/plugin-sdk-entrypoints.json +++ b/scripts/lib/plugin-sdk-entrypoints.json @@ -170,6 +170,7 @@ "native-command-registry", "nextcloud-talk", "nostr", + "qa-channel", "provider-auth", "provider-auth-runtime", "provider-auth-api-key", diff --git a/src/plugin-sdk/qa-channel.ts b/src/plugin-sdk/qa-channel.ts new file mode 100644 index 00000000000..7b4c8241486 --- /dev/null +++ b/src/plugin-sdk/qa-channel.ts @@ -0,0 +1,40 @@ +// Narrow plugin-sdk surface for the bundled qa-channel plugin. +// Keep this list additive and scoped to QA transport contracts. + +export { + buildQaTarget, + buildQaTarget as formatQaTarget, + createQaBusThread, + deleteQaBusMessage, + editQaBusMessage, + getQaBusState, + injectQaBusInboundMessage, + normalizeQaTarget, + parseQaTarget, + pollQaBus, + qaChannelPlugin, + reactToQaBusMessage, + readQaBusMessage, + searchQaBusMessages, + sendQaBusMessage, + setQaChannelRuntime, +} from "../../extensions/qa-channel/api.js"; +export type { + QaBusConversation, + QaBusConversationKind, + QaBusCreateThreadInput, + QaBusDeleteMessageInput, + QaBusEditMessageInput, + QaBusEvent, + QaBusInboundMessageInput, + QaBusMessage, + QaBusOutboundMessageInput, + QaBusPollInput, + QaBusPollResult, + QaBusReactToMessageInput, + QaBusReadMessageInput, + QaBusSearchMessagesInput, + QaBusStateSnapshot, + QaBusThread, + QaBusWaitForInput, +} from "../../extensions/qa-channel/api.js"; diff --git a/src/qa-e2e/bus-queries.ts b/src/qa-e2e/bus-queries.ts index fe92541e2d9..57d9d5c96c8 100644 --- a/src/qa-e2e/bus-queries.ts +++ b/src/qa-e2e/bus-queries.ts @@ -1,136 +1 @@ -import type { - QaBusConversation, - QaBusEvent, - QaBusMessage, - QaBusPollInput, - QaBusPollResult, - QaBusReadMessageInput, - QaBusSearchMessagesInput, - QaBusStateSnapshot, - QaBusThread, -} from "../../extensions/qa-channel/test-api.js"; - -export const DEFAULT_ACCOUNT_ID = "default"; - -export function normalizeAccountId(raw?: string): string { - const trimmed = raw?.trim(); - return trimmed || DEFAULT_ACCOUNT_ID; -} - -export function normalizeConversationFromTarget(target: string): { - conversation: QaBusConversation; - threadId?: string; -} { - const trimmed = target.trim(); - if (trimmed.startsWith("thread:")) { - const rest = trimmed.slice("thread:".length); - const slash = rest.indexOf("/"); - if (slash > 0) { - return { - conversation: { id: rest.slice(0, slash), kind: "channel" }, - threadId: rest.slice(slash + 1), - }; - } - } - if (trimmed.startsWith("channel:")) { - return { - conversation: { id: trimmed.slice("channel:".length), kind: "channel" }, - }; - } - if (trimmed.startsWith("dm:")) { - return { - conversation: { id: trimmed.slice("dm:".length), kind: "direct" }, - }; - } - return { - conversation: { id: trimmed, kind: "direct" }, - }; -} - -export function cloneMessage(message: QaBusMessage): QaBusMessage { - return { - ...message, - conversation: { ...message.conversation }, - reactions: message.reactions.map((reaction) => ({ ...reaction })), - }; -} - -export function cloneEvent(event: QaBusEvent): QaBusEvent { - switch (event.kind) { - case "inbound-message": - case "outbound-message": - case "message-edited": - case "message-deleted": - case "reaction-added": - return { ...event, message: cloneMessage(event.message) }; - case "thread-created": - return { ...event, thread: { ...event.thread } }; - } -} - -export function buildQaBusSnapshot(params: { - cursor: number; - conversations: Map; - threads: Map; - messages: Map; - events: QaBusEvent[]; -}): QaBusStateSnapshot { - return { - cursor: params.cursor, - conversations: Array.from(params.conversations.values()).map((conversation) => ({ - ...conversation, - })), - threads: Array.from(params.threads.values()).map((thread) => ({ ...thread })), - messages: Array.from(params.messages.values()).map((message) => cloneMessage(message)), - events: params.events.map((event) => cloneEvent(event)), - }; -} - -export function readQaBusMessage(params: { - messages: Map; - input: QaBusReadMessageInput; -}) { - const message = params.messages.get(params.input.messageId); - if (!message) { - throw new Error(`qa-bus message not found: ${params.input.messageId}`); - } - return cloneMessage(message); -} - -export function searchQaBusMessages(params: { - messages: Map; - input: QaBusSearchMessagesInput; -}) { - const accountId = normalizeAccountId(params.input.accountId); - const limit = Math.max(1, Math.min(params.input.limit ?? 20, 100)); - const query = params.input.query?.trim().toLowerCase(); - return Array.from(params.messages.values()) - .filter((message) => message.accountId === accountId) - .filter((message) => - params.input.conversationId ? message.conversation.id === params.input.conversationId : true, - ) - .filter((message) => - params.input.threadId ? message.threadId === params.input.threadId : true, - ) - .filter((message) => (query ? message.text.toLowerCase().includes(query) : true)) - .slice(-limit) - .map((message) => cloneMessage(message)); -} - -export function pollQaBusEvents(params: { - events: QaBusEvent[]; - cursor: number; - input?: QaBusPollInput; -}): QaBusPollResult { - const accountId = normalizeAccountId(params.input?.accountId); - const startCursor = params.input?.cursor ?? 0; - const limit = Math.max(1, Math.min(params.input?.limit ?? 100, 500)); - const matches = params.events - .filter((event) => event.accountId === accountId && event.cursor > startCursor) - .slice(0, limit) - .map((event) => cloneEvent(event)); - return { - cursor: params.cursor, - events: matches, - }; -} +export * from "../../extensions/qa-lab/api.js"; diff --git a/src/qa-e2e/bus-server.ts b/src/qa-e2e/bus-server.ts index efad93e647d..57d9d5c96c8 100644 --- a/src/qa-e2e/bus-server.ts +++ b/src/qa-e2e/bus-server.ts @@ -1,170 +1 @@ -import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; -import type { - QaBusCreateThreadInput, - QaBusDeleteMessageInput, - QaBusEditMessageInput, - QaBusInboundMessageInput, - QaBusOutboundMessageInput, - QaBusPollInput, - QaBusReactToMessageInput, - QaBusReadMessageInput, - QaBusSearchMessagesInput, - QaBusWaitForInput, -} from "../../extensions/qa-channel/test-api.js"; -import type { QaBusState } from "./bus-state.js"; - -async function readJson(req: IncomingMessage): Promise { - const chunks: Buffer[] = []; - for await (const chunk of req) { - chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); - } - const text = Buffer.concat(chunks).toString("utf8").trim(); - return text ? (JSON.parse(text) as unknown) : {}; -} - -function writeJson(res: ServerResponse, statusCode: number, body: unknown) { - const payload = JSON.stringify(body); - res.writeHead(statusCode, { - "content-type": "application/json; charset=utf-8", - "content-length": Buffer.byteLength(payload), - }); - res.end(payload); -} - -function writeError(res: ServerResponse, statusCode: number, error: unknown) { - writeJson(res, statusCode, { - error: error instanceof Error ? error.message : String(error), - }); -} - -async function handleRequest(params: { - req: IncomingMessage; - res: ServerResponse; - state: QaBusState; -}) { - const method = params.req.method ?? "GET"; - const url = new URL(params.req.url ?? "/", "http://127.0.0.1"); - - if (method === "GET" && url.pathname === "/health") { - writeJson(params.res, 200, { ok: true }); - return; - } - - if (method === "GET" && url.pathname === "/v1/state") { - writeJson(params.res, 200, params.state.getSnapshot()); - return; - } - - if (method !== "POST") { - writeError(params.res, 405, "method not allowed"); - return; - } - - const body = (await readJson(params.req)) as Record; - - try { - switch (url.pathname) { - case "/v1/reset": - params.state.reset(); - writeJson(params.res, 200, { ok: true }); - return; - case "/v1/inbound/message": - writeJson(params.res, 200, { - message: params.state.addInboundMessage(body as unknown as QaBusInboundMessageInput), - }); - return; - case "/v1/outbound/message": - writeJson(params.res, 200, { - message: params.state.addOutboundMessage(body as unknown as QaBusOutboundMessageInput), - }); - return; - case "/v1/actions/thread-create": - writeJson(params.res, 200, { - thread: params.state.createThread(body as unknown as QaBusCreateThreadInput), - }); - return; - case "/v1/actions/react": - writeJson(params.res, 200, { - message: params.state.reactToMessage(body as unknown as QaBusReactToMessageInput), - }); - return; - case "/v1/actions/edit": - writeJson(params.res, 200, { - message: params.state.editMessage(body as unknown as QaBusEditMessageInput), - }); - return; - case "/v1/actions/delete": - writeJson(params.res, 200, { - message: params.state.deleteMessage(body as unknown as QaBusDeleteMessageInput), - }); - return; - case "/v1/actions/read": - writeJson(params.res, 200, { - message: params.state.readMessage(body as unknown as QaBusReadMessageInput), - }); - return; - case "/v1/actions/search": - writeJson(params.res, 200, { - messages: params.state.searchMessages(body as unknown as QaBusSearchMessagesInput), - }); - return; - case "/v1/poll": { - const input = body as unknown as QaBusPollInput; - const timeoutMs = Math.max(0, Math.min(input.timeoutMs ?? 0, 30_000)); - const initial = params.state.poll(input); - if (initial.events.length > 0 || timeoutMs === 0) { - writeJson(params.res, 200, initial); - return; - } - try { - await params.state.waitFor({ - kind: "event-kind", - eventKind: "inbound-message", - timeoutMs, - }); - } catch { - // timeout is fine for long-poll. - } - writeJson(params.res, 200, params.state.poll(input)); - return; - } - case "/v1/wait": - writeJson(params.res, 200, { - match: await params.state.waitFor(body as unknown as QaBusWaitForInput), - }); - return; - default: - writeError(params.res, 404, "not found"); - } - } catch (error) { - writeError(params.res, 400, error); - } -} - -export function createQaBusServer(state: QaBusState): Server { - return createServer(async (req, res) => { - await handleRequest({ req, res, state }); - }); -} - -export async function startQaBusServer(params: { state: QaBusState; port?: number }) { - const server = createQaBusServer(params.state); - await new Promise((resolve, reject) => { - server.once("error", reject); - server.listen(params.port ?? 0, "127.0.0.1", () => resolve()); - }); - const address = server.address(); - if (!address || typeof address === "string") { - throw new Error("qa-bus failed to bind"); - } - return { - server, - port: address.port, - baseUrl: `http://127.0.0.1:${address.port}`, - async stop() { - await new Promise((resolve, reject) => - server.close((error) => (error ? reject(error) : resolve())), - ); - }, - }; -} +export * from "../../extensions/qa-lab/api.js"; diff --git a/src/qa-e2e/bus-state.test.ts b/src/qa-e2e/bus-state.test.ts index 64d0123bd33..3d1aa921746 100644 --- a/src/qa-e2e/bus-state.test.ts +++ b/src/qa-e2e/bus-state.test.ts @@ -62,4 +62,38 @@ describe("qa-bus state", () => { }); expect("id" in waited && waited.id).toBe(thread.id); }); + + it("replays fresh events after a reset rewinds the cursor", () => { + const state = createQaBusState(); + + state.addInboundMessage({ + conversation: { id: "alice", kind: "direct" }, + senderId: "alice", + text: "before reset", + }); + const beforeReset = state.poll({ + accountId: "default", + cursor: 0, + }); + expect(beforeReset.events).toHaveLength(1); + + state.reset(); + state.addInboundMessage({ + conversation: { id: "alice", kind: "direct" }, + senderId: "alice", + text: "after reset", + }); + + const afterReset = state.poll({ + accountId: "default", + cursor: beforeReset.cursor, + }); + expect(afterReset.events).toHaveLength(1); + expect(afterReset.events[0]?.kind).toBe("inbound-message"); + expect( + afterReset.events[0] && + "message" in afterReset.events[0] && + afterReset.events[0].message.text, + ).toBe("after reset"); + }); }); diff --git a/src/qa-e2e/bus-state.ts b/src/qa-e2e/bus-state.ts index 76241d997fa..57d9d5c96c8 100644 --- a/src/qa-e2e/bus-state.ts +++ b/src/qa-e2e/bus-state.ts @@ -1,256 +1 @@ -import { randomUUID } from "node:crypto"; -import { - type QaBusConversation, - type QaBusCreateThreadInput, - type QaBusDeleteMessageInput, - type QaBusEditMessageInput, - type QaBusEvent, - type QaBusInboundMessageInput, - type QaBusMessage, - type QaBusOutboundMessageInput, - type QaBusPollInput, - type QaBusReadMessageInput, - type QaBusReactToMessageInput, - type QaBusSearchMessagesInput, - type QaBusThread, - type QaBusWaitForInput, -} from "../../extensions/qa-channel/test-api.js"; -import { - buildQaBusSnapshot, - cloneMessage, - normalizeAccountId, - normalizeConversationFromTarget, - pollQaBusEvents, - readQaBusMessage, - searchQaBusMessages, -} from "./bus-queries.js"; -import { createQaBusWaiterStore } from "./bus-waiters.js"; - -const DEFAULT_BOT_ID = "openclaw"; -const DEFAULT_BOT_NAME = "OpenClaw QA"; - -type QaBusEventSeed = - | Omit, "cursor"> - | Omit, "cursor"> - | Omit, "cursor"> - | Omit, "cursor"> - | Omit, "cursor"> - | Omit, "cursor">; - -export function createQaBusState() { - const conversations = new Map(); - const threads = new Map(); - const messages = new Map(); - const events: QaBusEvent[] = []; - let cursor = 0; - const waiters = createQaBusWaiterStore(() => - buildQaBusSnapshot({ - cursor, - conversations, - threads, - messages, - events, - }), - ); - - const pushEvent = (event: QaBusEventSeed | ((cursor: number) => QaBusEventSeed)): QaBusEvent => { - cursor += 1; - const next = typeof event === "function" ? event(cursor) : event; - const finalized = { cursor, ...next } as QaBusEvent; - events.push(finalized); - waiters.settle(); - return finalized; - }; - - const ensureConversation = (conversation: QaBusConversation): QaBusConversation => { - const existing = conversations.get(conversation.id); - if (existing) { - if (!existing.title && conversation.title) { - existing.title = conversation.title; - } - return existing; - } - const created = { ...conversation }; - conversations.set(created.id, created); - return created; - }; - - const createMessage = (params: { - direction: QaBusMessage["direction"]; - accountId: string; - conversation: QaBusConversation; - senderId: string; - senderName?: string; - text: string; - timestamp?: number; - threadId?: string; - threadTitle?: string; - replyToId?: string; - }): QaBusMessage => { - const conversation = ensureConversation(params.conversation); - const message: QaBusMessage = { - id: randomUUID(), - accountId: params.accountId, - direction: params.direction, - conversation, - senderId: params.senderId, - senderName: params.senderName, - text: params.text, - timestamp: params.timestamp ?? Date.now(), - threadId: params.threadId, - threadTitle: params.threadTitle, - replyToId: params.replyToId, - reactions: [], - }; - messages.set(message.id, message); - return message; - }; - - return { - reset() { - conversations.clear(); - threads.clear(); - messages.clear(); - events.length = 0; - cursor = 0; - waiters.reset(); - }, - getSnapshot() { - return buildQaBusSnapshot({ - cursor, - conversations, - threads, - messages, - events, - }); - }, - addInboundMessage(input: QaBusInboundMessageInput) { - const accountId = normalizeAccountId(input.accountId); - const message = createMessage({ - direction: "inbound", - accountId, - conversation: input.conversation, - senderId: input.senderId, - senderName: input.senderName, - text: input.text, - timestamp: input.timestamp, - threadId: input.threadId, - threadTitle: input.threadTitle, - replyToId: input.replyToId, - }); - pushEvent({ - kind: "inbound-message", - accountId, - message: cloneMessage(message), - }); - return cloneMessage(message); - }, - addOutboundMessage(input: QaBusOutboundMessageInput) { - const accountId = normalizeAccountId(input.accountId); - const { conversation, threadId } = normalizeConversationFromTarget(input.to); - const message = createMessage({ - direction: "outbound", - accountId, - conversation, - senderId: input.senderId?.trim() || DEFAULT_BOT_ID, - senderName: input.senderName?.trim() || DEFAULT_BOT_NAME, - text: input.text, - timestamp: input.timestamp, - threadId: input.threadId ?? threadId, - replyToId: input.replyToId, - }); - pushEvent({ - kind: "outbound-message", - accountId, - message: cloneMessage(message), - }); - return cloneMessage(message); - }, - createThread(input: QaBusCreateThreadInput) { - const accountId = normalizeAccountId(input.accountId); - const thread: QaBusThread = { - id: `thread-${randomUUID()}`, - accountId, - conversationId: input.conversationId, - title: input.title, - createdAt: input.timestamp ?? Date.now(), - createdBy: input.createdBy?.trim() || DEFAULT_BOT_ID, - }; - threads.set(thread.id, thread); - ensureConversation({ - id: input.conversationId, - kind: "channel", - }); - pushEvent({ - kind: "thread-created", - accountId, - thread: { ...thread }, - }); - return { ...thread }; - }, - reactToMessage(input: QaBusReactToMessageInput) { - const accountId = normalizeAccountId(input.accountId); - const message = messages.get(input.messageId); - if (!message) { - throw new Error(`qa-bus message not found: ${input.messageId}`); - } - const reaction = { - emoji: input.emoji, - senderId: input.senderId?.trim() || DEFAULT_BOT_ID, - timestamp: input.timestamp ?? Date.now(), - }; - message.reactions.push(reaction); - pushEvent({ - kind: "reaction-added", - accountId, - message: cloneMessage(message), - emoji: reaction.emoji, - senderId: reaction.senderId, - }); - return cloneMessage(message); - }, - editMessage(input: QaBusEditMessageInput) { - const accountId = normalizeAccountId(input.accountId); - const message = messages.get(input.messageId); - if (!message) { - throw new Error(`qa-bus message not found: ${input.messageId}`); - } - message.text = input.text; - message.editedAt = input.timestamp ?? Date.now(); - pushEvent({ - kind: "message-edited", - accountId, - message: cloneMessage(message), - }); - return cloneMessage(message); - }, - deleteMessage(input: QaBusDeleteMessageInput) { - const accountId = normalizeAccountId(input.accountId); - const message = messages.get(input.messageId); - if (!message) { - throw new Error(`qa-bus message not found: ${input.messageId}`); - } - message.deleted = true; - pushEvent({ - kind: "message-deleted", - accountId, - message: cloneMessage(message), - }); - return cloneMessage(message); - }, - readMessage(input: QaBusReadMessageInput) { - return readQaBusMessage({ messages, input }); - }, - searchMessages(input: QaBusSearchMessagesInput) { - return searchQaBusMessages({ messages, input }); - }, - poll(input: QaBusPollInput = {}) { - return pollQaBusEvents({ events, cursor, input }); - }, - async waitFor(input: QaBusWaitForInput) { - return await waiters.waitFor(input); - }, - }; -} - -export type QaBusState = ReturnType; +export * from "../../extensions/qa-lab/api.js"; diff --git a/src/qa-e2e/bus-waiters.ts b/src/qa-e2e/bus-waiters.ts index 07c277a8af7..57d9d5c96c8 100644 --- a/src/qa-e2e/bus-waiters.ts +++ b/src/qa-e2e/bus-waiters.ts @@ -1,87 +1 @@ -import type { - QaBusEvent, - QaBusMessage, - QaBusStateSnapshot, - QaBusThread, - QaBusWaitForInput, -} from "../../extensions/qa-channel/test-api.js"; - -export const DEFAULT_WAIT_TIMEOUT_MS = 5_000; - -export type QaBusWaitMatch = QaBusEvent | QaBusMessage | QaBusThread; - -type Waiter = { - resolve: (event: QaBusWaitMatch) => void; - reject: (error: Error) => void; - timer: NodeJS.Timeout; - matcher: (snapshot: QaBusStateSnapshot) => QaBusWaitMatch | null; -}; - -function createQaBusMatcher( - input: QaBusWaitForInput, -): (snapshot: QaBusStateSnapshot) => QaBusWaitMatch | null { - return (snapshot) => { - if (input.kind === "event-kind") { - return snapshot.events.find((event) => event.kind === input.eventKind) ?? null; - } - if (input.kind === "thread-id") { - return snapshot.threads.find((thread) => thread.id === input.threadId) ?? null; - } - return ( - snapshot.messages.find( - (message) => - (!input.direction || message.direction === input.direction) && - message.text.includes(input.textIncludes), - ) ?? null - ); - }; -} - -export function createQaBusWaiterStore(getSnapshot: () => QaBusStateSnapshot) { - const waiters = new Set(); - - return { - reset(reason = "qa-bus reset") { - for (const waiter of waiters) { - clearTimeout(waiter.timer); - waiter.reject(new Error(reason)); - } - waiters.clear(); - }, - settle() { - if (waiters.size === 0) { - return; - } - const snapshot = getSnapshot(); - for (const waiter of Array.from(waiters)) { - const match = waiter.matcher(snapshot); - if (!match) { - continue; - } - clearTimeout(waiter.timer); - waiters.delete(waiter); - waiter.resolve(match); - } - }, - async waitFor(input: QaBusWaitForInput) { - const matcher = createQaBusMatcher(input); - const immediate = matcher(getSnapshot()); - if (immediate) { - return immediate; - } - return await new Promise((resolve, reject) => { - const timeoutMs = input.timeoutMs ?? DEFAULT_WAIT_TIMEOUT_MS; - const waiter: Waiter = { - resolve, - reject, - matcher, - timer: setTimeout(() => { - waiters.delete(waiter); - reject(new Error(`qa-bus wait timeout after ${timeoutMs}ms`)); - }, timeoutMs), - }; - waiters.add(waiter); - }); - }, - }; -} +export * from "../../extensions/qa-lab/api.js"; diff --git a/src/qa-e2e/harness-runtime.ts b/src/qa-e2e/harness-runtime.ts index 5c59ed59d0b..57d9d5c96c8 100644 --- a/src/qa-e2e/harness-runtime.ts +++ b/src/qa-e2e/harness-runtime.ts @@ -1,75 +1 @@ -import type { PluginRuntime } from "../plugins/runtime/types.js"; - -type SessionRecord = { - sessionKey: string; - body: string; -}; - -export function createQaRunnerRuntime(): PluginRuntime { - const sessions = new Map(); - return { - channel: { - routing: { - resolveAgentRoute({ - accountId, - peer, - }: { - accountId?: string | null; - peer?: { kind?: string; id?: string } | null; - }) { - return { - agentId: "qa-agent", - accountId: accountId ?? "default", - sessionKey: `qa-agent:${peer?.kind ?? "direct"}:${peer?.id ?? "default"}`, - mainSessionKey: "qa-agent:main", - lastRoutePolicy: "session", - matchedBy: "default", - channel: "qa-channel", - }; - }, - }, - session: { - resolveStorePath(_store: string | undefined, { agentId }: { agentId: string }) { - return agentId; - }, - readSessionUpdatedAt({ sessionKey }: { sessionKey: string }) { - return sessions.has(sessionKey) ? Date.now() : undefined; - }, - recordInboundSession({ - sessionKey, - ctx, - }: { - sessionKey: string; - ctx: { BodyForAgent?: string; Body?: string }; - }) { - sessions.set(sessionKey, { - sessionKey, - body: String(ctx.BodyForAgent ?? ctx.Body ?? ""), - }); - }, - }, - reply: { - resolveEnvelopeFormatOptions() { - return {}; - }, - formatAgentEnvelope({ body }: { body: string }) { - return body; - }, - finalizeInboundContext(ctx: Record) { - return ctx as typeof ctx & { CommandAuthorized: boolean }; - }, - async dispatchReplyWithBufferedBlockDispatcher({ - ctx, - dispatcherOptions, - }: { - ctx: { BodyForAgent?: string; Body?: string }; - dispatcherOptions: { deliver: (payload: { text: string }) => Promise }; - }) { - await dispatcherOptions.deliver({ - text: `qa-echo: ${String(ctx.BodyForAgent ?? ctx.Body ?? "")}`, - }); - }, - }, - }, - } as unknown as PluginRuntime; -} +export * from "../../extensions/qa-lab/api.js"; diff --git a/src/qa-e2e/report.ts b/src/qa-e2e/report.ts index ff8c254d92d..57d9d5c96c8 100644 --- a/src/qa-e2e/report.ts +++ b/src/qa-e2e/report.ts @@ -1,91 +1 @@ -export type QaReportCheck = { - name: string; - status: "pass" | "fail" | "skip"; - details?: string; -}; - -export type QaReportScenario = { - name: string; - status: "pass" | "fail" | "skip"; - details?: string; - steps?: QaReportCheck[]; -}; - -export function renderQaMarkdownReport(params: { - title: string; - startedAt: Date; - finishedAt: Date; - checks?: QaReportCheck[]; - scenarios?: QaReportScenario[]; - timeline?: string[]; - notes?: string[]; -}) { - const checks = params.checks ?? []; - const scenarios = params.scenarios ?? []; - const passCount = - checks.filter((check) => check.status === "pass").length + - scenarios.filter((scenario) => scenario.status === "pass").length; - const failCount = - checks.filter((check) => check.status === "fail").length + - scenarios.filter((scenario) => scenario.status === "fail").length; - - const lines = [ - `# ${params.title}`, - "", - `- Started: ${params.startedAt.toISOString()}`, - `- Finished: ${params.finishedAt.toISOString()}`, - `- Duration ms: ${params.finishedAt.getTime() - params.startedAt.getTime()}`, - `- Passed: ${passCount}`, - `- Failed: ${failCount}`, - "", - ]; - - if (checks.length > 0) { - lines.push("## Checks", ""); - for (const check of checks) { - lines.push(`- [${check.status === "pass" ? "x" : " "}] ${check.name}`); - if (check.details) { - lines.push(` - ${check.details}`); - } - } - } - - if (scenarios.length > 0) { - lines.push("", "## Scenarios", ""); - for (const scenario of scenarios) { - lines.push(`### ${scenario.name}`); - lines.push(""); - lines.push(`- Status: ${scenario.status}`); - if (scenario.details) { - lines.push(`- Details: ${scenario.details}`); - } - if (scenario.steps?.length) { - lines.push("- Steps:"); - for (const step of scenario.steps) { - lines.push(` - [${step.status === "pass" ? "x" : " "}] ${step.name}`); - if (step.details) { - lines.push(` - ${step.details}`); - } - } - } - lines.push(""); - } - } - - if (params.timeline && params.timeline.length > 0) { - lines.push("## Timeline", ""); - for (const item of params.timeline) { - lines.push(`- ${item}`); - } - } - - if (params.notes && params.notes.length > 0) { - lines.push("", "## Notes", ""); - for (const note of params.notes) { - lines.push(`- ${note}`); - } - } - - lines.push(""); - return lines.join("\n"); -} +export * from "../../extensions/qa-lab/api.js"; diff --git a/src/qa-e2e/runner.ts b/src/qa-e2e/runner.ts index 7903579fe79..57d9d5c96c8 100644 --- a/src/qa-e2e/runner.ts +++ b/src/qa-e2e/runner.ts @@ -1,124 +1 @@ -import fs from "node:fs/promises"; -import path from "node:path"; -import { qaChannelPlugin, setQaChannelRuntime } from "../../extensions/qa-channel/api.js"; -import type { OpenClawConfig } from "../config/config.js"; -import { startQaBusServer } from "./bus-server.js"; -import { createQaBusState } from "./bus-state.js"; -import { createQaRunnerRuntime } from "./harness-runtime.js"; -import { renderQaMarkdownReport } from "./report.js"; -import { runQaScenario } from "./scenario.js"; -import { createQaSelfCheckScenario } from "./self-check-scenario.js"; - -export async function runQaE2eSelfCheck(params?: { outputPath?: string }) { - const startedAt = new Date(); - const state = createQaBusState(); - const bus = await startQaBusServer({ state }); - const runtime = createQaRunnerRuntime(); - setQaChannelRuntime(runtime); - - const cfg: OpenClawConfig = { - channels: { - "qa-channel": { - enabled: true, - baseUrl: bus.baseUrl, - botUserId: "openclaw", - botDisplayName: "OpenClaw QA", - allowFrom: ["*"], - }, - }, - }; - - const account = qaChannelPlugin.config.resolveAccount(cfg, "default"); - const abort = new AbortController(); - - const task = qaChannelPlugin.gateway?.startAccount?.({ - accountId: account.accountId, - account, - cfg, - runtime: { - log: () => undefined, - error: () => undefined, - exit: () => undefined, - }, - abortSignal: abort.signal, - log: { - info: () => undefined, - warn: () => undefined, - error: () => undefined, - debug: () => undefined, - }, - getStatus: () => ({ - accountId: account.accountId, - configured: true, - enabled: true, - running: true, - }), - setStatus: () => undefined, - }); - - const checks: Array<{ name: string; status: "pass" | "fail"; details?: string }> = []; - let scenarioResult: Awaited> | undefined; - - try { - scenarioResult = await runQaScenario(createQaSelfCheckScenario(cfg), { state }); - checks.push({ - name: "QA self-check scenario", - status: scenarioResult.status, - details: `${scenarioResult.steps.filter((step) => step.status === "pass").length}/${scenarioResult.steps.length} steps passed`, - }); - } catch (error) { - checks.push({ - name: "QA self-check", - status: "fail", - details: error instanceof Error ? error.message : String(error), - }); - } finally { - abort.abort(); - await task; - await bus.stop(); - } - - const finishedAt = new Date(); - const snapshot = state.getSnapshot(); - const timeline = snapshot.events.map((event) => { - switch (event.kind) { - case "thread-created": - return `${event.cursor}. ${event.kind} ${event.thread.conversationId}/${event.thread.id}`; - case "reaction-added": - return `${event.cursor}. ${event.kind} ${event.message.id} ${event.emoji}`; - default: - return `${event.cursor}. ${event.kind} ${"message" in event ? event.message.id : ""}`.trim(); - } - }); - const report = renderQaMarkdownReport({ - title: "OpenClaw QA E2E Self-Check", - startedAt, - finishedAt, - checks, - scenarios: scenarioResult - ? [ - { - name: scenarioResult.name, - status: scenarioResult.status, - details: scenarioResult.details, - steps: scenarioResult.steps, - }, - ] - : undefined, - timeline, - notes: [ - "Vertical slice only: bus + bundled qa-channel + in-process runner runtime.", - "Full Docker orchestration and model/provider matrix remain follow-up work.", - ], - }); - - const outputPath = - params?.outputPath ?? path.join(process.cwd(), ".artifacts", "qa-e2e", "self-check.md"); - await fs.mkdir(path.dirname(outputPath), { recursive: true }); - await fs.writeFile(outputPath, report, "utf8"); - return { - outputPath, - report, - checks, - }; -} +export * from "../../extensions/qa-lab/api.js"; diff --git a/src/qa-e2e/scenario.ts b/src/qa-e2e/scenario.ts index 6e839875aeb..57d9d5c96c8 100644 --- a/src/qa-e2e/scenario.ts +++ b/src/qa-e2e/scenario.ts @@ -1,65 +1 @@ -import type { QaBusState } from "./bus-state.js"; - -export type QaScenarioStepContext = { - state: QaBusState; -}; - -export type QaScenarioStep = { - name: string; - run: (ctx: QaScenarioStepContext) => Promise; -}; - -export type QaScenarioDefinition = { - name: string; - steps: QaScenarioStep[]; -}; - -export type QaScenarioStepResult = { - name: string; - status: "pass" | "fail"; - details?: string; -}; - -export type QaScenarioResult = { - name: string; - status: "pass" | "fail"; - steps: QaScenarioStepResult[]; - details?: string; -}; - -export async function runQaScenario( - definition: QaScenarioDefinition, - ctx: QaScenarioStepContext, -): Promise { - const steps: QaScenarioStepResult[] = []; - - for (const step of definition.steps) { - try { - const details = await step.run(ctx); - steps.push({ - name: step.name, - status: "pass", - ...(details ? { details } : {}), - }); - } catch (error) { - const details = error instanceof Error ? error.message : String(error); - steps.push({ - name: step.name, - status: "fail", - details, - }); - return { - name: definition.name, - status: "fail", - steps, - details, - }; - } - } - - return { - name: definition.name, - status: "pass", - steps, - }; -} +export * from "../../extensions/qa-lab/api.js"; diff --git a/src/qa-e2e/self-check-scenario.ts b/src/qa-e2e/self-check-scenario.ts index 7aec1f81f60..57d9d5c96c8 100644 --- a/src/qa-e2e/self-check-scenario.ts +++ b/src/qa-e2e/self-check-scenario.ts @@ -1,122 +1 @@ -import { qaChannelPlugin } from "../../extensions/qa-channel/api.js"; -import type { OpenClawConfig } from "../config/config.js"; -import { extractToolPayload } from "../infra/outbound/tool-payload.js"; -import type { QaScenarioDefinition } from "./scenario.js"; - -export function createQaSelfCheckScenario(cfg: OpenClawConfig): QaScenarioDefinition { - return { - name: "Synthetic Slack-class roundtrip", - steps: [ - { - name: "DM echo roundtrip", - async run({ state }) { - state.addInboundMessage({ - conversation: { id: "alice", kind: "direct" }, - senderId: "alice", - senderName: "Alice", - text: "hello from qa", - }); - await state.waitFor({ - kind: "message-text", - textIncludes: "qa-echo: hello from qa", - direction: "outbound", - timeoutMs: 5_000, - }); - }, - }, - { - name: "Thread create and threaded echo", - async run({ state }) { - const threadResult = await qaChannelPlugin.actions?.handleAction?.({ - channel: "qa-channel", - action: "thread-create", - cfg, - accountId: "default", - params: { - channelId: "qa-room", - title: "QA thread", - }, - }); - const threadPayload = (threadResult ? extractToolPayload(threadResult) : undefined) as - | { thread?: { id?: string } } - | undefined; - const threadId = threadPayload?.thread?.id; - if (!threadId) { - throw new Error("thread-create did not return thread id"); - } - - state.addInboundMessage({ - conversation: { id: "qa-room", kind: "channel", title: "QA Room" }, - senderId: "alice", - senderName: "Alice", - text: "inside thread", - threadId, - threadTitle: "QA thread", - }); - await state.waitFor({ - kind: "message-text", - textIncludes: "qa-echo: inside thread", - direction: "outbound", - timeoutMs: 5_000, - }); - return threadId; - }, - }, - { - name: "Reaction, edit, delete lifecycle", - async run({ state }) { - const outbound = state - .searchMessages({ query: "qa-echo: inside thread", conversationId: "qa-room" }) - .at(-1); - if (!outbound) { - throw new Error("threaded outbound message not found"); - } - - await qaChannelPlugin.actions?.handleAction?.({ - channel: "qa-channel", - action: "react", - cfg, - accountId: "default", - params: { - messageId: outbound.id, - emoji: "white_check_mark", - }, - }); - const reacted = state.readMessage({ messageId: outbound.id }); - if (reacted.reactions.length === 0) { - throw new Error("reaction not recorded"); - } - - await qaChannelPlugin.actions?.handleAction?.({ - channel: "qa-channel", - action: "edit", - cfg, - accountId: "default", - params: { - messageId: outbound.id, - text: "qa-echo: inside thread (edited)", - }, - }); - const edited = state.readMessage({ messageId: outbound.id }); - if (!edited.text.includes("(edited)")) { - throw new Error("edit not recorded"); - } - - await qaChannelPlugin.actions?.handleAction?.({ - channel: "qa-channel", - action: "delete", - cfg, - accountId: "default", - params: { - messageId: outbound.id, - }, - }); - const deleted = state.readMessage({ messageId: outbound.id }); - if (!deleted.deleted) { - throw new Error("delete not recorded"); - } - }, - }, - ], - }; -} +export * from "../../extensions/qa-lab/api.js";