test: audit heartbeat config honor

This commit is contained in:
Tak Hoffman 2026-04-03 09:54:14 -05:00
parent 35cf7d0340
commit 759c81ceb8
No known key found for this signature in database
6 changed files with 407 additions and 0 deletions

View File

@ -0,0 +1,139 @@
import fs from "node:fs";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { GENERATED_BASE_CONFIG_SCHEMA } from "./schema.base.generated.js";
export type ConfigHonorInventoryRow = {
key: string;
schemaPaths: string[];
typePaths: string[];
mergePaths: string[];
consumerPaths: string[];
reloadPaths: string[];
testPaths: string[];
notes?: string[];
};
type ConfigHonorProofKey =
| "schemaPaths"
| "typePaths"
| "mergePaths"
| "consumerPaths"
| "reloadPaths"
| "testPaths";
export type ConfigHonorAuditResult = {
schemaKeys: string[];
missingKeys: string[];
extraKeys: string[];
missingSchemaPaths: string[];
missingFiles: string[];
missingProofs: Array<{
key: string;
missing: ConfigHonorProofKey[];
}>;
};
const REPO_ROOT = fileURLToPath(new URL("../../", import.meta.url));
function hasSchemaPath(schemaPath: string): boolean {
const segments = schemaPath.split(".");
let current: unknown = GENERATED_BASE_CONFIG_SCHEMA.schema;
for (const segment of segments) {
if (!current || typeof current !== "object") {
return false;
}
if (segment === "*") {
const items = (current as { items?: unknown }).items;
if (!items || typeof items !== "object") {
return false;
}
current = items;
continue;
}
const properties = (current as { properties?: Record<string, unknown> }).properties;
if (!properties || !Object.hasOwn(properties, segment)) {
return false;
}
current = properties[segment];
}
return true;
}
export function listSchemaLeafKeysForPrefixes(prefixes: string[]): string[] {
const keys = new Set<string>();
for (const prefix of prefixes) {
const segments = prefix.split(".");
let current: unknown = GENERATED_BASE_CONFIG_SCHEMA.schema;
for (const segment of segments) {
if (!current || typeof current !== "object") {
current = null;
break;
}
if (segment === "*") {
current = (current as { items?: unknown }).items ?? null;
continue;
}
current = (current as { properties?: Record<string, unknown> }).properties?.[segment] ?? null;
}
const properties = (current as { properties?: Record<string, unknown> } | null)?.properties;
if (!properties) {
continue;
}
for (const key of Object.keys(properties)) {
keys.add(key);
}
}
return [...keys].toSorted();
}
export function auditConfigHonorInventory(params: {
prefixes: string[];
rows: ConfigHonorInventoryRow[];
expectedKeys?: string[];
repoRoot?: string;
}): ConfigHonorAuditResult {
const repoRoot = params.repoRoot ?? REPO_ROOT;
const schemaKeys = listSchemaLeafKeysForPrefixes(params.prefixes);
const expectedKeys = new Set(params.expectedKeys ?? schemaKeys);
const rowKeys = new Set(params.rows.map((row) => row.key));
const missingKeys = [...expectedKeys].filter((key) => !rowKeys.has(key)).toSorted();
const extraKeys = params.rows
.map((row) => row.key)
.filter((key) => !expectedKeys.has(key))
.toSorted();
const missingSchemaPaths = params.rows.flatMap((row) =>
row.schemaPaths.filter((schemaPath) => !hasSchemaPath(schemaPath)),
);
const missingFiles = params.rows.flatMap((row) => {
const files = [...row.typePaths, ...row.mergePaths, ...row.consumerPaths, ...row.testPaths];
return files
.filter((relativePath) => !fs.existsSync(path.join(repoRoot, relativePath)))
.map((relativePath) => `${row.key}:${relativePath}`);
});
const missingProofs = params.rows
.map((row) => {
const missing: ConfigHonorProofKey[] = [
row.schemaPaths.length === 0 ? "schemaPaths" : null,
row.typePaths.length === 0 ? "typePaths" : null,
row.mergePaths.length === 0 ? "mergePaths" : null,
row.consumerPaths.length === 0 ? "consumerPaths" : null,
row.reloadPaths.length === 0 ? "reloadPaths" : null,
row.testPaths.length === 0 ? "testPaths" : null,
].filter((value): value is ConfigHonorProofKey => value !== null);
return missing.length > 0 ? { key: row.key, missing } : null;
})
.filter((row): row is NonNullable<typeof row> => row !== null);
return {
schemaKeys,
missingKeys,
extraKeys,
missingSchemaPaths,
missingFiles,
missingProofs,
};
}

View File

@ -0,0 +1,44 @@
import { describe, expect, it } from "vitest";
import { auditConfigHonorInventory, listSchemaLeafKeysForPrefixes } from "./config-honor-audit.js";
import {
HEARTBEAT_CONFIG_HONOR_INVENTORY,
HEARTBEAT_CONFIG_PREFIXES,
} from "./heartbeat-config-honor.inventory.js";
const EXPECTED_HEARTBEAT_KEYS = [
"every",
"model",
"prompt",
"ackMaxChars",
"suppressToolErrorWarnings",
"lightContext",
"isolatedSession",
"target",
"to",
"accountId",
"directPolicy",
"includeReasoning",
] as const;
describe("heartbeat config-honor inventory", () => {
it("keeps the planned heartbeat audit slice aligned with schema leaf keys", () => {
const schemaKeys = listSchemaLeafKeysForPrefixes([...HEARTBEAT_CONFIG_PREFIXES]);
for (const key of EXPECTED_HEARTBEAT_KEYS) {
expect(schemaKeys).toContain(key);
}
});
it("covers the planned heartbeat keys with runtime, reload, and test proofs", () => {
const audit = auditConfigHonorInventory({
prefixes: [...HEARTBEAT_CONFIG_PREFIXES],
expectedKeys: [...EXPECTED_HEARTBEAT_KEYS],
rows: HEARTBEAT_CONFIG_HONOR_INVENTORY,
});
expect(audit.missingKeys).toEqual([]);
expect(audit.extraKeys).toEqual([]);
expect(audit.missingSchemaPaths).toEqual([]);
expect(audit.missingFiles).toEqual([]);
expect(audit.missingProofs).toEqual([]);
});
});

View File

@ -0,0 +1,142 @@
import type { ConfigHonorInventoryRow } from "./config-honor-audit.js";
export const HEARTBEAT_CONFIG_PREFIXES = [
"agents.defaults.heartbeat",
"agents.list.*.heartbeat",
] as const;
export const HEARTBEAT_CONFIG_HONOR_INVENTORY: ConfigHonorInventoryRow[] = [
{
key: "every",
schemaPaths: ["agents.defaults.heartbeat.every", "agents.list.*.heartbeat.every"],
typePaths: ["src/config/types.agent-defaults.ts", "src/config/zod-schema.agent-runtime.ts"],
mergePaths: ["src/infra/heartbeat-runner.ts", "src/agents/acp-spawn.ts"],
consumerPaths: ["src/infra/heartbeat-runner.ts", "src/agents/acp-spawn.ts"],
reloadPaths: ["src/gateway/config-reload-plan.ts"],
testPaths: [
"src/infra/heartbeat-runner.returns-default-unset.test.ts",
"src/gateway/config-reload.test.ts",
],
},
{
key: "model",
schemaPaths: ["agents.defaults.heartbeat.model", "agents.list.*.heartbeat.model"],
typePaths: ["src/config/types.agent-defaults.ts", "src/config/zod-schema.agent-runtime.ts"],
mergePaths: ["src/infra/heartbeat-runner.ts"],
consumerPaths: ["src/infra/heartbeat-runner.ts"],
reloadPaths: ["src/gateway/config-reload-plan.ts"],
testPaths: [
"src/infra/heartbeat-runner.model-override.test.ts",
"src/gateway/config-reload.test.ts",
],
},
{
key: "prompt",
schemaPaths: ["agents.defaults.heartbeat.prompt", "agents.list.*.heartbeat.prompt"],
typePaths: ["src/config/types.agent-defaults.ts", "src/config/zod-schema.agent-runtime.ts"],
mergePaths: ["src/infra/heartbeat-runner.ts"],
consumerPaths: ["src/infra/heartbeat-runner.ts"],
reloadPaths: ["src/gateway/config-reload-plan.ts"],
testPaths: ["src/infra/heartbeat-runner.returns-default-unset.test.ts"],
},
{
key: "ackMaxChars",
schemaPaths: ["agents.defaults.heartbeat.ackMaxChars", "agents.list.*.heartbeat.ackMaxChars"],
typePaths: ["src/config/types.agent-defaults.ts", "src/config/zod-schema.agent-runtime.ts"],
mergePaths: ["src/infra/heartbeat-runner.ts"],
consumerPaths: ["src/infra/heartbeat-runner.ts"],
reloadPaths: ["src/gateway/config-reload-plan.ts"],
testPaths: ["src/infra/heartbeat-runner.respects-ackmaxchars-heartbeat-acks.test.ts"],
},
{
key: "suppressToolErrorWarnings",
schemaPaths: [
"agents.defaults.heartbeat.suppressToolErrorWarnings",
"agents.list.*.heartbeat.suppressToolErrorWarnings",
],
typePaths: ["src/config/types.agent-defaults.ts", "src/config/zod-schema.agent-runtime.ts"],
mergePaths: ["src/infra/heartbeat-runner.ts"],
consumerPaths: ["src/infra/heartbeat-runner.ts"],
reloadPaths: ["src/gateway/config-reload-plan.ts"],
testPaths: ["src/infra/heartbeat-runner.model-override.test.ts"],
},
{
key: "lightContext",
schemaPaths: ["agents.defaults.heartbeat.lightContext", "agents.list.*.heartbeat.lightContext"],
typePaths: ["src/config/types.agent-defaults.ts", "src/config/zod-schema.agent-runtime.ts"],
mergePaths: ["src/infra/heartbeat-runner.ts"],
consumerPaths: ["src/infra/heartbeat-runner.ts", "src/agents/bootstrap-files.ts"],
reloadPaths: ["src/gateway/config-reload-plan.ts"],
testPaths: [
"src/infra/heartbeat-runner.model-override.test.ts",
"src/agents/bootstrap-files.test.ts",
"src/gateway/config-reload.test.ts",
],
},
{
key: "isolatedSession",
schemaPaths: [
"agents.defaults.heartbeat.isolatedSession",
"agents.list.*.heartbeat.isolatedSession",
],
typePaths: ["src/config/types.agent-defaults.ts", "src/config/zod-schema.agent-runtime.ts"],
mergePaths: ["src/infra/heartbeat-runner.ts"],
consumerPaths: ["src/infra/heartbeat-runner.ts"],
reloadPaths: ["src/gateway/config-reload-plan.ts"],
testPaths: ["src/infra/heartbeat-runner.model-override.test.ts"],
},
{
key: "target",
schemaPaths: ["agents.defaults.heartbeat.target", "agents.list.*.heartbeat.target"],
typePaths: ["src/config/types.agent-defaults.ts", "src/config/zod-schema.agent-runtime.ts"],
mergePaths: ["src/infra/heartbeat-runner.ts", "src/infra/outbound/targets.ts"],
consumerPaths: ["src/infra/outbound/targets.ts", "src/infra/heartbeat-runner.ts"],
reloadPaths: ["src/gateway/config-reload-plan.ts"],
testPaths: [
"src/infra/heartbeat-runner.returns-default-unset.test.ts",
"src/cron/service.main-job-passes-heartbeat-target-last.test.ts",
],
},
{
key: "to",
schemaPaths: ["agents.defaults.heartbeat.to", "agents.list.*.heartbeat.to"],
typePaths: ["src/config/types.agent-defaults.ts", "src/config/zod-schema.agent-runtime.ts"],
mergePaths: ["src/infra/heartbeat-runner.ts", "src/infra/outbound/targets.ts"],
consumerPaths: ["src/infra/outbound/targets.ts"],
reloadPaths: ["src/gateway/config-reload-plan.ts"],
testPaths: ["src/infra/heartbeat-runner.returns-default-unset.test.ts"],
},
{
key: "accountId",
schemaPaths: ["agents.defaults.heartbeat.accountId", "agents.list.*.heartbeat.accountId"],
typePaths: ["src/config/types.agent-defaults.ts", "src/config/zod-schema.agent-runtime.ts"],
mergePaths: ["src/infra/heartbeat-runner.ts", "src/infra/outbound/targets.ts"],
consumerPaths: ["src/infra/outbound/targets.ts", "src/infra/heartbeat-runner.ts"],
reloadPaths: ["src/gateway/config-reload-plan.ts"],
testPaths: [
"src/infra/heartbeat-runner.returns-default-unset.test.ts",
"src/infra/heartbeat-runner.respects-ackmaxchars-heartbeat-acks.test.ts",
],
},
{
key: "directPolicy",
schemaPaths: ["agents.defaults.heartbeat.directPolicy", "agents.list.*.heartbeat.directPolicy"],
typePaths: ["src/config/types.agent-defaults.ts", "src/config/zod-schema.agent-runtime.ts"],
mergePaths: ["src/infra/heartbeat-runner.ts", "src/infra/outbound/targets.ts"],
consumerPaths: ["src/infra/outbound/targets.ts"],
reloadPaths: ["src/gateway/config-reload-plan.ts"],
testPaths: ["src/infra/heartbeat-runner.returns-default-unset.test.ts"],
},
{
key: "includeReasoning",
schemaPaths: [
"agents.defaults.heartbeat.includeReasoning",
"agents.list.*.heartbeat.includeReasoning",
],
typePaths: ["src/config/types.agent-defaults.ts", "src/config/zod-schema.agent-runtime.ts"],
mergePaths: ["src/infra/heartbeat-runner.ts"],
consumerPaths: ["src/infra/heartbeat-runner.ts"],
reloadPaths: ["src/gateway/config-reload-plan.ts"],
testPaths: ["src/infra/heartbeat-runner.returns-default-unset.test.ts"],
},
];

View File

@ -73,6 +73,11 @@ const BASE_RELOAD_RULES: ReloadRule[] = [
kind: "hot",
actions: ["restart-heartbeat"],
},
{
prefix: "agents.list",
kind: "hot",
actions: ["restart-heartbeat"],
},
{ prefix: "agent.heartbeat", kind: "hot", actions: ["restart-heartbeat"] },
{ prefix: "cron", kind: "hot", actions: ["restart-cron"] },
{ prefix: "browser", kind: "restart" },

View File

@ -68,6 +68,21 @@ describe("diffConfigPaths", () => {
};
expect(diffConfigPaths(prev, next)).toContain("memory.qmd.paths");
});
it("collapses changed agents.list heartbeat entries to agents.list", () => {
const prev = {
agents: {
list: [{ id: "ops", heartbeat: { every: "5m", lightContext: false } }],
},
};
const next = {
agents: {
list: [{ id: "ops", heartbeat: { every: "5m", lightContext: true } }],
},
};
expect(diffConfigPaths(prev, next)).toEqual(["agents.list"]);
});
});
describe("buildGatewayReloadPlan", () => {
@ -174,6 +189,14 @@ describe("buildGatewayReloadPlan", () => {
expect(plan.noopPaths).toEqual([]);
});
it("restarts heartbeat when agents.list entries change", () => {
const plan = buildGatewayReloadPlan(["agents.list"]);
expect(plan.restartGateway).toBe(false);
expect(plan.restartHeartbeat).toBe(true);
expect(plan.hotReasons).toContain("agents.list");
expect(plan.noopPaths).toEqual([]);
});
it("hot-reloads health monitor when channelHealthCheckMinutes changes", () => {
const plan = buildGatewayReloadPlan(["gateway.channelHealthCheckMinutes"]);
expect(plan.restartGateway).toBe(false);
@ -235,6 +258,12 @@ describe("buildGatewayReloadPlan", () => {
expectRestartGmailWatcher: true,
expectReloadHooks: true,
},
{
path: "agents.list",
expectRestartGateway: false,
expectHotPath: "agents.list",
expectRestartHeartbeat: true,
},
{
path: "gateway.remote.url",
expectRestartGateway: false,
@ -271,6 +300,9 @@ describe("buildGatewayReloadPlan", () => {
if (testCase.expectReloadHooks) {
expect(plan.reloadHooks).toBe(true);
}
if (testCase.expectRestartHeartbeat) {
expect(plan.restartHeartbeat).toBe(true);
}
});
});

View File

@ -235,6 +235,51 @@ describe("runHeartbeatOnce heartbeat model override", () => {
});
});
it("passes per-agent heartbeat lightContext override after merging defaults", async () => {
await withHeartbeatFixture(async ({ tmpDir, storePath, seedSession }) => {
const cfg: OpenClawConfig = {
agents: {
defaults: {
heartbeat: {
every: "30m",
lightContext: false,
},
},
list: [
{ id: "main", default: true },
{
id: "ops",
workspace: tmpDir,
heartbeat: {
every: "5m",
target: "whatsapp",
lightContext: true,
},
},
],
},
channels: { whatsapp: { allowFrom: ["*"] } },
session: { store: storePath },
};
const sessionKey = resolveAgentMainSessionKey({ cfg, agentId: "ops" });
const result = await runHeartbeatWithSeed({
seedSession,
cfg,
agentId: "ops",
sessionKey,
});
expect(result.replySpy).toHaveBeenCalledWith(
expect.any(Object),
expect.objectContaining({
isHeartbeat: true,
bootstrapContextMode: "lightweight",
}),
cfg,
);
});
});
it("does not pass heartbeatModelOverride when no heartbeat model is configured", async () => {
const replyOpts = await runDefaultsHeartbeat({ model: undefined });
expect(replyOpts).toEqual(