From af94a3a89bece1e1192e533e5fa3ad41d6c0f763 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sat, 4 Apr 2026 04:01:17 +0100 Subject: [PATCH] test: use native vitest root projects --- AGENTS.md | 4 ++-- docs/help/testing.md | 10 ++++----- docs/pi-dev.md | 4 ++-- docs/reference/test.md | 6 +++--- package.json | 16 +++++++-------- test/vitest-config.test.ts | 22 ++++++++++++++++++-- test/vitest-projects-config.test.ts | 11 +++------- vitest.acp.config.ts | 1 + vitest.agents.config.ts | 1 + vitest.auto-reply.config.ts | 1 + vitest.bundled.config.ts | 1 + vitest.channels.config.ts | 1 + vitest.commands.config.ts | 1 + vitest.config.ts | 23 +++++++++++++++------ vitest.contracts.config.ts | 3 +++ vitest.extension-channels.config.ts | 1 + vitest.extensions.config.ts | 1 + vitest.gateway.config.ts | 1 + vitest.scoped-config.ts | 2 ++ vitest.shared.config.ts | 32 ++++++++++++++++++++--------- vitest.ui.config.ts | 1 + vitest.unit.config.ts | 3 ++- 22 files changed, 99 insertions(+), 47 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 55e4ffabb9e..257b73530d3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -204,10 +204,10 @@ - Test performance guardrail: prefer narrow public SDK subpaths such as `models-provider-runtime`, `skill-commands-runtime`, and `reply-dispatch-runtime` over older broad helper barrels when both expose the needed helper. - Test performance guardrail: treat import-dominated test time as a boundary bug. Refactor the import surface before adding more cases to the slow file. - Agents MUST NOT modify baseline, inventory, ignore, snapshot, or expected-failure files to silence failing checks without explicit approval in this chat. -- For targeted/local debugging, keep using the wrapper: `pnpm test -- [vitest args...]` (for example `pnpm test -- src/commands/onboard-search.test.ts -t "shows registered plugin providers"`); do not default to raw `pnpm vitest run ...` because it bypasses wrapper config/profile/pool routing. +- For targeted/local debugging, use the native root-project entrypoint: `pnpm test [vitest args...]` (for example `pnpm test src/commands/onboard-search.test.ts -t "shows registered plugin providers"`); do not default to raw `pnpm vitest run ...` because it bypasses the repo's default config/profile/pool routing. - Do not set test workers above 16; tried already. - Keep Vitest on `forks` only. Do not introduce or reintroduce any non-`forks` Vitest pool or alternate execution mode in configs, wrapper scripts, or default test commands without explicit approval in this chat. This includes `threads`, `vmThreads`, `vmForks`, and any future/nonstandard pool variant. -- If local Vitest runs cause memory pressure, the wrapper now derives budgets from host capabilities (CPU, memory band, current load). For a conservative explicit override during land/gate runs, use `OPENCLAW_TEST_PROFILE=serial OPENCLAW_TEST_SERIAL_GATEWAY=1 pnpm test`. +- If local Vitest runs cause memory pressure, the default worker budget now derives from host capabilities (CPU, memory band, current load). For a conservative explicit override during land/gate runs, use `OPENCLAW_VITEST_MAX_WORKERS=1 pnpm test`. - Live tests (real keys): `OPENCLAW_LIVE_TEST=1 pnpm test:live` (OpenClaw-only) or `LIVE=1 pnpm test:live` (includes provider live tests). Docker: `pnpm test:docker:live-models`, `pnpm test:docker:live-gateway`. Onboarding Docker E2E: `pnpm test:docker:onboard`. - `pnpm test:live` defaults quiet now. Keep `[live]` progress; suppress profile/gateway chatter. Full logs: `OPENCLAW_LIVE_TEST_QUIET=0 pnpm test:live`. - Full kit + what’s covered: `docs/help/testing.md`. diff --git a/docs/help/testing.md b/docs/help/testing.md index 4034e9f80f7..ac7cfa7ccd6 100644 --- a/docs/help/testing.md +++ b/docs/help/testing.md @@ -25,7 +25,7 @@ Most days: - Full gate (expected before push): `pnpm build && pnpm check && pnpm test` - Faster local full-suite run on a roomy machine: `pnpm test:max` - Direct Vitest watch loop (modern projects config): `pnpm test:watch` -- Direct file targeting now routes extension/channel paths too: `pnpm test -- extensions/discord/src/monitor/message-handler.preflight.test.ts` +- Direct file targeting now routes extension/channel paths too: `pnpm test extensions/discord/src/monitor/message-handler.preflight.test.ts` When you touch tests or want extra confidence: @@ -46,7 +46,7 @@ Think of the suites as “increasing realism” (and increasing flakiness/cost): ### Unit / integration (default) - Command: `pnpm test` -- Config: native Vitest `projects` via `vitest.config.ts` (`unit` + `boundary`) +- Config: native Vitest `projects` via `vitest.config.ts` - Files: core/unit inventories under `src/**/*.test.ts`, `packages/**/*.test.ts`, `test/**/*.test.ts`, and the whitelisted `ui` node tests covered by `vitest.unit.config.ts` - Scope: - Pure unit tests @@ -57,9 +57,8 @@ Think of the suites as “increasing realism” (and increasing flakiness/cost): - No real keys required - Should be fast and stable - Projects note: - - `pnpm test` and `pnpm test:watch` both use the same native Vitest `projects` config now. - - The tiny script wrapper still keeps scheduling native, but it now reroutes direct `extensions/...` and channel-surface test paths onto the matching Vitest lane automatically. - - If you target mixed suites in one command, the wrapper runs those lanes sequentially under the same local heavy-check lock. + - `pnpm test`, `pnpm test:watch`, and `pnpm test:changed` all use the same native Vitest root `projects` config now. + - Direct file filters route natively through the root project graph, so `pnpm test extensions/discord/src/monitor/message-handler.preflight.test.ts` works without a custom wrapper. - Embedded runner note: - When you change message-tool discovery inputs or compaction runtime context, keep both levels of coverage. @@ -82,6 +81,7 @@ Think of the suites as “increasing realism” (and increasing flakiness/cost): - Fast-local iteration note: - `pnpm test:changed` runs the native projects config with `--changed origin/main`. - `pnpm test:max` and `pnpm test:changed:max` keep the same native projects config, just with a higher worker cap. + - Local worker auto-scaling is intentionally conservative now and also backs off when the host load average is already high, so multiple concurrent Vitest runs do less damage by default. - The base Vitest config marks the projects/config files as `forceRerunTriggers` so changed-mode reruns stay correct when test wiring changes. - The config keeps `OPENCLAW_VITEST_FS_MODULE_CACHE` enabled on supported hosts; set `OPENCLAW_VITEST_FS_MODULE_CACHE_PATH=/abs/path` if you want one explicit cache location for direct profiling. - Perf-debug note: diff --git a/docs/pi-dev.md b/docs/pi-dev.md index ee03c362e43..04df93e5357 100644 --- a/docs/pi-dev.md +++ b/docs/pi-dev.md @@ -22,7 +22,7 @@ This guide summarizes a sane workflow for working on the pi integration in OpenC Run the Pi-focused test set directly with Vitest: ```bash -pnpm test -- \ +pnpm test \ "src/agents/pi-*.test.ts" \ "src/agents/pi-embedded-*.test.ts" \ "src/agents/pi-tools*.test.ts" \ @@ -34,7 +34,7 @@ pnpm test -- \ To include the live provider exercise: ```bash -OPENCLAW_LIVE_TEST=1 pnpm test -- src/agents/pi-embedded-runner-extraparams.live.test.ts +OPENCLAW_LIVE_TEST=1 pnpm test src/agents/pi-embedded-runner-extraparams.live.test.ts ``` This covers the main Pi unit suites: diff --git a/docs/reference/test.md b/docs/reference/test.md index 80760215a50..5e248b23bf7 100644 --- a/docs/reference/test.md +++ b/docs/reference/test.md @@ -13,12 +13,12 @@ title: "Tests" - `pnpm test:coverage`: Runs the unit suite with V8 coverage (via `vitest.unit.config.ts`). Global thresholds are 70% lines/branches/functions/statements. Coverage excludes integration-heavy entrypoints (CLI wiring, gateway/telegram bridges, webchat static server) to keep the target focused on unit-testable logic. - `pnpm test:coverage:changed`: Runs unit coverage only for files changed since `origin/main`. - `pnpm test:changed`: runs the native Vitest projects config with `--changed origin/main`. The base config treats the projects/config files as `forceRerunTriggers` so wiring changes still rerun broadly when needed. -- `pnpm test`: runs the native Vitest projects config (`unit` + `boundary`) via a tiny passthrough wrapper so `pnpm test -- ` keeps working. +- `pnpm test`: runs the native Vitest root projects config directly. File filters work natively across the configured projects. - Unit, channel, and extension configs default to `pool: "forks"`. - `pnpm test:channels` runs `vitest.channels.config.ts`. - `pnpm test:extensions` runs `vitest.extensions.config.ts`. - `pnpm test:extensions`: runs extension/plugin suites. -- `pnpm test:perf:imports`: enables Vitest import-duration + import-breakdown reporting for the wrapper. +- `pnpm test:perf:imports`: enables Vitest import-duration + import-breakdown reporting for the native root projects run. - `pnpm test:perf:imports:changed`: same import profiling, but only for files changed since `origin/main`. - `pnpm test:perf:profile:main`: writes a CPU profile for the Vitest main thread (`.artifacts/vitest-main-profile`). - `pnpm test:perf:profile:runner`: writes CPU + heap profiles for the unit runner (`.artifacts/vitest-runner-profile`). @@ -37,7 +37,7 @@ For local PR land/gate checks, run: - `pnpm test` - `pnpm check:docs` -If `pnpm test` flakes on a loaded host, rerun once before treating it as a regression, then isolate with `pnpm test -- `. For memory-constrained hosts, use: +If `pnpm test` flakes on a loaded host, rerun once before treating it as a regression, then isolate with `pnpm test `. For memory-constrained hosts, use: - `OPENCLAW_VITEST_MAX_WORKERS=1 pnpm test` - `OPENCLAW_VITEST_FS_MODULE_CACHE_PATH=/tmp/openclaw-vitest-cache pnpm test:changed` diff --git a/package.json b/package.json index a8eb40e6407..b2222d56b40 100644 --- a/package.json +++ b/package.json @@ -1083,13 +1083,13 @@ "runtime-sidecars:gen": "node --import tsx scripts/generate-runtime-sidecar-paths-baseline.ts --write", "stage:bundled-plugin-runtime-deps": "node scripts/stage-bundled-plugin-runtime-deps.mjs", "start": "node scripts/run-node.mjs", - "test": "node scripts/test-projects.mjs", + "test": "vitest run --config vitest.config.ts", "test:all": "pnpm lint && pnpm build && pnpm test && pnpm test:e2e && pnpm test:live && pnpm test:docker:all", "test:auth:compat": "vitest run --config vitest.gateway.config.ts src/gateway/server.auth.compat-baseline.test.ts src/gateway/client.test.ts src/gateway/reconnect-gating.test.ts src/gateway/protocol/connect-error-details.test.ts", "test:build:singleton": "node scripts/test-built-plugin-singleton.mjs", "test:bundled": "vitest run --config vitest.bundled.config.ts", - "test:changed": "node scripts/test-projects.mjs --changed origin/main", - "test:changed:max": "OPENCLAW_VITEST_MAX_WORKERS=8 node scripts/test-projects.mjs --changed origin/main", + "test:changed": "vitest run --config vitest.config.ts --changed origin/main", + "test:changed:max": "OPENCLAW_VITEST_MAX_WORKERS=8 vitest run --config vitest.config.ts --changed origin/main", "test:channels": "vitest run --config vitest.channels.config.ts", "test:contracts": "pnpm test:contracts:channels && pnpm test:contracts:plugins", "test:contracts:channels": "pnpm exec vitest run --config vitest.contracts.config.ts --maxWorkers=1 src/channels/plugins/contracts", @@ -1127,19 +1127,19 @@ "test:live": "node scripts/test-live.mjs", "test:live:gateway-profiles": "node scripts/test-live.mjs -- src/gateway/gateway-models.profiles.live.test.ts", "test:live:models-profiles": "node scripts/test-live.mjs -- src/agents/models.profiles.live.test.ts", - "test:max": "OPENCLAW_VITEST_MAX_WORKERS=8 node scripts/test-projects.mjs", + "test:max": "OPENCLAW_VITEST_MAX_WORKERS=8 vitest run --config vitest.config.ts", "test:parallels:linux": "bash scripts/e2e/parallels-linux-smoke.sh", "test:parallels:macos": "bash scripts/e2e/parallels-macos-smoke.sh", "test:parallels:npm-update": "bash scripts/e2e/parallels-npm-update-smoke.sh", "test:parallels:windows": "bash scripts/e2e/parallels-windows-smoke.sh", "test:perf:budget": "node scripts/test-perf-budget.mjs", "test:perf:hotspots": "node scripts/test-hotspots.mjs", - "test:perf:imports": "OPENCLAW_VITEST_IMPORT_DURATIONS=1 OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN=1 node scripts/test-projects.mjs", - "test:perf:imports:changed": "OPENCLAW_VITEST_IMPORT_DURATIONS=1 OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN=1 node scripts/test-projects.mjs --changed origin/main", + "test:perf:imports": "OPENCLAW_VITEST_IMPORT_DURATIONS=1 OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN=1 vitest run --config vitest.config.ts", + "test:perf:imports:changed": "OPENCLAW_VITEST_IMPORT_DURATIONS=1 OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN=1 vitest run --config vitest.config.ts --changed origin/main", "test:perf:profile:main": "node scripts/run-vitest-profile.mjs main", "test:perf:profile:runner": "node scripts/run-vitest-profile.mjs runner", "test:sectriage": "pnpm exec vitest run --config vitest.gateway.config.ts && vitest run --config vitest.unit.config.ts --exclude src/daemon/launchd.integration.test.ts --exclude src/process/exec.test.ts", - "test:serial": "OPENCLAW_VITEST_MAX_WORKERS=1 node scripts/test-projects.mjs", + "test:serial": "OPENCLAW_VITEST_MAX_WORKERS=1 vitest run --config vitest.config.ts", "test:startup:bench": "node --import tsx scripts/bench-cli-startup.ts", "test:startup:bench:check": "node scripts/test-cli-startup-bench-budget.mjs", "test:startup:bench:save": "node --import tsx scripts/bench-cli-startup.ts --preset all --runs 5 --warmup 1 --output .artifacts/cli-startup-bench-all.json", @@ -1148,7 +1148,7 @@ "test:startup:memory": "node scripts/check-cli-startup-memory.mjs", "test:ui": "pnpm lint:ui:no-raw-window-open && pnpm --dir ui test", "test:voicecall:closedloop": "node scripts/test-voicecall-closedloop.mjs", - "test:watch": "node scripts/test-projects.mjs --watch", + "test:watch": "vitest --config vitest.config.ts", "ts-topology": "node --import tsx scripts/ts-topology.ts", "tsgo": "node scripts/run-tsgo.mjs", "tui": "node scripts/run-node.mjs tui", diff --git a/test/vitest-config.test.ts b/test/vitest-config.test.ts index 3e1120d6e5f..3b2b43cd2da 100644 --- a/test/vitest-config.test.ts +++ b/test/vitest-config.test.ts @@ -10,10 +10,11 @@ describe("resolveLocalVitestMaxWorkers", () => { }, { cpuCount: 10, + loadAverage1m: 0, totalMemoryBytes: 64 * 1024 ** 3, }, ), - ).toBe(4); + ).toBe(3); }); it("lets OPENCLAW_VITEST_MAX_WORKERS override the inferred cap", () => { @@ -24,6 +25,7 @@ describe("resolveLocalVitestMaxWorkers", () => { }, { cpuCount: 10, + loadAverage1m: 0, totalMemoryBytes: 128 * 1024 ** 3, platform: "darwin", }, @@ -39,6 +41,7 @@ describe("resolveLocalVitestMaxWorkers", () => { }, { cpuCount: 16, + loadAverage1m: 0, totalMemoryBytes: 128 * 1024 ** 3, }, ), @@ -51,6 +54,7 @@ describe("resolveLocalVitestMaxWorkers", () => { {}, { cpuCount: 16, + loadAverage1m: 0, totalMemoryBytes: 16 * 1024 ** 3, }, ), @@ -63,10 +67,24 @@ describe("resolveLocalVitestMaxWorkers", () => { {}, { cpuCount: 16, + loadAverage1m: 0, totalMemoryBytes: 128 * 1024 ** 3, }, ), - ).toBe(8); + ).toBe(4); + }); + + it("backs off further when the host is already busy", () => { + expect( + resolveLocalVitestMaxWorkers( + {}, + { + cpuCount: 16, + loadAverage1m: 16, + totalMemoryBytes: 128 * 1024 ** 3, + }, + ), + ).toBe(2); }); }); diff --git a/test/vitest-projects-config.test.ts b/test/vitest-projects-config.test.ts index 22783455377..846655def6b 100644 --- a/test/vitest-projects-config.test.ts +++ b/test/vitest-projects-config.test.ts @@ -1,13 +1,8 @@ import { describe, expect, it } from "vitest"; -import baseConfig from "../vitest.config.ts"; +import baseConfig, { rootVitestProjects } from "../vitest.config.ts"; describe("projects vitest config", () => { - it("defines unit, boundary, acp, and ui project config files at the root", () => { - expect(baseConfig.test?.projects).toEqual([ - "vitest.unit.config.ts", - "vitest.boundary.config.ts", - "vitest.acp.config.ts", - "vitest.ui.config.ts", - ]); + it("defines the native root project list for all non-live Vitest lanes", () => { + expect(baseConfig.test?.projects).toEqual([...rootVitestProjects]); }); }); diff --git a/vitest.acp.config.ts b/vitest.acp.config.ts index 1deded309ef..bfaac96c73a 100644 --- a/vitest.acp.config.ts +++ b/vitest.acp.config.ts @@ -4,6 +4,7 @@ export function createAcpVitestConfig(env?: Record) return createScopedVitestConfig(["src/acp/**/*.test.ts"], { dir: "src/acp", env, + name: "acp", }); } diff --git a/vitest.agents.config.ts b/vitest.agents.config.ts index 6eedaed9208..30a867c875e 100644 --- a/vitest.agents.config.ts +++ b/vitest.agents.config.ts @@ -4,6 +4,7 @@ export function createAgentsVitestConfig(env?: Record) { return createScopedVitestConfig( @@ -12,6 +13,8 @@ export function createContractsVitestConfig(env?: Record; exclude?: string[]; + name?: string; pool?: "threads" | "forks"; passWithNoTests?: boolean; setupFiles?: string[]; @@ -61,6 +62,7 @@ export function createScopedVitestConfig( ...base, test: { ...baseTest, + ...(options?.name ? { name: options.name } : {}), isolate, runner: "./test/non-isolated-runner.ts", setupFiles: [...new Set([...(baseTest.setupFiles ?? []), "test/setup-openclaw-runtime.ts"])], diff --git a/vitest.shared.config.ts b/vitest.shared.config.ts index 7e3e6f4be30..9c06b2ce31b 100644 --- a/vitest.shared.config.ts +++ b/vitest.shared.config.ts @@ -17,6 +17,7 @@ function parsePositiveInt(value: string | undefined): number | null { type VitestHostInfo = { cpuCount?: number; + loadAverage1m?: number; totalMemoryBytes?: number; }; @@ -24,6 +25,7 @@ function detectVitestHostInfo(): Required { return { cpuCount: typeof os.availableParallelism === "function" ? os.availableParallelism() : os.cpus().length, + loadAverage1m: os.loadavg()[0] ?? 0, totalMemoryBytes: os.totalmem(), }; } @@ -38,21 +40,32 @@ export function resolveLocalVitestMaxWorkers( } const cpuCount = Math.max(1, system.cpuCount ?? 1); + const loadAverage1m = Math.max(0, system.loadAverage1m ?? 0); const totalMemoryGb = (system.totalMemoryBytes ?? 0) / 1024 ** 3; - let inferred = cpuCount <= 4 ? cpuCount - 1 : Math.floor(cpuCount / 2); - inferred = clamp(inferred, 1, 8); + let inferred = + cpuCount <= 4 ? 1 : cpuCount <= 8 ? 2 : cpuCount <= 12 ? 3 : cpuCount <= 16 ? 4 : 6; if (totalMemoryGb <= 16) { - return Math.min(inferred, 2); + inferred = Math.min(inferred, 2); + } else if (totalMemoryGb <= 32) { + inferred = Math.min(inferred, 3); + } else if (totalMemoryGb <= 64) { + inferred = Math.min(inferred, 4); + } else if (totalMemoryGb <= 128) { + inferred = Math.min(inferred, 5); + } else { + inferred = Math.min(inferred, 6); } - if (totalMemoryGb <= 32) { - return Math.min(inferred, 3); + + const loadRatio = loadAverage1m > 0 ? loadAverage1m / cpuCount : 0; + if (loadRatio >= 1) { + inferred = Math.max(1, Math.floor(inferred / 2)); + } else if (loadRatio >= 0.75) { + inferred = Math.max(1, inferred - 1); } - if (totalMemoryGb <= 64) { - return Math.min(inferred, 4); - } - return Math.min(inferred, 8); + + return clamp(inferred, 1, 16); } const repoRoot = path.dirname(fileURLToPath(import.meta.url)); @@ -92,7 +105,6 @@ export const sharedVitestConfig = { "test/setup.shared.ts", "test/setup.extensions.ts", "test/setup-openclaw-runtime.ts", - "scripts/test-projects.mjs", "vitest.channel-paths.mjs", "vitest.channels.config.ts", "vitest.acp.config.ts", diff --git a/vitest.ui.config.ts b/vitest.ui.config.ts index 7c6895f963e..31af2db1dd6 100644 --- a/vitest.ui.config.ts +++ b/vitest.ui.config.ts @@ -4,6 +4,7 @@ export function createUiVitestConfig(env?: Record) { return createScopedVitestConfig(["ui/src/ui/**/*.test.ts"], { dir: "ui/src/ui", env, + name: "ui", }); } diff --git a/vitest.unit.config.ts b/vitest.unit.config.ts index 7ca1082faad..985f90cb13d 100644 --- a/vitest.unit.config.ts +++ b/vitest.unit.config.ts @@ -27,13 +27,14 @@ export function createUnitVitestConfigWithOptions( options: { includePatterns?: string[]; extraExcludePatterns?: string[]; + name?: string; } = {}, ) { return defineProject({ ...sharedVitestConfig, test: { ...sharedTest, - name: "unit", + name: options.name ?? "unit", isolate: resolveVitestIsolation(env), runner: "./test/non-isolated-runner.ts", setupFiles: [