diff --git a/package.json b/package.json index 30a47a0e476..d93011b8960 100644 --- a/package.json +++ b/package.json @@ -970,7 +970,7 @@ "android:run": "cd apps/android && ./gradlew :app:installPlayDebug && adb shell am start -n ai.openclaw.app/.MainActivity", "android:run:third-party": "cd apps/android && ./gradlew :app:installThirdPartyDebug && adb shell am start -n ai.openclaw.app/.MainActivity", "android:test": "cd apps/android && ./gradlew :app:testPlayDebugUnitTest", - "android:test:integration": "OPENCLAW_LIVE_TEST=1 OPENCLAW_LIVE_ANDROID_NODE=1 vitest run --config vitest.live.config.ts src/gateway/android-node.capabilities.live.test.ts", + "android:test:integration": "OPENCLAW_LIVE_TEST=1 OPENCLAW_LIVE_ANDROID_NODE=1 node scripts/run-vitest.mjs run --config vitest.live.config.ts src/gateway/android-node.capabilities.live.test.ts", "android:test:third-party": "cd apps/android && ./gradlew :app:testThirdPartyDebugUnitTest", "audit:seams": "node scripts/audit-seams.mjs", "build": "pnpm canvas:a2ui:bundle && node scripts/tsdown-build.mjs && node scripts/runtime-postbuild.mjs && node scripts/build-stamp.mjs && pnpm build:plugin-sdk:dts && node --import tsx scripts/write-plugin-sdk-entry-dts.ts && node scripts/check-plugin-sdk-exports.mjs && node --import tsx scripts/canvas-a2ui-copy.ts && node --import tsx scripts/copy-hook-metadata.ts && node --import tsx scripts/copy-export-html-templates.ts && node --import tsx scripts/write-build-info.ts && node --import tsx scripts/write-cli-startup-metadata.ts && node --import tsx scripts/write-cli-compat.ts", @@ -1093,17 +1093,17 @@ "start": "node scripts/run-node.mjs", "test": "node scripts/test-projects.mjs", "test:all": "pnpm lint && pnpm build && pnpm test && pnpm test:e2e && pnpm test:live && pnpm test:docker:all", - "test:auth:compat": "vitest run --config vitest.gateway.config.ts src/gateway/server.auth.compat-baseline.test.ts src/gateway/client.test.ts src/gateway/reconnect-gating.test.ts src/gateway/protocol/connect-error-details.test.ts", + "test:auth:compat": "node scripts/run-vitest.mjs run --config vitest.gateway.config.ts src/gateway/server.auth.compat-baseline.test.ts src/gateway/client.test.ts src/gateway/reconnect-gating.test.ts src/gateway/protocol/connect-error-details.test.ts", "test:build:singleton": "node scripts/test-built-plugin-singleton.mjs", - "test:bundled": "vitest run --config vitest.bundled.config.ts", - "test:changed": "vitest run --config vitest.config.ts --changed origin/main", - "test:changed:max": "OPENCLAW_VITEST_MAX_WORKERS=8 vitest run --config vitest.config.ts --changed origin/main", - "test:channels": "vitest run --config vitest.channels.config.ts", + "test:bundled": "node scripts/run-vitest.mjs run --config vitest.bundled.config.ts", + "test:changed": "node scripts/run-vitest.mjs run --config vitest.config.ts --changed origin/main", + "test:changed:max": "OPENCLAW_VITEST_MAX_WORKERS=8 node scripts/run-vitest.mjs run --config vitest.config.ts --changed origin/main", + "test:channels": "node scripts/run-vitest.mjs run --config vitest.channels.config.ts", "test:contracts": "pnpm test:contracts:channels && pnpm test:contracts:plugins", - "test:contracts:channels": "pnpm exec vitest run --config vitest.contracts.config.ts --maxWorkers=1 src/channels/plugins/contracts", - "test:contracts:plugins": "pnpm exec vitest run --config vitest.contracts.config.ts --maxWorkers=1 src/plugins/contracts", - "test:coverage": "vitest run --config vitest.unit.config.ts --coverage", - "test:coverage:changed": "vitest run --config vitest.unit.config.ts --coverage --changed origin/main", + "test:contracts:channels": "node scripts/run-vitest.mjs run --config vitest.contracts.config.ts --maxWorkers=1 src/channels/plugins/contracts", + "test:contracts:plugins": "node scripts/run-vitest.mjs run --config vitest.contracts.config.ts --maxWorkers=1 src/plugins/contracts", + "test:coverage": "node scripts/run-vitest.mjs run --config vitest.unit.config.ts --coverage", + "test:coverage:changed": "node scripts/run-vitest.mjs run --config vitest.unit.config.ts --coverage --changed origin/main", "test:docker:all": "pnpm test:docker:live-build && OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-models && OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:live-gateway && pnpm test:docker:openwebui && pnpm test:docker:onboard && pnpm test:docker:gateway-network && pnpm test:docker:mcp-channels && pnpm test:docker:qr && pnpm test:docker:doctor-switch && pnpm test:docker:plugins && pnpm test:docker:cleanup", "test:docker:cleanup": "bash scripts/test-cleanup-docker.sh", "test:docker:doctor-switch": "bash scripts/e2e/doctor-install-switch-docker.sh", @@ -1118,15 +1118,15 @@ "test:docker:openwebui": "bash scripts/e2e/openwebui-docker.sh", "test:docker:plugins": "bash scripts/e2e/plugins-docker.sh", "test:docker:qr": "bash scripts/e2e/qr-import-docker.sh", - "test:e2e": "vitest run --config vitest.e2e.config.ts", - "test:e2e:openshell": "OPENCLAW_E2E_OPENSHELL=1 vitest run --config vitest.e2e.config.ts test/openshell-sandbox.e2e.test.ts", + "test:e2e": "node scripts/run-vitest.mjs run --config vitest.e2e.config.ts", + "test:e2e:openshell": "OPENCLAW_E2E_OPENSHELL=1 node scripts/run-vitest.mjs run --config vitest.e2e.config.ts test/openshell-sandbox.e2e.test.ts", "test:extension": "node scripts/test-extension.mjs", - "test:extensions": "vitest run --config vitest.extensions.config.ts", + "test:extensions": "node scripts/run-vitest.mjs run --config vitest.extensions.config.ts", "test:extensions:batch": "node scripts/test-extension-batch.mjs", "test:extensions:memory": "node scripts/profile-extension-memory.mjs", - "test:fast": "vitest run --config vitest.unit.config.ts", + "test:fast": "node scripts/run-vitest.mjs run --config vitest.unit.config.ts", "test:force": "node --import tsx scripts/test-force.ts", - "test:gateway": "vitest run --config vitest.gateway.config.ts --pool=forks", + "test:gateway": "node scripts/run-vitest.mjs run --config vitest.gateway.config.ts --pool=forks", "test:gateway:watch-regression": "node scripts/check-gateway-watch-regression.mjs", "test:install:e2e": "bash scripts/test-install-sh-e2e-docker.sh", "test:install:e2e:anthropic": "OPENCLAW_E2E_MODELS=anthropic bash scripts/test-install-sh-e2e-docker.sh", @@ -1142,11 +1142,11 @@ "test:parallels:windows": "bash scripts/e2e/parallels-windows-smoke.sh", "test:perf:budget": "node scripts/test-perf-budget.mjs", "test:perf:hotspots": "node scripts/test-hotspots.mjs", - "test:perf:imports": "OPENCLAW_VITEST_IMPORT_DURATIONS=1 OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN=1 vitest run --config vitest.config.ts", - "test:perf:imports:changed": "OPENCLAW_VITEST_IMPORT_DURATIONS=1 OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN=1 vitest run --config vitest.config.ts --changed origin/main", + "test:perf:imports": "OPENCLAW_VITEST_IMPORT_DURATIONS=1 OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN=1 node scripts/run-vitest.mjs run --config vitest.config.ts", + "test:perf:imports:changed": "OPENCLAW_VITEST_IMPORT_DURATIONS=1 OPENCLAW_VITEST_PRINT_IMPORT_BREAKDOWN=1 node scripts/run-vitest.mjs run --config vitest.config.ts --changed origin/main", "test:perf:profile:main": "node scripts/run-vitest-profile.mjs main", "test:perf:profile:runner": "node scripts/run-vitest-profile.mjs runner", - "test:sectriage": "pnpm exec vitest run --config vitest.gateway.config.ts && vitest run --config vitest.unit.config.ts --exclude src/daemon/launchd.integration.test.ts --exclude src/process/exec.test.ts", + "test:sectriage": "node scripts/run-vitest.mjs run --config vitest.gateway.config.ts && node scripts/run-vitest.mjs run --config vitest.unit.config.ts --exclude src/daemon/launchd.integration.test.ts --exclude src/process/exec.test.ts", "test:serial": "OPENCLAW_VITEST_MAX_WORKERS=1 node scripts/test-projects.mjs", "test:startup:bench": "node --import tsx scripts/bench-cli-startup.ts", "test:startup:bench:check": "node scripts/test-cli-startup-bench-budget.mjs", diff --git a/scripts/run-vitest.mjs b/scripts/run-vitest.mjs new file mode 100644 index 00000000000..867585bd14f --- /dev/null +++ b/scripts/run-vitest.mjs @@ -0,0 +1,26 @@ +import { spawnPnpmRunner } from "./pnpm-runner.mjs"; + +const forwardedArgs = process.argv.slice(2); + +if (forwardedArgs.length === 0) { + console.error("usage: node scripts/run-vitest.mjs "); + process.exit(1); +} + +const child = spawnPnpmRunner({ + pnpmArgs: ["exec", "vitest", ...forwardedArgs], + env: process.env, +}); + +child.on("exit", (code, signal) => { + if (signal) { + process.kill(process.pid, signal); + return; + } + process.exit(code ?? 1); +}); + +child.on("error", (error) => { + console.error(error); + process.exit(1); +}); diff --git a/vitest.config.ts b/vitest.config.ts index 88e66992bff..9eac17abf8d 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -2,10 +2,11 @@ import { defineConfig } from "vitest/config"; import { resolveDefaultVitestPool, resolveLocalVitestMaxWorkers, + resolveLocalVitestScheduling, sharedVitestConfig, } from "./vitest.shared.config.ts"; -export { resolveDefaultVitestPool, resolveLocalVitestMaxWorkers }; +export { resolveDefaultVitestPool, resolveLocalVitestMaxWorkers, resolveLocalVitestScheduling }; export const rootVitestProjects = [ "vitest.unit.config.ts", diff --git a/vitest.shared.config.ts b/vitest.shared.config.ts index 05b8bf3c4bd..28097c2916a 100644 --- a/vitest.shared.config.ts +++ b/vitest.shared.config.ts @@ -7,6 +7,11 @@ import { BUNDLED_PLUGIN_TEST_GLOB, } from "./vitest.bundled-plugin-paths.ts"; import { loadVitestExperimentalConfig } from "./vitest.performance-config.ts"; +import { + detectVitestProcessStats, + shouldPrintVitestThrottle, + type VitestProcessStats, +} from "./vitest.system-load.ts"; const clamp = (value: number, min: number, max: number) => Math.max(min, Math.min(max, value)); @@ -15,6 +20,11 @@ function parsePositiveInt(value: string | undefined): number | null { return Number.isFinite(parsed) && parsed > 0 ? parsed : null; } +function isSystemThrottleDisabled(env: Record): boolean { + const normalized = env.OPENCLAW_VITEST_DISABLE_SYSTEM_THROTTLE?.trim().toLowerCase(); + return normalized === "1" || normalized === "true"; +} + type VitestHostInfo = { cpuCount?: number; loadAverage1m?: number; @@ -23,6 +33,12 @@ type VitestHostInfo = { export type OpenClawVitestPool = "forks"; +export type LocalVitestScheduling = { + maxWorkers: number; + fileParallelism: boolean; + throttledBySystem: boolean; +}; + export const jsdomOptimizedDeps = { optimizer: { web: { @@ -44,10 +60,26 @@ function detectVitestHostInfo(): Required { export function resolveLocalVitestMaxWorkers( env: Record = process.env, system: VitestHostInfo = detectVitestHostInfo(), + pool: OpenClawVitestPool = resolveDefaultVitestPool(env), + processStats: VitestProcessStats = detectVitestProcessStats(env), ): number { + return resolveLocalVitestScheduling(env, system, pool, processStats).maxWorkers; +} + +export function resolveLocalVitestScheduling( + env: Record = process.env, + system: VitestHostInfo = detectVitestHostInfo(), + pool: OpenClawVitestPool = resolveDefaultVitestPool(env), + processStats: VitestProcessStats = detectVitestProcessStats(env), +): LocalVitestScheduling { const override = parsePositiveInt(env.OPENCLAW_VITEST_MAX_WORKERS ?? env.OPENCLAW_TEST_WORKERS); if (override !== null) { - return clamp(override, 1, 16); + const maxWorkers = clamp(override, 1, 16); + return { + maxWorkers, + fileParallelism: maxWorkers > 1, + throttledBySystem: false, + }; } const cpuCount = Math.max(1, system.cpuCount ?? 1); @@ -76,7 +108,50 @@ export function resolveLocalVitestMaxWorkers( inferred = Math.max(1, inferred - 1); } - return clamp(inferred, 1, 16); + inferred = clamp(inferred, 1, 16); + + if (isSystemThrottleDisabled(env)) { + return { + maxWorkers: inferred, + fileParallelism: true, + throttledBySystem: false, + }; + } + + const highSystemContention = + loadRatio >= 1 || + processStats.otherVitestWorkerCount >= 2 || + processStats.otherVitestCpuPercent >= 150 || + processStats.otherVitestRootCount >= 2; + + if (highSystemContention) { + return { + maxWorkers: 1, + fileParallelism: false, + throttledBySystem: true, + }; + } + + const moderateSystemContention = + loadRatio >= 0.75 || + processStats.otherVitestWorkerCount >= 1 || + processStats.otherVitestCpuPercent >= 75 || + processStats.otherVitestRootCount >= 1; + + if (moderateSystemContention) { + const maxWorkers = Math.min(inferred, 2); + return { + maxWorkers, + fileParallelism: true, + throttledBySystem: maxWorkers < inferred, + }; + } + + return { + maxWorkers: inferred, + fileParallelism: true, + throttledBySystem: false, + }; } export function resolveDefaultVitestPool( @@ -93,9 +168,21 @@ const repoRoot = path.dirname(fileURLToPath(import.meta.url)); const isCI = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true"; const isWindows = process.platform === "win32"; const defaultPool = resolveDefaultVitestPool(); -const localWorkers = resolveLocalVitestMaxWorkers(process.env, detectVitestHostInfo()); +const localScheduling = resolveLocalVitestScheduling( + process.env, + detectVitestHostInfo(), + defaultPool, +); const ciWorkers = isWindows ? 2 : 3; +if (!isCI && localScheduling.throttledBySystem && shouldPrintVitestThrottle(process.env)) { + console.error( + `[vitest] throttling local workers to ${localScheduling.maxWorkers}${ + localScheduling.fileParallelism ? "" : " with file parallelism disabled" + } because the host already looks busy.`, + ); +} + export const sharedVitestConfig = { resolve: { alias: [ @@ -119,7 +206,8 @@ export const sharedVitestConfig = { unstubEnvs: true, unstubGlobals: true, pool: defaultPool, - maxWorkers: isCI ? ciWorkers : localWorkers, + maxWorkers: isCI ? ciWorkers : localScheduling.maxWorkers, + fileParallelism: isCI ? true : localScheduling.fileParallelism, forceRerunTriggers: [ "package.json", "pnpm-lock.yaml", diff --git a/vitest.system-load.ts b/vitest.system-load.ts new file mode 100644 index 00000000000..84234e5bf81 --- /dev/null +++ b/vitest.system-load.ts @@ -0,0 +1,122 @@ +import { spawnSync } from "node:child_process"; + +type EnvMap = Record; + +export type VitestProcessStats = { + otherVitestRootCount: number; + otherVitestWorkerCount: number; + otherVitestCpuPercent: number; +}; + +type PsResult = { + status: number | null; + stdout: string; +}; + +type DetectVitestProcessStatsOptions = { + platform?: NodeJS.Platform; + selfPid?: number; + runPs?: () => PsResult; +}; + +const EMPTY_VITEST_PROCESS_STATS: VitestProcessStats = { + otherVitestRootCount: 0, + otherVitestWorkerCount: 0, + otherVitestCpuPercent: 0, +}; + +const BOOLEAN_TRUE_VALUES = new Set(["1", "true"]); + +function isExplicitlyEnabled(value: string | undefined): boolean { + const normalized = value?.trim().toLowerCase(); + return normalized ? BOOLEAN_TRUE_VALUES.has(normalized) : false; +} + +function isVitestWorkerArgs(args: string): boolean { + return args.includes("/vitest/dist/workers/") || args.includes("\\vitest\\dist\\workers\\"); +} + +function isVitestRootArgs(args: string): boolean { + return ( + args.includes("node_modules/.bin/vitest") || + /\bvitest(?:\.(?:m?js|cmd|exe))?\b/u.test(args) || + args.includes("scripts/test-projects.mjs") || + args.includes("scripts/run-vitest.mjs") + ); +} + +function normalizeCpu(rawCpu: string): number { + const parsed = Number.parseFloat(rawCpu); + return Number.isFinite(parsed) && parsed > 0 ? parsed : 0; +} + +export function parseVitestProcessStats( + psOutput: string, + selfPid: number = process.pid, +): VitestProcessStats { + const stats = { ...EMPTY_VITEST_PROCESS_STATS }; + + for (const line of psOutput.split("\n")) { + const trimmed = line.trim(); + if (trimmed.length === 0) { + continue; + } + + const match = /^(\d+)\s+([0-9.]+)\s+(.*)$/u.exec(trimmed); + if (!match) { + continue; + } + + const [, rawPid, rawCpu, args] = match; + const pid = Number.parseInt(rawPid, 10); + if (!Number.isFinite(pid) || pid === selfPid) { + continue; + } + + if (!isVitestWorkerArgs(args) && !isVitestRootArgs(args)) { + continue; + } + + stats.otherVitestCpuPercent += normalizeCpu(rawCpu); + if (isVitestWorkerArgs(args)) { + stats.otherVitestWorkerCount += 1; + } else { + stats.otherVitestRootCount += 1; + } + } + + stats.otherVitestCpuPercent = Number.parseFloat(stats.otherVitestCpuPercent.toFixed(1)); + return stats; +} + +export function detectVitestProcessStats( + env: EnvMap = process.env, + options: DetectVitestProcessStatsOptions = {}, +): VitestProcessStats { + const platform = options.platform ?? process.platform; + if (platform === "win32") { + return { ...EMPTY_VITEST_PROCESS_STATS }; + } + + if (isExplicitlyEnabled(env.OPENCLAW_VITEST_DISABLE_SYSTEM_THROTTLE)) { + return { ...EMPTY_VITEST_PROCESS_STATS }; + } + + const result = + options.runPs?.() ?? + spawnSync("ps", ["-xao", "pid=,pcpu=,args="], { + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }); + + if (result.status === 0 && typeof result.stdout === "string" && result.stdout.length > 0) { + return parseVitestProcessStats(result.stdout, options.selfPid ?? process.pid); + } + + return { ...EMPTY_VITEST_PROCESS_STATS }; +} + +export function shouldPrintVitestThrottle(env: EnvMap = process.env): boolean { + const normalized = env.OPENCLAW_VITEST_PRINT_SYSTEM_THROTTLE?.trim().toLowerCase(); + return normalized ? BOOLEAN_TRUE_VALUES.has(normalized) : false; +}