nyrn
/
openclaw
mirror of https://github.com/openclaw/openclaw.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ci: shard extension fast checks

This commit is contained in:
Peter Steinberger 2026-04-03 18:26:14 +01:00
parent 230c61885d
commit ebdade0efc
No known key found for this signature in database
5 changed files with 321 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
.github/workflows/ci.yml vendored
View File

@ -36,7 +36,8 @@ jobs:
changed_extensions_matrix: ${{ steps.manifest.outputs.changed_extensions_matrix }}
run_build_artifacts: ${{ steps.manifest.outputs.run_build_artifacts }}
run_checks_fast: ${{ steps.manifest.outputs.run_checks_fast }}
checks_fast_matrix: ${{ steps.manifest.outputs.checks_fast_matrix }}
checks_fast_core_matrix: ${{ steps.manifest.outputs.checks_fast_core_matrix }}
checks_fast_extensions_matrix: ${{ steps.manifest.outputs.checks_fast_extensions_matrix }}
run_checks: ${{ steps.manifest.outputs.run_checks }}
checks_matrix: ${{ steps.manifest.outputs.checks_matrix }}
run_extension_fast: ${{ steps.manifest.outputs.run_extension_fast }}
@ -132,6 +133,10 @@ jobs:
run: |
node --input-type=module <<'EOF'
import { appendFileSync } from "node:fs";
import {
createExtensionTestShards,
DEFAULT_EXTENSION_TEST_SHARD_COUNT,
} from "./scripts/lib/extension-test-plan.mjs";
const parseBoolean = (value, fallback = false) => {
if (value === undefined) return fallback;
@ -165,6 +170,18 @@ jobs:
const changedExtensionsMatrix = hasChangedExtensions
? parseJson(process.env.OPENCLAW_CI_CHANGED_EXTENSIONS_MATRIX, { include: [] })
: { include: [] };
const extensionShardMatrix = createMatrix(
runNode
? createExtensionTestShards({
shardCount: DEFAULT_EXTENSION_TEST_SHARD_COUNT,
}).map((shard) => ({
check_name: shard.checkName,
extensions_csv: shard.extensionIds.join(","),
shard_index: shard.index + 1,
task: "extensions-batch",
}))
: [],
);
const manifest = {
docs_only: docsOnly,
@ -178,11 +195,10 @@ jobs:
changed_extensions_matrix: changedExtensionsMatrix,
run_build_artifacts: runNode,
run_checks_fast: runNode,
checks_fast_matrix: createMatrix(
checks_fast_core_matrix: createMatrix(
runNode
? [
{ check_name: "checks-fast-bundled", runtime: "node", task: "bundled" },
{ check_name: "checks-fast-extensions", runtime: "node", task: "extensions" },
{
check_name: "checks-fast-contracts-protocol",
runtime: "node",
@ -191,6 +207,7 @@ jobs:
]
: [],
),
checks_fast_extensions_matrix: extensionShardMatrix,
run_checks: runNode,
checks_matrix: createMatrix(
runNode
@ -404,7 +421,7 @@ jobs:
include-hidden-files: true
retention-days: 1
checks-fast:
checks-fast-core:
name: ${{ matrix.check_name }}
needs: [preflight]
if: needs.preflight.outputs.run_checks_fast == 'true'
@ -412,7 +429,7 @@ jobs:
timeout-minutes: 60
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.preflight.outputs.checks_fast_matrix) }}
matrix: ${{ fromJson(needs.preflight.outputs.checks_fast_core_matrix) }}
steps:
- name: Checkout
uses: actions/checkout@v6
@ -436,9 +453,6 @@ jobs:
bundled)
pnpm test:bundled
;;
extensions)
pnpm test:extensions
;;
contracts|contracts-protocol)
pnpm build
pnpm test:contracts
@ -450,6 +464,49 @@ jobs:
;;
esac
checks-fast-extensions-shard:
name: ${{ matrix.check_name }}
needs: [preflight]
if: needs.preflight.outputs.run_checks_fast == 'true'
runs-on: blacksmith-16vcpu-ubuntu-2404
timeout-minutes: 60
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.preflight.outputs.checks_fast_extensions_matrix) }}
steps:
- name: Checkout
uses: actions/checkout@v6
with:
persist-credentials: false
submodules: false
- name: Setup Node environment
uses: ./.github/actions/setup-node-env
with:
install-bun: "false"
use-sticky-disk: "false"
- name: Run extension shard
env:
OPENCLAW_EXTENSION_BATCH: ${{ matrix.extensions_csv }}
run: pnpm test:extensions:batch -- "$OPENCLAW_EXTENSION_BATCH"
checks-fast-extensions:
name: checks-fast-extensions
needs: [preflight, checks-fast-extensions-shard]
if: always() && needs.preflight.outputs.run_checks_fast == 'true'
runs-on: blacksmith-16vcpu-ubuntu-2404
timeout-minutes: 5
steps:
- name: Verify extension shards
env:
SHARD_RESULT: ${{ needs.checks-fast-extensions-shard.result }}
run: |
if [ "$SHARD_RESULT" != "success" ]; then
echo "Extension shard checks failed: $SHARD_RESULT" >&2
exit 1
fi
checks:
name: ${{ matrix.check_name }}
needs: [preflight, build-artifacts]

1
package.json
View File

@ -1086,6 +1086,7 @@
"test:e2e:openshell": "OPENCLAW_E2E_OPENSHELL=1 vitest run --config vitest.e2e.config.ts test/openshell-sandbox.e2e.test.ts",
"test:extension": "node scripts/test-extension.mjs",
"test:extensions": "vitest run --config vitest.extensions.config.ts",
"test:extensions:batch": "node scripts/test-extension-batch.mjs",
"test:extensions:memory": "node scripts/profile-extension-memory.mjs",
"test:fast": "vitest run --config vitest.unit.config.ts",
"test:force": "node --import tsx scripts/test-force.ts",

99
scripts/lib/extension-test-plan.mjs
View File

@ -2,8 +2,10 @@ import fs from "node:fs";
import path from "node:path";
import { channelTestRoots } from "../../vitest.channel-paths.mjs";
import { BUNDLED_PLUGIN_PATH_PREFIX, BUNDLED_PLUGIN_ROOT_DIR } from "./bundled-plugin-paths.mjs";
import { listAvailableExtensionIds } from "./changed-extensions.mjs";
const repoRoot = path.resolve(import.meta.dirname, "..", "..");
export const DEFAULT_EXTENSION_TEST_SHARD_COUNT = 6;
function normalizeRelative(inputPath) {
return inputPath.split(path.sep).join("/");
@ -102,3 +104,100 @@ export function resolveExtensionTestPlan(params = {}) {
testFileCount,
};
}
function mergeTestPlans(plans) {
const groupsByConfig = new Map();
for (const plan of plans) {
const current = groupsByConfig.get(plan.config) ?? {
config: plan.config,
extensionIds: [],
roots: [],
testFileCount: 0,
};
current.extensionIds.push(plan.extensionId);
current.roots.push(...plan.roots);
current.testFileCount += plan.testFileCount;
groupsByConfig.set(plan.config, current);
}
const planGroups = [...groupsByConfig.values()]
.map((group) => ({
...group,
extensionIds: group.extensionIds.toSorted((left, right) => left.localeCompare(right)),
roots: [...new Set(group.roots)],
}))
.toSorted((left, right) => left.config.localeCompare(right.config));
return {
extensionCount: plans.length,
extensionIds: plans
.map((plan) => plan.extensionId)
.toSorted((left, right) => left.localeCompare(right)),
hasTests: plans.length > 0,
planGroups,
testFileCount: plans.reduce((sum, plan) => sum + plan.testFileCount, 0),
};
}
export function resolveExtensionBatchPlan(params = {}) {
const cwd = params.cwd ?? process.cwd();
const extensionIds = params.extensionIds ?? listAvailableExtensionIds();
const plans = extensionIds
.map((extensionId) => resolveExtensionTestPlan({ cwd, targetArg: extensionId }))
.filter((plan) => plan.hasTests);
return mergeTestPlans(plans);
}
function pickLeastLoadedShard(shards) {
return shards.reduce((bestIndex, shard, index) => {
if (bestIndex === -1) {
return index;
}
const best = shards[bestIndex];
if (shard.testFileCount !== best.testFileCount) {
return shard.testFileCount < best.testFileCount ? index : bestIndex;
}
if (shard.plans.length !== best.plans.length) {
return shard.plans.length < best.plans.length ? index : bestIndex;
}
return index < bestIndex ? index : bestIndex;
}, -1);
}
export function createExtensionTestShards(params = {}) {
const cwd = params.cwd ?? process.cwd();
const extensionIds = params.extensionIds ?? listAvailableExtensionIds();
const shardCount = Math.max(1, Number.parseInt(String(params.shardCount ?? ""), 10) || 1);
const plans = extensionIds
.map((extensionId) => resolveExtensionTestPlan({ cwd, targetArg: extensionId }))
.filter((plan) => plan.hasTests)
.toSorted((left, right) => {
if (left.testFileCount !== right.testFileCount) {
return right.testFileCount - left.testFileCount;
}
return left.extensionId.localeCompare(right.extensionId);
});
const effectiveShardCount = Math.min(shardCount, Math.max(1, plans.length));
const shards = Array.from({ length: effectiveShardCount }, () => ({
plans: [],
testFileCount: 0,
}));
for (const plan of plans) {
const targetIndex = pickLeastLoadedShard(shards);
shards[targetIndex].plans.push(plan);
shards[targetIndex].testFileCount += plan.testFileCount;
}
return shards
.map((shard, index) => ({
index,
checkName: `checks-fast-extensions-shard-${index + 1}`,
...mergeTestPlans(shard.plans),
}))
.filter((shard) => shard.hasTests);
}

105
scripts/test-extension-batch.mjs Normal file
View File

@ -0,0 +1,105 @@
#!/usr/bin/env node
import { spawn } from "node:child_process";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { resolveExtensionBatchPlan } from "./lib/extension-test-plan.mjs";
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const repoRoot = path.resolve(__dirname, "..");
const pnpm = "pnpm";
async function runVitestBatch(params) {
return await new Promise((resolve, reject) => {
const child = spawn(
pnpm,
["exec", "vitest", "run", "--config", params.config, ...params.targets, ...params.args],
{
cwd: repoRoot,
stdio: "inherit",
shell: process.platform === "win32",
env: params.env,
},
);
child.on("error", reject);
child.on("exit", (code, signal) => {
if (signal) {
process.kill(process.pid, signal);
return;
}
resolve(code ?? 1);
});
});
}
function printUsage() {
console.error("Usage: pnpm test:extensions:batch <extension[,extension...]> [vitest args...]");
console.error(
" node scripts/test-extension-batch.mjs <extension[,extension...]> [vitest args...]",
);
}
function parseExtensionIds(rawArgs) {
const args = [...rawArgs];
const extensionIds = [];
while (args[0] && !args[0].startsWith("-")) {
extensionIds.push(
...args
.shift()
.split(",")
.map((value) => value.trim())
.filter(Boolean),
);
}
return { extensionIds, passthroughArgs: args };
}
async function run() {
const rawArgs = process.argv.slice(2);
if (rawArgs.includes("--help") || rawArgs.includes("-h")) {
printUsage();
return;
}
const passthroughArgs = rawArgs.filter((arg) => arg !== "--");
const { extensionIds, passthroughArgs: vitestArgs } = parseExtensionIds(passthroughArgs);
if (extensionIds.length === 0) {
printUsage();
process.exit(1);
}
const batchPlan = resolveExtensionBatchPlan({ cwd: process.cwd(), extensionIds });
if (!batchPlan.hasTests) {
console.log("[test-extension-batch] No tests found for the requested extensions. Skipping.");
return;
}
console.log(
`[test-extension-batch] Running ${batchPlan.testFileCount} test files across ${batchPlan.extensionCount} extensions`,
);
for (const group of batchPlan.planGroups) {
console.log(
`[test-extension-batch] ${group.config}: ${group.extensionIds.join(", ")} (${group.testFileCount} files)`,
);
const exitCode = await runVitestBatch({
args: vitestArgs,
config: group.config,
env: process.env,
targets: group.roots,
});
if (exitCode !== 0) {
process.exit(exitCode);
}
}
}
const entryHref = process.argv[1] ? pathToFileURL(path.resolve(process.argv[1])).href : "";
if (import.meta.url === entryHref) {
await run();
}

52
test/scripts/test-extension.test.ts
View File

@ -6,7 +6,12 @@ import {
listAvailableExtensionIds,
listChangedExtensionIds,
} from "../../scripts/lib/changed-extensions.mjs";
import { resolveExtensionTestPlan } from "../../scripts/lib/extension-test-plan.mjs";
import {
DEFAULT_EXTENSION_TEST_SHARD_COUNT,
createExtensionTestShards,
resolveExtensionBatchPlan,
resolveExtensionTestPlan,
} from "../../scripts/lib/extension-test-plan.mjs";
import { bundledPluginFile, bundledPluginRoot } from "../helpers/bundled-plugin-paths.js";
const scriptPath = path.join(process.cwd(), "scripts", "test-extension.mjs");
@ -103,6 +108,51 @@ describe("scripts/test-extension.mjs", () => {
expect(plan.testFileCount).toBe(0);
});
it("batches extensions into config-specific vitest invocations", () => {
const batch = resolveExtensionBatchPlan({
cwd: process.cwd(),
extensionIds: ["slack", "firecrawl", "line"],
});
expect(batch.extensionIds).toEqual(["firecrawl", "line", "slack"]);
expect(batch.planGroups).toEqual([
{
config: "vitest.channels.config.ts",
extensionIds: ["slack"],
roots: [bundledPluginRoot("slack")],
testFileCount: expect.any(Number),
},
{
config: "vitest.extensions.config.ts",
extensionIds: ["firecrawl", "line"],
roots: [bundledPluginRoot("firecrawl"), bundledPluginRoot("line")],
testFileCount: expect.any(Number),
},
]);
});
it("balances extension test shards by test file count", () => {
const shards = createExtensionTestShards({
cwd: process.cwd(),
shardCount: DEFAULT_EXTENSION_TEST_SHARD_COUNT,
});
expect(shards).toHaveLength(DEFAULT_EXTENSION_TEST_SHARD_COUNT);
const assigned = shards.flatMap((shard) => shard.extensionIds);
const uniqueAssigned = [...new Set(assigned)];
const expected = listAvailableExtensionIds().filter(
(extensionId) =>
resolveExtensionTestPlan({ cwd: process.cwd(), targetArg: extensionId }).hasTests,
);
expect(uniqueAssigned).toEqual(expected);
expect(assigned).toHaveLength(expected.length);
const totals = shards.map((shard) => shard.testFileCount);
expect(Math.max(...totals) - Math.min(...totals)).toBeLessThanOrEqual(1);
});
it("treats extensions without tests as a no-op by default", () => {
const extensionId = findExtensionWithoutTests();
const stdout = runScript([extensionId]);