nyrn
/
openclaw
mirror of https://github.com/openclaw/openclaw.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor: share cron restart catchup harness

This commit is contained in:
Peter Steinberger 2026-03-13 20:31:03 +00:00
parent e762a57d62
commit 9dafcd417d
1 changed files with 216 additions and 268 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

210
src/cron/service.restart-catchup.test.ts
View File

@ -47,15 +47,41 @@ describe("CronService restart catch-up", () => {
};
}
it("executes an overdue recurring job immediately on start", async () => {
async function withRestartedCron(
jobs: unknown[],
run: (params: {
cron: CronService;
enqueueSystemEvent: ReturnType<typeof vi.fn>;
requestHeartbeatNow: ReturnType<typeof vi.fn>;
}) => Promise<void>,
) {
const store = await makeStorePath();
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
await writeStoreJobs(store.storePath, jobs);
const cron = createRestartCronService({
storePath: store.storePath,
enqueueSystemEvent,
requestHeartbeatNow,
});
try {
await cron.start();
await run({ cron, enqueueSystemEvent, requestHeartbeatNow });
} finally {
cron.stop();
await store.cleanup();
}
}
it("executes an overdue recurring job immediately on start", async () => {
const dueAt = Date.parse("2025-12-13T15:00:00.000Z");
const lastRunAt = Date.parse("2025-12-12T15:00:00.000Z");
await writeStoreJobs(store.storePath, [
await withRestartedCron(
[
{
id: "restart-overdue-job",
name: "daily digest",
@ -72,41 +98,29 @@ describe("CronService restart catch-up", () => {
lastStatus: "ok",
},
},
]);
const cron = createRestartCronService({
storePath: store.storePath,
enqueueSystemEvent,
requestHeartbeatNow,
});
await cron.start();
],
async ({ cron, enqueueSystemEvent, requestHeartbeatNow }) => {
expect(enqueueSystemEvent).toHaveBeenCalledWith(
"digest now",
expect.objectContaining({ agentId: undefined }),
);
expect(requestHeartbeatNow).toHaveBeenCalled();
const jobs = await cron.list({ includeDisabled: true });
const updated = jobs.find((job) => job.id === "restart-overdue-job");
const listedJobs = await cron.list({ includeDisabled: true });
const updated = listedJobs.find((job) => job.id === "restart-overdue-job");
expect(updated?.state.lastStatus).toBe("ok");
expect(updated?.state.lastRunAtMs).toBe(Date.parse("2025-12-13T17:00:00.000Z"));
expect(updated?.state.nextRunAtMs).toBeGreaterThan(Date.parse("2025-12-13T17:00:00.000Z"));
cron.stop();
await store.cleanup();
},
);
});
it("clears stale running markers without replaying interrupted startup jobs", async () => {
const store = await makeStorePath();
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
const dueAt = Date.parse("2025-12-13T16:00:00.000Z");
const staleRunningAt = Date.parse("2025-12-13T16:30:00.000Z");
await writeStoreJobs(store.storePath, [
await withRestartedCron(
[
{
id: "restart-stale-running",
name: "daily stale marker",
@ -122,39 +136,29 @@ describe("CronService restart catch-up", () => {
runningAtMs: staleRunningAt,
},
},
]);
const cron = createRestartCronService({
storePath: store.storePath,
enqueueSystemEvent,
requestHeartbeatNow,
});
await cron.start();
],
async ({ cron, enqueueSystemEvent }) => {
expect(enqueueSystemEvent).not.toHaveBeenCalled();
expect(noopLogger.warn).toHaveBeenCalledWith(
expect.objectContaining({ jobId: "restart-stale-running" }),
"cron: clearing stale running marker on startup",
);
const jobs = await cron.list({ includeDisabled: true });
const updated = jobs.find((job) => job.id === "restart-stale-running");
const listedJobs = await cron.list({ includeDisabled: true });
const updated = listedJobs.find((job) => job.id === "restart-stale-running");
expect(updated?.state.runningAtMs).toBeUndefined();
expect(updated?.state.lastStatus).toBeUndefined();
expect(updated?.state.lastRunAtMs).toBeUndefined();
expect((updated?.state.nextRunAtMs ?? 0) > Date.parse("2025-12-13T17:00:00.000Z")).toBe(true);
cron.stop();
await store.cleanup();
expect((updated?.state.nextRunAtMs ?? 0) > Date.parse("2025-12-13T17:00:00.000Z")).toBe(
true,
);
},
);
});
it("replays the most recent missed cron slot after restart when nextRunAtMs already advanced", async () => {
vi.setSystemTime(new Date("2025-12-13T04:02:00.000Z"));
const store = await makeStorePath();
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
await writeStoreJobs(store.storePath, [
await withRestartedCron(
[
{
id: "restart-missed-slot",
name: "every ten minutes +1",
@ -173,39 +177,27 @@ describe("CronService restart catch-up", () => {
lastStatus: "ok",
},
},
]);
const cron = createRestartCronService({
storePath: store.storePath,
enqueueSystemEvent,
requestHeartbeatNow,
});
await cron.start();
],
async ({ cron, enqueueSystemEvent, requestHeartbeatNow }) => {
expect(enqueueSystemEvent).toHaveBeenCalledWith(
"catch missed slot",
expect.objectContaining({ agentId: undefined }),
);
expect(requestHeartbeatNow).toHaveBeenCalled();
const jobs = await cron.list({ includeDisabled: true });
const updated = jobs.find((job) => job.id === "restart-missed-slot");
const listedJobs = await cron.list({ includeDisabled: true });
const updated = listedJobs.find((job) => job.id === "restart-missed-slot");
expect(updated?.state.lastRunAtMs).toBe(Date.parse("2025-12-13T04:02:00.000Z"));
cron.stop();
await store.cleanup();
},
);
});
it("does not replay interrupted one-shot jobs on startup", async () => {
const store = await makeStorePath();
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
const dueAt = Date.parse("2025-12-13T16:00:00.000Z");
const staleRunningAt = Date.parse("2025-12-13T16:30:00.000Z");
await writeStoreJobs(store.storePath, [
await withRestartedCron(
[
{
id: "restart-stale-one-shot",
name: "one shot stale marker",
@ -221,34 +213,22 @@ describe("CronService restart catch-up", () => {
runningAtMs: staleRunningAt,
},
},
]);
const cron = createRestartCronService({
storePath: store.storePath,
enqueueSystemEvent,
requestHeartbeatNow,
});
await cron.start();
],
async ({ cron, enqueueSystemEvent, requestHeartbeatNow }) => {
expect(enqueueSystemEvent).not.toHaveBeenCalled();
expect(requestHeartbeatNow).not.toHaveBeenCalled();
const jobs = await cron.list({ includeDisabled: true });
const updated = jobs.find((job) => job.id === "restart-stale-one-shot");
const listedJobs = await cron.list({ includeDisabled: true });
const updated = listedJobs.find((job) => job.id === "restart-stale-one-shot");
expect(updated?.state.runningAtMs).toBeUndefined();
cron.stop();
await store.cleanup();
},
);
});
it("does not replay cron slot when the latest slot already ran before restart", async () => {
vi.setSystemTime(new Date("2025-12-13T04:02:00.000Z"));
const store = await makeStorePath();
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
await writeStoreJobs(store.storePath, [
await withRestartedCron(
[
{
id: "restart-no-duplicate-slot",
name: "every ten minutes +1 no duplicate",
@ -265,29 +245,18 @@ describe("CronService restart catch-up", () => {
lastStatus: "ok",
},
},
]);
const cron = createRestartCronService({
storePath: store.storePath,
enqueueSystemEvent,
requestHeartbeatNow,
});
await cron.start();
],
async ({ enqueueSystemEvent, requestHeartbeatNow }) => {
expect(enqueueSystemEvent).not.toHaveBeenCalled();
expect(requestHeartbeatNow).not.toHaveBeenCalled();
cron.stop();
await store.cleanup();
},
);
});
it("does not replay missed cron slots while error backoff is pending after restart", async () => {
vi.setSystemTime(new Date("2025-12-13T04:02:00.000Z"));
const store = await makeStorePath();
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
await writeStoreJobs(store.storePath, [
await withRestartedCron(
[
{
id: "restart-backoff-pending",
name: "backoff pending",
@ -306,30 +275,18 @@ describe("CronService restart catch-up", () => {
consecutiveErrors: 4,
},
},
]);
const cron = createRestartCronService({
storePath: store.storePath,
enqueueSystemEvent,
requestHeartbeatNow,
});
await cron.start();
],
async ({ enqueueSystemEvent, requestHeartbeatNow }) => {
expect(enqueueSystemEvent).not.toHaveBeenCalled();
expect(requestHeartbeatNow).not.toHaveBeenCalled();
cron.stop();
await store.cleanup();
},
);
});
it("replays missed cron slot after restart when error backoff has already elapsed", async () => {
vi.setSystemTime(new Date("2025-12-13T04:02:00.000Z"));
const store = await makeStorePath();
const enqueueSystemEvent = vi.fn();
const requestHeartbeatNow = vi.fn();
await writeStoreJobs(store.storePath, [
await withRestartedCron(
[
{
id: "restart-backoff-elapsed-replay",
name: "backoff elapsed replay",
@ -349,24 +306,15 @@ describe("CronService restart catch-up", () => {
consecutiveErrors: 1,
},
},
]);
const cron = createRestartCronService({
storePath: store.storePath,
enqueueSystemEvent,
requestHeartbeatNow,
});
await cron.start();
],
async ({ enqueueSystemEvent, requestHeartbeatNow }) => {
expect(enqueueSystemEvent).toHaveBeenCalledWith(
"replay after backoff elapsed",
expect.objectContaining({ agentId: undefined }),
);
expect(requestHeartbeatNow).toHaveBeenCalled();
cron.stop();
await store.cleanup();
},
);
});
it("reschedules deferred missed jobs from the post-catchup clock so they stay in the future", async () => {