From 78a842d055192a3f8cff42f2f088944b5ffc6d8a Mon Sep 17 00:00:00 2001 From: Gustavo Madeira Santana Date: Thu, 2 Apr 2026 21:43:58 -0400 Subject: [PATCH] fix(matrix): align IDB snapshot lock timing --- extensions/matrix/src/matrix/sdk.test.ts | 1 + extensions/matrix/src/matrix/sdk.ts | 3 +- .../src/matrix/sdk/idb-persistence-lock.ts | 51 +++++++++++++++ .../sdk/idb-persistence.lock-order.test.ts | 34 +++++++++- .../matrix/src/matrix/sdk/idb-persistence.ts | 65 +++++++++---------- 5 files changed, 118 insertions(+), 36 deletions(-) create mode 100644 extensions/matrix/src/matrix/sdk/idb-persistence-lock.ts diff --git a/extensions/matrix/src/matrix/sdk.test.ts b/extensions/matrix/src/matrix/sdk.test.ts index 94cf76d9a27..28922c888d5 100644 --- a/extensions/matrix/src/matrix/sdk.test.ts +++ b/extensions/matrix/src/matrix/sdk.test.ts @@ -1,3 +1,4 @@ +import "fake-indexeddb/auto"; import { EventEmitter } from "node:events"; import fs from "node:fs"; import os from "node:os"; diff --git a/extensions/matrix/src/matrix/sdk.ts b/extensions/matrix/src/matrix/sdk.ts index 6f17d615d34..6929eb87619 100644 --- a/extensions/matrix/src/matrix/sdk.ts +++ b/extensions/matrix/src/matrix/sdk.ts @@ -23,6 +23,7 @@ import type { MatrixCryptoFacade } from "./sdk/crypto-facade.js"; import type { MatrixDecryptBridge } from "./sdk/decrypt-bridge.js"; import { matrixEventToRaw, parseMxc } from "./sdk/event-helpers.js"; import { MatrixAuthedHttpClient } from "./sdk/http-client.js"; +import { MATRIX_IDB_PERSIST_INTERVAL_MS } from "./sdk/idb-persistence-lock.js"; import { ConsoleLogger, LogService, noop } from "./sdk/logger.js"; import { MatrixRecoveryKeyStore } from "./sdk/recovery-key-store.js"; import { createMatrixGuardedFetch, type HttpMethod, type QueryParams } from "./sdk/transport.js"; @@ -547,7 +548,7 @@ export class MatrixClient { snapshotPath: this.idbSnapshotPath, databasePrefix: this.cryptoDatabasePrefix, }).catch(noop); - }, 60_000); + }, MATRIX_IDB_PERSIST_INTERVAL_MS); } catch (err) { LogService.warn("MatrixClientLite", "Failed to initialize rust crypto:", err); } diff --git a/extensions/matrix/src/matrix/sdk/idb-persistence-lock.ts b/extensions/matrix/src/matrix/sdk/idb-persistence-lock.ts new file mode 100644 index 00000000000..37204a912f3 --- /dev/null +++ b/extensions/matrix/src/matrix/sdk/idb-persistence-lock.ts @@ -0,0 +1,51 @@ +import type { FileLockOptions } from "openclaw/plugin-sdk/infra-runtime"; + +export const MATRIX_IDB_PERSIST_INTERVAL_MS = 60_000; + +const IDB_SNAPSHOT_LOCK_STALE_MS = 5 * 60_000; +const IDB_SNAPSHOT_LOCK_RETRY_BASE = { + factor: 2, + minTimeout: 50, + maxTimeout: 5_000, + randomize: true, +} satisfies Omit; + +function computeRetryDelayMs(retries: FileLockOptions["retries"], attempt: number): number { + return Math.min( + retries.maxTimeout, + Math.max(retries.minTimeout, retries.minTimeout * retries.factor ** attempt), + ); +} + +export function computeMinimumRetryWindowMs(retries: FileLockOptions["retries"]): number { + let total = 0; + const attempts = Math.max(1, retries.retries + 1); + for (let attempt = 0; attempt < attempts - 1; attempt += 1) { + total += computeRetryDelayMs(retries, attempt); + } + return total; +} + +function resolveRetriesForMinimumWindowMs( + retries: Omit, + minimumWindowMs: number, +): FileLockOptions["retries"] { + const resolved: FileLockOptions["retries"] = { + ...retries, + retries: 0, + }; + while (computeMinimumRetryWindowMs(resolved) < minimumWindowMs) { + resolved.retries += 1; + } + return resolved; +} + +export const MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS: FileLockOptions = { + // Wait longer than one periodic persist interval so a concurrent restore + // or large snapshot dump finishes instead of forcing warn-and-continue. + retries: resolveRetriesForMinimumWindowMs( + IDB_SNAPSHOT_LOCK_RETRY_BASE, + MATRIX_IDB_PERSIST_INTERVAL_MS, + ), + stale: IDB_SNAPSHOT_LOCK_STALE_MS, +}; diff --git a/extensions/matrix/src/matrix/sdk/idb-persistence.lock-order.test.ts b/extensions/matrix/src/matrix/sdk/idb-persistence.lock-order.test.ts index 559b2dddc6c..f2ac5be8c51 100644 --- a/extensions/matrix/src/matrix/sdk/idb-persistence.lock-order.test.ts +++ b/extensions/matrix/src/matrix/sdk/idb-persistence.lock-order.test.ts @@ -3,6 +3,10 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { + computeMinimumRetryWindowMs, + MATRIX_IDB_PERSIST_INTERVAL_MS, +} from "./idb-persistence-lock.js"; import { clearAllIndexedDbState, seedDatabase } from "./idb-persistence.test-helpers.js"; const { withFileLockMock } = vi.hoisted(() => ({ @@ -20,9 +24,12 @@ vi.mock("openclaw/plugin-sdk/infra-runtime", async (importOriginal) => { }); let persistIdbToDisk: typeof import("./idb-persistence.js").persistIdbToDisk; +let restoreIdbFromDisk: typeof import("./idb-persistence.js").restoreIdbFromDisk; +type CapturedLockOptions = + typeof import("./idb-persistence-lock.js").MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS; beforeAll(async () => { - ({ persistIdbToDisk } = await import("./idb-persistence.js")); + ({ persistIdbToDisk, restoreIdbFromDisk } = await import("./idb-persistence.js")); }); describe("Matrix IndexedDB persistence lock ordering", () => { @@ -71,4 +78,29 @@ describe("Matrix IndexedDB persistence lock ordering", () => { const sessionsStore = data[0]?.stores.find((store) => store.name === "sessions"); expect(sessionsStore?.records).toEqual([{ key: "room-1", value: { session: "new-session" } }]); }); + + it("waits at least one persist interval before timing out on snapshot lock contention", async () => { + const snapshotPath = path.join(tmpDir, "crypto-idb-snapshot.json"); + const capturedOptions: CapturedLockOptions[] = []; + + withFileLockMock.mockImplementationOnce(async (_filePath, options) => { + capturedOptions.push(options as CapturedLockOptions); + return 0; + }); + await persistIdbToDisk({ snapshotPath, databasePrefix: "openclaw-matrix-test" }); + + withFileLockMock.mockImplementationOnce(async (_filePath, options) => { + capturedOptions.push(options as CapturedLockOptions); + return false; + }); + await restoreIdbFromDisk(snapshotPath); + + expect(capturedOptions).toHaveLength(2); + for (const options of capturedOptions) { + expect(computeMinimumRetryWindowMs(options.retries)).toBeGreaterThanOrEqual( + MATRIX_IDB_PERSIST_INTERVAL_MS, + ); + expect(options.stale).toBe(5 * 60_000); + } + }); }); diff --git a/extensions/matrix/src/matrix/sdk/idb-persistence.ts b/extensions/matrix/src/matrix/sdk/idb-persistence.ts index 238ab1d5b07..5f7fe6d8c7f 100644 --- a/extensions/matrix/src/matrix/sdk/idb-persistence.ts +++ b/extensions/matrix/src/matrix/sdk/idb-persistence.ts @@ -1,8 +1,8 @@ import fs from "node:fs"; import path from "node:path"; import { indexedDB as fakeIndexedDB } from "fake-indexeddb"; -import type { FileLockOptions } from "openclaw/plugin-sdk/infra-runtime"; import { withFileLock } from "openclaw/plugin-sdk/infra-runtime"; +import { MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS } from "./idb-persistence-lock.js"; import { LogService } from "./logger.js"; // Advisory lock options for IDB snapshot file access. Without locking, the @@ -11,17 +11,6 @@ import { LogService } from "./logger.js"; // Use a longer stale window than the generic 30s default because snapshot // restore and large crypto-store dumps can legitimately hold the lock for // longer, and reclaiming a live lock would reintroduce concurrent corruption. -const IDB_SNAPSHOT_LOCK_OPTIONS: FileLockOptions = { - retries: { - retries: 10, - factor: 2, - minTimeout: 50, - maxTimeout: 5_000, - randomize: true, - }, - stale: 5 * 60_000, -}; - type IdbStoreSnapshot = { name: string; keyPath: IDBObjectStoreParameters["keyPath"]; @@ -217,19 +206,23 @@ export async function restoreIdbFromDisk(snapshotPath?: string): Promise { - const data = fs.readFileSync(resolvedPath, "utf8"); - const snapshot = parseSnapshotPayload(data); - if (!snapshot) { - return false; - } - await restoreIndexedDatabases(snapshot); - LogService.info( - "IdbPersistence", - `Restored ${snapshot.length} IndexedDB database(s) from ${resolvedPath}`, - ); - return true; - }); + const restored = await withFileLock( + resolvedPath, + MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS, + async () => { + const data = fs.readFileSync(resolvedPath, "utf8"); + const snapshot = parseSnapshotPayload(data); + if (!snapshot) { + return false; + } + await restoreIndexedDatabases(snapshot); + LogService.info( + "IdbPersistence", + `Restored ${snapshot.length} IndexedDB database(s) from ${resolvedPath}`, + ); + return true; + }, + ); if (restored) { return true; } @@ -252,15 +245,19 @@ export async function persistIdbToDisk(params?: { const snapshotPath = params?.snapshotPath ?? resolveDefaultIdbSnapshotPath(); try { fs.mkdirSync(path.dirname(snapshotPath), { recursive: true }); - const persistedCount = await withFileLock(snapshotPath, IDB_SNAPSHOT_LOCK_OPTIONS, async () => { - const snapshot = await dumpIndexedDatabases(params?.databasePrefix); - if (snapshot.length === 0) { - return 0; - } - fs.writeFileSync(snapshotPath, JSON.stringify(snapshot)); - fs.chmodSync(snapshotPath, 0o600); - return snapshot.length; - }); + const persistedCount = await withFileLock( + snapshotPath, + MATRIX_IDB_SNAPSHOT_LOCK_OPTIONS, + async () => { + const snapshot = await dumpIndexedDatabases(params?.databasePrefix); + if (snapshot.length === 0) { + return 0; + } + fs.writeFileSync(snapshotPath, JSON.stringify(snapshot)); + fs.chmodSync(snapshotPath, 0o600); + return snapshot.length; + }, + ); if (persistedCount === 0) return; LogService.debug( "IdbPersistence",