diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b1abef4951..eeca9a510df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -295,6 +295,7 @@ Docs: https://docs.openclaw.ai - Plugins/Matrix: durably dedupe inbound room events across gateway restarts so previously handled Matrix messages are not replayed as new, while preserving clean-restart backlog delivery for unseen events. (#50922) thanks @gumadeiras - Agents/media replies: migrate the remaining browser, canvas, and nodes snapshot outputs onto `details.media` so generated media keeps attaching to assistant replies after the collect-then-attach refactor. (#51731) Thanks @christianklotz. - Android/contacts search: escape literal `%` and `_` in contact-name queries so searches like `100%` or `_id` no longer match unrelated contacts through SQL `LIKE` wildcards. (#41891) Thanks @Kaneki-x. +- Gateway/usage: include reset and deleted archived session transcripts in usage totals, session discovery, and archived-only session detail fallback so the Usage view no longer undercounts rotated sessions. (#43215) Thanks @rcrick. ## 2026.3.13 diff --git a/src/config/sessions/artifacts.test.ts b/src/config/sessions/artifacts.test.ts index b8c438a9eca..53eb2526e73 100644 --- a/src/config/sessions/artifacts.test.ts +++ b/src/config/sessions/artifacts.test.ts @@ -3,6 +3,8 @@ import { formatSessionArchiveTimestamp, isPrimarySessionTranscriptFileName, isSessionArchiveArtifactName, + isUsageCountedSessionTranscriptFileName, + parseUsageCountedSessionIdFromFileName, parseSessionArchiveTimestamp, } from "./artifacts.js"; @@ -25,6 +27,32 @@ describe("session artifact helpers", () => { expect(isPrimarySessionTranscriptFileName("sessions.json")).toBe(false); }); + it("classifies usage-counted transcript files", () => { + expect(isUsageCountedSessionTranscriptFileName("abc.jsonl")).toBe(true); + expect( + isUsageCountedSessionTranscriptFileName("abc.jsonl.reset.2026-01-01T00-00-00.000Z"), + ).toBe(true); + expect( + isUsageCountedSessionTranscriptFileName("abc.jsonl.deleted.2026-01-01T00-00-00.000Z"), + ).toBe(true); + expect(isUsageCountedSessionTranscriptFileName("abc.jsonl.bak.2026-01-01T00-00-00.000Z")).toBe( + false, + ); + }); + + it("parses usage-counted session ids from file names", () => { + expect(parseUsageCountedSessionIdFromFileName("abc.jsonl")).toBe("abc"); + expect(parseUsageCountedSessionIdFromFileName("abc.jsonl.reset.2026-01-01T00-00-00.000Z")).toBe( + "abc", + ); + expect( + parseUsageCountedSessionIdFromFileName("abc.jsonl.deleted.2026-01-01T00-00-00.000Z"), + ).toBe("abc"); + expect(parseUsageCountedSessionIdFromFileName("abc.jsonl.bak.2026-01-01T00-00-00.000Z")).toBe( + null, + ); + }); + it("formats and parses archive timestamps", () => { const now = Date.parse("2026-02-23T12:34:56.000Z"); const stamp = formatSessionArchiveTimestamp(now); diff --git a/src/config/sessions/artifacts.ts b/src/config/sessions/artifacts.ts index c851f7967fc..14edb495d15 100644 --- a/src/config/sessions/artifacts.ts +++ b/src/config/sessions/artifacts.ts @@ -34,6 +34,27 @@ export function isPrimarySessionTranscriptFileName(fileName: string): boolean { return !isSessionArchiveArtifactName(fileName); } +export function isUsageCountedSessionTranscriptFileName(fileName: string): boolean { + if (isPrimarySessionTranscriptFileName(fileName)) { + return true; + } + return hasArchiveSuffix(fileName, "reset") || hasArchiveSuffix(fileName, "deleted"); +} + +export function parseUsageCountedSessionIdFromFileName(fileName: string): string | null { + if (isPrimarySessionTranscriptFileName(fileName)) { + return fileName.slice(0, -".jsonl".length); + } + for (const reason of ["reset", "deleted"] as const) { + const marker = `.jsonl.${reason}.`; + const index = fileName.lastIndexOf(marker); + if (index > 0 && hasArchiveSuffix(fileName, reason)) { + return fileName.slice(0, index); + } + } + return null; +} + export function formatSessionArchiveTimestamp(nowMs = Date.now()): string { return new Date(nowMs).toISOString().replaceAll(":", "-"); } diff --git a/src/gateway/server-methods/usage.ts b/src/gateway/server-methods/usage.ts index f074a0c66b1..8298f34eb24 100644 --- a/src/gateway/server-methods/usage.ts +++ b/src/gateway/server-methods/usage.ts @@ -17,6 +17,7 @@ import { loadSessionCostSummary, loadSessionUsageTimeSeries, discoverAllSessions, + resolveExistingUsageSessionFile, type DiscoveredSession, } from "../../infra/session-cost-usage.js"; import { parseAgentSessionKey } from "../../routing/session-key.js"; @@ -425,13 +426,18 @@ export const usageHandlers: GatewayRequestHandlers = { const sessionId = storeEntry?.sessionId ?? keyRest; // Resolve the session file path - let sessionFile: string; + let sessionFile: string | undefined; try { const pathOpts = resolveSessionFilePathOptions({ storePath: storePath !== "(multiple)" ? storePath : undefined, agentId: agentIdFromKey, }); - sessionFile = resolveSessionFilePath(sessionId, storeEntry, pathOpts); + sessionFile = resolveExistingUsageSessionFile({ + sessionId, + sessionEntry: storeEntry, + sessionFile: resolveSessionFilePath(sessionId, storeEntry, pathOpts), + agentId: agentIdFromKey, + }); } catch { respond( false, @@ -441,20 +447,22 @@ export const usageHandlers: GatewayRequestHandlers = { return; } - try { - const stats = fs.statSync(sessionFile); - if (stats.isFile()) { - mergedEntries.push({ - key: resolvedStoreKey, - sessionId, - sessionFile, - label: storeEntry?.label, - updatedAt: storeEntry?.updatedAt ?? stats.mtimeMs, - storeEntry, - }); + if (sessionFile) { + try { + const stats = fs.statSync(sessionFile); + if (stats.isFile()) { + mergedEntries.push({ + key: resolvedStoreKey, + sessionId, + sessionFile, + label: storeEntry?.label, + updatedAt: storeEntry?.updatedAt ?? stats.mtimeMs, + storeEntry, + }); + } + } catch { + // File doesn't exist - no results for this key } - } catch { - // File doesn't exist - no results for this key } } else { // Full discovery for list view diff --git a/src/infra/session-cost-usage.test.ts b/src/infra/session-cost-usage.test.ts index ba9e10b1f4a..24e212e4f83 100644 --- a/src/infra/session-cost-usage.test.ts +++ b/src/infra/session-cost-usage.test.ts @@ -231,6 +231,304 @@ describe("session cost usage", () => { }); }); + it("counts reset and deleted transcripts in global usage summary, but excludes bak archives", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-usage-archives-")); + const sessionsDir = path.join(root, "agents", "main", "sessions"); + await fs.mkdir(sessionsDir, { recursive: true }); + + const timestamp = "2026-02-12T10:00:00.000Z"; + await fs.writeFile( + path.join(sessionsDir, "sess-active.jsonl"), + JSON.stringify({ + type: "message", + timestamp, + message: { + role: "assistant", + usage: { input: 1, output: 2, totalTokens: 3, cost: { total: 0.003 } }, + }, + }), + "utf-8", + ); + await fs.writeFile( + path.join(sessionsDir, "sess-reset.jsonl.reset.2026-02-12T11-00-00.000Z"), + JSON.stringify({ + type: "message", + timestamp, + message: { + role: "assistant", + usage: { input: 10, output: 20, totalTokens: 30, cost: { total: 0.03 } }, + }, + }), + "utf-8", + ); + await fs.writeFile( + path.join(sessionsDir, "sess-deleted.jsonl.deleted.2026-02-12T12-00-00.000Z"), + JSON.stringify({ + type: "message", + timestamp, + message: { + role: "assistant", + usage: { input: 4, output: 5, totalTokens: 9, cost: { total: 0.009 } }, + }, + }), + "utf-8", + ); + await fs.writeFile( + path.join(sessionsDir, "sess-bak.jsonl.bak.2026-02-12T13-00-00.000Z"), + JSON.stringify({ + type: "message", + timestamp, + message: { + role: "assistant", + usage: { input: 100, output: 200, totalTokens: 300, cost: { total: 0.3 } }, + }, + }), + "utf-8", + ); + + await withStateDir(root, async () => { + const summary = await loadCostUsageSummary({ + startMs: Date.UTC(2026, 1, 12), + endMs: Date.UTC(2026, 1, 12, 23, 59, 59, 999), + }); + expect(summary.totals.totalTokens).toBe(42); + expect(summary.totals.totalCost).toBeCloseTo(0.042, 8); + }); + }); + + it("discovers reset and deleted transcripts as usage sessions", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-discover-archives-")); + const sessionsDir = path.join(root, "agents", "main", "sessions"); + await fs.mkdir(sessionsDir, { recursive: true }); + + await fs.writeFile( + path.join(sessionsDir, "sess-reset.jsonl.reset.2026-02-12T11-00-00.000Z"), + JSON.stringify({ + type: "message", + timestamp: "2026-02-12T10:00:00.000Z", + message: { role: "user", content: "reset transcript" }, + }), + "utf-8", + ); + await fs.writeFile( + path.join(sessionsDir, "sess-deleted.jsonl.deleted.2026-02-12T12-00-00.000Z"), + JSON.stringify({ + type: "message", + timestamp: "2026-02-12T10:00:00.000Z", + message: { role: "user", content: "deleted transcript" }, + }), + "utf-8", + ); + + await withStateDir(root, async () => { + const sessions = await discoverAllSessions(); + expect(sessions.map((session) => session.sessionId)).toEqual(["sess-deleted", "sess-reset"]); + expect( + sessions + .map((session) => session.firstUserMessage) + .toSorted((a, b) => String(a).localeCompare(String(b))), + ).toEqual(["deleted transcript", "reset transcript"]); + }); + }); + + it("deduplicates discovered sessions by sessionId and keeps the newest archive", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-discover-dedupe-")); + const sessionsDir = path.join(root, "agents", "main", "sessions"); + await fs.mkdir(sessionsDir, { recursive: true }); + + const resetPath = path.join(sessionsDir, "sess-shared.jsonl.reset.2026-02-12T11-00-00.000Z"); + const deletedPath = path.join( + sessionsDir, + "sess-shared.jsonl.deleted.2026-02-12T12-00-00.000Z", + ); + + await fs.writeFile( + resetPath, + JSON.stringify({ + type: "message", + timestamp: "2026-02-12T10:00:00.000Z", + message: { role: "user", content: "older archive" }, + }), + "utf-8", + ); + await fs.writeFile( + deletedPath, + JSON.stringify({ + type: "message", + timestamp: "2026-02-12T10:05:00.000Z", + message: { role: "user", content: "newer archive" }, + }), + "utf-8", + ); + + const older = Date.UTC(2026, 1, 12, 11, 0, 0) / 1000; + const newer = Date.UTC(2026, 1, 12, 12, 0, 0) / 1000; + await fs.utimes(resetPath, older, older); + await fs.utimes(deletedPath, newer, newer); + + await withStateDir(root, async () => { + const sessions = await discoverAllSessions(); + expect(sessions).toHaveLength(1); + expect(sessions[0]?.sessionId).toBe("sess-shared"); + expect(sessions[0]?.sessionFile).toContain(".jsonl.deleted."); + expect(sessions[0]?.firstUserMessage).toBe("newer archive"); + }); + }); + + it("prefers the active transcript over archives during discovery dedupe", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-discover-active-preferred-")); + const sessionsDir = path.join(root, "agents", "main", "sessions"); + await fs.mkdir(sessionsDir, { recursive: true }); + + const activePath = path.join(sessionsDir, "sess-live.jsonl"); + const archivePath = path.join(sessionsDir, "sess-live.jsonl.deleted.2026-02-12T12-00-00.000Z"); + + await fs.writeFile( + activePath, + JSON.stringify({ + type: "message", + timestamp: "2026-02-12T10:00:00.000Z", + message: { role: "user", content: "active transcript" }, + }), + "utf-8", + ); + await fs.writeFile( + archivePath, + JSON.stringify({ + type: "message", + timestamp: "2026-02-12T10:05:00.000Z", + message: { role: "user", content: "archive transcript" }, + }), + "utf-8", + ); + + const older = Date.UTC(2026, 1, 12, 10, 0, 0) / 1000; + const newer = Date.UTC(2026, 1, 12, 12, 0, 0) / 1000; + await fs.utimes(activePath, older, older); + await fs.utimes(archivePath, newer, newer); + + await withStateDir(root, async () => { + const sessions = await discoverAllSessions(); + expect(sessions).toHaveLength(1); + expect(sessions[0]?.sessionId).toBe("sess-live"); + expect(sessions[0]?.sessionFile).toBe(activePath); + expect(sessions[0]?.firstUserMessage).toBe("active transcript"); + }); + }); + + it("falls back to archived reset transcripts for per-session detail queries", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-session-archive-fallback-")); + const sessionsDir = path.join(root, "agents", "main", "sessions"); + await fs.mkdir(sessionsDir, { recursive: true }); + + await fs.writeFile( + path.join(sessionsDir, "sess-reset.jsonl.reset.2026-02-12T11-00-00.000Z"), + JSON.stringify({ + type: "message", + timestamp: "2026-02-12T10:00:00.000Z", + message: { + role: "assistant", + content: "archived answer", + usage: { input: 6, output: 4, totalTokens: 10, cost: { total: 0.01 } }, + }, + }), + "utf-8", + ); + + await withStateDir(root, async () => { + const summary = await loadSessionCostSummary({ sessionId: "sess-reset" }); + const timeseries = await loadSessionUsageTimeSeries({ sessionId: "sess-reset" }); + const logs = await loadSessionLogs({ sessionId: "sess-reset" }); + + expect(summary?.totalTokens).toBe(10); + expect(summary?.sessionFile).toContain(".jsonl.reset."); + expect(timeseries?.points[0]?.totalTokens).toBe(10); + expect(logs).toHaveLength(1); + expect(logs?.[0]?.content).toContain("archived answer"); + }); + }); + + it("uses the candidate session directory for archived fallback lookups", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-session-custom-archive-")); + const customSessionsDir = path.join(root, "custom-store", "sessions"); + await fs.mkdir(customSessionsDir, { recursive: true }); + + const activePath = path.join(customSessionsDir, "sess-custom.jsonl"); + const archivePath = path.join( + customSessionsDir, + "sess-custom.jsonl.deleted.2026-02-12T12-00-00.000Z", + ); + + await fs.writeFile( + archivePath, + JSON.stringify({ + type: "message", + timestamp: "2026-02-12T12:00:00.000Z", + message: { + role: "assistant", + content: "custom archived answer", + usage: { input: 9, output: 3, totalTokens: 12, cost: { total: 0.012 } }, + }, + }), + "utf-8", + ); + + const summary = await loadSessionCostSummary({ + sessionId: "sess-custom", + sessionFile: activePath, + }); + const logs = await loadSessionLogs({ + sessionId: "sess-custom", + sessionFile: activePath, + }); + + expect(summary?.totalTokens).toBe(12); + expect(summary?.sessionFile).toBe(archivePath); + expect(logs?.[0]?.content).toContain("custom archived answer"); + }); + + it("picks the newest archive by timestamp when reset and deleted archives coexist", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-session-archive-order-")); + const sessionsDir = path.join(root, "agents", "main", "sessions"); + await fs.mkdir(sessionsDir, { recursive: true }); + + await fs.writeFile( + path.join(sessionsDir, "sess-mixed.jsonl.reset.2026-02-12T11-00-00.000Z"), + JSON.stringify({ + type: "message", + timestamp: "2026-02-12T11:00:00.000Z", + message: { + role: "assistant", + content: "older reset archive", + usage: { input: 6, output: 4, totalTokens: 10, cost: { total: 0.01 } }, + }, + }), + "utf-8", + ); + await fs.writeFile( + path.join(sessionsDir, "sess-mixed.jsonl.deleted.2026-02-12T12-00-00.000Z"), + JSON.stringify({ + type: "message", + timestamp: "2026-02-12T12:00:00.000Z", + message: { + role: "assistant", + content: "newer deleted archive", + usage: { input: 12, output: 8, totalTokens: 20, cost: { total: 0.02 } }, + }, + }), + "utf-8", + ); + + await withStateDir(root, async () => { + const summary = await loadSessionCostSummary({ sessionId: "sess-mixed" }); + const logs = await loadSessionLogs({ sessionId: "sess-mixed" }); + + expect(summary?.totalTokens).toBe(20); + expect(summary?.sessionFile).toContain(".jsonl.deleted."); + expect(logs?.[0]?.content).toContain("newer deleted archive"); + }); + }); + it("resolves non-main absolute sessionFile using explicit agentId for cost summary", async () => { const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-cost-agent-")); const workerSessionsDir = path.join(root, "agents", "worker1", "sessions"); diff --git a/src/infra/session-cost-usage.ts b/src/infra/session-cost-usage.ts index 4c021bcc72f..86f20a607a9 100644 --- a/src/infra/session-cost-usage.ts +++ b/src/infra/session-cost-usage.ts @@ -5,6 +5,13 @@ import type { NormalizedUsage, UsageLike } from "../agents/usage.js"; import { normalizeUsage } from "../agents/usage.js"; import { stripInboundMetadata } from "../auto-reply/reply/strip-inbound-meta.js"; import type { OpenClawConfig } from "../config/config.js"; +import { + isPrimarySessionTranscriptFileName, + isSessionArchiveArtifactName, + isUsageCountedSessionTranscriptFileName, + parseSessionArchiveTimestamp, + parseUsageCountedSessionIdFromFileName, +} from "../config/sessions/artifacts.js"; import { resolveSessionFilePath, resolveSessionTranscriptsDirForAgent, @@ -287,6 +294,69 @@ async function scanUsageFile(params: { }); } +export function resolveExistingUsageSessionFile(params: { + sessionId?: string; + sessionEntry?: SessionEntry; + sessionFile?: string; + agentId?: string; +}): string | undefined { + const candidate = + params.sessionFile ?? + (params.sessionId + ? resolveSessionFilePath(params.sessionId, params.sessionEntry, { + agentId: params.agentId, + }) + : undefined); + + if (candidate && fs.existsSync(candidate)) { + return candidate; + } + + const sessionId = params.sessionId?.trim(); + if (!sessionId) { + return candidate; + } + + try { + const sessionsDir = candidate + ? path.dirname(candidate) + : resolveSessionTranscriptsDirForAgent(params.agentId); + const baseFileName = `${sessionId}.jsonl`; + const entries = fs.readdirSync(sessionsDir, { withFileTypes: true }).filter((entry) => { + return ( + entry.isFile() && + (entry.name === baseFileName || + entry.name.startsWith(`${baseFileName}.reset.`) || + entry.name.startsWith(`${baseFileName}.deleted.`)) + ); + }); + + const primary = entries.find((entry) => entry.name === baseFileName); + if (primary) { + return path.join(sessionsDir, primary.name); + } + + const latestArchive = entries + .filter((entry) => isSessionArchiveArtifactName(entry.name)) + .map((entry) => entry.name) + .toSorted((a, b) => { + const tsA = + parseSessionArchiveTimestamp(a, "deleted") ?? + parseSessionArchiveTimestamp(a, "reset") ?? + 0; + const tsB = + parseSessionArchiveTimestamp(b, "deleted") ?? + parseSessionArchiveTimestamp(b, "reset") ?? + 0; + return tsB - tsA || b.localeCompare(a); + })[0]; + + return latestArchive ? path.join(sessionsDir, latestArchive) : candidate; + } catch { + return candidate; + } +} + export async function loadCostUsageSummary(params?: { startMs?: number; endMs?: number; @@ -318,7 +388,7 @@ export async function loadCostUsageSummary(params?: { const files = ( await Promise.all( entries - .filter((entry) => entry.isFile() && entry.name.endsWith(".jsonl")) + .filter((entry) => entry.isFile() && isUsageCountedSessionTranscriptFileName(entry.name)) .map(async (entry) => { const filePath = path.join(sessionsDir, entry.name); const stats = await fs.promises.stat(filePath).catch(() => null); @@ -390,10 +460,10 @@ export async function discoverAllSessions(params?: { const sessionsDir = resolveSessionTranscriptsDirForAgent(params?.agentId); const entries = await fs.promises.readdir(sessionsDir, { withFileTypes: true }).catch(() => []); - const discovered: DiscoveredSession[] = []; + const discovered = new Map(); for (const entry of entries) { - if (!entry.isFile() || !entry.name.endsWith(".jsonl")) { + if (!entry.isFile() || !isUsageCountedSessionTranscriptFileName(entry.name)) { continue; } @@ -409,8 +479,11 @@ export async function discoverAllSessions(params?: { } // Do not exclude by endMs: a session can have activity in range even if it continued later. - // Extract session ID from filename (remove .jsonl) - const sessionId = entry.name.slice(0, -6); + const sessionId = parseUsageCountedSessionIdFromFileName(entry.name); + if (!sessionId) { + continue; + } + const isPrimaryTranscript = isPrimarySessionTranscriptFileName(entry.name); // Try to read first user message for label extraction let firstUserMessage: string | undefined; @@ -447,16 +520,33 @@ export async function discoverAllSessions(params?: { // Ignore read errors } - discovered.push({ - sessionId, - sessionFile: filePath, - mtime: stats.mtimeMs, - firstUserMessage, - }); + const existing = discovered.get(sessionId); + const existingIsPrimary = existing + ? isPrimarySessionTranscriptFileName(path.basename(existing.sessionFile)) + : false; + const shouldReplace = + !existing || + (isPrimaryTranscript && !existingIsPrimary) || + (isPrimaryTranscript === existingIsPrimary && stats.mtimeMs >= existing.mtime); + + if (shouldReplace) { + discovered.set(sessionId, { + sessionId, + sessionFile: filePath, + mtime: stats.mtimeMs, + firstUserMessage: firstUserMessage ?? existing?.firstUserMessage, + }); + continue; + } + + if (!existing.firstUserMessage && firstUserMessage) { + existing.firstUserMessage = firstUserMessage; + discovered.set(sessionId, existing); + } } // Sort by mtime descending (most recent first) - return discovered.toSorted((a, b) => b.mtime - a.mtime); + return Array.from(discovered.values()).toSorted((a, b) => b.mtime - a.mtime); } export async function loadSessionCostSummary(params: { @@ -468,13 +558,7 @@ export async function loadSessionCostSummary(params: { startMs?: number; endMs?: number; }): Promise { - const sessionFile = - params.sessionFile ?? - (params.sessionId - ? resolveSessionFilePath(params.sessionId, params.sessionEntry, { - agentId: params.agentId, - }) - : undefined); + const sessionFile = resolveExistingUsageSessionFile(params); if (!sessionFile || !fs.existsSync(sessionFile)) { return null; } @@ -745,13 +829,7 @@ export async function loadSessionUsageTimeSeries(params: { agentId?: string; maxPoints?: number; }): Promise { - const sessionFile = - params.sessionFile ?? - (params.sessionId - ? resolveSessionFilePath(params.sessionId, params.sessionEntry, { - agentId: params.agentId, - }) - : undefined); + const sessionFile = resolveExistingUsageSessionFile(params); if (!sessionFile || !fs.existsSync(sessionFile)) { return null; } @@ -854,13 +932,7 @@ export async function loadSessionLogs(params: { agentId?: string; limit?: number; }): Promise { - const sessionFile = - params.sessionFile ?? - (params.sessionId - ? resolveSessionFilePath(params.sessionId, params.sessionEntry, { - agentId: params.agentId, - }) - : undefined); + const sessionFile = resolveExistingUsageSessionFile(params); if (!sessionFile || !fs.existsSync(sessionFile)) { return null; }