mirror of https://github.com/openclaw/openclaw.git
Gateway: add eager secrets runtime snapshot activation
This commit is contained in:
joshavant
Peter Steinberger
parent
2f3b919b94
commit
b50c4c2c44
|
|
@ -17,7 +17,10 @@ export {
|
|||
suggestOAuthProfileIdForLegacyDefault,
|
||||
} from "./auth-profiles/repair.js";
|
||||
export {
|
||||
clearRuntimeAuthProfileStoreSnapshots,
|
||||
ensureAuthProfileStore,
|
||||
loadAuthProfileStoreForRuntime,
|
||||
replaceRuntimeAuthProfileStoreSnapshots,
|
||||
loadAuthProfileStore,
|
||||
saveAuthProfileStore,
|
||||
} from "./auth-profiles/store.js";
|
||||
|
|
|
|||
|
|
@ -14,6 +14,65 @@ type RejectedCredentialEntry = { key: string; reason: CredentialRejectReason };
|
|||
|
||||
const AUTH_PROFILE_TYPES = new Set<AuthProfileCredential["type"]>(["api_key", "oauth", "token"]);
|
||||
|
||||
const runtimeAuthStoreSnapshots = new Map<string, AuthProfileStore>();
|
||||
|
||||
function resolveRuntimeStoreKey(agentDir?: string): string {
|
||||
return resolveAuthStorePath(agentDir);
|
||||
}
|
||||
|
||||
function cloneAuthProfileStore(store: AuthProfileStore): AuthProfileStore {
|
||||
return structuredClone(store);
|
||||
}
|
||||
|
||||
function resolveRuntimeAuthProfileStore(agentDir?: string): AuthProfileStore | null {
|
||||
if (runtimeAuthStoreSnapshots.size === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const mainKey = resolveRuntimeStoreKey(undefined);
|
||||
const requestedKey = resolveRuntimeStoreKey(agentDir);
|
||||
const mainStore = runtimeAuthStoreSnapshots.get(mainKey);
|
||||
const requestedStore = runtimeAuthStoreSnapshots.get(requestedKey);
|
||||
|
||||
if (!agentDir || requestedKey === mainKey) {
|
||||
if (!mainStore) {
|
||||
return null;
|
||||
}
|
||||
return cloneAuthProfileStore(mainStore);
|
||||
}
|
||||
|
||||
if (mainStore && requestedStore) {
|
||||
return mergeAuthProfileStores(
|
||||
cloneAuthProfileStore(mainStore),
|
||||
cloneAuthProfileStore(requestedStore),
|
||||
);
|
||||
}
|
||||
if (requestedStore) {
|
||||
return cloneAuthProfileStore(requestedStore);
|
||||
}
|
||||
if (mainStore) {
|
||||
return cloneAuthProfileStore(mainStore);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export function replaceRuntimeAuthProfileStoreSnapshots(
|
||||
entries: Array<{ agentDir?: string; store: AuthProfileStore }>,
|
||||
): void {
|
||||
runtimeAuthStoreSnapshots.clear();
|
||||
for (const entry of entries) {
|
||||
runtimeAuthStoreSnapshots.set(
|
||||
resolveRuntimeStoreKey(entry.agentDir),
|
||||
cloneAuthProfileStore(entry.store),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function clearRuntimeAuthProfileStoreSnapshots(): void {
|
||||
runtimeAuthStoreSnapshots.clear();
|
||||
}
|
||||
|
||||
export async function updateAuthProfileStoreWithLock(params: {
|
||||
agentDir?: string;
|
||||
updater: (store: AuthProfileStore) => boolean;
|
||||
|
|
@ -372,10 +431,30 @@ function loadAuthProfileStoreForAgent(
|
|||
return store;
|
||||
}
|
||||
|
||||
export function loadAuthProfileStoreForRuntime(
|
||||
agentDir?: string,
|
||||
options?: { allowKeychainPrompt?: boolean },
|
||||
): AuthProfileStore {
|
||||
const store = loadAuthProfileStoreForAgent(agentDir, options);
|
||||
const authPath = resolveAuthStorePath(agentDir);
|
||||
const mainAuthPath = resolveAuthStorePath();
|
||||
if (!agentDir || authPath === mainAuthPath) {
|
||||
return store;
|
||||
}
|
||||
|
||||
const mainStore = loadAuthProfileStoreForAgent(undefined, options);
|
||||
return mergeAuthProfileStores(mainStore, store);
|
||||
}
|
||||
|
||||
export function ensureAuthProfileStore(
|
||||
agentDir?: string,
|
||||
options?: { allowKeychainPrompt?: boolean },
|
||||
): AuthProfileStore {
|
||||
const runtimeStore = resolveRuntimeAuthProfileStore(agentDir);
|
||||
if (runtimeStore) {
|
||||
return runtimeStore;
|
||||
}
|
||||
|
||||
const store = loadAuthProfileStoreForAgent(agentDir, options);
|
||||
const authPath = resolveAuthStorePath(agentDir);
|
||||
const mainAuthPath = resolveAuthStorePath();
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
import type { OAuthCredentials } from "@mariozechner/pi-ai";
|
||||
import type { OpenClawConfig } from "../../config/config.js";
|
||||
import type { SecretRef } from "../../config/types.secrets.js";
|
||||
|
||||
export type ApiKeyCredential = {
|
||||
type: "api_key";
|
||||
provider: string;
|
||||
key?: string;
|
||||
keyRef?: SecretRef;
|
||||
email?: string;
|
||||
/** Optional provider-specific metadata (e.g., account IDs, gateway IDs). */
|
||||
metadata?: Record<string, string>;
|
||||
|
|
@ -18,6 +20,7 @@ export type TokenCredential = {
|
|||
type: "token";
|
||||
provider: string;
|
||||
token: string;
|
||||
tokenRef?: SecretRef;
|
||||
/** Optional expiry timestamp (ms since epoch). */
|
||||
expires?: number;
|
||||
email?: string;
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ import {
|
|||
KILOCODE_DEFAULT_MAX_TOKENS,
|
||||
KILOCODE_MODEL_CATALOG,
|
||||
} from "../providers/kilocode-shared.js";
|
||||
import { normalizeOptionalSecretInput } from "../utils/normalize-secret-input.js";
|
||||
import { ensureAuthProfileStore, listProfilesForProvider } from "./auth-profiles.js";
|
||||
import { discoverBedrockModels } from "./bedrock-discovery.js";
|
||||
import {
|
||||
|
|
@ -405,16 +406,17 @@ export function normalizeProviders(params: {
|
|||
for (const [key, provider] of Object.entries(providers)) {
|
||||
const normalizedKey = key.trim();
|
||||
let normalizedProvider = provider;
|
||||
const configuredApiKey = normalizedProvider.apiKey;
|
||||
|
||||
// Fix common misconfig: apiKey set to "${ENV_VAR}" instead of "ENV_VAR".
|
||||
if (
|
||||
normalizedProvider.apiKey &&
|
||||
normalizeApiKeyConfig(normalizedProvider.apiKey) !== normalizedProvider.apiKey
|
||||
typeof configuredApiKey === "string" &&
|
||||
normalizeApiKeyConfig(configuredApiKey) !== configuredApiKey
|
||||
) {
|
||||
mutated = true;
|
||||
normalizedProvider = {
|
||||
...normalizedProvider,
|
||||
apiKey: normalizeApiKeyConfig(normalizedProvider.apiKey),
|
||||
apiKey: normalizeApiKeyConfig(configuredApiKey),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -422,7 +424,9 @@ export function normalizeProviders(params: {
|
|||
// Fill it from the environment or auth profiles when possible.
|
||||
const hasModels =
|
||||
Array.isArray(normalizedProvider.models) && normalizedProvider.models.length > 0;
|
||||
if (hasModels && !normalizedProvider.apiKey?.trim()) {
|
||||
const normalizedApiKey = normalizeOptionalSecretInput(normalizedProvider.apiKey);
|
||||
const hasConfiguredApiKey = Boolean(normalizedApiKey || normalizedProvider.apiKey);
|
||||
if (hasModels && !hasConfiguredApiKey) {
|
||||
const authMode =
|
||||
normalizedProvider.auth ?? (normalizedKey === "amazon-bedrock" ? "aws-sdk" : undefined);
|
||||
if (authMode === "aws-sdk") {
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import type { OpenClawConfig } from "../config/config.js";
|
|||
import type { ModelProviderConfig } from "../config/types.models.js";
|
||||
import type { RuntimeEnv } from "../runtime.js";
|
||||
import { fetchWithTimeout } from "../utils/fetch-timeout.js";
|
||||
import { normalizeOptionalSecretInput } from "../utils/normalize-secret-input.js";
|
||||
import type { WizardPrompter } from "../wizard/prompts.js";
|
||||
import { applyPrimaryModel } from "./model-picker.js";
|
||||
import { normalizeAlias } from "./models/shared.js";
|
||||
|
|
@ -541,7 +542,7 @@ export function applyCustomApiConfig(params: ApplyCustomApiConfigParams): Custom
|
|||
const mergedModels = hasModel ? existingModels : [...existingModels, nextModel];
|
||||
const { apiKey: existingApiKey, ...existingProviderRest } = existingProvider ?? {};
|
||||
const normalizedApiKey =
|
||||
params.apiKey?.trim() || (existingApiKey ? existingApiKey.trim() : undefined);
|
||||
normalizeOptionalSecretInput(params.apiKey) ?? normalizeOptionalSecretInput(existingApiKey);
|
||||
|
||||
let config: OpenClawConfig = {
|
||||
...params.config,
|
||||
|
|
|
|||
|
|
@ -1,11 +1,14 @@
|
|||
export {
|
||||
clearConfigCache,
|
||||
clearRuntimeConfigSnapshot,
|
||||
createConfigIO,
|
||||
getRuntimeConfigSnapshot,
|
||||
loadConfig,
|
||||
parseConfigJson5,
|
||||
readConfigFileSnapshot,
|
||||
readConfigFileSnapshotForWrite,
|
||||
resolveConfigSnapshotHash,
|
||||
setRuntimeConfigSnapshot,
|
||||
writeConfigFile,
|
||||
} from "./io.js";
|
||||
export { migrateLegacyConfig } from "./legacy-migrate.js";
|
||||
|
|
|
|||
|
|
@ -1298,6 +1298,7 @@ let configCache: {
|
|||
expiresAt: number;
|
||||
config: OpenClawConfig;
|
||||
} | null = null;
|
||||
let runtimeConfigSnapshot: OpenClawConfig | null = null;
|
||||
|
||||
function resolveConfigCacheMs(env: NodeJS.ProcessEnv): number {
|
||||
const raw = env.OPENCLAW_CONFIG_CACHE_MS?.trim();
|
||||
|
|
@ -1325,7 +1326,24 @@ export function clearConfigCache(): void {
|
|||
configCache = null;
|
||||
}
|
||||
|
||||
export function setRuntimeConfigSnapshot(config: OpenClawConfig): void {
|
||||
runtimeConfigSnapshot = config;
|
||||
clearConfigCache();
|
||||
}
|
||||
|
||||
export function clearRuntimeConfigSnapshot(): void {
|
||||
runtimeConfigSnapshot = null;
|
||||
clearConfigCache();
|
||||
}
|
||||
|
||||
export function getRuntimeConfigSnapshot(): OpenClawConfig | null {
|
||||
return runtimeConfigSnapshot;
|
||||
}
|
||||
|
||||
export function loadConfig(): OpenClawConfig {
|
||||
if (runtimeConfigSnapshot) {
|
||||
return runtimeConfigSnapshot;
|
||||
}
|
||||
const io = createConfigIO();
|
||||
const configPath = io.configPath;
|
||||
const now = Date.now();
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
import type { SecretInput } from "./types.secrets.js";
|
||||
|
||||
export type ModelApi =
|
||||
| "openai-completions"
|
||||
| "openai-responses"
|
||||
|
|
@ -43,7 +45,7 @@ export type ModelDefinitionConfig = {
|
|||
|
||||
export type ModelProviderConfig = {
|
||||
baseUrl: string;
|
||||
apiKey?: string;
|
||||
apiKey?: SecretInput;
|
||||
auth?: ModelProviderAuthMode;
|
||||
api?: ModelApi;
|
||||
headers?: Record<string, string>;
|
||||
|
|
|
|||
|
|
@ -255,7 +255,7 @@ export function startGatewayConfigReloader(opts: {
|
|||
initialConfig: OpenClawConfig;
|
||||
readSnapshot: () => Promise<ConfigFileSnapshot>;
|
||||
onHotReload: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => Promise<void>;
|
||||
onRestart: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => void;
|
||||
onRestart: (plan: GatewayReloadPlan, nextConfig: OpenClawConfig) => void | Promise<void>;
|
||||
log: {
|
||||
info: (msg: string) => void;
|
||||
warn: (msg: string) => void;
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import { createDefaultDeps } from "../cli/deps.js";
|
|||
import { isRestartEnabled } from "../config/commands.js";
|
||||
import {
|
||||
CONFIG_PATH,
|
||||
type OpenClawConfig,
|
||||
isNixMode,
|
||||
loadConfig,
|
||||
migrateLegacyConfig,
|
||||
|
|
@ -45,6 +46,12 @@ import { createEmptyPluginRegistry } from "../plugins/registry.js";
|
|||
import type { PluginServicesHandle } from "../plugins/services.js";
|
||||
import { getTotalQueueSize } from "../process/command-queue.js";
|
||||
import type { RuntimeEnv } from "../runtime.js";
|
||||
import {
|
||||
activateSecretsRuntimeSnapshot,
|
||||
clearSecretsRuntimeSnapshot,
|
||||
getActiveSecretsRuntimeSnapshot,
|
||||
prepareSecretsRuntimeSnapshot,
|
||||
} from "../secrets/runtime.js";
|
||||
import { runOnboardingWizard } from "../wizard/onboarding.js";
|
||||
import { createAuthRateLimiter, type AuthRateLimiter } from "./auth-rate-limit.js";
|
||||
import { startChannelHealthMonitor } from "./channel-health-monitor.js";
|
||||
|
|
@ -107,6 +114,7 @@ const logReload = log.child("reload");
|
|||
const logHooks = log.child("hooks");
|
||||
const logPlugins = log.child("plugins");
|
||||
const logWsControl = log.child("ws");
|
||||
const logSecrets = log.child("secrets");
|
||||
const gatewayRuntime = runtimeForLogger(log);
|
||||
const canvasRuntime = runtimeForLogger(logCanvas);
|
||||
|
||||
|
|
@ -248,7 +256,64 @@ export async function startGatewayServer(
|
|||
}
|
||||
}
|
||||
|
||||
let cfgAtStart = loadConfig();
|
||||
let secretsDegraded = false;
|
||||
const activateRuntimeSecrets = async (
|
||||
config: OpenClawConfig,
|
||||
params: { reason: "startup" | "reload" | "restart-check"; activate: boolean },
|
||||
) => {
|
||||
try {
|
||||
const prepared = await prepareSecretsRuntimeSnapshot({ config });
|
||||
if (params.activate) {
|
||||
activateSecretsRuntimeSnapshot(prepared);
|
||||
}
|
||||
for (const warning of prepared.warnings) {
|
||||
logSecrets.warn(`[${warning.code}] ${warning.message}`);
|
||||
}
|
||||
if (secretsDegraded) {
|
||||
logSecrets.info(
|
||||
"[SECRETS_RELOADER_RECOVERED] Secret resolution recovered; runtime remained on last-known-good during the outage.",
|
||||
);
|
||||
}
|
||||
secretsDegraded = false;
|
||||
return prepared;
|
||||
} catch (err) {
|
||||
const details = String(err);
|
||||
if (!secretsDegraded) {
|
||||
logSecrets.error(`[SECRETS_RELOADER_DEGRADED] ${details}`);
|
||||
} else {
|
||||
logSecrets.warn(`[SECRETS_RELOADER_DEGRADED] ${details}`);
|
||||
}
|
||||
secretsDegraded = true;
|
||||
if (params.reason === "startup") {
|
||||
throw new Error(`Startup failed: required secrets are unavailable. ${details}`, {
|
||||
cause: err,
|
||||
});
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
};
|
||||
|
||||
// Fail fast before startup if required refs are unresolved.
|
||||
let cfgAtStart: OpenClawConfig;
|
||||
{
|
||||
const freshSnapshot = await readConfigFileSnapshot();
|
||||
if (!freshSnapshot.valid) {
|
||||
const issues =
|
||||
freshSnapshot.issues.length > 0
|
||||
? freshSnapshot.issues
|
||||
.map((issue) => `${issue.path || "<root>"}: ${issue.message}`)
|
||||
.join("\n")
|
||||
: "Unknown validation issue.";
|
||||
throw new Error(`Invalid config at ${freshSnapshot.path}.\n${issues}`);
|
||||
}
|
||||
const prepared = await activateRuntimeSecrets(freshSnapshot.config, {
|
||||
reason: "startup",
|
||||
activate: true,
|
||||
});
|
||||
cfgAtStart = prepared.config;
|
||||
}
|
||||
|
||||
cfgAtStart = loadConfig();
|
||||
const authBootstrap = await ensureGatewayStartupAuth({
|
||||
cfg: cfgAtStart,
|
||||
env: process.env,
|
||||
|
|
@ -268,6 +333,12 @@ export async function startGatewayServer(
|
|||
);
|
||||
}
|
||||
}
|
||||
cfgAtStart = (
|
||||
await activateRuntimeSecrets(cfgAtStart, {
|
||||
reason: "startup",
|
||||
activate: true,
|
||||
})
|
||||
).config;
|
||||
const diagnosticsEnabled = isDiagnosticsEnabled(cfgAtStart);
|
||||
if (diagnosticsEnabled) {
|
||||
startDiagnosticHeartbeat();
|
||||
|
|
@ -738,8 +809,27 @@ export async function startGatewayServer(
|
|||
return startGatewayConfigReloader({
|
||||
initialConfig: cfgAtStart,
|
||||
readSnapshot: readConfigFileSnapshot,
|
||||
onHotReload: applyHotReload,
|
||||
onRestart: requestGatewayRestart,
|
||||
onHotReload: async (plan, nextConfig) => {
|
||||
const previousSnapshot = getActiveSecretsRuntimeSnapshot();
|
||||
const prepared = await activateRuntimeSecrets(nextConfig, {
|
||||
reason: "reload",
|
||||
activate: true,
|
||||
});
|
||||
try {
|
||||
await applyHotReload(plan, prepared.config);
|
||||
} catch (err) {
|
||||
if (previousSnapshot) {
|
||||
activateSecretsRuntimeSnapshot(previousSnapshot);
|
||||
} else {
|
||||
clearSecretsRuntimeSnapshot();
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
},
|
||||
onRestart: async (plan, nextConfig) => {
|
||||
await activateRuntimeSecrets(nextConfig, { reason: "restart-check", activate: false });
|
||||
requestGatewayRestart(plan, nextConfig);
|
||||
},
|
||||
log: {
|
||||
info: (msg) => logReload.info(msg),
|
||||
warn: (msg) => logReload.warn(msg),
|
||||
|
|
@ -794,6 +884,7 @@ export async function startGatewayServer(
|
|||
authRateLimiter?.dispose();
|
||||
browserAuthRateLimiter.dispose();
|
||||
channelHealthMonitor?.stop();
|
||||
clearSecretsRuntimeSnapshot();
|
||||
await close(opts);
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -0,0 +1,159 @@
|
|||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { ensureAuthProfileStore } from "../agents/auth-profiles.js";
|
||||
import { loadConfig, type OpenClawConfig } from "../config/config.js";
|
||||
import {
|
||||
activateSecretsRuntimeSnapshot,
|
||||
clearSecretsRuntimeSnapshot,
|
||||
prepareSecretsRuntimeSnapshot,
|
||||
} from "./runtime.js";
|
||||
|
||||
const runExecMock = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("../process/exec.js", () => ({
|
||||
runExec: runExecMock,
|
||||
}));
|
||||
|
||||
describe("secrets runtime snapshot", () => {
|
||||
afterEach(() => {
|
||||
runExecMock.mockReset();
|
||||
clearSecretsRuntimeSnapshot();
|
||||
});
|
||||
|
||||
it("resolves env refs for config and auth profiles", async () => {
|
||||
const config: OpenClawConfig = {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
apiKey: { source: "env", id: "OPENAI_API_KEY" },
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const snapshot = await prepareSecretsRuntimeSnapshot({
|
||||
config,
|
||||
env: {
|
||||
OPENAI_API_KEY: "sk-env-openai",
|
||||
GITHUB_TOKEN: "ghp-env-token",
|
||||
},
|
||||
agentDirs: ["/tmp/openclaw-agent-main"],
|
||||
loadAuthStore: () => ({
|
||||
version: 1,
|
||||
profiles: {
|
||||
"openai:default": {
|
||||
type: "api_key",
|
||||
provider: "openai",
|
||||
key: "old-openai",
|
||||
keyRef: { source: "env", id: "OPENAI_API_KEY" },
|
||||
},
|
||||
"github-copilot:default": {
|
||||
type: "token",
|
||||
provider: "github-copilot",
|
||||
token: "old-gh",
|
||||
tokenRef: { source: "env", id: "GITHUB_TOKEN" },
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
expect(snapshot.config.models?.providers?.openai?.apiKey).toBe("sk-env-openai");
|
||||
expect(snapshot.warnings).toHaveLength(2);
|
||||
expect(snapshot.authStores[0]?.store.profiles["openai:default"]).toMatchObject({
|
||||
type: "api_key",
|
||||
key: "sk-env-openai",
|
||||
});
|
||||
expect(snapshot.authStores[0]?.store.profiles["github-copilot:default"]).toMatchObject({
|
||||
type: "token",
|
||||
token: "ghp-env-token",
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves file refs via sops json payload", async () => {
|
||||
runExecMock.mockResolvedValueOnce({
|
||||
stdout: JSON.stringify({
|
||||
providers: {
|
||||
openai: {
|
||||
apiKey: "sk-from-sops",
|
||||
},
|
||||
},
|
||||
}),
|
||||
stderr: "",
|
||||
});
|
||||
|
||||
const config: OpenClawConfig = {
|
||||
secrets: {
|
||||
sources: {
|
||||
file: {
|
||||
type: "sops",
|
||||
path: "~/.openclaw/secrets.enc.json",
|
||||
timeoutMs: 7000,
|
||||
},
|
||||
},
|
||||
},
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
apiKey: { source: "file", id: "/providers/openai/apiKey" },
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const snapshot = await prepareSecretsRuntimeSnapshot({
|
||||
config,
|
||||
agentDirs: ["/tmp/openclaw-agent-main"],
|
||||
loadAuthStore: () => ({ version: 1, profiles: {} }),
|
||||
});
|
||||
|
||||
expect(snapshot.config.models?.providers?.openai?.apiKey).toBe("sk-from-sops");
|
||||
expect(runExecMock).toHaveBeenCalledWith(
|
||||
"sops",
|
||||
["--decrypt", "--output-type", "json", expect.stringContaining("secrets.enc.json")],
|
||||
{
|
||||
timeoutMs: 7000,
|
||||
maxBuffer: 10 * 1024 * 1024,
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("activates runtime snapshots for loadConfig and ensureAuthProfileStore", async () => {
|
||||
const prepared = await prepareSecretsRuntimeSnapshot({
|
||||
config: {
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
apiKey: { source: "env", id: "OPENAI_API_KEY" },
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
env: { OPENAI_API_KEY: "sk-runtime" },
|
||||
agentDirs: ["/tmp/openclaw-agent-main"],
|
||||
loadAuthStore: () => ({
|
||||
version: 1,
|
||||
profiles: {
|
||||
"openai:default": {
|
||||
type: "api_key",
|
||||
provider: "openai",
|
||||
keyRef: { source: "env", id: "OPENAI_API_KEY" },
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
activateSecretsRuntimeSnapshot(prepared);
|
||||
|
||||
expect(loadConfig().models?.providers?.openai?.apiKey).toBe("sk-runtime");
|
||||
const store = ensureAuthProfileStore("/tmp/openclaw-agent-main");
|
||||
expect(store.profiles["openai:default"]).toMatchObject({
|
||||
type: "api_key",
|
||||
key: "sk-runtime",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,385 @@
|
|||
import { resolveOpenClawAgentDir } from "../agents/agent-paths.js";
|
||||
import { listAgentIds, resolveAgentDir } from "../agents/agent-scope.js";
|
||||
import type { AuthProfileCredential, AuthProfileStore } from "../agents/auth-profiles.js";
|
||||
import {
|
||||
clearRuntimeAuthProfileStoreSnapshots,
|
||||
loadAuthProfileStoreForRuntime,
|
||||
replaceRuntimeAuthProfileStoreSnapshots,
|
||||
} from "../agents/auth-profiles.js";
|
||||
import {
|
||||
clearRuntimeConfigSnapshot,
|
||||
setRuntimeConfigSnapshot,
|
||||
type OpenClawConfig,
|
||||
type SecretRef,
|
||||
} from "../config/config.js";
|
||||
import { runExec } from "../process/exec.js";
|
||||
import { resolveUserPath } from "../utils.js";
|
||||
|
||||
const DEFAULT_SOPS_TIMEOUT_MS = 5_000;
|
||||
const MAX_SOPS_OUTPUT_BYTES = 10 * 1024 * 1024;
|
||||
|
||||
type SecretResolverWarningCode = "SECRETS_REF_OVERRIDES_PLAINTEXT";
|
||||
|
||||
export type SecretResolverWarning = {
|
||||
code: SecretResolverWarningCode;
|
||||
path: string;
|
||||
message: string;
|
||||
};
|
||||
|
||||
export type PreparedSecretsRuntimeSnapshot = {
|
||||
config: OpenClawConfig;
|
||||
authStores: Array<{ agentDir: string; store: AuthProfileStore }>;
|
||||
warnings: SecretResolverWarning[];
|
||||
};
|
||||
|
||||
type ResolverContext = {
|
||||
config: OpenClawConfig;
|
||||
env: NodeJS.ProcessEnv;
|
||||
fileSecretsPromise: Promise<unknown> | null;
|
||||
};
|
||||
|
||||
type ProviderLike = {
|
||||
apiKey?: unknown;
|
||||
};
|
||||
|
||||
type GoogleChatAccountLike = {
|
||||
serviceAccount?: unknown;
|
||||
serviceAccountRef?: unknown;
|
||||
accounts?: Record<string, unknown>;
|
||||
};
|
||||
|
||||
type ApiKeyCredentialLike = AuthProfileCredential & {
|
||||
type: "api_key";
|
||||
key?: string;
|
||||
keyRef?: unknown;
|
||||
};
|
||||
|
||||
type TokenCredentialLike = AuthProfileCredential & {
|
||||
type: "token";
|
||||
token?: string;
|
||||
tokenRef?: unknown;
|
||||
};
|
||||
|
||||
let activeSnapshot: PreparedSecretsRuntimeSnapshot | null = null;
|
||||
|
||||
function cloneSnapshot(snapshot: PreparedSecretsRuntimeSnapshot): PreparedSecretsRuntimeSnapshot {
|
||||
return {
|
||||
config: structuredClone(snapshot.config),
|
||||
authStores: snapshot.authStores.map((entry) => ({
|
||||
agentDir: entry.agentDir,
|
||||
store: structuredClone(entry.store),
|
||||
})),
|
||||
warnings: snapshot.warnings.map((warning) => ({ ...warning })),
|
||||
};
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function isSecretRef(value: unknown): value is SecretRef {
|
||||
if (!isRecord(value)) {
|
||||
return false;
|
||||
}
|
||||
if (Object.keys(value).length !== 2) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
(value.source === "env" || value.source === "file") &&
|
||||
typeof value.id === "string" &&
|
||||
value.id.trim().length > 0
|
||||
);
|
||||
}
|
||||
|
||||
function isNonEmptyString(value: unknown): value is string {
|
||||
return typeof value === "string" && value.trim().length > 0;
|
||||
}
|
||||
|
||||
function decodeJsonPointerToken(token: string): string {
|
||||
return token.replace(/~1/g, "/").replace(/~0/g, "~");
|
||||
}
|
||||
|
||||
function readJsonPointer(root: unknown, pointer: string): unknown {
|
||||
if (!pointer.startsWith("/")) {
|
||||
throw new Error(
|
||||
`File-backed secret ids must be absolute JSON pointers (for example: /providers/openai/apiKey).`,
|
||||
);
|
||||
}
|
||||
|
||||
const tokens = pointer
|
||||
.slice(1)
|
||||
.split("/")
|
||||
.map((token) => decodeJsonPointerToken(token));
|
||||
|
||||
let current: unknown = root;
|
||||
for (const token of tokens) {
|
||||
if (Array.isArray(current)) {
|
||||
const index = Number.parseInt(token, 10);
|
||||
if (!Number.isFinite(index) || index < 0 || index >= current.length) {
|
||||
throw new Error(`JSON pointer segment "${token}" is out of bounds.`);
|
||||
}
|
||||
current = current[index];
|
||||
continue;
|
||||
}
|
||||
if (!isRecord(current)) {
|
||||
throw new Error(`JSON pointer segment "${token}" does not exist.`);
|
||||
}
|
||||
if (!Object.hasOwn(current, token)) {
|
||||
throw new Error(`JSON pointer segment "${token}" does not exist.`);
|
||||
}
|
||||
current = current[token];
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
async function decryptSopsFile(config: OpenClawConfig): Promise<unknown> {
|
||||
const fileSource = config.secrets?.sources?.file;
|
||||
if (!fileSource) {
|
||||
throw new Error(
|
||||
`Secret reference source "file" is not configured. Configure secrets.sources.file first.`,
|
||||
);
|
||||
}
|
||||
if (fileSource.type !== "sops") {
|
||||
throw new Error(`Unsupported secrets.sources.file.type "${String(fileSource.type)}".`);
|
||||
}
|
||||
|
||||
const resolvedPath = resolveUserPath(fileSource.path);
|
||||
const timeoutMs =
|
||||
typeof fileSource.timeoutMs === "number" && Number.isFinite(fileSource.timeoutMs)
|
||||
? Math.max(1, Math.floor(fileSource.timeoutMs))
|
||||
: DEFAULT_SOPS_TIMEOUT_MS;
|
||||
|
||||
try {
|
||||
const { stdout } = await runExec("sops", ["--decrypt", "--output-type", "json", resolvedPath], {
|
||||
timeoutMs,
|
||||
maxBuffer: MAX_SOPS_OUTPUT_BYTES,
|
||||
});
|
||||
return JSON.parse(stdout) as unknown;
|
||||
} catch (err) {
|
||||
const error = err as NodeJS.ErrnoException & { message?: string };
|
||||
if (error.code === "ENOENT") {
|
||||
throw new Error(
|
||||
`sops binary not found in PATH. Install sops >= 3.9.0 or disable secrets.sources.file.`,
|
||||
{ cause: err },
|
||||
);
|
||||
}
|
||||
if (typeof error.message === "string" && error.message.toLowerCase().includes("timed out")) {
|
||||
throw new Error(`sops decrypt timed out after ${timeoutMs}ms for ${resolvedPath}.`, {
|
||||
cause: err,
|
||||
});
|
||||
}
|
||||
throw new Error(`sops decrypt failed for ${resolvedPath}: ${String(error.message ?? err)}`, {
|
||||
cause: err,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveSecretRefValue(ref: SecretRef, context: ResolverContext): Promise<unknown> {
|
||||
const id = ref.id.trim();
|
||||
if (!id) {
|
||||
throw new Error(`Secret reference id is empty.`);
|
||||
}
|
||||
|
||||
if (ref.source === "env") {
|
||||
const envValue = context.env[id];
|
||||
if (!isNonEmptyString(envValue)) {
|
||||
throw new Error(`Environment variable "${id}" is missing or empty.`);
|
||||
}
|
||||
return envValue;
|
||||
}
|
||||
|
||||
if (ref.source === "file") {
|
||||
context.fileSecretsPromise ??= decryptSopsFile(context.config);
|
||||
const payload = await context.fileSecretsPromise;
|
||||
return readJsonPointer(payload, id);
|
||||
}
|
||||
|
||||
throw new Error(`Unsupported secret source "${String((ref as { source?: unknown }).source)}".`);
|
||||
}
|
||||
|
||||
async function resolveGoogleChatServiceAccount(
|
||||
target: GoogleChatAccountLike,
|
||||
path: string,
|
||||
context: ResolverContext,
|
||||
warnings: SecretResolverWarning[],
|
||||
): Promise<void> {
|
||||
const explicitRef = isSecretRef(target.serviceAccountRef) ? target.serviceAccountRef : null;
|
||||
const inlineRef = isSecretRef(target.serviceAccount) ? target.serviceAccount : null;
|
||||
const ref = explicitRef ?? inlineRef;
|
||||
if (!ref) {
|
||||
return;
|
||||
}
|
||||
if (explicitRef && target.serviceAccount !== undefined && !isSecretRef(target.serviceAccount)) {
|
||||
warnings.push({
|
||||
code: "SECRETS_REF_OVERRIDES_PLAINTEXT",
|
||||
path,
|
||||
message: `${path}: serviceAccountRef is set; runtime will ignore plaintext serviceAccount.`,
|
||||
});
|
||||
}
|
||||
target.serviceAccount = await resolveSecretRefValue(ref, context);
|
||||
}
|
||||
|
||||
async function resolveConfigSecretRefs(params: {
|
||||
config: OpenClawConfig;
|
||||
context: ResolverContext;
|
||||
warnings: SecretResolverWarning[];
|
||||
}): Promise<OpenClawConfig> {
|
||||
const resolved = structuredClone(params.config);
|
||||
const providers = resolved.models?.providers as Record<string, ProviderLike> | undefined;
|
||||
if (providers) {
|
||||
for (const [providerId, provider] of Object.entries(providers)) {
|
||||
if (!isSecretRef(provider.apiKey)) {
|
||||
continue;
|
||||
}
|
||||
const resolvedValue = await resolveSecretRefValue(provider.apiKey, params.context);
|
||||
if (!isNonEmptyString(resolvedValue)) {
|
||||
throw new Error(
|
||||
`models.providers.${providerId}.apiKey resolved to a non-string or empty value.`,
|
||||
);
|
||||
}
|
||||
provider.apiKey = resolvedValue;
|
||||
}
|
||||
}
|
||||
|
||||
const googleChat = resolved.channels?.googlechat as GoogleChatAccountLike | undefined;
|
||||
if (googleChat) {
|
||||
await resolveGoogleChatServiceAccount(
|
||||
googleChat,
|
||||
"channels.googlechat",
|
||||
params.context,
|
||||
params.warnings,
|
||||
);
|
||||
if (isRecord(googleChat.accounts)) {
|
||||
for (const [accountId, account] of Object.entries(googleChat.accounts)) {
|
||||
if (!isRecord(account)) {
|
||||
continue;
|
||||
}
|
||||
await resolveGoogleChatServiceAccount(
|
||||
account as GoogleChatAccountLike,
|
||||
`channels.googlechat.accounts.${accountId}`,
|
||||
params.context,
|
||||
params.warnings,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return resolved;
|
||||
}
|
||||
|
||||
async function resolveAuthStoreSecretRefs(params: {
|
||||
store: AuthProfileStore;
|
||||
context: ResolverContext;
|
||||
warnings: SecretResolverWarning[];
|
||||
agentDir: string;
|
||||
}): Promise<AuthProfileStore> {
|
||||
const resolvedStore = structuredClone(params.store);
|
||||
for (const [profileId, profile] of Object.entries(resolvedStore.profiles)) {
|
||||
if (profile.type === "api_key") {
|
||||
const apiProfile = profile as ApiKeyCredentialLike;
|
||||
const keyRef = isSecretRef(apiProfile.keyRef) ? apiProfile.keyRef : null;
|
||||
if (keyRef && isNonEmptyString(apiProfile.key)) {
|
||||
params.warnings.push({
|
||||
code: "SECRETS_REF_OVERRIDES_PLAINTEXT",
|
||||
path: `${params.agentDir}.auth-profiles.${profileId}.key`,
|
||||
message: `auth-profiles ${profileId}: keyRef is set; runtime will ignore plaintext key.`,
|
||||
});
|
||||
}
|
||||
if (keyRef) {
|
||||
const resolvedValue = await resolveSecretRefValue(keyRef, params.context);
|
||||
if (!isNonEmptyString(resolvedValue)) {
|
||||
throw new Error(`auth profile "${profileId}" keyRef resolved to an empty value.`);
|
||||
}
|
||||
apiProfile.key = resolvedValue;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
if (profile.type === "token") {
|
||||
const tokenProfile = profile as TokenCredentialLike;
|
||||
const tokenRef = isSecretRef(tokenProfile.tokenRef) ? tokenProfile.tokenRef : null;
|
||||
if (tokenRef && isNonEmptyString(tokenProfile.token)) {
|
||||
params.warnings.push({
|
||||
code: "SECRETS_REF_OVERRIDES_PLAINTEXT",
|
||||
path: `${params.agentDir}.auth-profiles.${profileId}.token`,
|
||||
message: `auth-profiles ${profileId}: tokenRef is set; runtime will ignore plaintext token.`,
|
||||
});
|
||||
}
|
||||
if (tokenRef) {
|
||||
const resolvedValue = await resolveSecretRefValue(tokenRef, params.context);
|
||||
if (!isNonEmptyString(resolvedValue)) {
|
||||
throw new Error(`auth profile "${profileId}" tokenRef resolved to an empty value.`);
|
||||
}
|
||||
tokenProfile.token = resolvedValue;
|
||||
}
|
||||
}
|
||||
}
|
||||
return resolvedStore;
|
||||
}
|
||||
|
||||
function collectCandidateAgentDirs(config: OpenClawConfig): string[] {
|
||||
const dirs = new Set<string>();
|
||||
dirs.add(resolveUserPath(resolveOpenClawAgentDir()));
|
||||
for (const agentId of listAgentIds(config)) {
|
||||
dirs.add(resolveUserPath(resolveAgentDir(config, agentId)));
|
||||
}
|
||||
return [...dirs];
|
||||
}
|
||||
|
||||
export async function prepareSecretsRuntimeSnapshot(params: {
|
||||
config: OpenClawConfig;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
agentDirs?: string[];
|
||||
loadAuthStore?: (agentDir?: string) => AuthProfileStore;
|
||||
}): Promise<PreparedSecretsRuntimeSnapshot> {
|
||||
const warnings: SecretResolverWarning[] = [];
|
||||
const context: ResolverContext = {
|
||||
config: params.config,
|
||||
env: params.env ?? process.env,
|
||||
fileSecretsPromise: null,
|
||||
};
|
||||
const resolvedConfig = await resolveConfigSecretRefs({
|
||||
config: params.config,
|
||||
context,
|
||||
warnings,
|
||||
});
|
||||
|
||||
const loadAuthStore = params.loadAuthStore ?? loadAuthProfileStoreForRuntime;
|
||||
const candidateDirs = params.agentDirs?.length
|
||||
? [...new Set(params.agentDirs.map((entry) => resolveUserPath(entry)))]
|
||||
: collectCandidateAgentDirs(resolvedConfig);
|
||||
const authStores: Array<{ agentDir: string; store: AuthProfileStore }> = [];
|
||||
for (const agentDir of candidateDirs) {
|
||||
const rawStore = loadAuthStore(agentDir);
|
||||
const resolvedStore = await resolveAuthStoreSecretRefs({
|
||||
store: rawStore,
|
||||
context,
|
||||
warnings,
|
||||
agentDir,
|
||||
});
|
||||
authStores.push({ agentDir, store: resolvedStore });
|
||||
}
|
||||
|
||||
return {
|
||||
config: resolvedConfig,
|
||||
authStores,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
|
||||
export function activateSecretsRuntimeSnapshot(snapshot: PreparedSecretsRuntimeSnapshot): void {
|
||||
const next = cloneSnapshot(snapshot);
|
||||
setRuntimeConfigSnapshot(next.config);
|
||||
replaceRuntimeAuthProfileStoreSnapshots(next.authStores);
|
||||
activeSnapshot = next;
|
||||
}
|
||||
|
||||
export function getActiveSecretsRuntimeSnapshot(): PreparedSecretsRuntimeSnapshot | null {
|
||||
return activeSnapshot ? cloneSnapshot(activeSnapshot) : null;
|
||||
}
|
||||
|
||||
export function clearSecretsRuntimeSnapshot(): void {
|
||||
activeSnapshot = null;
|
||||
clearRuntimeConfigSnapshot();
|
||||
clearRuntimeAuthProfileStoreSnapshots();
|
||||
}
|
||||
Loading…
Reference in New Issue