diff --git a/extensions/runway/index.ts b/extensions/runway/index.ts new file mode 100644 index 00000000000..29659956781 --- /dev/null +++ b/extensions/runway/index.ts @@ -0,0 +1,11 @@ +import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry"; +import { buildRunwayVideoGenerationProvider } from "./video-generation-provider.js"; + +export default definePluginEntry({ + id: "runway", + name: "Runway Provider", + description: "Bundled Runway video provider plugin", + register(api) { + api.registerVideoGenerationProvider(buildRunwayVideoGenerationProvider()); + }, +}); diff --git a/extensions/runway/openclaw.plugin.json b/extensions/runway/openclaw.plugin.json new file mode 100644 index 00000000000..41720342a37 --- /dev/null +++ b/extensions/runway/openclaw.plugin.json @@ -0,0 +1,30 @@ +{ + "id": "runway", + "enabledByDefault": true, + "providerAuthEnvVars": { + "runway": ["RUNWAYML_API_SECRET", "RUNWAY_API_KEY"] + }, + "providerAuthChoices": [ + { + "provider": "runway", + "method": "api-key", + "choiceId": "runway-api-key", + "choiceLabel": "Runway API key", + "groupId": "runway", + "groupLabel": "Runway", + "groupHint": "API key", + "optionKey": "runwayApiKey", + "cliFlag": "--runway-api-key", + "cliOption": "--runway-api-key ", + "cliDescription": "Runway API key" + } + ], + "contracts": { + "videoGenerationProviders": ["runway"] + }, + "configSchema": { + "type": "object", + "additionalProperties": false, + "properties": {} + } +} diff --git a/extensions/runway/package.json b/extensions/runway/package.json new file mode 100644 index 00000000000..ef7a2ff378f --- /dev/null +++ b/extensions/runway/package.json @@ -0,0 +1,12 @@ +{ + "name": "@openclaw/runway-provider", + "version": "2026.4.5", + "private": true, + "description": "OpenClaw Runway video provider plugin", + "type": "module", + "openclaw": { + "extensions": [ + "./index.ts" + ] + } +} diff --git a/extensions/runway/plugin-registration.contract.test.ts b/extensions/runway/plugin-registration.contract.test.ts new file mode 100644 index 00000000000..93cd97a0f73 --- /dev/null +++ b/extensions/runway/plugin-registration.contract.test.ts @@ -0,0 +1,7 @@ +import { describePluginRegistrationContract } from "../../test/helpers/plugins/plugin-registration-contract.js"; + +describePluginRegistrationContract({ + pluginId: "runway", + videoGenerationProviderIds: ["runway"], + requireGenerateVideo: true, +}); diff --git a/extensions/runway/video-generation-provider.test.ts b/extensions/runway/video-generation-provider.test.ts new file mode 100644 index 00000000000..4c4c7e66958 --- /dev/null +++ b/extensions/runway/video-generation-provider.test.ts @@ -0,0 +1,162 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { buildRunwayVideoGenerationProvider } from "./video-generation-provider.js"; + +const { + resolveApiKeyForProviderMock, + postJsonRequestMock, + fetchWithTimeoutMock, + assertOkOrThrowHttpErrorMock, + resolveProviderHttpRequestConfigMock, +} = vi.hoisted(() => ({ + resolveApiKeyForProviderMock: vi.fn(async () => ({ apiKey: "runway-key" })), + postJsonRequestMock: vi.fn(), + fetchWithTimeoutMock: vi.fn(), + assertOkOrThrowHttpErrorMock: vi.fn(async () => {}), + resolveProviderHttpRequestConfigMock: vi.fn((params) => ({ + baseUrl: params.baseUrl ?? params.defaultBaseUrl, + allowPrivateNetwork: false, + headers: new Headers(params.defaultHeaders), + dispatcherPolicy: undefined, + })), +})); + +vi.mock("openclaw/plugin-sdk/provider-auth-runtime", () => ({ + resolveApiKeyForProvider: resolveApiKeyForProviderMock, +})); + +vi.mock("openclaw/plugin-sdk/provider-http", () => ({ + assertOkOrThrowHttpError: assertOkOrThrowHttpErrorMock, + fetchWithTimeout: fetchWithTimeoutMock, + postJsonRequest: postJsonRequestMock, + resolveProviderHttpRequestConfig: resolveProviderHttpRequestConfigMock, +})); + +describe("runway video generation provider", () => { + afterEach(() => { + resolveApiKeyForProviderMock.mockClear(); + postJsonRequestMock.mockReset(); + fetchWithTimeoutMock.mockReset(); + assertOkOrThrowHttpErrorMock.mockClear(); + resolveProviderHttpRequestConfigMock.mockClear(); + }); + + it("submits a text-to-video task, polls it, and downloads the output", async () => { + postJsonRequestMock.mockResolvedValue({ + response: { + json: async () => ({ + id: "task-1", + }), + }, + release: vi.fn(async () => {}), + }); + fetchWithTimeoutMock + .mockResolvedValueOnce({ + json: async () => ({ + id: "task-1", + status: "SUCCEEDED", + output: ["https://example.com/out.mp4"], + }), + headers: new Headers(), + }) + .mockResolvedValueOnce({ + arrayBuffer: async () => Buffer.from("mp4-bytes"), + headers: new Headers({ "content-type": "video/mp4" }), + }); + + const provider = buildRunwayVideoGenerationProvider(); + const result = await provider.generateVideo({ + provider: "runway", + model: "gen4.5", + prompt: "a tiny lobster DJ under neon lights", + cfg: {}, + durationSeconds: 4, + aspectRatio: "16:9", + }); + + expect(postJsonRequestMock).toHaveBeenCalledWith( + expect.objectContaining({ + url: "https://api.dev.runwayml.com/v1/text_to_video", + body: { + model: "gen4.5", + promptText: "a tiny lobster DJ under neon lights", + ratio: "1280:720", + duration: 4, + }, + }), + ); + expect(fetchWithTimeoutMock).toHaveBeenNthCalledWith( + 1, + "https://api.dev.runwayml.com/v1/tasks/task-1", + expect.objectContaining({ method: "GET" }), + 120000, + fetch, + ); + expect(result.videos).toHaveLength(1); + expect(result.metadata).toEqual( + expect.objectContaining({ + taskId: "task-1", + status: "SUCCEEDED", + endpoint: "/v1/text_to_video", + }), + ); + }); + + it("accepts local image buffers by converting them into data URIs", async () => { + postJsonRequestMock.mockResolvedValue({ + response: { + json: async () => ({ id: "task-2" }), + }, + release: vi.fn(async () => {}), + }); + fetchWithTimeoutMock + .mockResolvedValueOnce({ + json: async () => ({ + id: "task-2", + status: "SUCCEEDED", + output: ["https://example.com/out.mp4"], + }), + headers: new Headers(), + }) + .mockResolvedValueOnce({ + arrayBuffer: async () => Buffer.from("mp4-bytes"), + headers: new Headers({ "content-type": "video/mp4" }), + }); + + const provider = buildRunwayVideoGenerationProvider(); + await provider.generateVideo({ + provider: "runway", + model: "gen4_turbo", + prompt: "animate this frame", + cfg: {}, + inputImages: [{ buffer: Buffer.from("png-bytes"), mimeType: "image/png" }], + aspectRatio: "1:1", + durationSeconds: 6, + }); + + expect(postJsonRequestMock).toHaveBeenCalledWith( + expect.objectContaining({ + url: "https://api.dev.runwayml.com/v1/image_to_video", + body: expect.objectContaining({ + promptImage: expect.stringMatching(/^data:image\/png;base64,/u), + ratio: "960:960", + duration: 6, + }), + }), + ); + }); + + it("requires gen4_aleph for video-to-video", async () => { + const provider = buildRunwayVideoGenerationProvider(); + + await expect( + provider.generateVideo({ + provider: "runway", + model: "gen4.5", + prompt: "restyle this clip", + cfg: {}, + inputVideos: [{ url: "https://example.com/input.mp4" }], + }), + ).rejects.toThrow("Runway video-to-video currently requires model gen4_aleph."); + expect(postJsonRequestMock).not.toHaveBeenCalled(); + }); +}); diff --git a/extensions/runway/video-generation-provider.ts b/extensions/runway/video-generation-provider.ts new file mode 100644 index 00000000000..a8ffb59d8f0 --- /dev/null +++ b/extensions/runway/video-generation-provider.ts @@ -0,0 +1,346 @@ +import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth"; +import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime"; +import { + assertOkOrThrowHttpError, + fetchWithTimeout, + postJsonRequest, + resolveProviderHttpRequestConfig, +} from "openclaw/plugin-sdk/provider-http"; +import type { + GeneratedVideoAsset, + VideoGenerationProvider, + VideoGenerationRequest, + VideoGenerationResult, + VideoGenerationSourceAsset, +} from "openclaw/plugin-sdk/video-generation"; + +const DEFAULT_RUNWAY_BASE_URL = "https://api.dev.runwayml.com"; +const DEFAULT_RUNWAY_MODEL = "gen4.5"; +const RUNWAY_API_VERSION = "2024-11-06"; +const DEFAULT_TIMEOUT_MS = 120_000; +const POLL_INTERVAL_MS = 5_000; +const MAX_POLL_ATTEMPTS = 120; +const MAX_DURATION_SECONDS = 10; + +type RunwayTaskStatus = "PENDING" | "RUNNING" | "THROTTLED" | "SUCCEEDED" | "FAILED" | "CANCELLED"; + +type RunwayTaskCreateResponse = { + id?: string; +}; + +type RunwayTaskDetailResponse = { + id?: string; + status?: RunwayTaskStatus; + output?: string[]; + failure?: string | { message?: string } | null; +}; + +const TEXT_ONLY_MODELS = new Set(["gen4.5", "veo3.1", "veo3.1_fast", "veo3"]); +const IMAGE_MODELS = new Set([ + "gen4.5", + "gen4_turbo", + "gen3a_turbo", + "veo3.1", + "veo3.1_fast", + "veo3", +]); +const VIDEO_MODELS = new Set(["gen4_aleph"]); + +function resolveRunwayBaseUrl(req: VideoGenerationRequest): string { + return req.cfg?.models?.providers?.runway?.baseUrl?.trim() || DEFAULT_RUNWAY_BASE_URL; +} + +function toDataUrl(buffer: Buffer, mimeType: string): string { + return `data:${mimeType};base64,${buffer.toString("base64")}`; +} + +function resolveSourceUri( + asset: VideoGenerationSourceAsset | undefined, + fallbackMimeType: string, +): string | undefined { + if (!asset) { + return undefined; + } + const url = asset.url?.trim(); + if (url) { + return url; + } + if (!asset.buffer) { + return undefined; + } + return toDataUrl(asset.buffer, asset.mimeType?.trim() || fallbackMimeType); +} + +function resolveDurationSeconds(value: number | undefined): number { + if (typeof value !== "number" || !Number.isFinite(value)) { + return 5; + } + return Math.max(2, Math.min(MAX_DURATION_SECONDS, Math.round(value))); +} + +function resolveRunwayRatio(req: VideoGenerationRequest): string { + const hasImageInput = (req.inputImages?.length ?? 0) > 0; + const requested = + req.size?.trim() || + (() => { + switch (req.aspectRatio?.trim()) { + case "9:16": + return "720:1280"; + case "16:9": + return "1280:720"; + case "1:1": + return "960:960"; + case "3:4": + return "832:1104"; + case "4:3": + return "1104:832"; + case "21:9": + return "1584:672"; + default: + return undefined; + } + })(); + if (requested) { + if (!hasImageInput && requested !== "1280:720" && requested !== "720:1280") { + throw new Error("Runway text-to-video currently supports only 16:9 or 9:16 output ratios."); + } + return requested; + } + return "1280:720"; +} + +function resolveEndpoint( + req: VideoGenerationRequest, +): "/v1/text_to_video" | "/v1/image_to_video" | "/v1/video_to_video" { + const imageCount = req.inputImages?.length ?? 0; + const videoCount = req.inputVideos?.length ?? 0; + if (imageCount > 0 && videoCount > 0) { + throw new Error("Runway video generation does not support image and video inputs together."); + } + if (imageCount > 1 || videoCount > 1) { + throw new Error("Runway video generation supports at most one input image or one input video."); + } + if (videoCount > 0) { + return "/v1/video_to_video"; + } + if (imageCount > 0) { + return "/v1/image_to_video"; + } + return "/v1/text_to_video"; +} + +function buildCreateBody(req: VideoGenerationRequest): Record { + const endpoint = resolveEndpoint(req); + const duration = resolveDurationSeconds(req.durationSeconds); + const ratio = resolveRunwayRatio(req); + const model = req.model?.trim() || DEFAULT_RUNWAY_MODEL; + if (endpoint === "/v1/text_to_video") { + if (!TEXT_ONLY_MODELS.has(model)) { + throw new Error( + `Runway text-to-video does not support model ${model}. Use one of: ${[...TEXT_ONLY_MODELS].join(", ")}.`, + ); + } + return { + model, + promptText: req.prompt, + ratio, + duration, + }; + } + + if (endpoint === "/v1/image_to_video") { + if (!IMAGE_MODELS.has(model)) { + throw new Error( + `Runway image-to-video does not support model ${model}. Use one of: ${[...IMAGE_MODELS].join(", ")}.`, + ); + } + const promptImage = resolveSourceUri(req.inputImages?.[0], "image/png"); + if (!promptImage) { + throw new Error("Runway image-to-video input is missing image data."); + } + return { + model, + promptText: req.prompt, + promptImage, + ratio, + duration, + }; + } + + if (!VIDEO_MODELS.has(model)) { + throw new Error("Runway video-to-video currently requires model gen4_aleph."); + } + const videoUri = resolveSourceUri(req.inputVideos?.[0], "video/mp4"); + if (!videoUri) { + throw new Error("Runway video-to-video input is missing video data."); + } + return { + model, + promptText: req.prompt, + videoUri, + ratio, + }; +} + +async function pollRunwayTask(params: { + taskId: string; + headers: Headers; + timeoutMs?: number; + baseUrl: string; + fetchFn: typeof fetch; +}): Promise { + for (let attempt = 0; attempt < MAX_POLL_ATTEMPTS; attempt += 1) { + const response = await fetchWithTimeout( + `${params.baseUrl}/v1/tasks/${params.taskId}`, + { + method: "GET", + headers: params.headers, + }, + params.timeoutMs ?? DEFAULT_TIMEOUT_MS, + params.fetchFn, + ); + await assertOkOrThrowHttpError(response, "Runway video status request failed"); + const payload = (await response.json()) as RunwayTaskDetailResponse; + switch (payload.status) { + case "SUCCEEDED": + return payload; + case "FAILED": + case "CANCELLED": + throw new Error( + (typeof payload.failure === "string" + ? payload.failure + : payload.failure?.message + )?.trim() || `Runway video generation ${payload.status.toLowerCase()}`, + ); + case "PENDING": + case "RUNNING": + case "THROTTLED": + default: + await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS)); + break; + } + } + throw new Error(`Runway video generation task ${params.taskId} did not finish in time`); +} + +async function downloadRunwayVideos(params: { + urls: string[]; + timeoutMs?: number; + fetchFn: typeof fetch; +}): Promise { + const videos: GeneratedVideoAsset[] = []; + for (const [index, url] of params.urls.entries()) { + const response = await fetchWithTimeout( + url, + { method: "GET" }, + params.timeoutMs ?? DEFAULT_TIMEOUT_MS, + params.fetchFn, + ); + await assertOkOrThrowHttpError(response, "Runway generated video download failed"); + const mimeType = response.headers.get("content-type")?.trim() || "video/mp4"; + const arrayBuffer = await response.arrayBuffer(); + videos.push({ + buffer: Buffer.from(arrayBuffer), + mimeType, + fileName: `video-${index + 1}.${mimeType.includes("webm") ? "webm" : "mp4"}`, + metadata: { sourceUrl: url }, + }); + } + return videos; +} + +export function buildRunwayVideoGenerationProvider(): VideoGenerationProvider { + return { + id: "runway", + label: "Runway", + defaultModel: DEFAULT_RUNWAY_MODEL, + models: ["gen4.5", "gen4_turbo", "gen4_aleph", "gen3a_turbo", "veo3.1", "veo3.1_fast", "veo3"], + isConfigured: ({ agentDir }) => + isProviderApiKeyConfigured({ + provider: "runway", + agentDir, + }), + capabilities: { + maxVideos: 1, + maxInputImages: 1, + maxInputVideos: 1, + maxDurationSeconds: MAX_DURATION_SECONDS, + supportsAspectRatio: true, + }, + async generateVideo(req): Promise { + const auth = await resolveApiKeyForProvider({ + provider: "runway", + cfg: req.cfg, + agentDir: req.agentDir, + store: req.authStore, + }); + if (!auth.apiKey) { + throw new Error("Runway API key missing"); + } + + const fetchFn = fetch; + const requestBody = buildCreateBody(req); + const endpoint = resolveEndpoint(req); + const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } = + resolveProviderHttpRequestConfig({ + baseUrl: resolveRunwayBaseUrl(req), + defaultBaseUrl: DEFAULT_RUNWAY_BASE_URL, + defaultHeaders: { + Authorization: `Bearer ${auth.apiKey}`, + "Content-Type": "application/json", + "X-Runway-Version": RUNWAY_API_VERSION, + }, + provider: "runway", + capability: "video", + transport: "http", + }); + const { response, release } = await postJsonRequest({ + url: `${baseUrl}${endpoint}`, + headers, + body: requestBody, + timeoutMs: req.timeoutMs, + fetchFn, + allowPrivateNetwork, + dispatcherPolicy, + }); + try { + await assertOkOrThrowHttpError(response, "Runway video generation failed"); + const submitted = (await response.json()) as RunwayTaskCreateResponse; + const taskId = submitted.id?.trim(); + if (!taskId) { + throw new Error("Runway video generation response missing task id"); + } + const completed = await pollRunwayTask({ + taskId, + headers, + timeoutMs: req.timeoutMs, + baseUrl, + fetchFn, + }); + const outputUrls = completed.output?.filter( + (value) => typeof value === "string" && value.trim(), + ); + if (!outputUrls?.length) { + throw new Error("Runway video generation completed without output URLs"); + } + const videos = await downloadRunwayVideos({ + urls: outputUrls, + timeoutMs: req.timeoutMs, + fetchFn, + }); + return { + videos, + model: req.model?.trim() || DEFAULT_RUNWAY_MODEL, + metadata: { + taskId, + status: completed.status, + endpoint, + outputUrls, + }, + }; + } finally { + await release(); + } + }, + }; +} diff --git a/extensions/video-generation-providers.live.test.ts b/extensions/video-generation-providers.live.test.ts index 1630a64afc7..010ef8782cd 100644 --- a/extensions/video-generation-providers.live.test.ts +++ b/extensions/video-generation-providers.live.test.ts @@ -20,6 +20,7 @@ import googlePlugin from "./google/index.js"; import minimaxPlugin from "./minimax/index.js"; import openaiPlugin from "./openai/index.js"; import qwenPlugin from "./qwen/index.js"; +import runwayPlugin from "./runway/index.js"; import togetherPlugin from "./together/index.js"; import xaiPlugin from "./xai/index.js"; @@ -57,6 +58,7 @@ const CASES: LiveProviderCase[] = [ }, { plugin: openaiPlugin, pluginId: "openai", pluginName: "OpenAI Provider", providerId: "openai" }, { plugin: qwenPlugin, pluginId: "qwen", pluginName: "Qwen Provider", providerId: "qwen" }, + { plugin: runwayPlugin, pluginId: "runway", pluginName: "Runway Provider", providerId: "runway" }, { plugin: togetherPlugin, pluginId: "together", diff --git a/src/video-generation/live-test-helpers.ts b/src/video-generation/live-test-helpers.ts index 3730a403f99..21007975929 100644 --- a/src/video-generation/live-test-helpers.ts +++ b/src/video-generation/live-test-helpers.ts @@ -9,6 +9,7 @@ export const DEFAULT_LIVE_VIDEO_MODELS: Record = { minimax: "minimax/MiniMax-Hailuo-2.3", openai: "openai/sora-2", qwen: "qwen/wan2.6-t2v", + runway: "runway/gen4.5", together: "together/Wan-AI/Wan2.2-T2V-A14B", xai: "xai/grok-imagine-video", };