feat(video): add runway provider

extensions/runway/index.ts

@@ -0,0 +1,11 @@
+import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";
+import { buildRunwayVideoGenerationProvider } from "./video-generation-provider.js";
+
+export default definePluginEntry({
+  id: "runway",
+  name: "Runway Provider",
+  description: "Bundled Runway video provider plugin",
+  register(api) {
+    api.registerVideoGenerationProvider(buildRunwayVideoGenerationProvider());
+  },
+});

extensions/runway/openclaw.plugin.json

@@ -0,0 +1,30 @@
+{
+  "id": "runway",
+  "enabledByDefault": true,
+  "providerAuthEnvVars": {
+    "runway": ["RUNWAYML_API_SECRET", "RUNWAY_API_KEY"]
+  },
+  "providerAuthChoices": [
+    {
+      "provider": "runway",
+      "method": "api-key",
+      "choiceId": "runway-api-key",
+      "choiceLabel": "Runway API key",
+      "groupId": "runway",
+      "groupLabel": "Runway",
+      "groupHint": "API key",
+      "optionKey": "runwayApiKey",
+      "cliFlag": "--runway-api-key",
+      "cliOption": "--runway-api-key <key>",
+      "cliDescription": "Runway API key"
+    }
+  ],
+  "contracts": {
+    "videoGenerationProviders": ["runway"]
+  },
+  "configSchema": {
+    "type": "object",
+    "additionalProperties": false,
+    "properties": {}
+  }
+}

extensions/runway/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "@openclaw/runway-provider",
+  "version": "2026.4.5",
+  "private": true,
+  "description": "OpenClaw Runway video provider plugin",
+  "type": "module",
+  "openclaw": {
+    "extensions": [
+      "./index.ts"
+    ]
+  }
+}

extensions/runway/plugin-registration.contract.test.ts

@@ -0,0 +1,7 @@
+import { describePluginRegistrationContract } from "../../test/helpers/plugins/plugin-registration-contract.js";
+
+describePluginRegistrationContract({
+  pluginId: "runway",
+  videoGenerationProviderIds: ["runway"],
+  requireGenerateVideo: true,
+});

extensions/runway/video-generation-provider.test.ts

@@ -0,0 +1,162 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { buildRunwayVideoGenerationProvider } from "./video-generation-provider.js";
+
+const {
+  resolveApiKeyForProviderMock,
+  postJsonRequestMock,
+  fetchWithTimeoutMock,
+  assertOkOrThrowHttpErrorMock,
+  resolveProviderHttpRequestConfigMock,
+} = vi.hoisted(() => ({
+  resolveApiKeyForProviderMock: vi.fn(async () => ({ apiKey: "runway-key" })),
+  postJsonRequestMock: vi.fn(),
+  fetchWithTimeoutMock: vi.fn(),
+  assertOkOrThrowHttpErrorMock: vi.fn(async () => {}),
+  resolveProviderHttpRequestConfigMock: vi.fn((params) => ({
+    baseUrl: params.baseUrl ?? params.defaultBaseUrl,
+    allowPrivateNetwork: false,
+    headers: new Headers(params.defaultHeaders),
+    dispatcherPolicy: undefined,
+  })),
+}));
+
+vi.mock("openclaw/plugin-sdk/provider-auth-runtime", () => ({
+  resolveApiKeyForProvider: resolveApiKeyForProviderMock,
+}));
+
+vi.mock("openclaw/plugin-sdk/provider-http", () => ({
+  assertOkOrThrowHttpError: assertOkOrThrowHttpErrorMock,
+  fetchWithTimeout: fetchWithTimeoutMock,
+  postJsonRequest: postJsonRequestMock,
+  resolveProviderHttpRequestConfig: resolveProviderHttpRequestConfigMock,
+}));
+
+describe("runway video generation provider", () => {
+  afterEach(() => {
+    resolveApiKeyForProviderMock.mockClear();
+    postJsonRequestMock.mockReset();
+    fetchWithTimeoutMock.mockReset();
+    assertOkOrThrowHttpErrorMock.mockClear();
+    resolveProviderHttpRequestConfigMock.mockClear();
+  });
+
+  it("submits a text-to-video task, polls it, and downloads the output", async () => {
+    postJsonRequestMock.mockResolvedValue({
+      response: {
+        json: async () => ({
+          id: "task-1",
+        }),
+      },
+      release: vi.fn(async () => {}),
+    });
+    fetchWithTimeoutMock
+      .mockResolvedValueOnce({
+        json: async () => ({
+          id: "task-1",
+          status: "SUCCEEDED",
+          output: ["https://example.com/out.mp4"],
+        }),
+        headers: new Headers(),
+      })
+      .mockResolvedValueOnce({
+        arrayBuffer: async () => Buffer.from("mp4-bytes"),
+        headers: new Headers({ "content-type": "video/mp4" }),
+      });
+
+    const provider = buildRunwayVideoGenerationProvider();
+    const result = await provider.generateVideo({
+      provider: "runway",
+      model: "gen4.5",
+      prompt: "a tiny lobster DJ under neon lights",
+      cfg: {},
+      durationSeconds: 4,
+      aspectRatio: "16:9",
+    });
+
+    expect(postJsonRequestMock).toHaveBeenCalledWith(
+      expect.objectContaining({
+        url: "https://api.dev.runwayml.com/v1/text_to_video",
+        body: {
+          model: "gen4.5",
+          promptText: "a tiny lobster DJ under neon lights",
+          ratio: "1280:720",
+          duration: 4,
+        },
+      }),
+    );
+    expect(fetchWithTimeoutMock).toHaveBeenNthCalledWith(
+      1,
+      "https://api.dev.runwayml.com/v1/tasks/task-1",
+      expect.objectContaining({ method: "GET" }),
+      120000,
+      fetch,
+    );
+    expect(result.videos).toHaveLength(1);
+    expect(result.metadata).toEqual(
+      expect.objectContaining({
+        taskId: "task-1",
+        status: "SUCCEEDED",
+        endpoint: "/v1/text_to_video",
+      }),
+    );
+  });
+
+  it("accepts local image buffers by converting them into data URIs", async () => {
+    postJsonRequestMock.mockResolvedValue({
+      response: {
+        json: async () => ({ id: "task-2" }),
+      },
+      release: vi.fn(async () => {}),
+    });
+    fetchWithTimeoutMock
+      .mockResolvedValueOnce({
+        json: async () => ({
+          id: "task-2",
+          status: "SUCCEEDED",
+          output: ["https://example.com/out.mp4"],
+        }),
+        headers: new Headers(),
+      })
+      .mockResolvedValueOnce({
+        arrayBuffer: async () => Buffer.from("mp4-bytes"),
+        headers: new Headers({ "content-type": "video/mp4" }),
+      });
+
+    const provider = buildRunwayVideoGenerationProvider();
+    await provider.generateVideo({
+      provider: "runway",
+      model: "gen4_turbo",
+      prompt: "animate this frame",
+      cfg: {},
+      inputImages: [{ buffer: Buffer.from("png-bytes"), mimeType: "image/png" }],
+      aspectRatio: "1:1",
+      durationSeconds: 6,
+    });
+
+    expect(postJsonRequestMock).toHaveBeenCalledWith(
+      expect.objectContaining({
+        url: "https://api.dev.runwayml.com/v1/image_to_video",
+        body: expect.objectContaining({
+          promptImage: expect.stringMatching(/^data:image\/png;base64,/u),
+          ratio: "960:960",
+          duration: 6,
+        }),
+      }),
+    );
+  });
+
+  it("requires gen4_aleph for video-to-video", async () => {
+    const provider = buildRunwayVideoGenerationProvider();
+
+    await expect(
+      provider.generateVideo({
+        provider: "runway",
+        model: "gen4.5",
+        prompt: "restyle this clip",
+        cfg: {},
+        inputVideos: [{ url: "https://example.com/input.mp4" }],
+      }),
+    ).rejects.toThrow("Runway video-to-video currently requires model gen4_aleph.");
+    expect(postJsonRequestMock).not.toHaveBeenCalled();
+  });
+});

extensions/runway/video-generation-provider.ts

@@ -0,0 +1,346 @@
+import { isProviderApiKeyConfigured } from "openclaw/plugin-sdk/provider-auth";
+import { resolveApiKeyForProvider } from "openclaw/plugin-sdk/provider-auth-runtime";
+import {
+  assertOkOrThrowHttpError,
+  fetchWithTimeout,
+  postJsonRequest,
+  resolveProviderHttpRequestConfig,
+} from "openclaw/plugin-sdk/provider-http";
+import type {
+  GeneratedVideoAsset,
+  VideoGenerationProvider,
+  VideoGenerationRequest,
+  VideoGenerationResult,
+  VideoGenerationSourceAsset,
+} from "openclaw/plugin-sdk/video-generation";
+
+const DEFAULT_RUNWAY_BASE_URL = "https://api.dev.runwayml.com";
+const DEFAULT_RUNWAY_MODEL = "gen4.5";
+const RUNWAY_API_VERSION = "2024-11-06";
+const DEFAULT_TIMEOUT_MS = 120_000;
+const POLL_INTERVAL_MS = 5_000;
+const MAX_POLL_ATTEMPTS = 120;
+const MAX_DURATION_SECONDS = 10;
+
+type RunwayTaskStatus = "PENDING" | "RUNNING" | "THROTTLED" | "SUCCEEDED" | "FAILED" | "CANCELLED";
+
+type RunwayTaskCreateResponse = {
+  id?: string;
+};
+
+type RunwayTaskDetailResponse = {
+  id?: string;
+  status?: RunwayTaskStatus;
+  output?: string[];
+  failure?: string | { message?: string } | null;
+};
+
+const TEXT_ONLY_MODELS = new Set(["gen4.5", "veo3.1", "veo3.1_fast", "veo3"]);
+const IMAGE_MODELS = new Set([
+  "gen4.5",
+  "gen4_turbo",
+  "gen3a_turbo",
+  "veo3.1",
+  "veo3.1_fast",
+  "veo3",
+]);
+const VIDEO_MODELS = new Set(["gen4_aleph"]);
+
+function resolveRunwayBaseUrl(req: VideoGenerationRequest): string {
+  return req.cfg?.models?.providers?.runway?.baseUrl?.trim() || DEFAULT_RUNWAY_BASE_URL;
+}
+
+function toDataUrl(buffer: Buffer, mimeType: string): string {
+  return `data:${mimeType};base64,${buffer.toString("base64")}`;
+}
+
+function resolveSourceUri(
+  asset: VideoGenerationSourceAsset | undefined,
+  fallbackMimeType: string,
+): string | undefined {
+  if (!asset) {
+    return undefined;
+  }
+  const url = asset.url?.trim();
+  if (url) {
+    return url;
+  }
+  if (!asset.buffer) {
+    return undefined;
+  }
+  return toDataUrl(asset.buffer, asset.mimeType?.trim() || fallbackMimeType);
+}
+
+function resolveDurationSeconds(value: number | undefined): number {
+  if (typeof value !== "number" || !Number.isFinite(value)) {
+    return 5;
+  }
+  return Math.max(2, Math.min(MAX_DURATION_SECONDS, Math.round(value)));
+}
+
+function resolveRunwayRatio(req: VideoGenerationRequest): string {
+  const hasImageInput = (req.inputImages?.length ?? 0) > 0;
+  const requested =
+    req.size?.trim() ||
+    (() => {
+      switch (req.aspectRatio?.trim()) {
+        case "9:16":
+          return "720:1280";
+        case "16:9":
+          return "1280:720";
+        case "1:1":
+          return "960:960";
+        case "3:4":
+          return "832:1104";
+        case "4:3":
+          return "1104:832";
+        case "21:9":
+          return "1584:672";
+        default:
+          return undefined;
+      }
+    })();
+  if (requested) {
+    if (!hasImageInput && requested !== "1280:720" && requested !== "720:1280") {
+      throw new Error("Runway text-to-video currently supports only 16:9 or 9:16 output ratios.");
+    }
+    return requested;
+  }
+  return "1280:720";
+}
+
+function resolveEndpoint(
+  req: VideoGenerationRequest,
+): "/v1/text_to_video" | "/v1/image_to_video" | "/v1/video_to_video" {
+  const imageCount = req.inputImages?.length ?? 0;
+  const videoCount = req.inputVideos?.length ?? 0;
+  if (imageCount > 0 && videoCount > 0) {
+    throw new Error("Runway video generation does not support image and video inputs together.");
+  }
+  if (imageCount > 1 || videoCount > 1) {
+    throw new Error("Runway video generation supports at most one input image or one input video.");
+  }
+  if (videoCount > 0) {
+    return "/v1/video_to_video";
+  }
+  if (imageCount > 0) {
+    return "/v1/image_to_video";
+  }
+  return "/v1/text_to_video";
+}
+
+function buildCreateBody(req: VideoGenerationRequest): Record<string, unknown> {
+  const endpoint = resolveEndpoint(req);
+  const duration = resolveDurationSeconds(req.durationSeconds);
+  const ratio = resolveRunwayRatio(req);
+  const model = req.model?.trim() || DEFAULT_RUNWAY_MODEL;
+  if (endpoint === "/v1/text_to_video") {
+    if (!TEXT_ONLY_MODELS.has(model)) {
+      throw new Error(
+        `Runway text-to-video does not support model ${model}. Use one of: ${[...TEXT_ONLY_MODELS].join(", ")}.`,
+      );
+    }
+    return {
+      model,
+      promptText: req.prompt,
+      ratio,
+      duration,
+    };
+  }
+
+  if (endpoint === "/v1/image_to_video") {
+    if (!IMAGE_MODELS.has(model)) {
+      throw new Error(
+        `Runway image-to-video does not support model ${model}. Use one of: ${[...IMAGE_MODELS].join(", ")}.`,
+      );
+    }
+    const promptImage = resolveSourceUri(req.inputImages?.[0], "image/png");
+    if (!promptImage) {
+      throw new Error("Runway image-to-video input is missing image data.");
+    }
+    return {
+      model,
+      promptText: req.prompt,
+      promptImage,
+      ratio,
+      duration,
+    };
+  }
+
+  if (!VIDEO_MODELS.has(model)) {
+    throw new Error("Runway video-to-video currently requires model gen4_aleph.");
+  }
+  const videoUri = resolveSourceUri(req.inputVideos?.[0], "video/mp4");
+  if (!videoUri) {
+    throw new Error("Runway video-to-video input is missing video data.");
+  }
+  return {
+    model,
+    promptText: req.prompt,
+    videoUri,
+    ratio,
+  };
+}
+
+async function pollRunwayTask(params: {
+  taskId: string;
+  headers: Headers;
+  timeoutMs?: number;
+  baseUrl: string;
+  fetchFn: typeof fetch;
+}): Promise<RunwayTaskDetailResponse> {
+  for (let attempt = 0; attempt < MAX_POLL_ATTEMPTS; attempt += 1) {
+    const response = await fetchWithTimeout(
+      `${params.baseUrl}/v1/tasks/${params.taskId}`,
+      {
+        method: "GET",
+        headers: params.headers,
+      },
+      params.timeoutMs ?? DEFAULT_TIMEOUT_MS,
+      params.fetchFn,
+    );
+    await assertOkOrThrowHttpError(response, "Runway video status request failed");
+    const payload = (await response.json()) as RunwayTaskDetailResponse;
+    switch (payload.status) {
+      case "SUCCEEDED":
+        return payload;
+      case "FAILED":
+      case "CANCELLED":
+        throw new Error(
+          (typeof payload.failure === "string"
+            ? payload.failure
+            : payload.failure?.message
+          )?.trim() || `Runway video generation ${payload.status.toLowerCase()}`,
+        );
+      case "PENDING":
+      case "RUNNING":
+      case "THROTTLED":
+      default:
+        await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL_MS));
+        break;
+    }
+  }
+  throw new Error(`Runway video generation task ${params.taskId} did not finish in time`);
+}
+
+async function downloadRunwayVideos(params: {
+  urls: string[];
+  timeoutMs?: number;
+  fetchFn: typeof fetch;
+}): Promise<GeneratedVideoAsset[]> {
+  const videos: GeneratedVideoAsset[] = [];
+  for (const [index, url] of params.urls.entries()) {
+    const response = await fetchWithTimeout(
+      url,
+      { method: "GET" },
+      params.timeoutMs ?? DEFAULT_TIMEOUT_MS,
+      params.fetchFn,
+    );
+    await assertOkOrThrowHttpError(response, "Runway generated video download failed");
+    const mimeType = response.headers.get("content-type")?.trim() || "video/mp4";
+    const arrayBuffer = await response.arrayBuffer();
+    videos.push({
+      buffer: Buffer.from(arrayBuffer),
+      mimeType,
+      fileName: `video-${index + 1}.${mimeType.includes("webm") ? "webm" : "mp4"}`,
+      metadata: { sourceUrl: url },
+    });
+  }
+  return videos;
+}
+
+export function buildRunwayVideoGenerationProvider(): VideoGenerationProvider {
+  return {
+    id: "runway",
+    label: "Runway",
+    defaultModel: DEFAULT_RUNWAY_MODEL,
+    models: ["gen4.5", "gen4_turbo", "gen4_aleph", "gen3a_turbo", "veo3.1", "veo3.1_fast", "veo3"],
+    isConfigured: ({ agentDir }) =>
+      isProviderApiKeyConfigured({
+        provider: "runway",
+        agentDir,
+      }),
+    capabilities: {
+      maxVideos: 1,
+      maxInputImages: 1,
+      maxInputVideos: 1,
+      maxDurationSeconds: MAX_DURATION_SECONDS,
+      supportsAspectRatio: true,
+    },
+    async generateVideo(req): Promise<VideoGenerationResult> {
+      const auth = await resolveApiKeyForProvider({
+        provider: "runway",
+        cfg: req.cfg,
+        agentDir: req.agentDir,
+        store: req.authStore,
+      });
+      if (!auth.apiKey) {
+        throw new Error("Runway API key missing");
+      }
+
+      const fetchFn = fetch;
+      const requestBody = buildCreateBody(req);
+      const endpoint = resolveEndpoint(req);
+      const { baseUrl, allowPrivateNetwork, headers, dispatcherPolicy } =
+        resolveProviderHttpRequestConfig({
+          baseUrl: resolveRunwayBaseUrl(req),
+          defaultBaseUrl: DEFAULT_RUNWAY_BASE_URL,
+          defaultHeaders: {
+            Authorization: `Bearer ${auth.apiKey}`,
+            "Content-Type": "application/json",
+            "X-Runway-Version": RUNWAY_API_VERSION,
+          },
+          provider: "runway",
+          capability: "video",
+          transport: "http",
+        });
+      const { response, release } = await postJsonRequest({
+        url: `${baseUrl}${endpoint}`,
+        headers,
+        body: requestBody,
+        timeoutMs: req.timeoutMs,
+        fetchFn,
+        allowPrivateNetwork,
+        dispatcherPolicy,
+      });
+      try {
+        await assertOkOrThrowHttpError(response, "Runway video generation failed");
+        const submitted = (await response.json()) as RunwayTaskCreateResponse;
+        const taskId = submitted.id?.trim();
+        if (!taskId) {
+          throw new Error("Runway video generation response missing task id");
+        }
+        const completed = await pollRunwayTask({
+          taskId,
+          headers,
+          timeoutMs: req.timeoutMs,
+          baseUrl,
+          fetchFn,
+        });
+        const outputUrls = completed.output?.filter(
+          (value) => typeof value === "string" && value.trim(),
+        );
+        if (!outputUrls?.length) {
+          throw new Error("Runway video generation completed without output URLs");
+        }
+        const videos = await downloadRunwayVideos({
+          urls: outputUrls,
+          timeoutMs: req.timeoutMs,
+          fetchFn,
+        });
+        return {
+          videos,
+          model: req.model?.trim() || DEFAULT_RUNWAY_MODEL,
+          metadata: {
+            taskId,
+            status: completed.status,
+            endpoint,
+            outputUrls,
+          },
+        };
+      } finally {
+        await release();
+      }
+    },
+  };
+}

extensions/video-generation-providers.live.test.ts

@@ -20,6 +20,7 @@ import googlePlugin from "./google/index.js";
 import minimaxPlugin from "./minimax/index.js";
 import openaiPlugin from "./openai/index.js";
 import qwenPlugin from "./qwen/index.js";
+import runwayPlugin from "./runway/index.js";
 import togetherPlugin from "./together/index.js";
 import xaiPlugin from "./xai/index.js";
 
@@ -57,6 +58,7 @@ const CASES: LiveProviderCase[] = [
   },
   { plugin: openaiPlugin, pluginId: "openai", pluginName: "OpenAI Provider", providerId: "openai" },
   { plugin: qwenPlugin, pluginId: "qwen", pluginName: "Qwen Provider", providerId: "qwen" },
+  { plugin: runwayPlugin, pluginId: "runway", pluginName: "Runway Provider", providerId: "runway" },
   {
     plugin: togetherPlugin,
     pluginId: "together",

src/video-generation/live-test-helpers.ts

@@ -9,6 +9,7 @@ export const DEFAULT_LIVE_VIDEO_MODELS: Record<string, string> = {
   minimax: "minimax/MiniMax-Hailuo-2.3",
   openai: "openai/sora-2",
   qwen: "qwen/wan2.6-t2v",
+  runway: "runway/gen4.5",
   together: "together/Wan-AI/Wan2.2-T2V-A14B",
   xai: "xai/grok-imagine-video",
 };