nyrn
/
openclaw
mirror of https://github.com/openclaw/openclaw.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor: share request url resolution

This commit is contained in:
Peter Steinberger 2026-03-14 01:39:37 +00:00
parent 757077d028
commit 9cfc2d4618
2 changed files with 20 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
src/agents/tools/web-tools.fetch.test.ts
View File

@ -1,6 +1,7 @@
import { EnvHttpProxyAgent } from "undici";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import * as ssrf from "../../infra/net/ssrf.js";
import { resolveRequestUrl } from "../../plugin-sdk/request-url.js";
import { withFetchPreconnect } from "../../test-utils/fetch-mock.js";
import { makeFetchHeaders } from "./web-fetch.test-harness.js";
import { createWebFetchTool } from "./web-tools.js";
@ -76,19 +77,6 @@ function errorHtmlResponse(
text: async () => html,
};
}
function requestUrl(input: RequestInfo | URL): string {
if (typeof input === "string") {
return input;
}
if (input instanceof URL) {
return input.toString();
}
if ("url" in input && typeof input.url === "string") {
return input.url;
}
return "";
}
function installMockFetch(
impl: (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>,
) {
@ -122,7 +110,7 @@ function installPlainTextFetch(text: string) {
status: 200,
headers: makeFetchHeaders({ "content-type": "text/plain" }),
text: async () => text,
url: requestUrl(input),
url: resolveRequestUrl(input),
} as Response),
);
}
@ -212,7 +200,7 @@ describe("web_fetch extraction fallbacks", () => {
status: 200,
headers: makeFetchHeaders({ "content-type": "text/plain" }),
text: async () => longText,
url: requestUrl(input),
url: resolveRequestUrl(input),
} as Response),
);
@ -274,7 +262,7 @@ describe("web_fetch extraction fallbacks", () => {
status: 200,
headers: makeFetchHeaders({ "content-type": "text/plain" }),
text: async () => "proxy body",
url: requestUrl(input),
url: resolveRequestUrl(input),
} as Response),
);
const tool = createFetchTool({ firecrawl: { enabled: false } });
@ -293,7 +281,7 @@ describe("web_fetch extraction fallbacks", () => {
it("falls back to firecrawl when readability returns no content", async () => {
installMockFetch((input: RequestInfo | URL) => {
const url = requestUrl(input);
const url = resolveRequestUrl(input);
if (url.includes("api.firecrawl.dev")) {
return Promise.resolve(firecrawlResponse("firecrawl content")) as Promise<Response>;
}
@ -311,7 +299,7 @@ describe("web_fetch extraction fallbacks", () => {
it("normalizes firecrawl Authorization header values", async () => {
const fetchSpy = installMockFetch((input: RequestInfo | URL) => {
const url = requestUrl(input);
const url = resolveRequestUrl(input);
if (url.includes("api.firecrawl.dev/v2/scrape")) {
return Promise.resolve(firecrawlResponse("firecrawl normalized")) as Promise<Response>;
}
@ -328,7 +316,7 @@ describe("web_fetch extraction fallbacks", () => {
expect(result?.details).toMatchObject({ extractor: "firecrawl" });
const firecrawlCall = fetchSpy.mock.calls.find((call) =>
requestUrl(call[0]).includes("/v2/scrape"),
resolveRequestUrl(call[0]).includes("/v2/scrape"),
);
expect(firecrawlCall).toBeTruthy();
const init = firecrawlCall?.[1];
@ -340,7 +328,7 @@ describe("web_fetch extraction fallbacks", () => {
installMockFetch(
(input: RequestInfo | URL) =>
Promise.resolve(
htmlResponse("<html><body>hi</body></html>", requestUrl(input)),
htmlResponse("<html><body>hi</body></html>", resolveRequestUrl(input)),
) as Promise<Response>,
);
@ -356,7 +344,7 @@ describe("web_fetch extraction fallbacks", () => {
it("throws when readability is empty and firecrawl fails", async () => {
installMockFetch((input: RequestInfo | URL) => {
const url = requestUrl(input);
const url = resolveRequestUrl(input);
if (url.includes("api.firecrawl.dev")) {
return Promise.resolve(firecrawlError()) as Promise<Response>;
}
@ -373,7 +361,7 @@ describe("web_fetch extraction fallbacks", () => {
it("uses firecrawl when direct fetch fails", async () => {
installMockFetch((input: RequestInfo | URL) => {
const url = requestUrl(input);
const url = resolveRequestUrl(input);
if (url.includes("api.firecrawl.dev")) {
return Promise.resolve(firecrawlResponse("firecrawl fallback", url)) as Promise<Response>;
}
@ -399,7 +387,7 @@ describe("web_fetch extraction fallbacks", () => {
const large = "a".repeat(80_000);
installMockFetch(
(input: RequestInfo | URL) =>
Promise.resolve(textResponse(large, requestUrl(input))) as Promise<Response>,
Promise.resolve(textResponse(large, resolveRequestUrl(input))) as Promise<Response>,
);
const tool = createFetchTool({
@ -427,7 +415,7 @@ describe("web_fetch extraction fallbacks", () => {
installMockFetch(
(input: RequestInfo | URL) =>
Promise.resolve(
errorHtmlResponse(html, 404, requestUrl(input), "Text/HTML; charset=utf-8"),
errorHtmlResponse(html, 404, resolveRequestUrl(input), "Text/HTML; charset=utf-8"),
) as Promise<Response>,
);
@ -450,7 +438,9 @@ describe("web_fetch extraction fallbacks", () => {
"<!DOCTYPE HTML><html><head><title>Oops</title></head><body><h1>Oops</h1></body></html>";
installMockFetch(
(input: RequestInfo | URL) =>
Promise.resolve(errorHtmlResponse(html, 500, requestUrl(input), null)) as Promise<Response>,
Promise.resolve(
errorHtmlResponse(html, 500, resolveRequestUrl(input), null),
) as Promise<Response>,
);
const tool = createFetchTool({ firecrawl: { enabled: false } });
@ -466,7 +456,7 @@ describe("web_fetch extraction fallbacks", () => {
it("wraps firecrawl error details", async () => {
installMockFetch((input: RequestInfo | URL) => {
const url = requestUrl(input);
const url = resolveRequestUrl(input);
if (url.includes("api.firecrawl.dev")) {
return Promise.resolve({
ok: false,

17
src/slack/monitor/media.ts
View File

@ -3,6 +3,7 @@ import { normalizeHostname } from "../../infra/net/hostname.js";
import type { FetchLike } from "../../media/fetch.js";
import { fetchRemoteMedia } from "../../media/fetch.js";
import { saveMediaBuffer } from "../../media/store.js";
import { resolveRequestUrl } from "../../plugin-sdk/request-url.js";
import type { SlackAttachment, SlackFile } from "../types.js";
function isSlackHostname(hostname: string): boolean {
@ -37,23 +38,13 @@ function assertSlackFileUrl(rawUrl: string): URL {
return parsed;
}
function resolveRequestUrl(input: RequestInfo | URL): string {
if (typeof input === "string") {
return input;
}
if (input instanceof URL) {
return input.toString();
}
if ("url" in input && typeof input.url === "string") {
return input.url;
}
throw new Error("Unsupported fetch input: expected string, URL, or Request");
}
function createSlackMediaFetch(token: string): FetchLike {
let includeAuth = true;
return async (input, init) => {
const url = resolveRequestUrl(input);
if (!url) {
throw new Error("Unsupported fetch input: expected string, URL, or Request");
}
const { headers: initHeaders, redirect: _redirect, ...rest } = init ?? {};
const headers = new Headers(initHeaders);