mirror of https://github.com/openclaw/openclaw.git
memory: normalize Gemini embeddings (#43409)
Merged via squash.
Prepared head SHA: 70613e0225
Co-authored-by: gumadeiras <5599352+gumadeiras@users.noreply.github.com>
Co-authored-by: gumadeiras <5599352+gumadeiras@users.noreply.github.com>
Reviewed-by: @gumadeiras
This commit is contained in:
parent
2a18cbb110
commit
01ffc5db24
|
|
@ -12,7 +12,7 @@ Docs: https://docs.openclaw.ai
|
|||
- Exec/child commands: mark child command environments with `OPENCLAW_CLI` so subprocesses can detect when they were launched from the OpenClaw CLI. (#41411) Thanks @vincentkoc.
|
||||
- iOS/Home canvas: add a bundled welcome screen with a live agent overview that refreshes on connect, reconnect, and foreground return, and move the compact connection pill off the top-left canvas overlay. (#42456) Thanks @ngutman.
|
||||
- iOS/Home canvas: replace floating controls with a docked toolbar, make the bundled home scaffold adapt to smaller phones, and open chat in the resolved main session instead of a synthetic `ios` session. (#42456) Thanks @ngutman.
|
||||
- Memory/Gemini: add `gemini-embedding-2-preview` memory-search support with configurable output dimensions and automatic reindexing when the configured dimensions change. (#42501) thanks @BillChirico.
|
||||
- Memory/Gemini: add `gemini-embedding-2-preview` memory-search support with configurable output dimensions and automatic reindexing when the configured dimensions change. (#42501) Thanks @BillChirico and @gumadeiras.
|
||||
- Discord/auto threads: add `autoArchiveDuration` channel config for auto-created threads so Discord thread archiving can stay at 1 hour, 1 day, 3 days, or 1 week instead of always using the 1-hour default. (#35065) Thanks @davidguttman.
|
||||
- OpenCode/onboarding: add new OpenCode Go provider, treat Zen and Go as one OpenCode setup in the wizard/docs while keeping the runtime providers split, store one shared OpenCode key for both profiles, and stop overriding the built-in `opencode-go` catalog routing. (#42313) Thanks @ImLukeF and @vincentkoc.
|
||||
- macOS/chat UI: add a chat model picker, persist explicit thinking-level selections across relaunch, and harden provider-aware session model sync for the shared chat composer. (#42314) Thanks @ImLukeF.
|
||||
|
|
@ -181,6 +181,7 @@ Docs: https://docs.openclaw.ai
|
|||
- SecretRef/models: harden custom/provider secret persistence and reuse across models.json snapshots, merge behavior, runtime headers, and secret audits. (#42554) Thanks @joshavant.
|
||||
- macOS/browser proxy: serialize non-GET browser proxy request bodies through `AnyCodable.foundationValue` so nested JSON bodies no longer crash the macOS app with `Invalid type in JSON write (__SwiftValue)`. (#43069) Thanks @Effet.
|
||||
- CLI/skills tables: keep terminal table borders aligned for wide graphemes, use full reported terminal width, and switch a few ambiguous skill icons to Terminal-safe emoji so `openclaw skills` renders more consistently in Terminal.app and iTerm. Thanks @vincentkoc.
|
||||
- Memory/Gemini: normalize returned Gemini embeddings across direct query, direct batch, and async batch paths so memory search uses consistent vector handling for Gemini too. (#43409) Thanks @gumadeiras.
|
||||
|
||||
## 2026.3.7
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,10 @@
|
|||
import { afterEach, beforeAll, describe, expect, it, vi } from "vitest";
|
||||
import type { GeminiEmbeddingClient } from "./embeddings-gemini.js";
|
||||
|
||||
function magnitude(values: number[]) {
|
||||
return Math.sqrt(values.reduce((sum, value) => sum + value * value, 0));
|
||||
}
|
||||
|
||||
describe("runGeminiEmbeddingBatches", () => {
|
||||
let runGeminiEmbeddingBatches: typeof import("./batch-gemini.js").runGeminiEmbeddingBatches;
|
||||
|
||||
|
|
@ -56,7 +60,7 @@ describe("runGeminiEmbeddingBatches", () => {
|
|||
return new Response(
|
||||
JSON.stringify({
|
||||
key: "req-1",
|
||||
response: { embedding: { values: [0.1, 0.2, 0.3] } },
|
||||
response: { embedding: { values: [3, 4] } },
|
||||
}),
|
||||
{
|
||||
status: 200,
|
||||
|
|
@ -88,7 +92,11 @@ describe("runGeminiEmbeddingBatches", () => {
|
|||
concurrency: 1,
|
||||
});
|
||||
|
||||
expect(results.get("req-1")).toEqual([0.1, 0.2, 0.3]);
|
||||
const embedding = results.get("req-1");
|
||||
expect(embedding).toBeDefined();
|
||||
expect(embedding?.[0]).toBeCloseTo(0.6, 5);
|
||||
expect(embedding?.[1]).toBeCloseTo(0.8, 5);
|
||||
expect(magnitude(embedding ?? [])).toBeCloseTo(1, 5);
|
||||
expect(fetchMock).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import {
|
|||
type EmbeddingBatchExecutionParams,
|
||||
} from "./batch-runner.js";
|
||||
import { buildBatchHeaders, normalizeBatchBaseUrl } from "./batch-utils.js";
|
||||
import { sanitizeAndNormalizeEmbedding } from "./embedding-vectors.js";
|
||||
import { debugEmbeddingsLog } from "./embeddings-debug.js";
|
||||
import type { GeminiEmbeddingClient, GeminiTextEmbeddingRequest } from "./embeddings-gemini.js";
|
||||
import { hashText } from "./internal.js";
|
||||
|
|
@ -346,7 +347,9 @@ export async function runGeminiEmbeddingBatches(
|
|||
errors.push(`${customId}: ${line.response.error.message}`);
|
||||
continue;
|
||||
}
|
||||
const embedding = line.embedding?.values ?? line.response?.embedding?.values ?? [];
|
||||
const embedding = sanitizeAndNormalizeEmbedding(
|
||||
line.embedding?.values ?? line.response?.embedding?.values ?? [],
|
||||
);
|
||||
if (embedding.length === 0) {
|
||||
errors.push(`${customId}: empty embedding`);
|
||||
continue;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,8 @@
|
|||
export function sanitizeAndNormalizeEmbedding(vec: number[]): number[] {
|
||||
const sanitized = vec.map((value) => (Number.isFinite(value) ? value : 0));
|
||||
const magnitude = Math.sqrt(sanitized.reduce((sum, value) => sum + value * value, 0));
|
||||
if (magnitude < 1e-10) {
|
||||
return sanitized;
|
||||
}
|
||||
return sanitized.map((value) => value / magnitude);
|
||||
}
|
||||
|
|
@ -44,6 +44,10 @@ function parseFetchBody(fetchMock: { mock: { calls: unknown[][] } }, callIndex =
|
|||
return JSON.parse((init?.body as string) ?? "{}") as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function magnitude(values: number[]) {
|
||||
return Math.sqrt(values.reduce((sum, value) => sum + value * value, 0));
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.resetAllMocks();
|
||||
vi.unstubAllGlobals();
|
||||
|
|
@ -224,6 +228,25 @@ describe("gemini-embedding-2-preview provider", () => {
|
|||
expect(body.content).toEqual({ parts: [{ text: "test query" }] });
|
||||
});
|
||||
|
||||
it("normalizes embedQuery response vectors", async () => {
|
||||
const fetchMock = createGeminiFetchMock([3, 4]);
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
mockResolvedProviderKey();
|
||||
|
||||
const { provider } = await createGeminiEmbeddingProvider({
|
||||
config: {} as never,
|
||||
provider: "gemini",
|
||||
model: "gemini-embedding-2-preview",
|
||||
fallback: "none",
|
||||
});
|
||||
|
||||
const embedding = await provider.embedQuery("test query");
|
||||
|
||||
expect(embedding[0]).toBeCloseTo(0.6, 5);
|
||||
expect(embedding[1]).toBeCloseTo(0.8, 5);
|
||||
expect(magnitude(embedding)).toBeCloseTo(1, 5);
|
||||
});
|
||||
|
||||
it("includes outputDimensionality in embedBatch request", async () => {
|
||||
const fetchMock = createGeminiBatchFetchMock(2);
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
|
@ -255,6 +278,28 @@ describe("gemini-embedding-2-preview provider", () => {
|
|||
]);
|
||||
});
|
||||
|
||||
it("normalizes embedBatch response vectors", async () => {
|
||||
const fetchMock = createGeminiBatchFetchMock(2, [3, 4]);
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
mockResolvedProviderKey();
|
||||
|
||||
const { provider } = await createGeminiEmbeddingProvider({
|
||||
config: {} as never,
|
||||
provider: "gemini",
|
||||
model: "gemini-embedding-2-preview",
|
||||
fallback: "none",
|
||||
});
|
||||
|
||||
const embeddings = await provider.embedBatch(["text1", "text2"]);
|
||||
|
||||
expect(embeddings).toHaveLength(2);
|
||||
for (const embedding of embeddings) {
|
||||
expect(embedding[0]).toBeCloseTo(0.6, 5);
|
||||
expect(embedding[1]).toBeCloseTo(0.8, 5);
|
||||
expect(magnitude(embedding)).toBeCloseTo(1, 5);
|
||||
}
|
||||
});
|
||||
|
||||
it("respects custom outputDimensionality", async () => {
|
||||
const fetchMock = createGeminiFetchMock();
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
|
@ -310,6 +355,28 @@ describe("gemini-embedding-2-preview provider", () => {
|
|||
).rejects.toThrow(/Invalid outputDimensionality 512/);
|
||||
});
|
||||
|
||||
it("sanitizes non-finite values before normalization", async () => {
|
||||
const fetchMock = createGeminiFetchMock([
|
||||
1,
|
||||
Number.NaN,
|
||||
Number.POSITIVE_INFINITY,
|
||||
Number.NEGATIVE_INFINITY,
|
||||
]);
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
mockResolvedProviderKey();
|
||||
|
||||
const { provider } = await createGeminiEmbeddingProvider({
|
||||
config: {} as never,
|
||||
provider: "gemini",
|
||||
model: "gemini-embedding-2-preview",
|
||||
fallback: "none",
|
||||
});
|
||||
|
||||
const embedding = await provider.embedQuery("test");
|
||||
|
||||
expect(embedding).toEqual([1, 0, 0, 0]);
|
||||
});
|
||||
|
||||
it("uses correct endpoint URL", async () => {
|
||||
const fetchMock = createGeminiFetchMock();
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import {
|
|||
import { requireApiKey, resolveApiKeyForProvider } from "../agents/model-auth.js";
|
||||
import { parseGeminiAuth } from "../infra/gemini-auth.js";
|
||||
import type { SsrFPolicy } from "../infra/net/ssrf.js";
|
||||
import { sanitizeAndNormalizeEmbedding } from "./embedding-vectors.js";
|
||||
import { debugEmbeddingsLog } from "./embeddings-debug.js";
|
||||
import type { EmbeddingProvider, EmbeddingProviderOptions } from "./embeddings.js";
|
||||
import { buildRemoteBaseUrlPolicy, withRemoteHttpResponse } from "./remote-http.js";
|
||||
|
|
@ -222,7 +223,7 @@ export async function createGeminiEmbeddingProvider(
|
|||
apiKeys: client.apiKeys,
|
||||
execute: (apiKey) => fetchWithGeminiAuth(apiKey, embedUrl, body),
|
||||
});
|
||||
return payload.embedding?.values ?? [];
|
||||
return sanitizeAndNormalizeEmbedding(payload.embedding?.values ?? []);
|
||||
};
|
||||
|
||||
const embedBatch = async (texts: string[]): Promise<number[][]> => {
|
||||
|
|
@ -244,7 +245,7 @@ export async function createGeminiEmbeddingProvider(
|
|||
execute: (apiKey) => fetchWithGeminiAuth(apiKey, batchUrl, batchBody),
|
||||
});
|
||||
const embeddings = Array.isArray(payload.embeddings) ? payload.embeddings : [];
|
||||
return texts.map((_, index) => embeddings[index]?.values ?? []);
|
||||
return texts.map((_, index) => sanitizeAndNormalizeEmbedding(embeddings[index]?.values ?? []));
|
||||
};
|
||||
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { resolveOllamaApiBase } from "../agents/ollama-models.js";
|
|||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
import type { SsrFPolicy } from "../infra/net/ssrf.js";
|
||||
import { normalizeOptionalSecretInput } from "../utils/normalize-secret-input.js";
|
||||
import { sanitizeAndNormalizeEmbedding } from "./embedding-vectors.js";
|
||||
import { normalizeEmbeddingModelWithPrefixes } from "./embeddings-model-normalize.js";
|
||||
import type { EmbeddingProvider, EmbeddingProviderOptions } from "./embeddings.js";
|
||||
import { buildRemoteBaseUrlPolicy, withRemoteHttpResponse } from "./remote-http.js";
|
||||
|
|
@ -19,15 +20,6 @@ type OllamaEmbeddingClientConfig = Omit<OllamaEmbeddingClient, "embedBatch">;
|
|||
|
||||
export const DEFAULT_OLLAMA_EMBEDDING_MODEL = "nomic-embed-text";
|
||||
|
||||
function sanitizeAndNormalizeEmbedding(vec: number[]): number[] {
|
||||
const sanitized = vec.map((value) => (Number.isFinite(value) ? value : 0));
|
||||
const magnitude = Math.sqrt(sanitized.reduce((sum, value) => sum + value * value, 0));
|
||||
if (magnitude < 1e-10) {
|
||||
return sanitized;
|
||||
}
|
||||
return sanitized.map((value) => value / magnitude);
|
||||
}
|
||||
|
||||
function normalizeOllamaModel(model: string): string {
|
||||
return normalizeEmbeddingModelWithPrefixes({
|
||||
model,
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import type { OpenClawConfig } from "../config/config.js";
|
|||
import type { SecretInput } from "../config/types.secrets.js";
|
||||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
import { resolveUserPath } from "../utils.js";
|
||||
import { sanitizeAndNormalizeEmbedding } from "./embedding-vectors.js";
|
||||
import {
|
||||
createGeminiEmbeddingProvider,
|
||||
type GeminiEmbeddingClient,
|
||||
|
|
@ -18,15 +19,6 @@ import { createOpenAiEmbeddingProvider, type OpenAiEmbeddingClient } from "./emb
|
|||
import { createVoyageEmbeddingProvider, type VoyageEmbeddingClient } from "./embeddings-voyage.js";
|
||||
import { importNodeLlamaCpp } from "./node-llama.js";
|
||||
|
||||
function sanitizeAndNormalizeEmbedding(vec: number[]): number[] {
|
||||
const sanitized = vec.map((value) => (Number.isFinite(value) ? value : 0));
|
||||
const magnitude = Math.sqrt(sanitized.reduce((sum, value) => sum + value * value, 0));
|
||||
if (magnitude < 1e-10) {
|
||||
return sanitized;
|
||||
}
|
||||
return sanitized.map((value) => value / magnitude);
|
||||
}
|
||||
|
||||
export type { GeminiEmbeddingClient } from "./embeddings-gemini.js";
|
||||
export type { MistralEmbeddingClient } from "./embeddings-mistral.js";
|
||||
export type { OpenAiEmbeddingClient } from "./embeddings-openai.js";
|
||||
|
|
|
|||
Loading…
Reference in New Issue