From aa80cdce2ecd615dfb99ec3eaba5c6535db4ae6e Mon Sep 17 00:00:00 2001 From: esengine <359807859@qq.com> Date: Sat, 17 Oct 2026 20:18:28 -0700 Subject: [PATCH] feat(gemini): effect test, docs, release note and the desktop kind label --- desktop/frontend-next/src/i18n/en.ts | 1 + desktop/frontend-next/src/ui/vendors.ts | 1 + docs/GUIDE.md | 49 ++++++++ docs/GUIDE.zh-CN.md | 49 ++++++++ internal/assembly/boot/boot_test.go | 1 + internal/assembly/boot/effect_gemini_test.go | 117 ++++++++++++++++++ .../studio/unreleased/13615-gemini-native.md | 2 + 7 files changed, 220 insertions(+) create mode 100644 internal/assembly/boot/effect_gemini_test.go create mode 100644 release-notes/studio/unreleased/13615-gemini-native.md diff --git a/desktop/frontend-next/src/i18n/en.ts b/desktop/frontend-next/src/i18n/en.ts index 34375aeec4..22bef256bc 100644 --- a/desktop/frontend-next/src/i18n/en.ts +++ b/desktop/frontend-next/src/i18n/en.ts @@ -375,6 +375,7 @@ export const EN: Record = { "失败": "Failed", "OpenAI 兼容": "OpenAI-compatible", "Anthropic 兼容": "Anthropic-compatible", + "Gemini 原生（实验）": "Gemini native (experimental)", "扩展": "Extension", "新增": "Added", "已删除": "Deleted", diff --git a/desktop/frontend-next/src/ui/vendors.ts b/desktop/frontend-next/src/ui/vendors.ts index 56402f00d4..9fe3825663 100644 --- a/desktop/frontend-next/src/ui/vendors.ts +++ b/desktop/frontend-next/src/ui/vendors.ts @@ -8,6 +8,7 @@ export const KIND_LABEL: Record = { openai: "OpenAI 兼容", anthropic: "Anthropic 兼容", responses: "Responses", + gemini: "Gemini 原生（实验）", typesafe: "决策 · System One", extension: "扩展", }; diff --git a/docs/GUIDE.md b/docs/GUIDE.md index 5c2899f30e..1b97986770 100644 --- a/docs/GUIDE.md +++ b/docs/GUIDE.md @@ -30,6 +30,7 @@ - [Embedded documentation retrieval](#embedded-documentation-retrieval) - [@ references](#-references) - [Two-model collaboration](#two-model-collaboration) +- [Google Gemini, native (experimental)](#google-gemini-native-experimental) ## Configuration @@ -2112,3 +2113,51 @@ Not built, and why: - Gemini: its video input is in the native Gemini API (inline data, the Files API, YouTube URLs); the OpenAI-compatible page does not list a video content part, and Reasonix reaches Gemini through that endpoint. - Zhipu GLM: the documentation shows `video_url` with a public HTTPS address only and says nothing about base64 or limits, so a local file cannot be sent without Reasonix uploading it somewhere. + +## Google Gemini, native (experimental) + +The `gemini` provider kind speaks Google's Interactions API. It is experimental: it was written from Google's documentation (read 2026-10-17) and has not been run against the real endpoint, so expect rough edges and report them. + +Install the **Google Gemini, native (experimental)** preset, or write the entry yourself: + +```toml +[[providers]] +name = "gemini-native" +kind = "gemini" +base_url = "https://generativelanguage.googleapis.com/v1beta" +api_key_env = "GEMINI_API_KEY" +models = ["gemini-3.8-flash"] +``` + +- Every request carries the whole conversation and `store` is false, so the transcript stays yours; Google keeps nothing of it. +- Thinking runs at the level the entry sets (`/effort`); the preset lists the levels each model documents. Thought summaries are streamed as reasoning, and the thought steps are sent back exactly as received, in order, on the next request. +- `temperature` and `top_p` are never sent: Google's reference page does not list them for this API. +- The preset lists only models whose own pages were read: `gemini-3.8-flash`, `gemini-4-argon`, `gemini-3.6-flash`, `gemini-3.5-flash-lite` and `gemini-3.1-pro-preview`, all with a 1,048,576-token window, image and video input. Others work if you list them and say what they read. +- The older **Google Gemini** preset uses Google's OpenAI-compatible endpoint. That endpoint documents no video input. + +### Watching video + +Declare the models and how a file reaches them, as for any provider (see Letting a model watch a video file): + +```toml +video_models = ["gemini-3.8-flash"] +video_protocol = "gemini_files" # or "gemini_inline" +``` + +- `gemini_inline` puts the file in the request as base64. Google's page says "under 20MB total request size" while its table says 100 MB; the narrower reading is used, so the file is held to 15,000,000 bytes (the encoded request is a third larger). You can only lower it with `[tools] video_native_max_bytes`. +- `gemini_files` uploads the file with the Files API resumable protocol, waits until Google reports it `ACTIVE`, refers to it by its uri, and deletes it when the request ends. Google allows 2 GB per file and 20 GB per project and removes files after 48 hours on its own. +- MP4, WebM and MOV are sent natively on both protocols; any other container is sampled into frames. A file Google reports as `FAILED`, or in a state this build does not know, is not sent and the model is told `video.upload_failed`. +- A public YouTube address cannot be given to the model through `read_video`: the host does not fetch addresses, and handing one to the vendor is a separate setting. + +### Cost + +Reasonix prices Google's list rates per 1M tokens (https://ai.google.dev/gemini-api/docs/pricing, read 2026-10-17): + +| Model | Input | Cached | Output (incl. thinking) | +|---|---|---|---| +| `gemini-3.8-flash`, `gemini-3.6-flash` | $0.75 | $0.075 | $3.75 | +| the same two from 2027-01-01 | $1.50 | $0.15 | $7.50 | +| `gemini-3.5-flash-lite` | $0.30 | $0.03 | $2.50 | +| `gemini-3.1-pro-preview` (up to 200k prompt tokens) | $2.00 | $0.20 | $12.00 | + +`gemini-3.1-pro-preview` costs more above 200k prompt tokens ($4.00 / $0.40 / $18.00), which one rate card cannot say, so a long prompt reads low. `gemini-4-argon` shows no price on the page and is not priced. Storage for explicit context caching is not counted. diff --git a/docs/GUIDE.zh-CN.md b/docs/GUIDE.zh-CN.md index 248f5bdf13..f9c0c093b7 100644 --- a/docs/GUIDE.zh-CN.md +++ b/docs/GUIDE.zh-CN.md @@ -30,6 +30,7 @@ - [内置文档检索](#内置文档检索) - [@ 引用](#-引用) - [双模型协同](#双模型协同) +- [Google Gemini 原生接口（实验）](#google-gemini-原生接口实验) ## 配置 @@ -1667,3 +1668,51 @@ video_protocol = "inline_data_url" # 或 "moonshot_file" - Gemini：视频输入在原生 Gemini 接口里（内联数据、Files API、YouTube 链接）；OpenAI 兼容页没有列视频内容块，而 Reasonix 是通过那个端点接 Gemini 的。 - 智谱 GLM：文档只给了公网 HTTPS 地址的 `video_url` 示例，没写 base64 与限制，所以不能在不替用户把文件传到别处的前提下发送本地文件。 + +## Google Gemini 原生接口（实验） + +`gemini` 提供方类型直接使用 Google 的 Interactions API。它是实验性的：按 Google 官方文档（2026-10-17 读取）编写，尚未用真实端点验证，可能有粗糙之处，欢迎反馈。 + +安装预设「Google Gemini, native (experimental)」，或自己写一项： + +```toml +[[providers]] +name = "gemini-native" +kind = "gemini" +base_url = "https://generativelanguage.googleapis.com/v1beta" +api_key_env = "GEMINI_API_KEY" +models = ["gemini-3.8-flash"] +``` + +- 每次请求都带上完整对话，且 `store` 为 false：对话记录归你自己，Google 那边不留存。 +- 思考强度按条目设置（`/effort`），预设里是每个模型文档列出的档位。思考摘要作为推理内容流出，思考步骤会按收到的原样、原顺序随下一次请求发回。 +- 永远不发送 `temperature` 和 `top_p`：Google 的参考页没有为这个接口列出它们。 +- 预设只列读过各自模型页的模型：`gemini-3.8-flash`、`gemini-4-argon`、`gemini-3.6-flash`、`gemini-3.5-flash-lite`、`gemini-3.1-pro-preview`，都是 1,048,576 token 窗口、支持图像和视频输入。其他模型自己列出并声明能读什么即可。 +- 原有的「Google Gemini」预设走 Google 的 OpenAI 兼容端点，该端点文档里没有视频输入。 + +### 看视频 + +像其他提供方一样，声明模型和文件的送达方式（见「让模型原生看视频文件」）： + +```toml +video_models = ["gemini-3.8-flash"] +video_protocol = "gemini_files" # 或 "gemini_inline" +``` + +- `gemini_inline` 把文件以 base64 放进请求。Google 页面正文写“请求总大小 20MB 以内”，表格却写 100 MB；这里取更窄的读法，文件上限 15,000,000 字节（编码后请求会大三分之一），只能用 `[tools] video_native_max_bytes` 调低。 +- `gemini_files` 用 Files API 的可恢复上传协议上传，等 Google 报告 `ACTIVE`，用 uri 引用，请求结束后立即删除。Google 允许单文件 2 GB、每项目 20 GB，文件 48 小时后自动清除。 +- 两种协议都原生发送 MP4、WebM、MOV；其他容器会抽帧。Google 报告 `FAILED` 或本版本不认识的状态时不发送，并告诉模型 `video.upload_failed`。 +- 公开的 YouTube 地址不能通过 `read_video` 交给模型：宿主不抓取地址，把地址交给厂商是单独的一项设置。 + +### 费用 + +Reasonix 按 Google 公布的标价计费，每百万 token（https://ai.google.dev/gemini-api/docs/pricing，2026-10-17 读取）： + +| 模型 | 输入 | 缓存命中 | 输出（含思考） | +|---|---|---|---| +| `gemini-3.8-flash`、`gemini-3.6-flash` | $0.75 | $0.075 | $3.75 | +| 这两个模型自 2027-01-01 起 | $1.50 | $0.15 | $7.50 | +| `gemini-3.5-flash-lite` | $0.30 | $0.03 | $2.50 | +| `gemini-3.1-pro-preview`（提示词不超过 200k token） | $2.00 | $0.20 | $12.00 | + +`gemini-3.1-pro-preview` 在提示词超过 200k token 后更贵（$4.00 / $0.40 / $18.00），一张费率卡表达不了，所以超长提示词会读低。`gemini-4-argon` 页面上没有价格，不计价。显式上下文缓存的存储费不计入。 diff --git a/internal/assembly/boot/boot_test.go b/internal/assembly/boot/boot_test.go index 2585d07c5f..0b399e0580 100644 --- a/internal/assembly/boot/boot_test.go +++ b/internal/assembly/boot/boot_test.go @@ -45,6 +45,7 @@ import ( // Blank import registers the provider kind the same way cmd/reasonix's main // does; importing builtin above registers the built-in tools. _ "reasonix/internal/model/anthropic" + _ "reasonix/internal/model/gemini" _ "reasonix/internal/model/openai" ) diff --git a/internal/assembly/boot/effect_gemini_test.go b/internal/assembly/boot/effect_gemini_test.go new file mode 100644 index 0000000000..b3b84aeb83 --- /dev/null +++ b/internal/assembly/boot/effect_gemini_test.go @@ -0,0 +1,117 @@ +package boot + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + + "reasonix/internal/contract/config" + "reasonix/internal/contract/event" +) + +// geminiServer answers the Interactions API with one thought and one answer, and +// records what each request carried. The wire is built from Google's documentation +// (https://ai.google.dev/gemini-api/docs/interactions/streaming, read 2026-10-17) +// and was not run against the real endpoint. +type geminiServer struct { + mu sync.Mutex + bodies []string + headers []http.Header + paths []string +} + +func (g *geminiServer) handler(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + g.mu.Lock() + g.bodies = append(g.bodies, string(b)) + g.headers = append(g.headers, r.Header.Clone()) + g.paths = append(g.paths, r.URL.Path) + n := len(g.bodies) + g.mu.Unlock() + w.Header().Set("Content-Type", "text/event-stream") + events := []string{ + `{"event_type":"interaction.created","interaction":{"id":"i","status":"in_progress"}}`, + `{"event_type":"step.start","index":0,"step":{"type":"thought"}}`, + fmt.Sprintf(`{"event_type":"step.delta","index":0,"delta":{"type":"thought_summary","content":{"type":"text","text":"thinking %d"}}}`, n), + fmt.Sprintf(`{"event_type":"step.delta","index":0,"delta":{"type":"thought_signature","signature":"SIG-%d"}}`, n), + `{"event_type":"step.stop","index":0}`, + `{"event_type":"step.start","index":1,"step":{"type":"model_output"}}`, + fmt.Sprintf(`{"event_type":"step.delta","index":1,"delta":{"type":"text","text":"answer %d"}}`, n), + `{"event_type":"step.stop","index":1}`, + `{"event_type":"interaction.completed","interaction":{"id":"i","status":"completed","usage":{"total_tokens":30,"total_input_tokens":10,"total_output_tokens":5,"total_thought_tokens":15,"total_cached_tokens":4}}}`, + } + for _, e := range events { + var probe struct { + EventType string `json:"event_type"` + } + _ = json.Unmarshal([]byte(e), &probe) + fmt.Fprintf(w, "event: %s\ndata: %s\n\n", probe.EventType, e) + } + fmt.Fprint(w, "event: done\ndata: [DONE]\n\n") +} + +// A thought the model produced rides the next request exactly as it came, in +// order, beside the answer it belonged to; nothing in the request is a sampling +// parameter, and the request is stateless. +func TestEffectAGeminiThoughtIsReplayedOnTheNextRequest(t *testing.T) { + isolateConfigHome(t) + dir := robustTempDir(t) + t.Chdir(dir) + g := &geminiServer{} + srv := httptest.NewServer(http.HandlerFunc(g.handler)) + defer srv.Close() + writeFile(t, filepath.Dir(config.UserConfigPath()), ".env", "GEMINI_EFFECT_KEY=k-effect\n") + writeFile(t, filepath.Dir(config.UserConfigPath()), "config.toml", fmt.Sprintf(`default_model = "gem/gemini-3.8-flash" +[agent] +system_prompt = "BASE" +[[providers]] +name = "gem" +kind = "gemini" +base_url = %q +model = "gemini-3.8-flash" +models = ["gemini-3.8-flash"] +api_key_env = "GEMINI_EFFECT_KEY" +`, srv.URL+"/v1beta")) + ctrl, err := Build(context.Background(), Options{Sink: event.Discard, Model: "gem/gemini-3.8-flash", WorkspaceRoot: dir}) + if err != nil { + t.Fatal(err) + } + defer ctrl.Close() + for _, prompt := range []string{"first question", "second question"} { + if err := ctrl.Run(context.Background(), prompt); err != nil { + t.Fatalf("Run(%q): %v", prompt, err) + } + } + if len(g.bodies) != 2 { + t.Fatalf("%d requests, want 2", len(g.bodies)) + } + for i, p := range g.paths { + if p != "/v1beta/interactions" || g.headers[i].Get("x-goog-api-key") != "k-effect" { + t.Fatalf("request %d: path=%q key=%q headers=%v", i, p, g.headers[i].Get("x-goog-api-key"), g.headers[i]) + } + } + first, second := g.bodies[0], g.bodies[1] + for _, banned := range []string{"temperature", "top_p"} { + if strings.Contains(first, banned) { + t.Fatalf("a sampling parameter reached the wire: %s", first) + } + } + for _, want := range []string{`"store":false`, `"stream":true`, `"system_instruction":"BASE`, `"type":"user_input"`, `"type":"function"`} { + if !strings.Contains(first, want) { + t.Fatalf("the first request lacks %s:\n%.600s", want, first) + } + } + replay := `{"type":"thought","summary":[{"type":"text","text":"thinking 1"}],"signature":"SIG-1"},` + + `{"type":"model_output","content":[{"type":"text","text":"answer 1"}]},` + + `{"type":"user_input","content":[` + if !strings.Contains(second, replay) || !strings.Contains(second, "second question") { + t.Fatalf("the first turn did not come back verbatim and in order:\n%s", second) + } +} diff --git a/release-notes/studio/unreleased/13615-gemini-native.md b/release-notes/studio/unreleased/13615-gemini-native.md new file mode 100644 index 0000000000..e25116e0e5 --- /dev/null +++ b/release-notes/studio/unreleased/13615-gemini-native.md @@ -0,0 +1,2 @@ +- 新增（实验性，未经真实端点验证）：原生 Gemini 提供方（Interactions API），支持思考、工具调用与视频（含 Files API）。关联 #13615。 +- Added (experimental, not run against the real endpoint): a native Gemini provider on the Interactions API with thinking, tools and video. Refs #13615.