feat(web_search): add configurable Tavily and Bocha provider backends

DuckDuckGo HTML scraping with Bing fallback remains the default
`web_search` backend — no API key required, no behaviour change for
installs that don't opt in. Users in regions where those scrapers
are rate-limited or unreliable can now set `[search]
provider = "tavily" | "bocha"` plus `api_key = "..."` in
`~/.deepseek/config.toml` (or via the `DEEPSEEK_SEARCH_PROVIDER` /
`DEEPSEEK_SEARCH_API_KEY` env vars) to route every `web_search`
call through the chosen API.

Tavily targets general AI-agent search; Bocha (博查) is the
mainland-China-friendly equivalent. Both providers are gated by
the existing `[network]` policy on their respective hosts
(`api.tavily.com`, `api.bochaai.com`) and surface a clear
`ToolError` (rather than a silent fallback to DuckDuckGo) when
the user has opted in but forgotten to set `api_key`. Test pins
the missing-key behaviour for both providers.

Resolved 3-way merge conflicts in `web_search.rs` (description
text and test module) so the contributor's helpers
(`truncate_error_body`, `sanitize_error_body`) coexist with the
v0.8.30 spam-detection and query-parsing tests already on `main`.
Folded the `SearchProvider::default()` impl into a `#[default]`
derive to satisfy the workspace `-D warnings` clippy gate, and
threaded `search: Option<SearchConfig>` through `merge_config` so
the multi-config layering doesn't break the field initialiser.

Harvested from PR #1294 by @sandofree

Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
This commit is contained in:
Hunter BownandClaude Opus 4.7 committed 2026-05-12 00:37:34 -05:00
1 parent a57cb4dfa7
commit 40df46c73d
9 files changed
+450 -4

No files matched your search

+16
View File
@@ -39,6 +39,22 @@ real world uses."
### Added
- **`web_search` supports Tavily and Bocha as configurable
backends** (harvested from PR #1294 by **@sandofree**). DuckDuckGo
with Bing fallback remains the default — no API key required —
but users in regions where those scrapers are unreliable can now
set `[search] provider = "tavily" | "bocha"` plus
`api_key = "..."` in `config.toml` (or via the
`DEEPSEEK_SEARCH_PROVIDER` / `DEEPSEEK_SEARCH_API_KEY` env vars)
to route every `web_search` call through the chosen API. Tavily
is an AI-search API targeted at general use; Bocha is the
mainland-China-friendly equivalent. Trust-boundary pins: an
unset `api_key` on an opted-in provider surfaces a clear
`ToolError` naming the missing key rather than silently falling
through to a different provider, the network policy gate
(`[network]`) is consulted for the provider host on every call,
and the default path is unchanged so no install sees provider
behaviour change unless they explicitly opt in.
- **`/change` slash command** displays the most recent
CHANGELOG.md version section from inside the TUI, so users can
see what they just upgraded into without leaving the chat
+19
View File
@@ -207,6 +207,25 @@ max_subagents = 10 # optional (1-20)
# base_url = "http://localhost:11434/v1"
# model = "deepseek-coder:1.3b" # or any local Ollama tag
# ─────────────────────────────────────────────────────────────────────────────────
# Web Search Provider
# ─────────────────────────────────────────────────────────────────────────────────
# Choose which backend `web_search` uses. Default is DuckDuckGo HTML scraping
# with Bing fallback — no API key needed but may be blocked in some regions.
# Switch to Tavily or Bocha for reliable search in mainland China.
#
# [search]
# provider = "duckduckgo" # duckduckgo | tavily | bocha
# # tavily: https://tavily.com — AI search, needs api_key
# # bocha: https://bochaai.com — 博查AI搜索,国内友好,需api_key
# api_key = "tvly-YOUR_KEY" # required for tavily and bocha
# # WARNING: treat config.toml like a secret file when
# # storing API keys. Use env vars or `auth set` instead.
#
# Env-var overrides:
# DEEPSEEK_SEARCH_PROVIDER → search.provider
# DEEPSEEK_SEARCH_API_KEY → search.api_key
# ─────────────────────────────────────────────────────────────────────────────────
# Network Policy (#135)
# ─────────────────────────────────────────────────────────────────────────────────
+43
View File
@@ -498,6 +498,42 @@ impl SnapshotsConfig {
}
}
/// Search provider enumeration — selects which backend `web_search` uses.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum SearchProvider {
/// DuckDuckGo HTML scraping with Bing fallback. No API key needed.
#[default]
#[serde(alias = "duckduckgo")]
DuckDuckGo,
/// Tavily AI Search API (https://tavily.com). Requires api_key.
Tavily,
/// Bocha AI Search API (https://bochaai.com). Requires api_key.
Bocha,
}
impl SearchProvider {
#[must_use]
pub fn as_str(self) -> &'static str {
match self {
Self::DuckDuckGo => "duckduckgo",
Self::Tavily => "tavily",
Self::Bocha => "bocha",
}
}
}
/// Web search provider configuration (`[search]` table in config.toml).
#[derive(Debug, Clone, Deserialize, Default)]
pub struct SearchConfig {
/// Search provider: `duckduckgo` | `tavily` | `bocha`. Default: `duckduckgo`.
#[serde(default)]
pub provider: Option<SearchProvider>,
/// API key for Tavily or Bocha. Not required for DuckDuckGo.
#[serde(default)]
pub api_key: Option<String>,
}
/// One configurable footer item.
///
/// Order in the user's `Vec<StatusItem>` is preserved: items in the left
@@ -826,6 +862,12 @@ pub struct Config {
#[serde(default)]
pub snapshots: Option<SnapshotsConfig>,
/// Web search provider configuration. When absent, defaults to DuckDuckGo
/// with Bing fallback. Set `provider` to `tavily` or `bocha` and provide
/// an `api_key` to use those services instead.
#[serde(default)]
pub search: Option<SearchConfig>,
/// User-level memory file (#489). Default behaviour is **opt-in**:
/// loading + injection happens only when `[memory] enabled = true` or
/// `DEEPSEEK_MEMORY=on` is set.
@@ -2543,6 +2585,7 @@ fn merge_config(base: Config, override_cfg: Config) -> Config {
network: override_cfg.network.or(base.network),
skills: override_cfg.skills.or(base.skills),
snapshots: override_cfg.snapshots.or(base.snapshots),
search: override_cfg.search.or(base.search),
memory: override_cfg.memory.or(base.memory),
lsp: override_cfg.lsp.or(base.lsp),
context: ContextConfig {
+10
View File
@@ -154,6 +154,10 @@ pub struct EngineConfig {
pub strict_tool_mode: bool,
/// Workshop / large-tool-output routing (#548). `None` disables routing.
pub workshop: Option<crate::tools::large_output_router::WorkshopConfig>,
/// Which search backend `web_search` should use. Default: DuckDuckGo.
pub search_provider: crate::config::SearchProvider,
/// API key for Tavily or Bocha. `None` for DuckDuckGo.
pub search_api_key: Option<String>,
}
impl Default for EngineConfig {
@@ -188,6 +192,8 @@ impl Default for EngineConfig {
goal_objective: None,
locale_tag: "en".to_string(),
workshop: None,
search_provider: crate::config::SearchProvider::default(),
search_api_key: None,
}
}
}
@@ -1512,6 +1518,10 @@ impl Engine {
ctx = ctx.with_sandbox_backend(std::sync::Arc::clone(backend));
}
// Wire search provider config.
ctx.search_provider = self.config.search_provider;
ctx.search_api_key = self.config.search_api_key.clone();
let policy = sandbox_policy_for_mode(mode, &self.session.workspace);
let mut ctx = ctx.with_elevated_sandbox_policy(policy);
if matches!(mode, AppMode::Plan) {
+6
View File
@@ -4373,6 +4373,12 @@ async fn run_exec_agent(
.tag()
.to_string(),
workshop: config.workshop.clone(),
search_provider: config
.search
.as_ref()
.and_then(|s| s.provider)
.unwrap_or_default(),
search_api_key: config.search.as_ref().and_then(|s| s.api_key.clone()),
};
let engine_handle = spawn_engine(engine_config, config);
+7
View File
@@ -1965,6 +1965,13 @@ impl RuntimeThreadManager {
.tag()
.to_string(),
workshop: self.config.workshop.clone(),
search_provider: self
.config
.search
.as_ref()
.and_then(|s| s.provider)
.unwrap_or_default(),
search_api_key: self.config.search.as_ref().and_then(|s| s.api_key.clone()),
};
let engine = spawn_engine(engine_cfg, &self.config);
+12
View File
@@ -132,6 +132,12 @@ pub struct ToolContext {
/// routing (e.g. in sub-agents and test contexts to avoid recursion).
pub large_output_router: Option<crate::tools::large_output_router::LargeOutputRouter>,
/// Which search backend `web_search` should use. Default: DuckDuckGo with
/// Bing fallback. Set via `[search] provider` in config.toml.
pub search_provider: crate::config::SearchProvider,
/// API key for Tavily or Bocha. `None` for DuckDuckGo.
pub search_api_key: Option<String>,
/// Per-session workshop variable store (#548). Holds the raw content of
/// the most recent large-tool routing event so the parent can call
/// `promote_to_context` later. `None` when the router is disabled.
@@ -168,6 +174,8 @@ impl ToolContext {
memory_path: None,
lsp_manager: None,
large_output_router: None,
search_provider: crate::config::SearchProvider::default(),
search_api_key: None,
workshop_vars: None,
}
}
@@ -202,6 +210,8 @@ impl ToolContext {
memory_path: None,
lsp_manager: None,
large_output_router: None,
search_provider: crate::config::SearchProvider::default(),
search_api_key: None,
workshop_vars: None,
}
}
@@ -236,6 +246,8 @@ impl ToolContext {
memory_path: None,
lsp_manager: None,
large_output_router: None,
search_provider: crate::config::SearchProvider::default(),
search_api_key: None,
workshop_vars: None,
}
}
+331 -4
View File
@@ -1,11 +1,17 @@
//! Web search tool backed by DuckDuckGo HTML results (with Bing fallback).
//! Web search tool backed by multiple providers: DuckDuckGo (HTML scrape
//! with Bing fallback), Tavily API, and Bocha (博查) API.
//!
//! This is the primary web search surface for agents. For browsing workflows
//! (page open, click, screenshot) use a direct URL approach instead.
//!
//! Set `[search]` in config.toml to switch providers:
//! provider = "tavily" # requires api_key
//! api_key = "tvly-..."
use super::spec::{
ApprovalRequirement, ToolCapability, ToolContext, ToolError, ToolResult, ToolSpec, optional_u64,
};
use crate::config::SearchProvider;
use crate::network_policy::{Decision, NetworkPolicyDecider};
use async_trait::async_trait;
use base64::{Engine as _, engine::general_purpose};
@@ -17,6 +23,9 @@ use std::time::Duration;
const DUCKDUCKGO_HOST: &str = "html.duckduckgo.com";
const BING_HOST: &str = "www.bing.com";
const TAVILY_ENDPOINT: &str = "https://api.tavily.com/search";
const BOCHA_ENDPOINT: &str = "https://api.bochaai.com/v1/ai/search";
const ERROR_BODY_PREVIEW_BYTES: usize = 512;
/// Returns `Ok(())` if the policy allows the call, or a `ToolError` otherwise.
/// Falls through silently when no policy is attached (back-compat).
@@ -115,7 +124,7 @@ impl ToolSpec for WebSearchTool {
}
fn description(&self) -> &'static str {
"Search the web (DuckDuckGo or Bing) and return ranked results with URLs and snippets. Use this instead of scraping search engines with `curl` in `exec_shell`. For a known canonical URL, prefer `fetch_url` directly."
"Search the web and return ranked results with URLs and snippets. Default backend is DuckDuckGo (with Bing fallback); set `[search] provider = \"tavily\" | \"bocha\"` in config.toml to switch backends. Use this instead of scraping search engines with `curl` in `exec_shell`. For a known canonical URL, prefer `fetch_url` directly."
}
fn input_schema(&self) -> Value {
@@ -172,6 +181,27 @@ impl ToolSpec for WebSearchTool {
let max_results = max_results.clamp(1, MAX_RESULTS);
let timeout_ms = optional_u64(&input, "timeout_ms", DEFAULT_TIMEOUT_MS).min(60_000);
// Dispatch to the configured search provider.
match context.search_provider {
SearchProvider::Tavily => {
let decider = context.network_policy.as_ref();
check_policy(decider, "api.tavily.com")?;
return self
.run_tavily_search(&query, max_results, timeout_ms, context)
.await;
}
SearchProvider::Bocha => {
let decider = context.network_policy.as_ref();
check_policy(decider, "api.bochaai.com")?;
return self
.run_bocha_search(&query, max_results, timeout_ms, context)
.await;
}
SearchProvider::DuckDuckGo => {
// fall through to existing DuckDuckGo + Bing fallback logic
}
}
// Per-domain network policy gate (#135). The "host" for web search is
// the upstream search engine domain — DuckDuckGo first, Bing on
// fallback. We gate DuckDuckGo here; Bing is gated separately inside
@@ -264,6 +294,235 @@ impl ToolSpec for WebSearchTool {
}
}
impl WebSearchTool {
/// Search via Tavily AI Search API (https://tavily.com).
async fn run_tavily_search(
&self,
query: &str,
max_results: usize,
timeout_ms: u64,
context: &ToolContext,
) -> Result<ToolResult, ToolError> {
let api_key = context
.search_api_key
.as_deref()
.ok_or_else(|| {
ToolError::execution_failed(
"Tavily search requires an API key. Set `[search] api_key = \"tvly-...\"` in config.toml.",
)
})?;
let client = reqwest::Client::builder()
.timeout(Duration::from_millis(timeout_ms))
.build()
.map_err(|e| {
ToolError::execution_failed(format!("Failed to build HTTP client: {e}"))
})?;
let payload = json!({
"api_key": api_key, // noqa: api-key-in-body
"query": query,
"search_depth": "basic",
"max_results": max_results,
});
let resp = client
.post(TAVILY_ENDPOINT)
.header("Content-Type", "application/json")
.json(&payload)
.send()
.await
.map_err(|e| {
ToolError::execution_failed(format!("Tavily search request failed: {e}"))
})?;
let status = resp.status();
let body = resp.text().await.map_err(|e| {
ToolError::execution_failed(format!("Failed to read Tavily response: {e}"))
})?;
if !status.is_success() {
let truncated = truncate_error_body(&body);
return Err(ToolError::execution_failed(format!(
"Tavily search failed: HTTP {} — {truncated}",
status.as_u16()
)));
}
let parsed: serde_json::Value = serde_json::from_str(&body).map_err(|e| {
ToolError::execution_failed(format!("Failed to parse Tavily response: {e}"))
})?;
let results: Vec<WebSearchEntry> = parsed
.get("results")
.and_then(|v| v.as_array())
.into_iter()
.flat_map(|arr| arr.iter())
.filter_map(|item| {
let title = item.get("title")?.as_str()?.to_string();
let url = item.get("url")?.as_str()?.to_string();
let snippet = item
.get("content")
.or_else(|| item.get("snippet"))
.and_then(|s| s.as_str())
.map(|s| s.to_string());
Some(WebSearchEntry {
title,
url,
snippet,
})
})
.take(max_results)
.collect();
let message = if results.is_empty() {
"No results found".to_string()
} else {
format!("Found {} result(s)", results.len())
};
let response = WebSearchResponse {
query: query.to_string(),
source: "tavily".to_string(),
count: results.len(),
message,
results,
};
ToolResult::json(&response).map_err(|e| ToolError::execution_failed(e.to_string()))
}
/// Search via Bocha AI Search API (https://bochaai.com).
async fn run_bocha_search(
&self,
query: &str,
max_results: usize,
timeout_ms: u64,
context: &ToolContext,
) -> Result<ToolResult, ToolError> {
let api_key = context
.search_api_key
.as_deref()
.ok_or_else(|| {
ToolError::execution_failed(
"Bocha search requires an API key. Set `[search] api_key = \"sk-...\"` in config.toml.",
)
})?;
let client = reqwest::Client::builder()
.timeout(Duration::from_millis(timeout_ms))
.build()
.map_err(|e| {
ToolError::execution_failed(format!("Failed to build HTTP client: {e}"))
})?;
let payload = json!({
"query": query,
"freshness": "noLimit",
"count": max_results,
});
let resp = client
.post(BOCHA_ENDPOINT)
.header("Content-Type", "application/json")
.header("Authorization", format!("Bearer {api_key}"))
.json(&payload)
.send()
.await
.map_err(|e| {
ToolError::execution_failed(format!("Bocha search request failed: {e}"))
})?;
let status = resp.status();
let body = resp.text().await.map_err(|e| {
ToolError::execution_failed(format!("Failed to read Bocha response: {e}"))
})?;
if !status.is_success() {
let truncated = truncate_error_body(&body);
return Err(ToolError::execution_failed(format!(
"Bocha search failed: HTTP {} — {truncated}",
status.as_u16()
)));
}
let parsed: serde_json::Value = serde_json::from_str(&body).map_err(|e| {
ToolError::execution_failed(format!("Failed to parse Bocha response: {e}"))
})?;
// Bocha returns `{"code": 200, "data": {"pages": [...]}}`
let results: Vec<WebSearchEntry> = parsed
.get("data")
.and_then(|d| d.get("pages"))
.or_else(|| parsed.get("pages"))
.and_then(|v| v.as_array())
.into_iter()
.flat_map(|arr| arr.iter())
.filter_map(|item| {
let title = item
.get("name")
.or_else(|| item.get("title"))
.and_then(|s| s.as_str())?
.to_string();
let url = item
.get("url")
.or_else(|| item.get("link"))
.and_then(|s| s.as_str())?
.to_string();
let snippet = item
.get("summary")
.or_else(|| item.get("snippet"))
.or_else(|| item.get("description"))
.and_then(|s| s.as_str())
.map(|s| s.to_string());
Some(WebSearchEntry {
title,
url,
snippet,
})
})
.take(max_results)
.collect();
let message = if results.is_empty() {
"No results found".to_string()
} else {
format!("Found {} result(s)", results.len())
};
let response = WebSearchResponse {
query: query.to_string(),
source: "bocha".to_string(),
count: results.len(),
message,
results,
};
ToolResult::json(&response).map_err(|e| ToolError::execution_failed(e.to_string()))
}
}
fn truncate_error_body(body: &str) -> String {
let stripped = sanitize_error_body(body);
if stripped.len() <= ERROR_BODY_PREVIEW_BYTES {
stripped
} else {
let mut end = ERROR_BODY_PREVIEW_BYTES;
while !stripped.is_char_boundary(end) {
end -= 1;
}
format!("{}...", &stripped[..end])
}
}
fn sanitize_error_body(body: &str) -> String {
let stripped = strip_html_tags(body);
stripped
.chars()
.filter(|c| !c.is_control() || c.is_ascii_whitespace())
.collect()
}
fn extract_search_query(input: &Value) -> Result<String, ToolError> {
for key in ["query", "q"] {
if let Some(value) = input.get(key) {
@@ -622,8 +881,9 @@ fn extract_query_param(url: &str, key: &str) -> Option<String> {
#[cfg(test)]
mod tests {
use super::{
WebSearchEntry, decode_html_entities, extract_search_query, is_likely_spam_results,
optional_search_max_results, root_domain,
ERROR_BODY_PREVIEW_BYTES, WebSearchEntry, WebSearchTool, decode_html_entities,
extract_search_query, is_likely_spam_results, optional_search_max_results, root_domain,
sanitize_error_body, truncate_error_body,
};
use serde_json::json;
@@ -868,4 +1128,71 @@ mod tests {
);
}
}
#[test]
fn truncate_error_body_truncates_long_body() {
let body = "a".repeat(ERROR_BODY_PREVIEW_BYTES + 100);
let truncated = truncate_error_body(&body);
assert!(truncated.len() <= ERROR_BODY_PREVIEW_BYTES + 3);
assert!(truncated.ends_with("..."));
}
#[test]
fn truncate_error_body_keeps_short_body_intact() {
let body = "short error";
assert_eq!(truncate_error_body(body), body);
}
#[test]
fn sanitize_error_body_strips_html_and_control_chars() {
let body = "<p>error</p>\x00\x01\x02";
let sanitized = sanitize_error_body(body);
assert_eq!(sanitized, "error");
}
#[tokio::test]
async fn tavily_provider_without_api_key_surfaces_clear_error_not_silent_fallback() {
// Trust-boundary pin: if a user has opted into Tavily but
// forgot the api_key, the tool must NOT silently fall through
// to DuckDuckGo (which would expose the query to a different
// provider than the user authorised). Instead it returns a
// ToolError that names the missing key explicitly.
use crate::config::SearchProvider;
use crate::tools::spec::{ToolContext, ToolSpec};
let tmp = tempfile::tempdir().expect("tempdir");
let mut ctx = ToolContext::new(tmp.path().to_path_buf());
ctx.search_provider = SearchProvider::Tavily;
ctx.search_api_key = None;
let err = WebSearchTool
.execute(json!({"query": "anything"}), &ctx)
.await
.expect_err("missing api_key must surface as ToolError");
let msg = err.to_string();
assert!(
msg.contains("Tavily") && msg.contains("API key"),
"error must name the provider and missing key; got `{msg}`"
);
}
#[tokio::test]
async fn bocha_provider_without_api_key_surfaces_clear_error_not_silent_fallback() {
// Same trust-boundary pin for Bocha.
use crate::config::SearchProvider;
use crate::tools::spec::{ToolContext, ToolSpec};
let tmp = tempfile::tempdir().expect("tempdir");
let mut ctx = ToolContext::new(tmp.path().to_path_buf());
ctx.search_provider = SearchProvider::Bocha;
ctx.search_api_key = None;
let err = WebSearchTool
.execute(json!({"query": "anything"}), &ctx)
.await
.expect_err("missing api_key must surface as ToolError");
let msg = err.to_string();
assert!(
msg.contains("Bocha") && msg.contains("API key"),
"error must name the provider and missing key; got `{msg}`"
);
}
}
+6
View File
@@ -595,6 +595,12 @@ fn build_engine_config(app: &App, config: &Config) -> EngineConfig {
goal_objective: app.goal.goal_objective.clone(),
locale_tag: app.ui_locale.tag().to_string(),
workshop: config.workshop.clone(),
search_provider: config
.search
.as_ref()
.and_then(|s| s.provider)
.unwrap_or_default(),
search_api_key: config.search.as_ref().and_then(|s| s.api_key.clone()),
}
}