fix(agent): align Binding file upload timeouts (#41027)

This commit is contained in:
Ingram Z
2026-08-20 18:59:44 +00:00
committed by GitHub
parent 785ca4b915
commit 0ca99312c1
20 changed files with 154 additions and 21 deletions
+2
View File
@@ -709,6 +709,8 @@ AGENT_BACKEND_BASE_URL=http://localhost:5050
AGENT_BACKEND_API_TOKEN=dify-agent-run-token-for-dev-only
AGENT_BACKEND_STREAM_READ_TIMEOUT_SECONDS=30
AGENT_BACKEND_STREAM_MAX_RECONNECTS=3
# Client deadline for converting a Binding file to a ToolFile through the Agent backend.
AGENT_BACKEND_BINDING_FILE_DOWNLOAD_TIMEOUT_SECONDS=240
# KnowledgeFS (Dataset 2.0)
KNOWLEDGE_FS_ENABLED=false
+13 -2
View File
@@ -8,10 +8,21 @@ from clients.agent_backend.client import AgentBackendRunClient, DifyAgentBackend
from clients.agent_backend.fake_client import FakeAgentBackendRunClient, FakeAgentBackendScenario
def create_agent_backend_client(*, base_url: str, api_token: str | None = None, stream_timeout: float = 30) -> Client:
def create_agent_backend_client(
*,
base_url: str,
api_token: str | None = None,
stream_timeout: float = 30,
binding_file_download_timeout: float = 240,
) -> Client:
api_token = api_token.strip() if api_token else None
headers = {"Authorization": f"Bearer {api_token}"} if api_token else None
return Client(base_url=base_url, stream_timeout=stream_timeout, headers=headers)
return Client(
base_url=base_url,
stream_timeout=stream_timeout,
binding_file_download_timeout=binding_file_download_timeout,
headers=headers,
)
def create_agent_backend_run_client(
@@ -37,6 +37,11 @@ class AgentBackendConfig(BaseSettings):
default=3,
)
AGENT_BACKEND_BINDING_FILE_DOWNLOAD_TIMEOUT_SECONDS: PositiveFloat = Field(
description="Client timeout for converting a Binding file to a ToolFile through the Agent backend.",
default=240,
)
AGENT_SHELL_ENABLED: bool = Field(
description=(
"Inject the Home, Workspace, Sandbox, and Shell runtime layers into Agent runs. "
@@ -476,6 +476,7 @@ def _default_client_factory() -> Client:
return create_agent_backend_client(
base_url=base_url,
api_token=dify_config.AGENT_BACKEND_API_TOKEN,
binding_file_download_timeout=dify_config.AGENT_BACKEND_BINDING_FILE_DOWNLOAD_TIMEOUT_SECONDS,
)
@@ -31,6 +31,7 @@ def test_create_agent_backend_client_forwards_authentication(
client_cls.assert_called_once_with(
base_url="http://agent-backend",
stream_timeout=30,
binding_file_download_timeout=240,
headers=headers,
)
@@ -51,23 +52,33 @@ def test_create_agent_backend_run_client_forwards_stream_read_timeout(create_cli
@pytest.mark.parametrize(
("factory", "module"),
("factory", "module", "extra_kwargs"),
[
(home_snapshot_service.AgentHomeSnapshotService._client, home_snapshot_service),
(workspace_service.AgentWorkspaceService._client, workspace_service),
(agent_app_sandbox_service._default_client_factory, agent_app_sandbox_service),
(home_snapshot_service.AgentHomeSnapshotService._client, home_snapshot_service, {}),
(workspace_service.AgentWorkspaceService._client, workspace_service, {}),
(
agent_app_sandbox_service._default_client_factory,
agent_app_sandbox_service,
{"binding_file_download_timeout": 123.5},
),
],
)
def test_default_agent_backend_clients_forward_authentication(
monkeypatch: pytest.MonkeyPatch,
factory: Callable[[], Client],
module: ModuleType,
extra_kwargs: dict[str, float],
) -> None:
monkeypatch.setattr(dify_config, "AGENT_BACKEND_BASE_URL", "http://agent-backend")
monkeypatch.setattr(dify_config, "AGENT_BACKEND_API_TOKEN", "secret-token")
monkeypatch.setattr(dify_config, "AGENT_BACKEND_BINDING_FILE_DOWNLOAD_TIMEOUT_SECONDS", 123.5)
create_client = MagicMock()
monkeypatch.setattr(module, "create_agent_backend_client", create_client)
factory()
create_client.assert_called_once_with(base_url="http://agent-backend", api_token="secret-token")
create_client.assert_called_once_with(
base_url="http://agent-backend",
api_token="secret-token",
**extra_kwargs,
)
@@ -0,0 +1,41 @@
from __future__ import annotations
from pathlib import Path
import pytest
from pydantic import ValidationError
from configs.extra.agent_backend_config import AgentBackendConfig
_REPOSITORY_ROOT = Path(__file__).resolve().parents[4]
_API_TIMEOUT_ENV = "AGENT_BACKEND_BINDING_FILE_DOWNLOAD_TIMEOUT_SECONDS"
_AGENT_TIMEOUT_ENV = "DIFY_AGENT_BINDING_FILE_DOWNLOAD_COMMAND_TIMEOUT_SECONDS"
def test_binding_file_download_timeout_defaults_to_240_seconds(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.delenv(_API_TIMEOUT_ENV, raising=False)
assert AgentBackendConfig().AGENT_BACKEND_BINDING_FILE_DOWNLOAD_TIMEOUT_SECONDS == 240.0
def test_binding_file_download_timeout_rejects_non_positive_values() -> None:
with pytest.raises(ValidationError):
AgentBackendConfig(AGENT_BACKEND_BINDING_FILE_DOWNLOAD_TIMEOUT_SECONDS=0)
def test_binding_file_timeout_docker_settings_use_their_service_env_files() -> None:
root_env_example = (_REPOSITORY_ROOT / "docker/.env.example").read_text(encoding="utf-8")
api_env_example = (_REPOSITORY_ROOT / "docker/envs/core-services/api.env.example").read_text(encoding="utf-8")
agent_env_example = (_REPOSITORY_ROOT / "docker/envs/core-services/dify-agent.env.example").read_text(
encoding="utf-8"
)
compose_template = (_REPOSITORY_ROOT / "docker/docker-compose-template.yaml").read_text(encoding="utf-8")
assert f"{_API_TIMEOUT_ENV}=" not in root_env_example
assert f"{_AGENT_TIMEOUT_ENV}=" not in root_env_example
assert f"{_API_TIMEOUT_ENV}=" in api_env_example
assert f"{_AGENT_TIMEOUT_ENV}=" not in api_env_example
assert f"{_API_TIMEOUT_ENV}=" not in agent_env_example
assert f"{_AGENT_TIMEOUT_ENV}=" in agent_env_example
assert f"{_API_TIMEOUT_ENV}:" not in compose_template
assert f"{_AGENT_TIMEOUT_ENV}:" not in compose_template
@@ -25,7 +25,10 @@ type HTTPClient struct {
var errUploadRequestAborted = errors.New("upload request aborted")
const agentStubAuthorizationExpiredCode = "agent_stub_authorization_expired"
const (
agentStubAuthorizationExpiredCode = "agent_stub_authorization_expired"
defaultUploadRequestTimeout = 180 * time.Second
)
type agentStubHTTPError struct {
statusCode int
@@ -70,7 +73,7 @@ func openUploadSource(path string) (io.ReadCloser, error) {
}
func doUploadRequest(req *http.Request) (*http.Response, error) {
return (&http.Client{Timeout: 120 * time.Second}).Do(req)
return (&http.Client{Timeout: defaultUploadRequestTimeout}).Do(req)
}
// postJSON sends a POST request with JSON body and returns the response body.
@@ -20,6 +20,12 @@ import (
const fifoTestDeadline = 3 * time.Second
func TestDefaultUploadRequestTimeout(t *testing.T) {
if defaultUploadRequestTimeout != 180*time.Second {
t.Fatalf("defaultUploadRequestTimeout = %s, want 180s", defaultUploadRequestTimeout)
}
}
func receiveWithin[T any](ch <-chan T, timeout time.Duration) (T, bool) {
timer := time.NewTimer(timeout)
defer timer.Stop()
+2
View File
@@ -60,6 +60,8 @@ DIFY_AGENT_STUB_API_BASE_URL=http://localhost:5050/agent-stub
DIFY_AGENT_SANDBOX_FILES_BASE_URL=http://localhost:5001
# Maximum Agent Stub upload size in MiB; forwarded to Dify API as a signed byte limit.
DIFY_AGENT_STUB_UPLOAD_FILE_SIZE_LIMIT=50
# Shell command deadline for converting a Binding file to a ToolFile.
DIFY_AGENT_BINDING_FILE_DOWNLOAD_COMMAND_TIMEOUT_SECONDS=210
# Server-wide root secret used to derive Agent Stub JWE keys.
# This is security-sensitive: it derives the JWE encryption key for Agent Stub bearer tokens.
# Replace this development default in production.
@@ -187,6 +187,12 @@ directly from the runtime to Dify's existing ToolFile endpoint. Dify Agent
returns only the canonical ToolFile reference and releases the lease before
Dify API signs a browser URL.
The default Binding-download deadline chain leaves each caller time to receive
and normalize the lower layer's result: the sandbox CLI upload is 180 seconds,
`DIFY_AGENT_BINDING_FILE_DOWNLOAD_COMMAND_TIMEOUT_SECONDS` is 210 seconds,
Dify API's `AGENT_BACKEND_BINDING_FILE_DOWNLOAD_TIMEOUT_SECONDS` is 240 seconds,
and `DIFY_AGENT_E2B_ACTIVE_TIMEOUT_SECONDS` is 3600 seconds.
`RuntimeLayout.home_dir` and `RuntimeLayout.workspace_dir` are canonical paths
inside the backend execution namespace. They are not host paths, product ids,
or request configuration. Shell commands start in `workspace_dir`, and `HOME`
@@ -36,6 +36,7 @@ also reads `.env` and `dify-agent/.env` when present.
| `DIFY_AGENT_SHUTDOWN_GRACE_SECONDS` | `30` | Seconds to wait for active local runs during graceful shutdown before cancellation. |
| `DIFY_AGENT_RUN_RETENTION_SECONDS` | `259200` | Seconds to retain Redis run records and per-run event streams; defaults to 3 days. |
| `DIFY_AGENT_RUN_TIMEOUT_SECONDS` | `3600` | Wall-clock deadline in seconds for the Pydantic AI `agent.run(...)` model/tool loop. Deadline failures use `agent_run_limit_exceeded`. Its default intentionally matches `DIFY_AGENT_E2B_ACTIVE_TIMEOUT_SECONDS`, but the settings are independently configurable. |
| `DIFY_AGENT_BINDING_FILE_DOWNLOAD_COMMAND_TIMEOUT_SECONDS` | `210` | Shell command deadline for running the sandbox `dify-agent file upload --no-download-link` conversion. Keep it above the CLI's 180-second upload deadline. |
| `DIFY_AGENT_API_TOKEN` | empty | Optional Bearer token required by private run, Execution Binding, Home Snapshot, and Binding file control-plane routes. Must match Dify API `AGENT_BACKEND_API_TOKEN`. |
| `DIFY_AGENT_PLUGIN_DAEMON_URL` | `http://localhost:5002` | Base URL for the Dify plugin daemon. |
| `DIFY_AGENT_PLUGIN_DAEMON_API_KEY` | empty | API key sent to the Dify plugin daemon. |
+5 -3
View File
@@ -52,7 +52,6 @@ from dify_agent.protocol import (
_ResponseModelT = TypeVar("_ResponseModelT", bound=BaseModel)
_TERMINAL_EVENT_TYPES = {"run_succeeded", "run_failed", "run_cancelled"}
_TERMINAL_RUN_STATUSES = {"succeeded", "failed", "cancelled"}
_BINDING_FILE_DOWNLOAD_TIMEOUT_SECONDS = 90.0
_function_tool_result_payload_key_cache: str | None = None
@@ -271,6 +270,7 @@ class Client:
_base_url: str
_timeout: float | httpx.Timeout
_stream_timeout: float | httpx.Timeout | None
_binding_file_download_timeout: float | httpx.Timeout
_headers: dict[str, str]
_sync_http_client: httpx.Client | None
_async_http_client: httpx.AsyncClient | None
@@ -285,6 +285,7 @@ class Client:
base_url: str,
timeout: float | httpx.Timeout = 30.0,
stream_timeout: float | httpx.Timeout | None = 30.0,
binding_file_download_timeout: float | httpx.Timeout = 240.0,
headers: dict[str, str] | None = None,
sync_http_client: httpx.Client | None = None,
async_http_client: httpx.AsyncClient | None = None,
@@ -292,6 +293,7 @@ class Client:
self._base_url = base_url.rstrip("/")
self._timeout = timeout
self._stream_timeout = stream_timeout
self._binding_file_download_timeout = binding_file_download_timeout
self._headers = dict(headers or {})
self._sync_http_client = sync_http_client
self._async_http_client = async_http_client
@@ -549,7 +551,7 @@ class Client:
"download_binding_file",
"/execution-bindings/files/download",
request,
timeout=_BINDING_FILE_DOWNLOAD_TIMEOUT_SECONDS,
timeout=self._binding_file_download_timeout,
)
return _parse_model_response(response, BindingFileDownloadResponse)
@@ -558,7 +560,7 @@ class Client:
"download_binding_file_sync",
"/execution-bindings/files/download",
request,
timeout=_BINDING_FILE_DOWNLOAD_TIMEOUT_SECONDS,
timeout=self._binding_file_download_timeout,
)
return _parse_model_response(response, BindingFileDownloadResponse)
+1
View File
@@ -76,6 +76,7 @@ def create_app(settings: ServerSettings | None = None) -> FastAPI:
execution_bindings=runtime_backend_profile.execution_bindings,
agent_stub_api_base_url=resolved_settings.agent_stub_api_base_url,
agent_stub_token_factory=agent_stub_token_factory,
download_command_timeout_seconds=resolved_settings.binding_file_download_command_timeout_seconds,
)
if runtime_backend_profile is not None
else None
@@ -39,7 +39,6 @@ logger = logging.getLogger(__name__)
_LIST_MAX_ENTRIES = 1000
_BROWSE_TIMEOUT_SECONDS = 60.0
_BROWSE_OUTPUT_MAX_BYTES = 1024 * 1024
_DOWNLOAD_TIMEOUT_SECONDS = 60.0
_DOWNLOAD_OUTPUT_MAX_BYTES = 32 * 1024
_PAYLOAD_BEGIN = "<<<DIFY_BINDING_FILE_BEGIN>>>"
_PAYLOAD_END = "<<<DIFY_BINDING_FILE_END>>>"
@@ -149,6 +148,7 @@ class BindingFileService:
execution_bindings: ExecutionBindingBackend
agent_stub_api_base_url: str | None
agent_stub_token_factory: ShellAgentStubTokenFactory | None
download_command_timeout_seconds: float
async def list_files(self, request: BindingFileListRequest) -> BindingFileListResponse:
try:
@@ -243,7 +243,7 @@ class BindingFileService:
f"dify-agent file upload --no-download-link {shlex.quote(resolved_path)}",
cwd=lease.layout.workspace_dir,
env=env,
timeout=_DOWNLOAD_TIMEOUT_SECONDS,
timeout=self.download_command_timeout_seconds,
max_output_bytes=_DOWNLOAD_OUTPUT_MAX_BYTES,
mode="stdio",
)
@@ -82,6 +82,7 @@ class ServerSettings(BaseSettings):
description="Maximum Agent Stub upload size in MiB",
validation_alias="DIFY_AGENT_STUB_UPLOAD_FILE_SIZE_LIMIT",
)
binding_file_download_command_timeout_seconds: float = Field(default=210.0, gt=0)
server_secret_key: str | None = None
api_token: str | None = None
shell_redact_patterns: str = ""
@@ -98,9 +98,9 @@ def _binding_file_download_request(path: str = "report.txt") -> BindingFileDownl
)
def _assert_binding_download_timeout(request: httpx.Request) -> None:
def _assert_binding_download_timeout(request: httpx.Request, expected: float = 240.0) -> None:
timeout = cast(dict[str, float], request.extensions["timeout"])
assert timeout == {"connect": 90.0, "read": 90.0, "write": 90.0, "pool": 90.0}
assert timeout == {"connect": expected, "read": expected, "write": expected, "pool": expected}
def _function_tool_result_payload(key: str) -> dict[str, object]:
@@ -398,13 +398,17 @@ def test_async_binding_file_methods_post_dtos_and_parse_responses() -> None:
200, json={"path": "note.txt", "size": 5, "truncated": False, "binary": False, "text": "hello"}
)
if request.url.path == "/execution-bindings/files/download":
_assert_binding_download_timeout(request)
_assert_binding_download_timeout(request, expected=123.5)
return httpx.Response(200, json={"reference": "dify-file-ref:file-1"})
raise AssertionError(f"unexpected request: {request.method} {request.url}")
async def scenario() -> None:
http_client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
client = Client(base_url="http://testserver", async_http_client=http_client)
client = Client(
base_url="http://testserver",
binding_file_download_timeout=123.5,
async_http_client=http_client,
)
listing = await client.list_binding_files("binding-ref", ".")
preview = await client.read_binding_file("binding-ref", "note.txt")
@@ -179,12 +179,18 @@ def _context() -> DifyExecutionContextLayerConfig:
)
def _service(commands: _Commands, *, configured: bool = True) -> tuple[BindingFileService, _Backend]:
def _service(
commands: _Commands,
*,
configured: bool = True,
download_command_timeout_seconds: float = 210.0,
) -> tuple[BindingFileService, _Backend]:
backend = _Backend(lease=cast(RuntimeLease, _Lease(commands=commands)))
service = BindingFileService(
execution_bindings=backend, # pyright: ignore[reportArgumentType]
agent_stub_api_base_url="http://stub/agent-stub" if configured else None,
agent_stub_token_factory=(lambda execution_context, *, session_id: "secret-jwe") if configured else None,
download_command_timeout_seconds=download_command_timeout_seconds,
)
return service, backend
@@ -208,6 +214,7 @@ def _local_service(tmp_path: Path) -> tuple[BindingFileService, _Backend, _Local
execution_bindings=backend, # pyright: ignore[reportArgumentType]
agent_stub_api_base_url=None,
agent_stub_token_factory=None,
download_command_timeout_seconds=210.0,
)
return service, backend, commands, workspace, home
@@ -415,7 +422,7 @@ async def test_download_shell_quotes_resolved_path_and_returns_only_reference_in
commands = _Commands(
outputs=[json.dumps({"transfer_method": "tool_file", "reference": _REFERENCE, "public_download_url": "bad"})]
)
service, backend = _service(commands)
service, backend = _service(commands, download_command_timeout_seconds=123.5)
issued_tokens: list[tuple[DifyExecutionContextLayerConfig, str | None]] = []
def issue_token(execution_context: DifyExecutionContextLayerConfig, *, session_id: str | None) -> str:
@@ -448,7 +455,7 @@ async def test_download_shell_quotes_resolved_path_and_returns_only_reference_in
"DIFY_AGENT_STUB_API_BASE_URL": "http://stub/agent-stub",
"DIFY_AGENT_STUB_AUTH_JWE": "secret-jwe",
}
assert timeout == pytest.approx(60.0, rel=0, abs=0.01)
assert timeout == pytest.approx(123.5, rel=0, abs=0.01)
assert issued_tokens == [(context, None)]
assert issued_tokens[0][0].model_dump() == context.model_dump()
assert backend.releases == 1
@@ -610,6 +617,7 @@ async def test_download_maps_binding_acquire_errors(
execution_bindings=backend, # pyright: ignore[reportArgumentType]
agent_stub_api_base_url="http://stub/agent-stub",
agent_stub_token_factory=lambda execution_context, *, session_id: "secret-jwe",
download_command_timeout_seconds=210.0,
)
with pytest.raises(BindingFileError) as exc_info:
@@ -79,6 +79,31 @@ def test_server_settings_rejects_non_positive_run_timeout() -> None:
_ = ServerSettings(run_timeout_seconds=0)
def test_server_settings_reads_binding_file_download_command_timeout_from_env(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("DIFY_AGENT_BINDING_FILE_DOWNLOAD_COMMAND_TIMEOUT_SECONDS", "123.5")
settings = ServerSettings()
assert settings.binding_file_download_command_timeout_seconds == 123.5
def test_server_settings_defaults_binding_file_download_command_timeout_to_210_seconds(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
monkeypatch.delenv("DIFY_AGENT_BINDING_FILE_DOWNLOAD_COMMAND_TIMEOUT_SECONDS", raising=False)
monkeypatch.chdir(tmp_path)
assert ServerSettings().binding_file_download_command_timeout_seconds == 210.0
def test_server_settings_rejects_non_positive_binding_file_download_command_timeout() -> None:
with pytest.raises(ValidationError, match="greater than 0"):
_ = ServerSettings(binding_file_download_command_timeout_seconds=0)
def test_server_settings_defaults_shellctl_auth_token_to_none(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
@@ -11,6 +11,8 @@ PLUGIN_REMOTE_INSTALL_PORT=5003
PLUGIN_MAX_PACKAGE_SIZE=52428800
PLUGIN_DAEMON_TIMEOUT=600.0
INNER_API_KEY_FOR_PLUGIN=QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1
# Client deadline for converting a Binding file to a ToolFile through the Agent backend.
AGENT_BACKEND_BINDING_FILE_DOWNLOAD_TIMEOUT_SECONDS=240
KNOWLEDGE_FS_ENABLED=${KNOWLEDGE_FS_ENABLED:-false}
KNOWLEDGE_FS_BASE_URL=
KNOWLEDGE_FS_JWT_SECRET=
@@ -13,6 +13,7 @@ DIFY_AGENT_SHUTDOWN_GRACE_SECONDS=30
DIFY_AGENT_RUN_RETENTION_SECONDS=259200
# Pydantic AI run deadline; its default matches the independently configurable E2B active timeout.
DIFY_AGENT_RUN_TIMEOUT_SECONDS=3600
DIFY_AGENT_BINDING_FILE_DOWNLOAD_COMMAND_TIMEOUT_SECONDS=210
# Leave empty to derive from PLUGIN_DAEMON_URL and PLUGIN_DAEMON_KEY in Docker Compose.
DIFY_AGENT_PLUGIN_DAEMON_URL=