mirror of
https://github.com/Significant-Gravitas/AutoGPT.git
synced 2026-11-10 04:41:59 -10:00
fix(backend): preserve explicit Responses tool strictness (#16145)
### Why / What / How Flattening Chat Completions function tools for the native Responses API discarded explicit `strict` values. In particular, `strict: false` never reached OpenAI, losing the caller's opt-out. Preserve both explicit booleans and keep missing/null values omitted without rewriting schemas or inputs. This matches [OpenAI's function-calling guide](https://developers.openai.com/api/docs/guides/function-calling#strict-mode): omission lets the provider attempt strict normalization, while explicit false keeps best-effort calling. This is a locally reproduced correctness fix; no production incident was established. ### Changes 🏗️ - Preserve sxl-lucky7's original implementation, helper regressions, both contributor commits, and authorship. - Integrate `dev` at `0dde66336bcab98f705078dc5bf12b91d5615ea4` using normal merges. - Add 13 regressions through `call_provider`, the real OpenAI SDK, and HTTPX MockTransport. Cover false/true/missing/null in sync and flex modes, optional nested schema preservation, built-in tool passthrough, absent tools, malformed names, and unchanged inputs. - Verify newer request contracts remain intact: disabled storage, encrypted reasoning inclusion, the parallel-call flag, token limit, optional temperature, and flex tier. - No configuration, dependency, API route, or database schema changes in the effective PR diff. ### Agents and large language models used - OpenAI Codex with GPT-6 (exact model variant unavailable), as disclosed by the original contributor, for the initial implementation and helper tests. - Codex with gpt-6-astra for takeover, current documentation/SDK verification, integration, transport regressions, and local validation. ### Checklist 📋 #### For code changes: - [x] I have clearly listed my changes in the PR description - [x] I have made a test plan - [x] I have tested my changes according to the test plan: - [x] Baseline regression: six expected failures and 32 passes against committed `dev`'s full converter. That converter is byte-identical at the integrated base. Four failures were real SDK serialized requests losing explicit booleans. - [x] Whole-backend Ruff lint fixes using the pre-commit pin, isort, and Black; shared-library formatting is checked by the unchanged CI lint command. No additional formatting changes. - [x] `poetry run lint --skip-pyright` over backend and shared libraries. - [x] Configured type checks: `poetry run pyright` and `poetry run pyright --project ../autogpt_libs`, both with zero errors/warnings. - [x] `poetry run pytest --noconftest backend/util/openai_responses_test.py backend/util/openai_responses_strict_test.py backend/util/llm/providers_test.py backend/util/llm/providers_responses_test.py -q`: **210 passed**, Python 3.12.3, OpenAI SDK 1.109.1. Tooling limits: lockfile Ruff 0.17.2 enables broader default lint rules than the pre-commit 0.7.2 pin. Unintended rewrites from the initial format attempt were restored; lint fixes use the pre-commit pin. A broader Pyright probe reached vendored browser examples with unavailable Playwright imports; the configured project checks pass. No lint/CI configuration or gates were relaxed. The focused tests disable repository conftests because their autouse server fixture requires the full service stack. They import the real application modules and intercept HTTP transport. No live/paid provider request, model schema enforcement, full database-backed suite, or browser journey was exercised. #### For configuration changes: - [x] `.env.default` is updated or already compatible with my changes - [x] `docker-compose.yml` is updated or already compatible with my changes - [x] I have included a list of my configuration changes in the PR description (under **Changes**) --------- Co-authored-by: GPT-6 (Codex) <[email protected]> Co-authored-by: Nicholas Tindle <[email protected]>
This commit is contained in:
3 files changed
+190
-2
No files matched your search
@@ -0,0 +1,147 @@
|
||||
"""Exercise the native Responses caller through the real SDK's HTTP transport."""
|
||||
|
||||
import json
|
||||
from copy import deepcopy
|
||||
|
||||
import httpx
|
||||
import openai
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from backend.util.llm import providers
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("execution_mode", ["sync", "flex"])
|
||||
@pytest.mark.parametrize(
|
||||
"strict_fields",
|
||||
[{"strict": False}, {"strict": True}, {}, {"strict": None}],
|
||||
ids=["false", "true", "missing", "null"],
|
||||
)
|
||||
async def test_responses_serializes_function_strictness(
|
||||
responses_requests, search_function, execution_mode, strict_fields
|
||||
):
|
||||
function = {**search_function, **strict_fields}
|
||||
tools = [
|
||||
{"type": "function", "function": function},
|
||||
{"type": "web_search_preview"},
|
||||
]
|
||||
original = deepcopy(tools)
|
||||
|
||||
result = await _call_responses(tools, execution_mode)
|
||||
|
||||
assert len(responses_requests) == 1
|
||||
request = responses_requests[0]
|
||||
assert request.method == "POST"
|
||||
assert request.url.path == "/v1/responses"
|
||||
payload = json.loads(request.content)
|
||||
expected_function = {"type": "function", **function}
|
||||
if strict_fields.get("strict") is None:
|
||||
expected_function.pop("strict", None)
|
||||
else:
|
||||
assert payload["tools"][0]["strict"] is strict_fields["strict"]
|
||||
assert payload["tools"] == [expected_function, {"type": "web_search_preview"}]
|
||||
assert tools == original
|
||||
assert payload["store"] is False
|
||||
assert payload["include"] == ["reasoning.encrypted_content"]
|
||||
assert payload["parallel_tool_calls"] is False
|
||||
assert payload["max_output_tokens"] == 32
|
||||
assert "temperature" not in payload
|
||||
if execution_mode == "flex":
|
||||
assert payload["service_tier"] == "flex"
|
||||
else:
|
||||
assert "service_tier" not in payload
|
||||
assert result.content == "ok"
|
||||
assert (result.prompt_tokens, result.completion_tokens) == (2, 1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("tools", [None, []])
|
||||
async def test_responses_omits_absent_tools(responses_requests, tools):
|
||||
await _call_responses(tools)
|
||||
|
||||
assert len(responses_requests) == 1
|
||||
assert "tools" not in json.loads(responses_requests[0].content)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("function", [{}, {"name": ""}, {"name": None}])
|
||||
async def test_responses_rejects_missing_function_name_before_request(
|
||||
responses_requests, function
|
||||
):
|
||||
tools = [{"type": "function", "function": {"strict": False, **function}}]
|
||||
original = deepcopy(tools)
|
||||
|
||||
with pytest.raises(ValueError, match="missing required 'name' field"):
|
||||
await _call_responses(tools)
|
||||
|
||||
assert responses_requests == []
|
||||
assert tools == original
|
||||
|
||||
|
||||
async def _call_responses(tools, execution_mode="sync"):
|
||||
return await providers.call_provider(
|
||||
provider="openai",
|
||||
model="gpt-4.1",
|
||||
api_key="test-not-a-real-key",
|
||||
messages=[{"role": "user", "content": "Search"}],
|
||||
max_tokens=32,
|
||||
tools=tools,
|
||||
parallel_tool_calls=False,
|
||||
execution_mode=execution_mode,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def search_function():
|
||||
return {
|
||||
"name": "search",
|
||||
"description": "Search with optional filters",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string"},
|
||||
"filters": {
|
||||
"type": "object",
|
||||
"properties": {"strict": {"type": "boolean", "default": False}},
|
||||
"additionalProperties": True,
|
||||
},
|
||||
},
|
||||
"required": ["query"],
|
||||
"additionalProperties": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def responses_requests(monkeypatch):
|
||||
requests: list[httpx.Request] = []
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
requests.append(request)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "resp_test",
|
||||
"object": "response",
|
||||
"created_at": 0,
|
||||
"model": "gpt-4.1",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "ok"}],
|
||||
}
|
||||
],
|
||||
"usage": {"input_tokens": 2, "output_tokens": 1, "total_tokens": 3},
|
||||
},
|
||||
)
|
||||
|
||||
async with openai.AsyncOpenAI(
|
||||
api_key="test-not-a-real-key",
|
||||
base_url="https://responses.invalid/v1",
|
||||
http_client=httpx.AsyncClient(transport=httpx.MockTransport(respond)),
|
||||
max_retries=0,
|
||||
) as client:
|
||||
monkeypatch.setattr(providers.openai, "AsyncOpenAI", lambda **kwargs: client)
|
||||
yield requests
|
||||
@@ -11,7 +11,7 @@ def convert_tools_to_responses_format(tools: list[dict] | None) -> list[dict]:
|
||||
"""Convert Chat Completions tool format to Responses API format.
|
||||
|
||||
The Responses API uses internally-tagged polymorphism (flatter structure)
|
||||
and functions are strict by default.
|
||||
and explicit function strictness must be preserved.
|
||||
|
||||
Chat Completions format:
|
||||
{"type": "function", "function": {"name": "...", "parameters": {...}}}
|
||||
@@ -40,8 +40,9 @@ def convert_tools_to_responses_format(tools: list[dict] | None) -> list[dict]:
|
||||
entry: dict[str, Any] = {
|
||||
"type": "function",
|
||||
"name": name,
|
||||
# Note: strict=True is default in Responses API
|
||||
}
|
||||
if func.get("strict") is not None:
|
||||
entry["strict"] = func["strict"]
|
||||
if func.get("description") is not None:
|
||||
entry["description"] = func["description"]
|
||||
if func.get("parameters") is not None:
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Explicit function schema policies must survive API format conversion."""
|
||||
|
||||
from copy import deepcopy
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.util.openai_responses import convert_tools_to_responses_format
|
||||
|
||||
|
||||
@pytest.mark.parametrize("strict", [False, True])
|
||||
def test_preserves_explicit_function_strictness(strict: bool):
|
||||
"""Preserve explicit strictness without changing the source tool schema."""
|
||||
function = {
|
||||
"name": "get_weather",
|
||||
"description": "Get the weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"location": {"type": "string"}},
|
||||
"required": ["location"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
"strict": strict,
|
||||
}
|
||||
tools = [{"type": "function", "function": function}]
|
||||
original = deepcopy(tools)
|
||||
|
||||
assert convert_tools_to_responses_format(tools) == [
|
||||
{"type": "function", **function}
|
||||
]
|
||||
assert tools == original
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fields", [{}, {"strict": None}])
|
||||
def test_leaves_unspecified_strictness_to_provider(fields: dict):
|
||||
"""Omit missing or null strictness so the provider keeps its default."""
|
||||
tools = [{"type": "function", "function": {"name": "simple", **fields}}]
|
||||
|
||||
assert convert_tools_to_responses_format(tools) == [
|
||||
{"type": "function", "name": "simple"}
|
||||
]
|
||||
Reference in new issue
Block a user