fix(core): delegate HTTP message handling to h11 (#365)

This commit is contained in:
WH-2099
2026-07-20 14:32:06 +00:00
committed by GitHub
parent e9e73df46c
commit 13a3c7a8c4
5 changed files with 195 additions and 307 deletions
+1 -1
View File
@@ -6,8 +6,8 @@ authors = [{ name = 'langgenius', email = 'hello@dify.ai' }]
dependencies = [
'Flask>=3.1.3',
'Werkzeug>=3.1.8',
'dpkt>=1.9.8',
'gevent>=26.4.0',
'h11>=0.16.0',
'httpx>=0.28.1',
'pydantic_settings>=2.14.2',
'pydantic>=2.13.4',
+96 -137
View File
@@ -1,167 +1,126 @@
from io import BytesIO
from urllib.parse import quote_from_bytes, unquote_to_bytes, urlsplit
import h11
from werkzeug import Request, Response
from werkzeug.datastructures import Headers
MIN_REQUEST_LINE_PARTS = 2
MIN_STATUS_LINE_PARTS = 2
CONTENT_HEADER_NAMES = frozenset({"CONTENT-TYPE", "CONTENT-LENGTH"})
def serialize_request(request: Request) -> bytes:
method = request.method
path = request.full_path.rstrip("?")
raw = f"{method} {path} HTTP/1.1\r\n".encode()
for name, value in request.headers.items():
raw += f"{name}: {value}\r\n".encode()
raw += b"\r\n"
body = request.get_data(as_text=False)
if body:
raw += body
return raw
def deserialize_request(raw_data: bytes) -> Request:
header_end = raw_data.find(b"\r\n\r\n")
if header_end == -1:
header_end = raw_data.find(b"\n\n")
if header_end == -1:
header_data = raw_data
body = b""
else:
header_data = raw_data[:header_end]
body = raw_data[header_end + 2 :]
else:
header_data = raw_data[:header_end]
body = raw_data[header_end + 4 :]
lines = header_data.split(b"\r\n")
if len(lines) == 1 and b"\n" in lines[0]:
lines = header_data.split(b"\n")
if not lines or not lines[0]:
if not raw_data:
msg = "Empty HTTP request"
raise ValueError(msg)
request_line = lines[0].decode("utf-8", errors="ignore")
parts = request_line.split(" ", 2)
if len(parts) < MIN_REQUEST_LINE_PARTS:
msg = f"Invalid request line: {request_line}"
header_data, separator, body = raw_data.partition(b"\r\n\r\n")
line_separator = b"\r\n"
if not separator:
header_data, separator, body = raw_data.partition(b"\n\n")
line_separator = b"\n"
if not separator:
line_separator = b"\r\n" if b"\r\n" in raw_data else b"\n"
lines = header_data.split(line_separator)
method, space, remainder = lines[0].partition(b" ")
target, version_space, protocol = remainder.rpartition(b" ")
if not space or not version_space:
msg = "Invalid HTTP request line"
raise ValueError(msg)
method = parts[0]
full_path = parts[1]
protocol = parts[2] if len(parts) > MIN_REQUEST_LINE_PARTS else "HTTP/1.1"
if protocol not in {b"HTTP/1.0", b"HTTP/1.1"}:
msg = "Invalid HTTP protocol"
raise ValueError(msg)
full_path = target.decode()
if not full_path.isprintable():
msg = "Invalid HTTP request target"
raise ValueError(msg)
if "?" in full_path:
path, query_string = full_path.split("?", 1)
else:
path = full_path
query_string = ""
normalized_target = quote_from_bytes(
target,
safe="/%:?#[]@!$&'()*+,;=",
).encode()
request_line = b" ".join((method, normalized_target, protocol))
request_head = b"\r\n".join((request_line, *lines[1:], b"", b""))
connection = h11.Connection(h11.SERVER)
try:
connection.receive_data(request_head)
request_event = connection.next_event()
except h11.ProtocolError as exc:
raise ValueError(str(exc)) from exc
if not isinstance(request_event, h11.Request):
msg = "Invalid HTTP request"
raise TypeError(msg)
if any(b"_" in name for name, _ in request_event.headers):
msg = "Invalid HTTP header"
raise ValueError(msg)
headers = Headers()
for line in lines[1:]:
if not line:
continue
line_str = line.decode("utf-8", errors="ignore")
if ":" not in line_str:
continue
name, value = line_str.split(":", 1)
headers.add(name, value.strip())
raw_path, _, query_string = full_path.partition("?")
host = headers.get("Host", "localhost")
if ":" in host:
server_name, server_port = host.rsplit(":", 1)
else:
server_name = host
server_port = "80"
header_values = dict(request_event.headers)
host = header_values.get(b"host", b"localhost").decode()
parsed_host = urlsplit(f"//{host}")
server_name = parsed_host.hostname or host
server_port = str(80 if parsed_host.port is None else parsed_host.port)
environ = {
"REQUEST_METHOD": method,
"PATH_INFO": path,
"REQUEST_METHOD": request_event.method.decode("ascii"),
"PATH_INFO": unquote_to_bytes(raw_path).decode("latin-1"),
"QUERY_STRING": query_string,
"SERVER_NAME": server_name,
"SERVER_PORT": server_port,
"SERVER_PROTOCOL": protocol,
"SERVER_PROTOCOL": f"HTTP/{request_event.http_version.decode('ascii')}",
"wsgi.input": BytesIO(body),
"wsgi.input_terminated": True,
"wsgi.url_scheme": "http",
}
if "Content-Type" in headers:
environ["CONTENT_TYPE"] = headers.get("Content-Type")
content_length = header_values.get(b"content-length")
transfer_encoding = header_values.get(b"transfer-encoding")
if content_length is not None and transfer_encoding is not None:
msg = "Ambiguous HTTP body framing"
raise ValueError(msg)
if content_length is not None and int(content_length) != len(body):
msg = "HTTP body does not match Content-Length"
raise ValueError(msg)
if "Content-Length" in headers:
environ["CONTENT_LENGTH"] = headers.get("Content-Length")
elif body:
environ["CONTENT_LENGTH"] = str(len(body))
for name, value in headers.items():
if name.upper() in CONTENT_HEADER_NAMES:
for name, value in request_event.headers:
if name == b"transfer-encoding":
continue
env_name = f"HTTP_{name.upper().replace('-', '_')}"
environ[env_name] = value
env_name = name.decode("ascii").upper().replace("-", "_")
key = (
env_name
if name in {b"content-type", b"content-length"}
else f"HTTP_{env_name}"
)
if key in environ and name in {b"content-type", b"content-length"}:
msg = f"Duplicate HTTP header: {name.decode('ascii')}"
raise ValueError(msg)
environ[key] = value.decode()
if content_length is None and body:
environ["CONTENT_LENGTH"] = str(len(body))
return Request(environ)
def serialize_response(response: Response) -> bytes:
raw = f"HTTP/1.1 {response.status}\r\n".encode()
for name, value in response.headers.items():
raw += f"{name}: {value}\r\n".encode()
raw += b"\r\n"
body = response.get_data(as_text=False)
if body:
raw += body
return raw
def deserialize_response(raw_data: bytes) -> Response:
header_end = raw_data.find(b"\r\n\r\n")
if header_end == -1:
header_end = raw_data.find(b"\n\n")
if header_end == -1:
header_data = raw_data
body = b""
else:
header_data = raw_data[:header_end]
body = raw_data[header_end + 2 :]
else:
header_data = raw_data[:header_end]
body = raw_data[header_end + 4 :]
lines = header_data.split(b"\r\n")
if len(lines) == 1 and b"\n" in lines[0]:
lines = header_data.split(b"\n")
if not lines or not lines[0]:
msg = "Empty HTTP response"
raise ValueError(msg)
status_line = lines[0].decode("utf-8", errors="ignore")
parts = status_line.split(" ", 2)
if len(parts) < MIN_STATUS_LINE_PARTS:
msg = f"Invalid status line: {status_line}"
raise ValueError(msg)
status_code = int(parts[1])
response = Response(response=body, status=status_code)
for line in lines[1:]:
if not line:
continue
line_str = line.decode("utf-8", errors="ignore")
if ":" not in line_str:
continue
name, value = line_str.split(":", 1)
response.headers[name] = value.strip()
return response
body = response.get_data()
try:
response_event = h11.Response(
status_code=response.status_code,
headers=[
(name.encode("ascii"), value.encode())
for name, value in response.headers
if name.lower() not in {"content-length", "transfer-encoding"}
]
+ [(b"Content-Length", str(len(body)).encode())],
)
connection = h11.Connection(h11.SERVER)
return b"".join(
chunk or b""
for chunk in (
connection.send(response_event),
connection.send(h11.Data(data=body)),
connection.send(h11.EndOfMessage()),
)
)
except h11.ProtocolError as exc:
raise ValueError(str(exc)) from exc
+3 -11
View File
@@ -1,7 +1,6 @@
import pathlib
import pytest
from yarl import URL
from dify_plugin.config.integration_config import IntegrationConfig, find_dify_cli_path
from dify_plugin.core.entities.plugin.request import (
@@ -19,10 +18,8 @@ from dify_plugin.integration.run import PluginRunner
import requests
_MARKETPLACE_API_URL = "https://marketplace.dify.ai"
_OPENAI_PLUGIN_IDENTIFIER = (
"langgenius/openai:1.0.0@"
"c645d52efece517b10435f28da529156a86050dfa28fc4fde8089c038eee7f5a"
_OPENAI_PLUGIN_URL = (
"https://marketplace.dify.ai/api/v1/plugins/langgenius/openai/1.0.0/download"
)
pytestmark = pytest.mark.skipif(
@@ -32,12 +29,7 @@ pytestmark = pytest.mark.skipif(
def test_invoke_llm(openai_mock_server: str) -> None:
url = str(
(URL(_MARKETPLACE_API_URL) / "api/v1/plugins/download").with_query(
unique_identifier=_OPENAI_PLUGIN_IDENTIFIER
)
)
response = requests.get(url, timeout=10)
response = requests.get(_OPENAI_PLUGIN_URL, timeout=10)
response.raise_for_status()
# save the response to a file
+93 -147
View File
@@ -1,163 +1,109 @@
# ruff:file-ignore[ambiguous-unicode-character-string]
import pytest
from werkzeug import Response
import json
from flask import Flask, jsonify, make_response
from flask import request as flask_request
from flask.typing import ResponseReturnValue
from dify_plugin.core.utils.http_parser import (
deserialize_request,
deserialize_response,
serialize_request,
serialize_response,
)
from dify_plugin.core.utils.http_parser import deserialize_request, serialize_response
def test_http_request_roundtrip() -> None:
"""Test all HTTP request attributes are preserved through serialization"""
app = Flask(__name__)
def test_deserialize_request() -> None:
request = deserialize_request(
b"POST /caf\xc3\xa9/a b%3Fc%23d?q[]=1 HTTP/1.1\r\n"
b"Host: example.com:8080\r\n"
b"Authorization: Bearer token\r\n"
b"Content-Type: application/json\r\n"
b"Transfer-Encoding: chunked\r\n"
b"X-Label: caf\xc3\xa9\r\n\r\n"
b'{"id":1}',
)
@app.route("/webhook", methods=["POST"])
def webhook() -> ResponseReturnValue:
original = flask_request
raw = serialize_request(original)
reconstructed = deserialize_request(raw)
assert request.method == "POST"
assert request.path == "/café/a b?c#d"
assert request.query_string == b"q[]=1"
assert request.host == "example.com:8080"
assert request.headers["Authorization"] == "Bearer token"
assert request.headers["X-Label"] == "café"
assert "Transfer-Encoding" not in request.headers
assert request.content_length == 8
assert request.get_json() == {"id": 1}
assert reconstructed.method == original.method
assert reconstructed.path == original.path
assert reconstructed.query_string == original.query_string
assert reconstructed.get_json() == original.get_json()
assert reconstructed.get_data() == original.get_data()
for key in ["Authorization", "X-Webhook-Signature", "User-Agent"]:
if key in original.headers:
assert reconstructed.headers.get(key) == original.headers.get(key)
def test_serialize_response() -> None:
response = Response(
b"\x00\xff", status=201, content_type="application/octet-stream"
)
response.headers["X-Test"] = "café"
for key in original.cookies:
assert reconstructed.cookies.get(key) == original.cookies.get(key)
assert serialize_response(response) == (
b"HTTP/1.1 201 \r\n"
b"Content-Type: application/octet-stream\r\n"
b"X-Test: caf\xc3\xa9\r\n"
b"Content-Length: 2\r\n\r\n"
b"\x00\xff"
)
return jsonify({"status": "ok"})
with app.test_client() as client:
client.set_cookie("session_id", "abc123")
client.set_cookie("user", "john")
def test_ignores_response_reason_phrase() -> None:
response = Response(status="200 Fine\r\nX-Injected: yes")
response = client.post(
"/webhook?param=value&array[]=1&array[]=2",
json={"event": "test.event", "data": {"id": 123, "items": [1, 2, 3]}},
headers={
"Authorization": "Bearer token",
"X-Webhook-Signature": "sha256=signature",
"User-Agent": "TestClient/1.0",
},
raw_response = serialize_response(response)
assert raw_response.startswith(b"HTTP/1.1 200 \r\n")
assert b"X-Injected" not in raw_response
def test_rejects_content_length_mismatch() -> None:
raw_request = (
b"POST / HTTP/1.1\r\nHost: example.com\r\nContent-Length: 1\r\n\r\nbody"
)
with pytest.raises(ValueError, match="Content-Length"):
deserialize_request(raw_request)
def test_rejects_decoded_newline_in_target() -> None:
with pytest.raises(ValueError, match="request line"):
deserialize_request(
b"GET /safe\r\nX-Injected: yes HTTP/1.1\r\nHost: example.com\r\n\r\n",
)
assert response.status_code == 200
def test_http_response_roundtrip() -> None:
"""Test all HTTP response attributes are preserved through serialization"""
app = Flask(__name__)
@app.route("/api/<path:path>")
def api(path: str) -> ResponseReturnValue:
if path == "error":
response = make_response(jsonify({"error": "Not found"}), 404)
response.headers["X-Error"] = "NOTFOUND"
else:
response = make_response(jsonify({"data": {"id": 1, "name": "test"}}), 200)
response.headers["X-Version"] = "v1"
response.headers["Cache-Control"] = "max-age=3600"
response.set_cookie("token", "new-token")
return response
with app.test_client() as client:
response = client.get("/api/data")
raw = serialize_response(response)
reconstructed = deserialize_response(raw)
assert reconstructed.status_code == 200
assert "X-Version" in reconstructed.headers
assert json.loads(reconstructed.get_data())["data"]["id"] == 1
error_response = client.get("/api/error")
raw_error = serialize_response(error_response)
reconstructed_error = deserialize_response(raw_error)
assert reconstructed_error.status_code == 404
assert "X-Error" in reconstructed_error.headers
def test_form_and_binary_data() -> None:
"""Test form data and binary content preservation"""
app = Flask(__name__)
@app.route("/upload", methods=["POST"])
def upload() -> ResponseReturnValue:
raw = serialize_request(flask_request)
reconstructed = deserialize_request(raw)
if flask_request.content_type == "application/x-www-form-urlencoded":
assert reconstructed.form.to_dict() == flask_request.form.to_dict()
else:
assert reconstructed.get_data() == flask_request.get_data()
binary_response = bytes(range(256))
response = make_response(binary_response)
response.headers["Content-Type"] = "application/octet-stream"
return response
with app.test_client() as client:
response = client.post("/upload", data={"field1": "value1", "field2": "value2"})
assert response.status_code == 200
binary_data = b"\x00\x01\x02\xff\xfe\xfd" * 100
response = client.post(
"/upload",
data=binary_data,
headers={"Content-Type": "application/octet-stream"},
def test_rejects_ambiguous_headers() -> None:
with pytest.raises(ValueError, match="header"):
deserialize_request(
b"POST / HTTP/1.1\r\n"
b"Host: example.com\r\n"
b"Content-Type: application/json\r\n"
b"Content_Type: text/plain\r\n"
b"Content-Length: 2\r\n\r\n{}",
)
raw_response = serialize_response(response)
reconstructed = deserialize_response(raw_response)
assert len(reconstructed.get_data()) == 256
def test_special_cases() -> None:
"""Test edge cases and special characters"""
app = Flask(__name__)
@app.route("/test", methods=["GET", "POST"])
def test() -> ResponseReturnValue:
if flask_request.method == "GET":
return "", 204
raw = serialize_request(flask_request)
reconstructed = deserialize_request(raw)
json_data = reconstructed.get_json()
assert json_data == flask_request.get_json()
return jsonify(json_data)
with app.test_client() as client:
response = client.get("/test")
assert response.status_code == 204
raw = serialize_response(response)
reconstructed = deserialize_response(raw)
assert reconstructed.status_code == 204
assert reconstructed.get_data() == b""
response = client.post(
"/test",
json={
"japanese": "こんにちは",
"emoji": "😀🎉",
"special": "café",
"symbols": "α β γ",
},
with pytest.raises(ValueError, match="Duplicate"):
deserialize_request(
b"POST / HTTP/1.1\r\n"
b"Host: example.com\r\n"
b"Content-Type: application/json\r\n"
b"Content-Type: text/plain\r\n"
b"Content-Length: 2\r\n\r\n{}",
)
assert response.status_code == 200
data = response.get_json()
assert data["emoji"] == "😀🎉"
with pytest.raises(ValueError, match="framing"):
deserialize_request(
b"POST / HTTP/1.1\r\n"
b"Host: example.com\r\n"
b"Transfer-Encoding: chunked\r\n"
b"Content-Length: 2\r\n\r\n{}",
)
@pytest.mark.parametrize("newline", ["\r", "\n"])
def test_rejects_newlines_in_header_names(newline: str) -> None:
name = f"X-Test{newline}Injected"
with pytest.raises(ValueError, match="header"):
deserialize_request(
f"GET / HTTP/1.1\r\nHost: example.com\r\n{name}: value\r\n\r\n".encode(),
)
response = Response()
response.headers[name] = "value"
with pytest.raises(ValueError, match="header"):
serialize_response(response)
Generated
+2 -11
View File
@@ -233,9 +233,9 @@ name = "dify-plugin"
version = "0.10.0"
source = { editable = "." }
dependencies = [
{ name = "dpkt" },
{ name = "flask" },
{ name = "gevent" },
{ name = "h11" },
{ name = "httpx" },
{ name = "packaging" },
{ name = "pydantic" },
@@ -261,9 +261,9 @@ dev = [
[package.metadata]
requires-dist = [
{ name = "dpkt", specifier = ">=1.9.8" },
{ name = "flask", specifier = ">=3.1.3" },
{ name = "gevent", specifier = ">=26.4.0" },
{ name = "h11", specifier = ">=0.16.0" },
{ name = "httpx", specifier = ">=0.28.1" },
{ name = "packaging", specifier = ">=26.2" },
{ name = "pydantic", specifier = ">=2.13.4" },
@@ -287,15 +287,6 @@ dev = [
{ name = "ty" },
]
[[package]]
name = "dpkt"
version = "1.9.8"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/c9/7d/52f17a794db52a66e46ebb0c7549bf2f035ed61d5a920ba4aaa127dd038e/dpkt-1.9.8.tar.gz", hash = "sha256:43f8686e455da5052835fd1eda2689d51de3670aac9799b1b00cfd203927ee45", size = 180073, upload-time = "2022-08-18T05:54:13.582Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/11/79/479e2194c9096b92aecdf33634ae948d2be306c6011673e98ee1917f32c2/dpkt-1.9.8-py3-none-any.whl", hash = "sha256:4da4d111d7bf67575b571f5c678c71bddd2d8a01a3d57d489faf0a92c748fbfd", size = 194973, upload-time = "2022-08-18T05:54:10.793Z" },
]
[[package]]
name = "flask"
version = "3.1.3"