Add json() method for HTTP Request and Response classes. (#4612)

* Add `json()` method for HTTP Request and Response classes.

* Raise errors when fetching content to decode as json.

* Update http.py

Co-authored-by: Maximilian Hils <github@maximilianhils.com>

CHANGELOG.md

@@ -68,6 +68,7 @@ If you depend on these features, please raise your voice in
 * New `flow.comment` command to add a comment to the flow. Add `~comment <regex>` filter syntax to search flow comments. (@rbdixon)
 * Fix multipart forms losing `boundary` values on edit (@roytu)
 * `Transfer-Encoding: chunked` HTTP message bodies are now retained if they are below the stream_large_bodies limit.
+* `json()` method for HTTP Request and Response instances will return decoded JSON body. (@rbdixon)
 * --- TODO: add new PRs above this line ---
 * ... and various other fixes, documentation improvements, dependency version bumps, etc.
 

mitmproxy/http.py

@@ -3,6 +3,7 @@ import os
 import re
 import time
 import urllib.parse
+import json
 from dataclasses import dataclass
 from dataclasses import fields
 from email.utils import formatdate
@@ -18,6 +19,7 @@ from typing import Optional
 from typing import Tuple
 from typing import Union
 from typing import cast
+from typing import Any
 
 from mitmproxy import flow
 from mitmproxy.websocket import WebSocketData
@@ -499,6 +501,25 @@ class Message(serializable.Serializable):
         if "content-encoding" not in self.headers:
             raise ValueError("Invalid content encoding {}".format(repr(encoding)))
 
+    def json(self, **kwargs: Any) -> Any:
+        """
+        Returns the JSON encoded content of the response, if any.
+        `**kwargs` are optional arguments that will be
+        passed to `json.loads()`.
+
+        Will raise if the content can not be decoded and then parsed as JSON.
+
+        *Raises:*
+         - `json.decoder.JSONDecodeError` if content is not valid JSON.
+         - `TypeError` if the content is not available, for example because the response
+            has been streamed.
+        """
+        content = self.get_content(strict=False)
+        if content is None:
+            raise TypeError('Message content is not available.')
+        else:
+            return json.loads(content, **kwargs)
+
 
 class Request(Message):
     """

test/mitmproxy/test_http.py

@@ -1,5 +1,6 @@
 import email
 import time
+import json
 from unittest import mock
 
 import pytest
@@ -1144,3 +1145,23 @@ class TestMessageText:
         r.text = '\udcff'
         assert r.headers["content-type"] == "text/html; charset=utf-8"
         assert r.raw_content == b"\xFF"
+
+    def test_get_json(self):
+        req = treq(content=None)
+        with pytest.raises(TypeError):
+            req.json()
+
+        req = treq(content=b'')
+        with pytest.raises(json.decoder.JSONDecodeError):
+            req.json()
+
+        req = treq(content=b'{}')
+        assert req.json() == {}
+
+        req = treq(content=b'{"a": 1}')
+        assert req.json() == {"a": 1}
+
+        req = treq(content=b'{')
+
+        with pytest.raises(json.decoder.JSONDecodeError):
+            req.json()