Add --show-ignored-hosts (#6720)

* Add `--show-ignored-hosts`

Maybe a bit counterintuitive, but mitmproxy is very nice even without
the MITM part.  When doing `--ignore-hosts '.*'` it is not possible to
see SNI's, so add new flag to show the raw TCP/UDP streams.

Fixes #6421

* Add tests for `--show-ignored-hosts`

* Changelog for `--show-ignored-hosts`

* [autofix.ci] apply automated fixes

* fixups

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Maximilian Hils <github@maximilianhils.com>

CHANGELOG.md

@@ -7,6 +7,9 @@
 
 ## Unreleased: mitmproxy next
 
+- Add `show_ignored_hosts` option to display ignored flows in the UI.
+  This option is implemented as a temporary workaround and will be removed in the future.
+  ([#6720](https://github.com/mitmproxy/mitmproxy/pull/6720), @NicolaiSoeborg)
 - mitmproxy now supports transparent HTTP/3 proxying.
   ([#7202](https://github.com/mitmproxy/mitmproxy/pull/7202), @errorxyz, @meitinger, @mhils)
 - Fix endless tnetstring parsing in case of very large tnetstring

mitmproxy/addons/next_layer.py

@@ -127,9 +127,9 @@ class NextLayer:
         # 1)  check for --ignore/--allow
         if self._ignore_connection(context, data_client, data_server):
             return (
-                layers.TCPLayer(context, ignore=True)
+                layers.TCPLayer(context, ignore=not ctx.options.show_ignored_hosts)
                 if tcp_based
-                else layers.UDPLayer(context, ignore=True)
+                else layers.UDPLayer(context, ignore=not ctx.options.show_ignored_hosts)
             )
 
         # 2)  Handle proxy modes with well-defined next protocol

mitmproxy/options.py

@@ -21,6 +21,16 @@ class Options(optmanager.OptManager):
             False,
             "Use the Host header to construct URLs for display.",
         )
+        self.add_option(
+            "show_ignored_hosts",
+            bool,
+            False,
+            """
+            Record ignored flows in the UI even if we do not perform TLS interception.
+            This option will keep ignored flows' contents in memory, which can greatly increase memory usage.
+            A future release will fix this issue, record ignored flows by default, and remove this option.
+            """,
+        )
 
         # Proxy options
         self.add_option(

mitmproxy/tools/cmdline.py

@@ -49,6 +49,7 @@ def common_options(parser, opts):
     opts.make_parser(parser, "mode", short="m")
     opts.make_parser(parser, "anticache")
     opts.make_parser(parser, "showhost")
+    opts.make_parser(parser, "show_ignored_hosts")
     opts.make_parser(parser, "rfile", metavar="PATH", short="r")
     opts.make_parser(parser, "scripts", metavar="SCRIPT", short="s")
     opts.make_parser(parser, "stickycookie", metavar="FILTER")

test/mitmproxy/addons/test_next_layer.py

@@ -382,6 +382,27 @@ class TestNextLayer:
             else:
                 assert nl._ignore_connection(ctx, data_client, b"") is result
 
+    def test_show_ignored_hosts(self, monkeypatch):
+        nl = NextLayer()
+
+        with taddons.context(nl) as tctx:
+            m = MagicMock()
+            m.context = Context(
+                Client(peername=("192.168.0.42", 51234), sockname=("0.0.0.0", 8080)),
+                tctx.options,
+            )
+            m.context.layers = [modes.TransparentProxy(m.context)]
+            m.context.server.address = ("example.com", 42)
+            tctx.configure(nl, ignore_hosts=["example.com"])
+
+            # Connection is ignored (not-MITM'ed)
+            assert nl._ignore_connection(m.context, http_get, b"") is True
+            # No flow is being set (i.e. nothing shown in UI)
+            assert nl._next_layer(m.context, http_get, b"").flow is None
+            # ... until `--show-ignored-hosts` is set:
+            tctx.configure(nl, show_ignored_hosts=True)
+            assert nl._next_layer(m.context, http_get, b"").flow is not None
+
     def test_next_layer(self, monkeypatch, caplog):
         caplog.set_level(logging.INFO)
         nl = NextLayer()

web/src/js/ducks/_options_gen.ts

@@ -67,6 +67,7 @@ export interface OptionsState {
     server_replay_refresh: boolean;
     server_replay_reuse: boolean;
     server_replay_use_headers: string[];
+    show_ignored_hosts: boolean;
     showhost: boolean;
     ssl_insecure: boolean;
     ssl_verify_upstream_trusted_ca: string | undefined;
@@ -169,6 +170,7 @@ export const defaultState: OptionsState = {
     server_replay_refresh: true,
     server_replay_reuse: false,
     server_replay_use_headers: [],
+    show_ignored_hosts: false,
     showhost: false,
     ssl_insecure: false,
     ssl_verify_upstream_trusted_ca: undefined,