From 69f455b962f95b48f723004bc7a697beac0e91eb Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 28 Oct 2024 22:04:39 +0100 Subject: [PATCH] Add `tun` proxy mode on Linux (#7278) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * simplify stream handling callback * add `tun` proxy mode from mitmproxy_rs * tun mode: tests++ * [autofix.ci] apply automated fixes * bump mitmproxy_rs * fix bugs * ci: use macOS 13 for builds as 12 is being phased out * test debugging * bump mitmproxy_rs * bump python version in ci, 3.13 is stable now * nits * is unshare to blame? * how about this? * coverage++ * [autofix.ci] apply automated fixes * debüg * debüüg * debüüüg * bump mitmproxy_rs --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> --- .github/workflows/main.yml | 8 +-- CHANGELOG.md | 2 + mitmproxy/proxy/mode_servers.py | 69 ++++++++++++++----- mitmproxy/proxy/mode_specs.py | 21 ++++++ pyproject.toml | 2 +- test/conftest.py | 5 ++ test/mitmproxy/proxy/test_mode_servers.py | 70 +++++++++++++++++--- test/mitmproxy/proxy/test_mode_specs.py | 2 + web/gen/state_js.py | 4 ++ web/src/js/__tests__/ducks/_tbackendstate.ts | 9 +++ web/src/js/ducks/backendState.ts | 1 + 11 files changed, 164 insertions(+), 29 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index de1d8704b..b68d4db9f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,11 +44,11 @@ jobs: matrix: include: - os: ubuntu-latest - py: "3.13-dev" + py: "3.13" - os: windows-latest - py: "3.13-dev" + py: "3.13" - os: macos-latest - py: "3.13-dev" + py: "3.13" - os: ubuntu-latest py: "3.12" - os: ubuntu-latest @@ -100,7 +100,7 @@ jobs: include: - image: macos-14 platform: macos-arm64 - - image: macos-12 + - image: macos-13 platform: macos-x86_64 - image: windows-2019 platform: windows diff --git a/CHANGELOG.md b/CHANGELOG.md index 7898e7cc1..978ac5b46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ ([#7242](https://github.com/mitmproxy/mitmproxy/pull/7242), @mhils) - Tighten HTTP detection heuristic to better support custom TCP-based protocols. ([#7228](https://github.com/mitmproxy/mitmproxy/pull/7228), @fatanugraha) +- Add a `tun` proxy mode that creates a virtual network device on Linux for transparent proxying. + ([#7278](https://github.com/mitmproxy/mitmproxy/pull/7278), @mhils) - Fix a bug where mitmproxy would incorrectly report that TLS 1.0 and 1.1 are not supported with the current OpenSSL build. ([#7241](https://github.com/mitmproxy/mitmproxy/pull/7241), @mhils) diff --git a/mitmproxy/proxy/mode_servers.py b/mitmproxy/proxy/mode_servers.py index fb6ab66df..12320bc32 100644 --- a/mitmproxy/proxy/mode_servers.py +++ b/mitmproxy/proxy/mode_servers.py @@ -184,8 +184,11 @@ class ServerInstance(Generic[M], metaclass=ABCMeta): async def handle_stream( self, reader: asyncio.StreamReader | mitmproxy_rs.Stream, - writer: asyncio.StreamWriter | mitmproxy_rs.Stream, + writer: asyncio.StreamWriter | mitmproxy_rs.Stream | None = None, ) -> None: + if writer is None: + assert isinstance(reader, mitmproxy_rs.Stream) + writer = reader handler = ProxyConnectionHandler( ctx.master, reader, writer, ctx.options, self.mode ) @@ -204,7 +207,8 @@ class ServerInstance(Generic[M], metaclass=ABCMeta): handler.layer.context.client.sockname = original_dst handler.layer.context.server.address = original_dst elif isinstance( - self.mode, (mode_specs.WireGuardMode, mode_specs.LocalMode) + self.mode, + (mode_specs.WireGuardMode, mode_specs.LocalMode, mode_specs.TunMode), ): # pragma: no cover on platforms without wg-test-client handler.layer.context.server.address = writer.get_extra_info( "remote_endpoint", handler.layer.context.client.sockname @@ -213,9 +217,6 @@ class ServerInstance(Generic[M], metaclass=ABCMeta): with self.manager.register_connection(handler.layer.context.client.id, handler): await handler.handle_client() - async def handle_udp_stream(self, stream: mitmproxy_rs.Stream) -> None: - await self.handle_stream(stream, stream) - class AsyncioServerInstance(ServerInstance[M], metaclass=ABCMeta): _servers: list[asyncio.Server | mitmproxy_rs.udp.UdpServer] @@ -308,14 +309,14 @@ class AsyncioServerInstance(ServerInstance[M], metaclass=ABCMeta): ipv4 = await mitmproxy_rs.udp.start_udp_server( "0.0.0.0", port, - self.handle_udp_stream, + self.handle_stream, ) servers.append(ipv4) try: ipv6 = await mitmproxy_rs.udp.start_udp_server( "[::]", ipv4.getsockname()[1], - self.handle_udp_stream, + self.handle_stream, ) servers.append(ipv6) # pragma: no cover except Exception: # pragma: no cover @@ -325,7 +326,7 @@ class AsyncioServerInstance(ServerInstance[M], metaclass=ABCMeta): await mitmproxy_rs.udp.start_udp_server( host, port, - self.handle_udp_stream, + self.handle_stream, ) ) @@ -392,8 +393,8 @@ class WireGuardServerInstance(ServerInstance[mode_specs.WireGuardMode]): port, self.server_key, [p], - self.wg_handle_stream, - self.wg_handle_stream, + self.handle_stream, + self.handle_stream, ) conf = self.client_conf() @@ -434,11 +435,6 @@ class WireGuardServerInstance(ServerInstance[mode_specs.WireGuardMode]): finally: self._server = None - async def wg_handle_stream( - self, stream: mitmproxy_rs.Stream - ) -> None: # pragma: no cover on platforms without wg-test-client - await self.handle_stream(stream, stream) - class LocalRedirectorInstance(ServerInstance[mode_specs.LocalMode]): _server: ClassVar[mitmproxy_rs.local.LocalRedirector | None] = None @@ -460,7 +456,7 @@ class LocalRedirectorInstance(ServerInstance[mode_specs.LocalMode]): stream: mitmproxy_rs.Stream, ) -> None: if cls._instance is not None: - await cls._instance.handle_stream(stream, stream) + await cls._instance.handle_stream(stream) async def _start(self) -> None: if self._instance: @@ -523,6 +519,47 @@ class DnsInstance(AsyncioServerInstance[mode_specs.DnsMode]): return layers.DNSLayer(context) +class TunInstance(ServerInstance[mode_specs.TunMode]): + _server: mitmproxy_rs.tun.TunInterface | None = None + listen_addrs = () + + def make_top_layer( + self, context: Context + ) -> Layer: # pragma: no cover mocked in tests + return layers.modes.TransparentProxy(context) + + @property + def is_running(self) -> bool: + return self._server is not None + + @property + def tun_name(self) -> str | None: + if self._server: + return self._server.tun_name() + else: + return None + + def to_json(self) -> dict: + return {"tun_name": self.tun_name, **super().to_json()} + + async def _start(self) -> None: + assert self._server is None + self._server = await mitmproxy_rs.tun.create_tun_interface( + self.handle_stream, + self.handle_stream, + tun_name=self.mode.data or None, + ) + logger.info(f"TUN interface created: {self._server.tun_name()}") + + async def _stop(self) -> None: + assert self._server is not None + try: + self._server.close() + await self._server.wait_closed() + finally: + self._server = None + + # class Http3Instance(AsyncioServerInstance[mode_specs.Http3Mode]): # def make_top_layer(self, context: Context) -> Layer: # return layers.modes.HttpProxy(context) diff --git a/mitmproxy/proxy/mode_specs.py b/mitmproxy/proxy/mode_specs.py index cc65aa847..c1fa60fed 100644 --- a/mitmproxy/proxy/mode_specs.py +++ b/mitmproxy/proxy/mode_specs.py @@ -23,6 +23,8 @@ Examples: from __future__ import annotations import dataclasses +import platform +import re import sys from abc import ABCMeta from abc import abstractmethod @@ -302,6 +304,25 @@ class LocalMode(ProxyMode): mitmproxy_rs.local.LocalRedirector.describe_spec(self.data) +class TunMode(ProxyMode): + """A Tun interface.""" + + description = "TUN interface" + default_port = None + transport_protocol = BOTH + + def __post_init__(self) -> None: + invalid_tun_name = self.data and ( + # The Rust side is Linux only for the moment, but eventually we may need this. + platform.system() == "Darwin" and not re.match(r"^utun\d+$", self.data) + ) + if invalid_tun_name: # pragma: no cover + raise ValueError( + f"Invalid tun name: {self.data}. " + f"On macOS, the tun name must be the form utunx where x is a number, such as utun3." + ) + + class OsProxyMode(ProxyMode): # pragma: no cover """Deprecated alias for LocalMode""" diff --git a/pyproject.toml b/pyproject.toml index af30cb0b7..2ec6c4bc1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,7 @@ dependencies = [ "hyperframe>=6.0,<=6.0.1", "kaitaistruct>=0.10,<=0.10", "ldap3>=2.8,<=2.9.1", - "mitmproxy_rs>=0.9.1,<0.10", # relaxed upper bound here: we control this + "mitmproxy_rs>=0.10.7,<0.11", # relaxed upper bound here: we control this "msgpack>=1.0.0,<=1.1.0", "passlib>=1.6.5,<=1.7.4", "protobuf>=5.27.2,<=5.28.2", diff --git a/test/conftest.py b/test/conftest.py index c830a5f7c..4ab6ac6c6 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -2,6 +2,7 @@ from __future__ import annotations import asyncio import os +import platform import socket import sys @@ -15,6 +16,10 @@ skip_not_windows = pytest.mark.skipif( os.name != "nt", reason="Skipping due to not Windows" ) +skip_not_linux = pytest.mark.skipif( + platform.system() != "Linux", reason="Skipping due to not Linux" +) + try: s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) s.bind(("::1", 0)) diff --git a/test/mitmproxy/proxy/test_mode_servers.py b/test/mitmproxy/proxy/test_mode_servers.py index 23e40d8bc..228363d7f 100644 --- a/test/mitmproxy/proxy/test_mode_servers.py +++ b/test/mitmproxy/proxy/test_mode_servers.py @@ -1,5 +1,6 @@ import asyncio import platform +import socket from unittest.mock import AsyncMock from unittest.mock import MagicMock from unittest.mock import Mock @@ -7,11 +8,13 @@ from unittest.mock import Mock import pytest from ...conftest import no_ipv6 +from ...conftest import skip_not_linux import mitmproxy.platform import mitmproxy_rs from mitmproxy.addons.proxyserver import Proxyserver from mitmproxy.proxy.mode_servers import LocalRedirectorInstance from mitmproxy.proxy.mode_servers import ServerInstance +from mitmproxy.proxy.mode_servers import TunInstance from mitmproxy.proxy.mode_servers import WireGuardServerInstance from mitmproxy.proxy.server import ConnectionHandler from mitmproxy.test import taddons @@ -130,17 +133,18 @@ async def test_transparent(failure, monkeypatch, caplog_async): assert await caplog_async.await_log("stopped") +async def _echo_server(self: ConnectionHandler): + t = self.transports[self.client] + data = await t.reader.read(65535) + t.writer.write(data.upper()) + await t.writer.drain() + t.writer.close() + + async def test_wireguard(tdata, monkeypatch, caplog): caplog.set_level("DEBUG") - async def handle_client(self: ConnectionHandler): - t = self.transports[self.client] - data = await t.reader.read(65535) - t.writer.write(data.upper()) - await t.writer.drain() - t.writer.close() - - monkeypatch.setattr(ConnectionHandler, "handle_client", handle_client) + monkeypatch.setattr(ConnectionHandler, "handle_client", _echo_server) system = platform.system() if system == "Linux": @@ -353,6 +357,56 @@ async def test_dns_start_stop(caplog_async, transport_protocol): assert await caplog_async.await_log("stopped") +@skip_not_linux +async def test_tun_mode(monkeypatch, caplog): + monkeypatch.setattr(ConnectionHandler, "handle_client", _echo_server) + + with taddons.context(Proxyserver()): + inst = TunInstance.make(f"tun", MagicMock()) + assert inst.tun_name is None + try: + await inst.start() + except RuntimeError as e: + if "Operation not permitted" in str(e): + return pytest.skip("tun mode test must be run as root") + raise + assert inst.tun_name + assert inst.is_running + assert "tun_name" in inst.to_json() + + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.setsockopt(socket.SOL_SOCKET, socket.SO_BINDTODEVICE, inst.tun_name.encode()) + await asyncio.get_running_loop().sock_connect(s, ("192.0.2.1", 1234)) + reader, writer = await asyncio.open_connection(sock=s) + writer.write(b"hello") + await writer.drain() + assert await reader.readexactly(5) == b"HELLO" + writer.close() + await writer.wait_closed() + await inst.stop() + + +async def test_tun_mode_mocked(monkeypatch): + tun_interface = Mock() + tun_interface.tun_name = lambda: "tun0" + tun_interface.wait_closed = AsyncMock() + create_tun_interface = AsyncMock(return_value=tun_interface) + monkeypatch.setattr(mitmproxy_rs.tun, "create_tun_interface", create_tun_interface) + + inst = TunInstance.make(f"tun", MagicMock()) + assert not inst.is_running + assert inst.tun_name is None + + await inst.start() + assert inst.is_running + assert inst.tun_name == "tun0" + assert inst.to_json()["tun_name"] == "tun0" + + await inst.stop() + assert not inst.is_running + assert inst.tun_name is None + + @pytest.fixture() def patched_local_redirector(monkeypatch): start_local_redirector = AsyncMock(return_value=Mock()) diff --git a/test/mitmproxy/proxy/test_mode_specs.py b/test/mitmproxy/proxy/test_mode_specs.py index 9c7889eb1..50b19cfa2 100644 --- a/test/mitmproxy/proxy/test_mode_specs.py +++ b/test/mitmproxy/proxy/test_mode_specs.py @@ -70,6 +70,8 @@ def test_parse_specific_modes(): assert ProxyMode.parse("wireguard") assert ProxyMode.parse("wireguard:foo.conf").data == "foo.conf" assert ProxyMode.parse("wireguard@51821").listen_port() == 51821 + assert ProxyMode.parse("tun") + assert ProxyMode.parse("tun:utun42") assert ProxyMode.parse("local") diff --git a/web/gen/state_js.py b/web/gen/state_js.py index 187b53ea7..0d3becf7c 100755 --- a/web/gen/state_js.py +++ b/web/gen/state_js.py @@ -31,11 +31,15 @@ async def make() -> str: si2 = ServerInstance.make("reverse:example.com", m.proxyserver) si2.last_exception = RuntimeError("I failed somehow.") si3 = ServerInstance.make("socks5", m.proxyserver) + si4 = ServerInstance.make("tun", m.proxyserver) + si4._server = Mock() + si4._server.tun_name = lambda: "tun0" m.proxyserver.servers._instances.update( { si1.mode: si1, si2.mode: si2, si3.mode: si3, + si4.mode: si4, } ) diff --git a/web/src/js/__tests__/ducks/_tbackendstate.ts b/web/src/js/__tests__/ducks/_tbackendstate.ts index 2cedb31d6..c87e049ba 100644 --- a/web/src/js/__tests__/ducks/_tbackendstate.ts +++ b/web/src/js/__tests__/ducks/_tbackendstate.ts @@ -41,6 +41,15 @@ export function TBackendState(): Required { "last_exception": null, "listen_addrs": [], "type": "socks5" + }, + "tun": { + "description": "TUN interface", + "full_spec": "tun", + "is_running": true, + "last_exception": null, + "listen_addrs": [], + "tun_name": "tun0", + "type": "tun" } }, "version": "1.2.3" diff --git a/web/src/js/ducks/backendState.ts b/web/src/js/ducks/backendState.ts index 01360e181..4ffeff793 100644 --- a/web/src/js/ducks/backendState.ts +++ b/web/src/js/ducks/backendState.ts @@ -16,6 +16,7 @@ export interface ServerInfo { listen_addrs: [string, number][]; type: string; wireguard_conf?: string; + tun_name?: string; } export interface BackendState {