Merge remote-tracking branch 'origin/main' into gsoc

.github/workflows/main.yml

@@ -66,6 +66,8 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          - os: ubuntu-latest
+            py: '3.10.0-rc - 3.10'
           - os: windows-latest
             py: 3.9
           - os: macos-latest

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Release History
 
+## Unreleased: mitmproxy next
+
+* fix some responses not being decoded properly if the encoding was uppercase #4735 (@Mattwmaster58)
+* Windows: Switch to Python's default asyncio event loop, which increases the number of sockets
+  that can be processed simultaneously.
+
 ## 4 August 2021: mitmproxy 7.0.2
 
 * Fix a WebSocket crash introduced in 7.0.1 (@mhils)

examples/addons/websocket-inject-message.py

@@ -13,9 +13,9 @@ from mitmproxy import ctx, http
 def websocket_message(flow: http.HTTPFlow):
     assert flow.websocket is not None  # make type checker happy
     last_message = flow.websocket.messages[-1]
-    if b"secret" in last_message.content:
+    if last_message.is_text and "secret" in last_message.text:
         last_message.drop()
-        ctx.master.commands.call("inject.websocket", flow, last_message.from_client, "ssssssh")
+        ctx.master.commands.call("inject.websocket", flow, last_message.from_client, "ssssssh".encode())
 
 
 # Complex example: Schedule a periodic timer
@@ -24,7 +24,7 @@ async def inject_async(flow: http.HTTPFlow):
     msg = "hello from mitmproxy! "
     assert flow.websocket is not None  # make type checker happy
     while flow.websocket.timestamp_end is None:
-        ctx.master.commands.call("inject.websocket", flow, True, msg)
+        ctx.master.commands.call("inject.websocket", flow, True, msg.encode())
         await asyncio.sleep(1)
         msg = msg[1:] + msg[:1]
 

examples/contrib/har_dump.py

@@ -154,7 +154,7 @@ def response(flow: mitmproxy.http.HTTPFlow):
         }
 
     if flow.server_conn.connected:
-        entry["serverIPAddress"] = str(flow.server_conn.ip_address[0])
+        entry["serverIPAddress"] = str(flow.server_conn.peername[0])
 
     HAR["log"]["entries"].append(entry)
 

mitmproxy/__init__.py

@@ -1,9 +0,0 @@
-import asyncio
-import sys
-
-if sys.platform == 'win32':
-    # workaround for
-    # https://github.com/tornadoweb/tornado/issues/2751
-    # https://www.tornadoweb.org/en/stable/index.html#installation
-    # (copied multiple times in the codebase, please remove all occurrences)
-    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())

mitmproxy/io/io.py

@@ -46,7 +46,7 @@ class FlowReader:
                 if mdata["type"] not in FLOW_TYPES:
                     raise exceptions.FlowReadException("Unknown flow type: {}".format(mdata["type"]))
                 yield FLOW_TYPES[mdata["type"]].from_state(mdata)
-        except (ValueError, TypeError) as e:
+        except (ValueError, TypeError, IndexError) as e:
             if str(e) == "not a tnetstring: empty file":
                 return  # Error is due to EOF
             raise exceptions.FlowReadException("Invalid data format.")

mitmproxy/net/encoding.py

@@ -52,6 +52,7 @@ def decode(
     """
     if encoded is None:
         return None
+    encoding = encoding.lower()
 
     global _cache
     cached = (
@@ -67,7 +68,7 @@ def decode(
             decoded = custom_decode[encoding](encoded)
         except KeyError:
             decoded = codecs.decode(encoded, encoding, errors)  # type: ignore
-        if encoding in ("gzip", "deflate", "br", "zstd"):
+        if encoding in ("gzip", "deflate", "deflateraw", "br", "zstd"):
             _cache = CachedDecode(encoded, encoding, errors, decoded)
         return decoded
     except TypeError:
@@ -108,6 +109,7 @@ def encode(decoded: Union[None, str, bytes], encoding, errors='strict') -> Union
     """
     if decoded is None:
         return None
+    encoding = encoding.lower()
 
     global _cache
     cached = (
@@ -123,7 +125,7 @@ def encode(decoded: Union[None, str, bytes], encoding, errors='strict') -> Union
             encoded = custom_encode[encoding](decoded)
         except KeyError:
             encoded = codecs.encode(decoded, encoding, errors)  # type: ignore
-        if encoding in ("gzip", "deflate", "br", "zstd"):
+        if encoding in ("gzip", "deflate", "deflateraw", "br", "zstd"):
             _cache = CachedDecode(encoded, encoding, errors, decoded)
         return encoded
     except TypeError:
@@ -216,7 +218,7 @@ custom_decode = {
     "identity": identity,
     "gzip": decode_gzip,
     "deflate": decode_deflate,
-    "deflateRaw": decode_deflate,
+    "deflateraw": decode_deflate,
     "br": decode_brotli,
     "zstd": decode_zstd,
 }
@@ -225,7 +227,7 @@ custom_encode = {
     "identity": identity,
     "gzip": encode_gzip,
     "deflate": encode_deflate,
-    "deflateRaw": encode_deflate,
+    "deflateraw": encode_deflate,
     "br": encode_brotli,
     "zstd": encode_zstd,
 }

mitmproxy/net/tls.py

@@ -289,7 +289,7 @@ class ClientHello:
         )
 
     @property
-    def cipher_suites(self):
+    def cipher_suites(self) -> List[int]:
         return self._client_hello.cipher_suites.cipher_suites
 
     @property

mitmproxy/proxy/events.py

@@ -72,9 +72,11 @@ class CommandCompleted(Event):
         return super().__new__(cls)
 
     def __init_subclass__(cls, **kwargs):
-        command_cls = cls.__annotations__["command"]
+        command_cls = cls.__annotations__.get("command", None)
         valid_command_subclass = (
-                issubclass(command_cls, commands.Command) and command_cls is not commands.Command
+                isinstance(command_cls, type)
+                and issubclass(command_cls, commands.Command)
+                and command_cls is not commands.Command
         )
         if not valid_command_subclass:
             warnings.warn(f"{command_cls} needs a properly annotated command attribute.", RuntimeWarning)

mitmproxy/proxy/layers/tls.py

@@ -100,7 +100,14 @@ HTTP_ALPNS = (b"h2",) + HTTP1_ALPNS
 @dataclass
 class ClientHelloData:
     context: context.Context
+    """The context object for this connection."""
+    client_hello: net_tls.ClientHello
+    """The entire parsed TLS ClientHello."""
     establish_server_tls_first: bool = False
+    """
+    If set to `True`, pause this handshake and establish TLS with an upstream server first.
+    This makes it possible to process the server certificate when generating an interception certificate.
+    """
 
 
 @dataclass
@@ -386,7 +393,7 @@ class ClientTLSLayer(_TLSLayer):
 
         self.conn.sni = client_hello.sni
         self.conn.alpn_offers = client_hello.alpn_protocols
-        tls_clienthello = ClientHelloData(self.context)
+        tls_clienthello = ClientHelloData(self.context, client_hello)
         yield TlsClienthelloHook(tls_clienthello)
 
         if tls_clienthello.establish_server_tls_first and not self.context.server.tls_established:

setup.py

@@ -100,7 +100,7 @@ setup(
             "hypothesis>=5.8,<7",
             "parver>=0.1,<2.0",
             "pdoc>=4.0.0",
-            "pyinstaller==4.5",
+            "pyinstaller==4.5.1",
             "pytest-asyncio>=0.10.0,<0.16,!=0.14",
             "pytest-cov>=2.7.1,<3",
             "pytest-timeout>=1.3.3,<2",
@@ -108,7 +108,7 @@ setup(
             "pytest>=6.1.0,<7",
             "requests>=2.9.1,<3",
             "tox>=3.5,<4",
-            "wheel>=0.36.2,<0.37"
+            "wheel>=0.36.2,<0.38"
         ],
     }
 )

test/mitmproxy/addons/test_tlsconfig.py

@@ -93,7 +93,7 @@ class TestTlsConfig:
         ta = tlsconfig.TlsConfig()
         with taddons.context(ta) as tctx:
             ctx = context.Context(connection.Client(("client", 1234), ("127.0.0.1", 8080), 1605699329), tctx.options)
-            ch = tls.ClientHelloData(ctx)
+            ch = tls.ClientHelloData(ctx, None)  # type: ignore
             ta.tls_clienthello(ch)
             assert not ch.establish_server_tls_first
 

test/mitmproxy/io/test_io.py

@@ -11,6 +11,7 @@ from mitmproxy.io import FlowReader, tnetstring
 class TestFlowReader:
     @given(binary())
     @example(b'51:11:12345678901#4:this,8:true!0:~,4:true!0:]4:\\x00,~}')
+    @example(b'0:')
     def test_fuzz(self, data):
         f = io.BytesIO(data)
         reader = FlowReader(f)

test/mitmproxy/net/test_encoding.py

@@ -17,6 +17,7 @@ def test_identity(encoder):
 
 @pytest.mark.parametrize("encoder", [
     'gzip',
+    'GZIP',
     'br',
     'deflate',
     'zstd',

test/mitmproxy/tools/web/test_app.py

@@ -4,7 +4,6 @@ import json
 import json as _json
 import logging
 import re
-import sys
 import typing
 from contextlib import redirect_stdout
 from pathlib import Path
@@ -14,12 +13,6 @@ import pytest
 
 from mitmproxy.http import Headers
 
-if sys.platform == 'win32':
-    # workaround for
-    # https://github.com/tornadoweb/tornado/issues/2751
-    # https://www.tornadoweb.org/en/stable/index.html#installation
-    # (copied multiple times in the codebase, please remove all occurrences)
-    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
 
 import tornado.testing  # noqa
 from tornado import httpclient  # noqa

tox.ini

@@ -31,11 +31,11 @@ commands =
 deps =
   mypy==0.910
   types-certifi==0.1.4
-  types-Flask==1.1.1
-  types-Werkzeug==1.0.2
-  types-requests==2.25.2
-  types-cryptography==3.3.3
-  types-pyOpenSSL==20.0.4
+  types-Flask==1.1.3
+  types-Werkzeug==1.0.5
+  types-requests==2.25.6
+  types-cryptography==3.3.5
+  types-pyOpenSSL==20.0.5
 
 commands =
   mypy {posargs}