From c35316f85aac1160702de7f375f4a4a3c72eaf9e Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 13 Dec 2020 13:51:17 +0100 Subject: [PATCH] remove pathod and pathoc --- CHANGELOG.md | 1 + MANIFEST.in | 1 - examples/pathod/libpathod_pathoc.py | 7 - examples/pathod/test_context.py | 23 - examples/pathod/test_setup.py | 30 - examples/pathod/test_setupall.py | 39 - mitmproxy/version.py | 1 - pathod/__init__.py | 18 - pathod/language/__init__.py | 117 -- pathod/language/actions.py | 129 -- pathod/language/base.py | 540 ------- pathod/language/exceptions.py | 21 - pathod/language/generators.py | 93 -- pathod/language/http.py | 393 ------ pathod/language/http2.py | 306 ---- pathod/language/message.py | 144 -- pathod/language/websockets.py | 255 ---- pathod/language/websockets_frame.py | 274 ---- pathod/language/writer.py | 68 - pathod/log.py | 89 -- pathod/pathoc.py | 585 -------- pathod/pathoc_cmdline.py | 228 --- pathod/pathod.py | 496 ------- pathod/pathod_cmdline.py | 236 ---- pathod/protocols/__init__.py | 7 - pathod/protocols/http.py | 47 - pathod/protocols/http2.py | 429 ------ pathod/protocols/websockets.py | 54 - pathod/test.py | 104 -- pathod/utils.py | 48 - release/cibuild.py | 1 - release/installbuilder/mitmproxy.xml | 2 - release/specs/pathoc | 6 - release/specs/pathod | 6 - setup.cfg | 17 - setup.py | 3 - test/filename_matching.py | 4 +- test/individual_coverage.py | 2 +- test/mitmproxy/fuzzing/.env | 4 - test/mitmproxy/fuzzing/README | 14 - test/mitmproxy/fuzzing/client_patterns | 4 - test/mitmproxy/fuzzing/go_proxy | 29 - test/mitmproxy/fuzzing/reverse_patterns | 9 - test/mitmproxy/fuzzing/straight_stream | 6 - .../fuzzing/straight_stream_patterns | 16 - test/mitmproxy/fuzzing/straight_stream_ssl | 6 - test/mitmproxy/proxy/__init__.py | 0 test/mitmproxy/proxy/modes/test_http_proxy.py | 1 - .../proxy/modes/test_reverse_proxy.py | 1 - .../mitmproxy/proxy/modes/test_socks_proxy.py | 1 - .../proxy/modes/test_transparent_proxy.py | 1 - test/mitmproxy/proxy/protocol/__init__.py | 0 test/mitmproxy/proxy/protocol/test_base.py | 1 - test/mitmproxy/proxy/protocol/test_http.py | 1 - test/mitmproxy/proxy/protocol/test_http1.py | 112 -- test/mitmproxy/proxy/protocol/test_http2.py | 1244 ----------------- test/mitmproxy/proxy/protocol/test_rawtcp.py | 1 - test/mitmproxy/proxy/protocol/test_tls.py | 0 .../proxy/protocol/test_websocket.py | 510 ------- test/mitmproxy/proxy/test_config.py | 28 - test/mitmproxy/proxy/test_root_context.py | 1 - test/mitmproxy/proxy/test_server.py | 1131 --------------- test/mitmproxy/test_connections.py | 30 - test/mitmproxy/test_fuzzing.py | 28 - test/mitmproxy/tservers.py | 387 ----- test/pathod/__init__.py | 0 test/pathod/data/clientcert/.gitignore | 3 - test/pathod/data/clientcert/client.cnf | 5 - test/pathod/data/clientcert/client.pem | 42 - test/pathod/data/clientcert/make | 8 - test/pathod/data/file | 1 - test/pathod/data/request | 1 - test/pathod/data/response | 1 - test/pathod/data/testkey.pem | 68 - test/pathod/language/__init__.py | 0 test/pathod/language/test_actions.py | 134 -- test/pathod/language/test_base.py | 351 ----- test/pathod/language/test_exceptions.py | 1 - test/pathod/language/test_generators.py | 35 - test/pathod/language/test_http.py | 355 ----- test/pathod/language/test_http2.py | 236 ---- test/pathod/language/test_message.py | 1 - test/pathod/language/test_websockets.py | 134 -- test/pathod/language/test_websockets_frame.py | 187 --- test/pathod/language/test_writer.py | 90 -- test/pathod/protocols/__init__.py | 0 test/pathod/protocols/test_http.py | 1 - test/pathod/protocols/test_http2.py | 500 ------- test/pathod/protocols/test_websockets.py | 1 - test/pathod/scripts/generate.sh | 17 - test/pathod/scripts/openssl.cnf | 39 - test/pathod/test_log.py | 25 - test/pathod/test_pathoc.py | 250 ---- test/pathod/test_pathoc_cmdline.py | 60 - test/pathod/test_pathod.py | 264 ---- test/pathod/test_pathod_cmdline.py | 89 -- test/pathod/test_test.py | 34 - test/pathod/test_utils.py | 17 - test/pathod/tservers.py | 150 -- test/release/test_cibuild.py | 2 - tox.ini | 8 +- 101 files changed, 7 insertions(+), 11493 deletions(-) delete mode 100644 examples/pathod/libpathod_pathoc.py delete mode 100644 examples/pathod/test_context.py delete mode 100644 examples/pathod/test_setup.py delete mode 100644 examples/pathod/test_setupall.py delete mode 100644 pathod/__init__.py delete mode 100644 pathod/language/__init__.py delete mode 100644 pathod/language/actions.py delete mode 100644 pathod/language/base.py delete mode 100644 pathod/language/exceptions.py delete mode 100644 pathod/language/generators.py delete mode 100644 pathod/language/http.py delete mode 100644 pathod/language/http2.py delete mode 100644 pathod/language/message.py delete mode 100644 pathod/language/websockets.py delete mode 100644 pathod/language/websockets_frame.py delete mode 100644 pathod/language/writer.py delete mode 100644 pathod/log.py delete mode 100644 pathod/pathoc.py delete mode 100644 pathod/pathoc_cmdline.py delete mode 100644 pathod/pathod.py delete mode 100644 pathod/pathod_cmdline.py delete mode 100644 pathod/protocols/__init__.py delete mode 100644 pathod/protocols/http.py delete mode 100644 pathod/protocols/http2.py delete mode 100644 pathod/protocols/websockets.py delete mode 100644 pathod/test.py delete mode 100644 pathod/utils.py delete mode 100644 release/specs/pathoc delete mode 100644 release/specs/pathod delete mode 100644 test/mitmproxy/fuzzing/.env delete mode 100644 test/mitmproxy/fuzzing/README delete mode 100644 test/mitmproxy/fuzzing/client_patterns delete mode 100644 test/mitmproxy/fuzzing/go_proxy delete mode 100644 test/mitmproxy/fuzzing/reverse_patterns delete mode 100644 test/mitmproxy/fuzzing/straight_stream delete mode 100644 test/mitmproxy/fuzzing/straight_stream_patterns delete mode 100644 test/mitmproxy/fuzzing/straight_stream_ssl delete mode 100644 test/mitmproxy/proxy/__init__.py delete mode 100644 test/mitmproxy/proxy/modes/test_http_proxy.py delete mode 100644 test/mitmproxy/proxy/modes/test_reverse_proxy.py delete mode 100644 test/mitmproxy/proxy/modes/test_socks_proxy.py delete mode 100644 test/mitmproxy/proxy/modes/test_transparent_proxy.py delete mode 100644 test/mitmproxy/proxy/protocol/__init__.py delete mode 100644 test/mitmproxy/proxy/protocol/test_base.py delete mode 100644 test/mitmproxy/proxy/protocol/test_http.py delete mode 100644 test/mitmproxy/proxy/protocol/test_http1.py delete mode 100644 test/mitmproxy/proxy/protocol/test_http2.py delete mode 100644 test/mitmproxy/proxy/protocol/test_rawtcp.py delete mode 100644 test/mitmproxy/proxy/protocol/test_tls.py delete mode 100644 test/mitmproxy/proxy/protocol/test_websocket.py delete mode 100644 test/mitmproxy/proxy/test_config.py delete mode 100644 test/mitmproxy/proxy/test_root_context.py delete mode 100644 test/mitmproxy/proxy/test_server.py delete mode 100644 test/mitmproxy/test_fuzzing.py delete mode 100644 test/mitmproxy/tservers.py delete mode 100644 test/pathod/__init__.py delete mode 100644 test/pathod/data/clientcert/.gitignore delete mode 100644 test/pathod/data/clientcert/client.cnf delete mode 100644 test/pathod/data/clientcert/client.pem delete mode 100644 test/pathod/data/clientcert/make delete mode 100644 test/pathod/data/file delete mode 100644 test/pathod/data/request delete mode 100644 test/pathod/data/response delete mode 100644 test/pathod/data/testkey.pem delete mode 100644 test/pathod/language/__init__.py delete mode 100644 test/pathod/language/test_actions.py delete mode 100644 test/pathod/language/test_base.py delete mode 100644 test/pathod/language/test_exceptions.py delete mode 100644 test/pathod/language/test_generators.py delete mode 100644 test/pathod/language/test_http.py delete mode 100644 test/pathod/language/test_http2.py delete mode 100644 test/pathod/language/test_message.py delete mode 100644 test/pathod/language/test_websockets.py delete mode 100644 test/pathod/language/test_websockets_frame.py delete mode 100644 test/pathod/language/test_writer.py delete mode 100644 test/pathod/protocols/__init__.py delete mode 100644 test/pathod/protocols/test_http.py delete mode 100644 test/pathod/protocols/test_http2.py delete mode 100644 test/pathod/protocols/test_websockets.py delete mode 100644 test/pathod/scripts/generate.sh delete mode 100644 test/pathod/scripts/openssl.cnf delete mode 100644 test/pathod/test_log.py delete mode 100644 test/pathod/test_pathoc.py delete mode 100644 test/pathod/test_pathoc_cmdline.py delete mode 100644 test/pathod/test_pathod.py delete mode 100644 test/pathod/test_pathod_cmdline.py delete mode 100644 test/pathod/test_test.py delete mode 100644 test/pathod/test_utils.py delete mode 100644 test/pathod/tservers.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c20ae067..f2c7f07ac 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ If you depend on these features, please raise your voice in ### Full Changelog +* Remove all deprecated pathod and pathoc tools and modules (@Kriechi) * --- TODO: add new PRs above this line --- * ... and various other fixes, documentation improvements, dependency version bumps, etc. diff --git a/MANIFEST.in b/MANIFEST.in index 404936e84..5e4c2cb29 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,2 @@ graft mitmproxy -graft pathod recursive-exclude * *.pyc *.pyo *.swo *.swp *.map diff --git a/examples/pathod/libpathod_pathoc.py b/examples/pathod/libpathod_pathoc.py deleted file mode 100644 index 4c2745711..000000000 --- a/examples/pathod/libpathod_pathoc.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python -from pathod import pathoc - -p = pathoc.Pathoc(("google.com", 80)) -p.connect() -print(p.request("get:/")) -print(p.request("get:/foo")) diff --git a/examples/pathod/test_context.py b/examples/pathod/test_context.py deleted file mode 100644 index 8db4331f7..000000000 --- a/examples/pathod/test_context.py +++ /dev/null @@ -1,23 +0,0 @@ -import requests -from pathod import test - - -def test_simple(): - """ - Testing the requests module with - a pathod context manager. - """ - # Start pathod in a separate thread - with test.Daemon() as d: - # Get a URL for a pathod spec - url = d.p("200:b@100") - # ... and request it - r = requests.put(url) - - # Check the returned data - assert r.status_code == 200 - assert len(r.content) == 100 - - # Check pathod's internal log - log = d.last_log()["request"] - assert log["method"] == "PUT" diff --git a/examples/pathod/test_setup.py b/examples/pathod/test_setup.py deleted file mode 100644 index 32fcb214f..000000000 --- a/examples/pathod/test_setup.py +++ /dev/null @@ -1,30 +0,0 @@ -import requests -from pathod import test - - -class Test: - """ - Testing the requests module with - a pathod instance started for - each test. - """ - - def setup(self): - self.d = test.Daemon() - - def teardown(self): - self.d.shutdown() - - def test_simple(self): - # Get a URL for a pathod spec - url = self.d.p("200:b@100") - # ... and request it - r = requests.put(url) - - # Check the returned data - assert r.status_code == 200 - assert len(r.content) == 100 - - # Check pathod's internal log - log = self.d.last_log()["request"] - assert log["method"] == "PUT" diff --git a/examples/pathod/test_setupall.py b/examples/pathod/test_setupall.py deleted file mode 100644 index cc0ec2e4a..000000000 --- a/examples/pathod/test_setupall.py +++ /dev/null @@ -1,39 +0,0 @@ -import requests -from pathod import test - - -class Test: - """ - Testing the requests module with - a single pathod instance started - for the test suite. - """ - - @classmethod - def setup_class(cls): - cls.d = test.Daemon() - - @classmethod - def teardown_class(cls): - cls.d.shutdown() - - def setup(self): - # Clear the pathod logs between tests - self.d.clear_log() - - def test_simple(self): - # Get a URL for a pathod spec - url = self.d.p("200:b@100") - # ... and request it - r = requests.put(url) - - # Check the returned data - assert r.status_code == 200 - assert len(r.content) == 100 - - # Check pathod's internal log - log = self.d.last_log()["request"] - assert log["method"] == "PUT" - - def test_two(self): - assert not self.d.log() diff --git a/mitmproxy/version.py b/mitmproxy/version.py index bf78f7454..a559d16ba 100644 --- a/mitmproxy/version.py +++ b/mitmproxy/version.py @@ -3,7 +3,6 @@ import subprocess import sys VERSION = "7.0.0.dev" -PATHOD = "pathod " + VERSION MITMPROXY = "mitmproxy " + VERSION # Serialization format version. This is displayed nowhere, it just needs to be incremented by one diff --git a/pathod/__init__.py b/pathod/__init__.py deleted file mode 100644 index 148e62aca..000000000 --- a/pathod/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -import os -import sys - -import warnings - -warnings.warn( - "pathod and pathoc modules are deprecated, see https://github.com/mitmproxy/mitmproxy/issues/4273", - DeprecationWarning, - stacklevel=2 -) - - -def print_tool_deprecation_message(): - print("####", file=sys.stderr) - print(f"### {os.path.basename(sys.argv[0])} is deprecated and will not be part of future mitmproxy releases!", file=sys.stderr) - print("### See https://github.com/mitmproxy/mitmproxy/issues/4273 for more information.", file=sys.stderr) - print("####", file=sys.stderr) - print("", file=sys.stderr) diff --git a/pathod/language/__init__.py b/pathod/language/__init__.py deleted file mode 100644 index 584e3f803..000000000 --- a/pathod/language/__init__.py +++ /dev/null @@ -1,117 +0,0 @@ -import itertools -import time - -import pyparsing as pp - -from . import http, http2, websockets, writer, exceptions - -from .exceptions import RenderError, FileAccessDenied, ParseException -from .base import Settings - -__all__ = [ - "RenderError", "FileAccessDenied", "ParseException", - "Settings", -] - - -def expand(msg): - times = getattr(msg, "times", None) - if times: - for j_ in range(int(times.value)): - yield msg.strike_token("times") - else: - yield msg - - -def parse_pathod(s, use_http2=False): - """ - May raise ParseException - """ - try: - s.encode("ascii") - except UnicodeError: - raise exceptions.ParseException("Spec must be valid ASCII.", 0, 0) - try: - if use_http2: - expressions = [ - # http2.Frame.expr(), - http2.Response.expr(), - ] - else: - expressions = [ - websockets.WebsocketFrame.expr(), - http.Response.expr(), - ] - reqs = pp.Or(expressions).parseString(s, parseAll=True) - except pp.ParseException as v: - raise exceptions.ParseException(v.msg, v.line, v.col) - return itertools.chain(*[expand(i) for i in reqs]) - - -def parse_pathoc(s, use_http2=False): - try: - s.encode("ascii") - except UnicodeError: - raise exceptions.ParseException("Spec must be valid ASCII.", 0, 0) - try: - if use_http2: - expressions = [ - # http2.Frame.expr(), - http2.Request.expr(), - ] - else: - expressions = [ - websockets.WebsocketClientFrame.expr(), - http.Request.expr(), - ] - reqs = pp.OneOrMore(pp.Or(expressions)).parseString(s, parseAll=True) - except pp.ParseException as v: - raise exceptions.ParseException(v.msg, v.line, v.col) - return itertools.chain(*[expand(i) for i in reqs]) - - -def parse_websocket_frame(s): - """ - May raise ParseException - """ - try: - reqs = pp.OneOrMore( - websockets.WebsocketFrame.expr() - ).parseString( - s, - parseAll=True - ) - except pp.ParseException as v: - raise exceptions.ParseException(v.msg, v.line, v.col) - return itertools.chain(*[expand(i) for i in reqs]) - - -def serve(msg, fp, settings): - """ - fp: The file pointer to write to. - - request_host: If this a request, this is the connecting host. If - None, we assume it's a response. Used to decide what standard - modifications to make if raw is not set. - - Calling this function may modify the object. - """ - msg = msg.resolve(settings) - started = time.time() - - vals = msg.values(settings) - vals.reverse() - - actions = sorted(msg.actions[:]) - actions.reverse() - actions = [i.intermediate(settings) for i in actions] - - disconnect = writer.write_values(fp, vals, actions[:]) - duration = time.time() - started - ret = dict( - disconnect=disconnect, - started=started, - duration=duration, - ) - ret.update(msg.log(settings)) - return ret diff --git a/pathod/language/actions.py b/pathod/language/actions.py deleted file mode 100644 index 9673871da..000000000 --- a/pathod/language/actions.py +++ /dev/null @@ -1,129 +0,0 @@ -import abc -import copy -import random -from functools import total_ordering -import pyparsing as pp -from . import base - - -@total_ordering # type: ignore -class _Action(base.Token): - - """ - An action that operates on the raw data stream of the message. All - actions have one thing in common: an offset that specifies where the - action should take place. - """ - - def __init__(self, offset): - self.offset = offset - - def resolve(self, settings, msg): - """ - Resolves offset specifications to a numeric offset. Returns a copy - of the action object. - """ - c = copy.copy(self) - l = msg.length(settings) - if c.offset == "r": - c.offset = random.randrange(l) - elif c.offset == "a": - c.offset = l + 1 - return c - - def __lt__(self, other): - return self.offset < other.offset - - def __eq__(self, other): - return self.offset == other.offset - - def __repr__(self): - return self.spec() - - @abc.abstractmethod - def spec(self): # pragma: no cover - pass - - @abc.abstractmethod - def intermediate(self, settings): # pragma: no cover - pass - - -class PauseAt(_Action): - unique_name = None - - def __init__(self, offset, seconds): - _Action.__init__(self, offset) - self.seconds = seconds - - @classmethod - def expr(cls): - e = pp.Literal("p").suppress() - e += base.TokOffset - e += pp.Literal(",").suppress() - e += pp.MatchFirst( - [ - base.v_integer, - pp.Literal("f") - ] - ) - return e.setParseAction(lambda x: cls(*x)) - - def spec(self): - return f"p{self.offset},{self.seconds}" - - def intermediate(self, settings): - return (self.offset, "pause", self.seconds) - - def freeze(self, settings_): - return self - - -class DisconnectAt(_Action): - - def __init__(self, offset): - _Action.__init__(self, offset) - - @classmethod - def expr(cls): - e = pp.Literal("d").suppress() - e += base.TokOffset - return e.setParseAction(lambda x: cls(*x)) - - def spec(self): - return "d%s" % self.offset - - def intermediate(self, settings): - return (self.offset, "disconnect") - - def freeze(self, settings_): - return self - - -class InjectAt(_Action): - unique_name = None # type: ignore - - def __init__(self, offset, value): - _Action.__init__(self, offset) - self.value = value - - @classmethod - def expr(cls): - e = pp.Literal("i").suppress() - e += base.TokOffset - e += pp.Literal(",").suppress() - e += base.TokValue - return e.setParseAction(lambda x: cls(*x)) - - def spec(self): - return f"i{self.offset},{self.value.spec()}" - - def intermediate(self, settings): - return ( - self.offset, - "inject", - self.value.get_generator(settings) - ) - - def freeze(self, settings): - return InjectAt(self.offset, self.value.freeze(settings)) diff --git a/pathod/language/base.py b/pathod/language/base.py deleted file mode 100644 index eb397819f..000000000 --- a/pathod/language/base.py +++ /dev/null @@ -1,540 +0,0 @@ -import operator -import os -import abc -import functools -import pyparsing as pp -from mitmproxy.utils import strutils -from mitmproxy.utils import human -import typing # noqa -from . import generators -from . import exceptions - - -class Settings: - - def __init__( - self, - is_client=False, - staticdir=None, - unconstrained_file_access=False, - request_host=None, - websocket_key=None, - protocol=None, - ): - self.is_client = is_client - self.staticdir = staticdir - self.unconstrained_file_access = unconstrained_file_access - self.request_host = request_host - self.websocket_key = websocket_key # TODO: refactor this into the protocol - self.protocol = protocol - - -Sep = pp.Optional(pp.Literal(":")).suppress() - - -v_integer = pp.Word(pp.nums)\ - .setName("integer")\ - .setParseAction(lambda toks: int(toks[0])) - - -v_literal = pp.MatchFirst( - [ - pp.QuotedString( - "\"", - unquoteResults=True, - multiline=True - ), - pp.QuotedString( - "'", - unquoteResults=True, - multiline=True - ), - ] -) - -v_naked_literal = pp.MatchFirst( - [ - v_literal, - pp.Word("".join(i for i in pp.printables if i not in ",:\n@\'\"")) - ] -) - - -class Token: - - """ - A token in the specification language. Tokens are immutable. The token - classes have no meaning in and of themselves, and are combined into - Components and Actions to build the language. - """ - __metaclass__ = abc.ABCMeta - - @classmethod - def expr(cls): # pragma: no cover - """ - A parse expression. - """ - return None - - @abc.abstractmethod - def spec(self): # pragma: no cover - """ - A parseable specification for this token. - """ - return None - - @property - def unique_name(self) -> typing.Optional[str]: - """ - Controls uniqueness constraints for tokens. No two tokens with the - same name will be allowed. If no uniquness should be applied, this - should be None. - """ - return self.__class__.__name__.lower() - - def resolve(self, settings_, msg_): - """ - Resolves this token to ready it for transmission. This means that - the calculated offsets of actions are fixed. - - settings: a language.Settings instance - msg: The containing message - """ - return self - - def __repr__(self): - return self.spec() - - -class _TokValueLiteral(Token): - - def __init__(self, val): - self.val = strutils.escaped_str_to_bytes(val) - - def get_generator(self, settings_): - return self.val - - def freeze(self, settings_): - return self - - -class TokValueLiteral(_TokValueLiteral): - - """ - A literal with Python-style string escaping - """ - @classmethod - def expr(cls): - e = v_literal.copy() - return e.setParseAction(cls.parseAction) - - @classmethod - def parseAction(cls, x): - v = cls(*x) - return v - - def spec(self): - inner = strutils.bytes_to_escaped_str(self.val) - inner = inner.replace(r"'", r"\x27") - return "'" + inner + "'" - - -class TokValueNakedLiteral(_TokValueLiteral): - - @classmethod - def expr(cls): - e = v_naked_literal.copy() - return e.setParseAction(lambda x: cls(*x)) - - def spec(self): - return strutils.bytes_to_escaped_str(self.val, escape_single_quotes=True) - - -class TokValueGenerate(Token): - - def __init__(self, usize, unit, datatype): - if not unit: - unit = "b" - self.usize, self.unit, self.datatype = usize, unit, datatype - - def bytes(self): - return self.usize * human.SIZE_UNITS[self.unit] - - def get_generator(self, settings_): - return generators.RandomGenerator(self.datatype, self.bytes()) - - def freeze(self, settings): - g = self.get_generator(settings) - return TokValueLiteral(strutils.bytes_to_escaped_str(g[:], escape_single_quotes=True)) - - @classmethod - def expr(cls): - e = pp.Literal("@").suppress() + v_integer - - u = functools.reduce( - operator.or_, - [pp.Literal(i) for i in human.SIZE_UNITS.keys()] - ).leaveWhitespace() - e = e + pp.Optional(u, default=None) - - s = pp.Literal(",").suppress() - s += functools.reduce( - operator.or_, - [pp.Literal(i) for i in generators.DATATYPES.keys()] - ) - e += pp.Optional(s, default="bytes") - return e.setParseAction(lambda x: cls(*x)) - - def spec(self): - s = "@%s" % self.usize - if self.unit != "b": - s += self.unit - if self.datatype != "bytes": - s += ",%s" % self.datatype - return s - - -class TokValueFile(Token): - - def __init__(self, path): - self.path = str(path) - - @classmethod - def expr(cls): - e = pp.Literal("<").suppress() - e = e + v_naked_literal - return e.setParseAction(lambda x: cls(*x)) - - def freeze(self, settings_): - return self - - def get_generator(self, settings): - if not settings.staticdir: - raise exceptions.FileAccessDenied("File access disabled.") - s = os.path.expanduser(self.path) - s = os.path.normpath( - os.path.abspath(os.path.join(settings.staticdir, s)) - ) - uf = settings.unconstrained_file_access - if not uf and not s.startswith(os.path.normpath(settings.staticdir)): - raise exceptions.FileAccessDenied( - "File access outside of configured directory" - ) - if not os.path.isfile(s): - raise exceptions.FileAccessDenied("File not readable") - return generators.FileGenerator(s) - - def spec(self): - return "<'%s'" % self.path - - -TokValue = pp.MatchFirst( - [ - TokValueGenerate.expr(), - TokValueFile.expr(), - TokValueLiteral.expr() - ] -) - - -TokNakedValue = pp.MatchFirst( - [ - TokValueGenerate.expr(), - TokValueFile.expr(), - TokValueLiteral.expr(), - TokValueNakedLiteral.expr(), - ] -) - - -TokOffset = pp.MatchFirst( - [ - v_integer, - pp.Literal("r"), - pp.Literal("a") - ] -) - - -class _Component(Token): - - """ - A value component of the primary specification of an message. - Components produce byte values describing the bytes of the message. - """ - - def values(self, settings): # pragma: no cover - """ - A sequence of values, which can either be strings or generators. - """ - pass - - def string(self, settings=None): - """ - A bytestring representation of the object. - """ - return b"".join(i[:] for i in self.values(settings or {})) - - -class KeyValue(_Component): - - """ - A key/value pair. - cls.preamble: leader - """ - - def __init__(self, key, value): - self.key, self.value = key, value - - @classmethod - def expr(cls): - e = pp.Literal(cls.preamble).suppress() - e += TokValue - e += pp.Literal("=").suppress() - e += TokValue - return e.setParseAction(lambda x: cls(*x)) - - def spec(self): - return f"{self.preamble}{self.key.spec()}={self.value.spec()}" - - def freeze(self, settings): - return self.__class__( - self.key.freeze(settings), self.value.freeze(settings) - ) - - -class CaselessLiteral(_Component): - - """ - A caseless token that can take only one value. - """ - - def __init__(self, value): - self.value = value - - @classmethod - def expr(cls): - spec = pp.CaselessLiteral(cls.TOK) - spec = spec.setParseAction(lambda x: cls(*x)) - return spec - - def values(self, settings): - return self.TOK - - def spec(self): - return self.TOK - - def freeze(self, settings_): - return self - - -class OptionsOrValue(_Component): - - """ - Can be any of a specified set of options, or a value specifier. - """ - preamble = "" - options: typing.List[str] = [] - - def __init__(self, value): - # If it's a string, we were passed one of the options, so we lower-case - # it to be canonical. The user can specify a different case by using a - # string value literal. - self.option_used = False - if isinstance(value, str): - for i in self.options: - # Find the exact option value in a case-insensitive way - if i.lower() == value.lower(): - self.option_used = True - value = TokValueLiteral(i) - break - self.value = value - - @classmethod - def expr(cls): - parts = [pp.CaselessLiteral(i) for i in cls.options] - m = pp.MatchFirst(parts) - spec = m | TokValue.copy() - spec = spec.setParseAction(lambda x: cls(*x)) - if cls.preamble: - spec = pp.Literal(cls.preamble).suppress() + spec - return spec - - def values(self, settings): - return [ - self.value.get_generator(settings) - ] - - def spec(self): - s = self.value.spec() - if s[1:-1].lower() in self.options: - s = s[1:-1].lower() - return f"{self.preamble}{s}" - - def freeze(self, settings): - return self.__class__(self.value.freeze(settings)) - - -class Integer(_Component): - bounds: typing.Tuple[typing.Optional[int], typing.Optional[int]] = (None, None) - preamble = "" - - def __init__(self, value): - v = int(value) - outofbounds = any([ - self.bounds[0] is not None and v < self.bounds[0], - self.bounds[1] is not None and v > self.bounds[1] - ]) - if outofbounds: - raise exceptions.ParseException( - "Integer value must be between %s and %s." % self.bounds, - 0, 0 - ) - self.value = str(value).encode() - - @classmethod - def expr(cls): - e = v_integer.copy() - if cls.preamble: - e = pp.Literal(cls.preamble).suppress() + e - return e.setParseAction(lambda x: cls(*x)) - - def values(self, settings): - return [self.value] - - def spec(self): - return f"{self.preamble}{self.value.decode()}" - - def freeze(self, settings_): - return self - - -class Value(_Component): - - """ - A value component lead by an optional preamble. - """ - preamble = "" - - def __init__(self, value): - self.value = value - - @classmethod - def expr(cls): - e = (TokValue | TokNakedValue) - if cls.preamble: - e = pp.Literal(cls.preamble).suppress() + e - return e.setParseAction(lambda x: cls(*x)) - - def values(self, settings): - return [self.value.get_generator(settings)] - - def spec(self): - return f"{self.preamble}{self.value.spec()}" - - def freeze(self, settings): - return self.__class__(self.value.freeze(settings)) - - -class FixedLengthValue(Value): - - """ - A value component lead by an optional preamble. - """ - preamble = "" - length: typing.Optional[int] = None - - def __init__(self, value): - Value.__init__(self, value) - lenguess = None - try: - lenguess = len(value.get_generator(Settings())) - except exceptions.RenderError: - pass - # This check will fail if we know the length upfront - if lenguess is not None and lenguess != self.length: - raise exceptions.RenderError( - "Invalid value length: '{}' is {} bytes, should be {}.".format( - self.spec(), - lenguess, - self.length - ) - ) - - def values(self, settings): - ret = Value.values(self, settings) - l = sum(len(i) for i in ret) - # This check will fail if we don't know the length upfront - i.e. for - # file inputs - if l != self.length: - raise exceptions.RenderError( - "Invalid value length: '{}' is {} bytes, should be {}.".format( - self.spec(), - l, - self.length - ) - ) - return ret - - -class Boolean(_Component): - - """ - A boolean flag. - name = true - -name = false - """ - name = "" - - def __init__(self, value): - self.value = value - - @classmethod - def expr(cls): - e = pp.Optional(pp.Literal("-"), default=True) - e += pp.Literal(cls.name).suppress() - - def parse(s_, loc_, toks): - val = True - if toks[0] == "-": - val = False - return cls(val) - - return e.setParseAction(parse) - - def spec(self): - return "{}{}".format("-" if not self.value else "", self.name) - - -class IntField(_Component): - - """ - An integer field, where values can optionally specified by name. - """ - names: typing.Dict[str, int] = {} - max = 16 - preamble = "" - - def __init__(self, value): - self.origvalue = value - self.value = self.names.get(value, value) - if self.value > self.max: - raise exceptions.ParseException( - "Value can't exceed %s" % self.max, 0, 0 - ) - - @classmethod - def expr(cls): - parts = [pp.CaselessLiteral(i) for i in cls.names.keys()] - m = pp.MatchFirst(parts) - spec = m | v_integer.copy() - spec = spec.setParseAction(lambda x: cls(*x)) - if cls.preamble: - spec = pp.Literal(cls.preamble).suppress() + spec - return spec - - def values(self, settings): - return [str(self.value)] - - def spec(self): - return f"{self.preamble}{self.origvalue}" diff --git a/pathod/language/exceptions.py b/pathod/language/exceptions.py deleted file mode 100644 index 618c4c26f..000000000 --- a/pathod/language/exceptions.py +++ /dev/null @@ -1,21 +0,0 @@ -class RenderError(Exception): - pass - - -class FileAccessDenied(RenderError): - pass - - -class ParseException(Exception): - - def __init__(self, msg, s, col): - Exception.__init__(self) - self.msg = msg - self.s = s - self.col = col - - def marked(self): - return "{}\n{}".format(self.s, " " * (self.col - 1) + "^") - - def __str__(self): - return f"{self.msg} at char {self.col}" diff --git a/pathod/language/generators.py b/pathod/language/generators.py deleted file mode 100644 index ab16348e4..000000000 --- a/pathod/language/generators.py +++ /dev/null @@ -1,93 +0,0 @@ -import os -import string -import random -import mmap -import sys - -DATATYPES = dict( - ascii_letters=string.ascii_letters.encode(), - ascii_lowercase=string.ascii_lowercase.encode(), - ascii_uppercase=string.ascii_uppercase.encode(), - digits=string.digits.encode(), - hexdigits=string.hexdigits.encode(), - octdigits=string.octdigits.encode(), - punctuation=string.punctuation.encode(), - whitespace=string.whitespace.encode(), - ascii=string.printable.encode(), - bytes=bytes(range(256)) -) - - -class TransformGenerator: - - """ - Perform a byte-by-byte transform another generator - that is, for each - input byte, the transformation must produce one output byte. - - gen: A generator to wrap - transform: A function (offset, data) -> transformed - """ - - def __init__(self, gen, transform): - self.gen = gen - self.transform = transform - - def __len__(self): - return len(self.gen) - - def __getitem__(self, x): - d = self.gen.__getitem__(x) - if isinstance(x, slice): - return self.transform(x.start, d) - return self.transform(x, d) - - def __repr__(self): - return "'transform(%s)'" % self.gen - - -def rand_byte(chars): - """ - Return a random character as byte from a charset. - """ - # bytearray has consistent behaviour on both Python 2 and 3 - # while bytes does not - return bytes([random.choice(chars)]) - - -class RandomGenerator: - - def __init__(self, dtype, length): - self.dtype = dtype - self.length = length - - def __len__(self): - return self.length - - def __getitem__(self, x): - chars = DATATYPES[self.dtype] - if isinstance(x, slice): - return b"".join(rand_byte(chars) for _ in range(*x.indices(min(self.length, sys.maxsize)))) - return rand_byte(chars) - - def __repr__(self): - return f"{self.length} random from {self.dtype}" - - -class FileGenerator: - def __init__(self, path): - self.path = os.path.expanduser(path) - - def __len__(self): - return os.path.getsize(self.path) - - def __getitem__(self, x): - with open(self.path, mode="rb") as f: - if isinstance(x, slice): - with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mapped: - return mapped.__getitem__(x) - else: - f.seek(x) - return f.read(1) - - def __repr__(self): - return "<%s" % self.path diff --git a/pathod/language/http.py b/pathod/language/http.py deleted file mode 100644 index dc6de11e9..000000000 --- a/pathod/language/http.py +++ /dev/null @@ -1,393 +0,0 @@ -import abc - -import pyparsing as pp - -from mitmproxy.net import websocket -from mitmproxy.net.http import status_codes, url, user_agents -from . import base, exceptions, actions, message - -# TODO: use mitmproxy.net.semantics.protocol assemble method, -# instead of duplicating the HTTP on-the-wire representation here. -# see http2 language for an example - - -class WS(base.CaselessLiteral): - TOK = "ws" - - -class Raw(base.CaselessLiteral): - TOK = "r" - - -class Path(base.Value): - pass - - -class StatusCode(base.Integer): - pass - - -class Reason(base.Value): - preamble = "m" - - -class Body(base.Value): - preamble = "b" - - -class Times(base.Integer): - preamble = "x" - - -class Method(base.OptionsOrValue): - options = [ - "GET", - "HEAD", - "POST", - "PUT", - "DELETE", - "OPTIONS", - "TRACE", - "CONNECT", - ] - - -class _HeaderMixin: - @property - def unique_name(self): - return None - - def format_header(self, key, value): - return [key, b": ", value, b"\r\n"] - - def values(self, settings): - return self.format_header( - self.key.get_generator(settings), - self.value.get_generator(settings), - ) - - -class Header(_HeaderMixin, base.KeyValue): - preamble = "h" - - -class ShortcutContentType(_HeaderMixin, base.Value): - preamble = "c" - key = base.TokValueLiteral("Content-Type") - - -class ShortcutLocation(_HeaderMixin, base.Value): - preamble = "l" - key = base.TokValueLiteral("Location") - - -class ShortcutUserAgent(_HeaderMixin, base.OptionsOrValue): - preamble = "u" - options = [i[1] for i in user_agents.UASTRINGS] - key = base.TokValueLiteral("User-Agent") - - def values(self, settings): - value = self.value.val - if self.option_used: - value = user_agents.get_by_shortcut(value.lower().decode())[2].encode() - - return self.format_header( - self.key.get_generator(settings), - value - ) - - -def get_header(val, headers): - """ - Header keys may be Values, so we have to "generate" them as we try the - match. - """ - for h in headers: - k = h.key.get_generator({}) - if len(k) == len(val) and k[:].lower() == val.lower(): - return h - return None - - -class _HTTPMessage(message.Message): - version = b"HTTP/1.1" - - @property - def actions(self): - return self.toks(actions._Action) - - @property - def raw(self): - return bool(self.tok(Raw)) - - @property - def body(self): - return self.tok(Body) - - @abc.abstractmethod - def preamble(self, settings): # pragma: no cover - pass - - @property - def headers(self): - return self.toks(_HeaderMixin) - - def values(self, settings): - vals = self.preamble(settings) - vals.append(b"\r\n") - for h in self.headers: - vals.extend(h.values(settings)) - vals.append(b"\r\n") - if self.body: - vals.extend(self.body.values(settings)) - return vals - - -class Response(_HTTPMessage): - unique_name = None # type: ignore - comps = ( - Header, - ShortcutContentType, - ShortcutLocation, - Raw, - Reason, - Body, - - actions.PauseAt, - actions.DisconnectAt, - actions.InjectAt, - ) - logattrs = ["status_code", "reason", "version", "body"] - - @property - def ws(self): - return self.tok(WS) - - @property - def status_code(self): - return self.tok(StatusCode) - - @property - def reason(self): - return self.tok(Reason) - - def preamble(self, settings): - l = [self.version, b" "] - l.extend(self.status_code.values(settings)) - status_code = int(self.status_code.value) - l.append(b" ") - if self.reason: - l.extend(self.reason.values(settings)) - else: - l.append( - status_codes.RESPONSES.get( - status_code, - "Unknown code" - ).encode() - ) - return l - - def resolve(self, settings, msg=None): - tokens = self.tokens[:] - if self.ws: - if not settings.websocket_key: - raise exceptions.RenderError( - "No websocket key - have we seen a client handshake?" - ) - if not self.status_code: - tokens.insert( - 1, - StatusCode(101) - ) - headers = websocket.server_handshake_headers( - settings.websocket_key - ) - for i in headers.fields: - if not get_header(i[0], self.headers): - tokens.append( - Header( - base.TokValueLiteral(i[0].decode()), - base.TokValueLiteral(i[1].decode())) - ) - if not self.raw: - if not get_header(b"Content-Length", self.headers): - if not self.body: - length = 0 - else: - length = sum( - len(i) for i in self.body.values(settings) - ) - tokens.append( - Header( - base.TokValueLiteral("Content-Length"), - base.TokValueLiteral(str(length)), - ) - ) - intermediate = self.__class__(tokens) - return self.__class__( - [i.resolve(settings, intermediate) for i in tokens] - ) - - @classmethod - def expr(cls): - parts = [i.expr() for i in cls.comps] - atom = pp.MatchFirst(parts) - resp = pp.And( - [ - pp.MatchFirst( - [ - WS.expr() + pp.Optional( - base.Sep + StatusCode.expr() - ), - StatusCode.expr(), - ] - ), - pp.ZeroOrMore(base.Sep + atom) - ] - ) - resp = resp.setParseAction(cls) - return resp - - def spec(self): - return ":".join([i.spec() for i in self.tokens]) - - -class NestedResponse(message.NestedMessage): - preamble = "s" - nest_type = Response - - -class Request(_HTTPMessage): - comps = ( - Header, - ShortcutContentType, - ShortcutUserAgent, - Raw, - NestedResponse, - Body, - Times, - - actions.PauseAt, - actions.DisconnectAt, - actions.InjectAt, - ) - logattrs = ["method", "path", "body"] - - @property - def ws(self): - return self.tok(WS) - - @property - def method(self): - return self.tok(Method) - - @property - def path(self): - return self.tok(Path) - - @property - def times(self): - return self.tok(Times) - - @property - def nested_response(self): - return self.tok(NestedResponse) - - def preamble(self, settings): - v = self.method.values(settings) - v.append(b" ") - v.extend(self.path.values(settings)) - if self.nested_response: - v.append(self.nested_response.parsed.spec()) - v.append(b" ") - v.append(self.version) - return v - - def resolve(self, settings, msg=None): - tokens = self.tokens[:] - if self.ws: - if not self.method: - tokens.insert( - 1, - Method("get") - ) - for i in websocket.client_handshake_headers().fields: - if not get_header(i[0], self.headers): - tokens.append( - Header( - base.TokValueLiteral(i[0].decode()), - base.TokValueLiteral(i[1].decode()) - ) - ) - if not self.raw: - if not get_header(b"Content-Length", self.headers): - if self.body: - length = sum( - len(i) for i in self.body.values(settings) - ) - tokens.append( - Header( - base.TokValueLiteral("Content-Length"), - base.TokValueLiteral(str(length)), - ) - ) - if settings.request_host: - if not get_header(b"Host", self.headers): - h = settings.request_host - if self.path: - path = b"".join(self.path.values({})).decode( - "ascii", errors="ignore" - ) - try: - _, h, _, _ = url.parse(path) - h = h.decode("ascii", errors="ignore") - except ValueError: - pass - tokens.append( - Header( - base.TokValueLiteral("Host"), - base.TokValueLiteral(h) - ) - ) - intermediate = self.__class__(tokens) - return self.__class__( - [i.resolve(settings, intermediate) for i in tokens] - ) - - @classmethod - def expr(cls): - parts = [i.expr() for i in cls.comps] - atom = pp.MatchFirst(parts) - resp = pp.And( - [ - pp.MatchFirst( - [ - WS.expr() + pp.Optional( - base.Sep + Method.expr() - ), - Method.expr(), - ] - ), - base.Sep, - Path.expr(), - pp.ZeroOrMore(base.Sep + atom) - ] - ) - resp = resp.setParseAction(cls) - return resp - - def spec(self): - return ":".join([i.spec() for i in self.tokens]) - - -def make_error_response(reason, body=None): - tokens = [ - StatusCode("800"), - Header( - base.TokValueLiteral("Content-Type"), - base.TokValueLiteral("text/plain") - ), - Reason(base.TokValueLiteral(reason)), - Body(base.TokValueLiteral("pathod error: " + (body or reason))), - ] - return Response(tokens) diff --git a/pathod/language/http2.py b/pathod/language/http2.py deleted file mode 100644 index c3c398be1..000000000 --- a/pathod/language/http2.py +++ /dev/null @@ -1,306 +0,0 @@ -import pyparsing as pp - -from mitmproxy.net import http -from mitmproxy.net.http import user_agents, Headers -from . import base, message - -""" - Normal HTTP requests: - ::
: - e.g.: - GET:/ - GET:/:h"foo"="bar" - POST:/:h"foo"="bar":b'content body payload' - - Normal HTTP responses: - :
: - e.g.: - 200 - 302:h"foo"="bar" - 404:h"foo"="bar":b'content body payload' - - Individual HTTP/2 frames: - h2f::::: - e.g.: - h2f:0:PING - h2f:42:HEADERS:END_HEADERS:0x1234567:foo=bar,host=example.com - h2f:42:DATA:END_STREAM,PADDED:0x1234567:'content body payload' -""" - - -def get_header(val, headers): - """ - Header keys may be Values, so we have to "generate" them as we try the - match. - """ - for h in headers: - k = h.key.get_generator({}) - if len(k) == len(val) and k[:].lower() == val.lower(): - return h - return None - - -class _HeaderMixin: - @property - def unique_name(self): - return None - - def values(self, settings): - return ( - self.key.get_generator(settings), - self.value.get_generator(settings), - ) - - -class _HTTP2Message(message.Message): - @property - def actions(self): - return [] # self.toks(actions._Action) - - @property - def headers(self): - headers = self.toks(_HeaderMixin) - - if not self.raw: - if not get_header(b"content-length", headers): - if not self.body: - length = 0 - else: - length = len(self.body.string()) - headers.append( - Header( - base.TokValueLiteral("content-length"), - base.TokValueLiteral(str(length)), - ) - ) - return headers - - @property - def raw(self): - return bool(self.tok(Raw)) - - @property - def body(self): - return self.tok(Body) - - def resolve(self, settings): - return self - - -class StatusCode(base.Integer): - pass - - -class Method(base.OptionsOrValue): - options = [ - "GET", - "HEAD", - "POST", - "PUT", - "DELETE", - ] - - -class Path(base.Value): - pass - - -class Header(_HeaderMixin, base.KeyValue): - preamble = "h" - - -class ShortcutContentType(_HeaderMixin, base.Value): - preamble = "c" - key = base.TokValueLiteral("content-type") - - -class ShortcutLocation(_HeaderMixin, base.Value): - preamble = "l" - key = base.TokValueLiteral("location") - - -class ShortcutUserAgent(_HeaderMixin, base.OptionsOrValue): - preamble = "u" - options = [i[1] for i in user_agents.UASTRINGS] - key = base.TokValueLiteral("user-agent") - - def values(self, settings): - value = self.value.val - if self.option_used: - value = user_agents.get_by_shortcut(value.lower().decode())[2].encode() - - return ( - self.key.get_generator(settings), - value - ) - - -class Raw(base.CaselessLiteral): - TOK = "r" - - -class Body(base.Value): - preamble = "b" - - -class Times(base.Integer): - preamble = "x" - - -class Response(_HTTP2Message): - unique_name = None - comps = ( - Header, - Body, - ShortcutContentType, - ShortcutLocation, - Raw, - ) - - def __init__(self, tokens): - super().__init__(tokens) - self.rendered_values = None - self.stream_id = 2 - - @property - def status_code(self): - return self.tok(StatusCode) - - @classmethod - def expr(cls): - parts = [i.expr() for i in cls.comps] - atom = pp.MatchFirst(parts) - resp = pp.And( - [ - StatusCode.expr(), - pp.ZeroOrMore(base.Sep + atom) - ] - ) - resp = resp.setParseAction(cls) - return resp - - def values(self, settings): - if self.rendered_values: - return self.rendered_values - else: - headers = Headers([header.values(settings) for header in self.headers]) - - body = self.body - if body: - body = body.string() - - resp = http.Response( - http_version=b'HTTP/2.0', - status_code=int(self.status_code.string()), - reason=b'', - headers=headers, - content=body, - trailers=None, - timestamp_start=0, - timestamp_end=0 - ) - resp.stream_id = self.stream_id - - self.rendered_values = settings.protocol.assemble(resp) - return self.rendered_values - - def spec(self): - return ":".join([i.spec() for i in self.tokens]) - - -class NestedResponse(message.NestedMessage): - preamble = "s" - nest_type = Response - - -class Request(_HTTP2Message): - comps = ( - Header, - ShortcutContentType, - ShortcutUserAgent, - Raw, - NestedResponse, - Body, - Times, - ) - logattrs = ["method", "path"] - - def __init__(self, tokens): - super().__init__(tokens) - self.rendered_values = None - self.stream_id = 1 - - @property - def method(self): - return self.tok(Method) - - @property - def path(self): - return self.tok(Path) - - @property - def nested_response(self): - return self.tok(NestedResponse) - - @property - def times(self): - return self.tok(Times) - - @classmethod - def expr(cls): - parts = [i.expr() for i in cls.comps] - atom = pp.MatchFirst(parts) - resp = pp.And( - [ - Method.expr(), - base.Sep, - Path.expr(), - pp.ZeroOrMore(base.Sep + atom) - ] - ) - resp = resp.setParseAction(cls) - return resp - - def values(self, settings): - if self.rendered_values: - return self.rendered_values - else: - path = self.path.string() - if self.nested_response: - path += self.nested_response.parsed.spec().encode() - - headers = Headers([header.values(settings) for header in self.headers]) - - body = self.body - if body: - body = body.string() - - req = http.Request( - "", - 0, - self.method.string(), - b'http', - b'', - path, - b"HTTP/2.0", - headers, - body, - None, - 0, - 0, - ) - req.stream_id = self.stream_id - - self.rendered_values = settings.protocol.assemble(req) - return self.rendered_values - - def spec(self): - return ":".join([i.spec() for i in self.tokens]) - - -def make_error_response(reason, body=None): - tokens = [ - StatusCode("800"), - Body(base.TokValueLiteral("pathod error: " + (body or reason))), - ] - return Response(tokens) diff --git a/pathod/language/message.py b/pathod/language/message.py deleted file mode 100644 index d66b48133..000000000 --- a/pathod/language/message.py +++ /dev/null @@ -1,144 +0,0 @@ -import abc -import typing # noqa - -import pyparsing as pp - -from mitmproxy.utils import strutils -from . import actions, exceptions, base - -LOG_TRUNCATE = 1024 - - -class Message: - __metaclass__ = abc.ABCMeta - logattrs: typing.List[str] = [] - - def __init__(self, tokens): - track = set() - for i in tokens: - if i.unique_name: - if i.unique_name in track: - raise exceptions.ParseException( - "Message has multiple %s clauses, " - "but should only have one." % i.unique_name, - 0, 0 - ) - else: - track.add(i.unique_name) - self.tokens = tokens - - def strike_token(self, name): - toks = [i for i in self.tokens if i.unique_name != name] - return self.__class__(toks) - - def toks(self, klass): - """ - Fetch all tokens that are instances of klass - """ - return [i for i in self.tokens if isinstance(i, klass)] - - def tok(self, klass): - """ - Fetch first token that is an instance of klass - """ - l = self.toks(klass) - if l: - return l[0] - - def length(self, settings): - """ - Calculate the length of the base message without any applied - actions. - """ - return sum(len(x) for x in self.values(settings)) - - def preview_safe(self): - """ - Return a copy of this message that is safe for previews. - """ - tokens = [i for i in self.tokens if not isinstance(i, actions.PauseAt)] - return self.__class__(tokens) - - def maximum_length(self, settings): - """ - Calculate the maximum length of the base message with all applied - actions. - """ - l = self.length(settings) - for i in self.actions: - if isinstance(i, actions.InjectAt): - l += len(i.value.get_generator(settings)) - return l - - @classmethod - def expr(cls): # pragma: no cover - pass - - def log(self, settings): - """ - A dictionary that should be logged if this message is served. - """ - ret = {} - for i in self.logattrs: - v = getattr(self, i) - # Careful not to log any VALUE specs without sanitizing them first. - # We truncate at 1k. - if hasattr(v, "values"): - v = [x[:LOG_TRUNCATE] for x in v.values(settings)] - v = strutils.bytes_to_escaped_str(b"".join(v)) - elif hasattr(v, "__len__"): - v = v[:LOG_TRUNCATE] - v = strutils.bytes_to_escaped_str(v) - ret[i] = v - ret["spec"] = self.spec() - return ret - - def freeze(self, settings): - r = self.resolve(settings) - return self.__class__([i.freeze(settings) for i in r.tokens]) - - def __repr__(self): - return self.spec() - - -class NestedMessage(base.Token): - """ - A nested message, as an escaped string with a preamble. - """ - preamble = "" - nest_type: typing.Optional[typing.Type[Message]] = None - - def __init__(self, value): - super().__init__() - self.value = value - try: - self.parsed = self.nest_type( - self.nest_type.expr().parseString( - value.val.decode(), - parseAll=True - ) - ) - except pp.ParseException as v: - raise exceptions.ParseException(v.msg, v.line, v.col) - - @classmethod - def expr(cls): - e = pp.Literal(cls.preamble).suppress() - e = e + base.TokValueLiteral.expr() - return e.setParseAction(lambda x: cls(*x)) - - def values(self, settings): - return [ - self.value.get_generator(settings), - ] - - def spec(self): - return f"{self.preamble}{self.value.spec()}" - - def freeze(self, settings): - f = self.parsed.freeze(settings).spec() - return self.__class__( - base.TokValueLiteral( - strutils.bytes_to_escaped_str(f.encode(), escape_single_quotes=True) - ) - ) diff --git a/pathod/language/websockets.py b/pathod/language/websockets.py deleted file mode 100644 index 772cff280..000000000 --- a/pathod/language/websockets.py +++ /dev/null @@ -1,255 +0,0 @@ -import random -import string -import typing # noqa - -import pyparsing as pp - -from wsproto.frame_protocol import Opcode - -from mitmproxy.utils import strutils -from . import base, generators, actions, message, websockets_frame - - -NESTED_LEADER = b"pathod!" - - -class WF(base.CaselessLiteral): - TOK = "wf" - - -class OpCode(base.IntField): - names: typing.Dict[str, int] = { - "continue": Opcode.CONTINUATION, - "text": Opcode.TEXT, - "binary": Opcode.BINARY, - "close": Opcode.CLOSE, - "ping": Opcode.PING, - "pong": Opcode.PONG, - } - max = 15 - preamble = "c" - - -class Body(base.Value): - preamble = "b" - - -class RawBody(base.Value): - unique_name = "body" - preamble = "r" - - -class Fin(base.Boolean): - name = "fin" - - -class RSV1(base.Boolean): - name = "rsv1" - - -class RSV2(base.Boolean): - name = "rsv2" - - -class RSV3(base.Boolean): - name = "rsv3" - - -class Mask(base.Boolean): - name = "mask" - - -class Key(base.FixedLengthValue): - preamble = "k" - length = 4 - - -class KeyNone(base.CaselessLiteral): - unique_name = "key" - TOK = "knone" - - -class Length(base.Integer): - bounds = (0, 1 << 64) - preamble = "l" - - -class Times(base.Integer): - preamble = "x" - - -COMPONENTS = [ - OpCode, - Length, - # Bit flags - Fin, - RSV1, - RSV2, - RSV3, - Mask, - actions.PauseAt, - actions.DisconnectAt, - actions.InjectAt, - KeyNone, - Key, - Times, - Body, - RawBody, -] - - -class WebsocketFrame(message.Message): - components: typing.List[typing.Type[base._Component]] = COMPONENTS - logattrs = ["body"] - # Used for nested frames - unique_name = "body" - - @property - def actions(self): - return self.toks(actions._Action) - - @property - def body(self): - return self.tok(Body) - - @property - def rawbody(self): - return self.tok(RawBody) - - @property - def opcode(self): - return self.tok(OpCode) - - @property - def fin(self): - return self.tok(Fin) - - @property - def rsv1(self): - return self.tok(RSV1) - - @property - def rsv2(self): - return self.tok(RSV2) - - @property - def rsv3(self): - return self.tok(RSV3) - - @property - def mask(self): - return self.tok(Mask) - - @property - def key(self): - return self.tok(Key) - - @property - def knone(self): - return self.tok(KeyNone) - - @property - def times(self): - return self.tok(Times) - - @property - def toklength(self): - return self.tok(Length) - - @classmethod - def expr(cls): - parts = [i.expr() for i in cls.components] - atom = pp.MatchFirst(parts) - resp = pp.And( - [ - WF.expr(), - base.Sep, - pp.ZeroOrMore(base.Sep + atom) - ] - ) - resp = resp.setParseAction(cls) - return resp - - @property - def nested_frame(self): - return self.tok(NestedFrame) - - def resolve(self, settings, msg=None): - tokens = self.tokens[:] - if not self.mask and settings.is_client: - tokens.append( - Mask(True) - ) - if not self.knone and self.mask and self.mask.value and not self.key: - allowed_chars = string.ascii_letters + string.digits - k = ''.join([allowed_chars[random.randrange(0, len(allowed_chars))] for i in range(4)]) - tokens.append( - Key(base.TokValueLiteral(k)) - ) - return self.__class__( - [i.resolve(settings, self) for i in tokens] - ) - - def values(self, settings): - if self.body: - bodygen = self.body.value.get_generator(settings) - length = len(self.body.value.get_generator(settings)) - elif self.rawbody: - bodygen = self.rawbody.value.get_generator(settings) - length = len(self.rawbody.value.get_generator(settings)) - elif self.nested_frame: - bodygen = NESTED_LEADER + strutils.always_bytes(self.nested_frame.parsed.spec()) - length = len(bodygen) - else: - bodygen = None - length = 0 - if self.toklength: - length = int(self.toklength.value) - frameparts = dict( - payload_length=length - ) - if self.mask and self.mask.value: - frameparts["mask"] = True - if self.knone: - frameparts["masking_key"] = None - elif self.key: - key = self.key.values(settings)[0][:] - frameparts["masking_key"] = key - for i in ["opcode", "fin", "rsv1", "rsv2", "rsv3", "mask"]: - v = getattr(self, i, None) - if v is not None: - frameparts[i] = v.value - - # import wsproto.frame_protocol - # wsproto.frame_protocol.Frame( - # opcode=frameparts["opcode"], - # payload=None, - # frame_finished=frameparts["fin"] - # ) - - frame = websockets_frame.FrameHeader(**frameparts) - vals = [bytes(frame)] - if bodygen: - if frame.masking_key and not self.rawbody: - masker = websockets_frame.Masker(frame.masking_key) - vals.append( - generators.TransformGenerator( - bodygen, - masker.mask - ) - ) - else: - vals.append(bodygen) - return vals - - def spec(self): - return ":".join([i.spec() for i in self.tokens]) - - -class NestedFrame(message.NestedMessage): - preamble = "f" - nest_type = WebsocketFrame - - -class WebsocketClientFrame(WebsocketFrame): - components = COMPONENTS + [NestedFrame] diff --git a/pathod/language/websockets_frame.py b/pathod/language/websockets_frame.py deleted file mode 100644 index 6bd1c91a6..000000000 --- a/pathod/language/websockets_frame.py +++ /dev/null @@ -1,274 +0,0 @@ -import sys -import os -import struct -import io - -from wsproto.frame_protocol import Opcode - -from mitmproxy.net import tcp -from mitmproxy.utils import bits, human, strutils - - -MAX_16_BIT_INT = (1 << 16) -MAX_64_BIT_INT = (1 << 64) - -DEFAULT = object() - - -class Masker: - """ - Data sent from the server must be masked to prevent malicious clients - from sending data over the wire in predictable patterns. - - Servers do not have to mask data they send to the client. - https://tools.ietf.org/html/rfc6455#section-5.3 - """ - - def __init__(self, key): - self.key = key - self.offset = 0 - - def mask(self, offset, data): - datalen = len(data) - offset_mod = offset % 4 - data = int.from_bytes(data, sys.byteorder) - num_keys = (datalen + offset_mod + 3) // 4 - mask = int.from_bytes((self.key * num_keys)[offset_mod:datalen + - offset_mod], sys.byteorder) - return (data ^ mask).to_bytes(datalen, sys.byteorder) - - def __call__(self, data): - ret = self.mask(self.offset, data) - self.offset += len(ret) - return ret - - -class FrameHeader: - - def __init__( - self, - opcode=Opcode.TEXT, - payload_length=0, - fin=False, - rsv1=False, - rsv2=False, - rsv3=False, - masking_key=DEFAULT, - mask=DEFAULT, - length_code=DEFAULT - ): - if not 0 <= opcode < 2 ** 4: - raise ValueError("opcode must be 0-16") - self.opcode = opcode - self.payload_length = payload_length - self.fin = fin - self.rsv1 = rsv1 - self.rsv2 = rsv2 - self.rsv3 = rsv3 - - if length_code is DEFAULT: - self.length_code = self._make_length_code(self.payload_length) - else: - self.length_code = length_code - - if (mask is DEFAULT and masking_key is DEFAULT) or mask == 0 or mask is False: - self.mask = False - self.masking_key = b"" - elif mask is DEFAULT: - self.mask = 1 - self.masking_key = masking_key - elif masking_key is DEFAULT: - self.mask = mask - self.masking_key = os.urandom(4) - else: - self.mask = mask - self.masking_key = masking_key - - if self.masking_key and len(self.masking_key) != 4: - raise ValueError("Masking key must be 4 bytes.") - - @classmethod - def _make_length_code(self, length): - """ - A WebSocket frame contains an initial length_code, and an optional - extended length code to represent the actual length if length code is - larger than 125 - """ - if length <= 125: - return length - elif length >= 126 and length <= 65535: - return 126 - else: - return 127 - - def __repr__(self): - vals = [ - "ws frame:", - Opcode(self.opcode).name.lower() - ] - flags = [] - for i in ["fin", "rsv1", "rsv2", "rsv3", "mask"]: - if getattr(self, i): - flags.append(i) - if flags: - vals.extend([":", "|".join(flags)]) - if self.masking_key: - vals.append(":key=%s" % repr(self.masking_key)) - if self.payload_length: - vals.append(" %s" % human.pretty_size(self.payload_length)) - return "".join(vals) - - def __bytes__(self): - first_byte = bits.setbit(0, 7, self.fin) - first_byte = bits.setbit(first_byte, 6, self.rsv1) - first_byte = bits.setbit(first_byte, 5, self.rsv2) - first_byte = bits.setbit(first_byte, 4, self.rsv3) - first_byte = first_byte | self.opcode - - second_byte = bits.setbit(self.length_code, 7, self.mask) - - b = bytes([first_byte, second_byte]) - - if self.payload_length < 126: - pass - elif self.payload_length < MAX_16_BIT_INT: - # '!H' pack as 16 bit unsigned short - # add 2 byte extended payload length - b += struct.pack('!H', self.payload_length) - elif self.payload_length < MAX_64_BIT_INT: - # '!Q' = pack as 64 bit unsigned long long - # add 8 bytes extended payload length - b += struct.pack('!Q', self.payload_length) - else: - raise ValueError("Payload length exceeds 64bit integer") - - if self.masking_key: - b += self.masking_key - return b - - @classmethod - def from_file(cls, fp): - """ - read a WebSocket frame header - """ - first_byte, second_byte = fp.safe_read(2) - fin = bits.getbit(first_byte, 7) - rsv1 = bits.getbit(first_byte, 6) - rsv2 = bits.getbit(first_byte, 5) - rsv3 = bits.getbit(first_byte, 4) - opcode = first_byte & 0xF - mask_bit = bits.getbit(second_byte, 7) - length_code = second_byte & 0x7F - - # payload_length > 125 indicates you need to read more bytes - # to get the actual payload length - if length_code <= 125: - payload_length = length_code - elif length_code == 126: - payload_length, = struct.unpack("!H", fp.safe_read(2)) - else: # length_code == 127: - payload_length, = struct.unpack("!Q", fp.safe_read(8)) - - # masking key only present if mask bit set - if mask_bit == 1: - masking_key = fp.safe_read(4) - else: - masking_key = None - - return cls( - fin=fin, - rsv1=rsv1, - rsv2=rsv2, - rsv3=rsv3, - opcode=opcode, - mask=mask_bit, - length_code=length_code, - payload_length=payload_length, - masking_key=masking_key, - ) - - def __eq__(self, other): - if isinstance(other, FrameHeader): - return bytes(self) == bytes(other) - return False - - -class Frame: - """ - Represents a single WebSocket frame. - Constructor takes human readable forms of the frame components. - from_bytes() reads from a file-like object to create a new Frame. - - WebSocket frame as defined in RFC6455 - - 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 - +-+-+-+-+-------+-+-------------+-------------------------------+ - |F|R|R|R| opcode|M| Payload len | Extended payload length | - |I|S|S|S| (4) |A| (7) | (16/64) | - |N|V|V|V| |S| | (if payload len==126/127) | - | |1|2|3| |K| | | - +-+-+-+-+-------+-+-------------+ - - - - - - - - - - - - - - - + - | Extended payload length continued, if payload len == 127 | - + - - - - - - - - - - - - - - - +-------------------------------+ - | |Masking-key, if MASK set to 1 | - +-------------------------------+-------------------------------+ - | Masking-key (continued) | Payload Data | - +-------------------------------- - - - - - - - - - - - - - - - + - : Payload Data continued ... : - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + - | Payload Data continued ... | - +---------------------------------------------------------------+ - """ - - def __init__(self, payload=b"", **kwargs): - self.payload = payload - kwargs["payload_length"] = kwargs.get("payload_length", len(payload)) - self.header = FrameHeader(**kwargs) - - @classmethod - def from_bytes(cls, bytestring): - """ - Construct a websocket frame from an in-memory bytestring - to construct a frame from a stream of bytes, use from_file() directly - """ - return cls.from_file(tcp.Reader(io.BytesIO(bytestring))) - - def __repr__(self): - ret = repr(self.header) - if self.payload: - ret = ret + "\nPayload:\n" + strutils.bytes_to_escaped_str(self.payload) - return ret - - def __bytes__(self): - """ - Serialize the frame to wire format. Returns a string. - """ - b = bytes(self.header) - if self.header.masking_key: - b += Masker(self.header.masking_key)(self.payload) - else: - b += self.payload - return b - - @classmethod - def from_file(cls, fp): - """ - read a WebSocket frame sent by a server or client - - fp is a "file like" object that could be backed by a network - stream or a disk or an in memory stream reader - """ - header = FrameHeader.from_file(fp) - payload = fp.safe_read(header.payload_length) - - if header.mask == 1 and header.masking_key: - payload = Masker(header.masking_key)(payload) - - frame = cls(payload) - frame.header = header - return frame - - def __eq__(self, other): - if isinstance(other, Frame): - return bytes(self) == bytes(other) - return False diff --git a/pathod/language/writer.py b/pathod/language/writer.py deleted file mode 100644 index cdb62c3a6..000000000 --- a/pathod/language/writer.py +++ /dev/null @@ -1,68 +0,0 @@ -import time -from mitmproxy import exceptions - -BLOCKSIZE = 1024 -# It's not clear what the upper limit for time.sleep is. It's lower than the -# maximum int or float. 1 year should do. -FOREVER = 60 * 60 * 24 * 365 - - -def send_chunk(fp, val, blocksize, start, end): - """ - (start, end): Inclusive lower bound, exclusive upper bound. - """ - for i in range(start, end, blocksize): - fp.write( - val[i:min(i + blocksize, end)] - ) - return end - start - - -def write_values(fp, vals, actions, sofar=0, blocksize=BLOCKSIZE): - """ - vals: A list of values, which may be strings or Value objects. - - actions: A list of (offset, action, arg) tuples. Action may be "inject", - "pause" or "disconnect". - - Both vals and actions are in reverse order, with the first items last. - - Return True if connection should disconnect. - """ - sofar = 0 - try: - while vals: - v = vals.pop() - offset = 0 - while actions and actions[-1][0] < (sofar + len(v)): - a = actions.pop() - offset += send_chunk( - fp, - v, - blocksize, - offset, - a[0] - sofar - offset - ) - if a[1] == "pause": - time.sleep( - FOREVER if a[2] == "f" else a[2] - ) - elif a[1] == "disconnect": - return True - elif a[1] == "inject": - send_chunk(fp, a[2], blocksize, 0, len(a[2])) - send_chunk(fp, v, blocksize, offset, len(v)) - sofar += len(v) - # Remainders - while actions: - a = actions.pop() - if a[1] == "pause": - time.sleep( - FOREVER if a[2] == "f" else a[2] - ) - elif a[1] == "disconnect": - return True - elif a[1] == "inject": - send_chunk(fp, a[2], blocksize, 0, len(a[2])) - except exceptions.TcpDisconnect: # pragma: no cover - return True diff --git a/pathod/log.py b/pathod/log.py deleted file mode 100644 index 1d7a1bda8..000000000 --- a/pathod/log.py +++ /dev/null @@ -1,89 +0,0 @@ -import time - -from mitmproxy.utils import strutils -from mitmproxy.utils import human - - -def write_raw(fp, lines, timestamp=True): - if fp: - if timestamp: - fp.write(human.format_timestamp(time.time())) - for i in lines: - fp.write(i) - fp.write("\n") - fp.flush() - - -class LogCtx: - - def __init__(self, fp, hex, timestamp, rfile, wfile): - self.lines = [] - self.fp = fp - self.suppressed = False - self.hex = hex - self.timestamp = timestamp - self.rfile, self.wfile = rfile, wfile - - def __enter__(self): - if self.wfile: - self.wfile.start_log() - if self.rfile: - self.rfile.start_log() - return self - - def __exit__(self, exc_type, exc_value, traceback): - wlog = self.wfile.get_log() if self.wfile else None - rlog = self.rfile.get_log() if self.rfile else None - if self.suppressed or not self.fp: - return - if wlog: - self("Bytes written:") - self.dump(wlog, self.hex) - if rlog: - self("Bytes read:") - self.dump(rlog, self.hex) - if self.lines: - write_raw( - self.fp, - [ - "\n".join(self.lines), - ], - timestamp = self.timestamp - ) - if exc_value: - raise exc_value - - def suppress(self): - self.suppressed = True - - def dump(self, data, hexdump): - if hexdump: - for line in strutils.hexdump(data): - self("\t%s %s %s" % line) - else: - data = strutils.always_str( - strutils.escape_control_characters( - data - .decode("ascii", "replace") - .replace("\ufffd", ".") - ) - ) - for i in data.split("\n"): - self("\t%s" % i) - - def __call__(self, line): - self.lines.append(line) - - -class ConnectionLogger: - def __init__(self, fp, hex, timestamp, rfile, wfile): - self.fp = fp - self.hex = hex - self.rfile, self.wfile = rfile, wfile - self.timestamp = timestamp - - def ctx(self): - return LogCtx(self.fp, self.hex, self.timestamp, self.rfile, self.wfile) - - def write(self, lines): - write_raw(self.fp, lines, timestamp=self.timestamp) diff --git a/pathod/pathoc.py b/pathod/pathoc.py deleted file mode 100644 index 272bc912a..000000000 --- a/pathod/pathoc.py +++ /dev/null @@ -1,585 +0,0 @@ -import contextlib -import sys -import os -import itertools -import hashlib -import queue -import random -import select -import time - -import OpenSSL.crypto -import logging - -from mitmproxy import certs, exceptions -from mitmproxy.net import tcp, tls, socks -from mitmproxy.net import http as net_http -from mitmproxy.coretypes import basethread -from mitmproxy.utils import strutils - -from pathod import language, log -from pathod.protocols import http2 - - -logging.getLogger("hpack").setLevel(logging.WARNING) - - -def xrepr(s): - return repr(s)[1:-1] - - -class PathocError(Exception): - pass - - -class SSLInfo: - - def __init__(self, certchain, cipher, alp): - self.certchain, self.cipher, self.alp = certchain, cipher, alp - - def __str__(self): - parts = [ - "Application Layer Protocol: %s" % strutils.always_str(self.alp, "utf8"), - "Cipher: %s, %s bit, %s" % self.cipher, - "SSL certificate chain:" - ] - for n, i in enumerate(self.certchain): - parts.append(" Certificate [%s]" % n) - parts.append("\tSubject: ") - for cn in i.get_subject().get_components(): - parts.append("\t\t{}={}".format( - strutils.always_str(cn[0], "utf8"), - strutils.always_str(cn[1], "utf8")) - ) - parts.append("\tIssuer: ") - for cn in i.get_issuer().get_components(): - parts.append("\t\t{}={}".format( - strutils.always_str(cn[0], "utf8"), - strutils.always_str(cn[1], "utf8")) - ) - parts.extend( - [ - "\tVersion: %s" % i.get_version(), - "\tValidity: {} - {}".format( - strutils.always_str(i.get_notBefore(), "utf8"), - strutils.always_str(i.get_notAfter(), "utf8") - ), - "\tSerial: %s" % i.get_serial_number(), - "\tAlgorithm: %s" % strutils.always_str(i.get_signature_algorithm(), "utf8") - ] - ) - pk = i.get_pubkey() - types = { - OpenSSL.crypto.TYPE_RSA: "RSA", - OpenSSL.crypto.TYPE_DSA: "DSA" - } - t = types.get(pk.type(), "Uknown") - parts.append(f"\tPubkey: {pk.bits()} bit {t}") - s = certs.Cert(i) - if s.altnames: - parts.append("\tSANs: %s" % " ".join(strutils.always_str(n, "utf8") for n in s.altnames)) - return "\n".join(parts) - - -class WebsocketFrameReader(basethread.BaseThread): - - def __init__( - self, - rfile, - logfp, - showresp, - hexdump, - ws_read_limit, - timeout - ): - basethread.BaseThread.__init__(self, "WebsocketFrameReader") - self.timeout = timeout - self.ws_read_limit = ws_read_limit - self.logfp = logfp - self.showresp = showresp - self.hexdump = hexdump - self.rfile = rfile - self.terminate = queue.Queue() - self.frames_queue = queue.Queue() - self.logger = log.ConnectionLogger( - self.logfp, - self.hexdump, - False, - rfile if showresp else None, - None - ) - - @contextlib.contextmanager - def terminator(self): - yield - self.frames_queue.put(None) - - def run(self): - starttime = time.time() - with self.terminator(): - while True: - if self.ws_read_limit == 0: - return - try: - r, _, _ = select.select([self.rfile], [], [], 0.05) - except OSError: # pragma: no cover - return # this is not reliably triggered due to its nature, so we exclude it from coverage. - delta = time.time() - starttime - if not r and self.timeout and delta > self.timeout: - return - try: - self.terminate.get_nowait() - return - except queue.Empty: - pass - for rfile in r: - with self.logger.ctx() as log: - try: - frm = language.websockets_frame.Frame.from_file(self.rfile) - except exceptions.TcpDisconnect: - return - self.frames_queue.put(frm) - log("<< %s" % repr(frm.header)) - if self.ws_read_limit is not None: - self.ws_read_limit -= 1 - starttime = time.time() - - -class Pathoc(tcp.TCPClient): - - def __init__( - self, - address, - - # SSL - ssl=None, - sni=None, - ssl_version=tls.DEFAULT_METHOD, - ssl_options=tls.DEFAULT_OPTIONS, - clientcert=None, - ciphers=None, - - # HTTP/2 - use_http2=False, - http2_skip_connection_preface=False, - http2_framedump=False, - - # Websockets - ws_read_limit=None, - - # Network - timeout=None, - - # Output control - showreq=False, - showresp=False, - explain=False, - hexdump=False, - ignorecodes=(), - ignoretimeout=False, - showsummary=False, - fp=sys.stdout - ): - """ - spec: A request specification - showreq: Print requests - showresp: Print responses - explain: Print request explanation - showssl: Print info on SSL connection - hexdump: When printing requests or responses, use hex dump output - showsummary: Show a summary of requests - ignorecodes: Sequence of return codes to ignore - """ - tcp.TCPClient.__init__(self, address) - - self.ssl, self.sni = ssl, sni - self.clientcert = clientcert - self.ssl_version = ssl_version - self.ssl_options = ssl_options - self.ciphers = ciphers - self.sslinfo = None - - self.use_http2 = use_http2 - self.http2_skip_connection_preface = http2_skip_connection_preface - self.http2_framedump = http2_framedump - - self.ws_read_limit = ws_read_limit - - self.timeout = timeout - - self.showreq = showreq - self.showresp = showresp - self.explain = explain - self.hexdump = hexdump - self.ignorecodes = ignorecodes - self.ignoretimeout = ignoretimeout - self.showsummary = showsummary - self.fp = fp - - self.ws_framereader = None - - if self.use_http2: - self.protocol = http2.HTTP2StateProtocol(self, dump_frames=self.http2_framedump) - else: - self.protocol = net_http.http1 - - self.settings = language.Settings( - is_client=True, - staticdir=os.getcwd(), - unconstrained_file_access=True, - request_host=self.address[0], - protocol=self.protocol, - ) - - def http_connect(self, connect_to): - req = net_http.Request( - host=connect_to[0], - port=connect_to[1], - method=b'CONNECT', - scheme=b"", - authority=f"{connect_to[0]}:{connect_to[1]}".encode(), - path=b"", - http_version=b'HTTP/1.1', - headers=((b"Host", connect_to[0].encode("idna")),), - content=b'', - trailers=None, - timestamp_start=0, - timestamp_end=0, - ) - self.wfile.write(net_http.http1.assemble_request(req)) - self.wfile.flush() - try: - resp = self.protocol.read_response(self.rfile, req) - if resp.status_code != 200: - raise exceptions.HttpException("Unexpected status code: %s" % resp.status_code) - except exceptions.HttpException as e: - raise PathocError( - "Proxy CONNECT failed: %s" % repr(e) - ) - - def socks_connect(self, connect_to): - try: - client_greet = socks.ClientGreeting( - socks.VERSION.SOCKS5, - [socks.METHOD.NO_AUTHENTICATION_REQUIRED] - ) - client_greet.to_file(self.wfile) - self.wfile.flush() - - server_greet = socks.ServerGreeting.from_file(self.rfile) - server_greet.assert_socks5() - if server_greet.method != socks.METHOD.NO_AUTHENTICATION_REQUIRED: - raise socks.SocksError( - socks.METHOD.NO_ACCEPTABLE_METHODS, - "pathoc only supports SOCKS without authentication" - ) - - connect_request = socks.Message( - socks.VERSION.SOCKS5, - socks.CMD.CONNECT, - socks.ATYP.DOMAINNAME, - connect_to, - ) - connect_request.to_file(self.wfile) - self.wfile.flush() - - connect_reply = socks.Message.from_file(self.rfile) - connect_reply.assert_socks5() - if connect_reply.msg != socks.REP.SUCCEEDED: - raise socks.SocksError( - connect_reply.msg, - "SOCKS server error" - ) - except (socks.SocksError, exceptions.TcpDisconnect) as e: - raise PathocError(str(e)) - - def connect(self, connect_to=None, showssl=False, fp=sys.stdout): - """ - connect_to: A (host, port) tuple, which will be connected to with - an HTTP CONNECT request. - """ - if self.use_http2 and not self.ssl: - raise NotImplementedError("HTTP2 without SSL is not supported.") - - with tcp.TCPClient.connect(self) as closer: - if connect_to: - self.http_connect(connect_to) - - self.sslinfo = None - if self.ssl: - try: - alpn_protos = [b'http/1.1'] - if self.use_http2: - alpn_protos.append(b'h2') - - self.convert_to_tls( - sni=self.sni, - cert=self.clientcert, - method=self.ssl_version, - options=self.ssl_options, - cipher_list=self.ciphers, - alpn_protos=alpn_protos - ) - except exceptions.TlsException as v: - raise PathocError(str(v)) - - self.sslinfo = SSLInfo( - self.connection.get_peer_cert_chain(), - self.get_current_cipher(), - self.get_alpn_proto_negotiated() - ) - if showssl: - print(str(self.sslinfo), file=fp) - - if self.use_http2: - self.protocol.check_alpn() - if not self.http2_skip_connection_preface: - self.protocol.perform_client_connection_preface() - - if self.timeout: - self.settimeout(self.timeout) - - return closer.pop() - - def stop(self): - if self.ws_framereader: - self.ws_framereader.terminate.put(None) - - def wait(self, timeout=0.01, finish=True): - """ - A generator that yields frames until Pathoc terminates. - - timeout: If specified None may be yielded instead if timeout is - reached. If timeout is None, wait forever. If timeout is 0, return - immediately if nothing is on the queue. - - finish: If true, consume messages until the reader shuts down. - Otherwise, return None on timeout. - """ - if self.ws_framereader: - while True: - try: - frm = self.ws_framereader.frames_queue.get( - timeout=timeout, - block=True if timeout != 0 else False - ) - except queue.Empty: - if finish: - continue - else: - return - if frm is None: - self.ws_framereader.join() - self.ws_framereader = None - return - yield frm - - def websocket_send_frame(self, r): - """ - Sends a single websocket frame. - """ - logger = log.ConnectionLogger( - self.fp, - self.hexdump, - False, - None, - self.wfile if self.showreq else None, - ) - with logger.ctx() as lg: - lg(">> %s" % r) - language.serve(r, self.wfile, self.settings) - self.wfile.flush() - - def websocket_start(self, r): - """ - Performs an HTTP request, and attempts to drop into websocket - connection. - """ - resp = self.http(r) - if resp.status_code == 101: - self.ws_framereader = WebsocketFrameReader( - self.rfile, - self.fp, - self.showresp, - self.hexdump, - self.ws_read_limit, - self.timeout - ) - self.ws_framereader.start() - return resp - - def http(self, r): - """ - Performs a single request. - - r: A language.http.Request object, or a string representing one - request. - - Returns Response if we have a non-ignored response. - - May raise a exceptions.NetlibException - """ - logger = log.ConnectionLogger( - self.fp, - self.hexdump, - False, - self.rfile if self.showresp else None, - self.wfile if self.showreq else None, - ) - with logger.ctx() as lg: - lg(">> %s" % r) - resp, req = None, None - try: - req = language.serve(r, self.wfile, self.settings) - self.wfile.flush() - - # build a dummy request to read the response - # ideally this would be returned directly from language.serve - dummy_req = net_http.Request( - host="localhost", - port=80, - method=req["method"], - scheme=b"http", - authority=b"", - path=b"/", - http_version=b"HTTP/1.1", - headers=(), - content=b'', - trailers=None, - timestamp_start=time.time(), - timestamp_end=None, - ) - - resp = self.protocol.read_response(self.rfile, dummy_req) - resp.sslinfo = self.sslinfo - except exceptions.HttpException as v: - lg("Invalid server response: %s" % v) - raise - except exceptions.TcpTimeout: - if self.ignoretimeout: - lg("Timeout (ignored)") - return None - lg("Timeout") - raise - finally: - if resp: - lg("<< {} {}: {} bytes".format( - resp.status_code, strutils.escape_control_characters(resp.reason) if resp.reason else "", len(resp.content) - )) - if resp.status_code in self.ignorecodes: - lg.suppress() - return resp - - def request(self, r): - """ - Performs a single request. - - r: A language.message.Message object, or a string representing - one. - - Returns Response if we have a non-ignored response. - - May raise a exceptions.NetlibException - """ - if isinstance(r, str): - r = next(language.parse_pathoc(r, self.use_http2)) - - if isinstance(r, language.http.Request): - if r.ws: - return self.websocket_start(r) - else: - return self.http(r) - elif isinstance(r, language.websockets.WebsocketFrame): - self.websocket_send_frame(r) - elif isinstance(r, language.http2.Request): - return self.http(r) - # elif isinstance(r, language.http2.Frame): - # TODO: do something - - -def main(args): # pragma: no cover - memo = set() - p = None - - if args.repeat == 1: - requests = args.requests - else: - # If we are replaying more than once, we must convert the request generators to lists - # or they will be exhausted after the first run. - # This is bad for the edge-case where get:/:x10000000 (see 0da3e51) is combined with -n 2, - # but does not matter otherwise. - requests = [list(x) for x in args.requests] - - try: - requests_done = 0 - while True: - if requests_done == args.repeat: - break - if args.wait and requests_done > 0: - time.sleep(args.wait) - - requests_done += 1 - if args.random: - playlist = random.choice(requests) - else: - playlist = itertools.chain.from_iterable(requests) - p = Pathoc( - (args.host, args.port), - ssl=args.ssl, - sni=args.sni, - ssl_version=args.ssl_version, - ssl_options=args.ssl_options, - clientcert=args.clientcert, - ciphers=args.ciphers, - use_http2=args.use_http2, - http2_skip_connection_preface=args.http2_skip_connection_preface, - http2_framedump=args.http2_framedump, - showreq=args.showreq, - showresp=args.showresp, - explain=args.explain, - hexdump=args.hexdump, - ignorecodes=args.ignorecodes, - timeout=args.timeout, - ignoretimeout=args.ignoretimeout, - showsummary=True - ) - trycount = 0 - try: - with p.connect(args.connect_to, args.showssl): - for spec in playlist: - if args.explain or args.memo: - spec = spec.freeze(p.settings) - if args.memo: - h = hashlib.sha256(spec.spec()).digest() - if h not in memo: - trycount = 0 - memo.add(h) - else: - trycount += 1 - if trycount > args.memolimit: - print("Memo limit exceeded...", file=sys.stderr) - return - else: - continue - try: - ret = p.request(spec) - if ret and args.oneshot: - return - # We consume the queue when we can, so it doesn't build up. - for _ in p.wait(timeout=0, finish=False): - pass - except exceptions.NetlibException: - break - for _ in p.wait(timeout=0.01, finish=True): - pass - except exceptions.TcpException as v: - print(str(v), file=sys.stderr) - continue - except PathocError as v: - print(str(v), file=sys.stderr) - sys.exit(1) - - except KeyboardInterrupt: - pass - if p: - p.stop() diff --git a/pathod/pathoc_cmdline.py b/pathod/pathoc_cmdline.py deleted file mode 100644 index a9500c2fe..000000000 --- a/pathod/pathoc_cmdline.py +++ /dev/null @@ -1,228 +0,0 @@ -import sys -import argparse -import os -import os.path - -from mitmproxy.net import tls -from mitmproxy import version -from mitmproxy.net.http import user_agents -from . import print_tool_deprecation_message, pathoc, language - - -def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr): - preparser = argparse.ArgumentParser(add_help=False) - preparser.add_argument( - "--show-uas", dest="showua", action="store_true", default=False, - help="Print user agent shortcuts and exit." - ) - pa = preparser.parse_known_args(argv)[0] - if pa.showua: - print("User agent strings:", file=stdout) - for i in user_agents.UASTRINGS: - print(" ", i[1], i[0], file=stdout) - sys.exit(0) - - parser = argparse.ArgumentParser( - description='A perverse HTTP client.', parents=[preparser] - ) - parser.add_argument( - '--version', - action='version', - version="pathoc " + version.VERSION - ) - parser.add_argument( - "-c", dest="connect_to", type=str, default=False, - metavar="HOST:PORT", - help="Issue an HTTP CONNECT to connect to the specified host." - ) - parser.add_argument( - "--memo-limit", dest='memolimit', default=5000, type=int, metavar="N", - help='Stop if we do not find a valid request after N attempts.' - ) - parser.add_argument( - "-m", dest='memo', action="store_true", default=False, - help=""" - Remember specs, and never play the same one twice. Note that this - means requests have to be rendered in memory, which means that - large generated data can cause issues. - """ - ) - parser.add_argument( - "-n", dest='repeat', default=1, type=int, metavar="N", - help='Repeat N times. Pass -1 to repeat infinitely.' - ) - parser.add_argument( - "-w", dest='wait', default=0, type=float, metavar="N", - help='Wait N seconds between each request.' - ) - parser.add_argument( - "-r", dest="random", action="store_true", default=False, - help=""" - Select a random request from those specified. If this is not specified, - requests are all played in sequence. - """ - ) - parser.add_argument( - "-t", dest="timeout", type=int, default=None, - help="Connection timeout" - ) - parser.add_argument( - "--http2", dest="use_http2", action="store_true", default=False, - help='Perform all requests over a single HTTP/2 connection.' - ) - parser.add_argument( - "--http2-skip-connection-preface", - dest="http2_skip_connection_preface", - action="store_true", - default=False, - help='Skips the HTTP/2 connection preface before sending requests.') - - parser.add_argument( - 'host', type=str, - metavar="host[:port]", - help='Host and port to connect to' - ) - parser.add_argument( - 'requests', type=str, nargs="+", - help=""" - Request specification, or path to a file containing request - specifcations - """ - ) - - group = parser.add_argument_group( - 'SSL', - ) - group.add_argument( - "-s", dest="ssl", action="store_true", default=False, - help="Connect with SSL" - ) - group.add_argument( - "-C", dest="clientcert", type=str, default=False, - help="Path to a file containing client certificate and private key" - ) - group.add_argument( - "-i", dest="sni", type=str, default=False, - help="SSL Server Name Indication" - ) - group.add_argument( - "--ciphers", dest="ciphers", type=str, default=False, - help="SSL cipher specification" - ) - group.add_argument( - "--ssl-version", dest="ssl_version", type=str, default="secure", - choices=tls.VERSION_CHOICES.keys(), - help="Set supported SSL/TLS versions. " - "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." - ) - - group = parser.add_argument_group( - 'Controlling Output', - """ - Some of these options expand generated values for logging - if - you're generating large data, use them with caution. - """ - ) - group.add_argument( - "-I", dest="ignorecodes", type=str, default="", - help="Comma-separated list of response codes to ignore" - ) - group.add_argument( - "-S", dest="showssl", action="store_true", default=False, - help="Show info on SSL connection" - ) - group.add_argument( - "-e", dest="explain", action="store_true", default=False, - help="Explain requests" - ) - group.add_argument( - "-o", dest="oneshot", action="store_true", default=False, - help="Oneshot - exit after first non-ignored response" - ) - group.add_argument( - "-q", dest="showreq", action="store_true", default=False, - help="Print full request" - ) - group.add_argument( - "-p", dest="showresp", action="store_true", default=False, - help="Print full response" - ) - group.add_argument( - "-T", dest="ignoretimeout", action="store_true", default=False, - help="Ignore timeouts" - ) - group.add_argument( - "-x", dest="hexdump", action="store_true", default=False, - help="Output in hexdump format" - ) - group.add_argument( - "--http2-framedump", dest="http2_framedump", action="store_true", default=False, - help="Output all received & sent HTTP/2 frames" - ) - - args = parser.parse_args(argv[1:]) - - args.ssl_version, args.ssl_options = tls.VERSION_CHOICES[args.ssl_version] - - args.port = None - if ":" in args.host: - h, p = args.host.rsplit(":", 1) - try: - p = int(p) - except ValueError: - return parser.error("Invalid port in host spec: %s" % args.host) - args.host = h - args.port = p - - if args.port is None: - args.port = 443 if args.ssl else 80 - - try: - args.ignorecodes = [int(i) for i in args.ignorecodes.split(",") if i] - except ValueError: - return parser.error( - "Invalid return code specification: %s" % - args.ignorecodes) - - if args.connect_to: - parts = args.connect_to.split(":") - if len(parts) != 2: - return parser.error( - "Invalid CONNECT specification: %s" % - args.connect_to) - try: - parts[1] = int(parts[1]) - except ValueError: - return parser.error( - "Invalid CONNECT specification: %s" % - args.connect_to) - args.connect_to = parts - else: - args.connect_to = None - - if args.http2_skip_connection_preface: - args.use_http2 = True - - if args.use_http2: - args.ssl = True - - reqs = [] - for r in args.requests: - r = os.path.expanduser(r) - if os.path.isfile(r): - with open(r) as f: - r = f.read() - try: - reqs.append(language.parse_pathoc(r, args.use_http2)) - except language.ParseException as v: - print("Error parsing request spec: %s" % v.msg, file=stderr) - print(v.marked(), file=stderr) - sys.exit(1) - args.requests = reqs - return args - - -def go_pathoc(): # pragma: no cover - print_tool_deprecation_message() - args = args_pathoc(sys.argv) - pathoc.main(args) diff --git a/pathod/pathod.py b/pathod/pathod.py deleted file mode 100644 index d89baf18d..000000000 --- a/pathod/pathod.py +++ /dev/null @@ -1,496 +0,0 @@ -import copy -import logging -import os -import sys -import threading -import urllib -import typing # noqa - -from mitmproxy import certs as mcerts, exceptions, version -from mitmproxy.net import tcp, tls, websocket - -from pathod import language, utils, log, protocols - - -DEFAULT_CERT_DOMAIN = b"pathod.net" -CONFDIR = "~/.mitmproxy" -CERTSTORE_BASENAME = "mitmproxy" -CA_CERT_NAME = "mitmproxy-ca.pem" -DEFAULT_CRAFT_ANCHOR = "/p/" -KEY_SIZE = 2048 - -logger = logging.getLogger('pathod') - - -class PathodError(Exception): - pass - - -class SSLOptions: - def __init__( - self, - confdir=CONFDIR, - cn=None, - sans=(), - not_after_connect=None, - request_client_cert=False, - ssl_version=tls.DEFAULT_METHOD, - ssl_options=tls.DEFAULT_OPTIONS, - ciphers=None, - certs=None, - alpn_select=b'h2', - ): - self.confdir = confdir - self.cn = cn - self.sans = sans - self.not_after_connect = not_after_connect - self.request_client_cert = request_client_cert - self.ssl_version = ssl_version - self.ssl_options = ssl_options - self.ciphers = ciphers - self.alpn_select = alpn_select - self.certstore = mcerts.CertStore.from_store( - os.path.expanduser(confdir), - CERTSTORE_BASENAME, - KEY_SIZE - ) - for i in certs or []: - self.certstore.add_cert_file(*i) - - def get_cert(self, name): - if self.cn: - name = self.cn - elif not name: - name = DEFAULT_CERT_DOMAIN - return self.certstore.get_cert(name, self.sans) - - -class PathodHandler(tcp.BaseHandler): - wbufsize = 0 - sni: typing.Union[str, None, bool] = None - - def __init__( - self, - connection, - address, - server, - logfp, - settings, - http2_framedump=False - ): - tcp.BaseHandler.__init__(self, connection, address, server) - self.logfp = logfp - self.settings = copy.copy(settings) - self.protocol = None - self.use_http2 = False - self.http2_framedump = http2_framedump - - def handle_sni(self, connection): - sni = connection.get_servername() - if sni: - sni = sni.decode("idna") - self.sni = sni - - def http_serve_crafted(self, crafted, logctx): - error, crafted = self.server.check_policy( - crafted, self.settings - ) - if error: - err = self.make_http_error_response(error) - language.serve(err, self.wfile, self.settings) - return None, dict( - type="error", - msg=error - ) - - if self.server.explain and not hasattr(crafted, 'is_error_response'): - crafted = crafted.freeze(self.settings) - logctx(">> Spec: %s" % crafted.spec()) - - response_log = language.serve( - crafted, - self.wfile, - self.settings - ) - if response_log["disconnect"]: - return None, response_log - return self.handle_http_request, response_log - - def handle_http_request(self, logger): - """ - Returns a (handler, log) tuple. - - handler: Handler for the next request, or None to disconnect - log: A dictionary, or None - """ - with logger.ctx() as lg: - try: - req = self.protocol.read_request(self.rfile) - except exceptions.HttpReadDisconnect: - return None, None - except exceptions.HttpException as s: - s = str(s) - lg(s) - return None, dict(type="error", msg=s) - - if req.method == 'CONNECT': - return self.protocol.handle_http_connect([req.host, req.port, req.http_version], lg) - - method = req.method - path = req.path - http_version = req.http_version - headers = req.headers - first_line_format = req.first_line_format - - clientcert = None - if self.clientcert: - clientcert = dict( - cn=self.clientcert.cn, - subject=self.clientcert.subject, - serial=self.clientcert.serial, - notbefore=self.clientcert.notbefore.isoformat(), - notafter=self.clientcert.notafter.isoformat(), - keyinfo=self.clientcert.keyinfo, - ) - - retlog = dict( - type="crafted", - protocol="http", - request=dict( - path=path, - method=method, - headers=headers.fields, - http_version=http_version, - sni=self.sni, - remote_address=self.address, - clientcert=clientcert, - first_line_format=first_line_format - ), - cipher=None, - ) - if self.tls_established: - retlog["cipher"] = self.get_current_cipher() - - m = utils.MemBool() - - valid_websocket_handshake = websocket.check_handshake(headers) - self.settings.websocket_key = websocket.get_client_key(headers) - - # If this is a websocket initiation, we respond with a proper - # server response, unless over-ridden. - if valid_websocket_handshake: - anchor_gen = language.parse_pathod("ws") - else: - anchor_gen = None - - for regex, spec in self.server.anchors: - if regex.match(path): - anchor_gen = language.parse_pathod(spec, self.use_http2) - break - else: - if m(path.startswith(self.server.craftanchor)): - spec = urllib.parse.unquote(path)[len(self.server.craftanchor):] - if spec: - try: - anchor_gen = language.parse_pathod(spec, self.use_http2) - except language.ParseException as v: - lg("Parse error: %s" % v.msg) - anchor_gen = iter([self.make_http_error_response( - "Parse Error", - "Error parsing response spec: %s\n" % ( - v.msg + v.marked() - ) - )]) - else: - if self.use_http2: - anchor_gen = iter([self.make_http_error_response( - "Spec Error", - "HTTP/2 only supports request/response with the craft anchor point: %s" % - self.server.craftanchor - )]) - - if not anchor_gen: - anchor_gen = iter([self.make_http_error_response( - "Not found", - "No valid craft request found" - )]) - - spec = next(anchor_gen) - - if self.use_http2 and isinstance(spec, language.http2.Response): - spec.stream_id = req.stream_id - - lg("crafting spec: %s" % spec) - nexthandler, retlog["response"] = self.http_serve_crafted( - spec, - lg - ) - if nexthandler and valid_websocket_handshake: - self.protocol = protocols.websockets.WebsocketsProtocol(self) - return self.protocol.handle_websocket, retlog - else: - return nexthandler, retlog - - def make_http_error_response(self, reason, body=None): - resp = self.protocol.make_error_response(reason, body) - resp.is_error_response = True - return resp - - def handle(self): - self.settimeout(self.server.timeout) - - if self.server.ssl: - try: - cert, key, _ = self.server.ssloptions.get_cert(None) - self.convert_to_tls( - cert, - key, - handle_sni=self.handle_sni, - request_client_cert=self.server.ssloptions.request_client_cert, - cipher_list=self.server.ssloptions.ciphers, - method=self.server.ssloptions.ssl_version, - options=self.server.ssloptions.ssl_options, - alpn_select=self.server.ssloptions.alpn_select, - ) - except exceptions.TlsException as v: - s = str(v) - self.server.add_log( - dict( - type="error", - msg=s - ) - ) - log.write_raw(self.logfp, s) - return - - alp = self.get_alpn_proto_negotiated() - if alp == b'h2': - self.protocol = protocols.http2.HTTP2Protocol(self) - self.use_http2 = True - - if not self.protocol: - self.protocol = protocols.http.HTTPProtocol(self) - - lr = self.rfile if self.server.logreq else None - lw = self.wfile if self.server.logresp else None - logger = log.ConnectionLogger(self.logfp, self.server.hexdump, True, lr, lw) - - self.settings.protocol = self.protocol - - handler = self.handle_http_request - - while not self.finished: - handler, l = handler(logger) - if l: - self.addlog(l) - if not handler: - return - - def addlog(self, log): - if self.server.logreq: - log["request_bytes"] = self.rfile.get_log() - if self.server.logresp: - log["response_bytes"] = self.wfile.get_log() - self.server.add_log(log) - - -class Pathod(tcp.TCPServer): - LOGBUF = 500 - - def __init__( - self, - addr, - ssl=False, - ssloptions=None, - craftanchor=DEFAULT_CRAFT_ANCHOR, - staticdir=None, - anchors=(), - sizelimit=None, - nocraft=False, - nohang=False, - timeout=None, - logreq=False, - logresp=False, - explain=False, - hexdump=False, - http2_framedump=False, - webdebug=False, - logfp=sys.stdout, - ): - """ - addr: (address, port) tuple. If port is 0, a free port will be - automatically chosen. - ssloptions: an SSLOptions object. - craftanchor: URL prefix specifying the path under which to anchor - response generation. - staticdir: path to a directory of static resources, or None. - anchors: List of (regex object, language.Request object) tuples, or - None. - sizelimit: Limit size of served data. - nocraft: Disable response crafting. - nohang: Disable pauses. - """ - tcp.TCPServer.__init__(self, addr) - self.ssl = ssl - self.ssloptions = ssloptions or SSLOptions() - self.staticdir = staticdir - self.craftanchor = craftanchor - self.sizelimit = sizelimit - self.nocraft = nocraft - self.nohang = nohang - self.timeout, self.logreq = timeout, logreq - self.logresp, self.hexdump = logresp, hexdump - self.http2_framedump = http2_framedump - self.explain = explain - self.logfp = logfp - - self.log = [] - self.logid = 0 - self.anchors = anchors - - self.settings = language.Settings( - staticdir=self.staticdir - ) - - self.loglock = threading.Lock() - - def check_policy(self, req, settings): - """ - A policy check that verifies the request size is within limits. - """ - if self.nocraft: - return "Crafting disabled.", None - try: - req = req.resolve(settings) - l = req.maximum_length(settings) - except language.FileAccessDenied: - return "File access denied.", None - if self.sizelimit and l > self.sizelimit: - return "Response too large.", None - pauses = [isinstance(i, language.actions.PauseAt) for i in req.actions] - if self.nohang and any(pauses): - return "Pauses have been disabled.", None - return None, req - - def handle_client_connection(self, request, client_address): - h = PathodHandler( - request, - client_address, - self, - self.logfp, - self.settings, - self.http2_framedump, - ) - try: - h.handle() - h.finish() - except exceptions.TcpDisconnect: # pragma: no cover - log.write_raw(self.logfp, "Disconnect") - self.add_log( - dict( - type="error", - msg="Disconnect" - ) - ) - return - except exceptions.TcpTimeout: - log.write_raw(self.logfp, "Timeout") - self.add_log( - dict( - type="timeout", - ) - ) - return - - def add_log(self, d): - with self.loglock: - d["id"] = self.logid - self.log.insert(0, d) - if len(self.log) > self.LOGBUF: - self.log.pop() - self.logid += 1 - return d["id"] - - def clear_log(self): - with self.loglock: - self.log = [] - - def log_by_id(self, identifier): - with self.loglock: - for i in self.log: - if i["id"] == identifier: - return i - - def get_log(self): - with self.loglock: - return self.log - - -def main(args): # pragma: no cover - ssloptions = SSLOptions( - cn=args.cn, - confdir=args.confdir, - not_after_connect=args.ssl_not_after_connect, - ciphers=args.ciphers, - ssl_version=args.ssl_version, - ssl_options=args.ssl_options, - certs=args.ssl_certs, - sans=args.sans, - ) - - root = logging.getLogger() - if root.handlers: - for handler in root.handlers: - root.removeHandler(handler) - - log = logging.getLogger('pathod') - log.setLevel(logging.DEBUG) - fmt = logging.Formatter( - '%(asctime)s: %(message)s', - datefmt='%d-%m-%y %H:%M:%S', - ) - if args.logfile: - fh = logging.handlers.WatchedFileHandler(args.logfile) - fh.setFormatter(fmt) - log.addHandler(fh) - if not args.daemonize: - sh = logging.StreamHandler() - sh.setFormatter(fmt) - log.addHandler(sh) - - try: - pd = Pathod( - (args.address, args.port), - craftanchor=args.craftanchor, - ssl=args.ssl, - ssloptions=ssloptions, - staticdir=args.staticdir, - anchors=args.anchors, - sizelimit=args.sizelimit, - nocraft=args.nocraft, - nohang=args.nohang, - timeout=args.timeout, - logreq=args.logreq, - logresp=args.logresp, - hexdump=args.hexdump, - http2_framedump=args.http2_framedump, - explain=args.explain, - webdebug=args.webdebug - ) - except PathodError as v: - print("Error: %s" % v, file=sys.stderr) - sys.exit(1) - except language.FileAccessDenied as v: - print("Error: %s" % v, file=sys.stderr) - - if args.daemonize: - utils.daemonize() - - try: - print("{} listening on {}".format( - version.PATHOD, - repr(pd.address) - )) - pd.serve_forever() - except KeyboardInterrupt: - pass diff --git a/pathod/pathod_cmdline.py b/pathod/pathod_cmdline.py deleted file mode 100644 index feda28f13..000000000 --- a/pathod/pathod_cmdline.py +++ /dev/null @@ -1,236 +0,0 @@ -import sys -import argparse -import os -import os.path -import re - -from mitmproxy.net import tls -from mitmproxy.utils import human -from mitmproxy import version -from . import print_tool_deprecation_message, pathod - - -def parse_anchor_spec(s): - """ - Return a tuple, or None on error. - """ - if "=" not in s: - return None - return tuple(s.split("=", 1)) - - -def args_pathod(argv, stdout_=sys.stdout, stderr_=sys.stderr): - parser = argparse.ArgumentParser( - description='A pathological HTTP/S daemon.' - ) - parser.add_argument( - '--version', - action='version', - version="pathod " + version.VERSION - ) - parser.add_argument( - "-p", - dest='port', - default=9999, - type=int, - help='Port. Specify 0 to pick an arbitrary empty port. (9999)' - ) - parser.add_argument( - "-l", - dest='address', - default="127.0.0.1", - type=str, - help='Listening address. (127.0.0.1)' - ) - parser.add_argument( - "-a", - dest='anchors', - default=[], - type=str, - action="append", - metavar="ANCHOR", - help=""" - Add an anchor. Specified as a string with the form - pattern=spec or pattern=filepath, where pattern is a regular - expression. - """ - ) - parser.add_argument( - "-c", dest='craftanchor', default=pathod.DEFAULT_CRAFT_ANCHOR, type=str, - help=""" - URL path specifying prefix for URL crafting - commands. (%s) - """ % pathod.DEFAULT_CRAFT_ANCHOR - ) - parser.add_argument( - "--confdir", - action="store", type=str, dest="confdir", default='~/.mitmproxy', - help="Configuration directory. (~/.mitmproxy)" - ) - parser.add_argument( - "-d", dest='staticdir', default=None, type=str, - help='Directory for static files.' - ) - parser.add_argument( - "-D", dest='daemonize', default=False, action="store_true", - help='Daemonize.' - ) - parser.add_argument( - "-t", dest="timeout", type=int, default=None, - help="Connection timeout" - ) - parser.add_argument( - "--limit-size", - dest='sizelimit', - default=None, - type=str, - help='Size limit of served responses. Understands size suffixes, i.e. 100k.') - parser.add_argument( - "--nohang", dest='nohang', default=False, action="store_true", - help='Disable pauses during crafted response generation.' - ) - parser.add_argument( - "--nocraft", - dest='nocraft', - default=False, - action="store_true", - help='Disable response crafting. If anchors are specified, they still work.') - parser.add_argument( - "--webdebug", dest='webdebug', default=False, action="store_true", - help='Debugging mode for the web app (dev only).' - ) - - group = parser.add_argument_group( - 'SSL', - ) - group.add_argument( - "-s", dest='ssl', default=False, action="store_true", - help='Run in HTTPS mode.' - ) - group.add_argument( - "--cn", - dest="cn", - type=str, - default=None, - help="CN for generated SSL certs. Default: %s" % - pathod.DEFAULT_CERT_DOMAIN) - group.add_argument( - "-C", dest='ssl_not_after_connect', default=False, action="store_true", - help="Don't expect SSL after a CONNECT request." - ) - group.add_argument( - "--cert", dest='ssl_certs', default=[], type=str, - metavar="SPEC", action="append", - help=""" - Add an SSL certificate. SPEC is of the form "[domain=]path". The domain - may include a wildcard, and is equal to "*" if not specified. The file - at path is a certificate in PEM format. If a private key is included in - the PEM, it is used, else the default key in the conf dir is used. Can - be passed multiple times. - """ - ) - group.add_argument( - "--ciphers", dest="ciphers", type=str, default=False, - help="SSL cipher specification" - ) - group.add_argument( - "--san", dest="sans", type=str, default=[], action="append", - metavar="SAN", - help=""" - Subject Altnernate Name to add to the server certificate. - May be passed multiple times. - """ - ) - group.add_argument( - "--ssl-version", dest="ssl_version", type=str, default="secure", - choices=tls.VERSION_CHOICES.keys(), - help="Set supported SSL/TLS versions. " - "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." - ) - - group = parser.add_argument_group( - 'Controlling Logging', - """ - Some of these options expand generated values for logging - if - you're generating large data, use them with caution. - """ - ) - group.add_argument( - "-e", dest="explain", action="store_true", default=False, - help="Explain responses" - ) - group.add_argument( - "-f", dest='logfile', default=None, type=str, - help='Log to file.' - ) - group.add_argument( - "-q", dest="logreq", action="store_true", default=False, - help="Log full request" - ) - group.add_argument( - "-r", dest="logresp", action="store_true", default=False, - help="Log full response" - ) - group.add_argument( - "-x", dest="hexdump", action="store_true", default=False, - help="Log request/response in hexdump format" - ) - group.add_argument( - "--http2-framedump", dest="http2_framedump", action="store_true", default=False, - help="Output all received & sent HTTP/2 frames" - ) - - args = parser.parse_args(argv[1:]) - - args.ssl_version, args.ssl_options = tls.VERSION_CHOICES[args.ssl_version] - - certs = [] - for i in args.ssl_certs: - parts = i.split("=", 1) - if len(parts) == 1: - parts = ["*", parts[0]] - parts[1] = os.path.expanduser(parts[1]) - if not os.path.isfile(parts[1]): - return parser.error( - "Certificate file does not exist: %s" % - parts[1]) - certs.append(parts) - args.ssl_certs = certs - - alst = [] - for i in args.anchors: - parts = parse_anchor_spec(i) - if not parts: - return parser.error("Invalid anchor specification: %s" % i) - alst.append(parts) - args.anchors = alst - - sizelimit = None - if args.sizelimit: - try: - sizelimit = human.parse_size(args.sizelimit) - except ValueError as v: - return parser.error(v) - args.sizelimit = sizelimit - - anchors = [] - for patt, spec in args.anchors: - spec = os.path.expanduser(spec) - if os.path.isfile(spec): - with open(spec) as f: - data = f.read() - spec = data - try: - arex = re.compile(patt) - except re.error: - return parser.error("Invalid regex in anchor: %s" % patt) - anchors.append((arex, spec)) - args.anchors = anchors - - return args - - -def go_pathod(): # pragma: no cover - print_tool_deprecation_message() - args = args_pathod(sys.argv) - pathod.main(args) diff --git a/pathod/protocols/__init__.py b/pathod/protocols/__init__.py deleted file mode 100644 index f8f3008fc..000000000 --- a/pathod/protocols/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from . import http, http2, websockets - -__all__ = [ - "http", - "http2", - "websockets", -] diff --git a/pathod/protocols/http.py b/pathod/protocols/http.py deleted file mode 100644 index 5fcb66182..000000000 --- a/pathod/protocols/http.py +++ /dev/null @@ -1,47 +0,0 @@ -from mitmproxy import version -from mitmproxy import exceptions -from mitmproxy.net.http import http1 -from .. import language - - -class HTTPProtocol: - def __init__(self, pathod_handler): - self.pathod_handler = pathod_handler - - def make_error_response(self, reason, body): - return language.http.make_error_response(reason, body) - - def handle_http_connect(self, connect, lg): - """ - Handle a CONNECT request. - """ - - self.pathod_handler.wfile.write( - b'HTTP/1.1 200 Connection established\r\n' + - (b'Proxy-agent: %s\r\n' % version.PATHOD.encode()) + - b'\r\n' - ) - self.pathod_handler.wfile.flush() - if not self.pathod_handler.server.ssloptions.not_after_connect: - try: - cert, key, chain_file_ = self.pathod_handler.server.ssloptions.get_cert( - connect[0].encode() - ) - self.pathod_handler.convert_to_tls( - cert, - key, - handle_sni=self.pathod_handler.handle_sni, - request_client_cert=self.pathod_handler.server.ssloptions.request_client_cert, - cipher_list=self.pathod_handler.server.ssloptions.ciphers, - method=self.pathod_handler.server.ssloptions.ssl_version, - options=self.pathod_handler.server.ssloptions.ssl_options, - alpn_select=self.pathod_handler.server.ssloptions.alpn_select, - ) - except exceptions.TlsException as v: - s = str(v) - lg(s) - return None, dict(type="error", msg=s) - return self.pathod_handler.handle_http_request, None - - def read_request(self, lg=None): - return http1.read_request(self.pathod_handler.rfile) diff --git a/pathod/protocols/http2.py b/pathod/protocols/http2.py deleted file mode 100644 index 966a0e460..000000000 --- a/pathod/protocols/http2.py +++ /dev/null @@ -1,429 +0,0 @@ -import itertools -import time - -import hyperframe.frame -from hpack.hpack import Decoder, Encoder - -import mitmproxy.net.http.headers -import mitmproxy.net.http.request -import mitmproxy.net.http.response -from mitmproxy.coretypes import bidi -from mitmproxy.net.http import http2, url -from .. import language - - -class TCPHandler: - - def __init__(self, rfile, wfile=None): - self.rfile = rfile - self.wfile = wfile - - -class HTTP2StateProtocol: - - ERROR_CODES = bidi.BiDi( - NO_ERROR=0x0, - PROTOCOL_ERROR=0x1, - INTERNAL_ERROR=0x2, - FLOW_CONTROL_ERROR=0x3, - SETTINGS_TIMEOUT=0x4, - STREAM_CLOSED=0x5, - FRAME_SIZE_ERROR=0x6, - REFUSED_STREAM=0x7, - CANCEL=0x8, - COMPRESSION_ERROR=0x9, - CONNECT_ERROR=0xa, - ENHANCE_YOUR_CALM=0xb, - INADEQUATE_SECURITY=0xc, - HTTP_1_1_REQUIRED=0xd - ) - - CLIENT_CONNECTION_PREFACE = b'PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n' - - HTTP2_DEFAULT_SETTINGS = { - hyperframe.frame.SettingsFrame.HEADER_TABLE_SIZE: 4096, - hyperframe.frame.SettingsFrame.ENABLE_PUSH: 1, - hyperframe.frame.SettingsFrame.MAX_CONCURRENT_STREAMS: None, - hyperframe.frame.SettingsFrame.INITIAL_WINDOW_SIZE: 2 ** 16 - 1, - hyperframe.frame.SettingsFrame.MAX_FRAME_SIZE: 2 ** 14, - hyperframe.frame.SettingsFrame.MAX_HEADER_LIST_SIZE: None, - } - - def __init__( - self, - tcp_handler=None, - rfile=None, - wfile=None, - is_server=False, - dump_frames=False, - encoder=None, - decoder=None, - unhandled_frame_cb=None, - ): - self.tcp_handler = tcp_handler or TCPHandler(rfile, wfile) - self.is_server = is_server - self.dump_frames = dump_frames - self.encoder = encoder or Encoder() - self.decoder = decoder or Decoder() - self.unhandled_frame_cb = unhandled_frame_cb - - self.http2_settings = self.HTTP2_DEFAULT_SETTINGS.copy() - self.current_stream_id = None - self.connection_preface_performed = False - - def read_request( - self, - __rfile, - include_body=True, - body_size_limit=None, - allow_empty=False, - ): - if body_size_limit is not None: - raise NotImplementedError() - - self.perform_connection_preface() - - timestamp_start = time.time() - if hasattr(self.tcp_handler.rfile, "reset_timestamps"): - self.tcp_handler.rfile.reset_timestamps() - - stream_id, headers, body = self._receive_transmission( - include_body=include_body, - ) - - if hasattr(self.tcp_handler.rfile, "first_byte_timestamp"): - # more accurate timestamp_start - timestamp_start = self.tcp_handler.rfile.first_byte_timestamp - - timestamp_end = time.time() - - # pseudo header must be present, see https://http2.github.io/http2-spec/#rfc.section.8.1.2.3 - authority = headers.pop(':authority', "") - method = headers.pop(':method', "") - scheme = headers.pop(':scheme', "") - path = headers.pop(':path', "") - - host, port = url.parse_authority(authority, check=False) - port = port or url.default_port(scheme) or 0 - - request = mitmproxy.net.http.Request( - host=host, - port=port, - method=method.encode(), - scheme=scheme.encode(), - authority=authority.encode(), - path=path.encode(), - http_version=b"HTTP/2.0", - headers=headers, - content=body, - trailers=None, - timestamp_start=timestamp_start, - timestamp_end=timestamp_end, - ) - request.stream_id = stream_id - - return request - - def read_response( - self, - __rfile, - request_method=b'', - body_size_limit=None, - include_body=True, - stream_id=None, - ): - if body_size_limit is not None: - raise NotImplementedError() - - self.perform_connection_preface() - - timestamp_start = time.time() - if hasattr(self.tcp_handler.rfile, "reset_timestamps"): - self.tcp_handler.rfile.reset_timestamps() - - stream_id, headers, body = self._receive_transmission( - stream_id=stream_id, - include_body=include_body, - ) - - if hasattr(self.tcp_handler.rfile, "first_byte_timestamp"): - # more accurate timestamp_start - timestamp_start = self.tcp_handler.rfile.first_byte_timestamp - - if include_body: - timestamp_end = time.time() - else: - timestamp_end = None - - response = mitmproxy.net.http.response.Response( - http_version=b"HTTP/2.0", - status_code=int(headers.get(':status', 502)), - reason=b'', - headers=headers, - content=body, - trailers=None, - timestamp_start=timestamp_start, - timestamp_end=timestamp_end, - ) - response.stream_id = stream_id - - return response - - def assemble(self, message): - if isinstance(message, mitmproxy.net.http.request.Request): - return self.assemble_request(message) - elif isinstance(message, mitmproxy.net.http.response.Response): - return self.assemble_response(message) - else: - raise ValueError("HTTP message not supported.") - - def assemble_request(self, request): - assert isinstance(request, mitmproxy.net.http.request.Request) - - authority = self.tcp_handler.sni if self.tcp_handler.sni else self.tcp_handler.address[0] - if self.tcp_handler.address[1] != 443: - authority += ":%d" % self.tcp_handler.address[1] - - headers = request.headers.copy() - - if ':authority' not in headers: - headers.insert(0, ':authority', authority) - headers.insert(0, ':scheme', request.scheme) - headers.insert(0, ':path', request.path) - headers.insert(0, ':method', request.method) - - if hasattr(request, 'stream_id'): - stream_id = request.stream_id - else: - stream_id = self._next_stream_id() - - return list(itertools.chain( - self._create_headers(headers, stream_id, end_stream=(request.content is None or len(request.content) == 0)), - self._create_body(request.content, stream_id))) - - def assemble_response(self, response): - assert isinstance(response, mitmproxy.net.http.response.Response) - - headers = response.headers.copy() - - if ':status' not in headers: - headers.insert(0, b':status', str(response.status_code).encode()) - - if hasattr(response, 'stream_id'): - stream_id = response.stream_id - else: - stream_id = self._next_stream_id() - - return list(itertools.chain( - self._create_headers(headers, stream_id, end_stream=(response.content is None or len(response.content) == 0)), - self._create_body(response.content, stream_id), - )) - - def perform_connection_preface(self, force=False): - if force or not self.connection_preface_performed: - if self.is_server: - self.perform_server_connection_preface(force) - else: - self.perform_client_connection_preface(force) - - def perform_server_connection_preface(self, force=False): - if force or not self.connection_preface_performed: - self.connection_preface_performed = True - - magic_length = len(self.CLIENT_CONNECTION_PREFACE) - magic = self.tcp_handler.rfile.safe_read(magic_length) - assert magic == self.CLIENT_CONNECTION_PREFACE - - frm = hyperframe.frame.SettingsFrame(settings={ - hyperframe.frame.SettingsFrame.ENABLE_PUSH: 0, - hyperframe.frame.SettingsFrame.MAX_CONCURRENT_STREAMS: 1, - }) - self.send_frame(frm, hide=True) - self._receive_settings(hide=True) - - def perform_client_connection_preface(self, force=False): - if force or not self.connection_preface_performed: - self.connection_preface_performed = True - - self.tcp_handler.wfile.write(self.CLIENT_CONNECTION_PREFACE) - - self.send_frame(hyperframe.frame.SettingsFrame(), hide=True) - self._receive_settings(hide=True) # server announces own settings - self._receive_settings(hide=True) # server acks my settings - - def send_frame(self, frm, hide=False): - raw_bytes = frm.serialize() - self.tcp_handler.wfile.write(raw_bytes) - self.tcp_handler.wfile.flush() - if not hide and self.dump_frames: # pragma: no cover - print(">> " + repr(frm)) - - def read_frame(self, hide=False): - while True: - frm, _ = http2.read_frame(self.tcp_handler.rfile) - - if not hide and self.dump_frames: # pragma: no cover - print("<< " + repr(frm)) - - if isinstance(frm, hyperframe.frame.PingFrame): - raw_bytes = hyperframe.frame.PingFrame(flags=['ACK'], payload=frm.payload).serialize() - self.tcp_handler.wfile.write(raw_bytes) - self.tcp_handler.wfile.flush() - continue - if isinstance(frm, hyperframe.frame.SettingsFrame) and 'ACK' not in frm.flags: - self._apply_settings(frm.settings, hide) - if isinstance(frm, hyperframe.frame.DataFrame) and frm.flow_controlled_length > 0: - self._update_flow_control_window(frm.stream_id, frm.flow_controlled_length) - return frm - - def check_alpn(self): - alp = self.tcp_handler.get_alpn_proto_negotiated() - if alp != b'h2': - raise NotImplementedError( - "HTTP2Protocol can not handle unknown ALPN value: %s" % alp) - return True - - def _handle_unexpected_frame(self, frm): - if isinstance(frm, hyperframe.frame.SettingsFrame): - return - if self.unhandled_frame_cb: - self.unhandled_frame_cb(frm) - - def _receive_settings(self, hide=False): - while True: - frm = self.read_frame(hide) - if isinstance(frm, hyperframe.frame.SettingsFrame): - break - else: - self._handle_unexpected_frame(frm) - - def _next_stream_id(self): - if self.current_stream_id is None: - if self.is_server: - # servers must use even stream ids - self.current_stream_id = 2 - else: - # clients must use odd stream ids - self.current_stream_id = 1 - else: - self.current_stream_id += 2 - return self.current_stream_id - - def _apply_settings(self, settings, hide=False): - for setting, value in settings.items(): - old_value = self.http2_settings[setting] - if not old_value: - old_value = '-' - self.http2_settings[setting] = value - - frm = hyperframe.frame.SettingsFrame(flags=['ACK']) - self.send_frame(frm, hide) - - def _update_flow_control_window(self, stream_id, increment): - frm = hyperframe.frame.WindowUpdateFrame(stream_id=0, window_increment=increment) - self.send_frame(frm) - frm = hyperframe.frame.WindowUpdateFrame(stream_id=stream_id, window_increment=increment) - self.send_frame(frm) - - def _create_headers(self, headers, stream_id, end_stream=True): - def frame_cls(chunks): - for i in chunks: - if i == 0: - yield hyperframe.frame.HeadersFrame, i - else: - yield hyperframe.frame.ContinuationFrame, i - - header_block_fragment = self.encoder.encode(headers.fields) - - chunk_size = self.http2_settings[hyperframe.frame.SettingsFrame.MAX_FRAME_SIZE] - chunks = range(0, len(header_block_fragment), chunk_size) - frms = [frm_cls( - flags=[], - stream_id=stream_id, - data=header_block_fragment[i:i + chunk_size]) for frm_cls, i in frame_cls(chunks)] - - frms[-1].flags.add('END_HEADERS') - if end_stream: - frms[0].flags.add('END_STREAM') - - if self.dump_frames: # pragma: no cover - for frm in frms: - print(">> ", repr(frm)) - - return [frm.serialize() for frm in frms] - - def _create_body(self, body, stream_id): - if body is None or len(body) == 0: - return b'' - - chunk_size = self.http2_settings[hyperframe.frame.SettingsFrame.MAX_FRAME_SIZE] - chunks = range(0, len(body), chunk_size) - frms = [hyperframe.frame.DataFrame( - flags=[], - stream_id=stream_id, - data=body[i:i + chunk_size]) for i in chunks] - frms[-1].flags.add('END_STREAM') - - if self.dump_frames: # pragma: no cover - for frm in frms: - print(">> ", repr(frm)) - - return [frm.serialize() for frm in frms] - - def _receive_transmission(self, stream_id=None, include_body=True): - if not include_body: - raise NotImplementedError() - - body_expected = True - - header_blocks = b'' - body = b'' - - while True: - frm = self.read_frame() - if ( - (isinstance(frm, hyperframe.frame.HeadersFrame) or isinstance(frm, hyperframe.frame.ContinuationFrame)) and - (stream_id is None or frm.stream_id == stream_id) - ): - stream_id = frm.stream_id - header_blocks += frm.data - if 'END_STREAM' in frm.flags: - body_expected = False - if 'END_HEADERS' in frm.flags: - break - else: - self._handle_unexpected_frame(frm) - - while body_expected: - frm = self.read_frame() - if isinstance(frm, hyperframe.frame.DataFrame) and frm.stream_id == stream_id: - body += frm.data - if 'END_STREAM' in frm.flags: - break - else: - self._handle_unexpected_frame(frm) - - headers = mitmproxy.net.http.headers.Headers( - [[k, v] for k, v in self.decoder.decode(header_blocks, raw=True)] - ) - - return stream_id, headers, body - - -class HTTP2Protocol: - - def __init__(self, pathod_handler): - self.pathod_handler = pathod_handler - self.wire_protocol = HTTP2StateProtocol( - self.pathod_handler, is_server=True, dump_frames=self.pathod_handler.http2_framedump - ) - - def make_error_response(self, reason, body): - return language.http2.make_error_response(reason, body) - - def read_request(self, lg=None): - self.wire_protocol.perform_server_connection_preface() - return self.wire_protocol.read_request(self.pathod_handler.rfile) - - def assemble(self, message): - return self.wire_protocol.assemble(message) diff --git a/pathod/protocols/websockets.py b/pathod/protocols/websockets.py deleted file mode 100644 index b1fb8a290..000000000 --- a/pathod/protocols/websockets.py +++ /dev/null @@ -1,54 +0,0 @@ -import time - -from pathod import language -from mitmproxy import exceptions - - -class WebsocketsProtocol: - - def __init__(self, pathod_handler): - self.pathod_handler = pathod_handler - - def handle_websocket(self, logger): - while True: - with logger.ctx() as lg: - started = time.time() - try: - frm = language.websockets_frame.Frame.from_file(self.pathod_handler.rfile) - except exceptions.NetlibException as e: - lg("Error reading websocket frame: %s" % e) - return None, None - ended = time.time() - lg(repr(frm)) - retlog = dict( - type="inbound", - protocol="websockets", - started=started, - duration=ended - started, - frame=dict( - ), - cipher=None, - ) - if self.pathod_handler.tls_established: - retlog["cipher"] = self.pathod_handler.get_current_cipher() - self.pathod_handler.addlog(retlog) - ld = language.websockets.NESTED_LEADER - if frm.payload.startswith(ld): - nest = frm.payload[len(ld):] - try: - wf_gen = language.parse_websocket_frame(nest.decode()) - except language.exceptions.ParseException as v: - logger.write( - "Parse error in reflected frame specifcation:" - " %s" % v.msg - ) - return None, None - for frm in wf_gen: - with logger.ctx() as lg: - frame_log = language.serve( - frm, - self.pathod_handler.wfile, - self.pathod_handler.settings - ) - lg("crafting websocket spec: %s" % frame_log["spec"]) - self.pathod_handler.addlog(frame_log) diff --git a/pathod/test.py b/pathod/test.py deleted file mode 100644 index 425891ea1..000000000 --- a/pathod/test.py +++ /dev/null @@ -1,104 +0,0 @@ -import io -import time -import queue -from . import pathod -from mitmproxy.coretypes import basethread -import typing # noqa - - -class Daemon: - IFACE = "127.0.0.1" - - def __init__(self, ssl=None, **daemonargs) -> None: - self.q: queue.Queue = queue.Queue() - self.logfp = io.StringIO() - daemonargs["logfp"] = self.logfp - self.thread = _PaThread(self.IFACE, self.q, ssl, daemonargs) - self.thread.start() - self.port = self.q.get(True, 5) - self.urlbase = "{}://{}:{}".format( - "https" if ssl else "http", - self.IFACE, - self.port - ) - - def __enter__(self): - return self - - def __exit__(self, type, value, traceback): - self.logfp.truncate(0) - self.shutdown() - return False - - def p(self, spec: str) -> str: - """ - Return a URL that will render the response in spec. - """ - return f"{self.urlbase}/p/{spec}" - - def text_log(self) -> str: - return self.logfp.getvalue() - - def wait_for_silence(self, timeout=5): - self.thread.server.wait_for_silence(timeout=timeout) - - def expect_log(self, n, timeout=5): - l = [] - start = time.time() - while True: - l = self.log() - if time.time() - start >= timeout: - return None - if len(l) >= n: - break - return l - - def last_log(self): - """ - Returns the last logged request, or None. - """ - l = self.expect_log(1) - if not l: - return None - return l[-1] - - def log(self) -> typing.List[typing.Dict]: - """ - Return the log buffer as a list of dictionaries. - """ - return self.thread.server.get_log() - - def clear_log(self): - """ - Clear the log. - """ - return self.thread.server.clear_log() - - def shutdown(self): - """ - Shut the daemon down, return after the thread has exited. - """ - self.thread.server.shutdown() - self.thread.join() - - -class _PaThread(basethread.BaseThread): - - def __init__(self, iface, q, ssl, daemonargs): - basethread.BaseThread.__init__(self, "PathodThread") - self.iface, self.q, self.ssl = iface, q, ssl - self.daemonargs = daemonargs - self.server = None - - def run(self): - self.server = pathod.Pathod( - (self.iface, 0), - ssl=self.ssl, - **self.daemonargs - ) - self.name = "PathodThread ({}:{})".format( - self.server.address[0], - self.server.address[1], - ) - self.q.put(self.server.address[1]) - self.server.serve_forever() diff --git a/pathod/utils.py b/pathod/utils.py deleted file mode 100644 index 4ef854398..000000000 --- a/pathod/utils.py +++ /dev/null @@ -1,48 +0,0 @@ -import os -import sys -from mitmproxy.utils import data as mdata -import typing # noqa - - -class MemBool: - - """ - Truth-checking with a memory, for use in chained if statements. - """ - - def __init__(self) -> None: - self.v: typing.Optional[bool] = None - - def __call__(self, v: bool) -> bool: - self.v = v - return bool(v) - - -# FIXME: change this name -data = mdata.Data(__name__) - - -def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'): # pragma: no cover - try: - pid = os.fork() - if pid > 0: - sys.exit(0) - except OSError as e: - sys.stderr.write("fork #1 failed: (%d) %s\n" % (e.errno, e.strerror)) - sys.exit(1) - os.chdir("/") - os.umask(0) - os.setsid() - try: - pid = os.fork() - if pid > 0: - sys.exit(0) - except OSError as e: - sys.stderr.write("fork #2 failed: (%d) %s\n" % (e.errno, e.strerror)) - sys.exit(1) - si = open(stdin, 'rb') - so = open(stdout, 'a+b') - se = open(stderr, 'a+b', 0) - os.dup2(si.fileno(), sys.stdin.fileno()) - os.dup2(so.fileno(), sys.stdout.fileno()) - os.dup2(se.fileno(), sys.stderr.fileno()) diff --git a/release/cibuild.py b/release/cibuild.py index fa14e4ba1..3af43e966 100755 --- a/release/cibuild.py +++ b/release/cibuild.py @@ -149,7 +149,6 @@ class BuildEnviron: def bdists(self): ret = { "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], - "pathod": ["pathoc", "pathod"] } if self.system == "Windows": ret["mitmproxy"].remove("mitmproxy") diff --git a/release/installbuilder/mitmproxy.xml b/release/installbuilder/mitmproxy.xml index e4ad00a3f..ed0c94733 100644 --- a/release/installbuilder/mitmproxy.xml +++ b/release/installbuilder/mitmproxy.xml @@ -31,7 +31,6 @@ 1 - */patho* ../build/binaries/${platform_name}/* @@ -132,4 +131,3 @@ - diff --git a/release/specs/pathoc b/release/specs/pathoc deleted file mode 100644 index ab62d6318..000000000 --- a/release/specs/pathoc +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python - -from pathod import pathoc_cmdline as cmdline - -if __name__ == "__main__": - cmdline.go_pathoc() diff --git a/release/specs/pathod b/release/specs/pathod deleted file mode 100644 index af0c118b3..000000000 --- a/release/specs/pathod +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env python - -from pathod import pathod_cmdline as cmdline - -if __name__ == "__main__": - cmdline.go_pathod() diff --git a/setup.cfg b/setup.cfg index 9e00816bb..7347e80b6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -47,10 +47,6 @@ exclude = mitmproxy/proxy/root_context.py mitmproxy/proxy/server.py mitmproxy/tools/ - pathod/pathoc.py - pathod/pathod.py - pathod/test.py - pathod/protocols/http2.py release/hooks @@ -97,17 +93,4 @@ exclude = mitmproxy/proxy2/server.py mitmproxy/proxy2/layers/tls.py mitmproxy/utils/bits.py - pathod/language/actions.py - pathod/language/base.py - pathod/language/exceptions.py - pathod/language/generators.py - pathod/language/http.py - pathod/language/message.py - pathod/log.py - pathod/pathoc.py - pathod/pathod.py - pathod/protocols/http.py - pathod/protocols/http2.py - pathod/protocols/websockets.py - pathod/test.py release/hooks diff --git a/setup.py b/setup.py index c7a13f493..4b6435e02 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,6 @@ setup( }, packages=find_packages(include=[ "mitmproxy", "mitmproxy.*", - "pathod", "pathod.*", ]), include_package_data=True, entry_points={ @@ -61,8 +60,6 @@ setup( "mitmproxy = mitmproxy.tools.main:mitmproxy", "mitmdump = mitmproxy.tools.main:mitmdump", "mitmweb = mitmproxy.tools.main:mitmweb", - "pathod = pathod.pathod_cmdline:go_pathod", - "pathoc = pathod.pathoc_cmdline:go_pathoc" ] }, python_requires='>=3.8', diff --git a/test/filename_matching.py b/test/filename_matching.py index 1257ce505..7cff79aaa 100755 --- a/test/filename_matching.py +++ b/test/filename_matching.py @@ -11,7 +11,7 @@ def check_src_files_have_test(): excluded = ['mitmproxy/contrib/', 'mitmproxy/io/proto/', 'mitmproxy/proxy2/layers/http', 'mitmproxy/test/', 'mitmproxy/tools/', 'mitmproxy/platform/'] - src_files = glob.glob('mitmproxy/**/*.py', recursive=True) + glob.glob('pathod/**/*.py', recursive=True) + src_files = glob.glob('mitmproxy/**/*.py', recursive=True) src_files = [f for f in src_files if os.path.basename(f) != '__init__.py'] src_files = [f for f in src_files if not any(os.path.normpath(p) in f for p in excluded)] for f in src_files: @@ -26,7 +26,7 @@ def check_test_files_have_src(): unknown_test_files = [] excluded = ['test/mitmproxy/data/', 'test/mitmproxy/net/data/', '/tservers.py', '/conftest.py'] - test_files = glob.glob('test/mitmproxy/**/*.py', recursive=True) + glob.glob('test/pathod/**/*.py', recursive=True) + test_files = glob.glob('test/mitmproxy/**/*.py', recursive=True) test_files = [f for f in test_files if os.path.basename(f) != '__init__.py'] test_files = [f for f in test_files if not any(os.path.normpath(p) in f for p in excluded)] for f in test_files: diff --git a/test/individual_coverage.py b/test/individual_coverage.py index 0811b8d2a..c969b4b82 100755 --- a/test/individual_coverage.py +++ b/test/individual_coverage.py @@ -63,7 +63,7 @@ def main(): no_individual_cov = [f.strip() for f in fs] excluded = ['mitmproxy/contrib/', 'mitmproxy/test/', 'mitmproxy/tools/', 'mitmproxy/platform/'] - src_files = glob.glob('mitmproxy/**/*.py', recursive=True) + glob.glob('pathod/**/*.py', recursive=True) + src_files = glob.glob('mitmproxy/**/*.py', recursive=True) src_files = [f for f in src_files if os.path.basename(f) != '__init__.py'] src_files = [f for f in src_files if not any(os.path.normpath(p) in f for p in excluded)] diff --git a/test/mitmproxy/fuzzing/.env b/test/mitmproxy/fuzzing/.env deleted file mode 100644 index 8923b8aea..000000000 --- a/test/mitmproxy/fuzzing/.env +++ /dev/null @@ -1,4 +0,0 @@ -MITMDUMP=mitmdump -PATHOD=pathod -PATHOC=pathoc -FUZZ_SETTINGS="-remTt 1 -n 0" diff --git a/test/mitmproxy/fuzzing/README b/test/mitmproxy/fuzzing/README deleted file mode 100644 index 2760506fc..000000000 --- a/test/mitmproxy/fuzzing/README +++ /dev/null @@ -1,14 +0,0 @@ - -A fuzzing architecture for mitmproxy -==================================== - -Quick start: - - honcho -f ./straight_stream start - - -Notes: - - - Processes are managed using honcho (pip install honcho) - - Paths and common settings live in .env - diff --git a/test/mitmproxy/fuzzing/client_patterns b/test/mitmproxy/fuzzing/client_patterns deleted file mode 100644 index 83457b6f3..000000000 --- a/test/mitmproxy/fuzzing/client_patterns +++ /dev/null @@ -1,4 +0,0 @@ -get:'http://localhost:9999/p/200':ir,"\n" -get:'http://localhost:9999/p/200':ir,"\0" -get:'http://localhost:9999/p/200':ir,@5 -get:'http://localhost:9999/p/200':dr diff --git a/test/mitmproxy/fuzzing/go_proxy b/test/mitmproxy/fuzzing/go_proxy deleted file mode 100644 index ea29400f7..000000000 --- a/test/mitmproxy/fuzzing/go_proxy +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh -# Assuming: -# mitmproxy/mitmdump is running on port 8080 in straight proxy mode. -# pathod is running on port 9999 - -BASE="../../../" -BASE_HTTP=$BASE"/pathod/pathoc -Tt 1 -e -I 200,400,405,502 -p 8080 localhost " -BASE_HTTPS=$BASE"/pathod/pathoc -sc localhost:9999 -Tt 1 -eo -I 200,400,404,405,502,800 -p 8080 localhost " - -#$BASE_HTTP -n 10000 "get:'http://localhost:9999':ir,@1" -#$BASE_HTTP -n 100 "get:'http://localhost:9999':dr" -#$BASE_HTTP -n 10000 "get:'http://localhost:9999/p/200':ir,@300" - -#$BASE_HTTP -n 10000 "get:'http://localhost:9999/p/200:ir,@1'" -#$BASE_HTTP -n 100 "get:'http://localhost:9999/p/200:dr'" -#$BASE_HTTP -n 10000 "get:'http://localhost:9999/p/200:ir,@100'" - - -# Assuming: -# mitmproxy/mitmdump is running on port 8080 in straight proxy mode. -# pathod with SSL enabled is running on port 9999 - -#$BASE_HTTPS -en 10000 "get:'/p/200:b@100:ir,@1'" -#$BASE_HTTPS -en 10000 "get:'/p/200:ir,@1'" - -#$BASE_HTTPS -n 100 "get:'/p/200:dr'" -#$BASE_HTTPS -n 10000 "get:'/p/200:ir,@3000'" -#$BASE_HTTPS -n 10000 "get:'/p/200:ir,\"\\n\"'" - diff --git a/test/mitmproxy/fuzzing/reverse_patterns b/test/mitmproxy/fuzzing/reverse_patterns deleted file mode 100644 index 8d1d76a20..000000000 --- a/test/mitmproxy/fuzzing/reverse_patterns +++ /dev/null @@ -1,9 +0,0 @@ -get:'/p/200':b@10:ir,"\n" -get:'/p/200':b@10:ir,"\r\n" -get:'/p/200':b@10:ir,"\0" -get:'/p/200':b@10:ir,@5 -get:'/p/200':b@10:dr - -get:'/p/200:b@10:ir,@1' -get:'/p/200:b@10:dr' -get:'/p/200:b@10:ir,@100' diff --git a/test/mitmproxy/fuzzing/straight_stream b/test/mitmproxy/fuzzing/straight_stream deleted file mode 100644 index 41e2a6e16..000000000 --- a/test/mitmproxy/fuzzing/straight_stream +++ /dev/null @@ -1,6 +0,0 @@ - -mitmdump: $MITMDUMP -pathod: $PATHOD -pathoc: sleep 2 && $PATHOC $FUZZ_SETTINGS localhost:8080 ./straight_stream_patterns -#pathoc: sleep 2 && $PATHOC localhost:8080 /tmp/err - diff --git a/test/mitmproxy/fuzzing/straight_stream_patterns b/test/mitmproxy/fuzzing/straight_stream_patterns deleted file mode 100644 index 235f2cc33..000000000 --- a/test/mitmproxy/fuzzing/straight_stream_patterns +++ /dev/null @@ -1,16 +0,0 @@ -get:'http://localhost:9999/p/':s'200:b"foo"':ir,'\n' -get:'http://localhost:9999/p/':s'200:b"foo"':ir,'a' -get:'http://localhost:9999/p/':s'200:b"foo"':ir,'9' -get:'http://localhost:9999/p/':s'200:b"foo"':ir,':' -get:'http://localhost:9999/p/':s'200:b"foo"':ir,'"' -get:'http://localhost:9999/p/':s'200:b"foo"':ir,'-' - -get:'http://localhost:9999/p/':s'200:b"foo":ir,"\n"' -get:'http://localhost:9999/p/':s'200:b"foo":ir,"a"' -get:'http://localhost:9999/p/':s'200:b"foo":ir,"9"' -get:'http://localhost:9999/p/':s'200:b"foo":ir,":"' -get:'http://localhost:9999/p/':s'200:b"foo":ir,"-"' -get:'http://localhost:9999/p/':s'200:b"foo":dr' - -get:'http://localhost:9999/p/':s'200:b"foo"':ir,@2 -get:'http://localhost:9999/p/':s'200:b"foo":ir,@2' diff --git a/test/mitmproxy/fuzzing/straight_stream_ssl b/test/mitmproxy/fuzzing/straight_stream_ssl deleted file mode 100644 index 708ff0b3d..000000000 --- a/test/mitmproxy/fuzzing/straight_stream_ssl +++ /dev/null @@ -1,6 +0,0 @@ - -mitmdump: $MITMDUMP -q --stream 1 -pathod: $PATHOD -pathoc: sleep 2 && $PATHOC $FUZZ_SETTINGS localhost:8080 ./straight_stream_patterns -#pathoc: sleep 2 && $PATHOC localhost:8080 /tmp/err - diff --git a/test/mitmproxy/proxy/__init__.py b/test/mitmproxy/proxy/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test/mitmproxy/proxy/modes/test_http_proxy.py b/test/mitmproxy/proxy/modes/test_http_proxy.py deleted file mode 100644 index 777ab4dd1..000000000 --- a/test/mitmproxy/proxy/modes/test_http_proxy.py +++ /dev/null @@ -1 +0,0 @@ -# TODO: write tests diff --git a/test/mitmproxy/proxy/modes/test_reverse_proxy.py b/test/mitmproxy/proxy/modes/test_reverse_proxy.py deleted file mode 100644 index 777ab4dd1..000000000 --- a/test/mitmproxy/proxy/modes/test_reverse_proxy.py +++ /dev/null @@ -1 +0,0 @@ -# TODO: write tests diff --git a/test/mitmproxy/proxy/modes/test_socks_proxy.py b/test/mitmproxy/proxy/modes/test_socks_proxy.py deleted file mode 100644 index 777ab4dd1..000000000 --- a/test/mitmproxy/proxy/modes/test_socks_proxy.py +++ /dev/null @@ -1 +0,0 @@ -# TODO: write tests diff --git a/test/mitmproxy/proxy/modes/test_transparent_proxy.py b/test/mitmproxy/proxy/modes/test_transparent_proxy.py deleted file mode 100644 index 777ab4dd1..000000000 --- a/test/mitmproxy/proxy/modes/test_transparent_proxy.py +++ /dev/null @@ -1 +0,0 @@ -# TODO: write tests diff --git a/test/mitmproxy/proxy/protocol/__init__.py b/test/mitmproxy/proxy/protocol/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test/mitmproxy/proxy/protocol/test_base.py b/test/mitmproxy/proxy/protocol/test_base.py deleted file mode 100644 index 777ab4dd1..000000000 --- a/test/mitmproxy/proxy/protocol/test_base.py +++ /dev/null @@ -1 +0,0 @@ -# TODO: write tests diff --git a/test/mitmproxy/proxy/protocol/test_http.py b/test/mitmproxy/proxy/protocol/test_http.py deleted file mode 100644 index 777ab4dd1..000000000 --- a/test/mitmproxy/proxy/protocol/test_http.py +++ /dev/null @@ -1 +0,0 @@ -# TODO: write tests diff --git a/test/mitmproxy/proxy/protocol/test_http1.py b/test/mitmproxy/proxy/protocol/test_http1.py deleted file mode 100644 index 14dba95e3..000000000 --- a/test/mitmproxy/proxy/protocol/test_http1.py +++ /dev/null @@ -1,112 +0,0 @@ -from unittest import mock -import pytest - -from mitmproxy.test import tflow -from mitmproxy.net.http import http1 -from mitmproxy.net.tcp import TCPClient -from mitmproxy.test.tutils import treq -from ... import tservers - - -class TestHTTPFlow: - - def test_repr(self): - f = tflow.tflow(resp=True, err=True) - assert repr(f) - - -class TestInvalidRequests(tservers.HTTPProxyTest): - ssl = True - - def test_double_connect(self): - p = self.pathoc() - with p.connect(): - r = p.request("connect:'{}:{}'".format("127.0.0.1", self.server2.port)) - assert r.status_code == 400 - assert b"Unexpected CONNECT" in r.content - - def test_relative_request(self): - p = self.pathoc_raw() - with p.connect(): - r = p.request("get:/p/200") - assert r.status_code == 400 - assert b"Invalid HTTP request form" in r.content - - -class TestProxyMisconfiguration(tservers.TransparentProxyTest): - - def test_absolute_request(self): - p = self.pathoc() - with p.connect(): - r = p.request("get:'http://localhost:%d/p/200'" % self.server.port) - assert r.status_code == 400 - assert b"misconfiguration" in r.content - - -class TestExpectHeader(tservers.HTTPProxyTest): - - def test_simple(self): - client = TCPClient(("127.0.0.1", self.proxy.port)) - client.connect() - - # call pathod server, wait a second to complete the request - client.wfile.write( - b"POST http://localhost:%d/p/200 HTTP/1.1\r\n" - b"Expect: 100-continue\r\n" - b"Content-Length: 16\r\n" - b"\r\n" % self.server.port - ) - client.wfile.flush() - - assert client.rfile.readline() == b"HTTP/1.1 100 Continue\r\n" - assert client.rfile.readline() == b"content-length: 0\r\n" - assert client.rfile.readline() == b"\r\n" - - client.wfile.write(b"0123456789abcdef\r\n") - client.wfile.flush() - - resp = http1.read_response(client.rfile, treq()) - assert resp.status_code == 200 - - client.finish() - client.close() - - -class TestHeadContentLength(tservers.HTTPProxyTest): - - def test_head_content_length(self): - p = self.pathoc() - with p.connect(): - resp = p.request( - """head:'%s/p/200:h"Content-Length"="42"'""" % self.server.urlbase - ) - assert resp.headers["Content-Length"] == "42" - - -class TestStreaming(tservers.HTTPProxyTest): - - @pytest.mark.parametrize('streaming', [True, False]) - def test_streaming(self, streaming): - - class Stream: - def requestheaders(self, f): - f.request.stream = streaming - - def responseheaders(self, f): - f.response.stream = streaming - - def assert_write(self, v): - if streaming: - assert len(v) <= 4096 - return self.o.write(v) - - self.master.addons.add(Stream()) - p = self.pathoc() - with p.connect(): - with mock.patch("mitmproxy.net.tcp.Writer.write", side_effect=assert_write, autospec=True): - # response with 10000 bytes - r = p.request("post:'%s/p/200:b@10000'" % self.server.urlbase) - assert len(r.content) == 10000 - - # request with 10000 bytes - assert p.request("post:'%s/p/200':b@10000" % self.server.urlbase) diff --git a/test/mitmproxy/proxy/protocol/test_http2.py b/test/mitmproxy/proxy/protocol/test_http2.py deleted file mode 100644 index f022f4f0f..000000000 --- a/test/mitmproxy/proxy/protocol/test_http2.py +++ /dev/null @@ -1,1244 +0,0 @@ -import os -import tempfile -import traceback -import pytest -import h2 - -from mitmproxy import options - -import mitmproxy.net -import mitmproxy.http -from ...net import tservers as net_tservers -from mitmproxy import exceptions -from mitmproxy.net.http import http1, http2 -from pathod.language import generators - -from ... import tservers - -import logging -logging.getLogger("hyper.packages.hpack.hpack").setLevel(logging.WARNING) -logging.getLogger("requests.packages.urllib3.connectionpool").setLevel(logging.WARNING) -logging.getLogger("passlib.utils.compat").setLevel(logging.WARNING) -logging.getLogger("passlib.registry").setLevel(logging.WARNING) - - -# inspect the log: -# for msg in self.proxy.tmaster.tlog: -# print(msg) - - -class _Http2ServerBase(net_tservers.ServerTestBase): - ssl = dict(alpn_select=b'h2') - - class handler(mitmproxy.net.tcp.BaseHandler): - - def handle(self): - config = h2.config.H2Configuration( - client_side=False, - validate_outbound_headers=False, - validate_inbound_headers=False) - h2_conn = h2.connection.H2Connection(config) - - preamble = self.rfile.read(24) - h2_conn.initiate_connection() - h2_conn.receive_data(preamble) - self.wfile.write(h2_conn.data_to_send()) - self.wfile.flush() - - if 'h2_server_settings' in self.kwargs: - h2_conn.update_settings(self.kwargs['h2_server_settings']) - self.wfile.write(h2_conn.data_to_send()) - self.wfile.flush() - - done = False - while not done: - try: - _, consumed_bytes = http2.read_frame(self.rfile, False) - events = h2_conn.receive_data(consumed_bytes) - except exceptions.HttpException: - print(traceback.format_exc()) - assert False - except exceptions.TcpDisconnect: - break - except: - print(traceback.format_exc()) - break - self.wfile.write(h2_conn.data_to_send()) - self.wfile.flush() - - for event in events: - try: - if not self.server.handle_server_event(event, h2_conn, self.rfile, self.wfile): - done = True - break - except exceptions.TcpDisconnect: - done = True - except: - done = True - print(traceback.format_exc()) - break - - def handle_server_event(self, event, h2_conn, rfile, wfile): - raise NotImplementedError() - - -class _Http2TestBase: - - @classmethod - def setup_class(cls): - cls.options = cls.get_options() - cls.proxy = tservers.ProxyThread(tservers.TestMaster, cls.options) - cls.proxy.start() - - @classmethod - def teardown_class(cls): - cls.proxy.shutdown() - - @classmethod - def get_options(cls): - opts = options.Options( - listen_port=0, - upstream_cert=True, - ssl_insecure=True - ) - opts.confdir = os.path.join(tempfile.gettempdir(), "mitmproxy") - return opts - - @property - def master(self): - return self.proxy.tmaster - - def setup(self): - self.master.reset([]) - self.server.server.handle_server_event = self.handle_server_event - - def teardown(self): - if self.client: - self.client.close() - self.server.server.wait_for_silence() - - def setup_connection(self): - self.client = mitmproxy.net.tcp.TCPClient(("127.0.0.1", self.proxy.port)) - self.client.connect() - - # send CONNECT request - self.client.wfile.write(http1.assemble_request( - mitmproxy.http.make_connect_request(("localhost", self.server.server.address[1])) - )) - self.client.wfile.flush() - - # read CONNECT response - while self.client.rfile.readline() != b"\r\n": - pass - - self.client.convert_to_tls(alpn_protos=[b'h2']) - - config = h2.config.H2Configuration( - client_side=True, - validate_outbound_headers=False, - validate_inbound_headers=False) - h2_conn = h2.connection.H2Connection(config) - h2_conn.initiate_connection() - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - return h2_conn - - def _send_request(self, - wfile, - h2_conn, - stream_id=1, - headers=None, - body=b'', - end_stream=None, - priority_exclusive=None, - priority_depends_on=None, - priority_weight=None, - streaming=False): - if headers is None: - headers = [] - if end_stream is None: - end_stream = (len(body) == 0) - - h2_conn.send_headers( - stream_id=stream_id, - headers=headers, - end_stream=end_stream, - priority_exclusive=priority_exclusive, - priority_depends_on=priority_depends_on, - priority_weight=priority_weight, - ) - if body: - h2_conn.send_data(stream_id, body) - if not streaming: - h2_conn.end_stream(stream_id) - wfile.write(h2_conn.data_to_send()) - wfile.flush() - - -class _Http2Test(_Http2TestBase, _Http2ServerBase): - - @classmethod - def setup_class(cls): - _Http2TestBase.setup_class() - _Http2ServerBase.setup_class() - - @classmethod - def teardown_class(cls): - _Http2TestBase.teardown_class() - _Http2ServerBase.teardown_class() - - -class TestSimpleRequestWithBody(_Http2Test): - request_body_buffer = b'' - - @classmethod - def handle_server_event(cls, event, h2_conn, rfile, wfile): - if isinstance(event, h2.events.ConnectionTerminated): - return False - elif isinstance(event, h2.events.RequestReceived): - assert (b'self.client-foo', b'self.client-bar-1') in event.headers - assert (b'self.client-foo', b'self.client-bar-2') in event.headers - elif isinstance(event, h2.events.StreamEnded): - import warnings - with warnings.catch_warnings(): - # Ignore UnicodeWarning: - # h2/utilities.py:64: UnicodeWarning: Unicode equal comparison - # failed to convert both arguments to Unicode - interpreting - # them as being unequal. - # elif header[0] in (b'cookie', u'cookie') and len(header[1]) < 20: - - warnings.simplefilter("ignore") - h2_conn.send_headers(event.stream_id, [ - (':status', '200'), - ('server-foo', 'server-bar'), - ('föo', 'bär'), - ('X-Stream-ID', str(event.stream_id)), - ]) - h2_conn.send_data(event.stream_id, b'response body') - h2_conn.end_stream(event.stream_id) - wfile.write(h2_conn.data_to_send()) - wfile.flush() - elif isinstance(event, h2.events.DataReceived): - cls.request_body_buffer += event.data - return True - - def test_simple_request_with_body(self): - response_body_buffer = b'' - h2_conn = self.setup_connection() - - self._send_request( - self.client.wfile, - h2_conn, - headers=[ - (':authority', "127.0.0.1:{}".format(self.server.server.address[1])), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/'), - ('self.client-FoO', 'self.client-bar-1'), - ('self.client-FoO', 'self.client-bar-2'), - ], - body=b'request body') - - done = False - while not done: - try: - _, consumed_bytes = http2.read_frame(self.client.rfile, False) - events = h2_conn.receive_data(consumed_bytes) - except exceptions.HttpException: - print(traceback.format_exc()) - assert False - - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - for event in events: - if isinstance(event, h2.events.DataReceived): - response_body_buffer += event.data - elif isinstance(event, h2.events.StreamEnded): - done = True - - h2_conn.close_connection() - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - assert len(self.master.state.flows) == 1 - assert self.master.state.flows[0].response.status_code == 200 - assert self.master.state.flows[0].response.headers['server-foo'] == 'server-bar' - assert self.master.state.flows[0].response.headers['föo'] == 'bär' - assert self.master.state.flows[0].response.content == b'response body' - assert self.request_body_buffer == b'request body' - assert response_body_buffer == b'response body' - - -class TestSimpleRequestWithoutBody(_Http2Test): - @classmethod - def handle_server_event(cls, event, h2_conn, rfile, wfile): - if isinstance(event, h2.events.ConnectionTerminated): - return False - elif isinstance(event, h2.events.RequestReceived): - assert (b'self.client-foo', b'self.client-bar-1') in event.headers - elif isinstance(event, h2.events.StreamEnded): - import warnings - with warnings.catch_warnings(): - # Ignore UnicodeWarning: - # h2/utilities.py:64: UnicodeWarning: Unicode equal comparison - # failed to convert both arguments to Unicode - interpreting - # them as being unequal. - # elif header[0] in (b'cookie', u'cookie') and len(header[1]) < 20: - - warnings.simplefilter("ignore") - h2_conn.send_headers(event.stream_id, [ - (':status', '200'), - ]) - h2_conn.send_data(event.stream_id, b'response body') - h2_conn.end_stream(event.stream_id) - wfile.write(h2_conn.data_to_send()) - wfile.flush() - elif isinstance(event, h2.events.DataReceived): - return False - return True - - def test_simple(self): - response_body_buffer = b'' - h2_conn = self.setup_connection() - - self._send_request( - self.client.wfile, - h2_conn, - headers=[ - (':authority', "127.0.0.1:{}".format(self.server.server.address[1])), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/'), - ('self.client-FoO', 'self.client-bar-1'), - ]) - - done = False - while not done: - try: - _, consumed_bytes = http2.read_frame(self.client.rfile, False) - events = h2_conn.receive_data(consumed_bytes) - except exceptions.HttpException: - print(traceback.format_exc()) - assert False - - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - for event in events: - if isinstance(event, h2.events.DataReceived): - response_body_buffer += event.data - elif isinstance(event, h2.events.StreamEnded): - done = True - - h2_conn.close_connection() - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - assert len(self.master.state.flows) == 1 - assert self.master.state.flows[0].response.status_code == 200 - assert self.master.state.flows[0].response.content == b'response body' - assert response_body_buffer == b'response body' - - -class TestRequestWithPriority(_Http2Test): - - @classmethod - def handle_server_event(cls, event, h2_conn, rfile, wfile): - if isinstance(event, h2.events.ConnectionTerminated): - return False - elif isinstance(event, h2.events.RequestReceived): - import warnings - with warnings.catch_warnings(): - # Ignore UnicodeWarning: - # h2/utilities.py:64: UnicodeWarning: Unicode equal comparison - # failed to convert both arguments to Unicode - interpreting - # them as being unequal. - # elif header[0] in (b'cookie', u'cookie') and len(header[1]) < 20: - - warnings.simplefilter("ignore") - - headers = [(':status', '200')] - if event.priority_updated: - headers.append(('priority_exclusive', str(event.priority_updated.exclusive).encode())) - headers.append(('priority_depends_on', str(event.priority_updated.depends_on).encode())) - headers.append(('priority_weight', str(event.priority_updated.weight).encode())) - h2_conn.send_headers(event.stream_id, headers) - h2_conn.end_stream(event.stream_id) - wfile.write(h2_conn.data_to_send()) - wfile.flush() - return True - - @pytest.mark.parametrize("http2_priority_enabled, priority, expected_priority", [ - (True, (True, 42424242, 42), ('True', '42424242', '42')), - (False, (True, 42424242, 42), (None, None, None)), - (True, (None, None, None), (None, None, None)), - (False, (None, None, None), (None, None, None)), - ]) - def test_request_with_priority(self, http2_priority_enabled, priority, expected_priority): - self.options.http2_priority = http2_priority_enabled - - h2_conn = self.setup_connection() - - self._send_request( - self.client.wfile, - h2_conn, - headers=[ - (':authority', "127.0.0.1:{}".format(self.server.server.address[1])), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/'), - ], - priority_exclusive=priority[0], - priority_depends_on=priority[1], - priority_weight=priority[2], - ) - - done = False - while not done: - try: - _, consumed_bytes = http2.read_frame(self.client.rfile, False) - events = h2_conn.receive_data(consumed_bytes) - except exceptions.HttpException: - print(traceback.format_exc()) - assert False - - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - for event in events: - if isinstance(event, h2.events.StreamEnded): - done = True - - h2_conn.close_connection() - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - assert len(self.master.state.flows) == 1 - - resp = self.master.state.flows[0].response - assert resp.headers.get('priority_exclusive', None) == expected_priority[0] - assert resp.headers.get('priority_depends_on', None) == expected_priority[1] - assert resp.headers.get('priority_weight', None) == expected_priority[2] - - -class TestPriority(_Http2Test): - - @classmethod - def handle_server_event(cls, event, h2_conn, rfile, wfile): - if isinstance(event, h2.events.ConnectionTerminated): - return False - elif isinstance(event, h2.events.PriorityUpdated): - cls.priority_data.append((event.exclusive, event.depends_on, event.weight)) - elif isinstance(event, h2.events.RequestReceived): - import warnings - with warnings.catch_warnings(): - # Ignore UnicodeWarning: - # h2/utilities.py:64: UnicodeWarning: Unicode equal comparison - # failed to convert both arguments to Unicode - interpreting - # them as being unequal. - # elif header[0] in (b'cookie', u'cookie') and len(header[1]) < 20: - - warnings.simplefilter("ignore") - - headers = [(':status', '200')] - h2_conn.send_headers(event.stream_id, headers) - h2_conn.end_stream(event.stream_id) - wfile.write(h2_conn.data_to_send()) - wfile.flush() - return True - - @pytest.mark.parametrize("prioritize_before", [True, False]) - @pytest.mark.parametrize("http2_priority_enabled, priority, expected_priority", [ - (True, (True, 42424242, 42), [(True, 42424242, 42)]), - (False, (True, 42424242, 42), []), - ]) - def test_priority(self, prioritize_before, http2_priority_enabled, priority, expected_priority): - self.options.http2_priority = http2_priority_enabled - self.__class__.priority_data = [] - - h2_conn = self.setup_connection() - - if prioritize_before: - h2_conn.prioritize(1, exclusive=priority[0], depends_on=priority[1], weight=priority[2]) - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - self._send_request( - self.client.wfile, - h2_conn, - headers=[ - (':authority', "127.0.0.1:{}".format(self.server.server.address[1])), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/'), - ], - end_stream=prioritize_before, - ) - - if not prioritize_before: - h2_conn.prioritize(1, exclusive=priority[0], depends_on=priority[1], weight=priority[2]) - h2_conn.end_stream(1) - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - done = False - while not done: - try: - _, consumed_bytes = http2.read_frame(self.client.rfile, False) - events = h2_conn.receive_data(consumed_bytes) - except exceptions.HttpException: - print(traceback.format_exc()) - assert False - - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - for event in events: - if isinstance(event, h2.events.StreamEnded): - done = True - - h2_conn.close_connection() - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - assert len(self.master.state.flows) == 1 - assert self.priority_data == expected_priority - - -class TestStreamResetFromServer(_Http2Test): - - @classmethod - def handle_server_event(cls, event, h2_conn, rfile, wfile): - if isinstance(event, h2.events.ConnectionTerminated): - return False - elif isinstance(event, h2.events.RequestReceived): - h2_conn.reset_stream(event.stream_id, 0x8) - wfile.write(h2_conn.data_to_send()) - wfile.flush() - return True - - def test_request_with_priority(self): - h2_conn = self.setup_connection() - - self._send_request( - self.client.wfile, - h2_conn, - headers=[ - (':authority', "127.0.0.1:{}".format(self.server.server.address[1])), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/'), - ], - ) - - done = False - while not done: - try: - _, consumed_bytes = http2.read_frame(self.client.rfile, False) - events = h2_conn.receive_data(consumed_bytes) - except exceptions.HttpException: - print(traceback.format_exc()) - assert False - - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - for event in events: - if isinstance(event, h2.events.StreamReset): - done = True - - h2_conn.close_connection() - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - assert len(self.master.state.flows) == 1 - assert self.master.state.flows[0].response is None - - -class TestAllStreamResetsFromServer(_Http2Test): - - current_error_name = None - current_error_code = None - - @classmethod - def handle_server_event(cls, event, h2_conn, rfile, wfile): - if isinstance(event, h2.events.ConnectionTerminated): - return False - elif isinstance(event, h2.events.RequestReceived): - h2_conn.reset_stream(event.stream_id, int(cls.current_error_code)) - wfile.write(h2_conn.data_to_send()) - wfile.flush() - return True - - def test_all_stream_reset_error_codes(self): - for error_name, error_code in h2.errors.ErrorCodes.__members__.items(): - self.__class__.current_error_name = error_name - self.__class__.current_error_code = error_code - try: - self.run_test_for_stream_reset() - except: - print('Exception occurred during test for error code {} ({})'.format( - error_name, error_code - )) - raise - - def run_test_for_stream_reset(self): - h2_conn = self.setup_connection() - - self._send_request( - self.client.wfile, - h2_conn, - headers=[ - (':authority', "127.0.0.1:{}".format(self.server.server.address[1])), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/'), - ], - ) - - self.client.rfile.o.settimeout(1) - - done = False - while not done: - try: - _, consumed_bytes = http2.read_frame(self.client.rfile, False) - events = h2_conn.receive_data(consumed_bytes) - except exceptions.HttpException: - print(traceback.format_exc()) - assert False - - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - for event in events: - if isinstance(event, h2.events.StreamReset): - assert event.error_code == int(self.current_error_code) - done = True - - h2_conn.close_connection() - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - -class TestBodySizeLimit(_Http2Test): - - @classmethod - def handle_server_event(cls, event, h2_conn, rfile, wfile): - if isinstance(event, h2.events.ConnectionTerminated): - return False - return True - - def test_body_size_limit(self): - self.options.body_size_limit = "20" - - h2_conn = self.setup_connection() - - self._send_request( - self.client.wfile, - h2_conn, - headers=[ - (':authority', "127.0.0.1:{}".format(self.server.server.address[1])), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/'), - ], - body=b'very long body over 20 characters long', - ) - - done = False - while not done: - try: - _, consumed_bytes = http2.read_frame(self.client.rfile, False) - events = h2_conn.receive_data(consumed_bytes) - except exceptions.HttpException: - print(traceback.format_exc()) - assert False - - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - for event in events: - if isinstance(event, h2.events.StreamReset): - done = True - - h2_conn.close_connection() - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - assert len(self.master.state.flows) == 0 - - -class TestPushPromise(_Http2Test): - - @classmethod - def handle_server_event(cls, event, h2_conn, rfile, wfile): - if isinstance(event, h2.events.ConnectionTerminated): - return False - elif isinstance(event, h2.events.RequestReceived): - if event.stream_id != 1: - # ignore requests initiated by push promises - return True - - h2_conn.send_headers(1, [(':status', '200')]) - wfile.write(h2_conn.data_to_send()) - wfile.flush() - - h2_conn.push_stream(1, 2, [ - (':authority', f"127.0.0.1:{cls.port}"), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/pushed_stream_foo'), - ('foo', 'bar') - ]) - wfile.write(h2_conn.data_to_send()) - wfile.flush() - - h2_conn.push_stream(1, 4, [ - (':authority', f"127.0.0.1:{cls.port}"), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/pushed_stream_bar'), - ('foo', 'bar') - ]) - wfile.write(h2_conn.data_to_send()) - wfile.flush() - - h2_conn.send_headers(2, [(':status', '200')]) - h2_conn.send_headers(4, [(':status', '200')]) - wfile.write(h2_conn.data_to_send()) - wfile.flush() - - h2_conn.send_data(1, b'regular_stream') - wfile.write(h2_conn.data_to_send()) - wfile.flush() - - h2_conn.end_stream(1) - wfile.write(h2_conn.data_to_send()) - wfile.flush() - - h2_conn.send_data(2, b'pushed_stream_foo') - h2_conn.end_stream(2) - wfile.write(h2_conn.data_to_send()) - wfile.flush() - - h2_conn.send_data(4, b'pushed_stream_bar') - h2_conn.end_stream(4) - wfile.write(h2_conn.data_to_send()) - wfile.flush() - - return True - - def test_push_promise(self): - h2_conn = self.setup_connection() - - self._send_request(self.client.wfile, h2_conn, stream_id=1, headers=[ - (':authority', "127.0.0.1:{}".format(self.server.server.address[1])), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/'), - ('foo', 'bar') - ]) - - done = False - ended_streams = 0 - pushed_streams = 0 - responses = 0 - while not done: - try: - _, consumed_bytes = http2.read_frame(self.client.rfile, False) - events = h2_conn.receive_data(consumed_bytes) - except exceptions.HttpException: - print(traceback.format_exc()) - assert False - except: - break - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - for event in events: - if isinstance(event, h2.events.StreamEnded): - ended_streams += 1 - elif isinstance(event, h2.events.PushedStreamReceived): - pushed_streams += 1 - elif isinstance(event, h2.events.ResponseReceived): - responses += 1 - if isinstance(event, h2.events.ConnectionTerminated): - done = True - - if responses == 3 and ended_streams == 3 and pushed_streams == 2: - done = True - - h2_conn.close_connection() - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - assert ended_streams == 3 - assert pushed_streams == 2 - - bodies = [flow.response.content for flow in self.master.state.flows] - assert len(bodies) == 3 - assert b'regular_stream' in bodies - assert b'pushed_stream_foo' in bodies - assert b'pushed_stream_bar' in bodies - - pushed_flows = [flow for flow in self.master.state.flows if 'h2-pushed-stream' in flow.metadata] - assert len(pushed_flows) == 2 - - def test_push_promise_reset(self): - h2_conn = self.setup_connection() - - self._send_request(self.client.wfile, h2_conn, stream_id=1, headers=[ - (':authority', "127.0.0.1:{}".format(self.server.server.address[1])), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/'), - ('foo', 'bar') - ]) - - done = False - ended_streams = 0 - pushed_streams = 0 - responses = 0 - while not done: - try: - _, consumed_bytes = http2.read_frame(self.client.rfile, False) - events = h2_conn.receive_data(consumed_bytes) - except exceptions.HttpException: - print(traceback.format_exc()) - assert False - - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - for event in events: - if isinstance(event, h2.events.StreamEnded) and event.stream_id == 1: - ended_streams += 1 - elif isinstance(event, h2.events.PushedStreamReceived): - pushed_streams += 1 - h2_conn.reset_stream(event.pushed_stream_id, error_code=0x8) - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - elif isinstance(event, h2.events.ResponseReceived): - responses += 1 - if isinstance(event, h2.events.ConnectionTerminated): - done = True - - if responses >= 1 and ended_streams >= 1 and pushed_streams == 2: - done = True - - h2_conn.close_connection() - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - bodies = [flow.response.content for flow in self.master.state.flows if flow.response] - assert len(bodies) >= 1 - assert b'regular_stream' in bodies - # the other two bodies might not be transmitted before the reset - - -class TestConnectionLost(_Http2Test): - - @classmethod - def handle_server_event(cls, event, h2_conn, rfile, wfile): - if isinstance(event, h2.events.RequestReceived): - h2_conn.send_headers(1, [(':status', '200')]) - wfile.write(h2_conn.data_to_send()) - wfile.flush() - return False - - def test_connection_lost(self): - h2_conn = self.setup_connection() - - self._send_request(self.client.wfile, h2_conn, stream_id=1, headers=[ - (':authority', "127.0.0.1:{}".format(self.server.server.address[1])), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/'), - ('foo', 'bar') - ]) - - done = False - while not done: - try: - _, consumed_bytes = http2.read_frame(self.client.rfile, False) - h2_conn.receive_data(consumed_bytes) - except exceptions.HttpException: - print(traceback.format_exc()) - assert False - except: - break - try: - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - except: - break - - if len(self.master.state.flows) == 1: - assert self.master.state.flows[0].response is None - - -class TestMaxConcurrentStreams(_Http2Test): - - @classmethod - def setup_class(cls): - _Http2TestBase.setup_class() - _Http2ServerBase.setup_class(h2_server_settings={h2.settings.SettingCodes.MAX_CONCURRENT_STREAMS: 2}) - - @classmethod - def handle_server_event(cls, event, h2_conn, rfile, wfile): - if isinstance(event, h2.events.ConnectionTerminated): - return False - elif isinstance(event, h2.events.RequestReceived): - h2_conn.send_headers(event.stream_id, [ - (':status', '200'), - ('X-Stream-ID', str(event.stream_id)), - ]) - h2_conn.send_data(event.stream_id, f'Stream-ID {event.stream_id}'.encode()) - h2_conn.end_stream(event.stream_id) - wfile.write(h2_conn.data_to_send()) - wfile.flush() - return True - - def test_max_concurrent_streams(self): - h2_conn = self.setup_connection() - new_streams = [1, 3, 5, 7, 9, 11] - for stream_id in new_streams: - # this will exceed MAX_CONCURRENT_STREAMS on the server connection - # and cause mitmproxy to throttle stream creation to the server - self._send_request(self.client.wfile, h2_conn, stream_id=stream_id, headers=[ - (':authority', "127.0.0.1:{}".format(self.server.server.address[1])), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/'), - ('X-Stream-ID', str(stream_id)), - ]) - - ended_streams = 0 - while ended_streams != len(new_streams): - try: - _, consumed_bytes = http2.read_frame(self.client.rfile, False) - events = h2_conn.receive_data(consumed_bytes) - except: - break - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - for event in events: - if isinstance(event, h2.events.StreamEnded): - ended_streams += 1 - - h2_conn.close_connection() - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - assert len(self.master.state.flows) == len(new_streams) - for flow in self.master.state.flows: - assert flow.response.status_code == 200 - assert b"Stream-ID " in flow.response.content - - -class TestConnectionTerminated(_Http2Test): - - @classmethod - def handle_server_event(cls, event, h2_conn, rfile, wfile): - if isinstance(event, h2.events.RequestReceived): - h2_conn.close_connection(error_code=5, last_stream_id=42, additional_data=b'foobar') - wfile.write(h2_conn.data_to_send()) - wfile.flush() - return True - - def test_connection_terminated(self): - h2_conn = self.setup_connection() - - self._send_request(self.client.wfile, h2_conn, headers=[ - (':authority', "127.0.0.1:{}".format(self.server.server.address[1])), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/'), - ]) - - done = False - connection_terminated_event = None - while not done: - try: - _, consumed_bytes = http2.read_frame(self.client.rfile, False) - events = h2_conn.receive_data(consumed_bytes) - for event in events: - if isinstance(event, h2.events.ConnectionTerminated): - connection_terminated_event = event - done = True - except: - break - - assert len(self.master.state.flows) == 1 - assert connection_terminated_event is not None - assert connection_terminated_event.error_code == 5 - assert connection_terminated_event.last_stream_id == 42 - assert connection_terminated_event.additional_data == b'foobar' - - -class TestRequestStreaming(_Http2Test): - - @classmethod - def handle_server_event(cls, event, h2_conn, rfile, wfile): - if isinstance(event, h2.events.ConnectionTerminated): - return False - elif isinstance(event, h2.events.DataReceived): - data = event.data - assert data - h2_conn.close_connection(error_code=5, last_stream_id=42, additional_data=data) - wfile.write(h2_conn.data_to_send()) - wfile.flush() - - return True - - @pytest.mark.parametrize('streaming', [True, False]) - def test_request_streaming(self, streaming): - class Stream: - def requestheaders(self, f): - f.request.stream = streaming - - self.master.addons.add(Stream()) - h2_conn = self.setup_connection() - body = generators.RandomGenerator("bytes", 100)[:] - self._send_request( - self.client.wfile, - h2_conn, - headers=[ - (':authority', "127.0.0.1:{}".format(self.server.server.address[1])), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/'), - - ], - body=body, - streaming=True - ) - done = False - connection_terminated_event = None - self.client.rfile.o.settimeout(2) - while not done: - try: - _, consumed_bytes = http2.read_frame(self.client.rfile, False) - events = h2_conn.receive_data(consumed_bytes) - - for event in events: - if isinstance(event, h2.events.ConnectionTerminated): - connection_terminated_event = event - done = True - except mitmproxy.exceptions.TcpTimeout: - if not streaming: - break # this is expected for this test case - else: - assert False - except: - print(traceback.format_exc()) - assert False - - if streaming: - assert connection_terminated_event.additional_data == body - else: - assert connection_terminated_event is None - - -class TestResponseStreaming(_Http2Test): - - @classmethod - def handle_server_event(cls, event, h2_conn, rfile, wfile): - if isinstance(event, h2.events.ConnectionTerminated): - return False - elif isinstance(event, h2.events.RequestReceived): - data = generators.RandomGenerator("bytes", 100)[:] - h2_conn.send_headers(event.stream_id, [ - (':status', '200'), - ('content-length', '100') - ]) - h2_conn.send_data(event.stream_id, data) - wfile.write(h2_conn.data_to_send()) - wfile.flush() - return True - - @pytest.mark.parametrize('streaming', [True, False]) - def test_response_streaming(self, streaming): - class Stream: - def responseheaders(self, f): - f.response.stream = streaming - - self.master.addons.add(Stream()) - h2_conn = self.setup_connection() - self._send_request( - self.client.wfile, - h2_conn, - headers=[ - (':authority', "127.0.0.1:{}".format(self.server.server.address[1])), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/'), - - ] - ) - done = False - self.client.rfile.o.settimeout(2) - data = None - while not done: - try: - _, consumed_bytes = http2.read_frame(self.client.rfile, False) - events = h2_conn.receive_data(consumed_bytes) - - for event in events: - if isinstance(event, h2.events.DataReceived): - data = event.data - done = True - except: - break - - if streaming: - assert data - else: - assert data is None - - -class TestRequestTrailers(_Http2Test): - server_trailers_received = False - - @classmethod - def handle_server_event(cls, event, h2_conn, rfile, wfile): - if isinstance(event, h2.events.RequestReceived): - # reset the value for a fresh test - cls.server_trailers_received = False - elif isinstance(event, h2.events.ConnectionTerminated): - return False - elif isinstance(event, h2.events.TrailersReceived): - cls.server_trailers_received = True - - elif isinstance(event, h2.events.StreamEnded): - h2_conn.send_headers(event.stream_id, [ - (':status', '200'), - ('x-my-trailer-request-received', 'success' if cls.server_trailers_received else "failure"), - ], end_stream=True) - wfile.write(h2_conn.data_to_send()) - wfile.flush() - return True - - @pytest.mark.parametrize('announce', [True, False]) - @pytest.mark.parametrize('body', [None, b"foobar"]) - def test_trailers(self, announce, body): - h2_conn = self.setup_connection() - stream_id = 1 - headers = [ - (':authority', "127.0.0.1:{}".format(self.server.server.address[1])), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/'), - ] - if announce: - headers.append(('trailer', 'x-my-trailers')) - h2_conn.send_headers( - stream_id=stream_id, - headers=headers, - ) - if body: - h2_conn.send_data(stream_id, body) - - # send trailers - h2_conn.send_headers(stream_id, [('x-my-trailers', 'foobar')], end_stream=True) - - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - done = False - while not done: - try: - _, consumed_bytes = http2.read_frame(self.client.rfile, False) - events = h2_conn.receive_data(consumed_bytes) - except exceptions.HttpException: - print(traceback.format_exc()) - assert False - - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - for event in events: - if isinstance(event, h2.events.StreamEnded): - done = True - - h2_conn.close_connection() - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - assert len(self.master.state.flows) == 1 - assert self.master.state.flows[0].request.trailers['x-my-trailers'] == 'foobar' - assert self.master.state.flows[0].response.status_code == 200 - assert self.master.state.flows[0].response.headers['x-my-trailer-request-received'] == 'success' - - -class TestResponseTrailers(_Http2Test): - - @classmethod - def handle_server_event(cls, event, h2_conn, rfile, wfile): - if isinstance(event, h2.events.ConnectionTerminated): - return False - elif isinstance(event, h2.events.StreamEnded): - headers = [ - (':status', '200'), - ] - if event.stream_id == 1: - # special stream_id to activate the Trailer announcement header - headers.append(('trailer', 'x-my-trailers')) - - h2_conn.send_headers(event.stream_id, headers) - h2_conn.send_data(event.stream_id, b'response body') - h2_conn.send_headers(event.stream_id, [('x-my-trailers', 'foobar')], end_stream=True) - wfile.write(h2_conn.data_to_send()) - wfile.flush() - return True - - @pytest.mark.parametrize('announce', [True, False]) - def test_trailers(self, announce): - response_body_buffer = b'' - h2_conn = self.setup_connection() - - self._send_request( - self.client.wfile, - h2_conn, - stream_id=(1 if announce else 3), - headers=[ - (':authority', "127.0.0.1:{}".format(self.server.server.address[1])), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/'), - ]) - - trailers_buffer = None - done = False - while not done: - try: - _, consumed_bytes = http2.read_frame(self.client.rfile, False) - events = h2_conn.receive_data(consumed_bytes) - except exceptions.HttpException: - print(traceback.format_exc()) - assert False - - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - for event in events: - if isinstance(event, h2.events.DataReceived): - response_body_buffer += event.data - elif isinstance(event, h2.events.TrailersReceived): - trailers_buffer = event.headers - elif isinstance(event, h2.events.StreamEnded): - done = True - - h2_conn.close_connection() - self.client.wfile.write(h2_conn.data_to_send()) - self.client.wfile.flush() - - assert len(self.master.state.flows) == 1 - assert self.master.state.flows[0].response.status_code == 200 - assert self.master.state.flows[0].response.content == b'response body' - assert response_body_buffer == b'response body' - assert self.master.state.flows[0].response.trailers['x-my-trailers'] == 'foobar' - assert trailers_buffer == [(b'x-my-trailers', b'foobar')] diff --git a/test/mitmproxy/proxy/protocol/test_rawtcp.py b/test/mitmproxy/proxy/protocol/test_rawtcp.py deleted file mode 100644 index 777ab4dd1..000000000 --- a/test/mitmproxy/proxy/protocol/test_rawtcp.py +++ /dev/null @@ -1 +0,0 @@ -# TODO: write tests diff --git a/test/mitmproxy/proxy/protocol/test_tls.py b/test/mitmproxy/proxy/protocol/test_tls.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test/mitmproxy/proxy/protocol/test_websocket.py b/test/mitmproxy/proxy/protocol/test_websocket.py deleted file mode 100644 index bbaeb1853..000000000 --- a/test/mitmproxy/proxy/protocol/test_websocket.py +++ /dev/null @@ -1,510 +0,0 @@ -import pytest -import os -import struct -import tempfile -import traceback - -from wsproto.frame_protocol import Opcode - -from mitmproxy import exceptions, options -from mitmproxy.http import HTTPFlow, make_connect_request -from mitmproxy.websocket import WebSocketFlow -from mitmproxy.net import http, tcp, websocket - -from pathod.language import websockets_frame - -from ...net import tservers as net_tservers -from ... import tservers - - -class _WebSocketServerBase(net_tservers.ServerTestBase): - - class handler(tcp.BaseHandler): - - def handle(self): - try: - request = http.http1.read_request(self.rfile) - assert websocket.check_handshake(request.headers) - - response = http.Response( - http_version=b"HTTP/1.1", - status_code=101, - reason=http.status_codes.RESPONSES.get(101).encode(), - headers=http.Headers( - connection='upgrade', - upgrade='websocket', - sec_websocket_accept=b'', - sec_websocket_extensions='permessage-deflate' if "permessage-deflate" in request.headers.values() else '' - ), - content=b'', - trailers=None, - timestamp_start=0, - timestamp_end=0, - ) - self.wfile.write(http.http1.assemble_response(response)) - self.wfile.flush() - - self.server.handle_websockets(self.rfile, self.wfile) - except: - traceback.print_exc() - - -class _WebSocketTestBase: - client = None - - @classmethod - def setup_class(cls): - cls.options = cls.get_options() - cls.proxy = tservers.ProxyThread(tservers.TestMaster, cls.options) - cls.proxy.start() - - @classmethod - def teardown_class(cls): - cls.proxy.shutdown() - - @classmethod - def get_options(cls): - opts = options.Options( - listen_port=0, - upstream_cert=True, - ssl_insecure=True, - websocket=True, - ) - opts.confdir = os.path.join(tempfile.gettempdir(), "mitmproxy") - return opts - - @property - def master(self): - return self.proxy.tmaster - - def setup(self): - self.master.reset([]) - self.server.server.handle_websockets = self.handle_websockets - - def teardown(self): - if self.client: - self.client.close() - - def setup_connection(self, extension=False): - self.client = tcp.TCPClient(("127.0.0.1", self.proxy.port)) - self.client.connect() - - request = make_connect_request(("127.0.0.1", self.server.server.address[1])) - self.client.wfile.write(http.http1.assemble_request(request)) - self.client.wfile.flush() - - response = http.http1.read_response(self.client.rfile, request) - - if self.ssl: - self.client.convert_to_tls() - assert self.client.tls_established - - request = http.Request( - host="127.0.0.1", - port=self.server.server.address[1], - method=b"GET", - scheme=b"http", - authority=b"", - path=b"/ws", - http_version=b"HTTP/1.1", - headers=http.Headers( - connection="upgrade", - upgrade="websocket", - sec_websocket_version="13", - sec_websocket_key="1234", - sec_websocket_extensions="permessage-deflate" if extension else "" - ), - content=b'', - trailers=None, - timestamp_start=0, - timestamp_end=0, - ) - self.client.wfile.write(http.http1.assemble_request(request)) - self.client.wfile.flush() - - response = http.http1.read_response(self.client.rfile, request) - assert websocket.check_handshake(response.headers) - - -class _WebSocketTest(_WebSocketTestBase, _WebSocketServerBase): - - @classmethod - def setup_class(cls): - _WebSocketTestBase.setup_class() - _WebSocketServerBase.setup_class(ssl=cls.ssl) - - @classmethod - def teardown_class(cls): - _WebSocketTestBase.teardown_class() - _WebSocketServerBase.teardown_class() - - -class TestSimple(_WebSocketTest): - - @classmethod - def handle_websockets(cls, rfile, wfile): - wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=Opcode.TEXT, payload=b'server-foobar'))) - wfile.flush() - - header, frame, _ = websocket.read_frame(rfile) - wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=header.opcode, payload=frame.payload))) - wfile.flush() - - header, frame, _ = websocket.read_frame(rfile) - wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=header.opcode, payload=frame.payload))) - wfile.flush() - - @pytest.mark.parametrize('streaming', [True, False]) - def test_simple(self, streaming): - class Stream: - def websocket_start(self, f): - f.stream = streaming - - self.proxy.set_addons(Stream()) - self.setup_connection() - - _, frame, _ = websocket.read_frame(self.client.rfile) - assert frame.payload == b'server-foobar' - - self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.TEXT, payload=b'self.client-foobar'))) - self.client.wfile.flush() - - _, frame, _ = websocket.read_frame(self.client.rfile) - assert frame.payload == b'self.client-foobar' - - self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.BINARY, payload=b'\xde\xad\xbe\xef'))) - self.client.wfile.flush() - - _, frame, _ = websocket.read_frame(self.client.rfile) - assert frame.payload == b'\xde\xad\xbe\xef' - - self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.CLOSE))) - self.client.wfile.flush() - - assert len(self.master.state.flows) == 2 - assert isinstance(self.master.state.flows[0], HTTPFlow) - assert isinstance(self.master.state.flows[1], WebSocketFlow) - assert len(self.master.state.flows[1].messages) == 5 - assert self.master.state.flows[1].messages[0].content == 'server-foobar' - assert self.master.state.flows[1].messages[0].type == Opcode.TEXT - assert self.master.state.flows[1].messages[1].content == 'self.client-foobar' - assert self.master.state.flows[1].messages[1].type == Opcode.TEXT - assert self.master.state.flows[1].messages[2].content == 'self.client-foobar' - assert self.master.state.flows[1].messages[2].type == Opcode.TEXT - assert self.master.state.flows[1].messages[3].content == b'\xde\xad\xbe\xef' - assert self.master.state.flows[1].messages[3].type == Opcode.BINARY - assert self.master.state.flows[1].messages[4].content == b'\xde\xad\xbe\xef' - assert self.master.state.flows[1].messages[4].type == Opcode.BINARY - - def test_change_payload(self): - class Addon: - def websocket_message(self, f): - f.messages[-1].content = "foo" - - self.proxy.set_addons(Addon()) - self.setup_connection() - - _, frame, _ = websocket.read_frame(self.client.rfile) - assert frame.payload == b'foo' - - self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.TEXT, payload=b'self.client-foobar'))) - self.client.wfile.flush() - - _, frame, _ = websocket.read_frame(self.client.rfile) - assert frame.payload == b'foo' - - self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.BINARY, payload=b'\xde\xad\xbe\xef'))) - self.client.wfile.flush() - - _, frame, _ = websocket.read_frame(self.client.rfile) - assert frame.payload == b'foo' - - -class TestKillFlow(_WebSocketTest): - - @classmethod - def handle_websockets(cls, rfile, wfile): - wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=Opcode.TEXT, payload=b'server-foobar'))) - wfile.flush() - - def test_kill(self): - class KillFlow: - def websocket_message(self, f): - f.kill() - - self.proxy.set_addons(KillFlow()) - self.setup_connection() - - with pytest.raises(exceptions.TcpDisconnect): - _ = websocket.read_frame(self.client.rfile, False) - - -class TestSimpleTLS(_WebSocketTest): - ssl = True - - @classmethod - def handle_websockets(cls, rfile, wfile): - wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=Opcode.TEXT, payload=b'server-foobar'))) - wfile.flush() - - header, frame, _ = websocket.read_frame(rfile) - wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=header.opcode, payload=frame.payload))) - wfile.flush() - - def test_simple_tls(self): - self.setup_connection() - - _, frame, _ = websocket.read_frame(self.client.rfile) - assert frame.payload == b'server-foobar' - - self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.TEXT, payload=b'self.client-foobar'))) - self.client.wfile.flush() - - _, frame, _ = websocket.read_frame(self.client.rfile) - assert frame.payload == b'self.client-foobar' - - self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.CLOSE))) - self.client.wfile.flush() - - -class TestPing(_WebSocketTest): - - @classmethod - def handle_websockets(cls, rfile, wfile): - wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=Opcode.PING, payload=b'foobar'))) - wfile.flush() - - header, frame, _ = websocket.read_frame(rfile) - assert header.opcode == Opcode.PONG - assert frame.payload == b'foobar' - - wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=Opcode.PONG, payload=b'done'))) - wfile.flush() - - wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=Opcode.CLOSE))) - wfile.flush() - _ = websocket.read_frame(rfile, False) - - @pytest.mark.asyncio - async def test_ping(self): - self.setup_connection() - - header, frame, _ = websocket.read_frame(self.client.rfile) - _ = websocket.read_frame(self.client.rfile, False) - self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.CLOSE))) - self.client.wfile.flush() - assert header.opcode == Opcode.PING - assert frame.payload == b'' # We don't send payload to other end - - assert await self.master.await_log("Pong Received from server", "info") - - -class TestPong(_WebSocketTest): - - @classmethod - def handle_websockets(cls, rfile, wfile): - header, frame, _ = websocket.read_frame(rfile) - assert header.opcode == Opcode.PING - assert frame.payload == b'' - - wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=Opcode.PONG, payload=frame.payload))) - wfile.flush() - - wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=Opcode.CLOSE))) - wfile.flush() - _ = websocket.read_frame(rfile) - - @pytest.mark.asyncio - async def test_pong(self): - self.setup_connection() - - self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.PING, payload=b'foobar'))) - self.client.wfile.flush() - - header, frame, _ = websocket.read_frame(self.client.rfile) - _ = websocket.read_frame(self.client.rfile) - self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.CLOSE))) - self.client.wfile.flush() - - assert header.opcode == Opcode.PONG - assert frame.payload == b'foobar' - assert await self.master.await_log("pong received") - - -class TestClose(_WebSocketTest): - - @classmethod - def handle_websockets(cls, rfile, wfile): - header, frame, _ = websocket.read_frame(rfile) - wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=header.opcode, payload=frame.payload))) - wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=Opcode.CLOSE))) - wfile.flush() - - with pytest.raises(exceptions.TcpDisconnect): - _ = websocket.read_frame(rfile) - - def test_close(self): - self.setup_connection() - - self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.CLOSE))) - self.client.wfile.flush() - - _ = websocket.read_frame(self.client.rfile) - with pytest.raises(exceptions.TcpDisconnect): - _ = websocket.read_frame(self.client.rfile) - - def test_close_payload_1(self): - self.setup_connection() - - self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.CLOSE, payload=b'\00\42'))) - self.client.wfile.flush() - - _ = websocket.read_frame(self.client.rfile) - with pytest.raises(exceptions.TcpDisconnect): - _ = websocket.read_frame(self.client.rfile) - - def test_close_payload_2(self): - self.setup_connection() - - self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.CLOSE, payload=b'\00\42foobar'))) - self.client.wfile.flush() - - _ = websocket.read_frame(self.client.rfile) - with pytest.raises(exceptions.TcpDisconnect): - _ = websocket.read_frame(self.client.rfile) - - -class TestInvalidFrame(_WebSocketTest): - - @classmethod - def handle_websockets(cls, rfile, wfile): - wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=15, payload=b'foobar'))) - wfile.flush() - - def test_invalid_frame(self): - self.setup_connection() - - _, frame, _ = websocket.read_frame(self.client.rfile) - code, = struct.unpack('!H', frame.payload[:2]) - assert code == 1002 - assert frame.payload[2:].startswith(b'Invalid opcode') - - -class TestStreaming(_WebSocketTest): - - @classmethod - def handle_websockets(cls, rfile, wfile): - wfile.write(bytes(websockets_frame.Frame(opcode=Opcode.TEXT, payload=b'server-foobar'))) - wfile.flush() - - @pytest.mark.parametrize('streaming', [True, False]) - def test_streaming(self, streaming): - class Stream: - def websocket_start(self, f): - f.stream = streaming - - self.proxy.set_addons(Stream()) - self.setup_connection() - - frame = None - if not streaming: - with pytest.raises(exceptions.TcpDisconnect): # Reader.safe_read get nothing as result - _, frame, _ = websocket.read_frame(self.client.rfile) - assert frame is None - - else: - _, frame, _ = websocket.read_frame(self.client.rfile) - - assert frame - assert self.master.state.flows[1].messages == [] # Message not appended as the final frame isn't received - - -class TestExtension(_WebSocketTest): - - @classmethod - def handle_websockets(cls, rfile, wfile): - wfile.write(b'\xc1\x0f*N-*K-\xd2M\xcb\xcfOJ,\x02\x00') - wfile.flush() - - header, _, _ = websocket.read_frame(rfile) - assert header.rsv.rsv1 - wfile.write(b'\xc1\nJ\xce\xc9L\xcd+\x81r\x00\x00') - wfile.flush() - - header, _, _ = websocket.read_frame(rfile) - assert header.rsv.rsv1 - wfile.write(b'\xc2\x07\xba\xb7v\xdf{\x00\x00') - wfile.flush() - - def test_extension(self): - self.setup_connection(True) - - header, _, _ = websocket.read_frame(self.client.rfile) - assert header.rsv.rsv1 - - self.client.wfile.write(b'\xc1\x8fQ\xb7vX\x1by\xbf\x14\x9c\x9c\xa7\x15\x9ax9\x12}\xb5v') - self.client.wfile.flush() - - header, _, _ = websocket.read_frame(self.client.rfile) - assert header.rsv.rsv1 - - self.client.wfile.write(b'\xc2\x87\xeb\xbb\x0csQ\x0cz\xac\x90\xbb\x0c') - self.client.wfile.flush() - - header, _, _ = websocket.read_frame(self.client.rfile) - assert header.rsv.rsv1 - - assert len(self.master.state.flows[1].messages) == 5 - assert self.master.state.flows[1].messages[0].content == 'server-foobar' - assert self.master.state.flows[1].messages[0].type == Opcode.TEXT - assert self.master.state.flows[1].messages[1].content == 'client-foobar' - assert self.master.state.flows[1].messages[1].type == Opcode.TEXT - assert self.master.state.flows[1].messages[2].content == 'client-foobar' - assert self.master.state.flows[1].messages[2].type == Opcode.TEXT - assert self.master.state.flows[1].messages[3].content == b'\xde\xad\xbe\xef' - assert self.master.state.flows[1].messages[3].type == Opcode.BINARY - assert self.master.state.flows[1].messages[4].content == b'\xde\xad\xbe\xef' - assert self.master.state.flows[1].messages[4].type == Opcode.BINARY - - -class TestInjectMessageClient(_WebSocketTest): - - @classmethod - def handle_websockets(cls, rfile, wfile): - pass - - def test_inject_message_client(self): - class Inject: - def websocket_start(self, flow): - flow.inject_message(flow.client_conn, 'This is an injected message!') - - self.proxy.set_addons(Inject()) - self.setup_connection() - - header, frame, _ = websocket.read_frame(self.client.rfile) - assert header.opcode == Opcode.TEXT - assert frame.payload == b'This is an injected message!' - - -class TestInjectMessageServer(_WebSocketTest): - - @classmethod - def handle_websockets(cls, rfile, wfile): - header, frame, _ = websocket.read_frame(rfile) - assert header.opcode == Opcode.TEXT - success = frame.payload == b'This is an injected message!' - - wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=Opcode.TEXT, payload=str(success).encode()))) - wfile.flush() - - def test_inject_message_server(self): - class Inject: - def websocket_start(self, flow): - flow.inject_message(flow.server_conn, 'This is an injected message!') - - self.proxy.set_addons(Inject()) - self.setup_connection() - - header, frame, _ = websocket.read_frame(self.client.rfile) - assert header.opcode == Opcode.TEXT - assert frame.payload == b'True' diff --git a/test/mitmproxy/proxy/test_config.py b/test/mitmproxy/proxy/test_config.py deleted file mode 100644 index 38a6e1ade..000000000 --- a/test/mitmproxy/proxy/test_config.py +++ /dev/null @@ -1,28 +0,0 @@ -import pytest - -from mitmproxy import options -from mitmproxy import exceptions -from mitmproxy.proxy.config import ProxyConfig - - -class TestProxyConfig: - def test_invalid_confdir(self): - opts = options.Options() - opts.confdir = "foo" - with pytest.raises(exceptions.OptionsError, match="parent directory does not exist"): - ProxyConfig(opts) - - def test_invalid_certificate(self, tdata): - opts = options.Options() - opts.certs = [tdata.path("mitmproxy/data/dumpfile-011.bin")] - with pytest.raises(exceptions.OptionsError, match="Invalid certificate format"): - ProxyConfig(opts) - - def test_cannot_set_both_allow_and_filter_options(self): - opts = options.Options() - opts.ignore_hosts = ["foo"] - opts.allow_hosts = ["bar"] - with pytest.raises(exceptions.OptionsError, match="--ignore-hosts and --allow-hosts are " - "mutually exclusive; please choose " - "one."): - ProxyConfig(opts) diff --git a/test/mitmproxy/proxy/test_root_context.py b/test/mitmproxy/proxy/test_root_context.py deleted file mode 100644 index 777ab4dd1..000000000 --- a/test/mitmproxy/proxy/test_root_context.py +++ /dev/null @@ -1 +0,0 @@ -# TODO: write tests diff --git a/test/mitmproxy/proxy/test_server.py b/test/mitmproxy/proxy/test_server.py deleted file mode 100644 index d092f73e5..000000000 --- a/test/mitmproxy/proxy/test_server.py +++ /dev/null @@ -1,1131 +0,0 @@ -import asyncio -import os -import socket -import time -from unittest import mock - -import pytest - -import mitmproxy.net.http -from mitmproxy import certs -from mitmproxy import exceptions -from mitmproxy import http -from mitmproxy import options -from mitmproxy.addons import script -from mitmproxy.net import socks -from mitmproxy.net import tcp -from mitmproxy.net.http import http1 -from mitmproxy.proxy.config import HostMatcher -from mitmproxy.utils import data -from pathod import pathoc -from pathod import pathod -from .. import tservers -from ...conftest import skip_appveyor - - -cdata = data.Data(__name__) - - -class CommonMixin: - - def test_large(self): - assert len(self.pathod("200:b@50k").content) == 1024 * 50 - - def test_http(self): - f = self.pathod("304") - assert f.status_code == 304 - - # In Upstream mode with SSL, we may already have a previous CONNECT - # request. - l = self.master.state.flows[-1] - assert l.client_conn.address - assert "host" in l.request.headers - assert l.response.status_code == 304 - - def test_invalid_http(self): - t = tcp.TCPClient(("127.0.0.1", self.proxy.port)) - with t.connect(): - t.wfile.write(b"invalid\r\n\r\n") - t.wfile.flush() - line = t.rfile.readline() - assert (b"Bad Request" in line) or (b"Bad Gateway" in line) - - def test_sni(self): - if not self.ssl: - return - - if getattr(self, 'reverse', False): - # In reverse proxy mode, we expect to use the upstream host as our SNI value - expected_sni = "127.0.0.1" - else: - expected_sni = "testserver.com" - - f = self.pathod("304", sni="testserver.com") - assert f.status_code == 304 - log = self.server.last_log() - assert log["request"]["sni"] == expected_sni - - -class TcpMixin: - - def _ignore_on(self): - assert not hasattr(self, "_ignore_backup") - self._ignore_backup = self.options.ignore_hosts - self.options.ignore_hosts = [".+:%s" % self.server.port] + self.options.ignore_hosts - - def _ignore_off(self): - assert hasattr(self, "_ignore_backup") - self.options.ignore_hosts = self._ignore_backup - del self._ignore_backup - - def _allow_on(self): - assert not hasattr(self, "_allow_backup") - self._allow_backup = self.options.allow_hosts - self.options.allow_hosts = ["(127.0.0.1|None):\\d+"] + self.options.allow_hosts - - def _allow_off(self): - assert hasattr(self, "_allow_backup") - self.options.allow_hosts = self._allow_backup - del self._allow_backup - - def test_ignore(self): - n = self.pathod("304") - self._ignore_on() - i = self.pathod("305") - i2 = self.pathod("306") - self._ignore_off() - - assert n.status_code == 304 - assert i.status_code == 305 - assert i2.status_code == 306 - assert any(f.response.status_code == 304 for f in self.master.state.flows) - assert not any(f.response.status_code == 305 for f in self.master.state.flows) - assert not any(f.response.status_code == 306 for f in self.master.state.flows) - - # Test that we get the original SSL cert - if self.ssl: - i_cert = certs.Cert(i.sslinfo.certchain[0]) - i2_cert = certs.Cert(i2.sslinfo.certchain[0]) - n_cert = certs.Cert(n.sslinfo.certchain[0]) - - assert i_cert == i2_cert - assert i_cert != n_cert - - # Test Non-HTTP traffic - spec = "200:i0,@100:d0" # this results in just 100 random bytes - # mitmproxy responds with bad gateway - assert self.pathod(spec).status_code == 502 - self._ignore_on() - with pytest.raises(exceptions.HttpException): - self.pathod(spec) # pathoc tries to parse answer as HTTP - - self._ignore_off() - - def test_allow(self): - n = self.pathod("304") - self._allow_on() - i = self.pathod("305") - i2 = self.pathod("306") - self._allow_off() - - assert n.status_code == 304 - assert i.status_code == 305 - assert i2.status_code == 306 - - assert any(f.response.status_code == 304 for f in self.master.state.flows) - assert any(f.response.status_code == 305 for f in self.master.state.flows) - assert any(f.response.status_code == 306 for f in self.master.state.flows) - - # Test that we get the original SSL cert - if self.ssl: - i_cert = certs.Cert(i.sslinfo.certchain[0]) - i2_cert = certs.Cert(i2.sslinfo.certchain[0]) - n_cert = certs.Cert(n.sslinfo.certchain[0]) - - assert i_cert == i2_cert - assert i_cert != n_cert - - # Test Non-HTTP traffic - spec = "200:i0,@100:d0" # this results in just 100 random bytes - # mitmproxy responds with bad gateway - assert self.pathod(spec).status_code == 502 - self._allow_on() - - self.pathod(spec) # pathoc parses answer as HTTP - - self._allow_off() - - def _tcpproxy_on(self): - assert not hasattr(self, "_tcpproxy_backup") - self._tcpproxy_backup = self.options.tcp_hosts - self.options.tcp_hosts = [".+:%s" % self.server.port] + self.options.tcp_hosts - - def _tcpproxy_off(self): - assert hasattr(self, "_tcpproxy_backup") - self.options.tcp_hosts = self._tcpproxy_backup - del self._tcpproxy_backup - - def test_tcp(self): - n = self.pathod("304") - self._tcpproxy_on() - i = self.pathod("305") - i2 = self.pathod("306") - self._tcpproxy_off() - - assert n.status_code == 304 - assert i.status_code == 305 - assert i2.status_code == 306 - assert any(f.response.status_code == 304 for f in self.master.state.flows if isinstance(f, http.HTTPFlow)) - assert not any(f.response.status_code == 305 for f in self.master.state.flows if isinstance(f, http.HTTPFlow)) - assert not any(f.response.status_code == 306 for f in self.master.state.flows if isinstance(f, http.HTTPFlow)) - - # Test that we get the original SSL cert - if self.ssl: - i_cert = certs.Cert(i.sslinfo.certchain[0]) - i2_cert = certs.Cert(i2.sslinfo.certchain[0]) - n_cert = certs.Cert(n.sslinfo.certchain[0]) - - assert i_cert == i2_cert - assert i_cert != n_cert - - # Make sure that TCP messages are in the event log. - # Re-enable and fix this when we start keeping TCPFlows in the state. - # assert any("305" in m for m in self.master.tlog) - # assert any("306" in m for m in self.master.tlog) - - -class TestHTTP(tservers.HTTPProxyTest, CommonMixin): - def test_invalid_connect(self): - t = tcp.TCPClient(("127.0.0.1", self.proxy.port)) - with t.connect(): - t.wfile.write(b"CONNECT invalid\n\n") - t.wfile.flush() - assert b"Bad Request" in t.rfile.readline() - - def test_upstream_ssl_error(self): - p = self.pathoc() - with p.connect(): - ret = p.request("get:'https://localhost:%s/'" % self.server.port) - assert ret.status_code == 502 - - def test_connection_close(self): - # Add a body, so we have a content-length header, which combined with - # HTTP1.1 means the connection is kept alive. - response = '%s/p/200:b@1' % self.server.urlbase - - # Lets sanity check that the connection does indeed stay open by - # issuing two requests over the same connection - p = self.pathoc() - with p.connect(): - assert p.request("get:'%s'" % response) - assert p.request("get:'%s'" % response) - - # Now check that the connection is closed as the client specifies - p = self.pathoc() - with p.connect(): - assert p.request("get:'%s':h'Connection'='close'" % response) - # There's a race here, which means we can get any of a number of errors. - # Rather than introduce yet another sleep into the test suite, we just - # relax the Exception specification. - with pytest.raises(Exception): - p.request("get:'%s'" % response) - - def test_reconnect(self): - req = "get:'%s/p/200:b@1:da'" % self.server.urlbase - p = self.pathoc() - - with p.connect(): - assert p.request(req) - # Server has disconnected. Mitmproxy should detect this, and reconnect. - assert p.request(req) - assert p.request(req) - - @pytest.mark.asyncio - async def test_get_connection_switching(self): - req = "get:'%s/p/200:b@1'" - p = self.pathoc() - with p.connect(): - assert p.request(req % self.server.urlbase) - assert p.request(req % self.server2.urlbase) - assert await self.proxy.tmaster.await_log("serverdisconnect") - - def test_blank_leading_line(self): - p = self.pathoc() - with p.connect(): - req = "get:'%s/p/201':i0,'\r\n'" - assert p.request(req % self.server.urlbase).status_code == 201 - - def test_invalid_headers(self): - p = self.pathoc() - with p.connect(): - resp = p.request("get:'http://foo':h':foo'='bar'") - assert resp.status_code == 400 - - @pytest.mark.asyncio - async def test_stream_modify(self, tdata): - s = script.Script( - tdata.path("mitmproxy/data/addonscripts/stream_modify.py"), - False, - ) - self.set_addons(s) - await self.master.await_log("stream_modify running") - - d = self.pathod('200:b"foo"') - assert d.content == b"bar" - - def test_first_line_rewrite(self): - """ - If mitmproxy is a regular HTTP proxy, it must rewrite an absolute-form request like - GET http://example.com/foo HTTP/1.0 - to - GET /foo HTTP/1.0 - when sending the request upstream. While any server should technically accept - the absolute form, this is not the case in practice. - """ - req = "get:'%s/p/200'" % self.server.urlbase - p = self.pathoc() - with p.connect(): - assert p.request(req).status_code == 200 - assert self.server.last_log()["request"]["first_line_format"] == "relative" - - -class TestHTTPS(tservers.HTTPProxyTest, CommonMixin, TcpMixin): - ssl = True - ssloptions = pathod.SSLOptions(request_client_cert=True) - - def test_clientcert_file(self, tdata): - try: - self.options.client_certs = os.path.join( - tdata.path("mitmproxy/data/clientcert"), "client.pem") - f = self.pathod("304") - assert f.status_code == 304 - assert self.server.last_log()["request"]["clientcert"]["keyinfo"] - finally: - self.options.client_certs = None - - def test_clientcert_dir(self, tdata): - try: - self.options.client_certs = tdata.path("mitmproxy/data/clientcert") - f = self.pathod("304") - assert f.status_code == 304 - assert self.server.last_log()["request"]["clientcert"]["keyinfo"] - finally: - self.options.client_certs = None - - def test_error_post_connect(self): - p = self.pathoc() - with p.connect(): - assert p.request("get:/:i0,'invalid\r\n\r\n'").status_code == 400 - - -class TestHTTPSCertfile(tservers.HTTPProxyTest, CommonMixin): - ssl = True - certfile = True - - def test_certfile(self): - assert self.pathod("304") - - -class TestHTTPSSecureByDefault: - def test_secure_by_default(self): - """ - Certificate verification should be turned on by default. - """ - default_opts = options.Options() - assert not default_opts.ssl_insecure - - -class TestHTTPSUpstreamServerVerificationWTrustedCert(tservers.HTTPProxyTest): - - """ - Test upstream server certificate verification with a trusted server cert. - """ - ssl = True - ssloptions = pathod.SSLOptions( - cn=b"example.mitmproxy.org", - certs=[ - ("example.mitmproxy.org", cdata.path("../data/servercert/trusted-leaf.pem")) - ] - ) - - def _request(self): - p = self.pathoc(sni="example.mitmproxy.org") - with p.connect(): - return p.request("get:/p/242") - - def test_verification_w_confdir(self, tdata): - self.options.update( - ssl_insecure=False, - ssl_verify_upstream_trusted_confdir=tdata.path( - "mitmproxy/data/servercert/" - ), - ssl_verify_upstream_trusted_ca=None, - ) - assert self._request().status_code == 242 - - def test_verification_w_pemfile(self, tdata): - self.options.update( - ssl_insecure=False, - ssl_verify_upstream_trusted_confdir=None, - ssl_verify_upstream_trusted_ca=tdata.path( - "mitmproxy/data/servercert/trusted-root.pem" - ), - ) - assert self._request().status_code == 242 - - -class TestHTTPSUpstreamServerVerificationWBadCert(tservers.HTTPProxyTest): - - """ - Test upstream server certificate verification with an untrusted server cert. - """ - ssl = True - ssloptions = pathod.SSLOptions( - cn=b"example.mitmproxy.org", - certs=[ - ("example.mitmproxy.org", cdata.path("../data/servercert/self-signed.pem")) - ]) - - def _request(self): - p = self.pathoc(sni="example.mitmproxy.org") - with p.connect(): - return p.request("get:/p/242") - - @classmethod - def get_options(cls): - opts = super().get_options() - opts.ssl_verify_upstream_trusted_ca = cdata.path( - "../data/servercert/trusted-root.pem" - ) - return opts - - def test_no_verification_w_bad_cert(self): - self.options.ssl_insecure = True - r = self._request() - assert r.status_code == 242 - - def test_verification_w_bad_cert(self): - # We only test for a single invalid cert here. - # Actual testing of different root-causes (invalid hostname, expired, ...) - # is done in mitmproxy.net. - self.options.ssl_insecure = False - r = self._request() - assert r.status_code == 502 - assert b"Certificate verification error" in r.raw_content - - -class TestHTTPSNoCommonName(tservers.HTTPProxyTest): - - """ - Test what happens if we get a cert without common name back. - """ - ssl = True - ssloptions = pathod.SSLOptions( - certs=[ - ("*", cdata.path("../data/no_common_name.pem")) - ] - ) - - def test_http(self): - f = self.pathod("202") - assert f.sslinfo.certchain[0].get_subject().CN == "127.0.0.1" - - -class TestReverse(tservers.ReverseProxyTest, CommonMixin, TcpMixin): - reverse = True - - def test_host_header(self): - self.options.keep_host_header = True - p = self.pathoc() - with p.connect(): - resp = p.request("get:/p/200:h'Host'='example.com'") - assert resp.status_code == 200 - - req = self.master.state.flows[0].request - assert req.host_header == "example.com" - - def test_overridden_host_header(self): - self.options.keep_host_header = False # default value - p = self.pathoc() - with p.connect(): - resp = p.request("get:/p/200:h'Host'='example.com'") - assert resp.status_code == 200 - - req = self.master.state.flows[0].request - assert req.host_header.startswith("127.0.0.1:") - - @pytest.mark.asyncio - async def test_selfconnection(self): - self.options.mode = "reverse:http://127.0.0.1:0" - - p = self.pathoc() - with p.connect(): - p.request("get:/") - assert await self.master.await_log("The proxy shall not connect to itself.") - - -class TestReverseSSL(tservers.ReverseProxyTest, CommonMixin, TcpMixin): - reverse = True - ssl = True - - -class TestSocks5(tservers.SocksModeTest): - - def test_simple(self): - p = self.pathoc() - with p.connect(): - p.socks_connect(("localhost", self.server.port)) - f = p.request("get:/p/200") - assert f.status_code == 200 - - def test_with_authentication_only(self): - p = self.pathoc() - with p.connect(): - f = p.request("get:/p/200") - assert f.status_code == 502 - assert b"SOCKS5 mode failure" in f.content - assert b"Invalid SOCKS version. Expected 0x05, got 0x47" in f.content - - def test_no_connect(self): - """ - mitmproxy doesn't support UDP or BIND SOCKS CMDs - """ - p = self.pathoc() - with p.connect(): - socks.ClientGreeting( - socks.VERSION.SOCKS5, - [socks.METHOD.NO_AUTHENTICATION_REQUIRED] - ).to_file(p.wfile) - socks.Message( - socks.VERSION.SOCKS5, - socks.CMD.BIND, - socks.ATYP.DOMAINNAME, - ("example.com", 8080) - ).to_file(p.wfile) - - p.wfile.flush() - p.rfile.read(2) # read server greeting - f = p.request("get:/p/200") # the request doesn't matter, error response from handshake will be read anyway. - assert f.status_code == 502 - assert b"SOCKS5 mode failure" in f.content - assert b"mitmproxy only supports SOCKS5 CONNECT" in f.content - - def test_with_authentication(self): - p = self.pathoc() - with p.connect(): - socks.ClientGreeting( - socks.VERSION.SOCKS5, - [socks.METHOD.USERNAME_PASSWORD] - ).to_file(p.wfile) - p.wfile.flush() - f = p.request("get:/p/200") # the request doesn't matter, error response from handshake will be read anyway. - assert f.status_code == 502 - assert b"SOCKS5 mode failure" in f.content - assert b"mitmproxy only supports SOCKS without authentication" in f.content - - -class TestSocks5SSL(tservers.SocksModeTest): - ssl = True - - def test_simple(self): - p = self.pathoc_raw() - with p.connect(): - p.socks_connect(("localhost", self.server.port)) - p.convert_to_tls() - f = p.request("get:/p/200") - assert f.status_code == 200 - - -class TestHttps2Http(tservers.ReverseProxyTest): - - @classmethod - def get_options(cls): - opts = super().get_options() - return opts - - def pathoc(self, ssl, sni=None): - """ - Returns a connected Pathoc instance. - """ - p = pathoc.Pathoc( - ("localhost", self.proxy.port), ssl=True, sni=sni, fp=None - ) - return p - - def test_all(self): - p = self.pathoc(ssl=True) - with p.connect(): - assert p.request("get:'/p/200'").status_code == 200 - - def test_sni(self): - p = self.pathoc(ssl=True, sni="example.com") - with p.connect(): - assert p.request("get:'/p/200'").status_code == 200 - - def test_http(self): - p = self.pathoc(ssl=False) - with p.connect(): - assert p.request("get:'/p/200'").status_code == 200 - - -class TestTransparent(tservers.TransparentProxyTest, CommonMixin, TcpMixin): - ssl = False - - def test_tcp_stream_modify(self, tdata): - s = script.Script( - tdata.path("mitmproxy/data/addonscripts/tcp_stream_modify.py"), - False, - ) - self.set_addons(s) - self._tcpproxy_on() - d = self.pathod('200:b"foo"') - self._tcpproxy_off() - assert d.content == b"bar" - - -class TestTransparentSSL(tservers.TransparentProxyTest, CommonMixin, TcpMixin): - ssl = True - - def test_sslerr(self): - p = pathoc.Pathoc(("localhost", self.proxy.port), fp=None) - p.connect() - r = p.request("get:/") - assert r.status_code == 502 - - -class TestProxy(tservers.HTTPProxyTest): - - def test_http(self): - f = self.pathod("304") - assert f.status_code == 304 - - f = self.master.state.flows[0] - assert f.client_conn.address - assert "host" in f.request.headers - assert f.response.status_code == 304 - - @skip_appveyor - def test_response_timestamps(self): - # test that we notice at least 1 sec delay between timestamps - # in response object - f = self.pathod("304:b@1k:p50,1") - assert f.status_code == 304 - - response = self.master.state.flows[0].response - # timestamp_start might fire a bit late, so we play safe and only require 300ms. - assert 0.3 <= response.timestamp_end - response.timestamp_start - - @skip_appveyor - def test_request_timestamps(self): - # test that we notice a delay between timestamps in request object - connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - connection.connect(("127.0.0.1", self.proxy.port)) - - # call pathod server, wait a second to complete the request - connection.send( - b"GET http://localhost:%d/p/304:b@1k HTTP/1.1\r\n" % - self.server.port) - time.sleep(1) - connection.send(b"\r\n") - connection.recv(50000) - connection.close() - - request, response = self.master.state.flows[ - 0].request, self.master.state.flows[0].response - assert response.status_code == 304 # sanity test for our low level request - # timestamp_start might fire a bit late, so we play safe and only require 300ms. - assert 0.3 <= request.timestamp_end - request.timestamp_start - - def test_request_tcp_setup_timestamp_presence(self): - # tests that the client_conn a tcp connection has a tcp_setup_timestamp - connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - connection.connect(("localhost", self.proxy.port)) - connection.send( - b"GET http://localhost:%d/p/200:b@1k HTTP/1.1\r\n" % - self.server.port) - connection.send(b"\r\n") - # a bit hacky: make sure that we don't just read the headers only. - recvd = 0 - while recvd < 1024: - recvd += len(connection.recv(5000)) - connection.send( - b"GET http://localhost:%d/p/200:b@1k HTTP/1.1\r\n" % - self.server.port) - connection.send(b"\r\nb") - recvd = 0 - while recvd < 1024: - recvd += len(connection.recv(5000)) - connection.close() - - first_flow = self.master.state.flows[0] - second_flow = self.master.state.flows[1] - assert first_flow.server_conn.timestamp_tcp_setup - assert first_flow.server_conn.timestamp_tls_setup is None - assert second_flow.server_conn.timestamp_tcp_setup - assert first_flow.server_conn.timestamp_tcp_setup == second_flow.server_conn.timestamp_tcp_setup - - def test_request_ip(self): - f = self.pathod("200:b@100") - assert f.status_code == 200 - f = self.master.state.flows[0] - assert f.server_conn.address == ("127.0.0.1", self.server.port) - - -class TestProxySSL(tservers.HTTPProxyTest): - ssl = True - - def test_request_tls_attribute_presence(self): - # tests that the ssl timestamp is present when ssl is used - f = self.pathod("304:b@10k") - assert f.status_code == 304 - first_flow = self.master.state.flows[0] - assert first_flow.server_conn.timestamp_tls_setup - assert first_flow.client_conn.tls_extensions - - def test_via(self): - # tests that the ssl timestamp is present when ssl is used - f = self.pathod("200:b@10") - assert f.status_code == 200 - first_flow = self.master.state.flows[0] - assert not first_flow.server_conn.via - - -class ARedirectRequest: - def __init__(self, redirect_port): - self.redirect_port = redirect_port - - def request(self, f): - if f.request.path == "/p/201": - # This part should have no impact, but it should also not cause any exceptions. - addr = f.live.server_conn.address - addr2 = ("127.0.0.1", self.redirect_port) - f.live.set_server(addr2) - f.live.set_server(addr) - - # This is the actual redirection. - f.request.port = self.redirect_port - - def response(self, f): - f.response.content = bytes(f.client_conn.address[1]) - f.response.headers["server-conn-id"] = str(f.server_conn.source_address[1]) - - -class TestRedirectRequest(tservers.HTTPProxyTest): - ssl = True - - def test_redirect(self): - """ - Imagine a single HTTPS connection with three requests: - - 1. First request should pass through unmodified - 2. Second request will be redirected to a different host by an inline script - 3. Third request should pass through unmodified - - This test verifies that the original destination is restored for the third request. - """ - self.set_addons(ARedirectRequest(self.server2.port)) - - p = self.pathoc() - with p.connect(): - self.server.clear_log() - self.server2.clear_log() - r1 = p.request("get:'/p/200'") - assert r1.status_code == 200 - assert self.server.last_log() - assert not self.server2.expect_log(1, 0.5) - - self.server.clear_log() - self.server2.clear_log() - r2 = p.request("get:'/p/201'") - assert r2.status_code == 201 - assert not self.server.expect_log(1, 0.5) - assert self.server2.last_log() - - self.server.clear_log() - self.server2.clear_log() - r3 = p.request("get:'/p/202'") - assert r3.status_code == 202 - assert self.server.last_log() - assert not self.server2.expect_log(1, 0.5) - - assert r1.content == r2.content == r3.content - - -class AStreamRequest: - - """ - Enables the stream flag on the flow for all requests - """ - def responseheaders(self, f): - f.response.stream = True - - -class TestStreamRequest(tservers.HTTPProxyTest): - def test_stream_simple(self): - self.set_addons(AStreamRequest()) - p = self.pathoc() - with p.connect(): - # a request with 100k of data but without content-length - r1 = p.request("get:'%s/p/200:r:b@100k:d102400'" % self.server.urlbase) - assert r1.status_code == 200 - assert len(r1.content) > 100000 - - def test_stream_multiple(self): - self.set_addons(AStreamRequest()) - p = self.pathoc() - with p.connect(): - # simple request with streaming turned on - r1 = p.request("get:'%s/p/200'" % self.server.urlbase) - assert r1.status_code == 200 - - # now send back 100k of data, streamed but not chunked - r1 = p.request("get:'%s/p/201:b@100k'" % self.server.urlbase) - assert r1.status_code == 201 - - def test_stream_chunked(self): - self.set_addons(AStreamRequest()) - connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - connection.connect(("127.0.0.1", self.proxy.port)) - fconn = connection.makefile("rb") - spec = '200:h"Transfer-Encoding"="chunked":r:b"4\\r\\nthis\\r\\n11\\r\\nisatest__reachhex\\r\\n0\\r\\n\\r\\n"' - connection.send( - b"GET %s/p/%s HTTP/1.1\r\n" % - (self.server.urlbase.encode(), spec.encode())) - connection.send(b"\r\n") - - resp = http1.read_response_head(fconn) - - assert resp.headers["Transfer-Encoding"] == 'chunked' - assert resp.status_code == 200 - - chunks = list(http1.read_body(fconn, None)) - assert chunks == [b"this", b"isatest__reachhex"] - - fconn.close() - connection.shutdown(socket.SHUT_RDWR) - connection.close() - - -class AFakeResponse: - def request(self, f): - f.response = mitmproxy.test.tutils.tresp() - - -class TestFakeResponse(tservers.HTTPProxyTest): - - def test_fake(self): - self.set_addons(AFakeResponse()) - f = self.pathod("200") - assert "header-response" in f.headers - - -class TestServerConnect(tservers.HTTPProxyTest): - ssl = True - - @classmethod - def get_options(cls): - opts = tservers.HTTPProxyTest.get_options() - opts.upstream_cert = False - return opts - - @pytest.mark.asyncio - async def test_unnecessary_serverconnect(self): - """A replayed/fake response with no upstream_cert should not connect to an upstream server""" - self.set_addons(AFakeResponse()) - assert self.pathod("200").status_code == 200 - await asyncio.sleep(0.1) - assert not self.proxy.tmaster.has_log("serverconnect") - - -class AKillRequest: - - def request(self, f): - f.reply.kill() - - -class TestKillRequest(tservers.HTTPProxyTest): - def test_kill(self): - self.set_addons(AKillRequest()) - with pytest.raises(exceptions.HttpReadDisconnect): - self.pathod("200") - # Nothing should have hit the server - assert not self.server.expect_log(1, 0.5) - - -class AKillResponse: - def response(self, f): - f.reply.kill() - - -class TestKillResponse(tservers.HTTPProxyTest): - def test_kill(self): - self.set_addons(AKillResponse()) - with pytest.raises(exceptions.HttpReadDisconnect): - self.pathod("200") - # The server should have seen a request - assert self.server.last_log() - - -class TestTransparentResolveError(tservers.TransparentProxyTest): - @mock.patch("mitmproxy.platform.original_addr") - def test_resolve_error(self, original_addr): - original_addr.side_effect = RuntimeError - assert self.pathod("304").status_code == 502 - - -class AIncomplete: - def request(self, f): - resp = mitmproxy.test.tutils.tresp() - resp.content = None - f.response = resp - - -class TestIncompleteResponse(tservers.HTTPProxyTest): - def test_incomplete(self): - self.set_addons(AIncomplete()) - assert self.pathod("200").status_code == 502 - - -class TestUpstreamProxy(tservers.HTTPUpstreamProxyTest, CommonMixin): - ssl = False - - -class TestUpstreamProxySSL( - tservers.HTTPUpstreamProxyTest, - CommonMixin, - TcpMixin): - ssl = True - - def _host_pattern_on(self, attr): - """ - Updates config.check_tcp or check_filter, depending on attr. - """ - assert not hasattr(self, "_ignore_%s_backup" % attr) - backup = [] - handle = attr - attr = "filter" if attr in ["allow", "ignore"] else attr - for proxy in self.chain: - old_matcher = getattr( - proxy.tmaster.server.config, - "check_%s" % - attr) - backup.append(old_matcher) - setattr( - proxy.tmaster.server.config, - "check_%s" % attr, - HostMatcher(handle, [".+:%s" % self.server.port] + old_matcher.patterns) - ) - - setattr(self, "_ignore_%s_backup" % attr, backup) - - def _host_pattern_off(self, attr): - attr = "filter" if attr in ["allow", "ignore"] else attr - backup = getattr(self, "_ignore_%s_backup" % attr) - for proxy in reversed(self.chain): - setattr( - proxy.tmaster.server.config, - "check_%s" % attr, - backup.pop() - ) - - assert not backup - delattr(self, "_ignore_%s_backup" % attr) - - def _ignore_on(self): - super()._ignore_on() - self._host_pattern_on("ignore") - - def _ignore_off(self): - super()._ignore_off() - self._host_pattern_off("ignore") - - def _tcpproxy_on(self): - super()._tcpproxy_on() - self._host_pattern_on("tcp") - - def _tcpproxy_off(self): - super()._tcpproxy_off() - self._host_pattern_off("tcp") - - def test_simple(self): - p = self.pathoc() - with p.connect(): - req = p.request("get:'/p/418:b\"content\"'") - assert req.content == b"content" - assert req.status_code == 418 - - # CONNECT from pathoc to chain[0], - assert len(self.proxy.tmaster.state.flows) == 1 - assert self.proxy.tmaster.state.flows[0].server_conn.via - # request from pathoc to chain[0] - # CONNECT from proxy to chain[1], - assert len(self.chain[0].tmaster.state.flows) == 1 - assert self.chain[0].tmaster.state.flows[0].server_conn.via - # request from proxy to chain[1] - # request from chain[0] (regular proxy doesn't store CONNECTs) - assert not self.chain[1].tmaster.state.flows[0].server_conn.via - assert len(self.chain[1].tmaster.state.flows) == 1 - - def test_change_upstream_proxy_connect(self): - # skip chain[0]. - self.set_addons( - UpstreamProxyChanger( - ("127.0.0.1", self.chain[1].port) - ) - ) - p = self.pathoc() - with p.connect(): - req = p.request("get:'/p/418'") - - assert req.status_code == 418 - assert len(self.chain[0].tmaster.state.flows) == 0 - assert len(self.chain[1].tmaster.state.flows) == 1 - - def test_connect_https_to_http(self): - """ - https://github.com/mitmproxy/mitmproxy/issues/2329 - - Client <- HTTPS -> Proxy <- HTTP -> Proxy <- HTTPS -> Server - """ - self.set_addons(RewriteToHttp()) - self.chain[1].set_addons(RewriteToHttps()) - p = self.pathoc() - with p.connect(): - resp = p.request("get:'/p/418'") - - assert self.proxy.tmaster.state.flows[0].client_conn.tls_established - assert not self.proxy.tmaster.state.flows[0].server_conn.tls_established - assert not self.chain[1].tmaster.state.flows[0].client_conn.tls_established - assert self.chain[1].tmaster.state.flows[0].server_conn.tls_established - assert resp.status_code == 418 - - -class RewriteToHttp: - def http_connect(self, f): - f.request.scheme = "http" - - def request(self, f): - f.request.scheme = "http" - - -class RewriteToHttps: - def http_connect(self, f): - f.request.scheme = "https" - - def request(self, f): - f.request.scheme = "https" - - -class UpstreamProxyChanger: - def __init__(self, addr): - self.address = addr - - def request(self, f): - f.live.change_upstream_proxy_server(self.address) - - -class RequestKiller: - def __init__(self, exclude): - self.exclude = exclude - self.k = 0 - - def request(self, f): - self.k += 1 - if self.k not in self.exclude: - f.reply.kill() - - -class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxyTest): - ssl = True - - def test_reconnect(self): - """ - Tests proper functionality of ConnectionHandler.server_reconnect mock. - If we have a disconnect on a secure connection that's transparently - proxified to an upstream http proxy, we need to send the CONNECT - request again. - """ - - self.chain[0].set_addons(RequestKiller([1, 2])) - self.chain[1].set_addons(RequestKiller([1])) - - p = self.pathoc() - with p.connect(): - req = p.request("get:'/p/418:b\"content\"'") - assert req.content == b"content" - assert req.status_code == 418 - - # First request goes through all three proxies exactly once - assert len(self.proxy.tmaster.state.flows) == 1 - assert len(self.chain[0].tmaster.state.flows) == 1 - assert len(self.chain[1].tmaster.state.flows) == 1 - - req = p.request("get:'/p/418:b\"content2\"'") - - assert req.status_code == 502 - - assert len(self.proxy.tmaster.state.flows) == 2 - assert len(self.chain[0].tmaster.state.flows) == 2 - # Upstream sees two requests due to reconnection attempt - assert len(self.chain[1].tmaster.state.flows) == 3 - assert not self.chain[1].tmaster.state.flows[-1].response - assert not self.chain[1].tmaster.state.flows[-2].response - - # Reconnection failed, so we're now disconnected - with pytest.raises(exceptions.HttpException): - p.request("get:'/p/418:b\"content3\"'") - - -class AddUpstreamCertsToClientChainMixin: - - ssl = True - servercert = cdata.path("../data/servercert/trusted-root.pem") - ssloptions = pathod.SSLOptions( - cn=b"example.mitmproxy.org", - certs=[ - ("example.mitmproxy.org", servercert) - ] - ) - - def test_add_upstream_certs_to_client_chain(self): - with open(self.servercert, "rb") as f: - d = f.read() - upstreamCert = certs.Cert.from_pem(d) - p = self.pathoc() - with p.connect(): - upstream_cert_found_in_client_chain = False - for receivedCert in p.server_certs: - if receivedCert.digest('sha256') == upstreamCert.digest('sha256'): - upstream_cert_found_in_client_chain = True - break - assert(upstream_cert_found_in_client_chain == self.master.options.add_upstream_certs_to_client_chain) - - -class TestHTTPSAddUpstreamCertsToClientChainTrue( - AddUpstreamCertsToClientChainMixin, - tservers.HTTPProxyTest -): - """ - If --add-server-certs-to-client-chain is True, then the client should - receive the upstream server's certificates - """ - @classmethod - def get_options(cls): - opts = super().get_options() - opts.add_upstream_certs_to_client_chain = True - return opts - - -class TestHTTPSAddUpstreamCertsToClientChainFalse( - AddUpstreamCertsToClientChainMixin, - tservers.HTTPProxyTest -): - """ - If --add-server-certs-to-client-chain is False, then the client should not - receive the upstream server's certificates - """ - @classmethod - def get_options(cls): - opts = super().get_options() - opts.add_upstream_certs_to_client_chain = False - return opts diff --git a/test/mitmproxy/test_connections.py b/test/mitmproxy/test_connections.py index 154df4491..79f8bc85f 100644 --- a/test/mitmproxy/test_connections.py +++ b/test/mitmproxy/test_connections.py @@ -11,7 +11,6 @@ from mitmproxy.net import tcp from mitmproxy.net.http import http1 from mitmproxy.test import tflow from .net import tservers -from pathod import test from ..conftest import skip_new_proxy_core @@ -135,35 +134,6 @@ class TestServerConnection: c = connections.ServerConnection.make_dummy(('foobar', 1234)) assert c.address == ('foobar', 1234) - def test_simple(self): - d = test.Daemon() - c = connections.ServerConnection((d.IFACE, d.port)) - c.connect() - f = tflow.tflow() - f.server_conn = c - f.request.path = "/p/200:da" - - # use this protocol just to assemble - not for actual sending - c.wfile.write(http1.assemble_request(f.request)) - c.wfile.flush() - - assert http1.read_response(c.rfile, f.request, 1000) - assert d.last_log() - - c.finish() - c.close() - d.shutdown() - - def test_terminate_error(self): - d = test.Daemon() - c = connections.ServerConnection((d.IFACE, d.port)) - c.connect() - c.close() - c.connection = mock.Mock() - c.connection.recv = mock.Mock(return_value=False) - c.connection.flush = mock.Mock(side_effect=exceptions.TcpDisconnect) - d.shutdown() - def test_sni(self): c = connections.ServerConnection(('', 1234)) with pytest.raises(ValueError, match='sni must be str, not '): diff --git a/test/mitmproxy/test_fuzzing.py b/test/mitmproxy/test_fuzzing.py deleted file mode 100644 index 57d0ca552..000000000 --- a/test/mitmproxy/test_fuzzing.py +++ /dev/null @@ -1,28 +0,0 @@ -from . import tservers - -""" - A collection of errors turned up by fuzzing. Errors are integrated here - after being fixed to check for regressions. -""" - - -class TestFuzzy(tservers.HTTPProxyTest): - - def test_idna_err(self): - req = r'get:"http://localhost:%s":i10,"\xc6"' - p = self.pathoc() - with p.connect(): - assert p.request(req % self.server.port).status_code == 400 - - def test_nullbytes(self): - req = r'get:"http://localhost:%s":i19,"\x00"' - p = self.pathoc() - with p.connect(): - assert p.request(req % self.server.port).status_code == 400 - - def test_invalid_ipv6_url(self): - req = 'get:"http://localhost:%s":i13,"["' - p = self.pathoc() - with p.connect(): - resp = p.request(req % self.server.port) - assert resp.status_code == 400 \ No newline at end of file diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py deleted file mode 100644 index d1dd9d8e6..000000000 --- a/test/mitmproxy/tservers.py +++ /dev/null @@ -1,387 +0,0 @@ -import os.path -import threading -import tempfile -import sys -import time -from unittest import mock -import asyncio - -import mitmproxy.platform -from mitmproxy.addons import core -from mitmproxy.proxy.config import ProxyConfig -from mitmproxy.proxy.server import ProxyServer -from mitmproxy import controller -from mitmproxy import options -from mitmproxy import exceptions -from mitmproxy import io -from mitmproxy.utils import human -import pathod.test -import pathod.pathoc - -from mitmproxy import eventsequence -from mitmproxy.test import tflow -from mitmproxy.test import tutils -from mitmproxy.test import taddons - - -class MasterTest: - - async def cycle(self, master, content): - f = tflow.tflow(req=tutils.treq(content=content)) - layer = mock.Mock("mitmproxy.proxy.protocol.base.Layer") - layer.client_conn = f.client_conn - layer.reply = controller.DummyReply() - await master.addons.handle_lifecycle("clientconnect", layer) - for i in eventsequence.iterate(f): - await master.addons.handle_lifecycle(*i) - await master.addons.handle_lifecycle("clientdisconnect", layer) - return f - - async def dummy_cycle(self, master, n, content): - for i in range(n): - await self.cycle(master, content) - await master._shutdown() - - def flowfile(self, path): - with open(path, "wb") as f: - fw = io.FlowWriter(f) - t = tflow.tflow(resp=True) - fw.add(t) - - -class TestState: - def __init__(self): - self.flows = [] - - def request(self, f): - if f not in self.flows: - self.flows.append(f) - - def response(self, f): - if f not in self.flows: - self.flows.append(f) - - def websocket_start(self, f): - if f not in self.flows: - self.flows.append(f) - - -class TestMaster(taddons.RecordingMaster): - - def __init__(self, opts): - super().__init__(opts) - config = ProxyConfig(opts) - self.server = ProxyServer(config) - - def clear_addons(self, addons): - self.addons.clear() - self.state = TestState() - self.addons.add(self.state) - self.addons.add(*addons) - self.addons.trigger("configure", self.options.keys()) - self.addons.trigger("running") - - def reset(self, addons): - self.clear_addons(addons) - self.clear() - - -class ProxyThread(threading.Thread): - - def __init__(self, masterclass, options): - threading.Thread.__init__(self) - self.masterclass = masterclass - self.options = options - self.tmaster = None - self.event_loop = None - controller.should_exit = False - - @property - def port(self): - return self.tmaster.server.address[1] - - @property - def tlog(self): - return self.tmaster.logs - - def shutdown(self): - self.tmaster.shutdown() - - def run(self): - self.event_loop = asyncio.new_event_loop() - asyncio.set_event_loop(self.event_loop) - self.tmaster = self.masterclass(self.options) - self.tmaster.addons.add(core.Core()) - self.name = "ProxyThread (%s)" % human.format_address(self.tmaster.server.address) - self.tmaster.run() - - def set_addons(self, *addons): - self.tmaster.reset(addons) - - def start(self): - super().start() - while True: - if self.tmaster: - break - time.sleep(0.01) - - -class ProxyTestBase: - # Test Configuration - ssl = None - ssloptions = False - masterclass = TestMaster - - add_upstream_certs_to_client_chain = False - - @classmethod - def setup_class(cls): - cls.server = pathod.test.Daemon( - ssl=cls.ssl, - ssloptions=cls.ssloptions) - cls.server2 = pathod.test.Daemon( - ssl=cls.ssl, - ssloptions=cls.ssloptions) - - cls.options = cls.get_options() - cls.proxy = ProxyThread(cls.masterclass, cls.options) - cls.proxy.start() - - @classmethod - def teardown_class(cls): - # perf: we want to run tests in parallel - # should this ever cause an error, travis should catch it. - # shutil.rmtree(cls.confdir) - cls.proxy.shutdown() - cls.server.shutdown() - cls.server2.shutdown() - - def teardown(self): - try: - self.server.wait_for_silence() - except exceptions.Timeout: - # FIXME: Track down the Windows sync issues - if sys.platform != "win32": - raise - - def setup(self): - self.master.reset(self.addons()) - self.server.clear_log() - self.server2.clear_log() - - @property - def master(self): - return self.proxy.tmaster - - @classmethod - def get_options(cls): - cls.confdir = os.path.join(tempfile.gettempdir(), "mitmproxy") - return options.Options( - listen_port=0, - confdir=cls.confdir, - add_upstream_certs_to_client_chain=cls.add_upstream_certs_to_client_chain, - ssl_insecure=True, - ) - - def set_addons(self, *addons): - self.proxy.set_addons(*addons) - - def addons(self): - """ - Can be over-ridden to add a standard set of addons to tests. - """ - return [] - - -class LazyPathoc(pathod.pathoc.Pathoc): - def __init__(self, lazy_connect, *args, **kwargs): - self.lazy_connect = lazy_connect - pathod.pathoc.Pathoc.__init__(self, *args, **kwargs) - - def connect(self): - return pathod.pathoc.Pathoc.connect(self, self.lazy_connect) - - -class HTTPProxyTest(ProxyTestBase): - - def pathoc_raw(self): - return pathod.pathoc.Pathoc(("127.0.0.1", self.proxy.port), fp=None) - - def pathoc(self, sni=None): - """ - Returns a connected Pathoc instance. - """ - if self.ssl: - conn = ("127.0.0.1", self.server.port) - else: - conn = None - return LazyPathoc( - conn, - ("localhost", self.proxy.port), ssl=self.ssl, sni=sni, fp=None - ) - - def pathod(self, spec, sni=None): - """ - Constructs a pathod GET request, with the appropriate base and proxy. - """ - p = self.pathoc(sni=sni) - if self.ssl: - q = "get:'/p/%s'" % spec - else: - q = f"get:'{self.server.urlbase}/p/{spec}'" - with p.connect(): - return p.request(q) - - def app(self, page): - if self.ssl: - p = pathod.pathoc.Pathoc( - ("127.0.0.1", self.proxy.port), True, fp=None - ) - with p.connect((self.master.options.onboarding_host, self.master.options.onbarding_port)): - return p.request("get:'%s'" % page) - else: - p = self.pathoc() - with p.connect(): - return p.request(f"get:'http://{self.master.options.onboarding_host}{page}'") - - -class TransparentProxyTest(ProxyTestBase): - ssl = None - - @classmethod - def setup_class(cls): - cls._init_transparent_mode = mitmproxy.platform.init_transparent_mode - cls._original_addr = mitmproxy.platform.original_addr - mitmproxy.platform.init_transparent_mode = lambda: True - mitmproxy.platform.original_addr = lambda sock: ("127.0.0.1", cls.server.port) - super().setup_class() - - @classmethod - def teardown_class(cls): - super().teardown_class() - mitmproxy.platform.init_transparent_mode = cls._init_transparent_mode - mitmproxy.platform.original_addr = cls._original_addr - - @classmethod - def get_options(cls): - opts = ProxyTestBase.get_options() - opts.mode = "transparent" - return opts - - def pathod(self, spec, sni=None): - """ - Constructs a pathod GET request, with the appropriate base and proxy. - """ - if self.ssl: - p = self.pathoc(sni=sni) - q = "get:'/p/%s'" % spec - else: - p = self.pathoc() - q = "get:'/p/%s'" % spec - with p.connect(): - return p.request(q) - - def pathoc(self, sni=None): - """ - Returns a connected Pathoc instance. - """ - p = pathod.pathoc.Pathoc( - ("localhost", self.proxy.port), ssl=self.ssl, sni=sni, fp=None - ) - return p - - -class ReverseProxyTest(ProxyTestBase): - ssl = None - - @classmethod - def get_options(cls): - opts = ProxyTestBase.get_options() - s = "".join( - [ - "https" if cls.ssl else "http", - "://", - "127.0.0.1:", - str(cls.server.port) - ] - ) - opts.mode = "reverse:" + s - return opts - - def pathoc(self, sni=None): - """ - Returns a connected Pathoc instance. - """ - p = pathod.pathoc.Pathoc( - ("localhost", self.proxy.port), ssl=self.ssl, sni=sni, fp=None - ) - return p - - def pathod(self, spec, sni=None): - """ - Constructs a pathod GET request, with the appropriate base and proxy. - """ - if self.ssl: - p = self.pathoc(sni=sni) - q = "get:'/p/%s'" % spec - else: - p = self.pathoc() - q = "get:'/p/%s'" % spec - with p.connect(): - return p.request(q) - - -class SocksModeTest(HTTPProxyTest): - - @classmethod - def get_options(cls): - opts = ProxyTestBase.get_options() - opts.mode = "socks5" - return opts - - -class HTTPUpstreamProxyTest(HTTPProxyTest): - """ - Chain three instances of mitmproxy in a row to test upstream mode. - Proxy order is cls.proxy -> cls.chain[0] -> cls.chain[1] - cls.proxy and cls.chain[0] are in upstream mode, - cls.chain[1] is in regular mode. - """ - chain = None - n = 2 - - @classmethod - def setup_class(cls): - # We need to initialize the chain first so that the normal server gets a correct config. - cls.chain = [] - for _ in range(cls.n): - opts = cls.get_options() - proxy = ProxyThread(cls.masterclass, opts) - proxy.start() - cls.chain.insert(0, proxy) - while True: - if proxy.event_loop and proxy.event_loop.is_running(): - break - - super().setup_class() - - @classmethod - def teardown_class(cls): - super().teardown_class() - for proxy in cls.chain: - proxy.shutdown() - - def setup(self): - super().setup() - for proxy in self.chain: - proxy.tmaster.reset(self.addons()) - - @classmethod - def get_options(cls): - opts = super().get_options() - if cls.chain: # First proxy is in normal mode. - s = "http://127.0.0.1:%s" % cls.chain[0].port - opts.update( - mode="upstream:" + s, - ) - return opts diff --git a/test/pathod/__init__.py b/test/pathod/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test/pathod/data/clientcert/.gitignore b/test/pathod/data/clientcert/.gitignore deleted file mode 100644 index 07bc53d25..000000000 --- a/test/pathod/data/clientcert/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -client.crt -client.key -client.req diff --git a/test/pathod/data/clientcert/client.cnf b/test/pathod/data/clientcert/client.cnf deleted file mode 100644 index 5046a9440..000000000 --- a/test/pathod/data/clientcert/client.cnf +++ /dev/null @@ -1,5 +0,0 @@ -[ ssl_client ] -basicConstraints = CA:FALSE -nsCertType = client -keyUsage = digitalSignature, keyEncipherment -extendedKeyUsage = clientAuth diff --git a/test/pathod/data/clientcert/client.pem b/test/pathod/data/clientcert/client.pem deleted file mode 100644 index 4927bca28..000000000 --- a/test/pathod/data/clientcert/client.pem +++ /dev/null @@ -1,42 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEAzCpoRjSTfIN24kkNap/GYmP9zVWj0Gk8R5BB/PvvN0OB1Zk0 -EEYPsWCcuhEdK0ehiDZX030doF0DOncKKa6mop/d0x2o+ts42peDhZM6JNUrm6d+ -ZWQVtio33mpp77UMhR093vaA+ExDnmE26kBTVijJ1+fRAVDXG/cmQINEri91Kk/G -3YJ5e45UrohGI5seBZ4vV0xbHtmczFRhYFlGOvYsoIe4Lvz/eFS2pIrTIpYQ2VM/ -SQQl+JFy+NlQRsWG2NrxtKOzMnnDE7YN4I3z5D5eZFo1EtwZ48LNCeSwrEOdfuzP -G5q5qbs5KpE/x85H9umuRwSCIArbMwBYV8a8JwIDAQABAoIBAFE3FV/IDltbmHEP -iky93hbJm+6QgKepFReKpRVTyqb7LaygUvueQyPWQMIriKTsy675nxo8DQr7tQsO -y3YlSZgra/xNMikIB6e82c7K8DgyrDQw/rCqjZB3Xt4VCqsWJDLXnQMSn98lx0g7 -d7Lbf8soUpKWXqfdVpSDTi4fibSX6kshXyfSTpcz4AdoncEpViUfU1xkEEmZrjT8 -1GcCsDC41xdNmzCpqRuZX7DKSFRoB+0hUzsC1oiqM7FD5kixonRd4F5PbRXImIzt -6YCsT2okxTA04jX7yByis7LlOLTlkmLtKQYuc3erOFvwx89s4vW+AeFei+GGNitn -tHfSwbECgYEA7SzV+nN62hAERHlg8cEQT4TxnsWvbronYWcc/ev44eHSPDWL5tPi -GHfSbW6YAq5Wa0I9jMWfXyhOYEC3MZTC5EEeLOB71qVrTwcy/sY66rOrcgjFI76Q -5JFHQ4wy3SWU50KxE0oWJO9LIowprG+pW1vzqC3VF0T7q0FqESrY4LUCgYEA3F7Z -80ndnCUlooJAb+Hfotv7peFf1o6+m1PTRcz1lLnVt5R5lXj86kn+tXEpYZo1RiGR -2rE2N0seeznWCooakHcsBN7/qmFIhhooJNF7yW+JP2I4P2UV5+tJ+8bcs/voUkQD -1x+rGOuMn8nvHBd2+Vharft8eGL2mgooPVI2XusCgYEAlMZpO3+w8pTVeHaDP2MR -7i/AuQ3cbCLNjSX3Y7jgGCFllWspZRRIYXzYPNkA9b2SbBnTLjjRLgnEkFBIGgvs -7O2EFjaCuDRvydUEQhjq4ErwIsopj7B8h0QyZcbOKTbn3uFQ3n68wVJx2Sv/ADHT -FIHrp/WIE96r19Niy34LKXkCgYB2W59VsuOKnMz01l5DeR5C+0HSWxS9SReIl2IO -yEFSKullWyJeLIgyUaGy0990430feKI8whcrZXYumuah7IDN/KOwzhCk8vEfzWao -N7bzfqtJVrh9HA7C7DVlO+6H4JFrtcoWPZUIomJ549w/yz6EN3ckoMC+a/Ck1TW9 -ka1QFwKBgQCywG6TrZz0UmOjyLQZ+8Q4uvZklSW5NAKBkNnyuQ2kd5rzyYgMPE8C -Er8T88fdVIKvkhDyHhwcI7n58xE5Gr7wkwsrk/Hbd9/ZB2GgAPY3cATskK1v1McU -YeX38CU0fUS4aoy26hWQXkViB47IGQ3jWo3ZCtzIJl8DI9/RsBWTnw== ------END RSA PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIICYDCCAckCAQEwDQYJKoZIhvcNAQEFBQAwKDESMBAGA1UEAxMJbWl0bXByb3h5 -MRIwEAYDVQQKEwltaXRtcHJveHkwHhcNMTMwMTIwMDEwODEzWhcNMTUxMDE3MDEw -ODEzWjBFMQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UE -ChMYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC -AQ8AMIIBCgKCAQEAzCpoRjSTfIN24kkNap/GYmP9zVWj0Gk8R5BB/PvvN0OB1Zk0 -EEYPsWCcuhEdK0ehiDZX030doF0DOncKKa6mop/d0x2o+ts42peDhZM6JNUrm6d+ -ZWQVtio33mpp77UMhR093vaA+ExDnmE26kBTVijJ1+fRAVDXG/cmQINEri91Kk/G -3YJ5e45UrohGI5seBZ4vV0xbHtmczFRhYFlGOvYsoIe4Lvz/eFS2pIrTIpYQ2VM/ -SQQl+JFy+NlQRsWG2NrxtKOzMnnDE7YN4I3z5D5eZFo1EtwZ48LNCeSwrEOdfuzP -G5q5qbs5KpE/x85H9umuRwSCIArbMwBYV8a8JwIDAQABMA0GCSqGSIb3DQEBBQUA -A4GBAFvI+cd47B85PQ970n2dU/PlA2/Hb1ldrrXh2guR4hX6vYx/uuk5yRI/n0Rd -KOXJ3czO0bd2Fpe3ZoNpkW0pOSDej/Q+58ScuJd0gWCT/Sh1eRk6ZdC0kusOuWoY -bPOPMkG45LPgUMFOnZEsfJP6P5mZIxlbCvSMFC25nPHWlct7 ------END CERTIFICATE----- diff --git a/test/pathod/data/clientcert/make b/test/pathod/data/clientcert/make deleted file mode 100644 index d1caea813..000000000 --- a/test/pathod/data/clientcert/make +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -openssl genrsa -out client.key 2048 -openssl req -key client.key -new -out client.req -openssl x509 -req -days 365 -in client.req -signkey client.key -out client.crt -extfile client.cnf -extensions ssl_client -openssl x509 -req -days 1000 -in client.req -CA ~/.mitmproxy/mitmproxy-ca.pem -CAkey ~/.mitmproxy/mitmproxy-ca.pem -set_serial 00001 -out client.crt -extensions ssl_client -cat client.key client.crt > client.pem -openssl x509 -text -noout -in client.pem diff --git a/test/pathod/data/file b/test/pathod/data/file deleted file mode 100644 index 26918572e..000000000 --- a/test/pathod/data/file +++ /dev/null @@ -1 +0,0 @@ -testfile diff --git a/test/pathod/data/request b/test/pathod/data/request deleted file mode 100644 index c4c90e76f..000000000 --- a/test/pathod/data/request +++ /dev/null @@ -1 +0,0 @@ -get:/foo diff --git a/test/pathod/data/response b/test/pathod/data/response deleted file mode 100644 index 8f897c857..000000000 --- a/test/pathod/data/response +++ /dev/null @@ -1 +0,0 @@ -202 diff --git a/test/pathod/data/testkey.pem b/test/pathod/data/testkey.pem deleted file mode 100644 index b804bd4c2..000000000 --- a/test/pathod/data/testkey.pem +++ /dev/null @@ -1,68 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIG5QIBAAKCAYEAwvtKxoZvBV2AxPAkCx8PXbuE7KeqK9bBvk8x+JchPMdf/KZj -sdu2v6Gm8Hi053i7ZGxouFvonJxHAiK6cwk9OYQwa9fbOFf2mgWKEBO4fbCH93tW -DCTdWVxFyNViAvxGHlJs3/IU03pIG29AgUnhRW8pGbabAfx8emcOZJZ3ykEuimaC -4s7mRwdc63GXnbcjTtRkrJsBATI+xvPwuR2+4daX7sPCf0kel3bN2jMpwXfvk/Ww -kJ2BIEeZCg0qIvyMjH9qrUirUnsmQnpPln0CGBbQEBsW9yMfGoFdREiMYls5jZeq -NxjWNv1RTRIm/4RjMwyxnoTA9eDS9wwO2NnJS4vfXAnUTP4BYx8Pe4ZMA2Gm6YrC -ysT6YA1xdHNpcuHXClxwmPj/cm8Z5kIg5clbNIK60ts9yFr/Ao3KPPYJ2GBv8/Oe -ApPBJuubews+/9/13Ew/SJ1t2u28+sPbgXUG8dC2n4vWTvJwKf6Duqxgnm82zdzj -SZoXRQsP984qiN7NAgMBAAECggGBALB6rqWdzCL5DLI0AQun40qdjaR95UKksNvF -5p7we379nl2ZZKb5DSHJ+MWzG1pfJo2wqeAkIBiQQp0mPcgdVrMWeJVD3QHUbDng -RaRjlRr+izJvCeUYANj+8ZLjwECfgf+z7yOLg1oeVeGvAp2C90jXYkYJx6c2lpxb -ZuWYY3hHIw7V1iXfywIDIhFg0TBJMMYK68xmx7QDfFqrNPj4eWsDxqSvvv1iezPw -rkWPBX49RjWPrW5XgSZsZ5J3c+oS1rZmIY7EAgopTWB/3wJjZR1Idz/9l9LIWlBP -6zVC27CIZzSEeGguqNVeyzJ0TPWh5idYNRmSZr6eTUF0245LNO/gqvWKgRSNIZko -HoBa2F1AvCiB67S1kxjwS5y3VkudZE4jkgGKcC2Ws/9QmOZ0HAsjI8VAMp2hj6iN -0HdPMTNtsLgbhKyXsoZuW4YmwfSTPxGi2gvcI7GUozpTz84n1cOneJnz1ygx6Uru -v8DpQg+VX6xTy4X6AK1F8OYNMZ/jaQKBwQDv30NevQStnGbTmcSr+0fd4rmWFklK -V6B2X7zWynVpSGvCWkqVSp3mG6aiZItAltVMRL/9LT6zIheyClyd+vXIjR2+W210 -XMxrvz7A8qCXkvB2dyEhrMdCfZ7p8+kf+eD2c/Mnxb7VpmDfHYLx30JeQoBwjrwU -Eul+dE1P+r8bWBaLTjlsipTya74yItWWAToXAo+s1BXBtXhEsLoe4FghlC0u724d -ucjDaeICdLcerApdvg6Q6p4kVHaoF6ka6I8CgcEA0Bdc05ery9gLC6CclV+BhA5Q -dfDq2P7qhc7e1ipwNRrQo2gy5HhgOkTL3dJWc+8rV6CBP/JfchnsW40tDOnPCTLT -gg3n7vv3RHrtncApXuhIFR+B5xjohTPBzxRUMiAOre2d0F5b6eBXFjptf/1i2tQ+ -qdqJoyOGOZP0hKVslGIfz+CKc6WEkIqX7c91Msdr5myeaWDI5TsurfuKRBH395T3 -BMAi6oinAAEb1rdySenLO2A/0kVmBVlTpaN3TNjjAoHBAMvS4uQ1qSv8okNbfgrF -UqPwa9JkzZImM2tinovFLU9xAl/7aTTCWrmU9Vs4JDuV71kHcjwnngeJCKl4tIpp -HUB06Lk/5xnhYLKNpz087cjeSwXe5IBA2HBfXhFd+NH6+nVwwUUieq4A2n+8C/CK -zVJbH9iE8Lv99fpFyQwU/R63EzD8Hz9j4ny7oLnpb6QvFrVGr98jt/kJwlBb+0sR -RtIBnwMq4F7R5w5lgm6jzpZ5ibVuMeJh+k7Ulp7uu/rpcQKBwQDE3sWIvf7f7PaO -OpbJz0CmYjCHVLWrNIlGrPAv6Jid9U+cuXEkrCpGFl5V77CxIH59+bEuga0BMztl -ZkxP4khoqHhom6VpeWJ3nGGAFJRPYS0JJvTsYalilBPxSYdaoO+iZ6MdxpfozcE2 -m3KLW3uSEqlyYvpCqNJNWQhGEoeGXstADWyPevHPGgAhElwL/ZW8u9inU9Tc4sAI -BGnMer+BsaJ+ERU3lK+Clony+z2aZiFLfIUE93lM6DT2CZBN2QcCgcAVk4L0bfA6 -HFnP/ZWNlnYWpOVFKcq57PX+J5/k7Tf34e2cYM2P0eqYggWZbzVd8qoCOQCHrAx0 -aZSSvEyKAVvzRNeqbm1oXaMojksMnrSX5henHjPbZlr1EmM7+zMnSTMkfVOx/6g1 -97sASej31XdOAgKCBJGymrwvYrCLW+P5cHqd+D8v/PvfpRIQM54p5ixRt3EYZvtR -zGrzsr0OGyOLZtj1DB0a3kvajAAOCl3TawJSzviKo2mwc+/xj28MCQM= ------END RSA PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIIE4TCCA0mgAwIBAgIJALONCAWZxPhUMA0GCSqGSIb3DQEBCwUAMEExCzAJBgNV -BAYTAk5aMQ4wDAYDVQQIDAVPdGFnbzEPMA0GA1UECgwGUGF0aG9kMREwDwYDVQQD -DAh0ZXN0LmNvbTAeFw0xNTA0MTgyMjA0NTNaFw00MjA5MDIyMjA0NTNaMEExCzAJ -BgNVBAYTAk5aMQ4wDAYDVQQIDAVPdGFnbzEPMA0GA1UECgwGUGF0aG9kMREwDwYD -VQQDDAh0ZXN0LmNvbTCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAML7 -SsaGbwVdgMTwJAsfD127hOynqivWwb5PMfiXITzHX/ymY7Hbtr+hpvB4tOd4u2Rs -aLhb6JycRwIiunMJPTmEMGvX2zhX9poFihATuH2wh/d7Vgwk3VlcRcjVYgL8Rh5S -bN/yFNN6SBtvQIFJ4UVvKRm2mwH8fHpnDmSWd8pBLopmguLO5kcHXOtxl523I07U -ZKybAQEyPsbz8LkdvuHWl+7Dwn9JHpd2zdozKcF375P1sJCdgSBHmQoNKiL8jIx/ -aq1Iq1J7JkJ6T5Z9AhgW0BAbFvcjHxqBXURIjGJbOY2XqjcY1jb9UU0SJv+EYzMM -sZ6EwPXg0vcMDtjZyUuL31wJ1Ez+AWMfD3uGTANhpumKwsrE+mANcXRzaXLh1wpc -cJj4/3JvGeZCIOXJWzSCutLbPcha/wKNyjz2Cdhgb/PzngKTwSbrm3sLPv/f9dxM -P0idbdrtvPrD24F1BvHQtp+L1k7ycCn+g7qsYJ5vNs3c40maF0ULD/fOKojezQID -AQABo4HbMIHYMAsGA1UdDwQEAwIFoDAdBgNVHQ4EFgQUbEgfTauEqEP/bnBtby1K -bihJvcswcQYDVR0jBGowaIAUbEgfTauEqEP/bnBtby1KbihJvcuhRaRDMEExCzAJ -BgNVBAYTAk5aMQ4wDAYDVQQIDAVPdGFnbzEPMA0GA1UECgwGUGF0aG9kMREwDwYD -VQQDDAh0ZXN0LmNvbYIJALONCAWZxPhUMAwGA1UdEwQFMAMBAf8wKQYDVR0RBCIw -IIIIdGVzdC5jb22CCXRlc3QyLmNvbYIJdGVzdDMuY29tMA0GCSqGSIb3DQEBCwUA -A4IBgQBcTedXtUb91DxQRtg73iomz7cQ4niZntUBW8iE5rpoA7prtQNGHMCbHwaX -tbWFkzBmL5JTBWvd/6AQ2LtiB3rYB3W/iRhbpsNJ501xaoOguPEQ9720Ph8TEveM -208gNzGsEOcNALwyXj2y9M19NGu9zMa8eu1Tc3IsQaVaGKHx8XZn5HTNUx8EdcwI -Z/Ji9ETDCL7+e5INv0tqfFSazWaQUwxM4IzPMkKTYRcMuN/6eog609k9r9pp32Ut -rKlzc6GIkAlgJJ0Wkoz1V46DmJNJdJG7eLu/mtsB85j6hytIQeWTf1fll5YnMZLF -HgNZtfYn8Q0oTdBQ0ZOaZeQCfZ8emYBdLJf2YB83uGRMjQ1FoeIxzQqiRq8WHRdb -9Q45i0DINMnNp0DbLMA4numZ7wT9SQb6sql9eUyuCNDw7nGIWTHUNfLtU1Er3h1d -icJuApx9+//UN/pGh0yTXb3fZbiI4IehRmkpnIWonIAwaVGm6JZU04wiIn8CuBho -/qQdlS8= ------END CERTIFICATE----- diff --git a/test/pathod/language/__init__.py b/test/pathod/language/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test/pathod/language/test_actions.py b/test/pathod/language/test_actions.py deleted file mode 100644 index 9740e5c79..000000000 --- a/test/pathod/language/test_actions.py +++ /dev/null @@ -1,134 +0,0 @@ -import io - -from pathod.language import actions, parse_pathoc, parse_pathod, serve - - -def parse_request(s): - return next(parse_pathoc(s)) - - -def test_unique_name(): - assert not actions.PauseAt(0, "f").unique_name - assert actions.DisconnectAt(0).unique_name - - -class TestDisconnects: - - def test_parse_pathod(self): - a = next(parse_pathod("400:d0")).actions[0] - assert a.spec() == "d0" - a = next(parse_pathod("400:dr")).actions[0] - assert a.spec() == "dr" - - def test_at(self): - e = actions.DisconnectAt.expr() - v = e.parseString("d0")[0] - assert isinstance(v, actions.DisconnectAt) - assert v.offset == 0 - - v = e.parseString("d100")[0] - assert v.offset == 100 - - e = actions.DisconnectAt.expr() - v = e.parseString("dr")[0] - assert v.offset == "r" - - def test_spec(self): - assert actions.DisconnectAt("r").spec() == "dr" - assert actions.DisconnectAt(10).spec() == "d10" - - -class TestInject: - - def test_parse_pathod(self): - a = next(parse_pathod("400:ir,@100")).actions[0] - assert a.offset == "r" - assert a.value.datatype == "bytes" - assert a.value.usize == 100 - - a = next(parse_pathod("400:ia,@100")).actions[0] - assert a.offset == "a" - - def test_at(self): - e = actions.InjectAt.expr() - v = e.parseString("i0,'foo'")[0] - assert v.value.val == b"foo" - assert v.offset == 0 - assert isinstance(v, actions.InjectAt) - - v = e.parseString("ir,'foo'")[0] - assert v.offset == "r" - - def test_serve(self): - s = io.BytesIO() - r = next(parse_pathod("400:i0,'foo'")) - assert serve(r, s, {}) - - def test_spec(self): - e = actions.InjectAt.expr() - v = e.parseString("i0,'foo'")[0] - assert v.spec() == "i0,'foo'" - - def test_spec2(self): - e = actions.InjectAt.expr() - v = e.parseString("i0,@100")[0] - v2 = v.freeze({}) - v3 = v2.freeze({}) - assert v2.value.val == v3.value.val - - -class TestPauses: - - def test_parse_pathod(self): - e = actions.PauseAt.expr() - v = e.parseString("p10,10")[0] - assert v.seconds == 10 - assert v.offset == 10 - - v = e.parseString("p10,f")[0] - assert v.seconds == "f" - - v = e.parseString("pr,f")[0] - assert v.offset == "r" - - v = e.parseString("pa,f")[0] - assert v.offset == "a" - - def test_request(self): - r = next(parse_pathod('400:p10,10')) - assert r.actions[0].spec() == "p10,10" - - def test_spec(self): - assert actions.PauseAt("r", 5).spec() == "pr,5" - assert actions.PauseAt(0, 5).spec() == "p0,5" - assert actions.PauseAt(0, "f").spec() == "p0,f" - - def test_freeze(self): - l = actions.PauseAt("r", 5) - assert l.freeze({}).spec() == l.spec() - - -class Test_Action: - - def test_cmp(self): - a = actions.DisconnectAt(0) - b = actions.DisconnectAt(1) - c = actions.DisconnectAt(0) - assert a < b - assert a == c - l = sorted([b, a]) - assert l[0].offset == 0 - - def test_resolve(self): - r = parse_request('GET:"/foo"') - e = actions.DisconnectAt("r") - ret = e.resolve({}, r) - assert isinstance(ret.offset, int) - - def test_repr(self): - e = actions.DisconnectAt("r") - assert repr(e) - - def test_freeze(self): - l = actions.DisconnectAt(5) - assert l.freeze({}).spec() == l.spec() diff --git a/test/pathod/language/test_base.py b/test/pathod/language/test_base.py deleted file mode 100644 index 910d298a6..000000000 --- a/test/pathod/language/test_base.py +++ /dev/null @@ -1,351 +0,0 @@ -import pytest - -from pathod import language -from pathod.language import base, exceptions - - -def parse_request(s): - return language.parse_pathoc(s).next() - - -def test_times(): - reqs = list(language.parse_pathoc("get:/:x5")) - assert len(reqs) == 5 - assert not reqs[0].times - - -def test_caseless_literal(): - class CL(base.CaselessLiteral): - TOK = "foo" - v = CL("foo") - assert v.expr() - assert v.values(language.Settings()) - - -class TestTokValueNakedLiteral: - - def test_expr(self): - v = base.TokValueNakedLiteral("foo") - assert v.expr() - - def test_spec(self): - v = base.TokValueNakedLiteral("foo") - assert v.spec() == repr(v) == "foo" - - v = base.TokValueNakedLiteral("f\x00oo") - assert v.spec() == repr(v) == r"f\x00oo" - - -class TestTokValueLiteral: - - def test_expr(self): - v = base.TokValueLiteral("foo") - assert v.expr() - assert v.val == b"foo" - - v = base.TokValueLiteral("foo\n") - assert v.expr() - assert v.val == b"foo\n" - assert repr(v) - - def test_spec(self): - v = base.TokValueLiteral("foo") - assert v.spec() == r"'foo'" - - v = base.TokValueLiteral("f\x00oo") - assert v.spec() == repr(v) == r"'f\x00oo'" - - v = base.TokValueLiteral('"') - assert v.spec() == repr(v) == """ '"' """.strip() - - # While pyparsing has a escChar argument for QuotedString, - # escChar only performs scapes single-character escapes and does not work for e.g. r"\x02". - # Thus, we cannot use that option, which means we cannot have single quotes in strings. - # To fix this, we represent single quotes as r"\x07". - v = base.TokValueLiteral("'") - assert v.spec() == r"'\x27'" - - def roundtrip(self, spec): - e = base.TokValueLiteral.expr() - v = base.TokValueLiteral(spec) - v2 = e.parseString(v.spec()) - assert v.val == v2[0].val - assert v.spec() == v2[0].spec() - - def test_roundtrip(self): - self.roundtrip("'") - self.roundtrip(r"\'") - self.roundtrip("a") - self.roundtrip("\"") - # self.roundtrip("\\") - self.roundtrip("200:b'foo':i23,'\\''") - self.roundtrip("\a") - - -class TestTokValueGenerate: - - def test_basic(self): - v = base.TokValue.parseString("@10b")[0] - assert v.usize == 10 - assert v.unit == "b" - assert v.bytes() == 10 - v = base.TokValue.parseString("@10")[0] - assert v.unit == "b" - v = base.TokValue.parseString("@10k")[0] - assert v.bytes() == 10240 - v = base.TokValue.parseString("@10g")[0] - assert v.bytes() == 1024 ** 3 * 10 - - v = base.TokValue.parseString("@10g,digits")[0] - assert v.datatype == "digits" - g = v.get_generator({}) - assert g[:100] - - v = base.TokValue.parseString("@10,digits")[0] - assert v.unit == "b" - assert v.datatype == "digits" - - def test_spec(self): - v = base.TokValueGenerate(1, "b", "bytes") - assert v.spec() == repr(v) == "@1" - - v = base.TokValueGenerate(1, "k", "bytes") - assert v.spec() == repr(v) == "@1k" - - v = base.TokValueGenerate(1, "k", "ascii") - assert v.spec() == repr(v) == "@1k,ascii" - - v = base.TokValueGenerate(1, "b", "ascii") - assert v.spec() == repr(v) == "@1,ascii" - - def test_freeze(self): - v = base.TokValueGenerate(100, "b", "ascii") - f = v.freeze(language.Settings()) - assert len(f.val) == 100 - - -class TestTokValueFile: - - def test_file_value(self): - v = base.TokValue.parseString("<'one two'")[0] - assert str(v) - assert v.path == "one two" - - v = base.TokValue.parseString(" 100 - - def test_path_generator(self): - r = parse_request("GET:@100").freeze(language.Settings()) - assert len(r.spec()) > 100 - - def test_websocket(self): - r = parse_request('ws:/path/') - res = r.resolve(language.Settings()) - assert res.method.string().lower() == b"get" - assert res.tok(http.Path).value.val == b"/path/" - assert res.tok(http.Method).value.val.lower() == b"get" - assert http.get_header(b"Upgrade", res.headers).value.val == b"websocket" - - r = parse_request('ws:put:/path/') - res = r.resolve(language.Settings()) - assert r.method.string().lower() == b"put" - assert res.tok(http.Path).value.val == b"/path/" - assert res.tok(http.Method).value.val.lower() == b"put" - assert http.get_header(b"Upgrade", res.headers).value.val == b"websocket" - - -class TestResponse: - - def dummy_response(self): - return next(language.parse_pathod("400'msg'")) - - def test_response(self): - r = next(language.parse_pathod("400:m'msg'")) - assert r.status_code.string() == b"400" - assert r.reason.string() == b"msg" - - r = next(language.parse_pathod("400:m'msg':b@100b")) - assert r.reason.string() == b"msg" - assert r.body.values({}) - assert str(r) - - r = next(language.parse_pathod("200")) - assert r.status_code.string() == b"200" - assert not r.reason - assert b"OK" in [i[:] for i in r.preamble({})] - - def test_render(self): - s = io.BytesIO() - r = next(language.parse_pathod("400:m'msg'")) - assert language.serve(r, s, {}) - - r = next(language.parse_pathod("400:p0,100:dr")) - assert "p0" in r.spec() - s = r.preview_safe() - assert "p0" not in s.spec() - - def test_raw(self): - s = io.BytesIO() - r = next(language.parse_pathod("400:b'foo'")) - language.serve(r, s, {}) - v = s.getvalue() - assert b"Content-Length" in v - - s = io.BytesIO() - r = next(language.parse_pathod("400:b'foo':r")) - language.serve(r, s, {}) - v = s.getvalue() - assert b"Content-Length" not in v - - def test_length(self): - def testlen(x): - s = io.BytesIO() - x = next(x) - language.serve(x, s, language.Settings()) - assert x.length(language.Settings()) == len(s.getvalue()) - testlen(language.parse_pathod("400:m'msg':r")) - testlen(language.parse_pathod("400:m'msg':h'foo'='bar':r")) - testlen(language.parse_pathod("400:m'msg':h'foo'='bar':b@100b:r")) - - def test_maximum_length(self): - def testlen(x): - x = next(x) - s = io.BytesIO() - m = x.maximum_length({}) - language.serve(x, s, {}) - assert m >= len(s.getvalue()) - - r = language.parse_pathod("400:m'msg':b@100:d0") - testlen(r) - - r = language.parse_pathod("400:m'msg':b@100:d0:i0,'foo'") - testlen(r) - - r = language.parse_pathod("400:m'msg':b@100:d0:i0,'foo'") - testlen(r) - - def test_parse_err(self): - with pytest.raises(language.ParseException): - language.parse_pathod("400:msg,b:") - try: - language.parse_pathod("400'msg':b:") - except language.ParseException as v: - assert v.marked() - assert str(v) - - def test_nonascii(self): - with pytest.raises(Exception, match="ASCII"): - language.parse_pathod("foo:b\xf0") - - def test_parse_header(self): - r = next(language.parse_pathod('400:h"foo"="bar"')) - assert http.get_header(b"foo", r.headers) - - def test_parse_pause_before(self): - r = next(language.parse_pathod("400:p0,10")) - assert r.actions[0].spec() == "p0,10" - - def test_parse_pause_after(self): - r = next(language.parse_pathod("400:pa,10")) - assert r.actions[0].spec() == "pa,10" - - def test_parse_pause_random(self): - r = next(language.parse_pathod("400:pr,10")) - assert r.actions[0].spec() == "pr,10" - - def test_parse_stress(self): - # While larger values are known to work on linux, len() technically - # returns an int and a python 2.7 int on windows has 32bit precision. - # Therefore, we should keep the body length < 2147483647 bytes in our - # tests. - r = next(language.parse_pathod("400:b@1g")) - assert r.length({}) - - def test_spec(self): - def rt(s): - s = next(language.parse_pathod(s)).spec() - assert next(language.parse_pathod(s)).spec() == s - rt("400:b@100g") - rt("400") - rt("400:da") - - def test_websockets(self): - r = next(language.parse_pathod("ws")) - with pytest.raises(Exception, match="No websocket key"): - r.resolve(language.Settings()) - res = r.resolve(language.Settings(websocket_key=b"foo")) - assert res.status_code.string() == b"101" - - -def test_ctype_shortcut(): - e = http.ShortcutContentType.expr() - v = e.parseString("c'foo'")[0] - assert v.key.val == b"Content-Type" - assert v.value.val == b"foo" - - s = v.spec() - assert s == e.parseString(s)[0].spec() - - e = http.ShortcutContentType.expr() - v = e.parseString("c@100")[0] - v2 = v.freeze({}) - v3 = v2.freeze({}) - assert v2.value.val == v3.value.val - - -def test_location_shortcut(): - e = http.ShortcutLocation.expr() - v = e.parseString("l'foo'")[0] - assert v.key.val == b"Location" - assert v.value.val == b"foo" - - s = v.spec() - assert s == e.parseString(s)[0].spec() - - e = http.ShortcutLocation.expr() - v = e.parseString("l@100")[0] - v2 = v.freeze({}) - v3 = v2.freeze({}) - assert v2.value.val == v3.value.val - - -def test_shortcuts(): - assert next(language.parse_pathod( - "400:c'foo'")).headers[0].key.val == b"Content-Type" - assert next(language.parse_pathod( - "400:l'foo'")).headers[0].key.val == b"Location" - - assert b"Android" in tservers.render(parse_request("get:/:ua")) - assert b"User-Agent" in tservers.render(parse_request("get:/:ua")) - - -def test_user_agent(): - e = http.ShortcutUserAgent.expr() - v = e.parseString("ua")[0] - assert b"Android" in v.string() - - e = http.ShortcutUserAgent.expr() - v = e.parseString("u'a'")[0] - assert b"Android" not in v.string() - - v = e.parseString("u@100'")[0] - assert len(str(v.freeze({}).value)) > 100 - v2 = v.freeze({}) - v3 = v2.freeze({}) - assert v2.value.val == v3.value.val - - -def test_nested_response(): - e = http.NestedResponse.expr() - v = e.parseString("s'200'")[0] - assert v.value.val == b"200" - with pytest.raises(language.ParseException): - e.parseString("s'foo'") - - v = e.parseString('s"200:b@1"')[0] - assert "@1" in v.spec() - f = v.freeze({}) - assert "@1" not in f.spec() - - -def test_nested_response_freeze(): - e = http.NestedResponse( - base.TokValueLiteral( - r"200:b\'foo\':i10,\'\\x27\'" - ) - ) - assert e.freeze({}) - assert e.values({}) - - -def test_unique_components(): - with pytest.raises(Exception, match="multiple body clauses"): - language.parse_pathod("400:b@1:b@1") diff --git a/test/pathod/language/test_http2.py b/test/pathod/language/test_http2.py deleted file mode 100644 index 4f89adb80..000000000 --- a/test/pathod/language/test_http2.py +++ /dev/null @@ -1,236 +0,0 @@ -import io -import pytest - -from mitmproxy.net import tcp -from mitmproxy.net.http import user_agents - -from pathod import language -from pathod.language import http2 -from pathod.protocols.http2 import HTTP2StateProtocol - - -def parse_request(s): - return next(language.parse_pathoc(s, True)) - - -def parse_response(s): - return next(language.parse_pathod(s, True)) - - -def default_settings(): - return language.Settings( - request_host="foo.com", - protocol=HTTP2StateProtocol(tcp.TCPClient(('localhost', 1234))) - ) - - -def test_make_error_response(): - d = io.BytesIO() - s = http2.make_error_response("foo", "bar") - language.serve(s, d, default_settings()) - - -class TestRequest: - - def test_cached_values(self): - req = parse_request("get:/") - req_id = id(req) - assert req_id == id(req.resolve(default_settings())) - assert req.values(default_settings()) == req.values(default_settings()) - - def test_nonascii(self): - with pytest.raises(Exception, match="ASCII"): - parse_request("get:\xf0") - - def test_err(self): - with pytest.raises(language.ParseException): - parse_request('GET') - - def test_simple(self): - r = parse_request('GET:"/foo"') - assert r.method.string() == b"GET" - assert r.path.string() == b"/foo" - r = parse_request('GET:/foo') - assert r.path.string() == b"/foo" - - def test_multiple(self): - r = list(language.parse_pathoc("GET:/ PUT:/")) - assert r[0].method.string() == b"GET" - assert r[1].method.string() == b"PUT" - assert len(r) == 2 - - l = """ - GET - "/foo" - - PUT - - "/foo - - - - bar" - """ - r = list(language.parse_pathoc(l, True)) - assert len(r) == 2 - assert r[0].method.string() == b"GET" - assert r[1].method.string() == b"PUT" - - l = """ - get:"http://localhost:9999/p/200" - get:"http://localhost:9999/p/200" - """ - r = list(language.parse_pathoc(l, True)) - assert len(r) == 2 - assert r[0].method.string() == b"GET" - assert r[1].method.string() == b"GET" - - def test_render_simple(self): - s = io.BytesIO() - r = parse_request("GET:'/foo'") - assert language.serve( - r, - s, - default_settings(), - ) - - def test_raw_content_length(self): - r = parse_request('GET:/:r') - assert len(r.headers) == 0 - - r = parse_request('GET:/:r:b"foobar"') - assert len(r.headers) == 0 - - r = parse_request('GET:/') - assert len(r.headers) == 1 - assert r.headers[0].values(default_settings()) == (b"content-length", b"0") - - r = parse_request('GET:/:b"foobar"') - assert len(r.headers) == 1 - assert r.headers[0].values(default_settings()) == (b"content-length", b"6") - - r = parse_request('GET:/:b"foobar":h"content-length"="42"') - assert len(r.headers) == 1 - assert r.headers[0].values(default_settings()) == (b"content-length", b"42") - - r = parse_request('GET:/:r:b"foobar":h"content-length"="42"') - assert len(r.headers) == 1 - assert r.headers[0].values(default_settings()) == (b"content-length", b"42") - - def test_content_type(self): - r = parse_request('GET:/:r:c"foobar"') - assert len(r.headers) == 1 - assert r.headers[0].values(default_settings()) == (b"content-type", b"foobar") - - def test_user_agent(self): - r = parse_request('GET:/:r:ua') - assert len(r.headers) == 1 - assert r.headers[0].values(default_settings()) == (b"user-agent", user_agents.get_by_shortcut('a')[2].encode()) - - def test_render_with_headers(self): - s = io.BytesIO() - r = parse_request('GET:/foo:h"foo"="bar"') - assert language.serve( - r, - s, - default_settings(), - ) - - def test_nested_response(self): - l = "get:/p/:s'200'" - r = parse_request(l) - assert len(r.tokens) == 3 - assert isinstance(r.tokens[2], http2.NestedResponse) - assert r.values(default_settings()) - - def test_render_with_body(self): - s = io.BytesIO() - r = parse_request("GET:'/foo':bfoobar") - assert language.serve( - r, - s, - default_settings(), - ) - - def test_spec(self): - def rt(s): - s = parse_request(s).spec() - assert parse_request(s).spec() == s - rt("get:/foo") - - -class TestResponse: - - def test_cached_values(self): - res = parse_response("200") - res_id = id(res) - assert res_id == id(res.resolve(default_settings())) - assert res.values(default_settings()) == res.values(default_settings()) - - def test_nonascii(self): - with pytest.raises(Exception, match="ASCII"): - parse_response("200:\xf0") - - def test_err(self): - with pytest.raises(language.ParseException): - parse_response('GET:/') - - def test_raw_content_length(self): - r = parse_response('200:r') - assert len(r.headers) == 0 - - r = parse_response('200') - assert len(r.headers) == 1 - assert r.headers[0].values(default_settings()) == (b"content-length", b"0") - - def test_content_type(self): - r = parse_response('200:r:c"foobar"') - assert len(r.headers) == 1 - assert r.headers[0].values(default_settings()) == (b"content-type", b"foobar") - - def test_simple(self): - r = parse_response('200:r:h"foo"="bar"') - assert r.status_code.string() == b"200" - assert len(r.headers) == 1 - assert r.headers[0].values(default_settings()) == (b"foo", b"bar") - assert r.body is None - - r = parse_response('200:r:h"foo"="bar":bfoobar:h"bla"="fasel"') - assert r.status_code.string() == b"200" - assert len(r.headers) == 2 - assert r.headers[0].values(default_settings()) == (b"foo", b"bar") - assert r.headers[1].values(default_settings()) == (b"bla", b"fasel") - assert r.body.string() == b"foobar" - - def test_render_simple(self): - s = io.BytesIO() - r = parse_response('200') - assert language.serve( - r, - s, - default_settings(), - ) - - def test_render_with_headers(self): - s = io.BytesIO() - r = parse_response('200:h"foo"="bar"') - assert language.serve( - r, - s, - default_settings(), - ) - - def test_render_with_body(self): - s = io.BytesIO() - r = parse_response('200:bfoobar') - assert language.serve( - r, - s, - default_settings(), - ) - - def test_spec(self): - def rt(s): - s = parse_response(s).spec() - assert parse_response(s).spec() == s - rt("200:bfoobar") diff --git a/test/pathod/language/test_message.py b/test/pathod/language/test_message.py deleted file mode 100644 index 777ab4dd1..000000000 --- a/test/pathod/language/test_message.py +++ /dev/null @@ -1 +0,0 @@ -# TODO: write tests diff --git a/test/pathod/language/test_websockets.py b/test/pathod/language/test_websockets.py deleted file mode 100644 index 9cb17f75d..000000000 --- a/test/pathod/language/test_websockets.py +++ /dev/null @@ -1,134 +0,0 @@ -import pytest - -from pathod import language - -from wsproto.frame_protocol import Opcode - -from .. import tservers - - -def parse_request(s): - return next(language.parse_pathoc(s)) - - -class TestWebsocketFrame: - - def _test_messages(self, specs, message_klass): - for i in specs: - wf = parse_request(i) - assert isinstance(wf, message_klass) - assert wf - assert wf.values(language.Settings()) - assert wf.resolve(language.Settings()) - - spec = wf.spec() - wf2 = parse_request(spec) - assert wf2.spec() == spec - - def test_server_values(self): - specs = [ - "wf", - "wf:dr", - "wf:b'foo'", - "wf:mask:r'foo'", - "wf:l1024:b'foo'", - "wf:cbinary", - "wf:c1", - "wf:mask:knone", - "wf:fin", - "wf:fin:rsv1:rsv2:rsv3:mask", - "wf:-fin:-rsv1:-rsv2:-rsv3:-mask", - "wf:k@4", - "wf:x10", - ] - self._test_messages(specs, language.websockets.WebsocketFrame) - - def test_parse_websocket_frames(self): - wf = language.parse_websocket_frame("wf:x10") - assert len(list(wf)) == 10 - with pytest.raises(language.ParseException): - language.parse_websocket_frame("wf:x") - - def test_client_values(self): - specs = [ - "wf:f'wf'", - ] - self._test_messages(specs, language.websockets.WebsocketClientFrame) - - def test_nested_frame(self): - wf = parse_request("wf:f'wf'") - assert wf.nested_frame - - def test_flags(self): - wf = parse_request("wf:fin:mask:rsv1:rsv2:rsv3") - frm = language.websockets_frame.Frame.from_bytes(tservers.render(wf)) - assert frm.header.fin - assert frm.header.mask - assert frm.header.rsv1 - assert frm.header.rsv2 - assert frm.header.rsv3 - - wf = parse_request("wf:-fin:-mask:-rsv1:-rsv2:-rsv3") - frm = language.websockets_frame.Frame.from_bytes(tservers.render(wf)) - assert not frm.header.fin - assert not frm.header.mask - assert not frm.header.rsv1 - assert not frm.header.rsv2 - assert not frm.header.rsv3 - - def fr(self, spec, **kwargs): - settings = language.base.Settings(**kwargs) - wf = parse_request(spec) - return language.websockets_frame.Frame.from_bytes(tservers.render(wf, settings)) - - def test_construction(self): - assert self.fr("wf:c1").header.opcode == 1 - assert self.fr("wf:c0").header.opcode == 0 - assert self.fr("wf:cbinary").header.opcode == Opcode.BINARY - assert self.fr("wf:ctext").header.opcode == Opcode.TEXT - - def test_rawbody(self): - frm = self.fr("wf:mask:r'foo'") - assert len(frm.payload) == 3 - assert frm.payload != b"foo" - - assert self.fr("wf:r'foo'").payload == b"foo" - - def test_construction_2(self): - # Simple server frame - frm = self.fr("wf:b'foo'") - assert not frm.header.mask - assert not frm.header.masking_key - - # Simple client frame - frm = self.fr("wf:b'foo'", is_client=True) - assert frm.header.mask - assert frm.header.masking_key - frm = self.fr("wf:b'foo':k'abcd'", is_client=True) - assert frm.header.mask - assert frm.header.masking_key == b'abcd' - - # Server frame, mask explicitly set - frm = self.fr("wf:b'foo':mask") - assert frm.header.mask - assert frm.header.masking_key - frm = self.fr("wf:b'foo':k'abcd'") - assert frm.header.mask - assert frm.header.masking_key == b'abcd' - - # Client frame, mask explicitly unset - frm = self.fr("wf:b'foo':-mask", is_client=True) - assert not frm.header.mask - assert not frm.header.masking_key - - def test_knone(self): - with pytest.raises(Exception, match="Expected 4 bytes"): - self.fr("wf:b'foo':mask:knone") - - def test_length(self): - assert self.fr("wf:l3:b'foo'").header.payload_length == 3 - frm = self.fr("wf:l2:b'foo'") - assert frm.header.payload_length == 2 - assert frm.payload == b"fo" - with pytest.raises(Exception, match="Expected 1024 bytes"): - self.fr("wf:l1024:b'foo'") diff --git a/test/pathod/language/test_websockets_frame.py b/test/pathod/language/test_websockets_frame.py deleted file mode 100644 index 8905a6486..000000000 --- a/test/pathod/language/test_websockets_frame.py +++ /dev/null @@ -1,187 +0,0 @@ -import os -import codecs -import pytest - -from wsproto.frame_protocol import Opcode - -from pathod.language import websockets_frame -from mitmproxy.test import tutils - - -class TestMasker: - - @pytest.mark.parametrize("input,expected", [ - ([b"a"], '00'), - ([b"four"], '070d1616'), - ([b"fourf"], '070d161607'), - ([b"fourfive"], '070d1616070b1501'), - ([b"a", b"aasdfasdfa", b"asdf"], '000302170504021705040205120605'), - ([b"a" * 50, b"aasdfasdfa", b"asdf"], '00030205000302050003020500030205000302050003020500030205000302050003020500030205000302050003020500030205120605051206050500110702'), # noqa - ]) - def test_masker(self, input, expected): - m = websockets_frame.Masker(b"abcd") - data = b"".join([m(t) for t in input]) - assert data == codecs.decode(expected, 'hex') - - data = websockets_frame.Masker(b"abcd")(data) - assert data == b"".join(input) - - -class TestFrameHeader: - - @pytest.mark.parametrize("input,expected", [ - (0, '0100'), - (125, '017D'), - (126, '017E007E'), - (127, '017E007F'), - (142, '017E008E'), - (65534, '017EFFFE'), - (65535, '017EFFFF'), - (65536, '017F0000000000010000'), - (8589934591, '017F00000001FFFFFFFF'), - (2 ** 64 - 1, '017FFFFFFFFFFFFFFFFF'), - ]) - def test_serialization_length(self, input, expected): - h = websockets_frame.FrameHeader( - opcode=Opcode.TEXT, - payload_length=input, - ) - assert bytes(h) == codecs.decode(expected, 'hex') - - def test_serialization_too_large(self): - h = websockets_frame.FrameHeader( - payload_length=2 ** 64 + 1, - ) - with pytest.raises(ValueError): - bytes(h) - - @pytest.mark.parametrize("input,expected", [ - ('0100', 0), - ('017D', 125), - ('017E007E', 126), - ('017E007F', 127), - ('017E008E', 142), - ('017EFFFE', 65534), - ('017EFFFF', 65535), - ('017F0000000000010000', 65536), - ('017F00000001FFFFFFFF', 8589934591), - ('017FFFFFFFFFFFFFFFFF', 2 ** 64 - 1), - ]) - def test_deserialization_length(self, input, expected): - h = websockets_frame.FrameHeader.from_file(tutils.treader(codecs.decode(input, 'hex'))) - assert h.payload_length == expected - - @pytest.mark.parametrize("input,expected", [ - ('0100', (False, None)), - ('018000000000', (True, '00000000')), - ('018012345678', (True, '12345678')), - ]) - def test_deserialization_masking(self, input, expected): - h = websockets_frame.FrameHeader.from_file(tutils.treader(codecs.decode(input, 'hex'))) - assert h.mask == expected[0] - if h.mask: - assert h.masking_key == codecs.decode(expected[1], 'hex') - - def test_equality(self): - h = websockets_frame.FrameHeader(mask=True, masking_key=b'1234') - h2 = websockets_frame.FrameHeader(mask=True, masking_key=b'1234') - assert h == h2 - - h = websockets_frame.FrameHeader(fin=True) - h2 = websockets_frame.FrameHeader(fin=False) - assert h != h2 - - assert h != 'foobar' - - def test_roundtrip(self): - def round(*args, **kwargs): - h = websockets_frame.FrameHeader(*args, **kwargs) - h2 = websockets_frame.FrameHeader.from_file(tutils.treader(bytes(h))) - assert h == h2 - - round() - round(fin=True) - round(rsv1=True) - round(rsv2=True) - round(rsv3=True) - round(payload_length=1) - round(payload_length=100) - round(payload_length=1000) - round(payload_length=10000) - round(opcode=Opcode.PING) - round(masking_key=b"test") - - def test_human_readable(self): - f = websockets_frame.FrameHeader( - masking_key=b"test", - fin=True, - payload_length=10 - ) - assert repr(f) - - f = websockets_frame.FrameHeader() - assert repr(f) - - def test_funky(self): - f = websockets_frame.FrameHeader(masking_key=b"test", mask=False) - raw = bytes(f) - f2 = websockets_frame.FrameHeader.from_file(tutils.treader(raw)) - assert not f2.mask - - def test_violations(self): - with pytest.raises(Exception, match="opcode"): - websockets_frame.FrameHeader(opcode=17) - with pytest.raises(Exception, match="Masking key"): - websockets_frame.FrameHeader(masking_key=b"x") - - def test_automask(self): - f = websockets_frame.FrameHeader(mask=True) - assert f.masking_key - - f = websockets_frame.FrameHeader(masking_key=b"foob") - assert f.mask - - f = websockets_frame.FrameHeader(masking_key=b"foob", mask=0) - assert not f.mask - assert not f.masking_key - - -class TestFrame: - def test_equality(self): - f = websockets_frame.Frame(payload=b'1234') - f2 = websockets_frame.Frame(payload=b'1234') - assert f == f2 - - assert f != b'1234' - - def test_roundtrip(self): - def round(*args, **kwargs): - f = websockets_frame.Frame(*args, **kwargs) - raw = bytes(f) - f2 = websockets_frame.Frame.from_file(tutils.treader(raw)) - assert f == f2 - round(b"test") - round(b"test", fin=1) - round(b"test", rsv1=1) - round(b"test", opcode=Opcode.PING) - round(b"test", masking_key=b"test") - - def test_human_readable(self): - f = websockets_frame.Frame() - assert repr(f) - - f = websockets_frame.Frame(b"foobar") - assert "foobar" in repr(f) - - @pytest.mark.parametrize("masked", [True, False]) - @pytest.mark.parametrize("length", [100, 50000, 150000]) - def test_serialization_bijection(self, masked, length): - frame = websockets_frame.Frame( - os.urandom(length), - fin=True, - opcode=Opcode.TEXT, - mask=int(masked), - masking_key=(os.urandom(4) if masked else None) - ) - serialized = bytes(frame) - assert frame == websockets_frame.Frame.from_bytes(serialized) diff --git a/test/pathod/language/test_writer.py b/test/pathod/language/test_writer.py deleted file mode 100644 index 7feb985d5..000000000 --- a/test/pathod/language/test_writer.py +++ /dev/null @@ -1,90 +0,0 @@ -import io -from pathod import language -from pathod.language import writer - - -def test_send_chunk(): - v = b"foobarfoobar" - for bs in range(1, len(v) + 2): - s = io.BytesIO() - writer.send_chunk(s, v, bs, 0, len(v)) - assert s.getvalue() == v - for start in range(len(v)): - for end in range(len(v)): - s = io.BytesIO() - writer.send_chunk(s, v, bs, start, end) - assert s.getvalue() == v[start:end] - - -def test_write_values_inject(): - tst = b"foo" - - s = io.BytesIO() - writer.write_values(s, [tst], [(0, "inject", b"aaa")], blocksize=5) - assert s.getvalue() == b"aaafoo" - - s = io.BytesIO() - writer.write_values(s, [tst], [(1, "inject", b"aaa")], blocksize=5) - assert s.getvalue() == b"faaaoo" - - s = io.BytesIO() - writer.write_values(s, [tst], [(1, "inject", b"aaa")], blocksize=5) - assert s.getvalue() == b"faaaoo" - - -def test_write_values_disconnects(): - s = io.BytesIO() - tst = b"foo" * 100 - writer.write_values(s, [tst], [(0, "disconnect")], blocksize=5) - assert not s.getvalue() - - -def test_write_values(): - tst = b"foobarvoing" - s = io.BytesIO() - writer.write_values(s, [tst], []) - assert s.getvalue() == tst - - for bs in range(1, len(tst) + 2): - for off in range(len(tst)): - s = io.BytesIO() - writer.write_values( - s, [tst], [(off, "disconnect")], blocksize=bs - ) - assert s.getvalue() == tst[:off] - - -def test_write_values_pauses(): - tst = "".join(str(i) for i in range(10)).encode() - for i in range(2, 10): - s = io.BytesIO() - writer.write_values( - s, [tst], [(2, "pause", 0), (1, "pause", 0)], blocksize=i - ) - assert s.getvalue() == tst - - for i in range(2, 10): - s = io.BytesIO() - writer.write_values(s, [tst], [(1, "pause", 0)], blocksize=i) - assert s.getvalue() == tst - - tst = [tst] * 5 - for i in range(2, 10): - s = io.BytesIO() - writer.write_values(s, tst[:], [(1, "pause", 0)], blocksize=i) - assert s.getvalue() == b"".join(tst) - - -def test_write_values_after(): - s = io.BytesIO() - r = next(language.parse_pathod("400:da")) - language.serve(r, s, {}) - - s = io.BytesIO() - r = next(language.parse_pathod("400:pa,0")) - language.serve(r, s, {}) - - s = io.BytesIO() - r = next(language.parse_pathod("400:ia,'xx'")) - language.serve(r, s, {}) - assert s.getvalue().endswith(b'xx') diff --git a/test/pathod/protocols/__init__.py b/test/pathod/protocols/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/test/pathod/protocols/test_http.py b/test/pathod/protocols/test_http.py deleted file mode 100644 index 777ab4dd1..000000000 --- a/test/pathod/protocols/test_http.py +++ /dev/null @@ -1 +0,0 @@ -# TODO: write tests diff --git a/test/pathod/protocols/test_http2.py b/test/pathod/protocols/test_http2.py deleted file mode 100644 index b0fffe731..000000000 --- a/test/pathod/protocols/test_http2.py +++ /dev/null @@ -1,500 +0,0 @@ -from unittest import mock - -import hyperframe -import pytest - -from mitmproxy import exceptions -from mitmproxy.net import http, tcp -from mitmproxy.net.http import http2 -from pathod.protocols.http2 import HTTP2StateProtocol, TCPHandler -from ...mitmproxy.net import tservers as net_tservers - - -class TestTCPHandlerWrapper: - def test_wrapped(self): - h = TCPHandler(rfile='foo', wfile='bar') - p = HTTP2StateProtocol(h) - assert p.tcp_handler.rfile == 'foo' - assert p.tcp_handler.wfile == 'bar' - - def test_direct(self): - p = HTTP2StateProtocol(rfile='foo', wfile='bar') - assert isinstance(p.tcp_handler, TCPHandler) - assert p.tcp_handler.rfile == 'foo' - assert p.tcp_handler.wfile == 'bar' - - -class EchoHandler(tcp.BaseHandler): - sni = None - - def handle(self): - while True: - v = self.rfile.safe_read(1) - self.wfile.write(v) - self.wfile.flush() - - -class TestProtocol: - @mock.patch("pathod.protocols.http2.HTTP2StateProtocol.perform_server_connection_preface") - @mock.patch("pathod.protocols.http2.HTTP2StateProtocol.perform_client_connection_preface") - def test_perform_connection_preface(self, mock_client_method, mock_server_method): - protocol = HTTP2StateProtocol(is_server=False) - protocol.connection_preface_performed = True - - protocol.perform_connection_preface() - assert not mock_client_method.called - assert not mock_server_method.called - - protocol.perform_connection_preface(force=True) - assert mock_client_method.called - assert not mock_server_method.called - - @mock.patch("pathod.protocols.http2.HTTP2StateProtocol.perform_server_connection_preface") - @mock.patch("pathod.protocols.http2.HTTP2StateProtocol.perform_client_connection_preface") - def test_perform_connection_preface_server(self, mock_client_method, mock_server_method): - protocol = HTTP2StateProtocol(is_server=True) - protocol.connection_preface_performed = True - - protocol.perform_connection_preface() - assert not mock_client_method.called - assert not mock_server_method.called - - protocol.perform_connection_preface(force=True) - assert not mock_client_method.called - assert mock_server_method.called - - -class TestCheckALPNMatch(net_tservers.ServerTestBase): - handler = EchoHandler - ssl = dict( - alpn_select=b'h2', - ) - - def test_check_alpn(self): - c = tcp.TCPClient(("127.0.0.1", self.port)) - with c.connect(): - c.convert_to_tls(alpn_protos=[b'h2']) - protocol = HTTP2StateProtocol(c) - assert protocol.check_alpn() - - -class TestCheckALPNMismatch(net_tservers.ServerTestBase): - handler = EchoHandler - ssl = dict( - alpn_select=None, - ) - - def test_check_alpn(self): - c = tcp.TCPClient(("127.0.0.1", self.port)) - with c.connect(): - c.convert_to_tls(alpn_protos=[b'h2']) - protocol = HTTP2StateProtocol(c) - with pytest.raises(NotImplementedError): - protocol.check_alpn() - - -class TestPerformServerConnectionPreface(net_tservers.ServerTestBase): - class handler(tcp.BaseHandler): - - def handle(self): - # send magic - self.wfile.write(bytes.fromhex("505249202a20485454502f322e300d0a0d0a534d0d0a0d0a")) - self.wfile.flush() - - # send empty settings frame - self.wfile.write(bytes.fromhex("000000040000000000")) - self.wfile.flush() - - # check empty settings frame - _, consumed_bytes = http2.read_frame(self.rfile, False) - assert consumed_bytes == bytes.fromhex("00000c040000000000000200000000000300000001") - - # check settings acknowledgement - _, consumed_bytes = http2.read_frame(self.rfile, False) - assert consumed_bytes == bytes.fromhex("000000040100000000") - - # send settings acknowledgement - self.wfile.write(bytes.fromhex("000000040100000000")) - self.wfile.flush() - - def test_perform_server_connection_preface(self): - c = tcp.TCPClient(("127.0.0.1", self.port)) - with c.connect(): - protocol = HTTP2StateProtocol(c) - - assert not protocol.connection_preface_performed - protocol.perform_server_connection_preface() - assert protocol.connection_preface_performed - - with pytest.raises(exceptions.TcpReadIncomplete): - protocol.perform_server_connection_preface(force=True) - - -class TestPerformClientConnectionPreface(net_tservers.ServerTestBase): - class handler(tcp.BaseHandler): - - def handle(self): - # check magic - assert self.rfile.read(24) == HTTP2StateProtocol.CLIENT_CONNECTION_PREFACE - - # check empty settings frame - assert self.rfile.read(9) ==\ - bytes.fromhex("000000040000000000") - - # send empty settings frame - self.wfile.write(bytes.fromhex("000000040000000000")) - self.wfile.flush() - - # check settings acknowledgement - assert self.rfile.read(9) == \ - bytes.fromhex("000000040100000000") - - # send settings acknowledgement - self.wfile.write(bytes.fromhex("000000040100000000")) - self.wfile.flush() - - def test_perform_client_connection_preface(self): - c = tcp.TCPClient(("127.0.0.1", self.port)) - with c.connect(): - protocol = HTTP2StateProtocol(c) - - assert not protocol.connection_preface_performed - protocol.perform_client_connection_preface() - assert protocol.connection_preface_performed - - -class TestClientStreamIds: - c = tcp.TCPClient(("127.0.0.1", 0)) - protocol = HTTP2StateProtocol(c) - - def test_client_stream_ids(self): - assert self.protocol.current_stream_id is None - assert self.protocol._next_stream_id() == 1 - assert self.protocol.current_stream_id == 1 - assert self.protocol._next_stream_id() == 3 - assert self.protocol.current_stream_id == 3 - assert self.protocol._next_stream_id() == 5 - assert self.protocol.current_stream_id == 5 - - -class TestserverstreamIds: - c = tcp.TCPClient(("127.0.0.1", 0)) - protocol = HTTP2StateProtocol(c, is_server=True) - - def test_server_stream_ids(self): - assert self.protocol.current_stream_id is None - assert self.protocol._next_stream_id() == 2 - assert self.protocol.current_stream_id == 2 - assert self.protocol._next_stream_id() == 4 - assert self.protocol.current_stream_id == 4 - assert self.protocol._next_stream_id() == 6 - assert self.protocol.current_stream_id == 6 - - -class TestApplySettings(net_tservers.ServerTestBase): - class handler(tcp.BaseHandler): - def handle(self): - # check settings acknowledgement - assert self.rfile.read(9) == bytes.fromhex("000000040100000000") - self.wfile.write(b"OK") - self.wfile.flush() - self.rfile.safe_read(9) # just to keep the connection alive a bit longer - - ssl = True - - def test_apply_settings(self): - c = tcp.TCPClient(("127.0.0.1", self.port)) - with c.connect(): - c.convert_to_tls() - protocol = HTTP2StateProtocol(c) - - protocol._apply_settings({ - hyperframe.frame.SettingsFrame.ENABLE_PUSH: 'foo', - hyperframe.frame.SettingsFrame.MAX_CONCURRENT_STREAMS: 'bar', - hyperframe.frame.SettingsFrame.INITIAL_WINDOW_SIZE: 'deadbeef', - }) - - assert c.rfile.safe_read(2) == b"OK" - - assert protocol.http2_settings[ - hyperframe.frame.SettingsFrame.ENABLE_PUSH] == 'foo' - assert protocol.http2_settings[ - hyperframe.frame.SettingsFrame.MAX_CONCURRENT_STREAMS] == 'bar' - assert protocol.http2_settings[ - hyperframe.frame.SettingsFrame.INITIAL_WINDOW_SIZE] == 'deadbeef' - - -class TestCreateHeaders: - c = tcp.TCPClient(("127.0.0.1", 0)) - - def test_create_headers(self): - headers = http.Headers([ - (b':method', b'GET'), - (b':path', b'index.html'), - (b':scheme', b'https'), - (b'foo', b'bar')]) - - data = HTTP2StateProtocol(self.c)._create_headers( - headers, 1, end_stream=True) - assert b''.join(data) == bytes.fromhex("000014010500000001824488355217caf3a69a3f87408294e7838c767f") - - data = HTTP2StateProtocol(self.c)._create_headers( - headers, 1, end_stream=False) - assert b''.join(data) == bytes.fromhex("000014010400000001824488355217caf3a69a3f87408294e7838c767f") - - def test_create_headers_multiple_frames(self): - headers = http.Headers([ - (b':method', b'GET'), - (b':path', b'/'), - (b':scheme', b'https'), - (b'foo', b'bar'), - (b'server', b'version')]) - - protocol = HTTP2StateProtocol(self.c) - protocol.http2_settings[hyperframe.frame.SettingsFrame.MAX_FRAME_SIZE] = 8 - data = protocol._create_headers(headers, 1, end_stream=True) - assert len(data) == 3 - assert data[0] == bytes.fromhex("000008010100000001828487408294e783") - assert data[1] == bytes.fromhex("0000080900000000018c767f7685ee5b10") - assert data[2] == bytes.fromhex("00000209040000000163d5") - - -class TestCreateBody: - c = tcp.TCPClient(("127.0.0.1", 0)) - - def test_create_body_empty(self): - protocol = HTTP2StateProtocol(self.c) - bytes = protocol._create_body(b'', 1) - assert b''.join(bytes) == b'' - - def test_create_body_single_frame(self): - protocol = HTTP2StateProtocol(self.c) - data = protocol._create_body(b'foobar', 1) - assert b''.join(data) == bytes.fromhex("000006000100000001666f6f626172") - - def test_create_body_multiple_frames(self): - protocol = HTTP2StateProtocol(self.c) - protocol.http2_settings[hyperframe.frame.SettingsFrame.MAX_FRAME_SIZE] = 5 - data = protocol._create_body(b'foobarmehm42', 1) - assert len(data) == 3 - assert data[0] == bytes.fromhex("000005000000000001666f6f6261") - assert data[1] == bytes.fromhex("000005000000000001726d65686d") - assert data[2] == bytes.fromhex("0000020001000000013432") - - -class TestReadRequest(net_tservers.ServerTestBase): - class handler(tcp.BaseHandler): - - def handle(self): - self.wfile.write( - bytes.fromhex("000003010400000001828487")) - self.wfile.write( - bytes.fromhex("000006000100000001666f6f626172")) - self.wfile.flush() - self.rfile.safe_read(9) # just to keep the connection alive a bit longer - - ssl = True - - def test_read_request(self): - c = tcp.TCPClient(("127.0.0.1", self.port)) - with c.connect(): - c.convert_to_tls() - protocol = HTTP2StateProtocol(c, is_server=True) - protocol.connection_preface_performed = True - - req = protocol.read_request(NotImplemented) - - assert req.stream_id - assert req.headers.fields == () - assert req.method == "GET" - assert req.path == "/" - assert req.scheme == "https" - assert req.content == b'foobar' - - -class TestReadRequestRelative(net_tservers.ServerTestBase): - class handler(tcp.BaseHandler): - def handle(self): - self.wfile.write( - bytes.fromhex("00000c0105000000014287d5af7e4d5a777f4481f9")) - self.wfile.flush() - - ssl = True - - def test_asterisk_form(self): - c = tcp.TCPClient(("127.0.0.1", self.port)) - with c.connect(): - c.convert_to_tls() - protocol = HTTP2StateProtocol(c, is_server=True) - protocol.connection_preface_performed = True - - req = protocol.read_request(NotImplemented) - - assert req.first_line_format == "relative" - assert req.method == "OPTIONS" - assert req.path == "*" - - -class TestReadResponse(net_tservers.ServerTestBase): - class handler(tcp.BaseHandler): - def handle(self): - self.wfile.write( - bytes.fromhex("00000801040000002a88628594e78c767f")) - self.wfile.write( - bytes.fromhex("00000600010000002a666f6f626172")) - self.wfile.flush() - self.rfile.safe_read(9) # just to keep the connection alive a bit longer - - ssl = True - - def test_read_response(self): - c = tcp.TCPClient(("127.0.0.1", self.port)) - with c.connect(): - c.convert_to_tls() - protocol = HTTP2StateProtocol(c) - protocol.connection_preface_performed = True - - resp = protocol.read_response(NotImplemented, stream_id=42) - - assert resp.http_version == "HTTP/2.0" - assert resp.status_code == 200 - assert resp.reason == '' - assert resp.headers.fields == ((b':status', b'200'), (b'etag', b'foobar')) - assert resp.content == b'foobar' - assert resp.timestamp_end - - -class TestReadEmptyResponse(net_tservers.ServerTestBase): - class handler(tcp.BaseHandler): - def handle(self): - self.wfile.write( - bytes.fromhex("00000801050000002a88628594e78c767f")) - self.wfile.flush() - - ssl = True - - def test_read_empty_response(self): - c = tcp.TCPClient(("127.0.0.1", self.port)) - with c.connect(): - c.convert_to_tls() - protocol = HTTP2StateProtocol(c) - protocol.connection_preface_performed = True - - resp = protocol.read_response(NotImplemented, stream_id=42) - - assert resp.stream_id == 42 - assert resp.http_version == "HTTP/2.0" - assert resp.status_code == 200 - assert resp.reason == '' - assert resp.headers.fields == ((b':status', b'200'), (b'etag', b'foobar')) - assert resp.content == b'' - - -class TestAssembleRequest: - c = tcp.TCPClient(("127.0.0.1", 0)) - - def test_request_simple(self): - data = HTTP2StateProtocol(self.c).assemble_request(http.Request( - host="", - port=0, - method=b'GET', - scheme=b'https', - authority=b'', - path=b'/', - http_version=b"HTTP/2.0", - headers=(), - content=None, - trailers=None, - timestamp_start=0, - timestamp_end=0 - )) - assert len(data) == 1 - assert data[0] == bytes.fromhex('00000d0105000000018284874188089d5c0b8170dc07') - - def test_request_with_stream_id(self): - req = http.Request( - host="", - port=0, - method=b'GET', - scheme=b'https', - authority=b'', - path=b'/', - http_version=b"HTTP/2.0", - headers=(), - content=None, - trailers=None, - timestamp_start=0, - timestamp_end=0 - ) - req.stream_id = 0x42 - data = HTTP2StateProtocol(self.c).assemble_request(req) - assert len(data) == 1 - assert data[0] == bytes.fromhex('00000d0105000000428284874188089d5c0b8170dc07') - - def test_request_with_body(self): - data = HTTP2StateProtocol(self.c).assemble_request(http.Request( - host="", - port=0, - method=b'GET', - scheme=b'https', - authority=b'', - path=b'/', - http_version=b"HTTP/2.0", - headers=http.Headers([(b'foo', b'bar')]), - content=b'foobar', - trailers=None, - timestamp_start=0, - timestamp_end=None, - )) - assert len(data) == 2 - assert data[0] == bytes.fromhex("0000150104000000018284874188089d5c0b8170dc07408294e7838c767f") - assert data[1] == bytes.fromhex("000006000100000001666f6f626172") - - -class TestAssembleResponse: - c = tcp.TCPClient(("127.0.0.1", 0)) - - def test_simple(self): - data = HTTP2StateProtocol(self.c, is_server=True).assemble_response(http.Response( - http_version=b"HTTP/2.0", - status_code=200, - reason=b"", - headers=(), - content=b"", - trailers=None, - timestamp_start=0, - timestamp_end=0, - )) - assert len(data) == 1 - assert data[0] == bytes.fromhex("00000101050000000288") - - def test_with_stream_id(self): - resp = http.Response( - http_version=b"HTTP/2.0", - status_code=200, - reason=b"", - headers=(), - content=b"", - trailers=None, - timestamp_start=0, - timestamp_end=0, - ) - resp.stream_id = 0x42 - data = HTTP2StateProtocol(self.c, is_server=True).assemble_response(resp) - assert len(data) == 1 - assert data[0] == bytes.fromhex("00000101050000004288") - - def test_with_body(self): - data = HTTP2StateProtocol(self.c, is_server=True).assemble_response(http.Response( - http_version=b"HTTP/2.0", - status_code=200, - reason=b'', - headers=http.Headers(foo=b"bar"), - content=b'foobar', - trailers=None, - timestamp_start=0, - timestamp_end=0, - )) - assert len(data) == 2 - assert data[0] == bytes.fromhex("00000901040000000288408294e7838c767f") - assert data[1] == bytes.fromhex("000006000100000002666f6f626172") diff --git a/test/pathod/protocols/test_websockets.py b/test/pathod/protocols/test_websockets.py deleted file mode 100644 index 777ab4dd1..000000000 --- a/test/pathod/protocols/test_websockets.py +++ /dev/null @@ -1 +0,0 @@ -# TODO: write tests diff --git a/test/pathod/scripts/generate.sh b/test/pathod/scripts/generate.sh deleted file mode 100644 index eec3077da..000000000 --- a/test/pathod/scripts/generate.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -if [ ! -f ./private.key ] -then - openssl genrsa -out private.key 3072 -fi -openssl req \ - -batch \ - -new -x509 \ - -key private.key \ - -sha256 \ - -out cert.pem \ - -days 9999 \ - -config ./openssl.cnf -openssl x509 -in cert.pem -text -noout -cat ./private.key ./cert.pem > testcert.pem -rm ./private.key ./cert.pem diff --git a/test/pathod/scripts/openssl.cnf b/test/pathod/scripts/openssl.cnf deleted file mode 100644 index 5c8903541..000000000 --- a/test/pathod/scripts/openssl.cnf +++ /dev/null @@ -1,39 +0,0 @@ -[ req ] -default_bits = 1024 -default_keyfile = privkey.pem -distinguished_name = req_distinguished_name -x509_extensions = v3_ca - -[ req_distinguished_name ] -countryName = Country Name (2 letter code) -countryName_default = NZ -countryName_min = 2 -countryName_max = 2 -stateOrProvinceName = State or Province Name (full name) -stateOrProvinceName_default = Otago -localityName = Locality Name (eg, city) -0.organizationName = Organization Name (eg, company) -0.organizationName_default = Pathod -commonName = Common Name (e.g. server FQDN or YOUR name) -commonName_default = test.com -commonName_max = 64 - -[ v3_req ] - -basicConstraints = CA:FALSE -keyUsage = nonRepudiation, digitalSignature, keyEncipherment - -[ v3_ca ] - -keyUsage = digitalSignature, keyEncipherment -subjectKeyIdentifier=hash -authorityKeyIdentifier=keyid:always,issuer:always -basicConstraints = CA:true -subjectAltName = @alternate_names - - -[ alternate_names ] - -DNS.1 = test.com -DNS.2 = test2.com -DNS.3 = test3.com diff --git a/test/pathod/test_log.py b/test/pathod/test_log.py deleted file mode 100644 index 8890e7d96..000000000 --- a/test/pathod/test_log.py +++ /dev/null @@ -1,25 +0,0 @@ -import io - -from pathod import log -from mitmproxy import exceptions - - -class DummyIO(io.StringIO): - - def start_log(self, *args, **kwargs): - pass - - def get_log(self, *args, **kwargs): - return "" - - -def test_disconnect(): - outf = DummyIO() - rw = DummyIO() - l = log.ConnectionLogger(outf, False, True, rw, rw) - try: - with l.ctx() as lg: - lg("Test") - except exceptions.TcpDisconnect: - pass - assert "Test" in outf.getvalue() diff --git a/test/pathod/test_pathoc.py b/test/pathod/test_pathoc.py deleted file mode 100644 index 0fcbc2e30..000000000 --- a/test/pathod/test_pathoc.py +++ /dev/null @@ -1,250 +0,0 @@ -import io -from unittest.mock import Mock - -import pytest - -from mitmproxy import exceptions -from mitmproxy.net.http import http1 -from mitmproxy.test import tutils -from pathod import language, pathoc -from pathod.protocols.http2 import HTTP2StateProtocol -from . import tservers - - -class PathocTestDaemon(tservers.DaemonTests): - def tval(self, requests, timeout=None, showssl=False, **kwargs): - s = io.StringIO() - c = pathoc.Pathoc( - ("127.0.0.1", self.d.port), - ssl=self.ssl, - fp=s, - **kwargs - ) - with c.connect(showssl=showssl, fp=s): - if timeout: - c.settimeout(timeout) - for i in requests: - r = next(language.parse_pathoc(i)) - if kwargs.get("explain"): - r = r.freeze(language.Settings()) - try: - c.request(r) - except exceptions.NetlibException: - pass - self.d.wait_for_silence() - return s.getvalue() - - -class TestDaemonSSL(PathocTestDaemon): - ssl = True - ssloptions = dict( - request_client_cert=True, - sans=[b"test1.com", b"test2.com"], - alpn_select=b'h2', - ) - - def test_sni(self): - self.tval( - ["get:/p/200"], - sni="foobar.com" - ) - log = self.d.log() - assert log[0]["request"]["sni"] == "foobar.com" - - def test_showssl(self): - assert "certificate chain" in self.tval(["get:/p/200"], showssl=True) - - def test_clientcert(self, tdata): - self.tval( - ["get:/p/200"], - clientcert=tdata.path("pathod/data/clientcert/client.pem"), - ) - log = self.d.log() - assert log[0]["request"]["clientcert"]["keyinfo"] - - def test_http2_without_ssl(self): - fp = io.StringIO() - c = pathoc.Pathoc( - ("127.0.0.1", self.d.port), - use_http2=True, - ssl=False, - fp=fp - ) - with pytest.raises(NotImplementedError): - c.connect() - - -class TestDaemon(PathocTestDaemon): - ssl = False - - def test_ssl_error(self): - c = pathoc.Pathoc(("127.0.0.1", self.d.port), ssl=True, fp=None) - try: - with c.connect(): - pass - except Exception as e: - assert "SSL" in str(e) - else: - raise AssertionError("No exception raised.") - - def test_showssl(self): - assert "certificate chain" not in self.tval( - ["get:/p/200"], - showssl=True) - - def test_ignorecodes(self): - assert "200" in self.tval(["get:'/p/200:b@1'"]) - assert "200" in self.tval(["get:'/p/200:b@1'"]) - assert "200" in self.tval(["get:'/p/200:b@1'"]) - assert "200" not in self.tval(["get:'/p/200:b@1'"], ignorecodes=[200]) - assert "200" not in self.tval( - ["get:'/p/200:b@1'"], - ignorecodes=[ - 200, - 201]) - assert "202" in self.tval(["get:'/p/202:b@1'"], ignorecodes=[200, 201]) - - def _test_timeout(self): - assert "Timeout" in self.tval(["get:'/p/200:p0,100'"], timeout=0.01) - assert "HTTP" in self.tval( - ["get:'/p/200:p5,100'"], - showresp=True, - timeout=1 - ) - assert "HTTP" not in self.tval( - ["get:'/p/200:p3,100'"], - showresp=True, - timeout=1, - ignoretimeout=True - ) - - def test_showresp(self): - reqs = ["get:/p/200:da", "get:/p/200:da"] - assert self.tval(reqs).count("200 OK") == 2 - assert self.tval(reqs, showresp=True).count("HTTP/1.1 200 OK") == 2 - assert self.tval( - reqs, showresp=True, hexdump=True - ).count("0000000000") == 2 - - def test_showresp_httperr(self): - v = self.tval(["get:'/p/200:d20'"], showresp=True, showsummary=True) - assert "Invalid header" in v - assert "HTTP/" in v - - def test_explain(self): - reqs = ["get:/p/200:b@100"] - assert "b@100" not in self.tval(reqs, explain=True) - - def test_showreq(self): - reqs = ["get:/p/200:da", "get:/p/200:da"] - assert self.tval(reqs, showreq=True).count("GET /p/200") == 2 - assert self.tval( - reqs, showreq=True, hexdump=True - ).count("0000000000") == 2 - - def test_conn_err(self): - assert "Invalid server response" in self.tval(["get:'/p/200:d2'"]) - - def test_websocket_shutdown(self): - self.tval(["ws:/"]) - - def test_wait_finish(self): - c = pathoc.Pathoc( - ("127.0.0.1", self.d.port), - fp=None, - ws_read_limit=1 - ) - with c.connect(): - c.request("ws:/") - c.request("wf:f'wf'") - # This should read a frame and close the websocket reader - assert len([i for i in c.wait(timeout=5, finish=False)]) == 1 - assert not [i for i in c.wait(timeout=0)] - - def test_connect_fail(self): - to = ("foobar", 80) - c = pathoc.Pathoc(("127.0.0.1", self.d.port), fp=None) - c.rfile, c.wfile = io.BytesIO(), io.BytesIO() - with pytest.raises(Exception, match="CONNECT failed"): - c.http_connect(to) - c.rfile = io.BytesIO( - b"HTTP/1.1 500 OK\r\n" - ) - with pytest.raises(Exception, match="CONNECT failed"): - c.http_connect(to) - c.rfile = io.BytesIO( - b"HTTP/1.1 200 OK\r\n" - ) - c.http_connect(to) - - def test_socks_connect(self): - to = ("foobar", 80) - c = pathoc.Pathoc(("127.0.0.1", self.d.port), fp=None) - c.rfile, c.wfile = tutils.treader(b""), io.BytesIO() - with pytest.raises(pathoc.PathocError): - c.socks_connect(to) - - c.rfile = tutils.treader( - b"\x05\xEE" - ) - with pytest.raises(Exception, match="SOCKS without authentication"): - c.socks_connect(("example.com", 0xDEAD)) - - c.rfile = tutils.treader( - b"\x05\x00" + - b"\x05\xEE\x00\x03\x0bexample.com\xDE\xAD" - ) - with pytest.raises(Exception, match="SOCKS server error"): - c.socks_connect(("example.com", 0xDEAD)) - - c.rfile = tutils.treader( - b"\x05\x00" + - b"\x05\x00\x00\x03\x0bexample.com\xDE\xAD" - ) - c.socks_connect(("example.com", 0xDEAD)) - - -class TestDaemonHTTP2(PathocTestDaemon): - ssl = True - explain = False - - def test_http2(self): - c = pathoc.Pathoc( - ("127.0.0.1", self.d.port), - fp=None, - ssl=True, - use_http2=True, - ) - assert isinstance(c.protocol, HTTP2StateProtocol) - - c = pathoc.Pathoc( - ("127.0.0.1", self.d.port), - ) - assert c.protocol == http1 - - def test_http2_alpn(self): - c = pathoc.Pathoc( - ("127.0.0.1", self.d.port), - fp=None, - ssl=True, - use_http2=True, - http2_skip_connection_preface=True, - ) - - tmp_convert_to_tls = c.convert_to_tls - c.convert_to_tls = Mock() - c.convert_to_tls.side_effect = tmp_convert_to_tls - with c.connect(): - _, kwargs = c.convert_to_tls.call_args - assert set(kwargs['alpn_protos']) == {b'http/1.1', b'h2'} - - def test_request(self): - c = pathoc.Pathoc( - ("127.0.0.1", self.d.port), - fp=None, - ssl=True, - use_http2=True, - ) - with c.connect(): - resp = c.request("get:/p/200") - assert resp.status_code == 200 diff --git a/test/pathod/test_pathoc_cmdline.py b/test/pathod/test_pathoc_cmdline.py deleted file mode 100644 index fecebe3d8..000000000 --- a/test/pathod/test_pathoc_cmdline.py +++ /dev/null @@ -1,60 +0,0 @@ -import io -import pytest -from unittest import mock - -from pathod import pathoc_cmdline as cmdline - - -@mock.patch("argparse.ArgumentParser.error") -def test_pathoc(perror, tdata): - assert cmdline.args_pathoc(["pathoc", "foo.com", "get:/"]) - s = io.StringIO() - with pytest.raises(SystemExit): - cmdline.args_pathoc(["pathoc", "--show-uas"], s, s) - - a = cmdline.args_pathoc(["pathoc", "foo.com:8888", "get:/"]) - assert a.port == 8888 - - a = cmdline.args_pathoc(["pathoc", "foo.com:xxx", "get:/"]) - assert perror.called - perror.reset_mock() - - a = cmdline.args_pathoc(["pathoc", "-I", "10, 20", "foo.com:8888", "get:/"]) - assert a.ignorecodes == [10, 20] - - a = cmdline.args_pathoc(["pathoc", "-I", "xx, 20", "foo.com:8888", "get:/"]) - assert perror.called - perror.reset_mock() - - a = cmdline.args_pathoc(["pathoc", "-c", "foo:10", "foo.com:8888", "get:/"]) - assert a.connect_to == ["foo", 10] - - a = cmdline.args_pathoc(["pathoc", "foo.com", "get:/", "--http2"]) - assert a.use_http2 is True - assert a.ssl is True - - a = cmdline.args_pathoc(["pathoc", "foo.com", "get:/", "--http2-skip-connection-preface"]) - assert a.use_http2 is True - assert a.ssl is True - assert a.http2_skip_connection_preface is True - - a = cmdline.args_pathoc(["pathoc", "-c", "foo", "foo.com:8888", "get:/"]) - assert perror.called - perror.reset_mock() - - a = cmdline.args_pathoc( - ["pathoc", "-c", "foo:bar", "foo.com:8888", "get:/"]) - assert perror.called - perror.reset_mock() - - a = cmdline.args_pathoc( - [ - "pathoc", - "foo.com:8888", - tdata.path("pathod/data/request") - ] - ) - assert len(list(a.requests)) == 1 - - with pytest.raises(SystemExit): - cmdline.args_pathoc(["pathoc", "foo.com", "invalid"], s, s) diff --git a/test/pathod/test_pathod.py b/test/pathod/test_pathod.py deleted file mode 100644 index 246bff3b8..000000000 --- a/test/pathod/test_pathod.py +++ /dev/null @@ -1,264 +0,0 @@ -import io - -import pytest - -from pathod import pathod -from mitmproxy.net import tcp -from mitmproxy import exceptions -from mitmproxy.utils import data - -from . import tservers - - -cdata = data.Data(__name__) - - -class TestPathod: - - def test_logging(self): - s = io.StringIO() - p = pathod.Pathod(("127.0.0.1", 0), logfp=s) - assert len(p.get_log()) == 0 - id = p.add_log(dict(s="foo")) - assert p.log_by_id(id) - assert len(p.get_log()) == 1 - p.clear_log() - assert len(p.get_log()) == 0 - - for _ in range(p.LOGBUF + 1): - p.add_log(dict(s="foo")) - assert len(p.get_log()) <= p.LOGBUF - - -class TestTimeout(tservers.DaemonTests): - timeout = 0.01 - - def test_timeout(self): - # FIXME: Add float values to spec language, reduce test timeout to - # increase test performance - # This is a bodge - we have some platform difference that causes - # different exceptions to be raised here. - with pytest.raises(Exception): - self.pathoc(["get:/:p1,1"]) - assert self.d.last_log()["type"] == "timeout" - - -class TestNotAfterConnect(tservers.DaemonTests): - ssl = False - ssloptions = dict( - not_after_connect=True - ) - - def test_connect(self): - r, _ = self.pathoc( - [r"get:'http://foo.com/p/202':da"], - connect_to=("localhost", self.d.port) - ) - assert r[0].status_code == 202 - - -class TestCustomCert(tservers.DaemonTests): - ssl = True - ssloptions = dict( - certs=[("*", cdata.path("data/testkey.pem"))], - ) - - def test_connect(self): - r, _ = self.pathoc([r"get:/p/202"]) - r = r[0] - assert r.status_code == 202 - assert r.sslinfo - assert "test.com" in str(r.sslinfo.certchain[0].get_subject()) - - -class TestSSLCN(tservers.DaemonTests): - ssl = True - ssloptions = dict( - cn=b"foo.com" - ) - - def test_connect(self): - r, _ = self.pathoc([r"get:/p/202"]) - r = r[0] - assert r.status_code == 202 - assert r.sslinfo - assert r.sslinfo.certchain[0].get_subject().CN == "foo.com" - - -class TestNohang(tservers.DaemonTests): - nohang = True - - def test_nohang(self): - r = self.get("200:p0,0") - assert r.status_code == 800 - l = self.d.last_log() - assert "Pauses have been disabled" in l["response"]["msg"] - - -class TestHexdump(tservers.DaemonTests): - hexdump = True - - def test_hexdump(self): - assert self.get(r"200:b'\xf0'") - - -class TestNocraft(tservers.DaemonTests): - nocraft = True - - def test_nocraft(self): - r = self.get(r"200:b'\xf0'") - assert r.status_code == 800 - assert b"Crafting disabled" in r.content - - -class CommonTests(tservers.DaemonTests): - - def test_binarydata(self): - assert self.get(r"200:b'\xf0'") - assert self.d.last_log() - # FIXME: Other binary data elements - - def test_sizelimit(self): - r = self.get("200:b@1g") - assert r.status_code == 800 - l = self.d.last_log() - assert "too large" in l["response"]["msg"] - - def test_preline(self): - r, _ = self.pathoc([r"get:'/p/200':i0,'\r\n'"]) - assert r[0].status_code == 200 - - def test_logs(self): - self.d.clear_log() - assert self.get("202:da") - assert self.d.expect_log(1) - self.d.clear_log() - assert len(self.d.log()) == 0 - - def test_disconnect(self): - with pytest.raises(Exception, match="Unexpected EOF"): - self.get("202:b@100k:d200") - - def test_parserr(self): - rsp = self.get("400:msg,b:") - assert rsp.status_code == 800 - - def test_static(self): - rsp = self.get("200:b 0 - - -class TestHTTP2(tservers.DaemonTests): - ssl = True - nohang = True - - def test_http2(self): - r, _ = self.pathoc(["GET:/"], ssl=True, use_http2=True) - assert r[0].status_code == 800 diff --git a/test/pathod/test_pathod_cmdline.py b/test/pathod/test_pathod_cmdline.py deleted file mode 100644 index 374271796..000000000 --- a/test/pathod/test_pathod_cmdline.py +++ /dev/null @@ -1,89 +0,0 @@ -from unittest import mock - -from pathod import pathod_cmdline as cmdline - - -def test_parse_anchor_spec(): - assert cmdline.parse_anchor_spec("foo=200") == ("foo", "200") - assert cmdline.parse_anchor_spec("foo") is None - - -@mock.patch("argparse.ArgumentParser.error") -def test_pathod(perror, tdata): - assert cmdline.args_pathod(["pathod"]) - - a = cmdline.args_pathod( - [ - "pathod", - "--cert", - tdata.path("pathod/data/testkey.pem") - ] - ) - assert a.ssl_certs - - a = cmdline.args_pathod( - [ - "pathod", - "--cert", - "nonexistent" - ] - ) - assert perror.called - perror.reset_mock() - - a = cmdline.args_pathod( - [ - "pathod", - "-a", - "foo=200" - ] - ) - assert a.anchors - - a = cmdline.args_pathod( - [ - "pathod", - "-a", - "foo=" + tdata.path("pathod/data/response") - ] - ) - assert a.anchors - - a = cmdline.args_pathod( - [ - "pathod", - "-a", - "?=200" - ] - ) - assert perror.called - perror.reset_mock() - - a = cmdline.args_pathod( - [ - "pathod", - "-a", - "foo" - ] - ) - assert perror.called - perror.reset_mock() - - a = cmdline.args_pathod( - [ - "pathod", - "--limit-size", - "200k" - ] - ) - assert a.sizelimit - - a = cmdline.args_pathod( - [ - "pathod", - "--limit-size", - "q" - ] - ) - assert perror.called - perror.reset_mock() diff --git a/test/pathod/test_test.py b/test/pathod/test_test.py deleted file mode 100644 index d51a2c7a3..000000000 --- a/test/pathod/test_test.py +++ /dev/null @@ -1,34 +0,0 @@ -import os -import requests -import pytest - -from pathod import test -from pathod.pathod import SSLOptions, CA_CERT_NAME - - -class TestDaemonManual: - - def test_simple(self): - with test.Daemon() as d: - rsp = requests.get("http://localhost:%s/p/202:da" % d.port) - assert rsp.ok - assert rsp.status_code == 202 - with pytest.raises(requests.ConnectionError): - requests.get("http://localhost:%s/p/202:da" % d.port) - - @pytest.mark.parametrize('not_after_connect', [True, False]) - def test_startstop_ssl(self, not_after_connect): - ssloptions = SSLOptions( - cn=b'localhost', - sans=[b'localhost', b'127.0.0.1'], - not_after_connect=not_after_connect, - ) - d = test.Daemon(ssl=True, ssloptions=ssloptions) - rsp = requests.get( - "https://localhost:%s/p/202:da" % d.port, - verify=os.path.expanduser(os.path.join(d.thread.server.ssloptions.confdir, CA_CERT_NAME))) - assert rsp.ok - assert rsp.status_code == 202 - d.shutdown() - with pytest.raises(requests.ConnectionError): - requests.get("http://localhost:%s/p/202:da" % d.port) diff --git a/test/pathod/test_utils.py b/test/pathod/test_utils.py deleted file mode 100644 index 28443e249..000000000 --- a/test/pathod/test_utils.py +++ /dev/null @@ -1,17 +0,0 @@ -import pytest - -from pathod import utils - - -def test_membool(): - m = utils.MemBool() - assert not m.v - assert m(1) - assert m.v == 1 - assert m(2) - assert m.v == 2 - - -def test_data_path(): - with pytest.raises(ValueError): - utils.data.path("nonexistent") diff --git a/test/pathod/tservers.py b/test/pathod/tservers.py deleted file mode 100644 index 7bea05ecc..000000000 --- a/test/pathod/tservers.py +++ /dev/null @@ -1,150 +0,0 @@ -import os -import tempfile -import re -import shutil -import requests -import io -import urllib - - -from mitmproxy.net import tcp -from mitmproxy.utils import data - -from pathod import language -from pathod import pathoc -from pathod import pathod -from pathod import test -from pathod.pathod import CA_CERT_NAME - - -cdata = data.Data(__name__) - - -def treader(bytes): - """ - Construct a tcp.Read object from bytes. - """ - fp = io.BytesIO(bytes) - return tcp.Reader(fp) - - -class DaemonTests: - nohang = False - ssl = False - timeout = None - hexdump = False - ssloptions = None - nocraft = False - explain = True - - @classmethod - def setup_class(cls): - opts = cls.ssloptions or {} - cls.confdir = tempfile.mkdtemp() - opts["confdir"] = cls.confdir - so = pathod.SSLOptions(**opts) - cls.d = test.Daemon( - staticdir=cdata.path("data"), - anchors=[ - (re.compile("/anchor/.*"), "202:da") - ], - ssl=cls.ssl, - ssloptions=so, - sizelimit=1 * 1024 * 1024, - nohang=cls.nohang, - timeout=cls.timeout, - hexdump=cls.hexdump, - nocraft=cls.nocraft, - logreq=True, - logresp=True, - explain=cls.explain - ) - - @classmethod - def teardown_class(cls): - cls.d.shutdown() - shutil.rmtree(cls.confdir) - - def teardown(self): - self.d.wait_for_silence() - self.d.clear_log() - - def _getpath(self, path, params=None): - scheme = "https" if self.ssl else "http" - resp = requests.get( - "{}://localhost:{}/{}".format( - scheme, - self.d.port, - path - ), - verify=os.path.join(self.d.thread.server.ssloptions.confdir, CA_CERT_NAME), - params=params - ) - return resp - - def getpath(self, path, params=None): - logfp = io.StringIO() - c = pathoc.Pathoc( - ("localhost", self.d.port), - ssl=self.ssl, - fp=logfp, - ) - with c.connect(): - if params: - path = path + "?" + urllib.parse.urlencode(params) - resp = c.request("get:%s" % path) - return resp - - def get(self, spec): - logfp = io.StringIO() - c = pathoc.Pathoc( - ("localhost", self.d.port), - ssl=self.ssl, - fp=logfp, - ) - with c.connect(): - resp = c.request( - "get:/p/%s" % urllib.parse.quote(spec) - ) - return resp - - def pathoc( - self, - specs, - timeout=None, - connect_to=None, - ssl=None, - ws_read_limit=None, - use_http2=False, - ): - """ - Returns a (messages, text log) tuple. - """ - if ssl is None: - ssl = self.ssl - logfp = io.StringIO() - c = pathoc.Pathoc( - ("localhost", self.d.port), - ssl=ssl, - ws_read_limit=ws_read_limit, - timeout=timeout, - fp=logfp, - use_http2=use_http2, - ) - with c.connect(connect_to): - ret = [] - for i in specs: - resp = c.request(i) - if resp: - ret.append(resp) - for frm in c.wait(): - ret.append(frm) - c.stop() - return ret, logfp.getvalue() - - -def render(r, settings=language.Settings()): - r = r.resolve(settings) - s = io.BytesIO() - assert language.serve(r, s, settings) - return s.getvalue() diff --git a/test/release/test_cibuild.py b/test/release/test_cibuild.py index d4ed32b09..3831e78f3 100644 --- a/test/release/test_cibuild.py +++ b/test/release/test_cibuild.py @@ -222,7 +222,6 @@ def test_buildenviron_osx(tmpdir): assert be.platform_tag == "osx" assert be.bdists == { "mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"], - "pathod": ["pathoc", "pathod"], } assert be.archive_name("mitmproxy") == "mitmproxy-0.0.1-osx.tar.gz" @@ -241,7 +240,6 @@ def test_buildenviron_windows(tmpdir): assert be.platform_tag == "windows" assert be.bdists == { "mitmproxy": ["mitmdump", "mitmweb"], - "pathod": ["pathoc", "pathod"], } assert be.archive_name("mitmproxy") == "mitmproxy-0.0.1-windows.zip" diff --git a/tox.ini b/tox.ini index 9a1eb0257..fc9bd2cab 100644 --- a/tox.ini +++ b/tox.ini @@ -10,14 +10,14 @@ setenv = HOME = {envtmpdir} commands = mitmdump --version pytest --timeout 60 -vv --cov-report xml \ - --cov=mitmproxy --cov=pathod --cov=release \ - --full-cov=mitmproxy/ --full-cov=pathod/ \ + --cov=mitmproxy --cov=release \ + --full-cov=mitmproxy/ \ {posargs} [testenv:flake8] deps = flake8==3.8.4 commands = - flake8 --jobs 8 mitmproxy pathod examples test release {posargs} + flake8 --jobs 8 mitmproxy examples test release {posargs} [testenv:filename_matching] deps = @@ -59,8 +59,6 @@ commands = mitmproxy --version mitmdump --version mitmweb --version - pathod --version - pathoc --version [testenv:docs] passenv = GITHUB_* AWS_*