Grasscutters/mitmproxy
1
0
Fork 0
mirror of https://github.com/Grasscutters/mitmproxy.git synced 2024-11-23 04:19:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

remove pathod and pathoc

Browse Source
This commit is contained in:
Thomas Kriechbaumer 2020-12-13 13:51:17 +01:00
parent b7efe9b2d4
commit c35316f85a
101 changed files with 7 additions and 11493 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -37,6 +37,7 @@ If you depend on these features, please raise your voice in
### Full Changelog
* Remove all deprecated pathod and pathoc tools and modules (@Kriechi)
* --- TODO: add new PRs above this line ---
* ... and various other fixes, documentation improvements, dependency version bumps, etc.

1
MANIFEST.in
View File

@ -1,3 +1,2 @@
graft mitmproxy
graft pathod
recursive-exclude * *.pyc *.pyo *.swo *.swp *.map

7
examples/pathod/libpathod_pathoc.py
View File

@ -1,7 +0,0 @@
#!/usr/bin/env python
from pathod import pathoc
p = pathoc.Pathoc(("google.com", 80))
p.connect()
print(p.request("get:/"))
print(p.request("get:/foo"))

23
examples/pathod/test_context.py
View File

@ -1,23 +0,0 @@
import requests
from pathod import test
def test_simple():
"""
Testing the requests module with
a pathod context manager.
"""
# Start pathod in a separate thread
with test.Daemon() as d:
# Get a URL for a pathod spec
url = d.p("200:b@100")
# ... and request it
r = requests.put(url)
# Check the returned data
assert r.status_code == 200
assert len(r.content) == 100
# Check pathod's internal log
log = d.last_log()["request"]
assert log["method"] == "PUT"

30
examples/pathod/test_setup.py
View File

@ -1,30 +0,0 @@
import requests
from pathod import test
class Test:
"""
Testing the requests module with
a pathod instance started for
each test.
"""
def setup(self):
self.d = test.Daemon()
def teardown(self):
self.d.shutdown()
def test_simple(self):
# Get a URL for a pathod spec
url = self.d.p("200:b@100")
# ... and request it
r = requests.put(url)
# Check the returned data
assert r.status_code == 200
assert len(r.content) == 100
# Check pathod's internal log
log = self.d.last_log()["request"]
assert log["method"] == "PUT"

39
examples/pathod/test_setupall.py
View File

@ -1,39 +0,0 @@
import requests
from pathod import test
class Test:
"""
Testing the requests module with
a single pathod instance started
for the test suite.
"""
@classmethod
def setup_class(cls):
cls.d = test.Daemon()
@classmethod
def teardown_class(cls):
cls.d.shutdown()
def setup(self):
# Clear the pathod logs between tests
self.d.clear_log()
def test_simple(self):
# Get a URL for a pathod spec
url = self.d.p("200:b@100")
# ... and request it
r = requests.put(url)
# Check the returned data
assert r.status_code == 200
assert len(r.content) == 100
# Check pathod's internal log
log = self.d.last_log()["request"]
assert log["method"] == "PUT"
def test_two(self):
assert not self.d.log()

1
mitmproxy/version.py
View File

@ -3,7 +3,6 @@ import subprocess
import sys
VERSION = "7.0.0.dev"
PATHOD = "pathod " + VERSION
MITMPROXY = "mitmproxy " + VERSION
# Serialization format version. This is displayed nowhere, it just needs to be incremented by one

18
pathod/__init__.py
View File

@ -1,18 +0,0 @@
import os
import sys
import warnings
warnings.warn(
"pathod and pathoc modules are deprecated, see https://github.com/mitmproxy/mitmproxy/issues/4273",
DeprecationWarning,
stacklevel=2
)
def print_tool_deprecation_message():
print("####", file=sys.stderr)
print(f"### {os.path.basename(sys.argv[0])} is deprecated and will not be part of future mitmproxy releases!", file=sys.stderr)
print("### See https://github.com/mitmproxy/mitmproxy/issues/4273 for more information.", file=sys.stderr)
print("####", file=sys.stderr)
print("", file=sys.stderr)

117
pathod/language/__init__.py
View File

@ -1,117 +0,0 @@
import itertools
import time
import pyparsing as pp
from . import http, http2, websockets, writer, exceptions
from .exceptions import RenderError, FileAccessDenied, ParseException
from .base import Settings
__all__ = [
"RenderError", "FileAccessDenied", "ParseException",
"Settings",
]
def expand(msg):
times = getattr(msg, "times", None)
if times:
for j_ in range(int(times.value)):
yield msg.strike_token("times")
else:
yield msg
def parse_pathod(s, use_http2=False):
"""
May raise ParseException
"""
try:
s.encode("ascii")
except UnicodeError:
raise exceptions.ParseException("Spec must be valid ASCII.", 0, 0)
try:
if use_http2:
expressions = [
# http2.Frame.expr(),
http2.Response.expr(),
]
else:
expressions = [
websockets.WebsocketFrame.expr(),
http.Response.expr(),
]
reqs = pp.Or(expressions).parseString(s, parseAll=True)
except pp.ParseException as v:
raise exceptions.ParseException(v.msg, v.line, v.col)
return itertools.chain(*[expand(i) for i in reqs])
def parse_pathoc(s, use_http2=False):
try:
s.encode("ascii")
except UnicodeError:
raise exceptions.ParseException("Spec must be valid ASCII.", 0, 0)
try:
if use_http2:
expressions = [
# http2.Frame.expr(),
http2.Request.expr(),
]
else:
expressions = [
websockets.WebsocketClientFrame.expr(),
http.Request.expr(),
]
reqs = pp.OneOrMore(pp.Or(expressions)).parseString(s, parseAll=True)
except pp.ParseException as v:
raise exceptions.ParseException(v.msg, v.line, v.col)
return itertools.chain(*[expand(i) for i in reqs])
def parse_websocket_frame(s):
"""
May raise ParseException
"""
try:
reqs = pp.OneOrMore(
websockets.WebsocketFrame.expr()
).parseString(
s,
parseAll=True
)
except pp.ParseException as v:
raise exceptions.ParseException(v.msg, v.line, v.col)
return itertools.chain(*[expand(i) for i in reqs])
def serve(msg, fp, settings):
"""
fp: The file pointer to write to.
request_host: If this a request, this is the connecting host. If
None, we assume it's a response. Used to decide what standard
modifications to make if raw is not set.
Calling this function may modify the object.
"""
msg = msg.resolve(settings)
started = time.time()
vals = msg.values(settings)
vals.reverse()
actions = sorted(msg.actions[:])
actions.reverse()
actions = [i.intermediate(settings) for i in actions]
disconnect = writer.write_values(fp, vals, actions[:])
duration = time.time() - started
ret = dict(
disconnect=disconnect,
started=started,
duration=duration,
)
ret.update(msg.log(settings))
return ret

129
pathod/language/actions.py
View File

@ -1,129 +0,0 @@
import abc
import copy
import random
from functools import total_ordering
import pyparsing as pp
from . import base
@total_ordering # type: ignore
class _Action(base.Token):
"""
An action that operates on the raw data stream of the message. All
actions have one thing in common: an offset that specifies where the
action should take place.
"""
def __init__(self, offset):
self.offset = offset
def resolve(self, settings, msg):
"""
Resolves offset specifications to a numeric offset. Returns a copy
of the action object.
"""
c = copy.copy(self)
l = msg.length(settings)
if c.offset == "r":
c.offset = random.randrange(l)
elif c.offset == "a":
c.offset = l + 1
return c
def __lt__(self, other):
return self.offset < other.offset
def __eq__(self, other):
return self.offset == other.offset
def __repr__(self):
return self.spec()
@abc.abstractmethod
def spec(self): # pragma: no cover
pass
@abc.abstractmethod
def intermediate(self, settings): # pragma: no cover
pass
class PauseAt(_Action):
unique_name = None
def __init__(self, offset, seconds):
_Action.__init__(self, offset)
self.seconds = seconds
@classmethod
def expr(cls):
e = pp.Literal("p").suppress()
e += base.TokOffset
e += pp.Literal(",").suppress()
e += pp.MatchFirst(
[
base.v_integer,
pp.Literal("f")
]
)
return e.setParseAction(lambda x: cls(*x))
def spec(self):
return f"p{self.offset},{self.seconds}"
def intermediate(self, settings):
return (self.offset, "pause", self.seconds)
def freeze(self, settings_):
return self
class DisconnectAt(_Action):
def __init__(self, offset):
_Action.__init__(self, offset)
@classmethod
def expr(cls):
e = pp.Literal("d").suppress()
e += base.TokOffset
return e.setParseAction(lambda x: cls(*x))
def spec(self):
return "d%s" % self.offset
def intermediate(self, settings):
return (self.offset, "disconnect")
def freeze(self, settings_):
return self
class InjectAt(_Action):
unique_name = None # type: ignore
def __init__(self, offset, value):
_Action.__init__(self, offset)
self.value = value
@classmethod
def expr(cls):
e = pp.Literal("i").suppress()
e += base.TokOffset
e += pp.Literal(",").suppress()
e += base.TokValue
return e.setParseAction(lambda x: cls(*x))
def spec(self):
return f"i{self.offset},{self.value.spec()}"
def intermediate(self, settings):
return (
self.offset,
"inject",
self.value.get_generator(settings)
)
def freeze(self, settings):
return InjectAt(self.offset, self.value.freeze(settings))

540
pathod/language/base.py
View File

@ -1,540 +0,0 @@
import operator
import os
import abc
import functools
import pyparsing as pp
from mitmproxy.utils import strutils
from mitmproxy.utils import human
import typing # noqa
from . import generators
from . import exceptions
class Settings:
def __init__(
self,
is_client=False,
staticdir=None,
unconstrained_file_access=False,
request_host=None,
websocket_key=None,
protocol=None,
):
self.is_client = is_client
self.staticdir = staticdir
self.unconstrained_file_access = unconstrained_file_access
self.request_host = request_host
self.websocket_key = websocket_key # TODO: refactor this into the protocol
self.protocol = protocol
Sep = pp.Optional(pp.Literal(":")).suppress()
v_integer = pp.Word(pp.nums)\
.setName("integer")\
.setParseAction(lambda toks: int(toks[0]))
v_literal = pp.MatchFirst(
[
pp.QuotedString(
"\"",
unquoteResults=True,
multiline=True
),
pp.QuotedString(
"'",
unquoteResults=True,
multiline=True
),
]
)
v_naked_literal = pp.MatchFirst(
[
v_literal,
pp.Word("".join(i for i in pp.printables if i not in ",:\n@\'\""))
]
)
class Token:
"""
A token in the specification language. Tokens are immutable. The token
classes have no meaning in and of themselves, and are combined into
Components and Actions to build the language.
"""
__metaclass__ = abc.ABCMeta
@classmethod
def expr(cls): # pragma: no cover
"""
A parse expression.
"""
return None
@abc.abstractmethod
def spec(self): # pragma: no cover
"""
A parseable specification for this token.
"""
return None
@property
def unique_name(self) -> typing.Optional[str]:
"""
Controls uniqueness constraints for tokens. No two tokens with the
same name will be allowed. If no uniquness should be applied, this
should be None.
"""
return self.__class__.__name__.lower()
def resolve(self, settings_, msg_):
"""
Resolves this token to ready it for transmission. This means that
the calculated offsets of actions are fixed.
settings: a language.Settings instance
msg: The containing message
"""
return self
def __repr__(self):
return self.spec()
class _TokValueLiteral(Token):
def __init__(self, val):
self.val = strutils.escaped_str_to_bytes(val)
def get_generator(self, settings_):
return self.val
def freeze(self, settings_):
return self
class TokValueLiteral(_TokValueLiteral):
"""
A literal with Python-style string escaping
"""
@classmethod
def expr(cls):
e = v_literal.copy()
return e.setParseAction(cls.parseAction)
@classmethod
def parseAction(cls, x):
v = cls(*x)
return v
def spec(self):
inner = strutils.bytes_to_escaped_str(self.val)
inner = inner.replace(r"'", r"\x27")
return "'" + inner + "'"
class TokValueNakedLiteral(_TokValueLiteral):
@classmethod
def expr(cls):
e = v_naked_literal.copy()
return e.setParseAction(lambda x: cls(*x))
def spec(self):
return strutils.bytes_to_escaped_str(self.val, escape_single_quotes=True)
class TokValueGenerate(Token):
def __init__(self, usize, unit, datatype):
if not unit:
unit = "b"
self.usize, self.unit, self.datatype = usize, unit, datatype
def bytes(self):
return self.usize * human.SIZE_UNITS[self.unit]
def get_generator(self, settings_):
return generators.RandomGenerator(self.datatype, self.bytes())
def freeze(self, settings):
g = self.get_generator(settings)
return TokValueLiteral(strutils.bytes_to_escaped_str(g[:], escape_single_quotes=True))
@classmethod
def expr(cls):
e = pp.Literal("@").suppress() + v_integer
u = functools.reduce(
operator.or_,
[pp.Literal(i) for i in human.SIZE_UNITS.keys()]
).leaveWhitespace()
e = e + pp.Optional(u, default=None)
s = pp.Literal(",").suppress()
s += functools.reduce(
operator.or_,
[pp.Literal(i) for i in generators.DATATYPES.keys()]
)
e += pp.Optional(s, default="bytes")
return e.setParseAction(lambda x: cls(*x))
def spec(self):
s = "@%s" % self.usize
if self.unit != "b":
s += self.unit
if self.datatype != "bytes":
s += ",%s" % self.datatype
return s
class TokValueFile(Token):
def __init__(self, path):
self.path = str(path)
@classmethod
def expr(cls):
e = pp.Literal("<").suppress()
e = e + v_naked_literal
return e.setParseAction(lambda x: cls(*x))
def freeze(self, settings_):
return self
def get_generator(self, settings):
if not settings.staticdir:
raise exceptions.FileAccessDenied("File access disabled.")
s = os.path.expanduser(self.path)
s = os.path.normpath(
os.path.abspath(os.path.join(settings.staticdir, s))
)
uf = settings.unconstrained_file_access
if not uf and not s.startswith(os.path.normpath(settings.staticdir)):
raise exceptions.FileAccessDenied(
"File access outside of configured directory"
)
if not os.path.isfile(s):
raise exceptions.FileAccessDenied("File not readable")
return generators.FileGenerator(s)
def spec(self):
return "<'%s'" % self.path
TokValue = pp.MatchFirst(
[
TokValueGenerate.expr(),
TokValueFile.expr(),
TokValueLiteral.expr()
]
)
TokNakedValue = pp.MatchFirst(
[
TokValueGenerate.expr(),
TokValueFile.expr(),
TokValueLiteral.expr(),
TokValueNakedLiteral.expr(),
]
)
TokOffset = pp.MatchFirst(
[
v_integer,
pp.Literal("r"),
pp.Literal("a")
]
)
class _Component(Token):
"""
A value component of the primary specification of an message.
Components produce byte values describing the bytes of the message.
"""
def values(self, settings): # pragma: no cover
"""
A sequence of values, which can either be strings or generators.
"""
pass
def string(self, settings=None):
"""
A bytestring representation of the object.
"""
return b"".join(i[:] for i in self.values(settings or {}))
class KeyValue(_Component):
"""
A key/value pair.
cls.preamble: leader
"""
def __init__(self, key, value):
self.key, self.value = key, value
@classmethod
def expr(cls):
e = pp.Literal(cls.preamble).suppress()
e += TokValue
e += pp.Literal("=").suppress()
e += TokValue
return e.setParseAction(lambda x: cls(*x))
def spec(self):
return f"{self.preamble}{self.key.spec()}={self.value.spec()}"
def freeze(self, settings):
return self.__class__(
self.key.freeze(settings), self.value.freeze(settings)
)
class CaselessLiteral(_Component):
"""
A caseless token that can take only one value.
"""
def __init__(self, value):
self.value = value
@classmethod
def expr(cls):
spec = pp.CaselessLiteral(cls.TOK)
spec = spec.setParseAction(lambda x: cls(*x))
return spec
def values(self, settings):
return self.TOK
def spec(self):
return self.TOK
def freeze(self, settings_):
return self
class OptionsOrValue(_Component):
"""
Can be any of a specified set of options, or a value specifier.
"""
preamble = ""
options: typing.List[str] = []
def __init__(self, value):
# If it's a string, we were passed one of the options, so we lower-case
# it to be canonical. The user can specify a different case by using a
# string value literal.
self.option_used = False
if isinstance(value, str):
for i in self.options:
# Find the exact option value in a case-insensitive way
if i.lower() == value.lower():
self.option_used = True
value = TokValueLiteral(i)
break
self.value = value
@classmethod
def expr(cls):
parts = [pp.CaselessLiteral(i) for i in cls.options]
m = pp.MatchFirst(parts)
spec = m | TokValue.copy()
spec = spec.setParseAction(lambda x: cls(*x))
if cls.preamble:
spec = pp.Literal(cls.preamble).suppress() + spec
return spec
def values(self, settings):
return [
self.value.get_generator(settings)
]
def spec(self):
s = self.value.spec()
if s[1:-1].lower() in self.options:
s = s[1:-1].lower()
return f"{self.preamble}{s}"
def freeze(self, settings):
return self.__class__(self.value.freeze(settings))
class Integer(_Component):
bounds: typing.Tuple[typing.Optional[int], typing.Optional[int]] = (None, None)
preamble = ""
def __init__(self, value):
v = int(value)
outofbounds = any([
self.bounds[0] is not None and v < self.bounds[0],
self.bounds[1] is not None and v > self.bounds[1]
])
if outofbounds:
raise exceptions.ParseException(
"Integer value must be between %s and %s." % self.bounds,
0, 0
)
self.value = str(value).encode()
@classmethod
def expr(cls):
e = v_integer.copy()
if cls.preamble:
e = pp.Literal(cls.preamble).suppress() + e
return e.setParseAction(lambda x: cls(*x))
def values(self, settings):
return [self.value]
def spec(self):
return f"{self.preamble}{self.value.decode()}"
def freeze(self, settings_):
return self
class Value(_Component):
"""
A value component lead by an optional preamble.
"""
preamble = ""
def __init__(self, value):
self.value = value
@classmethod
def expr(cls):
e = (TokValue | TokNakedValue)
if cls.preamble:
e = pp.Literal(cls.preamble).suppress() + e
return e.setParseAction(lambda x: cls(*x))
def values(self, settings):
return [self.value.get_generator(settings)]
def spec(self):
return f"{self.preamble}{self.value.spec()}"
def freeze(self, settings):
return self.__class__(self.value.freeze(settings))
class FixedLengthValue(Value):
"""
A value component lead by an optional preamble.
"""
preamble = ""
length: typing.Optional[int] = None
def __init__(self, value):
Value.__init__(self, value)
lenguess = None
try:
lenguess = len(value.get_generator(Settings()))
except exceptions.RenderError:
pass
# This check will fail if we know the length upfront
if lenguess is not None and lenguess != self.length:
raise exceptions.RenderError(
"Invalid value length: '{}' is {} bytes, should be {}.".format(
self.spec(),
lenguess,
self.length
)
)
def values(self, settings):
ret = Value.values(self, settings)
l = sum(len(i) for i in ret)
# This check will fail if we don't know the length upfront - i.e. for
# file inputs
if l != self.length:
raise exceptions.RenderError(
"Invalid value length: '{}' is {} bytes, should be {}.".format(
self.spec(),
l,
self.length
)
)
return ret
class Boolean(_Component):
"""
A boolean flag.
name = true
-name = false
"""
name = ""
def __init__(self, value):
self.value = value
@classmethod
def expr(cls):
e = pp.Optional(pp.Literal("-"), default=True)
e += pp.Literal(cls.name).suppress()
def parse(s_, loc_, toks):
val = True
if toks[0] == "-":
val = False
return cls(val)
return e.setParseAction(parse)
def spec(self):
return "{}{}".format("-" if not self.value else "", self.name)
class IntField(_Component):
"""
An integer field, where values can optionally specified by name.
"""
names: typing.Dict[str, int] = {}
max = 16
preamble = ""
def __init__(self, value):
self.origvalue = value
self.value = self.names.get(value, value)
if self.value > self.max:
raise exceptions.ParseException(
"Value can't exceed %s" % self.max, 0, 0
)
@classmethod
def expr(cls):
parts = [pp.CaselessLiteral(i) for i in cls.names.keys()]
m = pp.MatchFirst(parts)
spec = m | v_integer.copy()
spec = spec.setParseAction(lambda x: cls(*x))
if cls.preamble:
spec = pp.Literal(cls.preamble).suppress() + spec
return spec
def values(self, settings):
return [str(self.value)]
def spec(self):
return f"{self.preamble}{self.origvalue}"

21
pathod/language/exceptions.py
View File

@ -1,21 +0,0 @@
class RenderError(Exception):
pass
class FileAccessDenied(RenderError):
pass
class ParseException(Exception):
def __init__(self, msg, s, col):
Exception.__init__(self)
self.msg = msg
self.s = s
self.col = col
def marked(self):
return "{}\n{}".format(self.s, " " * (self.col - 1) + "^")
def __str__(self):
return f"{self.msg} at char {self.col}"

93
pathod/language/generators.py
View File

@ -1,93 +0,0 @@
import os
import string
import random
import mmap
import sys
DATATYPES = dict(
ascii_letters=string.ascii_letters.encode(),
ascii_lowercase=string.ascii_lowercase.encode(),
ascii_uppercase=string.ascii_uppercase.encode(),
digits=string.digits.encode(),
hexdigits=string.hexdigits.encode(),
octdigits=string.octdigits.encode(),
punctuation=string.punctuation.encode(),
whitespace=string.whitespace.encode(),
ascii=string.printable.encode(),
bytes=bytes(range(256))
)
class TransformGenerator:
"""
Perform a byte-by-byte transform another generator - that is, for each
input byte, the transformation must produce one output byte.
gen: A generator to wrap
transform: A function (offset, data) -> transformed
"""
def __init__(self, gen, transform):
self.gen = gen
self.transform = transform
def __len__(self):
return len(self.gen)
def __getitem__(self, x):
d = self.gen.__getitem__(x)
if isinstance(x, slice):
return self.transform(x.start, d)
return self.transform(x, d)
def __repr__(self):
return "'transform(%s)'" % self.gen
def rand_byte(chars):
"""
Return a random character as byte from a charset.
"""
# bytearray has consistent behaviour on both Python 2 and 3
# while bytes does not
return bytes([random.choice(chars)])
class RandomGenerator:
def __init__(self, dtype, length):
self.dtype = dtype
self.length = length
def __len__(self):
return self.length
def __getitem__(self, x):
chars = DATATYPES[self.dtype]
if isinstance(x, slice):
return b"".join(rand_byte(chars) for _ in range(*x.indices(min(self.length, sys.maxsize))))
return rand_byte(chars)
def __repr__(self):
return f"{self.length} random from {self.dtype}"
class FileGenerator:
def __init__(self, path):
self.path = os.path.expanduser(path)
def __len__(self):
return os.path.getsize(self.path)
def __getitem__(self, x):
with open(self.path, mode="rb") as f:
if isinstance(x, slice):
with mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ) as mapped:
return mapped.__getitem__(x)
else:
f.seek(x)
return f.read(1)
def __repr__(self):
return "<%s" % self.path

393
pathod/language/http.py
View File

@ -1,393 +0,0 @@
import abc
import pyparsing as pp
from mitmproxy.net import websocket
from mitmproxy.net.http import status_codes, url, user_agents
from . import base, exceptions, actions, message
# TODO: use mitmproxy.net.semantics.protocol assemble method,
# instead of duplicating the HTTP on-the-wire representation here.
# see http2 language for an example
class WS(base.CaselessLiteral):
TOK = "ws"
class Raw(base.CaselessLiteral):
TOK = "r"
class Path(base.Value):
pass
class StatusCode(base.Integer):
pass
class Reason(base.Value):
preamble = "m"
class Body(base.Value):
preamble = "b"
class Times(base.Integer):
preamble = "x"
class Method(base.OptionsOrValue):
options = [
"GET",
"HEAD",
"POST",
"PUT",
"DELETE",
"OPTIONS",
"TRACE",
"CONNECT",
]
class _HeaderMixin:
@property
def unique_name(self):
return None
def format_header(self, key, value):
return [key, b": ", value, b"\r\n"]
def values(self, settings):
return self.format_header(
self.key.get_generator(settings),
self.value.get_generator(settings),
)
class Header(_HeaderMixin, base.KeyValue):
preamble = "h"
class ShortcutContentType(_HeaderMixin, base.Value):
preamble = "c"
key = base.TokValueLiteral("Content-Type")
class ShortcutLocation(_HeaderMixin, base.Value):
preamble = "l"
key = base.TokValueLiteral("Location")
class ShortcutUserAgent(_HeaderMixin, base.OptionsOrValue):
preamble = "u"
options = [i[1] for i in user_agents.UASTRINGS]
key = base.TokValueLiteral("User-Agent")
def values(self, settings):
value = self.value.val
if self.option_used:
value = user_agents.get_by_shortcut(value.lower().decode())[2].encode()
return self.format_header(
self.key.get_generator(settings),
value
)
def get_header(val, headers):
"""
Header keys may be Values, so we have to "generate" them as we try the
match.
"""
for h in headers:
k = h.key.get_generator({})
if len(k) == len(val) and k[:].lower() == val.lower():
return h
return None
class _HTTPMessage(message.Message):
version = b"HTTP/1.1"
@property
def actions(self):
return self.toks(actions._Action)
@property
def raw(self):
return bool(self.tok(Raw))
@property
def body(self):
return self.tok(Body)
@abc.abstractmethod
def preamble(self, settings): # pragma: no cover
pass
@property
def headers(self):
return self.toks(_HeaderMixin)
def values(self, settings):
vals = self.preamble(settings)
vals.append(b"\r\n")
for h in self.headers:
vals.extend(h.values(settings))
vals.append(b"\r\n")
if self.body:
vals.extend(self.body.values(settings))
return vals
class Response(_HTTPMessage):
unique_name = None # type: ignore
comps = (
Header,
ShortcutContentType,
ShortcutLocation,
Raw,
Reason,
Body,
actions.PauseAt,
actions.DisconnectAt,
actions.InjectAt,
)
logattrs = ["status_code", "reason", "version", "body"]
@property
def ws(self):
return self.tok(WS)
@property
def status_code(self):
return self.tok(StatusCode)
@property
def reason(self):
return self.tok(Reason)
def preamble(self, settings):
l = [self.version, b" "]
l.extend(self.status_code.values(settings))
status_code = int(self.status_code.value)
l.append(b" ")
if self.reason:
l.extend(self.reason.values(settings))
else:
l.append(
status_codes.RESPONSES.get(
status_code,
"Unknown code"
).encode()
)
return l
def resolve(self, settings, msg=None):
tokens = self.tokens[:]
if self.ws:
if not settings.websocket_key:
raise exceptions.RenderError(
"No websocket key - have we seen a client handshake?"
)
if not self.status_code:
tokens.insert(
1,
StatusCode(101)
)
headers = websocket.server_handshake_headers(
settings.websocket_key
)
for i in headers.fields:
if not get_header(i[0], self.headers):
tokens.append(
Header(
base.TokValueLiteral(i[0].decode()),
base.TokValueLiteral(i[1].decode()))
)
if not self.raw:
if not get_header(b"Content-Length", self.headers):
if not self.body:
length = 0
else:
length = sum(
len(i) for i in self.body.values(settings)
)
tokens.append(
Header(
base.TokValueLiteral("Content-Length"),
base.TokValueLiteral(str(length)),
)
)
intermediate = self.__class__(tokens)
return self.__class__(
[i.resolve(settings, intermediate) for i in tokens]
)
@classmethod
def expr(cls):
parts = [i.expr() for i in cls.comps]
atom = pp.MatchFirst(parts)
resp = pp.And(
[
pp.MatchFirst(
[
WS.expr() + pp.Optional(
base.Sep + StatusCode.expr()
),
StatusCode.expr(),
]
),
pp.ZeroOrMore(base.Sep + atom)
]
)
resp = resp.setParseAction(cls)
return resp
def spec(self):
return ":".join([i.spec() for i in self.tokens])
class NestedResponse(message.NestedMessage):
preamble = "s"
nest_type = Response
class Request(_HTTPMessage):
comps = (
Header,
ShortcutContentType,
ShortcutUserAgent,
Raw,
NestedResponse,
Body,
Times,
actions.PauseAt,
actions.DisconnectAt,
actions.InjectAt,
)
logattrs = ["method", "path", "body"]
@property
def ws(self):
return self.tok(WS)
@property
def method(self):
return self.tok(Method)
@property
def path(self):
return self.tok(Path)
@property
def times(self):
return self.tok(Times)
@property
def nested_response(self):
return self.tok(NestedResponse)
def preamble(self, settings):
v = self.method.values(settings)
v.append(b" ")
v.extend(self.path.values(settings))
if self.nested_response:
v.append(self.nested_response.parsed.spec())
v.append(b" ")
v.append(self.version)
return v
def resolve(self, settings, msg=None):
tokens = self.tokens[:]
if self.ws:
if not self.method:
tokens.insert(
1,
Method("get")
)
for i in websocket.client_handshake_headers().fields:
if not get_header(i[0], self.headers):
tokens.append(
Header(
base.TokValueLiteral(i[0].decode()),
base.TokValueLiteral(i[1].decode())
)
)
if not self.raw:
if not get_header(b"Content-Length", self.headers):
if self.body:
length = sum(
len(i) for i in self.body.values(settings)
)
tokens.append(
Header(
base.TokValueLiteral("Content-Length"),
base.TokValueLiteral(str(length)),
)
)
if settings.request_host:
if not get_header(b"Host", self.headers):
h = settings.request_host
if self.path:
path = b"".join(self.path.values({})).decode(
"ascii", errors="ignore"
)
try:
_, h, _, _ = url.parse(path)
h = h.decode("ascii", errors="ignore")
except ValueError:
pass
tokens.append(
Header(
base.TokValueLiteral("Host"),
base.TokValueLiteral(h)
)
)
intermediate = self.__class__(tokens)
return self.__class__(
[i.resolve(settings, intermediate) for i in tokens]
)
@classmethod
def expr(cls):
parts = [i.expr() for i in cls.comps]
atom = pp.MatchFirst(parts)
resp = pp.And(
[
pp.MatchFirst(
[
WS.expr() + pp.Optional(
base.Sep + Method.expr()
),
Method.expr(),
]
),
base.Sep,
Path.expr(),
pp.ZeroOrMore(base.Sep + atom)
]
)
resp = resp.setParseAction(cls)
return resp
def spec(self):
return ":".join([i.spec() for i in self.tokens])
def make_error_response(reason, body=None):
tokens = [
StatusCode("800"),
Header(
base.TokValueLiteral("Content-Type"),
base.TokValueLiteral("text/plain")
),
Reason(base.TokValueLiteral(reason)),
Body(base.TokValueLiteral("pathod error: " + (body or reason))),
]
return Response(tokens)

306
pathod/language/http2.py
View File

@ -1,306 +0,0 @@
import pyparsing as pp
from mitmproxy.net import http
from mitmproxy.net.http import user_agents, Headers
from . import base, message
"""
Normal HTTP requests:
<method>:<path>:<header>:<body>
e.g.:
GET:/
GET:/:h"foo"="bar"
POST:/:h"foo"="bar":b'content body payload'
Normal HTTP responses:
<code>:<header>:<body>
e.g.:
200
302:h"foo"="bar"
404:h"foo"="bar":b'content body payload'
Individual HTTP/2 frames:
h2f:<payload_length>:<type>:<flags>:<stream_id>:<payload>
e.g.:
h2f:0:PING
h2f:42:HEADERS:END_HEADERS:0x1234567:foo=bar,host=example.com
h2f:42:DATA:END_STREAM,PADDED:0x1234567:'content body payload'
"""
def get_header(val, headers):
"""
Header keys may be Values, so we have to "generate" them as we try the
match.
"""
for h in headers:
k = h.key.get_generator({})
if len(k) == len(val) and k[:].lower() == val.lower():
return h
return None
class _HeaderMixin:
@property
def unique_name(self):
return None
def values(self, settings):
return (
self.key.get_generator(settings),
self.value.get_generator(settings),
)
class _HTTP2Message(message.Message):
@property
def actions(self):
return [] # self.toks(actions._Action)
@property
def headers(self):
headers = self.toks(_HeaderMixin)
if not self.raw:
if not get_header(b"content-length", headers):
if not self.body:
length = 0
else:
length = len(self.body.string())
headers.append(
Header(
base.TokValueLiteral("content-length"),
base.TokValueLiteral(str(length)),
)
)
return headers
@property
def raw(self):
return bool(self.tok(Raw))
@property
def body(self):
return self.tok(Body)
def resolve(self, settings):
return self
class StatusCode(base.Integer):
pass
class Method(base.OptionsOrValue):
options = [
"GET",
"HEAD",
"POST",
"PUT",
"DELETE",
]
class Path(base.Value):
pass
class Header(_HeaderMixin, base.KeyValue):
preamble = "h"
class ShortcutContentType(_HeaderMixin, base.Value):
preamble = "c"
key = base.TokValueLiteral("content-type")
class ShortcutLocation(_HeaderMixin, base.Value):
preamble = "l"
key = base.TokValueLiteral("location")
class ShortcutUserAgent(_HeaderMixin, base.OptionsOrValue):
preamble = "u"
options = [i[1] for i in user_agents.UASTRINGS]
key = base.TokValueLiteral("user-agent")
def values(self, settings):
value = self.value.val
if self.option_used:
value = user_agents.get_by_shortcut(value.lower().decode())[2].encode()
return (
self.key.get_generator(settings),
value
)
class Raw(base.CaselessLiteral):
TOK = "r"
class Body(base.Value):
preamble = "b"
class Times(base.Integer):
preamble = "x"
class Response(_HTTP2Message):
unique_name = None
comps = (
Header,
Body,
ShortcutContentType,
ShortcutLocation,
Raw,
)
def __init__(self, tokens):
super().__init__(tokens)
self.rendered_values = None
self.stream_id = 2
@property
def status_code(self):
return self.tok(StatusCode)
@classmethod
def expr(cls):
parts = [i.expr() for i in cls.comps]
atom = pp.MatchFirst(parts)
resp = pp.And(
[
StatusCode.expr(),
pp.ZeroOrMore(base.Sep + atom)
]
)
resp = resp.setParseAction(cls)
return resp
def values(self, settings):
if self.rendered_values:
return self.rendered_values
else:
headers = Headers([header.values(settings) for header in self.headers])
body = self.body
if body:
body = body.string()
resp = http.Response(
http_version=b'HTTP/2.0',
status_code=int(self.status_code.string()),
reason=b'',
headers=headers,
content=body,
trailers=None,
timestamp_start=0,
timestamp_end=0
)
resp.stream_id = self.stream_id
self.rendered_values = settings.protocol.assemble(resp)
return self.rendered_values
def spec(self):
return ":".join([i.spec() for i in self.tokens])
class NestedResponse(message.NestedMessage):
preamble = "s"
nest_type = Response
class Request(_HTTP2Message):
comps = (
Header,
ShortcutContentType,
ShortcutUserAgent,
Raw,
NestedResponse,
Body,
Times,
)
logattrs = ["method", "path"]
def __init__(self, tokens):
super().__init__(tokens)
self.rendered_values = None
self.stream_id = 1
@property
def method(self):
return self.tok(Method)
@property
def path(self):
return self.tok(Path)
@property
def nested_response(self):
return self.tok(NestedResponse)
@property
def times(self):
return self.tok(Times)
@classmethod
def expr(cls):
parts = [i.expr() for i in cls.comps]
atom = pp.MatchFirst(parts)
resp = pp.And(
[
Method.expr(),
base.Sep,
Path.expr(),
pp.ZeroOrMore(base.Sep + atom)
]
)
resp = resp.setParseAction(cls)
return resp
def values(self, settings):
if self.rendered_values:
return self.rendered_values
else:
path = self.path.string()
if self.nested_response:
path += self.nested_response.parsed.spec().encode()
headers = Headers([header.values(settings) for header in self.headers])
body = self.body
if body:
body = body.string()
req = http.Request(
"",
0,
self.method.string(),
b'http',
b'',
path,
b"HTTP/2.0",
headers,
body,
None,
0,
0,
)
req.stream_id = self.stream_id
self.rendered_values = settings.protocol.assemble(req)
return self.rendered_values
def spec(self):
return ":".join([i.spec() for i in self.tokens])
def make_error_response(reason, body=None):
tokens = [
StatusCode("800"),
Body(base.TokValueLiteral("pathod error: " + (body or reason))),
]
return Response(tokens)

144
pathod/language/message.py
View File

@ -1,144 +0,0 @@
import abc
import typing # noqa
import pyparsing as pp
from mitmproxy.utils import strutils
from . import actions, exceptions, base
LOG_TRUNCATE = 1024
class Message:
__metaclass__ = abc.ABCMeta
logattrs: typing.List[str] = []
def __init__(self, tokens):
track = set()
for i in tokens:
if i.unique_name:
if i.unique_name in track:
raise exceptions.ParseException(
"Message has multiple %s clauses, "
"but should only have one." % i.unique_name,
0, 0
)
else:
track.add(i.unique_name)
self.tokens = tokens
def strike_token(self, name):
toks = [i for i in self.tokens if i.unique_name != name]
return self.__class__(toks)
def toks(self, klass):
"""
Fetch all tokens that are instances of klass
"""
return [i for i in self.tokens if isinstance(i, klass)]
def tok(self, klass):
"""
Fetch first token that is an instance of klass
"""
l = self.toks(klass)
if l:
return l[0]
def length(self, settings):
"""
Calculate the length of the base message without any applied
actions.
"""
return sum(len(x) for x in self.values(settings))
def preview_safe(self):
"""
Return a copy of this message that is safe for previews.
"""
tokens = [i for i in self.tokens if not isinstance(i, actions.PauseAt)]
return self.__class__(tokens)
def maximum_length(self, settings):
"""
Calculate the maximum length of the base message with all applied
actions.
"""
l = self.length(settings)
for i in self.actions:
if isinstance(i, actions.InjectAt):
l += len(i.value.get_generator(settings))
return l
@classmethod
def expr(cls): # pragma: no cover
pass
def log(self, settings):
"""
A dictionary that should be logged if this message is served.
"""
ret = {}
for i in self.logattrs:
v = getattr(self, i)
# Careful not to log any VALUE specs without sanitizing them first.
# We truncate at 1k.
if hasattr(v, "values"):
v = [x[:LOG_TRUNCATE] for x in v.values(settings)]
v = strutils.bytes_to_escaped_str(b"".join(v))
elif hasattr(v, "__len__"):
v = v[:LOG_TRUNCATE]
v = strutils.bytes_to_escaped_str(v)
ret[i] = v
ret["spec"] = self.spec()
return ret
def freeze(self, settings):
r = self.resolve(settings)
return self.__class__([i.freeze(settings) for i in r.tokens])
def __repr__(self):
return self.spec()
class NestedMessage(base.Token):
"""
A nested message, as an escaped string with a preamble.
"""
preamble = ""
nest_type: typing.Optional[typing.Type[Message]] = None
def __init__(self, value):
super().__init__()
self.value = value
try:
self.parsed = self.nest_type(
self.nest_type.expr().parseString(
value.val.decode(),
parseAll=True
)
)
except pp.ParseException as v:
raise exceptions.ParseException(v.msg, v.line, v.col)
@classmethod
def expr(cls):
e = pp.Literal(cls.preamble).suppress()
e = e + base.TokValueLiteral.expr()
return e.setParseAction(lambda x: cls(*x))
def values(self, settings):
return [
self.value.get_generator(settings),
]
def spec(self):
return f"{self.preamble}{self.value.spec()}"
def freeze(self, settings):
f = self.parsed.freeze(settings).spec()
return self.__class__(
base.TokValueLiteral(
strutils.bytes_to_escaped_str(f.encode(), escape_single_quotes=True)
)
)

255
pathod/language/websockets.py
View File

@ -1,255 +0,0 @@
import random
import string
import typing # noqa
import pyparsing as pp
from wsproto.frame_protocol import Opcode
from mitmproxy.utils import strutils
from . import base, generators, actions, message, websockets_frame
NESTED_LEADER = b"pathod!"
class WF(base.CaselessLiteral):
TOK = "wf"
class OpCode(base.IntField):
names: typing.Dict[str, int] = {
"continue": Opcode.CONTINUATION,
"text": Opcode.TEXT,
"binary": Opcode.BINARY,
"close": Opcode.CLOSE,
"ping": Opcode.PING,
"pong": Opcode.PONG,
}
max = 15
preamble = "c"
class Body(base.Value):
preamble = "b"
class RawBody(base.Value):
unique_name = "body"
preamble = "r"
class Fin(base.Boolean):
name = "fin"
class RSV1(base.Boolean):
name = "rsv1"
class RSV2(base.Boolean):
name = "rsv2"
class RSV3(base.Boolean):
name = "rsv3"
class Mask(base.Boolean):
name = "mask"
class Key(base.FixedLengthValue):
preamble = "k"
length = 4
class KeyNone(base.CaselessLiteral):
unique_name = "key"
TOK = "knone"
class Length(base.Integer):
bounds = (0, 1 << 64)
preamble = "l"
class Times(base.Integer):
preamble = "x"
COMPONENTS = [
OpCode,
Length,
# Bit flags
Fin,
RSV1,
RSV2,
RSV3,
Mask,
actions.PauseAt,
actions.DisconnectAt,
actions.InjectAt,
KeyNone,
Key,
Times,
Body,
RawBody,
]
class WebsocketFrame(message.Message):
components: typing.List[typing.Type[base._Component]] = COMPONENTS
logattrs = ["body"]
# Used for nested frames
unique_name = "body"
@property
def actions(self):
return self.toks(actions._Action)
@property
def body(self):
return self.tok(Body)
@property
def rawbody(self):
return self.tok(RawBody)
@property
def opcode(self):
return self.tok(OpCode)
@property
def fin(self):
return self.tok(Fin)
@property
def rsv1(self):
return self.tok(RSV1)
@property
def rsv2(self):
return self.tok(RSV2)
@property
def rsv3(self):
return self.tok(RSV3)
@property
def mask(self):
return self.tok(Mask)
@property
def key(self):
return self.tok(Key)
@property
def knone(self):
return self.tok(KeyNone)
@property
def times(self):
return self.tok(Times)
@property
def toklength(self):
return self.tok(Length)
@classmethod
def expr(cls):
parts = [i.expr() for i in cls.components]
atom = pp.MatchFirst(parts)
resp = pp.And(
[
WF.expr(),
base.Sep,
pp.ZeroOrMore(base.Sep + atom)
]
)
resp = resp.setParseAction(cls)
return resp
@property
def nested_frame(self):
return self.tok(NestedFrame)
def resolve(self, settings, msg=None):
tokens = self.tokens[:]
if not self.mask and settings.is_client:
tokens.append(
Mask(True)
)
if not self.knone and self.mask and self.mask.value and not self.key:
allowed_chars = string.ascii_letters + string.digits
k = ''.join([allowed_chars[random.randrange(0, len(allowed_chars))] for i in range(4)])
tokens.append(
Key(base.TokValueLiteral(k))
)
return self.__class__(
[i.resolve(settings, self) for i in tokens]
)
def values(self, settings):
if self.body:
bodygen = self.body.value.get_generator(settings)
length = len(self.body.value.get_generator(settings))
elif self.rawbody:
bodygen = self.rawbody.value.get_generator(settings)
length = len(self.rawbody.value.get_generator(settings))
elif self.nested_frame:
bodygen = NESTED_LEADER + strutils.always_bytes(self.nested_frame.parsed.spec())
length = len(bodygen)
else:
bodygen = None
length = 0
if self.toklength:
length = int(self.toklength.value)
frameparts = dict(
payload_length=length
)
if self.mask and self.mask.value:
frameparts["mask"] = True
if self.knone:
frameparts["masking_key"] = None
elif self.key:
key = self.key.values(settings)[0][:]
frameparts["masking_key"] = key
for i in ["opcode", "fin", "rsv1", "rsv2", "rsv3", "mask"]:
v = getattr(self, i, None)
if v is not None:
frameparts[i] = v.value
# import wsproto.frame_protocol
# wsproto.frame_protocol.Frame(
# opcode=frameparts["opcode"],
# payload=None,
# frame_finished=frameparts["fin"]
# )
frame = websockets_frame.FrameHeader(**frameparts)
vals = [bytes(frame)]
if bodygen:
if frame.masking_key and not self.rawbody:
masker = websockets_frame.Masker(frame.masking_key)
vals.append(
generators.TransformGenerator(
bodygen,
masker.mask
)
)
else:
vals.append(bodygen)
return vals
def spec(self):
return ":".join([i.spec() for i in self.tokens])
class NestedFrame(message.NestedMessage):
preamble = "f"
nest_type = WebsocketFrame
class WebsocketClientFrame(WebsocketFrame):
components = COMPONENTS + [NestedFrame]

274
pathod/language/websockets_frame.py
View File

@ -1,274 +0,0 @@
import sys
import os
import struct
import io
from wsproto.frame_protocol import Opcode
from mitmproxy.net import tcp
from mitmproxy.utils import bits, human, strutils
MAX_16_BIT_INT = (1 << 16)
MAX_64_BIT_INT = (1 << 64)
DEFAULT = object()
class Masker:
"""
Data sent from the server must be masked to prevent malicious clients
from sending data over the wire in predictable patterns.
Servers do not have to mask data they send to the client.
https://tools.ietf.org/html/rfc6455#section-5.3
"""
def __init__(self, key):
self.key = key
self.offset = 0
def mask(self, offset, data):
datalen = len(data)
offset_mod = offset % 4
data = int.from_bytes(data, sys.byteorder)
num_keys = (datalen + offset_mod + 3) // 4
mask = int.from_bytes((self.key * num_keys)[offset_mod:datalen +
offset_mod], sys.byteorder)
return (data ^ mask).to_bytes(datalen, sys.byteorder)
def __call__(self, data):
ret = self.mask(self.offset, data)
self.offset += len(ret)
return ret
class FrameHeader:
def __init__(
self,
opcode=Opcode.TEXT,
payload_length=0,
fin=False,
rsv1=False,
rsv2=False,
rsv3=False,
masking_key=DEFAULT,
mask=DEFAULT,
length_code=DEFAULT
):
if not 0 <= opcode < 2 ** 4:
raise ValueError("opcode must be 0-16")
self.opcode = opcode
self.payload_length = payload_length
self.fin = fin
self.rsv1 = rsv1
self.rsv2 = rsv2
self.rsv3 = rsv3
if length_code is DEFAULT:
self.length_code = self._make_length_code(self.payload_length)
else:
self.length_code = length_code
if (mask is DEFAULT and masking_key is DEFAULT) or mask == 0 or mask is False:
self.mask = False
self.masking_key = b""
elif mask is DEFAULT:
self.mask = 1
self.masking_key = masking_key
elif masking_key is DEFAULT:
self.mask = mask
self.masking_key = os.urandom(4)
else:
self.mask = mask
self.masking_key = masking_key
if self.masking_key and len(self.masking_key) != 4:
raise ValueError("Masking key must be 4 bytes.")
@classmethod
def _make_length_code(self, length):
"""
A WebSocket frame contains an initial length_code, and an optional
extended length code to represent the actual length if length code is
larger than 125
"""
if length <= 125:
return length
elif length >= 126 and length <= 65535:
return 126
else:
return 127
def __repr__(self):
vals = [
"ws frame:",
Opcode(self.opcode).name.lower()
]
flags = []
for i in ["fin", "rsv1", "rsv2", "rsv3", "mask"]:
if getattr(self, i):
flags.append(i)
if flags:
vals.extend([":", "|".join(flags)])
if self.masking_key:
vals.append(":key=%s" % repr(self.masking_key))
if self.payload_length:
vals.append(" %s" % human.pretty_size(self.payload_length))
return "".join(vals)
def __bytes__(self):
first_byte = bits.setbit(0, 7, self.fin)
first_byte = bits.setbit(first_byte, 6, self.rsv1)
first_byte = bits.setbit(first_byte, 5, self.rsv2)
first_byte = bits.setbit(first_byte, 4, self.rsv3)
first_byte = first_byte | self.opcode
second_byte = bits.setbit(self.length_code, 7, self.mask)
b = bytes([first_byte, second_byte])
if self.payload_length < 126:
pass
elif self.payload_length < MAX_16_BIT_INT:
# '!H' pack as 16 bit unsigned short
# add 2 byte extended payload length
b += struct.pack('!H', self.payload_length)
elif self.payload_length < MAX_64_BIT_INT:
# '!Q' = pack as 64 bit unsigned long long
# add 8 bytes extended payload length
b += struct.pack('!Q', self.payload_length)
else:
raise ValueError("Payload length exceeds 64bit integer")
if self.masking_key:
b += self.masking_key
return b
@classmethod
def from_file(cls, fp):
"""
read a WebSocket frame header
"""
first_byte, second_byte = fp.safe_read(2)
fin = bits.getbit(first_byte, 7)
rsv1 = bits.getbit(first_byte, 6)
rsv2 = bits.getbit(first_byte, 5)
rsv3 = bits.getbit(first_byte, 4)
opcode = first_byte & 0xF
mask_bit = bits.getbit(second_byte, 7)
length_code = second_byte & 0x7F
# payload_length > 125 indicates you need to read more bytes
# to get the actual payload length
if length_code <= 125:
payload_length = length_code
elif length_code == 126:
payload_length, = struct.unpack("!H", fp.safe_read(2))
else: # length_code == 127:
payload_length, = struct.unpack("!Q", fp.safe_read(8))
# masking key only present if mask bit set
if mask_bit == 1:
masking_key = fp.safe_read(4)
else:
masking_key = None
return cls(
fin=fin,
rsv1=rsv1,
rsv2=rsv2,
rsv3=rsv3,
opcode=opcode,
mask=mask_bit,
length_code=length_code,
payload_length=payload_length,
masking_key=masking_key,
)
def __eq__(self, other):
if isinstance(other, FrameHeader):
return bytes(self) == bytes(other)
return False
class Frame:
"""
Represents a single WebSocket frame.
Constructor takes human readable forms of the frame components.
from_bytes() reads from a file-like object to create a new Frame.
WebSocket frame as defined in RFC6455
0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+-+-+-+-+-------+-+-------------+-------------------------------+
|F|R|R|R| opcode|M| Payload len | Extended payload length |
|I|S|S|S| (4) |A| (7) | (16/64) |
|N|V|V|V| |S| | (if payload len==126/127) |
| |1|2|3| |K| | |
+-+-+-+-+-------+-+-------------+ - - - - - - - - - - - - - - - +
| Extended payload length continued, if payload len == 127 |
+ - - - - - - - - - - - - - - - +-------------------------------+
| |Masking-key, if MASK set to 1 |
+-------------------------------+-------------------------------+
| Masking-key (continued) | Payload Data |
+-------------------------------- - - - - - - - - - - - - - - - +
: Payload Data continued ... :
+ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +
| Payload Data continued ... |
+---------------------------------------------------------------+
"""
def __init__(self, payload=b"", **kwargs):
self.payload = payload
kwargs["payload_length"] = kwargs.get("payload_length", len(payload))
self.header = FrameHeader(**kwargs)
@classmethod
def from_bytes(cls, bytestring):
"""
Construct a websocket frame from an in-memory bytestring
to construct a frame from a stream of bytes, use from_file() directly
"""
return cls.from_file(tcp.Reader(io.BytesIO(bytestring)))
def __repr__(self):
ret = repr(self.header)
if self.payload:
ret = ret + "\nPayload:\n" + strutils.bytes_to_escaped_str(self.payload)
return ret
def __bytes__(self):
"""
Serialize the frame to wire format. Returns a string.
"""
b = bytes(self.header)
if self.header.masking_key:
b += Masker(self.header.masking_key)(self.payload)
else:
b += self.payload
return b
@classmethod
def from_file(cls, fp):
"""
read a WebSocket frame sent by a server or client
fp is a "file like" object that could be backed by a network
stream or a disk or an in memory stream reader
"""
header = FrameHeader.from_file(fp)
payload = fp.safe_read(header.payload_length)
if header.mask == 1 and header.masking_key:
payload = Masker(header.masking_key)(payload)
frame = cls(payload)
frame.header = header
return frame
def __eq__(self, other):
if isinstance(other, Frame):
return bytes(self) == bytes(other)
return False

68
pathod/language/writer.py
View File

@ -1,68 +0,0 @@
import time
from mitmproxy import exceptions
BLOCKSIZE = 1024
# It's not clear what the upper limit for time.sleep is. It's lower than the
# maximum int or float. 1 year should do.
FOREVER = 60 * 60 * 24 * 365
def send_chunk(fp, val, blocksize, start, end):
"""
(start, end): Inclusive lower bound, exclusive upper bound.
"""
for i in range(start, end, blocksize):
fp.write(
val[i:min(i + blocksize, end)]
)
return end - start
def write_values(fp, vals, actions, sofar=0, blocksize=BLOCKSIZE):
"""
vals: A list of values, which may be strings or Value objects.
actions: A list of (offset, action, arg) tuples. Action may be "inject",
"pause" or "disconnect".
Both vals and actions are in reverse order, with the first items last.
Return True if connection should disconnect.
"""
sofar = 0
try:
while vals:
v = vals.pop()
offset = 0
while actions and actions[-1][0] < (sofar + len(v)):
a = actions.pop()
offset += send_chunk(
fp,
v,
blocksize,
offset,
a[0] - sofar - offset
)
if a[1] == "pause":
time.sleep(
FOREVER if a[2] == "f" else a[2]
)
elif a[1] == "disconnect":
return True
elif a[1] == "inject":
send_chunk(fp, a[2], blocksize, 0, len(a[2]))
send_chunk(fp, v, blocksize, offset, len(v))
sofar += len(v)
# Remainders
while actions:
a = actions.pop()
if a[1] == "pause":
time.sleep(
FOREVER if a[2] == "f" else a[2]
)
elif a[1] == "disconnect":
return True
elif a[1] == "inject":
send_chunk(fp, a[2], blocksize, 0, len(a[2]))
except exceptions.TcpDisconnect: # pragma: no cover
return True

89
pathod/log.py
View File

@ -1,89 +0,0 @@
import time
from mitmproxy.utils import strutils
from mitmproxy.utils import human
def write_raw(fp, lines, timestamp=True):
if fp:
if timestamp:
fp.write(human.format_timestamp(time.time()))
for i in lines:
fp.write(i)
fp.write("\n")
fp.flush()
class LogCtx:
def __init__(self, fp, hex, timestamp, rfile, wfile):
self.lines = []
self.fp = fp
self.suppressed = False
self.hex = hex
self.timestamp = timestamp
self.rfile, self.wfile = rfile, wfile
def __enter__(self):
if self.wfile:
self.wfile.start_log()
if self.rfile:
self.rfile.start_log()
return self
def __exit__(self, exc_type, exc_value, traceback):
wlog = self.wfile.get_log() if self.wfile else None
rlog = self.rfile.get_log() if self.rfile else None
if self.suppressed or not self.fp:
return
if wlog:
self("Bytes written:")
self.dump(wlog, self.hex)
if rlog:
self("Bytes read:")
self.dump(rlog, self.hex)
if self.lines:
write_raw(
self.fp,
[
"\n".join(self.lines),
],
timestamp = self.timestamp
)
if exc_value:
raise exc_value
def suppress(self):
self.suppressed = True
def dump(self, data, hexdump):
if hexdump:
for line in strutils.hexdump(data):
self("\t%s %s %s" % line)
else:
data = strutils.always_str(
strutils.escape_control_characters(
data
.decode("ascii", "replace")
.replace("\ufffd", ".")
)
)
for i in data.split("\n"):
self("\t%s" % i)
def __call__(self, line):
self.lines.append(line)
class ConnectionLogger:
def __init__(self, fp, hex, timestamp, rfile, wfile):
self.fp = fp
self.hex = hex
self.rfile, self.wfile = rfile, wfile
self.timestamp = timestamp
def ctx(self):
return LogCtx(self.fp, self.hex, self.timestamp, self.rfile, self.wfile)
def write(self, lines):
write_raw(self.fp, lines, timestamp=self.timestamp)

585
pathod/pathoc.py
View File

@ -1,585 +0,0 @@
import contextlib
import sys
import os
import itertools
import hashlib
import queue
import random
import select
import time
import OpenSSL.crypto
import logging
from mitmproxy import certs, exceptions
from mitmproxy.net import tcp, tls, socks
from mitmproxy.net import http as net_http
from mitmproxy.coretypes import basethread
from mitmproxy.utils import strutils
from pathod import language, log
from pathod.protocols import http2
logging.getLogger("hpack").setLevel(logging.WARNING)
def xrepr(s):
return repr(s)[1:-1]
class PathocError(Exception):
pass
class SSLInfo:
def __init__(self, certchain, cipher, alp):
self.certchain, self.cipher, self.alp = certchain, cipher, alp
def __str__(self):
parts = [
"Application Layer Protocol: %s" % strutils.always_str(self.alp, "utf8"),
"Cipher: %s, %s bit, %s" % self.cipher,
"SSL certificate chain:"
]
for n, i in enumerate(self.certchain):
parts.append(" Certificate [%s]" % n)
parts.append("\tSubject: ")
for cn in i.get_subject().get_components():
parts.append("\t\t{}={}".format(
strutils.always_str(cn[0], "utf8"),
strutils.always_str(cn[1], "utf8"))
)
parts.append("\tIssuer: ")
for cn in i.get_issuer().get_components():
parts.append("\t\t{}={}".format(
strutils.always_str(cn[0], "utf8"),
strutils.always_str(cn[1], "utf8"))
)
parts.extend(
[
"\tVersion: %s" % i.get_version(),
"\tValidity: {} - {}".format(
strutils.always_str(i.get_notBefore(), "utf8"),
strutils.always_str(i.get_notAfter(), "utf8")
),
"\tSerial: %s" % i.get_serial_number(),
"\tAlgorithm: %s" % strutils.always_str(i.get_signature_algorithm(), "utf8")
]
)
pk = i.get_pubkey()
types = {
OpenSSL.crypto.TYPE_RSA: "RSA",
OpenSSL.crypto.TYPE_DSA: "DSA"
}
t = types.get(pk.type(), "Uknown")
parts.append(f"\tPubkey: {pk.bits()} bit {t}")
s = certs.Cert(i)
if s.altnames:
parts.append("\tSANs: %s" % " ".join(strutils.always_str(n, "utf8") for n in s.altnames))
return "\n".join(parts)
class WebsocketFrameReader(basethread.BaseThread):
def __init__(
self,
rfile,
logfp,
showresp,
hexdump,
ws_read_limit,
timeout
):
basethread.BaseThread.__init__(self, "WebsocketFrameReader")
self.timeout = timeout
self.ws_read_limit = ws_read_limit
self.logfp = logfp
self.showresp = showresp
self.hexdump = hexdump
self.rfile = rfile
self.terminate = queue.Queue()
self.frames_queue = queue.Queue()
self.logger = log.ConnectionLogger(
self.logfp,
self.hexdump,
False,
rfile if showresp else None,
None
)
@contextlib.contextmanager
def terminator(self):
yield
self.frames_queue.put(None)
def run(self):
starttime = time.time()
with self.terminator():
while True:
if self.ws_read_limit == 0:
return
try:
r, _, _ = select.select([self.rfile], [], [], 0.05)
except OSError: # pragma: no cover
return # this is not reliably triggered due to its nature, so we exclude it from coverage.
delta = time.time() - starttime
if not r and self.timeout and delta > self.timeout:
return
try:
self.terminate.get_nowait()
return
except queue.Empty:
pass
for rfile in r:
with self.logger.ctx() as log:
try:
frm = language.websockets_frame.Frame.from_file(self.rfile)
except exceptions.TcpDisconnect:
return
self.frames_queue.put(frm)
log("<< %s" % repr(frm.header))
if self.ws_read_limit is not None:
self.ws_read_limit -= 1
starttime = time.time()
class Pathoc(tcp.TCPClient):
def __init__(
self,
address,
# SSL
ssl=None,
sni=None,
ssl_version=tls.DEFAULT_METHOD,
ssl_options=tls.DEFAULT_OPTIONS,
clientcert=None,
ciphers=None,
# HTTP/2
use_http2=False,
http2_skip_connection_preface=False,
http2_framedump=False,
# Websockets
ws_read_limit=None,
# Network
timeout=None,
# Output control
showreq=False,
showresp=False,
explain=False,
hexdump=False,
ignorecodes=(),
ignoretimeout=False,
showsummary=False,
fp=sys.stdout
):
"""
spec: A request specification
showreq: Print requests
showresp: Print responses
explain: Print request explanation
showssl: Print info on SSL connection
hexdump: When printing requests or responses, use hex dump output
showsummary: Show a summary of requests
ignorecodes: Sequence of return codes to ignore
"""
tcp.TCPClient.__init__(self, address)
self.ssl, self.sni = ssl, sni
self.clientcert = clientcert
self.ssl_version = ssl_version
self.ssl_options = ssl_options
self.ciphers = ciphers
self.sslinfo = None
self.use_http2 = use_http2
self.http2_skip_connection_preface = http2_skip_connection_preface
self.http2_framedump = http2_framedump
self.ws_read_limit = ws_read_limit
self.timeout = timeout
self.showreq = showreq
self.showresp = showresp
self.explain = explain
self.hexdump = hexdump
self.ignorecodes = ignorecodes
self.ignoretimeout = ignoretimeout
self.showsummary = showsummary
self.fp = fp
self.ws_framereader = None
if self.use_http2:
self.protocol = http2.HTTP2StateProtocol(self, dump_frames=self.http2_framedump)
else:
self.protocol = net_http.http1
self.settings = language.Settings(
is_client=True,
staticdir=os.getcwd(),
unconstrained_file_access=True,
request_host=self.address[0],
protocol=self.protocol,
)
def http_connect(self, connect_to):
req = net_http.Request(
host=connect_to[0],
port=connect_to[1],
method=b'CONNECT',
scheme=b"",
authority=f"{connect_to[0]}:{connect_to[1]}".encode(),
path=b"",
http_version=b'HTTP/1.1',
headers=((b"Host", connect_to[0].encode("idna")),),
content=b'',
trailers=None,
timestamp_start=0,
timestamp_end=0,
)
self.wfile.write(net_http.http1.assemble_request(req))
self.wfile.flush()
try:
resp = self.protocol.read_response(self.rfile, req)
if resp.status_code != 200:
raise exceptions.HttpException("Unexpected status code: %s" % resp.status_code)
except exceptions.HttpException as e:
raise PathocError(
"Proxy CONNECT failed: %s" % repr(e)
)
def socks_connect(self, connect_to):
try:
client_greet = socks.ClientGreeting(
socks.VERSION.SOCKS5,
[socks.METHOD.NO_AUTHENTICATION_REQUIRED]
)
client_greet.to_file(self.wfile)
self.wfile.flush()
server_greet = socks.ServerGreeting.from_file(self.rfile)
server_greet.assert_socks5()
if server_greet.method != socks.METHOD.NO_AUTHENTICATION_REQUIRED:
raise socks.SocksError(
socks.METHOD.NO_ACCEPTABLE_METHODS,
"pathoc only supports SOCKS without authentication"
)
connect_request = socks.Message(
socks.VERSION.SOCKS5,
socks.CMD.CONNECT,
socks.ATYP.DOMAINNAME,
connect_to,
)
connect_request.to_file(self.wfile)
self.wfile.flush()
connect_reply = socks.Message.from_file(self.rfile)
connect_reply.assert_socks5()
if connect_reply.msg != socks.REP.SUCCEEDED:
raise socks.SocksError(
connect_reply.msg,
"SOCKS server error"
)
except (socks.SocksError, exceptions.TcpDisconnect) as e:
raise PathocError(str(e))
def connect(self, connect_to=None, showssl=False, fp=sys.stdout):
"""
connect_to: A (host, port) tuple, which will be connected to with
an HTTP CONNECT request.
"""
if self.use_http2 and not self.ssl:
raise NotImplementedError("HTTP2 without SSL is not supported.")
with tcp.TCPClient.connect(self) as closer:
if connect_to:
self.http_connect(connect_to)
self.sslinfo = None
if self.ssl:
try:
alpn_protos = [b'http/1.1']
if self.use_http2:
alpn_protos.append(b'h2')
self.convert_to_tls(
sni=self.sni,
cert=self.clientcert,
method=self.ssl_version,
options=self.ssl_options,
cipher_list=self.ciphers,
alpn_protos=alpn_protos
)
except exceptions.TlsException as v:
raise PathocError(str(v))
self.sslinfo = SSLInfo(
self.connection.get_peer_cert_chain(),
self.get_current_cipher(),
self.get_alpn_proto_negotiated()
)
if showssl:
print(str(self.sslinfo), file=fp)
if self.use_http2:
self.protocol.check_alpn()
if not self.http2_skip_connection_preface:
self.protocol.perform_client_connection_preface()
if self.timeout:
self.settimeout(self.timeout)
return closer.pop()
def stop(self):
if self.ws_framereader:
self.ws_framereader.terminate.put(None)
def wait(self, timeout=0.01, finish=True):
"""
A generator that yields frames until Pathoc terminates.
timeout: If specified None may be yielded instead if timeout is
reached. If timeout is None, wait forever. If timeout is 0, return
immediately if nothing is on the queue.
finish: If true, consume messages until the reader shuts down.
Otherwise, return None on timeout.
"""
if self.ws_framereader:
while True:
try:
frm = self.ws_framereader.frames_queue.get(
timeout=timeout,
block=True if timeout != 0 else False
)
except queue.Empty:
if finish:
continue
else:
return
if frm is None:
self.ws_framereader.join()
self.ws_framereader = None
return
yield frm
def websocket_send_frame(self, r):
"""
Sends a single websocket frame.
"""
logger = log.ConnectionLogger(
self.fp,
self.hexdump,
False,
None,
self.wfile if self.showreq else None,
)
with logger.ctx() as lg:
lg(">> %s" % r)
language.serve(r, self.wfile, self.settings)
self.wfile.flush()
def websocket_start(self, r):
"""
Performs an HTTP request, and attempts to drop into websocket
connection.
"""
resp = self.http(r)
if resp.status_code == 101:
self.ws_framereader = WebsocketFrameReader(
self.rfile,
self.fp,
self.showresp,
self.hexdump,
self.ws_read_limit,
self.timeout
)
self.ws_framereader.start()
return resp
def http(self, r):
"""
Performs a single request.
r: A language.http.Request object, or a string representing one
request.
Returns Response if we have a non-ignored response.
May raise a exceptions.NetlibException
"""
logger = log.ConnectionLogger(
self.fp,
self.hexdump,
False,
self.rfile if self.showresp else None,
self.wfile if self.showreq else None,
)
with logger.ctx() as lg:
lg(">> %s" % r)
resp, req = None, None
try:
req = language.serve(r, self.wfile, self.settings)
self.wfile.flush()
# build a dummy request to read the response
# ideally this would be returned directly from language.serve
dummy_req = net_http.Request(
host="localhost",
port=80,
method=req["method"],
scheme=b"http",
authority=b"",
path=b"/",
http_version=b"HTTP/1.1",
headers=(),
content=b'',
trailers=None,
timestamp_start=time.time(),
timestamp_end=None,
)
resp = self.protocol.read_response(self.rfile, dummy_req)
resp.sslinfo = self.sslinfo
except exceptions.HttpException as v:
lg("Invalid server response: %s" % v)
raise
except exceptions.TcpTimeout:
if self.ignoretimeout:
lg("Timeout (ignored)")
return None
lg("Timeout")
raise
finally:
if resp:
lg("<< {} {}: {} bytes".format(
resp.status_code, strutils.escape_control_characters(resp.reason) if resp.reason else "", len(resp.content)
))
if resp.status_code in self.ignorecodes:
lg.suppress()
return resp
def request(self, r):
"""
Performs a single request.
r: A language.message.Message object, or a string representing
one.
Returns Response if we have a non-ignored response.
May raise a exceptions.NetlibException
"""
if isinstance(r, str):
r = next(language.parse_pathoc(r, self.use_http2))
if isinstance(r, language.http.Request):
if r.ws:
return self.websocket_start(r)
else:
return self.http(r)
elif isinstance(r, language.websockets.WebsocketFrame):
self.websocket_send_frame(r)
elif isinstance(r, language.http2.Request):
return self.http(r)
# elif isinstance(r, language.http2.Frame):
# TODO: do something
def main(args): # pragma: no cover
memo = set()
p = None
if args.repeat == 1:
requests = args.requests
else:
# If we are replaying more than once, we must convert the request generators to lists
# or they will be exhausted after the first run.
# This is bad for the edge-case where get:/:x10000000 (see 0da3e51) is combined with -n 2,
# but does not matter otherwise.
requests = [list(x) for x in args.requests]
try:
requests_done = 0
while True:
if requests_done == args.repeat:
break
if args.wait and requests_done > 0:
time.sleep(args.wait)
requests_done += 1
if args.random:
playlist = random.choice(requests)
else:
playlist = itertools.chain.from_iterable(requests)
p = Pathoc(
(args.host, args.port),
ssl=args.ssl,
sni=args.sni,
ssl_version=args.ssl_version,
ssl_options=args.ssl_options,
clientcert=args.clientcert,
ciphers=args.ciphers,
use_http2=args.use_http2,
http2_skip_connection_preface=args.http2_skip_connection_preface,
http2_framedump=args.http2_framedump,
showreq=args.showreq,
showresp=args.showresp,
explain=args.explain,
hexdump=args.hexdump,
ignorecodes=args.ignorecodes,
timeout=args.timeout,
ignoretimeout=args.ignoretimeout,
showsummary=True
)
trycount = 0
try:
with p.connect(args.connect_to, args.showssl):
for spec in playlist:
if args.explain or args.memo:
spec = spec.freeze(p.settings)
if args.memo:
h = hashlib.sha256(spec.spec()).digest()
if h not in memo:
trycount = 0
memo.add(h)
else:
trycount += 1
if trycount > args.memolimit:
print("Memo limit exceeded...", file=sys.stderr)
return
else:
continue
try:
ret = p.request(spec)
if ret and args.oneshot:
return
# We consume the queue when we can, so it doesn't build up.
for _ in p.wait(timeout=0, finish=False):
pass
except exceptions.NetlibException:
break
for _ in p.wait(timeout=0.01, finish=True):
pass
except exceptions.TcpException as v:
print(str(v), file=sys.stderr)
continue
except PathocError as v:
print(str(v), file=sys.stderr)
sys.exit(1)
except KeyboardInterrupt:
pass
if p:
p.stop()

228
pathod/pathoc_cmdline.py
View File

@ -1,228 +0,0 @@
import sys
import argparse
import os
import os.path
from mitmproxy.net import tls
from mitmproxy import version
from mitmproxy.net.http import user_agents
from . import print_tool_deprecation_message, pathoc, language
def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr):
preparser = argparse.ArgumentParser(add_help=False)
preparser.add_argument(
"--show-uas", dest="showua", action="store_true", default=False,
help="Print user agent shortcuts and exit."
)
pa = preparser.parse_known_args(argv)[0]
if pa.showua:
print("User agent strings:", file=stdout)
for i in user_agents.UASTRINGS:
print(" ", i[1], i[0], file=stdout)
sys.exit(0)
parser = argparse.ArgumentParser(
description='A perverse HTTP client.', parents=[preparser]
)
parser.add_argument(
'--version',
action='version',
version="pathoc " + version.VERSION
)
parser.add_argument(
"-c", dest="connect_to", type=str, default=False,
metavar="HOST:PORT",
help="Issue an HTTP CONNECT to connect to the specified host."
)
parser.add_argument(
"--memo-limit", dest='memolimit', default=5000, type=int, metavar="N",
help='Stop if we do not find a valid request after N attempts.'
)
parser.add_argument(
"-m", dest='memo', action="store_true", default=False,
help="""
Remember specs, and never play the same one twice. Note that this
means requests have to be rendered in memory, which means that
large generated data can cause issues.
"""
)
parser.add_argument(
"-n", dest='repeat', default=1, type=int, metavar="N",
help='Repeat N times. Pass -1 to repeat infinitely.'
)
parser.add_argument(
"-w", dest='wait', default=0, type=float, metavar="N",
help='Wait N seconds between each request.'
)
parser.add_argument(
"-r", dest="random", action="store_true", default=False,
help="""
Select a random request from those specified. If this is not specified,
requests are all played in sequence.
"""
)
parser.add_argument(
"-t", dest="timeout", type=int, default=None,
help="Connection timeout"
)
parser.add_argument(
"--http2", dest="use_http2", action="store_true", default=False,
help='Perform all requests over a single HTTP/2 connection.'
)
parser.add_argument(
"--http2-skip-connection-preface",
dest="http2_skip_connection_preface",
action="store_true",
default=False,
help='Skips the HTTP/2 connection preface before sending requests.')
parser.add_argument(
'host', type=str,
metavar="host[:port]",
help='Host and port to connect to'
)
parser.add_argument(
'requests', type=str, nargs="+",
help="""
Request specification, or path to a file containing request
specifcations
"""
)
group = parser.add_argument_group(
'SSL',
)
group.add_argument(
"-s", dest="ssl", action="store_true", default=False,
help="Connect with SSL"
)
group.add_argument(
"-C", dest="clientcert", type=str, default=False,
help="Path to a file containing client certificate and private key"
)
group.add_argument(
"-i", dest="sni", type=str, default=False,
help="SSL Server Name Indication"
)
group.add_argument(
"--ciphers", dest="ciphers", type=str, default=False,
help="SSL cipher specification"
)
group.add_argument(
"--ssl-version", dest="ssl_version", type=str, default="secure",
choices=tls.VERSION_CHOICES.keys(),
help="Set supported SSL/TLS versions. "
"SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+."
)
group = parser.add_argument_group(
'Controlling Output',
"""
Some of these options expand generated values for logging - if
you're generating large data, use them with caution.
"""
)
group.add_argument(
"-I", dest="ignorecodes", type=str, default="",
help="Comma-separated list of response codes to ignore"
)
group.add_argument(
"-S", dest="showssl", action="store_true", default=False,
help="Show info on SSL connection"
)
group.add_argument(
"-e", dest="explain", action="store_true", default=False,
help="Explain requests"
)
group.add_argument(
"-o", dest="oneshot", action="store_true", default=False,
help="Oneshot - exit after first non-ignored response"
)
group.add_argument(
"-q", dest="showreq", action="store_true", default=False,
help="Print full request"
)
group.add_argument(
"-p", dest="showresp", action="store_true", default=False,
help="Print full response"
)
group.add_argument(
"-T", dest="ignoretimeout", action="store_true", default=False,
help="Ignore timeouts"
)
group.add_argument(
"-x", dest="hexdump", action="store_true", default=False,
help="Output in hexdump format"
)
group.add_argument(
"--http2-framedump", dest="http2_framedump", action="store_true", default=False,
help="Output all received & sent HTTP/2 frames"
)
args = parser.parse_args(argv[1:])
args.ssl_version, args.ssl_options = tls.VERSION_CHOICES[args.ssl_version]
args.port = None
if ":" in args.host:
h, p = args.host.rsplit(":", 1)
try:
p = int(p)
except ValueError:
return parser.error("Invalid port in host spec: %s" % args.host)
args.host = h
args.port = p
if args.port is None:
args.port = 443 if args.ssl else 80
try:
args.ignorecodes = [int(i) for i in args.ignorecodes.split(",") if i]
except ValueError:
return parser.error(
"Invalid return code specification: %s" %
args.ignorecodes)
if args.connect_to:
parts = args.connect_to.split(":")
if len(parts) != 2:
return parser.error(
"Invalid CONNECT specification: %s" %
args.connect_to)
try:
parts[1] = int(parts[1])
except ValueError:
return parser.error(
"Invalid CONNECT specification: %s" %
args.connect_to)
args.connect_to = parts
else:
args.connect_to = None
if args.http2_skip_connection_preface:
args.use_http2 = True
if args.use_http2:
args.ssl = True
reqs = []
for r in args.requests:
r = os.path.expanduser(r)
if os.path.isfile(r):
with open(r) as f:
r = f.read()
try:
reqs.append(language.parse_pathoc(r, args.use_http2))
except language.ParseException as v:
print("Error parsing request spec: %s" % v.msg, file=stderr)
print(v.marked(), file=stderr)
sys.exit(1)
args.requests = reqs
return args
def go_pathoc(): # pragma: no cover
print_tool_deprecation_message()
args = args_pathoc(sys.argv)
pathoc.main(args)

496
pathod/pathod.py
View File

@ -1,496 +0,0 @@
import copy
import logging
import os
import sys
import threading
import urllib
import typing # noqa
from mitmproxy import certs as mcerts, exceptions, version
from mitmproxy.net import tcp, tls, websocket
from pathod import language, utils, log, protocols
DEFAULT_CERT_DOMAIN = b"pathod.net"
CONFDIR = "~/.mitmproxy"
CERTSTORE_BASENAME = "mitmproxy"
CA_CERT_NAME = "mitmproxy-ca.pem"
DEFAULT_CRAFT_ANCHOR = "/p/"
KEY_SIZE = 2048
logger = logging.getLogger('pathod')
class PathodError(Exception):
pass
class SSLOptions:
def __init__(
self,
confdir=CONFDIR,
cn=None,
sans=(),
not_after_connect=None,
request_client_cert=False,
ssl_version=tls.DEFAULT_METHOD,
ssl_options=tls.DEFAULT_OPTIONS,
ciphers=None,
certs=None,
alpn_select=b'h2',
):
self.confdir = confdir
self.cn = cn
self.sans = sans
self.not_after_connect = not_after_connect
self.request_client_cert = request_client_cert
self.ssl_version = ssl_version
self.ssl_options = ssl_options
self.ciphers = ciphers
self.alpn_select = alpn_select
self.certstore = mcerts.CertStore.from_store(
os.path.expanduser(confdir),
CERTSTORE_BASENAME,
KEY_SIZE
)
for i in certs or []:
self.certstore.add_cert_file(*i)
def get_cert(self, name):
if self.cn:
name = self.cn
elif not name:
name = DEFAULT_CERT_DOMAIN
return self.certstore.get_cert(name, self.sans)
class PathodHandler(tcp.BaseHandler):
wbufsize = 0
sni: typing.Union[str, None, bool] = None
def __init__(
self,
connection,
address,
server,
logfp,
settings,
http2_framedump=False
):
tcp.BaseHandler.__init__(self, connection, address, server)
self.logfp = logfp
self.settings = copy.copy(settings)
self.protocol = None
self.use_http2 = False
self.http2_framedump = http2_framedump
def handle_sni(self, connection):
sni = connection.get_servername()
if sni:
sni = sni.decode("idna")
self.sni = sni
def http_serve_crafted(self, crafted, logctx):
error, crafted = self.server.check_policy(
crafted, self.settings
)
if error:
err = self.make_http_error_response(error)
language.serve(err, self.wfile, self.settings)
return None, dict(
type="error",
msg=error
)
if self.server.explain and not hasattr(crafted, 'is_error_response'):
crafted = crafted.freeze(self.settings)
logctx(">> Spec: %s" % crafted.spec())
response_log = language.serve(
crafted,
self.wfile,
self.settings
)
if response_log["disconnect"]:
return None, response_log
return self.handle_http_request, response_log
def handle_http_request(self, logger):
"""
Returns a (handler, log) tuple.
handler: Handler for the next request, or None to disconnect
log: A dictionary, or None
"""
with logger.ctx() as lg:
try:
req = self.protocol.read_request(self.rfile)
except exceptions.HttpReadDisconnect:
return None, None
except exceptions.HttpException as s:
s = str(s)
lg(s)
return None, dict(type="error", msg=s)
if req.method == 'CONNECT':
return self.protocol.handle_http_connect([req.host, req.port, req.http_version], lg)
method = req.method
path = req.path
http_version = req.http_version
headers = req.headers
first_line_format = req.first_line_format
clientcert = None
if self.clientcert:
clientcert = dict(
cn=self.clientcert.cn,
subject=self.clientcert.subject,
serial=self.clientcert.serial,
notbefore=self.clientcert.notbefore.isoformat(),
notafter=self.clientcert.notafter.isoformat(),
keyinfo=self.clientcert.keyinfo,
)
retlog = dict(
type="crafted",
protocol="http",
request=dict(
path=path,
method=method,
headers=headers.fields,
http_version=http_version,
sni=self.sni,
remote_address=self.address,
clientcert=clientcert,
first_line_format=first_line_format
),
cipher=None,
)
if self.tls_established:
retlog["cipher"] = self.get_current_cipher()
m = utils.MemBool()
valid_websocket_handshake = websocket.check_handshake(headers)
self.settings.websocket_key = websocket.get_client_key(headers)
# If this is a websocket initiation, we respond with a proper
# server response, unless over-ridden.
if valid_websocket_handshake:
anchor_gen = language.parse_pathod("ws")
else:
anchor_gen = None
for regex, spec in self.server.anchors:
if regex.match(path):
anchor_gen = language.parse_pathod(spec, self.use_http2)
break
else:
if m(path.startswith(self.server.craftanchor)):
spec = urllib.parse.unquote(path)[len(self.server.craftanchor):]
if spec:
try:
anchor_gen = language.parse_pathod(spec, self.use_http2)
except language.ParseException as v:
lg("Parse error: %s" % v.msg)
anchor_gen = iter([self.make_http_error_response(
"Parse Error",
"Error parsing response spec: %s\n" % (
v.msg + v.marked()
)
)])
else:
if self.use_http2:
anchor_gen = iter([self.make_http_error_response(
"Spec Error",
"HTTP/2 only supports request/response with the craft anchor point: %s" %
self.server.craftanchor
)])
if not anchor_gen:
anchor_gen = iter([self.make_http_error_response(
"Not found",
"No valid craft request found"
)])
spec = next(anchor_gen)
if self.use_http2 and isinstance(spec, language.http2.Response):
spec.stream_id = req.stream_id
lg("crafting spec: %s" % spec)
nexthandler, retlog["response"] = self.http_serve_crafted(
spec,
lg
)
if nexthandler and valid_websocket_handshake:
self.protocol = protocols.websockets.WebsocketsProtocol(self)
return self.protocol.handle_websocket, retlog
else:
return nexthandler, retlog
def make_http_error_response(self, reason, body=None):
resp = self.protocol.make_error_response(reason, body)
resp.is_error_response = True
return resp
def handle(self):
self.settimeout(self.server.timeout)
if self.server.ssl:
try:
cert, key, _ = self.server.ssloptions.get_cert(None)
self.convert_to_tls(
cert,
key,
handle_sni=self.handle_sni,
request_client_cert=self.server.ssloptions.request_client_cert,
cipher_list=self.server.ssloptions.ciphers,
method=self.server.ssloptions.ssl_version,
options=self.server.ssloptions.ssl_options,
alpn_select=self.server.ssloptions.alpn_select,
)
except exceptions.TlsException as v:
s = str(v)
self.server.add_log(
dict(
type="error",
msg=s
)
)
log.write_raw(self.logfp, s)
return
alp = self.get_alpn_proto_negotiated()
if alp == b'h2':
self.protocol = protocols.http2.HTTP2Protocol(self)
self.use_http2 = True
if not self.protocol:
self.protocol = protocols.http.HTTPProtocol(self)
lr = self.rfile if self.server.logreq else None
lw = self.wfile if self.server.logresp else None
logger = log.ConnectionLogger(self.logfp, self.server.hexdump, True, lr, lw)
self.settings.protocol = self.protocol
handler = self.handle_http_request
while not self.finished:
handler, l = handler(logger)
if l:
self.addlog(l)
if not handler:
return
def addlog(self, log):
if self.server.logreq:
log["request_bytes"] = self.rfile.get_log()
if self.server.logresp:
log["response_bytes"] = self.wfile.get_log()
self.server.add_log(log)
class Pathod(tcp.TCPServer):
LOGBUF = 500
def __init__(
self,
addr,
ssl=False,
ssloptions=None,
craftanchor=DEFAULT_CRAFT_ANCHOR,
staticdir=None,
anchors=(),
sizelimit=None,
nocraft=False,
nohang=False,
timeout=None,
logreq=False,
logresp=False,
explain=False,
hexdump=False,
http2_framedump=False,
webdebug=False,
logfp=sys.stdout,
):
"""
addr: (address, port) tuple. If port is 0, a free port will be
automatically chosen.
ssloptions: an SSLOptions object.
craftanchor: URL prefix specifying the path under which to anchor
response generation.
staticdir: path to a directory of static resources, or None.
anchors: List of (regex object, language.Request object) tuples, or
None.
sizelimit: Limit size of served data.
nocraft: Disable response crafting.
nohang: Disable pauses.
"""
tcp.TCPServer.__init__(self, addr)
self.ssl = ssl
self.ssloptions = ssloptions or SSLOptions()
self.staticdir = staticdir
self.craftanchor = craftanchor
self.sizelimit = sizelimit
self.nocraft = nocraft
self.nohang = nohang
self.timeout, self.logreq = timeout, logreq
self.logresp, self.hexdump = logresp, hexdump
self.http2_framedump = http2_framedump
self.explain = explain
self.logfp = logfp
self.log = []
self.logid = 0
self.anchors = anchors
self.settings = language.Settings(
staticdir=self.staticdir
)
self.loglock = threading.Lock()
def check_policy(self, req, settings):
"""
A policy check that verifies the request size is within limits.
"""
if self.nocraft:
return "Crafting disabled.", None
try:
req = req.resolve(settings)
l = req.maximum_length(settings)
except language.FileAccessDenied:
return "File access denied.", None
if self.sizelimit and l > self.sizelimit:
return "Response too large.", None
pauses = [isinstance(i, language.actions.PauseAt) for i in req.actions]
if self.nohang and any(pauses):
return "Pauses have been disabled.", None
return None, req
def handle_client_connection(self, request, client_address):
h = PathodHandler(
request,
client_address,
self,
self.logfp,
self.settings,
self.http2_framedump,
)
try:
h.handle()
h.finish()
except exceptions.TcpDisconnect: # pragma: no cover
log.write_raw(self.logfp, "Disconnect")
self.add_log(
dict(
type="error",
msg="Disconnect"
)
)
return
except exceptions.TcpTimeout:
log.write_raw(self.logfp, "Timeout")
self.add_log(
dict(
type="timeout",
)
)
return
def add_log(self, d):
with self.loglock:
d["id"] = self.logid
self.log.insert(0, d)
if len(self.log) > self.LOGBUF:
self.log.pop()
self.logid += 1
return d["id"]
def clear_log(self):
with self.loglock:
self.log = []
def log_by_id(self, identifier):
with self.loglock:
for i in self.log:
if i["id"] == identifier:
return i
def get_log(self):
with self.loglock:
return self.log
def main(args): # pragma: no cover
ssloptions = SSLOptions(
cn=args.cn,
confdir=args.confdir,
not_after_connect=args.ssl_not_after_connect,
ciphers=args.ciphers,
ssl_version=args.ssl_version,
ssl_options=args.ssl_options,
certs=args.ssl_certs,
sans=args.sans,
)
root = logging.getLogger()
if root.handlers:
for handler in root.handlers:
root.removeHandler(handler)
log = logging.getLogger('pathod')
log.setLevel(logging.DEBUG)
fmt = logging.Formatter(
'%(asctime)s: %(message)s',
datefmt='%d-%m-%y %H:%M:%S',
)
if args.logfile:
fh = logging.handlers.WatchedFileHandler(args.logfile)
fh.setFormatter(fmt)
log.addHandler(fh)
if not args.daemonize:
sh = logging.StreamHandler()
sh.setFormatter(fmt)
log.addHandler(sh)
try:
pd = Pathod(
(args.address, args.port),
craftanchor=args.craftanchor,
ssl=args.ssl,
ssloptions=ssloptions,
staticdir=args.staticdir,
anchors=args.anchors,
sizelimit=args.sizelimit,
nocraft=args.nocraft,
nohang=args.nohang,
timeout=args.timeout,
logreq=args.logreq,
logresp=args.logresp,
hexdump=args.hexdump,
http2_framedump=args.http2_framedump,
explain=args.explain,
webdebug=args.webdebug
)
except PathodError as v:
print("Error: %s" % v, file=sys.stderr)
sys.exit(1)
except language.FileAccessDenied as v:
print("Error: %s" % v, file=sys.stderr)
if args.daemonize:
utils.daemonize()
try:
print("{} listening on {}".format(
version.PATHOD,
repr(pd.address)
))
pd.serve_forever()
except KeyboardInterrupt:
pass

236
pathod/pathod_cmdline.py
View File

@ -1,236 +0,0 @@
import sys
import argparse
import os
import os.path
import re
from mitmproxy.net import tls
from mitmproxy.utils import human
from mitmproxy import version
from . import print_tool_deprecation_message, pathod
def parse_anchor_spec(s):
"""
Return a tuple, or None on error.
"""
if "=" not in s:
return None
return tuple(s.split("=", 1))
def args_pathod(argv, stdout_=sys.stdout, stderr_=sys.stderr):
parser = argparse.ArgumentParser(
description='A pathological HTTP/S daemon.'
)
parser.add_argument(
'--version',
action='version',
version="pathod " + version.VERSION
)
parser.add_argument(
"-p",
dest='port',
default=9999,
type=int,
help='Port. Specify 0 to pick an arbitrary empty port. (9999)'
)
parser.add_argument(
"-l",
dest='address',
default="127.0.0.1",
type=str,
help='Listening address. (127.0.0.1)'
)
parser.add_argument(
"-a",
dest='anchors',
default=[],
type=str,
action="append",
metavar="ANCHOR",
help="""
Add an anchor. Specified as a string with the form
pattern=spec or pattern=filepath, where pattern is a regular
expression.
"""
)
parser.add_argument(
"-c", dest='craftanchor', default=pathod.DEFAULT_CRAFT_ANCHOR, type=str,
help="""
URL path specifying prefix for URL crafting
commands. (%s)
""" % pathod.DEFAULT_CRAFT_ANCHOR
)
parser.add_argument(
"--confdir",
action="store", type=str, dest="confdir", default='~/.mitmproxy',
help="Configuration directory. (~/.mitmproxy)"
)
parser.add_argument(
"-d", dest='staticdir', default=None, type=str,
help='Directory for static files.'
)
parser.add_argument(
"-D", dest='daemonize', default=False, action="store_true",
help='Daemonize.'
)
parser.add_argument(
"-t", dest="timeout", type=int, default=None,
help="Connection timeout"
)
parser.add_argument(
"--limit-size",
dest='sizelimit',
default=None,
type=str,
help='Size limit of served responses. Understands size suffixes, i.e. 100k.')
parser.add_argument(
"--nohang", dest='nohang', default=False, action="store_true",
help='Disable pauses during crafted response generation.'
)
parser.add_argument(
"--nocraft",
dest='nocraft',
default=False,
action="store_true",
help='Disable response crafting. If anchors are specified, they still work.')
parser.add_argument(
"--webdebug", dest='webdebug', default=False, action="store_true",
help='Debugging mode for the web app (dev only).'
)
group = parser.add_argument_group(
'SSL',
)
group.add_argument(
"-s", dest='ssl', default=False, action="store_true",
help='Run in HTTPS mode.'
)
group.add_argument(
"--cn",
dest="cn",
type=str,
default=None,
help="CN for generated SSL certs. Default: %s" %
pathod.DEFAULT_CERT_DOMAIN)
group.add_argument(
"-C", dest='ssl_not_after_connect', default=False, action="store_true",
help="Don't expect SSL after a CONNECT request."
)
group.add_argument(
"--cert", dest='ssl_certs', default=[], type=str,
metavar="SPEC", action="append",
help="""
Add an SSL certificate. SPEC is of the form "[domain=]path". The domain
may include a wildcard, and is equal to "*" if not specified. The file
at path is a certificate in PEM format. If a private key is included in
the PEM, it is used, else the default key in the conf dir is used. Can
be passed multiple times.
"""
)
group.add_argument(
"--ciphers", dest="ciphers", type=str, default=False,
help="SSL cipher specification"
)
group.add_argument(
"--san", dest="sans", type=str, default=[], action="append",
metavar="SAN",
help="""
Subject Altnernate Name to add to the server certificate.
May be passed multiple times.
"""
)
group.add_argument(
"--ssl-version", dest="ssl_version", type=str, default="secure",
choices=tls.VERSION_CHOICES.keys(),
help="Set supported SSL/TLS versions. "
"SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+."
)
group = parser.add_argument_group(
'Controlling Logging',
"""
Some of these options expand generated values for logging - if
you're generating large data, use them with caution.
"""
)
group.add_argument(
"-e", dest="explain", action="store_true", default=False,
help="Explain responses"
)
group.add_argument(
"-f", dest='logfile', default=None, type=str,
help='Log to file.'
)
group.add_argument(
"-q", dest="logreq", action="store_true", default=False,
help="Log full request"
)
group.add_argument(
"-r", dest="logresp", action="store_true", default=False,
help="Log full response"
)
group.add_argument(
"-x", dest="hexdump", action="store_true", default=False,
help="Log request/response in hexdump format"
)
group.add_argument(
"--http2-framedump", dest="http2_framedump", action="store_true", default=False,
help="Output all received & sent HTTP/2 frames"
)
args = parser.parse_args(argv[1:])
args.ssl_version, args.ssl_options = tls.VERSION_CHOICES[args.ssl_version]
certs = []
for i in args.ssl_certs:
parts = i.split("=", 1)
if len(parts) == 1:
parts = ["*", parts[0]]
parts[1] = os.path.expanduser(parts[1])
if not os.path.isfile(parts[1]):
return parser.error(
"Certificate file does not exist: %s" %
parts[1])
certs.append(parts)
args.ssl_certs = certs
alst = []
for i in args.anchors:
parts = parse_anchor_spec(i)
if not parts:
return parser.error("Invalid anchor specification: %s" % i)
alst.append(parts)
args.anchors = alst
sizelimit = None
if args.sizelimit:
try:
sizelimit = human.parse_size(args.sizelimit)
except ValueError as v:
return parser.error(v)
args.sizelimit = sizelimit
anchors = []
for patt, spec in args.anchors:
spec = os.path.expanduser(spec)
if os.path.isfile(spec):
with open(spec) as f:
data = f.read()
spec = data
try:
arex = re.compile(patt)
except re.error:
return parser.error("Invalid regex in anchor: %s" % patt)
anchors.append((arex, spec))
args.anchors = anchors
return args
def go_pathod(): # pragma: no cover
print_tool_deprecation_message()
args = args_pathod(sys.argv)
pathod.main(args)

7
pathod/protocols/__init__.py
View File

@ -1,7 +0,0 @@
from . import http, http2, websockets
__all__ = [
"http",
"http2",
"websockets",
]

47
pathod/protocols/http.py
View File

@ -1,47 +0,0 @@
from mitmproxy import version
from mitmproxy import exceptions
from mitmproxy.net.http import http1
from .. import language
class HTTPProtocol:
def __init__(self, pathod_handler):
self.pathod_handler = pathod_handler
def make_error_response(self, reason, body):
return language.http.make_error_response(reason, body)
def handle_http_connect(self, connect, lg):
"""
Handle a CONNECT request.
"""
self.pathod_handler.wfile.write(
b'HTTP/1.1 200 Connection established\r\n' +
(b'Proxy-agent: %s\r\n' % version.PATHOD.encode()) +
b'\r\n'
)
self.pathod_handler.wfile.flush()
if not self.pathod_handler.server.ssloptions.not_after_connect:
try:
cert, key, chain_file_ = self.pathod_handler.server.ssloptions.get_cert(
connect[0].encode()
)
self.pathod_handler.convert_to_tls(
cert,
key,
handle_sni=self.pathod_handler.handle_sni,
request_client_cert=self.pathod_handler.server.ssloptions.request_client_cert,
cipher_list=self.pathod_handler.server.ssloptions.ciphers,
method=self.pathod_handler.server.ssloptions.ssl_version,
options=self.pathod_handler.server.ssloptions.ssl_options,
alpn_select=self.pathod_handler.server.ssloptions.alpn_select,
)
except exceptions.TlsException as v:
s = str(v)
lg(s)
return None, dict(type="error", msg=s)
return self.pathod_handler.handle_http_request, None
def read_request(self, lg=None):
return http1.read_request(self.pathod_handler.rfile)

429
pathod/protocols/http2.py
View File

@ -1,429 +0,0 @@
import itertools
import time
import hyperframe.frame
from hpack.hpack import Decoder, Encoder
import mitmproxy.net.http.headers
import mitmproxy.net.http.request
import mitmproxy.net.http.response
from mitmproxy.coretypes import bidi
from mitmproxy.net.http import http2, url
from .. import language
class TCPHandler:
def __init__(self, rfile, wfile=None):
self.rfile = rfile
self.wfile = wfile
class HTTP2StateProtocol:
ERROR_CODES = bidi.BiDi(
NO_ERROR=0x0,
PROTOCOL_ERROR=0x1,
INTERNAL_ERROR=0x2,
FLOW_CONTROL_ERROR=0x3,
SETTINGS_TIMEOUT=0x4,
STREAM_CLOSED=0x5,
FRAME_SIZE_ERROR=0x6,
REFUSED_STREAM=0x7,
CANCEL=0x8,
COMPRESSION_ERROR=0x9,
CONNECT_ERROR=0xa,
ENHANCE_YOUR_CALM=0xb,
INADEQUATE_SECURITY=0xc,
HTTP_1_1_REQUIRED=0xd
)
CLIENT_CONNECTION_PREFACE = b'PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n'
HTTP2_DEFAULT_SETTINGS = {
hyperframe.frame.SettingsFrame.HEADER_TABLE_SIZE: 4096,
hyperframe.frame.SettingsFrame.ENABLE_PUSH: 1,
hyperframe.frame.SettingsFrame.MAX_CONCURRENT_STREAMS: None,
hyperframe.frame.SettingsFrame.INITIAL_WINDOW_SIZE: 2 ** 16 - 1,
hyperframe.frame.SettingsFrame.MAX_FRAME_SIZE: 2 ** 14,
hyperframe.frame.SettingsFrame.MAX_HEADER_LIST_SIZE: None,
}
def __init__(
self,
tcp_handler=None,
rfile=None,
wfile=None,
is_server=False,
dump_frames=False,
encoder=None,
decoder=None,
unhandled_frame_cb=None,
):
self.tcp_handler = tcp_handler or TCPHandler(rfile, wfile)
self.is_server = is_server
self.dump_frames = dump_frames
self.encoder = encoder or Encoder()
self.decoder = decoder or Decoder()
self.unhandled_frame_cb = unhandled_frame_cb
self.http2_settings = self.HTTP2_DEFAULT_SETTINGS.copy()
self.current_stream_id = None
self.connection_preface_performed = False
def read_request(
self,
__rfile,
include_body=True,
body_size_limit=None,
allow_empty=False,
):
if body_size_limit is not None:
raise NotImplementedError()
self.perform_connection_preface()
timestamp_start = time.time()
if hasattr(self.tcp_handler.rfile, "reset_timestamps"):
self.tcp_handler.rfile.reset_timestamps()
stream_id, headers, body = self._receive_transmission(
include_body=include_body,
)
if hasattr(self.tcp_handler.rfile, "first_byte_timestamp"):
# more accurate timestamp_start
timestamp_start = self.tcp_handler.rfile.first_byte_timestamp
timestamp_end = time.time()
# pseudo header must be present, see https://http2.github.io/http2-spec/#rfc.section.8.1.2.3
authority = headers.pop(':authority', "")
method = headers.pop(':method', "")
scheme = headers.pop(':scheme', "")
path = headers.pop(':path', "")
host, port = url.parse_authority(authority, check=False)
port = port or url.default_port(scheme) or 0
request = mitmproxy.net.http.Request(
host=host,
port=port,
method=method.encode(),
scheme=scheme.encode(),
authority=authority.encode(),
path=path.encode(),
http_version=b"HTTP/2.0",
headers=headers,
content=body,
trailers=None,
timestamp_start=timestamp_start,
timestamp_end=timestamp_end,
)
request.stream_id = stream_id
return request
def read_response(
self,
__rfile,
request_method=b'',
body_size_limit=None,
include_body=True,
stream_id=None,
):
if body_size_limit is not None:
raise NotImplementedError()
self.perform_connection_preface()
timestamp_start = time.time()
if hasattr(self.tcp_handler.rfile, "reset_timestamps"):
self.tcp_handler.rfile.reset_timestamps()
stream_id, headers, body = self._receive_transmission(
stream_id=stream_id,
include_body=include_body,
)
if hasattr(self.tcp_handler.rfile, "first_byte_timestamp"):
# more accurate timestamp_start
timestamp_start = self.tcp_handler.rfile.first_byte_timestamp
if include_body:
timestamp_end = time.time()
else:
timestamp_end = None
response = mitmproxy.net.http.response.Response(
http_version=b"HTTP/2.0",
status_code=int(headers.get(':status', 502)),
reason=b'',
headers=headers,
content=body,
trailers=None,
timestamp_start=timestamp_start,
timestamp_end=timestamp_end,
)
response.stream_id = stream_id
return response
def assemble(self, message):
if isinstance(message, mitmproxy.net.http.request.Request):
return self.assemble_request(message)
elif isinstance(message, mitmproxy.net.http.response.Response):
return self.assemble_response(message)
else:
raise ValueError("HTTP message not supported.")
def assemble_request(self, request):
assert isinstance(request, mitmproxy.net.http.request.Request)
authority = self.tcp_handler.sni if self.tcp_handler.sni else self.tcp_handler.address[0]
if self.tcp_handler.address[1] != 443:
authority += ":%d" % self.tcp_handler.address[1]
headers = request.headers.copy()
if ':authority' not in headers:
headers.insert(0, ':authority', authority)
headers.insert(0, ':scheme', request.scheme)
headers.insert(0, ':path', request.path)
headers.insert(0, ':method', request.method)
if hasattr(request, 'stream_id'):
stream_id = request.stream_id
else:
stream_id = self._next_stream_id()
return list(itertools.chain(
self._create_headers(headers, stream_id, end_stream=(request.content is None or len(request.content) == 0)),
self._create_body(request.content, stream_id)))
def assemble_response(self, response):
assert isinstance(response, mitmproxy.net.http.response.Response)
headers = response.headers.copy()
if ':status' not in headers:
headers.insert(0, b':status', str(response.status_code).encode())
if hasattr(response, 'stream_id'):
stream_id = response.stream_id
else:
stream_id = self._next_stream_id()
return list(itertools.chain(
self._create_headers(headers, stream_id, end_stream=(response.content is None or len(response.content) == 0)),
self._create_body(response.content, stream_id),
))
def perform_connection_preface(self, force=False):
if force or not self.connection_preface_performed:
if self.is_server:
self.perform_server_connection_preface(force)
else:
self.perform_client_connection_preface(force)
def perform_server_connection_preface(self, force=False):
if force or not self.connection_preface_performed:
self.connection_preface_performed = True
magic_length = len(self.CLIENT_CONNECTION_PREFACE)
magic = self.tcp_handler.rfile.safe_read(magic_length)
assert magic == self.CLIENT_CONNECTION_PREFACE
frm = hyperframe.frame.SettingsFrame(settings={
hyperframe.frame.SettingsFrame.ENABLE_PUSH: 0,
hyperframe.frame.SettingsFrame.MAX_CONCURRENT_STREAMS: 1,
})
self.send_frame(frm, hide=True)
self._receive_settings(hide=True)
def perform_client_connection_preface(self, force=False):
if force or not self.connection_preface_performed:
self.connection_preface_performed = True
self.tcp_handler.wfile.write(self.CLIENT_CONNECTION_PREFACE)
self.send_frame(hyperframe.frame.SettingsFrame(), hide=True)
self._receive_settings(hide=True) # server announces own settings
self._receive_settings(hide=True) # server acks my settings
def send_frame(self, frm, hide=False):
raw_bytes = frm.serialize()
self.tcp_handler.wfile.write(raw_bytes)
self.tcp_handler.wfile.flush()
if not hide and self.dump_frames: # pragma: no cover
print(">> " + repr(frm))
def read_frame(self, hide=False):
while True:
frm, _ = http2.read_frame(self.tcp_handler.rfile)
if not hide and self.dump_frames: # pragma: no cover
print("<< " + repr(frm))
if isinstance(frm, hyperframe.frame.PingFrame):
raw_bytes = hyperframe.frame.PingFrame(flags=['ACK'], payload=frm.payload).serialize()
self.tcp_handler.wfile.write(raw_bytes)
self.tcp_handler.wfile.flush()
continue
if isinstance(frm, hyperframe.frame.SettingsFrame) and 'ACK' not in frm.flags:
self._apply_settings(frm.settings, hide)
if isinstance(frm, hyperframe.frame.DataFrame) and frm.flow_controlled_length > 0:
self._update_flow_control_window(frm.stream_id, frm.flow_controlled_length)
return frm
def check_alpn(self):
alp = self.tcp_handler.get_alpn_proto_negotiated()
if alp != b'h2':
raise NotImplementedError(
"HTTP2Protocol can not handle unknown ALPN value: %s" % alp)
return True
def _handle_unexpected_frame(self, frm):
if isinstance(frm, hyperframe.frame.SettingsFrame):
return
if self.unhandled_frame_cb:
self.unhandled_frame_cb(frm)
def _receive_settings(self, hide=False):
while True:
frm = self.read_frame(hide)
if isinstance(frm, hyperframe.frame.SettingsFrame):
break
else:
self._handle_unexpected_frame(frm)
def _next_stream_id(self):
if self.current_stream_id is None:
if self.is_server:
# servers must use even stream ids
self.current_stream_id = 2
else:
# clients must use odd stream ids
self.current_stream_id = 1
else:
self.current_stream_id += 2
return self.current_stream_id
def _apply_settings(self, settings, hide=False):
for setting, value in settings.items():
old_value = self.http2_settings[setting]
if not old_value:
old_value = '-'
self.http2_settings[setting] = value
frm = hyperframe.frame.SettingsFrame(flags=['ACK'])
self.send_frame(frm, hide)
def _update_flow_control_window(self, stream_id, increment):
frm = hyperframe.frame.WindowUpdateFrame(stream_id=0, window_increment=increment)
self.send_frame(frm)
frm = hyperframe.frame.WindowUpdateFrame(stream_id=stream_id, window_increment=increment)
self.send_frame(frm)
def _create_headers(self, headers, stream_id, end_stream=True):
def frame_cls(chunks):
for i in chunks:
if i == 0:
yield hyperframe.frame.HeadersFrame, i
else:
yield hyperframe.frame.ContinuationFrame, i
header_block_fragment = self.encoder.encode(headers.fields)
chunk_size = self.http2_settings[hyperframe.frame.SettingsFrame.MAX_FRAME_SIZE]
chunks = range(0, len(header_block_fragment), chunk_size)
frms = [frm_cls(
flags=[],
stream_id=stream_id,
data=header_block_fragment[i:i + chunk_size]) for frm_cls, i in frame_cls(chunks)]
frms[-1].flags.add('END_HEADERS')
if end_stream:
frms[0].flags.add('END_STREAM')
if self.dump_frames: # pragma: no cover
for frm in frms:
print(">> ", repr(frm))
return [frm.serialize() for frm in frms]
def _create_body(self, body, stream_id):
if body is None or len(body) == 0:
return b''
chunk_size = self.http2_settings[hyperframe.frame.SettingsFrame.MAX_FRAME_SIZE]
chunks = range(0, len(body), chunk_size)
frms = [hyperframe.frame.DataFrame(
flags=[],
stream_id=stream_id,
data=body[i:i + chunk_size]) for i in chunks]
frms[-1].flags.add('END_STREAM')
if self.dump_frames: # pragma: no cover
for frm in frms:
print(">> ", repr(frm))
return [frm.serialize() for frm in frms]
def _receive_transmission(self, stream_id=None, include_body=True):
if not include_body:
raise NotImplementedError()
body_expected = True
header_blocks = b''
body = b''
while True:
frm = self.read_frame()
if (
(isinstance(frm, hyperframe.frame.HeadersFrame) or isinstance(frm, hyperframe.frame.ContinuationFrame)) and
(stream_id is None or frm.stream_id == stream_id)
):
stream_id = frm.stream_id
header_blocks += frm.data
if 'END_STREAM' in frm.flags:
body_expected = False
if 'END_HEADERS' in frm.flags:
break
else:
self._handle_unexpected_frame(frm)
while body_expected:
frm = self.read_frame()
if isinstance(frm, hyperframe.frame.DataFrame) and frm.stream_id == stream_id:
body += frm.data
if 'END_STREAM' in frm.flags:
break
else:
self._handle_unexpected_frame(frm)
headers = mitmproxy.net.http.headers.Headers(
[[k, v] for k, v in self.decoder.decode(header_blocks, raw=True)]
)
return stream_id, headers, body
class HTTP2Protocol:
def __init__(self, pathod_handler):
self.pathod_handler = pathod_handler
self.wire_protocol = HTTP2StateProtocol(
self.pathod_handler, is_server=True, dump_frames=self.pathod_handler.http2_framedump
)
def make_error_response(self, reason, body):
return language.http2.make_error_response(reason, body)
def read_request(self, lg=None):
self.wire_protocol.perform_server_connection_preface()
return self.wire_protocol.read_request(self.pathod_handler.rfile)
def assemble(self, message):
return self.wire_protocol.assemble(message)

54
pathod/protocols/websockets.py
View File

@ -1,54 +0,0 @@
import time
from pathod import language
from mitmproxy import exceptions
class WebsocketsProtocol:
def __init__(self, pathod_handler):
self.pathod_handler = pathod_handler
def handle_websocket(self, logger):
while True:
with logger.ctx() as lg:
started = time.time()
try:
frm = language.websockets_frame.Frame.from_file(self.pathod_handler.rfile)
except exceptions.NetlibException as e:
lg("Error reading websocket frame: %s" % e)
return None, None
ended = time.time()
lg(repr(frm))
retlog = dict(
type="inbound",
protocol="websockets",
started=started,
duration=ended - started,
frame=dict(
),
cipher=None,
)
if self.pathod_handler.tls_established:
retlog["cipher"] = self.pathod_handler.get_current_cipher()
self.pathod_handler.addlog(retlog)
ld = language.websockets.NESTED_LEADER
if frm.payload.startswith(ld):
nest = frm.payload[len(ld):]
try:
wf_gen = language.parse_websocket_frame(nest.decode())
except language.exceptions.ParseException as v:
logger.write(
"Parse error in reflected frame specifcation:"
" %s" % v.msg
)
return None, None
for frm in wf_gen:
with logger.ctx() as lg:
frame_log = language.serve(
frm,
self.pathod_handler.wfile,
self.pathod_handler.settings
)
lg("crafting websocket spec: %s" % frame_log["spec"])
self.pathod_handler.addlog(frame_log)

104
pathod/test.py
View File

@ -1,104 +0,0 @@
import io
import time
import queue
from . import pathod
from mitmproxy.coretypes import basethread
import typing # noqa
class Daemon:
IFACE = "127.0.0.1"
def __init__(self, ssl=None, **daemonargs) -> None:
self.q: queue.Queue = queue.Queue()
self.logfp = io.StringIO()
daemonargs["logfp"] = self.logfp
self.thread = _PaThread(self.IFACE, self.q, ssl, daemonargs)
self.thread.start()
self.port = self.q.get(True, 5)
self.urlbase = "{}://{}:{}".format(
"https" if ssl else "http",
self.IFACE,
self.port
)
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.logfp.truncate(0)
self.shutdown()
return False
def p(self, spec: str) -> str:
"""
Return a URL that will render the response in spec.
"""
return f"{self.urlbase}/p/{spec}"
def text_log(self) -> str:
return self.logfp.getvalue()
def wait_for_silence(self, timeout=5):
self.thread.server.wait_for_silence(timeout=timeout)
def expect_log(self, n, timeout=5):
l = []
start = time.time()
while True:
l = self.log()
if time.time() - start >= timeout:
return None
if len(l) >= n:
break
return l
def last_log(self):
"""
Returns the last logged request, or None.
"""
l = self.expect_log(1)
if not l:
return None
return l[-1]
def log(self) -> typing.List[typing.Dict]:
"""
Return the log buffer as a list of dictionaries.
"""
return self.thread.server.get_log()
def clear_log(self):
"""
Clear the log.
"""
return self.thread.server.clear_log()
def shutdown(self):
"""
Shut the daemon down, return after the thread has exited.
"""
self.thread.server.shutdown()
self.thread.join()
class _PaThread(basethread.BaseThread):
def __init__(self, iface, q, ssl, daemonargs):
basethread.BaseThread.__init__(self, "PathodThread")
self.iface, self.q, self.ssl = iface, q, ssl
self.daemonargs = daemonargs
self.server = None
def run(self):
self.server = pathod.Pathod(
(self.iface, 0),
ssl=self.ssl,
**self.daemonargs
)
self.name = "PathodThread ({}:{})".format(
self.server.address[0],
self.server.address[1],
)
self.q.put(self.server.address[1])
self.server.serve_forever()

48
pathod/utils.py
View File

@ -1,48 +0,0 @@
import os
import sys
from mitmproxy.utils import data as mdata
import typing # noqa
class MemBool:
"""
Truth-checking with a memory, for use in chained if statements.
"""
def __init__(self) -> None:
self.v: typing.Optional[bool] = None
def __call__(self, v: bool) -> bool:
self.v = v
return bool(v)
# FIXME: change this name
data = mdata.Data(__name__)
def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'): # pragma: no cover
try:
pid = os.fork()
if pid > 0:
sys.exit(0)
except OSError as e:
sys.stderr.write("fork #1 failed: (%d) %s\n" % (e.errno, e.strerror))
sys.exit(1)
os.chdir("/")
os.umask(0)
os.setsid()
try:
pid = os.fork()
if pid > 0:
sys.exit(0)
except OSError as e:
sys.stderr.write("fork #2 failed: (%d) %s\n" % (e.errno, e.strerror))
sys.exit(1)
si = open(stdin, 'rb')
so = open(stdout, 'a+b')
se = open(stderr, 'a+b', 0)
os.dup2(si.fileno(), sys.stdin.fileno())
os.dup2(so.fileno(), sys.stdout.fileno())
os.dup2(se.fileno(), sys.stderr.fileno())

1
release/cibuild.py
View File

@ -149,7 +149,6 @@ class BuildEnviron:
def bdists(self):
ret = {
"mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
"pathod": ["pathoc", "pathod"]
}
if self.system == "Windows":
ret["mitmproxy"].remove("mitmproxy")

2
release/installbuilder/mitmproxy.xml
View File

@ -31,7 +31,6 @@
<distributionFileList>
<distributionFile>
<allowWildcards>1</allowWildcards>
<excludeFiles>*/patho*</excludeFiles>
<origin>../build/binaries/${platform_name}/*</origin>
</distributionFile>
</distributionFileList>
@ -132,4 +131,3 @@
</directoryParameter>
</parameterList>
</project>

6
release/specs/pathoc
View File

@ -1,6 +0,0 @@
#!/usr/bin/env python
from pathod import pathoc_cmdline as cmdline
if __name__ == "__main__":
cmdline.go_pathoc()

6
release/specs/pathod
View File

@ -1,6 +0,0 @@
#!/usr/bin/env python
from pathod import pathod_cmdline as cmdline
if __name__ == "__main__":
cmdline.go_pathod()

17
setup.cfg
View File

@ -47,10 +47,6 @@ exclude =
mitmproxy/proxy/root_context.py
mitmproxy/proxy/server.py
mitmproxy/tools/
pathod/pathoc.py
pathod/pathod.py
pathod/test.py
pathod/protocols/http2.py
release/hooks
@ -97,17 +93,4 @@ exclude =
mitmproxy/proxy2/server.py
mitmproxy/proxy2/layers/tls.py
mitmproxy/utils/bits.py
pathod/language/actions.py
pathod/language/base.py
pathod/language/exceptions.py
pathod/language/generators.py
pathod/language/http.py
pathod/language/message.py
pathod/log.py
pathod/pathoc.py
pathod/pathod.py
pathod/protocols/http.py
pathod/protocols/http2.py
pathod/protocols/websockets.py
pathod/test.py
release/hooks

3
setup.py
View File

@ -53,7 +53,6 @@ setup(
},
packages=find_packages(include=[
"mitmproxy", "mitmproxy.*",
"pathod", "pathod.*",
]),
include_package_data=True,
entry_points={
@ -61,8 +60,6 @@ setup(
"mitmproxy = mitmproxy.tools.main:mitmproxy",
"mitmdump = mitmproxy.tools.main:mitmdump",
"mitmweb = mitmproxy.tools.main:mitmweb",
"pathod = pathod.pathod_cmdline:go_pathod",
"pathoc = pathod.pathoc_cmdline:go_pathoc"
]
},
python_requires='>=3.8',

4
test/filename_matching.py
View File

@ -11,7 +11,7 @@ def check_src_files_have_test():
excluded = ['mitmproxy/contrib/', 'mitmproxy/io/proto/', 'mitmproxy/proxy2/layers/http',
'mitmproxy/test/', 'mitmproxy/tools/', 'mitmproxy/platform/']
src_files = glob.glob('mitmproxy/**/*.py', recursive=True) + glob.glob('pathod/**/*.py', recursive=True)
src_files = glob.glob('mitmproxy/**/*.py', recursive=True)
src_files = [f for f in src_files if os.path.basename(f) != '__init__.py']
src_files = [f for f in src_files if not any(os.path.normpath(p) in f for p in excluded)]
for f in src_files:
@ -26,7 +26,7 @@ def check_test_files_have_src():
unknown_test_files = []
excluded = ['test/mitmproxy/data/', 'test/mitmproxy/net/data/', '/tservers.py', '/conftest.py']
test_files = glob.glob('test/mitmproxy/**/*.py', recursive=True) + glob.glob('test/pathod/**/*.py', recursive=True)
test_files = glob.glob('test/mitmproxy/**/*.py', recursive=True)
test_files = [f for f in test_files if os.path.basename(f) != '__init__.py']
test_files = [f for f in test_files if not any(os.path.normpath(p) in f for p in excluded)]
for f in test_files:

2
test/individual_coverage.py
View File

@ -63,7 +63,7 @@ def main():
no_individual_cov = [f.strip() for f in fs]
excluded = ['mitmproxy/contrib/', 'mitmproxy/test/', 'mitmproxy/tools/', 'mitmproxy/platform/']
src_files = glob.glob('mitmproxy/**/*.py', recursive=True) + glob.glob('pathod/**/*.py', recursive=True)
src_files = glob.glob('mitmproxy/**/*.py', recursive=True)
src_files = [f for f in src_files if os.path.basename(f) != '__init__.py']
src_files = [f for f in src_files if not any(os.path.normpath(p) in f for p in excluded)]

4
test/mitmproxy/fuzzing/.env
View File

@ -1,4 +0,0 @@
MITMDUMP=mitmdump
PATHOD=pathod
PATHOC=pathoc
FUZZ_SETTINGS="-remTt 1 -n 0"

14
test/mitmproxy/fuzzing/README
View File

@ -1,14 +0,0 @@
A fuzzing architecture for mitmproxy
====================================
Quick start:
honcho -f ./straight_stream start
Notes:
- Processes are managed using honcho (pip install honcho)
- Paths and common settings live in .env

4
test/mitmproxy/fuzzing/client_patterns
View File

@ -1,4 +0,0 @@
get:'http://localhost:9999/p/200':ir,"\n"
get:'http://localhost:9999/p/200':ir,"\0"
get:'http://localhost:9999/p/200':ir,@5
get:'http://localhost:9999/p/200':dr

29
test/mitmproxy/fuzzing/go_proxy
View File

@ -1,29 +0,0 @@
#!/bin/sh
# Assuming:
# mitmproxy/mitmdump is running on port 8080 in straight proxy mode.
# pathod is running on port 9999
BASE="../../../"
BASE_HTTP=$BASE"/pathod/pathoc -Tt 1 -e -I 200,400,405,502 -p 8080 localhost "
BASE_HTTPS=$BASE"/pathod/pathoc -sc localhost:9999 -Tt 1 -eo -I 200,400,404,405,502,800 -p 8080 localhost "
#$BASE_HTTP -n 10000 "get:'http://localhost:9999':ir,@1"
#$BASE_HTTP -n 100 "get:'http://localhost:9999':dr"
#$BASE_HTTP -n 10000 "get:'http://localhost:9999/p/200':ir,@300"
#$BASE_HTTP -n 10000 "get:'http://localhost:9999/p/200:ir,@1'"
#$BASE_HTTP -n 100 "get:'http://localhost:9999/p/200:dr'"
#$BASE_HTTP -n 10000 "get:'http://localhost:9999/p/200:ir,@100'"
# Assuming:
# mitmproxy/mitmdump is running on port 8080 in straight proxy mode.
# pathod with SSL enabled is running on port 9999
#$BASE_HTTPS -en 10000 "get:'/p/200:b@100:ir,@1'"
#$BASE_HTTPS -en 10000 "get:'/p/200:ir,@1'"
#$BASE_HTTPS -n 100 "get:'/p/200:dr'"
#$BASE_HTTPS -n 10000 "get:'/p/200:ir,@3000'"
#$BASE_HTTPS -n 10000 "get:'/p/200:ir,\"\\n\"'"

9
test/mitmproxy/fuzzing/reverse_patterns
View File

@ -1,9 +0,0 @@
get:'/p/200':b@10:ir,"\n"
get:'/p/200':b@10:ir,"\r\n"
get:'/p/200':b@10:ir,"\0"
get:'/p/200':b@10:ir,@5
get:'/p/200':b@10:dr
get:'/p/200:b@10:ir,@1'
get:'/p/200:b@10:dr'
get:'/p/200:b@10:ir,@100'

6
test/mitmproxy/fuzzing/straight_stream
View File

@ -1,6 +0,0 @@
mitmdump: $MITMDUMP
pathod: $PATHOD
pathoc: sleep 2 && $PATHOC $FUZZ_SETTINGS localhost:8080 ./straight_stream_patterns
#pathoc: sleep 2 && $PATHOC localhost:8080 /tmp/err

16
test/mitmproxy/fuzzing/straight_stream_patterns
View File

@ -1,16 +0,0 @@
get:'http://localhost:9999/p/':s'200:b"foo"':ir,'\n'
get:'http://localhost:9999/p/':s'200:b"foo"':ir,'a'
get:'http://localhost:9999/p/':s'200:b"foo"':ir,'9'
get:'http://localhost:9999/p/':s'200:b"foo"':ir,':'
get:'http://localhost:9999/p/':s'200:b"foo"':ir,'"'
get:'http://localhost:9999/p/':s'200:b"foo"':ir,'-'
get:'http://localhost:9999/p/':s'200:b"foo":ir,"\n"'
get:'http://localhost:9999/p/':s'200:b"foo":ir,"a"'
get:'http://localhost:9999/p/':s'200:b"foo":ir,"9"'
get:'http://localhost:9999/p/':s'200:b"foo":ir,":"'
get:'http://localhost:9999/p/':s'200:b"foo":ir,"-"'
get:'http://localhost:9999/p/':s'200:b"foo":dr'
get:'http://localhost:9999/p/':s'200:b"foo"':ir,@2
get:'http://localhost:9999/p/':s'200:b"foo":ir,@2'

6
test/mitmproxy/fuzzing/straight_stream_ssl
View File

@ -1,6 +0,0 @@
mitmdump: $MITMDUMP -q --stream 1
pathod: $PATHOD
pathoc: sleep 2 && $PATHOC $FUZZ_SETTINGS localhost:8080 ./straight_stream_patterns
#pathoc: sleep 2 && $PATHOC localhost:8080 /tmp/err

0
test/mitmproxy/proxy/__init__.py
View File

1
test/mitmproxy/proxy/modes/test_http_proxy.py
View File

@ -1 +0,0 @@
# TODO: write tests

1
test/mitmproxy/proxy/modes/test_reverse_proxy.py
View File

@ -1 +0,0 @@
# TODO: write tests

1
test/mitmproxy/proxy/modes/test_socks_proxy.py
View File

@ -1 +0,0 @@
# TODO: write tests

1
test/mitmproxy/proxy/modes/test_transparent_proxy.py
View File

@ -1 +0,0 @@
# TODO: write tests

0
test/mitmproxy/proxy/protocol/__init__.py
View File

1
test/mitmproxy/proxy/protocol/test_base.py
View File

@ -1 +0,0 @@
# TODO: write tests

1
test/mitmproxy/proxy/protocol/test_http.py
View File

@ -1 +0,0 @@
# TODO: write tests

112
test/mitmproxy/proxy/protocol/test_http1.py
View File

@ -1,112 +0,0 @@
from unittest import mock
import pytest
from mitmproxy.test import tflow
from mitmproxy.net.http import http1
from mitmproxy.net.tcp import TCPClient
from mitmproxy.test.tutils import treq
from ... import tservers
class TestHTTPFlow:
def test_repr(self):
f = tflow.tflow(resp=True, err=True)
assert repr(f)
class TestInvalidRequests(tservers.HTTPProxyTest):
ssl = True
def test_double_connect(self):
p = self.pathoc()
with p.connect():
r = p.request("connect:'{}:{}'".format("127.0.0.1", self.server2.port))
assert r.status_code == 400
assert b"Unexpected CONNECT" in r.content
def test_relative_request(self):
p = self.pathoc_raw()
with p.connect():
r = p.request("get:/p/200")
assert r.status_code == 400
assert b"Invalid HTTP request form" in r.content
class TestProxyMisconfiguration(tservers.TransparentProxyTest):
def test_absolute_request(self):
p = self.pathoc()
with p.connect():
r = p.request("get:'http://localhost:%d/p/200'" % self.server.port)
assert r.status_code == 400
assert b"misconfiguration" in r.content
class TestExpectHeader(tservers.HTTPProxyTest):
def test_simple(self):
client = TCPClient(("127.0.0.1", self.proxy.port))
client.connect()
# call pathod server, wait a second to complete the request
client.wfile.write(
b"POST http://localhost:%d/p/200 HTTP/1.1\r\n"
b"Expect: 100-continue\r\n"
b"Content-Length: 16\r\n"
b"\r\n" % self.server.port
)
client.wfile.flush()
assert client.rfile.readline() == b"HTTP/1.1 100 Continue\r\n"
assert client.rfile.readline() == b"content-length: 0\r\n"
assert client.rfile.readline() == b"\r\n"
client.wfile.write(b"0123456789abcdef\r\n")
client.wfile.flush()
resp = http1.read_response(client.rfile, treq())
assert resp.status_code == 200
client.finish()
client.close()
class TestHeadContentLength(tservers.HTTPProxyTest):
def test_head_content_length(self):
p = self.pathoc()
with p.connect():
resp = p.request(
"""head:'%s/p/200:h"Content-Length"="42"'""" % self.server.urlbase
)
assert resp.headers["Content-Length"] == "42"
class TestStreaming(tservers.HTTPProxyTest):
@pytest.mark.parametrize('streaming', [True, False])
def test_streaming(self, streaming):
class Stream:
def requestheaders(self, f):
f.request.stream = streaming
def responseheaders(self, f):
f.response.stream = streaming
def assert_write(self, v):
if streaming:
assert len(v) <= 4096
return self.o.write(v)
self.master.addons.add(Stream())
p = self.pathoc()
with p.connect():
with mock.patch("mitmproxy.net.tcp.Writer.write", side_effect=assert_write, autospec=True):
# response with 10000 bytes
r = p.request("post:'%s/p/200:b@10000'" % self.server.urlbase)
assert len(r.content) == 10000
# request with 10000 bytes
assert p.request("post:'%s/p/200':b@10000" % self.server.urlbase)

1244
test/mitmproxy/proxy/protocol/test_http2.py
View File

File diff suppressed because it is too large Load Diff

1
test/mitmproxy/proxy/protocol/test_rawtcp.py
View File

@ -1 +0,0 @@
# TODO: write tests

0
test/mitmproxy/proxy/protocol/test_tls.py
View File

510
test/mitmproxy/proxy/protocol/test_websocket.py
View File

@ -1,510 +0,0 @@
import pytest
import os
import struct
import tempfile
import traceback
from wsproto.frame_protocol import Opcode
from mitmproxy import exceptions, options
from mitmproxy.http import HTTPFlow, make_connect_request
from mitmproxy.websocket import WebSocketFlow
from mitmproxy.net import http, tcp, websocket
from pathod.language import websockets_frame
from ...net import tservers as net_tservers
from ... import tservers
class _WebSocketServerBase(net_tservers.ServerTestBase):
class handler(tcp.BaseHandler):
def handle(self):
try:
request = http.http1.read_request(self.rfile)
assert websocket.check_handshake(request.headers)
response = http.Response(
http_version=b"HTTP/1.1",
status_code=101,
reason=http.status_codes.RESPONSES.get(101).encode(),
headers=http.Headers(
connection='upgrade',
upgrade='websocket',
sec_websocket_accept=b'',
sec_websocket_extensions='permessage-deflate' if "permessage-deflate" in request.headers.values() else ''
),
content=b'',
trailers=None,
timestamp_start=0,
timestamp_end=0,
)
self.wfile.write(http.http1.assemble_response(response))
self.wfile.flush()
self.server.handle_websockets(self.rfile, self.wfile)
except:
traceback.print_exc()
class _WebSocketTestBase:
client = None
@classmethod
def setup_class(cls):
cls.options = cls.get_options()
cls.proxy = tservers.ProxyThread(tservers.TestMaster, cls.options)
cls.proxy.start()
@classmethod
def teardown_class(cls):
cls.proxy.shutdown()
@classmethod
def get_options(cls):
opts = options.Options(
listen_port=0,
upstream_cert=True,
ssl_insecure=True,
websocket=True,
)
opts.confdir = os.path.join(tempfile.gettempdir(), "mitmproxy")
return opts
@property
def master(self):
return self.proxy.tmaster
def setup(self):
self.master.reset([])
self.server.server.handle_websockets = self.handle_websockets
def teardown(self):
if self.client:
self.client.close()
def setup_connection(self, extension=False):
self.client = tcp.TCPClient(("127.0.0.1", self.proxy.port))
self.client.connect()
request = make_connect_request(("127.0.0.1", self.server.server.address[1]))
self.client.wfile.write(http.http1.assemble_request(request))
self.client.wfile.flush()
response = http.http1.read_response(self.client.rfile, request)
if self.ssl:
self.client.convert_to_tls()
assert self.client.tls_established
request = http.Request(
host="127.0.0.1",
port=self.server.server.address[1],
method=b"GET",
scheme=b"http",
authority=b"",
path=b"/ws",
http_version=b"HTTP/1.1",
headers=http.Headers(
connection="upgrade",
upgrade="websocket",
sec_websocket_version="13",
sec_websocket_key="1234",
sec_websocket_extensions="permessage-deflate" if extension else ""
),
content=b'',
trailers=None,
timestamp_start=0,
timestamp_end=0,
)
self.client.wfile.write(http.http1.assemble_request(request))
self.client.wfile.flush()
response = http.http1.read_response(self.client.rfile, request)
assert websocket.check_handshake(response.headers)
class _WebSocketTest(_WebSocketTestBase, _WebSocketServerBase):
@classmethod
def setup_class(cls):
_WebSocketTestBase.setup_class()
_WebSocketServerBase.setup_class(ssl=cls.ssl)
@classmethod
def teardown_class(cls):
_WebSocketTestBase.teardown_class()
_WebSocketServerBase.teardown_class()
class TestSimple(_WebSocketTest):
@classmethod
def handle_websockets(cls, rfile, wfile):
wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=Opcode.TEXT, payload=b'server-foobar')))
wfile.flush()
header, frame, _ = websocket.read_frame(rfile)
wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=header.opcode, payload=frame.payload)))
wfile.flush()
header, frame, _ = websocket.read_frame(rfile)
wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=header.opcode, payload=frame.payload)))
wfile.flush()
@pytest.mark.parametrize('streaming', [True, False])
def test_simple(self, streaming):
class Stream:
def websocket_start(self, f):
f.stream = streaming
self.proxy.set_addons(Stream())
self.setup_connection()
_, frame, _ = websocket.read_frame(self.client.rfile)
assert frame.payload == b'server-foobar'
self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.TEXT, payload=b'self.client-foobar')))
self.client.wfile.flush()
_, frame, _ = websocket.read_frame(self.client.rfile)
assert frame.payload == b'self.client-foobar'
self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.BINARY, payload=b'\xde\xad\xbe\xef')))
self.client.wfile.flush()
_, frame, _ = websocket.read_frame(self.client.rfile)
assert frame.payload == b'\xde\xad\xbe\xef'
self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.CLOSE)))
self.client.wfile.flush()
assert len(self.master.state.flows) == 2
assert isinstance(self.master.state.flows[0], HTTPFlow)
assert isinstance(self.master.state.flows[1], WebSocketFlow)
assert len(self.master.state.flows[1].messages) == 5
assert self.master.state.flows[1].messages[0].content == 'server-foobar'
assert self.master.state.flows[1].messages[0].type == Opcode.TEXT
assert self.master.state.flows[1].messages[1].content == 'self.client-foobar'
assert self.master.state.flows[1].messages[1].type == Opcode.TEXT
assert self.master.state.flows[1].messages[2].content == 'self.client-foobar'
assert self.master.state.flows[1].messages[2].type == Opcode.TEXT
assert self.master.state.flows[1].messages[3].content == b'\xde\xad\xbe\xef'
assert self.master.state.flows[1].messages[3].type == Opcode.BINARY
assert self.master.state.flows[1].messages[4].content == b'\xde\xad\xbe\xef'
assert self.master.state.flows[1].messages[4].type == Opcode.BINARY
def test_change_payload(self):
class Addon:
def websocket_message(self, f):
f.messages[-1].content = "foo"
self.proxy.set_addons(Addon())
self.setup_connection()
_, frame, _ = websocket.read_frame(self.client.rfile)
assert frame.payload == b'foo'
self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.TEXT, payload=b'self.client-foobar')))
self.client.wfile.flush()
_, frame, _ = websocket.read_frame(self.client.rfile)
assert frame.payload == b'foo'
self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.BINARY, payload=b'\xde\xad\xbe\xef')))
self.client.wfile.flush()
_, frame, _ = websocket.read_frame(self.client.rfile)
assert frame.payload == b'foo'
class TestKillFlow(_WebSocketTest):
@classmethod
def handle_websockets(cls, rfile, wfile):
wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=Opcode.TEXT, payload=b'server-foobar')))
wfile.flush()
def test_kill(self):
class KillFlow:
def websocket_message(self, f):
f.kill()
self.proxy.set_addons(KillFlow())
self.setup_connection()
with pytest.raises(exceptions.TcpDisconnect):
_ = websocket.read_frame(self.client.rfile, False)
class TestSimpleTLS(_WebSocketTest):
ssl = True
@classmethod
def handle_websockets(cls, rfile, wfile):
wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=Opcode.TEXT, payload=b'server-foobar')))
wfile.flush()
header, frame, _ = websocket.read_frame(rfile)
wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=header.opcode, payload=frame.payload)))
wfile.flush()
def test_simple_tls(self):
self.setup_connection()
_, frame, _ = websocket.read_frame(self.client.rfile)
assert frame.payload == b'server-foobar'
self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.TEXT, payload=b'self.client-foobar')))
self.client.wfile.flush()
_, frame, _ = websocket.read_frame(self.client.rfile)
assert frame.payload == b'self.client-foobar'
self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.CLOSE)))
self.client.wfile.flush()
class TestPing(_WebSocketTest):
@classmethod
def handle_websockets(cls, rfile, wfile):
wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=Opcode.PING, payload=b'foobar')))
wfile.flush()
header, frame, _ = websocket.read_frame(rfile)
assert header.opcode == Opcode.PONG
assert frame.payload == b'foobar'
wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=Opcode.PONG, payload=b'done')))
wfile.flush()
wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=Opcode.CLOSE)))
wfile.flush()
_ = websocket.read_frame(rfile, False)
@pytest.mark.asyncio
async def test_ping(self):
self.setup_connection()
header, frame, _ = websocket.read_frame(self.client.rfile)
_ = websocket.read_frame(self.client.rfile, False)
self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.CLOSE)))
self.client.wfile.flush()
assert header.opcode == Opcode.PING
assert frame.payload == b'' # We don't send payload to other end
assert await self.master.await_log("Pong Received from server", "info")
class TestPong(_WebSocketTest):
@classmethod
def handle_websockets(cls, rfile, wfile):
header, frame, _ = websocket.read_frame(rfile)
assert header.opcode == Opcode.PING
assert frame.payload == b''
wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=Opcode.PONG, payload=frame.payload)))
wfile.flush()
wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=Opcode.CLOSE)))
wfile.flush()
_ = websocket.read_frame(rfile)
@pytest.mark.asyncio
async def test_pong(self):
self.setup_connection()
self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.PING, payload=b'foobar')))
self.client.wfile.flush()
header, frame, _ = websocket.read_frame(self.client.rfile)
_ = websocket.read_frame(self.client.rfile)
self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.CLOSE)))
self.client.wfile.flush()
assert header.opcode == Opcode.PONG
assert frame.payload == b'foobar'
assert await self.master.await_log("pong received")
class TestClose(_WebSocketTest):
@classmethod
def handle_websockets(cls, rfile, wfile):
header, frame, _ = websocket.read_frame(rfile)
wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=header.opcode, payload=frame.payload)))
wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=Opcode.CLOSE)))
wfile.flush()
with pytest.raises(exceptions.TcpDisconnect):
_ = websocket.read_frame(rfile)
def test_close(self):
self.setup_connection()
self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.CLOSE)))
self.client.wfile.flush()
_ = websocket.read_frame(self.client.rfile)
with pytest.raises(exceptions.TcpDisconnect):
_ = websocket.read_frame(self.client.rfile)
def test_close_payload_1(self):
self.setup_connection()
self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.CLOSE, payload=b'\00\42')))
self.client.wfile.flush()
_ = websocket.read_frame(self.client.rfile)
with pytest.raises(exceptions.TcpDisconnect):
_ = websocket.read_frame(self.client.rfile)
def test_close_payload_2(self):
self.setup_connection()
self.client.wfile.write(bytes(websockets_frame.Frame(fin=1, mask=1, opcode=Opcode.CLOSE, payload=b'\00\42foobar')))
self.client.wfile.flush()
_ = websocket.read_frame(self.client.rfile)
with pytest.raises(exceptions.TcpDisconnect):
_ = websocket.read_frame(self.client.rfile)
class TestInvalidFrame(_WebSocketTest):
@classmethod
def handle_websockets(cls, rfile, wfile):
wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=15, payload=b'foobar')))
wfile.flush()
def test_invalid_frame(self):
self.setup_connection()
_, frame, _ = websocket.read_frame(self.client.rfile)
code, = struct.unpack('!H', frame.payload[:2])
assert code == 1002
assert frame.payload[2:].startswith(b'Invalid opcode')
class TestStreaming(_WebSocketTest):
@classmethod
def handle_websockets(cls, rfile, wfile):
wfile.write(bytes(websockets_frame.Frame(opcode=Opcode.TEXT, payload=b'server-foobar')))
wfile.flush()
@pytest.mark.parametrize('streaming', [True, False])
def test_streaming(self, streaming):
class Stream:
def websocket_start(self, f):
f.stream = streaming
self.proxy.set_addons(Stream())
self.setup_connection()
frame = None
if not streaming:
with pytest.raises(exceptions.TcpDisconnect): # Reader.safe_read get nothing as result
_, frame, _ = websocket.read_frame(self.client.rfile)
assert frame is None
else:
_, frame, _ = websocket.read_frame(self.client.rfile)
assert frame
assert self.master.state.flows[1].messages == [] # Message not appended as the final frame isn't received
class TestExtension(_WebSocketTest):
@classmethod
def handle_websockets(cls, rfile, wfile):
wfile.write(b'\xc1\x0f*N-*K-\xd2M\xcb\xcfOJ,\x02\x00')
wfile.flush()
header, _, _ = websocket.read_frame(rfile)
assert header.rsv.rsv1
wfile.write(b'\xc1\nJ\xce\xc9L\xcd+\x81r\x00\x00')
wfile.flush()
header, _, _ = websocket.read_frame(rfile)
assert header.rsv.rsv1
wfile.write(b'\xc2\x07\xba\xb7v\xdf{\x00\x00')
wfile.flush()
def test_extension(self):
self.setup_connection(True)
header, _, _ = websocket.read_frame(self.client.rfile)
assert header.rsv.rsv1
self.client.wfile.write(b'\xc1\x8fQ\xb7vX\x1by\xbf\x14\x9c\x9c\xa7\x15\x9ax9\x12}\xb5v')
self.client.wfile.flush()
header, _, _ = websocket.read_frame(self.client.rfile)
assert header.rsv.rsv1
self.client.wfile.write(b'\xc2\x87\xeb\xbb\x0csQ\x0cz\xac\x90\xbb\x0c')
self.client.wfile.flush()
header, _, _ = websocket.read_frame(self.client.rfile)
assert header.rsv.rsv1
assert len(self.master.state.flows[1].messages) == 5
assert self.master.state.flows[1].messages[0].content == 'server-foobar'
assert self.master.state.flows[1].messages[0].type == Opcode.TEXT
assert self.master.state.flows[1].messages[1].content == 'client-foobar'
assert self.master.state.flows[1].messages[1].type == Opcode.TEXT
assert self.master.state.flows[1].messages[2].content == 'client-foobar'
assert self.master.state.flows[1].messages[2].type == Opcode.TEXT
assert self.master.state.flows[1].messages[3].content == b'\xde\xad\xbe\xef'
assert self.master.state.flows[1].messages[3].type == Opcode.BINARY
assert self.master.state.flows[1].messages[4].content == b'\xde\xad\xbe\xef'
assert self.master.state.flows[1].messages[4].type == Opcode.BINARY
class TestInjectMessageClient(_WebSocketTest):
@classmethod
def handle_websockets(cls, rfile, wfile):
pass
def test_inject_message_client(self):
class Inject:
def websocket_start(self, flow):
flow.inject_message(flow.client_conn, 'This is an injected message!')
self.proxy.set_addons(Inject())
self.setup_connection()
header, frame, _ = websocket.read_frame(self.client.rfile)
assert header.opcode == Opcode.TEXT
assert frame.payload == b'This is an injected message!'
class TestInjectMessageServer(_WebSocketTest):
@classmethod
def handle_websockets(cls, rfile, wfile):
header, frame, _ = websocket.read_frame(rfile)
assert header.opcode == Opcode.TEXT
success = frame.payload == b'This is an injected message!'
wfile.write(bytes(websockets_frame.Frame(fin=1, opcode=Opcode.TEXT, payload=str(success).encode())))
wfile.flush()
def test_inject_message_server(self):
class Inject:
def websocket_start(self, flow):
flow.inject_message(flow.server_conn, 'This is an injected message!')
self.proxy.set_addons(Inject())
self.setup_connection()
header, frame, _ = websocket.read_frame(self.client.rfile)
assert header.opcode == Opcode.TEXT
assert frame.payload == b'True'

28
test/mitmproxy/proxy/test_config.py
View File

@ -1,28 +0,0 @@
import pytest
from mitmproxy import options
from mitmproxy import exceptions
from mitmproxy.proxy.config import ProxyConfig
class TestProxyConfig:
def test_invalid_confdir(self):
opts = options.Options()
opts.confdir = "foo"
with pytest.raises(exceptions.OptionsError, match="parent directory does not exist"):
ProxyConfig(opts)
def test_invalid_certificate(self, tdata):
opts = options.Options()
opts.certs = [tdata.path("mitmproxy/data/dumpfile-011.bin")]
with pytest.raises(exceptions.OptionsError, match="Invalid certificate format"):
ProxyConfig(opts)
def test_cannot_set_both_allow_and_filter_options(self):
opts = options.Options()
opts.ignore_hosts = ["foo"]
opts.allow_hosts = ["bar"]
with pytest.raises(exceptions.OptionsError, match="--ignore-hosts and --allow-hosts are "
"mutually exclusive; please choose "
"one."):
ProxyConfig(opts)

1
test/mitmproxy/proxy/test_root_context.py
View File

@ -1 +0,0 @@
# TODO: write tests

1131
test/mitmproxy/proxy/test_server.py
View File

File diff suppressed because it is too large Load Diff

30
test/mitmproxy/test_connections.py
View File

@ -11,7 +11,6 @@ from mitmproxy.net import tcp
from mitmproxy.net.http import http1
from mitmproxy.test import tflow
from .net import tservers
from pathod import test
from ..conftest import skip_new_proxy_core
@ -135,35 +134,6 @@ class TestServerConnection:
c = connections.ServerConnection.make_dummy(('foobar', 1234))
assert c.address == ('foobar', 1234)
def test_simple(self):
d = test.Daemon()
c = connections.ServerConnection((d.IFACE, d.port))
c.connect()
f = tflow.tflow()
f.server_conn = c
f.request.path = "/p/200:da"
# use this protocol just to assemble - not for actual sending
c.wfile.write(http1.assemble_request(f.request))
c.wfile.flush()
assert http1.read_response(c.rfile, f.request, 1000)
assert d.last_log()
c.finish()
c.close()
d.shutdown()
def test_terminate_error(self):
d = test.Daemon()
c = connections.ServerConnection((d.IFACE, d.port))
c.connect()
c.close()
c.connection = mock.Mock()
c.connection.recv = mock.Mock(return_value=False)
c.connection.flush = mock.Mock(side_effect=exceptions.TcpDisconnect)
d.shutdown()
def test_sni(self):
c = connections.ServerConnection(('', 1234))
with pytest.raises(ValueError, match='sni must be str, not '):

28
test/mitmproxy/test_fuzzing.py
View File

@ -1,28 +0,0 @@
from . import tservers
"""
A collection of errors turned up by fuzzing. Errors are integrated here
after being fixed to check for regressions.
"""
class TestFuzzy(tservers.HTTPProxyTest):
def test_idna_err(self):
req = r'get:"http://localhost:%s":i10,"\xc6"'
p = self.pathoc()
with p.connect():
assert p.request(req % self.server.port).status_code == 400
def test_nullbytes(self):
req = r'get:"http://localhost:%s":i19,"\x00"'
p = self.pathoc()
with p.connect():
assert p.request(req % self.server.port).status_code == 400
def test_invalid_ipv6_url(self):
req = 'get:"http://localhost:%s":i13,"["'
p = self.pathoc()
with p.connect():
resp = p.request(req % self.server.port)
assert resp.status_code == 400

387
test/mitmproxy/tservers.py
View File

@ -1,387 +0,0 @@
import os.path
import threading
import tempfile
import sys
import time
from unittest import mock
import asyncio
import mitmproxy.platform
from mitmproxy.addons import core
from mitmproxy.proxy.config import ProxyConfig
from mitmproxy.proxy.server import ProxyServer
from mitmproxy import controller
from mitmproxy import options
from mitmproxy import exceptions
from mitmproxy import io
from mitmproxy.utils import human
import pathod.test
import pathod.pathoc
from mitmproxy import eventsequence
from mitmproxy.test import tflow
from mitmproxy.test import tutils
from mitmproxy.test import taddons
class MasterTest:
async def cycle(self, master, content):
f = tflow.tflow(req=tutils.treq(content=content))
layer = mock.Mock("mitmproxy.proxy.protocol.base.Layer")
layer.client_conn = f.client_conn
layer.reply = controller.DummyReply()
await master.addons.handle_lifecycle("clientconnect", layer)
for i in eventsequence.iterate(f):
await master.addons.handle_lifecycle(*i)
await master.addons.handle_lifecycle("clientdisconnect", layer)
return f
async def dummy_cycle(self, master, n, content):
for i in range(n):
await self.cycle(master, content)
await master._shutdown()
def flowfile(self, path):
with open(path, "wb") as f:
fw = io.FlowWriter(f)
t = tflow.tflow(resp=True)
fw.add(t)
class TestState:
def __init__(self):
self.flows = []
def request(self, f):
if f not in self.flows:
self.flows.append(f)
def response(self, f):
if f not in self.flows:
self.flows.append(f)
def websocket_start(self, f):
if f not in self.flows:
self.flows.append(f)
class TestMaster(taddons.RecordingMaster):
def __init__(self, opts):
super().__init__(opts)
config = ProxyConfig(opts)
self.server = ProxyServer(config)
def clear_addons(self, addons):
self.addons.clear()
self.state = TestState()
self.addons.add(self.state)
self.addons.add(*addons)
self.addons.trigger("configure", self.options.keys())
self.addons.trigger("running")
def reset(self, addons):
self.clear_addons(addons)
self.clear()
class ProxyThread(threading.Thread):
def __init__(self, masterclass, options):
threading.Thread.__init__(self)
self.masterclass = masterclass
self.options = options
self.tmaster = None
self.event_loop = None
controller.should_exit = False
@property
def port(self):
return self.tmaster.server.address[1]
@property
def tlog(self):
return self.tmaster.logs
def shutdown(self):
self.tmaster.shutdown()
def run(self):
self.event_loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.event_loop)
self.tmaster = self.masterclass(self.options)
self.tmaster.addons.add(core.Core())
self.name = "ProxyThread (%s)" % human.format_address(self.tmaster.server.address)
self.tmaster.run()
def set_addons(self, *addons):
self.tmaster.reset(addons)
def start(self):
super().start()
while True:
if self.tmaster:
break
time.sleep(0.01)
class ProxyTestBase:
# Test Configuration
ssl = None
ssloptions = False
masterclass = TestMaster
add_upstream_certs_to_client_chain = False
@classmethod
def setup_class(cls):
cls.server = pathod.test.Daemon(
ssl=cls.ssl,
ssloptions=cls.ssloptions)
cls.server2 = pathod.test.Daemon(
ssl=cls.ssl,
ssloptions=cls.ssloptions)
cls.options = cls.get_options()
cls.proxy = ProxyThread(cls.masterclass, cls.options)
cls.proxy.start()
@classmethod
def teardown_class(cls):
# perf: we want to run tests in parallel
# should this ever cause an error, travis should catch it.
# shutil.rmtree(cls.confdir)
cls.proxy.shutdown()
cls.server.shutdown()
cls.server2.shutdown()
def teardown(self):
try:
self.server.wait_for_silence()
except exceptions.Timeout:
# FIXME: Track down the Windows sync issues
if sys.platform != "win32":
raise
def setup(self):
self.master.reset(self.addons())
self.server.clear_log()
self.server2.clear_log()
@property
def master(self):
return self.proxy.tmaster
@classmethod
def get_options(cls):
cls.confdir = os.path.join(tempfile.gettempdir(), "mitmproxy")
return options.Options(
listen_port=0,
confdir=cls.confdir,
add_upstream_certs_to_client_chain=cls.add_upstream_certs_to_client_chain,
ssl_insecure=True,
)
def set_addons(self, *addons):
self.proxy.set_addons(*addons)
def addons(self):
"""
Can be over-ridden to add a standard set of addons to tests.
"""
return []
class LazyPathoc(pathod.pathoc.Pathoc):
def __init__(self, lazy_connect, *args, **kwargs):
self.lazy_connect = lazy_connect
pathod.pathoc.Pathoc.__init__(self, *args, **kwargs)
def connect(self):
return pathod.pathoc.Pathoc.connect(self, self.lazy_connect)
class HTTPProxyTest(ProxyTestBase):
def pathoc_raw(self):
return pathod.pathoc.Pathoc(("127.0.0.1", self.proxy.port), fp=None)
def pathoc(self, sni=None):
"""
Returns a connected Pathoc instance.
"""
if self.ssl:
conn = ("127.0.0.1", self.server.port)
else:
conn = None
return LazyPathoc(
conn,
("localhost", self.proxy.port), ssl=self.ssl, sni=sni, fp=None
)
def pathod(self, spec, sni=None):
"""
Constructs a pathod GET request, with the appropriate base and proxy.
"""
p = self.pathoc(sni=sni)
if self.ssl:
q = "get:'/p/%s'" % spec
else:
q = f"get:'{self.server.urlbase}/p/{spec}'"
with p.connect():
return p.request(q)
def app(self, page):
if self.ssl:
p = pathod.pathoc.Pathoc(
("127.0.0.1", self.proxy.port), True, fp=None
)
with p.connect((self.master.options.onboarding_host, self.master.options.onbarding_port)):
return p.request("get:'%s'" % page)
else:
p = self.pathoc()
with p.connect():
return p.request(f"get:'http://{self.master.options.onboarding_host}{page}'")
class TransparentProxyTest(ProxyTestBase):
ssl = None
@classmethod
def setup_class(cls):
cls._init_transparent_mode = mitmproxy.platform.init_transparent_mode
cls._original_addr = mitmproxy.platform.original_addr
mitmproxy.platform.init_transparent_mode = lambda: True
mitmproxy.platform.original_addr = lambda sock: ("127.0.0.1", cls.server.port)
super().setup_class()
@classmethod
def teardown_class(cls):
super().teardown_class()
mitmproxy.platform.init_transparent_mode = cls._init_transparent_mode
mitmproxy.platform.original_addr = cls._original_addr
@classmethod
def get_options(cls):
opts = ProxyTestBase.get_options()
opts.mode = "transparent"
return opts
def pathod(self, spec, sni=None):
"""
Constructs a pathod GET request, with the appropriate base and proxy.
"""
if self.ssl:
p = self.pathoc(sni=sni)
q = "get:'/p/%s'" % spec
else:
p = self.pathoc()
q = "get:'/p/%s'" % spec
with p.connect():
return p.request(q)
def pathoc(self, sni=None):
"""
Returns a connected Pathoc instance.
"""
p = pathod.pathoc.Pathoc(
("localhost", self.proxy.port), ssl=self.ssl, sni=sni, fp=None
)
return p
class ReverseProxyTest(ProxyTestBase):
ssl = None
@classmethod
def get_options(cls):
opts = ProxyTestBase.get_options()
s = "".join(
[
"https" if cls.ssl else "http",
"://",
"127.0.0.1:",
str(cls.server.port)
]
)
opts.mode = "reverse:" + s
return opts
def pathoc(self, sni=None):
"""
Returns a connected Pathoc instance.
"""
p = pathod.pathoc.Pathoc(
("localhost", self.proxy.port), ssl=self.ssl, sni=sni, fp=None
)
return p
def pathod(self, spec, sni=None):
"""
Constructs a pathod GET request, with the appropriate base and proxy.
"""
if self.ssl:
p = self.pathoc(sni=sni)
q = "get:'/p/%s'" % spec
else:
p = self.pathoc()
q = "get:'/p/%s'" % spec
with p.connect():
return p.request(q)
class SocksModeTest(HTTPProxyTest):
@classmethod
def get_options(cls):
opts = ProxyTestBase.get_options()
opts.mode = "socks5"
return opts
class HTTPUpstreamProxyTest(HTTPProxyTest):
"""
Chain three instances of mitmproxy in a row to test upstream mode.
Proxy order is cls.proxy -> cls.chain[0] -> cls.chain[1]
cls.proxy and cls.chain[0] are in upstream mode,
cls.chain[1] is in regular mode.
"""
chain = None
n = 2
@classmethod
def setup_class(cls):
# We need to initialize the chain first so that the normal server gets a correct config.
cls.chain = []
for _ in range(cls.n):
opts = cls.get_options()
proxy = ProxyThread(cls.masterclass, opts)
proxy.start()
cls.chain.insert(0, proxy)
while True:
if proxy.event_loop and proxy.event_loop.is_running():
break
super().setup_class()
@classmethod
def teardown_class(cls):
super().teardown_class()
for proxy in cls.chain:
proxy.shutdown()
def setup(self):
super().setup()
for proxy in self.chain:
proxy.tmaster.reset(self.addons())
@classmethod
def get_options(cls):
opts = super().get_options()
if cls.chain: # First proxy is in normal mode.
s = "http://127.0.0.1:%s" % cls.chain[0].port
opts.update(
mode="upstream:" + s,
)
return opts

0
test/pathod/__init__.py
View File

3
test/pathod/data/clientcert/.gitignore vendored
View File

@ -1,3 +0,0 @@
client.crt
client.key
client.req

5
test/pathod/data/clientcert/client.cnf
View File

@ -1,5 +0,0 @@
[ ssl_client ]
basicConstraints = CA:FALSE
nsCertType = client
keyUsage = digitalSignature, keyEncipherment
extendedKeyUsage = clientAuth

42
test/pathod/data/clientcert/client.pem
View File

@ -1,42 +0,0 @@
-----BEGIN RSA PRIVATE KEY-----
MIIEpAIBAAKCAQEAzCpoRjSTfIN24kkNap/GYmP9zVWj0Gk8R5BB/PvvN0OB1Zk0
EEYPsWCcuhEdK0ehiDZX030doF0DOncKKa6mop/d0x2o+ts42peDhZM6JNUrm6d+
ZWQVtio33mpp77UMhR093vaA+ExDnmE26kBTVijJ1+fRAVDXG/cmQINEri91Kk/G
3YJ5e45UrohGI5seBZ4vV0xbHtmczFRhYFlGOvYsoIe4Lvz/eFS2pIrTIpYQ2VM/
SQQl+JFy+NlQRsWG2NrxtKOzMnnDE7YN4I3z5D5eZFo1EtwZ48LNCeSwrEOdfuzP
G5q5qbs5KpE/x85H9umuRwSCIArbMwBYV8a8JwIDAQABAoIBAFE3FV/IDltbmHEP
iky93hbJm+6QgKepFReKpRVTyqb7LaygUvueQyPWQMIriKTsy675nxo8DQr7tQsO
y3YlSZgra/xNMikIB6e82c7K8DgyrDQw/rCqjZB3Xt4VCqsWJDLXnQMSn98lx0g7
d7Lbf8soUpKWXqfdVpSDTi4fibSX6kshXyfSTpcz4AdoncEpViUfU1xkEEmZrjT8
1GcCsDC41xdNmzCpqRuZX7DKSFRoB+0hUzsC1oiqM7FD5kixonRd4F5PbRXImIzt
6YCsT2okxTA04jX7yByis7LlOLTlkmLtKQYuc3erOFvwx89s4vW+AeFei+GGNitn
tHfSwbECgYEA7SzV+nN62hAERHlg8cEQT4TxnsWvbronYWcc/ev44eHSPDWL5tPi
GHfSbW6YAq5Wa0I9jMWfXyhOYEC3MZTC5EEeLOB71qVrTwcy/sY66rOrcgjFI76Q
5JFHQ4wy3SWU50KxE0oWJO9LIowprG+pW1vzqC3VF0T7q0FqESrY4LUCgYEA3F7Z
80ndnCUlooJAb+Hfotv7peFf1o6+m1PTRcz1lLnVt5R5lXj86kn+tXEpYZo1RiGR
2rE2N0seeznWCooakHcsBN7/qmFIhhooJNF7yW+JP2I4P2UV5+tJ+8bcs/voUkQD
1x+rGOuMn8nvHBd2+Vharft8eGL2mgooPVI2XusCgYEAlMZpO3+w8pTVeHaDP2MR
7i/AuQ3cbCLNjSX3Y7jgGCFllWspZRRIYXzYPNkA9b2SbBnTLjjRLgnEkFBIGgvs
7O2EFjaCuDRvydUEQhjq4ErwIsopj7B8h0QyZcbOKTbn3uFQ3n68wVJx2Sv/ADHT
FIHrp/WIE96r19Niy34LKXkCgYB2W59VsuOKnMz01l5DeR5C+0HSWxS9SReIl2IO
yEFSKullWyJeLIgyUaGy0990430feKI8whcrZXYumuah7IDN/KOwzhCk8vEfzWao
N7bzfqtJVrh9HA7C7DVlO+6H4JFrtcoWPZUIomJ549w/yz6EN3ckoMC+a/Ck1TW9
ka1QFwKBgQCywG6TrZz0UmOjyLQZ+8Q4uvZklSW5NAKBkNnyuQ2kd5rzyYgMPE8C
Er8T88fdVIKvkhDyHhwcI7n58xE5Gr7wkwsrk/Hbd9/ZB2GgAPY3cATskK1v1McU
YeX38CU0fUS4aoy26hWQXkViB47IGQ3jWo3ZCtzIJl8DI9/RsBWTnw==
-----END RSA PRIVATE KEY-----
-----BEGIN CERTIFICATE-----
MIICYDCCAckCAQEwDQYJKoZIhvcNAQEFBQAwKDESMBAGA1UEAxMJbWl0bXByb3h5
MRIwEAYDVQQKEwltaXRtcHJveHkwHhcNMTMwMTIwMDEwODEzWhcNMTUxMDE3MDEw
ODEzWjBFMQswCQYDVQQGEwJBVTETMBEGA1UECBMKU29tZS1TdGF0ZTEhMB8GA1UE
ChMYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOC
AQ8AMIIBCgKCAQEAzCpoRjSTfIN24kkNap/GYmP9zVWj0Gk8R5BB/PvvN0OB1Zk0
EEYPsWCcuhEdK0ehiDZX030doF0DOncKKa6mop/d0x2o+ts42peDhZM6JNUrm6d+
ZWQVtio33mpp77UMhR093vaA+ExDnmE26kBTVijJ1+fRAVDXG/cmQINEri91Kk/G
3YJ5e45UrohGI5seBZ4vV0xbHtmczFRhYFlGOvYsoIe4Lvz/eFS2pIrTIpYQ2VM/
SQQl+JFy+NlQRsWG2NrxtKOzMnnDE7YN4I3z5D5eZFo1EtwZ48LNCeSwrEOdfuzP
G5q5qbs5KpE/x85H9umuRwSCIArbMwBYV8a8JwIDAQABMA0GCSqGSIb3DQEBBQUA
A4GBAFvI+cd47B85PQ970n2dU/PlA2/Hb1ldrrXh2guR4hX6vYx/uuk5yRI/n0Rd
KOXJ3czO0bd2Fpe3ZoNpkW0pOSDej/Q+58ScuJd0gWCT/Sh1eRk6ZdC0kusOuWoY
bPOPMkG45LPgUMFOnZEsfJP6P5mZIxlbCvSMFC25nPHWlct7
-----END CERTIFICATE-----

8
test/pathod/data/clientcert/make
View File

@ -1,8 +0,0 @@
#!/bin/sh
openssl genrsa -out client.key 2048
openssl req -key client.key -new -out client.req
openssl x509 -req -days 365 -in client.req -signkey client.key -out client.crt -extfile client.cnf -extensions ssl_client
openssl x509 -req -days 1000 -in client.req -CA ~/.mitmproxy/mitmproxy-ca.pem -CAkey ~/.mitmproxy/mitmproxy-ca.pem -set_serial 00001 -out client.crt -extensions ssl_client
cat client.key client.crt > client.pem
openssl x509 -text -noout -in client.pem

1
test/pathod/data/file
View File

@ -1 +0,0 @@
testfile

1
test/pathod/data/request
View File

@ -1 +0,0 @@
get:/foo

1
test/pathod/data/response
View File

@ -1 +0,0 @@
202

68
test/pathod/data/testkey.pem
View File

@ -1,68 +0,0 @@
-----BEGIN RSA PRIVATE KEY-----
MIIG5QIBAAKCAYEAwvtKxoZvBV2AxPAkCx8PXbuE7KeqK9bBvk8x+JchPMdf/KZj
sdu2v6Gm8Hi053i7ZGxouFvonJxHAiK6cwk9OYQwa9fbOFf2mgWKEBO4fbCH93tW
DCTdWVxFyNViAvxGHlJs3/IU03pIG29AgUnhRW8pGbabAfx8emcOZJZ3ykEuimaC
4s7mRwdc63GXnbcjTtRkrJsBATI+xvPwuR2+4daX7sPCf0kel3bN2jMpwXfvk/Ww
kJ2BIEeZCg0qIvyMjH9qrUirUnsmQnpPln0CGBbQEBsW9yMfGoFdREiMYls5jZeq
NxjWNv1RTRIm/4RjMwyxnoTA9eDS9wwO2NnJS4vfXAnUTP4BYx8Pe4ZMA2Gm6YrC
ysT6YA1xdHNpcuHXClxwmPj/cm8Z5kIg5clbNIK60ts9yFr/Ao3KPPYJ2GBv8/Oe
ApPBJuubews+/9/13Ew/SJ1t2u28+sPbgXUG8dC2n4vWTvJwKf6Duqxgnm82zdzj
SZoXRQsP984qiN7NAgMBAAECggGBALB6rqWdzCL5DLI0AQun40qdjaR95UKksNvF
5p7we379nl2ZZKb5DSHJ+MWzG1pfJo2wqeAkIBiQQp0mPcgdVrMWeJVD3QHUbDng
RaRjlRr+izJvCeUYANj+8ZLjwECfgf+z7yOLg1oeVeGvAp2C90jXYkYJx6c2lpxb
ZuWYY3hHIw7V1iXfywIDIhFg0TBJMMYK68xmx7QDfFqrNPj4eWsDxqSvvv1iezPw
rkWPBX49RjWPrW5XgSZsZ5J3c+oS1rZmIY7EAgopTWB/3wJjZR1Idz/9l9LIWlBP
6zVC27CIZzSEeGguqNVeyzJ0TPWh5idYNRmSZr6eTUF0245LNO/gqvWKgRSNIZko
HoBa2F1AvCiB67S1kxjwS5y3VkudZE4jkgGKcC2Ws/9QmOZ0HAsjI8VAMp2hj6iN
0HdPMTNtsLgbhKyXsoZuW4YmwfSTPxGi2gvcI7GUozpTz84n1cOneJnz1ygx6Uru
v8DpQg+VX6xTy4X6AK1F8OYNMZ/jaQKBwQDv30NevQStnGbTmcSr+0fd4rmWFklK
V6B2X7zWynVpSGvCWkqVSp3mG6aiZItAltVMRL/9LT6zIheyClyd+vXIjR2+W210
XMxrvz7A8qCXkvB2dyEhrMdCfZ7p8+kf+eD2c/Mnxb7VpmDfHYLx30JeQoBwjrwU
Eul+dE1P+r8bWBaLTjlsipTya74yItWWAToXAo+s1BXBtXhEsLoe4FghlC0u724d
ucjDaeICdLcerApdvg6Q6p4kVHaoF6ka6I8CgcEA0Bdc05ery9gLC6CclV+BhA5Q
dfDq2P7qhc7e1ipwNRrQo2gy5HhgOkTL3dJWc+8rV6CBP/JfchnsW40tDOnPCTLT
gg3n7vv3RHrtncApXuhIFR+B5xjohTPBzxRUMiAOre2d0F5b6eBXFjptf/1i2tQ+
qdqJoyOGOZP0hKVslGIfz+CKc6WEkIqX7c91Msdr5myeaWDI5TsurfuKRBH395T3
BMAi6oinAAEb1rdySenLO2A/0kVmBVlTpaN3TNjjAoHBAMvS4uQ1qSv8okNbfgrF
UqPwa9JkzZImM2tinovFLU9xAl/7aTTCWrmU9Vs4JDuV71kHcjwnngeJCKl4tIpp
HUB06Lk/5xnhYLKNpz087cjeSwXe5IBA2HBfXhFd+NH6+nVwwUUieq4A2n+8C/CK
zVJbH9iE8Lv99fpFyQwU/R63EzD8Hz9j4ny7oLnpb6QvFrVGr98jt/kJwlBb+0sR
RtIBnwMq4F7R5w5lgm6jzpZ5ibVuMeJh+k7Ulp7uu/rpcQKBwQDE3sWIvf7f7PaO
OpbJz0CmYjCHVLWrNIlGrPAv6Jid9U+cuXEkrCpGFl5V77CxIH59+bEuga0BMztl
ZkxP4khoqHhom6VpeWJ3nGGAFJRPYS0JJvTsYalilBPxSYdaoO+iZ6MdxpfozcE2
m3KLW3uSEqlyYvpCqNJNWQhGEoeGXstADWyPevHPGgAhElwL/ZW8u9inU9Tc4sAI
BGnMer+BsaJ+ERU3lK+Clony+z2aZiFLfIUE93lM6DT2CZBN2QcCgcAVk4L0bfA6
HFnP/ZWNlnYWpOVFKcq57PX+J5/k7Tf34e2cYM2P0eqYggWZbzVd8qoCOQCHrAx0
aZSSvEyKAVvzRNeqbm1oXaMojksMnrSX5henHjPbZlr1EmM7+zMnSTMkfVOx/6g1
97sASej31XdOAgKCBJGymrwvYrCLW+P5cHqd+D8v/PvfpRIQM54p5ixRt3EYZvtR
zGrzsr0OGyOLZtj1DB0a3kvajAAOCl3TawJSzviKo2mwc+/xj28MCQM=
-----END RSA PRIVATE KEY-----
-----BEGIN CERTIFICATE-----
MIIE4TCCA0mgAwIBAgIJALONCAWZxPhUMA0GCSqGSIb3DQEBCwUAMEExCzAJBgNV
BAYTAk5aMQ4wDAYDVQQIDAVPdGFnbzEPMA0GA1UECgwGUGF0aG9kMREwDwYDVQQD
DAh0ZXN0LmNvbTAeFw0xNTA0MTgyMjA0NTNaFw00MjA5MDIyMjA0NTNaMEExCzAJ
BgNVBAYTAk5aMQ4wDAYDVQQIDAVPdGFnbzEPMA0GA1UECgwGUGF0aG9kMREwDwYD
VQQDDAh0ZXN0LmNvbTCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAML7
SsaGbwVdgMTwJAsfD127hOynqivWwb5PMfiXITzHX/ymY7Hbtr+hpvB4tOd4u2Rs
aLhb6JycRwIiunMJPTmEMGvX2zhX9poFihATuH2wh/d7Vgwk3VlcRcjVYgL8Rh5S
bN/yFNN6SBtvQIFJ4UVvKRm2mwH8fHpnDmSWd8pBLopmguLO5kcHXOtxl523I07U
ZKybAQEyPsbz8LkdvuHWl+7Dwn9JHpd2zdozKcF375P1sJCdgSBHmQoNKiL8jIx/
aq1Iq1J7JkJ6T5Z9AhgW0BAbFvcjHxqBXURIjGJbOY2XqjcY1jb9UU0SJv+EYzMM
sZ6EwPXg0vcMDtjZyUuL31wJ1Ez+AWMfD3uGTANhpumKwsrE+mANcXRzaXLh1wpc
cJj4/3JvGeZCIOXJWzSCutLbPcha/wKNyjz2Cdhgb/PzngKTwSbrm3sLPv/f9dxM
P0idbdrtvPrD24F1BvHQtp+L1k7ycCn+g7qsYJ5vNs3c40maF0ULD/fOKojezQID
AQABo4HbMIHYMAsGA1UdDwQEAwIFoDAdBgNVHQ4EFgQUbEgfTauEqEP/bnBtby1K
bihJvcswcQYDVR0jBGowaIAUbEgfTauEqEP/bnBtby1KbihJvcuhRaRDMEExCzAJ
BgNVBAYTAk5aMQ4wDAYDVQQIDAVPdGFnbzEPMA0GA1UECgwGUGF0aG9kMREwDwYD
VQQDDAh0ZXN0LmNvbYIJALONCAWZxPhUMAwGA1UdEwQFMAMBAf8wKQYDVR0RBCIw
IIIIdGVzdC5jb22CCXRlc3QyLmNvbYIJdGVzdDMuY29tMA0GCSqGSIb3DQEBCwUA
A4IBgQBcTedXtUb91DxQRtg73iomz7cQ4niZntUBW8iE5rpoA7prtQNGHMCbHwaX
tbWFkzBmL5JTBWvd/6AQ2LtiB3rYB3W/iRhbpsNJ501xaoOguPEQ9720Ph8TEveM
208gNzGsEOcNALwyXj2y9M19NGu9zMa8eu1Tc3IsQaVaGKHx8XZn5HTNUx8EdcwI
Z/Ji9ETDCL7+e5INv0tqfFSazWaQUwxM4IzPMkKTYRcMuN/6eog609k9r9pp32Ut
rKlzc6GIkAlgJJ0Wkoz1V46DmJNJdJG7eLu/mtsB85j6hytIQeWTf1fll5YnMZLF
HgNZtfYn8Q0oTdBQ0ZOaZeQCfZ8emYBdLJf2YB83uGRMjQ1FoeIxzQqiRq8WHRdb
9Q45i0DINMnNp0DbLMA4numZ7wT9SQb6sql9eUyuCNDw7nGIWTHUNfLtU1Er3h1d
icJuApx9+//UN/pGh0yTXb3fZbiI4IehRmkpnIWonIAwaVGm6JZU04wiIn8CuBho
/qQdlS8=
-----END CERTIFICATE-----

0
test/pathod/language/__init__.py
View File

134
test/pathod/language/test_actions.py
View File

@ -1,134 +0,0 @@
import io
from pathod.language import actions, parse_pathoc, parse_pathod, serve
def parse_request(s):
return next(parse_pathoc(s))
def test_unique_name():
assert not actions.PauseAt(0, "f").unique_name
assert actions.DisconnectAt(0).unique_name
class TestDisconnects:
def test_parse_pathod(self):
a = next(parse_pathod("400:d0")).actions[0]
assert a.spec() == "d0"
a = next(parse_pathod("400:dr")).actions[0]
assert a.spec() == "dr"
def test_at(self):
e = actions.DisconnectAt.expr()
v = e.parseString("d0")[0]
assert isinstance(v, actions.DisconnectAt)
assert v.offset == 0
v = e.parseString("d100")[0]
assert v.offset == 100
e = actions.DisconnectAt.expr()
v = e.parseString("dr")[0]
assert v.offset == "r"
def test_spec(self):
assert actions.DisconnectAt("r").spec() == "dr"
assert actions.DisconnectAt(10).spec() == "d10"
class TestInject:
def test_parse_pathod(self):
a = next(parse_pathod("400:ir,@100")).actions[0]
assert a.offset == "r"
assert a.value.datatype == "bytes"
assert a.value.usize == 100
a = next(parse_pathod("400:ia,@100")).actions[0]
assert a.offset == "a"
def test_at(self):
e = actions.InjectAt.expr()
v = e.parseString("i0,'foo'")[0]
assert v.value.val == b"foo"
assert v.offset == 0
assert isinstance(v, actions.InjectAt)
v = e.parseString("ir,'foo'")[0]
assert v.offset == "r"
def test_serve(self):
s = io.BytesIO()
r = next(parse_pathod("400:i0,'foo'"))
assert serve(r, s, {})
def test_spec(self):
e = actions.InjectAt.expr()
v = e.parseString("i0,'foo'")[0]
assert v.spec() == "i0,'foo'"
def test_spec2(self):
e = actions.InjectAt.expr()
v = e.parseString("i0,@100")[0]
v2 = v.freeze({})
v3 = v2.freeze({})
assert v2.value.val == v3.value.val
class TestPauses:
def test_parse_pathod(self):
e = actions.PauseAt.expr()
v = e.parseString("p10,10")[0]
assert v.seconds == 10
assert v.offset == 10
v = e.parseString("p10,f")[0]
assert v.seconds == "f"
v = e.parseString("pr,f")[0]
assert v.offset == "r"
v = e.parseString("pa,f")[0]
assert v.offset == "a"
def test_request(self):
r = next(parse_pathod('400:p10,10'))
assert r.actions[0].spec() == "p10,10"
def test_spec(self):
assert actions.PauseAt("r", 5).spec() == "pr,5"
assert actions.PauseAt(0, 5).spec() == "p0,5"
assert actions.PauseAt(0, "f").spec() == "p0,f"
def test_freeze(self):
l = actions.PauseAt("r", 5)
assert l.freeze({}).spec() == l.spec()
class Test_Action:
def test_cmp(self):
a = actions.DisconnectAt(0)
b = actions.DisconnectAt(1)
c = actions.DisconnectAt(0)
assert a < b
assert a == c
l = sorted([b, a])
assert l[0].offset == 0
def test_resolve(self):
r = parse_request('GET:"/foo"')
e = actions.DisconnectAt("r")
ret = e.resolve({}, r)
assert isinstance(ret.offset, int)
def test_repr(self):
e = actions.DisconnectAt("r")
assert repr(e)
def test_freeze(self):
l = actions.DisconnectAt(5)
assert l.freeze({}).spec() == l.spec()

351
test/pathod/language/test_base.py
View File

@ -1,351 +0,0 @@
import pytest
from pathod import language
from pathod.language import base, exceptions
def parse_request(s):
return language.parse_pathoc(s).next()
def test_times():
reqs = list(language.parse_pathoc("get:/:x5"))
assert len(reqs) == 5
assert not reqs[0].times
def test_caseless_literal():
class CL(base.CaselessLiteral):
TOK = "foo"
v = CL("foo")
assert v.expr()
assert v.values(language.Settings())
class TestTokValueNakedLiteral:
def test_expr(self):
v = base.TokValueNakedLiteral("foo")
assert v.expr()
def test_spec(self):
v = base.TokValueNakedLiteral("foo")
assert v.spec() == repr(v) == "foo"
v = base.TokValueNakedLiteral("f\x00oo")
assert v.spec() == repr(v) == r"f\x00oo"
class TestTokValueLiteral:
def test_expr(self):
v = base.TokValueLiteral("foo")
assert v.expr()
assert v.val == b"foo"
v = base.TokValueLiteral("foo\n")
assert v.expr()
assert v.val == b"foo\n"
assert repr(v)
def test_spec(self):
v = base.TokValueLiteral("foo")
assert v.spec() == r"'foo'"
v = base.TokValueLiteral("f\x00oo")
assert v.spec() == repr(v) == r"'f\x00oo'"
v = base.TokValueLiteral('"')
assert v.spec() == repr(v) == """ '"' """.strip()
# While pyparsing has a escChar argument for QuotedString,
# escChar only performs scapes single-character escapes and does not work for e.g. r"\x02".
# Thus, we cannot use that option, which means we cannot have single quotes in strings.
# To fix this, we represent single quotes as r"\x07".
v = base.TokValueLiteral("'")
assert v.spec() == r"'\x27'"
def roundtrip(self, spec):
e = base.TokValueLiteral.expr()
v = base.TokValueLiteral(spec)
v2 = e.parseString(v.spec())
assert v.val == v2[0].val
assert v.spec() == v2[0].spec()
def test_roundtrip(self):
self.roundtrip("'")
self.roundtrip(r"\'")
self.roundtrip("a")
self.roundtrip("\"")
# self.roundtrip("\\")
self.roundtrip("200:b'foo':i23,'\\''")
self.roundtrip("\a")
class TestTokValueGenerate:
def test_basic(self):
v = base.TokValue.parseString("@10b")[0]
assert v.usize == 10
assert v.unit == "b"
assert v.bytes() == 10
v = base.TokValue.parseString("@10")[0]
assert v.unit == "b"
v = base.TokValue.parseString("@10k")[0]
assert v.bytes() == 10240
v = base.TokValue.parseString("@10g")[0]
assert v.bytes() == 1024 ** 3 * 10
v = base.TokValue.parseString("@10g,digits")[0]
assert v.datatype == "digits"
g = v.get_generator({})
assert g[:100]
v = base.TokValue.parseString("@10,digits")[0]
assert v.unit == "b"
assert v.datatype == "digits"
def test_spec(self):
v = base.TokValueGenerate(1, "b", "bytes")
assert v.spec() == repr(v) == "@1"
v = base.TokValueGenerate(1, "k", "bytes")
assert v.spec() == repr(v) == "@1k"
v = base.TokValueGenerate(1, "k", "ascii")
assert v.spec() == repr(v) == "@1k,ascii"
v = base.TokValueGenerate(1, "b", "ascii")
assert v.spec() == repr(v) == "@1,ascii"
def test_freeze(self):
v = base.TokValueGenerate(100, "b", "ascii")
f = v.freeze(language.Settings())
assert len(f.val) == 100
class TestTokValueFile:
def test_file_value(self):
v = base.TokValue.parseString("<'one two'")[0]
assert str(v)
assert v.path == "one two"
v = base.TokValue.parseString("<path")[0]
assert v.path == "path"
def test_access_control(self, tmpdir):
v = base.TokValue.parseString("<path")[0]
f = tmpdir.join("path")
f.write(b"x" * 10000)
assert v.get_generator(language.Settings(staticdir=str(tmpdir)))
v = base.TokValue.parseString("<path2")[0]
with pytest.raises(exceptions.FileAccessDenied):
v.get_generator(language.Settings(staticdir=str(tmpdir)))
with pytest.raises(Exception, match="access disabled"):
v.get_generator(language.Settings())
v = base.TokValue.parseString("</outside")[0]
with pytest.raises(Exception, match="outside"):
v.get_generator(language.Settings(staticdir=str(tmpdir)))
def test_spec(self):
v = base.TokValue.parseString("<'one two'")[0]
v2 = base.TokValue.parseString(v.spec())[0]
assert v2.path == "one two"
def test_freeze(self):
v = base.TokValue.parseString("<'one two'")[0]
v2 = v.freeze({})
assert v2.path == v.path
class TestMisc:
def test_generators(self):
v = base.TokValue.parseString("'val'")[0]
g = v.get_generator({})
assert g[:] == b"val"
def test_value(self):
assert base.TokValue.parseString("'val'")[0].val == b"val"
assert base.TokValue.parseString('"val"')[0].val == b"val"
assert base.TokValue.parseString('"\'val\'"')[0].val == b"'val'"
def test_value2(self):
class TT(base.Value):
preamble = "m"
e = TT.expr()
v = e.parseString("m'msg'")[0]
assert v.value.val == b"msg"
s = v.spec()
assert s == e.parseString(s)[0].spec()
v = e.parseString("m@100")[0]
v2 = v.freeze({})
v3 = v2.freeze({})
assert v2.value.val == v3.value.val
def test_fixedlengthvalue(self, tmpdir):
class TT(base.FixedLengthValue):
preamble = "m"
length = 4
e = TT.expr()
assert e.parseString("m@4")
with pytest.raises(Exception, match="Invalid value length"):
e.parseString("m@100")
with pytest.raises(Exception, match="Invalid value length"):
e.parseString("m@1")
s = base.Settings(staticdir=str(tmpdir))
with open(str(tmpdir.join("path")), 'wb') as f:
f.write(b"a" * 20)
v = e.parseString("m<path")[0]
with pytest.raises(Exception, match="Invalid value length"):
v.values(s)
with open(str(tmpdir.join("path2")), 'wb') as f:
f.write(b"a" * 4)
v = e.parseString("m<path2")[0]
assert v.values(s)
class TKeyValue(base.KeyValue):
preamble = "h"
def values(self, settings):
return [
self.key.get_generator(settings),
": ",
self.value.get_generator(settings),
"\r\n",
]
class TestKeyValue:
def test_simple(self):
e = TKeyValue.expr()
v = e.parseString("h'foo'='bar'")[0]
assert v.key.val == b"foo"
assert v.value.val == b"bar"
v2 = e.parseString(v.spec())[0]
assert v2.key.val == v.key.val
assert v2.value.val == v.value.val
s = v.spec()
assert s == e.parseString(s)[0].spec()
def test_freeze(self):
e = TKeyValue.expr()
v = e.parseString("h@10=@10'")[0]
v2 = v.freeze({})
v3 = v2.freeze({})
assert v2.key.val == v3.key.val
assert v2.value.val == v3.value.val
def test_intfield():
class TT(base.IntField):
preamble = "t"
names = {
"one": 1,
"two": 2,
"three": 3
}
max = 4
e = TT.expr()
v = e.parseString("tone")[0]
assert v.value == 1
assert v.spec() == "tone"
assert v.values(language.Settings())
v = e.parseString("t1")[0]
assert v.value == 1
assert v.spec() == "t1"
v = e.parseString("t4")[0]
assert v.value == 4
assert v.spec() == "t4"
with pytest.raises(Exception, match="can't exceed"):
e.parseString("t5")
def test_options_or_value():
class TT(base.OptionsOrValue):
options = [
"one",
"two",
"three"
]
e = TT.expr()
assert e.parseString("one")[0].value.val == b"one"
assert e.parseString("'foo'")[0].value.val == b"foo"
assert e.parseString("'get'")[0].value.val == b"get"
assert e.parseString("one")[0].spec() == "one"
assert e.parseString("'foo'")[0].spec() == "'foo'"
s = e.parseString("one")[0].spec()
assert s == e.parseString(s)[0].spec()
s = e.parseString("'foo'")[0].spec()
assert s == e.parseString(s)[0].spec()
v = e.parseString("@100")[0]
v2 = v.freeze({})
v3 = v2.freeze({})
assert v2.value.val == v3.value.val
def test_integer():
e = base.Integer.expr()
v = e.parseString("200")[0]
assert v.string() == b"200"
assert v.spec() == "200"
assert v.freeze({}).value == v.value
class BInt(base.Integer):
bounds = (1, 5)
with pytest.raises(Exception, match="must be between"):
BInt(0)
with pytest.raises(Exception, match="must be between"):
BInt(6)
assert BInt(5)
assert BInt(1)
assert BInt(3)
class TBoolean(base.Boolean):
name = "test"
def test_unique_name():
b = TBoolean(True)
assert b.unique_name
class test_boolean:
e = TBoolean.expr()
assert e.parseString("test")[0].value
assert not e.parseString("-test")[0].value
def roundtrip(s):
e = TBoolean.expr()
s2 = e.parseString(s)[0].spec()
v1 = e.parseString(s)[0].value
v2 = e.parseString(s2)[0].value
assert s == s2
assert v1 == v2
roundtrip("test")
roundtrip("-test")

1
test/pathod/language/test_exceptions.py
View File

@ -1 +0,0 @@
# TODO: write tests

35
test/pathod/language/test_generators.py
View File

@ -1,35 +0,0 @@
from pathod.language import generators
def test_randomgenerator():
g = generators.RandomGenerator("bytes", 100)
assert repr(g)
assert g[0]
assert len(g[0]) == 1
assert len(g[:10]) == 10
assert len(g[1:10]) == 9
assert len(g[:1000]) == 100
assert len(g[1000:1001]) == 0
def test_filegenerator(tmpdir):
f = tmpdir.join("foo")
f.write(b"abcdefghijklmnopqrstuvwxyz" * 1000)
g = generators.FileGenerator(str(f))
assert len(g) == 26000
assert g[0] == b"a"
assert g[2:7] == b"cdefg"
assert len(g[1:10]) == 9
assert len(g[26000:26001]) == 0
assert repr(g)
def test_transform_generator():
def trans(offset, data):
return "a" * len(data)
g = "one"
t = generators.TransformGenerator(g, trans)
assert len(t) == len(g)
assert t[0] == "a"
assert t[:] == "a" * len(g)
assert repr(t)

355
test/pathod/language/test_http.py
View File

@ -1,355 +0,0 @@
import io
import pytest
from pathod import language
from pathod.language import http, base
from .. import tservers
def parse_request(s):
return next(language.parse_pathoc(s))
def test_make_error_response():
d = io.BytesIO()
s = http.make_error_response("foo")
language.serve(s, d, {})
class TestRequest:
def test_nonascii(self):
with pytest.raises(Exception, match="ASCII"):
parse_request("get:\xf0")
def test_err(self):
with pytest.raises(language.ParseException):
parse_request('GET')
def test_simple(self):
r = parse_request('GET:"/foo"')
assert r.method.string() == b"GET"
assert r.path.string() == b"/foo"
r = parse_request('GET:/foo')
assert r.path.string() == b"/foo"
r = parse_request('GET:@1k')
assert len(r.path.string()) == 1024
def test_multiple(self):
r = list(language.parse_pathoc("GET:/ PUT:/"))
assert r[0].method.string() == b"GET"
assert r[1].method.string() == b"PUT"
assert len(r) == 2
l = """
GET
"/foo"
ir,@1
PUT
"/foo
bar"
ir,@1
"""
r = list(language.parse_pathoc(l))
assert len(r) == 2
assert r[0].method.string() == b"GET"
assert r[1].method.string() == b"PUT"
l = """
get:"http://localhost:9999/p/200":ir,@1
get:"http://localhost:9999/p/200":ir,@2
"""
r = list(language.parse_pathoc(l))
assert len(r) == 2
assert r[0].method.string() == b"GET"
assert r[1].method.string() == b"GET"
def test_nested_response(self):
l = "get:/p:s'200'"
r = list(language.parse_pathoc(l))
assert len(r) == 1
assert len(r[0].tokens) == 3
assert isinstance(r[0].tokens[2], http.NestedResponse)
assert r[0].values({})
def test_render(self):
s = io.BytesIO()
r = parse_request("GET:'/foo'")
assert language.serve(
r,
s,
language.Settings(request_host="foo.com")
)
def test_multiline(self):
l = """
GET
"/foo"
ir,@1
"""
r = parse_request(l)
assert r.method.string() == b"GET"
assert r.path.string() == b"/foo"
assert r.actions
l = """
GET
"/foo
bar"
ir,@1
"""
r = parse_request(l)
assert r.method.string() == b"GET"
assert r.path.string().endswith(b"bar")
assert r.actions
def test_spec(self):
def rt(s):
s = parse_request(s).spec()
assert parse_request(s).spec() == s
rt("get:/foo")
rt("get:/foo:da")
def test_freeze(self):
r = parse_request("GET:/:b@100").freeze(language.Settings())
assert len(r.spec()) > 100
def test_path_generator(self):
r = parse_request("GET:@100").freeze(language.Settings())
assert len(r.spec()) > 100
def test_websocket(self):
r = parse_request('ws:/path/')
res = r.resolve(language.Settings())
assert res.method.string().lower() == b"get"
assert res.tok(http.Path).value.val == b"/path/"
assert res.tok(http.Method).value.val.lower() == b"get"
assert http.get_header(b"Upgrade", res.headers).value.val == b"websocket"
r = parse_request('ws:put:/path/')
res = r.resolve(language.Settings())
assert r.method.string().lower() == b"put"
assert res.tok(http.Path).value.val == b"/path/"
assert res.tok(http.Method).value.val.lower() == b"put"
assert http.get_header(b"Upgrade", res.headers).value.val == b"websocket"
class TestResponse:
def dummy_response(self):
return next(language.parse_pathod("400'msg'"))
def test_response(self):
r = next(language.parse_pathod("400:m'msg'"))
assert r.status_code.string() == b"400"
assert r.reason.string() == b"msg"
r = next(language.parse_pathod("400:m'msg':b@100b"))
assert r.reason.string() == b"msg"
assert r.body.values({})
assert str(r)
r = next(language.parse_pathod("200"))
assert r.status_code.string() == b"200"
assert not r.reason
assert b"OK" in [i[:] for i in r.preamble({})]
def test_render(self):
s = io.BytesIO()
r = next(language.parse_pathod("400:m'msg'"))
assert language.serve(r, s, {})
r = next(language.parse_pathod("400:p0,100:dr"))
assert "p0" in r.spec()
s = r.preview_safe()
assert "p0" not in s.spec()
def test_raw(self):
s = io.BytesIO()
r = next(language.parse_pathod("400:b'foo'"))
language.serve(r, s, {})
v = s.getvalue()
assert b"Content-Length" in v
s = io.BytesIO()
r = next(language.parse_pathod("400:b'foo':r"))
language.serve(r, s, {})
v = s.getvalue()
assert b"Content-Length" not in v
def test_length(self):
def testlen(x):
s = io.BytesIO()
x = next(x)
language.serve(x, s, language.Settings())
assert x.length(language.Settings()) == len(s.getvalue())
testlen(language.parse_pathod("400:m'msg':r"))
testlen(language.parse_pathod("400:m'msg':h'foo'='bar':r"))
testlen(language.parse_pathod("400:m'msg':h'foo'='bar':b@100b:r"))
def test_maximum_length(self):
def testlen(x):
x = next(x)
s = io.BytesIO()
m = x.maximum_length({})
language.serve(x, s, {})
assert m >= len(s.getvalue())
r = language.parse_pathod("400:m'msg':b@100:d0")
testlen(r)
r = language.parse_pathod("400:m'msg':b@100:d0:i0,'foo'")
testlen(r)
r = language.parse_pathod("400:m'msg':b@100:d0:i0,'foo'")
testlen(r)
def test_parse_err(self):
with pytest.raises(language.ParseException):
language.parse_pathod("400:msg,b:")
try:
language.parse_pathod("400'msg':b:")
except language.ParseException as v:
assert v.marked()
assert str(v)
def test_nonascii(self):
with pytest.raises(Exception, match="ASCII"):
language.parse_pathod("foo:b\xf0")
def test_parse_header(self):
r = next(language.parse_pathod('400:h"foo"="bar"'))
assert http.get_header(b"foo", r.headers)
def test_parse_pause_before(self):
r = next(language.parse_pathod("400:p0,10"))
assert r.actions[0].spec() == "p0,10"
def test_parse_pause_after(self):
r = next(language.parse_pathod("400:pa,10"))
assert r.actions[0].spec() == "pa,10"
def test_parse_pause_random(self):
r = next(language.parse_pathod("400:pr,10"))
assert r.actions[0].spec() == "pr,10"
def test_parse_stress(self):
# While larger values are known to work on linux, len() technically
# returns an int and a python 2.7 int on windows has 32bit precision.
# Therefore, we should keep the body length < 2147483647 bytes in our
# tests.
r = next(language.parse_pathod("400:b@1g"))
assert r.length({})
def test_spec(self):
def rt(s):
s = next(language.parse_pathod(s)).spec()
assert next(language.parse_pathod(s)).spec() == s
rt("400:b@100g")
rt("400")
rt("400:da")
def test_websockets(self):
r = next(language.parse_pathod("ws"))
with pytest.raises(Exception, match="No websocket key"):
r.resolve(language.Settings())
res = r.resolve(language.Settings(websocket_key=b"foo"))
assert res.status_code.string() == b"101"
def test_ctype_shortcut():
e = http.ShortcutContentType.expr()
v = e.parseString("c'foo'")[0]
assert v.key.val == b"Content-Type"
assert v.value.val == b"foo"
s = v.spec()
assert s == e.parseString(s)[0].spec()
e = http.ShortcutContentType.expr()
v = e.parseString("c@100")[0]
v2 = v.freeze({})
v3 = v2.freeze({})
assert v2.value.val == v3.value.val
def test_location_shortcut():
e = http.ShortcutLocation.expr()
v = e.parseString("l'foo'")[0]
assert v.key.val == b"Location"
assert v.value.val == b"foo"
s = v.spec()
assert s == e.parseString(s)[0].spec()
e = http.ShortcutLocation.expr()
v = e.parseString("l@100")[0]
v2 = v.freeze({})
v3 = v2.freeze({})
assert v2.value.val == v3.value.val
def test_shortcuts():
assert next(language.parse_pathod(
"400:c'foo'")).headers[0].key.val == b"Content-Type"
assert next(language.parse_pathod(
"400:l'foo'")).headers[0].key.val == b"Location"
assert b"Android" in tservers.render(parse_request("get:/:ua"))
assert b"User-Agent" in tservers.render(parse_request("get:/:ua"))
def test_user_agent():
e = http.ShortcutUserAgent.expr()
v = e.parseString("ua")[0]
assert b"Android" in v.string()
e = http.ShortcutUserAgent.expr()
v = e.parseString("u'a'")[0]
assert b"Android" not in v.string()
v = e.parseString("u@100'")[0]
assert len(str(v.freeze({}).value)) > 100
v2 = v.freeze({})
v3 = v2.freeze({})
assert v2.value.val == v3.value.val
def test_nested_response():
e = http.NestedResponse.expr()
v = e.parseString("s'200'")[0]
assert v.value.val == b"200"
with pytest.raises(language.ParseException):
e.parseString("s'foo'")
v = e.parseString('s"200:b@1"')[0]
assert "@1" in v.spec()
f = v.freeze({})
assert "@1" not in f.spec()
def test_nested_response_freeze():
e = http.NestedResponse(
base.TokValueLiteral(
r"200:b\'foo\':i10,\'\\x27\'"
)
)
assert e.freeze({})
assert e.values({})
def test_unique_components():
with pytest.raises(Exception, match="multiple body clauses"):
language.parse_pathod("400:b@1:b@1")

236
test/pathod/language/test_http2.py
View File

@ -1,236 +0,0 @@
import io
import pytest
from mitmproxy.net import tcp
from mitmproxy.net.http import user_agents
from pathod import language
from pathod.language import http2
from pathod.protocols.http2 import HTTP2StateProtocol
def parse_request(s):
return next(language.parse_pathoc(s, True))
def parse_response(s):
return next(language.parse_pathod(s, True))
def default_settings():
return language.Settings(
request_host="foo.com",
protocol=HTTP2StateProtocol(tcp.TCPClient(('localhost', 1234)))
)
def test_make_error_response():
d = io.BytesIO()
s = http2.make_error_response("foo", "bar")
language.serve(s, d, default_settings())
class TestRequest:
def test_cached_values(self):
req = parse_request("get:/")
req_id = id(req)
assert req_id == id(req.resolve(default_settings()))
assert req.values(default_settings()) == req.values(default_settings())
def test_nonascii(self):
with pytest.raises(Exception, match="ASCII"):
parse_request("get:\xf0")
def test_err(self):
with pytest.raises(language.ParseException):
parse_request('GET')
def test_simple(self):
r = parse_request('GET:"/foo"')
assert r.method.string() == b"GET"
assert r.path.string() == b"/foo"
r = parse_request('GET:/foo')
assert r.path.string() == b"/foo"
def test_multiple(self):
r = list(language.parse_pathoc("GET:/ PUT:/"))
assert r[0].method.string() == b"GET"
assert r[1].method.string() == b"PUT"
assert len(r) == 2
l = """
GET
"/foo"
PUT
"/foo
bar"
"""
r = list(language.parse_pathoc(l, True))
assert len(r) == 2
assert r[0].method.string() == b"GET"
assert r[1].method.string() == b"PUT"
l = """
get:"http://localhost:9999/p/200"
get:"http://localhost:9999/p/200"
"""
r = list(language.parse_pathoc(l, True))
assert len(r) == 2
assert r[0].method.string() == b"GET"
assert r[1].method.string() == b"GET"
def test_render_simple(self):
s = io.BytesIO()
r = parse_request("GET:'/foo'")
assert language.serve(
r,
s,
default_settings(),
)
def test_raw_content_length(self):
r = parse_request('GET:/:r')
assert len(r.headers) == 0
r = parse_request('GET:/:r:b"foobar"')
assert len(r.headers) == 0
r = parse_request('GET:/')
assert len(r.headers) == 1
assert r.headers[0].values(default_settings()) == (b"content-length", b"0")
r = parse_request('GET:/:b"foobar"')
assert len(r.headers) == 1
assert r.headers[0].values(default_settings()) == (b"content-length", b"6")
r = parse_request('GET:/:b"foobar":h"content-length"="42"')
assert len(r.headers) == 1
assert r.headers[0].values(default_settings()) == (b"content-length", b"42")
r = parse_request('GET:/:r:b"foobar":h"content-length"="42"')
assert len(r.headers) == 1
assert r.headers[0].values(default_settings()) == (b"content-length", b"42")
def test_content_type(self):
r = parse_request('GET:/:r:c"foobar"')
assert len(r.headers) == 1
assert r.headers[0].values(default_settings()) == (b"content-type", b"foobar")
def test_user_agent(self):
r = parse_request('GET:/:r:ua')
assert len(r.headers) == 1
assert r.headers[0].values(default_settings()) == (b"user-agent", user_agents.get_by_shortcut('a')[2].encode())
def test_render_with_headers(self):
s = io.BytesIO()
r = parse_request('GET:/foo:h"foo"="bar"')
assert language.serve(
r,
s,
default_settings(),
)
def test_nested_response(self):
l = "get:/p/:s'200'"
r = parse_request(l)
assert len(r.tokens) == 3
assert isinstance(r.tokens[2], http2.NestedResponse)
assert r.values(default_settings())
def test_render_with_body(self):
s = io.BytesIO()
r = parse_request("GET:'/foo':bfoobar")
assert language.serve(
r,
s,
default_settings(),
)
def test_spec(self):
def rt(s):
s = parse_request(s).spec()
assert parse_request(s).spec() == s
rt("get:/foo")
class TestResponse:
def test_cached_values(self):
res = parse_response("200")
res_id = id(res)
assert res_id == id(res.resolve(default_settings()))
assert res.values(default_settings()) == res.values(default_settings())
def test_nonascii(self):
with pytest.raises(Exception, match="ASCII"):
parse_response("200:\xf0")
def test_err(self):
with pytest.raises(language.ParseException):
parse_response('GET:/')
def test_raw_content_length(self):
r = parse_response('200:r')
assert len(r.headers) == 0
r = parse_response('200')
assert len(r.headers) == 1
assert r.headers[0].values(default_settings()) == (b"content-length", b"0")
def test_content_type(self):
r = parse_response('200:r:c"foobar"')
assert len(r.headers) == 1
assert r.headers[0].values(default_settings()) == (b"content-type", b"foobar")
def test_simple(self):
r = parse_response('200:r:h"foo"="bar"')
assert r.status_code.string() == b"200"
assert len(r.headers) == 1
assert r.headers[0].values(default_settings()) == (b"foo", b"bar")
assert r.body is None
r = parse_response('200:r:h"foo"="bar":bfoobar:h"bla"="fasel"')
assert r.status_code.string() == b"200"
assert len(r.headers) == 2
assert r.headers[0].values(default_settings()) == (b"foo", b"bar")
assert r.headers[1].values(default_settings()) == (b"bla", b"fasel")
assert r.body.string() == b"foobar"
def test_render_simple(self):
s = io.BytesIO()
r = parse_response('200')
assert language.serve(
r,
s,
default_settings(),
)
def test_render_with_headers(self):
s = io.BytesIO()
r = parse_response('200:h"foo"="bar"')
assert language.serve(
r,
s,
default_settings(),
)
def test_render_with_body(self):
s = io.BytesIO()
r = parse_response('200:bfoobar')
assert language.serve(
r,
s,
default_settings(),
)
def test_spec(self):
def rt(s):
s = parse_response(s).spec()
assert parse_response(s).spec() == s
rt("200:bfoobar")

1
test/pathod/language/test_message.py
View File

@ -1 +0,0 @@
# TODO: write tests

134
test/pathod/language/test_websockets.py
View File

@ -1,134 +0,0 @@
import pytest
from pathod import language
from wsproto.frame_protocol import Opcode
from .. import tservers
def parse_request(s):
return next(language.parse_pathoc(s))
class TestWebsocketFrame:
def _test_messages(self, specs, message_klass):
for i in specs:
wf = parse_request(i)
assert isinstance(wf, message_klass)
assert wf
assert wf.values(language.Settings())
assert wf.resolve(language.Settings())
spec = wf.spec()
wf2 = parse_request(spec)
assert wf2.spec() == spec
def test_server_values(self):
specs = [
"wf",
"wf:dr",
"wf:b'foo'",
"wf:mask:r'foo'",
"wf:l1024:b'foo'",
"wf:cbinary",
"wf:c1",
"wf:mask:knone",
"wf:fin",
"wf:fin:rsv1:rsv2:rsv3:mask",
"wf:-fin:-rsv1:-rsv2:-rsv3:-mask",
"wf:k@4",
"wf:x10",
]
self._test_messages(specs, language.websockets.WebsocketFrame)
def test_parse_websocket_frames(self):
wf = language.parse_websocket_frame("wf:x10")
assert len(list(wf)) == 10
with pytest.raises(language.ParseException):
language.parse_websocket_frame("wf:x")
def test_client_values(self):
specs = [
"wf:f'wf'",
]
self._test_messages(specs, language.websockets.WebsocketClientFrame)
def test_nested_frame(self):
wf = parse_request("wf:f'wf'")
assert wf.nested_frame
def test_flags(self):
wf = parse_request("wf:fin:mask:rsv1:rsv2:rsv3")
frm = language.websockets_frame.Frame.from_bytes(tservers.render(wf))
assert frm.header.fin
assert frm.header.mask
assert frm.header.rsv1
assert frm.header.rsv2
assert frm.header.rsv3
wf = parse_request("wf:-fin:-mask:-rsv1:-rsv2:-rsv3")
frm = language.websockets_frame.Frame.from_bytes(tservers.render(wf))
assert not frm.header.fin
assert not frm.header.mask
assert not frm.header.rsv1
assert not frm.header.rsv2
assert not frm.header.rsv3
def fr(self, spec, **kwargs):
settings = language.base.Settings(**kwargs)
wf = parse_request(spec)
return language.websockets_frame.Frame.from_bytes(tservers.render(wf, settings))
def test_construction(self):
assert self.fr("wf:c1").header.opcode == 1
assert self.fr("wf:c0").header.opcode == 0
assert self.fr("wf:cbinary").header.opcode == Opcode.BINARY
assert self.fr("wf:ctext").header.opcode == Opcode.TEXT
def test_rawbody(self):
frm = self.fr("wf:mask:r'foo'")
assert len(frm.payload) == 3
assert frm.payload != b"foo"
assert self.fr("wf:r'foo'").payload == b"foo"
def test_construction_2(self):
# Simple server frame
frm = self.fr("wf:b'foo'")
assert not frm.header.mask
assert not frm.header.masking_key
# Simple client frame
frm = self.fr("wf:b'foo'", is_client=True)
assert frm.header.mask
assert frm.header.masking_key
frm = self.fr("wf:b'foo':k'abcd'", is_client=True)
assert frm.header.mask
assert frm.header.masking_key == b'abcd'
# Server frame, mask explicitly set
frm = self.fr("wf:b'foo':mask")
assert frm.header.mask
assert frm.header.masking_key
frm = self.fr("wf:b'foo':k'abcd'")
assert frm.header.mask
assert frm.header.masking_key == b'abcd'
# Client frame, mask explicitly unset
frm = self.fr("wf:b'foo':-mask", is_client=True)
assert not frm.header.mask
assert not frm.header.masking_key
def test_knone(self):
with pytest.raises(Exception, match="Expected 4 bytes"):
self.fr("wf:b'foo':mask:knone")
def test_length(self):
assert self.fr("wf:l3:b'foo'").header.payload_length == 3
frm = self.fr("wf:l2:b'foo'")
assert frm.header.payload_length == 2
assert frm.payload == b"fo"
with pytest.raises(Exception, match="Expected 1024 bytes"):
self.fr("wf:l1024:b'foo'")

187
test/pathod/language/test_websockets_frame.py
View File

@ -1,187 +0,0 @@
import os
import codecs
import pytest
from wsproto.frame_protocol import Opcode
from pathod.language import websockets_frame
from mitmproxy.test import tutils
class TestMasker:
@pytest.mark.parametrize("input,expected", [
([b"a"], '00'),
([b"four"], '070d1616'),
([b"fourf"], '070d161607'),
([b"fourfive"], '070d1616070b1501'),
([b"a", b"aasdfasdfa", b"asdf"], '000302170504021705040205120605'),
([b"a" * 50, b"aasdfasdfa", b"asdf"], '00030205000302050003020500030205000302050003020500030205000302050003020500030205000302050003020500030205120605051206050500110702'), # noqa
])
def test_masker(self, input, expected):
m = websockets_frame.Masker(b"abcd")
data = b"".join([m(t) for t in input])
assert data == codecs.decode(expected, 'hex')
data = websockets_frame.Masker(b"abcd")(data)
assert data == b"".join(input)
class TestFrameHeader:
@pytest.mark.parametrize("input,expected", [
(0, '0100'),
(125, '017D'),
(126, '017E007E'),
(127, '017E007F'),
(142, '017E008E'),
(65534, '017EFFFE'),
(65535, '017EFFFF'),
(65536, '017F0000000000010000'),
(8589934591, '017F00000001FFFFFFFF'),
(2 ** 64 - 1, '017FFFFFFFFFFFFFFFFF'),
])
def test_serialization_length(self, input, expected):
h = websockets_frame.FrameHeader(
opcode=Opcode.TEXT,
payload_length=input,
)
assert bytes(h) == codecs.decode(expected, 'hex')
def test_serialization_too_large(self):
h = websockets_frame.FrameHeader(
payload_length=2 ** 64 + 1,
)
with pytest.raises(ValueError):
bytes(h)
@pytest.mark.parametrize("input,expected", [
('0100', 0),
('017D', 125),
('017E007E', 126),
('017E007F', 127),
('017E008E', 142),
('017EFFFE', 65534),
('017EFFFF', 65535),
('017F0000000000010000', 65536),
('017F00000001FFFFFFFF', 8589934591),
('017FFFFFFFFFFFFFFFFF', 2 ** 64 - 1),
])
def test_deserialization_length(self, input, expected):
h = websockets_frame.FrameHeader.from_file(tutils.treader(codecs.decode(input, 'hex')))
assert h.payload_length == expected
@pytest.mark.parametrize("input,expected", [
('0100', (False, None)),
('018000000000', (True, '00000000')),
('018012345678', (True, '12345678')),
])
def test_deserialization_masking(self, input, expected):
h = websockets_frame.FrameHeader.from_file(tutils.treader(codecs.decode(input, 'hex')))
assert h.mask == expected[0]
if h.mask:
assert h.masking_key == codecs.decode(expected[1], 'hex')
def test_equality(self):
h = websockets_frame.FrameHeader(mask=True, masking_key=b'1234')
h2 = websockets_frame.FrameHeader(mask=True, masking_key=b'1234')
assert h == h2
h = websockets_frame.FrameHeader(fin=True)
h2 = websockets_frame.FrameHeader(fin=False)
assert h != h2
assert h != 'foobar'
def test_roundtrip(self):
def round(*args, **kwargs):
h = websockets_frame.FrameHeader(*args, **kwargs)
h2 = websockets_frame.FrameHeader.from_file(tutils.treader(bytes(h)))
assert h == h2
round()
round(fin=True)
round(rsv1=True)
round(rsv2=True)
round(rsv3=True)
round(payload_length=1)
round(payload_length=100)
round(payload_length=1000)
round(payload_length=10000)
round(opcode=Opcode.PING)
round(masking_key=b"test")
def test_human_readable(self):
f = websockets_frame.FrameHeader(
masking_key=b"test",
fin=True,
payload_length=10
)
assert repr(f)
f = websockets_frame.FrameHeader()
assert repr(f)
def test_funky(self):
f = websockets_frame.FrameHeader(masking_key=b"test", mask=False)
raw = bytes(f)
f2 = websockets_frame.FrameHeader.from_file(tutils.treader(raw))
assert not f2.mask
def test_violations(self):
with pytest.raises(Exception, match="opcode"):
websockets_frame.FrameHeader(opcode=17)
with pytest.raises(Exception, match="Masking key"):
websockets_frame.FrameHeader(masking_key=b"x")
def test_automask(self):
f = websockets_frame.FrameHeader(mask=True)
assert f.masking_key
f = websockets_frame.FrameHeader(masking_key=b"foob")
assert f.mask
f = websockets_frame.FrameHeader(masking_key=b"foob", mask=0)
assert not f.mask
assert not f.masking_key
class TestFrame:
def test_equality(self):
f = websockets_frame.Frame(payload=b'1234')
f2 = websockets_frame.Frame(payload=b'1234')
assert f == f2
assert f != b'1234'
def test_roundtrip(self):
def round(*args, **kwargs):
f = websockets_frame.Frame(*args, **kwargs)
raw = bytes(f)
f2 = websockets_frame.Frame.from_file(tutils.treader(raw))
assert f == f2
round(b"test")
round(b"test", fin=1)
round(b"test", rsv1=1)
round(b"test", opcode=Opcode.PING)
round(b"test", masking_key=b"test")
def test_human_readable(self):
f = websockets_frame.Frame()
assert repr(f)
f = websockets_frame.Frame(b"foobar")
assert "foobar" in repr(f)
@pytest.mark.parametrize("masked", [True, False])
@pytest.mark.parametrize("length", [100, 50000, 150000])
def test_serialization_bijection(self, masked, length):
frame = websockets_frame.Frame(
os.urandom(length),
fin=True,
opcode=Opcode.TEXT,
mask=int(masked),
masking_key=(os.urandom(4) if masked else None)
)
serialized = bytes(frame)
assert frame == websockets_frame.Frame.from_bytes(serialized)

90
test/pathod/language/test_writer.py
View File

@ -1,90 +0,0 @@
import io
from pathod import language
from pathod.language import writer
def test_send_chunk():
v = b"foobarfoobar"
for bs in range(1, len(v) + 2):
s = io.BytesIO()
writer.send_chunk(s, v, bs, 0, len(v))
assert s.getvalue() == v
for start in range(len(v)):
for end in range(len(v)):
s = io.BytesIO()
writer.send_chunk(s, v, bs, start, end)
assert s.getvalue() == v[start:end]
def test_write_values_inject():
tst = b"foo"
s = io.BytesIO()
writer.write_values(s, [tst], [(0, "inject", b"aaa")], blocksize=5)
assert s.getvalue() == b"aaafoo"
s = io.BytesIO()
writer.write_values(s, [tst], [(1, "inject", b"aaa")], blocksize=5)
assert s.getvalue() == b"faaaoo"
s = io.BytesIO()
writer.write_values(s, [tst], [(1, "inject", b"aaa")], blocksize=5)
assert s.getvalue() == b"faaaoo"
def test_write_values_disconnects():
s = io.BytesIO()
tst = b"foo" * 100
writer.write_values(s, [tst], [(0, "disconnect")], blocksize=5)
assert not s.getvalue()
def test_write_values():
tst = b"foobarvoing"
s = io.BytesIO()
writer.write_values(s, [tst], [])
assert s.getvalue() == tst
for bs in range(1, len(tst) + 2):
for off in range(len(tst)):
s = io.BytesIO()
writer.write_values(
s, [tst], [(off, "disconnect")], blocksize=bs
)
assert s.getvalue() == tst[:off]
def test_write_values_pauses():
tst = "".join(str(i) for i in range(10)).encode()
for i in range(2, 10):
s = io.BytesIO()
writer.write_values(
s, [tst], [(2, "pause", 0), (1, "pause", 0)], blocksize=i
)
assert s.getvalue() == tst
for i in range(2, 10):
s = io.BytesIO()
writer.write_values(s, [tst], [(1, "pause", 0)], blocksize=i)
assert s.getvalue() == tst
tst = [tst] * 5
for i in range(2, 10):
s = io.BytesIO()
writer.write_values(s, tst[:], [(1, "pause", 0)], blocksize=i)
assert s.getvalue() == b"".join(tst)
def test_write_values_after():
s = io.BytesIO()
r = next(language.parse_pathod("400:da"))
language.serve(r, s, {})
s = io.BytesIO()
r = next(language.parse_pathod("400:pa,0"))
language.serve(r, s, {})
s = io.BytesIO()
r = next(language.parse_pathod("400:ia,'xx'"))
language.serve(r, s, {})
assert s.getvalue().endswith(b'xx')

0
test/pathod/protocols/__init__.py
View File

1
test/pathod/protocols/test_http.py
View File

@ -1 +0,0 @@
# TODO: write tests

500
test/pathod/protocols/test_http2.py
View File

@ -1,500 +0,0 @@
from unittest import mock
import hyperframe
import pytest
from mitmproxy import exceptions
from mitmproxy.net import http, tcp
from mitmproxy.net.http import http2
from pathod.protocols.http2 import HTTP2StateProtocol, TCPHandler
from ...mitmproxy.net import tservers as net_tservers
class TestTCPHandlerWrapper:
def test_wrapped(self):
h = TCPHandler(rfile='foo', wfile='bar')
p = HTTP2StateProtocol(h)
assert p.tcp_handler.rfile == 'foo'
assert p.tcp_handler.wfile == 'bar'
def test_direct(self):
p = HTTP2StateProtocol(rfile='foo', wfile='bar')
assert isinstance(p.tcp_handler, TCPHandler)
assert p.tcp_handler.rfile == 'foo'
assert p.tcp_handler.wfile == 'bar'
class EchoHandler(tcp.BaseHandler):
sni = None
def handle(self):
while True:
v = self.rfile.safe_read(1)
self.wfile.write(v)
self.wfile.flush()
class TestProtocol:
@mock.patch("pathod.protocols.http2.HTTP2StateProtocol.perform_server_connection_preface")
@mock.patch("pathod.protocols.http2.HTTP2StateProtocol.perform_client_connection_preface")
def test_perform_connection_preface(self, mock_client_method, mock_server_method):
protocol = HTTP2StateProtocol(is_server=False)
protocol.connection_preface_performed = True
protocol.perform_connection_preface()
assert not mock_client_method.called
assert not mock_server_method.called
protocol.perform_connection_preface(force=True)
assert mock_client_method.called
assert not mock_server_method.called
@mock.patch("pathod.protocols.http2.HTTP2StateProtocol.perform_server_connection_preface")
@mock.patch("pathod.protocols.http2.HTTP2StateProtocol.perform_client_connection_preface")
def test_perform_connection_preface_server(self, mock_client_method, mock_server_method):
protocol = HTTP2StateProtocol(is_server=True)
protocol.connection_preface_performed = True
protocol.perform_connection_preface()
assert not mock_client_method.called
assert not mock_server_method.called
protocol.perform_connection_preface(force=True)
assert not mock_client_method.called
assert mock_server_method.called
class TestCheckALPNMatch(net_tservers.ServerTestBase):
handler = EchoHandler
ssl = dict(
alpn_select=b'h2',
)
def test_check_alpn(self):
c = tcp.TCPClient(("127.0.0.1", self.port))
with c.connect():
c.convert_to_tls(alpn_protos=[b'h2'])
protocol = HTTP2StateProtocol(c)
assert protocol.check_alpn()
class TestCheckALPNMismatch(net_tservers.ServerTestBase):
handler = EchoHandler
ssl = dict(
alpn_select=None,
)
def test_check_alpn(self):
c = tcp.TCPClient(("127.0.0.1", self.port))
with c.connect():
c.convert_to_tls(alpn_protos=[b'h2'])
protocol = HTTP2StateProtocol(c)
with pytest.raises(NotImplementedError):
protocol.check_alpn()
class TestPerformServerConnectionPreface(net_tservers.ServerTestBase):
class handler(tcp.BaseHandler):
def handle(self):
# send magic
self.wfile.write(bytes.fromhex("505249202a20485454502f322e300d0a0d0a534d0d0a0d0a"))
self.wfile.flush()
# send empty settings frame
self.wfile.write(bytes.fromhex("000000040000000000"))
self.wfile.flush()
# check empty settings frame
_, consumed_bytes = http2.read_frame(self.rfile, False)
assert consumed_bytes == bytes.fromhex("00000c040000000000000200000000000300000001")
# check settings acknowledgement
_, consumed_bytes = http2.read_frame(self.rfile, False)
assert consumed_bytes == bytes.fromhex("000000040100000000")
# send settings acknowledgement
self.wfile.write(bytes.fromhex("000000040100000000"))
self.wfile.flush()
def test_perform_server_connection_preface(self):
c = tcp.TCPClient(("127.0.0.1", self.port))
with c.connect():
protocol = HTTP2StateProtocol(c)
assert not protocol.connection_preface_performed
protocol.perform_server_connection_preface()
assert protocol.connection_preface_performed
with pytest.raises(exceptions.TcpReadIncomplete):
protocol.perform_server_connection_preface(force=True)
class TestPerformClientConnectionPreface(net_tservers.ServerTestBase):
class handler(tcp.BaseHandler):
def handle(self):
# check magic
assert self.rfile.read(24) == HTTP2StateProtocol.CLIENT_CONNECTION_PREFACE
# check empty settings frame
assert self.rfile.read(9) ==\
bytes.fromhex("000000040000000000")
# send empty settings frame
self.wfile.write(bytes.fromhex("000000040000000000"))
self.wfile.flush()
# check settings acknowledgement
assert self.rfile.read(9) == \
bytes.fromhex("000000040100000000")
# send settings acknowledgement
self.wfile.write(bytes.fromhex("000000040100000000"))
self.wfile.flush()
def test_perform_client_connection_preface(self):
c = tcp.TCPClient(("127.0.0.1", self.port))
with c.connect():
protocol = HTTP2StateProtocol(c)
assert not protocol.connection_preface_performed
protocol.perform_client_connection_preface()
assert protocol.connection_preface_performed
class TestClientStreamIds:
c = tcp.TCPClient(("127.0.0.1", 0))
protocol = HTTP2StateProtocol(c)
def test_client_stream_ids(self):
assert self.protocol.current_stream_id is None
assert self.protocol._next_stream_id() == 1
assert self.protocol.current_stream_id == 1
assert self.protocol._next_stream_id() == 3
assert self.protocol.current_stream_id == 3
assert self.protocol._next_stream_id() == 5
assert self.protocol.current_stream_id == 5
class TestserverstreamIds:
c = tcp.TCPClient(("127.0.0.1", 0))
protocol = HTTP2StateProtocol(c, is_server=True)
def test_server_stream_ids(self):
assert self.protocol.current_stream_id is None
assert self.protocol._next_stream_id() == 2
assert self.protocol.current_stream_id == 2
assert self.protocol._next_stream_id() == 4
assert self.protocol.current_stream_id == 4
assert self.protocol._next_stream_id() == 6
assert self.protocol.current_stream_id == 6
class TestApplySettings(net_tservers.ServerTestBase):
class handler(tcp.BaseHandler):
def handle(self):
# check settings acknowledgement
assert self.rfile.read(9) == bytes.fromhex("000000040100000000")
self.wfile.write(b"OK")
self.wfile.flush()
self.rfile.safe_read(9) # just to keep the connection alive a bit longer
ssl = True
def test_apply_settings(self):
c = tcp.TCPClient(("127.0.0.1", self.port))
with c.connect():
c.convert_to_tls()
protocol = HTTP2StateProtocol(c)
protocol._apply_settings({
hyperframe.frame.SettingsFrame.ENABLE_PUSH: 'foo',
hyperframe.frame.SettingsFrame.MAX_CONCURRENT_STREAMS: 'bar',
hyperframe.frame.SettingsFrame.INITIAL_WINDOW_SIZE: 'deadbeef',
})
assert c.rfile.safe_read(2) == b"OK"
assert protocol.http2_settings[
hyperframe.frame.SettingsFrame.ENABLE_PUSH] == 'foo'
assert protocol.http2_settings[
hyperframe.frame.SettingsFrame.MAX_CONCURRENT_STREAMS] == 'bar'
assert protocol.http2_settings[
hyperframe.frame.SettingsFrame.INITIAL_WINDOW_SIZE] == 'deadbeef'
class TestCreateHeaders:
c = tcp.TCPClient(("127.0.0.1", 0))
def test_create_headers(self):
headers = http.Headers([
(b':method', b'GET'),
(b':path', b'index.html'),
(b':scheme', b'https'),
(b'foo', b'bar')])
data = HTTP2StateProtocol(self.c)._create_headers(
headers, 1, end_stream=True)
assert b''.join(data) == bytes.fromhex("000014010500000001824488355217caf3a69a3f87408294e7838c767f")
data = HTTP2StateProtocol(self.c)._create_headers(
headers, 1, end_stream=False)
assert b''.join(data) == bytes.fromhex("000014010400000001824488355217caf3a69a3f87408294e7838c767f")
def test_create_headers_multiple_frames(self):
headers = http.Headers([
(b':method', b'GET'),
(b':path', b'/'),
(b':scheme', b'https'),
(b'foo', b'bar'),
(b'server', b'version')])
protocol = HTTP2StateProtocol(self.c)
protocol.http2_settings[hyperframe.frame.SettingsFrame.MAX_FRAME_SIZE] = 8
data = protocol._create_headers(headers, 1, end_stream=True)
assert len(data) == 3
assert data[0] == bytes.fromhex("000008010100000001828487408294e783")
assert data[1] == bytes.fromhex("0000080900000000018c767f7685ee5b10")
assert data[2] == bytes.fromhex("00000209040000000163d5")
class TestCreateBody:
c = tcp.TCPClient(("127.0.0.1", 0))
def test_create_body_empty(self):
protocol = HTTP2StateProtocol(self.c)
bytes = protocol._create_body(b'', 1)
assert b''.join(bytes) == b''
def test_create_body_single_frame(self):
protocol = HTTP2StateProtocol(self.c)
data = protocol._create_body(b'foobar', 1)
assert b''.join(data) == bytes.fromhex("000006000100000001666f6f626172")
def test_create_body_multiple_frames(self):
protocol = HTTP2StateProtocol(self.c)
protocol.http2_settings[hyperframe.frame.SettingsFrame.MAX_FRAME_SIZE] = 5
data = protocol._create_body(b'foobarmehm42', 1)
assert len(data) == 3
assert data[0] == bytes.fromhex("000005000000000001666f6f6261")
assert data[1] == bytes.fromhex("000005000000000001726d65686d")
assert data[2] == bytes.fromhex("0000020001000000013432")
class TestReadRequest(net_tservers.ServerTestBase):
class handler(tcp.BaseHandler):
def handle(self):
self.wfile.write(
bytes.fromhex("000003010400000001828487"))
self.wfile.write(
bytes.fromhex("000006000100000001666f6f626172"))
self.wfile.flush()
self.rfile.safe_read(9) # just to keep the connection alive a bit longer
ssl = True
def test_read_request(self):
c = tcp.TCPClient(("127.0.0.1", self.port))
with c.connect():
c.convert_to_tls()
protocol = HTTP2StateProtocol(c, is_server=True)
protocol.connection_preface_performed = True
req = protocol.read_request(NotImplemented)
assert req.stream_id
assert req.headers.fields == ()
assert req.method == "GET"
assert req.path == "/"
assert req.scheme == "https"
assert req.content == b'foobar'
class TestReadRequestRelative(net_tservers.ServerTestBase):
class handler(tcp.BaseHandler):
def handle(self):
self.wfile.write(
bytes.fromhex("00000c0105000000014287d5af7e4d5a777f4481f9"))
self.wfile.flush()
ssl = True
def test_asterisk_form(self):
c = tcp.TCPClient(("127.0.0.1", self.port))
with c.connect():
c.convert_to_tls()
protocol = HTTP2StateProtocol(c, is_server=True)
protocol.connection_preface_performed = True
req = protocol.read_request(NotImplemented)
assert req.first_line_format == "relative"
assert req.method == "OPTIONS"
assert req.path == "*"
class TestReadResponse(net_tservers.ServerTestBase):
class handler(tcp.BaseHandler):
def handle(self):
self.wfile.write(
bytes.fromhex("00000801040000002a88628594e78c767f"))
self.wfile.write(
bytes.fromhex("00000600010000002a666f6f626172"))
self.wfile.flush()
self.rfile.safe_read(9) # just to keep the connection alive a bit longer
ssl = True
def test_read_response(self):
c = tcp.TCPClient(("127.0.0.1", self.port))
with c.connect():
c.convert_to_tls()
protocol = HTTP2StateProtocol(c)
protocol.connection_preface_performed = True
resp = protocol.read_response(NotImplemented, stream_id=42)
assert resp.http_version == "HTTP/2.0"
assert resp.status_code == 200
assert resp.reason == ''
assert resp.headers.fields == ((b':status', b'200'), (b'etag', b'foobar'))
assert resp.content == b'foobar'
assert resp.timestamp_end
class TestReadEmptyResponse(net_tservers.ServerTestBase):
class handler(tcp.BaseHandler):
def handle(self):
self.wfile.write(
bytes.fromhex("00000801050000002a88628594e78c767f"))
self.wfile.flush()
ssl = True
def test_read_empty_response(self):
c = tcp.TCPClient(("127.0.0.1", self.port))
with c.connect():
c.convert_to_tls()
protocol = HTTP2StateProtocol(c)
protocol.connection_preface_performed = True
resp = protocol.read_response(NotImplemented, stream_id=42)
assert resp.stream_id == 42
assert resp.http_version == "HTTP/2.0"
assert resp.status_code == 200
assert resp.reason == ''
assert resp.headers.fields == ((b':status', b'200'), (b'etag', b'foobar'))
assert resp.content == b''
class TestAssembleRequest:
c = tcp.TCPClient(("127.0.0.1", 0))
def test_request_simple(self):
data = HTTP2StateProtocol(self.c).assemble_request(http.Request(
host="",
port=0,
method=b'GET',
scheme=b'https',
authority=b'',
path=b'/',
http_version=b"HTTP/2.0",
headers=(),
content=None,
trailers=None,
timestamp_start=0,
timestamp_end=0
))
assert len(data) == 1
assert data[0] == bytes.fromhex('00000d0105000000018284874188089d5c0b8170dc07')
def test_request_with_stream_id(self):
req = http.Request(
host="",
port=0,
method=b'GET',
scheme=b'https',
authority=b'',
path=b'/',
http_version=b"HTTP/2.0",
headers=(),
content=None,
trailers=None,
timestamp_start=0,
timestamp_end=0
)
req.stream_id = 0x42
data = HTTP2StateProtocol(self.c).assemble_request(req)
assert len(data) == 1
assert data[0] == bytes.fromhex('00000d0105000000428284874188089d5c0b8170dc07')
def test_request_with_body(self):
data = HTTP2StateProtocol(self.c).assemble_request(http.Request(
host="",
port=0,
method=b'GET',
scheme=b'https',
authority=b'',
path=b'/',
http_version=b"HTTP/2.0",
headers=http.Headers([(b'foo', b'bar')]),
content=b'foobar',
trailers=None,
timestamp_start=0,
timestamp_end=None,
))
assert len(data) == 2
assert data[0] == bytes.fromhex("0000150104000000018284874188089d5c0b8170dc07408294e7838c767f")
assert data[1] == bytes.fromhex("000006000100000001666f6f626172")
class TestAssembleResponse:
c = tcp.TCPClient(("127.0.0.1", 0))
def test_simple(self):
data = HTTP2StateProtocol(self.c, is_server=True).assemble_response(http.Response(
http_version=b"HTTP/2.0",
status_code=200,
reason=b"",
headers=(),
content=b"",
trailers=None,
timestamp_start=0,
timestamp_end=0,
))
assert len(data) == 1
assert data[0] == bytes.fromhex("00000101050000000288")
def test_with_stream_id(self):
resp = http.Response(
http_version=b"HTTP/2.0",
status_code=200,
reason=b"",
headers=(),
content=b"",
trailers=None,
timestamp_start=0,
timestamp_end=0,
)
resp.stream_id = 0x42
data = HTTP2StateProtocol(self.c, is_server=True).assemble_response(resp)
assert len(data) == 1
assert data[0] == bytes.fromhex("00000101050000004288")
def test_with_body(self):
data = HTTP2StateProtocol(self.c, is_server=True).assemble_response(http.Response(
http_version=b"HTTP/2.0",
status_code=200,
reason=b'',
headers=http.Headers(foo=b"bar"),
content=b'foobar',
trailers=None,
timestamp_start=0,
timestamp_end=0,
))
assert len(data) == 2
assert data[0] == bytes.fromhex("00000901040000000288408294e7838c767f")
assert data[1] == bytes.fromhex("000006000100000002666f6f626172")

1
test/pathod/protocols/test_websockets.py
View File

@ -1 +0,0 @@
# TODO: write tests

17
test/pathod/scripts/generate.sh
View File

@ -1,17 +0,0 @@
#!/bin/bash
if [ ! -f ./private.key ]
then
openssl genrsa -out private.key 3072
fi
openssl req \
-batch \
-new -x509 \
-key private.key \
-sha256 \
-out cert.pem \
-days 9999 \
-config ./openssl.cnf
openssl x509 -in cert.pem -text -noout
cat ./private.key ./cert.pem > testcert.pem
rm ./private.key ./cert.pem

39
test/pathod/scripts/openssl.cnf
View File

@ -1,39 +0,0 @@
[ req ]
default_bits = 1024
default_keyfile = privkey.pem
distinguished_name = req_distinguished_name
x509_extensions = v3_ca
[ req_distinguished_name ]
countryName = Country Name (2 letter code)
countryName_default = NZ
countryName_min = 2
countryName_max = 2
stateOrProvinceName = State or Province Name (full name)
stateOrProvinceName_default = Otago
localityName = Locality Name (eg, city)
0.organizationName = Organization Name (eg, company)
0.organizationName_default = Pathod
commonName = Common Name (e.g. server FQDN or YOUR name)
commonName_default = test.com
commonName_max = 64
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
[ v3_ca ]
keyUsage = digitalSignature, keyEncipherment
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid:always,issuer:always
basicConstraints = CA:true
subjectAltName = @alternate_names
[ alternate_names ]
DNS.1 = test.com
DNS.2 = test2.com
DNS.3 = test3.com

25
test/pathod/test_log.py
View File

@ -1,25 +0,0 @@
import io
from pathod import log
from mitmproxy import exceptions
class DummyIO(io.StringIO):
def start_log(self, *args, **kwargs):
pass
def get_log(self, *args, **kwargs):
return ""
def test_disconnect():
outf = DummyIO()
rw = DummyIO()
l = log.ConnectionLogger(outf, False, True, rw, rw)
try:
with l.ctx() as lg:
lg("Test")
except exceptions.TcpDisconnect:
pass
assert "Test" in outf.getvalue()

250
test/pathod/test_pathoc.py
View File

@ -1,250 +0,0 @@
import io
from unittest.mock import Mock
import pytest
from mitmproxy import exceptions
from mitmproxy.net.http import http1
from mitmproxy.test import tutils
from pathod import language, pathoc
from pathod.protocols.http2 import HTTP2StateProtocol
from . import tservers
class PathocTestDaemon(tservers.DaemonTests):
def tval(self, requests, timeout=None, showssl=False, **kwargs):
s = io.StringIO()
c = pathoc.Pathoc(
("127.0.0.1", self.d.port),
ssl=self.ssl,
fp=s,
**kwargs
)
with c.connect(showssl=showssl, fp=s):
if timeout:
c.settimeout(timeout)
for i in requests:
r = next(language.parse_pathoc(i))
if kwargs.get("explain"):
r = r.freeze(language.Settings())
try:
c.request(r)
except exceptions.NetlibException:
pass
self.d.wait_for_silence()
return s.getvalue()
class TestDaemonSSL(PathocTestDaemon):
ssl = True
ssloptions = dict(
request_client_cert=True,
sans=[b"test1.com", b"test2.com"],
alpn_select=b'h2',
)
def test_sni(self):
self.tval(
["get:/p/200"],
sni="foobar.com"
)
log = self.d.log()
assert log[0]["request"]["sni"] == "foobar.com"
def test_showssl(self):
assert "certificate chain" in self.tval(["get:/p/200"], showssl=True)
def test_clientcert(self, tdata):
self.tval(
["get:/p/200"],
clientcert=tdata.path("pathod/data/clientcert/client.pem"),
)
log = self.d.log()
assert log[0]["request"]["clientcert"]["keyinfo"]
def test_http2_without_ssl(self):
fp = io.StringIO()
c = pathoc.Pathoc(
("127.0.0.1", self.d.port),
use_http2=True,
ssl=False,
fp=fp
)
with pytest.raises(NotImplementedError):
c.connect()
class TestDaemon(PathocTestDaemon):
ssl = False
def test_ssl_error(self):
c = pathoc.Pathoc(("127.0.0.1", self.d.port), ssl=True, fp=None)
try:
with c.connect():
pass
except Exception as e:
assert "SSL" in str(e)
else:
raise AssertionError("No exception raised.")
def test_showssl(self):
assert "certificate chain" not in self.tval(
["get:/p/200"],
showssl=True)
def test_ignorecodes(self):
assert "200" in self.tval(["get:'/p/200:b@1'"])
assert "200" in self.tval(["get:'/p/200:b@1'"])
assert "200" in self.tval(["get:'/p/200:b@1'"])
assert "200" not in self.tval(["get:'/p/200:b@1'"], ignorecodes=[200])
assert "200" not in self.tval(
["get:'/p/200:b@1'"],
ignorecodes=[
200,
201])
assert "202" in self.tval(["get:'/p/202:b@1'"], ignorecodes=[200, 201])
def _test_timeout(self):
assert "Timeout" in self.tval(["get:'/p/200:p0,100'"], timeout=0.01)
assert "HTTP" in self.tval(
["get:'/p/200:p5,100'"],
showresp=True,
timeout=1
)
assert "HTTP" not in self.tval(
["get:'/p/200:p3,100'"],
showresp=True,
timeout=1,
ignoretimeout=True
)
def test_showresp(self):
reqs = ["get:/p/200:da", "get:/p/200:da"]
assert self.tval(reqs).count("200 OK") == 2
assert self.tval(reqs, showresp=True).count("HTTP/1.1 200 OK") == 2
assert self.tval(
reqs, showresp=True, hexdump=True
).count("0000000000") == 2
def test_showresp_httperr(self):
v = self.tval(["get:'/p/200:d20'"], showresp=True, showsummary=True)
assert "Invalid header" in v
assert "HTTP/" in v
def test_explain(self):
reqs = ["get:/p/200:b@100"]
assert "b@100" not in self.tval(reqs, explain=True)
def test_showreq(self):
reqs = ["get:/p/200:da", "get:/p/200:da"]
assert self.tval(reqs, showreq=True).count("GET /p/200") == 2
assert self.tval(
reqs, showreq=True, hexdump=True
).count("0000000000") == 2
def test_conn_err(self):
assert "Invalid server response" in self.tval(["get:'/p/200:d2'"])
def test_websocket_shutdown(self):
self.tval(["ws:/"])
def test_wait_finish(self):
c = pathoc.Pathoc(
("127.0.0.1", self.d.port),
fp=None,
ws_read_limit=1
)
with c.connect():
c.request("ws:/")
c.request("wf:f'wf'")
# This should read a frame and close the websocket reader
assert len([i for i in c.wait(timeout=5, finish=False)]) == 1
assert not [i for i in c.wait(timeout=0)]
def test_connect_fail(self):
to = ("foobar", 80)
c = pathoc.Pathoc(("127.0.0.1", self.d.port), fp=None)
c.rfile, c.wfile = io.BytesIO(), io.BytesIO()
with pytest.raises(Exception, match="CONNECT failed"):
c.http_connect(to)
c.rfile = io.BytesIO(
b"HTTP/1.1 500 OK\r\n"
)
with pytest.raises(Exception, match="CONNECT failed"):
c.http_connect(to)
c.rfile = io.BytesIO(
b"HTTP/1.1 200 OK\r\n"
)
c.http_connect(to)
def test_socks_connect(self):
to = ("foobar", 80)
c = pathoc.Pathoc(("127.0.0.1", self.d.port), fp=None)
c.rfile, c.wfile = tutils.treader(b""), io.BytesIO()
with pytest.raises(pathoc.PathocError):
c.socks_connect(to)
c.rfile = tutils.treader(
b"\x05\xEE"
)
with pytest.raises(Exception, match="SOCKS without authentication"):
c.socks_connect(("example.com", 0xDEAD))
c.rfile = tutils.treader(
b"\x05\x00" +
b"\x05\xEE\x00\x03\x0bexample.com\xDE\xAD"
)
with pytest.raises(Exception, match="SOCKS server error"):
c.socks_connect(("example.com", 0xDEAD))
c.rfile = tutils.treader(
b"\x05\x00" +
b"\x05\x00\x00\x03\x0bexample.com\xDE\xAD"
)
c.socks_connect(("example.com", 0xDEAD))
class TestDaemonHTTP2(PathocTestDaemon):
ssl = True
explain = False
def test_http2(self):
c = pathoc.Pathoc(
("127.0.0.1", self.d.port),
fp=None,
ssl=True,
use_http2=True,
)
assert isinstance(c.protocol, HTTP2StateProtocol)
c = pathoc.Pathoc(
("127.0.0.1", self.d.port),
)
assert c.protocol == http1
def test_http2_alpn(self):
c = pathoc.Pathoc(
("127.0.0.1", self.d.port),
fp=None,
ssl=True,
use_http2=True,
http2_skip_connection_preface=True,
)
tmp_convert_to_tls = c.convert_to_tls
c.convert_to_tls = Mock()
c.convert_to_tls.side_effect = tmp_convert_to_tls
with c.connect():
_, kwargs = c.convert_to_tls.call_args
assert set(kwargs['alpn_protos']) == {b'http/1.1', b'h2'}
def test_request(self):
c = pathoc.Pathoc(
("127.0.0.1", self.d.port),
fp=None,
ssl=True,
use_http2=True,
)
with c.connect():
resp = c.request("get:/p/200")
assert resp.status_code == 200

60
test/pathod/test_pathoc_cmdline.py
View File

@ -1,60 +0,0 @@
import io
import pytest
from unittest import mock
from pathod import pathoc_cmdline as cmdline
@mock.patch("argparse.ArgumentParser.error")
def test_pathoc(perror, tdata):
assert cmdline.args_pathoc(["pathoc", "foo.com", "get:/"])
s = io.StringIO()
with pytest.raises(SystemExit):
cmdline.args_pathoc(["pathoc", "--show-uas"], s, s)
a = cmdline.args_pathoc(["pathoc", "foo.com:8888", "get:/"])
assert a.port == 8888
a = cmdline.args_pathoc(["pathoc", "foo.com:xxx", "get:/"])
assert perror.called
perror.reset_mock()
a = cmdline.args_pathoc(["pathoc", "-I", "10, 20", "foo.com:8888", "get:/"])
assert a.ignorecodes == [10, 20]
a = cmdline.args_pathoc(["pathoc", "-I", "xx, 20", "foo.com:8888", "get:/"])
assert perror.called
perror.reset_mock()
a = cmdline.args_pathoc(["pathoc", "-c", "foo:10", "foo.com:8888", "get:/"])
assert a.connect_to == ["foo", 10]
a = cmdline.args_pathoc(["pathoc", "foo.com", "get:/", "--http2"])
assert a.use_http2 is True
assert a.ssl is True
a = cmdline.args_pathoc(["pathoc", "foo.com", "get:/", "--http2-skip-connection-preface"])
assert a.use_http2 is True
assert a.ssl is True
assert a.http2_skip_connection_preface is True
a = cmdline.args_pathoc(["pathoc", "-c", "foo", "foo.com:8888", "get:/"])
assert perror.called
perror.reset_mock()
a = cmdline.args_pathoc(
["pathoc", "-c", "foo:bar", "foo.com:8888", "get:/"])
assert perror.called
perror.reset_mock()
a = cmdline.args_pathoc(
[
"pathoc",
"foo.com:8888",
tdata.path("pathod/data/request")
]
)
assert len(list(a.requests)) == 1
with pytest.raises(SystemExit):
cmdline.args_pathoc(["pathoc", "foo.com", "invalid"], s, s)

264
test/pathod/test_pathod.py
View File

@ -1,264 +0,0 @@
import io
import pytest
from pathod import pathod
from mitmproxy.net import tcp
from mitmproxy import exceptions
from mitmproxy.utils import data
from . import tservers
cdata = data.Data(__name__)
class TestPathod:
def test_logging(self):
s = io.StringIO()
p = pathod.Pathod(("127.0.0.1", 0), logfp=s)
assert len(p.get_log()) == 0
id = p.add_log(dict(s="foo"))
assert p.log_by_id(id)
assert len(p.get_log()) == 1
p.clear_log()
assert len(p.get_log()) == 0
for _ in range(p.LOGBUF + 1):
p.add_log(dict(s="foo"))
assert len(p.get_log()) <= p.LOGBUF
class TestTimeout(tservers.DaemonTests):
timeout = 0.01
def test_timeout(self):
# FIXME: Add float values to spec language, reduce test timeout to
# increase test performance
# This is a bodge - we have some platform difference that causes
# different exceptions to be raised here.
with pytest.raises(Exception):
self.pathoc(["get:/:p1,1"])
assert self.d.last_log()["type"] == "timeout"
class TestNotAfterConnect(tservers.DaemonTests):
ssl = False
ssloptions = dict(
not_after_connect=True
)
def test_connect(self):
r, _ = self.pathoc(
[r"get:'http://foo.com/p/202':da"],
connect_to=("localhost", self.d.port)
)
assert r[0].status_code == 202
class TestCustomCert(tservers.DaemonTests):
ssl = True
ssloptions = dict(
certs=[("*", cdata.path("data/testkey.pem"))],
)
def test_connect(self):
r, _ = self.pathoc([r"get:/p/202"])
r = r[0]
assert r.status_code == 202
assert r.sslinfo
assert "test.com" in str(r.sslinfo.certchain[0].get_subject())
class TestSSLCN(tservers.DaemonTests):
ssl = True
ssloptions = dict(
cn=b"foo.com"
)
def test_connect(self):
r, _ = self.pathoc([r"get:/p/202"])
r = r[0]
assert r.status_code == 202
assert r.sslinfo
assert r.sslinfo.certchain[0].get_subject().CN == "foo.com"
class TestNohang(tservers.DaemonTests):
nohang = True
def test_nohang(self):
r = self.get("200:p0,0")
assert r.status_code == 800
l = self.d.last_log()
assert "Pauses have been disabled" in l["response"]["msg"]
class TestHexdump(tservers.DaemonTests):
hexdump = True
def test_hexdump(self):
assert self.get(r"200:b'\xf0'")
class TestNocraft(tservers.DaemonTests):
nocraft = True
def test_nocraft(self):
r = self.get(r"200:b'\xf0'")
assert r.status_code == 800
assert b"Crafting disabled" in r.content
class CommonTests(tservers.DaemonTests):
def test_binarydata(self):
assert self.get(r"200:b'\xf0'")
assert self.d.last_log()
# FIXME: Other binary data elements
def test_sizelimit(self):
r = self.get("200:b@1g")
assert r.status_code == 800
l = self.d.last_log()
assert "too large" in l["response"]["msg"]
def test_preline(self):
r, _ = self.pathoc([r"get:'/p/200':i0,'\r\n'"])
assert r[0].status_code == 200
def test_logs(self):
self.d.clear_log()
assert self.get("202:da")
assert self.d.expect_log(1)
self.d.clear_log()
assert len(self.d.log()) == 0
def test_disconnect(self):
with pytest.raises(Exception, match="Unexpected EOF"):
self.get("202:b@100k:d200")
def test_parserr(self):
rsp = self.get("400:msg,b:")
assert rsp.status_code == 800
def test_static(self):
rsp = self.get("200:b<file")
assert rsp.status_code == 200
assert rsp.content.strip() == b"testfile"
def test_anchor(self):
rsp = self.getpath("/anchor/foo")
assert rsp.status_code == 202
def test_invalid_first_line(self):
c = tcp.TCPClient(("localhost", self.d.port))
with c.connect():
if self.ssl:
c.convert_to_tls()
c.wfile.write(b"foo\n\n\n")
c.wfile.flush()
l = self.d.last_log()
assert l["type"] == "error"
assert "foo" in l["msg"]
def test_invalid_content_length(self):
with pytest.raises(exceptions.HttpException):
self.pathoc(["get:/:h'content-length'='foo'"])
l = self.d.last_log()
assert l["type"] == "error"
assert "Unparseable Content Length" in l["msg"]
def test_invalid_headers(self):
with pytest.raises(exceptions.HttpException):
self.pathoc(["get:/:h'\t'='foo'"])
l = self.d.last_log()
assert l["type"] == "error"
assert "Invalid headers" in l["msg"]
def test_access_denied(self):
rsp = self.get("=nonexistent")
assert rsp.status_code == 800
def test_source_access_denied(self):
rsp = self.get("200:b</foo")
assert rsp.status_code == 800
assert b"File access denied" in rsp.content
def test_proxy(self):
r, _ = self.pathoc([r"get:'http://foo.com/p/202':da"])
assert r[0].status_code == 202
def test_websocket(self):
r, _ = self.pathoc(["ws:/p/"], ws_read_limit=0)
assert r[0].status_code == 101
r, _ = self.pathoc(["ws:/p/ws"], ws_read_limit=0)
assert r[0].status_code == 101
def test_websocket_frame(self):
r, _ = self.pathoc(
["ws:/p/", "wf:f'wf:b\"test\"':pa,1"],
ws_read_limit=1
)
assert r[1].payload == b"test"
def test_websocket_frame_reflect_error(self):
r, _ = self.pathoc(
["ws:/p/", "wf:-mask:knone:f'wf:b@10':i13,'a'"],
ws_read_limit=1,
timeout=1
)
# FIXME: Race Condition?
assert "Parse error" in self.d.text_log()
def test_websocket_frame_disconnect_error(self):
self.pathoc(["ws:/p/", "wf:b@10:d3"], ws_read_limit=0)
assert self.d.last_log()
class TestDaemon(CommonTests):
ssl = False
def test_connect(self):
r, _ = self.pathoc(
[r"get:'http://foo.com/p/202':da"],
connect_to=("localhost", self.d.port),
ssl=True
)
assert r[0].status_code == 202
def test_connect_err(self):
with pytest.raises(exceptions.HttpException):
self.pathoc([r"get:'http://foo.com/p/202':da"], connect_to=("localhost", self.d.port))
class TestDaemonSSL(CommonTests):
ssl = True
def test_ssl_conn_failure(self):
c = tcp.TCPClient(("localhost", self.d.port))
c.rbufsize = 0
c.wbufsize = 0
with c.connect():
c.wfile.write(b"\0\0\0\0")
with pytest.raises(exceptions.TlsException):
c.convert_to_tls()
l = self.d.last_log()
assert l["type"] == "error"
assert "SSL" in l["msg"]
def test_ssl_cipher(self):
r, _ = self.pathoc([r"get:/p/202"])
assert r[0].status_code == 202
assert self.d.last_log()["cipher"][1] > 0
class TestHTTP2(tservers.DaemonTests):
ssl = True
nohang = True
def test_http2(self):
r, _ = self.pathoc(["GET:/"], ssl=True, use_http2=True)
assert r[0].status_code == 800

89
test/pathod/test_pathod_cmdline.py
View File

@ -1,89 +0,0 @@
from unittest import mock
from pathod import pathod_cmdline as cmdline
def test_parse_anchor_spec():
assert cmdline.parse_anchor_spec("foo=200") == ("foo", "200")
assert cmdline.parse_anchor_spec("foo") is None
@mock.patch("argparse.ArgumentParser.error")
def test_pathod(perror, tdata):
assert cmdline.args_pathod(["pathod"])
a = cmdline.args_pathod(
[
"pathod",
"--cert",
tdata.path("pathod/data/testkey.pem")
]
)
assert a.ssl_certs
a = cmdline.args_pathod(
[
"pathod",
"--cert",
"nonexistent"
]
)
assert perror.called
perror.reset_mock()
a = cmdline.args_pathod(
[
"pathod",
"-a",
"foo=200"
]
)
assert a.anchors
a = cmdline.args_pathod(
[
"pathod",
"-a",
"foo=" + tdata.path("pathod/data/response")
]
)
assert a.anchors
a = cmdline.args_pathod(
[
"pathod",
"-a",
"?=200"
]
)
assert perror.called
perror.reset_mock()
a = cmdline.args_pathod(
[
"pathod",
"-a",
"foo"
]
)
assert perror.called
perror.reset_mock()
a = cmdline.args_pathod(
[
"pathod",
"--limit-size",
"200k"
]
)
assert a.sizelimit
a = cmdline.args_pathod(
[
"pathod",
"--limit-size",
"q"
]
)
assert perror.called
perror.reset_mock()

34
test/pathod/test_test.py
View File

@ -1,34 +0,0 @@
import os
import requests
import pytest
from pathod import test
from pathod.pathod import SSLOptions, CA_CERT_NAME
class TestDaemonManual:
def test_simple(self):
with test.Daemon() as d:
rsp = requests.get("http://localhost:%s/p/202:da" % d.port)
assert rsp.ok
assert rsp.status_code == 202
with pytest.raises(requests.ConnectionError):
requests.get("http://localhost:%s/p/202:da" % d.port)
@pytest.mark.parametrize('not_after_connect', [True, False])
def test_startstop_ssl(self, not_after_connect):
ssloptions = SSLOptions(
cn=b'localhost',
sans=[b'localhost', b'127.0.0.1'],
not_after_connect=not_after_connect,
)
d = test.Daemon(ssl=True, ssloptions=ssloptions)
rsp = requests.get(
"https://localhost:%s/p/202:da" % d.port,
verify=os.path.expanduser(os.path.join(d.thread.server.ssloptions.confdir, CA_CERT_NAME)))
assert rsp.ok
assert rsp.status_code == 202
d.shutdown()
with pytest.raises(requests.ConnectionError):
requests.get("http://localhost:%s/p/202:da" % d.port)

17
test/pathod/test_utils.py
View File

@ -1,17 +0,0 @@
import pytest
from pathod import utils
def test_membool():
m = utils.MemBool()
assert not m.v
assert m(1)
assert m.v == 1
assert m(2)
assert m.v == 2
def test_data_path():
with pytest.raises(ValueError):
utils.data.path("nonexistent")

150
test/pathod/tservers.py
View File

@ -1,150 +0,0 @@
import os
import tempfile
import re
import shutil
import requests
import io
import urllib
from mitmproxy.net import tcp
from mitmproxy.utils import data
from pathod import language
from pathod import pathoc
from pathod import pathod
from pathod import test
from pathod.pathod import CA_CERT_NAME
cdata = data.Data(__name__)
def treader(bytes):
"""
Construct a tcp.Read object from bytes.
"""
fp = io.BytesIO(bytes)
return tcp.Reader(fp)
class DaemonTests:
nohang = False
ssl = False
timeout = None
hexdump = False
ssloptions = None
nocraft = False
explain = True
@classmethod
def setup_class(cls):
opts = cls.ssloptions or {}
cls.confdir = tempfile.mkdtemp()
opts["confdir"] = cls.confdir
so = pathod.SSLOptions(**opts)
cls.d = test.Daemon(
staticdir=cdata.path("data"),
anchors=[
(re.compile("/anchor/.*"), "202:da")
],
ssl=cls.ssl,
ssloptions=so,
sizelimit=1 * 1024 * 1024,
nohang=cls.nohang,
timeout=cls.timeout,
hexdump=cls.hexdump,
nocraft=cls.nocraft,
logreq=True,
logresp=True,
explain=cls.explain
)
@classmethod
def teardown_class(cls):
cls.d.shutdown()
shutil.rmtree(cls.confdir)
def teardown(self):
self.d.wait_for_silence()
self.d.clear_log()
def _getpath(self, path, params=None):
scheme = "https" if self.ssl else "http"
resp = requests.get(
"{}://localhost:{}/{}".format(
scheme,
self.d.port,
path
),
verify=os.path.join(self.d.thread.server.ssloptions.confdir, CA_CERT_NAME),
params=params
)
return resp
def getpath(self, path, params=None):
logfp = io.StringIO()
c = pathoc.Pathoc(
("localhost", self.d.port),
ssl=self.ssl,
fp=logfp,
)
with c.connect():
if params:
path = path + "?" + urllib.parse.urlencode(params)
resp = c.request("get:%s" % path)
return resp
def get(self, spec):
logfp = io.StringIO()
c = pathoc.Pathoc(
("localhost", self.d.port),
ssl=self.ssl,
fp=logfp,
)
with c.connect():
resp = c.request(
"get:/p/%s" % urllib.parse.quote(spec)
)
return resp
def pathoc(
self,
specs,
timeout=None,
connect_to=None,
ssl=None,
ws_read_limit=None,
use_http2=False,
):
"""
Returns a (messages, text log) tuple.
"""
if ssl is None:
ssl = self.ssl
logfp = io.StringIO()
c = pathoc.Pathoc(
("localhost", self.d.port),
ssl=ssl,
ws_read_limit=ws_read_limit,
timeout=timeout,
fp=logfp,
use_http2=use_http2,
)
with c.connect(connect_to):
ret = []
for i in specs:
resp = c.request(i)
if resp:
ret.append(resp)
for frm in c.wait():
ret.append(frm)
c.stop()
return ret, logfp.getvalue()
def render(r, settings=language.Settings()):
r = r.resolve(settings)
s = io.BytesIO()
assert language.serve(r, s, settings)
return s.getvalue()

2
test/release/test_cibuild.py
View File

@ -222,7 +222,6 @@ def test_buildenviron_osx(tmpdir):
assert be.platform_tag == "osx"
assert be.bdists == {
"mitmproxy": ["mitmproxy", "mitmdump", "mitmweb"],
"pathod": ["pathoc", "pathod"],
}
assert be.archive_name("mitmproxy") == "mitmproxy-0.0.1-osx.tar.gz"
@ -241,7 +240,6 @@ def test_buildenviron_windows(tmpdir):
assert be.platform_tag == "windows"
assert be.bdists == {
"mitmproxy": ["mitmdump", "mitmweb"],
"pathod": ["pathoc", "pathod"],
}
assert be.archive_name("mitmproxy") == "mitmproxy-0.0.1-windows.zip"

Some files were not shown because too many files have changed in this diff Show More