Magic-Mirror/mitmproxy
1
0
Fork 0
mirror of https://github.com/mitmproxy/mitmproxy.git synced 2024-11-23 13:19:48 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Replace configargparse with argparse, enable new options loading

Browse Source 
This commit ditches configargparse and enables the new config file format. The
default location is ~/.mitmproxy/config.yaml. Unifying all of the various
Options objects will follow in the next patch.
This commit is contained in:
Aldo Cortesi 2016-12-05 15:55:43 +13:00
parent 6aacd27ab2
commit d742d4fb8c
7 changed files with 126 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -1,6 +1,6 @@
.DS_Store
MANIFEST
*/tmp
**/tmp
/venv*
*.py[cdo]
*.swp

1
mitmproxy/optmanager.py
View File

@ -238,6 +238,7 @@ class OptManager(metaclass=_DefaultsMeta):
OptionsError.
"""
for p in paths:
p = os.path.expanduser(p)
if os.path.exists(p) and os.path.isfile(p):
txt = open(p, "r").read()
self.load(txt)

7
mitmproxy/platform/windows.py
View File

@ -7,7 +7,7 @@ import struct
import threading
import time
import configargparse
import argparse
import pydivert
import pydivert.consts
import pickle
@ -386,8 +386,9 @@ class TransparentProxy:
if __name__ == "__main__":
parser = configargparse.ArgumentParser(
description="Windows Transparent Proxy")
parser = argparse.ArgumentParser(
description="Windows Transparent Proxy"
)
parser.add_argument(
'--mode',
choices=[

168
mitmproxy/tools/cmdline.py
View File

@ -1,6 +1,7 @@
import configargparse
import os
import argparse
import re
import os
from mitmproxy import exceptions
from mitmproxy import flowfilter
from mitmproxy import options
@ -11,6 +12,9 @@ from mitmproxy import version
from mitmproxy.addons import view
CONFIG_PATH = os.path.join(options.CA_DIR, "config.yaml")
class ParseException(Exception):
pass
@ -113,13 +117,13 @@ def get_common_options(args):
stream_large_bodies = human.parse_size(stream_large_bodies)
reps = []
for i in args.replace:
for i in args.replace or []:
try:
p = parse_replace_hook(i)
except ParseException as e:
raise exceptions.OptionsError(e)
reps.append(p)
for i in args.replace_file:
for i in args.replace_file or []:
try:
patt, rex, path = parse_replace_hook(i)
except ParseException as e:
@ -133,7 +137,7 @@ def get_common_options(args):
reps.append((patt, rex, v))
setheaders = []
for i in args.setheader:
for i in args.setheader or []:
try:
p = parse_setheader(i)
except ParseException as e:
@ -154,7 +158,7 @@ def get_common_options(args):
# Proxy config
certs = []
for i in args.certs:
for i in args.certs or []:
parts = i.split("=", 1)
if len(parts) == 1:
parts = ["*", parts[0]]
@ -287,8 +291,7 @@ def basic_options(parser):
)
parser.add_argument(
"--anticache",
action="store_true", dest="anticache", default=False,
action="store_true", dest="anticache",
help="""
Strip out request headers that might cause the server to return
304-not-modified.
@ -296,12 +299,12 @@ def basic_options(parser):
)
parser.add_argument(
"--cadir",
action="store", type=str, dest="cadir", default=options.CA_DIR,
action="store", type=str, dest="cadir",
help="Location of the default mitmproxy CA files. (%s)" % options.CA_DIR
)
parser.add_argument(
"--host",
action="store_true", dest="showhost", default=False,
action="store_true", dest="showhost",
help="Use the Host header to construct URLs for display."
)
parser.add_argument(
@ -311,12 +314,12 @@ def basic_options(parser):
)
parser.add_argument(
"-r", "--read-flows",
action="store", dest="rfile", default=None,
action="store", dest="rfile",
help="Read flows from file."
)
parser.add_argument(
"-s", "--script",
action="append", type=str, dest="scripts", default=[],
action="append", type=str, dest="scripts",
metavar='"script.py --bar"',
help="""
Run a script. Surround with quotes to pass script arguments. Can be
@ -327,18 +330,17 @@ def basic_options(parser):
"-t", "--stickycookie",
action="store",
dest="stickycookie_filt",
default=None,
metavar="FILTER",
help="Set sticky cookie filter. Matched against requests."
)
parser.add_argument(
"-u", "--stickyauth",
action="store", dest="stickyauth_filt", default=None, metavar="FILTER",
action="store", dest="stickyauth_filt", metavar="FILTER",
help="Set sticky auth filter. Matched against requests."
)
parser.add_argument(
"-v", "--verbose",
action="store_const", dest="verbose", default=2, const=3,
action="store_const", dest="verbose", const=3,
help="Increase log verbosity."
)
streamfile = parser.add_mutually_exclusive_group()
@ -354,19 +356,19 @@ def basic_options(parser):
)
parser.add_argument(
"-z", "--anticomp",
action="store_true", dest="anticomp", default=False,
action="store_true", dest="anticomp",
help="Try to convince servers to send us un-compressed data."
)
parser.add_argument(
"-Z", "--body-size-limit",
action="store", dest="body_size_limit", default=None,
action="store", dest="body_size_limit",
metavar="SIZE",
help="Byte size limit of HTTP request and response bodies."
" Understands k/m/g suffixes, i.e. 3m for 3 megabytes."
)
parser.add_argument(
"--stream",
action="store", dest="stream_large_bodies", default=None,
action="store", dest="stream_large_bodies",
metavar="SIZE",
help="""
Stream data to the client if response body exceeds the given
@ -383,7 +385,6 @@ def proxy_modes(parser):
action="store",
type=str,
dest="reverse_proxy",
default=None,
help="""
Forward all requests to upstream HTTP server:
http[s]://host[:port]. Clients can always connect both
@ -393,12 +394,12 @@ def proxy_modes(parser):
)
group.add_argument(
"--socks",
action="store_true", dest="socks_proxy", default=False,
action="store_true", dest="socks_proxy",
help="Set SOCKS5 proxy mode."
)
group.add_argument(
"-T", "--transparent",
action="store_true", dest="transparent_proxy", default=False,
action="store_true", dest="transparent_proxy",
help="Set transparent proxy mode."
)
group.add_argument(
@ -406,7 +407,6 @@ def proxy_modes(parser):
action="store",
type=str,
dest="upstream_proxy",
default=None,
help="Forward all requests to upstream proxy server: http://host[:port]"
)
@ -415,12 +415,12 @@ def proxy_options(parser):
group = parser.add_argument_group("Proxy Options")
group.add_argument(
"-b", "--bind-address",
action="store", type=str, dest="addr", default='',
action="store", type=str, dest="addr",
help="Address to bind proxy to (defaults to all interfaces)"
)
group.add_argument(
"-I", "--ignore",
action="append", type=str, dest="ignore_hosts", default=[],
action="append", type=str, dest="ignore_hosts",
metavar="HOST",
help="""
Ignore host and forward all traffic without processing it. In
@ -433,7 +433,7 @@ def proxy_options(parser):
)
group.add_argument(
"--tcp",
action="append", type=str, dest="tcp_hosts", default=[],
action="append", type=str, dest="tcp_hosts",
metavar="HOST",
help="""
Generic TCP SSL proxy mode for all hosts that match the pattern.
@ -448,7 +448,7 @@ def proxy_options(parser):
)
group.add_argument(
"-p", "--port",
action="store", type=int, dest="port", default=options.LISTEN_PORT,
action="store", type=int, dest="port",
help="Proxy service port."
)
group.add_argument(
@ -467,7 +467,7 @@ def proxy_options(parser):
parser.add_argument(
"--upstream-auth",
action="store", dest="upstream_auth", default=None,
action="store", dest="upstream_auth",
type=str,
help="""
Add HTTP Basic authentcation to upstream proxy and reverse proxy
@ -491,7 +491,7 @@ def proxy_options(parser):
)
group.add_argument(
"--upstream-bind-address",
action="store", type=str, dest="upstream_bind_address", default='',
action="store", type=str, dest="upstream_bind_address",
help="Address to bind upstream requests to (defaults to none)"
)
@ -502,7 +502,6 @@ def proxy_ssl_options(parser):
group.add_argument(
"--cert",
dest='certs',
default=[],
type=str,
metavar="SPEC",
action="append",
@ -514,56 +513,55 @@ def proxy_ssl_options(parser):
'as the first entry. Can be passed multiple times.')
group.add_argument(
"--ciphers-client", action="store",
type=str, dest="ciphers_client", default=options.DEFAULT_CLIENT_CIPHERS,
help="Set supported ciphers for client connections. (OpenSSL Syntax)"
)
group.add_argument(
"--ciphers-server", action="store",
type=str, dest="ciphers_server", default=None,
type=str, dest="ciphers_server",
help="Set supported ciphers for server connections. (OpenSSL Syntax)"
)
group.add_argument(
"--client-certs", action="store",
type=str, dest="clientcerts", default=None,
type=str, dest="clientcerts",
help="Client certificate file or directory."
)
group.add_argument(
"--no-upstream-cert", default=False,
"--no-upstream-cert",
action="store_true", dest="no_upstream_cert",
help="Don't connect to upstream server to look up certificate details."
)
group.add_argument(
"--add-upstream-certs-to-client-chain", default=False,
"--add-upstream-certs-to-client-chain",
action="store_true", dest="add_upstream_certs_to_client_chain",
help="Add all certificates of the upstream server to the certificate chain "
"that will be served to the proxy client, as extras."
)
group.add_argument(
"--insecure", default=False,
"--insecure",
action="store_true", dest="ssl_insecure",
help="Do not verify upstream server SSL/TLS certificates."
)
group.add_argument(
"--upstream-trusted-cadir", default=None, action="store",
"--upstream-trusted-cadir", action="store",
dest="ssl_verify_upstream_trusted_cadir",
help="Path to a directory of trusted CA certificates for upstream "
"server verification prepared using the c_rehash tool."
)
group.add_argument(
"--upstream-trusted-ca", default=None, action="store",
"--upstream-trusted-ca", action="store",
dest="ssl_verify_upstream_trusted_ca",
help="Path to a PEM formatted trusted CA certificate."
)
group.add_argument(
"--ssl-version-client", dest="ssl_version_client",
default="secure", action="store",
action="store",
choices=tcp.sslversion_choices.keys(),
help="Set supported SSL/TLS versions for client connections. "
"SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+."
)
group.add_argument(
"--ssl-version-server", dest="ssl_version_server",
default="secure", action="store",
action="store",
choices=tcp.sslversion_choices.keys(),
help="Set supported SSL/TLS versions for server connections. "
"SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+."
@ -574,12 +572,12 @@ def onboarding_app(parser):
group = parser.add_argument_group("Onboarding App")
group.add_argument(
"--noapp",
action="store_false", dest="app", default=True,
action="store_false", dest="app",
help="Disable the mitmproxy onboarding app."
)
group.add_argument(
"--app-host",
action="store", dest="app_host", default=options.APP_HOST, metavar="host",
action="store", dest="app_host",
help="""
Domain to serve the onboarding app from. For transparent mode, use
an IP when a DNS entry for the app domain is not present. Default:
@ -590,7 +588,6 @@ def onboarding_app(parser):
"--app-port",
action="store",
dest="app_port",
default=options.APP_PORT,
type=int,
metavar="80",
help="Port to serve the onboarding app from."
@ -601,7 +598,7 @@ def client_replay(parser):
group = parser.add_argument_group("Client Replay")
group.add_argument(
"-c", "--client-replay",
action="append", dest="client_replay", default=[], metavar="PATH",
action="append", dest="client_replay", metavar="PATH",
help="Replay client requests from a saved file."
)
@ -610,23 +607,23 @@ def server_replay(parser):
group = parser.add_argument_group("Server Replay")
group.add_argument(
"-S", "--server-replay",
action="append", dest="server_replay", default=[], metavar="PATH",
action="append", dest="server_replay", metavar="PATH",
help="Replay server responses from a saved file."
)
group.add_argument(
"-k", "--replay-kill-extra",
action="store_true", dest="replay_kill_extra", default=False,
action="store_true", dest="replay_kill_extra",
help="Kill extra requests during replay."
)
group.add_argument(
"--server-replay-use-header",
action="append", dest="server_replay_use_headers", type=str, default=[],
action="append", dest="server_replay_use_headers", type=str,
help="Request headers to be considered during replay. "
"Can be passed multiple times."
)
group.add_argument(
"--norefresh",
action="store_true", dest="norefresh", default=False,
action="store_true", dest="norefresh",
help="""
Disable response refresh, which updates times in cookies and headers
for replayed responses.
@ -634,21 +631,21 @@ def server_replay(parser):
)
group.add_argument(
"--no-pop",
action="store_true", dest="server_replay_nopop", default=False,
action="store_true", dest="server_replay_nopop",
help="Disable response pop from response flow. "
"This makes it possible to replay same response multiple times."
)
payload = group.add_mutually_exclusive_group()
payload.add_argument(
"--replay-ignore-content",
action="store_true", dest="server_replay_ignore_content", default=False,
action="store_true", dest="server_replay_ignore_content",
help="""
Ignore request's content while searching for a saved flow to replay
"""
)
payload.add_argument(
"--replay-ignore-payload-param",
action="append", dest="server_replay_ignore_payload_params", type=str, default=[],
action="append", dest="server_replay_ignore_payload_params", type=str,
help="""
Request's payload parameters (application/x-www-form-urlencoded or multipart/form-data) to
be ignored while searching for a saved flow to replay.
@ -658,7 +655,7 @@ def server_replay(parser):
group.add_argument(
"--replay-ignore-param",
action="append", dest="server_replay_ignore_params", type=str, default=[],
action="append", dest="server_replay_ignore_params", type=str,
help="""
Request's parameters to be ignored while searching for a saved flow
to replay. Can be passed multiple times.
@ -668,7 +665,6 @@ def server_replay(parser):
"--replay-ignore-host",
action="store_true",
dest="server_replay_ignore_host",
default=False,
help="Ignore request's destination host while searching for a saved flow to replay")
@ -683,13 +679,13 @@ def replacements(parser):
)
group.add_argument(
"--replace",
action="append", type=str, dest="replace", default=[],
action="append", type=str, dest="replace",
metavar="PATTERN",
help="Replacement pattern."
)
group.add_argument(
"--replace-from-file",
action="append", type=str, dest="replace_file", default=[],
action="append", type=str, dest="replace_file",
metavar="PATH",
help="""
Replacement pattern, where the replacement clause is a path to a
@ -709,7 +705,7 @@ def set_headers(parser):
)
group.add_argument(
"--setheader",
action="append", type=str, dest="setheader", default=[],
action="append", type=str, dest="setheader",
metavar="PATTERN",
help="Header set pattern."
)
@ -747,6 +743,15 @@ def proxy_authentication(parser):
def common_options(parser):
parser.add_argument(
"--conf",
type=str, dest="conf", default=CONFIG_PATH,
metavar="PATH",
help="""
Configuration file
"""
)
basic_options(parser)
proxy_modes(parser)
proxy_options(parser)
@ -764,26 +769,17 @@ def mitmproxy():
# platforms.
from .console import palettes
parser = configargparse.ArgumentParser(
usage="%(prog)s [options]",
args_for_setting_config_path=["--conf"],
default_config_files=[
os.path.join(options.CA_DIR, "common.conf"),
os.path.join(options.CA_DIR, "mitmproxy.conf")
],
add_config_file_help=True,
add_env_var_help=True
)
parser = argparse.ArgumentParser(usage="%(prog)s [options]")
common_options(parser)
parser.add_argument(
"--palette", type=str, default=palettes.DEFAULT,
"--palette", type=str,
action="store", dest="palette",
choices=sorted(palettes.palettes.keys()),
help="Select color palette: " + ", ".join(palettes.palettes.keys())
)
parser.add_argument(
"--palette-transparent",
action="store_true", dest="palette_transparent", default=False,
action="store_true", dest="palette_transparent",
help="Set transparent background for palette."
)
parser.add_argument(
@ -798,7 +794,7 @@ def mitmproxy():
)
parser.add_argument(
"--order",
type=str, dest="order", default=None,
type=str, dest="order",
choices=[o[1] for o in view.orders],
help="Flow sort order."
)
@ -813,33 +809,24 @@ def mitmproxy():
)
group.add_argument(
"-i", "--intercept", action="store",
type=str, dest="intercept", default=None,
type=str, dest="intercept",
help="Intercept filter expression."
)
group.add_argument(
"-f", "--filter", action="store",
type=str, dest="filter", default=None,
type=str, dest="filter",
help="Filter view expression."
)
return parser
def mitmdump():
parser = configargparse.ArgumentParser(
usage="%(prog)s [options] [filter]",
args_for_setting_config_path=["--conf"],
default_config_files=[
os.path.join(options.CA_DIR, "common.conf"),
os.path.join(options.CA_DIR, "mitmdump.conf")
],
add_config_file_help=True,
add_env_var_help=True
)
parser = argparse.ArgumentParser(usage="%(prog)s [options] [filter]")
common_options(parser)
parser.add_argument(
"--keepserving",
action="store_true", dest="keepserving", default=False,
action="store_true", dest="keepserving",
help="""
Continue serving after client playback or file read. We exit by
default.
@ -847,7 +834,7 @@ def mitmdump():
)
parser.add_argument(
"-d", "--detail",
action="count", dest="flow_detail", default=1,
action="count", dest="flow_detail",
help="Increase flow detail display level. Can be passed multiple times."
)
parser.add_argument(
@ -862,16 +849,7 @@ def mitmdump():
def mitmweb():
parser = configargparse.ArgumentParser(
usage="%(prog)s [options]",
args_for_setting_config_path=["--conf"],
default_config_files=[
os.path.join(options.CA_DIR, "common.conf"),
os.path.join(options.CA_DIR, "mitmweb.conf")
],
add_config_file_help=True,
add_env_var_help=True
)
parser = argparse.ArgumentParser(usage="%(prog)s [options]")
group = parser.add_argument_group("Mitmweb")
group.add_argument(
@ -881,13 +859,13 @@ def mitmweb():
)
group.add_argument(
"--wport",
action="store", type=int, dest="wport", default=8081,
action="store", type=int, dest="wport",
metavar="PORT",
help="Mitmweb port."
)
group.add_argument(
"--wiface",
action="store", dest="wiface", default="127.0.0.1",
action="store", dest="wiface",
metavar="IFACE",
help="Mitmweb interface."
)
@ -904,7 +882,7 @@ def mitmweb():
)
group.add_argument(
"-i", "--intercept", action="store",
type=str, dest="intercept", default=None,
type=str, dest="intercept",
help="Intercept filter expression."
)
return parser

66
mitmproxy/tools/main.py
View File

@ -17,6 +17,14 @@ from mitmproxy.utils import version_check # noqa
from mitmproxy.utils import debug # noqa
def notnone(d):
ret = {}
for k, v in d.items():
if v is not None:
ret[k] = v
return ret
def assert_utf8_env():
spec = ""
for i in ["LANG", "LC_CTYPE", "LC_ALL"]:
@ -63,17 +71,21 @@ def mitmproxy(args=None): # pragma: no cover
args = parser.parse_args(args)
try:
console_options = console.master.Options(
**cmdline.get_common_options(args)
console_options = console.master.Options()
console_options.load_paths(args.conf)
console_options.update(**notnone(cmdline.get_common_options(args)))
console_options.update(
**notnone(dict(
palette = args.palette,
palette_transparent = args.palette_transparent,
eventlog = args.eventlog,
focus_follow = args.focus_follow,
intercept = args.intercept,
filter = args.filter,
no_mouse = args.no_mouse,
order = args.order,
))
)
console_options.palette = args.palette
console_options.palette_transparent = args.palette_transparent
console_options.eventlog = args.eventlog
console_options.focus_follow = args.focus_follow
console_options.intercept = args.intercept
console_options.filter = args.filter
console_options.no_mouse = args.no_mouse
console_options.order = args.order
server = process_options(parser, console_options, args)
m = console.master.ConsoleMaster(console_options, server)
@ -98,10 +110,17 @@ def mitmdump(args=None): # pragma: no cover
master = None
try:
dump_options = dump.Options(**cmdline.get_common_options(args))
dump_options.flow_detail = args.flow_detail
dump_options.keepserving = args.keepserving
dump_options.filtstr = " ".join(args.filter) if args.filter else None
dump_options = dump.Options()
dump_options.load_paths(args.conf)
dump_options.update(**notnone(cmdline.get_common_options(args)))
dump_options.update(
**notnone(dict(
flow_detail = args.flow_detail,
keepserving = args.keepserving,
filtstr = " ".join(args.filter) if args.filter else None,
))
)
server = process_options(parser, dump_options, args)
master = dump.DumpMaster(dump_options, server)
@ -130,13 +149,18 @@ def mitmweb(args=None): # pragma: no cover
args = parser.parse_args(args)
try:
web_options = web.master.Options(**cmdline.get_common_options(args))
web_options.intercept = args.intercept
web_options.open_browser = args.open_browser
web_options.wdebug = args.wdebug
web_options.wiface = args.wiface
web_options.wport = args.wport
web_options = web.master.Options()
web_options.load_paths(args.conf)
web_options.update(**notnone(cmdline.get_common_options(args)))
web_options.update(
**notnone(dict(
intercept = args.intercept,
open_browser = args.open_browser,
wdebug = args.wdebug,
wiface = args.wiface,
wport = args.wport,
))
)
server = process_options(parser, web_options, args)
m = web.master.WebMaster(web_options, server)
except exceptions.OptionsError as e:

1
setup.py
View File

@ -62,7 +62,6 @@ setup(
"blinker>=1.4, <1.5",
"click>=6.2, <7.0",
"certifi>=2015.11.20.1", # no semver here - this should always be on the last release!
"configargparse>=0.10, <0.12",
"construct>=2.8, <2.9",
"cryptography>=1.3, <1.7",
"cssutils>=1.0.1, <1.1",

3
test/mitmproxy/test_proxy.py
View File

@ -5,6 +5,7 @@ import argparse
from OpenSSL import SSL
from mitmproxy.tools import cmdline
from mitmproxy.tools import main
from mitmproxy import options
from mitmproxy.proxy import ProxyConfig
from mitmproxy import connections
@ -76,7 +77,7 @@ class TestProcessProxyOptions:
cmdline.common_options(parser)
args = parser.parse_args(args=args)
opts = cmdline.get_common_options(args)
pconf = config.ProxyConfig(options.Options(**opts))
pconf = config.ProxyConfig(options.Options(**main.notnone(opts)))
return parser, pconf
def assert_err(self, err, *args):