docs: add auto-generated options reference

.travis.yml

@@ -42,15 +42,15 @@ matrix:
           yarn: true
           directories:
               - web/node_modules
-    - language: python
-      env:
-        - NAME=docs
+    - python: 3.6
+      env: NAME=docs TOXENV=docs
       install:
         - wget https://github.com/gohugoio/hugo/releases/download/v0.37/hugo_0.37_Linux-64bit.deb
         - sudo dpkg -i hugo*.deb
-        - pip install awscli
+        - pip install tox virtualenv setuptools
+        - pyenv global system 3.6
       script:
-        - cd docs && ./ci
+        - tox
       after_success:
         - echo done
 

docs/.gitignore

@@ -1,3 +1,4 @@
+generated/
 src/public
 node_modules
 public

docs/build-current

@@ -1,5 +1,12 @@
-#!/bin/sh
+#!/usr/bin/env bash
+
 set -e
 
+for script in scripts/* ; do
+  echo "Generating output for $script ..."
+  output="${script##*/}"
+  "$script" > "src/generated/${output%.*}.html"
+done
+
 cd src
 hugo

docs/scripts/filters.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 from mitmproxy import flowfilter
 
@@ -6,4 +6,4 @@ from mitmproxy import flowfilter
 print("<table class=\"table filtertable\"><tbody>")
 for i in flowfilter.help:
     print("<tr><th>%s</th><td>%s</td></tr>" % i)
-print("</tbody></table>")
+print("</tbody></table>")

docs/scripts/options.py

@@ -1,4 +1,5 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
+
 from mitmproxy import options, optmanager
 from mitmproxy.tools import dump, console, web
 
@@ -8,7 +9,45 @@ masters = {
     "mitmweb": web.master.WebMaster
 }
 
-for name, master in masters.items():
+unified_options = {}
+
+for tool_name, master in masters.items():
     opts = options.Options()
     inst = master(opts)
-    print(optmanager.dump_dicts(opts))
+    for key, option in optmanager.dump_dicts(opts).items():
+        if key in unified_options:
+            unified_options[key]['tools'].append(tool_name)
+        else:
+            unified_options[key] = option
+            unified_options[key]['tools'] = [tool_name]
+
+print("""
+      <table class=\"table optiontable\">
+        <thead>
+          <tr>
+            <th>Name</th>
+            <th>Type</th>
+            <th>Description</th>
+        </tr>
+        </thead>
+        <tbody>
+      """.strip())
+for key, option in sorted(unified_options.items(), key=lambda t: t[0]):
+    print("""
+          <tr>
+          <th>{}<br/>{}</th>
+          <td>{}</td>
+          <td>{}<br/>
+            Default: {}
+            {}
+          </td>
+          </tr>
+          """.strip().format(
+              key,
+              ' '.join(["<span class='badge'>{}</span>".format(t) for t in option['tools']]),
+              option['type'],
+              option['help'],
+              option['default'],
+              "<br/>Choices: {}".format(', '.join(option['choices'])) if option['choices'] else "",
+    ))
+print("</tbody></table>")

docs/src/content/concepts-filters.md

@@ -10,37 +10,7 @@ menu:
 Many commands in the mitmproxy tool make use of filter expressions. Filter
 expressions consist of the following operators:
 
-
-<table class="table filtertable"><tbody>
-<tr><th>~a</th><td>Match asset in response: CSS, Javascript, Flash, images.</td></tr>
-<tr><th>~b regex</th><td>Body</td></tr>
-<tr><th>~bq regex</th><td>Request body</td></tr>
-<tr><th>~bs regex</th><td>Response body</td></tr>
-<tr><th>~c int</th><td>HTTP response code</td></tr>
-<tr><th>~d regex</th><td>Domain</td></tr>
-<tr><th>~dst regex</th><td>Match destination address</td></tr>
-<tr><th>~e</th><td>Match error</td></tr>
-<tr><th>~h regex</th><td>Header</td></tr>
-<tr><th>~hq regex</th><td>Request header</td></tr>
-<tr><th>~hs regex</th><td>Response header</td></tr>
-<tr><th>~http</th><td>Match HTTP flows</td></tr>
-<tr><th>~m regex</th><td>Method</td></tr>
-<tr><th>~marked</th><td>Match marked flows</td></tr>
-<tr><th>~q</th><td>Match request with no response</td></tr>
-<tr><th>~s</th><td>Match response</td></tr>
-<tr><th>~src regex</th><td>Match source address</td></tr>
-<tr><th>~t regex</th><td>Content-type header</td></tr>
-<tr><th>~tcp</th><td>Match TCP flows</td></tr>
-<tr><th>~tq regex</th><td>Request Content-Type header</td></tr>
-<tr><th>~ts regex</th><td>Response Content-Type header</td></tr>
-<tr><th>~u regex</th><td>URL</td></tr>
-<tr><th>~websocket</th><td>Match WebSocket flows</td></tr>
-<tr><th>!</th><td>unary not</td></tr>
-<tr><th>&</th><td>and</td></tr>
-<tr><th>|</th><td>or</td></tr>
-<tr><th>(...)</th><td>grouping</td></tr>
-</tbody></table>
-
+{{< readfile file="/generated/filters.html" >}}
 
 - Regexes are Python-style
 - Regexes can be specified as quoted strings

docs/src/content/concepts-options.md

@@ -34,33 +34,14 @@ interactively have immediate effect in the running instance, and can be made
 persistent by saving the settings out to a YAML configuration file (please see
 the specific tool's interactive help for details on how to do this).
 
-For all tools, options can be set directly by name using the `--set` command-line
-option. Please see the command-line help (`--help`) for usage.
+For all tools, options can be set directly by name using the `--set`
+command-line option. Please see the command-line help (`--help`) for usage.
 
 
-## Example
-
-Here is an excerpt showing the first few lines of the complete `--options`
-output:
-
-{{< highlight yaml  >}}
-
-# Add all certificates of the upstream server to the certificate chain
-# that will be served to the proxy client, as extras. Type bool.
-add_upstream_certs_to_client_chain: false
-
-# Allow remote clients to connect to proxy. If set to false, client will
-# not be able to connect to proxy unless it is on the same network or
-# the proxyauth option is set Type bool.
-allow_remote: false
-
-# Strip out request headers that might cause the server to return
-# 304-not-modified. Type bool.
-anticache: false
-
-# Try to convince servers to send us un-compressed data. Type bool.
-anticomp: false
-
-{{< /highlight >}}
+## Available Options
 
+This list might not reflect what is actually available in your current mitmproxy
+environment. For an up-to-date list please use the `--options` flag for each of
+the mitmproxy tools.
 
+{{< readfile file="/generated/options.html" >}}

docs/src/layouts/shortcodes/readfile.html

@@ -0,0 +1,6 @@
+{{$file := .Get "file"}}
+{{- if eq (.Get "markdown") "true" -}}
+{{- $file  | readFile | markdownify -}}
+{{- else -}}
+{{ $file  | readFile | safeHTML }}
+{{- end -}}

docs/src/themes/mitmproxydocs/static/css/style.css

@@ -250,6 +250,21 @@
 .chroma .gu {
   color: #75715e; }
 
+.badge {
+  color: #fff;
+  background-color: #6c757d;
+  display: inline-block;
+  padding: .25em .4em;
+  font-size: 75%;
+  font-weight: 1;
+  line-height: 1;
+  text-align: center;
+  white-space: nowrap;
+  vertical-align: baseline;
+  border-radius: 0.25rem; }
+  .badge:empty {
+    display: none; }
+
 @keyframes spinAround {
   from {
     transform: rotate(0deg); }

docs/style/badge.scss

@@ -0,0 +1,18 @@
+.badge {
+  color: #fff;
+  background-color: #6c757d;
+  display: inline-block;
+  padding: .25em .4em;
+  font-size: 75%;
+  font-weight: 1;
+  line-height: 1;
+  text-align: center;
+  white-space: nowrap;
+  vertical-align: baseline;
+  border-radius: 0.25rem;
+
+  // Empty badges collapse automatically
+  &:empty {
+    display: none;
+  }
+}

docs/style/style.scss

@@ -1,4 +1,5 @@
 @import "./syntax";
+@import "./badge";
 
 $primary: #C93312;
 $warning-invert: #FFFFFF;

tox.ini

@@ -41,3 +41,12 @@ deps =
 commands =
   mitmdump --version
   python ./release/ci.py {posargs}
+
+[testenv:docs]
+passenv = TRAVIS_* AWS_* APPVEYOR_* RTOOL_KEY WHEEL
+deps =
+  -rrequirements.txt
+  awscli
+changedir = docs
+commands =
+  ./ci