Merge pull request #8003 from projectdiscovery/feat/har-input

add har input mode

README.md

@@ -150,7 +150,7 @@ TARGET:
    -mup, -max-urls-per-pattern int   scan at most this many urls sharing one structural shape, e.g. /user/1 and /user/2 (0 to scan all)
 
 TARGET-FORMAT:
-   -im, -input-mode string        mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list")
+   -im, -input-mode string        mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list")
    -ro, -required-only            use only required fields in input format when generating requests
    -sfv, -skip-format-validation  skip format validation (like missing vars) when parsing input file
    -vtt, -vars-text-templating    enable text templating for vars in input file (only for yaml input mode)

README_CN.md

@@ -148,7 +148,7 @@ TARGET:
    -iv, -ip-version string[]     IP version to scan of hostname (4,6) - (default 4)
 
 TARGET-FORMAT:
-   -im, -input-mode string        mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list")
+   -im, -input-mode string        mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list")
    -ro, -required-only            use only required fields in input format when generating requests
    -sfv, -skip-format-validation  skip format validation (like missing vars) when parsing input file
    -vtt, -vars-text-templating    enable text templating for vars in input file (only for yaml input mode)

README_ES.md

@@ -148,7 +148,7 @@ TARGET:
    -iv, -ip-version string[]     IP version to scan of hostname (4,6) - (default 4)
 
 TARGET-FORMAT:
-   -im, -input-mode string        mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list")
+   -im, -input-mode string        mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list")
    -ro, -required-only            use only required fields in input format when generating requests
    -sfv, -skip-format-validation  skip format validation (like missing vars) when parsing input file
    -vtt, -vars-text-templating    enable text templating for vars in input file (only for yaml input mode)

README_FA.md

@@ -165,7 +165,7 @@ TARGET:
    -mup, -max-urls-per-pattern int   scan at most this many urls sharing one structural shape, e.g. /user/1 and /user/2 (0 to scan all)
 
 TARGET-FORMAT:
-   -im, -input-mode string        mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list")
+   -im, -input-mode string        mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list")
    -ro, -required-only            use only required fields in input format when generating requests
    -sfv, -skip-format-validation  skip format validation (like missing vars) when parsing input file
    -vtt, -vars-text-templating    enable text templating for vars in input file (only for yaml input mode)

README_ID.md

@@ -148,7 +148,7 @@ TARGET:
    -iv, -ip-version string[]     IP version to scan of hostname (4,6) - (default 4)
 
 TARGET-FORMAT:
-   -im, -input-mode string        mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list")
+   -im, -input-mode string        mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list")
    -ro, -required-only            use only required fields in input format when generating requests
    -sfv, -skip-format-validation  skip format validation (like missing vars) when parsing input file
    -vtt, -vars-text-templating    enable text templating for vars in input file (only for yaml input mode)

README_JP.md

@@ -148,7 +148,7 @@ TARGET:
    -iv, -ip-version string[]     IP version to scan of hostname (4,6) - (default 4)
 
 TARGET-FORMAT:
-   -im, -input-mode string        mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list")
+   -im, -input-mode string        mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list")
    -ro, -required-only            use only required fields in input format when generating requests
    -sfv, -skip-format-validation  skip format validation (like missing vars) when parsing input file
    -vtt, -vars-text-templating    enable text templating for vars in input file (only for yaml input mode)

README_KR.md

@@ -148,7 +148,7 @@ TARGET:
    -iv, -ip-version string[]     IP version to scan of hostname (4,6) - (default 4)
 
 TARGET-FORMAT:
-   -im, -input-mode string        mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list")
+   -im, -input-mode string        mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list")
    -ro, -required-only            use only required fields in input format when generating requests
    -sfv, -skip-format-validation  skip format validation (like missing vars) when parsing input file
    -vtt, -vars-text-templating    enable text templating for vars in input file (only for yaml input mode)

README_PT-BR.md

@@ -148,7 +148,7 @@ TARGET:
    -iv, -ip-version string[]     IP version to scan of hostname (4,6) - (default 4)
 
 TARGET-FORMAT:
-   -im, -input-mode string        mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list")
+   -im, -input-mode string        mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list")
    -ro, -required-only            use only required fields in input format when generating requests
    -sfv, -skip-format-validation  skip format validation (like missing vars) when parsing input file
    -vtt, -vars-text-templating    enable text templating for vars in input file (only for yaml input mode)

README_TR.md

@@ -148,7 +148,7 @@ TARGET:
    -iv, -ip-version string[]     IP version to scan of hostname (4,6) - (default 4)
 
 TARGET-FORMAT:
-   -im, -input-mode string        mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list")
+   -im, -input-mode string        mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list")
    -ro, -required-only            use only required fields in input format when generating requests
    -sfv, -skip-format-validation  skip format validation (like missing vars) when parsing input file
    -vtt, -vars-text-templating    enable text templating for vars in input file (only for yaml input mode)

pkg/input/formats/README.md

@@ -5,6 +5,7 @@ Formats implements support for passing a number of request source as input provi
 Currently the following formats are implemented - 
 
 - Burp Suite XML Request/Response file
+- HAR (HTTP Archive) file exported by browsers and proxies
 - Proxify JSONL output file
 - OpenAPI Specification file
 - Postman Collection file

pkg/input/formats/har/har.go

@@ -0,0 +1,137 @@
+package har
+
+import (
+	"encoding/base64"
+	"io"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/projectdiscovery/gologger"
+	"github.com/projectdiscovery/nuclei/v3/pkg/input/formats"
+	"github.com/projectdiscovery/nuclei/v3/pkg/input/types"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/json"
+)
+
+// HARFormat is an HTTP Archive (HAR 1.2) parser, the format browsers and
+// proxies export captured traffic in.
+type HARFormat struct {
+	opts formats.InputFormatOptions
+}
+
+// New creates a new HAR parser
+func New() *HARFormat {
+	return &HARFormat{}
+}
+
+var _ formats.Format = &HARFormat{}
+
+type archive struct {
+	Log struct {
+		Entries []entry `json:"entries"`
+	} `json:"log"`
+}
+
+type entry struct {
+	Request struct {
+		Method   string   `json:"method"`
+		URL      string   `json:"url"`
+		Headers  []header `json:"headers"`
+		PostData *struct {
+			Text string `json:"text"`
+		} `json:"postData"`
+	} `json:"request"`
+	Response struct {
+		Status     int      `json:"status"`
+		StatusText string   `json:"statusText"`
+		Headers    []header `json:"headers"`
+		Content    struct {
+			Text     string `json:"text"`
+			Encoding string `json:"encoding"`
+		} `json:"content"`
+	} `json:"response"`
+}
+
+type header struct {
+	Name  string `json:"name"`
+	Value string `json:"value"`
+}
+
+// Name returns the name of the format
+func (h *HARFormat) Name() string {
+	return "har"
+}
+
+func (h *HARFormat) SetOptions(options formats.InputFormatOptions) {
+	h.opts = options
+}
+
+// Parse parses the input and calls the provided callback
+// function for each RawRequest it discovers.
+func (h *HARFormat) Parse(input io.Reader, resultsCb formats.ParseReqRespCallback, filePath string) error {
+	var har archive
+	if err := json.NewDecoder(input).Decode(&har); err != nil {
+		return errors.Wrap(err, "could not decode har file")
+	}
+
+	for _, e := range har.Log.Entries {
+		// Browsers also log data:, blob: and extension URLs, which have no
+		// host to send a request to.
+		if !strings.HasPrefix(e.Request.URL, "http://") && !strings.HasPrefix(e.Request.URL, "https://") {
+			continue
+		}
+		rawRequest, err := types.ParseRawRequestWithURL(rawRequestOf(e), e.Request.URL)
+		if err != nil {
+			gologger.Warning().Msgf("har: Could not parse request %s: %s\n", e.Request.URL, err)
+			continue
+		}
+		// Status 0 is a request that was blocked or never answered.
+		if e.Response.Status != 0 {
+			rawRequest.Response = &types.HttpResponse{Raw: rawResponseOf(e)}
+		}
+		resultsCb(rawRequest)
+	}
+	return nil
+}
+
+// rawRequestOf rebuilds the request as HTTP/1.1 with an absolute target, which
+// carries the authority that HTTP/2 captures only keep in pseudo-headers.
+func rawRequestOf(e entry) string {
+	var b strings.Builder
+	b.WriteString(e.Request.Method + " " + e.Request.URL + " HTTP/1.1\r\n")
+	writeHeaders(&b, e.Request.Headers)
+	if e.Request.PostData != nil {
+		b.WriteString(e.Request.PostData.Text)
+	}
+	return b.String()
+}
+
+func rawResponseOf(e entry) string {
+	statusText := e.Response.StatusText
+	if statusText == "" {
+		statusText = http.StatusText(e.Response.Status)
+	}
+	var b strings.Builder
+	b.WriteString("HTTP/1.1 " + strconv.Itoa(e.Response.Status) + " " + statusText + "\r\n")
+	writeHeaders(&b, e.Response.Headers)
+	body := e.Response.Content.Text
+	if e.Response.Content.Encoding == "base64" {
+		if decoded, err := base64.StdEncoding.DecodeString(body); err == nil {
+			body = string(decoded)
+		}
+	}
+	b.WriteString(body)
+	return b.String()
+}
+
+func writeHeaders(b *strings.Builder, headers []header) {
+	for _, h := range headers {
+		// HTTP/2 pseudo-headers (:authority, :path, ...) are not headers.
+		if strings.HasPrefix(h.Name, ":") {
+			continue
+		}
+		b.WriteString(h.Name + ": " + h.Value + "\r\n")
+	}
+	b.WriteString("\r\n")
+}

pkg/input/formats/har/har_test.go

@@ -0,0 +1,71 @@
+package har
+
+import (
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/input/types"
+	"github.com/stretchr/testify/require"
+)
+
+const chromeHAR = "../testdata/chrome.har"
+
+func parse(t *testing.T, input string) []*types.RequestResponse {
+	t.Helper()
+	file, err := os.Open(input)
+	require.NoError(t, err)
+	defer func() { _ = file.Close() }()
+
+	var got []*types.RequestResponse
+	require.NoError(t, New().Parse(file, func(rr *types.RequestResponse) bool {
+		got = append(got, rr)
+		return false
+	}, input))
+	return got
+}
+
+func TestHARParse(t *testing.T) {
+	got := parse(t, chromeHAR)
+
+	urls := make([]string, 0, len(got))
+	for _, rr := range got {
+		urls = append(urls, rr.URL.String())
+	}
+	require.Equal(t, []string{
+		"https://ginandjuice.shop/catalog?category=Juice",
+		"https://ginandjuice.shop/login",
+		"https://ginandjuice.shop/blocked.js",
+	}, urls, "non-http entries such as data: URLs are skipped")
+}
+
+func TestHARParseHTTP2Request(t *testing.T) {
+	rr := parse(t, chromeHAR)[0]
+
+	require.Equal(t, "GET", rr.Request.Method)
+	cookie, _ := rr.Request.Headers.Get("cookie")
+	require.Equal(t, "session=abc123", cookie)
+	for _, name := range rr.Request.Headers.GetKeys() {
+		require.False(t, strings.HasPrefix(name, ":"), "pseudo-header %s is not a real header", name)
+	}
+
+	req, err := rr.BuildRequest()
+	require.NoError(t, err)
+	require.Equal(t, "ginandjuice.shop", req.Host)
+}
+
+func TestHARParseBodyAndResponse(t *testing.T) {
+	got := parse(t, chromeHAR)
+
+	login := got[1]
+	require.Equal(t, "POST", login.Request.Method)
+	require.Equal(t, "username=carlos&password=hunter2", login.Request.Body)
+	require.NotNil(t, login.Response)
+	require.Contains(t, login.Response.Raw, "302 Found")
+	require.Contains(t, login.Response.Raw, "Location: /my-account")
+
+	require.NotNil(t, got[0].Response)
+	require.Contains(t, got[0].Response.Raw, "<h1>Juice</h1>", "base64 content is decoded")
+
+	require.Nil(t, got[2].Response, "a request that never got an answer has no response")
+}

pkg/input/formats/testdata/chrome.har

@@ -0,0 +1,71 @@
+{
+  "log": {
+    "version": "1.2",
+    "creator": { "name": "WebInspector", "version": "537.36" },
+    "entries": [
+      {
+        "request": {
+          "method": "GET",
+          "url": "https://ginandjuice.shop/catalog?category=Juice",
+          "httpVersion": "http/2.0",
+          "headers": [
+            { "name": ":authority", "value": "ginandjuice.shop" },
+            { "name": ":method", "value": "GET" },
+            { "name": ":path", "value": "/catalog?category=Juice" },
+            { "name": ":scheme", "value": "https" },
+            { "name": "accept", "value": "text/html" },
+            { "name": "cookie", "value": "session=abc123" }
+          ],
+          "queryString": [{ "name": "category", "value": "Juice" }],
+          "cookies": [],
+          "headersSize": -1,
+          "bodySize": 0
+        },
+        "response": {
+          "status": 200,
+          "statusText": "",
+          "httpVersion": "http/2.0",
+          "headers": [{ "name": "content-type", "value": "text/html; charset=utf-8" }],
+          "content": { "size": 19, "mimeType": "text/html", "text": "PGgxPkp1aWNlPC9oMT4=", "encoding": "base64" }
+        }
+      },
+      {
+        "request": {
+          "method": "POST",
+          "url": "https://ginandjuice.shop/login",
+          "httpVersion": "HTTP/1.1",
+          "headers": [
+            { "name": "Host", "value": "ginandjuice.shop" },
+            { "name": "Content-Type", "value": "application/x-www-form-urlencoded" }
+          ],
+          "postData": { "mimeType": "application/x-www-form-urlencoded", "text": "username=carlos&password=hunter2" }
+        },
+        "response": {
+          "status": 302,
+          "statusText": "Found",
+          "httpVersion": "HTTP/1.1",
+          "headers": [{ "name": "Location", "value": "/my-account" }],
+          "content": { "size": 0, "mimeType": "" }
+        }
+      },
+      {
+        "request": {
+          "method": "GET",
+          "url": "data:image/png;base64,iVBORw0KGgo=",
+          "httpVersion": "",
+          "headers": []
+        },
+        "response": { "status": 200, "statusText": "", "httpVersion": "", "headers": [], "content": { "size": 0 } }
+      },
+      {
+        "request": {
+          "method": "GET",
+          "url": "https://ginandjuice.shop/blocked.js",
+          "httpVersion": "",
+          "headers": []
+        },
+        "response": { "status": 0, "statusText": "", "httpVersion": "", "headers": [], "content": { "size": 0 } }
+      }
+    ]
+  }
+}

pkg/input/provider/http/multiformat.go

@@ -10,6 +10,7 @@ import (
 	"github.com/projectdiscovery/gologger"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/formats"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/formats/burp"
+	"github.com/projectdiscovery/nuclei/v3/pkg/input/formats/har"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/formats/json"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/formats/openapi"
 	"github.com/projectdiscovery/nuclei/v3/pkg/input/formats/raw"
@@ -142,6 +143,7 @@ func (i *HttpInputProvider) Close() {}
 // Supported Providers
 var providersList = []formats.Format{
 	burp.New(),
+	har.New(),
 	json.New(),
 	yaml.New(),
 	openapi.New(),