From 684d55eeedff7847e4a432e7a3cddc04636703ca Mon Sep 17 00:00:00 2001 From: Dogan Can Bakir <65292895+dogancanbakir@users.noreply.github.com> Date: Mon, 2 Nov 2026 09:31:10 +0300 Subject: [PATCH] add har input mode --- README.md | 2 +- README_CN.md | 2 +- README_ES.md | 2 +- README_FA.md | 2 +- README_ID.md | 2 +- README_JP.md | 2 +- README_KR.md | 2 +- README_PT-BR.md | 2 +- README_TR.md | 2 +- pkg/input/formats/README.md | 1 + pkg/input/formats/har/har.go | 137 +++++++++++++++++++++++++ pkg/input/formats/har/har_test.go | 71 +++++++++++++ pkg/input/formats/testdata/chrome.har | 71 +++++++++++++ pkg/input/provider/http/multiformat.go | 2 + 14 files changed, 291 insertions(+), 9 deletions(-) create mode 100644 pkg/input/formats/har/har.go create mode 100644 pkg/input/formats/har/har_test.go create mode 100644 pkg/input/formats/testdata/chrome.har diff --git a/README.md b/README.md index 5c4235f10..a6980a5e4 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ TARGET: -mup, -max-urls-per-pattern int scan at most this many urls sharing one structural shape, e.g. /user/1 and /user/2 (0 to scan all) TARGET-FORMAT: - -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list") + -im, -input-mode string mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list") -ro, -required-only use only required fields in input format when generating requests -sfv, -skip-format-validation skip format validation (like missing vars) when parsing input file -vtt, -vars-text-templating enable text templating for vars in input file (only for yaml input mode) diff --git a/README_CN.md b/README_CN.md index 67db9f295..df6a0dffb 100644 --- a/README_CN.md +++ b/README_CN.md @@ -148,7 +148,7 @@ TARGET: -iv, -ip-version string[] IP version to scan of hostname (4,6) - (default 4) TARGET-FORMAT: - -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list") + -im, -input-mode string mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list") -ro, -required-only use only required fields in input format when generating requests -sfv, -skip-format-validation skip format validation (like missing vars) when parsing input file -vtt, -vars-text-templating enable text templating for vars in input file (only for yaml input mode) diff --git a/README_ES.md b/README_ES.md index d70278075..88f4567ed 100644 --- a/README_ES.md +++ b/README_ES.md @@ -148,7 +148,7 @@ TARGET: -iv, -ip-version string[] IP version to scan of hostname (4,6) - (default 4) TARGET-FORMAT: - -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list") + -im, -input-mode string mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list") -ro, -required-only use only required fields in input format when generating requests -sfv, -skip-format-validation skip format validation (like missing vars) when parsing input file -vtt, -vars-text-templating enable text templating for vars in input file (only for yaml input mode) diff --git a/README_FA.md b/README_FA.md index 5e2f41d5f..7594dbba4 100644 --- a/README_FA.md +++ b/README_FA.md @@ -165,7 +165,7 @@ TARGET: -mup, -max-urls-per-pattern int scan at most this many urls sharing one structural shape, e.g. /user/1 and /user/2 (0 to scan all) TARGET-FORMAT: - -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list") + -im, -input-mode string mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list") -ro, -required-only use only required fields in input format when generating requests -sfv, -skip-format-validation skip format validation (like missing vars) when parsing input file -vtt, -vars-text-templating enable text templating for vars in input file (only for yaml input mode) diff --git a/README_ID.md b/README_ID.md index 9e09eb00b..84d1f3fc8 100644 --- a/README_ID.md +++ b/README_ID.md @@ -148,7 +148,7 @@ TARGET: -iv, -ip-version string[] IP version to scan of hostname (4,6) - (default 4) TARGET-FORMAT: - -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list") + -im, -input-mode string mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list") -ro, -required-only use only required fields in input format when generating requests -sfv, -skip-format-validation skip format validation (like missing vars) when parsing input file -vtt, -vars-text-templating enable text templating for vars in input file (only for yaml input mode) diff --git a/README_JP.md b/README_JP.md index 5931ae6bf..a2282ab88 100644 --- a/README_JP.md +++ b/README_JP.md @@ -148,7 +148,7 @@ TARGET: -iv, -ip-version string[] IP version to scan of hostname (4,6) - (default 4) TARGET-FORMAT: - -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list") + -im, -input-mode string mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list") -ro, -required-only use only required fields in input format when generating requests -sfv, -skip-format-validation skip format validation (like missing vars) when parsing input file -vtt, -vars-text-templating enable text templating for vars in input file (only for yaml input mode) diff --git a/README_KR.md b/README_KR.md index 064eb2c92..b5ee8287d 100644 --- a/README_KR.md +++ b/README_KR.md @@ -148,7 +148,7 @@ TARGET: -iv, -ip-version string[] IP version to scan of hostname (4,6) - (default 4) TARGET-FORMAT: - -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list") + -im, -input-mode string mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list") -ro, -required-only use only required fields in input format when generating requests -sfv, -skip-format-validation skip format validation (like missing vars) when parsing input file -vtt, -vars-text-templating enable text templating for vars in input file (only for yaml input mode) diff --git a/README_PT-BR.md b/README_PT-BR.md index d9b95c892..80117f182 100644 --- a/README_PT-BR.md +++ b/README_PT-BR.md @@ -148,7 +148,7 @@ TARGET: -iv, -ip-version string[] IP version to scan of hostname (4,6) - (default 4) TARGET-FORMAT: - -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list") + -im, -input-mode string mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list") -ro, -required-only use only required fields in input format when generating requests -sfv, -skip-format-validation skip format validation (like missing vars) when parsing input file -vtt, -vars-text-templating enable text templating for vars in input file (only for yaml input mode) diff --git a/README_TR.md b/README_TR.md index c0316a274..6eb8f94e4 100644 --- a/README_TR.md +++ b/README_TR.md @@ -148,7 +148,7 @@ TARGET: -iv, -ip-version string[] IP version to scan of hostname (4,6) - (default 4) TARGET-FORMAT: - -im, -input-mode string mode of input file (list, burp, jsonl, yaml, openapi, swagger, http) (default "list") + -im, -input-mode string mode of input file (list, burp, har, jsonl, yaml, openapi, swagger, http) (default "list") -ro, -required-only use only required fields in input format when generating requests -sfv, -skip-format-validation skip format validation (like missing vars) when parsing input file -vtt, -vars-text-templating enable text templating for vars in input file (only for yaml input mode) diff --git a/pkg/input/formats/README.md b/pkg/input/formats/README.md index 9f22077e0..8303baa85 100644 --- a/pkg/input/formats/README.md +++ b/pkg/input/formats/README.md @@ -5,6 +5,7 @@ Formats implements support for passing a number of request source as input provi Currently the following formats are implemented - - Burp Suite XML Request/Response file +- HAR (HTTP Archive) file exported by browsers and proxies - Proxify JSONL output file - OpenAPI Specification file - Postman Collection file diff --git a/pkg/input/formats/har/har.go b/pkg/input/formats/har/har.go new file mode 100644 index 000000000..c26217f4b --- /dev/null +++ b/pkg/input/formats/har/har.go @@ -0,0 +1,137 @@ +package har + +import ( + "encoding/base64" + "io" + "net/http" + "strconv" + "strings" + + "github.com/pkg/errors" + "github.com/projectdiscovery/gologger" + "github.com/projectdiscovery/nuclei/v3/pkg/input/formats" + "github.com/projectdiscovery/nuclei/v3/pkg/input/types" + "github.com/projectdiscovery/nuclei/v3/pkg/utils/json" +) + +// HARFormat is an HTTP Archive (HAR 1.2) parser, the format browsers and +// proxies export captured traffic in. +type HARFormat struct { + opts formats.InputFormatOptions +} + +// New creates a new HAR parser +func New() *HARFormat { + return &HARFormat{} +} + +var _ formats.Format = &HARFormat{} + +type archive struct { + Log struct { + Entries []entry `json:"entries"` + } `json:"log"` +} + +type entry struct { + Request struct { + Method string `json:"method"` + URL string `json:"url"` + Headers []header `json:"headers"` + PostData *struct { + Text string `json:"text"` + } `json:"postData"` + } `json:"request"` + Response struct { + Status int `json:"status"` + StatusText string `json:"statusText"` + Headers []header `json:"headers"` + Content struct { + Text string `json:"text"` + Encoding string `json:"encoding"` + } `json:"content"` + } `json:"response"` +} + +type header struct { + Name string `json:"name"` + Value string `json:"value"` +} + +// Name returns the name of the format +func (h *HARFormat) Name() string { + return "har" +} + +func (h *HARFormat) SetOptions(options formats.InputFormatOptions) { + h.opts = options +} + +// Parse parses the input and calls the provided callback +// function for each RawRequest it discovers. +func (h *HARFormat) Parse(input io.Reader, resultsCb formats.ParseReqRespCallback, filePath string) error { + var har archive + if err := json.NewDecoder(input).Decode(&har); err != nil { + return errors.Wrap(err, "could not decode har file") + } + + for _, e := range har.Log.Entries { + // Browsers also log data:, blob: and extension URLs, which have no + // host to send a request to. + if !strings.HasPrefix(e.Request.URL, "http://") && !strings.HasPrefix(e.Request.URL, "https://") { + continue + } + rawRequest, err := types.ParseRawRequestWithURL(rawRequestOf(e), e.Request.URL) + if err != nil { + gologger.Warning().Msgf("har: Could not parse request %s: %s\n", e.Request.URL, err) + continue + } + // Status 0 is a request that was blocked or never answered. + if e.Response.Status != 0 { + rawRequest.Response = &types.HttpResponse{Raw: rawResponseOf(e)} + } + resultsCb(rawRequest) + } + return nil +} + +// rawRequestOf rebuilds the request as HTTP/1.1 with an absolute target, which +// carries the authority that HTTP/2 captures only keep in pseudo-headers. +func rawRequestOf(e entry) string { + var b strings.Builder + b.WriteString(e.Request.Method + " " + e.Request.URL + " HTTP/1.1\r\n") + writeHeaders(&b, e.Request.Headers) + if e.Request.PostData != nil { + b.WriteString(e.Request.PostData.Text) + } + return b.String() +} + +func rawResponseOf(e entry) string { + statusText := e.Response.StatusText + if statusText == "" { + statusText = http.StatusText(e.Response.Status) + } + var b strings.Builder + b.WriteString("HTTP/1.1 " + strconv.Itoa(e.Response.Status) + " " + statusText + "\r\n") + writeHeaders(&b, e.Response.Headers) + body := e.Response.Content.Text + if e.Response.Content.Encoding == "base64" { + if decoded, err := base64.StdEncoding.DecodeString(body); err == nil { + body = string(decoded) + } + } + b.WriteString(body) + return b.String() +} + +func writeHeaders(b *strings.Builder, headers []header) { + for _, h := range headers { + // HTTP/2 pseudo-headers (:authority, :path, ...) are not headers. + if strings.HasPrefix(h.Name, ":") { + continue + } + b.WriteString(h.Name + ": " + h.Value + "\r\n") + } + b.WriteString("\r\n") +} diff --git a/pkg/input/formats/har/har_test.go b/pkg/input/formats/har/har_test.go new file mode 100644 index 000000000..3a7a842eb --- /dev/null +++ b/pkg/input/formats/har/har_test.go @@ -0,0 +1,71 @@ +package har + +import ( + "os" + "strings" + "testing" + + "github.com/projectdiscovery/nuclei/v3/pkg/input/types" + "github.com/stretchr/testify/require" +) + +const chromeHAR = "../testdata/chrome.har" + +func parse(t *testing.T, input string) []*types.RequestResponse { + t.Helper() + file, err := os.Open(input) + require.NoError(t, err) + defer func() { _ = file.Close() }() + + var got []*types.RequestResponse + require.NoError(t, New().Parse(file, func(rr *types.RequestResponse) bool { + got = append(got, rr) + return false + }, input)) + return got +} + +func TestHARParse(t *testing.T) { + got := parse(t, chromeHAR) + + urls := make([]string, 0, len(got)) + for _, rr := range got { + urls = append(urls, rr.URL.String()) + } + require.Equal(t, []string{ + "https://ginandjuice.shop/catalog?category=Juice", + "https://ginandjuice.shop/login", + "https://ginandjuice.shop/blocked.js", + }, urls, "non-http entries such as data: URLs are skipped") +} + +func TestHARParseHTTP2Request(t *testing.T) { + rr := parse(t, chromeHAR)[0] + + require.Equal(t, "GET", rr.Request.Method) + cookie, _ := rr.Request.Headers.Get("cookie") + require.Equal(t, "session=abc123", cookie) + for _, name := range rr.Request.Headers.GetKeys() { + require.False(t, strings.HasPrefix(name, ":"), "pseudo-header %s is not a real header", name) + } + + req, err := rr.BuildRequest() + require.NoError(t, err) + require.Equal(t, "ginandjuice.shop", req.Host) +} + +func TestHARParseBodyAndResponse(t *testing.T) { + got := parse(t, chromeHAR) + + login := got[1] + require.Equal(t, "POST", login.Request.Method) + require.Equal(t, "username=carlos&password=hunter2", login.Request.Body) + require.NotNil(t, login.Response) + require.Contains(t, login.Response.Raw, "302 Found") + require.Contains(t, login.Response.Raw, "Location: /my-account") + + require.NotNil(t, got[0].Response) + require.Contains(t, got[0].Response.Raw, "

Juice

", "base64 content is decoded") + + require.Nil(t, got[2].Response, "a request that never got an answer has no response") +} diff --git a/pkg/input/formats/testdata/chrome.har b/pkg/input/formats/testdata/chrome.har new file mode 100644 index 000000000..483556f96 --- /dev/null +++ b/pkg/input/formats/testdata/chrome.har @@ -0,0 +1,71 @@ +{ + "log": { + "version": "1.2", + "creator": { "name": "WebInspector", "version": "537.36" }, + "entries": [ + { + "request": { + "method": "GET", + "url": "https://ginandjuice.shop/catalog?category=Juice", + "httpVersion": "http/2.0", + "headers": [ + { "name": ":authority", "value": "ginandjuice.shop" }, + { "name": ":method", "value": "GET" }, + { "name": ":path", "value": "/catalog?category=Juice" }, + { "name": ":scheme", "value": "https" }, + { "name": "accept", "value": "text/html" }, + { "name": "cookie", "value": "session=abc123" } + ], + "queryString": [{ "name": "category", "value": "Juice" }], + "cookies": [], + "headersSize": -1, + "bodySize": 0 + }, + "response": { + "status": 200, + "statusText": "", + "httpVersion": "http/2.0", + "headers": [{ "name": "content-type", "value": "text/html; charset=utf-8" }], + "content": { "size": 19, "mimeType": "text/html", "text": "PGgxPkp1aWNlPC9oMT4=", "encoding": "base64" } + } + }, + { + "request": { + "method": "POST", + "url": "https://ginandjuice.shop/login", + "httpVersion": "HTTP/1.1", + "headers": [ + { "name": "Host", "value": "ginandjuice.shop" }, + { "name": "Content-Type", "value": "application/x-www-form-urlencoded" } + ], + "postData": { "mimeType": "application/x-www-form-urlencoded", "text": "username=carlos&password=hunter2" } + }, + "response": { + "status": 302, + "statusText": "Found", + "httpVersion": "HTTP/1.1", + "headers": [{ "name": "Location", "value": "/my-account" }], + "content": { "size": 0, "mimeType": "" } + } + }, + { + "request": { + "method": "GET", + "url": "data:image/png;base64,iVBORw0KGgo=", + "httpVersion": "", + "headers": [] + }, + "response": { "status": 200, "statusText": "", "httpVersion": "", "headers": [], "content": { "size": 0 } } + }, + { + "request": { + "method": "GET", + "url": "https://ginandjuice.shop/blocked.js", + "httpVersion": "", + "headers": [] + }, + "response": { "status": 0, "statusText": "", "httpVersion": "", "headers": [], "content": { "size": 0 } } + } + ] + } +} diff --git a/pkg/input/provider/http/multiformat.go b/pkg/input/provider/http/multiformat.go index 6b8d476f6..4109cf54a 100644 --- a/pkg/input/provider/http/multiformat.go +++ b/pkg/input/provider/http/multiformat.go @@ -10,6 +10,7 @@ import ( "github.com/projectdiscovery/gologger" "github.com/projectdiscovery/nuclei/v3/pkg/input/formats" "github.com/projectdiscovery/nuclei/v3/pkg/input/formats/burp" + "github.com/projectdiscovery/nuclei/v3/pkg/input/formats/har" "github.com/projectdiscovery/nuclei/v3/pkg/input/formats/json" "github.com/projectdiscovery/nuclei/v3/pkg/input/formats/openapi" "github.com/projectdiscovery/nuclei/v3/pkg/input/formats/raw" @@ -142,6 +143,7 @@ func (i *HttpInputProvider) Close() {} // Supported Providers var providersList = []formats.Format{ burp.New(), + har.New(), json.New(), yaml.New(), openapi.New(),