Merge branch 'dev' into feat-6567-memory-leak

pkg/protocols/http/body_heuristic.go

@@ -0,0 +1,6 @@
+package http
+
+const (
+	// MinimalBodySize is used when body is not needed - enough for headers and a small preview
+	MinimalBodySize = 4 * 1024 // 4KB
+)

pkg/protocols/http/http.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"fmt"
 	"math"
+	"net/http"
 	"strings"
 	"time"
 
@@ -29,6 +30,8 @@ import (
 	"github.com/projectdiscovery/rawhttp"
 	"github.com/projectdiscovery/retryablehttp-go"
 	fileutil "github.com/projectdiscovery/utils/file"
+	mapsutil "github.com/projectdiscovery/utils/maps"
+	unitutils "github.com/projectdiscovery/utils/unit"
 )
 
 // Request contains a http request to be made from a template
@@ -536,6 +539,49 @@ const (
 	SetThreadToCountZero = "set-thread-count-to-zero"
 )
 
+var (
+	// VerboseTargetThreshold is the threshold for marking a target as verbose
+	VerboseTargetThreshold = 2 * unitutils.Mega
+	VerboseTargetMaxSize   = 4 * unitutils.Kilo
+)
+
+var (
+	// verboseTargetRegistry stores targets that have been identified as verbose
+	// This allows us to apply lower response size limits to verbose targets
+	verboseTargetRegistry = mapsutil.NewSyncLockMap[string, bool]()
+)
+
+// MarkVerboseTarget marks a target URL as verbose
+func MarkVerboseTarget(url string) {
+	_ = verboseTargetRegistry.Set(url, true)
+}
+
+// IsVerboseTarget checks if a target URL is marked as verbose
+func IsVerboseTarget(url string) bool {
+	value, ok := verboseTargetRegistry.Get(url)
+	if !ok {
+		return false
+	}
+	return value
+}
+
+// CheckAndMarkVerbose checks the response headers/content-length and marks the target as verbose if needed
+// This should be called when a response is received to detect verbose targets early
+func CheckAndMarkVerbose(resp *http.Response, url string) {
+	if resp == nil {
+		return
+	}
+
+	// Check Content-Length header first (most reliable if present)
+	if cl := resp.ContentLength; cl > 0 && cl > int64(VerboseTargetThreshold) {
+		MarkVerboseTarget(url)
+		return
+	}
+
+	// If Content-Length is missing or 0, we'll check actual body size
+	// after first read (handled in request.go)
+}
+
 func init() {
 	stats.NewEntry(SetThreadToCountZero, "Setting thread count to 0 for %d templates, dynamic extractors are not supported with payloads yet")
 }

pkg/protocols/http/operators.go

@@ -31,7 +31,8 @@ func (request *Request) Match(data map[string]interface{}, matcher *matchers.Mat
 		if !ok {
 			return false, []string{}
 		}
-		return matcher.Result(matcher.MatchStatusCode(statusCode)), []string{responsehighlighter.CreateStatusCodeSnippet(data["response"].(string), statusCode)}
+		responseStr := types.ToString(data["response"])
+		return matcher.Result(matcher.MatchStatusCode(statusCode)), []string{responsehighlighter.CreateStatusCodeSnippet(responseStr, statusCode)}
 	case matchers.SizeMatcher:
 		return matcher.Result(matcher.MatchSize(len(item))), []string{}
 	case matchers.WordsMatcher:
@@ -93,8 +94,10 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (st
 
 	if part == "all" {
 		builder := &strings.Builder{}
-		builder.WriteString(types.ToString(data["body"]))
-		builder.WriteString(types.ToString(data["all_headers"]))
+		bodyVal := data["body"]
+		headersVal := data["all_headers"]
+		builder.WriteString(types.ToString(bodyVal))
+		builder.WriteString(types.ToString(headersVal))
 		itemStr = builder.String()
 	} else {
 		item, ok := data[part]
@@ -107,7 +110,8 @@ func (request *Request) getMatchPart(part string, data output.InternalEvent) (st
 }
 
 // responseToDSLMap converts an HTTP response to a map for use in DSL matching
-func (request *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq, rawResp, body, headers string, duration time.Duration, extra map[string]interface{}) output.InternalEvent {
+// body and rawResp can be strings or hash markers (string starting with "hash:")
+func (request *Request) responseToDSLMap(resp *http.Response, host, matched, rawReq string, rawResp interface{}, body interface{}, headers string, duration time.Duration, extra map[string]interface{}) output.InternalEvent {
 	data := make(output.InternalEvent, 12+len(extra)+len(resp.Header)+len(resp.Cookies()))
 	maps.Copy(data, extra)
 	for _, cookie := range resp.Cookies() {
@@ -120,10 +124,12 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw
 	data["host"] = host
 	data["type"] = request.Type().String()
 	data["matched"] = matched
-	request.setHashOrDefault(data, "request", rawReq)
-	request.setHashOrDefault(data, "response", rawResp)
+	request.setHashOrDefault(data, "request", types.ToString(rawReq))
+	// Store rawResp (can be hash marker or actual string)
+	data["response"] = rawResp
 	data["status_code"] = resp.StatusCode
-	request.setHashOrDefault(data, "body", body)
+	// Store body (can be hash marker or actual string)
+	data["body"] = body
 	request.setHashOrDefault(data, "all_headers", headers)
 	request.setHashOrDefault(data, "header", headers)
 	data["duration"] = duration.Seconds()
@@ -131,7 +137,9 @@ func (request *Request) responseToDSLMap(resp *http.Response, host, matched, raw
 	data["template-info"] = request.options.TemplateInfo
 	data["template-path"] = request.options.TemplatePath
 
-	data["content_length"] = utils.CalculateContentLength(resp.ContentLength, int64(len(body)))
+	// Calculate content length - resolve hash if needed
+	bodyStr := types.ToString(body)
+	data["content_length"] = utils.CalculateContentLength(resp.ContentLength, int64(len(bodyStr)))
 
 	if request.StopAtFirstMatch || request.options.StopAtFirstMatch {
 		data["stop-at-first-match"] = true

pkg/protocols/http/request.go

@@ -55,6 +55,9 @@ const (
 	// max unique errors to store & combine
 	// when executing requests in parallel
 	maxErrorsWhenParallel = 3
+	// HashThreshold is the minimum response body size (in bytes) before using hash markers instead of actual values
+	// This prevents small responses from being stored as hashes, which breaks template matching
+	HashThreshold = 5 * unitutils.Mega // 5MB
 )
 
 var (
@@ -927,11 +930,22 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 
 	duration := time.Since(timeStart)
 
+	// Check if target is verbose and mark it if needed (before creating ResponseChain)
+	if resp != nil {
+		CheckAndMarkVerbose(resp, formedURL)
+	}
+
 	// define max body read limit
-	maxBodylimit := MaxBodyRead // 10MB
+	maxBodylimit := MaxBodyRead // 10MB default
+	// Apply half of max body read limit for verbose targets to prevent memory exhaustion
+	if IsVerboseTarget(formedURL) {
+		maxBodylimit = VerboseTargetMaxSize // cap to 4KB
+	}
+	// Template-specific MaxSize override takes precedence
 	if request.MaxSize > 0 {
 		maxBodylimit = request.MaxSize
 	}
+	// User-configured ResponseReadSize override takes highest precedence
 	if request.options.Options.ResponseReadSize != 0 {
 		maxBodylimit = request.options.Options.ResponseReadSize
 	}
@@ -960,6 +974,17 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 			return errors.Wrap(err, "could not generate response chain")
 		}
 
+		// Check actual body size and mark target as verbose if Content-Length was missing
+		// This handles cases where Content-Length header was absent but body is large
+		if !IsVerboseTarget(formedURL) {
+			bodySize := int64(respChain.Body().Len())
+			if bodySize > int64(VerboseTargetThreshold) {
+				MarkVerboseTarget(formedURL)
+				// Note: This marking is for future requests to this target
+				// Current request already has maxBodylimit set, but this will help subsequent templates
+			}
+		}
+
 		// log request stats
 		request.options.Output.RequestStatsLog(strconv.Itoa(respChain.Response().StatusCode), respChain.FullResponse().String())
 
@@ -1002,7 +1027,12 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 			}
 		}
 
-		outputEvent := request.responseToDSLMap(respChain.Response(), input.MetaInput.Input, matchedURL, convUtil.String(dumpedRequest), respChain.FullResponse().String(), respChain.Body().String(), respChain.Headers().String(), duration, generatedRequest.meta)
+		// Pass hashes when cached to save memory in DSL map
+		// Only use hashes if response body is larger than threshold to avoid breaking templates
+		var bodyVal, fullRespVal interface{}
+		bodyVal = respChain.Body().String()
+		fullRespVal = respChain.FullResponse().String()
+		outputEvent := request.responseToDSLMap(respChain.Response(), input.MetaInput.Input, matchedURL, convUtil.String(dumpedRequest), fullRespVal, bodyVal, respChain.Headers().String(), duration, generatedRequest.meta)
 		// add response fields to template context and merge templatectx variables to output event
 		request.options.AddTemplateVars(input.MetaInput, request.Type(), request.ID, outputEvent)
 		if request.options.HasTemplateCtx(input.MetaInput) {