Merge pull request #7980 from kamilikamil/request-body-budget

Bound request body memory held by concurrent HTTP requests

README.md

@@ -256,6 +256,7 @@ CONFIGURATIONS:
    -rsr, -response-size-read int         max response size to read in bytes
    -rss, -response-size-save int         max response size to read in bytes (default 1048576)
    -rqss, -request-size-save int         max request body size to save in bytes (0 to save the whole body) (default 32768)
+   -mrbi, -max-request-body-inflight int max request body bytes held by in-flight http requests; requests with bodies of 1MB or more queue past it (0 disables)
    -reset                                reset removes all nuclei configuration and data files (including nuclei-templates)
    -tlsi, -tls-impersonate               enable experimental client hello (ja3) tls randomization
    -hae, -http-api-endpoint string       experimental http api endpoint

_typos.toml

@@ -22,8 +22,6 @@ extend-exclude = [
     "go.sum",
     # Generated history. Commit SHAs are not words.
     "SECURITY_CONTEXT.md",
-    # Generated technology graph. Entries are third-party product identifiers.
-    "pkg/protocols/common/automaticscan/techgraph/tech-graph.json",
 ]
 
 [default.extend-identifiers]

cmd/nuclei/main.go

@@ -307,7 +307,6 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.BoolVarP(&options.NewTemplates, "new-templates", "nt", false, "run only new templates added in latest nuclei-templates release"),
 		flagSet.StringSliceVarP(&options.NewTemplatesWithVersion, "new-templates-version", "ntv", nil, "run new templates added in specific version", goflags.CommaSeparatedStringSliceOptions),
 		flagSet.BoolVarP(&options.AutomaticScan, "automatic-scan", "as", false, "automatic web scan using wappalyzer technology detection to tags mapping"),
-		flagSet.StringVarP(&options.AutomaticScanCoverage, "automatic-scan-coverage", "asc", "balanced", "automatic scan coverage tier: lean, balanced, or thorough"),
 		flagSet.StringSliceVarP(&options.Templates, "templates", "t", nil, "list of template or template directory to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
 		flagSet.StringSliceVarP(&options.TemplateURLs, "template-url", "turl", nil, "template url or list containing template urls to run (comma-separated, file)", goflags.FileCommaSeparatedStringSliceOptions),
 		flagSet.StringVarP(&options.AITemplatePrompt, "prompt", "ai", "", "generate and run template using ai prompt"),
@@ -408,6 +407,7 @@ on extensive configurability, massive extensibility and ease of use.`)
 		flagSet.IntVarP(&options.ResponseReadSize, "response-size-read", "rsr", 0, "max response size to read in bytes"),
 		flagSet.IntVarP(&options.ResponseSaveSize, "response-size-save", "rss", unitutils.Mega, "max response size to read in bytes"),
 		flagSet.IntVarP(&options.RequestSaveSize, "request-size-save", "rqss", 32*unitutils.Kilo, "max request body size to save in bytes (0 to save the whole body)"),
+		flagSet.IntVarP(&options.MaxRequestBodyInFlight, "max-request-body-inflight", "mrbi", 0, "max request body bytes held by in-flight http requests; requests with bodies of 1MB or more queue past it (0 disables)"),
 		flagSet.CallbackVar(func() { resetRequested = true }, "reset", "reset removes all nuclei configuration and data files (including nuclei-templates)"),
 		flagSet.BoolVarP(&options.TlsImpersonate, "tls-impersonate", "tlsi", false, "enable experimental client hello (ja3) tls randomization"),
 		flagSet.StringVarP(&options.HttpApiEndpoint, "http-api-endpoint", "hae", "", "experimental http api endpoint"),

internal/runner/runner.go

@@ -73,6 +73,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/stats"
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
 	"github.com/projectdiscovery/retryablehttp-go"
@@ -676,6 +677,7 @@ func (r *Runner) RunEnumeration() error {
 		Catalog:             r.catalog,
 		IssuesClient:        r.issuesClient,
 		RateLimiter:         r.rateLimiter,
+		RequestBodyBudget:   httpbody.NewBudget(int64(r.options.MaxRequestBodyInFlight)),
 		Interactsh:          r.interactsh,
 		ProjectFile:         r.projectFile,
 		Browser:             r.browser,

lib/multi.go

@@ -39,12 +39,14 @@ func createEphemeralObjects(ctx context.Context, base *NucleiEngine, opts *types
 		Catalog:      base.catalog,
 		IssuesClient: base.rc,
 		RateLimiter:  base.rateLimiter,
-		Interactsh:   base.interactshClient,
-		Colorizer:    aurora.New(aurora.WithColors(true)),
-		ResumeCfg:    types.NewResumeCfg(),
-		Parser:       base.parser,
-		Browser:      base.browserInstance,
-		DoNotCache:   opts.DoNotCacheTemplates,
+
+		RequestBodyBudget: base.executerOpts.RequestBodyBudget,
+		Interactsh:        base.interactshClient,
+		Colorizer:         aurora.New(aurora.WithColors(true)),
+		ResumeCfg:         types.NewResumeCfg(),
+		Parser:            base.parser,
+		Browser:           base.browserInstance,
+		DoNotCache:        opts.DoNotCacheTemplates,
 	}
 	if base.interactshClient != nil {
 		u.executerOpts.InteractshScope = base.interactshClient.NewRequestScope()

lib/sdk_private.go

@@ -36,6 +36,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/templates"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
 	nucleiUtils "github.com/projectdiscovery/nuclei/v3/pkg/utils"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
 	"github.com/projectdiscovery/ratelimit"
 	pdcpauth "github.com/projectdiscovery/utils/auth/pdcp"
 )
@@ -279,6 +280,7 @@ func (e *NucleiEngine) init(ctx context.Context) error {
 		Catalog:            e.catalog,
 		IssuesClient:       e.rc,
 		RateLimiter:        e.rateLimiter,
+		RequestBodyBudget:  httpbody.NewBudget(int64(e.opts.MaxRequestBodyInFlight)),
 		Interactsh:         e.interactshClient,
 		Colorizer:          aurora.New(aurora.WithColors(true)),
 		ResumeCfg:          types.NewResumeCfg(),

pkg/protocols/common/automaticscan/automaticscan.go

@@ -6,7 +6,6 @@ import (
 	"net/http"
 	"os"
 	"path/filepath"
-	"sort"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -21,7 +20,6 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/output"
 	"github.com/projectdiscovery/nuclei/v3/pkg/progress"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/automaticscan/techgraph"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/writer"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/http/httpclientpool"
@@ -64,14 +62,6 @@ type Service struct {
 	techTemplates      []*templates.Template
 	ServiceOpts        Options
 	hasResults         *atomic.Bool
-
-	// tech-graph based selection (automatic scan v2). When selector is nil the
-	// service falls back to the legacy tag-overlap selection.
-	selector     *techgraph.Selector
-	tier         techgraph.Tier
-	universe     map[string]*templates.Template
-	universeErr  error
-	universeOnce sync.Once
 }
 
 // New takes options and returns a new automatic scan service
@@ -113,7 +103,7 @@ func New(opts Options) (*Service, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "could not get http client")
 	}
-	service := &Service{
+	return &Service{
 		opts:               opts.ExecuterOpts,
 		store:              opts.Store,
 		engine:             opts.Engine,
@@ -125,22 +115,7 @@ func New(opts Options) (*Service, error) {
 		techTemplates:      techDetectTemplates,
 		ServiceOpts:        opts,
 		hasResults:         &atomic.Bool{},
-	}
-	tier, err := techgraph.ParseTier(opts.ExecuterOpts.Options.AutomaticScanCoverage)
-	if err != nil {
-		return nil, err
-	}
-	service.tier = tier
-
-	// load the embedded tech-graph for v2 selection; on failure we silently fall
-	// back to legacy tag-overlap so behaviour never regresses.
-	if graph, err := techgraph.Embedded(); err == nil {
-		service.selector = techgraph.NewSelector(graph)
-	} else {
-		gologger.Warning().Msgf("automatic scan: tech-graph unavailable, using legacy selection: %s", err)
-	}
-
-	return service, nil
+	}, nil
 }
 
 // Close closes the service
@@ -204,9 +179,7 @@ func (s *Service) executeAutomaticScanOnTarget(ctx context.Context, input *conte
 	finalTags = append(finalTags, s.opts.Options.Tags...)
 	finalTags = sliceutil.Dedupe(finalTags)
 
-	// Lean has no baseline, so an empty signal set has nothing to run. Balanced
-	// and thorough still run the tech-agnostic baseline.
-	if len(finalTags) == 0 && (s.selector == nil || s.tier == techgraph.TierLean) {
+	if len(finalTags) == 0 {
 		gologger.Warning().Msgf("Skipping automatic scan since no tags were found on %v\n", input.Input)
 		return
 	}
@@ -214,19 +187,9 @@ func (s *Service) executeAutomaticScanOnTarget(ctx context.Context, input *conte
 		gologger.Print().Msgf("Final tags identified for %v: %+v\n", input.Input, finalTags)
 	}
 
-	var finalTemplates []*templates.Template
-	if s.selector != nil {
-		finalTemplates = s.selectWithGraph(input, finalTags)
-	} else {
-		var err error
-		finalTemplates, err = LoadTemplatesWithTags(s.ServiceOpts, s.templateDirs, finalTags, false)
-		if err != nil {
-			gologger.Error().Msgf("%v Error loading templates: %s\n", input.Input, err)
-			return
-		}
-	}
-	if len(finalTemplates) == 0 {
-		gologger.Warning().Msgf("Skipping automatic scan since no templates were selected on %v\n", input.Input)
+	finalTemplates, err := LoadTemplatesWithTags(s.ServiceOpts, s.templateDirs, finalTags, false)
+	if err != nil {
+		gologger.Error().Msgf("%v Error loading templates: %s\n", input.Input, err)
 		return
 	}
 	gologger.Info().Msgf("Executing %d templates on %v", len(finalTemplates), input.Input)
@@ -242,137 +205,12 @@ func (s *Service) executeAutomaticScanOnTarget(ctx context.Context, input *conte
 	s.hasResults.Store(tmp.Load())
 }
 
-// ensureUniverse lazily loads every selectable template once and indexes it by
-// id. This is the pool from which graph-based selection draws (precise + fuzzy),
-// avoiding a per-target loader pass.
-func (s *Service) ensureUniverse() (map[string]*templates.Template, error) {
-	s.universeOnce.Do(func() {
-		all, err := s.store.LoadTemplates(s.templateDirs)
-		if err != nil {
-			s.universeErr = err
-			return
-		}
-		s.universe = make(map[string]*templates.Template, len(all))
-		for _, t := range all {
-			if t != nil {
-				s.universe[t.ID] = t
-			}
-		}
-	})
-	return s.universe, s.universeErr
-}
-
-// selectWithGraph resolves detected signals to a precise fire-set via the
-// tech-graph, augmented (except at the lean tier) by legacy tag-overlap for any
-// signals not present in the graph, so recall never falls below legacy behaviour.
-func (s *Service) selectWithGraph(input *contextargs.MetaInput, finalTags []string) []*templates.Template {
-	universe, err := s.ensureUniverse()
-	if err != nil || len(universe) == 0 {
-		if len(finalTags) == 0 {
-			return nil
-		}
-		if err != nil {
-			gologger.Warning().Msgf("automatic scan: template load failed, using tag selection: %s", err)
-		}
-		tpls, loadErr := LoadTemplatesWithTags(s.ServiceOpts, s.templateDirs, finalTags, false)
-		if loadErr != nil {
-			gologger.Error().Msgf("%v Error loading templates: %s\n", input.Input, loadErr)
-			return nil
-		}
-		return tpls
-	}
-
-	techIDs, unresolved := s.selector.Resolve(finalTags)
-	selected := map[string]*templates.Template{}
-
-	graph := 0
-	for _, id := range s.selector.FireSet(s.tier, techIDs) {
-		if t, ok := universe[id]; ok {
-			selected[id] = t
-			graph++
-		}
-	}
-
-	added := 0
-	if s.tier != techgraph.TierLean && len(techIDs) > 0 {
-		candidates := make([]techgraph.Candidate, 0, len(universe))
-		for id, t := range universe {
-			if _, ok := selected[id]; ok {
-				continue
-			}
-			candidates = append(candidates, techgraph.Candidate{
-				ID:       id,
-				Tags:     t.Info.Tags.ToSlice(),
-				Category: categoryFromPath(t.Path),
-			})
-		}
-		for _, id := range s.selector.AbsentDependents(techIDs, candidates) {
-			if t, ok := universe[id]; ok {
-				selected[id] = t
-				added++
-			}
-		}
-	}
-
-	fuzzy := 0
-	if s.tier != techgraph.TierLean && len(unresolved) > 0 {
-		want := map[string]struct{}{}
-		for _, tag := range unresolved {
-			want[strings.ToLower(tag)] = struct{}{}
-		}
-		for id, t := range universe {
-			if _, ok := selected[id]; ok {
-				continue
-			}
-			for _, tag := range t.Info.Tags.ToSlice() {
-				if _, ok := want[strings.ToLower(tag)]; ok {
-					selected[id] = t
-					fuzzy++
-					break
-				}
-			}
-		}
-	}
-
-	out := make([]*templates.Template, 0, len(selected))
-	for _, t := range selected {
-		out = append(out, t)
-	}
-	sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID })
-
-	if s.opts.Options.VerboseVerbose {
-		gologger.Print().Msgf("%v selection [tier=%s]: %d techs, %d graph + %d new + %d fuzzy = %d templates\n",
-			input.Input, s.tier, len(techIDs), graph, added, fuzzy, len(out))
-	}
-
-	if !s.opts.Options.DisableClustering {
-		out, _, _ = templates.ClusterTemplates(out, s.opts)
-	}
-	return out
-}
-
 type sharedProgress struct {
 	progress.Progress
 }
 
 func (p *sharedProgress) Init(hostCount int64, rulesCount int, requestCount int64) {}
 
-// categoryFromPath returns the nuclei category directory for a template path
-// (http/cves/2024/x.yaml -> cves). An empty result means the category is unknown
-// and the template is not eligible for the post-graph supplement.
-func categoryFromPath(path string) string {
-	parts := strings.Split(filepath.ToSlash(path), "/")
-	for i, part := range parts {
-		switch part {
-		case "http", "network", "dns", "ssl", "file", "headless", "javascript", "code", "cloud", "dast":
-			if i+1 < len(parts) {
-				return parts[i+1]
-			}
-		}
-	}
-	return ""
-}
-
 // getTagsUsingWappalyzer returns tags using wappalyzer by fingerprinting target
 // and utilizing the mapping data
 func (s *Service) getTagsUsingWappalyzer(input *contextargs.MetaInput) []string {

pkg/protocols/common/automaticscan/automaticscan_test.go

@@ -13,9 +13,3 @@ func TestNormalizeAppName(t *testing.T) {
 	appName = normalizeAppName("JBoss:2.3.5")
 	require.Equal(t, "jboss", appName, "could not get normalized name")
 }
-
-func TestCategoryFromPath(t *testing.T) {
-	require.Equal(t, "cves", categoryFromPath("/home/templates/http/cves/2024/CVE-2024-1.yaml"))
-	require.Equal(t, "fuzzing", categoryFromPath("http/fuzzing/xss.yaml"))
-	require.Equal(t, "", categoryFromPath("README.md"))
-}

pkg/protocols/common/automaticscan/techgraph/baseline.go

@@ -1,73 +0,0 @@
-package techgraph
-
-import "strings"
-
-// excludedCategories never participate in automatic scan (opt-in/specialized).
-var excludedCategories = map[string]struct{}{
-	"fuzzing":     {},
-	"osint":       {},
-	"token-spray": {},
-	"malware":     {},
-	"cloud":       {},
-}
-
-// baselineCategories are the only categories whose product-less templates may be
-// considered for the tech-agnostic baseline. Product-specific categories (cves,
-// vulnerabilities, exposed-panels, default-logins, cnvd) are intentionally absent:
-// an unmappable template there must go to unmapped (fallback), never baseline.
-var baselineCategories = map[string]struct{}{
-	"exposures":        {},
-	"misconfiguration": {},
-	"miscellaneous":    {},
-	"iot":              {},
-	"takeovers":        {}, // subdomain takeover signatures are run host-agnostically
-}
-
-// genericBaselineTags positively identify a tech-agnostic check.
-var genericBaselineTags = map[string]struct{}{
-	"cors": {}, "crlf": {}, "open-redirect": {}, "ds_store": {}, "dsstore": {},
-	"phpinfo": {}, "directory-listing": {}, "listing": {}, "debug": {},
-	"robots": {}, "sitemap": {}, "git": {}, "svn": {}, "dotenv": {},
-	"host-header-injection": {}, "http-trace": {}, "options-method": {},
-	"well-known": {}, "trace": {},
-}
-
-// genericBaselineIDSubstrings positively identify generic checks by id.
-var genericBaselineIDSubstrings = []string{
-	"git-config", "gitignore", "git-credentials", "svn-", ".env", "dotenv",
-	"phpinfo", "directory-listing", "dir-listing", "ds-store", "ds_store",
-	"robots-txt", "sitemap-", "cors-", "crlf-", "open-redirect", "trace-method",
-	"options-method", "well-known", "backup-files", "sql-dump",
-}
-
-// classifyBaseline decides whether an otherwise-unmapped template is a genuinely
-// generic, tech-agnostic baseline check. It is a positive allowlist: anything not
-// recognised as generic is left for the caller to mark unmapped.
-func classifyBaseline(info templateInfo) (Tier, bool) {
-	// tech-agnostic "generic" buckets (e.g. vulnerabilities/generic/* holds
-	// cors/sqli/xss probes that apply to any target) qualify regardless of
-	// category. The logs/files/backups subdir heuristic was removed: it leaked
-	// product-specific exposures (bitrix, magento, oracle-ebs, ...) into baseline.
-	if info.Subdir == "generic" {
-		return TierThorough, true
-	}
-	if _, ok := baselineCategories[info.Category]; !ok {
-		return "", false
-	}
-	// takeovers are wholesale host-agnostic.
-	if info.Category == "takeovers" {
-		return TierBalanced, true
-	}
-	for _, tag := range info.Tags {
-		if _, ok := genericBaselineTags[normToken(tag)]; ok {
-			return TierBalanced, true
-		}
-	}
-	idl := strings.ToLower(info.ID)
-	for _, sub := range genericBaselineIDSubstrings {
-		if strings.Contains(idl, sub) {
-			return TierBalanced, true
-		}
-	}
-	return "", false
-}

pkg/protocols/common/automaticscan/techgraph/build.go

@@ -1,353 +0,0 @@
-package techgraph
-
-import (
-	"crypto/sha256"
-	"encoding/json"
-	"fmt"
-	"io/fs"
-	"os"
-	"path/filepath"
-	"sort"
-	"strings"
-	"time"
-)
-
-// Build walks the templates root and produces the tech-graph artifact.
-func Build(root string) (*Graph, error) {
-	root = filepath.Clean(root)
-	g := &Graph{
-		Version:     ArtifactVersion,
-		GeneratedAt: time.Now().UTC().Format(time.RFC3339),
-		Techs:       map[string]*Tech{},
-		Synonyms:    map[string]string{},
-	}
-
-	infos, hash, parseErr, err := collect(root)
-	if err != nil {
-		return nil, err
-	}
-	g.Stats.ParseError = parseErr
-	g.SourceHash = fmt.Sprintf("sha256:%x", hash)
-
-	var deferred []templateInfo
-
-	// Pass 1: build tech nodes from authoritative signals (detection anchors,
-	// cpe/product, platform). Everything else is deferred so it can be resolved
-	// against the fully-built node set in pass 2.
-	for _, info := range infos {
-		g.Stats.Total++
-
-		// Excluded categories never enter detection or the fire-set. Gating
-		// this before detection classification keeps opt-in categories such as
-		// cloud out of the automatic-scan detector list.
-		if _, ok := excludedCategories[info.Category]; ok {
-			g.Stats.Excluded++
-			continue
-		}
-		if isDetection(info) && isDetectionSeverity(info) {
-			id := techID(info.Vendor, info.Product)
-			if id != "" {
-				g.anchorTech(id, info)
-			}
-			g.Detection = append(g.Detection, DetectionEntry{ID: info.ID, Tech: id})
-			g.Stats.Detection++
-			continue
-		}
-		if p, ok := detectPlatform(info); ok {
-			g.addDependent(p, info, "platform")
-			g.Stats.Dependents++
-			g.Stats.DependentsPlat++
-			continue
-		}
-		if id := techID(info.Vendor, info.Product); id != "" {
-			g.addDependent(id, info, "cpe")
-			g.Stats.Dependents++
-			g.Stats.DependentsCPE++
-			continue
-		}
-		deferred = append(deferred, info)
-	}
-
-	// Pass 2: resolve deferred templates to a known tech BEFORE considering
-	// baseline. Order: directory grouping -> tag -> id token. Only genuinely
-	// generic leftovers become baseline; the rest are unmapped (runtime fallback).
-	aliasIdx := g.buildAliasIndex()
-	for _, info := range deferred {
-		if id, ok := g.resolveToken(info.Subdir, aliasIdx); ok {
-			g.addDependent(id, info, "dir")
-			g.Stats.Dependents++
-			g.Stats.DependentsDir++
-			continue
-		}
-		if id, ok := reconcileByTag(g, info, aliasIdx); ok {
-			g.addDependent(id, info, "tag")
-			g.Stats.Dependents++
-			g.Stats.DependentsTag++
-			continue
-		}
-		if id, ok := reconcileByID(g, info, aliasIdx); ok {
-			g.addDependent(id, info, "id")
-			g.Stats.Dependents++
-			g.Stats.DependentsID++
-			continue
-		}
-		if tier, ok := classifyBaseline(info); ok {
-			g.Baseline = append(g.Baseline, BaselineEntry{
-				ID: info.ID, Category: info.Category, Severity: info.Severity, Tier: tier,
-			})
-			g.Stats.Baseline++
-			continue
-		}
-		g.Stats.Unmapped++
-	}
-
-	// ship validated seed synonyms in the artifact so the runtime selector (which
-	// only reads the artifact) can resolve acronyms like aem -> adobe:experience_manager.
-	for alias, id := range seedSynonyms {
-		if _, ok := g.Techs[id]; ok {
-			g.Synonyms[alias] = id
-		}
-	}
-
-	g.Stats.Techs = len(g.Techs)
-	g.sortStable()
-	return g, nil
-}
-
-// collect walks the tree once and returns parsed infos plus a content hash.
-func collect(root string) ([]templateInfo, []byte, int, error) {
-	var infos []templateInfo
-	var parseErr int
-	hasher := sha256.New()
-	err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
-		if err != nil {
-			return nil
-		}
-		if d.IsDir() {
-			if skipDir(d.Name()) {
-				return filepath.SkipDir
-			}
-			return nil
-		}
-		if !strings.HasSuffix(path, ".yaml") && !strings.HasSuffix(path, ".yml") {
-			return nil
-		}
-		info, ok := extractTemplate(path, root)
-		if !ok {
-			parseErr++
-			return nil
-		}
-		if _, err := fmt.Fprintf(hasher, "%s|%s|%s|%s\n", info.ID, info.Vendor, info.Product, info.CPE); err != nil {
-			return err
-		}
-		infos = append(infos, info)
-		return nil
-	})
-	if err != nil {
-		return nil, nil, 0, err
-	}
-	sort.Slice(infos, func(i, j int) bool { return infos[i].ID < infos[j].ID })
-	return infos, hasher.Sum(nil), parseErr, nil
-}
-
-// anchorTech creates/updates a tech node's metadata without adding a fireable
-// template (used by detector templates that define a tech).
-func (g *Graph) anchorTech(id string, info templateInfo) *Tech {
-	tech, ok := g.Techs[id]
-	if !ok {
-		tech = &Tech{ID: id, Vendor: normToken(info.Vendor), Product: normToken(info.Product), CPE: info.CPE}
-		g.Techs[id] = tech
-	}
-	if tech.CPE == "" && info.CPE != "" {
-		tech.CPE = info.CPE
-	}
-	mergeAliases(tech, aliasesFor(info.Vendor, info.Product, info.Tags))
-	for _, a := range tech.Aliases {
-		if existing, ok := g.Synonyms[a]; !ok || existing == id {
-			g.Synonyms[a] = id
-		}
-	}
-	return tech
-}
-
-// addDependent registers a fireable template under its canonical tech.
-//
-// Only "cpe" attachments contribute aliases (the template's own vendor/product
-// genuinely define the tech). Platform/dir/tag/id attachments must NOT pollute
-// the node's aliases with the sub-product's tokens (e.g. a wp-plugin's name must
-// not become a wordpress alias), which would corrupt reconciliation.
-func (g *Graph) addDependent(id string, info templateInfo, source string) {
-	var tech *Tech
-	if source == "cpe" {
-		tech = g.anchorTech(id, info)
-	} else {
-		tech = g.ensureTech(id, info)
-	}
-	ref := TemplateRef{ID: info.ID, Category: info.Category, Severity: info.Severity, Source: source}
-	if source == "platform" || source == "dir" {
-		ref.Product = normToken(info.Product)
-	}
-	tech.Templates = append(tech.Templates, ref)
-}
-
-// ensureTech returns an existing node, or creates a bare one without merging the
-// template's aliases (used for non-cpe attachments).
-func (g *Graph) ensureTech(id string, info templateInfo) *Tech {
-	if tech, ok := g.Techs[id]; ok {
-		return tech
-	}
-	vendor, product := "", ""
-	if v, p, ok := splitTechID(id); ok {
-		vendor, product = v, p
-	}
-	tech := &Tech{ID: id, Vendor: vendor, Product: product}
-	g.Techs[id] = tech
-	return tech
-}
-
-// splitTechID reverses techID for bare node creation.
-func splitTechID(id string) (vendor, product string, ok bool) {
-	if i := strings.IndexByte(id, ':'); i > 0 {
-		return id[:i], id[i+1:], true
-	}
-	return "", id, true
-}
-
-// buildAliasIndex maps a single alias/product token to a unique tech id. Tokens
-// that resolve to more than one tech, are too short, or are generic are dropped
-// so tag reconciliation stays conservative.
-func (g *Graph) buildAliasIndex() map[string]string {
-	counts := map[string]map[string]struct{}{}
-	add := func(tok, id string) {
-		tok = normToken(tok)
-		if len(tok) < 5 || genericTokens[tok] {
-			return
-		}
-		if counts[tok] == nil {
-			counts[tok] = map[string]struct{}{}
-		}
-		counts[tok][id] = struct{}{}
-	}
-	for id, tech := range g.Techs {
-		add(id, id)
-		add(tech.Product, id)
-		for _, a := range tech.Aliases {
-			add(a, id)
-		}
-	}
-	idx := map[string]string{}
-	for tok, ids := range counts {
-		if len(ids) == 1 {
-			for id := range ids {
-				idx[tok] = id
-			}
-		}
-	}
-	return idx
-}
-
-// resolveToken maps a single token to a tech id via the seed synonym table
-// first (validated against existing nodes), then the alias index.
-func (g *Graph) resolveToken(tok string, idx map[string]string) (string, bool) {
-	tok = normToken(tok)
-	if tok == "" || genericTokens[tok] {
-		return "", false
-	}
-	if id, ok := seedSynonyms[tok]; ok {
-		if _, exists := g.Techs[id]; exists {
-			return id, true
-		}
-	}
-	if id, ok := idx[tok]; ok {
-		return id, true
-	}
-	return "", false
-}
-
-// reconcileByTag attaches a product-less template to a known tech when its tags
-// resolve to exactly one unique tech node.
-func reconcileByTag(g *Graph, info templateInfo, idx map[string]string) (string, bool) {
-	matches := map[string]struct{}{}
-	for _, tag := range info.Tags {
-		if id, ok := g.resolveToken(tag, idx); ok {
-			matches[id] = struct{}{}
-		}
-	}
-	if len(matches) == 1 {
-		for id := range matches {
-			return id, true
-		}
-	}
-	return "", false
-}
-
-// reconcileByID attaches a template when its id tokens resolve to exactly one
-// unique tech node (e.g. "aem-jcr-exposure" -> adobe:experience_manager).
-//
-// This is the riskiest signal (id tokens are common English words), so it is
-// guarded: a token must be >=6 chars AND also appear in the template name, which
-// requires the product to be the template's actual subject rather than an
-// incidental id suffix like "-dashboard"/"-gateway".
-func reconcileByID(g *Graph, info templateInfo, idx map[string]string) (string, bool) {
-	name := strings.ToLower(info.Name)
-	matches := map[string]struct{}{}
-	for _, tok := range strings.Split(strings.ToLower(info.ID), "-") {
-		if len(tok) < 6 || !strings.Contains(name, tok) {
-			continue
-		}
-		if id, ok := g.resolveToken(tok, idx); ok {
-			matches[id] = struct{}{}
-		}
-	}
-	if len(matches) == 1 {
-		for id := range matches {
-			return id, true
-		}
-	}
-	return "", false
-}
-
-func mergeAliases(tech *Tech, aliases []string) {
-	have := map[string]struct{}{}
-	for _, a := range tech.Aliases {
-		have[a] = struct{}{}
-	}
-	for _, a := range aliases {
-		if a == "" {
-			continue
-		}
-		if _, ok := have[a]; ok {
-			continue
-		}
-		have[a] = struct{}{}
-		tech.Aliases = append(tech.Aliases, a)
-	}
-}
-
-func (g *Graph) sortStable() {
-	for _, tech := range g.Techs {
-		sort.Strings(tech.Aliases)
-		sort.Slice(tech.Templates, func(i, j int) bool {
-			return tech.Templates[i].ID < tech.Templates[j].ID
-		})
-	}
-	sort.Slice(g.Baseline, func(i, j int) bool { return g.Baseline[i].ID < g.Baseline[j].ID })
-	sort.Slice(g.Detection, func(i, j int) bool { return g.Detection[i].ID < g.Detection[j].ID })
-}
-
-// WriteFile serialises the graph as indented JSON.
-func (g *Graph) WriteFile(path string) error {
-	data, err := json.MarshalIndent(g, "", "  ")
-	if err != nil {
-		return err
-	}
-	return os.WriteFile(path, append(data, '\n'), 0o644)
-}
-
-func skipDir(name string) bool {
-	switch name {
-	case ".git", ".github", "helpers", "profiles", "workflows":
-		return true
-	}
-	return false
-}

pkg/protocols/common/automaticscan/techgraph/canonical.go

@@ -1,121 +0,0 @@
-package techgraph
-
-import "strings"
-
-// seedSynonyms maps common acronyms/aliases to a canonical tech id. This is the
-// curated seed of the synonym table that the offline AI step later expands.
-// Targets are validated against existing nodes at resolve time, so a stale entry
-// simply has no effect.
-var seedSynonyms = map[string]string{
-	"aem":         "adobe:experience_manager",
-	"coldfusion":  "adobe:coldfusion",
-	"iis":         "microsoft:iis",
-	"k8s":         "kubernetes:kubernetes",
-	"springboot":  "vmware:spring_boot",
-	"spring-boot": "vmware:spring_boot",
-	"wls":         "oracle:weblogic_server",
-	"weblogic":    "oracle:weblogic_server",
-	"adfs":        "microsoft:active_directory_federation_services",
-	// platform roots: ambiguous in the alias index (collide with sub-product
-	// nodes like keydatas:wordpress) so pin them to the canonical platform node.
-	"wordpress": "wordpress",
-	"drupal":    "drupal",
-	"joomla":    "joomla",
-	"magento":   "magento",
-}
-
-// genericTokens are tags/aliases that are not products and must never be used as
-// a tag/id/dir-reconciliation key (they would incorrectly attach templates to random
-// techs). This list intentionally includes common English words that also happen
-// to be CPE product names (dashboard, gateway, console, ...), platform/OS tokens
-// (linux, windows, android, ...), and category labels (generic, audit, ...),
-// since those were the dominant source of false attachments.
-var genericTokens = map[string]bool{
-	// vuln classes / scan vocabulary
-	"cve": true, "cves": true, "vuln": true, "vulnerability": true, "tech": true,
-	"detect": true, "detection": true, "panel": true, "panels": true, "login": true, "cms": true,
-	"exposure": true, "exposures": true, "misconfig": true, "misconfiguration": true,
-	"rce": true, "lfi": true, "xss": true, "sqli": true, "ssrf": true, "ssti": true,
-	"redirect": true, "disclosure": true, "default": true, "auth": true, "bypass": true,
-	"injection": true, "traversal": true, "config": true, "files": true, "file": true,
-	"backup": true, "takeover": true, "oast": true, "intrusive": true, "unauth": true,
-	"router": true, "iot": true, "network": true, "http": true, "json": true, "api": true,
-	"server": true, "service": true, "framework": true, "plugin": true, "theme": true,
-	"webserver": true, "database": true, "cloud": true, "devops": true, "tools": true,
-	"edb": true, "kev": true, "packetstorm": true, "seclists": true, "osint": true,
-	// category labels
-	"generic": true, "audit": true, "miscellaneous": true, "misc": true, "enumeration": true,
-	"vulnerabilities": true, "exposed": true, "keys": true, "secret": true, "secrets": true,
-	// platform / OS / cloud-provider tokens (not products)
-	"linux": true, "windows": true, "macos": true, "unix": true, "android": true, "ios": true,
-	"aws": true, "azure": true, "gcp": true, "alibaba": true, "kubernetes": true, "k8s": true,
-	"docker": true, "mobile": true,
-	// common English words that collide with product names
-	"dashboard": true, "gateway": true, "path": true, "console": true, "firewall": true,
-	"monitor": true, "control": true, "permissions": true, "oauth": true, "analytics": true,
-	"enterprise": true, "report": true, "reports": true, "help": true, "printer": true,
-	"core": true, "commerce": true, "anchor": true, "nova": true, "element": true,
-	"webclient": true, "memcached": true, "manager": true, "management": true, "admin": true,
-	"client": true, "agent": true, "portal": true, "gateway2": true, "status": true,
-	"user": true, "users": true, "account": true, "token": true, "tokens": true,
-	"web": true, "app": true, "application": true, "system": true, "platform": true,
-}
-
-// normToken lowercases and trims a vendor/product token. CPE uses '_' for
-// spaces; we keep it as-is so the id stays stable and machine-joinable.
-func normToken(s string) string {
-	s = strings.ToLower(strings.TrimSpace(s))
-	// CPE wildcards / NA markers are not real tokens.
-	if s == "*" || s == "-" {
-		return ""
-	}
-	// URLs / paths / multi-word strings are not clean product tokens (guards
-	// against junk nodes like "https://apisix.apache.org").
-	if strings.ContainsAny(s, "/ ") {
-		return ""
-	}
-	return s
-}
-
-// techID builds the canonical technology id from vendor/product. When vendor is
-// empty or equal to product we collapse to a single token to avoid duplicate
-// nodes like "wordpress:wordpress" vs "wordpress".
-func techID(vendor, product string) string {
-	vendor = normToken(vendor)
-	product = normToken(product)
-	switch {
-	case product == "":
-		return ""
-	case vendor == "" || vendor == product:
-		return product
-	default:
-		return vendor + ":" + product
-	}
-}
-
-// aliasesFor returns alias tokens that runtime detection (wappalyzer names,
-// tags) might surface for a tech, so they can be reconciled to the canonical id.
-// Pillar 1 derives these from product/vendor tokens; AI augments this offline.
-func aliasesFor(vendor, product string, tags []string) []string {
-	seen := map[string]struct{}{}
-	var out []string
-	add := func(s string) {
-		s = normToken(s)
-		if s == "" {
-			return
-		}
-		if _, ok := seen[s]; ok {
-			return
-		}
-		seen[s] = struct{}{}
-		out = append(out, s)
-	}
-	// Only whole product/vendor tokens are safe aliases. Splitting on separators
-	// produces ambiguous fragments (manager, server, token, web, ...) that cause
-	// false tag/id reconciliation, so we deliberately avoid it.
-	add(product)
-	if vendor != product {
-		add(vendor)
-	}
-	return out
-}

pkg/protocols/common/automaticscan/techgraph/cmd/techgraph-gen/main.go

@@ -1,59 +0,0 @@
-// Command techgraph-gen builds the offline tech-graph.json artifact from a local
-// nuclei-templates checkout. See pkg/protocols/common/automaticscan/AUTOMATIC_SCAN_V2.md.
-package main
-
-import (
-	"flag"
-	"fmt"
-	"os"
-
-	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/automaticscan/techgraph"
-)
-
-func main() {
-	var (
-		templates = flag.String("templates", "", "path to nuclei-templates directory (required)")
-		output    = flag.String("output", "tech-graph.json", "output artifact path")
-		htmlOut   = flag.String("html", "", "also write a standalone visualizer HTML with data inlined")
-	)
-	flag.Parse()
-
-	if *templates == "" {
-		fmt.Fprintln(os.Stderr, "error: -templates is required")
-		flag.Usage()
-		os.Exit(2)
-	}
-
-	graph, err := techgraph.Build(*templates)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "error: build failed: %v\n", err)
-		os.Exit(1)
-	}
-	if err := graph.WriteFile(*output); err != nil {
-		fmt.Fprintf(os.Stderr, "error: write failed: %v\n", err)
-		os.Exit(1)
-	}
-	if *htmlOut != "" {
-		html, err := techgraph.StandaloneHTML(graph)
-		if err != nil {
-			fmt.Fprintf(os.Stderr, "error: html render failed: %v\n", err)
-			os.Exit(1)
-		}
-		if err := os.WriteFile(*htmlOut, []byte(html), 0o644); err != nil {
-			fmt.Fprintf(os.Stderr, "error: html write failed: %v\n", err)
-			os.Exit(1)
-		}
-		fmt.Printf("standalone visualizer written to %s\n", *htmlOut)
-	}
-
-	s := graph.Stats
-	fmt.Printf("tech-graph written to %s\n", *output)
-	fmt.Printf("  templates:  %d (parse-skipped %d)\n", s.Total, s.ParseError)
-	fmt.Printf("  dependents: %d (cpe %d, platform %d, dir %d, tag %d, id %d) across %d techs\n",
-		s.Dependents, s.DependentsCPE, s.DependentsPlat, s.DependentsDir, s.DependentsTag, s.DependentsID, s.Techs)
-	fmt.Printf("  baseline:   %d\n", s.Baseline)
-	fmt.Printf("  detection:  %d\n", s.Detection)
-	fmt.Printf("  excluded:   %d\n", s.Excluded)
-	fmt.Printf("  unmapped:   %d\n", s.Unmapped)
-	fmt.Printf("  source:     %s\n", graph.SourceHash)
-}

pkg/protocols/common/automaticscan/techgraph/embed.go

@@ -1,39 +0,0 @@
-package techgraph
-
-import (
-	_ "embed"
-	"encoding/json"
-	"strings"
-)
-
-// embeddedGraph is the prebuilt tech-graph artifact shipped with nuclei. Run the
-// techgraph-gen command to regenerate it against a nuclei-templates checkout.
-//
-//go:embed tech-graph.json
-var embeddedGraph []byte
-
-//go:embed visualizer.html
-var visualizerHTML string
-
-// Embedded returns the tech-graph artifact compiled into the binary.
-func Embedded() (*Graph, error) {
-	g := &Graph{}
-	if err := json.Unmarshal(embeddedGraph, g); err != nil {
-		return nil, err
-	}
-	return g, nil
-}
-
-// EmbeddedRaw returns the raw embedded artifact bytes.
-func EmbeddedRaw() []byte { return embeddedGraph }
-
-// StandaloneHTML returns the visualizer with the given graph inlined, so it can
-// be opened directly in a browser without a server or sidecar file.
-func StandaloneHTML(g *Graph) (string, error) {
-	data, err := json.Marshal(g)
-	if err != nil {
-		return "", err
-	}
-	inject := "<script>window.__TECHGRAPH__=" + string(data) + ";</script>\n</head>"
-	return strings.Replace(visualizerHTML, "</head>", inject, 1), nil
-}

pkg/protocols/common/automaticscan/techgraph/extract.go

@@ -1,165 +0,0 @@
-package techgraph
-
-import (
-	"os"
-	"path/filepath"
-	"strings"
-
-	"gopkg.in/yaml.v3"
-)
-
-// templateInfo is the subset of a nuclei template we need for graph building.
-// We deliberately avoid the full template parser: only the info block is read,
-// which keeps the generator fast and dependency-light over ~13k files.
-type templateInfo struct {
-	ID       string
-	Name     string
-	Severity string
-	Category string // derived from directory (cves, exposed-panels, ...)
-	Subdir   string // directory under the category (e.g. "aem" in misconfiguration/aem/)
-	Tags     []string
-	Vendor   string
-	Product  string
-	CPE      string
-}
-
-// rawTemplate mirrors the YAML structure for lenient decoding.
-type rawTemplate struct {
-	ID   string `yaml:"id"`
-	Info struct {
-		Name           string         `yaml:"name"`
-		Severity       string         `yaml:"severity"`
-		Tags           any            `yaml:"tags"`
-		Metadata       map[string]any `yaml:"metadata"`
-		Classification struct {
-			CPE string `yaml:"cpe"`
-		} `yaml:"classification"`
-	} `yaml:"info"`
-}
-
-// extractTemplate reads a single template file and returns its info subset.
-// ok is false when the file is not a parseable template (e.g. workflow, helper).
-func extractTemplate(path, root string) (templateInfo, bool) {
-	data, err := os.ReadFile(path)
-	if err != nil {
-		return templateInfo{}, false
-	}
-	var raw rawTemplate
-	if err := yaml.Unmarshal(data, &raw); err != nil {
-		return templateInfo{}, false
-	}
-	if raw.ID == "" {
-		return templateInfo{}, false
-	}
-
-	category, subdir := derivePath(path, root)
-	info := templateInfo{
-		ID:       raw.ID,
-		Name:     raw.Info.Name,
-		Severity: strings.ToLower(strings.TrimSpace(raw.Info.Severity)),
-		Category: category,
-		Subdir:   subdir,
-		Tags:     coerceStringSlice(raw.Info.Tags),
-		CPE:      strings.TrimSpace(raw.Info.Classification.CPE),
-	}
-
-	if raw.Info.Metadata != nil {
-		info.Vendor = firstNonEmpty(coerceStringSlice(raw.Info.Metadata["vendor"]))
-		info.Product = firstNonEmpty(coerceStringSlice(raw.Info.Metadata["product"]))
-	}
-
-	// Prefer CPE for vendor/product when metadata is missing.
-	if cv, cp, ok := parseCPE(info.CPE); ok {
-		if info.Vendor == "" {
-			info.Vendor = cv
-		}
-		if info.Product == "" {
-			info.Product = cp
-		}
-	}
-	return info, true
-}
-
-// derivePath returns the nuclei category and the immediate subdirectory under
-// it, e.g. http/misconfiguration/aem/x.yaml -> ("misconfiguration", "aem").
-// The subdir is often a product grouping and is used as a tech signal.
-func derivePath(path, root string) (category, subdir string) {
-	rel, err := filepath.Rel(root, path)
-	if err != nil {
-		rel = path
-	}
-	parts := strings.Split(filepath.ToSlash(rel), "/")
-	switch {
-	case len(parts) >= 4:
-		// protocol/category/subdir/.../file
-		return parts[1], parts[2]
-	case len(parts) == 3:
-		return parts[1], ""
-	case len(parts) == 2:
-		return parts[0], ""
-	default:
-		return "", ""
-	}
-}
-
-// parseCPE extracts vendor and product from CPE 2.3 (cpe:2.3:a:vendor:product:..)
-// or CPE 2.2 URI (cpe:/a:vendor:product:..) forms.
-func parseCPE(cpe string) (vendor, product string, ok bool) {
-	cpe = strings.TrimSpace(cpe)
-	if cpe == "" {
-		return "", "", false
-	}
-	switch {
-	case strings.HasPrefix(cpe, "cpe:2.3:"):
-		parts := strings.Split(cpe, ":")
-		// cpe:2.3:part:vendor:product:...
-		if len(parts) >= 5 {
-			return normToken(parts[3]), normToken(parts[4]), true
-		}
-	case strings.HasPrefix(cpe, "cpe:/"):
-		parts := strings.Split(strings.TrimPrefix(cpe, "cpe:/"), ":")
-		// part:vendor:product:...
-		if len(parts) >= 3 {
-			return normToken(parts[1]), normToken(parts[2]), true
-		}
-	}
-	return "", "", false
-}
-
-// coerceStringSlice normalises tags/metadata values that may be a string, a
-// comma-separated string, or a YAML list into a clean slice.
-func coerceStringSlice(v any) []string {
-	var out []string
-	add := func(s string) {
-		for _, part := range strings.Split(s, ",") {
-			part = strings.TrimSpace(part)
-			if part != "" {
-				out = append(out, part)
-			}
-		}
-	}
-	switch t := v.(type) {
-	case string:
-		add(t)
-	case []any:
-		for _, item := range t {
-			if s, ok := item.(string); ok {
-				add(s)
-			}
-		}
-	case []string:
-		for _, s := range t {
-			add(s)
-		}
-	}
-	return out
-}
-
-func firstNonEmpty(values []string) string {
-	for _, v := range values {
-		if strings.TrimSpace(v) != "" {
-			return strings.TrimSpace(v)
-		}
-	}
-	return ""
-}

pkg/protocols/common/automaticscan/techgraph/platform.go

@@ -1,77 +0,0 @@
-package techgraph
-
-// platforms are technologies with large plugin/extension ecosystems whose
-// add-on templates carry the add-on as `product` (often with the platform as
-// `vendor` or only as a tag). Runtime detection sees the platform, not each
-// plugin, so those templates must be reachable from the platform node.
-var platforms = map[string]string{
-	"wordpress":  "wordpress",
-	"joomla":     "joomla",
-	"drupal":     "drupal",
-	"magento":    "magento",
-	"prestashop": "prestashop",
-	"opencart":   "opencart",
-	"moodle":     "moodle",
-	"typo3":      "typo3",
-	"dotnetnuke": "dotnetnuke",
-	"concrete5":  "concrete5",
-	"shopware":   "shopware",
-	"vbulletin":  "vbulletin",
-	"mybb":       "mybb",
-	"phpbb":      "phpbb",
-}
-
-// platformTagAliases maps secondary tag tokens to a platform key.
-var platformTagAliases = map[string]string{
-	"wp":        "wordpress",
-	"wp-plugin": "wordpress",
-	"wp-theme":  "wordpress",
-}
-
-// detectPlatform returns the platform node id when a template belongs to a
-// platform's add-on ecosystem (and is not the platform core itself).
-func detectPlatform(info templateInfo) (string, bool) {
-	vendor := normToken(info.Vendor)
-	product := normToken(info.Product)
-
-	if p, ok := platforms[vendor]; ok && product != "" && product != p {
-		return p, true
-	}
-	for _, tag := range info.Tags {
-		t := normToken(tag)
-		if p, ok := platforms[t]; ok {
-			if product != "" && product != p {
-				return p, true
-			}
-		}
-		if p, ok := platformTagAliases[t]; ok {
-			if product != p {
-				return p, true
-			}
-		}
-	}
-	return "", false
-}
-
-// isDetection reports whether a template is a phase-1 detector.
-func isDetection(info templateInfo) bool {
-	for _, tag := range info.Tags {
-		switch normToken(tag) {
-		case "tech", "detect", "favicon":
-			return true
-		}
-	}
-	return false
-}
-
-// isDetectionSeverity guards the detection bucket against vuln/exposure templates
-// that merely carry a "tech"/"detect" tag. Real fingerprint templates are
-// informational; medium+ severity indicates an actual finding, not detection.
-func isDetectionSeverity(info templateInfo) bool {
-	switch info.Severity {
-	case "", "info", "low":
-		return true
-	default:
-		return false
-	}
-}

pkg/protocols/common/automaticscan/techgraph/selector.go

@@ -1,221 +0,0 @@
-package techgraph
-
-import (
-	"fmt"
-	"sort"
-	"strings"
-)
-
-// Selector resolves detected technology signals into a concrete fire-set of
-// template ids using a tech-graph artifact. It is pure and deterministic.
-type Selector struct {
-	g       *Graph
-	byAlias map[string]string // alias/synonym/id/product -> canonical tech id
-}
-
-// NewSelector builds a selector with an alias index for fast signal resolution.
-func NewSelector(g *Graph) *Selector {
-	s := &Selector{g: g, byAlias: make(map[string]string)}
-	// synonyms first (authoritative), then ids/products/aliases without clobbering.
-	for a, id := range g.Synonyms {
-		s.byAlias[normToken(a)] = id
-	}
-	for id, t := range g.Techs {
-		if _, ok := s.byAlias[normToken(id)]; !ok {
-			s.byAlias[normToken(id)] = id
-		}
-		if t.Product != "" {
-			if _, ok := s.byAlias[normToken(t.Product)]; !ok {
-				s.byAlias[normToken(t.Product)] = id
-			}
-		}
-		for _, a := range t.Aliases {
-			if _, ok := s.byAlias[normToken(a)]; !ok {
-				s.byAlias[normToken(a)] = id
-			}
-		}
-	}
-	return s
-}
-
-// Resolve maps detected signal tokens (wappalyzer names, detection tags, cpe
-// products) to canonical tech ids. Tokens that match no node are returned as
-// unresolved so the caller can apply tag-overlap fallback.
-func (s *Selector) Resolve(signals []string) (techIDs, unresolved []string) {
-	seenTech := map[string]struct{}{}
-	seenUnres := map[string]struct{}{}
-	for _, sig := range signals {
-		tok := normToken(sig)
-		if tok == "" || genericTokens[tok] {
-			continue
-		}
-		if id, ok := s.byAlias[tok]; ok {
-			if _, dup := seenTech[id]; !dup {
-				seenTech[id] = struct{}{}
-				techIDs = append(techIDs, id)
-			}
-			continue
-		}
-		if _, dup := seenUnres[tok]; !dup {
-			seenUnres[tok] = struct{}{}
-			unresolved = append(unresolved, tok)
-		}
-	}
-	sort.Strings(techIDs)
-	sort.Strings(unresolved)
-	return techIDs, unresolved
-}
-
-// FireSet returns the precise template ids to execute for the given techs at the
-// requested coverage tier (dependents + tier-gated baseline).
-func (s *Selector) FireSet(tier Tier, techIDs []string) []string {
-	ids := map[string]struct{}{}
-	for _, id := range techIDs {
-		if t, ok := s.g.Techs[id]; ok {
-			for _, r := range t.Templates {
-				ids[r.ID] = struct{}{}
-			}
-		}
-	}
-	for _, b := range s.g.Baseline {
-		if tierIncludesBaseline(tier, b.Tier) {
-			ids[b.ID] = struct{}{}
-		}
-	}
-	out := make([]string, 0, len(ids))
-	for id := range ids {
-		out = append(out, id)
-	}
-	sort.Strings(out)
-	return out
-}
-
-// BaselineIDs returns just the tier-gated baseline ids.
-func (s *Selector) BaselineIDs(tier Tier) []string {
-	var out []string
-	for _, b := range s.g.Baseline {
-		if tierIncludesBaseline(tier, b.Tier) {
-			out = append(out, b.ID)
-		}
-	}
-	sort.Strings(out)
-	return out
-}
-
-// tierIncludesBaseline gates baseline membership by coverage tier:
-//   - lean: no baseline (dependents only)
-//   - balanced: balanced baseline only
-//   - thorough: balanced + thorough baseline
-func tierIncludesBaseline(scan, entry Tier) bool {
-	switch scan {
-	case TierLean:
-		return false
-	case TierThorough:
-		return true
-	default: // balanced
-		return entry == TierBalanced
-	}
-}
-
-// NormalizeTier coerces arbitrary input to a valid tier, defaulting to balanced.
-// Case and surrounding whitespace are ignored.
-func NormalizeTier(s string) Tier {
-	tier, err := ParseTier(s)
-	if err != nil {
-		return TierBalanced
-	}
-	return tier
-}
-
-// ParseTier validates a coverage tier. Empty input defaults to balanced.
-// Unknown values return an error instead of silently changing behavior.
-func ParseTier(s string) (Tier, error) {
-	switch Tier(strings.ToLower(strings.TrimSpace(s))) {
-	case "", TierBalanced:
-		return TierBalanced, nil
-	case TierLean:
-		return TierLean, nil
-	case TierThorough:
-		return TierThorough, nil
-	default:
-		return "", fmt.Errorf("invalid automatic scan coverage %q (allowed: lean, balanced, thorough)", s)
-	}
-}
-
-// Candidate is a loaded template that the artifact may not know about yet.
-type Candidate struct {
-	ID       string
-	Tags     []string
-	Category string
-}
-
-// AbsentDependents returns ids of templates the graph does not contain whose
-// tags resolve to exactly one of the detected techs. Ambiguous tags and
-// excluded categories are left out, so this only recovers templates added
-// after the artifact was generated.
-func (s *Selector) AbsentDependents(detected []string, candidates []Candidate) []string {
-	if s == nil || s.g == nil || len(detected) == 0 || len(candidates) == 0 {
-		return nil
-	}
-	want := make(map[string]struct{}, len(detected))
-	for _, id := range detected {
-		want[id] = struct{}{}
-	}
-	known := s.knownIDs()
-	var out []string
-	for _, c := range candidates {
-		if c.ID == "" {
-			continue
-		}
-		if _, ok := known[c.ID]; ok {
-			continue
-		}
-		// An unknown category is skipped: excluded categories such as fuzzing
-		// and cloud must not be pulled in just because the path did not parse.
-		if _, excluded := excludedCategories[c.Category]; excluded || c.Category == "" {
-			continue
-		}
-		matches := map[string]struct{}{}
-		for _, tag := range c.Tags {
-			tok := normToken(tag)
-			if len(tok) < 5 || genericTokens[tok] {
-				continue
-			}
-			id, ok := s.byAlias[tok]
-			if !ok {
-				continue
-			}
-			matches[id] = struct{}{}
-		}
-		if len(matches) != 1 {
-			continue
-		}
-		var only string
-		for id := range matches {
-			only = id
-		}
-		if _, ok := want[only]; !ok {
-			continue
-		}
-		out = append(out, c.ID)
-	}
-	sort.Strings(out)
-	return out
-}
-
-// knownIDs is every template id the artifact already classifies.
-func (s *Selector) knownIDs() map[string]struct{} {
-	ids := make(map[string]struct{})
-	for _, tech := range s.g.Techs {
-		for _, ref := range tech.Templates {
-			ids[ref.ID] = struct{}{}
-		}
-	}
-	for _, b := range s.g.Baseline {
-		ids[b.ID] = struct{}{}
-	}
-	for _, d := range s.g.Detection {
-		ids[d.ID] = struct{}{}
-	}
-	return ids
-}

pkg/protocols/common/automaticscan/techgraph/selector_test.go

@@ -1,91 +0,0 @@
-package techgraph
-
-import (
-	"reflect"
-	"testing"
-)
-
-func testGraph() *Graph {
-	return &Graph{
-		Techs: map[string]*Tech{
-			"wordpress": {ID: "wordpress", Product: "wordpress", Aliases: []string{"wordpress"},
-				Templates: []TemplateRef{{ID: "wp-cve-1"}, {ID: "wp-plugin-1", Source: "platform"}}},
-			"atlassian:confluence": {ID: "atlassian:confluence", Product: "confluence", Aliases: []string{"confluence"},
-				Templates: []TemplateRef{{ID: "confluence-cve-1"}}},
-		},
-		Baseline: []BaselineEntry{
-			{ID: "git-config", Tier: TierBalanced},
-			{ID: "noisy-osinty", Tier: TierThorough},
-		},
-		Synonyms: map[string]string{"aem": "adobe:experience_manager"},
-	}
-}
-
-func TestResolve(t *testing.T) {
-	s := NewSelector(testGraph())
-	tech, unres := s.Resolve([]string{"WordPress", "confluence", "php", "cve"})
-	if len(tech) != 2 {
-		t.Fatalf("want 2 techs, got %v", tech)
-	}
-	if len(unres) != 1 || unres[0] != "php" {
-		t.Fatalf("want [php] unresolved, got %v", unres)
-	}
-}
-
-func TestFireSetTiers(t *testing.T) {
-	s := NewSelector(testGraph())
-	techs := []string{"wordpress"}
-
-	lean := s.FireSet(TierLean, techs)
-	if want := []string{"wp-cve-1", "wp-plugin-1"}; !reflect.DeepEqual(lean, want) {
-		t.Fatalf("lean: want %v, got %v", want, lean)
-	}
-	bal := s.FireSet(TierBalanced, techs)
-	if want := []string{"git-config", "wp-cve-1", "wp-plugin-1"}; !reflect.DeepEqual(bal, want) {
-		t.Fatalf("balanced: want %v, got %v", want, bal)
-	}
-	tho := s.FireSet(TierThorough, techs)
-	if want := []string{"git-config", "noisy-osinty", "wp-cve-1", "wp-plugin-1"}; !reflect.DeepEqual(tho, want) {
-		t.Fatalf("thorough: want %v, got %v", want, tho)
-	}
-}
-
-func TestNormalizeTier(t *testing.T) {
-	for in, want := range map[string]Tier{"lean": TierLean, " THOROUGH ": TierThorough, "": TierBalanced, "x": TierBalanced} {
-		if got := NormalizeTier(in); got != want {
-			t.Fatalf("NormalizeTier(%q)=%q want %q", in, got, want)
-		}
-	}
-}
-
-func TestParseTier(t *testing.T) {
-	if _, err := ParseTier("not-a-tier"); err == nil {
-		t.Fatal("expected error for unknown tier")
-	}
-	got, err := ParseTier(" Lean ")
-	if err != nil || got != TierLean {
-		t.Fatalf("ParseTier(Lean)=%q %v", got, err)
-	}
-}
-
-func TestAbsentDependents(t *testing.T) {
-	s := NewSelector(testGraph())
-	candidates := []Candidate{
-		{ID: "new-wp", Tags: []string{"wordpress"}, Category: "cves"},
-		{ID: "ambiguous", Tags: []string{"wordpress", "confluence"}, Category: "cves"},
-		{ID: "fuzzed", Tags: []string{"wordpress"}, Category: "fuzzing"},
-		{ID: "wp-cve-1", Tags: []string{"wordpress"}, Category: "cves"},
-		{ID: "short", Tags: []string{"wp"}, Category: "cves"},
-		{ID: "other-tech", Tags: []string{"wordpress"}, Category: "cves"},
-	}
-	got := s.AbsentDependents([]string{"wordpress"}, candidates[:5])
-	if want := []string{"new-wp"}; !reflect.DeepEqual(got, want) {
-		t.Fatalf("want %v, got %v", want, got)
-	}
-	if got := s.AbsentDependents([]string{"atlassian:confluence"}, candidates[5:]); len(got) != 0 {
-		t.Fatalf("wordpress tag must not attach to confluence, got %v", got)
-	}
-	if got := s.AbsentDependents([]string{"wordpress"}, []Candidate{{ID: "no-path", Tags: []string{"wordpress"}}}); len(got) != 0 {
-		t.Fatalf("unknown category must not be added, got %v", got)
-	}
-}

pkg/protocols/common/automaticscan/techgraph/types.go

@@ -1,84 +0,0 @@
-// Package techgraph builds the offline technology->template mapping artifact
-// (tech-graph.json) consumed by automatic scan v2. See AUTOMATIC_SCAN_V2.md.
-package techgraph
-
-// ArtifactVersion is the schema version of the emitted artifact.
-const ArtifactVersion = 1
-
-// Tier gates how aggressively a template is selected at runtime.
-type Tier string
-
-const (
-	TierLean     Tier = "lean"
-	TierBalanced Tier = "balanced"
-	TierThorough Tier = "thorough"
-)
-
-// Graph is the top-level tech-graph artifact.
-type Graph struct {
-	Version     int               `json:"version"`
-	GeneratedAt string            `json:"generated_at"`
-	SourceHash  string            `json:"source_hash,omitempty"`
-	Techs       map[string]*Tech  `json:"techs"`
-	Baseline    []BaselineEntry   `json:"baseline"`
-	Detection   []DetectionEntry  `json:"detection"`
-	Synonyms    map[string]string `json:"synonyms"`
-	Stats       Stats             `json:"stats"`
-}
-
-// DetectionEntry is a phase-1 detector template (tagged tech/detect/favicon).
-// Running these fingerprints the target before selection.
-type DetectionEntry struct {
-	ID   string `json:"id"`
-	Tech string `json:"tech,omitempty"` // canonical tech this detector anchors, if any
-}
-
-// Tech is a single canonical technology and the templates that depend on it.
-type Tech struct {
-	ID        string        `json:"id"`
-	Vendor    string        `json:"vendor,omitempty"`
-	Product   string        `json:"product,omitempty"`
-	CPE       string        `json:"cpe,omitempty"`
-	Aliases   []string      `json:"aliases,omitempty"`
-	Templates []TemplateRef `json:"templates"`
-}
-
-// TemplateRef references a template that depends on a tech.
-type TemplateRef struct {
-	ID       string `json:"id"`
-	Category string `json:"category,omitempty"`
-	Severity string `json:"severity,omitempty"`
-	// Product is the original sub-product when attached to a platform node via
-	// platform grouping (e.g. plugin "wp-optimize" under platform "wordpress").
-	Product string `json:"product,omitempty"`
-	// Source records how the template was attached: "cpe", "platform", "tag".
-	Source string `json:"source,omitempty"`
-	// VersionRange is reserved for the AI version-parsing milestone. Empty means
-	// "fire regardless of detected version".
-	VersionRange string `json:"version_range,omitempty"`
-}
-
-// BaselineEntry is a tech-agnostic template that runs on every applicable target.
-type BaselineEntry struct {
-	ID       string `json:"id"`
-	Category string `json:"category,omitempty"`
-	Severity string `json:"severity,omitempty"`
-	Tier     Tier   `json:"tier"`
-}
-
-// Stats summarises a build run.
-type Stats struct {
-	Total          int `json:"total"`
-	Dependents     int `json:"dependents"`
-	DependentsCPE  int `json:"dependents_cpe"`
-	DependentsPlat int `json:"dependents_platform"`
-	DependentsDir  int `json:"dependents_dir"`
-	DependentsTag  int `json:"dependents_tag"`
-	DependentsID   int `json:"dependents_id"`
-	Baseline       int `json:"baseline"`
-	Detection      int `json:"detection"`
-	Excluded       int `json:"excluded"`
-	Unmapped       int `json:"unmapped"`
-	Techs          int `json:"techs"`
-	ParseError     int `json:"parse_error"`
-}

pkg/protocols/common/automaticscan/techgraph/visualizer.html

@@ -1,225 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-<meta charset="utf-8" />
-<meta name="viewport" content="width=device-width, initial-scale=1" />
-<title>nuclei tech-graph visualizer</title>
-<style>
-  :root {
-    --bg: #0c0f17; --panel: #141926; --panel2: #1b2233; --line: #263149;
-    --txt: #e6edf6; --muted: #8b97ad; --accent: #5b8cff;
-    --cpe:#36c98a; --platform:#b07cff; --dir:#3bc4c4; --tag:#5b8cff; --id:#f0a23c;
-    --crit:#ff5470; --high:#ff8a3d; --med:#f5c451; --low:#5b8cff; --info:#7d8aa3;
-  }
-  * { box-sizing: border-box; }
-  body { margin:0; background:var(--bg); color:var(--txt);
-    font:14px/1.5 ui-sans-serif,system-ui,-apple-system,"Segoe UI",Roboto,sans-serif; }
-  header { padding:18px 24px; border-bottom:1px solid var(--line);
-    display:flex; align-items:center; gap:16px; flex-wrap:wrap;
-    background:linear-gradient(180deg,#10141f,#0c0f17); position:sticky; top:0; z-index:5; }
-  h1 { font-size:16px; margin:0; font-weight:600; letter-spacing:.3px; }
-  h1 span { color:var(--accent); }
-  .meta { color:var(--muted); font-size:12px; }
-  .btn { background:var(--panel2); color:var(--txt); border:1px solid var(--line);
-    padding:7px 13px; border-radius:8px; cursor:pointer; font-size:13px; }
-  .btn:hover { border-color:var(--accent); }
-  main { padding:20px 24px; max-width:1240px; margin:0 auto; }
-  .cards { display:grid; grid-template-columns:repeat(auto-fit,minmax(140px,1fr)); gap:12px; margin-bottom:18px; }
-  .card { background:var(--panel); border:1px solid var(--line); border-radius:12px; padding:14px 16px; }
-  .card .n { font-size:24px; font-weight:700; }
-  .card .l { color:var(--muted); font-size:12px; text-transform:uppercase; letter-spacing:.5px; }
-  .srcbar { display:flex; height:10px; border-radius:6px; overflow:hidden; margin:4px 0 16px; border:1px solid var(--line); }
-  .srcbar div { height:100%; }
-  .legend { display:flex; gap:14px; flex-wrap:wrap; font-size:12px; color:var(--muted); margin-bottom:18px; }
-  .legend i { display:inline-block; width:10px; height:10px; border-radius:3px; margin-right:5px; vertical-align:middle; }
-  .tabs { display:flex; gap:6px; border-bottom:1px solid var(--line); margin-bottom:16px; }
-  .tab { padding:9px 15px; cursor:pointer; color:var(--muted); border-bottom:2px solid transparent; }
-  .tab.active { color:var(--txt); border-bottom-color:var(--accent); }
-  .toolbar { display:flex; gap:10px; margin-bottom:14px; flex-wrap:wrap; align-items:center; }
-  input[type=search], select { background:var(--panel2); border:1px solid var(--line); color:var(--txt);
-    padding:8px 12px; border-radius:8px; font-size:13px; }
-  input[type=search] { flex:1; min-width:220px; }
-  .tech { background:var(--panel); border:1px solid var(--line); border-radius:10px; margin-bottom:8px; overflow:hidden; }
-  .tech > summary { padding:11px 15px; cursor:pointer; display:flex; align-items:center; gap:12px; list-style:none; }
-  .tech > summary::-webkit-details-marker { display:none; }
-  .tech .id { font-weight:600; }
-  .tech .cnt { margin-left:auto; color:var(--muted); font-size:12px; background:var(--panel2);
-    padding:2px 9px; border-radius:20px; }
-  .cpe { color:var(--muted); font-size:11px; font-family:ui-monospace,monospace; }
-  .tpls { border-top:1px solid var(--line); padding:6px 15px 12px; }
-  .row { display:flex; align-items:center; gap:8px; padding:5px 0; border-bottom:1px dashed #1e2740; font-size:13px; }
-  .row:last-child { border-bottom:none; }
-  .row .tid { font-family:ui-monospace,monospace; font-size:12px; }
-  .badge { font-size:10px; padding:1px 7px; border-radius:20px; text-transform:uppercase; letter-spacing:.4px; font-weight:600; }
-  .b-cpe{background:rgba(54,201,138,.16);color:var(--cpe)} .b-platform{background:rgba(176,124,255,.16);color:var(--platform)}
-  .b-dir{background:rgba(59,196,196,.16);color:var(--dir)} .b-tag{background:rgba(91,140,255,.16);color:var(--tag)}
-  .b-id{background:rgba(240,162,60,.16);color:var(--id)}
-  .sev { width:7px; height:7px; border-radius:50%; flex:none; }
-  .s-critical{background:var(--crit)} .s-high{background:var(--high)} .s-medium{background:var(--med)}
-  .s-low{background:var(--low)} .s-info{background:var(--info)} .s-unknown{background:#444}
-  .cat { color:var(--muted); font-size:11px; }
-  .drop { border:2px dashed var(--line); border-radius:14px; padding:60px; text-align:center; color:var(--muted); }
-  .drop.hot { border-color:var(--accent); color:var(--txt); }
-  .muted { color:var(--muted); }
-  .hidden { display:none; }
-  .alias { color:var(--muted); font-size:11px; }
-</style>
-</head>
-<body>
-<header>
-  <h1>nuclei <span>tech-graph</span> visualizer</h1>
-  <span class="meta" id="meta"></span>
-  <button class="btn" id="pick" style="margin-left:auto">Load tech-graph.json</button>
-  <input type="file" id="file" accept="application/json,.json" class="hidden" />
-</header>
-<main>
-  <div id="empty" class="drop">
-    Drop <b>tech-graph.json</b> here, or click <b>Load</b>.<br/>
-    <span class="muted">Generate it with <code>techgraph-gen -templates &lt;dir&gt; -output tech-graph.json</code></span>
-  </div>
-
-  <div id="app" class="hidden">
-    <div class="cards" id="cards"></div>
-    <div class="srcbar" id="srcbar"></div>
-    <div class="legend" id="legend"></div>
-
-    <div class="tabs">
-      <div class="tab active" data-tab="techs">Technologies</div>
-      <div class="tab" data-tab="baseline">Baseline</div>
-      <div class="tab" data-tab="detection">Detection</div>
-      <div class="tab" data-tab="synonyms">Synonyms</div>
-    </div>
-
-    <div class="toolbar">
-      <input type="search" id="q" placeholder="Search tech, template id, alias, cpe…" />
-      <select id="sort">
-        <option value="count">Sort: most templates</option>
-        <option value="name">Sort: name</option>
-      </select>
-      <select id="srcFilter">
-        <option value="">Source: all</option>
-        <option value="cpe">cpe</option><option value="platform">platform</option>
-        <option value="dir">dir</option><option value="tag">tag</option><option value="id">id</option>
-      </select>
-      <span class="muted" id="resultCount"></span>
-    </div>
-
-    <div id="view"></div>
-  </div>
-</main>
-
-<script>
-const SEV = ["critical","high","medium","low","info"];
-const SRC = ["cpe","platform","dir","tag","id"];
-const SRC_COLOR = {cpe:"--cpe",platform:"--platform",dir:"--dir",tag:"--tag",id:"--id"};
-let G = null;
-
-const $ = s => document.querySelector(s);
-const el = (t,c,h) => { const e=document.createElement(t); if(c)e.className=c; if(h!=null)e.innerHTML=h; return e; };
-const esc = s => String(s==null?"":s).replace(/[&<>]/g,m=>({"&":"&amp;","<":"&lt;",">":"&gt;"}[m]));
-
-function load(json){
-  G = json;
-  $("#empty").classList.add("hidden");
-  $("#app").classList.remove("hidden");
-  $("#meta").textContent = `v${json.version} · ${json.generated_at||""} · ${json.source_hash? json.source_hash.slice(0,18)+"…":""}`;
-  renderCards(); renderTabs(); render();
-}
-
-function renderCards(){
-  const s = G.stats || {};
-  const cards = [
-    ["templates", s.total], ["dependents", s.dependents], ["techs", s.techs],
-    ["baseline", s.baseline], ["detection", s.detection], ["excluded", s.excluded], ["unmapped", s.unmapped],
-  ];
-  $("#cards").innerHTML = cards.map(([l,n])=>`<div class="card"><div class="n">${n??0}</div><div class="l">${l}</div></div>`).join("");
-  // source bar from per-source dependent counts
-  const bySrc = {cpe:s.dependents_cpe,platform:s.dependents_platform,dir:s.dependents_dir,tag:s.dependents_tag,id:s.dependents_id};
-  const tot = SRC.reduce((a,k)=>a+(bySrc[k]||0),0)||1;
-  $("#srcbar").innerHTML = SRC.map(k=>`<div style="width:${100*(bySrc[k]||0)/tot}%;background:var(${SRC_COLOR[k]})" title="${k}: ${bySrc[k]||0}"></div>`).join("");
-  $("#legend").innerHTML = SRC.map(k=>`<span><i style="background:var(${SRC_COLOR[k]})"></i>${k} ${bySrc[k]||0}</span>`).join("")
-    + ` <span style="margin-left:auto" class="muted">dependent attachment source</span>`;
-}
-
-let TAB = "techs";
-function renderTabs(){
-  document.querySelectorAll(".tab").forEach(t=>{
-    t.onclick = ()=>{ TAB=t.dataset.tab; document.querySelectorAll(".tab").forEach(x=>x.classList.toggle("active",x===t)); render(); };
-  });
-}
-["#q","#sort","#srcFilter"].forEach(s=>document.addEventListener("input",e=>{ if(e.target.matches(s)) render(); }));
-
-function render(){
-  const q = $("#q").value.trim().toLowerCase();
-  const v = $("#view"); v.innerHTML="";
-  if(TAB==="techs") return renderTechs(q,v);
-  if(TAB==="baseline") return renderList(G.baseline||[], q, v, b=>[b.id,b.category], b=>rowHTML(b.id,null,b.category,b.severity,b.tier));
-  if(TAB==="detection") return renderList(G.detection||[], q, v, d=>[d.id,d.tech], d=>rowHTML(d.id,null,"detector → "+(d.tech||"—")));
-  if(TAB==="synonyms"){
-    const ents=Object.entries(G.synonyms||{}).filter(([a,t])=>!q||a.includes(q)||t.includes(q)).sort();
-    $("#resultCount").textContent = ents.length+" synonyms";
-    ents.forEach(([a,t])=>{ const r=el("div","row"); r.innerHTML=`<span class="tid">${esc(a)}</span><span class="muted">→</span><span class="badge b-tag">${esc(t)}</span>`; v.appendChild(r); });
-  }
-}
-
-function renderTechs(q,v){
-  const srcF = $("#srcFilter").value;
-  let techs = Object.values(G.techs||{});
-  techs = techs.filter(t=>{
-    const tpls = t.templates||[];
-    if(srcF && !tpls.some(x=>x.source===srcF)) return false;
-    if(!q) return true;
-    if(t.id.toLowerCase().includes(q) || (t.cpe||"").toLowerCase().includes(q)) return true;
-    if((t.aliases||[]).some(a=>a.includes(q))) return true;
-    return tpls.some(x=>x.id.toLowerCase().includes(q));
-  });
-  techs.sort($("#sort").value==="name"
-    ? (a,b)=>a.id.localeCompare(b.id)
-    : (a,b)=>(b.templates||[]).length-(a.templates||[]).length);
-  $("#resultCount").textContent = `${techs.length} techs · ${techs.reduce((n,t)=>n+(t.templates||[]).length,0)} templates`;
-  const frag=document.createDocumentFragment();
-  for(const t of techs.slice(0,400)){
-    const d=el("details","tech");
-    const tpls=(t.templates||[]);
-    d.innerHTML=`<summary><span class="id">${esc(t.id)}</span>`
-      +(t.cpe?`<span class="cpe">${esc(t.cpe)}</span>`:"")
-      +(t.aliases&&t.aliases.length?`<span class="alias">${esc(t.aliases.slice(0,4).join(", "))}</span>`:"")
-      +`<span class="cnt">${tpls.length}</span></summary>`;
-    const body=el("div","tpls");
-    let list=tpls; if(srcF) list=list.filter(x=>x.source===srcF);
-    if(q) list=list.filter(x=>x.id.toLowerCase().includes(q)||t.id.toLowerCase().includes(q)||(t.aliases||[]).some(a=>a.includes(q)));
-    body.innerHTML = list.slice(0,300).map(x=>rowHTML(x.id,x.source,x.category,x.severity)).join("");
-    d.appendChild(body); frag.appendChild(d);
-  }
-  v.appendChild(frag);
-  if(techs.length>400) v.appendChild(el("div","muted",`Showing first 400 of ${techs.length}. Refine your search.`));
-}
-
-function renderList(items,q,v,keys,rowFn){
-  const f=items.filter(it=>!q||keys(it).some(k=>String(k||"").toLowerCase().includes(q)));
-  $("#resultCount").textContent = f.length+" items";
-  v.innerHTML = `<div class="tech"><div class="tpls">`+f.slice(0,1000).map(rowFn).join("")+`</div></div>`;
-}
-
-function rowHTML(id,source,category,severity,extra){
-  const sev=(severity||"unknown").toLowerCase();
-  return `<div class="row"><span class="sev s-${SEV.includes(sev)?sev:"unknown"}"></span>`
-    + `<span class="tid">${esc(id)}</span>`
-    + (source?`<span class="badge b-${source}">${esc(source)}</span>`:"")
-    + (category?`<span class="cat">${esc(category)}</span>`:"")
-    + (extra?`<span class="cat">· ${esc(extra)}</span>`:"")
-    + `</div>`;
-}
-
-// ---- loading: file picker, drag/drop, auto-fetch ----
-$("#pick").onclick=()=>$("#file").click();
-$("#file").onchange=e=>{ const f=e.target.files[0]; if(f) f.text().then(t=>load(JSON.parse(t))); };
-const dz=$("#empty");
-["dragover","dragenter"].forEach(ev=>dz.addEventListener(ev,e=>{e.preventDefault();dz.classList.add("hot");}));
-["dragleave","drop"].forEach(ev=>dz.addEventListener(ev,e=>{e.preventDefault();dz.classList.remove("hot");}));
-dz.addEventListener("drop",e=>{ const f=e.dataTransfer.files[0]; if(f) f.text().then(t=>load(JSON.parse(t))); });
-if(window.__TECHGRAPH__){ load(window.__TECHGRAPH__); }
-else { fetch("tech-graph.json").then(r=>r.ok?r.json():Promise.reject()).then(load).catch(()=>{}); }
-</script>
-</body>
-</html>

pkg/protocols/http/body_budget.go

@@ -0,0 +1,103 @@
+package http
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
+)
+
+// bodySizeEstimate is the largest request body a template has built so far.
+type bodySizeEstimate struct {
+	known    atomic.Bool
+	max      atomic.Int64
+	learning sync.Mutex
+}
+
+type bodyReservation struct {
+	budget  *httpbody.Budget
+	size    int64
+	release func()
+	unlock  func()
+}
+
+func noop() {}
+
+// reserveBody reserves the template's largest body size before the body is
+// built. Until a size is known, builds run one at a time.
+func (request *Request) reserveBody(ctx context.Context) (*bodyReservation, error) {
+	r := &bodyReservation{budget: request.options.RequestBodyBudget, release: noop, unlock: noop}
+	estimate := request.bodySize
+	if r.budget == nil || estimate == nil {
+		return r, nil
+	}
+	if !estimate.known.Load() {
+		estimate.learning.Lock()
+		if !estimate.known.Load() {
+			r.unlock = estimate.learning.Unlock
+			return r, nil
+		}
+		estimate.learning.Unlock()
+	}
+	r.size = estimate.max.Load()
+	release, err := r.budget.Acquire(ctx, r.size)
+	if err != nil {
+		return nil, err
+	}
+	r.release = release
+	return r, nil
+}
+
+// makeReserved builds the next request inside a body reservation, waiting
+// for budget until waitCtx ends. On success the caller owns the reservation
+// and calls done once the send ends.
+func (request *Request) makeReserved(waitCtx context.Context, generator *requestGenerator, input *contextargs.Context, data string, payloads, dynamicValues map[string]interface{}) (*generatedRequest, *bodyReservation, error) {
+	reservation, err := request.reserveBody(waitCtx)
+	if err != nil {
+		return nil, nil, err
+	}
+	generated, err := generator.Make(input.Context(), input, data, payloads, dynamicValues)
+	if err == nil {
+		err = reservation.built(waitCtx, request, generated.bodyLen())
+	}
+	if err != nil {
+		reservation.done()
+		return nil, nil, err
+	}
+	generated.bodyReserved = true
+	return generated, reservation, nil
+}
+
+// built grows the reservation to the body's real size. It releases before
+// acquiring again: waiting while holding part of the budget can deadlock.
+func (r *bodyReservation) built(ctx context.Context, request *Request, size int64) error {
+	if estimate := request.bodySize; estimate != nil {
+		for {
+			seen := estimate.max.Load()
+			if size <= seen || estimate.max.CompareAndSwap(seen, size) {
+				break
+			}
+		}
+		estimate.known.Store(true)
+	}
+	r.unlock()
+	r.unlock = noop
+	if size <= r.size {
+		return nil
+	}
+	r.release()
+	r.release = noop
+	release, err := r.budget.Acquire(ctx, size)
+	if err != nil {
+		return err
+	}
+	r.release, r.size = release, size
+	return nil
+}
+
+func (r *bodyReservation) done() {
+	r.unlock()
+	r.release()
+}

pkg/protocols/http/body_budget_memory_test.go

@@ -0,0 +1,125 @@
+//go:build !race
+
+package http
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"runtime"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
+	"github.com/projectdiscovery/nuclei/v3/pkg/model"
+	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
+	"github.com/projectdiscovery/nuclei/v3/pkg/output"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
+)
+
+// Threads and pipeline build bodies ahead of the send. Built bodies must stay
+// within the budget while the server holds the first request.
+func TestRequestBodyBudgetBoundsBodiesBuiltAhead(t *testing.T) {
+	const bodySize = 8 << 20
+	const payloads = 12
+
+	for _, mode := range []string{"threads", "pipeline"} {
+		t.Run(mode, func(t *testing.T) {
+			if mode == "pipeline" {
+				t.Skip("needs projectdiscovery/rawhttp#551: the pipeline client hangs after 10s idle")
+			}
+			arrived := make(chan struct{}, payloads)
+			unblock := make(chan struct{})
+			var received atomic.Int32
+			// net/http drops pipelined connections with large bodies, so serve
+			// Content-Length framed requests in order by hand.
+			listener, err := net.Listen("tcp", "127.0.0.1:0")
+			require.NoError(t, err)
+			defer func() { _ = listener.Close() }()
+			go func() {
+				for {
+					conn, err := listener.Accept()
+					if err != nil {
+						return
+					}
+					go func() {
+						defer func() { _ = conn.Close() }()
+						reader := bufio.NewReader(conn)
+						for {
+							req, err := http.ReadRequest(reader)
+							if err != nil {
+								return
+							}
+							_, _ = io.Copy(io.Discard, req.Body)
+							received.Add(1)
+							arrived <- struct{}{}
+							<-unblock
+							if _, err := io.WriteString(conn, "HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok"); err != nil {
+								return
+							}
+						}
+					}()
+				}
+			}()
+			target := "http://" + listener.Addr().String()
+
+			options := testutils.DefaultOptions.Copy()
+			testutils.Init(options)
+
+			values := make([]string, payloads)
+			for i := range values {
+				values[i] = fmt.Sprint(i)
+			}
+			request := &Request{
+				ID:       "body-budget-" + mode,
+				Payloads: map[string]interface{}{"v": values},
+			}
+			body := `{{repeat("A", 8388608)}}`
+			if mode == "threads" {
+				request.Threads = 8
+				request.Path = []string{"{{BaseURL}}/?v={{v}}"}
+				request.Method = HTTPMethodTypeHolder{MethodType: HTTPPost}
+				request.Body = body
+			} else {
+				request.Pipeline = true
+				request.PipelineConcurrentConnections = 1
+				request.PipelineRequestsPerConnection = payloads
+				request.Raw = []string{"POST /?v={{v}} HTTP/1.1\nHost: {{Hostname}}\nContent-Length: 8388608\n\n" + body}
+			}
+			executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+				ID:   request.ID,
+				Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
+			})
+			executerOpts.RequestBodyBudget = httpbody.NewBudget(bodySize)
+			require.NoError(t, request.Compile(executerOpts))
+
+			var before runtime.MemStats
+			runtime.GC()
+			runtime.ReadMemStats(&before)
+
+			done := make(chan error, 1)
+			go func() {
+				done <- request.ExecuteWithResults(contextargs.NewWithInput(context.Background(), target), make(output.InternalEvent), make(output.InternalEvent), func(*output.InternalWrappedEvent) {})
+			}()
+			<-arrived
+			time.Sleep(300 * time.Millisecond)
+			var held runtime.MemStats
+			runtime.GC()
+			runtime.ReadMemStats(&held)
+			close(unblock)
+			require.NoError(t, <-done)
+
+			grown := int64(held.HeapAlloc) - int64(before.HeapAlloc)
+			t.Logf("heap held: %d MB", grown>>20)
+			require.Less(t, grown, int64(4*bodySize), "bodies built ahead of the send must stay within the budget")
+			require.EqualValues(t, payloads, received.Load())
+		})
+	}
+}

pkg/protocols/http/body_budget_test.go

@@ -0,0 +1,200 @@
+package http
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/projectdiscovery/nuclei/v3/internal/tests/testutils"
+	"github.com/projectdiscovery/nuclei/v3/pkg/model"
+	"github.com/projectdiscovery/nuclei/v3/pkg/model/types/severity"
+	"github.com/projectdiscovery/nuclei/v3/pkg/operators"
+	"github.com/projectdiscovery/nuclei/v3/pkg/operators/matchers"
+	"github.com/projectdiscovery/nuclei/v3/pkg/output"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols"
+	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/contextargs"
+	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
+)
+
+// The third request queues ~2.4s, so it only beats the 3s timeout if the
+// timeout starts after admission.
+func TestRequestBodyBudgetQueuesHeavyRequests(t *testing.T) {
+	const bodySize = 1 << 20
+	const requests = 3
+
+	var inFlight, maxInFlight atomic.Int32
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		n := inFlight.Add(1)
+		defer inFlight.Add(-1)
+		for {
+			seen := maxInFlight.Load()
+			if n <= seen || maxInFlight.CompareAndSwap(seen, n) {
+				break
+			}
+		}
+		body, _ := io.ReadAll(r.Body)
+		time.Sleep(1200 * time.Millisecond)
+		if len(body) == bodySize {
+			_, _ = io.WriteString(w, "full body")
+		}
+	}))
+	defer ts.Close()
+
+	options := testutils.DefaultOptions.Copy()
+	options.SetTimeouts(&types.Timeouts{DialTimeout: time.Second, HttpTimeout: 3 * time.Second})
+	testutils.Init(options)
+
+	request := &Request{
+		ID:     "body-budget",
+		Path:   []string{"{{BaseURL}}"},
+		Method: HTTPMethodTypeHolder{MethodType: HTTPPost},
+		Body:   `{{repeat("A", 1048576)}}`,
+		Operators: operators.Operators{
+			Matchers: []*matchers.Matcher{{
+				Part:  "body",
+				Type:  matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
+				Words: []string{"full body"},
+			}},
+		},
+	}
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   request.ID,
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
+	})
+	executerOpts.RequestBodyBudget = httpbody.NewBudget(bodySize)
+	require.NoError(t, request.Compile(executerOpts))
+
+	var matched atomic.Int32
+	var wg sync.WaitGroup
+	for range requests {
+		wg.Go(func() {
+			err := request.ExecuteWithResults(contextargs.NewWithInput(context.Background(), ts.URL), make(output.InternalEvent), make(output.InternalEvent), func(event *output.InternalWrappedEvent) {
+				if event.OperatorsResult != nil && event.OperatorsResult.Matched {
+					matched.Add(1)
+				}
+			})
+			require.NoError(t, err)
+		})
+	}
+	wg.Wait()
+
+	require.EqualValues(t, 1, maxInFlight.Load(), "budget must hold heavy requests to one in flight")
+	require.EqualValues(t, requests, matched.Load(), "queued requests must not time out while waiting")
+}
+
+func TestReserveBodyQueuesBuildsOnLearnedSize(t *testing.T) {
+	const bodySize = 4 << 20
+	request := &Request{
+		options:  &protocols.ExecutorOptions{RequestBodyBudget: httpbody.NewBudget(6 << 20)},
+		bodySize: &bodySizeEstimate{},
+	}
+
+	first, err := request.reserveBody(t.Context())
+	require.NoError(t, err)
+	blocked := make(chan struct{})
+	go func() {
+		second, err := request.reserveBody(t.Context())
+		require.NoError(t, err)
+		second.done()
+		close(blocked)
+	}()
+	select {
+	case <-blocked:
+		t.Fatal("second build started before the first recorded its size")
+	case <-time.After(20 * time.Millisecond):
+	}
+	require.NoError(t, first.built(t.Context(), request, bodySize))
+	first.done()
+	<-blocked
+
+	held, err := request.reserveBody(t.Context())
+	require.NoError(t, err)
+	ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
+	defer cancel()
+	_, err = request.reserveBody(ctx)
+	require.ErrorIs(t, err, context.DeadlineExceeded)
+
+	held.done()
+	next, err := request.reserveBody(t.Context())
+	require.NoError(t, err)
+	next.done()
+}
+
+func TestReserveBodyGrowsToLargerBody(t *testing.T) {
+	request := &Request{
+		options:  &protocols.ExecutorOptions{RequestBodyBudget: httpbody.NewBudget(6 << 20)},
+		bodySize: &bodySizeEstimate{},
+	}
+	learn, err := request.reserveBody(t.Context())
+	require.NoError(t, err)
+	require.NoError(t, learn.built(t.Context(), request, 1<<20))
+	learn.done()
+
+	grown, err := request.reserveBody(t.Context())
+	require.NoError(t, err)
+	require.NoError(t, grown.built(t.Context(), request, 5<<20))
+
+	ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
+	defer cancel()
+	_, err = request.reserveBody(ctx)
+	require.ErrorIs(t, err, context.DeadlineExceeded)
+	grown.done()
+}
+
+// Stop-at-first-match ends the workers with tasks still queued. Their
+// reservations must go back to the shared budget.
+func TestRequestBodyBudgetStopAtFirstMatchReleasesQueued(t *testing.T) {
+	const budget = 16 << 20
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = io.Copy(io.Discard, r.Body)
+		time.Sleep(100 * time.Millisecond)
+		_, _ = io.WriteString(w, "match")
+	}))
+	defer ts.Close()
+
+	options := testutils.DefaultOptions.Copy()
+	testutils.Init(options)
+
+	values := make([]string, 64)
+	for i := range values {
+		values[i] = fmt.Sprint(i)
+	}
+	request := &Request{
+		ID:               "body-budget-stop-first",
+		Method:           HTTPMethodTypeHolder{MethodType: HTTPPost},
+		Path:             []string{"{{BaseURL}}/?v={{v}}"},
+		Body:             `{{repeat("A", 1048576)}}`,
+		Threads:          4,
+		Payloads:         map[string]interface{}{"v": values},
+		StopAtFirstMatch: true,
+		Operators: operators.Operators{Matchers: []*matchers.Matcher{{
+			Part:  "body",
+			Type:  matchers.MatcherTypeHolder{MatcherType: matchers.WordsMatcher},
+			Words: []string{"match"},
+		}}},
+	}
+	executerOpts := testutils.NewMockExecuterOptions(options, &testutils.TemplateInfo{
+		ID:   request.ID,
+		Info: model.Info{SeverityHolder: severity.Holder{Severity: severity.Low}, Name: "test"},
+	})
+	executerOpts.RequestBodyBudget = httpbody.NewBudget(budget)
+	require.NoError(t, request.Compile(executerOpts))
+
+	err := request.ExecuteWithResults(contextargs.NewWithInput(context.Background(), ts.URL), make(output.InternalEvent), make(output.InternalEvent), func(*output.InternalWrappedEvent) {})
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithTimeout(t.Context(), time.Second)
+	defer cancel()
+	release, err := executerOpts.RequestBodyBudget.Acquire(ctx, budget)
+	require.NoError(t, err, "every reservation must be released")
+	release()
+}

pkg/protocols/http/build_request.go

@@ -82,6 +82,9 @@ type generatedRequest struct {
 	dynamicValues        map[string]interface{}
 	interactshURLs       []string
 	customCancelFunction context.CancelFunc
+	// sendTimeout starts after the body budget admits the request.
+	sendTimeout  time.Duration
+	bodyReserved bool
 	// requestURLPattern tracks unmodified request url pattern without values ( it is used for constant vuln_hash)
 	// ex: {{BaseURL}}/api/exp?param={{randstr}}
 	requestURLPattern string

pkg/protocols/http/http.go

@@ -147,6 +147,7 @@ type Request struct {
 	rawhttpClient     *rawhttp.Client
 	replayClient      *http.Client // set when -replay-proxy is given
 	dialer            *fastdialer.Dialer
+	bodySize          *bodySizeEstimate
 
 	// description: |
 	//   SelfContained specifies if the request is self-contained.
@@ -416,6 +417,7 @@ func (request *Request) Compile(options *protocols.ExecutorOptions) error {
 		return errors.Wrap(err, "could not get dialer")
 	}
 	request.dialer = dialer
+	request.bodySize = &bodySizeEstimate{}
 
 	request.options = options
 	for _, option := range request.options.Options.CustomHeaders {

pkg/protocols/http/operators.go

@@ -15,6 +15,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/helpers/responsehighlighter"
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/utils"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	convUtil "github.com/projectdiscovery/utils/conversion"
 )
 
 // Match matches a generic data response again a given matcher
@@ -225,7 +226,7 @@ func (request *Request) savedRequest(dump string) string {
 	if request.options == nil || request.options.Options == nil {
 		return dump
 	}
-	return string(capDumpBody([]byte(dump), request.options.Options.RequestSaveSize))
+	return string(capDumpBody(convUtil.Bytes(dump), request.options.Options.RequestSaveSize))
 }
 
 // eventResponse returns the full response of event. Events matched after the

pkg/protocols/http/request.go

@@ -331,6 +331,7 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 	// bounded worker-pool to avoid spawning one goroutine per payload
 	type task struct {
 		req                *generatedRequest
+		body               *bodyReservation
 		updatedInput       *contextargs.Context
 		hasInteractMarkers bool
 	}
@@ -338,72 +339,87 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 	var workersWg sync.WaitGroup
 	currentWorkers := maxWorkers
 	tasks := make(chan task, maxWorkers)
+	// Workers exit on cancel without draining, and the budget is shared, so
+	// release what they left queued.
+	stopWorkers := func() {
+		close(tasks)
+		workersWg.Wait()
+		for t := range tasks {
+			t.body.done()
+		}
+	}
 	spawnWorker := func(ctx context.Context) {
 		workersWg.Add(1)
 		go func() {
 			defer workersWg.Done()
 			for t := range tasks {
-				select {
-				case <-ctx.Done():
-					return
-				default:
-				}
-				if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(t.updatedInput) || spmHandler.Cancelled() {
-					continue
-				}
-				if err := spmHandler.Acquire(); err != nil {
-					continue
-				}
-				if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(t.updatedInput) || spmHandler.Cancelled() {
-					spmHandler.Release()
-					continue
-				}
-				// Full URL, so https and an explicit port stay on the same key the observation uses.
-				// The handler context ends the wait when stop-at-first-match fires.
-				waitCtx, cancelWait := stopWithHandler(ctx, spmHandler)
-				err := request.rateLimitTake(waitCtx, hostBackoffKey(t.req, t.updatedInput))
-				cancelWait()
-				if err != nil {
+				if func() (exit bool) {
+					defer t.body.done()
+					select {
+					case <-ctx.Done():
+						return true
+					default:
+					}
+					if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(t.updatedInput) || spmHandler.Cancelled() {
+						return false
+					}
+					if err := spmHandler.Acquire(); err != nil {
+						return false
+					}
+					if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(t.updatedInput) || spmHandler.Cancelled() {
+						spmHandler.Release()
+						return false
+					}
+					// Full URL, so https and an explicit port stay on the same key the observation uses.
+					// The handler context ends the wait when stop-at-first-match fires.
+					waitCtx, cancelWait := stopWithHandler(ctx, spmHandler)
+					err := request.rateLimitTake(waitCtx, hostBackoffKey(t.req, t.updatedInput))
+					cancelWait()
+					if err != nil {
+						select {
+						case <-spmHandler.Done():
+							spmHandler.Release()
+							return false
+						case <-ctx.Done():
+							spmHandler.Release()
+							return false
+						case spmHandler.ResultChan <- err:
+							spmHandler.Release()
+							return false
+						}
+					}
+					if ctx.Err() != nil || spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(t.updatedInput) || spmHandler.Cancelled() {
+						spmHandler.Release()
+						return false
+					}
+					hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
+					needsRequestEvent := hasInteractMatchers && request.NeedsRequestCondition()
+					err = request.executeRequest(t.updatedInput, t.req, make(map[string]interface{}), hasInteractMatchers, func(event *output.InternalWrappedEvent) {
+						if (t.hasInteractMarkers || needsRequestEvent) && request.options.Interactsh != nil {
+							requestData := &interactsh.RequestData{
+								MakeResultFunc: request.MakeResultEvent,
+								Event:          event,
+								Operators:      request.CompiledOperators,
+								MatchFunc:      request.Match,
+								ExtractFunc:    request.Extract,
+							}
+							allOASTUrls := httputils.GetInteractshURLSFromEvent(event.InternalEvent)
+							allOASTUrls = append(allOASTUrls, t.req.interactshURLs...)
+							request.options.RegisterInteractshRequest(sliceutil.Dedupe(allOASTUrls), requestData)
+						}
+						wrappedCallback(event)
+					}, 0)
 					select {
 					case <-spmHandler.Done():
 						spmHandler.Release()
-						continue
-					case <-ctx.Done():
-						spmHandler.Release()
-						continue
+						return false
 					case spmHandler.ResultChan <- err:
+						request.recordHostResultAndCancelIfUnresponsive(t.updatedInput, err, spmHandler.Cancel)
 						spmHandler.Release()
-						continue
 					}
-				}
-				if ctx.Err() != nil || spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(t.updatedInput) || spmHandler.Cancelled() {
-					spmHandler.Release()
-					continue
-				}
-				hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
-				needsRequestEvent := hasInteractMatchers && request.NeedsRequestCondition()
-				err = request.executeRequest(t.updatedInput, t.req, make(map[string]interface{}), hasInteractMatchers, func(event *output.InternalWrappedEvent) {
-					if (t.hasInteractMarkers || needsRequestEvent) && request.options.Interactsh != nil {
-						requestData := &interactsh.RequestData{
-							MakeResultFunc: request.MakeResultEvent,
-							Event:          event,
-							Operators:      request.CompiledOperators,
-							MatchFunc:      request.Match,
-							ExtractFunc:    request.Extract,
-						}
-						allOASTUrls := httputils.GetInteractshURLSFromEvent(event.InternalEvent)
-						allOASTUrls = append(allOASTUrls, t.req.interactshURLs...)
-						request.options.RegisterInteractshRequest(sliceutil.Dedupe(allOASTUrls), requestData)
-					}
-					wrappedCallback(event)
-				}, 0)
-				select {
-				case <-spmHandler.Done():
-					spmHandler.Release()
-					continue
-				case spmHandler.ResultChan <- err:
-					request.recordHostResultAndCancelIfUnresponsive(t.updatedInput, err, spmHandler.Cancel)
-					spmHandler.Release()
+					return false
+				}() {
+					return
 				}
 			}
 		}()
@@ -422,8 +438,7 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 
 		select {
 		case <-input.Context().Done():
-			close(tasks)
-			workersWg.Wait()
+			stopWorkers()
 			return input.Context().Err()
 		default:
 		}
@@ -431,8 +446,7 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 		// resize check point - nop if there are no changes
 		if shouldFollowGlobal && spmHandler.Size() != request.options.Options.PayloadConcurrency {
 			if err := spmHandler.Resize(input.Context(), request.options.Options.PayloadConcurrency); err != nil {
-				close(tasks)
-				workersWg.Wait()
+				stopWorkers()
 				return err
 			}
 			// if payload concurrency increased, add more workers
@@ -449,15 +463,22 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 			break
 		}
 
-		ctx := input.Context()
-		generatedHttpRequest, err := generator.Make(ctx, input, inputData, payloads, dynamicValues)
+		// Queued tasks hold budget and are only released once the workers
+		// stop, so the wait has to end with them.
+		generatedHttpRequest, reservation, err := request.makeReserved(workerCtx, generator, input, inputData, payloads, dynamicValues)
 		if err != nil {
 			if err == types.ErrNoMoreRequests {
 				break
 			}
+			if workerCtx.Err() != nil {
+				stopWorkers()
+				if spmHandler.FoundFirstMatch() {
+					return nil
+				}
+				return context.Cause(workerCtx)
+			}
 			request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
-			close(tasks)
-			workersWg.Wait()
+			stopWorkers()
 			return err
 		}
 		hasInteractMarkers := interactsh.HasMarkers(inputData) || len(generatedHttpRequest.interactshURLs) > 0
@@ -467,33 +488,32 @@ func (request *Request) executeParallelHTTP(input *contextargs.Context, dynamicV
 		updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL())
 		if request.isUnresponsiveAddress(updatedInput) {
 			// skip on unresponsive host no need to continue
+			reservation.done()
 			spmHandler.Cancel()
-			close(tasks)
-			workersWg.Wait()
+			stopWorkers()
 			return nil
 		}
 		select {
 		case <-spmHandler.Done():
-			close(tasks)
-			workersWg.Wait()
+			reservation.done()
+			stopWorkers()
 			spmHandler.Wait()
 			if spmHandler.FoundFirstMatch() {
 				return nil
 			}
 			return multierr.Combine(spmHandler.CombinedResults()...)
 		case <-workerCtx.Done():
-			close(tasks)
-			workersWg.Wait()
+			reservation.done()
+			stopWorkers()
 			if spmHandler.FoundFirstMatch() {
 				return nil
 			}
 			return context.Cause(workerCtx)
-		case tasks <- task{req: generatedHttpRequest, updatedInput: updatedInput, hasInteractMarkers: hasInteractMarkers}:
+		case tasks <- task{req: generatedHttpRequest, body: reservation, updatedInput: updatedInput, hasInteractMarkers: hasInteractMarkers}:
 		}
 		request.options.Progress.IncrementRequests()
 	}
-	close(tasks)
-	workersWg.Wait()
+	stopWorkers()
 	spmHandler.Wait()
 	if spmHandler.FoundFirstMatch() {
 		// ignore any context cancellation and in-transit execution errors
@@ -582,7 +602,7 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
 		}
 
 		ctx := input.Context()
-		generatedHttpRequest, err := generator.Make(ctx, input, inputData, payloads, dynamicValues)
+		generatedHttpRequest, reservation, err := request.makeReserved(ctx, generator, input, inputData, payloads, dynamicValues)
 		if err != nil {
 			request.options.Progress.IncrementFailedRequestsBy(int64(generator.Total()))
 			return err
@@ -593,15 +613,18 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
 		updatedInput := contextargs.GetCopyIfHostOutdated(input, generatedHttpRequest.URL())
 		if request.isUnresponsiveAddress(updatedInput) {
 			// skip on unresponsive host no need to continue
+			reservation.done()
 			spmHandler.Cancel()
 			return nil
 		}
 		generatedHttpRequest.pipelinedClient = pipeClient
 		if err := spmHandler.Acquire(); err != nil {
+			reservation.done()
 			break
 		}
-		go func(httpRequest *generatedRequest, requestInput *contextargs.Context) {
+		go func(httpRequest *generatedRequest, body *bodyReservation, requestInput *contextargs.Context) {
 			defer spmHandler.Release()
+			defer body.done()
 			if spmHandler.FoundFirstMatch() || request.isUnresponsiveAddress(requestInput) {
 				// skip if first match is found
 				return
@@ -626,7 +649,7 @@ func (request *Request) executeTurboHTTP(input *contextargs.Context, dynamicValu
 				request.recordHostResultAndCancelIfUnresponsive(requestInput, err, spmHandler.Cancel)
 				return
 			}
-		}(generatedHttpRequest, updatedInput)
+		}(generatedHttpRequest, reservation, updatedInput)
 		request.options.Progress.IncrementRequests()
 	}
 	spmHandler.Wait()
@@ -676,16 +699,17 @@ executionLoop:
 			hasInteractMatchers := interactsh.HasMatchers(request.CompiledOperators)
 
 			ctx := input.Context()
-			ctxWithTimeout, cancel := context.WithTimeoutCause(ctx, request.options.Options.GetTimeouts().HttpTimeout, ErrHttpEngineRequestDeadline)
-			defer cancel()
-
-			generatedHttpRequest, err := generator.Make(ctxWithTimeout, input, data, payloads, dynamicValue)
+			generatedHttpRequest, reservation, err := request.makeReserved(ctx, generator, input, data, payloads, dynamicValue)
 			if err != nil {
 				if err == types.ErrNoMoreRequests {
 					return false, true, nil, nil
 				}
 				return false, true, nil, err
 			}
+			defer reservation.done()
+			if generatedHttpRequest.customCancelFunction == nil {
+				generatedHttpRequest.sendTimeout = request.options.Options.GetTimeouts().HttpTimeout
+			}
 			// ideally if http template used a custom port or hostname
 			// we would want to update it in input but currently templateCtx logic
 			// is closely tied to contextargs.Context so we are temporarily creating
@@ -1060,6 +1084,20 @@ func (request *Request) executeRequest(input *contextargs.Context, generatedRequ
 		return fmt.Errorf("hostErrorsCache : host %s is unresponsive", input.MetaInput.Input)
 	}
 
+	// Race requests must all be in flight together to reach the open gate.
+	if !generatedRequest.original.Race && !generatedRequest.bodyReserved {
+		releaseBody, err := request.options.RequestBodyBudget.Acquire(input.Context(), generatedRequest.bodyLen())
+		if err != nil {
+			return err
+		}
+		defer releaseBody()
+	}
+	if generatedRequest.sendTimeout > 0 && generatedRequest.request != nil {
+		ctx, cancel := context.WithTimeoutCause(generatedRequest.request.Context(), generatedRequest.sendTimeout, ErrHttpEngineRequestDeadline)
+		defer cancel()
+		generatedRequest.request = generatedRequest.request.WithContext(ctx)
+	}
+
 	// wrap one more callback for validation and fixing event
 	callback := func(event *output.InternalWrappedEvent) {
 		// validateNFixEvent performs necessary validation on generated event

pkg/protocols/http/utils.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"slices"
 	"strings"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/protocols/common/generators"
@@ -82,6 +83,9 @@ func dumpRequest(req *retryablehttp.Request, maxBody int) (requestDump, error) {
 	}
 
 	capture := &cappedWriter{headerEnd: -1, maxBody: maxBody}
+	if maxBody <= 0 {
+		capture.buf = make([]byte, 0, len(body)+dumpHeaderRoom)
+	}
 	if err := clone.Write(capture); err != nil {
 		return requestDump{}, err
 	}
@@ -89,7 +93,7 @@ func dumpRequest(req *retryablehttp.Request, maxBody int) (requestDump, error) {
 	// http.Transport asks for gzip unless the caller set an encoding.
 	if clone.Header.Get("Accept-Encoding") == "" && clone.Header.Get("Range") == "" && clone.Method != http.MethodHead && capture.headerEnd >= 2 {
 		end := capture.headerEnd - 2
-		data = append(data[:end:end], append([]byte("Accept-Encoding: gzip\r\n"), data[end:]...)...)
+		data = slices.Insert(data, end, acceptEncodingGzip...)
 	}
 	return requestDump{data: appendTruncationMarker(data, capture.dropped), truncated: capture.dropped}, nil
 }
@@ -123,6 +127,10 @@ func capDumpBody(dump []byte, maxBody int) []byte {
 
 var headerTerminator = []byte("\r\n\r\n")
 
+var acceptEncodingGzip = []byte("Accept-Encoding: gzip\r\n")
+
+const dumpHeaderRoom = 8 * 1024
+
 // cappedWriter keeps the request line and headers in full and at most
 // maxBody bytes of the body that follows, counting the rest.
 type cappedWriter struct {
@@ -177,6 +185,19 @@ func getHTTPProjectCacheScope(requestDump requestDump, body []byte, scheme, host
 	return scoped.Bytes()
 }
 
+func (g *generatedRequest) bodyLen() int64 {
+	if g.rawRequest != nil {
+		return int64(len(g.rawRequest.Data))
+	}
+	if g.request == nil || g.request.Body == nil {
+		return 0
+	}
+	if body, ok := g.request.Body.(*httpbody.Body); ok {
+		return int64(body.Len())
+	}
+	return g.request.ContentLength
+}
+
 // body returns the request body without copying it.
 func (g *generatedRequest) body() []byte {
 	if g.request != nil {

pkg/protocols/protocols.go

@@ -44,6 +44,7 @@ import (
 	"github.com/projectdiscovery/nuclei/v3/pkg/scan"
 	templateTypes "github.com/projectdiscovery/nuclei/v3/pkg/templates/types"
 	"github.com/projectdiscovery/nuclei/v3/pkg/types"
+	"github.com/projectdiscovery/nuclei/v3/pkg/utils/httpbody"
 	unitutils "github.com/projectdiscovery/utils/unit"
 )
 
@@ -106,6 +107,8 @@ type ExecutorOptions struct {
 	Progress progress.Progress
 	// RateLimiter is a rate-limiter for limiting sent number of requests.
 	RateLimiter *ratelimit.Limiter
+	// RequestBodyBudget bounds request-body bytes in flight; nil is no limit.
+	RequestBodyBudget *httpbody.Budget
 	// Catalog is a template catalog implementation for nuclei
 	Catalog catalog.Catalog
 	// ProjectFile is the project file for nuclei
@@ -395,6 +398,7 @@ func (e *ExecutorOptions) Copy() *ExecutorOptions {
 		IssuesClient:                 e.IssuesClient,
 		Progress:                     e.Progress,
 		RateLimiter:                  e.RateLimiter,
+		RequestBodyBudget:            e.RequestBodyBudget,
 		Catalog:                      e.Catalog,
 		ProjectFile:                  e.ProjectFile,
 		Browser:                      e.Browser,
@@ -706,6 +710,7 @@ func (e *ExecutorOptions) ApplyNewEngineOptions(n *ExecutorOptions) {
 	e.IssuesClient = n.IssuesClient
 	e.Progress = n.Progress
 	e.RateLimiter = n.RateLimiter
+	e.RequestBodyBudget = n.RequestBodyBudget
 	e.Catalog = n.Catalog
 	e.ProjectFile = n.ProjectFile
 	e.Browser = n.Browser

pkg/types/types.go

@@ -247,9 +247,6 @@ type Options struct {
 	LeaveDefaultPorts bool
 	// AutomaticScan enables automatic tech based template execution
 	AutomaticScan bool
-	// AutomaticScanCoverage tunes automatic scan selection breadth:
-	// "lean" (dependents only), "balanced" (default), or "thorough".
-	AutomaticScanCoverage string
 	// Silent suppresses any extra text and only writes found URLs on screen.
 	Silent bool
 	// Validate validates the templates passed to nuclei.
@@ -373,6 +370,8 @@ type Options struct {
 	// RequestSaveSize is the maximum request body kept in saved output and
 	// debug dumps. Matchers see the full request. 0 keeps the whole body.
 	RequestSaveSize int
+	// MaxRequestBodyInFlight bounds request-body bytes in flight; 0 disables it.
+	MaxRequestBodyInFlight int
 	// Health Check
 	HealthCheck bool
 	// Time to wait between each input read operation before closing the stream
@@ -708,7 +707,6 @@ func (options *Options) Copy() *Options {
 		AutoCalibrate:                  options.AutoCalibrate,
 		LeaveDefaultPorts:              options.LeaveDefaultPorts,
 		AutomaticScan:                  options.AutomaticScan,
-		AutomaticScanCoverage:          options.AutomaticScanCoverage,
 		Silent:                         options.Silent,
 		Validate:                       options.Validate,
 		NoStrictSyntax:                 options.NoStrictSyntax,
@@ -769,6 +767,7 @@ func (options *Options) Copy() *Options {
 		ResponseReadSize:               options.ResponseReadSize,
 		ResponseSaveSize:               options.ResponseSaveSize,
 		RequestSaveSize:                options.RequestSaveSize,
+		MaxRequestBodyInFlight:         options.MaxRequestBodyInFlight,
 		HealthCheck:                    options.HealthCheck,
 		InputReadTimeout:               options.InputReadTimeout,
 		DisableStdin:                   options.DisableStdin,

pkg/utils/httpbody/budget.go

@@ -0,0 +1,38 @@
+package httpbody
+
+import (
+	"context"
+
+	"golang.org/x/sync/semaphore"
+)
+
+// MinBudgetedBody is the smallest body that waits on a Budget, so small
+// requests never queue behind a heavy one.
+const MinBudgetedBody = 1 << 20
+
+// Budget bounds the request-body bytes held by requests in flight.
+type Budget struct {
+	size int64
+	sem  *semaphore.Weighted
+}
+
+// NewBudget returns nil when size <= 0; a nil budget never blocks.
+func NewBudget(size int64) *Budget {
+	if size <= 0 {
+		return nil
+	}
+	return &Budget{size: size, sem: semaphore.NewWeighted(size)}
+}
+
+// Acquire blocks until n bytes fit or ctx is done. n larger than the budget
+// is clamped, so that request runs alone.
+func (b *Budget) Acquire(ctx context.Context, n int64) (release func(), err error) {
+	if b == nil || n < MinBudgetedBody {
+		return func() {}, nil
+	}
+	n = min(n, b.size)
+	if err := b.sem.Acquire(ctx, n); err != nil {
+		return nil, err
+	}
+	return func() { b.sem.Release(n) }, nil
+}

pkg/utils/httpbody/budget_test.go

@@ -0,0 +1,68 @@
+package httpbody
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestBudgetQueuesBodiesOverSize(t *testing.T) {
+	budget := NewBudget(48 << 20)
+
+	first, err := budget.Acquire(t.Context(), 20<<20)
+	require.NoError(t, err)
+	second, err := budget.Acquire(t.Context(), 20<<20)
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
+	defer cancel()
+	_, err = budget.Acquire(ctx, 20<<20)
+	require.ErrorIs(t, err, context.DeadlineExceeded)
+
+	first()
+	third, err := budget.Acquire(t.Context(), 20<<20)
+	require.NoError(t, err)
+	third()
+	second()
+}
+
+func TestBudgetRunsBodyLargerThanBudgetAlone(t *testing.T) {
+	budget := NewBudget(8 << 20)
+
+	huge, err := budget.Acquire(t.Context(), 64<<20)
+	require.NoError(t, err)
+
+	ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
+	defer cancel()
+	_, err = budget.Acquire(ctx, 2<<20)
+	require.ErrorIs(t, err, context.DeadlineExceeded)
+
+	huge()
+	next, err := budget.Acquire(t.Context(), 2<<20)
+	require.NoError(t, err)
+	next()
+}
+
+func TestBudgetSkipsSmallBodies(t *testing.T) {
+	budget := NewBudget(MinBudgetedBody)
+	full, err := budget.Acquire(t.Context(), MinBudgetedBody)
+	require.NoError(t, err)
+	defer full()
+
+	ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond)
+	defer cancel()
+	release, err := budget.Acquire(ctx, MinBudgetedBody-1)
+	require.NoError(t, err)
+	release()
+}
+
+func TestNilBudgetNeverBlocks(t *testing.T) {
+	require.Nil(t, NewBudget(0))
+
+	var budget *Budget
+	release, err := budget.Acquire(t.Context(), 1<<30)
+	require.NoError(t, err)
+	release()
+}