Compare commits

...
Author SHA1 Message Date
knakul853 507a4feaee only save what needed 2025-06-10 19:42:04 +05:30
Dogan Can Bakir a4859df5e9 Merge pull request #6243 from tongjicoder/dev
refactor: use slices.Contains to simplify code
2025-05-27 15:48:20 +03:00
Dogan Can Bakir 85c709ea22 Merge pull request #6245 from projectdiscovery/bump_dsl_pkg
bump dsl pkg
2025-05-27 15:44:39 +03:00
Doğan Can Bakır ec353f534c bump dsl pkg 2025-05-27 21:42:33 +09:00
tongjicoder 3be29abfc9 refactor: use slices.Contains to simplify code
Signed-off-by: tongjicoder <[email protected]>
2025-05-27 17:16:26 +08:00
Reynaldo Jarro 8a13639b62 fixing missing symbol (#6242) 2025-05-27 14:32:25 +05:30
Dogan Can Bakir 37fa0c69ec Merge pull request #6206 from 23kbps/dev
Fix ingress template in helm chart
2025-05-24 16:31:49 +03:00
Dogan Can Bakir 160eab998c Merge pull request #6222 from fourcube/fix/slow-headless-start-and-shutdown
fix: improve headless engine startup and shutdown
2025-05-19 16:42:38 +03:00
Dogan Can Bakir 9dce36a0c8 Merge pull request #6233 from projectdiscovery/dwisiswant0/ci/adds-stale-workflow
ci: adds stale workflow
2025-05-19 16:13:09 +03:00
Nakul BhartiandDwi Siswanto 242b1e1636 increase file descriptor limits (#6230)
* add missing file

* increase file descriptor limit

* removed debugging code

* fixed lower case

* test: tweaks on script

* uses CI runtime env vars (`RUNNER_OS` &
  `RUNNER_DEBUG`)
* restores originial `ulimit`

Signed-off-by: Dwi Siswanto <[email protected]>

---------

Signed-off-by: Dwi Siswanto <[email protected]>
Co-authored-by: Dwi Siswanto <[email protected]>
2025-05-18 20:09:41 +05:30
Dwi Siswanto 21d376f194 ci: adds stale workflow
Signed-off-by: Dwi Siswanto <[email protected]>
2025-05-18 19:46:14 +07:00
Chris Grieger bc551fc3f1 fix: improve headless engine startup and shutdown
Fixes #6221

Instead of enumerating all chrome processes to determine
which ones need to be killed on shutdown, use the launcher.Kill()
method to terminate the process that was launched for this
browser instance.
2025-05-14 16:14:21 +02:00
23kbps a7a009084c Fix ingress template in helm chart 2025-05-03 15:28:18 +07:00
23 changed files with 506 additions and 93 deletions

No files matched your search

-27
View File
@@ -1,27 +0,0 @@
# Number of days of inactivity before an issue becomes stale
daysUntilStale: 7
# Number of days of inactivity before a stale issue is closed
daysUntilClose: 7
# Issues with these labels will never be considered stale
# exemptLabels:
# - pinned
# - security
# Only issues or pull requests with all of these labels are check if stale.
onlyLabels:
- "Status: Abandoned"
- "Type: Question"
# Label to use when marking as stale
staleLabel: stale
# Comment to post when marking an issue as stale. Set to `false` to disable
markComment: >
This issue has been automatically marked as stale because it has not had
recent activity. It will be closed if no further activity occurs. Thank you
for your contributions.
# Comment to post when closing a stale issue. Set to `false` to disable
closeComment: false
+41
View File
@@ -0,0 +1,41 @@
name: 💤 Stale
on:
schedule:
- cron: '0 0 * * 0' # Weekly
jobs:
stale:
runs-on: ubuntu-latest
permissions:
actions: write
contents: write # only for delete-branch option
issues: write
pull-requests: write
steps:
- uses: actions/stale@v9
with:
days-before-stale: 90
days-before-close: 7
stale-issue-label: "Status: Stale"
stale-pr-label: "Status: Stale"
stale-issue-message: >
This issue has been automatically marked as stale because it has not
had recent activity. It will be closed in 7 days if no further
activity occurs. Thank you for your contributions!
stale-pr-message: >
This pull request has been automatically marked as stale due to
inactivity. It will be closed in 7 days if no further activity
occurs. Please update if you wish to keep it open.
close-issue-message: >
This issue has been automatically closed due to inactivity. If you
think this is a mistake or would like to continue the discussion,
please comment or feel free to reopen it.
close-pr-message: >
This pull request has been automatically closed due to inactivity.
If you think this is a mistake or would like to continue working on
it, please comment or feel free to reopen it.
close-issue-label: "Status: Abandoned"
close-pr-label: "Status: Abandoned"
exempt-issue-labels: "Status: Abandoned"
exempt-pr-labels: "Status: Abandoned"
+1 -1
View File
@@ -94,7 +94,7 @@ jobs:
- uses: actions/checkout@v4
- uses: projectdiscovery/actions/setup/go@v1
- uses: projectdiscovery/actions/setup/python@v1
- run: bash run.sh "${{ matrix.os }}"
- run: bash run.sh
env:
GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
working-directory: cmd/functional-test/
+1 -1
View File
@@ -7,7 +7,7 @@
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_KR.md">`Korean`</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ID.md">`Indonesia`</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ES.md">`Spanish`</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_JP.md">`日本語`</a>
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_JP.md">`日本語`</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_PT-BR.md">`Portuguese`</a>
</div>
+1 -1
View File
@@ -33,7 +33,7 @@
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_CN.md">中文</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_KR.md">Korean</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ID.md">Indonesia</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ES.md">Spanish</a>
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ES.md">Spanish</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_PT-BR.md">Portuguese</a>
</p>
+1 -1
View File
@@ -31,7 +31,7 @@
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_CN.md">中文</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_KR.md">Korean</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ID.md">Indonesia</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ES.md">Spanish</a>
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ES.md">Spanish</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_PT-BR.md">Portuguese</a>
</p>
+1 -1
View File
@@ -33,7 +33,7 @@
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_CN.md">中文</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_KR.md">Korean</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ID.md">Indonesia</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ES.md">Spanish</a>
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ES.md">Spanish</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_PT-BR.md">Portuguese</a>
</p>
+1 -1
View File
@@ -30,7 +30,7 @@
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_CN.md">中国語</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_KR.md">韓国語</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ID.md">インドネシア語</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ES.md">スペイン語</a>
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ES.md">スペイン語</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_PT-BR.md">ポルトガル語</a>
</p>
+1 -1
View File
@@ -31,7 +31,7 @@
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README.md">English</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_CN.md">中文</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_KR.md">한국어</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ES.md">스페인어</a>
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ES.md">스페인어</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_PT-BR.md">포르투갈어</a>
</p>
+1 -1
View File
@@ -31,7 +31,7 @@
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_CN.md">中文</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_KR.md">Korean</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ID.md">Indonesia</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ES.md">Spanish</a>
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_ES.md">Spanish</a> •
<a href="https://github.com/projectdiscovery/nuclei/blob/main/README_PT-BR.md">Portuguese</a>
</p>
+1 -1
View File
@@ -27,7 +27,7 @@ var (
func main() {
flag.Parse()
debug := os.Getenv("DEBUG") == "true"
debug := os.Getenv("DEBUG") == "true" || os.Getenv("RUNNER_DEBUG") == "1"
if err, errored := runFunctionalTests(debug); err != nil {
log.Fatalf("Could not run functional tests: %s\n", err)
+29 -13
View File
@@ -1,27 +1,43 @@
#!/bin/bash
# reading os type from arguments
CURRENT_OS=$1
if [ "${RUNNER_OS}" == "Windows" ]; then
EXT=".exe"
elif [ "${RUNNER_OS}" == "macOS" ]; then
if [ "${CI}" == "true" ]; then
sudo sysctl -w kern.maxfiles{,perproc}=524288
sudo launchctl limit maxfiles 65536 524288
fi
if [ "${CURRENT_OS}" == "windows-latest" ];then
extension=.exe
ORIGINAL_ULIMIT="$(ulimit -n)"
ulimit -n 65536 || true
fi
mkdir -p .nuclei-config/nuclei/
touch .nuclei-config/nuclei/.nuclei-ignore
echo "::group::Building functional-test binary"
go build -o functional-test$extension
go build -o "functional-test${EXT}"
echo "::endgroup::"
echo "::group::Building Nuclei binary from current branch"
go build -o nuclei_dev$extension ../nuclei
echo "::endgroup::"
echo "::group::Installing nuclei templates"
./nuclei_dev$extension -update-templates
go build -o "nuclei-dev${EXT}" ../nuclei
echo "::endgroup::"
echo "::group::Building latest release of nuclei"
go build -o nuclei$extension -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei
go build -o "nuclei${EXT}" -v github.com/projectdiscovery/nuclei/v3/cmd/nuclei
echo "::endgroup::"
echo 'Starting Nuclei functional test'
./functional-test$extension -main ./nuclei$extension -dev ./nuclei_dev$extension -testcases testcases.txt
echo "::group::Installing nuclei templates"
eval "./nuclei-dev${EXT} -update-templates"
echo "::endgroup::"
echo "::group::Validating templates"
eval "./nuclei-dev${EXT} -validate"
echo "::endgroup::"
echo "Starting Nuclei functional test"
eval "./functional-test${EXT} -main ./nuclei${EXT} -dev ./nuclei-dev${EXT} -testcases testcases.txt"
if [ "${RUNNER_OS}" == "macOS" ]; then
ulimit -n "${ORIGINAL_ULIMIT}" || true
fi
+1 -1
View File
@@ -86,7 +86,7 @@ require (
github.com/microsoft/go-mssqldb v1.6.0
github.com/ory/dockertest/v3 v3.10.0
github.com/praetorian-inc/fingerprintx v1.1.9
github.com/projectdiscovery/dsl v0.4.2
github.com/projectdiscovery/dsl v0.4.3
github.com/projectdiscovery/fasttemplate v0.0.2
github.com/projectdiscovery/go-smb2 v0.0.0-20240129202741-052cc450c6cb
github.com/projectdiscovery/goflags v0.1.74
+2
View File
@@ -860,6 +860,8 @@ github.com/projectdiscovery/clistats v0.1.1 h1:8mwbdbwTU4aT88TJvwIzTpiNeow3XnAB7
github.com/projectdiscovery/clistats v0.1.1/go.mod h1:4LtTC9Oy//RiuT1+76MfTg8Hqs7FQp1JIGBM3nHK6a0=
github.com/projectdiscovery/dsl v0.4.2 h1:9PnD6EyDAZFvpQmJ0700gkQ96Fqlzl+lnTdcVHAagXI=
github.com/projectdiscovery/dsl v0.4.2/go.mod h1:J1RizRF6O3lvk2v8p/tLAYqaxWg6N52OWc+uS5ZmO2U=
github.com/projectdiscovery/dsl v0.4.3 h1:ZrbRkyK38hRiYMX7s6ohaTorDpq321ErqJuBUDmh49g=
github.com/projectdiscovery/dsl v0.4.3/go.mod h1:cyt2IaYhS5SlyZ1D2BdK0QwIBXQW/u9zaBmRAKYKAmk=
github.com/projectdiscovery/fastdialer v0.4.0 h1:licZKyq+Shd5lLDb8uPd60Jp43K4NFE8cr67XD2eg7w=
github.com/projectdiscovery/fastdialer v0.4.0/go.mod h1:Q0YLArvpx9GAfY/NcTPMCA9qZuVOGnuVoNYWzKBwxdQ=
github.com/projectdiscovery/fasttemplate v0.0.2 h1:h2cISk5xDhlJEinlBQS6RRx0vOlOirB2y3Yu4PJzpiA=
+4 -3
View File
@@ -1,6 +1,7 @@
{{- if .Values.interactsh.ingress.enabled -}}
{{- $fullName := include "nuclei.fullname" . -}}
{{- $svcPort := .Values.service.port -}}
{{- $svcPort := .Values.interactsh.service.port -}}
{{- $svcName := .Values.interactsh.service.name -}}
{{- if and .Values.interactsh.ingress.className (not (semverCompare ">=1.20-0" .Capabilities.KubeVersion.GitVersion)) }}
{{- if not (hasKey .Values.interactsh.ingress.annotations "kubernetes.io/ingress.class") }}
{{- $_ := set .Values.interactsh.ingress.annotations "kubernetes.io/ingress.class" .Values.interactsh.ingress.className}}
@@ -49,11 +50,11 @@ spec:
backend:
{{- if semverCompare ">=1.20-0" $.Capabilities.KubeVersion.GitVersion }}
service:
name: {{ $fullName }}
name: {{ $svcName }}
port:
number: {{ $svcPort }}
{{- else }}
serviceName: {{ $fullName }}
serviceName: {{ $svcName }}
servicePort: {{ $svcPort }}
{{- end }}
{{- end }}
+3 -4
View File
@@ -7,6 +7,7 @@ import (
"fmt"
"io"
"path"
"slices"
"strings"
"github.com/aws/aws-sdk-go-v2/aws"
@@ -140,10 +141,8 @@ func (c Catalog) ResolvePath(templateName, second string) (string, error) {
}
// check if templateName is already an absolute path to c key
for _, key := range keys {
if key == templateName {
return templateName, nil
}
if slices.Contains(keys, templateName) {
return templateName, nil
}
return "", fmt.Errorf("no such path found: %s%s for keys: %v", second, templateName, keys)
+2 -7
View File
@@ -3,6 +3,7 @@ package aws
import (
"io"
"reflect"
"slices"
"strings"
"testing"
@@ -250,13 +251,7 @@ func (m mocks3svc) getAllKeys() ([]string, error) {
}
func (m mocks3svc) downloadKey(name string) (io.ReadCloser, error) {
found := false
for _, key := range m.keys {
if key == name {
found = true
break
}
}
found := slices.Contains(m.keys, name)
if !found {
return nil, errors.New("key not found")
}
+2 -6
View File
@@ -7,6 +7,7 @@ import (
"log"
"os"
"path/filepath"
"slices"
"strings"
"github.com/projectdiscovery/goflags"
@@ -334,12 +335,7 @@ func (c *Config) copyIgnoreFile() {
// this could be a feature specific to debugging like PPROF or printing stats
// of max host error etc
func (c *Config) IsDebugArgEnabled(arg string) bool {
for _, v := range c.debugArgs {
if v == arg {
return true
}
}
return false
return slices.Contains(c.debugArgs, arg)
}
// parseDebugArgs from string
+2 -7
View File
@@ -2,6 +2,7 @@ package openapi
import (
"fmt"
"slices"
"github.com/getkin/kin-openapi/openapi3"
"github.com/pkg/errors"
@@ -84,13 +85,7 @@ func excludeFromMode(schema *openapi3.Schema) bool {
// isRequired checks whether a key is actually required.
func isRequired(schema *openapi3.Schema, key string) bool {
for _, req := range schema.Required {
if req == key {
return true
}
}
return false
return slices.Contains(schema.Required, key)
}
type cachedSchema struct {
+7 -9
View File
@@ -15,16 +15,15 @@ import (
"github.com/projectdiscovery/nuclei/v3/pkg/types"
fileutil "github.com/projectdiscovery/utils/file"
osutils "github.com/projectdiscovery/utils/os"
processutil "github.com/projectdiscovery/utils/process"
)
// Browser is a browser structure for nuclei headless module
type Browser struct {
customAgent string
tempDir string
previousPIDs map[int32]struct{} // track already running PIDs
engine *rod.Browser
options *types.Options
customAgent string
tempDir string
engine *rod.Browser
options *types.Options
launcher *launcher.Launcher
// use getHTTPClient to get the http client
httpClient *http.Client
httpClientOnce *sync.Once
@@ -36,7 +35,6 @@ func New(options *types.Options) (*Browser, error) {
if err != nil {
return nil, errors.Wrap(err, "could not create temporary directory")
}
previousPIDs := processutil.FindProcesses(processutil.IsChromeProcess)
chromeLauncher := launcher.New().
Leakless(false).
@@ -110,8 +108,8 @@ func New(options *types.Options) (*Browser, error) {
engine: browser,
options: options,
httpClientOnce: &sync.Once{},
launcher: chromeLauncher,
}
engine.previousPIDs = previousPIDs
return engine, nil
}
@@ -143,6 +141,6 @@ func (b *Browser) getHTTPClient() (*http.Client, error) {
// Close closes the browser engine
func (b *Browser) Close() {
b.engine.Close()
b.launcher.Kill()
os.RemoveAll(b.tempDir)
processutil.CloseProcesses(processutil.IsChromeProcess, b.previousPIDs)
}
+2 -6
View File
@@ -2,6 +2,7 @@ package http
import (
"regexp"
"slices"
)
var (
@@ -32,10 +33,5 @@ func (request *Request) NeedsRequestCondition() bool {
}
func checkRequestConditionExpressions(expressions ...string) bool {
for _, expression := range expressions {
if reRequestCondition.MatchString(expression) {
return true
}
}
return false
return slices.ContainsFunc(expressions, reRequestCondition.MatchString)
}
+43 -1
View File
@@ -3,6 +3,7 @@ package multiproto
import (
"strconv"
"strings"
"sync"
"sync/atomic"
"github.com/projectdiscovery/nuclei/v3/pkg/output"
@@ -14,6 +15,12 @@ import (
stringsutil "github.com/projectdiscovery/utils/strings"
)
var builderPool = sync.Pool{
New: func() any {
return &strings.Builder{}
},
}
// Mutliprotocol is a template executer engine that executes multiple protocols
// with logic in between
type MultiProtocol struct {
@@ -64,6 +71,32 @@ func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error {
previous := mapsutil.NewSyncLockMap[string, any]()
// This map contains a list of keys to explicitly include for storage in the `previous`
// event map. This is a security and performance measure to prevent memory exhaustion.
// Only small, scalar values should be included here. If a user needs a larger value
// from a previous request (like a body or header), they must use an `extractor` with `internal: true`.
var includeInPrevious = map[string]struct{}{
// Generic
"host": {},
"port": {},
"path": {},
"scheme": {},
"url": {},
"ip": {},
"type": {},
"matched-at": {},
// HTTP
"status_code": {},
"content_length": {},
"content_type": {},
// DNS
"rcode": {},
"question": {},
"qtype": {},
}
// template context: contains values extracted using `internal` extractor from previous protocols
// these values are extracted from each protocol in queue and are passed to next protocol in queue
// instead of adding seperator field to handle such cases these values are appended to `dynamicValues` (which are meant to be used in workflows)
@@ -92,8 +125,17 @@ func (m *MultiProtocol) ExecuteWithResults(ctx *scan.ScanContext) error {
ID := req.GetID()
if ID != "" {
builder := &strings.Builder{}
builder := builderPool.Get().(*strings.Builder)
builder.Reset()
defer builderPool.Put(builder)
for k, v := range event.InternalEvent {
// Only store keys that are explicitly on the include list.
// This is a critical optimization to prevent memory exhaustion.
if _, included := includeInPrevious[k]; !included {
continue
}
builder.WriteString(ID)
builder.WriteString("_")
builder.WriteString(k)
+359
View File
@@ -2,8 +2,11 @@ package multiproto_test
import (
"context"
"fmt"
"log"
"os"
"runtime"
"sync"
"testing"
"time"
@@ -23,6 +26,57 @@ import (
var executerOpts protocols.ExecutorOptions
// MemorySnapshot represents a point-in-time memory measurement
type MemorySnapshot struct {
Timestamp time.Time
Alloc uint64
TotalAlloc uint64
Sys uint64
NumGC uint32
}
// TakeMemorySnapshot captures current memory statistics
func TakeMemorySnapshot() *MemorySnapshot {
var stats runtime.MemStats
runtime.ReadMemStats(&stats)
return &MemorySnapshot{
Timestamp: time.Now(),
Alloc: stats.Alloc,
TotalAlloc: stats.TotalAlloc,
Sys: stats.Sys,
NumGC: stats.NumGC,
}
}
// MemoryDiff represents the difference between two memory snapshots
type MemoryDiff struct {
Duration time.Duration
AllocDiff int64
TotalDiff int64
SysDiff int64
GCDiff int64
}
// Compare compares two memory snapshots and returns the difference
func (s *MemorySnapshot) Compare(other *MemorySnapshot) *MemoryDiff {
return &MemoryDiff{
Duration: other.Timestamp.Sub(s.Timestamp),
AllocDiff: int64(other.Alloc) - int64(s.Alloc),
TotalDiff: int64(other.TotalAlloc) - int64(s.TotalAlloc),
SysDiff: int64(other.Sys) - int64(s.Sys),
GCDiff: int64(other.NumGC) - int64(s.NumGC),
}
}
// String returns a formatted string representation of the memory difference
func (d *MemoryDiff) String() string {
return fmt.Sprintf(
"Duration: %v, Alloc: %+d bytes (%+.2f MB), Total: %+d bytes, Sys: %+d bytes, GC: %+d",
d.Duration, d.AllocDiff, float64(d.AllocDiff)/1024/1024, d.TotalDiff, d.SysDiff, d.GCDiff,
)
}
func setup() {
options := testutils.DefaultOptions
testutils.Init(options)
@@ -47,6 +101,311 @@ func setup() {
executerOpts.WorkflowLoader = workflowLoader
}
// Helper function to create a test template execution
func executeTemplate(templatePath string, target string) error {
template, err := templates.Parse(templatePath, nil, executerOpts)
if err != nil {
return fmt.Errorf("could not parse template: %w", err)
}
err = template.Executer.Compile()
if err != nil {
return fmt.Errorf("could not compile template: %w", err)
}
input := contextargs.NewWithInput(context.Background(), target)
ctx := scan.NewScanContext(context.Background(), input)
_, err = template.Executer.Execute(ctx)
if err != nil {
return fmt.Errorf("could not execute template: %w", err)
}
return nil
}
// TestMemoryLeakDetection tests for memory leaks in multiprotocol execution
func TestMemoryLeakDetection(t *testing.T) {
tests := []struct {
name string
templatePath string
target string
numExecutions int
maxMemoryMB int
description string
}{
{
name: "SmallScale_DynamicExtractor",
templatePath: "testcases/multiprotodynamic.yaml",
target: "http://scanme.sh",
numExecutions: 10,
maxMemoryMB: 10,
description: "Basic memory leak detection with dynamic extractor",
},
{
name: "MediumScale_ProtoPrefix",
templatePath: "testcases/multiprotowithprefix.yaml",
target: "https://cloud.projectdiscovery.io/sign-in",
numExecutions: 50,
maxMemoryMB: 25,
description: "Medium scale execution with protocol prefix",
},
{
name: "LargeScale_Mixed",
templatePath: "testcases/multiprotodynamic.yaml",
target: "http://scanme.sh",
numExecutions: 100,
maxMemoryMB: 50,
description: "Large scale execution for memory leak detection",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Force GC and get baseline
runtime.GC()
runtime.GC() // Call twice to ensure cleanup
time.Sleep(100 * time.Millisecond) // Allow GC to complete
baseline := TakeMemorySnapshot()
t.Logf("Baseline memory: Alloc=%d bytes (%.2f MB)",
baseline.Alloc, float64(baseline.Alloc)/1024/1024)
// Execute multiple times
for i := 0; i < tt.numExecutions; i++ {
err := executeTemplate(tt.templatePath, tt.target)
if err != nil {
t.Logf("Execution %d failed (continuing): %v", i, err)
continue
}
// Periodic memory checks
if i%10 == 0 && i > 0 {
runtime.GC()
current := TakeMemorySnapshot()
diff := baseline.Compare(current)
memoryMB := float64(diff.AllocDiff) / 1024 / 1024
t.Logf("Execution %d: %s", i, diff.String())
if memoryMB > float64(tt.maxMemoryMB) {
t.Errorf("Memory usage exceeded limit: %.2f MB > %d MB",
memoryMB, tt.maxMemoryMB)
}
}
}
// Final memory check
runtime.GC()
runtime.GC()
time.Sleep(100 * time.Millisecond)
final := TakeMemorySnapshot()
finalDiff := baseline.Compare(final)
finalMemoryMB := float64(finalDiff.AllocDiff) / 1024 / 1024
t.Logf("Final memory analysis: %s", finalDiff.String())
if finalMemoryMB > float64(tt.maxMemoryMB) {
t.Errorf("Final memory usage exceeded limit: %.2f MB > %d MB",
finalMemoryMB, tt.maxMemoryMB)
} else {
t.Logf("✅ Memory usage within acceptable limits: %.2f MB <= %d MB",
finalMemoryMB, tt.maxMemoryMB)
}
})
}
}
// TestTemplateContextCleanup verifies that template contexts are properly cleaned up
func TestTemplateContextCleanup(t *testing.T) {
// Create executor options with template context store
opts := executerOpts
opts.CreateTemplateCtxStore()
template, err := templates.Parse("testcases/multiprotodynamic.yaml", nil, opts)
require.NoError(t, err, "could not parse template")
err = template.Executer.Compile()
require.NoError(t, err, "could not compile template")
// Create multiple contexts and execute
numContexts := 20
inputs := make([]*contextargs.Context, numContexts)
for i := 0; i < numContexts; i++ {
target := fmt.Sprintf("http://test%d.example.com", i)
inputs[i] = contextargs.NewWithInput(context.Background(), target)
ctx := scan.NewScanContext(context.Background(), inputs[i])
_, err := template.Executer.Execute(ctx)
if err != nil {
t.Logf("Execution %d failed (continuing): %v", i, err)
}
}
// Verify contexts exist
contextCount := 0
for _, input := range inputs {
if opts.HasTemplateCtx(input.MetaInput) {
contextCount++
}
}
t.Logf("Active template contexts: %d", contextCount)
// Clean up contexts manually (simulating proper cleanup)
for _, input := range inputs {
opts.RemoveTemplateCtx(input.MetaInput)
}
// Verify cleanup
remainingContexts := 0
for _, input := range inputs {
if opts.HasTemplateCtx(input.MetaInput) {
remainingContexts++
}
}
if remainingContexts > 0 {
t.Errorf("Template contexts not properly cleaned up: %d remaining", remainingContexts)
} else {
t.Logf("✅ All template contexts properly cleaned up")
}
}
// TestConcurrentExecution tests memory usage under concurrent execution
func TestConcurrentExecution(t *testing.T) {
if testing.Short() {
t.Skip("Skipping concurrent test in short mode")
}
const (
numGoroutines = 5
numExecutions = 20
maxMemoryMB = 100
)
baseline := TakeMemorySnapshot()
var wg sync.WaitGroup
errChan := make(chan error, numGoroutines)
// Start concurrent executions
for i := 0; i < numGoroutines; i++ {
wg.Add(1)
go func(id int) {
defer wg.Done()
for j := 0; j < numExecutions; j++ {
target := fmt.Sprintf("http://test%d-%d.example.com", id, j)
err := executeTemplate("testcases/multiprotodynamic.yaml", target)
if err != nil {
t.Logf("Goroutine %d execution %d failed: %v", id, j, err)
continue
}
// Periodic memory check
if j%5 == 0 {
var stats runtime.MemStats
runtime.ReadMemStats(&stats)
memoryMB := float64(stats.Alloc) / 1024 / 1024
if memoryMB > maxMemoryMB {
errChan <- fmt.Errorf("goroutine %d: memory exceeded: %.2f MB", id, memoryMB)
return
}
}
}
}(i)
}
// Wait for completion with timeout
done := make(chan struct{})
go func() {
wg.Wait()
close(done)
}()
select {
case err := <-errChan:
t.Fatal(err)
case <-done:
runtime.GC()
final := TakeMemorySnapshot()
diff := baseline.Compare(final)
t.Logf("✅ Concurrent execution completed successfully: %s", diff.String())
case <-time.After(2 * time.Minute):
t.Fatal("Test timed out")
}
}
// BenchmarkMultiProtocolExecution benchmarks multiprotocol execution performance
func BenchmarkMultiProtocolExecution(b *testing.B) {
benchmarks := []struct {
name string
templatePath string
target string
}{
{
name: "DynamicExtractor",
templatePath: "testcases/multiprotodynamic.yaml",
target: "http://scanme.sh",
},
{
name: "ProtoPrefix",
templatePath: "testcases/multiprotowithprefix.yaml",
target: "https://example.com",
},
}
for _, bm := range benchmarks {
b.Run(bm.name, func(b *testing.B) {
template, err := templates.Parse(bm.templatePath, nil, executerOpts)
if err != nil {
b.Fatalf("could not parse template: %v", err)
}
err = template.Executer.Compile()
if err != nil {
b.Fatalf("could not compile template: %v", err)
}
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
input := contextargs.NewWithInput(context.Background(), bm.target)
ctx := scan.NewScanContext(context.Background(), input)
_, err := template.Executer.Execute(ctx)
if err != nil {
b.Logf("Execution failed (continuing): %v", err)
}
}
})
}
}
// BenchmarkMemoryAllocation specifically benchmarks memory allocation patterns
func BenchmarkMemoryAllocation(b *testing.B) {
template, err := templates.Parse("testcases/multiprotodynamic.yaml", nil, executerOpts)
if err != nil {
b.Fatalf("could not parse template: %v", err)
}
err = template.Executer.Compile()
if err != nil {
b.Fatalf("could not compile template: %v", err)
}
b.ResetTimer()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
input := contextargs.NewWithInput(context.Background(), "http://scanme.sh")
ctx := scan.NewScanContext(context.Background(), input)
_, _ = template.Executer.Execute(ctx)
}
}
func TestMultiProtoWithDynamicExtractor(t *testing.T) {
Template, err := templates.Parse("testcases/multiprotodynamic.yaml", nil, executerOpts)
require.Nil(t, err, "could not parse template")