started implementing the HTTP client selection

config/resolvers.go

@@ -17,7 +17,7 @@ import (
 	"strings"
 
 	"github.com/caffix/stringset"
-	"github.com/owasp-amass/amass/v5/internal/net/http"
+	amasshttp "github.com/owasp-amass/amass/v5/internal/net/http"
 )
 
 // DefaultQueriesPerPublicResolver is the number of queries sent to each public DNS resolver per second.
@@ -54,7 +54,8 @@ var PublicResolvers []string
 // GetPublicDNSResolvers obtains the public DNS server addresses from public-dns.info and assigns them to PublicResolvers.
 func GetPublicDNSResolvers() error {
 	url := "https://public-dns.info/nameservers-all.csv"
-	resp, err := http.RequestWebPage(context.Background(), &http.Request{URL: url})
+	resp, err := amasshttp.RequestWebPage(context.Background(),
+		amasshttp.DefaultClient, &amasshttp.Request{URL: url})
 	if err != nil || resp.StatusCode < 200 || resp.StatusCode >= 400 {
 		return fmt.Errorf("failed to obtain the Public DNS csv file at %s: %v", url, err)
 	}

engine/plugins/api/binaryedge.go

@@ -16,7 +16,7 @@ import (
 	"github.com/caffix/stringset"
 	"github.com/owasp-amass/amass/v5/engine/plugins/support"
 	et "github.com/owasp-amass/amass/v5/engine/types"
-	"github.com/owasp-amass/amass/v5/internal/net/http"
+	amasshttp "github.com/owasp-amass/amass/v5/internal/net/http"
 	dbt "github.com/owasp-amass/asset-db/types"
 	oam "github.com/owasp-amass/open-asset-model"
 	oamdns "github.com/owasp-amass/open-asset-model/dns"
@@ -121,8 +121,8 @@ loop:
 			ctx, cancel := context.WithTimeout(e.Session.Ctx(), 5*time.Second)
 			defer cancel()
 
-			resp, err := http.RequestWebPage(ctx, &http.Request{
-				Header: http.Header{"X-KEY": []string{key}},
+			resp, err := amasshttp.RequestWebPage(ctx, e.Session.Clients().General, &amasshttp.Request{
+				Header: amasshttp.Header{"X-KEY": []string{key}},
 				URL:    "https://api.binaryedge.io/v2/query/domains/subdomain/" + name + "?page=" + strconv.Itoa(pagenum),
 			})
 			if err != nil || resp.Body == "" {

engine/plugins/api/passivetotal.go

@@ -14,8 +14,8 @@ import (
 	"github.com/owasp-amass/amass/v5/config"
 	"github.com/owasp-amass/amass/v5/engine/plugins/support"
 	et "github.com/owasp-amass/amass/v5/engine/types"
-	"github.com/owasp-amass/amass/v5/internal/net/dns"
-	"github.com/owasp-amass/amass/v5/internal/net/http"
+	amassdns "github.com/owasp-amass/amass/v5/internal/net/dns"
+	amasshttp "github.com/owasp-amass/amass/v5/internal/net/http"
 	dbt "github.com/owasp-amass/asset-db/types"
 	oam "github.com/owasp-amass/open-asset-model"
 	oamdns "github.com/owasp-amass/open-asset-model/dns"
@@ -122,13 +122,15 @@ loop:
 			ctx, cancel := context.WithTimeout(e.Session.Ctx(), 5*time.Second)
 			defer cancel()
 
-			resp, err := http.RequestWebPage(ctx, &http.Request{
+			e.Session.NetSem().Acquire()
+			resp, err := amasshttp.RequestWebPage(ctx, e.Session.Clients().General, &amasshttp.Request{
 				URL: url,
-				Auth: &http.BasicAuth{
+				Auth: &amasshttp.BasicAuth{
 					Username: cr.Username,
 					Password: cr.Apikey,
 				},
 			})
+			e.Session.NetSem().Release()
 			if err != nil || resp.Body == "" {
 				continue
 			}
@@ -143,7 +145,7 @@ loop:
 			}
 
 			for _, sub := range result.Subdomains {
-				n := dns.RemoveAsteriskLabel(http.CleanName(sub + "." + name))
+				n := amassdns.RemoveAsteriskLabel(amasshttp.CleanName(sub + "." + name))
 				// if the subdomain is not in scope, skip it
 				if _, conf := e.Session.Scope().IsAssetInScope(&oamdns.FQDN{Name: n}, 0); conf > 0 {
 					names.Insert(n)

engine/plugins/api/prospeo.go

@@ -14,7 +14,7 @@ import (
 
 	"github.com/owasp-amass/amass/v5/engine/plugins/support"
 	et "github.com/owasp-amass/amass/v5/engine/types"
-	"github.com/owasp-amass/amass/v5/internal/net/http"
+	amasshttp "github.com/owasp-amass/amass/v5/internal/net/http"
 	dbt "github.com/owasp-amass/asset-db/types"
 	oam "github.com/owasp-amass/open-asset-model"
 	oamdns "github.com/owasp-amass/open-asset-model/dns"
@@ -108,27 +108,29 @@ func (p *Prospeo) query(e *et.Event, name string) []*dbt.Entity {
 		return nil
 	}
 
-	rcreds, err := p.accountType(e.Session.Ctx(), key)
+	rcreds, err := p.accountType(e.Session, key)
 	if err != nil || key == "" {
 		return nil
 	}
 
-	count, err := p.count(e.Session.Ctx(), name, key)
+	count, err := p.count(e.Session, name, key)
 	if err != nil {
 		return nil
 	}
 	limit := min(rcreds*50, count)
 
 	_ = p.rlimit.Wait(e.Session.Ctx())
+	e.Session.NetSem().Acquire()
 	ctx, cancel := context.WithTimeout(e.Session.Ctx(), 5*time.Second)
 	defer cancel()
 
-	resp, err := http.RequestWebPage(ctx, &http.Request{
+	resp, err := amasshttp.RequestWebPage(ctx, e.Session.Clients().General, &amasshttp.Request{
 		Method: "POST",
 		Body:   `{"company": "` + name + `", "limit": ` + strconv.Itoa(limit) + `}`,
 		URL:    p.queryurl,
-		Header: http.Header{"Content-Type": []string{"application/json"}, "X-KEY": []string{key}},
+		Header: amasshttp.Header{"Content-Type": []string{"application/json"}, "X-KEY": []string{key}},
 	})
+	e.Session.NetSem().Release()
 	if err != nil {
 		return nil
 	}
@@ -159,17 +161,19 @@ func (p *Prospeo) process(e *et.Event, assets []*dbt.Entity) {
 	support.ProcessEmailsWithSource(e, assets, p.source)
 }
 
-func (p *Prospeo) accountType(ctx context.Context, key string) (int, error) {
-	_ = p.rlimit.Wait(ctx)
+func (p *Prospeo) accountType(sess et.Session, key string) (int, error) {
+	_ = p.rlimit.Wait(sess.Ctx())
 
-	rctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	sess.NetSem().Acquire()
+	rctx, cancel := context.WithTimeout(sess.Ctx(), 5*time.Second)
 	defer cancel()
 
-	resp, err := http.RequestWebPage(rctx, &http.Request{
+	resp, err := amasshttp.RequestWebPage(rctx, sess.Clients().General, &amasshttp.Request{
 		Method: "POST",
 		URL:    p.accturl,
-		Header: http.Header{"Content-Type": []string{"application/json"}, "X-KEY": []string{key}},
+		Header: amasshttp.Header{"Content-Type": []string{"application/json"}, "X-KEY": []string{key}},
 	})
+	sess.NetSem().Release()
 	if err != nil {
 		return 0, err
 	}
@@ -185,18 +189,20 @@ func (p *Prospeo) accountType(ctx context.Context, key string) (int, error) {
 	return r.Response.RemainingCredits, nil
 }
 
-func (p *Prospeo) count(ctx context.Context, domain string, key string) (int, error) {
-	_ = p.rlimit.Wait(ctx)
+func (p *Prospeo) count(sess et.Session, domain string, key string) (int, error) {
+	_ = p.rlimit.Wait(sess.Ctx())
 
-	rctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	sess.NetSem().Acquire()
+	rctx, cancel := context.WithTimeout(sess.Ctx(), 5*time.Second)
 	defer cancel()
 
-	resp, err := http.RequestWebPage(rctx, &http.Request{
+	resp, err := amasshttp.RequestWebPage(rctx, sess.Clients().General, &amasshttp.Request{
 		Method: "POST",
 		Body:   `{"domain": "` + domain + `"}`,
 		URL:    p.counturl,
-		Header: http.Header{"Content-Type": []string{"application/json"}, "X-KEY": []string{key}},
+		Header: amasshttp.Header{"Content-Type": []string{"application/json"}, "X-KEY": []string{key}},
 	})
+	sess.NetSem().Release()
 	if err != nil {
 		return 0, err
 	}

engine/plugins/api/zetalytics.go

@@ -15,8 +15,8 @@ import (
 
 	"github.com/owasp-amass/amass/v5/engine/plugins/support"
 	et "github.com/owasp-amass/amass/v5/engine/types"
-	"github.com/owasp-amass/amass/v5/internal/net/dns"
-	"github.com/owasp-amass/amass/v5/internal/net/http"
+	amassdns "github.com/owasp-amass/amass/v5/internal/net/dns"
+	amasshttp "github.com/owasp-amass/amass/v5/internal/net/http"
 	dbt "github.com/owasp-amass/asset-db/types"
 	oam "github.com/owasp-amass/open-asset-model"
 	oamdns "github.com/owasp-amass/open-asset-model/dns"
@@ -122,7 +122,9 @@ func (z *zetalytics) query(e *et.Event, name string, keys []string) []*dbt.Entit
 		ctx, cancel := context.WithTimeout(e.Session.Ctx(), 5*time.Second)
 		defer cancel()
 
-		resp, err := http.RequestWebPage(ctx, &http.Request{URL: url})
+		e.Session.NetSem().Acquire()
+		resp, err := amasshttp.RequestWebPage(ctx, e.Session.Clients().General, &amasshttp.Request{URL: url})
+		e.Session.NetSem().Release()
 		if err != nil || resp.Body == "" {
 			continue
 		}
@@ -141,7 +143,7 @@ func (z *zetalytics) query(e *et.Event, name string, keys []string) []*dbt.Entit
 		}
 
 		for _, s := range result.Subdomains {
-			name := strings.ToLower(strings.TrimSpace(dns.RemoveAsteriskLabel(http.CleanName(s.FQDN))))
+			name := strings.ToLower(strings.TrimSpace(amassdns.RemoveAsteriskLabel(amasshttp.CleanName(s.FQDN))))
 			// if the subdomain is not in scope, skip it
 			if _, conf := e.Session.Scope().IsAssetInScope(&oamdns.FQDN{Name: name}, 0); conf > 0 {
 				names.Insert(name)

engine/plugins/archive/wayback.go

@@ -15,8 +15,8 @@ import (
 	"github.com/caffix/stringset"
 	"github.com/owasp-amass/amass/v5/engine/plugins/support"
 	et "github.com/owasp-amass/amass/v5/engine/types"
-	"github.com/owasp-amass/amass/v5/internal/net/dns"
-	"github.com/owasp-amass/amass/v5/internal/net/http"
+	amassdns "github.com/owasp-amass/amass/v5/internal/net/dns"
+	amasshttp "github.com/owasp-amass/amass/v5/internal/net/http"
 	dbt "github.com/owasp-amass/asset-db/types"
 	oam "github.com/owasp-amass/open-asset-model"
 	oamdns "github.com/owasp-amass/open-asset-model/dns"
@@ -106,7 +106,10 @@ func (w *wayback) query(e *et.Event, name string) []*dbt.Entity {
 	ctx, cancel := context.WithTimeout(e.Session.Ctx(), 30*time.Second)
 	defer cancel()
 
-	resp, err := http.RequestWebPage(ctx, &http.Request{URL: w.URL + end})
+	e.Session.NetSem().Acquire()
+	resp, err := amasshttp.RequestWebPage(ctx,
+		e.Session.Clients().General, &amasshttp.Request{URL: w.URL + end})
+	e.Session.NetSem().Release()
 	if err != nil {
 		return nil
 	}
@@ -125,7 +128,7 @@ func (w *wayback) query(e *et.Event, name string) []*dbt.Entity {
 		}
 		u := url[0]
 
-		if n := dns.AnySubdomainRegex().FindString(u); n != "" {
+		if n := amassdns.AnySubdomainRegex().FindString(u); n != "" {
 			// if the subdomain is not in scope, skip it
 			if _, conf := e.Session.Scope().IsAssetInScope(&oamdns.FQDN{Name: n}, 0); conf > 0 {
 				subs.Insert(n)

engine/plugins/scrape/bing.go

@@ -15,7 +15,7 @@ import (
 	"github.com/caffix/stringset"
 	"github.com/owasp-amass/amass/v5/engine/plugins/support"
 	et "github.com/owasp-amass/amass/v5/engine/types"
-	"github.com/owasp-amass/amass/v5/internal/net/http"
+	amasshttp "github.com/owasp-amass/amass/v5/internal/net/http"
 	dbt "github.com/owasp-amass/asset-db/types"
 	oam "github.com/owasp-amass/open-asset-model"
 	oamdns "github.com/owasp-amass/open-asset-model/dns"
@@ -109,7 +109,10 @@ func (b *bing) query(e *et.Event, name string) []*dbt.Entity {
 		ctx, cancel := context.WithTimeout(e.Session.Ctx(), 10*time.Second)
 		defer cancel()
 
-		resp, err := http.RequestWebPage(ctx, &http.Request{URL: fmt.Sprintf(b.fmtstr, i, name, name)})
+		e.Session.NetSem().Acquire()
+		resp, err := amasshttp.RequestWebPage(ctx, e.Session.Clients().General,
+			&amasshttp.Request{URL: fmt.Sprintf(b.fmtstr, i, name, name)})
+		e.Session.NetSem().Release()
 		if err != nil || resp.Body == "" {
 			break
 		}

engine/plugins/scrape/dnshistory.go

@@ -15,7 +15,7 @@ import (
 	"github.com/caffix/stringset"
 	"github.com/owasp-amass/amass/v5/engine/plugins/support"
 	et "github.com/owasp-amass/amass/v5/engine/types"
-	"github.com/owasp-amass/amass/v5/internal/net/http"
+	amasshttp "github.com/owasp-amass/amass/v5/internal/net/http"
 	dbt "github.com/owasp-amass/asset-db/types"
 	oam "github.com/owasp-amass/open-asset-model"
 	oamdns "github.com/owasp-amass/open-asset-model/dns"
@@ -109,7 +109,10 @@ func (d *dnsHistory) query(e *et.Event, name string) []*dbt.Entity {
 		ctx, cancel := context.WithTimeout(e.Session.Ctx(), 10*time.Second)
 		defer cancel()
 
-		resp, err := http.RequestWebPage(ctx, &http.Request{URL: fmt.Sprintf(d.fmtstr, i, name)})
+		e.Session.NetSem().Acquire()
+		resp, err := amasshttp.RequestWebPage(ctx,
+			e.Session.Clients().General, &amasshttp.Request{URL: fmt.Sprintf(d.fmtstr, i, name)})
+		e.Session.NetSem().Release()
 		if err != nil || resp.Body == "" {
 			break
 		}

engine/plugins/scrape/duckduckgo.go

@@ -15,7 +15,7 @@ import (
 	"github.com/caffix/stringset"
 	"github.com/owasp-amass/amass/v5/engine/plugins/support"
 	et "github.com/owasp-amass/amass/v5/engine/types"
-	"github.com/owasp-amass/amass/v5/internal/net/http"
+	amasshttp "github.com/owasp-amass/amass/v5/internal/net/http"
 	dbt "github.com/owasp-amass/asset-db/types"
 	oam "github.com/owasp-amass/open-asset-model"
 	oamdns "github.com/owasp-amass/open-asset-model/dns"
@@ -107,7 +107,10 @@ func (d *duckDuckGo) query(e *et.Event, name string) []*dbt.Entity {
 	ctx, cancel := context.WithTimeout(e.Session.Ctx(), 10*time.Second)
 	defer cancel()
 
-	resp, err := http.RequestWebPage(ctx, &http.Request{URL: fmt.Sprintf(d.fmtstr, name, name)})
+	e.Session.NetSem().Acquire()
+	resp, err := amasshttp.RequestWebPage(ctx,
+		e.Session.Clients().General, &amasshttp.Request{URL: fmt.Sprintf(d.fmtstr, name, name)})
+	e.Session.NetSem().Release()
 	if err != nil || resp.Body == "" {
 		return nil
 	}

engine/plugins/scrape/ipverse.go

@@ -14,7 +14,7 @@ import (
 
 	"github.com/owasp-amass/amass/v5/engine/plugins/support"
 	et "github.com/owasp-amass/amass/v5/engine/types"
-	"github.com/owasp-amass/amass/v5/internal/net/http"
+	amasshttp "github.com/owasp-amass/amass/v5/internal/net/http"
 	dbt "github.com/owasp-amass/asset-db/types"
 	oam "github.com/owasp-amass/open-asset-model"
 	oamnet "github.com/owasp-amass/open-asset-model/network"
@@ -105,12 +105,15 @@ type record struct {
 
 func (v *ipverse) query(sess et.Session, asset *dbt.Entity) *record {
 	_ = v.rlimit.Wait(sess.Ctx())
+	as := asset.Asset.(*oamnet.AutonomousSystem)
 
 	ctx, cancel := context.WithTimeout(sess.Ctx(), 30*time.Second)
 	defer cancel()
 
-	as := asset.Asset.(*oamnet.AutonomousSystem)
-	resp, err := http.RequestWebPage(ctx, &http.Request{URL: fmt.Sprintf(v.fmtstr, as.Number)})
+	sess.NetSem().Acquire()
+	resp, err := amasshttp.RequestWebPage(ctx,
+		sess.Clients().General, &amasshttp.Request{URL: fmt.Sprintf(v.fmtstr, as.Number)})
+	sess.NetSem().Release()
 	if err != nil || resp.Body == "" {
 		return nil
 	}

engine/plugins/scrape/rapiddns.go

@@ -15,7 +15,7 @@ import (
 	"github.com/caffix/stringset"
 	"github.com/owasp-amass/amass/v5/engine/plugins/support"
 	et "github.com/owasp-amass/amass/v5/engine/types"
-	"github.com/owasp-amass/amass/v5/internal/net/http"
+	amasshttp "github.com/owasp-amass/amass/v5/internal/net/http"
 	dbt "github.com/owasp-amass/asset-db/types"
 	oam "github.com/owasp-amass/open-asset-model"
 	oamdns "github.com/owasp-amass/open-asset-model/dns"
@@ -107,7 +107,10 @@ func (rd *rapidDNS) query(e *et.Event, name string) []*dbt.Entity {
 	ctx, cancel := context.WithTimeout(e.Session.Ctx(), 10*time.Second)
 	defer cancel()
 
-	resp, err := http.RequestWebPage(ctx, &http.Request{URL: fmt.Sprintf(rd.fmtstr, name)})
+	e.Session.NetSem().Acquire()
+	resp, err := amasshttp.RequestWebPage(ctx,
+		e.Session.Clients().General, &amasshttp.Request{URL: fmt.Sprintf(rd.fmtstr, name)})
+	e.Session.NetSem().Release()
 	if err != nil || resp.Body == "" {
 		return nil
 	}

engine/plugins/scrape/sitedossier.go

@@ -15,7 +15,7 @@ import (
 	"github.com/caffix/stringset"
 	"github.com/owasp-amass/amass/v5/engine/plugins/support"
 	et "github.com/owasp-amass/amass/v5/engine/types"
-	"github.com/owasp-amass/amass/v5/internal/net/http"
+	amasshttp "github.com/owasp-amass/amass/v5/internal/net/http"
 	dbt "github.com/owasp-amass/asset-db/types"
 	oam "github.com/owasp-amass/open-asset-model"
 	oamdns "github.com/owasp-amass/open-asset-model/dns"
@@ -109,7 +109,10 @@ func (sd *siteDossier) query(e *et.Event, name string) []*dbt.Entity {
 		ctx, cancel := context.WithTimeout(e.Session.Ctx(), 10*time.Second)
 		defer cancel()
 
-		resp, err := http.RequestWebPage(ctx, &http.Request{URL: fmt.Sprintf(sd.fmtstr, name, i)})
+		e.Session.NetSem().Acquire()
+		resp, err := amasshttp.RequestWebPage(ctx,
+			e.Session.Clients().General, &amasshttp.Request{URL: fmt.Sprintf(sd.fmtstr, name, i)})
+		e.Session.NetSem().Release()
 		if err != nil || resp.Body == "" {
 			break
 		}

engine/plugins/service_discovery/http_probes/plugin.go

@@ -15,7 +15,7 @@ import (
 
 	"github.com/owasp-amass/amass/v5/engine/plugins/support"
 	et "github.com/owasp-amass/amass/v5/engine/types"
-	"github.com/owasp-amass/amass/v5/internal/net/http"
+	amasshttp "github.com/owasp-amass/amass/v5/internal/net/http"
 	dbt "github.com/owasp-amass/asset-db/types"
 	oam "github.com/owasp-amass/open-asset-model"
 	oamcert "github.com/owasp-amass/open-asset-model/certificate"
@@ -106,7 +106,7 @@ func (hp *httpProbing) query(e *et.Event, entity *dbt.Entity, target string, por
 	defer cancel()
 
 	e.Session.NetSem().Acquire()
-	resp, err := http.RequestWebPage(ctx, &http.Request{URL: target})
+	resp, err := amasshttp.RequestWebPage(ctx, e.Session.Clients().Probe, &amasshttp.Request{URL: target})
 	e.Session.NetSem().Release()
 
 	if err == nil && resp != nil {
@@ -115,7 +115,7 @@ func (hp *httpProbing) query(e *et.Event, entity *dbt.Entity, target string, por
 	return findings
 }
 
-func (hp *httpProbing) store(e *et.Event, resp *http.Response, entity *dbt.Entity, port int) []*support.Finding {
+func (hp *httpProbing) store(e *et.Event, resp *amasshttp.Response, entity *dbt.Entity, port int) []*support.Finding {
 	addr := entity.Asset.Key()
 
 	hp.servlock.Lock()
@@ -174,7 +174,7 @@ func (hp *httpProbing) store(e *et.Event, resp *http.Response, entity *dbt.Entit
 	return findings
 }
 
-func (hp *httpProbing) createCertificates(sess et.Session, resp *http.Response) (*dbt.Entity, *x509.Certificate, []*support.Finding) {
+func (hp *httpProbing) createCertificates(sess et.Session, resp *amasshttp.Response) (*dbt.Entity, *x509.Certificate, []*support.Finding) {
 	var findings []*support.Finding
 
 	if resp.TLS == nil || !resp.TLS.HandshakeComplete {

engine/plugins/support/fingerprinting.go

@@ -12,7 +12,7 @@ import (
 	"time"
 
 	jarm "github.com/caffix/jarm-go"
-	"github.com/owasp-amass/amass/v5/internal/net"
+	amassnet "github.com/owasp-amass/amass/v5/internal/net"
 	oam "github.com/owasp-amass/open-asset-model"
 	oamdns "github.com/owasp-amass/open-asset-model/dns"
 	"github.com/owasp-amass/open-asset-model/general"
@@ -43,7 +43,8 @@ func JARMFingerprint(target oam.Asset, portrel *general.PortRelation) (string, e
 		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 		defer cancel()
 
-		c, err := net.DialContext(ctx, "tcp", addr)
+		dial := amassnet.NewDialContext(5 * time.Second)
+		c, err := dial(ctx, "tcp", addr)
 		if err != nil {
 			return "", err
 		}

engine/plugins/support/location.go

@@ -127,7 +127,7 @@ func postalServerParseAddress(address string) ([]parsedComponent, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
 	defer cancel()
 
-	resp, err := amasshttp.RequestWebPage(ctx, &amasshttp.Request{
+	resp, err := amasshttp.RequestWebPage(ctx, amasshttp.DefaultClient, &amasshttp.Request{
 		Method: "POST",
 		URL:    "http://" + postalHost + ":" + postalPort + "/parse",
 		Body:   string(reqJSON),
@@ -147,7 +147,7 @@ func checkPostalServerHealth() bool {
 	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
 	defer cancel()
 
-	if resp, err := amasshttp.RequestWebPage(ctx, &amasshttp.Request{
+	if resp, err := amasshttp.RequestWebPage(ctx, amasshttp.DefaultClient, &amasshttp.Request{
 		URL: "http://" + postalHost + ":" + postalPort + "/health",
 	}); err != nil || resp == nil || resp.StatusCode != http.StatusOK {
 		return false

engine/plugins/whois/bgptools/plugin.go

@@ -129,13 +129,14 @@ type bgpToolsRecord struct {
 }
 
 func (bt *bgpTools) whois(ctx context.Context, ipstr string) (*bgpToolsRecord, error) {
+	dial := amassnet.NewDialContext(3 * time.Second)
 	addr := net.JoinHostPort(bt.addr, strconv.Itoa(bt.port))
 
 	_ = bt.rlimit.Wait(ctx)
 	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
 	defer cancel()
 
-	conn, err := amassnet.DialContext(ctx, "tcp", addr)
+	conn, err := dial(ctx, "tcp", addr)
 	if err != nil {
 		return nil, fmt.Errorf("failed to establish a connection with the WHOIS server: %v", err)
 	}

engine/sessions/http_clients.go

@@ -6,27 +6,28 @@ package sessions
 
 import (
 	"crypto/tls"
-	"net"
 	"net/http"
 	"net/http/cookiejar"
 	"time"
+
+	amassnet "github.com/owasp-amass/amass/v5/internal/net"
 )
 
 // Clients bundles the three clients + transports so you can close idle conns.
 type Clients struct {
-	API    *http.Client
-	Probe  *http.Client
-	Crawl  *http.Client
-	apiTr  *http.Transport
-	probTr *http.Transport
-	crwlTr *http.Transport
+	General *http.Client
+	Probe   *http.Client
+	Crawl   *http.Client
+	genTr   *http.Transport
+	probTr  *http.Transport
+	crwlTr  *http.Transport
 }
 
 // CloseIdleConnections is useful on session/engine shutdown.
 // It does not kill in-flight requests, but it releases keep-alive sockets.
 func (c *Clients) CloseIdleConnections() {
-	if c.apiTr != nil {
-		c.apiTr.CloseIdleConnections()
+	if c.genTr != nil {
+		c.genTr.CloseIdleConnections()
 	}
 	if c.probTr != nil {
 		c.probTr.CloseIdleConnections()
@@ -37,16 +38,18 @@ func (c *Clients) CloseIdleConnections() {
 }
 
 // NewClients returns three tuned clients: API, Probe, Crawl.
-func NewClients() (*Clients, error) {
-	apiTr := newDefaultTransport()
-	probTr := newProbeTransport(0)
-	crwlTr, err := newCrawlTransport()
+func NewClients(perHost int) (*Clients, error) {
+	genTr := newGeneralTransport()
+	probTr := newProbeTransport(perHost)
+	crwlTr := newCrawlTransport()
+
+	jar, err := cookiejar.New(nil)
 	if err != nil {
 		return nil, err
 	}
 
 	return &Clients{
-		API: &http.Client{Transport: apiTr, Timeout: 20 * time.Second},
+		General: &http.Client{Transport: genTr, Timeout: 20 * time.Second},
 		Probe: &http.Client{
 			Transport: probTr,
 			// for probes, prefer per-request context timeouts; keep a hard cap anyway
@@ -56,20 +59,18 @@ func NewClients() (*Clients, error) {
 			Transport: crwlTr,
 			// crawls can legitimately take longer. Use request contexts to bound if needed
 			Timeout: 45 * time.Second,
+			Jar:     jar,
 		},
-		apiTr:  apiTr,
+		genTr:  genTr,
 		probTr: probTr,
 		crwlTr: crwlTr,
 	}, nil
 }
 
-func newDefaultTransport() *http.Transport {
-	d := baseDialer(5*time.Second, 30*time.Second)
-
+func newGeneralTransport() *http.Transport {
 	return &http.Transport{
-		Proxy: http.ProxyFromEnvironment,
-
-		DialContext:           d.DialContext,
+		Proxy:                 http.ProxyFromEnvironment,
+		DialContext:           amassnet.NewDialContext(5 * time.Second),
 		ForceAttemptHTTP2:     true,
 		MaxIdleConns:          200,
 		MaxIdleConnsPerHost:   20,
@@ -78,98 +79,53 @@ func newDefaultTransport() *http.Transport {
 		TLSHandshakeTimeout:   5 * time.Second,
 		ExpectContinueTimeout: 1 * time.Second,
 		ResponseHeaderTimeout: 8 * time.Second,
-
 		// prefer correct TLS verification for APIs
 		TLSClientConfig: &tls.Config{
 			MinVersion: tls.VersionTLS12,
 		},
-
 		DisableCompression: false, // allow gzip
 	}
 }
 
-// ---- Probing client: short-lived, low reuse, high fan-out ----
-//
-// Intended for: HEAD/GET for status, banner grabs, lightweight fetches.
-// Strategy: limit keep-alives so you don't retain lots of idle sockets,
-// keep timeouts tight, and cap per-host concurrency.
-
 func newProbeTransport(perHost int) *http.Transport {
-	d := baseDialer(2*time.Second, 10*time.Second)
-
 	return &http.Transport{
-		Proxy: http.ProxyFromEnvironment,
-
-		DialContext:       d.DialContext,
+		Proxy:             http.ProxyFromEnvironment,
+		DialContext:       amassnet.NewDialContext(2 * time.Second),
 		ForceAttemptHTTP2: true,
-
 		// keep this lower: probes spray across many hosts; idle pools become “memory”
-		MaxIdleConns:        128,
-		MaxIdleConnsPerHost: 2,
-		MaxConnsPerHost:     10,
-		IdleConnTimeout:     20 * time.Second,
-
+		MaxIdleConns:          128,
+		MaxIdleConnsPerHost:   2,
+		MaxConnsPerHost:       10,
+		IdleConnTimeout:       20 * time.Second,
 		TLSHandshakeTimeout:   3 * time.Second,
 		ExpectContinueTimeout: 0,
 		ResponseHeaderTimeout: 4 * time.Second,
-
 		// often you’ll hit junk certs during probing; keep verification on by default.
 		// if you *must* allow insecure probing, fork a separate transport with InsecureSkipVerify=true
 		// but be intentional about it
 		TLSClientConfig: &tls.Config{
 			MinVersion: tls.VersionTLS12,
 		},
-
 		DisableCompression: true, // avoid spending CPU on gzip for tiny probe responses
 	}
 }
 
-// ---- Crawling client: polite, reusable connections, cookies, longer reads ----
-//
-// Intended for: fetching pages/assets, following redirects (default policy),
-// dealing with cookies, compression.
-//
-// Note: Cookie jar can grow; if you don’t need cookies, remove the jar.
-
-func newCrawlTransport() (*http.Transport, error) {
-	d := baseDialer(6*time.Second, 30*time.Second)
-
-	// crawlers often benefit from cookies (sessions, pagination, etc.).
-	// if you don't want cookies, you can omit the jar and still use the transport
-	_, err := cookiejar.New(nil)
-	if err != nil {
-		return nil, err
-	}
-
+func newCrawlTransport() *http.Transport {
 	return &http.Transport{
-		Proxy: http.ProxyFromEnvironment,
-
-		DialContext:       d.DialContext,
+		Proxy:             http.ProxyFromEnvironment,
+		DialContext:       amassnet.NewDialContext(6 * time.Second),
 		ForceAttemptHTTP2: true,
-
 		// crawling usually hits same hosts repeatedly; keep-alives pay off
-		MaxIdleConns:        512,
-		MaxIdleConnsPerHost: 32,
-		MaxConnsPerHost:     32,
-		IdleConnTimeout:     120 * time.Second,
-
+		MaxIdleConns:          512,
+		MaxIdleConnsPerHost:   32,
+		MaxConnsPerHost:       32,
+		IdleConnTimeout:       120 * time.Second,
 		TLSHandshakeTimeout:   6 * time.Second,
 		ExpectContinueTimeout: 1 * time.Second,
 		ResponseHeaderTimeout: 12 * time.Second,
-
 		TLSClientConfig: &tls.Config{
 			MinVersion: tls.VersionTLS12,
 		},
-
 		DisableCompression: false, // allow gzip to reduce bandwidth for HTML/JS/CSS
-	}, nil
-}
-
-// baseDialer returns a dialer with sane defaults.
-// keepAlive controls TCP keepalive probes at OS layer; it is NOT HTTP keep-alive.
-func baseDialer(timeout, keepAlive time.Duration) *net.Dialer {
-	return &net.Dialer{
-		Timeout:   timeout,
-		KeepAlive: keepAlive,
 	}
 }

engine/sessions/session.go

@@ -45,6 +45,7 @@ type Session struct {
 	pipelines    et.SessionPipelines
 	dsn          string
 	dbtype       string
+	clients      *Clients
 	ranger       cidranger.Ranger
 	tmpdir       string
 	stats        *et.SessionStats
@@ -62,6 +63,11 @@ func CreateSession(mgr *manager, reg et.Registry, cfg *config.Config) (et.Sessio
 		cfg = config.NewConfig()
 	}
 
+	clients, err := NewClients(len(cfg.Scope.Ports))
+	if err != nil {
+		return nil, err
+	}
+
 	startTime := time.Now()
 	numOfSessions := mgr.NumOfSessions() + 1
 	ctx, cancel := context.WithCancel(context.Background())
@@ -73,6 +79,7 @@ func CreateSession(mgr *manager, reg et.Registry, cfg *config.Config) (et.Sessio
 		cancel:       cancel,
 		cfg:          cfg,
 		start:        startTime,
+		clients:      clients,
 		ranger:       NewAmassRanger(),
 		ps:           pubsub.NewLogger(),
 		stats:        new(et.SessionStats),
@@ -83,7 +90,7 @@ func CreateSession(mgr *manager, reg et.Registry, cfg *config.Config) (et.Sessio
 	s.scope = scope.CreateFromConfigScope(s)
 	s.log = slog.New(slog.NewJSONHandler(s.ps, nil)).With("session", s.id)
 
-	err := s.setupDB()
+	err = s.setupDB()
 	if err != nil {
 		s.Kill()
 		return nil, err
@@ -160,7 +167,11 @@ func (s *Session) Pipelines() et.SessionPipelines {
 }
 
 func (s *Session) Clients() *et.SessionHTTPClients {
-	return nil
+	return &et.SessionHTTPClients{
+		General: s.clients.General,
+		Probe:   s.clients.Probe,
+		Crawl:   s.clients.Crawl,
+	}
 }
 
 func (s *Session) CIDRanger() cidranger.Ranger {
@@ -189,6 +200,7 @@ func (s *Session) Kill() {
 
 	s.cancel()
 	s.finished = true
+	s.clients.CloseIdleConnections()
 }
 
 func (s *Session) setupDB() error {

engine/types/sessions.go

@@ -33,7 +33,7 @@ type SessionSemaphone interface {
 }
 
 type SessionHTTPClients struct {
-	Default *http.Client
+	General *http.Client
 	Probe   *http.Client
 	Crawl   *http.Client
 }

internal/net/http/http.go

@@ -170,7 +170,7 @@ func RespToAmassResponse(resp *http.Response) *Response {
 }
 
 // RequestWebPage returns the response headers, body, and status code for the provided URL when successful.
-func RequestWebPage(ctx context.Context, r *Request) (*Response, error) {
+func RequestWebPage(ctx context.Context, client *http.Client, r *Request) (*Response, error) {
 	if r == nil {
 		return nil, errors.New("failed to provide a valid Amass HTTP request")
 	}
@@ -201,7 +201,7 @@ func RequestWebPage(ctx context.Context, r *Request) (*Response, error) {
 	}
 
 	amassnet.MaxNetConnSem.Acquire()
-	resp, err := DefaultClient.Do(req)
+	resp, err := client.Do(req)
 	amassnet.MaxNetConnSem.Release()
 	if err != nil {
 		return nil, err

internal/net/http/http_test.go

@@ -50,7 +50,7 @@ func TestRequestWebPage(t *testing.T) {
 	}))
 	defer ts.Close()
 
-	resp, err := RequestWebPage(context.TODO(), &Request{
+	resp, err := RequestWebPage(context.TODO(), DefaultClient, &Request{
 		URL:  ts.URL,
 		Auth: &BasicAuth{name, pass},
 	})
@@ -58,7 +58,7 @@ func TestRequestWebPage(t *testing.T) {
 		t.Error("Failed to detect the bad request")
 	}
 
-	resp, err = RequestWebPage(context.TODO(), &Request{
+	resp, err = RequestWebPage(context.TODO(), DefaultClient, &Request{
 		URL:    ts.URL,
 		Method: "POST",
 		Header: Header{hkey: []string{name}},
@@ -72,7 +72,7 @@ func TestRequestWebPage(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	cancel()
 
-	resp, err = RequestWebPage(ctx, &Request{URL: ts.URL})
+	resp, err = RequestWebPage(ctx, DefaultClient, &Request{URL: ts.URL})
 	if err == nil || resp != nil {
 		t.Error("Failed to detect the expired context")
 	}