diff --git a/config/resolvers.go b/config/resolvers.go index 88dbe2da..a61b40cc 100644 --- a/config/resolvers.go +++ b/config/resolvers.go @@ -17,7 +17,7 @@ import ( "strings" "github.com/caffix/stringset" - "github.com/owasp-amass/amass/v5/internal/net/http" + amasshttp "github.com/owasp-amass/amass/v5/internal/net/http" ) // DefaultQueriesPerPublicResolver is the number of queries sent to each public DNS resolver per second. @@ -54,7 +54,8 @@ var PublicResolvers []string // GetPublicDNSResolvers obtains the public DNS server addresses from public-dns.info and assigns them to PublicResolvers. func GetPublicDNSResolvers() error { url := "https://public-dns.info/nameservers-all.csv" - resp, err := http.RequestWebPage(context.Background(), &http.Request{URL: url}) + resp, err := amasshttp.RequestWebPage(context.Background(), + amasshttp.DefaultClient, &amasshttp.Request{URL: url}) if err != nil || resp.StatusCode < 200 || resp.StatusCode >= 400 { return fmt.Errorf("failed to obtain the Public DNS csv file at %s: %v", url, err) } diff --git a/engine/plugins/api/binaryedge.go b/engine/plugins/api/binaryedge.go index 066d73aa..025c53c6 100644 --- a/engine/plugins/api/binaryedge.go +++ b/engine/plugins/api/binaryedge.go @@ -16,7 +16,7 @@ import ( "github.com/caffix/stringset" "github.com/owasp-amass/amass/v5/engine/plugins/support" et "github.com/owasp-amass/amass/v5/engine/types" - "github.com/owasp-amass/amass/v5/internal/net/http" + amasshttp "github.com/owasp-amass/amass/v5/internal/net/http" dbt "github.com/owasp-amass/asset-db/types" oam "github.com/owasp-amass/open-asset-model" oamdns "github.com/owasp-amass/open-asset-model/dns" @@ -121,8 +121,8 @@ loop: ctx, cancel := context.WithTimeout(e.Session.Ctx(), 5*time.Second) defer cancel() - resp, err := http.RequestWebPage(ctx, &http.Request{ - Header: http.Header{"X-KEY": []string{key}}, + resp, err := amasshttp.RequestWebPage(ctx, e.Session.Clients().General, &amasshttp.Request{ + Header: amasshttp.Header{"X-KEY": []string{key}}, URL: "https://api.binaryedge.io/v2/query/domains/subdomain/" + name + "?page=" + strconv.Itoa(pagenum), }) if err != nil || resp.Body == "" { diff --git a/engine/plugins/api/passivetotal.go b/engine/plugins/api/passivetotal.go index 92848c14..13094420 100644 --- a/engine/plugins/api/passivetotal.go +++ b/engine/plugins/api/passivetotal.go @@ -14,8 +14,8 @@ import ( "github.com/owasp-amass/amass/v5/config" "github.com/owasp-amass/amass/v5/engine/plugins/support" et "github.com/owasp-amass/amass/v5/engine/types" - "github.com/owasp-amass/amass/v5/internal/net/dns" - "github.com/owasp-amass/amass/v5/internal/net/http" + amassdns "github.com/owasp-amass/amass/v5/internal/net/dns" + amasshttp "github.com/owasp-amass/amass/v5/internal/net/http" dbt "github.com/owasp-amass/asset-db/types" oam "github.com/owasp-amass/open-asset-model" oamdns "github.com/owasp-amass/open-asset-model/dns" @@ -122,13 +122,15 @@ loop: ctx, cancel := context.WithTimeout(e.Session.Ctx(), 5*time.Second) defer cancel() - resp, err := http.RequestWebPage(ctx, &http.Request{ + e.Session.NetSem().Acquire() + resp, err := amasshttp.RequestWebPage(ctx, e.Session.Clients().General, &amasshttp.Request{ URL: url, - Auth: &http.BasicAuth{ + Auth: &amasshttp.BasicAuth{ Username: cr.Username, Password: cr.Apikey, }, }) + e.Session.NetSem().Release() if err != nil || resp.Body == "" { continue } @@ -143,7 +145,7 @@ loop: } for _, sub := range result.Subdomains { - n := dns.RemoveAsteriskLabel(http.CleanName(sub + "." + name)) + n := amassdns.RemoveAsteriskLabel(amasshttp.CleanName(sub + "." + name)) // if the subdomain is not in scope, skip it if _, conf := e.Session.Scope().IsAssetInScope(&oamdns.FQDN{Name: n}, 0); conf > 0 { names.Insert(n) diff --git a/engine/plugins/api/prospeo.go b/engine/plugins/api/prospeo.go index e2b3ac16..3932642e 100644 --- a/engine/plugins/api/prospeo.go +++ b/engine/plugins/api/prospeo.go @@ -14,7 +14,7 @@ import ( "github.com/owasp-amass/amass/v5/engine/plugins/support" et "github.com/owasp-amass/amass/v5/engine/types" - "github.com/owasp-amass/amass/v5/internal/net/http" + amasshttp "github.com/owasp-amass/amass/v5/internal/net/http" dbt "github.com/owasp-amass/asset-db/types" oam "github.com/owasp-amass/open-asset-model" oamdns "github.com/owasp-amass/open-asset-model/dns" @@ -108,27 +108,29 @@ func (p *Prospeo) query(e *et.Event, name string) []*dbt.Entity { return nil } - rcreds, err := p.accountType(e.Session.Ctx(), key) + rcreds, err := p.accountType(e.Session, key) if err != nil || key == "" { return nil } - count, err := p.count(e.Session.Ctx(), name, key) + count, err := p.count(e.Session, name, key) if err != nil { return nil } limit := min(rcreds*50, count) _ = p.rlimit.Wait(e.Session.Ctx()) + e.Session.NetSem().Acquire() ctx, cancel := context.WithTimeout(e.Session.Ctx(), 5*time.Second) defer cancel() - resp, err := http.RequestWebPage(ctx, &http.Request{ + resp, err := amasshttp.RequestWebPage(ctx, e.Session.Clients().General, &amasshttp.Request{ Method: "POST", Body: `{"company": "` + name + `", "limit": ` + strconv.Itoa(limit) + `}`, URL: p.queryurl, - Header: http.Header{"Content-Type": []string{"application/json"}, "X-KEY": []string{key}}, + Header: amasshttp.Header{"Content-Type": []string{"application/json"}, "X-KEY": []string{key}}, }) + e.Session.NetSem().Release() if err != nil { return nil } @@ -159,17 +161,19 @@ func (p *Prospeo) process(e *et.Event, assets []*dbt.Entity) { support.ProcessEmailsWithSource(e, assets, p.source) } -func (p *Prospeo) accountType(ctx context.Context, key string) (int, error) { - _ = p.rlimit.Wait(ctx) +func (p *Prospeo) accountType(sess et.Session, key string) (int, error) { + _ = p.rlimit.Wait(sess.Ctx()) - rctx, cancel := context.WithTimeout(ctx, 5*time.Second) + sess.NetSem().Acquire() + rctx, cancel := context.WithTimeout(sess.Ctx(), 5*time.Second) defer cancel() - resp, err := http.RequestWebPage(rctx, &http.Request{ + resp, err := amasshttp.RequestWebPage(rctx, sess.Clients().General, &amasshttp.Request{ Method: "POST", URL: p.accturl, - Header: http.Header{"Content-Type": []string{"application/json"}, "X-KEY": []string{key}}, + Header: amasshttp.Header{"Content-Type": []string{"application/json"}, "X-KEY": []string{key}}, }) + sess.NetSem().Release() if err != nil { return 0, err } @@ -185,18 +189,20 @@ func (p *Prospeo) accountType(ctx context.Context, key string) (int, error) { return r.Response.RemainingCredits, nil } -func (p *Prospeo) count(ctx context.Context, domain string, key string) (int, error) { - _ = p.rlimit.Wait(ctx) +func (p *Prospeo) count(sess et.Session, domain string, key string) (int, error) { + _ = p.rlimit.Wait(sess.Ctx()) - rctx, cancel := context.WithTimeout(ctx, 5*time.Second) + sess.NetSem().Acquire() + rctx, cancel := context.WithTimeout(sess.Ctx(), 5*time.Second) defer cancel() - resp, err := http.RequestWebPage(rctx, &http.Request{ + resp, err := amasshttp.RequestWebPage(rctx, sess.Clients().General, &amasshttp.Request{ Method: "POST", Body: `{"domain": "` + domain + `"}`, URL: p.counturl, - Header: http.Header{"Content-Type": []string{"application/json"}, "X-KEY": []string{key}}, + Header: amasshttp.Header{"Content-Type": []string{"application/json"}, "X-KEY": []string{key}}, }) + sess.NetSem().Release() if err != nil { return 0, err } diff --git a/engine/plugins/api/zetalytics.go b/engine/plugins/api/zetalytics.go index 53bc80aa..bdeee2cd 100644 --- a/engine/plugins/api/zetalytics.go +++ b/engine/plugins/api/zetalytics.go @@ -15,8 +15,8 @@ import ( "github.com/owasp-amass/amass/v5/engine/plugins/support" et "github.com/owasp-amass/amass/v5/engine/types" - "github.com/owasp-amass/amass/v5/internal/net/dns" - "github.com/owasp-amass/amass/v5/internal/net/http" + amassdns "github.com/owasp-amass/amass/v5/internal/net/dns" + amasshttp "github.com/owasp-amass/amass/v5/internal/net/http" dbt "github.com/owasp-amass/asset-db/types" oam "github.com/owasp-amass/open-asset-model" oamdns "github.com/owasp-amass/open-asset-model/dns" @@ -122,7 +122,9 @@ func (z *zetalytics) query(e *et.Event, name string, keys []string) []*dbt.Entit ctx, cancel := context.WithTimeout(e.Session.Ctx(), 5*time.Second) defer cancel() - resp, err := http.RequestWebPage(ctx, &http.Request{URL: url}) + e.Session.NetSem().Acquire() + resp, err := amasshttp.RequestWebPage(ctx, e.Session.Clients().General, &amasshttp.Request{URL: url}) + e.Session.NetSem().Release() if err != nil || resp.Body == "" { continue } @@ -141,7 +143,7 @@ func (z *zetalytics) query(e *et.Event, name string, keys []string) []*dbt.Entit } for _, s := range result.Subdomains { - name := strings.ToLower(strings.TrimSpace(dns.RemoveAsteriskLabel(http.CleanName(s.FQDN)))) + name := strings.ToLower(strings.TrimSpace(amassdns.RemoveAsteriskLabel(amasshttp.CleanName(s.FQDN)))) // if the subdomain is not in scope, skip it if _, conf := e.Session.Scope().IsAssetInScope(&oamdns.FQDN{Name: name}, 0); conf > 0 { names.Insert(name) diff --git a/engine/plugins/archive/wayback.go b/engine/plugins/archive/wayback.go index 3fa1fcec..fd9931da 100644 --- a/engine/plugins/archive/wayback.go +++ b/engine/plugins/archive/wayback.go @@ -15,8 +15,8 @@ import ( "github.com/caffix/stringset" "github.com/owasp-amass/amass/v5/engine/plugins/support" et "github.com/owasp-amass/amass/v5/engine/types" - "github.com/owasp-amass/amass/v5/internal/net/dns" - "github.com/owasp-amass/amass/v5/internal/net/http" + amassdns "github.com/owasp-amass/amass/v5/internal/net/dns" + amasshttp "github.com/owasp-amass/amass/v5/internal/net/http" dbt "github.com/owasp-amass/asset-db/types" oam "github.com/owasp-amass/open-asset-model" oamdns "github.com/owasp-amass/open-asset-model/dns" @@ -106,7 +106,10 @@ func (w *wayback) query(e *et.Event, name string) []*dbt.Entity { ctx, cancel := context.WithTimeout(e.Session.Ctx(), 30*time.Second) defer cancel() - resp, err := http.RequestWebPage(ctx, &http.Request{URL: w.URL + end}) + e.Session.NetSem().Acquire() + resp, err := amasshttp.RequestWebPage(ctx, + e.Session.Clients().General, &amasshttp.Request{URL: w.URL + end}) + e.Session.NetSem().Release() if err != nil { return nil } @@ -125,7 +128,7 @@ func (w *wayback) query(e *et.Event, name string) []*dbt.Entity { } u := url[0] - if n := dns.AnySubdomainRegex().FindString(u); n != "" { + if n := amassdns.AnySubdomainRegex().FindString(u); n != "" { // if the subdomain is not in scope, skip it if _, conf := e.Session.Scope().IsAssetInScope(&oamdns.FQDN{Name: n}, 0); conf > 0 { subs.Insert(n) diff --git a/engine/plugins/scrape/bing.go b/engine/plugins/scrape/bing.go index 068a5654..495f2335 100644 --- a/engine/plugins/scrape/bing.go +++ b/engine/plugins/scrape/bing.go @@ -15,7 +15,7 @@ import ( "github.com/caffix/stringset" "github.com/owasp-amass/amass/v5/engine/plugins/support" et "github.com/owasp-amass/amass/v5/engine/types" - "github.com/owasp-amass/amass/v5/internal/net/http" + amasshttp "github.com/owasp-amass/amass/v5/internal/net/http" dbt "github.com/owasp-amass/asset-db/types" oam "github.com/owasp-amass/open-asset-model" oamdns "github.com/owasp-amass/open-asset-model/dns" @@ -109,7 +109,10 @@ func (b *bing) query(e *et.Event, name string) []*dbt.Entity { ctx, cancel := context.WithTimeout(e.Session.Ctx(), 10*time.Second) defer cancel() - resp, err := http.RequestWebPage(ctx, &http.Request{URL: fmt.Sprintf(b.fmtstr, i, name, name)}) + e.Session.NetSem().Acquire() + resp, err := amasshttp.RequestWebPage(ctx, e.Session.Clients().General, + &amasshttp.Request{URL: fmt.Sprintf(b.fmtstr, i, name, name)}) + e.Session.NetSem().Release() if err != nil || resp.Body == "" { break } diff --git a/engine/plugins/scrape/dnshistory.go b/engine/plugins/scrape/dnshistory.go index 6dfc6879..2c6c7714 100644 --- a/engine/plugins/scrape/dnshistory.go +++ b/engine/plugins/scrape/dnshistory.go @@ -15,7 +15,7 @@ import ( "github.com/caffix/stringset" "github.com/owasp-amass/amass/v5/engine/plugins/support" et "github.com/owasp-amass/amass/v5/engine/types" - "github.com/owasp-amass/amass/v5/internal/net/http" + amasshttp "github.com/owasp-amass/amass/v5/internal/net/http" dbt "github.com/owasp-amass/asset-db/types" oam "github.com/owasp-amass/open-asset-model" oamdns "github.com/owasp-amass/open-asset-model/dns" @@ -109,7 +109,10 @@ func (d *dnsHistory) query(e *et.Event, name string) []*dbt.Entity { ctx, cancel := context.WithTimeout(e.Session.Ctx(), 10*time.Second) defer cancel() - resp, err := http.RequestWebPage(ctx, &http.Request{URL: fmt.Sprintf(d.fmtstr, i, name)}) + e.Session.NetSem().Acquire() + resp, err := amasshttp.RequestWebPage(ctx, + e.Session.Clients().General, &amasshttp.Request{URL: fmt.Sprintf(d.fmtstr, i, name)}) + e.Session.NetSem().Release() if err != nil || resp.Body == "" { break } diff --git a/engine/plugins/scrape/duckduckgo.go b/engine/plugins/scrape/duckduckgo.go index 7eaef9d6..da565893 100644 --- a/engine/plugins/scrape/duckduckgo.go +++ b/engine/plugins/scrape/duckduckgo.go @@ -15,7 +15,7 @@ import ( "github.com/caffix/stringset" "github.com/owasp-amass/amass/v5/engine/plugins/support" et "github.com/owasp-amass/amass/v5/engine/types" - "github.com/owasp-amass/amass/v5/internal/net/http" + amasshttp "github.com/owasp-amass/amass/v5/internal/net/http" dbt "github.com/owasp-amass/asset-db/types" oam "github.com/owasp-amass/open-asset-model" oamdns "github.com/owasp-amass/open-asset-model/dns" @@ -107,7 +107,10 @@ func (d *duckDuckGo) query(e *et.Event, name string) []*dbt.Entity { ctx, cancel := context.WithTimeout(e.Session.Ctx(), 10*time.Second) defer cancel() - resp, err := http.RequestWebPage(ctx, &http.Request{URL: fmt.Sprintf(d.fmtstr, name, name)}) + e.Session.NetSem().Acquire() + resp, err := amasshttp.RequestWebPage(ctx, + e.Session.Clients().General, &amasshttp.Request{URL: fmt.Sprintf(d.fmtstr, name, name)}) + e.Session.NetSem().Release() if err != nil || resp.Body == "" { return nil } diff --git a/engine/plugins/scrape/ipverse.go b/engine/plugins/scrape/ipverse.go index 32e3e81b..dc2d4a4d 100644 --- a/engine/plugins/scrape/ipverse.go +++ b/engine/plugins/scrape/ipverse.go @@ -14,7 +14,7 @@ import ( "github.com/owasp-amass/amass/v5/engine/plugins/support" et "github.com/owasp-amass/amass/v5/engine/types" - "github.com/owasp-amass/amass/v5/internal/net/http" + amasshttp "github.com/owasp-amass/amass/v5/internal/net/http" dbt "github.com/owasp-amass/asset-db/types" oam "github.com/owasp-amass/open-asset-model" oamnet "github.com/owasp-amass/open-asset-model/network" @@ -105,12 +105,15 @@ type record struct { func (v *ipverse) query(sess et.Session, asset *dbt.Entity) *record { _ = v.rlimit.Wait(sess.Ctx()) + as := asset.Asset.(*oamnet.AutonomousSystem) ctx, cancel := context.WithTimeout(sess.Ctx(), 30*time.Second) defer cancel() - as := asset.Asset.(*oamnet.AutonomousSystem) - resp, err := http.RequestWebPage(ctx, &http.Request{URL: fmt.Sprintf(v.fmtstr, as.Number)}) + sess.NetSem().Acquire() + resp, err := amasshttp.RequestWebPage(ctx, + sess.Clients().General, &amasshttp.Request{URL: fmt.Sprintf(v.fmtstr, as.Number)}) + sess.NetSem().Release() if err != nil || resp.Body == "" { return nil } diff --git a/engine/plugins/scrape/rapiddns.go b/engine/plugins/scrape/rapiddns.go index f36e5c10..bb59caa1 100644 --- a/engine/plugins/scrape/rapiddns.go +++ b/engine/plugins/scrape/rapiddns.go @@ -15,7 +15,7 @@ import ( "github.com/caffix/stringset" "github.com/owasp-amass/amass/v5/engine/plugins/support" et "github.com/owasp-amass/amass/v5/engine/types" - "github.com/owasp-amass/amass/v5/internal/net/http" + amasshttp "github.com/owasp-amass/amass/v5/internal/net/http" dbt "github.com/owasp-amass/asset-db/types" oam "github.com/owasp-amass/open-asset-model" oamdns "github.com/owasp-amass/open-asset-model/dns" @@ -107,7 +107,10 @@ func (rd *rapidDNS) query(e *et.Event, name string) []*dbt.Entity { ctx, cancel := context.WithTimeout(e.Session.Ctx(), 10*time.Second) defer cancel() - resp, err := http.RequestWebPage(ctx, &http.Request{URL: fmt.Sprintf(rd.fmtstr, name)}) + e.Session.NetSem().Acquire() + resp, err := amasshttp.RequestWebPage(ctx, + e.Session.Clients().General, &amasshttp.Request{URL: fmt.Sprintf(rd.fmtstr, name)}) + e.Session.NetSem().Release() if err != nil || resp.Body == "" { return nil } diff --git a/engine/plugins/scrape/sitedossier.go b/engine/plugins/scrape/sitedossier.go index 0740cf42..3c11c1cb 100644 --- a/engine/plugins/scrape/sitedossier.go +++ b/engine/plugins/scrape/sitedossier.go @@ -15,7 +15,7 @@ import ( "github.com/caffix/stringset" "github.com/owasp-amass/amass/v5/engine/plugins/support" et "github.com/owasp-amass/amass/v5/engine/types" - "github.com/owasp-amass/amass/v5/internal/net/http" + amasshttp "github.com/owasp-amass/amass/v5/internal/net/http" dbt "github.com/owasp-amass/asset-db/types" oam "github.com/owasp-amass/open-asset-model" oamdns "github.com/owasp-amass/open-asset-model/dns" @@ -109,7 +109,10 @@ func (sd *siteDossier) query(e *et.Event, name string) []*dbt.Entity { ctx, cancel := context.WithTimeout(e.Session.Ctx(), 10*time.Second) defer cancel() - resp, err := http.RequestWebPage(ctx, &http.Request{URL: fmt.Sprintf(sd.fmtstr, name, i)}) + e.Session.NetSem().Acquire() + resp, err := amasshttp.RequestWebPage(ctx, + e.Session.Clients().General, &amasshttp.Request{URL: fmt.Sprintf(sd.fmtstr, name, i)}) + e.Session.NetSem().Release() if err != nil || resp.Body == "" { break } diff --git a/engine/plugins/service_discovery/http_probes/plugin.go b/engine/plugins/service_discovery/http_probes/plugin.go index 56985d21..fdaf8a4e 100644 --- a/engine/plugins/service_discovery/http_probes/plugin.go +++ b/engine/plugins/service_discovery/http_probes/plugin.go @@ -15,7 +15,7 @@ import ( "github.com/owasp-amass/amass/v5/engine/plugins/support" et "github.com/owasp-amass/amass/v5/engine/types" - "github.com/owasp-amass/amass/v5/internal/net/http" + amasshttp "github.com/owasp-amass/amass/v5/internal/net/http" dbt "github.com/owasp-amass/asset-db/types" oam "github.com/owasp-amass/open-asset-model" oamcert "github.com/owasp-amass/open-asset-model/certificate" @@ -106,7 +106,7 @@ func (hp *httpProbing) query(e *et.Event, entity *dbt.Entity, target string, por defer cancel() e.Session.NetSem().Acquire() - resp, err := http.RequestWebPage(ctx, &http.Request{URL: target}) + resp, err := amasshttp.RequestWebPage(ctx, e.Session.Clients().Probe, &amasshttp.Request{URL: target}) e.Session.NetSem().Release() if err == nil && resp != nil { @@ -115,7 +115,7 @@ func (hp *httpProbing) query(e *et.Event, entity *dbt.Entity, target string, por return findings } -func (hp *httpProbing) store(e *et.Event, resp *http.Response, entity *dbt.Entity, port int) []*support.Finding { +func (hp *httpProbing) store(e *et.Event, resp *amasshttp.Response, entity *dbt.Entity, port int) []*support.Finding { addr := entity.Asset.Key() hp.servlock.Lock() @@ -174,7 +174,7 @@ func (hp *httpProbing) store(e *et.Event, resp *http.Response, entity *dbt.Entit return findings } -func (hp *httpProbing) createCertificates(sess et.Session, resp *http.Response) (*dbt.Entity, *x509.Certificate, []*support.Finding) { +func (hp *httpProbing) createCertificates(sess et.Session, resp *amasshttp.Response) (*dbt.Entity, *x509.Certificate, []*support.Finding) { var findings []*support.Finding if resp.TLS == nil || !resp.TLS.HandshakeComplete { diff --git a/engine/plugins/support/fingerprinting.go b/engine/plugins/support/fingerprinting.go index a8cc5cc8..ed31a5f1 100644 --- a/engine/plugins/support/fingerprinting.go +++ b/engine/plugins/support/fingerprinting.go @@ -12,7 +12,7 @@ import ( "time" jarm "github.com/caffix/jarm-go" - "github.com/owasp-amass/amass/v5/internal/net" + amassnet "github.com/owasp-amass/amass/v5/internal/net" oam "github.com/owasp-amass/open-asset-model" oamdns "github.com/owasp-amass/open-asset-model/dns" "github.com/owasp-amass/open-asset-model/general" @@ -43,7 +43,8 @@ func JARMFingerprint(target oam.Asset, portrel *general.PortRelation) (string, e ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - c, err := net.DialContext(ctx, "tcp", addr) + dial := amassnet.NewDialContext(5 * time.Second) + c, err := dial(ctx, "tcp", addr) if err != nil { return "", err } diff --git a/engine/plugins/support/location.go b/engine/plugins/support/location.go index 0ec395d6..851913af 100644 --- a/engine/plugins/support/location.go +++ b/engine/plugins/support/location.go @@ -127,7 +127,7 @@ func postalServerParseAddress(address string) ([]parsedComponent, error) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - resp, err := amasshttp.RequestWebPage(ctx, &amasshttp.Request{ + resp, err := amasshttp.RequestWebPage(ctx, amasshttp.DefaultClient, &amasshttp.Request{ Method: "POST", URL: "http://" + postalHost + ":" + postalPort + "/parse", Body: string(reqJSON), @@ -147,7 +147,7 @@ func checkPostalServerHealth() bool { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - if resp, err := amasshttp.RequestWebPage(ctx, &amasshttp.Request{ + if resp, err := amasshttp.RequestWebPage(ctx, amasshttp.DefaultClient, &amasshttp.Request{ URL: "http://" + postalHost + ":" + postalPort + "/health", }); err != nil || resp == nil || resp.StatusCode != http.StatusOK { return false diff --git a/engine/plugins/whois/bgptools/plugin.go b/engine/plugins/whois/bgptools/plugin.go index 0ca8568f..4869d781 100644 --- a/engine/plugins/whois/bgptools/plugin.go +++ b/engine/plugins/whois/bgptools/plugin.go @@ -129,13 +129,14 @@ type bgpToolsRecord struct { } func (bt *bgpTools) whois(ctx context.Context, ipstr string) (*bgpToolsRecord, error) { + dial := amassnet.NewDialContext(3 * time.Second) addr := net.JoinHostPort(bt.addr, strconv.Itoa(bt.port)) _ = bt.rlimit.Wait(ctx) ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() - conn, err := amassnet.DialContext(ctx, "tcp", addr) + conn, err := dial(ctx, "tcp", addr) if err != nil { return nil, fmt.Errorf("failed to establish a connection with the WHOIS server: %v", err) } diff --git a/engine/sessions/http_clients.go b/engine/sessions/http_clients.go index 953ea8f4..568cc87c 100644 --- a/engine/sessions/http_clients.go +++ b/engine/sessions/http_clients.go @@ -6,27 +6,28 @@ package sessions import ( "crypto/tls" - "net" "net/http" "net/http/cookiejar" "time" + + amassnet "github.com/owasp-amass/amass/v5/internal/net" ) // Clients bundles the three clients + transports so you can close idle conns. type Clients struct { - API *http.Client - Probe *http.Client - Crawl *http.Client - apiTr *http.Transport - probTr *http.Transport - crwlTr *http.Transport + General *http.Client + Probe *http.Client + Crawl *http.Client + genTr *http.Transport + probTr *http.Transport + crwlTr *http.Transport } // CloseIdleConnections is useful on session/engine shutdown. // It does not kill in-flight requests, but it releases keep-alive sockets. func (c *Clients) CloseIdleConnections() { - if c.apiTr != nil { - c.apiTr.CloseIdleConnections() + if c.genTr != nil { + c.genTr.CloseIdleConnections() } if c.probTr != nil { c.probTr.CloseIdleConnections() @@ -37,16 +38,18 @@ func (c *Clients) CloseIdleConnections() { } // NewClients returns three tuned clients: API, Probe, Crawl. -func NewClients() (*Clients, error) { - apiTr := newDefaultTransport() - probTr := newProbeTransport(0) - crwlTr, err := newCrawlTransport() +func NewClients(perHost int) (*Clients, error) { + genTr := newGeneralTransport() + probTr := newProbeTransport(perHost) + crwlTr := newCrawlTransport() + + jar, err := cookiejar.New(nil) if err != nil { return nil, err } return &Clients{ - API: &http.Client{Transport: apiTr, Timeout: 20 * time.Second}, + General: &http.Client{Transport: genTr, Timeout: 20 * time.Second}, Probe: &http.Client{ Transport: probTr, // for probes, prefer per-request context timeouts; keep a hard cap anyway @@ -56,20 +59,18 @@ func NewClients() (*Clients, error) { Transport: crwlTr, // crawls can legitimately take longer. Use request contexts to bound if needed Timeout: 45 * time.Second, + Jar: jar, }, - apiTr: apiTr, + genTr: genTr, probTr: probTr, crwlTr: crwlTr, }, nil } -func newDefaultTransport() *http.Transport { - d := baseDialer(5*time.Second, 30*time.Second) - +func newGeneralTransport() *http.Transport { return &http.Transport{ - Proxy: http.ProxyFromEnvironment, - - DialContext: d.DialContext, + Proxy: http.ProxyFromEnvironment, + DialContext: amassnet.NewDialContext(5 * time.Second), ForceAttemptHTTP2: true, MaxIdleConns: 200, MaxIdleConnsPerHost: 20, @@ -78,98 +79,53 @@ func newDefaultTransport() *http.Transport { TLSHandshakeTimeout: 5 * time.Second, ExpectContinueTimeout: 1 * time.Second, ResponseHeaderTimeout: 8 * time.Second, - // prefer correct TLS verification for APIs TLSClientConfig: &tls.Config{ MinVersion: tls.VersionTLS12, }, - DisableCompression: false, // allow gzip } } -// ---- Probing client: short-lived, low reuse, high fan-out ---- -// -// Intended for: HEAD/GET for status, banner grabs, lightweight fetches. -// Strategy: limit keep-alives so you don't retain lots of idle sockets, -// keep timeouts tight, and cap per-host concurrency. - func newProbeTransport(perHost int) *http.Transport { - d := baseDialer(2*time.Second, 10*time.Second) - return &http.Transport{ - Proxy: http.ProxyFromEnvironment, - - DialContext: d.DialContext, + Proxy: http.ProxyFromEnvironment, + DialContext: amassnet.NewDialContext(2 * time.Second), ForceAttemptHTTP2: true, - // keep this lower: probes spray across many hosts; idle pools become “memory” - MaxIdleConns: 128, - MaxIdleConnsPerHost: 2, - MaxConnsPerHost: 10, - IdleConnTimeout: 20 * time.Second, - + MaxIdleConns: 128, + MaxIdleConnsPerHost: 2, + MaxConnsPerHost: 10, + IdleConnTimeout: 20 * time.Second, TLSHandshakeTimeout: 3 * time.Second, ExpectContinueTimeout: 0, ResponseHeaderTimeout: 4 * time.Second, - // often you’ll hit junk certs during probing; keep verification on by default. // if you *must* allow insecure probing, fork a separate transport with InsecureSkipVerify=true // but be intentional about it TLSClientConfig: &tls.Config{ MinVersion: tls.VersionTLS12, }, - DisableCompression: true, // avoid spending CPU on gzip for tiny probe responses } } -// ---- Crawling client: polite, reusable connections, cookies, longer reads ---- -// -// Intended for: fetching pages/assets, following redirects (default policy), -// dealing with cookies, compression. -// -// Note: Cookie jar can grow; if you don’t need cookies, remove the jar. - -func newCrawlTransport() (*http.Transport, error) { - d := baseDialer(6*time.Second, 30*time.Second) - - // crawlers often benefit from cookies (sessions, pagination, etc.). - // if you don't want cookies, you can omit the jar and still use the transport - _, err := cookiejar.New(nil) - if err != nil { - return nil, err - } - +func newCrawlTransport() *http.Transport { return &http.Transport{ - Proxy: http.ProxyFromEnvironment, - - DialContext: d.DialContext, + Proxy: http.ProxyFromEnvironment, + DialContext: amassnet.NewDialContext(6 * time.Second), ForceAttemptHTTP2: true, - // crawling usually hits same hosts repeatedly; keep-alives pay off - MaxIdleConns: 512, - MaxIdleConnsPerHost: 32, - MaxConnsPerHost: 32, - IdleConnTimeout: 120 * time.Second, - + MaxIdleConns: 512, + MaxIdleConnsPerHost: 32, + MaxConnsPerHost: 32, + IdleConnTimeout: 120 * time.Second, TLSHandshakeTimeout: 6 * time.Second, ExpectContinueTimeout: 1 * time.Second, ResponseHeaderTimeout: 12 * time.Second, - TLSClientConfig: &tls.Config{ MinVersion: tls.VersionTLS12, }, - DisableCompression: false, // allow gzip to reduce bandwidth for HTML/JS/CSS - }, nil -} - -// baseDialer returns a dialer with sane defaults. -// keepAlive controls TCP keepalive probes at OS layer; it is NOT HTTP keep-alive. -func baseDialer(timeout, keepAlive time.Duration) *net.Dialer { - return &net.Dialer{ - Timeout: timeout, - KeepAlive: keepAlive, } } diff --git a/engine/sessions/session.go b/engine/sessions/session.go index eac0b876..a9b1a4bc 100644 --- a/engine/sessions/session.go +++ b/engine/sessions/session.go @@ -45,6 +45,7 @@ type Session struct { pipelines et.SessionPipelines dsn string dbtype string + clients *Clients ranger cidranger.Ranger tmpdir string stats *et.SessionStats @@ -62,6 +63,11 @@ func CreateSession(mgr *manager, reg et.Registry, cfg *config.Config) (et.Sessio cfg = config.NewConfig() } + clients, err := NewClients(len(cfg.Scope.Ports)) + if err != nil { + return nil, err + } + startTime := time.Now() numOfSessions := mgr.NumOfSessions() + 1 ctx, cancel := context.WithCancel(context.Background()) @@ -73,6 +79,7 @@ func CreateSession(mgr *manager, reg et.Registry, cfg *config.Config) (et.Sessio cancel: cancel, cfg: cfg, start: startTime, + clients: clients, ranger: NewAmassRanger(), ps: pubsub.NewLogger(), stats: new(et.SessionStats), @@ -83,7 +90,7 @@ func CreateSession(mgr *manager, reg et.Registry, cfg *config.Config) (et.Sessio s.scope = scope.CreateFromConfigScope(s) s.log = slog.New(slog.NewJSONHandler(s.ps, nil)).With("session", s.id) - err := s.setupDB() + err = s.setupDB() if err != nil { s.Kill() return nil, err @@ -160,7 +167,11 @@ func (s *Session) Pipelines() et.SessionPipelines { } func (s *Session) Clients() *et.SessionHTTPClients { - return nil + return &et.SessionHTTPClients{ + General: s.clients.General, + Probe: s.clients.Probe, + Crawl: s.clients.Crawl, + } } func (s *Session) CIDRanger() cidranger.Ranger { @@ -189,6 +200,7 @@ func (s *Session) Kill() { s.cancel() s.finished = true + s.clients.CloseIdleConnections() } func (s *Session) setupDB() error { diff --git a/engine/types/sessions.go b/engine/types/sessions.go index b53c5f6d..65c26f14 100644 --- a/engine/types/sessions.go +++ b/engine/types/sessions.go @@ -33,7 +33,7 @@ type SessionSemaphone interface { } type SessionHTTPClients struct { - Default *http.Client + General *http.Client Probe *http.Client Crawl *http.Client } diff --git a/internal/net/http/http.go b/internal/net/http/http.go index daf44782..9b34210c 100644 --- a/internal/net/http/http.go +++ b/internal/net/http/http.go @@ -170,7 +170,7 @@ func RespToAmassResponse(resp *http.Response) *Response { } // RequestWebPage returns the response headers, body, and status code for the provided URL when successful. -func RequestWebPage(ctx context.Context, r *Request) (*Response, error) { +func RequestWebPage(ctx context.Context, client *http.Client, r *Request) (*Response, error) { if r == nil { return nil, errors.New("failed to provide a valid Amass HTTP request") } @@ -201,7 +201,7 @@ func RequestWebPage(ctx context.Context, r *Request) (*Response, error) { } amassnet.MaxNetConnSem.Acquire() - resp, err := DefaultClient.Do(req) + resp, err := client.Do(req) amassnet.MaxNetConnSem.Release() if err != nil { return nil, err diff --git a/internal/net/http/http_test.go b/internal/net/http/http_test.go index d9a47c16..274484a3 100644 --- a/internal/net/http/http_test.go +++ b/internal/net/http/http_test.go @@ -50,7 +50,7 @@ func TestRequestWebPage(t *testing.T) { })) defer ts.Close() - resp, err := RequestWebPage(context.TODO(), &Request{ + resp, err := RequestWebPage(context.TODO(), DefaultClient, &Request{ URL: ts.URL, Auth: &BasicAuth{name, pass}, }) @@ -58,7 +58,7 @@ func TestRequestWebPage(t *testing.T) { t.Error("Failed to detect the bad request") } - resp, err = RequestWebPage(context.TODO(), &Request{ + resp, err = RequestWebPage(context.TODO(), DefaultClient, &Request{ URL: ts.URL, Method: "POST", Header: Header{hkey: []string{name}}, @@ -72,7 +72,7 @@ func TestRequestWebPage(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - resp, err = RequestWebPage(ctx, &Request{URL: ts.URL}) + resp, err = RequestWebPage(ctx, DefaultClient, &Request{URL: ts.URL}) if err == nil || resp != nil { t.Error("Failed to detect the expired context") }