Fix SJ requests carrying no User-Agent when a document declares one as a header parameter

internal/api/handlers_spec.go

@@ -21,6 +21,11 @@ type specLoadRequest struct {
 	Text string `json:"text"` // the document verbatim
 	URL  string `json:"url"`  // fetched through Joro's proxy
 	Name string `json:"name"`
+
+	// UserAgent is the tab's, when a tab is doing the loading — a discovery
+	// sweep probes with it, so the fetch of what it found uses it too. Empty on
+	// a first load, where no tab exists yet.
+	UserAgent string `json:"userAgent,omitempty"`
 }
 
 // handleSpecLoad parses a document and stores it.
@@ -51,7 +56,7 @@ func (s *APIServer) handleSpecLoad(w http.ResponseWriter, r *http.Request) {
 		data = []byte(req.Text)
 		sourceURL = req.URL
 	case req.URL != "":
-		fetched, ct, err := s.fetchSpecViaProxy(r, req.URL)
+		fetched, ct, err := s.fetchSpecViaProxy(r, req.URL, req.UserAgent)
 		if err != nil {
 			writeError(w, http.StatusBadGateway, err.Error())
 			return
@@ -87,7 +92,7 @@ func (s *APIServer) handleSpecLoad(w http.ResponseWriter, r *http.Request) {
 // is captured into History, filtered by scope and routed through SOCKS exactly
 // as browser traffic is — and so the document is recoverable later from History
 // without re-fetching it.
-func (s *APIServer) fetchSpecViaProxy(r *http.Request, rawURL string) ([]byte, string, error) {
+func (s *APIServer) fetchSpecViaProxy(r *http.Request, rawURL, userAgent string) ([]byte, string, error) {
 	u, err := url.Parse(strings.TrimSpace(rawURL))
 	if err != nil || u.Host == "" {
 		return nil, "", fmt.Errorf("could not parse %q as a URL", rawURL)
@@ -97,10 +102,10 @@ func (s *APIServer) fetchSpecViaProxy(r *http.Request, rawURL string) ([]byte, s
 	}
 
 	target := u.RequestURI()
-	// The default rather than an operator value: a load carries no profile and
-	// no tab exists yet to have set one.
+	// A load carries no profile, so the tab's User-Agent is the only operator
+	// value there is; UserAgentOr falls back for the first load, which has none.
 	raw := "GET " + target + " HTTP/1.1\r\nHost: " + u.Host + "\r\n" +
-		"User-Agent: " + apispec.DefaultUserAgent + "\r\n" +
+		"User-Agent: " + apispec.UserAgentOr(userAgent) + "\r\n" +
 		"Accept: application/json, text/yaml, application/yaml, */*\r\n\r\n"
 
 	ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)

internal/apispec/render.go

@@ -57,9 +57,10 @@ type Auth struct {
 
 // RenderOptions tunes one render.
 type RenderOptions struct {
-	// UserAgent is sent as-is. Empty means DefaultUserAgent, and a User-Agent
-	// the operator named through an auth profile or an "in: header" parameter
-	// outranks both — the request carries exactly one either way.
+	// UserAgent is one rung of the ladder Render resolves, most specific first:
+	// an auth profile, a value the operator typed for a declared "in: header"
+	// parameter, this, the document's generated placeholder, DefaultUserAgent.
+	// The request carries exactly one whatever happens.
 	UserAgent string
 
 	// Accept defaults to a permissive value when empty.
@@ -93,6 +94,9 @@ const defaultAccept = "application/json, text/plain, */*"
 // advertises a Go program to the target while showing the operator nothing,
 // which is the one combination worth ruling out: a rendered request is only
 // trustworthy if it is what leaves the machine.
+//
+// So a named User-Agent that writeHeader would drop, or that net/http would
+// strip as blank, does not count as named — see usableUA.
 const DefaultUserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.85 Safari/537.36"
 
 // UserAgentOr returns ua when it is usable as a header value and
@@ -103,12 +107,64 @@ const DefaultUserAgent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/5
 // Routing an operator-supplied value through here is what keeps a newline in one
 // from splitting the header block.
 func UserAgentOr(ua string) string {
-	if ua != "" && validHeaderValue(ua) {
-		return ua
+	if v := usableUA(ua); v != "" {
+		return v
 	}
 	return DefaultUserAgent
 }
 
+// isUserAgent reports whether name is the User-Agent header, however spelled.
+// A document chooses that spelling, not Joro, so this is case- and
+// space-insensitive.
+func isUserAgent(name string) bool {
+	return strings.EqualFold(strings.TrimSpace(name), "User-Agent")
+}
+
+// usableUA returns value when it will survive both writeHeader and net/http's
+// own serialization, and "" otherwise.
+//
+// Blank and invalid are as good as unnamed: treating either as named suppresses
+// DefaultUserAgent, and the request then reaches the target carrying no
+// User-Agent at all or net/http's own.
+func usableUA(value string) string {
+	if strings.TrimSpace(value) == "" || !validHeaderValue(value) {
+		return ""
+	}
+	return value
+}
+
+// firstUsableUA picks the first candidate that will reach the wire intact,
+// which is how the rungs of the ladder fall through to one another, and
+// DefaultUserAgent is the floor none of them can drop past.
+func firstUsableUA(candidates ...string) string {
+	for _, c := range candidates {
+		if v := usableUA(c); v != "" {
+			return v
+		}
+	}
+	return DefaultUserAgent
+}
+
+// authUserAgent returns the usable User-Agent a profile names, or "". The two
+// loops are in applyAuth's order, so last-wins agrees with dedupeHeaders.
+func authUserAgent(auth Auth) string {
+	ua := ""
+	for _, c := range auth.Credentials {
+		if c.Kind != AuthAPIKey || c.In == InQuery || c.In == InCookie {
+			continue
+		}
+		if c.Name != "" && c.Value != "" && isUserAgent(c.Name) && validHeaderName(c.Name) {
+			ua = usableUA(c.Value)
+		}
+	}
+	for _, h := range auth.Headers {
+		if isUserAgent(h.Name) && validHeaderName(h.Name) {
+			ua = usableUA(h.Value)
+		}
+	}
+	return ua
+}
+
 // Render builds the raw HTTP/1.1 bytes for one operation.
 //
 // These bytes are the executable truth. The Spec is the editing document, and
@@ -144,6 +200,11 @@ func Render(op *Operation, srv Server, vals Values, auth Auth, opts RenderOption
 	headers := make([]Header, 0, len(op.Params)+8)
 	cookies := make([]Cookie, 0, 4)
 
+	// The two document-side rungs of the User-Agent ladder, filled below: what
+	// the operator typed for a declared parameter, and the placeholder the
+	// parser generated for one.
+	var typedUA, docUA string
+
 	// Exploded object properties are regrouped so the style can be honored: the
 	// three forms below are three different requests, and which one the server
 	// accepts is not negotiable.
@@ -188,7 +249,15 @@ func Render(op *Operation, srv Server, vals Values, auth Auth, opts RenderOption
 			}
 			queryPairs = append(queryPairs, renderScalarQuery(prm, raw, key, mark)...)
 		case InHeader:
-			headers = append(headers, Header{Name: prm.Name, Value: mark(key, raw)})
+			value := mark(key, raw)
+			if isUserAgent(prm.Name) && validHeaderName(prm.Name) {
+				if overridden {
+					typedUA = usableUA(value)
+				} else {
+					docUA = usableUA(value)
+				}
+			}
+			headers = append(headers, Header{Name: prm.Name, Value: value})
 		case InCookie:
 			cookies = append(cookies, Cookie{Name: prm.Name, Value: mark(key, raw)})
 		}
@@ -204,6 +273,10 @@ func Render(op *Operation, srv Server, vals Values, auth Auth, opts RenderOption
 	// placeholder for the same name.
 	headers, queryPairs, cookies = applyAuth(auth, headers, queryPairs, cookies)
 
+	// Resolved here rather than at the write site so there is one answer, and
+	// one place to read the precedence off.
+	userAgent := firstUsableUA(authUserAgent(auth), typedUA, opts.UserAgent, docUA)
+
 	target := path
 	if len(queryPairs) > 0 {
 		target += "?" + strings.Join(queryPairs, "&")
@@ -225,9 +298,7 @@ func Render(op *Operation, srv Server, vals Values, auth Auth, opts RenderOption
 	buf.WriteString(op.Method + " " + target + " HTTP/1.1\r\n")
 
 	writeHeader(&buf, "Host", host)
-	if !hasUserAgent(headers) {
-		writeHeader(&buf, "User-Agent", UserAgentOr(opts.UserAgent))
-	}
+	writeHeader(&buf, "User-Agent", userAgent)
 	accept := opts.Accept
 	if accept == "" {
 		accept = defaultAccept
@@ -235,7 +306,9 @@ func Render(op *Operation, srv Server, vals Values, auth Auth, opts RenderOption
 	writeHeader(&buf, "Accept", accept)
 
 	for _, h := range headers {
-		if reservedHeader(h.Name) {
+		// User-Agent is skipped because it was resolved and written above; a
+		// second line here is what the ladder exists to prevent.
+		if reservedHeader(h.Name) || isUserAgent(h.Name) {
 			continue
 		}
 		writeHeader(&buf, h.Name, h.Value)
@@ -664,22 +737,6 @@ func validHeaderValue(v string) bool {
 	return true
 }
 
-// hasUserAgent reports whether the operator already named one, through an auth
-// profile's extra headers or a document's own "in: header" parameter.
-//
-// Both land in the same slice, which applyAuth appends to without deduping and
-// writeHeader emits without case-folding — so this check is what makes a named
-// User-Agent an override rather than a second header line. It is deliberately
-// case- and space-insensitive: a document chooses that spelling, not Joro.
-func hasUserAgent(headers []Header) bool {
-	for _, h := range headers {
-		if strings.EqualFold(strings.TrimSpace(h.Name), "User-Agent") {
-			return true
-		}
-	}
-	return false
-}
-
 // buildCookieHeader folds every cookie into one header.
 //
 // Values are not percent-encoded wholesale — that would corrupt a legitimate

web/src/components/sj/SJBruteView.tsx

@@ -68,7 +68,7 @@ export default function SJBruteView({
     // never classified as a document and the operator is trying by hand.
     const already = run?.found.find((f) => f.sourceUrl === url)
     try {
-      const res = already ? await api.sjGetSpec(already.specId) : await api.sjLoad({ url })
+      const res = already ? await api.sjGetSpec(already.specId) : await api.sjLoad({ url, userAgent: tab.userAgent || undefined })
       store.setSpec(id, res.spec)
     } catch (e) {
       onFailure(id, e)

web/src/components/sj/SJDocumentBar.tsx

@@ -134,7 +134,8 @@ export default function SJDocumentBar({
             className="bg-surface-input text-xs px-2 py-1.5 rounded-sm border border-border w-28"
           />
           {/* Reaches every sub-tab from here, so a run and a single send agree.
-              A User-Agent set on an auth profile outranks it. */}
+              Outranked only by an auth profile and by a value typed for a
+              User-Agent the document itself declares as a parameter. */}
           <input
             value={tab.userAgent}
             onChange={(e) => store.updateTab(tab.id, { userAgent: e.target.value })}

web/src/components/sj/SJDocumentModal.tsx

@@ -68,7 +68,9 @@ export default function SJDocumentModal({
     store.updateTab(tab.id, { loading: true, loadError: '', htmlHint: null })
     try {
       const body =
-        source === 'url' ? { url: url.trim() } : { text, name: fileName || undefined }
+        source === 'url'
+          ? { url: url.trim(), userAgent: tab.userAgent || undefined }
+          : { text, name: fileName || undefined }
       const res = await api.sjLoad(body)
       store.setSpec(tab.id, res.spec)
       setSpecSource('')

web/src/lib/api.ts

@@ -1218,7 +1218,7 @@ export const api = {
   // A document is loaded from exactly one source. `url` is fetched server-side
   // through Joro's own proxy, so the retrieval is captured into History like any
   // other traffic; the browser never fetches a target directly.
-  sjLoad: (body: { raw?: string; text?: string; url?: string; name?: string }) =>
+  sjLoad: (body: { raw?: string; text?: string; url?: string; name?: string; userAgent?: string }) =>
     req<{ spec: SJSpec }>('POST', '/spec/load', body),
   sjListSpecs: () => req<{ specs: SJSpecSummary[] }>('GET', '/spec/specs'),
   sjGetSpec: (id: string) => req<{ spec: SJSpec }>('GET', `/spec/specs/${id}`),

web/src/stores/sjStore.ts

@@ -196,8 +196,8 @@ export interface SJTab {
   concurrency: number
   ratePerSec: number
   /** Empty means the server's default, which is a browser value rather than no
-   *  header — see apispec.DefaultUserAgent. Held per tab because it describes
-   *  the document's target, not one sub-tab's run. */
+   *  header — see apispec.DefaultUserAgent for the precedence ladder. Held per
+   *  tab because it describes the document's target, not one sub-tab's run. */
   userAgent: string
   allowDestructive: boolean
   /** Brute runs hotter than Automate — unauthenticated GETs against one host —