fix: Escape parameter values in URLs and generated curl commands

This commit is contained in:
Tony West committed 2026-09-29 16:58:32 -04:00
1 parent 001f8f16b1
commit c9f4472553
5 files changed
+419 -54

No files matched your search

+59 -2
View File
@@ -9,8 +9,10 @@ import (
"net/http"
"net/url"
"os"
"slices"
"strings"
"time"
"unicode"
)
var (
@@ -112,6 +114,59 @@ func MakeRequest(client http.Client, method, target string, timeout int64, reqDa
return bodyBytes, bodyString, status
}
// splitWords lowercases s and splits it into words on non-alphanumeric
// characters and camelCase boundaries, so "uploadImage" yields
// ["upload", "image"] while "border" stays a single word.
func splitWords(s string) []string {
var words []string
var current []rune
flush := func() {
if len(current) > 0 {
words = append(words, strings.ToLower(string(current)))
current = nil
}
}
var prev rune
for _, r := range s {
switch {
case unicode.IsUpper(r) && unicode.IsLower(prev):
flush()
current = append(current, r)
case unicode.IsLetter(r) || unicode.IsDigit(r):
current = append(current, r)
default:
flush()
}
prev = r
}
flush()
return words
}
// requestWords returns the set of words appearing in a request's path and
// query string. It reads the decoded forms so the dangerous-keyword scan
// behaves identically whether or not a value required percent-encoding.
func requestWords(u *url.URL) map[string]bool {
words := make(map[string]bool)
add := func(s string) {
for _, w := range splitWords(s) {
words[w] = true
}
}
add(u.Path)
for name, values := range u.Query() {
add(name)
for _, v := range values {
add(v)
}
}
return words
}
// MakeRequestFull also returns the response Content-Type and the User-Agent sent.
// Taking body and Content-Type from one response avoids a second request per URL.
func MakeRequestFull(client http.Client, method, target string, timeout int64, reqData io.Reader) ([]byte, string, int, string, string) {
@@ -125,9 +180,11 @@ func MakeRequestFull(client http.Client, method, target string, timeout int64, r
printWarn("Error parsing URL '%s': %v - skipping request.", target, err)
return nil, "", 0, "", ""
}
endpoint := u.RawPath + "?" + u.RawQuery
// Match whole words rather than substrings so "/store/order" is flagged
// while "/border" is not.
words := requestWords(u)
for _, v := range dangerousStrings {
if os.Args[1] == "automate" && !force && strings.Contains(endpoint, v) && !strings.Contains(strings.Join(safeWords, ","), v) {
if os.Args[1] == "automate" && !force && words[v] && !slices.ContainsFunc(safeWords, func(s string) bool { return strings.EqualFold(s, v) }) {
userChoice = ""
if avoidDangerousRequests == "y" {
return nil, "", 0, "", ""
+64
View File
@@ -0,0 +1,64 @@
package cmd
import (
"net/url"
"strings"
"testing"
)
func TestSplitWords(t *testing.T) {
cases := []struct {
in string
want string
}{
{in: "", want: ""},
{in: "/store/order", want: "store,order"},
{in: "uploadImage", want: "upload,image"},
{in: "createWithList", want: "create,with,list"},
{in: "border", want: "border"},
{in: "/pet/{petId}/uploadImage", want: "pet,pet,id,upload,image"},
{in: "delete_all-now.txt", want: "delete,all,now,txt"},
{in: "v2", want: "v2"},
}
for _, tc := range cases {
got := strings.Join(splitWords(tc.in), ",")
if got != tc.want {
t.Errorf("splitWords(%q): expected %q, got %q", tc.in, tc.want, got)
}
}
}
func TestRequestWords(t *testing.T) {
cases := []struct {
in string
present []string
absent []string
}{
{in: "https://h/v2/store/order", present: []string{"store", "order"}},
{in: "https://h/v2/pet/1/uploadImage", present: []string{"upload", "image"}},
{in: "https://h/border", absent: []string{"order"}},
{in: "https://h/dataset", absent: []string{"set"}},
{in: "https://h/search?action=delete", present: []string{"action", "delete"}},
// A percent-encoded value must be scanned in its decoded form.
{in: "https://h/search?q=please%20delete%20me", present: []string{"delete"}},
}
for _, tc := range cases {
u, err := url.Parse(tc.in)
if err != nil {
t.Fatalf("could not parse %q: %v", tc.in, err)
}
words := requestWords(u)
for _, w := range tc.present {
if !words[w] {
t.Errorf("requestWords(%q): expected to find %q", tc.in, w)
}
}
for _, w := range tc.absent {
if words[w] {
t.Errorf("requestWords(%q): did not expect to find %q", tc.in, w)
}
}
}
}
+2
View File
@@ -19,6 +19,7 @@ var replayProxy string
var quiet bool
var randomUserAgent bool
var rateLimit int
var rawValues bool
var safeWords []string
var swaggerURL string
var timeout int64
@@ -81,6 +82,7 @@ func init() {
rootCmd.PersistentFlags().StringVar(&replayProxy, "replay-proxy", "", "Replay matched requests using this proxy.")
rootCmd.PersistentFlags().BoolVarP(&quiet, "quiet", "q", false, "Do not prompt for user input - uses default values for all requests.")
rootCmd.PersistentFlags().BoolVar(&randomUserAgent, "randomize-user-agent", false, "Randomizes the user agent string. Default is 'false'.")
rootCmd.PersistentFlags().BoolVar(&rawValues, "raw-values", false, "Do not URL-encode generated parameter names and values. Useful for parser-differential and WAF testing.")
// rootCmd.PersistentFlags().IntVarP(&rateLimit, "rate", "r", 15, "Limit the number of requests per second.") // NEED TO RE-IMPLEMENT RATE LIMIT
rootCmd.PersistentFlags().StringArrayVarP(&safeWords, "safe-word", "s", nil, "Avoids 'dangerous word' check for the specified word(s). Multiple flags are accepted.")
rootCmd.PersistentFlags().StringVarP(&apiTarget, "target", "T", "", "Manually set a target for the requests to be made if separate from the host the documentation resides on.")
+112 -52
View File
@@ -3,7 +3,9 @@ package cmd
import (
"bytes"
"encoding/json"
"encoding/xml"
"fmt"
"maps"
"mime/multipart"
"net/http"
"net/url"
@@ -101,6 +103,10 @@ func BuildRequestsFromPaths(spec map[string]interface{}, client http.Client, rep
targetURL := fmt.Sprintf("%s%s%s", apiTarget, basePath, pathName)
curl := fmt.Sprintf("curl -X %s \"%s\"", strings.ToUpper(method), targetURL)
var bodyData string
// curlBodyArg is the shell-quoted rendering of bodyData that is
// appended to curl once, after every parameter is known.
var curlBodyArg string
var multipartBody bool
// Extracts the expected parameters from the parameters object
if params, ok := opMap["parameters"].([]interface{}); ok {
@@ -136,27 +142,14 @@ func BuildRequestsFromPaths(spec map[string]interface{}, client http.Client, rep
// Handle based on parameter location
if in == "query" {
// Query params with object schema: add each property to query string
for propertyItem, propertyValue := range exampleMap {
pVal := fmt.Sprintf("%v", propertyValue)
if strings.Contains(curl, "?") || strings.Contains(targetURL, "?") {
targetURL += fmt.Sprintf("&%s=%s", propertyItem, pVal)
} else {
targetURL += fmt.Sprintf("?%s=%s", propertyItem, pVal)
}
for _, propertyItem := range slices.Sorted(maps.Keys(exampleMap)) {
targetURL = appendQueryParam(targetURL, propertyItem, exampleMap[propertyItem])
}
handledAsObject = true
} else if in == "body" {
// Body params with object schema: add each property to body data
for propertyItem, propertyValue := range exampleMap {
pVal := fmt.Sprintf("%v", propertyValue)
if strings.Contains(curl, "-d '") {
bodyData += fmt.Sprintf("&%s=%s", propertyItem, pVal)
curl = strings.TrimSuffix(curl, "'")
curl += fmt.Sprintf("&%s=%s'", propertyItem, pVal)
} else {
bodyData += fmt.Sprintf("%s=%s", propertyItem, pVal)
curl += fmt.Sprintf(" -d '%s=%s'", propertyItem, pVal)
}
for _, propertyItem := range slices.Sorted(maps.Keys(exampleMap)) {
bodyData = appendFormField(bodyData, propertyItem, exampleMap[propertyItem])
}
handledAsObject = true
}
@@ -199,24 +192,15 @@ func BuildRequestsFromPaths(spec map[string]interface{}, client http.Client, rep
if !handledAsObject {
switch in {
case "query":
if strings.Contains(curl, "?") || strings.Contains(targetURL, "?") {
targetURL += fmt.Sprintf("&%s=%s", name, pValue)
} else {
targetURL += fmt.Sprintf("?%s=%s", name, pValue)
}
targetURL = appendQueryParam(targetURL, name, pValue)
case "path":
targetURL = strings.Replace(targetURL, "{"+name+"}", pValue, 1)
targetURL = replacePathParam(targetURL, name, pValue)
case "header":
curl += fmt.Sprintf(" -H \"%s: %s\"", name, pValue)
// A header value is not a URL component, so it is shell-quoted
// for the printed command but never percent-encoded.
curl += " -H " + shellSingleQuote(name+": "+pValue)
case "body":
if strings.Contains(curl, "-d '") {
bodyData += fmt.Sprintf("&%s=%s", name, pValue)
curl = strings.TrimSuffix(curl, "'")
curl += fmt.Sprintf("&%s=%s'", name, pValue)
} else {
bodyData += fmt.Sprintf("%s=%s", name, pValue)
curl += fmt.Sprintf(" -d '%s=%s'", name, pValue)
}
bodyData = appendFormField(bodyData, name, pValue)
}
}
}
@@ -253,40 +237,39 @@ func BuildRequestsFromPaths(spec map[string]interface{}, client http.Client, rep
bodyBytes, err := json.Marshal(example)
if err == nil {
bodyData = string(bodyBytes)
curl += fmt.Sprintf(" -H \"Content-Type: application/json\" -d '%s'", bodyBytes)
curl += " -H " + shellSingleQuote("Content-Type: application/json")
}
}
if cType == "application/xml" || cType == "text/xml" {
if obj, ok := example.(map[string]interface{}); ok {
xml := XmlFromObject(obj)
bodyData = xml
curl += fmt.Sprintf(" -H \"Content-Type: %s\" -d '%s'", cType, xml)
bodyData = XmlFromObject(obj)
curl += " -H " + shellSingleQuote("Content-Type: "+cType)
}
}
if cType == "application/x-www-form-urlencoded" {
if obj, ok := example.(map[string]interface{}); ok {
var formParts []string
for k, v := range obj {
formParts = append(formParts, fmt.Sprintf("%s=%v", k, v))
}
bodyData = strings.Join(formParts, "&")
curl += fmt.Sprintf(" -H \"Content-Type: %s\" -d '%s'", cType, bodyData)
bodyData = encodeFormBody(obj)
curl += " -H " + shellSingleQuote("Content-Type: "+cType)
}
}
if cType == "multipart/form-data" {
if obj, ok := example.(map[string]interface{}); ok {
keys := slices.Sorted(maps.Keys(obj))
var buf bytes.Buffer
mw := multipart.NewWriter(&buf)
for k, v := range obj {
_ = mw.WriteField(k, fmt.Sprintf("%v", v))
for _, k := range keys {
_ = mw.WriteField(k, fmt.Sprintf("%v", obj[k]))
}
_ = mw.Close()
bodyData = buf.String()
// multipart.Writer owns this wire format, so the body is
// never appended to curl as a -d argument.
multipartBody = true
// Replace the bare multipart/form-data Content-Type
// (set earlier in the loop) with a boundary-aware one.
EnforceSingleContentType(mw.FormDataContentType())
for k, v := range obj {
curl += fmt.Sprintf(" -F \"%s=%v\"", k, v)
for _, k := range keys {
curl += " -F " + shellSingleQuote(fmt.Sprintf("%s=%v", k, obj[k]))
}
}
}
@@ -296,6 +279,13 @@ func BuildRequestsFromPaths(spec map[string]interface{}, client http.Client, rep
}
}
// Compose the body argument once, from bodyData, so the bytes that are
// sent and the printed curl command cannot drift apart.
if bodyData != "" && !multipartBody {
curlBodyArg = shellSingleQuote(bodyData)
curl += " -d " + curlBodyArg
}
// Update the curl command with the final targetURL (which may have been modified with query params)
// Extract and replace the URL in quotes
curlParts := strings.SplitN(curl, "\"", 3)
@@ -366,8 +356,8 @@ func BuildRequestsFromPaths(spec map[string]interface{}, client http.Client, rep
if strings.ToLower(prepareFor) == "sqlmap" {
preparedCommand = strings.Replace(preparedCommand, "curl", "sqlmap", 1)
preparedCommand = strings.Replace(preparedCommand, "-X "+strings.ToUpper(method), "--method="+strings.ToUpper(method)+" -u", 1)
if bodyData != "" {
preparedCommand = strings.Replace(preparedCommand, "-d '"+bodyData+"'", "--data='"+bodyData+"'", 1)
if curlBodyArg != "" {
preparedCommand = strings.Replace(preparedCommand, "-d "+curlBodyArg, "--data="+curlBodyArg, 1)
}
preparedCommand = "$ " + preparedCommand
} else if prepareFor == "curl" {
@@ -755,6 +745,75 @@ func normalizeBasePath(path string) string {
return path
}
// encodePair returns an encoded "name=value" pair for a query string or an
// x-www-form-urlencoded body. This and replacePathParam are the only places
// the --raw-values escape hatch is honored, so the printed curl command and
// the request that is actually sent can never disagree.
func encodePair(name string, value interface{}) string {
v := fmt.Sprintf("%v", value)
if rawValues {
return name + "=" + v
}
return url.QueryEscape(name) + "=" + url.QueryEscape(v)
}
// appendQueryParam returns rawURL with an encoded pair appended, using "?" or
// "&" depending on whether rawURL already carries a query string.
func appendQueryParam(rawURL, name string, value interface{}) string {
sep := "?"
if strings.Contains(rawURL, "?") {
sep = "&"
}
return rawURL + sep + encodePair(name, value)
}
// replacePathParam substitutes the first {name} placeholder in rawURL with a
// path-escaped value so a "/", "?" or "#" cannot re-segment the URL.
func replacePathParam(rawURL, name, value string) string {
if !rawValues {
value = url.PathEscape(value)
}
return strings.Replace(rawURL, "{"+name+"}", value, 1)
}
// appendFormField returns an x-www-form-urlencoded body with an encoded pair
// appended, separated by "&" when body is not empty.
func appendFormField(body, name string, value interface{}) string {
pair := encodePair(name, value)
if body == "" {
return pair
}
return body + "&" + pair
}
// encodeFormBody serializes obj as an x-www-form-urlencoded body with keys in
// sorted order so repeated runs produce identical output.
func encodeFormBody(obj map[string]interface{}) string {
var body string
for _, k := range slices.Sorted(maps.Keys(obj)) {
body = appendFormField(body, k, obj[k])
}
return body
}
// shellSingleQuote wraps s in single quotes for a printed shell command,
// escaping any embedded single quote the POSIX way (close, escape, reopen) so
// the command stays runnable. This is presentation only and is not affected
// by --raw-values.
func shellSingleQuote(s string) string {
return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
}
// escapeXmlText returns s with XML metacharacters escaped so a value
// containing '<', '>' or '&' cannot break out of its element.
func escapeXmlText(s string) string {
var b bytes.Buffer
if err := xml.EscapeText(&b, []byte(s)); err != nil {
return ""
}
return b.String()
}
// ResolveRefWithContext resolves a reference and returns both the resolved schema and the spec it came from
func ResolveRefWithContext(spec map[string]interface{}, ref string) (map[string]interface{}, map[string]interface{}) {
// Handle external references (e.g., "./schemas/user.yaml#/User")
@@ -940,8 +999,9 @@ func TrimHostScheme(apiTarget, fullUrlHost string) (host string) {
func XmlFromObject(obj map[string]interface{}) string {
var b strings.Builder
for k, v := range obj {
switch val := v.(type) {
// Sorted keys keep the generated body identical between runs.
for _, k := range slices.Sorted(maps.Keys(obj)) {
switch val := obj[k].(type) {
case map[string]interface{}:
b.WriteString("<" + k + ">")
b.WriteString(XmlFromObject(val))
@@ -952,12 +1012,12 @@ func XmlFromObject(obj map[string]interface{}) string {
if m, ok := item.(map[string]interface{}); ok {
b.WriteString(XmlFromObject(m))
} else {
b.WriteString(XmlFromObject(m))
b.WriteString(escapeXmlText(fmt.Sprintf("%v", item)))
}
b.WriteString("</" + k + ">")
}
default:
b.WriteString(fmt.Sprintf("<%s>%v</%s>", k, val, k))
b.WriteString("<" + k + ">" + escapeXmlText(fmt.Sprintf("%v", val)) + "</" + k + ">")
}
}
+182
View File
@@ -918,3 +918,185 @@ func TestExpandSchemaSelfCycleBounded(t *testing.T) {
t.Errorf("recursive Node ref should collapse to a bare object, got properties %v", level2.Properties)
}
}
func TestEncodePair(t *testing.T) {
cases := []struct {
name string
value interface{}
want string
}{
{name: "q", value: "hello world", want: "q=hello+world"},
{name: "a&b", value: "c=d", want: "a%26b=c%3Dd"},
{name: "url", value: "https://bishopfox.com", want: "url=https%3A%2F%2Fbishopfox.com"},
{name: "email", value: "[email protected]", want: "email=noreply%40localhost.localdomain"},
{name: "p", value: "a+b", want: "p=a%2Bb"},
{name: "frag", value: "a#b", want: "frag=a%23b"},
{name: "u", value: "naïve", want: "u=na%C3%AFve"},
{name: "n", value: 1, want: "n=1"},
{name: "t", value: true, want: "t=true"},
{name: "plain", value: "bishopfox", want: "plain=bishopfox"},
}
for _, tc := range cases {
got := encodePair(tc.name, tc.value)
if got != tc.want {
t.Errorf("encodePair(%q, %v): expected %q, got %q", tc.name, tc.value, tc.want, got)
}
}
}
func TestAppendQueryParam(t *testing.T) {
cases := []struct {
url string
name string
value interface{}
want string
}{
{url: "https://h/pet", name: "status", value: "sold", want: "https://h/pet?status=sold"},
{url: "https://h/pet?a=1", name: "b", value: "2", want: "https://h/pet?a=1&b=2"},
}
for _, tc := range cases {
got := appendQueryParam(tc.url, tc.name, tc.value)
if got != tc.want {
t.Errorf("appendQueryParam(%q, %q, %v): expected %q, got %q", tc.url, tc.name, tc.value, tc.want, got)
}
}
// A hostile value must not be able to add parameters or a fragment.
got := appendQueryParam("https://h/pet", "q", "x&injected=1#frag?y=2")
u, err := url.Parse(got)
if err != nil {
t.Fatalf("appendQueryParam produced an unparseable URL %q: %v", got, err)
}
if u.Fragment != "" {
t.Errorf("expected no fragment in %q, got %q", got, u.Fragment)
}
if len(u.Query()) != 1 {
t.Errorf("expected exactly 1 query parameter in %q, got %d", got, len(u.Query()))
}
if u.Query().Get("q") != "x&injected=1#frag?y=2" {
t.Errorf("value did not round-trip: got %q", u.Query().Get("q"))
}
}
func TestReplacePathParam(t *testing.T) {
const base = "https://h/v2/pet/{petId}"
got := replacePathParam(base, "petId", "a/b?c#d")
u, err := url.Parse(got)
if err != nil {
t.Fatalf("replacePathParam produced an unparseable URL %q: %v", got, err)
}
if want := strings.Count("/v2/pet/x", "/"); strings.Count(u.EscapedPath(), "/") != want {
t.Errorf("value re-segmented the path: %q has %d separators, expected %d", u.EscapedPath(), strings.Count(u.EscapedPath(), "/"), want)
}
if u.RawQuery != "" || u.Fragment != "" {
t.Errorf("value leaked into query/fragment: query=%q fragment=%q", u.RawQuery, u.Fragment)
}
if u.Path != "/v2/pet/a/b?c#d" {
t.Errorf("value did not round-trip: got %q", u.Path)
}
if got := replacePathParam(base, "other", "x"); got != base {
t.Errorf("an unmatched placeholder should be left intact, got %q", got)
}
if got := replacePathParam("https://h/{a}/{a}", "a", "x"); got != "https://h/x/{a}" {
t.Errorf("expected only the first placeholder to be replaced, got %q", got)
}
}
func TestAppendFormField(t *testing.T) {
got := appendFormField("", "name", "a b")
if want := "name=a+b"; got != want {
t.Errorf("appendFormField on an empty body: expected %q, got %q", want, got)
}
got = appendFormField(got, "note", "x&y=z")
if want := "name=a+b&note=x%26y%3Dz"; got != want {
t.Errorf("appendFormField on an existing body: expected %q, got %q", want, got)
}
}
func TestEncodeFormBody(t *testing.T) {
obj := map[string]interface{}{
"zeta": "last",
"alpha": "first",
"mid": 2,
}
want := "alpha=first&mid=2&zeta=last"
for i := 0; i < 50; i++ {
if got := encodeFormBody(obj); got != want {
t.Fatalf("encodeFormBody is not deterministic: expected %q, got %q", want, got)
}
}
}
func TestShellSingleQuote(t *testing.T) {
cases := []struct {
in string
want string
}{
{in: "", want: "''"},
{in: "abc", want: "'abc'"},
{in: `it's`, want: `'it'\''s'`},
{in: `{"a":"b"}`, want: `'{"a":"b"}'`},
}
for _, tc := range cases {
got := shellSingleQuote(tc.in)
if got != tc.want {
t.Errorf("shellSingleQuote(%q): expected %q, got %q", tc.in, tc.want, got)
}
}
}
func TestXmlFromObjectEscapesValues(t *testing.T) {
got := XmlFromObject(map[string]interface{}{"note": "a < b & c"})
if want := "<note>a &lt; b &amp; c</note>"; got != want {
t.Errorf("expected %q, got %q", want, got)
}
// A scalar array element used to be emitted as an empty element.
got = XmlFromObject(map[string]interface{}{"tag": []interface{}{"x", "y"}})
if want := "<tag>x</tag><tag>y</tag>"; got != want {
t.Errorf("expected %q, got %q", want, got)
}
obj := map[string]interface{}{"zeta": 1, "alpha": 2, "mid": 3}
want := "<alpha>2</alpha><mid>3</mid><zeta>1</zeta>"
for i := 0; i < 50; i++ {
if got := XmlFromObject(obj); got != want {
t.Fatalf("XmlFromObject is not deterministic: expected %q, got %q", want, got)
}
}
}
func TestRawValuesSkipsEncoding(t *testing.T) {
oldRawValues := rawValues
rawValues = true
defer func() { rawValues = oldRawValues }()
if got, want := encodePair("a b", "c&d"), "a b=c&d"; got != want {
t.Errorf("encodePair: expected %q, got %q", want, got)
}
if got, want := appendQueryParam("https://h/p", "q", "a b"), "https://h/p?q=a b"; got != want {
t.Errorf("appendQueryParam: expected %q, got %q", want, got)
}
if got, want := appendFormField("", "q", "a b"), "q=a b"; got != want {
t.Errorf("appendFormField: expected %q, got %q", want, got)
}
if got, want := replacePathParam("https://h/{id}", "id", "a/b"), "https://h/a/b"; got != want {
t.Errorf("replacePathParam: expected %q, got %q", want, got)
}
// --raw-values covers percent-encoding only.
if got, want := shellSingleQuote(`it's`), `'it'\''s'`; got != want {
t.Errorf("shellSingleQuote should still escape: expected %q, got %q", want, got)
}
if got, want := escapeXmlText("a < b"), "a &lt; b"; got != want {
t.Errorf("escapeXmlText should still escape: expected %q, got %q", want, got)
}
}