From 55030eaa86e793083592e2b3fe12e7bd1778482d Mon Sep 17 00:00:00 2001 From: Pablo Garcia Date: Fri, 7 Aug 2026 14:53:18 +0200 Subject: [PATCH 01/23] fix(hcl): coerce non-string object keys to strings instead of panicking (#2795) HCL object keys may be non-string literals (e.g. a number like 1). The decoder called cty.Value.AsString on the evaluated key, which panics with 'not a string' for any non-string key type. Mirror OpenTofu/Terragrunt by silently converting non-string keys to their string representation via cty/convert. Add decode scenarios covering numeric and boolean keys. --- pkg/yqlib/decoder_hcl.go | 31 ++++++++++++++++++++++++++++++- pkg/yqlib/hcl_test.go | 14 ++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/pkg/yqlib/decoder_hcl.go b/pkg/yqlib/decoder_hcl.go index 11cd0030..dcd01557 100644 --- a/pkg/yqlib/decoder_hcl.go +++ b/pkg/yqlib/decoder_hcl.go @@ -13,6 +13,7 @@ import ( "github.com/hashicorp/hcl/v2" "github.com/hashicorp/hcl/v2/hclsyntax" "github.com/zclconf/go-cty/cty" + "github.com/zclconf/go-cty/cty/convert" ) type hclDecoder struct { @@ -252,6 +253,22 @@ func addBlockToMapping(parent *CandidateNode, block *hclsyntax.Block, src []byte } } +// hclKeyValueAsString converts an evaluated HCL key expression to its string +// representation. HCL object keys may be non-string literals (e.g. a number +// like `1`); calling AsString on those panics. Mirroring OpenTofu/Terragrunt, +// we silently coerce non-string keys to their string form ("1" -> "1", +// true -> "true") instead of panicking. +func hclKeyValueAsString(keyVal cty.Value) (string, error) { + if keyVal.Type() == cty.String { + return keyVal.AsString(), nil + } + strVal, err := convert.Convert(keyVal, cty.String) + if err != nil { + return "", err + } + return strVal.AsString(), nil +} + func convertHclExprToNode(expr hclsyntax.Expression, src []byte) *CandidateNode { // handle literal values directly switch e := expr.(type) { @@ -338,7 +355,19 @@ func convertHclExprToNode(expr hclsyntax.Expression, src []byte) *CandidateNode } continue } - keyStr := keyVal.AsString() + keyStr, err := hclKeyValueAsString(keyVal) + if err != nil { + // fallback: try to extract key from source + r := item.KeyExpr.Range() + start := r.Start.Byte + end := r.End.Byte + if start >= 0 && end >= start && end <= len(src) { + keyNode := createStringScalarNode(strings.TrimSpace(string(src[start:end]))) + valNode := convertHclExprToNode(item.ValueExpr, src) + m.AddKeyValueChild(keyNode, valNode) + } + continue + } keyNode := createStringScalarNode(keyStr) valNode := convertHclExprToNode(item.ValueExpr, src) m.AddKeyValueChild(keyNode, valNode) diff --git a/pkg/yqlib/hcl_test.go b/pkg/yqlib/hcl_test.go index a8f22065..d543fd03 100644 --- a/pkg/yqlib/hcl_test.go +++ b/pkg/yqlib/hcl_test.go @@ -472,6 +472,20 @@ var hclFormatScenarios = []formatScenario{ expected: "service {\n optional_field = null\n}\n", scenarioType: "roundtrip", }, + { + description: "Non-string object keys are coerced to strings", + skipDoc: true, + input: `intdict = { 1 = {} }`, + expected: "intdict: {\"1\": {}}\n", + scenarioType: "decode", + }, + { + description: "Mixed non-string object keys are coerced to strings", + skipDoc: true, + input: `d = { 1 = "a", 2 = "b", true = "c" }`, + expected: "d: {\"1\": \"a\", \"2\": \"b\", \"true\": \"c\"}\n", + scenarioType: "decode", + }, } func testHclScenario(t *testing.T, s formatScenario) { From b13737ef7087657e86f20391327a6f8fed3848ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 13:34:12 +1000 Subject: [PATCH 02/23] build(deps): bump github/codeql-action/upload-sarif (#2803) Bumps [github/codeql-action/upload-sarif](https://github.com/github/codeql-action) from 4.36.2 to 4.37.5. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/8aad20d150bbac5944a9f9d289da16a4b0d87c1e...d1ba80a13dd99fba24a470575428917156a28b43) --- updated-dependencies: - dependency-name: github/codeql-action/upload-sarif dependency-version: 4.37.5 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/scorecard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 73de9837..22525d70 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -73,6 +73,6 @@ jobs: # Upload the results to GitHub's code scanning dashboard (optional). # Commenting out will disable upload of results to your repo's Code Scanning dashboard - name: "Upload to code-scanning" - uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2 + uses: github/codeql-action/upload-sarif@d1ba80a13dd99fba24a470575428917156a28b43 # v4.37.5 with: sarif_file: results.sarif From 0a64318a22d25374183fd75699d0d2077674c779 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 13:34:22 +1000 Subject: [PATCH 03/23] build(deps): bump ossf/scorecard-action from 2.4.3 to 2.4.4 (#2801) Bumps [ossf/scorecard-action](https://github.com/ossf/scorecard-action) from 2.4.3 to 2.4.4. - [Release notes](https://github.com/ossf/scorecard-action/releases) - [Changelog](https://github.com/ossf/scorecard-action/blob/main/RELEASE.md) - [Commits](https://github.com/ossf/scorecard-action/compare/4eaacf0543bb3f2c246792bd56e8cdeffafb205a...2d1146689b8cda280b9bc96326124645441f03bc) --- updated-dependencies: - dependency-name: ossf/scorecard-action dependency-version: 2.4.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/scorecard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 22525d70..9ecb17ac 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -39,7 +39,7 @@ jobs: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3 + uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc # v2.4.4 with: results_file: results.sarif results_format: sarif From e65fe663bf7679f6a3b98ad759bb1f80c4a3c22d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 13:34:41 +1000 Subject: [PATCH 04/23] build(deps): bump github.com/magiconair/properties (#2799) Bumps [github.com/magiconair/properties](https://github.com/magiconair/properties) from 1.8.10 to 1.18.11. - [Release notes](https://github.com/magiconair/properties/releases) - [Commits](https://github.com/magiconair/properties/compare/v1.8.10...v1.18.11) --- updated-dependencies: - dependency-name: github.com/magiconair/properties dependency-version: 1.18.11 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2ebd547f..96915efd 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/goccy/go-yaml v1.19.2 github.com/hashicorp/hcl/v2 v2.24.0 github.com/jinzhu/copier v0.4.0 - github.com/magiconair/properties v1.8.10 + github.com/magiconair/properties v1.18.11 github.com/pelletier/go-toml/v2 v2.4.3 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e github.com/spf13/cobra v1.10.2 diff --git a/go.sum b/go.sum index e99d3473..b45ed73f 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= -github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= -github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/magiconair/properties v1.18.11 h1:j5ozYZl0zCjG7ahMDH0GWIobOvvUzT0BdAguG0ViKy0= +github.com/magiconair/properties v1.18.11/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= From d6543ac3fecbebc7c1f1a68052b3bce67cf34350 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 19 Aug 2026 13:34:49 +1000 Subject: [PATCH 05/23] build(deps): bump github.com/alecthomas/repr from 0.5.2 to 0.5.4 (#2798) Bumps [github.com/alecthomas/repr](https://github.com/alecthomas/repr) from 0.5.2 to 0.5.4. - [Commits](https://github.com/alecthomas/repr/compare/v0.5.2...v0.5.4) --- updated-dependencies: - dependency-name: github.com/alecthomas/repr dependency-version: 0.5.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 96915efd..7fcb643a 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/mikefarah/yq/v4 require ( github.com/a8m/envsubst v1.4.3 github.com/alecthomas/participle/v2 v2.1.4 - github.com/alecthomas/repr v0.5.2 + github.com/alecthomas/repr v0.5.4 github.com/dimchansky/utfbom v1.1.1 github.com/elliotchance/orderedmap v1.8.0 github.com/fatih/color v1.19.0 diff --git a/go.sum b/go.sum index b45ed73f..720f6f45 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,8 @@ github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8v github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k= github.com/alecthomas/participle/v2 v2.1.4 h1:W/H79S8Sat/krZ3el6sQMvMaahJ+XcM9WSI2naI7w2U= github.com/alecthomas/participle/v2 v2.1.4/go.mod h1:8tqVbpTX20Ru4NfYQgZf4mP18eXPTBViyMWiArNEgGI= -github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs= -github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= +github.com/alecthomas/repr v0.5.4 h1:OVP7JEcuzU9CCDsT6STCr3rg17oQfWILtPWd2EG0uN4= +github.com/alecthomas/repr v0.5.4/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= github.com/apparentlymart/go-textseg/v17 v17.0.1 h1:bpMXRgQ5cEoRNuQke1a80/Nl6w3G5eoIbWo9f3gXkAs= From 5c302ee313dab3cb22a09397e0a24b3fa06f8501 Mon Sep 17 00:00:00 2001 From: MsfPablo <129399053+MsfPablo@users.noreply.github.com> Date: Wed, 19 Aug 2026 05:43:22 +0200 Subject: [PATCH 06/23] perf: use io.WriteString in writeString to avoid heap allocation (#2809) writeString converted every string to []byte before calling io.Writer.Write, which Go escape analysis reports as escaping to the heap. io.WriteString uses the io.StringWriter fast path when available (the standard printer writer is a *bufio.Writer) and falls back to Write([]byte(txt)) otherwise. Fixes #2807 Co-authored-by: Pablo Garcia --- pkg/yqlib/utils.go | 2 +- pkg/yqlib/utils_test.go | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 pkg/yqlib/utils_test.go diff --git a/pkg/yqlib/utils.go b/pkg/yqlib/utils.go index 2843e42d..e4036f03 100644 --- a/pkg/yqlib/utils.go +++ b/pkg/yqlib/utils.go @@ -50,7 +50,7 @@ func readStream(filename string) (io.Reader, error) { } func writeString(writer io.Writer, txt string) error { - _, errorWriting := writer.Write([]byte(txt)) + _, errorWriting := io.WriteString(writer, txt) return errorWriting } diff --git a/pkg/yqlib/utils_test.go b/pkg/yqlib/utils_test.go new file mode 100644 index 00000000..fc1b0026 --- /dev/null +++ b/pkg/yqlib/utils_test.go @@ -0,0 +1,41 @@ +package yqlib + +import ( + "bufio" + "bytes" + "io" + "testing" + + "github.com/mikefarah/yq/v4/test" +) + +// plainWriter only implements io.Writer, so io.WriteString must fall back to Write. +type plainWriter struct { + buf bytes.Buffer +} + +func (w *plainWriter) Write(p []byte) (int, error) { + return w.buf.Write(p) +} + +func TestWriteStringToStringWriter(t *testing.T) { + var buf bytes.Buffer + writer := bufio.NewWriter(&buf) + test.AssertResult(t, nil, writeString(writer, "hello world")) + test.AssertResult(t, nil, writer.Flush()) + test.AssertResult(t, "hello world", buf.String()) +} + +func TestWriteStringToPlainWriter(t *testing.T) { + writer := &plainWriter{} + test.AssertResult(t, nil, writeString(writer, "hello world")) + test.AssertResult(t, "hello world", writer.buf.String()) +} + +func TestWriteStringDoesNotAllocate(t *testing.T) { + writer := bufio.NewWriter(io.Discard) + allocations := testing.AllocsPerRun(100, func() { + _ = writeString(writer, "hello world") + }) + test.AssertResult(t, 0.0, allocations) +} From 6a182b6a3091d1baf0645978afacd0b066808ced Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Wed, 19 Aug 2026 13:50:34 +1000 Subject: [PATCH 07/23] Fixing HCL non string key issue #2795 --- pkg/yqlib/decoder_hcl.go | 8 +++----- pkg/yqlib/hcl_test.go | 35 +++++++++++++++++++++++++++++++++++ pkg/yqlib/toml_test.go | 2 +- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/pkg/yqlib/decoder_hcl.go b/pkg/yqlib/decoder_hcl.go index 11cd0030..321f1156 100644 --- a/pkg/yqlib/decoder_hcl.go +++ b/pkg/yqlib/decoder_hcl.go @@ -300,8 +300,7 @@ func convertHclExprToNode(expr hclsyntax.Expression, src []byte) *CandidateNode it := v.ElementIterator() for it.Next() { key, val := it.Element() - keyStr := key.AsString() - keyNode := createStringScalarNode(keyStr) + keyNode := convertCtyValueToNode(key) valNode := convertCtyValueToNode(val) m.AddKeyValueChild(keyNode, valNode) } @@ -338,8 +337,7 @@ func convertHclExprToNode(expr hclsyntax.Expression, src []byte) *CandidateNode } continue } - keyStr := keyVal.AsString() - keyNode := createStringScalarNode(keyStr) + keyNode := convertCtyValueToNode(keyVal) valNode := convertHclExprToNode(item.ValueExpr, src) m.AddKeyValueChild(keyNode, valNode) } @@ -460,7 +458,7 @@ func convertCtyValueToNode(v cty.Value) *CandidateNode { it := v.ElementIterator() for it.Next() { key, val := it.Element() - keyNode := createStringScalarNode(key.AsString()) + keyNode := convertCtyValueToNode(key) valNode := convertCtyValueToNode(val) m.AddKeyValueChild(keyNode, valNode) } diff --git a/pkg/yqlib/hcl_test.go b/pkg/yqlib/hcl_test.go index a8f22065..08618446 100644 --- a/pkg/yqlib/hcl_test.go +++ b/pkg/yqlib/hcl_test.go @@ -176,6 +176,41 @@ var hclFormatScenarios = []formatScenario{ expected: "obj: {a: 1, b: \"two\"}\n", scenarioType: "decode", }, + { + description: "object with integer keys", + skipDoc: true, + input: `obj = { 1 = "one", 2 = "two" }`, + expected: "obj: {1: \"one\", 2: \"two\"}\n", + scenarioType: "decode", + }, + { + description: "object with boolean keys", + skipDoc: true, + input: `obj = { (true) = "yes", (false) = "no" }`, + expected: "obj: {true: \"yes\", false: \"no\"}\n", + scenarioType: "decode", + }, + { + description: "object with float keys", + skipDoc: true, + input: `obj = { (3.14) = "pi" }`, + expected: "obj: {3.14: \"pi\"}\n", + scenarioType: "decode", + }, + { + description: "object with mixed scalar keys", + skipDoc: true, + input: `obj = { a = 1, 1 = "one", (true) = "yes" }`, + expected: "obj: {a: 1, 1: \"one\", true: \"yes\"}\n", + scenarioType: "decode", + }, + { + description: "nested object with integer keys", + skipDoc: true, + input: `config = { levels = { 1 = "debug", 2 = "info" } }`, + expected: "config: {levels: {1: \"debug\", 2: \"info\"}}\n", + scenarioType: "decode", + }, { description: "nested block", skipDoc: true, diff --git a/pkg/yqlib/toml_test.go b/pkg/yqlib/toml_test.go index d36c10cd..634dd15c 100644 --- a/pkg/yqlib/toml_test.go +++ b/pkg/yqlib/toml_test.go @@ -369,7 +369,7 @@ var tomlScenarios = []formatScenario{ skipDoc: true, description: "blank", input: `A = "hello`, - expectedError: `bad file 'sample.yml': basic string not terminated by "`, + expectedError: `bad file 'sample.yml': unterminated basic string`, scenarioType: "decode-error", }, { From cb7b707457a3c963a3f16d3075b4cbea2989535b Mon Sep 17 00:00:00 2001 From: MsfPablo <129399053+MsfPablo@users.noreply.github.com> Date: Wed, 19 Aug 2026 06:17:53 +0200 Subject: [PATCH 08/23] Close input files after processing each one (#2796) (#2808) readStream opened an *os.File but returned a *bufio.Reader, so the *os.File type switch in streamEvaluator.EvaluateFiles never matched and no input file was ever closed. allAtOnceEvaluator.EvaluateFiles did not attempt to close them at all. Descriptors were only released at process exit or by the os.File finaliser, so a multi-file run could exhaust the file descriptor limit. readStream now returns an explicit cleanup function alongside the reader, which both evaluators call once the file has been processed. Co-authored-by: Pablo Garcia Co-authored-by: Mike Farah Co-authored-by: Cursor --- pkg/yqlib/all_at_once_evaluator.go | 3 +- pkg/yqlib/front_matter_test.go | 3 +- pkg/yqlib/stream_evaluator.go | 9 +--- pkg/yqlib/utils.go | 25 +++++----- pkg/yqlib/utils_test.go | 78 ++++++++++++++++++++++++++++++ 5 files changed, 96 insertions(+), 22 deletions(-) diff --git a/pkg/yqlib/all_at_once_evaluator.go b/pkg/yqlib/all_at_once_evaluator.go index bdeddd20..cfc0624b 100644 --- a/pkg/yqlib/all_at_once_evaluator.go +++ b/pkg/yqlib/all_at_once_evaluator.go @@ -49,12 +49,13 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string var allDocuments = list.New() for _, filename := range filenames { - reader, err := readStream(filename) + reader, cleanup, err := readStream(filename) if err != nil { return err } fileDocuments, err := readDocuments(reader, filename, fileIndex, decoder) + cleanup() if err != nil { return err } diff --git a/pkg/yqlib/front_matter_test.go b/pkg/yqlib/front_matter_test.go index 2680aa28..7de929ae 100644 --- a/pkg/yqlib/front_matter_test.go +++ b/pkg/yqlib/front_matter_test.go @@ -136,10 +136,11 @@ Some content test.AssertResult(t, originalFilename, resolved) // Read documents using the temp file, verify they get the original filename - reader, err := readStream(tempFilename) + reader, cleanup, err := readStream(tempFilename) if err != nil { panic(err) } + defer cleanup() decoder := NewYamlDecoder(ConfiguredYamlPreferences) docs, err := readDocuments(reader, tempFilename, 0, decoder) if err != nil { diff --git a/pkg/yqlib/stream_evaluator.go b/pkg/yqlib/stream_evaluator.go index ad147894..657c0401 100644 --- a/pkg/yqlib/stream_evaluator.go +++ b/pkg/yqlib/stream_evaluator.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "os" ) // A yaml expression evaluator that runs the expression multiple times for each given yaml document. @@ -50,21 +49,17 @@ func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, p } for _, filename := range filenames { - reader, err := readStream(filename) + reader, cleanup, err := readStream(filename) if err != nil { return err } processedDocs, err := s.Evaluate(filename, reader, node, printer, decoder) + cleanup() if err != nil { return err } totalProcessDocs = totalProcessDocs + processedDocs - - switch reader := reader.(type) { - case *os.File: - safelyCloseFile(reader) - } } if totalProcessDocs == 0 { diff --git a/pkg/yqlib/utils.go b/pkg/yqlib/utils.go index e4036f03..d7eb6928 100644 --- a/pkg/yqlib/utils.go +++ b/pkg/yqlib/utils.go @@ -32,21 +32,20 @@ func resolveFilename(filename string) string { return filename } -func readStream(filename string) (io.Reader, error) { - var reader *bufio.Reader +// readStream returns a reader for the given file, along with a cleanup function +// that must be called once the reader is no longer needed. The cleanup is a no-op +// for stdin. +func readStream(filename string) (io.Reader, func(), error) { if filename == "-" { - reader = bufio.NewReader(os.Stdin) - } else { - // ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory, - // and ensuring that it's not possible to give a path to a file outside that directory. - file, err := os.Open(filename) // #nosec - if err != nil { - return nil, err - } - reader = bufio.NewReader(file) + return bufio.NewReader(os.Stdin), func() {}, nil } - return reader, nil - + // ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory, + // and ensuring that it's not possible to give a path to a file outside that directory. + file, err := os.Open(filename) // #nosec + if err != nil { + return nil, nil, err + } + return bufio.NewReader(file), func() { safelyCloseFile(file) }, nil } func writeString(writer io.Writer, txt string) error { diff --git a/pkg/yqlib/utils_test.go b/pkg/yqlib/utils_test.go index fc1b0026..19d50069 100644 --- a/pkg/yqlib/utils_test.go +++ b/pkg/yqlib/utils_test.go @@ -4,11 +4,89 @@ import ( "bufio" "bytes" "io" + "os" + "path/filepath" + "runtime/debug" + "strconv" "testing" "github.com/mikefarah/yq/v4/test" ) +// countOpenFileDescriptors returns the number of file descriptors this process +// currently holds open, or -1 if the platform does not expose them. +func countOpenFileDescriptors() int { + for _, dir := range []string{"/proc/self/fd", "/dev/fd"} { + // Readdirnames avoids stat-ing each entry, which races with descriptors + // (including this directory handle) being closed underneath us. + handle, err := os.Open(dir) + if err != nil { + continue + } + names, err := handle.Readdirnames(-1) + safelyCloseFile(handle) + if err != nil { + continue + } + // discount the directory handle itself + return len(names) - 1 + } + return -1 +} + +func writeSampleFiles(t *testing.T, count int) []string { + t.Helper() + dir := t.TempDir() + filenames := make([]string, count) + for i := 0; i < count; i++ { + filename := filepath.Join(dir, "sample-"+strconv.Itoa(i)+".yml") + if err := os.WriteFile(filename, []byte("a: apple\n"), 0600); err != nil { + t.Fatalf("failed to write sample file: %v", err) + } + filenames[i] = filename + } + return filenames +} + +func discardingPrinter() Printer { + return NewPrinter(NewYamlEncoder(ConfiguredYamlPreferences), NewSinglePrinterWriter(bufio.NewWriter(io.Discard))) +} + +func assertNoLeakedFileDescriptors(t *testing.T, evaluate func(filenames []string) error) { + t.Helper() + InitExpressionParser() + + // os.File finalisers close leaked descriptors on collection, which would + // let a genuine leak pass unnoticed. + defer debug.SetGCPercent(debug.SetGCPercent(-1)) + + before := countOpenFileDescriptors() + if before < 0 { + t.Skip("file descriptors are not observable on this platform") + } + + if err := evaluate(writeSampleFiles(t, 50)); err != nil { + t.Fatalf("failed to evaluate files: %v", err) + } + + after := countOpenFileDescriptors() + if after > before { + t.Errorf("expected no additional open file descriptors, had %d before and %d after", before, after) + } +} + +func TestStreamEvaluatorClosesInputFiles(t *testing.T) { + assertNoLeakedFileDescriptors(t, func(filenames []string) error { + return NewStreamEvaluator().EvaluateFiles(".a", filenames, discardingPrinter(), NewYamlDecoder(ConfiguredYamlPreferences)) + }) +} + +func TestAllAtOnceEvaluatorClosesInputFiles(t *testing.T) { + assertNoLeakedFileDescriptors(t, func(filenames []string) error { + return NewAllAtOnceEvaluator().EvaluateFiles(".a", filenames, discardingPrinter(), NewYamlDecoder(ConfiguredYamlPreferences)) + }) +} + // plainWriter only implements io.Writer, so io.WriteString must fall back to Write. type plainWriter struct { buf bytes.Buffer From 62ac0c6b07b3c8a33762d8a8f3bc660745310394 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Wed, 19 Aug 2026 14:29:08 +1000 Subject: [PATCH 09/23] Release notes --- release_notes.txt | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/release_notes.txt b/release_notes.txt index b4c70e44..8a34b7a5 100644 --- a/release_notes.txt +++ b/release_notes.txt @@ -1,3 +1,19 @@ +4.53.4: +- Close input files after processing each one (#2796) (#2808) Thanks @MsfPablo +- Fixed non string HCL string keys (#2795) +- Fixed heap allocation bug (#2809) Thanks @MsfPablo +- Fix deleting commented empty list YAML output (#2765) Thanks @dpersek +- Fix (has) return false for negative array indices (#2769) Thanks @maximilize +- Fix (sort): avoid int64 overflow comparing large integers (#2771) Thanks @maximilize +- Fix: preserve correct parent references in explode merge anchor reconstruction (#2730) Thanks @vomba +- Improve Guard ExpressionParser initialization with sync.Once (#2789) Thanks @arcaven +- Fix: reject negative indent instead of panicking (#2746) Thanks @StressTestor +- Fix: parse signed hex and octal integers (#2749) Thanks @StressTestor +- Fix: skip UTF-8 BOM when processing front matter (#2751) Thanks @StressTestor +- Fix !!merge tag regression for yq (#2705) Thanks @W-Floyd +- Default to yaml when a file's extension is not a recognised format (#2785) Thanks @devthedevil +- Bumped dependencies + 4.53.3: - Add `--ini-preserve-quotes` flag for INI round-trip quote preservation (#2728) Thanks @toller892! - Fix: reset INI decoder state on init (#2719) Thanks @xieby1! From ce7026e3195aceac6b6a56a6b81fca4e8e466e36 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Wed, 19 Aug 2026 14:32:17 +1000 Subject: [PATCH 10/23] Bumping version --- cmd/version.go | 2 +- snap/snapcraft.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/version.go b/cmd/version.go index 1eb38b1a..a9a23f3a 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -11,7 +11,7 @@ var ( GitDescribe string // Version is main version number that is being run at the moment. - Version = "v4.53.3" + Version = "v4.53.4" // VersionPrerelease is a pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index d33f1677..d1828317 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,5 +1,5 @@ name: yq -version: 'v4.53.3' +version: 'v4.53.4' summary: A lightweight and portable command-line data file processor description: | `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml, json, xml, csv, properties and TOML files. @@ -32,6 +32,6 @@ parts: build-environment: - CGO_ENABLED: 0 source: https://github.com/mikefarah/yq.git - source-tag: v4.53.3 + source-tag: v4.53.4 build-snaps: - go/latest/stable From ce3131723ac76a96962c893b0273159779b7cec0 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Thu, 20 Aug 2026 15:59:11 +1000 Subject: [PATCH 11/23] Pr 2825 (#2836) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * disable hard line wrapping The formatting defaults of the used YAML library changed. Since yq has no mechanism for the user to override them, change them back to what we used before. * Clamp YAML indent and close dumper after encoding. Clamp indent to the library's 2-9 range, wrap dumper setup errors with %w, and always close the dumper after dump. Co-authored-by: Cursor --------- Co-authored-by: Maximilian Gaß Co-authored-by: Cursor --- pkg/yqlib/encoder_yaml.go | 27 +++++++++++++++++++++------ pkg/yqlib/operator_load_test.go | 2 +- pkg/yqlib/operator_multiply_test.go | 4 ++-- pkg/yqlib/operator_select_test.go | 2 +- pkg/yqlib/operator_sort_keys_test.go | 2 +- pkg/yqlib/yaml_test.go | 6 ++++++ 6 files changed, 32 insertions(+), 11 deletions(-) diff --git a/pkg/yqlib/encoder_yaml.go b/pkg/yqlib/encoder_yaml.go index b46ae444..8f84edaf 100644 --- a/pkg/yqlib/encoder_yaml.go +++ b/pkg/yqlib/encoder_yaml.go @@ -2,6 +2,7 @@ package yqlib import ( "bytes" + "fmt" "io" "strings" @@ -49,23 +50,37 @@ func (ye *yamlEncoder) Encode(writer io.Writer, node *CandidateNode) error { destination = tempBuffer } - var encoder = yaml.NewEncoder(destination) + indent := ye.prefs.Indent + if indent < 2 { + indent = 2 + } else if indent > 9 { + indent = 9 + } - encoder.SetIndent(ye.prefs.Indent) - if ye.prefs.CompactSequenceIndent { - encoder.CompactSeqIndent() + dumper, err := yaml.NewDumper(destination, + yaml.WithV3Defaults(), + yaml.WithIndent(indent), + yaml.WithCompactSeqIndent(ye.prefs.CompactSequenceIndent), + yaml.WithLineWidth(-1), + ) + if err != nil { + return fmt.Errorf("configure YAML encoding: %w", err) } target, err := node.MarshalYAML() - if err != nil { + _ = dumper.Close() return err } trailingContent := target.FootComment target.FootComment = "" - if err := encoder.Encode(target); err != nil { + err = dumper.Dump(target) + if closeErr := dumper.Close(); err == nil { + err = closeErr + } + if err != nil { return err } diff --git a/pkg/yqlib/operator_load_test.go b/pkg/yqlib/operator_load_test.go index c7a8480a..aab885ae 100644 --- a/pkg/yqlib/operator_load_test.go +++ b/pkg/yqlib/operator_load_test.go @@ -77,7 +77,7 @@ var loadScenarios = []expressionScenario{ document: `{something: {file: "thing.yml"}, over: {here: [{file: "thing.yml"}]}}`, expression: `(.. | select(has("file"))) |= load("../../examples/" + .file)`, expected: []string{ - "D0, P[], (!!map)::{something: {a: apple is included, b: cool.}, over: {here: [{a: apple is included,\n b: cool.}]}}\n", + "D0, P[], (!!map)::{something: {a: apple is included, b: cool.}, over: {here: [{a: apple is included, b: cool.}]}}\n", }, }, { diff --git a/pkg/yqlib/operator_multiply_test.go b/pkg/yqlib/operator_multiply_test.go index fbd556bf..721a1f5b 100644 --- a/pkg/yqlib/operator_multiply_test.go +++ b/pkg/yqlib/operator_multiply_test.go @@ -83,7 +83,7 @@ var nodeWithFooter = `a: apple var document = `a: &cat {name: cat} b: {name: dog} -c: +c: <<: *cat ` @@ -514,7 +514,7 @@ var multiplyOperatorScenarios = []expressionScenario{ environmentVariables: map[string]string{"originalPath": ".myArray", "otherPath": ".newArray", "idPath": ".a"}, expression: mergeExpression, expected: []string{ - "D0, P[], (!!map)::{myArray: [{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC},\n {a: dingo, c: dingoC}], something: else}\n", + "D0, P[], (!!map)::{myArray: [{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC}, {a: dingo, c: dingoC}], something: else}\n", }, }, { diff --git a/pkg/yqlib/operator_select_test.go b/pkg/yqlib/operator_select_test.go index b9bcbbcd..35214f39 100644 --- a/pkg/yqlib/operator_select_test.go +++ b/pkg/yqlib/operator_select_test.go @@ -98,7 +98,7 @@ var selectOperatorScenarios = []expressionScenario{ document: `[{animal: cat, legs: {cool: true}}, {animal: fish}]`, expression: `(.[] | select(.legs.cool == true).canWalk) = true | (.[] | .alive.things) = "yes"`, expected: []string{ - "D0, P[], (!!seq)::[{animal: cat, legs: {cool: true}, canWalk: true, alive: {things: yes}}, {animal: fish,\n alive: {things: yes}}]\n", + "D0, P[], (!!seq)::[{animal: cat, legs: {cool: true}, canWalk: true, alive: {things: yes}}, {animal: fish, alive: {things: yes}}]\n", }, }, { diff --git a/pkg/yqlib/operator_sort_keys_test.go b/pkg/yqlib/operator_sort_keys_test.go index 562994d4..c895b194 100644 --- a/pkg/yqlib/operator_sort_keys_test.go +++ b/pkg/yqlib/operator_sort_keys_test.go @@ -37,7 +37,7 @@ var sortKeysOperatorScenarios = []expressionScenario{ document: `{bParent: {c: dog, array: [3,1,2]}, aParent: {z: donkey, x: [{c: yum, b: delish}, {b: ew, a: apple}]}}`, expression: `sort_keys(..)`, expected: []string{ - "D0, P[], (!!map)::{aParent: {x: [{b: delish, c: yum}, {a: apple, b: ew}], z: donkey}, bParent: {array: [\n 3, 1, 2], c: dog}}\n", + "D0, P[], (!!map)::{aParent: {x: [{b: delish, c: yum}, {a: apple, b: ew}], z: donkey}, bParent: {array: [3, 1, 2], c: dog}}\n", }, }, } diff --git a/pkg/yqlib/yaml_test.go b/pkg/yqlib/yaml_test.go index bee7856f..15c9a826 100644 --- a/pkg/yqlib/yaml_test.go +++ b/pkg/yqlib/yaml_test.go @@ -106,6 +106,12 @@ var yamlFormatScenarios = []formatScenario{ input: "[1, 2]", expected: "[1, 2]\n", }, + { + description: "long line", + skipDoc: true, + input: "field: Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt", + expected: "field: Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt\n", + }, } var yamlParseScenarios = []expressionScenario{ From 8a95461ec25eaa72bbcdce681213c788dedccc57 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:59:50 +1000 Subject: [PATCH 12/23] build(deps): bump golang from 1.26.5 to 1.26.6 (#2832) Bumps golang from 1.26.5 to 1.26.6. --- updated-dependencies: - dependency-name: golang dependency-version: 1.26.6 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- Dockerfile.dev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d6c72181..d1260184 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.26.5@sha256:079e59808d2d252516e27e3f3a9c003740dee7f75e55aa71528766d52bcfc16a AS builder +FROM golang:1.26.6@sha256:0d1d3a794be25f809dd2cb3160d8c73276c4056a9f8242a138e908ddeee7b6b6 AS builder WORKDIR /go/src/mikefarah/yq diff --git a/Dockerfile.dev b/Dockerfile.dev index e21b2bbc..6890e784 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM golang:1.26.5@sha256:079e59808d2d252516e27e3f3a9c003740dee7f75e55aa71528766d52bcfc16a +FROM golang:1.26.6@sha256:0d1d3a794be25f809dd2cb3160d8c73276c4056a9f8242a138e908ddeee7b6b6 COPY scripts/devtools.sh /opt/devtools.sh From 8dc5ae1229a1c3672af23beb6c0a581b20ac9f7b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:01:28 +1000 Subject: [PATCH 13/23] build(deps): bump golang.org/x/mod from 0.38.0 to 0.40.0 (#2831) Bumps [golang.org/x/mod](https://github.com/golang/mod) from 0.38.0 to 0.40.0. - [Commits](https://github.com/golang/mod/compare/v0.38.0...v0.40.0) --- updated-dependencies: - dependency-name: golang.org/x/mod dependency-version: 0.40.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 7fcb643a..1259bcfd 100644 --- a/go.mod +++ b/go.mod @@ -20,9 +20,9 @@ require ( github.com/yuin/gopher-lua v1.1.2 github.com/zclconf/go-cty v1.19.0 go.yaml.in/yaml/v4 v4.0.0-rc.6 - golang.org/x/mod v0.38.0 - golang.org/x/net v0.57.0 - golang.org/x/text v0.40.0 + golang.org/x/mod v0.40.0 + golang.org/x/net v0.58.0 + golang.org/x/text v0.41.0 ) require ( @@ -36,7 +36,7 @@ require ( github.com/mitchellh/go-wordwrap v1.0.1 // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.47.0 // indirect - golang.org/x/tools v0.47.0 // indirect + golang.org/x/tools v0.49.0 // indirect ) go 1.25.0 diff --git a/go.sum b/go.sum index 720f6f45..45e5d2ca 100644 --- a/go.sum +++ b/go.sum @@ -72,19 +72,19 @@ github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmB go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4= go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= -golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= -golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= -golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= -golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= +golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= -golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= -golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From b1d6df4217755074da146631e185c2f6b2855b09 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:04:20 +1000 Subject: [PATCH 14/23] build(deps): bump docker/setup-buildx-action from 4.1.0 to 4.3.0 (#2829) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 4.1.0 to 4.3.0. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5...37fe631027851001ddb9b187196cc803df7f5f0e) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-version: 4.3.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker-githubaction.yml | 2 +- .github/workflows/docker-release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-githubaction.yml b/.github/workflows/docker-githubaction.yml index cf9e1122..97dc6853 100644 --- a/.github/workflows/docker-githubaction.yml +++ b/.github/workflows/docker-githubaction.yml @@ -24,7 +24,7 @@ jobs: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 with: version: latest diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index b16e54fb..ac8f309d 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -28,7 +28,7 @@ jobs: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 with: version: latest From 1f5d6e6656a32e1cc643e5edc431e7cfeff70d70 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Thu, 20 Aug 2026 16:15:37 +1000 Subject: [PATCH 15/23] Release notes --- release_notes.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/release_notes.txt b/release_notes.txt index 8a34b7a5..47f01274 100644 --- a/release_notes.txt +++ b/release_notes.txt @@ -1,3 +1,7 @@ +4.53.5: + - Fixed line wrapping bug (#2824, #2823) Thanks mxey + - Bumped dependencies + 4.53.4: - Close input files after processing each one (#2796) (#2808) Thanks @MsfPablo - Fixed non string HCL string keys (#2795) From e95064cd018c0e3d71ef83e4272b4722571dfa64 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Thu, 20 Aug 2026 16:16:13 +1000 Subject: [PATCH 16/23] Bumping version --- cmd/version.go | 2 +- snap/snapcraft.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/version.go b/cmd/version.go index a9a23f3a..2d118195 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -11,7 +11,7 @@ var ( GitDescribe string // Version is main version number that is being run at the moment. - Version = "v4.53.4" + Version = "v4.53.5" // VersionPrerelease is a pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index d1828317..bcee365f 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,5 +1,5 @@ name: yq -version: 'v4.53.4' +version: 'v4.53.5' summary: A lightweight and portable command-line data file processor description: | `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml, json, xml, csv, properties and TOML files. @@ -32,6 +32,6 @@ parts: build-environment: - CGO_ENABLED: 0 source: https://github.com/mikefarah/yq.git - source-tag: v4.53.4 + source-tag: v4.53.5 build-snaps: - go/latest/stable From ed4125b9b184e58e7d01cd2429aee785c2f1c32a Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Thu, 20 Aug 2026 19:07:20 +1000 Subject: [PATCH 17/23] Fixing release build --- .github/workflows/release.yml | 2 +- release_notes.txt | 3 +++ scripts/xcompile.sh | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fecc5900..51b8d07c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,7 +48,7 @@ jobs: - name: Cross compile run: | sudo apt-get install rhash -y - go install github.com/goreleaser/goreleaser/v2@v2.16.0 + go install github.com/goreleaser/goreleaser/v2@v2.17.1 ./scripts/xcompile.sh - name: Sign checksums diff --git a/release_notes.txt b/release_notes.txt index 47f01274..2f7c72bf 100644 --- a/release_notes.txt +++ b/release_notes.txt @@ -1,3 +1,6 @@ +4.53.6: + - Fixing release build issue + 4.53.5: - Fixed line wrapping bug (#2824, #2823) Thanks mxey - Bumped dependencies diff --git a/scripts/xcompile.sh b/scripts/xcompile.sh index efada51e..56bd341c 100755 --- a/scripts/xcompile.sh +++ b/scripts/xcompile.sh @@ -2,7 +2,7 @@ set -eo pipefail -# You may need to go install github.com/goreleaser/goreleaser/v2@v2.16.0 first +# You may need to go install github.com/goreleaser/goreleaser/v2@v2.17.1 first GORELEASER="goreleaser build --clean" if [ -z "$CI" ] || [[ "${GITHUB_REF_NAME:-}" == draft-* ]]; then GORELEASER+=" --snapshot" From c14f446382944492701b16c1ddb48bb9dbe683e3 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Thu, 20 Aug 2026 19:07:38 +1000 Subject: [PATCH 18/23] Bumping version --- cmd/version.go | 2 +- snap/snapcraft.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/version.go b/cmd/version.go index 2d118195..e8b0415b 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -11,7 +11,7 @@ var ( GitDescribe string // Version is main version number that is being run at the moment. - Version = "v4.53.5" + Version = "v4.53.6" // VersionPrerelease is a pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index bcee365f..af3ceed5 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,5 +1,5 @@ name: yq -version: 'v4.53.5' +version: 'v4.53.6' summary: A lightweight and portable command-line data file processor description: | `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml, json, xml, csv, properties and TOML files. @@ -32,6 +32,6 @@ parts: build-environment: - CGO_ENABLED: 0 source: https://github.com/mikefarah/yq.git - source-tag: v4.53.5 + source-tag: v4.53.6 build-snaps: - go/latest/stable From 8b5af0694bb82b41d4ae180fac9972029066f90a Mon Sep 17 00:00:00 2001 From: Siddharth Date: Tue, 25 Aug 2026 15:49:22 +0530 Subject: [PATCH 19/23] fix(test): update expected format for unrecognised extension to yaml (#2837) When unrecognised extensions are passed, FormatStringFromFilename defaults to 'yaml' (as handled in pkg/yqlib/format.go and tested in pkg/yqlib/format_test.go). This updates the test expectation in test/format_test.go for 'file.unknown' to 'yaml' so that go test ./test passes. --- test/format_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/format_test.go b/test/format_test.go index ffa41ee1..4ad4daee 100644 --- a/test/format_test.go +++ b/test/format_test.go @@ -17,7 +17,7 @@ func TestFormatStringFromFilename(t *testing.T) { {"FILE.JSON", "json"}, {"file.properties", "properties"}, {"file.xml", "xml"}, - {"file.unknown", "unknown"}, + {"file.unknown", "yaml"}, // filenames without extensions {"file", "yaml"}, From 6fc8fb4e3b321ab189dcf6e82e483aeb048662d1 Mon Sep 17 00:00:00 2001 From: M Laraib Ali Date: Tue, 15 Sep 2026 09:23:58 +0500 Subject: [PATCH 20/23] fix: /dev/null was triggering colour output (#2852) os.ModeCharDevice matches /dev/null too, so redirecting there silently turned on colourised (and much slower) output. Use isatty instead, like fatih/color already does internally - no new dependency needed. Tests cover a regular file, /dev/null, pipes and a real tty. --- cmd/utils.go | 8 ++-- cmd/utils_test.go | 100 ++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 2 +- 3 files changed, 106 insertions(+), 4 deletions(-) diff --git a/cmd/utils.go b/cmd/utils.go index 26631fc4..6352ee13 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -7,6 +7,7 @@ import ( "os" "strings" + "github.com/mattn/go-isatty" "github.com/mikefarah/yq/v4/pkg/yqlib" "github.com/spf13/cobra" ) @@ -45,9 +46,10 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) { } func setupColors() { - fileInfo, _ := os.Stdout.Stat() - - if forceColor || (!forceNoColor && (fileInfo.Mode()&os.ModeCharDevice) != 0) { + fd := os.Stdout.Fd() + // os.Stdout can be a regular file, pipe, or other character device such + // as /dev/null - only a real terminal should get colourised output. + if forceColor || (!forceNoColor && (isatty.IsTerminal(fd) || isatty.IsCygwinTerminal(fd))) { colorsEnabled = true } } diff --git a/cmd/utils_test.go b/cmd/utils_test.go index dd1fa61a..95fe2dce 100644 --- a/cmd/utils_test.go +++ b/cmd/utils_test.go @@ -3,7 +3,10 @@ package cmd import ( "fmt" "os" + "path/filepath" + "runtime" "strings" + "syscall" "testing" "github.com/mikefarah/yq/v4/pkg/yqlib" @@ -964,6 +967,103 @@ func TestSetupColors(t *testing.T) { } } +// terminalCase is a file/device to test setupColors against, paired with +// whether it should be treated as an interactive terminal - so what does +// and doesn't get colourised output is visible in a single place. +type terminalCase struct { + open func(t *testing.T) *os.File + isTerm bool +} + +func terminalTestCases() map[string]terminalCase { + return map[string]terminalCase{ + "regular file": {isTerm: false, open: func(t *testing.T) *os.File { + f, err := os.CreateTemp(t.TempDir(), "isterminal") + if err != nil { + t.Fatalf("failed to create temp file: %v", err) + } + return f + }}, + "/dev/null": {isTerm: false, open: func(t *testing.T) *os.File { + f, err := os.OpenFile(os.DevNull, os.O_RDWR, 0644) + if err != nil { + t.Fatalf("failed to open %s: %v", os.DevNull, err) + } + return f + }}, + "unnamed pipe, read end": {isTerm: false, open: func(t *testing.T) *os.File { + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("failed to create pipe: %v", err) + } + w.Close() + return r + }}, + "unnamed pipe, write end (e.g. piped to a pager)": {isTerm: false, open: func(t *testing.T) *os.File { + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("failed to create pipe: %v", err) + } + r.Close() + return w + }}, + "named pipe (FIFO)": {isTerm: false, open: func(t *testing.T) *os.File { + if runtime.GOOS == "windows" { + t.Skip("named pipes not supported by this test on windows") + } + path := filepath.Join(t.TempDir(), "fifo") + if err := syscall.Mkfifo(path, 0600); err != nil { + t.Fatalf("failed to create named pipe: %v", err) + } + // O_RDWR avoids blocking on open: a FIFO opened only for reading + // (or only writing) blocks until a peer opens the other end. + f, err := os.OpenFile(path, os.O_RDWR, os.ModeNamedPipe) + if err != nil { + t.Fatalf("failed to open named pipe: %v", err) + } + return f + }}, + "controlling terminal (/dev/tty)": {isTerm: true, open: func(t *testing.T) *os.File { + f, err := os.OpenFile("/dev/tty", os.O_RDWR, 0) + if err != nil { + t.Skipf("no controlling terminal available in this environment: %v", err) + } + return f + }}, + } +} + +func TestSetupColorsForFileKind(t *testing.T) { + for name, tc := range terminalTestCases() { + t.Run(name, func(t *testing.T) { + f := tc.open(t) + defer f.Close() + + originalStdout := os.Stdout + originalForceColor := forceColor + originalForceNoColor := forceNoColor + originalColorsEnabled := colorsEnabled + defer func() { + os.Stdout = originalStdout + forceColor = originalForceColor + forceNoColor = originalForceNoColor + colorsEnabled = originalColorsEnabled + }() + + os.Stdout = f + forceColor = false + forceNoColor = false + colorsEnabled = false + + setupColors() + + if colorsEnabled != tc.isTerm { + t.Errorf("setupColors() colorsEnabled = %v, want %v for %s", colorsEnabled, tc.isTerm, name) + } + }) + } +} + func TestLoadSplitFileExpression(t *testing.T) { // Create a temporary file with expression content tempFile, err := os.CreateTemp("", "split") diff --git a/go.mod b/go.mod index 1259bcfd..212ac467 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/hashicorp/hcl/v2 v2.24.0 github.com/jinzhu/copier v0.4.0 github.com/magiconair/properties v1.18.11 + github.com/mattn/go-isatty v0.0.20 github.com/pelletier/go-toml/v2 v2.4.3 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e github.com/spf13/cobra v1.10.2 @@ -32,7 +33,6 @@ require ( github.com/google/go-cmp v0.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.47.0 // indirect From e27a15e50a8902e47f205d5c5d4bbd20ee2b7864 Mon Sep 17 00:00:00 2001 From: MsfPablo <129399053+MsfPablo@users.noreply.github.com> Date: Tue, 15 Sep 2026 06:30:28 +0200 Subject: [PATCH 21/23] fix: error on int64 overflow in +, -, * instead of silently wrapping (#2826) Integer arithmetic (+, -, *) on two !!int values previously wrapped on int64 overflow, producing silently-wrong results: yq -n '3000000000 * 4000000000' -> -6446744073709551616 yq -n '9223372036854775807 + 1' -> -9223372036854775808 This matches neither user expectations nor yq's own behaviour elsewhere: parseInt64 already refuses out-of-range integer literals, and string repetition guards against len*count overflow. Arithmetic should be consistent and refuse out-of-range results too. Add checkedInt64Arithmetic, which computes the exact result with big.Int and errors when it falls outside [MinInt64, MaxInt64]. Wire it into the !!int branches of addScalars, subtractScalars and multiplyIntegers. The error propagates through both the plain operators and the compound assign forms (+=, -=, *=). Non-overflowing arithmetic, formatting (%v/0x%X/0o%o) and the int/float mixed paths are unchanged. Fixes #2820. Co-authored-by: Pablo Garcia --- pkg/yqlib/lib.go | 29 +++++++++++++++++++++++++++++ pkg/yqlib/operator_add.go | 5 ++++- pkg/yqlib/operator_add_test.go | 9 +++++++++ pkg/yqlib/operator_multiply.go | 6 +++++- pkg/yqlib/operator_multiply_test.go | 17 +++++++++++++++++ pkg/yqlib/operator_subtract.go | 5 ++++- pkg/yqlib/operator_subtract_test.go | 9 +++++++++ 7 files changed, 77 insertions(+), 3 deletions(-) diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 1503572b..02608bb3 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -6,6 +6,7 @@ import ( "fmt" "log/slog" "math" + "math/big" "strconv" "strings" "sync" @@ -204,6 +205,34 @@ func parseInt(numberString string) (int, error) { return int(parsed), err } +// checkedInt64Arithmetic performs '+', '-' or '*' on two int64 values and +// returns the exact result only when it still fits in an int64. yq refuses to +// silently wrap on int64 overflow (see issue #2820): a result outside +// [math.MinInt64, math.MaxInt64] is reported as an error rather than returned +// as a wrapped (and therefore wrong) value. big.Int is used so the exact +// product/sum/difference is computed before the range check, which avoids the +// sign-handling pitfalls of doing the overflow detection directly on the +// wrapped int64 result. +func checkedInt64Arithmetic(op byte, lhsNum, rhsNum int64) (int64, error) { + lhs := new(big.Int).SetInt64(lhsNum) + rhs := new(big.Int).SetInt64(rhsNum) + result := new(big.Int) + switch op { + case '+': + result.Add(lhs, rhs) + case '-': + result.Sub(lhs, rhs) + case '*': + result.Mul(lhs, rhs) + default: + return 0, fmt.Errorf("unknown arithmetic operator %q", op) + } + if !result.IsInt64() { + return 0, fmt.Errorf("%v %c %v overflows int64", lhsNum, op, rhsNum) + } + return result.Int64(), nil +} + func processEscapeCharacters(original string) string { if original == "" { return original diff --git a/pkg/yqlib/operator_add.go b/pkg/yqlib/operator_add.go index 538b858c..ced543ff 100644 --- a/pkg/yqlib/operator_add.go +++ b/pkg/yqlib/operator_add.go @@ -122,7 +122,10 @@ func addScalars(context Context, target *CandidateNode, lhs *CandidateNode, rhs if err != nil { return err } - sum := lhsNum + rhsNum + sum, err := checkedInt64Arithmetic('+', lhsNum, rhsNum) + if err != nil { + return err + } target.Tag = lhs.Tag target.Value = fmt.Sprintf(format, sum) } else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") { diff --git a/pkg/yqlib/operator_add_test.go b/pkg/yqlib/operator_add_test.go index d5cff38d..af848102 100644 --- a/pkg/yqlib/operator_add_test.go +++ b/pkg/yqlib/operator_add_test.go @@ -539,6 +539,15 @@ var addOperatorScenarios = []expressionScenario{ "D0, P[], (!!map)::? [{~: ~}]\n: v1\n? [{2: ~}]\n: v2\n", }, }, + { + // int64 + int64 that overflows int64. See issue #2820: previously + // this silently wrapped (MaxInt64 + 1 -> MinInt64). + description: "Add integers that overflow int64", + skipDoc: true, + document: `a: 9223372036854775807`, + expression: `.a + 1`, + expectedError: "9223372036854775807 + 1 overflows int64", + }, } func TestAddOperatorScenarios(t *testing.T) { diff --git a/pkg/yqlib/operator_multiply.go b/pkg/yqlib/operator_multiply.go index dc69c57f..4efcd96d 100644 --- a/pkg/yqlib/operator_multiply.go +++ b/pkg/yqlib/operator_multiply.go @@ -133,7 +133,11 @@ func multiplyIntegers(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, e if err != nil { return nil, err } - target.Value = fmt.Sprintf(format, lhsNum*rhsNum) + product, err := checkedInt64Arithmetic('*', lhsNum, rhsNum) + if err != nil { + return nil, err + } + target.Value = fmt.Sprintf(format, product) return target, nil } diff --git a/pkg/yqlib/operator_multiply_test.go b/pkg/yqlib/operator_multiply_test.go index 721a1f5b..a5150163 100644 --- a/pkg/yqlib/operator_multiply_test.go +++ b/pkg/yqlib/operator_multiply_test.go @@ -716,6 +716,23 @@ var multiplyOperatorScenarios = []expressionScenario{ expression: fmt.Sprintf(`"ab" * %d`, 1<<(bits.UintSize-2)), expectedError: fmt.Sprintf("result of repeating string (2 bytes) by %d would exceed 10485760 bytes", 1<<(bits.UintSize-2)), }, + { + // int64 * int64 that overflows int64. See issue #2820: previously + // this silently wrapped to a wrong (negative) value. + description: "Multiply integers that overflow int64", + skipDoc: true, + document: `a: 3000000000`, + expression: `.a * 4000000000`, + expectedError: "3000000000 * 4000000000 overflows int64", + }, + { + // MaxInt64 * 2 wraps to -2 without the overflow guard. + description: "Multiply max int by 2 overflows int64", + skipDoc: true, + document: `a: 9223372036854775807`, + expression: `.a * 2`, + expectedError: "9223372036854775807 * 2 overflows int64", + }, } func TestMultiplyOperatorScenarios(t *testing.T) { diff --git a/pkg/yqlib/operator_subtract.go b/pkg/yqlib/operator_subtract.go index ab23bcee..43842841 100644 --- a/pkg/yqlib/operator_subtract.go +++ b/pkg/yqlib/operator_subtract.go @@ -104,7 +104,10 @@ func subtractScalars(context Context, target *CandidateNode, lhs *CandidateNode, if err != nil { return err } - result := lhsNum - rhsNum + result, err := checkedInt64Arithmetic('-', lhsNum, rhsNum) + if err != nil { + return err + } target.Tag = lhs.Tag target.Value = fmt.Sprintf(format, result) } else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") { diff --git a/pkg/yqlib/operator_subtract_test.go b/pkg/yqlib/operator_subtract_test.go index 99c81b18..d10c6da8 100644 --- a/pkg/yqlib/operator_subtract_test.go +++ b/pkg/yqlib/operator_subtract_test.go @@ -157,6 +157,15 @@ var subtractOperatorScenarios = []expressionScenario{ "D0, P[], (!!seq)::[!goat {a: b}]\n", }, }, + { + // int64 - int64 that overflows int64. See issue #2820: previously + // this silently wrapped (MinInt64 - 1 -> MaxInt64). + description: "Subtract integers that overflow int64", + skipDoc: true, + document: `a: -9223372036854775808`, + expression: `.a - 1`, + expectedError: "-9223372036854775808 - 1 overflows int64", + }, } func TestSubtractOperatorScenarios(t *testing.T) { From 0442602b3215fec6043799cb0b3637f8d5c65f27 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Tue, 15 Sep 2026 14:33:50 +1000 Subject: [PATCH 22/23] Adding missing test --- pkg/yqlib/doc/operators/system-operators.md | 4 +-- pkg/yqlib/lib_test.go | 40 +++++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/pkg/yqlib/doc/operators/system-operators.md b/pkg/yqlib/doc/operators/system-operators.md index df1a76cd..de0a8caf 100644 --- a/pkg/yqlib/doc/operators/system-operators.md +++ b/pkg/yqlib/doc/operators/system-operators.md @@ -51,7 +51,7 @@ country: Australia ``` then ```bash -yq --security-enable-system-operator '.country = system("/usr/bin/echo"; "test")' sample.yml +yq --security-enable-system-operator '.country = system("/bin/echo"; "test")' sample.yml ``` will output ```yaml @@ -67,7 +67,7 @@ a: hello ``` then ```bash -yq --security-enable-system-operator '.a = system("/usr/bin/echo")' sample.yml +yq --security-enable-system-operator '.a = system("/bin/echo")' sample.yml ``` will output ```yaml diff --git a/pkg/yqlib/lib_test.go b/pkg/yqlib/lib_test.go index 4e031dfd..6727d8b5 100644 --- a/pkg/yqlib/lib_test.go +++ b/pkg/yqlib/lib_test.go @@ -2,6 +2,7 @@ package yqlib import ( "fmt" + "math" "strings" "sync" "testing" @@ -173,6 +174,45 @@ func TestParseInt64(t *testing.T) { } } +func TestCheckedInt64Arithmetic(t *testing.T) { + type checkedInt64ArithmeticScenario struct { + description string + op byte + lhs int64 + rhs int64 + expectedResult int64 + expectedError string + } + + scenarios := []checkedInt64ArithmeticScenario{ + {description: "add, no overflow", op: '+', lhs: 3, rhs: 4, expectedResult: 7}, + {description: "add, overflows MaxInt64", op: '+', lhs: math.MaxInt64, rhs: 1, expectedError: "9223372036854775807 + 1 overflows int64"}, + {description: "subtract, no overflow", op: '-', lhs: 10, rhs: 3, expectedResult: 7}, + {description: "subtract, underflows MinInt64", op: '-', lhs: math.MinInt64, rhs: 1, expectedError: "-9223372036854775808 - 1 overflows int64"}, + {description: "multiply, no overflow", op: '*', lhs: 6, rhs: 7, expectedResult: 42}, + {description: "multiply, overflows MaxInt64", op: '*', lhs: math.MaxInt64, rhs: 2, expectedError: "9223372036854775807 * 2 overflows int64"}, + {description: "multiply, MinInt64 by -1 overflows (no positive counterpart exists)", op: '*', lhs: math.MinInt64, rhs: -1, expectedError: "-9223372036854775808 * -1 overflows int64"}, + {description: "multiply, MinInt64 by 1 does not overflow", op: '*', lhs: math.MinInt64, rhs: 1, expectedResult: math.MinInt64}, + {description: "unknown operator", op: '/', lhs: 4, rhs: 2, expectedError: "unknown arithmetic operator"}, + } + + for _, tt := range scenarios { + actualResult, err := checkedInt64Arithmetic(tt.op, tt.lhs, tt.rhs) + if tt.expectedError != "" { + if err == nil { + t.Errorf("%v: expected error containing '%s' but got none", tt.description, tt.expectedError) + } else if !strings.Contains(err.Error(), tt.expectedError) { + t.Errorf("%v: expected error containing '%s', got '%s'", tt.description, tt.expectedError, err.Error()) + } + continue + } + if err != nil { + t.Errorf("%v: unexpected error: %v", tt.description, err) + } + test.AssertResultComplexWithContext(t, tt.expectedResult, actualResult, tt.description) + } +} + func TestGetContentValueByKey(t *testing.T) { // Create content with key-value pairs key1 := createStringScalarNode("key1") From 5efa5ffcd89e694f8357e031f8f1d4862fd81536 Mon Sep 17 00:00:00 2001 From: MsfPablo <129399053+MsfPablo@users.noreply.github.com> Date: Tue, 15 Sep 2026 07:54:05 +0200 Subject: [PATCH 23/23] =?UTF-8?q?feat:=20add=20@base64url/@base64urld=20op?= =?UTF-8?q?erators=20(RFC=204648=20=C2=A75)=20(#2827)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the URL- and filename-safe Base64 variant alongside the existing @base64/@base64d operators. The URL-safe alphabet replaces + and / with - and _ so encoded values can be embedded in URLs, filenames, or other contexts where + and / have special meaning. Adds: - Base64UrlFormat registered in the Formats slice (also exposes base64url as a top-level --input/--output-format) - NewBase64URLEncoder / NewBase64URLDecoder backed by base64.URLEncoding - @base64url / @base64urld lexer tokens (placed before @base64/@base64d so the longer patterns match first under Go's leftmost-first regex) - no_base64.go stubs for the yq_nobase64 build tag - operator tests covering encode, decode, round-trip, and missing-padding decode Closes #2794 Co-authored-by: Pablo Garcia --- pkg/yqlib/decoder_base64.go | 7 +++++ pkg/yqlib/doc/operators/encode-decode.md | 30 ++++++++++++++++++++ pkg/yqlib/encoder_base64.go | 6 ++++ pkg/yqlib/format.go | 10 +++++++ pkg/yqlib/lexer_participle.go | 3 ++ pkg/yqlib/no_base64.go | 8 ++++++ pkg/yqlib/operator_encoder_decoder_test.go | 33 ++++++++++++++++++++++ 7 files changed, 97 insertions(+) diff --git a/pkg/yqlib/decoder_base64.go b/pkg/yqlib/decoder_base64.go index 852878f1..56ee05c0 100644 --- a/pkg/yqlib/decoder_base64.go +++ b/pkg/yqlib/decoder_base64.go @@ -20,6 +20,13 @@ func NewBase64Decoder() Decoder { return &base64Decoder{finished: false, encoding: *base64.StdEncoding} } +// NewBase64URLDecoder returns a decoder for the URL- and filename-safe +// Base64 variant (RFC 4648 §5): `-` and `_` replace `+` and `/`. +// The shared padding logic in Init tolerates inputs with or without padding. +func NewBase64URLDecoder() Decoder { + return &base64Decoder{finished: false, encoding: *base64.URLEncoding} +} + func (dec *base64Decoder) Init(reader io.Reader) error { // Read all data from the reader and strip leading/trailing whitespace // This is necessary because base64 decoding needs to see the complete input diff --git a/pkg/yqlib/doc/operators/encode-decode.md b/pkg/yqlib/doc/operators/encode-decode.md index b3b02aad..0e4f1c8d 100644 --- a/pkg/yqlib/doc/operators/encode-decode.md +++ b/pkg/yqlib/doc/operators/encode-decode.md @@ -514,3 +514,33 @@ coolData: a: apple ``` +## Encode a string to base64url +Given a sample.yml file of: +```yaml +coolData: "Works with UTF-16 \U0001F60A" +``` +then +```bash +yq '.coolData | @base64url' sample.yml +``` +will output +```yaml +V29ya3Mgd2l0aCBVVEYtMTYg8J-Yig== +``` + +## Decode a base64url encoded string +URL-safe variant uses `-` and `_` in place of `+` and `/` (RFC 4648 §5). + +Given a sample.yml file of: +```yaml +coolData: V29ya3Mgd2l0aCBVVEYtMTYg8J-Yig== +``` +then +```bash +yq '.coolData | @base64urld' sample.yml +``` +will output +```yaml +Works with UTF-16 😊 +``` + diff --git a/pkg/yqlib/encoder_base64.go b/pkg/yqlib/encoder_base64.go index cfefbcd0..ecb4b83a 100644 --- a/pkg/yqlib/encoder_base64.go +++ b/pkg/yqlib/encoder_base64.go @@ -16,6 +16,12 @@ func NewBase64Encoder() Encoder { return &base64Encoder{encoding: *base64.StdEncoding} } +// NewBase64URLEncoder returns an encoder for the URL- and filename-safe +// Base64 variant (RFC 4648 §5): `-` and `_` replace `+` and `/`. +func NewBase64URLEncoder() Encoder { + return &base64Encoder{encoding: *base64.URLEncoding} +} + func (e *base64Encoder) CanHandleAliases() bool { return false } diff --git a/pkg/yqlib/format.go b/pkg/yqlib/format.go index 406ab6dd..f90bb29c 100644 --- a/pkg/yqlib/format.go +++ b/pkg/yqlib/format.go @@ -58,6 +58,15 @@ var Base64Format = &Format{"base64", []string{}, func() Decoder { return NewBase64Decoder() }, } +// Base64UrlFormat is the URL- and filename-safe Base64 variant defined by +// RFC 4648 §5: it uses `-` and `_` in place of `+` and `/`. Padding is kept +// (matching the standard Base64Format), so round-tripping with @base64urld +// works whether or not the input was padded. +var Base64UrlFormat = &Format{"base64url", []string{}, + func() Encoder { return NewBase64URLEncoder() }, + func() Decoder { return NewBase64URLDecoder() }, +} + var UriFormat = &Format{"uri", []string{}, func() Encoder { return NewUriEncoder() }, func() Decoder { return NewUriDecoder() }, @@ -102,6 +111,7 @@ var Formats = []*Format{ TSVFormat, XMLFormat, Base64Format, + Base64UrlFormat, UriFormat, ShFormat, TomlFormat, diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go index 938b8a71..c0be591c 100644 --- a/pkg/yqlib/lexer_participle.go +++ b/pkg/yqlib/lexer_participle.go @@ -81,6 +81,9 @@ var participleYqRules = []*participleYqRule{ {"TSVDecode", `from_?tsv|@tsvd`, decodeOp(TSVFormat), 0}, {"TSVEncode", `to_?tsv|@tsv`, encodeWithIndent(TSVFormat, 0), 0}, + {"Base64Urld", `@base64urld`, decodeOp(Base64UrlFormat), 0}, + {"Base64Url", `@base64url`, encodeWithIndent(Base64UrlFormat, 0), 0}, + {"Base64d", `@base64d`, decodeOp(Base64Format), 0}, {"Base64", `@base64`, encodeWithIndent(Base64Format, 0), 0}, diff --git a/pkg/yqlib/no_base64.go b/pkg/yqlib/no_base64.go index b6fe32bc..e64b99dd 100644 --- a/pkg/yqlib/no_base64.go +++ b/pkg/yqlib/no_base64.go @@ -9,3 +9,11 @@ func NewBase64Decoder() Decoder { func NewBase64Encoder() Encoder { return nil } + +func NewBase64URLDecoder() Decoder { + return nil +} + +func NewBase64URLEncoder() Encoder { + return nil +} diff --git a/pkg/yqlib/operator_encoder_decoder_test.go b/pkg/yqlib/operator_encoder_decoder_test.go index 6280f0cc..745fea07 100644 --- a/pkg/yqlib/operator_encoder_decoder_test.go +++ b/pkg/yqlib/operator_encoder_decoder_test.go @@ -326,6 +326,39 @@ var encoderDecoderOperatorScenarios = []expressionScenario{ "D0, P[], (!!str)::cats\n", }, }, + { + description: "Encode a string to base64url", + document: "coolData: Works with UTF-16 😊", + expression: ".coolData | @base64url", + expected: []string{ + "D0, P[coolData], (!!str)::V29ya3Mgd2l0aCBVVEYtMTYg8J-Yig==\n", + }, + }, + { + description: "Decode a base64url encoded string", + subdescription: "URL-safe variant uses `-` and `_` in place of `+` and `/` (RFC 4648 §5).", + document: "coolData: V29ya3Mgd2l0aCBVVEYtMTYg8J-Yig==", + expression: ".coolData | @base64urld", + expected: []string{ + "D0, P[coolData], (!!str)::Works with UTF-16 😊\n", + }, + }, + { + description: "base64url round trip preserves bytes with + and /", + skipDoc: true, + expression: `">>>???" | @base64url | @base64urld`, + expected: []string{ + "D0, P[], (!!str)::>>>???\n", + }, + }, + { + description: "base64url missing padding decode", + skipDoc: true, + expression: `"V29ya3Mgd2l0aCBVVEYtMTYg8J-Yig" | @base64urld`, + expected: []string{ + "D0, P[], (!!str)::Works with UTF-16 😊\n", + }, + }, { requiresFormat: "xml", description: "empty xml decode",